{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloading Legacy Survey DR4\n",
    "\n",
    "This notebook describes the downloading of the 'Legacy Survey' DR4\n",
    "\n",
    "http://legacysurvey.org/dr4/description/\n",
    "\n",
    "There are still edge effects due to using the brick centre to check if it is in HELP. TODO: develop a nearMoc function which says if a given ra/dec is near a given moc (within a given distance). This function already exists within Stilts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "from astropy.table import Table\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.utils import inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "log_file = open('log.txt', 'a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Download the list of all bricks\n",
    "subprocess.call(['wget', \n",
    "                    'ftp://archive.noao.edu/public/hlsp/ls/dr4/survey-bricks-dr4.fits.gz', '-P', './data'], \n",
    "                    stdout=log_file)\n",
    "\n",
    "#Download the list of all bricks\n",
    "subprocess.call(['wget', \n",
    "                    'ftp://archive.noao.edu/public/hlsp/ls/dr4/survey-bricks.fits.gz', '-P', './data'], \n",
    "                    stdout=log_file)\n",
    "\n",
    "#Download the list of all bricks\n",
    "subprocess.call(['wget', \n",
    "                    'ftp://archive.noao.edu/public/hlsp/ls/dr3/survey-bricks-dr3.fits.gz', '-P', './data'], \n",
    "                    stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total bricks on whole sky: 662174\n",
      "Total bricks on DECaLS DR3: 149464\n",
      "Total bricks on Legacy Survey DR4: 65543\n"
     ]
    }
   ],
   "source": [
    "dr4_bricks = Table.read('./data/survey-bricks-dr4.fits.gz')\n",
    "dr3_bricks = Table.read('./data/survey-bricks-dr3.fits.gz')\n",
    "total_bricks = Table.read('./data/survey-bricks.fits.gz')\n",
    "print('Total bricks on whole sky: ' + str(len(total_bricks)))\n",
    "print('Total bricks on DECaLS DR3: ' + str(len(dr3_bricks)))\n",
    "print('Total bricks on Legacy Survey DR4: ' + str(len(dr4_bricks)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<Table length=10>\n",
       "<table id=\"table4545790136-347353\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>brickname</th><th>ra</th><th>dec</th><th>nexp_g</th><th>nexp_r</th><th>nexp_z</th><th>nexphist_g [6]</th><th>nexphist_r [6]</th><th>nexphist_z [6]</th><th>nobjs</th><th>npsf</th><th>nsimp</th><th>nexp</th><th>ndev</th><th>ncomp</th><th>psfsize_g</th><th>psfsize_r</th><th>psfsize_z</th><th>psfdepth_g</th><th>psfdepth_r</th><th>psfdepth_z</th><th>galdepth_g</th><th>galdepth_r</th><th>galdepth_z</th><th>ebv</th><th>trans_g</th><th>trans_r</th><th>trans_z</th><th>ext_g</th><th>ext_r</th><th>ext_z</th><th>wise_nobs [4]</th><th>trans_wise [4]</th><th>ext_w1</th><th>ext_w2</th><th>ext_w3</th><th>ext_w4</th></tr></thead>\n",
       "<tr><td>0</td><td>0556p332</td><td>55.671641791</td><td>33.25</td><td>0</td><td>0</td><td>8</td><td>0 .. 0</td><td>0 .. 0</td><td>3983213 .. 6869231</td><td>2434</td><td>1716</td><td>424</td><td>188</td><td>100</td><td>6</td><td>0.0</td><td>0.0</td><td>1.10044</td><td>0.0</td><td>0.0</td><td>23.5382</td><td>0.0</td><td>0.0</td><td>23.1585</td><td>0.498528</td><td>0.228609</td><td>0.370062</td><td>0.573473</td><td>1.60227</td><td>1.07931</td><td>0.603717</td><td>70 .. 12</td><td>0.918985 .. 0.99583</td><td>0.0917291</td><td>0.0563336</td><td>0.0120145</td><td>0.0045366</td></tr>\n",
       "<tr><td>1</td><td>0557p330</td><td>55.7851239669</td><td>33.0</td><td>0</td><td>0</td><td>6</td><td>0 .. 0</td><td>0 .. 0</td><td>3515900 .. 6151845</td><td>2308</td><td>1646</td><td>381</td><td>178</td><td>99</td><td>4</td><td>0.0</td><td>0.0</td><td>1.12777</td><td>0.0</td><td>0.0</td><td>23.5938</td><td>0.0</td><td>0.0</td><td>23.2322</td><td>0.528384</td><td>0.209271</td><td>0.348674</td><td>0.554691</td><td>1.69823</td><td>1.14395</td><td>0.639873</td><td>74 .. 12</td><td>0.914347 .. 0.995581</td><td>0.0972226</td><td>0.0597073</td><td>0.012734</td><td>0.00480827</td></tr>\n",
       "<tr><td>2</td><td>0557p335</td><td>55.7641196013</td><td>33.5</td><td>0</td><td>0</td><td>10</td><td>0 .. 0</td><td>0 .. 0</td><td>335689 .. 8918538</td><td>3409</td><td>2322</td><td>567</td><td>297</td><td>208</td><td>15</td><td>0.0</td><td>0.0</td><td>1.08877</td><td>0.0</td><td>0.0</td><td>23.6377</td><td>0.0</td><td>0.0</td><td>23.2558</td><td>0.55702</td><td>0.192263</td><td>0.329322</td><td>0.537254</td><td>1.79026</td><td>1.20595</td><td>0.674551</td><td>73 .. 12</td><td>0.90992 .. 0.995342</td><td>0.102492</td><td>0.0629433</td><td>0.0134242</td><td>0.0050689</td></tr>\n",
       "<tr><td>3</td><td>0559p332</td><td>55.9701492537</td><td>33.25</td><td>0</td><td>0</td><td>10</td><td>0 .. 0</td><td>0 .. 0</td><td>8565 .. 11524237</td><td>3843</td><td>2595</td><td>694</td><td>332</td><td>210</td><td>12</td><td>0.0</td><td>0.0</td><td>1.10383</td><td>0.0</td><td>0.0</td><td>23.5416</td><td>0.0</td><td>0.0</td><td>23.1777</td><td>0.567837</td><td>0.186204</td><td>0.322295</td><td>0.530811</td><td>1.82503</td><td>1.22937</td><td>0.687651</td><td>72 .. 13</td><td>0.908254 .. 0.995252</td><td>0.104482</td><td>0.0641656</td><td>0.0136849</td><td>0.00516734</td></tr>\n",
       "<tr><td>4</td><td>0560p330</td><td>56.0826446281</td><td>33.0</td><td>0</td><td>0</td><td>6</td><td>0 .. 0</td><td>0 .. 0</td><td>3395289 .. 6321674</td><td>2243</td><td>1566</td><td>416</td><td>159</td><td>98</td><td>4</td><td>0.0</td><td>0.0</td><td>1.15255</td><td>0.0</td><td>0.0</td><td>23.4639</td><td>0.0</td><td>0.0</td><td>23.1147</td><td>0.616166</td><td>0.161383</td><td>0.292685</td><td>0.502955</td><td>1.98036</td><td>1.334</td><td>0.746177</td><td>74 .. 12</td><td>0.900845 .. 0.994849</td><td>0.113374</td><td>0.0696267</td><td>0.0148496</td><td>0.00560712</td></tr>\n",
       "<tr><td>5</td><td>0560p335</td><td>56.0631229236</td><td>33.5</td><td>0</td><td>0</td><td>10</td><td>0 .. 0</td><td>0 .. 0</td><td>60892 .. 9782339</td><td>3390</td><td>2384</td><td>590</td><td>256</td><td>145</td><td>15</td><td>0.0</td><td>0.0</td><td>1.09524</td><td>0.0</td><td>0.0</td><td>23.6286</td><td>0.0</td><td>0.0</td><td>23.2468</td><td>0.559281</td><td>0.19098</td><td>0.327841</td><td>0.535901</td><td>1.79753</td><td>1.21084</td><td>0.677289</td><td>73 .. 13</td><td>0.909572 .. 0.995323</td><td>0.102908</td><td>0.0631987</td><td>0.0134787</td><td>0.00508944</td></tr>\n",
       "<tr><td>6</td><td>0562p332</td><td>56.2686567164</td><td>33.25</td><td>0</td><td>0</td><td>9</td><td>0 .. 0</td><td>0 .. 0</td><td>1228886 .. 9607509</td><td>3287</td><td>2354</td><td>528</td><td>270</td><td>128</td><td>7</td><td>0.0</td><td>0.0</td><td>1.13715</td><td>0.0</td><td>0.0</td><td>23.5552</td><td>0.0</td><td>0.0</td><td>23.2001</td><td>0.633924</td><td>0.153118</td><td>0.282502</td><td>0.493091</td><td>2.03743</td><td>1.37245</td><td>0.767682</td><td>74 .. 13</td><td>0.898138 .. 0.994701</td><td>0.116642</td><td>0.0716334</td><td>0.0152776</td><td>0.00576871</td></tr>\n",
       "<tr><td>7</td><td>0563p330</td><td>56.3801652893</td><td>33.0</td><td>0</td><td>0</td><td>0</td><td>0 .. 0</td><td>0 .. 0</td><td>7450967 .. 2821842</td><td>1156</td><td>820</td><td>187</td><td>96</td><td>46</td><td>7</td><td>0.0</td><td>0.0</td><td>1.17665</td><td>0.0</td><td>0.0</td><td>23.4711</td><td>0.0</td><td>0.0</td><td>23.132</td><td>0.647852</td><td>0.146934</td><td>0.274764</td><td>0.48549</td><td>2.0822</td><td>1.4026</td><td>0.784549</td><td>73 .. 13</td><td>0.896021 .. 0.994585</td><td>0.119205</td><td>0.0732073</td><td>0.0156133</td><td>0.00589546</td></tr>\n",
       "<tr><td>8</td><td>0563p335</td><td>56.3621262458</td><td>33.5</td><td>0</td><td>0</td><td>2</td><td>0 .. 0</td><td>0 .. 0</td><td>5186227 .. 4945981</td><td>1841</td><td>1402</td><td>244</td><td>119</td><td>71</td><td>5</td><td>0.0</td><td>0.0</td><td>1.12285</td><td>0.0</td><td>0.0</td><td>23.6427</td><td>0.0</td><td>0.0</td><td>23.2754</td><td>0.562955</td><td>0.188914</td><td>0.325448</td><td>0.533709</td><td>1.80934</td><td>1.2188</td><td>0.681738</td><td>74 .. 13</td><td>0.909006 .. 0.995293</td><td>0.103584</td><td>0.0636139</td><td>0.0135672</td><td>0.00512286</td></tr>\n",
       "<tr><td>9</td><td>0644p677</td><td>64.4727272727</td><td>67.75</td><td>0</td><td>1</td><td>0</td><td>0 .. 0</td><td>4711324 .. 0</td><td>0 .. 0</td><td>1044</td><td>925</td><td>66</td><td>19</td><td>30</td><td>4</td><td>0.0</td><td>1.40017</td><td>0.0</td><td>0.0</td><td>22.0005</td><td>0.0</td><td>0.0</td><td>21.3913</td><td>0.0</td><td>0.516102</td><td>0.21702</td><td>0.357319</td><td>0.562342</td><td>1.65875</td><td>1.11736</td><td>0.625</td><td>113 .. 16</td><td>0.916252 .. 0.995684</td><td>0.0949628</td><td>0.0583195</td><td>0.0124381</td><td>0.00469654</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table4545790136-347353').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table4545790136-347353').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr4_bricks[:10].show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#TODO: Maybe check for edge effects and add add extra leeway\n",
    "fields= [\n",
    "# Field                RAtyp    DECtyp   RAmin   RAmax   DECmin  DECmax\n",
    "['AKARI-NEP',          274.65,  65.796,  263.6,  266.5,  68.48,  69.51 ],\n",
    "['AKARI-SEP',          72.231, -54.380,   66.1,  75.5,  -55.96, -51.61 ],\n",
    "['Bootes',             216.43,  32.401,  215.7,  220.7,  32.12,  36.18 ],\n",
    "['CDFS-SWIRE',         51.022, -29.818,   50.7,  55.5,  -30.50, -25.92 ],\n",
    "['COSMOS',             149.29,  1.0821,  148.6,  151.6,  0.72,   3.71  ],\n",
    "['EGS',                217.27,  53.644,  212.3,  217.6,  51.08,  54.31 ],\n",
    "['ELAIS-N1',           247.09,  55.175,  237.9,  247.9,  52.37,  57.59 ],\n",
    "['ELAIS-N2',           248.42,  39.127,  246.1,  252.3,  39.02,  43.02 ],\n",
    "['ELAIS-S1',           7.1062, -43.863,    6.3,  11.3,  -45.60, -41.53 ],\n",
    "['GAMA-09',            129.07, -2.2317,  127.1,  142.2, -2.53,   3.55  ],\n",
    "['GAMA-12',            172.84,  -0.482,  172.2,  187.4, -3.54,   2.59  ],\n",
    "['GAMA-15',            211.75, -2.2857,  209.9,  225.3, -2.57,   3.5   ],\n",
    "['HDF-N',              190.25,  62.205,  188.0,  190.4,  61.69,  62.78 ],\n",
    "['Herschel-Stripe-82', 353.75, -7.1089,  348.3,  19.1,  -9.5,    9.25  ],\n",
    "['Lockman-SWIRE',      161.94,  59.056,  154.7,  167.8,  54.95,  60.89 ],\n",
    "['NGP',                192.89,  22.099,  189.8,  209.3,  21.62,  36.16 ],\n",
    "['SA13',               197.89,  42.440,  197.5,  198.6,  42.34,  43.09 ],\n",
    "['SGP',                334.29, -34.503,  333.5,  28.2,  -37.1,  -26.0  ],\n",
    "['SPIRE-NEP',          266.33,  68.790,  264.6,  275.4,  64.41,  68.63 ],\n",
    "['SSDF',               341.57, -59.186,  340.5,  2.8,   -61.2,  -48.6  ],\n",
    "['xFLS',               261.38,  58.018,  255.6,  262.5,  57.85,  60.84 ],\n",
    "['XMM-13hr',           203.31,  37.474,  202.8,  204.4,  37.31,  38.52 ],\n",
    "['XMM-LSS',            32.941, -6.0229,   32.1,  38.2,  -7.53,  -1.52  ]]\n",
    "   \n",
    "    \n",
    "#Has a field been removed?  ( RA BETWEEN 13.4 AND 36.3 AND DEC BETWEEN -2.32 AND 2.49 )\n",
    "       \n",
    "    \n",
    "    \n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "help_moc = MOC()\n",
    "field_mocs = []\n",
    "for field in fields:\n",
    "    help_moc += MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0])) \n",
    "    field_mocs.append(MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0])))\n",
    "help_moc.write(\"../../dmu2/help_coverage_MOC.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "help_mask = inMoc(dr4_bricks['ra'],\n",
    "                  dr4_bricks['dec'],\n",
    "                  MOC(filename=\"../../dmu2/help_coverage_MOC.fits\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1505 bricks in Legacy Survey DR4 in HELP fields\n",
      "There are 0 bricks in Legacy Survey DR4 for AKARI-NEP\n",
      "There are 0 bricks in Legacy Survey DR4 for AKARI-SEP\n",
      "There are 175 bricks in Legacy Survey DR4 for Bootes\n",
      "There are 0 bricks in Legacy Survey DR4 for CDFS-SWIRE\n",
      "There are 0 bricks in Legacy Survey DR4 for COSMOS\n",
      "There are 57 bricks in Legacy Survey DR4 for EGS\n",
      "There are 215 bricks in Legacy Survey DR4 for ELAIS-N1\n",
      "There are 147 bricks in Legacy Survey DR4 for ELAIS-N2\n",
      "There are 0 bricks in Legacy Survey DR4 for ELAIS-S1\n",
      "There are 0 bricks in Legacy Survey DR4 for GAMA-09\n",
      "There are 0 bricks in Legacy Survey DR4 for GAMA-12\n",
      "There are 0 bricks in Legacy Survey DR4 for GAMA-15\n",
      "There are 0 bricks in Legacy Survey DR4 for HDF-N\n",
      "There are 0 bricks in Legacy Survey DR4 for Herschel-Stripe-82\n",
      "There are 182 bricks in Legacy Survey DR4 for Lockman-SWIRE\n",
      "There are 618 bricks in Legacy Survey DR4 for NGP\n",
      "There are 5 bricks in Legacy Survey DR4 for SA13\n",
      "There are 0 bricks in Legacy Survey DR4 for SGP\n",
      "There are 0 bricks in Legacy Survey DR4 for SPIRE-NEP\n",
      "There are 0 bricks in Legacy Survey DR4 for SSDF\n",
      "There are 93 bricks in Legacy Survey DR4 for xFLS\n",
      "There are 13 bricks in Legacy Survey DR4 for XMM-13hr\n",
      "There are 0 bricks in Legacy Survey DR4 for XMM-LSS\n"
     ]
    }
   ],
   "source": [
    "print('There are ' + str(len(dr4_bricks[help_mask])) + ' bricks in Legacy Survey DR4 in HELP fields')\n",
    "help_bricks_file = open(\"./data/help_bricks.lis\", \"w\")\n",
    "help_downloads_file = open(\"./data/help_bricks_downloads.lis\", \"w\")\n",
    "field_coverages = {}\n",
    "\n",
    "for field in fields:\n",
    "    \n",
    "    #TODO: creplace inMoc with new function nearMoc to avoid any edge effects\n",
    "    field_mask = inMoc(dr4_bricks['ra'],\n",
    "                       dr4_bricks['dec'],\n",
    "                       MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0])))\n",
    "    print('There are ' + str(len(dr4_bricks[field_mask])) + ' bricks in Legacy Survey DR4 for ' + field[0])\n",
    "    field_coverages[field[0]] = len(dr4_bricks[field_mask])\n",
    "    field_bricks_file  = open(\"./data/{}_bricks.lis\".format(field[0]), \"w\")\n",
    "    field_downloads_file  = open(\"./data/{}_bricks_downloads.lis\".format(field[0]), \"w\")\n",
    "    for brick in dr4_bricks[field_mask]:\n",
    "        \n",
    "        brick_string = (\"public/hlsp/ls/dr4/tractor/\" \n",
    "                      + brick['brickname'][:3] \n",
    "                      + '/tractor-' \n",
    "                      + brick['brickname'] \n",
    "                      + '.fits\\n')\n",
    "        field_bricks_file.write(brick_string)\n",
    "        field_downloads_file.write(\"ftp://archive.noao.edu/\" + brick_string)\n",
    "        help_bricks_file.write(brick_string)\n",
    "        help_downloads_file.write(\"ftp://archive.noao.edu/\" + brick_string)\n",
    "    field_bricks_file.close()\n",
    "    field_downloads_file.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the RA/DEC limits\n",
    "Because of edge effects it is better to use the conservative ra/dec bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print('There are ' + str(len(dr4_bricks[help_mask])) + ' bricks in Legacy Survey DR4 in HELP fields')\n",
    "help_bricks_file = open(\"./data/help_bricks.lis\", \"w\")\n",
    "help_downloads_file = open(\"./data/help_bricks_downloads.lis\", \"w\")\n",
    "field_coverages = {}\n",
    "\n",
    "def inRectangle(RAmin, RAmax, DECmin, DECmax, ra, dec):\n",
    "    if RAmin < RAmax:\n",
    "        if ra > RAmin and ra < RAmax and dec > DECmin and dec < DECmax:\n",
    "            output = True\n",
    "        else:\n",
    "            output = False\n",
    "    elif RAmin > RAmax:\n",
    "        if (ra > RAmin or ra < RAmax) and (dec > DECmin and dec < DECmax):\n",
    "            output = True\n",
    "        else:\n",
    "            output = False\n",
    "        \n",
    "    return output\n",
    "        \n",
    "\n",
    "for field in fields:\n",
    "    \n",
    "    #TODO: creplace inMoc with new function nearMoc to avoid any edge effects\n",
    "    field_mask = inMoc(dr4_bricks['ra'],\n",
    "                       dr4_bricks['dec'],\n",
    "                       MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0])))\n",
    "    print('There are ' + str(len(dr4_bricks[field_mask])) + ' bricks in Legacy Survey DR4 for ' + field[0])\n",
    "    field_coverages[field[0]] = len(dr4_bricks[field_mask])\n",
    "    field_bricks_file  = open(\"./data/{}_bricks_wider.lis\".format(field[0]), \"w\")\n",
    "    field_downloads_file  = open(\"./data/{}_bricks_downloads_wider.lis\".format(field[0]), \"w\")\n",
    "    for brick in dr4_bricks:\n",
    "        if inRectangle(field[3], field[4], field[5], field[6], brick['ra'], brick['dec'])\n",
    "        \n",
    "            brick_string = (\"public/hlsp/ls/dr4/tractor/\" \n",
    "                      + brick['brickname'][:3] \n",
    "                      + '/tractor-' \n",
    "                      + brick['brickname'] \n",
    "                      + '.fits\\n')\n",
    "            field_bricks_file.write(brick_string)\n",
    "            field_downloads_file.write(\"ftp://archive.noao.edu/\" + brick_string)\n",
    "            help_bricks_file.write(brick_string)\n",
    "            help_downloads_file.write(\"ftp://archive.noao.edu/\" + brick_string)\n",
    "    field_bricks_file.close()\n",
    "    field_downloads_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparison to DECaLS DR3\n",
    "We were originally confused about the relation between Legacy Survey Dr4 and DECaLS DR3 and wanted to compare coverages. It seems that they only marginally overlap and that DECaLS DR3 is officially included in Legacy Survey DR4 but that the actual data provided is only from BASS and MzLS."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 bricks in AKARI-NEP\n",
      "There are 0 bricks in AKARI-SEP\n",
      "There are 152 bricks in Bootes\n",
      "There are 0 bricks in CDFS-SWIRE\n",
      "There are 83 bricks in COSMOS\n",
      "There are 0 bricks in EGS\n",
      "There are 0 bricks in ELAIS-N1\n",
      "There are 0 bricks in ELAIS-N2\n",
      "There are 0 bricks in ELAIS-S1\n",
      "There are 966 bricks in GAMA-09\n",
      "There are 1006 bricks in GAMA-12\n",
      "There are 986 bricks in GAMA-15\n",
      "There are 0 bricks in HDF-N\n",
      "There are 4908 bricks in Herschel-Stripe-82\n",
      "There are 0 bricks in Lockman-SWIRE\n",
      "There are 2591 bricks in NGP\n",
      "There are 0 bricks in SA13\n",
      "There are 0 bricks in SGP\n",
      "There are 0 bricks in SPIRE-NEP\n",
      "There are 0 bricks in SSDF\n",
      "There are 0 bricks in xFLS\n",
      "There are 0 bricks in XMM-13hr\n",
      "There are 349 bricks in XMM-LSS\n"
     ]
    }
   ],
   "source": [
    "for field in fields:\n",
    "    field_mask = inMoc(dr3_bricks['ra'],\n",
    "                       dr3_bricks['dec'],\n",
    "                       MOC(filename=\"../../dmu2/dmu2_field_coverages/{}_MOC.fits\".format(field[0])))\n",
    "    print('There are ' + str(len(dr3_bricks[field_mask])) + ' bricks in DECaLS DR3 for ' + field[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Download only the bricks for help fields\n",
    "#for field in fields\n",
    "#    subprocess.call(['wget', \n",
    "#                    '-irx', '{}_bricks.lis'.format(field[0]), '-P', './data'], \n",
    "#                    stdout=log_file)\n",
    "\n",
    "#Download just the files with HELP coverage\n",
    "subprocess.call(['wget', \n",
    "                    '-irx', './data/help_bricks_downloads.lis', '-P', './data'], \n",
    "                    stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Concatenate all the fits files into one fits file\n",
    "for field in fields:\n",
    "    if field_coverages[field[0]] == 0:\n",
    "        continue\n",
    "    #First creat the overall fits files by concatenating all the relevant tables\n",
    "    subprocess.call(['stilts', \n",
    "                    'tcat', 'in=@{}_bricks.lis'.format(field[0]), 'out=LegacySurvey-dr4_{}.fits'.format(field[0])], \n",
    "                    stdout=log_file)\n",
    "    \n",
    "    #TODO: Remove all the sources which are not in HELP (Due to edge effects where part of the bricks go outside)\n",
    "    #Label the points by Field\n",
    "    #subprocess.call(['stilts', \n",
    "    #                'addcol', \n",
    "    #                'Field',\n",
    "    #                '\\\"inMoc(\\\"../../dmu2/field_coverages/{}_MOC.fits\\\") ?\\\"COSMOS\\\" : \\\"NOFIELD\\\"\\\"'], \n",
    "    #                stdout=log_file)\n",
    "    # \n",
    "    #Remove the points with \"NOFIELD\"\n",
    "    #subprocess.call(['stilts', \n",
    "    #                'select', \n",
    "    #                 '\\\"Field',\n",
    "    #                 '!=',\n",
    "    #                 '\\\"NOFIELD\\\"\\\"'\n",
    "    #                ], \n",
    "    #                stdout=log_file)\n",
    "    \n",
    "    # Then generate the MOCs\n",
    "    subprocess.call(['stilts', \n",
    "                    'pixfoot', \n",
    "                     'in=LegacySurvey-dr4_{}.fits'.format(field[0]), \n",
    "                     'out=LegacySurvey-dr4_{}_MOC.fits'.format(field[0]),\n",
    "                     'order=12' #Order 12 seems to avoid any holes\n",
    "                     ],  \n",
    "                    stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: UnitsWarning: '1/deg^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]\n",
      "WARNING: UnitsWarning: 'nanomaggy' did not parse as fits unit: At col 0, Unit 'nanomaggy' not supported by the FITS standard.  [astropy.units.core]\n",
      "WARNING: UnitsWarning: '1/nanomaggy^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]\n",
      "WARNING: UnitsWarning: '1/arcsec^2' did not parse as fits unit: Numeric factor not supported by FITS [astropy.units.core]\n"
     ]
    }
   ],
   "source": [
    "#Can also be done with astropy\n",
    "for field in fields:\n",
    "    if field_coverages[field[0]] == 0:\n",
    "        continue\n",
    "    field_catalogue = Table.read('./data/LegacySurvey-dr4_{}.fits'.format(field[0]))\n",
    "    mask = inMoc(field_catalogue['ra'],\n",
    "                 field_catalogue['dec'],\n",
    "                 MOC(filename='../../dmu2/dmu2_field_coverages/{}_MOC.fits'.format(field[0])))\n",
    "    field_catalogue.add_column(Column(data=field[0], name=\"Field\")) #Does it work without providing an array\n",
    "    field_catalogue[mask].write('./data/LegacySurvey-dr4_{}_help_only.fits'.format(field[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Alternative method\n",
    "This descibes an alternate method using the filter and tag shell script. This downloads everything and then tags it. It doesn't suffer from any edge effects but requires very large memory resources (both RAM and storage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Download all the files (Total ~270 Gb)\n",
    "subprocess.call(['wget', \n",
    "                 '-r', \n",
    "                 'ftp://archive.noao.edu/public/hlsp/ls/dr4/tractor/', \n",
    "                 '-P', \n",
    "                 './data'], \n",
    "                 stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Create a list of all the fits files \n",
    "subprocess.call(['ls', \n",
    "                 './data/archive.noao.edu/public/hlsp/ls/dr4/tractor/*/*.fits', \n",
    "                 '>', \n",
    "                 '.data/tractor_files.lis'], \n",
    "                    stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "subprocess.call(['stilts', \n",
    "                 'tcat', \n",
    "                 'in=@tractor_files.lis'.format(field[0]), \n",
    "                 'out=LegacySurvey-dr4.fits'.format(field[0])], \n",
    "                 stdout=log_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Use the tag script to filter in to HELP fields\n",
    "subprocess.call(['./filterAndTag.sh', \n",
    "                 'LegacySurvey_dr4.fits',\n",
    "                 'LegacySurvey_dr4_HELP-coverage.fits'], \n",
    "                 stdout=log_file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:herschelhelp_internal]",
   "language": "python",
   "name": "conda-env-herschelhelp_internal-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}