{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Masterlist overview\n",
    "\n",
    "This notebook summarises some important properties of the data presented here. This includes\n",
    "\n",
    "- The area covered by each field\n",
    "- The number of total sources in each field\n",
    "- The number of bands which we have data for in each field\n",
    "- The size of the masterlist in each field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/matplotlib/__init__.py:855: MatplotlibDeprecationWarning: \n",
      "examples.directory is deprecated; in the future, examples will be found relative to the 'datapath' directory.\n",
      "  \"found relative to the 'datapath' directory.\".format(key))\n",
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/matplotlib/__init__.py:846: MatplotlibDeprecationWarning: \n",
      "The text.latex.unicode rcparam was deprecated in Matplotlib 2.2 and will be removed in 3.1.\n",
      "  \"2.2\", name=key, obj_type=\"rcparam\", addendum=addendum)\n",
      "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/seaborn/apionly.py:9: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "from astropy.table import Table, Column\n",
    "from astropy import units as u\n",
    "import numpy as np\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.masterlist import find_last_ml_suffix\n",
    "\n",
    "import os\n",
    "import humanfriendly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The definition of HELP DR1\n",
    "Here we define DR1, which should be considered the principle definition in terms of the dated masterlist file that defines the object list on a given field. In some cases it is different to the masterlist used by an intermediate step because it was in progress as that step began. Nevertheless these files define DR1 so any discrepencies will force these objects lists through. The most likely upshot if this is that there may be objects in the dr1 masterlist that were not used in further steps and therefore lack, photozs, fir fluxes, or Cigale fits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1 = [\n",
    "# Field                suffix\n",
    "['AKARI-NEP',          '20180215' ],\n",
    "['AKARI-SEP',          '20180221' ],\n",
    "['Bootes',             '20190201' ], \n",
    "['CDFS-SWIRE',         '20180613' ],\n",
    "['COSMOS',             '20190402' ],\n",
    "['EGS',                '20180501' ],\n",
    "['ELAIS-N1',           '20171016' ],\n",
    "['ELAIS-N2',           '20180218' ],\n",
    "['ELAIS-S1',           '20180416' ],\n",
    "['GAMA-09',            '20180601' ],\n",
    "['GAMA-12',            '20180218' ],\n",
    "['GAMA-15',            '20180213' ],\n",
    "['HDF-N',              '20180427' ],\n",
    "['Herschel-Stripe-82', '20180307' ],\n",
    "['Lockman-SWIRE',      '20180219' ],\n",
    "['NGP',                '20180219' ],\n",
    "['SA13',               '20180501' ],\n",
    "['SGP',                '20180221' ],\n",
    "['SPIRE-NEP',          '20180220' ],\n",
    "['SSDF',               '20180221' ],\n",
    "['xFLS',               '20180501' ],\n",
    "['XMM-13hr',           '20180501' ],\n",
    "['XMM-LSS',            '20190328' ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "masterlist_summary = Table()\n",
    "masterlist_summary.add_column(Column(data = np.array(dr1)[:,0], name = 'field'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), '', dtype=\"S20\"), name = 'dr1_suffix'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), '', dtype=\"S20\"), name = 'last_suffix'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), 0, dtype=int), name = 'objects'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), np.nan, dtype=float), name = 'area_sq_degrees'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), 0, dtype=int), name = 'file_size_bytes'))\n",
    "masterlist_summary.add_column(Column(data =np.full(len(masterlist_summary), '', dtype=\"S20\"), name = 'file_size_readable'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total HELP area: 1269.6205101879136 sqare degrees\n"
     ]
    }
   ],
   "source": [
    "help_moc = MOC(filename='../dmu2/help_coverage_MOC.fits')\n",
    "print('Total HELP area: {} sqare degrees'.format(help_moc.area_sq_deg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for field in dr1:\n",
    "    number_objects = len(Table.read('dmu1_ml_{}/data/master_list_cross_ident_{}_{}.fits'.format(field[0], \n",
    "                                                                                      field[0].lower(), \n",
    "                                                                                      field[1])))\n",
    "    masterlist_summary['objects'][masterlist_summary['field'] == field[0]] = number_objects\n",
    "    \n",
    "    \n",
    "    moc = MOC(filename='../dmu2/dmu2_field_coverages/{}_MOC.fits'.format(field[0]))\n",
    "    area = moc.area_sq_deg\n",
    "    masterlist_summary['area_sq_degrees'][masterlist_summary['field'] == field[0]] = area\n",
    "    \n",
    "    masterlist_summary['dr1_suffix'][masterlist_summary['field'] == field[0]] = field[1]\n",
    "    masterlist_summary['last_suffix'][masterlist_summary['field'] == field[0]] = find_last_ml_suffix('dmu1_ml_{}/data/'.format(field[0]))\n",
    "    \n",
    "    size = os.stat('dmu1_ml_{}/data/master_catalogue_{}_{}.fits'.format(field[0], \n",
    "                                                                                      field[0].lower(), \n",
    "                                                                                      field[1])).st_size\n",
    "\n",
    "    \n",
    "    \n",
    "    #Can't install packages on Vulcain\n",
    "    #size_readable = humanfriendly.format_size(size)\n",
    "    masterlist_summary['file_size_bytes'][masterlist_summary['field'] == field[0]] = size\n",
    "    masterlist_summary['file_size_readable'][masterlist_summary['field'] == field[0]] = humanfriendly.format_size(size)\n",
    "    \n",
    "\n",
    "    \n",
    "    #TODO:\n",
    "    num_surveys = 0.\n",
    "    \n",
    "    #Mean depth in g\n",
    "    mean_depth = 0.\n",
    "    \n",
    "    num_bands = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=23</i>\n",
       "<table id=\"table140479636728128-344524\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>field</th><th>dr1_suffix</th><th>last_suffix</th><th>objects</th><th>area_sq_degrees</th><th>file_size_bytes</th><th>file_size_readable</th></tr></thead>\n",
       "<tr><td>0</td><td>AKARI-NEP</td><td>20180215</td><td>20180215</td><td>531746</td><td>9.194732358779467</td><td>348848640</td><td>348.85 MB</td></tr>\n",
       "<tr><td>1</td><td>AKARI-SEP</td><td>20180221</td><td>20180221</td><td>844172</td><td>8.713306475131118</td><td>538606080</td><td>538.61 MB</td></tr>\n",
       "<tr><td>2</td><td>Bootes</td><td>20180520</td><td>20190201</td><td>3481661</td><td>11.42815299095486</td><td>5002044480</td><td>5 GB</td></tr>\n",
       "<tr><td>3</td><td>CDFS-SWIRE</td><td>20180613</td><td>20180613</td><td>2171051</td><td>12.971246403717068</td><td>5705593920</td><td>5.71 GB</td></tr>\n",
       "<tr><td>4</td><td>COSMOS</td><td>20190402</td><td>20190402</td><td>2599374</td><td>5.083863478496816</td><td>10517175360</td><td>10.52 GB</td></tr>\n",
       "<tr><td>5</td><td>EGS</td><td>20180501</td><td>20180501</td><td>1412613</td><td>3.566383275122158</td><td>3808480320</td><td>3.81 GB</td></tr>\n",
       "<tr><td>6</td><td>ELAIS-N1</td><td>20171016</td><td>20180216</td><td>4026292</td><td>13.507484555454763</td><td>5242279680</td><td>5.24 GB</td></tr>\n",
       "<tr><td>7</td><td>ELAIS-N2</td><td>20180218</td><td>20180218</td><td>1783240</td><td>9.167479903991113</td><td>2007964800</td><td>2.01 GB</td></tr>\n",
       "<tr><td>8</td><td>ELAIS-S1</td><td>20180416</td><td>20180416</td><td>1655564</td><td>9.002940646885508</td><td>1970156160</td><td>1.97 GB</td></tr>\n",
       "<tr><td>9</td><td>GAMA-09</td><td>20180601</td><td>20190123</td><td>12937982</td><td>62.01393417284915</td><td>19911611520</td><td>19.91 GB</td></tr>\n",
       "<tr><td>10</td><td>GAMA-12</td><td>20180218</td><td>20190128</td><td>12369415</td><td>62.70933280772107</td><td>14633066880</td><td>14.63 GB</td></tr>\n",
       "<tr><td>11</td><td>GAMA-15</td><td>20180213</td><td>20190201</td><td>14232880</td><td>61.70114565980842</td><td>16595585280</td><td>16.6 GB</td></tr>\n",
       "<tr><td>12</td><td>HDF-N</td><td>20180427</td><td>20180427</td><td>130679</td><td>0.6717832558166041</td><td>185996160</td><td>186 MB</td></tr>\n",
       "<tr><td>13</td><td>Herschel-Stripe-82</td><td>20180307</td><td>20190205</td><td>50196455</td><td>363.22747930641884</td><td>127047306240</td><td>127.05 GB</td></tr>\n",
       "<tr><td>14</td><td>Lockman-SWIRE</td><td>20180219</td><td>20180219</td><td>4366298</td><td>22.413863402949218</td><td>5195931840</td><td>5.2 GB</td></tr>\n",
       "<tr><td>15</td><td>NGP</td><td>20180219</td><td>20190204</td><td>6759591</td><td>177.6981971077162</td><td>4954806720</td><td>4.95 GB</td></tr>\n",
       "<tr><td>16</td><td>SA13</td><td>20180501</td><td>20180501</td><td>9799</td><td>0.27416379328431184</td><td>2721600</td><td>2.72 MB</td></tr>\n",
       "<tr><td>17</td><td>SGP</td><td>20180221</td><td>20180221</td><td>29790690</td><td>294.5684540900642</td><td>45192530880</td><td>45.19 GB</td></tr>\n",
       "<tr><td>18</td><td>SPIRE-NEP</td><td>20180220</td><td>20180220</td><td>2674</td><td>0.1280660469377391</td><td>1203840</td><td>1.2 MB</td></tr>\n",
       "<tr><td>19</td><td>SSDF</td><td>20180221</td><td>20180221</td><td>12661903</td><td>111.11624949343539</td><td>8078319360</td><td>8.08 GB</td></tr>\n",
       "<tr><td>20</td><td>xFLS</td><td>20180501</td><td>20180501</td><td>977148</td><td>7.44043242140754</td><td>1050474240</td><td>1.05 GB</td></tr>\n",
       "<tr><td>21</td><td>XMM-13hr</td><td>20180501</td><td>20180501</td><td>38629</td><td>0.7570240166583633</td><td>10676160</td><td>10.68 MB</td></tr>\n",
       "<tr><td>22</td><td>XMM-LSS</td><td>20190328</td><td>20190328</td><td>8705837</td><td>21.75457939331374</td><td>28241838720</td><td>28.24 GB</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table140479636728128-344524').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table140479636728128-344524').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 4, 5, 6], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "masterlist_summary.show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of objects in HELP: 171685693\n"
     ]
    }
   ],
   "source": [
    "print('Total number of objects in HELP: {}'.format(np.sum(masterlist_summary['objects'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total size of masterlist files: 306.24 GB\n"
     ]
    }
   ],
   "source": [
    "print('Total size of masterlist files: {}'.format(humanfriendly.format_size(np.sum(masterlist_summary['file_size_bytes']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "masterlist_summary.write('dr1_overview.fits', overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}