{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final stage of HELP data processing\n",
    "\n",
    "This notebook collates the final output files ready for writing to csv for ingestion to a VO server. At the bottom of the notebook we also summarise the pipeline products which are processed on a given field. This are generated using the dmu32 meta_main.yml files which contain links to the XID+, CIGALE and photo-z catalogues which feed in to the final catalogues for publishing.\n",
    "\n",
    "Summary of notebook:\n",
    "\n",
    "- Take DR1 masterlist suffixes from overview table\n",
    "- Find dmu32 full table names and write to a file\n",
    "- Create summary of all the data products per field using the dmu32 meta_main.yml files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This notebook was run with herschelhelp_internal version: \n",
      "017bb1e (Mon Jun 18 14:58:59 2018 +0100)\n",
      "This notebook was executed on: \n",
      "2018-07-12 17:19:01.808314\n"
     ]
    }
   ],
   "source": [
    "from  herschelhelp_internal  import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n",
    "import datetime\n",
    "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/seaborn/apionly.py:6: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "from astropy.table import Table, Column\n",
    "from astropy import units as u\n",
    "import numpy as np\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.masterlist import find_last_ml_suffix\n",
    "\n",
    "import yaml\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "import humanfriendly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TODAY = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The definition of HELP DR1\n",
    "Here we take the DR1 definition from dmu1. We then propagate those suffixes through to dmu32. Note that this does not include a specification of how the masterlist made its way through the whole HELP process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1 = Table.read(\"../dmu1/dr1_overview.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=23</i>\n",
       "<table id=\"table139627909783448-786296\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>field</th><th>dr1_suffix</th><th>last_suffix</th><th>objects</th><th>area_sq_degrees</th><th>file_size_bytes</th><th>file_size_readable</th></tr></thead>\n",
       "<tr><td>0</td><td>AKARI-NEP</td><td>20180215</td><td>20180215</td><td>531746</td><td>9.194732358779467</td><td>348848640</td><td>348.85 MB</td></tr>\n",
       "<tr><td>1</td><td>AKARI-SEP</td><td>20180221</td><td>20180221</td><td>844172</td><td>8.713306475131118</td><td>538606080</td><td>538.61 MB</td></tr>\n",
       "<tr><td>2</td><td>Bootes</td><td>20180520</td><td>20180520</td><td>3481661</td><td>11.42815299095486</td><td>5002044480</td><td>5 GB</td></tr>\n",
       "<tr><td>3</td><td>CDFS-SWIRE</td><td>20180613</td><td>20180613</td><td>2171051</td><td>12.971246403717068</td><td>5705593920</td><td>5.71 GB</td></tr>\n",
       "<tr><td>4</td><td>COSMOS</td><td>20180619</td><td>20180619</td><td>2599374</td><td>5.083863478496816</td><td>10517175360</td><td>10.52 GB</td></tr>\n",
       "<tr><td>5</td><td>EGS</td><td>20180501</td><td>20180501</td><td>1412613</td><td>3.566383275122158</td><td>3808480320</td><td>3.81 GB</td></tr>\n",
       "<tr><td>6</td><td>ELAIS-N1</td><td>20171016</td><td>20180216</td><td>4026292</td><td>13.507484555454763</td><td>5242279680</td><td>5.24 GB</td></tr>\n",
       "<tr><td>7</td><td>ELAIS-N2</td><td>20180218</td><td>20180218</td><td>1783240</td><td>9.167479903991113</td><td>2007964800</td><td>2.01 GB</td></tr>\n",
       "<tr><td>8</td><td>ELAIS-S1</td><td>20180416</td><td>20180416</td><td>1655564</td><td>9.002940646885508</td><td>1970156160</td><td>1.97 GB</td></tr>\n",
       "<tr><td>9</td><td>GAMA-09</td><td>20180601</td><td>20180601</td><td>12937982</td><td>62.01393417284915</td><td>19911611520</td><td>19.91 GB</td></tr>\n",
       "<tr><td>10</td><td>GAMA-12</td><td>20180218</td><td>20180218</td><td>12369415</td><td>62.70933280772107</td><td>14633066880</td><td>14.63 GB</td></tr>\n",
       "<tr><td>11</td><td>GAMA-15</td><td>20180213</td><td>20180213</td><td>14232880</td><td>61.70114565980842</td><td>16595585280</td><td>16.6 GB</td></tr>\n",
       "<tr><td>12</td><td>HDF-N</td><td>20180427</td><td>20180427</td><td>130679</td><td>0.6717832558166041</td><td>185996160</td><td>186 MB</td></tr>\n",
       "<tr><td>13</td><td>Herschel-Stripe-82</td><td>20180307</td><td>20180307</td><td>50196455</td><td>363.22747930641884</td><td>127047306240</td><td>127.05 GB</td></tr>\n",
       "<tr><td>14</td><td>Lockman-SWIRE</td><td>20180219</td><td>20180219</td><td>4366298</td><td>22.413863402949218</td><td>5195931840</td><td>5.2 GB</td></tr>\n",
       "<tr><td>15</td><td>NGP</td><td>20180219</td><td>20180501</td><td>6759591</td><td>177.6981971077162</td><td>4954806720</td><td>4.95 GB</td></tr>\n",
       "<tr><td>16</td><td>SA13</td><td>20180501</td><td>20180501</td><td>9799</td><td>0.27416379328431184</td><td>2721600</td><td>2.72 MB</td></tr>\n",
       "<tr><td>17</td><td>SGP</td><td>20180221</td><td>20180221</td><td>29790690</td><td>294.5684540900642</td><td>45192530880</td><td>45.19 GB</td></tr>\n",
       "<tr><td>18</td><td>SPIRE-NEP</td><td>20180220</td><td>20180220</td><td>2674</td><td>0.1280660469377391</td><td>1203840</td><td>1.2 MB</td></tr>\n",
       "<tr><td>19</td><td>SSDF</td><td>20180221</td><td>20180221</td><td>12661903</td><td>111.11624949343539</td><td>8078319360</td><td>8.08 GB</td></tr>\n",
       "<tr><td>20</td><td>xFLS</td><td>20180501</td><td>20180501</td><td>977148</td><td>7.44043242140754</td><td>1050474240</td><td>1.05 GB</td></tr>\n",
       "<tr><td>21</td><td>XMM-13hr</td><td>20180501</td><td>20180501</td><td>38629</td><td>0.7570240166583633</td><td>10676160</td><td>10.68 MB</td></tr>\n",
       "<tr><td>22</td><td>XMM-LSS</td><td>20180504</td><td>20180504</td><td>8705837</td><td>21.75457939331374</td><td>28241838720</td><td>28.24 GB</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table139627909783448-786296').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table139627909783448-786296').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 4, 5, 6], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr1.show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./dmu32_AKARI-NEP/data/AKARI-NEP_20180215_cigale.fits\n",
      "./dmu32_AKARI-SEP/data/AKARI-SEP_20180221_cigale.fits\n",
      "./dmu32_Bootes/data/Bootes_20180520_cigale.fits\n",
      "./dmu32_CDFS-SWIRE/data/CDFS-SWIRE_20180613.fits\n",
      "./dmu32_COSMOS/data/COSMOS_20180619.fits\n",
      "./dmu32_EGS/data/EGS_20180501_cigale.fits\n",
      "./dmu32_ELAIS-N1/data/ELAIS-N1_20171016.fits\n",
      "./dmu32_ELAIS-N2/data/ELAIS-N2_20180218_cigale.fits\n",
      "./dmu32_ELAIS-S1/data/ELAIS-S1_20180416.fits\n",
      "./dmu32_GAMA-09/data/GAMA-09_20180601.fits\n",
      "./dmu32_GAMA-12/data/GAMA-12_20180218.fits\n",
      "./dmu32_GAMA-15/data/GAMA-15_20180213.fits\n",
      "./dmu32_HDF-N/data/HDF-N_20180427_cigale.fits\n",
      "./dmu32_Herschel-Stripe-82/data/Herschel-Stripe-82_20180307_cigale.fits\n",
      "./dmu32_Lockman-SWIRE/data/Lockman-SWIRE_20180219.fits\n",
      "./dmu32_NGP/data/NGP_20180219_cigale.fits\n",
      "./dmu32_SA13/data/SA13_20180501_cigale.fits\n",
      "./dmu32_SGP/data/SGP_20180221.fits\n",
      "./dmu32_SPIRE-NEP/data/SPIRE-NEP_20180220_cigale.fits\n",
      "./dmu32_SSDF/data/SSDF_20180221_cigale.fits\n",
      "./dmu32_xFLS/data/xFLS_20180501_cigale.fits\n",
      "./dmu32_XMM-13hr/data/XMM-13hr_20180501_cigale.fits\n",
      "./dmu32_XMM-LSS/data/XMM-LSS_20180504_cigale.fits\n"
     ]
    }
   ],
   "source": [
    "GAVO_FOLDER = '/mnt/hedam/data_vo/'\n",
    "stilts_command = 'stilts tpipe {in_file} omode=out ofmt=csv out={GAVO_FOLDER}{out_file}'\n",
    "\n",
    "final_data = open('help_to_vo.sh', 'w+')\n",
    "for field in dr1:\n",
    "    final_help_product = './dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1])\n",
    "    cigale_input = './dmu32_{}/data/{}_{}_cigale.fits'.format(field[0], field[0], field[1])\n",
    "\n",
    "    if os.path.exists(final_help_product):\n",
    "        print(final_help_product)\n",
    "    \n",
    "        #Test with Cigale input files\n",
    "        final_data.write(stilts_command.format(in_file=final_help_product, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='herschelhelp/main/{}_{}_all.csv \\n'.format(field[0], \n",
    "                                                                                                field[1])))\n",
    "    elif os.path.exists(cigale_input):\n",
    "        print(cigale_input)\n",
    "        final_data.write(stilts_command.format(in_file=cigale_input, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='herschelhelp/main/{}_{}_incomplete.csv \\n'.format(field[0], \n",
    "                                                                                                       field[1])))\n",
    "        \n",
    "    else:\n",
    "        final_data.write('# No data for {} \\n'.format(field[0]))\n",
    "        \n",
    "    #final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))\n",
    "    \n",
    "final_data.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The out put of this notebook is a shell script which will write all the fits files to csv files in the vo folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../dmu1/dmu1_ml_AKARI-NEP/data/depths_akari-nep_20180215.fits\n",
      "../dmu1/dmu1_ml_AKARI-SEP/data/depths_akari-sep_20180221.fits\n",
      "../dmu1/dmu1_ml_Bootes/data/depths_bootes_20180520.fits\n",
      "../dmu1/dmu1_ml_CDFS-SWIRE/data/depths_cdfs-swire_20180613.fits\n",
      "../dmu1/dmu1_ml_COSMOS/data/depths_cosmos_20180619.fits\n",
      "../dmu1/dmu1_ml_EGS/data/depths_egs_20180501.fits\n",
      "../dmu1/dmu1_ml_ELAIS-N1/data/depths_elais-n1_20171016.fits\n",
      "../dmu1/dmu1_ml_ELAIS-N2/data/depths_elais-n2_20180218.fits\n",
      "../dmu1/dmu1_ml_ELAIS-S1/data/depths_elais-s1_20180416.fits\n",
      "../dmu1/dmu1_ml_GAMA-09/data/depths_gama-09_20180601.fits\n",
      "../dmu1/dmu1_ml_GAMA-12/data/depths_gama-12_20180218.fits\n",
      "../dmu1/dmu1_ml_GAMA-15/data/depths_gama-15_20180213.fits\n",
      "../dmu1/dmu1_ml_HDF-N/data/depths_hdf-n_20180427.fits\n",
      "../dmu1/dmu1_ml_Herschel-Stripe-82/data/depths_herschel-stripe-82_20180307.fits\n",
      "../dmu1/dmu1_ml_Lockman-SWIRE/data/depths_lockman-swire_20180219.fits\n",
      "../dmu1/dmu1_ml_NGP/data/depths_ngp_20180219.fits\n",
      "../dmu1/dmu1_ml_SA13/data/depths_sa13_20180501.fits\n",
      "../dmu1/dmu1_ml_SGP/data/depths_sgp_20180221.fits\n",
      "../dmu1/dmu1_ml_SPIRE-NEP/data/depths_spire-nep_20180220.fits\n",
      "../dmu1/dmu1_ml_SSDF/data/depths_ssdf_20180221.fits\n",
      "../dmu1/dmu1_ml_xFLS/data/depths_xfls_20180501.fits\n",
      "../dmu1/dmu1_ml_XMM-13hr/data/depths_xmm-13hr_20180501.fits\n",
      "../dmu1/dmu1_ml_XMM-LSS/data/depths_xmm-lss_20180504.fits\n"
     ]
    }
   ],
   "source": [
    "depths_to_vo = open('depths_to_vo.sh', 'w+')\n",
    "for field in dr1:\n",
    "    final_depth_product = '../dmu1/dmu1_ml_{}/data/depths_{}_{}.fits'.format(field[0], field[0].lower(), field[1])\n",
    "    \n",
    "\n",
    "    if os.path.exists(final_depth_product):\n",
    "        print(final_depth_product)\n",
    "    \n",
    "        #Test with Cigale input files\n",
    "        depths_to_vo.write(stilts_command.format(in_file=final_depth_product, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='depth/{}_{}.csv \\n'.format(field[0], \n",
    "                                                                                                field[1])))\n",
    "\n",
    "        \n",
    "    else:\n",
    "        depths_to_vo.write('# No depths for {} \\n'.format(field[0]))\n",
    "        \n",
    "    #final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))\n",
    "    \n",
    "depths_to_vo.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarise completeness of HELP data sets\n",
    "\n",
    "Here we get information about what is available on each field to summarise the data products available per field. We take the cigale, xid+ and photo-z filenames from the per field meta_main.yml files here and check they are there and how large they are. This then given a summary of all the data present."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1_data_products = dr1.copy()\n",
    "\n",
    "fields_info = yaml.load(open(\"../dmu2/meta_main.yml\", 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'xid_objects'))\n",
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'photoz_objects'))\n",
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'cigale_objects'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AKARI-NEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "AKARI-SEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "Bootes:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "CDFS-SWIRE:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_CDFS-SWIRE/data/dmu26_XID+SPIRE_CDFS-SWIRE_cat_20170919.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_CDFS-SWIRE/data/dmu26_XID+MIPS_CDFS-SWIRE_cat_20170901.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_CDFS-SWIRE/data/dmu26_XID+PACS_CDFS-SWIRE_cat_20171019.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_CDFS-SWIRE/data/master_catalogue_cdfs-swire_20171103_photoz_20180210.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_CDFS-SWIRE/data/zphot/HELP_final_results.fits\n",
      "COSMOS:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_COSMOS/data/dmu26_XID+MIPS_COSMOS_20170213.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_COSMOS/data/dmu26_XID+PACS_COSMOS_20170303.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_COSMOS/data/dmu26_XID+SPIRE_COSMOS_20161129.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_COSMOS/data/COSMOS2015-HELP_selected_20160613_photoz_v1.0.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_COSMOS/data/zphot/final_results.fits\n",
      "EGS:\n",
      "  - xid not done.\n",
      "  - photoz: dmu_products/dmu24/dmu24_EGS/data/master_catalogue_egs_20180501_photoz_20180608.fits\n",
      "  - cigale not done.\n",
      "ELAIS-N1:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_ELAIS-N1/data/dmu26_XID+MIPS_ELAIS-N1_SERVS_cat_20170725.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_ELAIS-N1/data/dmu26_XID+PACS_ELAIS-N1_SWIRE_cat_20170808.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_ELAIS-N1/data/dmu26_XID+SPIRE_ELAIS-N1_SERVS_cat_20170725.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_ELAIS-N1/data/master_catalogue_elais-n1_20170706_photoz_20170725_irac1_optimised.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_ELAIS-N1/data/zphot/HELP_final_results.fits\n",
      "ELAIS-N2:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "ELAIS-S1:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_ELAIS-S1/data/dmu26_XID+MIPS_ELAIS-S1_20180215.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_ELAIS-S1/data/dmu26_XID+PACS_ELAIS-S1_cat_20180416.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_ELAIS-S1/data/dmu26_XID+SPIRE_ELAIS-S1_20180327.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_ELAIS-S1/data/master_catalogue_elais-s1_20180221_photoz_20180412.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_ELAIS-S1/data/zphot/HELP_final_results.fits\n",
      "GAMA-09:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_GAMA-09/data/dmu26_XID+SPIRE_GAMA-09_20180508.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_GAMA-09/data/master_catalogue_gama-09_20171206_photoz_20180213_r_optimised.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_GAMA-09/data/zphot/HELP_final_results.fits\n",
      "GAMA-12:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_GAMA-12/data/dmu26_XID+SPIRE_GAMA_12_20180508.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_GAMA-12/data/master_catalogue_gama-12_20171210_photoz_20180410_r_optimised.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_GAMA-12/data/zphot/HELP_final_results.fits\n",
      "GAMA-15:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_GAMA-15/data/dmu26_XID+SPIRE_GAMA-15_cat_20180416.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_GAMA-15/data/master_catalogue_gama-15_20180119_photoz_20180210_r_optimised.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_GAMA-15/data/zphot/HELP_final_results.fits\n",
      "HDF-N:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "Herschel-Stripe-82:\n",
      "  - xid not done.\n",
      "  - photoz: dmu_products/dmu24/dmu24_Herschel-Stripe-82/data/master_catalogue_herschel-stripe-82_20180307_photoz_20180509.fits\n",
      "  - cigale not done.\n",
      "Lockman-SWIRE:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_Lockman-SWIRE/data/dmu26_XID+MIPS_Lockman-SWIRE_cat_20171214.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_Lockman-SWIRE/data/dmu26_XID+PACS_Lockman-SWIRE_cat_20180220.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_Lockman-SWIRE/data/dmu26_XID+SPIRE_Lockman-SWIRE_20180220.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_Lockman-SWIRE/data/master_catalogue_Lockman-SWIRE_20170710_photoz_20170802_r_and_irac1_optimised_UPDATED_IDs_20180219.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_Lockman-SWIRE/data/zphot/best_extcor/HELP_final_results.fits\n",
      "NGP:\n",
      "  - xid not done.\n",
      "  - photoz: dmu_products/dmu24/dmu24_NGP/data/master_catalogue_ngp_20180501_photoz_20180601_r_optimised.fits\n",
      "  - cigale not done.\n",
      "SA13:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SGP:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_SGP/data/dmu26_XID+SPIRE_SGP_20180625.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_SGP/data/master_catalogue_sgp_20180221_photoz_20180502_r_optimised.fits\n",
      "  - cigale: dmu_products/dmu28/dmu28_SGP/data/zphot/HELP_final_results.fits\n",
      "SPIRE-NEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SSDF:\n",
      "  - xid not done.\n",
      "  - photoz: dmu_products/dmu24/dmu24_SSDF/data/master_catalogue_ssdf_20180221_photoz_20180612_r_optimised.fits\n",
      "  - cigale not done.\n",
      "xFLS:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "XMM-13hr:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "XMM-LSS:\n",
      "  - xid not done.\n",
      "  - photoz: dmu_products/dmu24/dmu24_XMM-LSS/data/master_catalogue_xmm-lss_20180221_photoz_20180518_r_optimised.fits\n",
      "  - cigale not done.\n"
     ]
    }
   ],
   "source": [
    "for field in fields_info['fields']:\n",
    "    print(field['name'] + ':')\n",
    "    xid_objects = 0\n",
    "    photoz_objects = 0\n",
    "    cigale_objects = 0\n",
    "    dmu32_info = yaml.load(open('./dmu32_{}/meta_main.yml'.format(field['name']), 'r'))\n",
    "\n",
    "        \n",
    "    try: \n",
    "        for n, xid_file in enumerate(dmu32_info['xid']):\n",
    "            #print(n,xid_file,xid_file.replace('dmu_products', '..'))\n",
    "            xid_objects = len(Table.read(xid_file.replace('dmu_products', '..')))\n",
    "            print(\"  - xid: {}\".format(dmu32_info['xid'][n]))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - xid not done.\".format(field['name']))\n",
    "        \n",
    "    try: \n",
    "        #print(dmu32_info['photoz'].replace('dmu_products', '..'))\n",
    "        photoz_objects = len(Table.read(dmu32_info['photoz'].replace('dmu_products', '..')))\n",
    "        print(\"  - photoz: {}\".format(dmu32_info['photoz']))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - photoz not done.\".format(field['name']))\n",
    "        \n",
    "    try: \n",
    "        cigale_objects = len(Table.read(dmu32_info['cigale'].replace('dmu_products', '..')))\n",
    "        print(\"  - cigale: {}\".format(dmu32_info['cigale']))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - cigale not done.\".format(field['name']))\n",
    " \n",
    "                   \n",
    "    dr1_data_products['xid_objects'][dr1_data_products['field'] == field['name']] = xid_objects\n",
    "    dr1_data_products['photoz_objects'][dr1_data_products['field'] == field['name']] = photoz_objects\n",
    "    dr1_data_products['cigale_objects'][dr1_data_products['field'] == field['name']] = cigale_objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=23</i>\n",
       "<table id=\"table139627909782104-588179\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>field</th><th>dr1_suffix</th><th>last_suffix</th><th>objects</th><th>area_sq_degrees</th><th>file_size_bytes</th><th>file_size_readable</th><th>xid_objects</th><th>photoz_objects</th><th>cigale_objects</th></tr></thead>\n",
       "<tr><td>0</td><td>AKARI-NEP</td><td>20180215</td><td>20180215</td><td>531746</td><td>9.194732358779467</td><td>348848640</td><td>348.85 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>AKARI-SEP</td><td>20180221</td><td>20180221</td><td>844172</td><td>8.713306475131118</td><td>538606080</td><td>538.61 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>2</td><td>Bootes</td><td>20180520</td><td>20180520</td><td>3481661</td><td>11.42815299095486</td><td>5002044480</td><td>5 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>3</td><td>CDFS-SWIRE</td><td>20180613</td><td>20180613</td><td>2171051</td><td>12.971246403717068</td><td>5705593920</td><td>5.71 GB</td><td>240489</td><td>139532</td><td>9308</td></tr>\n",
       "<tr><td>4</td><td>COSMOS</td><td>20180619</td><td>20180619</td><td>2599374</td><td>5.083863478496816</td><td>10517175360</td><td>10.52 GB</td><td>44310</td><td>694478</td><td>15747</td></tr>\n",
       "<tr><td>5</td><td>EGS</td><td>20180501</td><td>20180501</td><td>1412613</td><td>3.566383275122158</td><td>3808480320</td><td>3.81 GB</td><td>0</td><td>1183490</td><td>0</td></tr>\n",
       "<tr><td>6</td><td>ELAIS-N1</td><td>20171016</td><td>20180216</td><td>4026292</td><td>13.507484555454763</td><td>5242279680</td><td>5.24 GB</td><td>120282</td><td>2885116</td><td>50129</td></tr>\n",
       "<tr><td>7</td><td>ELAIS-N2</td><td>20180218</td><td>20180218</td><td>1783240</td><td>9.167479903991113</td><td>2007964800</td><td>2.01 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>8</td><td>ELAIS-S1</td><td>20180416</td><td>20180416</td><td>1655564</td><td>9.002940646885508</td><td>1970156160</td><td>1.97 GB</td><td>194276</td><td>1102319</td><td>25393</td></tr>\n",
       "<tr><td>9</td><td>GAMA-09</td><td>20180601</td><td>20180601</td><td>12937982</td><td>62.01393417284915</td><td>19911611520</td><td>19.91 GB</td><td>1386659</td><td>8834023</td><td>130293</td></tr>\n",
       "<tr><td>10</td><td>GAMA-12</td><td>20180218</td><td>20180218</td><td>12369415</td><td>62.70933280772107</td><td>14633066880</td><td>14.63 GB</td><td>1099477</td><td>8591676</td><td>108139</td></tr>\n",
       "<tr><td>11</td><td>GAMA-15</td><td>20180213</td><td>20180213</td><td>14232880</td><td>61.70114565980842</td><td>16595585280</td><td>16.6 GB</td><td>1236395</td><td>10109159</td><td>117234</td></tr>\n",
       "<tr><td>12</td><td>HDF-N</td><td>20180427</td><td>20180427</td><td>130679</td><td>0.6717832558166041</td><td>185996160</td><td>186 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>13</td><td>Herschel-Stripe-82</td><td>20180307</td><td>20180307</td><td>50196455</td><td>363.22747930641884</td><td>127047306240</td><td>127.05 GB</td><td>0</td><td>21613821</td><td>0</td></tr>\n",
       "<tr><td>14</td><td>Lockman-SWIRE</td><td>20180219</td><td>20180219</td><td>4366298</td><td>22.413863402949218</td><td>5195931840</td><td>5.2 GB</td><td>242065</td><td>1382438</td><td>46719</td></tr>\n",
       "<tr><td>15</td><td>NGP</td><td>20180219</td><td>20180501</td><td>6759591</td><td>177.6981971077162</td><td>4954806720</td><td>4.95 GB</td><td>0</td><td>3175339</td><td>0</td></tr>\n",
       "<tr><td>16</td><td>SA13</td><td>20180501</td><td>20180501</td><td>9799</td><td>0.27416379328431184</td><td>2721600</td><td>2.72 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>17</td><td>SGP</td><td>20180221</td><td>20180221</td><td>29790690</td><td>294.5684540900642</td><td>45192530880</td><td>45.19 GB</td><td>3511594</td><td>17057212</td><td>352804</td></tr>\n",
       "<tr><td>18</td><td>SPIRE-NEP</td><td>20180220</td><td>20180220</td><td>2674</td><td>0.1280660469377391</td><td>1203840</td><td>1.2 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>19</td><td>SSDF</td><td>20180221</td><td>20180221</td><td>12661903</td><td>111.11624949343539</td><td>8078319360</td><td>8.08 GB</td><td>0</td><td>9268783</td><td>0</td></tr>\n",
       "<tr><td>20</td><td>xFLS</td><td>20180501</td><td>20180501</td><td>977148</td><td>7.44043242140754</td><td>1050474240</td><td>1.05 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>21</td><td>XMM-13hr</td><td>20180501</td><td>20180501</td><td>38629</td><td>0.7570240166583633</td><td>10676160</td><td>10.68 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>22</td><td>XMM-LSS</td><td>20180504</td><td>20180504</td><td>8705837</td><td>21.75457939331374</td><td>28241838720</td><td>28.24 GB</td><td>0</td><td>6152920</td><td>0</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table139627909782104-588179').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table139627909782104-588179').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 4, 5, 6, 8, 9, 10], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr1_data_products.show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XID+: There are 9 fields with XID+ fluxes computed totalling 543.97 square degrees.\n",
      "      This is a total of 8075547 objects with an average of 10% of objects on processed areas.\n",
      "Photo-zs: There are 14 fields with photozs computed totalling 1221.34 square degrees.\n",
      "      This is a total of 92190306 objects with an average of 56% of objects on processed areas.\n",
      "CIGALE SEDs: There are 9 fields with SEDs computed totalling 543.97 square degrees.\n",
      "      This is a total of 855766 objects with an average of 1% of objects on processed areas.\n"
     ]
    }
   ],
   "source": [
    "has_xid = dr1_data_products['xid_objects'] != 0\n",
    "has_photoz = dr1_data_products['photoz_objects'] != 0\n",
    "has_cigale = dr1_data_products['cigale_objects'] != 0\n",
    "\n",
    "print(\"XID+: There are {} fields with XID+ fluxes computed totalling {area:.2f} square degrees.\".format(\n",
    "    np.sum(has_xid),\n",
    "    area=np.sum(dr1_data_products['area_sq_degrees'][has_xid])))\n",
    "print(\"      This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.\".format(\n",
    "    np.sum(dr1_data_products['xid_objects'][has_xid]),\n",
    "    perc=100 * np.sum(dr1_data_products['xid_objects'][has_xid])/np.sum(dr1_data_products['objects'][has_xid])))\n",
    "\n",
    "print(\"Photo-zs: There are {} fields with photozs computed totalling {area:.2f} square degrees.\".format(\n",
    "    np.sum(has_photoz),\n",
    "    area=np.sum(dr1_data_products['area_sq_degrees'][has_photoz])))\n",
    "print(\"      This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.\".format(\n",
    "    np.sum(dr1_data_products['photoz_objects'][has_photoz]),\n",
    "    perc=100 * np.sum(dr1_data_products['photoz_objects'][has_photoz])/np.sum(dr1_data_products['objects'][has_photoz])))\n",
    "\n",
    "print(\"CIGALE SEDs: There are {} fields with SEDs computed totalling {area:.2f} square degrees.\".format(\n",
    "    np.sum(has_cigale),\n",
    "    area=np.sum(dr1_data_products['area_sq_degrees'][has_cigale])))\n",
    "print(\"      This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.\".format(\n",
    "    np.sum(dr1_data_products['cigale_objects'][has_cigale]),\n",
    "    perc=100 * np.sum(dr1_data_products['cigale_objects'][has_cigale])/np.sum(dr1_data_products['objects'][has_cigale])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1_data_products.write('dr1_data_products_overview{}.csv'.format(TODAY), overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}