{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final stage of HELP data processing\n",
    "\n",
    "This notebook performs collates the final output files ready for writing to csv for ingestion to a VO server\n",
    "\n",
    "- Take DR1 masterlist suffixes from overview table\n",
    "- Find dmu32 full table names and write to a file\n",
    "- Create summary of all the data products per field using the dmu32 meta_main.yml files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from  herschelhelp_internal  import git_version\n",
    "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n",
    "import datetime\n",
    "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.table import Table, Column\n",
    "from astropy import units as u\n",
    "import numpy as np\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.masterlist import find_last_ml_suffix\n",
    "\n",
    "import yaml\n",
    "\n",
    "import os\n",
    "import time\n",
    "\n",
    "import humanfriendly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "TODAY = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The definition of HELP DR1\n",
    "Here we take the DR1 definition from dmu1. We then propagate those suffixes through to dmu32. Note that this does not include a specification of how the masterlist made its way through the whole HELP process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1 = Table.read(\"../dmu1/dr1_overview.fits\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=23</i>\n",
       "<table id=\"table4709190344-918720\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>field</th><th>dr1_suffix</th><th>last_suffix</th><th>objects</th><th>area_sq_degrees</th><th>file_size_bytes</th><th>file_size_readable</th></tr></thead>\n",
       "<tr><td>0</td><td>AKARI-NEP</td><td>20180215</td><td>20180215</td><td>531746</td><td>9.19473235878</td><td>348848640</td><td>348.85 MB</td></tr>\n",
       "<tr><td>1</td><td>AKARI-SEP</td><td>20180221</td><td>20180221</td><td>844172</td><td>8.71330647513</td><td>538606080</td><td>538.61 MB</td></tr>\n",
       "<tr><td>2</td><td>Bootes</td><td>20180520</td><td>20180520</td><td>3481661</td><td>11.428152991</td><td>5002044480</td><td>5 GB</td></tr>\n",
       "<tr><td>3</td><td>CDFS-SWIRE</td><td>20180613</td><td>20180613</td><td>2171051</td><td>12.9712464037</td><td>5705593920</td><td>5.71 GB</td></tr>\n",
       "<tr><td>4</td><td>COSMOS</td><td>20180619</td><td>20180619</td><td>2599374</td><td>5.0838634785</td><td>10517175360</td><td>10.52 GB</td></tr>\n",
       "<tr><td>5</td><td>EGS</td><td>20180501</td><td>20180501</td><td>1412613</td><td>3.56638327512</td><td>3808480320</td><td>3.81 GB</td></tr>\n",
       "<tr><td>6</td><td>ELAIS-N1</td><td>20171016</td><td>20180216</td><td>4026292</td><td>13.5074845555</td><td>5242279680</td><td>5.24 GB</td></tr>\n",
       "<tr><td>7</td><td>ELAIS-N2</td><td>20180218</td><td>20180218</td><td>1783240</td><td>9.16747990399</td><td>2007964800</td><td>2.01 GB</td></tr>\n",
       "<tr><td>8</td><td>ELAIS-S1</td><td>20180416</td><td>20180416</td><td>1655564</td><td>9.00294064689</td><td>1970156160</td><td>1.97 GB</td></tr>\n",
       "<tr><td>9</td><td>GAMA-09</td><td>20180601</td><td>20180601</td><td>12937982</td><td>62.0139341728</td><td>19911611520</td><td>19.91 GB</td></tr>\n",
       "<tr><td>10</td><td>GAMA-12</td><td>20180218</td><td>20180218</td><td>12369415</td><td>62.7093328077</td><td>14633066880</td><td>14.63 GB</td></tr>\n",
       "<tr><td>11</td><td>GAMA-15</td><td>20180213</td><td>20180213</td><td>14232880</td><td>61.7011456598</td><td>16595585280</td><td>16.6 GB</td></tr>\n",
       "<tr><td>12</td><td>HDF-N</td><td>20180427</td><td>20180427</td><td>130679</td><td>0.671783255817</td><td>185996160</td><td>186 MB</td></tr>\n",
       "<tr><td>13</td><td>Herschel-Stripe-82</td><td>20180307</td><td>20180307</td><td>50196455</td><td>363.227479306</td><td>127047306240</td><td>127.05 GB</td></tr>\n",
       "<tr><td>14</td><td>Lockman-SWIRE</td><td>20180219</td><td>20180219</td><td>4366298</td><td>22.4138634029</td><td>5195931840</td><td>5.2 GB</td></tr>\n",
       "<tr><td>15</td><td>NGP</td><td>20180219</td><td>20180501</td><td>6759591</td><td>177.698197108</td><td>4954806720</td><td>4.95 GB</td></tr>\n",
       "<tr><td>16</td><td>SA13</td><td>20180501</td><td>20180501</td><td>9799</td><td>0.274163793284</td><td>2721600</td><td>2.72 MB</td></tr>\n",
       "<tr><td>17</td><td>SGP</td><td>20180221</td><td>20180221</td><td>29790690</td><td>294.56845409</td><td>45192530880</td><td>45.19 GB</td></tr>\n",
       "<tr><td>18</td><td>SPIRE-NEP</td><td>20180220</td><td>20180220</td><td>2674</td><td>0.128066046938</td><td>1203840</td><td>1.2 MB</td></tr>\n",
       "<tr><td>19</td><td>SSDF</td><td>20180221</td><td>20180221</td><td>12661903</td><td>111.116249493</td><td>8078319360</td><td>8.08 GB</td></tr>\n",
       "<tr><td>20</td><td>xFLS</td><td>20180501</td><td>20180501</td><td>977148</td><td>7.44043242141</td><td>1050474240</td><td>1.05 GB</td></tr>\n",
       "<tr><td>21</td><td>XMM-13hr</td><td>20180501</td><td>20180501</td><td>38629</td><td>0.757024016658</td><td>10676160</td><td>10.68 MB</td></tr>\n",
       "<tr><td>22</td><td>XMM-LSS</td><td>20180504</td><td>20180504</td><td>8705837</td><td>21.7545793933</td><td>28241838720</td><td>28.24 GB</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table4709190344-918720').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table4709190344-918720').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 4, 5, 6], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr1.show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./dmu32_AKARI-NEP/data/AKARI-NEP_20180215_cigale.fits\n",
      "./dmu32_AKARI-SEP/data/AKARI-SEP_20180221_cigale.fits\n",
      "./dmu32_Bootes/data/Bootes_20180520_cigale.fits\n",
      "./dmu32_CDFS-SWIRE/data/CDFS-SWIRE_20180613.fits\n",
      "./dmu32_COSMOS/data/COSMOS_20180619.fits\n",
      "./dmu32_EGS/data/EGS_20180501_cigale.fits\n",
      "./dmu32_ELAIS-N1/data/ELAIS-N1_20171016.fits\n",
      "./dmu32_ELAIS-N2/data/ELAIS-N2_20180218_cigale.fits\n",
      "./dmu32_ELAIS-S1/data/ELAIS-S1_20180416.fits\n",
      "./dmu32_GAMA-09/data/GAMA-09_20180601.fits\n",
      "./dmu32_GAMA-12/data/GAMA-12_20180218.fits\n",
      "./dmu32_GAMA-15/data/GAMA-15_20180213.fits\n",
      "./dmu32_HDF-N/data/HDF-N_20180427_cigale.fits\n",
      "./dmu32_Herschel-Stripe-82/data/Herschel-Stripe-82_20180307_cigale.fits\n",
      "./dmu32_Lockman-SWIRE/data/Lockman-SWIRE_20180219.fits\n",
      "./dmu32_NGP/data/NGP_20180219_cigale.fits\n",
      "./dmu32_SA13/data/SA13_20180501_cigale.fits\n",
      "./dmu32_SGP/data/SGP_20180221_cigale.fits\n",
      "./dmu32_SPIRE-NEP/data/SPIRE-NEP_20180220_cigale.fits\n",
      "./dmu32_SSDF/data/SSDF_20180221_cigale.fits\n",
      "./dmu32_xFLS/data/xFLS_20180501_cigale.fits\n",
      "./dmu32_XMM-13hr/data/XMM-13hr_20180501_cigale.fits\n",
      "./dmu32_XMM-LSS/data/XMM-LSS_20180504_cigale.fits\n"
     ]
    }
   ],
   "source": [
    "GAVO_FOLDER = '/mnt/hedam/data_vo/'\n",
    "stilts_command = 'stilts tpipe {in_file} omode=out ofmt=csv out={GAVO_FOLDER}{out_file}'\n",
    "\n",
    "final_data = open('help_to_vo.sh', 'w+')\n",
    "for field in dr1:\n",
    "    final_help_product = './dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1])\n",
    "    cigale_input = './dmu32_{}/data/{}_{}_cigale.fits'.format(field[0], field[0], field[1])\n",
    "\n",
    "    if os.path.exists(final_help_product):\n",
    "        print(final_help_product)\n",
    "    \n",
    "        #Test with Cigale input files\n",
    "        final_data.write(stilts_command.format(in_file=final_help_product, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='herschelhelp/main/{}_{}_all.csv \\n'.format(field[0], \n",
    "                                                                                                field[1])))\n",
    "    elif os.path.exists(cigale_input):\n",
    "        print(cigale_input)\n",
    "        final_data.write(stilts_command.format(in_file=cigale_input, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='herschelhelp/main/{}_{}_incomplete.csv \\n'.format(field[0], \n",
    "                                                                                                       field[1])))\n",
    "        \n",
    "    else:\n",
    "        final_data.write('# No data for {} \\n'.format(field[0]))\n",
    "        \n",
    "    #final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))\n",
    "    \n",
    "final_data.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The out put of this notebook is a shell script which will write all the fits files to csv files in the vo folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../dmu1/dmu1_ml_AKARI-NEP/data/depths_akari-nep_20180215.fits\n",
      "../dmu1/dmu1_ml_AKARI-SEP/data/depths_akari-sep_20180221.fits\n",
      "../dmu1/dmu1_ml_Bootes/data/depths_bootes_20180520.fits\n",
      "../dmu1/dmu1_ml_CDFS-SWIRE/data/depths_cdfs-swire_20180613.fits\n",
      "../dmu1/dmu1_ml_COSMOS/data/depths_cosmos_20180619.fits\n",
      "../dmu1/dmu1_ml_EGS/data/depths_egs_20180501.fits\n",
      "../dmu1/dmu1_ml_ELAIS-N1/data/depths_elais-n1_20171016.fits\n",
      "../dmu1/dmu1_ml_ELAIS-N2/data/depths_elais-n2_20180218.fits\n",
      "../dmu1/dmu1_ml_ELAIS-S1/data/depths_elais-s1_20180416.fits\n",
      "../dmu1/dmu1_ml_GAMA-09/data/depths_gama-09_20180601.fits\n",
      "../dmu1/dmu1_ml_GAMA-12/data/depths_gama-12_20180218.fits\n",
      "../dmu1/dmu1_ml_GAMA-15/data/depths_gama-15_20180213.fits\n",
      "../dmu1/dmu1_ml_HDF-N/data/depths_hdf-n_20180427.fits\n",
      "../dmu1/dmu1_ml_Herschel-Stripe-82/data/depths_herschel-stripe-82_20180307.fits\n",
      "../dmu1/dmu1_ml_Lockman-SWIRE/data/depths_lockman-swire_20180219.fits\n",
      "../dmu1/dmu1_ml_NGP/data/depths_ngp_20180219.fits\n",
      "../dmu1/dmu1_ml_SA13/data/depths_sa13_20180501.fits\n",
      "../dmu1/dmu1_ml_SGP/data/depths_sgp_20180221.fits\n",
      "../dmu1/dmu1_ml_SPIRE-NEP/data/depths_spire-nep_20180220.fits\n",
      "../dmu1/dmu1_ml_SSDF/data/depths_ssdf_20180221.fits\n",
      "../dmu1/dmu1_ml_xFLS/data/depths_xfls_20180501.fits\n",
      "../dmu1/dmu1_ml_XMM-13hr/data/depths_xmm-13hr_20180501.fits\n",
      "../dmu1/dmu1_ml_XMM-LSS/data/depths_xmm-lss_20180504.fits\n"
     ]
    }
   ],
   "source": [
    "depths_to_vo = open('depths_to_vo.sh', 'w+')\n",
    "for field in dr1:\n",
    "    final_depth_product = '../dmu1/dmu1_ml_{}/data/depths_{}_{}.fits'.format(field[0], field[0].lower(), field[1])\n",
    "    \n",
    "\n",
    "    if os.path.exists(final_depth_product):\n",
    "        print(final_depth_product)\n",
    "    \n",
    "        #Test with Cigale input files\n",
    "        depths_to_vo.write(stilts_command.format(in_file=final_depth_product, \n",
    "                                               GAVO_FOLDER=GAVO_FOLDER, \n",
    "                                               out_file='depth/{}_{}.csv \\n'.format(field[0], \n",
    "                                                                                                field[1])))\n",
    "\n",
    "        \n",
    "    else:\n",
    "        depths_to_vo.write('# No depths for {} \\n'.format(field[0]))\n",
    "        \n",
    "    #final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))\n",
    "    \n",
    "depths_to_vo.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarise completeness of HELP data sets\n",
    "\n",
    "Here we get information about what is available on each field to summarise the data products available per field. We take the cigale, xid+ and photo-z filenames from the per field meta_main.yml files here and check they are there and how large they are. This then given a summary of all the data present."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1_data_products = dr1.copy()\n",
    "\n",
    "fields_info = yaml.load(open(\"../dmu2/meta_main.yml\", 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'xid_objects'))\n",
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'photoz_objects'))\n",
    "dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'cigale_objects'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AKARI-NEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "AKARI-SEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "Bootes:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "CDFS-SWIRE:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_CDFS-SWIRE/data/dmu26_XID+SPIRE_CDFS-SWIRE_cat_20170919.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_CDFS-SWIRE/data/dmu26_XID+MIPS_CDFS-SWIRE_cat_20170901.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_CDFS-SWIRE/data/dmu26_XID+PACS_CDFS-SWIRE_cat_20171019.fits\n",
      "  - photoz: dmu_products/dmu24/dmu24_CDFS-SWIRE/data/master_catalogue_cdfs-swire_20171103_photoz_20180210.fits\n",
      "  - cigale not done.\n",
      "COSMOS:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_COSMOS/data/dmu26_XID+MIPS_COSMOS_20170213.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_COSMOS/data/dmu26_XID+PACS_COSMOS_20170303.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_COSMOS/data/dmu26_XID+SPIRE_COSMOS_20161129.fits\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "EGS:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "ELAIS-N1:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale: dmu_products/dmu28/dmu28_ELAIS-N1/data/zphot/HELP_final_results.fits\n",
      "ELAIS-N2:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "ELAIS-S1:\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+MIPS_ELAIS-S1/data/dmu26_XID+MIPS_ELAIS-S1_20180215.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+PACS_ELAIS-S1/data/dmu26_XID+PACS_ELAIS-S1_cat_20180416.fits\n",
      "  - xid: dmu_products/dmu26/dmu26_XID+SPIRE_ELAIS-S1/data/dmu26_XID+SPIRE_ELAIS-S1_20180327.fits\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "GAMA-09:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "GAMA-12:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "GAMA-15:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "HDF-N:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "Herschel-Stripe-82:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "Lockman-SWIRE:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "NGP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SA13:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SGP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SPIRE-NEP:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "SSDF:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "xFLS:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "XMM-13hr:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n",
      "XMM-LSS:\n",
      "  - xid not done.\n",
      "  - photoz not done.\n",
      "  - cigale not done.\n"
     ]
    }
   ],
   "source": [
    "for field in fields_info['fields']:\n",
    "    print(field['name'] + ':')\n",
    "    xid_objects = 0\n",
    "    photoz_objects = 0\n",
    "    cigale_objects = 0\n",
    "    dmu32_info = yaml.load(open('./dmu32_{}/meta_main.yml'.format(field['name']), 'r'))\n",
    "\n",
    "        \n",
    "    try: \n",
    "        for n, xid_file in enumerate(dmu32_info['xid']):\n",
    "            xid_objects = len(Table.read(xid_file.replace('dmu_products', '..')))\n",
    "            print(\"  - xid: {}\".format(dmu32_info['xid'][n]))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - xid not done.\".format(field['name']))\n",
    "        \n",
    "    try: \n",
    "        photoz_objects = len(Table.read(dmu32_info['photoz'].replace('dmu_products', '..')))\n",
    "        print(\"  - photoz: {}\".format(dmu32_info['photoz']))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - photoz not done.\".format(field['name']))\n",
    "        \n",
    "    try: \n",
    "        cigale_objects = len(Table.read(dmu32_info['cigale'].replace('dmu_products', '..')))\n",
    "        print(\"  - cigale: {}\".format(dmu32_info['cigale']))\n",
    "    except FileNotFoundError:\n",
    "        print(\"  - cigale not done.\".format(field['name']))\n",
    " \n",
    "                   \n",
    "    dr1_data_products['xid_objects'][dr1_data_products['field'] == field['name']] = xid_objects\n",
    "    dr1_data_products['photoz_objects'][dr1_data_products['field'] == field['name']] = photoz_objects\n",
    "    dr1_data_products['cigale_objects'][dr1_data_products['field'] == field['name']] = cigale_objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=23</i>\n",
       "<table id=\"table4717607344\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>field</th><th>dr1_suffix</th><th>last_suffix</th><th>objects</th><th>area_sq_degrees</th><th>file_size_bytes</th><th>file_size_readable</th><th>xid_objects</th><th>photoz_objects</th><th>cigale_objects</th></tr></thead>\n",
       "<thead><tr><th>bytes18</th><th>bytes20</th><th>bytes20</th><th>int64</th><th>float64</th><th>int64</th><th>bytes20</th><th>int64</th><th>int64</th><th>int64</th></tr></thead>\n",
       "<tr><td>AKARI-NEP</td><td>20180215</td><td>20180215</td><td>531746</td><td>9.19473235878</td><td>348848640</td><td>348.85 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>AKARI-SEP</td><td>20180221</td><td>20180221</td><td>844172</td><td>8.71330647513</td><td>538606080</td><td>538.61 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>Bootes</td><td>20180520</td><td>20180520</td><td>3481661</td><td>11.428152991</td><td>5002044480</td><td>5 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>CDFS-SWIRE</td><td>20180613</td><td>20180613</td><td>2171051</td><td>12.9712464037</td><td>5705593920</td><td>5.71 GB</td><td>240489</td><td>139532</td><td>0</td></tr>\n",
       "<tr><td>COSMOS</td><td>20180619</td><td>20180619</td><td>2599374</td><td>5.0838634785</td><td>10517175360</td><td>10.52 GB</td><td>44310</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>EGS</td><td>20180501</td><td>20180501</td><td>1412613</td><td>3.56638327512</td><td>3808480320</td><td>3.81 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>ELAIS-N1</td><td>20171016</td><td>20180216</td><td>4026292</td><td>13.5074845555</td><td>5242279680</td><td>5.24 GB</td><td>0</td><td>0</td><td>50129</td></tr>\n",
       "<tr><td>ELAIS-N2</td><td>20180218</td><td>20180218</td><td>1783240</td><td>9.16747990399</td><td>2007964800</td><td>2.01 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>ELAIS-S1</td><td>20180416</td><td>20180416</td><td>1655564</td><td>9.00294064689</td><td>1970156160</td><td>1.97 GB</td><td>194276</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>GAMA-09</td><td>20180601</td><td>20180601</td><td>12937982</td><td>62.0139341728</td><td>19911611520</td><td>19.91 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>Herschel-Stripe-82</td><td>20180307</td><td>20180307</td><td>50196455</td><td>363.227479306</td><td>127047306240</td><td>127.05 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>Lockman-SWIRE</td><td>20180219</td><td>20180219</td><td>4366298</td><td>22.4138634029</td><td>5195931840</td><td>5.2 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>NGP</td><td>20180219</td><td>20180501</td><td>6759591</td><td>177.698197108</td><td>4954806720</td><td>4.95 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>SA13</td><td>20180501</td><td>20180501</td><td>9799</td><td>0.274163793284</td><td>2721600</td><td>2.72 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>SGP</td><td>20180221</td><td>20180221</td><td>29790690</td><td>294.56845409</td><td>45192530880</td><td>45.19 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>SPIRE-NEP</td><td>20180220</td><td>20180220</td><td>2674</td><td>0.128066046938</td><td>1203840</td><td>1.2 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>SSDF</td><td>20180221</td><td>20180221</td><td>12661903</td><td>111.116249493</td><td>8078319360</td><td>8.08 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>xFLS</td><td>20180501</td><td>20180501</td><td>977148</td><td>7.44043242141</td><td>1050474240</td><td>1.05 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>XMM-13hr</td><td>20180501</td><td>20180501</td><td>38629</td><td>0.757024016658</td><td>10676160</td><td>10.68 MB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "<tr><td>XMM-LSS</td><td>20180504</td><td>20180504</td><td>8705837</td><td>21.7545793933</td><td>28241838720</td><td>28.24 GB</td><td>0</td><td>0</td><td>0</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=23>\n",
       "      field        dr1_suffix last_suffix ... photoz_objects cigale_objects\n",
       "     bytes18        bytes20     bytes20   ...     int64          int64     \n",
       "------------------ ---------- ----------- ... -------------- --------------\n",
       "         AKARI-NEP   20180215    20180215 ...              0              0\n",
       "         AKARI-SEP   20180221    20180221 ...              0              0\n",
       "            Bootes   20180520    20180520 ...              0              0\n",
       "        CDFS-SWIRE   20180613    20180613 ...         139532              0\n",
       "            COSMOS   20180619    20180619 ...              0              0\n",
       "               EGS   20180501    20180501 ...              0              0\n",
       "          ELAIS-N1   20171016    20180216 ...              0          50129\n",
       "          ELAIS-N2   20180218    20180218 ...              0              0\n",
       "          ELAIS-S1   20180416    20180416 ...              0              0\n",
       "           GAMA-09   20180601    20180601 ...              0              0\n",
       "               ...        ...         ... ...            ...            ...\n",
       "Herschel-Stripe-82   20180307    20180307 ...              0              0\n",
       "     Lockman-SWIRE   20180219    20180219 ...              0              0\n",
       "               NGP   20180219    20180501 ...              0              0\n",
       "              SA13   20180501    20180501 ...              0              0\n",
       "               SGP   20180221    20180221 ...              0              0\n",
       "         SPIRE-NEP   20180220    20180220 ...              0              0\n",
       "              SSDF   20180221    20180221 ...              0              0\n",
       "              xFLS   20180501    20180501 ...              0              0\n",
       "          XMM-13hr   20180501    20180501 ...              0              0\n",
       "           XMM-LSS   20180504    20180504 ...              0              0"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dr1_data_products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "dr1_data_products.write('dr1_data_products_overview{}.csv'.format(TODAY), overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}