{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make VO table of extreme starbursts\n",
    "\n",
    "Reprocessing the manually made csv file from https://docs.google.com/spreadsheets/d/1hF9ls8AMzzY--2I5tukqBmJf6xf7LO8lB4NTr3sFu08/edit#gid=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.table import Table, Column\n",
    "from astropy.coordinates import SkyCoord\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import yaml\n",
    "\n",
    "from pymoc import MOC\n",
    "\n",
    "from herschelhelp_internal.utils import coords_to_hpidx,  gen_help_id, inMoc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants = Table.read('./data/Known_Monsters_Sheet1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table masked=True length=130</i>\n",
       "<table id=\"table34727843656\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>Name</th><th>RA</th><th>Dec</th><th>Redshift</th><th>Redshift source</th><th>Lbol/Lsun intrinsic</th><th>Lensed?</th><th>Original survey</th><th>Reference</th><th>Notes</th></tr></thead>\n",
       "<thead><tr><th>str24</th><th>str22</th><th>str13</th><th>str10</th><th>str14</th><th>str11</th><th>str9</th><th>str17</th><th>str106</th><th>str152</th></tr></thead>\n",
       "<tr><td>9io9</td><td>02h09m41.3s</td><td>00d15m58.6s</td><td>2.553</td><td>CO, H-alpha</td><td>1.00E+13</td><td>Y mu~10</td><td>Planck/ACT/VICS82</td><td>http://adsabs.harvard.edu/abs/2015MNRAS.452..502G</td><td>--</td></tr>\n",
       "<tr><td>WISE J132934.18+224327.3</td><td>13h29m34.18s</td><td>22d43m27.3s</td><td>2.0439</td><td>H-alpha</td><td>1.30E+13</td><td>Y mu~11</td><td>WISE/Planck</td><td>http://adsabs.harvard.edu/abs/2017ApJ...843L..22D</td><td>--</td></tr>\n",
       "<tr><td>Orochi</td><td>02h18m30.7s</td><td>-05d31m31s</td><td>3.4</td><td>CO</td><td>~1.30e13</td><td>Y mu~5</td><td>AzTEC-ASTE</td><td>http://adsabs.harvard.edu/abs/2011MNRAS.415.3081I</td><td>--</td></tr>\n",
       "<tr><td>MMJ154506.4−344318</td><td>15h45m06.35s</td><td>-34d43m18.2s</td><td>04-May</td><td>est</td><td>&gt;1e13</td><td>?</td><td>AzTEC-ASTE</td><td>http://adsabs.harvard.edu/abs/2015ApJ...808..121T</td><td>--</td></tr>\n",
       "<tr><td>MMJ154132.7−350320</td><td>15h41m32.71s</td><td>-35d03m23.3s</td><td>3</td><td>est</td><td>13.7-13.9</td><td>Y mu~1.2</td><td>AzTEC-ASTE</td><td>http://adsabs.harvard.edu/abs/2015ApJ...808..121T</td><td>--</td></tr>\n",
       "<tr><td>GN20</td><td>12h37m12s</td><td>62d22m12.100s</td><td>4.06</td><td>CO</td><td>2.90E+13</td><td>N</td><td>GOODS-N</td><td>http://adsabs.harvard.edu/abs/2009ApJ...694.1517D</td><td>--</td></tr>\n",
       "<tr><td>SPT0103-45</td><td>15.797917</td><td>-45.648306</td><td>3.0917</td><td>ALMA 3mm CO</td><td>1.67E+13</td><td>mu = 5.34</td><td>SPT-SZ</td><td>https://ui.adsabs.harvard.edu/#abs/2016ApJ...826..112S/abstract</td><td>Contact: jspilker@utexas.edu</td></tr>\n",
       "<tr><td>SPT0125-47</td><td>21.2795</td><td>-47.398889</td><td>2.5148</td><td>ALMA 3mm CO</td><td>3.33E+13</td><td>mu = 5.47</td><td>SPT-SZ</td><td>https://ui.adsabs.harvard.edu/#abs/2016ApJ...826..112S/abstract</td><td>Contact: jspilker@utexas.edu</td></tr>\n",
       "<tr><td>SPT0243-49</td><td>40.786708</td><td>-46.35675</td><td>5.699</td><td>ALMA 3mm CO</td><td>1.63E+13</td><td>mu = 5.09</td><td>SPT-SZ</td><td>https://ui.adsabs.harvard.edu/#abs/2016ApJ...826..112S/abstract</td><td>Contact: jspilker@utexas.edu</td></tr>\n",
       "<tr><td>SPT0300-46</td><td>45.018208</td><td>-46.35675</td><td>3.5954</td><td>ALMA 3mm CO</td><td>1.26E+13</td><td>mu = 6.30</td><td>SPT-SZ</td><td>https://ui.adsabs.harvard.edu/#abs/2016ApJ...826..112S/abstract</td><td>Contact: jspilker@utexas.edu</td></tr>\n",
       "<tr><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td><td>...</td></tr>\n",
       "<tr><td>--</td><td>138.158467</td><td>2.01306424</td><td>3.04128003</td><td>spec-z</td><td>1.32E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>9.17275683</td><td>-43.343878</td><td>0.13801</td><td>spec-z</td><td>1.99E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>8.32921426</td><td>-45.388595</td><td>0.13500001</td><td>spec-z</td><td>1.12E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>8.68245338</td><td>-45.189448</td><td>0.57800001</td><td>spec-z</td><td>1.39E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>242.824989</td><td>54.3402405</td><td>0.59622699</td><td>spec-z</td><td>2.30E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>240.811248</td><td>54.3738727</td><td>1.75600004</td><td>spec-z</td><td>1.37E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>245.194985</td><td>55.4695492</td><td>0.60170501</td><td>spec-z</td><td>1.78E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>242.830416</td><td>56.5343897</td><td>3.49065995</td><td>spec-z</td><td>1.56E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>241.414947</td><td>54.7630619</td><td>2.44812989</td><td>spec-z</td><td>1.24E+13</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>Sources selected from the HELP databse using the code on GITHUB</td></tr>\n",
       "<tr><td>--</td><td>--</td><td>--</td><td>--</td><td>phot-z</td><td>--</td><td>--</td><td>HELP</td><td>https://github.com/StevenDuivenvoorden/FIR-group/blob/master/Ian/plotting_tools/HELP_ULIRG_selection.ipynb</td><td>There are an additional ~1900 sources whose solar luminosity is &gt;10^13 solar luminosities in the HELP databse but which only have photometric redshifts.</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table masked=True length=130>\n",
       "          Name           ...\n",
       "         str24           ...\n",
       "------------------------ ...\n",
       "                    9io9 ...\n",
       "WISE J132934.18+224327.3 ...\n",
       "                  Orochi ...\n",
       "      MMJ154506.4−344318 ...\n",
       "      MMJ154132.7−350320 ...\n",
       "                    GN20 ...\n",
       "              SPT0103-45 ...\n",
       "              SPT0125-47 ...\n",
       "              SPT0243-49 ...\n",
       "              SPT0300-46 ...\n",
       "                     ... ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ...\n",
       "                      -- ..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "giants"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Insist on RA Dec\n",
    "We will also convert all RA Dec to decimal degrees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants['RA'].fill_value = ''\n",
    "giants['Dec'].fill_value = ''\n",
    "giants = giants.filled()\n",
    "giants = giants[(giants['RA'] != '' ) & (giants['Dec'] != '')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants.add_column(Column(np.full(len(giants), np.nan, dtype='float64'),\n",
    "                                   name=\"sfr\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "02h09m41.3s\n",
      "32.42208333333333 0.2662777777777778\n",
      "13h29m34.18s\n",
      "202.39241666666663 22.724249999999998\n",
      "02h18m30.7s\n",
      "34.62791666666666 -5.525277777777777\n",
      "15h45m06.35s\n",
      "236.2764583333333 -34.721722222222226\n",
      "15h41m32.71s\n",
      "235.38629166666664 -35.05647222222222\n",
      "12h37m12s\n",
      "189.29999999999998 62.37002777777778\n",
      "logSFR= will not transform to SFR\n",
      "logSFR= will not transform to SFR\n",
      "logSFR= will not transform to SFR\n",
      "logSFR= will not transform to SFR\n",
      "logSFR= will not transform to SFR\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n",
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n",
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n",
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n",
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n",
      "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/row.py:54: StringTruncateWarning: truncated right side string(s) longer than 13 character(s) during assignment\n",
      "  self._table.columns[item][self._index] = val\n"
     ]
    }
   ],
   "source": [
    "for gal in giants:\n",
    "    if 'h' in gal['RA']:\n",
    "        print(gal['RA'])\n",
    "        hours = gal['RA'].split('h')[0]\n",
    "        minutes = gal['RA'].split('h')[1].split('m')[0]\n",
    "        seconds = gal['RA'].split('h')[1].split('m')[1].split('s')[0]\n",
    "        coords = SkyCoord(gal['RA'].replace(' ', ''), gal['Dec'].replace(' ', ''))\n",
    "        print(coords.ra.value, coords.dec.value)\n",
    "        gal['RA'] = str(coords.ra.value)\n",
    "        gal['Dec'] = str(coords.dec.value)\n",
    "        \n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('m', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('u', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace(' ', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('N', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('Y', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('~', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('?', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('=', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('A', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('/', '')\n",
    "    gal['Lensed?'] = gal['Lensed?'].replace('nlensed', '')\n",
    "    \n",
    "    if 'logSFR' in gal['Lbol/Lsun intrinsic']:\n",
    "        try:\n",
    "            gal['sfr'] = 10**float(gal['Lbol/Lsun intrinsic'].replace('logSFR=', ''))\n",
    "        except:\n",
    "            print('{} will not transform to SFR'.format(gal['Lbol/Lsun intrinsic']))\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants['RA'] = giants['RA'].astype(float)\n",
    "giants['Dec'] = giants['Dec'].astype(float)\n",
    "\n",
    "giants['Lbol/Lsun intrinsic'] = giants['Lbol/Lsun intrinsic'].astype('<U22')\n",
    "for gal in giants:\n",
    "    try:\n",
    "        test = float(gal['Redshift'])\n",
    "    except:\n",
    "        gal['Redshift'] = '-99'\n",
    "    gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('~', '')\n",
    "    gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('>', '')\n",
    "    gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].split('-')[0]\n",
    "    if 'logFIR~' in gal['Lbol/Lsun intrinsic']:\n",
    "        gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('logFIR~', '')   \n",
    "    if 'log' in gal['Lbol/Lsun intrinsic']:\n",
    "        gal['Lbol/Lsun intrinsic'] = '-99'\n",
    "        #gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('logSFR', '')\n",
    "        #gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('logFIR', '')\n",
    "        #gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('=', '')\n",
    "        #gal['Lbol/Lsun intrinsic'] = gal['Lbol/Lsun intrinsic'].replace('~', '')\n",
    "    elif not (('e' in gal['Lbol/Lsun intrinsic']) \n",
    "            or ('E' in gal['Lbol/Lsun intrinsic'])):\n",
    "        gal['Lbol/Lsun intrinsic'] = str(10** float(gal['Lbol/Lsun intrinsic']) )\n",
    "\n",
    "\n",
    "giants['Redshift'] = giants['Redshift'].astype(float)\n",
    "giants['Redshift'][giants['Redshift'] < 0] =np.nan\n",
    "\n",
    "giants['Lbol/Lsun intrinsic'][giants['Lbol/Lsun intrinsic'] == ''] = '-99'\n",
    "giants['Lbol/Lsun intrinsic'] = giants['Lbol/Lsun intrinsic'].astype(float)\n",
    "giants['Lbol/Lsun intrinsic'][giants['Lbol/Lsun intrinsic'] < 0] =np.nan\n",
    "\n",
    "giants['Lensed?'][giants['Lensed?'] == ''] = '-99'\n",
    "giants['Lensed?'] = giants['Lensed?'].astype(float)\n",
    "giants['Lensed?'][giants['Lensed?'] < 0] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for gal in giants['Lbol/Lsun intrinsic'] :\n",
    "    try:\n",
    "        test = float(gal)\n",
    "    except:\n",
    "        print(gal)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Column name=&apos;Lbol/Lsun intrinsic&apos; dtype=&apos;float64&apos; length=105&gt;\n",
       "<table>\n",
       "<tr><td>10000000000000.0</td></tr>\n",
       "<tr><td>10100000000000.0</td></tr>\n",
       "<tr><td>10200000000000.0</td></tr>\n",
       "<tr><td>10600000000000.0</td></tr>\n",
       "<tr><td>10700000000000.0</td></tr>\n",
       "<tr><td>10800000000000.0</td></tr>\n",
       "<tr><td>11000000000000.0</td></tr>\n",
       "<tr><td>11100000000000.0</td></tr>\n",
       "<tr><td>11200000000000.0</td></tr>\n",
       "<tr><td>11300000000000.0</td></tr>\n",
       "<tr><td>11500000000000.0</td></tr>\n",
       "<tr><td>11600000000000.0</td></tr>\n",
       "<tr><td>...</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "<tr><td>nan</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Column name='Lbol/Lsun intrinsic' dtype='float64' length=105>\n",
       "10000000000000.0\n",
       "10100000000000.0\n",
       "10200000000000.0\n",
       "10600000000000.0\n",
       "10700000000000.0\n",
       "10800000000000.0\n",
       "11000000000000.0\n",
       "11100000000000.0\n",
       "11200000000000.0\n",
       "11300000000000.0\n",
       "11500000000000.0\n",
       "11600000000000.0\n",
       "             ...\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan\n",
       "             nan"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(giants['Lbol/Lsun intrinsic'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants['RA'].name = 'ra'\n",
    "giants['Dec'].name = 'dec'\n",
    "giants['Redshift'].name = 'redshift'\n",
    "giants['Lbol/Lsun intrinsic'].name = 'lbol_intrinsic'\n",
    "giants['Lensed?'].name = 'mu_lens'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants.add_column(Column(gen_help_id(giants['ra'], giants['dec']),\n",
    "                                   name=\"help_id\"))\n",
    "giants.add_column(Column(np.full(len(giants), \"\", dtype='<U18'),\n",
    "                                   name=\"field\"))\n",
    "\n",
    "giants.add_column(Column(\n",
    "    data=coords_to_hpidx(giants['ra'], giants['dec'], order=13),\n",
    "    name=\"hp_idx\"\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants['help_id'] = [s.replace('HELP', 'HELP_rare') for s in giants['help_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields_info = yaml.load(open(\"../../dmu2/meta_main.yml\", 'r'))\n",
    "\n",
    "for field in fields_info['fields']:\n",
    "\n",
    "    field_moc = MOC(filename='../../dmu2/dmu2_field_coverages/{}_MOC.fits'.format(field['name']))\n",
    "    giants['field'][inMoc(giants['ra'], giants['dec'], field_moc)] = field['name']\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=10</i>\n",
       "<table id=\"table34728150912-918895\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>help_id</th><th>hp_idx</th><th>field</th><th>ra</th><th>dec</th><th>redshift</th><th>lbol_intrinsic</th><th>mu_lens</th><th>sfr</th></tr></thead>\n",
       "<tr><td>0</td><td>HELP_rare_J020941.300+001558.600</td><td>291882535</td><td>Herschel-Stripe-82</td><td>32.42208333333333</td><td>0.26627777777</td><td>2.553</td><td>10000000000000.0</td><td>10.0</td><td>nan</td></tr>\n",
       "<tr><td>1</td><td>HELP_rare_J132934.180+224327.300</td><td>167978479</td><td></td><td>202.39241666666663</td><td>22.7242499999</td><td>2.0439</td><td>13000000000000.0</td><td>11.0</td><td>nan</td></tr>\n",
       "<tr><td>2</td><td>HELP_rare_J021830.700-053131.000</td><td>289810432</td><td>XMM-LSS</td><td>34.62791666666666</td><td>-5.5252777777</td><td>3.4</td><td>13000000000000.0</td><td>5.0</td><td>nan</td></tr>\n",
       "<tr><td>3</td><td>HELP_rare_J154506.350-344318.200</td><td>700199708</td><td></td><td>236.2764583333333</td><td>-34.721722222</td><td>nan</td><td>10000000000000.0</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>4</td><td>HELP_rare_J154132.710-350323.300</td><td>700044386</td><td></td><td>235.38629166666664</td><td>-35.056472222</td><td>3.0</td><td>50118723362727.15</td><td>1.2</td><td>nan</td></tr>\n",
       "<tr><td>5</td><td>HELP_rare_J123712.000+622212.100</td><td>184354581</td><td>HDF-N</td><td>189.29999999999998</td><td>62.3700277777</td><td>4.06</td><td>29000000000000.0</td><td>nan</td><td>nan</td></tr>\n",
       "<tr><td>6</td><td>HELP_rare_J010311.500-453853.902</td><td>579939617</td><td></td><td>15.797917</td><td>-45.648306</td><td>3.0917</td><td>16700000000000.0</td><td>5.34</td><td>nan</td></tr>\n",
       "<tr><td>7</td><td>HELP_rare_J012507.080-472356.000</td><td>574017511</td><td></td><td>21.2795</td><td>-47.398889</td><td>2.5148</td><td>33300000000000.0</td><td>5.47</td><td>nan</td></tr>\n",
       "<tr><td>8</td><td>HELP_rare_J024308.810-462124.300</td><td>553364920</td><td></td><td>40.786708</td><td>-46.35675</td><td>5.699</td><td>16300000000000.0</td><td>5.09</td><td>nan</td></tr>\n",
       "<tr><td>9</td><td>HELP_rare_J030004.370-462124.300</td><td>553435566</td><td></td><td>45.018208</td><td>-46.35675</td><td>3.5954</td><td>12600000000000.0</td><td>6.3</td><td>nan</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table34728150912-918895').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table34728150912-918895').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 2, 4, 5, 6, 7, 8, 9], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "giants['help_id', \n",
    "        'hp_idx',\n",
    "        'field', \n",
    "        'ra', \n",
    "        'dec', \n",
    "        'redshift', \n",
    "        'lbol_intrinsic',\n",
    "        'mu_lens', \n",
    "        'sfr'][:10].show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(giants)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(giants['lbol_intrinsic']  == np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "giants['help_id', \n",
    "        'hp_idx',\n",
    "        'field', \n",
    "        'ra', \n",
    "        'dec', \n",
    "        'redshift', \n",
    "        'lbol_intrinsic',\n",
    "        'mu_lens',\n",
    "        'sfr'].write('./data/rare_objects.csv', overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python (herschelhelp_internal)",
   "language": "python",
   "name": "helpint"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}