{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# XMM-LSS - Merging HELP data products\n", "\n", "This notebook merges the various HELP data products on XMM-LSS. Due to the large size of HS82 we must run the code over individual tiles as produced at the masterlist stage.\n", "\n", "It is first used to create a catalogue that will be used for SED fitting by CIGALE by merging the optical master list, the photo-z and the XID+ far infrared fluxes. Then, this notebook is used to incorporate the CIGALE physical parameter estimations and generate the final HELP data product on the field." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "\n", "from astropy.table import Column, MaskedColumn, Table, join, vstack\n", "\n", "import glob\n", "\n", "from herschelhelp.filters import get_filter_meta_table\n", "\n", "from herschelhelp_internal.utils import add_column_meta\n", "\n", "filter_mean_lambda = {\n", " item['filter_id']: item['mean_wavelength'] for item in\n", " get_filter_meta_table()\n", "}" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Set this to true to produce only the catalogue for CIGALE and to false \n", "# to continue and merge the CIGALE results too.\n", "MAKE_CIGALE = False\n", "MAKE_FINAL = True\n", "\n", "\n", "SUFFIX = '20190328'" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# On Herschel-Stripe-82 we have a list of tiles due to the large size of the field. \n", "# Therefore, we here process each in turn\n", "file_list = glob.glob('../../dmu1/dmu1_ml_XMM-LSS/data/tiles/sub_catalogue_xmm-lss_{}_*.fits'.format(SUFFIX))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Reading the masterlist, XID+, and photo-z catalogues" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# # Master list\n", "\n", "# ml = Table.read(\n", "# \"../../dmu1/dmu1_ml_CDFS-SWIRE/data/master_catalogue_cdfs-swire_20180129.fits\")\n", "# ml.meta = None" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='./output/Tile_18095854_11_MIPS24_cat.fits' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='./output/Tile_18095854_11_MIPS24_cat.fits'\n", "WARNING: MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-01-06T14:40:25' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-01-06T14:40:25'\n" ] } ], "source": [ "# # XID+ MIPS24\n", "\n", "\n", "\n", "xid_mips24_1 = Table.read(\"../../dmu26/dmu26_XID+MIPS_XMM-LSS/data/\"\n", " \"dmu26_XID+MIPS_XMM-LSS_SPUDS_cat_20181210.fits\")\n", "\n", "xid_mips24_2 = Table.read(\"../../dmu26/dmu26_XID+MIPS_XMM-LSS/data/\"\n", " \"dmu26_XID+MIPS_XMM-LSS_SWIREnSPUDS_cat_20190106.fits\")\n", "\n", "xid_mips24 = vstack([xid_mips24_1, xid_mips24_2])\n", "\n", "xid_mips24.meta = None\n", "\n", "# Adding the error column\n", "xid_mips24.add_column(Column(\n", " data=np.max([xid_mips24['FErr_MIPS_24_u'] - xid_mips24['F_MIPS_24'],\n", " xid_mips24['F_MIPS_24'] - xid_mips24['FErr_MIPS_24_l']],\n", " axis=0),\n", " name=\"ferr_mips_24\"\n", "))\n", "xid_mips24['F_MIPS_24'].name = \"f_mips_24\"\n", "xid_mips24 = xid_mips24['help_id', 'f_mips_24', 'ferr_mips_24', 'flag_mips_24']" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='./data/SWIRE/Tile_18095854_11_PACS_SWIRE_cat.fits' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='./data/SWIRE/Tile_18095854_11_PACS_SWIRE_cat.fits'\n", "WARNING: MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-02-15T10:23:51' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-02-15T10:23:51'\n" ] } ], "source": [ "# XID+ PACS\n", "\n", "xid_pacs_1 = Table.read(\"../../dmu26/dmu26_XID+PACS_XMM-LSS/data/\"\n", " \"dmu26_XID+PACS_XMM-LSS_SPUDS_cat20190215.fits\")\n", "\n", "xid_pacs_2 = Table.read(\"../../dmu26/dmu26_XID+PACS_XMM-LSS/data/\"\n", " \"dmu26_XID+PACS_XMM-LSS_SWIREnSPUDS_cat20190215.fits\")\n", "\n", "xid_pacs = vstack([xid_pacs_1, xid_pacs_2])\n", "\n", "xid_pacs.meta = None\n", "\n", "# Convert from mJy to μJy\n", "for col in [\"F_PACS_100\", \"FErr_PACS_100_u\", \"FErr_PACS_100_l\",\n", " \"F_PACS_160\", \"FErr_PACS_160_u\", \"FErr_PACS_160_l\"]:\n", " xid_pacs[col] *= 1000\n", "\n", "xid_pacs.add_column(Column(\n", " data=np.max([xid_pacs['FErr_PACS_100_u'] - xid_pacs['F_PACS_100'],\n", " xid_pacs['F_PACS_100'] - xid_pacs['FErr_PACS_100_l']],\n", " axis=0),\n", " name=\"ferr_pacs_green\"\n", "))\n", "xid_pacs['F_PACS_100'].name = \"f_pacs_green\"\n", "xid_pacs['flag_PACS_100'].name = \"flag_pacs_green\"\n", "\n", "xid_pacs.add_column(Column(\n", " data=np.max([xid_pacs['FErr_PACS_160_u'] - xid_pacs['F_PACS_160'],\n", " xid_pacs['F_PACS_160'] - xid_pacs['FErr_PACS_160_l']],\n", " axis=0),\n", " name=\"ferr_pacs_red\"\n", "))\n", "xid_pacs['F_PACS_160'].name = \"f_pacs_red\"\n", "xid_pacs['flag_PACS_160'].name = \"flag_pacs_red\"\n", "\n", "xid_pacs = xid_pacs['help_id', 'f_pacs_green', 'ferr_pacs_green',\n", " 'flag_pacs_green', 'f_pacs_red', 'ferr_pacs_red',\n", " 'flag_pacs_red']\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='Tile_1130990_9_SPIRE_cat.fits' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'EXTNAME' types and , choosing EXTNAME='Tile_1130990_9_SPIRE_cat.fits'\n", "WARNING: MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-01-05T13:28:52' [astropy.utils.metadata]\n", "WARNING:astropy:MergeConflictWarning: Cannot merge meta key 'DATE-HDU' types and , choosing DATE-HDU='2019-01-05T13:28:52'\n" ] } ], "source": [ "# XID+ SPIRE\n", "\n", "xid_spire_1 = Table.read(\"../../dmu26/dmu26_XID+SPIRE_XMM-LSS/data/\"\n", " \"dmu26_XID+SPIRE_XMM-LSS_SPUDS_cat_20190107.fits\")\n", "\n", "xid_spire_2 = Table.read(\"../../dmu26/dmu26_XID+SPIRE_XMM-LSS/data/\"\n", " \"dmu26_XID+SPIRE_XMM-LSS_SWIREnSPUDS_cat_20190107.fits\")\n", "\n", "xid_spire = vstack([xid_spire_1, xid_spire_2])\n", "\n", "xid_spire.meta = None\n", "\n", "xid_spire['HELP_ID'].name = \"help_id\"\n", "\n", "# Convert from mJy to μJy\n", "for col in [\"F_SPIRE_250\", \"FErr_SPIRE_250_u\", \"FErr_SPIRE_250_l\",\n", " \"F_SPIRE_350\", \"FErr_SPIRE_350_u\", \"FErr_SPIRE_350_l\",\n", " \"F_SPIRE_500\", \"FErr_SPIRE_500_u\", \"FErr_SPIRE_500_l\"]:\n", " xid_spire[col] *= 1000\n", "\n", "xid_spire.add_column(Column(\n", " data=np.max([xid_spire['FErr_SPIRE_250_u'] - xid_spire['F_SPIRE_250'],\n", " xid_spire['F_SPIRE_250'] - xid_spire['FErr_SPIRE_250_l']],\n", " axis=0),\n", " name=\"ferr_spire_250\"\n", "))\n", "xid_spire['F_SPIRE_250'].name = \"f_spire_250\"\n", "xid_spire.add_column(Column(\n", " data=np.max([xid_spire['FErr_SPIRE_350_u'] - xid_spire['F_SPIRE_350'],\n", " xid_spire['F_SPIRE_350'] - xid_spire['FErr_SPIRE_350_l']],\n", " axis=0),\n", " name=\"ferr_spire_350\"\n", "))\n", "xid_spire['F_SPIRE_350'].name = \"f_spire_350\"\n", "xid_spire.add_column(Column(\n", " data=np.max([xid_spire['FErr_SPIRE_500_u'] - xid_spire['F_SPIRE_500'],\n", " xid_spire['F_SPIRE_500'] - xid_spire['FErr_SPIRE_500_l']],\n", " axis=0),\n", " name=\"ferr_spire_500\"\n", "))\n", "xid_spire['F_SPIRE_500'].name = \"f_spire_500\"\n", "\n", "xid_spire = xid_spire['help_id',\n", " 'f_spire_250', 'ferr_spire_250', 'flag_spire_250',\n", " 'f_spire_350', 'ferr_spire_350', 'flag_spire_350',\n", " 'f_spire_500', 'ferr_spire_500', 'flag_spire_500']" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# Photo-z\n", "\n", "photoz = Table.read(\"../../dmu24/dmu24_XMM-LSS/data/master_catalogue_xmm-lss_20180221_photoz_20180518_r_optimised.fits\")\n", "photoz.meta = None\n", "\n", "photoz = photoz['help_id', 'z1_median']\n", "photoz['z1_median'].name = 'redshift'\n", "\n", "photoz['redshift'][photoz['redshift'] < 0] = np.nan # -99 used for missing values" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "# # Temp spec-z\n", "\n", "# ml['zspec'].name = 'redshift'\n", "\n", "# ml['redshift'][ml['redshift'] < 0] = np.nan # -99 used for missing values" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Merging" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def help_merge(table1, table2):\n", " out_table = join(table1, table2, join_type='left', keys='help_id')\n", " for col in out_table.colnames:\n", " if out_table[col].dtype == 'float64':\n", " out_table[col].fill_value = np.nan\n", " elif out_table[col].dtype == 'float32':\n", " out_table[col].fill_value = np.nan\n", " elif out_table[col].dtype == '>f8':\n", " out_table[col].fill_value = np.nan\n", " elif out_table[col].dtype == '|S27':\n", " out_table[col].fill_value = ''\n", " elif out_table[col].dtype == '|S18':\n", " out_table[col].fill_value = ''\n", " elif out_table[col].dtype == '|S18':\n", " out_table[col].fill_value = ''\n", " elif out_table[col].dtype == '>i8':\n", " out_table[col].fill_value = -1\n", " elif out_table[col].dtype == 'bool':\n", " out_table[col].fill_value = False\n", "\n", "\n", " return out_table.filled()\n", "\n", "\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# merged_table = join(ml, xid_mips24, join_type='left')\n", "\n", "# # Fill values\n", "# for col in xid_mips24.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "# merged_table = join(merged_table, xid_pacs, join_type='left')\n", " \n", "# # Fill values\n", "# for col in xid_pacs.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# merged_table = join(merged_table, xid_spire, join_type='left')\n", " \n", "# # Fill values\n", "# for col in xid_spire.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "#merged_table = join(merged_table, photoz, join_type='left')\n", "\n", "# Fill values\n", "#merged_table['redshift'].fill_value = np.nan\n", "#merged_table = merged_table.filled()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Saving the catalogue for CIGALE (first run)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "if MAKE_CIGALE:\n", " \n", " for tile in file_list:\n", " \n", " file_num = tile.split('_')[6].split('.')[0]\n", " ml = Table.read(tile)\n", " ml.meta = None\n", " \n", " \n", "\n", " ml['zspec'][ml['zspec'] < 0] = np.nan # -99 used for missing values\n", " \n", " \n", "\n", " \n", " \n", " \n", " #ADD in XID+ merging here\n", " merged_table = help_merge(ml, xid_mips24)\n", " merged_table = help_merge(merged_table, xid_pacs)\n", " merged_table = help_merge(merged_table, xid_spire)\n", " \n", " #photo-z merging\n", " merged_table = help_merge(merged_table, photoz)\n", " \n", " #flag merging\n", " flags = Table.read(\"../../dmu6/dmu6_v_XMM-LSS/data/tiles/xmm-lss_20180504_{}_flags.fits\".format(file_num))\n", " for col in flags.colnames:\n", " if 'flag' in col:\n", " try:\n", " merged_table.remove_column(col)\n", " except KeyError:\n", " print(\"Column: {} not in masterlist.\".format(col))\n", " \n", " merged_table = join(merged_table, flags, join_type='left')\n", " # Fill values\n", " for col in merged_table.colnames:\n", " if 'flag' in col:\n", " merged_table[col].fill_value = False\n", " merged_table = merged_table.filled()\n", " \n", " \n", " \n", " \n", " \n", " # Sorting the columns\n", " bands_tot = [col[2:] for col in merged_table.colnames\n", " if col.startswith('f_') and not col.startswith('f_ap')]\n", " bands_ap = [col[5:] for col in merged_table.colnames\n", " if col.startswith('f_ap_') ]\n", " bands = list(set(bands_tot) | set(bands_ap))\n", " bands.sort(key=lambda x: filter_mean_lambda[x])\n", " \n", " columns = ['help_id', 'field', 'ra', 'dec', 'hp_idx', 'ebv', 'redshift', \n", " 'zspec']\n", " for band in bands:\n", " for col_tpl in ['f_{}', 'ferr_{}', 'f_ap_{}', 'ferr_ap_{}',\n", " 'm_{}', 'merr_{}', 'm_ap_{}', 'merr_ap_{}',\n", " 'flag_{}']:\n", " colname = col_tpl.format(band)\n", " if colname in merged_table.colnames:\n", " columns.append(colname)\n", " columns += ['stellarity', 'stellarity_origin', 'flag_cleaned',\n", " 'flag_merged', 'flag_gaia', 'flag_optnir_obs',\n", " 'flag_optnir_det', 'zspec_qual', 'zspec_association_flag']\n", " \n", "\n", " # Check that we did not forget any column\n", " print( set(columns) - set(merged_table.colnames))\n", " print( set(merged_table.colnames) - set(columns) )\n", " \n", " merged_table = add_column_meta(merged_table, '../columns.yml')\n", " merged_table[columns].write(\"data/tiles/xmm-lss_{}_cigale_{}.fits\".format(SUFFIX, file_num), overwrite=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Merging CIGALE outputs\n", "\n", "We merge the CIGALE outputs to the main catalogue. The CIGALE products provides several χ² with associated thresholds. For simplicity, we convert these two values to flags." ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "if MAKE_FINAL:\n", "\n", " # Cigale outputs\n", " cigale = Table.read(\"../../dmu28/dmu28_XMM-LSS/data/zphot/HELP_final_results.fits\")\n", " cigale['id'].name = \"help_id\"\n", "\n", " # We convert the various Chi2 and threshold to flags\n", " flag_cigale_opt = cigale[\"UVoptIR_OPTchi2\"] <= cigale[\"UVoptIR_OPTchi2_threshold\"]\n", " flag_cigale_ir = cigale[\"UVoptIR_IRchi2\"] <= cigale[\"UVoptIR_IRchi2_threshold\"]\n", " flag_cigale = (\n", " (cigale[\"UVoptIR_best.reduced_chi_square\"] \n", " <= cigale[\"UVoptIR_best.reduced_chi_square_threshold\"]) &\n", " flag_cigale_opt & flag_cigale_ir)\n", " flag_cigale_ironly = cigale[\"IRonly_IRchi2\"] <= cigale[\"IRonly_IRchi2_threshold\"]\n", "\n", " cigale.add_columns([\n", " MaskedColumn(flag_cigale, \"flag_cigale\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_opt, \"flag_cigale_opt\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_ir, \"flag_cigale_ir\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_ironly, \"flag_cigale_ironly\", \n", " dtype=int, fill_value=-1)\n", " ])\n", "\n", " cigale['UVoptIR_bayes.stellar.m_star'].name = \"cigale_mstar\"\n", " cigale['UVoptIR_bayes.stellar.m_star_err'].name = \"cigale_mstar_err\"\n", " cigale['UVoptIR_bayes.sfh.sfr10Myrs'].name = \"cigale_sfr\"\n", " cigale['UVoptIR_bayes.sfh.sfr10Myrs_err'].name = \"cigale_sfr_err\"\n", " cigale['UVoptIR_bayes.dust.luminosity'].name = \"cigale_dustlumin\"\n", " cigale['UVoptIR_bayes.dust.luminosity_err'].name = \"cigale_dustlumin_err\"\n", " cigale['IR_bayes.dust.luminosity'].name = \"cigale_dustlumin_ironly\"\n", " cigale['IR_bayes.dust.luminosity_err'].name = \"cigale_dustlumin_ironly_err\"\n", "\n", " cigale = cigale['help_id', 'cigale_mstar', 'cigale_mstar_err', 'cigale_sfr',\n", " 'cigale_sfr_err', 'cigale_dustlumin', 'cigale_dustlumin_err', \n", " 'cigale_dustlumin_ironly', 'cigale_dustlumin_ironly_err', \n", " 'flag_cigale', 'flag_cigale_opt', 'flag_cigale_ir', \n", " 'flag_cigale_ironly']" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/opt/pyenv/versions/3.7.2/lib/python3.7/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'ferr_hawki_y'}\n", "Column ferr_hawki_y has missing data\n" ] } ], "source": [ "if MAKE_FINAL:\n", "\n", " for tile in file_list:\n", " \n", " file_num = tile.split('_')[6].split('.')[0]\n", " ml = Table.read(tile)\n", " ml.meta = None\n", " \n", " \n", "\n", " ml['zspec'][ml['zspec'] < 0] = np.nan # -99 used for missing values\n", " \n", " \n", "\n", " \n", " \n", " \n", " #ADD in XID+ merging here\n", " merged_table = help_merge(ml, xid_mips24)\n", " merged_table = help_merge(merged_table, xid_pacs)\n", " merged_table = help_merge(merged_table, xid_spire)\n", " merged_table = help_merge(merged_table, cigale)\n", " \n", " #photo-z merging\n", " merged_table = help_merge(merged_table, photoz)\n", " \n", " #flag merging\n", " flags = Table.read(\"../../dmu6/dmu6_v_XMM-LSS/data/tiles/xmm-lss_20180504_{}_flags.fits\".format(file_num))\n", " for col in flags.colnames:\n", " if 'flag' in col:\n", " try:\n", " merged_table.remove_column(col)\n", " except KeyError:\n", " print(\"Column: {} not in masterlist.\".format(col))\n", " \n", " merged_table = join(merged_table, flags, join_type='left')\n", " # Fill values\n", " for col in merged_table.colnames:\n", " if 'flag' in col:\n", " merged_table[col].fill_value = False\n", " merged_table = merged_table.filled()\n", " \n", " \n", " \n", " \n", " \n", " # Sorting the columns\n", " bands_tot = [col[2:] for col in merged_table.colnames\n", " if col.startswith('f_') and not col.startswith('f_ap')]\n", " bands_ap = [col[5:] for col in merged_table.colnames\n", " if col.startswith('f_ap_') ]\n", " bands = list(set(bands_tot) | set(bands_ap))\n", " bands.sort(key=lambda x: filter_mean_lambda[x])\n", " \n", " columns = ['help_id', 'field', 'ra', 'dec', 'hp_idx', 'ebv', 'redshift', \n", " 'zspec']\n", " for band in bands:\n", " for col_tpl in ['f_{}', 'ferr_{}', 'f_ap_{}', 'ferr_ap_{}',\n", " 'm_{}', 'merr_{}', 'm_ap_{}', 'merr_ap_{}',\n", " 'flag_{}']:\n", " colname = col_tpl.format(band)\n", " if colname in merged_table.colnames:\n", " columns.append(colname)\n", " columns += ['cigale_mstar', 'cigale_mstar_err', 'cigale_sfr',\n", " 'cigale_sfr_err', 'cigale_dustlumin', 'cigale_dustlumin_err', \n", " 'cigale_dustlumin_ironly', 'cigale_dustlumin_ironly_err', \n", " 'flag_cigale', 'flag_cigale_opt', 'flag_cigale_ir', \n", " 'flag_cigale_ironly','stellarity', 'stellarity_origin', 'flag_cleaned',\n", " 'flag_merged', 'flag_gaia', 'flag_optnir_obs',\n", " 'flag_optnir_det', 'zspec_qual', 'zspec_association_flag']\n", " \n", "\n", " # Check that we did not forget any column\n", " print( set(columns) - set(merged_table.colnames))\n", " print( set(merged_table.colnames) - set(columns) )\n", " \n", " merged_table = add_column_meta(merged_table, '../columns.yml')\n", " merged_table[columns].write(\"data/tiles/xmm-lss_{}_{}.fits\".format(SUFFIX, file_num), overwrite=True)" ] } ], "metadata": { "anaconda-cloud": {}, "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.2" } }, "nbformat": 4, "nbformat_minor": 2 }