{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# SPIRE-NEP - Merging HELP data products\n", "\n", "This notebook merges the various HELP data products on SPIRE-NEP.\n", "\n", "It is first used to create a catalogue that will be used for SED fitting by CIGALE by merging the optical master list, the photo-z and the XID+ far infrared fluxes. Then, this notebook is used to incorporate the CIGALE physical parameter estimations and generate the final HELP data product on the field." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Set this to true to produce only the catalogue for CIGALE and to false \n", "# to continue and merge the CIGALE results too.\n", "FIRST_RUN_FOR_CIGALE = True" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import numpy as np\n", "\n", "from astropy.table import Column, MaskedColumn, Table, join, vstack\n", "\n", "from herschelhelp.filters import get_filter_meta_table\n", "\n", "from herschelhelp_internal.utils import add_column_meta\n", "\n", "filter_mean_lambda = {\n", " item['filter_id']: item['mean_wavelength'] for item in\n", " get_filter_meta_table()\n", "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Reading the masterlist, XID+, and photo-z catalogues" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Master list\n", "SUFFIX = '20180220' #??\n", "ml = Table.read(\n", " \"../../dmu1/dmu1_ml_SPIRE-NEP/data/master_catalogue_spire-nep_{}.fits\".format(SUFFIX))\n", "ml.meta = None" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# # XID+ MIPS24\n", "\n", "# xid_mips24 = Table.read(\"../../dmu26/dmu26_XID+MIPS_CDFS-SWIRE/data/\"\n", "# \"dmu26_XID+MIPS_CDFS-SWIRE_cat_20170901.fits\")\n", "# xid_mips24.meta = None\n", "\n", "# # Adding the error column\n", "# xid_mips24.add_column(Column(\n", "# data=np.max([xid_mips24['FErr_MIPS_24_u'] - xid_mips24['F_MIPS_24'],\n", "# xid_mips24['F_MIPS_24'] - xid_mips24['FErr_MIPS_24_l']],\n", "# axis=0),\n", "# name=\"ferr_mips_24\"\n", "# ))\n", "# xid_mips24['F_MIPS_24'].name = \"f_mips_24\"\n", "# xid_mips24 = xid_mips24['help_id', 'f_mips_24', 'ferr_mips_24', 'flag_mips_24']" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# # XID+ PACS\n", "\n", "# xid_pacs = Table.read(\"../../dmu26/dmu26_XID+PACS_CDFS-SWIRE/data/\"\n", "# \"dmu26_XID+PACS_CDFS-SWIRE_cat_20171019.fits\")\n", "# xid_pacs.meta = None\n", "\n", "# # Convert from mJy to μJy\n", "# for col in [\"F_PACS_100\", \"FErr_PACS_100_u\", \"FErr_PACS_100_l\",\n", "# \"F_PACS_160\", \"FErr_PACS_160_u\", \"FErr_PACS_160_l\"]:\n", "# xid_pacs[col] *= 1000\n", "\n", "# xid_pacs.add_column(Column(\n", "# data=np.max([xid_pacs['FErr_PACS_100_u'] - xid_pacs['F_PACS_100'],\n", "# xid_pacs['F_PACS_100'] - xid_pacs['FErr_PACS_100_l']],\n", "# axis=0),\n", "# name=\"ferr_pacs_green\"\n", "# ))\n", "# xid_pacs['F_PACS_100'].name = \"f_pacs_green\"\n", "# xid_pacs['flag_PACS_100'].name = \"flag_pacs_green\"\n", "\n", "# xid_pacs.add_column(Column(\n", "# data=np.max([xid_pacs['FErr_PACS_160_u'] - xid_pacs['F_PACS_160'],\n", "# xid_pacs['F_PACS_160'] - xid_pacs['FErr_PACS_160_l']],\n", "# axis=0),\n", "# name=\"ferr_pacs_red\"\n", "# ))\n", "# xid_pacs['F_PACS_160'].name = \"f_pacs_red\"\n", "# xid_pacs['flag_PACS_160'].name = \"flag_pacs_red\"\n", "\n", "# xid_pacs = xid_pacs['help_id', 'f_pacs_green', 'ferr_pacs_green',\n", "# 'flag_pacs_green', 'f_pacs_red', 'ferr_pacs_red',\n", "# 'flag_pacs_red']\n" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# # XID+ SPIRE\n", "\n", "# xid_spire = Table.read(\"../../dmu26/dmu26_XID+SPIRE_CDFS-SWIRE/data/\"\n", "# \"dmu26_XID+SPIRE_CDFS-SWIRE_cat_20170919.fits\")\n", "# xid_spire.meta = None\n", "\n", "# xid_spire['HELP_ID'].name = \"help_id\"\n", "\n", "# # Convert from mJy to μJy\n", "# for col in [\"F_SPIRE_250\", \"FErr_SPIRE_250_u\", \"FErr_SPIRE_250_l\",\n", "# \"F_SPIRE_350\", \"FErr_SPIRE_350_u\", \"FErr_SPIRE_350_l\",\n", "# \"F_SPIRE_500\", \"FErr_SPIRE_500_u\", \"FErr_SPIRE_500_l\"]:\n", "# xid_spire[col] *= 1000\n", "\n", "# xid_spire.add_column(Column(\n", "# data=np.max([xid_spire['FErr_SPIRE_250_u'] - xid_spire['F_SPIRE_250'],\n", "# xid_spire['F_SPIRE_250'] - xid_spire['FErr_SPIRE_250_l']],\n", "# axis=0),\n", "# name=\"ferr_spire_250\"\n", "# ))\n", "# xid_spire['F_SPIRE_250'].name = \"f_spire_250\"\n", "# xid_spire.add_column(Column(\n", "# data=np.max([xid_spire['FErr_SPIRE_350_u'] - xid_spire['F_SPIRE_350'],\n", "# xid_spire['F_SPIRE_350'] - xid_spire['FErr_SPIRE_350_l']],\n", "# axis=0),\n", "# name=\"ferr_spire_350\"\n", "# ))\n", "# xid_spire['F_SPIRE_350'].name = \"f_spire_350\"\n", "# xid_spire.add_column(Column(\n", "# data=np.max([xid_spire['FErr_SPIRE_500_u'] - xid_spire['F_SPIRE_500'],\n", "# xid_spire['F_SPIRE_500'] - xid_spire['FErr_SPIRE_500_l']],\n", "# axis=0),\n", "# name=\"ferr_spire_500\"\n", "# ))\n", "# xid_spire['F_SPIRE_500'].name = \"f_spire_500\"\n", "\n", "# xid_spire = xid_spire['help_id',\n", "# 'f_spire_250', 'ferr_spire_250', 'flag_spire_250',\n", "# 'f_spire_350', 'ferr_spire_350', 'flag_spire_350',\n", "# 'f_spire_500', 'ferr_spire_500', 'flag_spire_500']" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Photo-z\n", "\n", "#photoz = Table.read(\"../../dmu24/dmu24_/data/\")\n", "#photoz.meta = None\n", "\n", "#photoz = photoz['help_id', 'z1_median']\n", "#photoz['z1_median'].name = 'redshift'\n", "\n", "#photoz['redshift'][photoz['redshift'] < 0] = np.nan # -99 used for missing values" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:965: RuntimeWarning: invalid value encountered in less\n", " return getattr(self.data, op)(other)\n" ] } ], "source": [ "# Temp spec-z\n", "\n", "ml['zspec'][ml['zspec'] < 0] = np.nan # -99 used for missing values" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Flags\n", "flags = Table.read(\"../../dmu6/dmu6_v_SPIRE-NEP/data/spire-nep_20180220_flags.fits\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Merging" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# merged_table = join(ml, xid_mips24, join_type='left')\n", "\n", "# # Fill values\n", "# for col in xid_mips24.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# merged_table = join(merged_table, xid_pacs, join_type='left')\n", " \n", "# # Fill values\n", "# for col in xid_pacs.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# merged_table = join(merged_table, xid_spire, join_type='left')\n", " \n", "# # Fill values\n", "# for col in xid_spire.colnames:\n", "# if col.startswith(\"f_\") or col.startswith(\"ferr_\"):\n", "# merged_table[col].fill_value = np.nan\n", "# elif col.startswith(\"flag_\"):\n", "# merged_table[col].fill_value = False\n", "# merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": true }, "outputs": [], "source": [ "#merged_table = join(merged_table, photoz, join_type='left')\n", "\n", "# Fill values\n", "#merged_table['redshift'].fill_value = np.nan\n", "#merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": true }, "outputs": [], "source": [ "merged_table = ml" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Column: flag_gpc1_g_ap not in masterlist.\n", "Column: flag_gpc1_r_ap not in masterlist.\n", "Column: flag_gpc1_i_ap not in masterlist.\n", "Column: flag_gpc1_z_ap not in masterlist.\n", "Column: flag_gpc1_y_ap not in masterlist.\n" ] } ], "source": [ "for col in flags.colnames:\n", " if 'flag' in col:\n", " try:\n", " merged_table.remove_column(col)\n", " except KeyError:\n", " print(\"Column: {} not in masterlist.\".format(col))\n", " \n", "merged_table = join(merged_table, flags, join_type='left')\n", "# Fill values\n", "for col in merged_table.colnames:\n", " if 'flag' in col:\n", " merged_table[col].fill_value = False\n", "merged_table = merged_table.filled()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Saving the catalogue for CIGALE (first run)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "collapsed": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "set()\n", "{'flag_gpc1_i_ap', 'flag_gpc1_z_ap', 'flag_gpc1_g_ap', 'flag_gpc1_r_ap', 'flag_gpc1_y_ap'}\n" ] } ], "source": [ "if FIRST_RUN_FOR_CIGALE:\n", " \n", " # Sorting the columns\n", " bands_tot = [col[2:] for col in merged_table.colnames\n", " if col.startswith('f_') and not col.startswith('f_ap')]\n", " bands_ap = [col[5:] for col in merged_table.colnames\n", " if col.startswith('f_ap_') ]\n", " bands = list(set(bands_tot) | set(bands_ap))\n", " bands.sort(key=lambda x: filter_mean_lambda[x])\n", " \n", " columns = ['help_id', 'field', 'ra', 'dec', 'hp_idx', 'ebv', #'redshift', \n", " 'zspec']\n", " for band in bands:\n", " for col_tpl in ['f_{}', 'ferr_{}', 'f_ap_{}', 'ferr_ap_{}',\n", " 'm_{}', 'merr_{}', 'm_ap_{}', 'merr_ap_{}',\n", " 'flag_{}']:\n", " colname = col_tpl.format(band)\n", " if colname in merged_table.colnames:\n", " columns.append(colname)\n", " columns += [# 'stellarity', 'stellarity_origin', \n", " 'flag_cleaned',\n", " # 'flag_merged', \n", " 'flag_gaia', 'flag_optnir_obs',\n", " 'flag_optnir_det', 'zspec_qual', 'zspec_association_flag']\n", " \n", "\n", " # Check that we did not forget any column\n", " print(set(columns) - set(merged_table.colnames))\n", " print( set(merged_table.colnames) - set(columns) )\n", " \n", " merged_table = add_column_meta(merged_table, '../columns.yml')\n", " merged_table[columns].write(\"data/SPIRE-NEP_{}_cigale.fits\".format(SUFFIX), overwrite=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Merging CIGALE outputs\n", "\n", "We merge the CIGALE outputs to the main catalogue. The CIGALE products provides several χ² with associated thresholds. For simplicity, we convert these two values to flags." ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " # Cigale outputs\n", " cigale = Table.read(\"../../dmu28/dmu28_CDFS-SWIRE/data/HELP_final_results.fits\")\n", " cigale['id'].name = \"help_id\"\n", "\n", " # We convert the various Chi2 and threshold to flags\n", " flag_cigale_opt = cigale[\"UVoptIR_OPTchi2\"] <= cigale[\"UVoptIR_OPTchi2_threshold\"]\n", " flag_cigale_ir = cigale[\"UVoptIR_IRchi2\"] <= cigale[\"UVoptIR_IRchi2_threshold\"]\n", " flag_cigale = (\n", " (cigale[\"UVoptIR_best.reduced_chi_square\"] \n", " <= cigale[\"UVoptIR_best.reduced_chi_square_threshold\"]) &\n", " flag_cigale_opt & flag_cigale_ir)\n", " flag_cigale_ironly = cigale[\"IRonly_IRchi2\"] <= cigale[\"IRonly_IRchi2_threshold\"]\n", "\n", " cigale.add_columns([\n", " MaskedColumn(flag_cigale, \"flag_cigale\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_opt, \"flag_cigale_opt\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_ir, \"flag_cigale_ir\", \n", " dtype=int, fill_value=-1),\n", " MaskedColumn(flag_cigale_ironly, \"flag_cigale_ironly\", \n", " dtype=int, fill_value=-1)\n", " ])\n", "\n", " cigale['UVoptIR_bayes.stellar.m_star'].name = \"cigale_mstar\"\n", " cigale['UVoptIR_bayes.stellar.m_star_err'].name = \"cigale_mstar_err\"\n", " cigale['UVoptIR_bayes.sfh.sfr10Myrs'].name = \"cigale_sfr\"\n", " cigale['UVoptIR_bayes.sfh.sfr10Myrs_err'].name = \"cigale_sfr_err\"\n", " cigale['UVoptIR_bayes.dust.luminosity'].name = \"cigale_dustlumin\"\n", " cigale['UVoptIR_bayes.dust.luminosity_err'].name = \"cigale_dustlumin_err\"\n", " cigale['IR_bayes.dust.luminosity'].name = \"cigale_dustlumin_ironly\"\n", " cigale['IR_bayes.dust.luminosity_err'].name = \"cigale_dustlumin_ironly_err\"\n", "\n", " cigale = cigale['help_id', 'cigale_mstar', 'cigale_mstar_err', 'cigale_sfr',\n", " 'cigale_sfr_err', 'cigale_dustlumin', 'cigale_dustlumin_err', \n", " 'cigale_dustlumin_ironly', 'cigale_dustlumin_ironly_err', \n", " 'flag_cigale', 'flag_cigale_opt', 'flag_cigale_ir', \n", " 'flag_cigale_ironly']" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " merged_table = join(merged_table, cigale, join_type='left')\n", "\n", " # Fill values\n", " for col in cigale.colnames:\n", " if col.startswith(\"cigale_\"):\n", " merged_table[col].fill_value = np.nan\n", " elif col.startswith(\"flag_\"):\n", " merged_table[col].fill_value = -1\n", " merged_table = merged_table.filled()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Sorting columns\n", "\n", "We sort the columns by increasing band wavelength." ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " bands = [col[2:] for col in merged_table.colnames\n", " if col.startswith('f_') and not col.startswith('f_ap')]\n", " bands.sort(key=lambda x: filter_mean_lambda[x])" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " columns = ['help_id', 'field', 'ra', 'dec', 'hp_idx', 'ebv', 'redshift', 'zspec']\n", " for band in bands:\n", " for col_tpl in ['f_{}', 'ferr_{}', 'f_ap_{}', 'ferr_ap_{}',\n", " 'm_{}', 'merr_{}', 'm_ap_{}', 'merr_ap_{}',\n", " 'flag_{}']:\n", " colname = col_tpl.format(band)\n", " if colname in merged_table.colnames:\n", " columns.append(colname)\n", " columns += ['cigale_mstar', 'cigale_mstar_err', 'cigale_sfr', 'cigale_sfr_err',\n", " 'cigale_dustlumin', 'cigale_dustlumin_err', 'cigale_dustlumin_ironly', \n", " 'cigale_dustlumin_ironly_err', 'flag_cigale', 'flag_cigale_opt', \n", " 'flag_cigale_ir', 'flag_cigale_ironly', 'stellarity', \n", " 'stellarity_origin', 'flag_cleaned', 'flag_merged', 'flag_gaia', \n", " 'flag_optnir_obs', 'flag_optnir_det', 'zspec_qual', \n", " 'zspec_association_flag']" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " # Check that we did not forget any column\n", " assert set(columns) == set(merged_table.colnames)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Saving" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "collapsed": true }, "outputs": [], "source": [ "if not FIRST_RUN_FOR_CIGALE:\n", "\n", " merged_table[columns].write(\"data/CDFS-SWIRE.fits\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }