{ "cells": [ { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "# Final Bootes master catalogue\n", "\n", "This notebook presents the combination of the HELP standard catalogue with the aperture matched photometry from Brown which gives superior photometric redhsifts." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "708e28f (Tue May 8 18:05:21 2018 +0100)\n", "This notebook was executed on: \n", "2018-05-16 18:12:48.187415\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/seaborn/apionly.py:6: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n", " warnings.warn(msg, UserWarning)\n" ] } ], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "import os\n", "import time\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table, join\n", "import numpy as np\n", "from pymoc import MOC\n", "\n", "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n", "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n", "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n", "\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass" ] }, { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "## I - Reading the prepared pristine catalogues" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "brown_catalogue_name =\"master_catalogue_bootes_brown_20180517.fits\"\n", "help_catalogue_name =\"master_catalogue_bootes_help_20180517.fits\"\n", "\n", "brown_cross_id_name = \"master_list_cross_ident_bootes_brown_20180517.fits\"\n", "help_cross_id_name = \"master_list_cross_ident_bootes_help_20180517.fits\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "brown_catalogue = Table.read(\"{}/{}\".format(OUT_DIR, brown_catalogue_name))\n", "help_catalogue = Table.read(\"{}/{}\".format(OUT_DIR, help_catalogue_name))" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "brown_cross_id = Table.read(\"{}/{}\".format(OUT_DIR,brown_cross_id_name))['full_help_id', 'help_id']\n", "brown_cross_id['help_id'].name = 'brown_help_id'\n", "brown_cross_id['full_help_id'].name = 'help_help_id'" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "in_brown = set(brown_catalogue.colnames)\n", "in_help = set(help_catalogue.colnames)\n", "in_both = in_help.intersection(in_brown)\n", "only_in_brown = in_brown - in_help\n", "\n", "for col in brown_catalogue.colnames:\n", " brown_catalogue[col].name = \"brown_{}\".format(col)\n", " \n", "for col in help_catalogue.colnames:\n", " help_catalogue[col].name = \"help_{}\".format(col)" ] }, { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "## 2 - Join the two catalogues taking brown if both are available" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [], "source": [ "master_catalogue = join(brown_cross_id, brown_catalogue, join_type='left')\n", "master_catalogue = join(master_catalogue, help_catalogue, join_type='left')\n", " \n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "in_both.remove('help_id')\n", "for col in in_both:\n", " has_brown = (master_catalogue['brown_help_id'] != '-1')\n", " \n", " try:\n", " has_brown = has_brown & ~np.isnan(master_catalogue[\"brown_{}\".format(col)])\n", " except TypeError:\n", " print(\"brown_{} is not a column of floats\".format(col))\n", " master_catalogue[\"help_{}\".format(col)][has_brown] = master_catalogue[\"brown_{}\".format(col)][has_brown]\n", " master_catalogue[\"help_{}\".format(col)].name = col\n", " master_catalogue.remove_column(\"brown_{}\".format(col))\n", " \n", "#Deal with ids seperately\n", "has_brown = (master_catalogue['brown_help_id'] != '-1')\n", "master_catalogue[\"help_help_id\"][has_brown] = master_catalogue[\"brown_help_id\"][has_brown]\n", "master_catalogue[\"help_help_id\"].name = \"help_id\"\n", "master_catalogue.remove_column('brown_help_id')\n", " \n", "for col in only_in_brown:\n", " master_catalogue[\"brown_{}\".format(col)].name = col" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "for col in master_catalogue.colnames:\n", " if \"m_\" in col or \"merr_\" in col or \"f_\" in col or \"ferr_\" in col or \"stellarity\" in col:\n", " master_catalogue[col].fill_value = np.nan\n", " elif \"flag\" in col:\n", " master_catalogue[col].fill_value = 0\n", " elif \"id\" in col:\n", " master_catalogue[col].fill_value = -1\n", " \n", "master_catalogue = master_catalogue.filled()" ] }, { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "## 3 - Crop to HELP area\n", "\n", "The Bootes Brown catalogue slightly goes outside the HELP Bootes field." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "help_bootes_moc = MOC(filename=\"../../dmu2/dmu2_field_coverages/Bootes_MOC.fits\")\n", "in_HELP_mask = inMoc(\n", " master_catalogue['ra'], master_catalogue['dec'],\n", " help_bootes_moc \n", ")\n", "\n", "master_catalogue = master_catalogue[in_HELP_mask]" ] }, { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "## 4 - Check and save" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "master_catalogue[:10].show_in_notebook()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "columns = [\"help_id\", \"field\", \"ra\", \"dec\", \"hp_idx\"]\n", "\n", "bands = [column[5:] for column in master_catalogue.colnames if 'f_ap' in column]\n", "for band in bands:\n", " columns += [\"f_ap_{}\".format(band), \"ferr_ap_{}\".format(band),\n", " \"m_ap_{}\".format(band), \"merr_ap_{}\".format(band),\n", " \"f_{}\".format(band), \"ferr_{}\".format(band),\n", " \"m_{}\".format(band), \"merr_{}\".format(band),\n", " #\"flag_{}\".format(band)\n", " ] \n", " \n", "columns += [\"stellarity\", \"flag_cleaned\", \"flag_merged\", \"flag_gaia\", \"flag_optnir_obs\", \n", " \"flag_optnir_det\", \"ebv\", 'zspec_association_flag', 'zspec_qual', 'zspec', \"stellarity_origin\"] " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "collapsed": true, "deletable": true, "editable": true }, "outputs": [], "source": [ "master_catalogue = master_catalogue.filled()\n", "print(columns)\n", "master_catalogue.write(\"{}/master_catalogue_bootes{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)" ] }, { "cell_type": "markdown", "metadata": { "deletable": true, "editable": true }, "source": [ "## 5 - Make final cross ID catalogue" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [], "source": [ "brown_cross_id = Table.read(\"{}/{}\".format(OUT_DIR,brown_cross_id_name))\n", "brown_cross_id['help_id'].name = 'brown_help_id'\n", "brown_cross_id['full_help_id'].name = 'help_help_id'\n", "\n", "\n", "help_cross_id = Table.read(\"{}/{}\".format(OUT_DIR,help_cross_id_name))\n", "help_cross_id.remove_columns(['specz_id', 'sdss_id'])\n", "help_cross_id['help_id'].name = 'help_help_id'\n", "\n", "final_cross_id = join(brown_cross_id, help_cross_id, join_type='left')\n", "\n", "\n", "\n", "\n", "\n", "for col in final_cross_id.colnames:\n", " final_cross_id[col].fill_value = -1\n", "final_cross_id = final_cross_id.filled()\n", "\n", "final_cross_id['specz_id'][final_cross_id['specz_id'] == ''] = '-1'\n", "\n", "final_cross_id.add_column(Column(data=final_cross_id['help_help_id'], name = 'help_id'))\n", "have_brown_id = (final_cross_id['brown_help_id'] != '-1')\n", "final_cross_id['help_id'][have_brown_id] = final_cross_id['brown_help_id'][have_brown_id]" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "data": { "text/plain": [ "['brown_i_id',\n", " 'brown_irac_id',\n", " 'help_help_id',\n", " 'brown_help_id',\n", " 'specz_id',\n", " 'sdss_id',\n", " 'ps1_id',\n", " 'legacy_id',\n", " 'ibis_id',\n", " 'ndwfs_id',\n", " 'uhs_id',\n", " 'zbootes_id',\n", " 'datafusion_intid',\n", " 'help_id']" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "final_cross_id.colnames" ] }, { "cell_type": "code", "execution_count": 35, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "set()\n" ] } ], "source": [ "columns = ['help_id',\n", " 'brown_help_id',\n", " 'help_help_id',\n", " 'brown_i_id',\n", " 'brown_irac_id',\n", " 'specz_id',\n", " 'sdss_id',\n", " 'ps1_id',\n", " 'legacy_id',\n", " 'ibis_id',\n", " 'ndwfs_id',\n", " 'uhs_id',\n", " 'zbootes_id',\n", " 'datafusion_intid']\n", "print(set(columns) - set(final_cross_id.colnames))\n", "final_cross_id = final_cross_id[columns]" ] }, { "cell_type": "code", "execution_count": 36, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "data": { "text/html": [ "Table length=10\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
idxhelp_idbrown_help_idhelp_help_idbrown_i_idbrown_irac_idspecz_idsdss_idps1_idlegacy_idibis_idndwfs_iduhs_idzbootes_iddatafusion_intid
0HELP_J142500.285+331035.838HELP_J142500.285+331035.838-1394263-1-1-1-1-1-1-1-1-1-1
1HELP_J143318.051+332158.118HELP_J143318.051+332158.118-1473154-1-1-1-1-1-1-1-1-1-1
2HELP_J142939.235+354635.105HELP_J142939.235+354635.105-12019839-1-1-1-1-1-1-1-1-1-1
3HELP_J142446.142+331756.854HELP_J142446.142+331756.854-1445407-1-1-1-1-1-1-1-1-1-1
4HELP_J142639.609+354724.470HELP_J142639.609+354724.470-12027735-1-1-1-1-1-1-1-1-1-1
5HELP_J142446.456+324136.432HELP_J142446.456+324136.432-1161462-1-1-1-1-1-1-1-1-1-1
6HELP_J142440.580+354503.991HELP_J142440.580+354503.991-12005436-1-1-1-1-1-1-1-1-1-1
7HELP_J142742.433+325234.332HELP_J142742.433+325234.332-1242716-1-1-1-1-1-1-1-1-1-1
8HELP_J142510.784+344929.132HELP_J142510.784+344929.132-11446199-1-1-1-1-1-1-1-1-1-1
9HELP_J143221.937+354511.185HELP_J143221.937+354511.185-12006597-1-1-1-1-1-1-1-1-1-1
\n", "\n" ], "text/plain": [ "" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "final_cross_id[:10].show_in_notebook()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": false, "deletable": true, "editable": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "False\n", "False\n", "False\n", "False\n", "False\n", "False\n" ] } ], "source": [ "\n", "final_cross_id.write(\"master_list_cross_ident_bootes{}.fits\".format(SUFFIX), overwrite=True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }