{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# HDF-N master catalogue\n", "## Preparation of CANDELS-GOODS-N data\n", "\n", "CANDELS-GOODS-N catalogue: the catalogue comes from `dmu0_CANDELS-GOODS-N`.\n", "\n", "In the catalogue, we keep:\n", "\n", "- The identifier (it's unique in the catalogue);\n", "- The position;\n", "- The stellarity;\n", "- The total magnitude.\n", "\n", "We don't know when the maps have been observed. We will use the year of the reference paper." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, flux_to_mag" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"candels-gn_ra\"\n", "DEC_COL = \"candels-gn_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: UnitsWarning: '0 use, gt 1 no-use' did not parse as fits unit: At col 2, Unit 'use' not supported by the FITS standard. Did you mean uS or us? [astropy.units.core]\n", "WARNING: UnitsWarning: '0 galaxy, 1 star' did not parse as fits unit: At col 2, Unit 'galaxy' not supported by the FITS standard. [astropy.units.core]\n" ] } ], "source": [ "imported_columns = OrderedDict({\n", " 'ID': \"candels-gn_id\",\n", " 'RA': \"candels-gn_ra\",\n", " 'DEC': \"candels-gn_dec\",\n", " 'CLASS_STAR': \"candels-gn_stellarity\",\n", " #HST data\n", " 'ACS_F435W_FLUX': \"f_acs_f435w\", \n", " 'ACS_F435W_FLUXERR': \"ferr_acs_f435w\", \n", " 'ACS_F606W_FLUX': \"f_acs_f606w\", \n", " 'ACS_F606W_FLUXERR': \"ferr_acs_f606w\", \n", " 'ACS_F775W_FLUX': \"f_acs_f775w\", \n", " 'ACS_F775W_FLUXERR': \"ferr_acs_f775w\", \n", " 'ACS_F814W_FLUX': \"f_acs_f814w\", \n", " 'ACS_F814W_FLUXERR': \"ferr_acs_f814w\", \n", " 'ACS_F850LP_FLUX': \"f_acs_f850lp\", \n", " 'ACS_F850LP_FLUXERR': \"ferr_acs_f850lp\", \n", " \n", " 'WFC3_F105W_FLUX': \"f_wfc3_f105w\", \n", " 'WFC3_F105W_FLUXERR': \"ferr_wfc3_f105w\", \n", " 'WFC3_F125W_FLUX': \"f_wfc3_f125w\", \n", " 'WFC3_F125W_FLUXERR': \"ferr_wfc3_f125w\", \n", " 'WFC3_F140W_FLUX': \"f_wfc3_f140w\", \n", " 'WFC3_F140W_FLUXERR': \"ferr_wfc3_f140w\", \n", " 'WFC3_F160W_FLUX': \"f_wfc3_f160w\", \n", " 'WFC3_F160W_FLUXERR': \"ferr_wfc3_f160w\", \n", " #MOIRCS. There is a Ks and K. We believe this to be K.\n", " 'MOIRCS_K_FLUX':\"f_moircs_k\",\n", " 'MOIRCS_K_FLUXERR':\"ferr_moircs_k\",\n", " #CFHT WIRCAM\n", " 'CFHT_Ks_FLUX': \"f_candels-wircam_k\",# 33 WIRCAM_K_FLUX Flux density (in μJy) in the Ks-band (CFHT/WIRCam) (3)\n", " 'CFHT_Ks_FLUXERR': \"ferr_candels-wircam_k\",# 34 WIRCAM_K_FLUXERR \n", " #Spitzer/IRAC\n", " 'IRAC_CH1_SCANDELS_FLUX': \"f_candels-irac_i1\",# 47 IRAC_CH1_FLUX Flux density (in μJy) in the 3.6μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH1_SCANDELS_FLUXERR': \"ferr_candels-irac_i1\",# 48 IRAC_CH1_FLUXERR Flux uncertainty (in μJy) in the 3.6μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH2_SCANDELS_FLUX': \"f_candels-irac_i2\",# 49 IRAC_CH2_FLUX Flux density (in μJy) in the 4.5μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH2_SCANDELS_FLUXERR': \"ferr_candels-irac_i2\",# 50 IRAC_CH2_FLUXERR Flux uncertainty (in μJy) in the 4.5μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH3_FLUX': \"f_candels-irac_i3\",# 51 IRAC_CH3_FLUX Flux density (in μJy) in the 5.8μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH3_FLUXERR': \"ferr_candels-irac_i3\",# 52 IRAC_CH3_FLUXERR Flux uncertainty (in μJy) in the 5.8μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH4_FLUX': \"f_candels-irac_i4\",# 53 IRAC_CH4_FLUX Flux density (in μJy) in the 8.0μm-band (Spitzer/IRAC) (3)\n", " 'IRAC_CH4_FLUXERR': \"ferr_candels-irac_i4\"# 54 IRAC_CH4_FLUXERR\n", " \n", " \n", " })\n", "\n", "\n", "catalogue = Table.read(\"../../dmu0/dmu0_CANDELS-GOODS-N/data/CANDELS.GOODSN.F160W.v1_1.photom.fits\")[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", "\n", "epoch = 2011\n", "\n", "# Clean table metadata\n", "catalogue.meta = None" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: divide by zero encountered in log10\n", " magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6\n", "/opt/herschelhelp_internal/herschelhelp_internal/utils.py:76: RuntimeWarning: invalid value encountered in log10\n", " magnitudes = 2.5 * (23 - np.log10(fluxes)) - 48.6\n", "/opt/herschelhelp_internal/herschelhelp_internal/utils.py:80: RuntimeWarning: divide by zero encountered in true_divide\n", " errors = 2.5 / np.log(10) * errors_on_fluxes / fluxes\n" ] } ], "source": [ "# Adding flux and band-flag columns\n", "for col in catalogue.colnames:\n", " if col.startswith('f_'):\n", " \n", " errcol = \"ferr{}\".format(col[1:])\n", " \n", " # Some object have a magnitude to 0, we suppose this means missing value\n", " #catalogue[col][catalogue[col] <= 0] = np.nan\n", " #catalogue[errcol][catalogue[errcol] <= 0] = np.nan \n", " \n", "\n", " mag, error = flux_to_mag(np.array(catalogue[col])*1.e-6, np.array(catalogue[errcol])*1.e-6)\n", " \n", " # Fluxes are added in µJy\n", " catalogue.add_column(Column(mag, name=\"m{}\".format(col[1:])))\n", " catalogue.add_column(Column(error, name=\"m{}\".format(errcol[1:])))\n", " \n", "\n", " \n", " # Band-flag column\n", " if \"ap\" not in col:\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag{}\".format(col[1:])))\n", " \n", "# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<Table length=10>\n", "
idx | candels-gn_id | candels-gn_ra | candels-gn_dec | candels-gn_stellarity | f_acs_f435w | ferr_acs_f435w | f_acs_f606w | ferr_acs_f606w | f_acs_f775w | ferr_acs_f775w | f_acs_f814w | ferr_acs_f814w | f_acs_f850lp | ferr_acs_f850lp | f_wfc3_f105w | ferr_wfc3_f105w | f_wfc3_f125w | ferr_wfc3_f125w | f_wfc3_f140w | ferr_wfc3_f140w | f_wfc3_f160w | ferr_wfc3_f160w | f_moircs_k | ferr_moircs_k | f_candels-wircam_k | ferr_candels-wircam_k | f_candels-irac_i1 | ferr_candels-irac_i1 | f_candels-irac_i2 | ferr_candels-irac_i2 | f_candels-irac_i3 | ferr_candels-irac_i3 | f_candels-irac_i4 | ferr_candels-irac_i4 | m_acs_f435w | merr_acs_f435w | flag_acs_f435w | m_acs_f606w | merr_acs_f606w | flag_acs_f606w | m_acs_f775w | merr_acs_f775w | flag_acs_f775w | m_acs_f814w | merr_acs_f814w | flag_acs_f814w | m_acs_f850lp | merr_acs_f850lp | flag_acs_f850lp | m_wfc3_f105w | merr_wfc3_f105w | flag_wfc3_f105w | m_wfc3_f125w | merr_wfc3_f125w | flag_wfc3_f125w | m_wfc3_f140w | merr_wfc3_f140w | flag_wfc3_f140w | m_wfc3_f160w | merr_wfc3_f160w | flag_wfc3_f160w | m_moircs_k | merr_moircs_k | flag_moircs_k | m_candels-wircam_k | merr_candels-wircam_k | flag_candels-wircam_k | m_candels-irac_i1 | merr_candels-irac_i1 | flag_candels-irac_i1 | m_candels-irac_i2 | merr_candels-irac_i2 | flag_candels-irac_i2 | m_candels-irac_i3 | merr_candels-irac_i3 | flag_candels-irac_i3 | m_candels-irac_i4 | merr_candels-irac_i4 | flag_candels-irac_i4 |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 galaxy, 1 star | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
0 | 1 | 189.15216178 | 62.09177764 | 0 | -0.03041 | 0.03046 | -0.01637 | 0.03921 | 0.036 | 0.0591 | -0.00105 | 64894254393.4 | -0.07984 | 0.07474 | -0.07928 | 0.05425 | 0.00583 | 0.03354 | -2e-05 | 40607499435.5 | 0.73155 | 0.37962 | 0.01084 | 1444.70263 | -0.20261 | 0.09029 | -0.08455 | 0.13695 | -0.18952 | 0.13653 | -0.37712 | 1.00009 | -0.35061 | 1.52074 | nan | -1.08752136787 | False | nan | -2.60059356069 | False | 27.5092437481 | 1.78241693614 | False | nan | -6.71028966435e+13 | False | nan | -1.01638181292 | False | nan | -0.742951426692 | False | 29.4858286131 | 6.2462422483 | False | nan | -2.20445161609e+15 | False | 24.2393899634 | 0.563416277835 | False | 28.8124267945 | 144701.65595 | False | nan | -0.483841478346 | False | nan | -1.75862298334 | False | nan | -0.782163170302 | False | nan | -2.8792795954 | False | nan | -4.70928517733 | False |
1 | 2 | 189.14883356 | 62.09202743 | 0 | -0.04039 | 0.12865 | 0.02556 | 0.15445 | 0.05139 | 0.23871 | -0.00188 | 264842218594.0 | -0.18039 | 0.3039 | -0.0424 | 0.25053 | 0.23843 | 0.15215 | -0.00038 | 165724690769.0 | 1.09767 | 0.38818 | 0.00992 | 1395.40524 | 0.00836 | 0.08557 | 0.05314 | 0.13861 | 0.62588 | 0.13547 | -0.54797 | 0.99801 | -1.1307 | 1.33768 | nan | -3.45828083046 | False | 27.8810978763 | 6.56071818564 | False | 27.1228034558 | 5.04331756057 | False | nan | -1.5295148153e+14 | False | nan | -1.82912152905 | False | nan | -6.41531819288 | False | 25.456597753 | 0.692843868448 | False | nan | -4.73508675765e+14 | False | 23.7988205129 | 0.383959732855 | False | 28.9087208196 | 152726.006994 | False | 29.0944843064 | 11.1132113686 | False | 27.0864461248 | 2.8320266342 | False | 24.4087723153 | 0.235004607367 | False | nan | -1.97743597224 | False | nan | -1.28448536869 | False |
2 | 3 | 189.15133834 | 62.09213218 | 0 | 0.01698 | 0.0183 | -0.01012 | 0.0216 | -0.05957 | 0.03347 | -0.00107 | 37153131237.0 | -0.03646 | 0.04346 | 0.02456 | 0.03178 | -0.00887 | 0.01767 | -2e-05 | 23193730332.7 | 0.36765 | 0.04515 | 0.01055 | 1448.69502 | -0.01034 | 0.08998 | 0.11392 | 0.14662 | 0.27129 | 0.14499 | 0.90009 | 1.01676 | 0.47453 | 1.55453 | 28.3251557852 | 1.17013972598 | False | nan | -2.3173816228 | False | nan | -0.61003174036 | False | nan | -3.76995324338e+13 | False | nan | -1.29418802685 | False | 27.9244290938 | 1.40491435616 | False | nan | -2.1629040291 | False | nan | -1.25911363728e+15 | False | 24.9864135742 | 0.133336025146 | False | 28.8418688509 | 149090.107381 | False | nan | -9.44821505843 | False | 26.2585000593 | 1.39738976775 | False | 25.3164155361 | 0.580267950635 | False | 24.0142851582 | 1.22646973475 | False | 24.7093408152 | 3.55680252541 | False |
3 | 4 | 189.14960603 | 62.09211704 | 0 | -0.02057 | 0.06794 | 0.06738 | 0.08303 | -0.08115 | 0.12626 | -0.00202 | 142937077016.0 | -0.0238 | 0.16321 | 0.16267 | 0.12396 | -0.08133 | 0.06801 | -0.00034 | 89442699180.2 | 2.03812 | 0.36995 | 0.01363 | 1618.73109 | 0.0057 | 0.09325 | 0.55563 | 0.13483 | 0.30043 | 0.13159 | -0.06876 | 0.97509 | 1.99878 | 1.36655 | nan | -3.58604364372 | False | 26.8286724834 | 1.33791447137 | False | nan | -1.68927976849 | False | nan | -7.68277027319e+13 | False | nan | -7.44550445288 | False | 25.8717313333 | 0.827367430638 | False | nan | -0.907917364879 | False | nan | -2.85621108092e+14 | False | 23.1269256232 | 0.197077752512 | False | 28.5637603604 | 128944.603828 | False | 29.5103128608 | 17.7622633498 | False | 24.5380357839 | 0.263466357986 | False | 25.2056417555 | 0.475558456826 | False | nan | -15.3968952283 | False | 23.148087512 | 0.742309213927 | False |
4 | 5 | 189.14808454 | 62.09249699 | 0 | 0.03149 | 0.04246 | -0.05597 | 0.05101 | -0.02036 | 0.07891 | -0.00161 | 87996244750.6 | -0.06562 | 0.10256 | 0.22797 | 0.08041 | 0.15102 | 0.04451 | -0.00032 | 55063541332.5 | 1.88213 | 0.40744 | 0.01539 | 1705.73592 | 0.41502 | 0.10039 | 8.86312 | 0.12774 | 4.06394 | 0.12548 | 4.95255 | 0.96467 | 2.10368 | 1.29263 | 27.6545683484 | 1.4639682202 | False | nan | -0.989519453363 | False | nan | -4.20802769732 | False | nan | -5.93420551606e+13 | False | nan | -1.69693851204 | False | 25.5053057519 | 0.382962882066 | False | 25.9524138352 | 0.319998135835 | False | nan | -1.86826501209e+14 | False | 23.2133759571 | 0.235038153192 | False | 28.4319034504 | 120336.53308 | False | 24.8548274348 | 0.26263085537 | False | 21.5310334266 | 0.0156482077187 | False | 22.3776317815 | 0.033523668896 | False | 22.162927828 | 0.211482396875 | False | 23.0925508048 | 0.667142906885 | False |
5 | 6 | 189.14870539 | 62.09269735 | 0 | 0.00072 | 0.05542 | -0.00956 | 0.06588 | -0.04616 | 0.10169 | -0.0039 | 114216919405.0 | -0.16295 | 0.12953 | -0.02333 | 0.09892 | 0.02109 | 0.05702 | -0.00022 | 71471096072.1 | 1.07439 | 0.24742 | 0.01249 | 1542.03164 | 0.14265 | 0.09545 | -4.62312 | 0.12194 | -1.85403 | 0.11972 | -3.1253 | 0.87925 | -2.1586 | 1.23912 | 31.7566687589 | 83.5715284274 | False | nan | -7.48203987128 | False | nan | -2.39186556893 | False | nan | -3.17972934857e+13 | False | nan | -0.863058672 | False | nan | -4.60355873873 | False | 28.0898085507 | 2.93545179684 | False | nan | -3.52721620905e+14 | False | 23.8220951064 | 0.250032904049 | False | 28.6585939041 | 134046.403557 | False | 26.01432056 | 0.726488052886 | False | nan | -0.028637515965 | False | nan | -0.0701090804537 | False | nan | -0.305453415043 | False | nan | -0.623254630798 | False |
6 | 7 | 189.14636359 | 62.09288181 | 0 | 0.11011 | 0.14499 | 0.00357 | 0.16921 | 1.09122 | 0.27022 | -0.01357 | 292273460385.0 | 1.98403 | 0.35271 | 2.74672 | 0.32787 | 3.65231 | 0.15878 | -0.00097 | 182889756397.0 | 4.56103 | 2.85484 | 0.01711 | 1758.97257 | 2.1676 | 0.10799 | 51.32974 | 0.09993 | 39.13622 | 0.10128 | 33.16982 | 0.7209 | 22.66882 | 1.04269 | 26.2954330934 | 1.42966935181 | False | 30.0183294597 | 51.4614630832 | False | 23.805219207 | 0.268862041797 | False | nan | -2.3384810437e+13 | False | 23.1561294132 | 0.193016243091 | False | 22.8029640255 | 0.129601972336 | False | 22.493580919 | 0.0472011397147 | False | nan | -2.04711371134e+14 | False | 22.252342678 | 0.679584029658 | False | 28.3168749761 | 111617.779218 | False | 23.0600521441 | 0.0540914618711 | False | 19.6240773389 | 0.00211373794104 | False | 19.9185528082 | 0.00280975942025 | False | 20.0981422131 | 0.0235969694744 | False | 20.511427715 | 0.0499402387658 | False |
7 | 8 | 189.14509981 | 62.09340188 | 0 | 0.0178 | 0.02649 | -0.02002 | 0.03159 | -0.01046 | 0.04839 | -0.00229 | 52742979131.0 | -0.04317 | 0.06216 | 0.03731 | 0.04688 | 0.01265 | 0.02578 | -5e-05 | 33003853966.8 | 1.4936 | 0.21241 | 0.01142 | 1471.11023 | -0.10997 | 0.08489 | 0.46067 | 0.14588 | 0.20961 | 0.13888 | 2.10853 | 0.90842 | 5.19303 | 1.35599 | 28.2739499942 | 1.61579505978 | False | nan | -1.71320712829 | False | nan | -5.02282743291 | False | nan | -2.50065336198e+13 | False | nan | -1.56333941366 | False | 27.4704368774 | 1.36422710477 | False | 28.6447736862 | 2.21267030503 | False | nan | -7.16669582967e+14 | False | 23.4644142376 | 0.154406284984 | False | 28.7558347402 | 139863.190709 | False | nan | -0.838120818604 | False | 24.741525173 | 0.343819214514 | False | 25.5964700051 | 0.71936951537 | False | 23.0900505387 | 0.467768769297 | False | 22.1114479212 | 0.283504512065 | False |
8 | 9 | 189.14428042 | 62.09353138 | 0 | -0.00789 | 0.04863 | -0.02897 | 0.05807 | -0.09221 | 0.08945 | -0.00363 | 101034871370.0 | 0.06439 | 0.11654 | 0.25504 | 0.09269 | 0.05482 | 0.0502 | 0.0001 | 63222445815.3 | 1.96327 | 0.45921 | 0.01395 | 1625.22483 | -0.0231 | 0.09401 | 1.17544 | 0.17931 | 1.10055 | 0.16543 | 1.08135 | 0.94592 | -2.24309 | 1.45174 | nan | -6.69193303389 | False | nan | -2.17634454299 | False | nan | -1.05323829862 | False | nan | -3.02196192257e+13 | False | 26.8779539373 | 1.96508304554 | False | 25.3834792507 | 0.394592569083 | False | 27.052652422 | 0.994234904758 | False | 33.9 | 6.8642898375e+14 | False | 23.1675499241 | 0.253954332612 | False | 28.538564481 | 126492.146151 | False | nan | -4.41861734239 | False | 23.724498836 | 0.16562594337 | False | 23.7959755547 | 0.16320325324 | False | 23.8150842886 | 0.949756869473 | False | nan | -0.702694353724 | False |
9 | 10 | 189.15491217 | 62.09411161 | 0 | 0.02194 | 0.00754 | 0.085 | 0.00929 | 0.25559 | 0.01553 | 0.28142 | 0.04471 | 0.34715 | 0.0201 | 0.40775 | 0.02116 | 0.50832 | 0.00891 | 2e-05 | 9395352460.95 | 0.4747 | 0.03888 | 0.01088 | 1479.21105 | 0.54232 | 0.09135 | -0.02745 | 0.15104 | 0.06979 | 0.14957 | -0.63339 | 1.0126 | -1.79954 | 1.52496 | 28.0469084419 | 0.373129032993 | False | 26.5764526857 | 0.118664580497 | False | 25.3811403551 | 0.065970825384 | False | 25.2766126032 | 0.172494015048 | False | 25.048707076 | 0.0628641731691 | False | 24.8740150757 | 0.0563437844088 | False | 24.6346570063 | 0.0190311409828 | False | 35.6474250108 | 5.10043716166e+14 | False | 24.7089519207 | 0.088926529684 | False | 28.8084277616 | 147613.326421 | False | 24.5643609477 | 0.182884648002 | False | nan | -5.97412008622 | False | 26.7905170042 | 2.3268887254 | False | nan | -1.73576545405 | False | nan | -0.920070841886 | False |