{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# EGS master catalogue\n", "## Preparation of Canada France Hawaii Telescope WIRDS Survey (CFHT-WIRDS) data\n", "\n", "The catalogue is in `dmu0_CFHT-WIRDS`.\n", "\n", "In the catalogue, we keep:\n", "\n", "- The position;\n", "- The stellarity;\n", "- The aperture magnitude (3 arcsec).\n", "- The total magnitude (Kron like aperture magnitude).\n", "\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]\n", "This notebook was executed on: \n", "2018-02-07 19:28:20.058540\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table, join\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, mag_to_flux\n", "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"wirds_ra\"\n", "DEC_COL = \"wirds_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.\n", "Check the NumPy 1.11 release notes for more information.\n", " ma.MaskedArray.__setitem__(self, index, value)\n", "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:63: RuntimeWarning: invalid value encountered in less\n", "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:64: RuntimeWarning: invalid value encountered in less\n" ] } ], "source": [ "#We have to import and combine the H, J and Ks catalogues separately. \n", "#Fluxes are given in counts sowe compute them fresh from the magnitudes\n", "\n", "epoch = 2007\n", "\n", "\n", "\n", "imported_columns = OrderedDict({\n", " 'id': \"wirds_id\",\n", " 'ra': \"wirds_ra\",\n", " 'dec': \"wirds_dec\",\n", "# ugriz are ks selected from cfhtls\n", " 'utot': \"m_wirds_u\",\n", " 'uterr': \"merr_wirds_u\",\n", " 'u': \"m_ap_wirds_u\",\n", " 'uerr': \"merr_ap_wirds_u\",\n", " 'gtot': \"m_wirds_g\",\n", " 'gterr': \"merr_wirds_g\",\n", " 'g': \"m_ap_wirds_g\",\n", " 'gerr': \"merr_ap_wirds_g\",\n", " 'rtot': \"m_wirds_r\",\n", " 'rterr': \"merr_wirds_r\",\n", " 'r': \"m_ap_wirds_r\",\n", " 'rerr': \"merr_ap_wirds_r\",\n", " 'itot': \"m_wirds_i\",\n", " 'iterr': \"merr_wirds_i\",\n", " 'i': \"m_ap_wirds_i\",\n", " 'ierr': \"merr_ap_wirds_i\",\n", " 'ztot': \"m_wirds_z\",\n", " 'zterr': \"merr_wirds_z\",\n", " 'z': \"m_ap_wirds_z\",\n", " 'zerr': \"merr_ap_wirds_z\",\n", " 'jtot': \"m_wirds_j\", \n", " 'jterr': \"merr_wirds_j\",\n", " 'j': \"m_ap_wirds_j\",\n", " 'jerr': \"merr_ap_wirds_j\",\n", " 'htot': \"m_wirds_h\",\n", " 'hterr': \"merr_wirds_h\",\n", " 'h': \"m_ap_wirds_h\",\n", " 'herr': \"merr_ap_wirds_h\",\n", " 'kstot': \"m_wirds_k\",\n", " 'ksterr': \"merr_wirds_k\",\n", " 'ks': \"m_ap_wirds_k\",\n", " 'kserr': \"merr_ap_wirds_k\"\n", " \n", " })\n", "\n", "\n", "catalogue = Table.read(\"../../dmu0/dmu0_CFHT-WIRDS/data/EGS_Ks-priors.fits\")[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", "\n", "for col in catalogue.colnames:\n", " if col.startswith('m_'):\n", " \n", " errcol = \"merr{}\".format(col[1:])\n", " #catalogue_h[col].name = imported_columns_h[col]\n", " \n", " #REplace 99.0 with nan\n", " catalogue[col][catalogue[col] > 90.] = np.nan\n", " catalogue[errcol][catalogue[errcol] > 90.] = np.nan \n", " #Replace -99.0 with nan\n", " catalogue[col][catalogue[col] < -90.] = np.nan\n", " catalogue[errcol][catalogue[errcol] < -90.] = np.nan \n", " \n", " flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))\n", " \n", " # Fluxes are added in µJy\n", " catalogue.add_column(Column(flux * 1.e6, name=\"f{}\".format(col[1:])))\n", " catalogue.add_column(Column(error * 1.e6, name=\"f{}\".format(errcol[1:])))\n", "\n", " # Band-flag column\n", " if \"ap\" not in col:\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag{}\".format(col[1:])))\n", "# Clean table metadata\n", "catalogue.meta = None\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<Table masked=True length=10>\n", "
idx | wirds_id | wirds_ra | wirds_dec | m_wirds_u | merr_wirds_u | m_ap_wirds_u | merr_ap_wirds_u | m_wirds_g | merr_wirds_g | m_ap_wirds_g | merr_ap_wirds_g | m_wirds_r | merr_wirds_r | m_ap_wirds_r | merr_ap_wirds_r | m_wirds_i | merr_wirds_i | m_ap_wirds_i | merr_ap_wirds_i | m_wirds_z | merr_wirds_z | m_ap_wirds_z | merr_ap_wirds_z | m_wirds_j | merr_wirds_j | m_ap_wirds_j | merr_ap_wirds_j | m_wirds_h | merr_wirds_h | m_ap_wirds_h | merr_ap_wirds_h | m_wirds_k | merr_wirds_k | m_ap_wirds_k | merr_ap_wirds_k | f_wirds_u | ferr_wirds_u | flag_wirds_u | f_ap_wirds_u | ferr_ap_wirds_u | f_wirds_g | ferr_wirds_g | flag_wirds_g | f_ap_wirds_g | ferr_ap_wirds_g | f_wirds_r | ferr_wirds_r | flag_wirds_r | f_ap_wirds_r | ferr_ap_wirds_r | f_wirds_i | ferr_wirds_i | flag_wirds_i | f_ap_wirds_i | ferr_ap_wirds_i | f_wirds_z | ferr_wirds_z | flag_wirds_z | f_ap_wirds_z | ferr_ap_wirds_z | f_wirds_j | ferr_wirds_j | flag_wirds_j | f_ap_wirds_j | ferr_ap_wirds_j | f_wirds_h | ferr_wirds_h | flag_wirds_h | f_ap_wirds_h | ferr_ap_wirds_h | f_wirds_k | ferr_wirds_k | flag_wirds_k | f_ap_wirds_k | ferr_ap_wirds_k |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 214.8727562 | 52.4056305 | 13.4962 | 0.0 | 23.9186 | 0.0103 | 11.7267 | 0.0 | 21.8535 | 0.0012 | 11.2138 | 0.0 | 20.9675 | 0.0007 | 11.2314 | 0.0 | 20.3067 | 0.0005 | 10.2368 | 0.0 | 19.9116 | 0.001 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 14505.1 | 0.0 | False | 0.983015 | 0.00932551505457 | 74015.0 | 0.0 | False | 6.58567 | 0.00727874648874 | 118708.0 | 0.0 | False | 14.8936 | 0.00960225206654 | 116799.0 | 0.0 | False | 27.3728 | 0.0126056556837 | 291931.0 | 0.0 | False | 39.3876 | 0.0362773425877 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
1 | 2 | 214.7786347 | 52.3817822 | 26.7294 | 0.4428 | 26.6443 | 0.111 | 23.7498 | 0.0206 | 24.5104 | 0.0114 | 22.1588 | 0.0063 | 22.9083 | 0.0035 | 20.8597 | 0.0027 | 21.5705 | 0.0015 | 20.4159 | 0.0048 | 21.1032 | 0.0025 | 19.9463 | 0.0217 | 21.261 | 0.0179 | 19.847 | 0.0534 | 20.8171 | 0.0349 | 19.0268 | 0.0166 | 19.9874 | 0.0099 | 0.0738311 | 0.0301108272396 | False | 0.0798509 | 0.00816353949773 | 1.14837 | 0.0217882813786 | False | 0.569953 | 0.00598438978159 | 4.97141 | 0.0288466930215 | False | 2.49276 | 0.00803569855634 | 16.4483 | 0.0409034255426 | False | 8.54673 | 0.0118077414299 | 24.7537 | 0.109435184277 | False | 13.1438 | 0.0302646230921 | 38.1487 | 0.762456320444 | False | 11.3658 | 0.187382430431 | 41.8022 | 2.05596907836 | False | 17.1064 | 0.549870776376 | 88.9774 | 1.36039038334 | False | 36.7316 | 0.334927243239 |
2 | 3 | 214.7717041 | 52.3815262 | 21.4995 | 0.0043 | 22.7167 | 0.0036 | 20.2183 | 0.001 | 21.3137 | 0.0008 | 19.3399 | 0.0006 | 20.3348 | 0.0005 | 18.8427 | 0.0005 | 19.7788 | 0.0003 | 18.5782 | 0.001 | 19.4954 | 0.0007 | 18.339 | 0.0062 | 20.0539 | 0.0056 | 17.9911 | 0.0094 | 19.3824 | 0.0059 | 17.7361 | 0.0051 | 18.8522 | 0.0031 | 9.12431 | 0.0361363211596 | False | 2.97386 | 0.00986048316918 | 29.6948 | 0.0273499008472 | False | 10.8273 | 0.00797784960014 | 66.6869 | 0.0368525303202 | False | 26.6735 | 0.0122835936054 | 105.419 | 0.0485473792651 | False | 44.5123 | 0.0122992129036 | 134.499 | 0.123878446175 | False | 57.7883 | 0.0372574882931 | 167.648 | 0.957341602771 | False | 34.5493 | 0.178198254434 | 230.972 | 1.99969260138 | False | 64.1268 | 0.348471639518 | 292.119 | 1.3721641124 | False | 104.501 | 0.298371497774 |
3 | 4 | 214.7488553 | 52.3800777 | 24.799 | 0.0601 | 27.5776 | 0.2211 | 24.3902 | 0.0298 | 27.1959 | 0.1117 | 23.9469 | 0.0265 | 26.6719 | 0.0923 | 23.6326 | 0.0274 | 26.4548 | 0.1044 | 23.8265 | 0.0915 | 27.7452 | 0.9547 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 0.436918 | 0.024185221622 | False | 0.0338034 | 0.00688373531403 | 0.636678 | 0.0174747794631 | False | 0.0480441 | 0.00494274892162 | 0.957724 | 0.0233755486079 | False | 0.0778467 | 0.00661785890088 | 1.27926 | 0.0322838705642 | False | 0.095078 | 0.00914231107458 | 1.07004 | 0.0901771750819 | False | 0.0289681 | 0.0254719899724 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
4 | 5 | 214.4955751 | 52.3818769 | 21.6788 | 0.0027 | 22.7279 | 0.0031 | 20.6868 | 0.0008 | 21.6604 | 0.0009 | 20.0348 | 0.0006 | 20.9352 | 0.0006 | 19.6802 | 0.0006 | 20.5205 | 0.0005 | 19.4692 | 0.0013 | 20.2856 | 0.0011 | 19.2251 | 0.0061 | 19.9921 | 0.0046 | 18.9598 | 0.0159 | 19.7092 | 0.0133 | 18.8211 | 0.0081 | 19.4783 | 0.0056 | 7.73535 | 0.0192361939753 | False | 2.94334 | 0.00840383490868 | 19.2877 | 0.0142116798088 | False | 7.86755 | 0.00652165558677 | 35.1625 | 0.0194315303816 | False | 15.3433 | 0.00847904047987 | 48.7438 | 0.0269368349109 | False | 22.4802 | 0.0103525089798 | 59.1997 | 0.0708824674803 | False | 27.91 | 0.0282766601231 | 74.1242 | 0.416452354693 | False | 36.5729 | 0.154950677097 | 94.641 | 1.385964988 | False | 47.4592 | 0.581363016317 | 107.537 | 0.802269736596 | False | 58.7056 | 0.302791257855 |
5 | 6 | 214.7796292 | 52.3801438 | 24.7872 | 0.0871 | 30.1952 | 2.9281 | 24.2704 | 0.0387 | nan | nan | 23.6836 | 0.0298 | nan | nan | 22.7355 | 0.0173 | 27.4542 | 0.3073 | 22.6148 | 0.042 | 26.955 | 0.5286 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 0.441692 | 0.0354334544767 | False | 0.00303333 | 0.00818053706761 | 0.710952 | 0.0253411704421 | False | nan | nan | 1.22056 | 0.0335005712714 | False | nan | nan | 2.9228 | 0.0465716503186 | False | 0.0378721 | 0.0107190866967 | 3.26648 | 0.126358662328 | False | 0.0599791 | 0.0292013368622 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
6 | 7 | 214.6754875 | 52.3814928 | 23.731 | 0.0167 | 24.326 | 0.0115 | 22.7691 | 0.005 | 23.3111 | 0.0033 | 21.5475 | 0.0022 | 22.0744 | 0.0015 | 20.4975 | 0.0012 | 20.979 | 0.0008 | 20.1201 | 0.0023 | 20.6057 | 0.0014 | 19.4274 | 0.012 | 20.0375 | 0.0064 | 19.1627 | 0.0147 | 19.6558 | 0.0085 | 18.9141 | 0.0154 | 19.3197 | 0.0063 | 1.16842 | 0.0179717932724 | False | 0.67546 | 0.00715440236831 | 2.83374 | 0.013049844938 | False | 1.72012 | 0.00522816817465 | 8.7297 | 0.0176887793714 | False | 5.37328 | 0.00742346719562 | 22.9615 | 0.025377975544 | False | 14.7367 | 0.0108583910333 | 32.5057 | 0.0688594476742 | False | 20.7836 | 0.0267993123998 | 61.5233 | 0.679980628775 | False | 35.0752 | 0.206754636019 | 78.509 | 1.06294900761 | False | 49.8517 | 0.39027848652 | 98.7096 | 1.40008982853 | False | 67.9391 | 0.394217755093 |
7 | 8 | 214.4828076 | 52.3855683 | 23.2934 | 0.0117 | 24.0164 | 0.0087 | 22.0459 | 0.0028 | 22.7515 | 0.0022 | 21.2106 | 0.0017 | 21.8924 | 0.0013 | 20.7792 | 0.0016 | 21.4547 | 0.0011 | 20.5532 | 0.0036 | 21.2222 | 0.0025 | 20.4067 | 0.0186 | 21.0031 | 0.0117 | 20.1036 | 0.0273 | 20.5912 | 0.0155 | 19.9227 | 0.0233 | 20.5544 | 0.0152 | 1.7484 | 0.0188408962799 | False | 0.898339 | 0.00719838557188 | 5.51619 | 0.0142256894833 | False | 2.88005 | 0.00583577493671 | 11.9058 | 0.0186416565157 | False | 6.35389 | 0.0076077914855 | 17.7141 | 0.0261045235675 | False | 9.50867 | 0.00963359107118 | 21.8132 | 0.0723265911802 | False | 11.7793 | 0.0271228577731 | 24.9643 | 0.427669650162 | False | 14.4132 | 0.155317927147 | 33.0035 | 0.829847461864 | False | 21.063 | 0.300695607621 | 38.987 | 0.836664038798 | False | 21.7891 | 0.305041848333 |
8 | 9 | 214.7444736 | 52.3805918 | 26.1915 | 0.1948 | 28.4687 | 0.501 | 25.0418 | 0.0487 | 26.2718 | 0.0479 | 24.7054 | 0.0478 | 25.816 | 0.0421 | 24.4045 | 0.05 | 25.6152 | 0.0482 | 23.9556 | 0.0926 | 25.3713 | 0.1076 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 0.121171 | 0.0217402423743 | False | 0.0148771 | 0.00686488242785 | 0.349366 | 0.0156705680013 | False | 0.112533 | 0.0049646792327 | 0.476255 | 0.0209673398274 | False | 0.171238 | 0.00663983770295 | 0.628348 | 0.0289364862738 | False | 0.206025 | 0.00914624302197 | 0.950078 | 0.0810300384728 | False | 0.257917 | 0.0255604148947 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
9 | 10 | 214.7806265 | 52.3804941 | 23.5868 | 0.0127 | 24.0273 | 0.0107 | 23.0777 | 0.0056 | 23.5144 | 0.0046 | 22.4582 | 0.0042 | 22.9064 | 0.0035 | 21.8087 | 0.0032 | 22.2767 | 0.0027 | 21.6366 | 0.0073 | 22.0868 | 0.006 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 1.33438 | 0.0156084338073 | False | 0.889365 | 0.00876474856568 | 2.13265 | 0.01099976962 | False | 1.42639 | 0.00604328649843 | 3.77329 | 0.0145963659634 | False | 2.49712 | 0.00804976525615 | 6.86309 | 0.0202276467462 | False | 4.45985 | 0.0110907153157 | 8.04192 | 0.0540702360468 | False | 5.31227 | 0.0293566663458 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |