{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# XMM-LSS master catalogue\n", "## Preparation of Canada France Hawaii Telescope WIRDS Survey (CFHT-WIRDS) data\n", "\n", "The catalogue is in `dmu0_CFHT-WIRDS`.\n", "\n", "In the catalogue, we keep:\n", "\n", "- The position;\n", "- The stellarity;\n", "- The aperture magnitude (3 arcsec).\n", "- The total magnitude (Kron like aperture magnitude).\n", "\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table, join\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, mag_to_flux\n", "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"wirds_ra\"\n", "DEC_COL = \"wirds_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.\n", "Check the NumPy 1.11 release notes for more information.\n", " ma.MaskedArray.__setitem__(self, index, value)\n", "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:63: RuntimeWarning: invalid value encountered in less\n", "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:64: RuntimeWarning: invalid value encountered in less\n" ] } ], "source": [ "#We have to import and combine the H, J and Ks catalogues separately. \n", "#Fluxes are given in counts sowe compute them fresh from the magnitudes\n", "\n", "epoch = 2007\n", "\n", "\n", "\n", "imported_columns = OrderedDict({\n", " 'id': \"wirds_id\",\n", " 'ra': \"wirds_ra\",\n", " 'dec': \"wirds_dec\",\n", "# ugriz are ks selected from cfhtls\n", " 'utot': \"m_wirds_u\",\n", " 'uterr': \"merr_wirds_u\",\n", " 'u': \"m_ap_wirds_u\",\n", " 'uerr': \"merr_ap_wirds_u\",\n", " 'gtot': \"m_wirds_g\",\n", " 'gterr': \"merr_wirds_g\",\n", " 'g': \"m_ap_wirds_g\",\n", " 'gerr': \"merr_ap_wirds_g\",\n", " 'rtot': \"m_wirds_r\",\n", " 'rterr': \"merr_wirds_r\",\n", " 'r': \"m_ap_wirds_r\",\n", " 'rerr': \"merr_ap_wirds_r\",\n", " 'itot': \"m_wirds_i\",\n", " 'iterr': \"merr_wirds_i\",\n", " 'i': \"m_ap_wirds_i\",\n", " 'ierr': \"merr_ap_wirds_i\",\n", " 'ztot': \"m_wirds_z\",\n", " 'zterr': \"merr_wirds_z\",\n", " 'z': \"m_ap_wirds_z\",\n", " 'zerr': \"merr_ap_wirds_z\",\n", " 'jtot': \"m_wirds_j\", \n", " 'jterr': \"merr_wirds_j\",\n", " 'j': \"m_ap_wirds_j\",\n", " 'jerr': \"merr_ap_wirds_j\",\n", " 'htot': \"m_wirds_h\",\n", " 'hterr': \"merr_wirds_h\",\n", " 'h': \"m_ap_wirds_h\",\n", " 'herr': \"merr_ap_wirds_h\",\n", " 'kstot': \"m_wirds_k\",\n", " 'ksterr': \"merr_wirds_k\",\n", " 'ks': \"m_ap_wirds_k\",\n", " 'kserr': \"merr_ap_wirds_k\"\n", " \n", " })\n", "\n", "\n", "catalogue = Table.read(\"../../dmu0/dmu0_CFHT-WIRDS/data/COSMOS_Ks-priors.fits\")[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", "\n", "for col in catalogue.colnames:\n", " if col.startswith('m_'):\n", " \n", " errcol = \"merr{}\".format(col[1:])\n", " #catalogue_h[col].name = imported_columns_h[col]\n", " \n", " #REplace 99.0 with nan\n", " catalogue[col][catalogue[col] > 90.] = np.nan\n", " catalogue[errcol][catalogue[errcol] > 90.] = np.nan \n", " #Replace -99.0 with nan\n", " catalogue[col][catalogue[col] < -90.] = np.nan\n", " catalogue[errcol][catalogue[errcol] < -90.] = np.nan \n", " \n", " flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))\n", " \n", " # Fluxes are added in µJy\n", " catalogue.add_column(Column(flux * 1.e6, name=\"f{}\".format(col[1:])))\n", " catalogue.add_column(Column(error * 1.e6, name=\"f{}\".format(errcol[1:])))\n", "\n", " # Band-flag column\n", " if \"ap\" not in col:\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag{}\".format(col[1:])))\n", "# Clean table metadata\n", "catalogue.meta = None\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<Table masked=True length=10>\n", "
idx | wirds_id | wirds_ra | wirds_dec | m_wirds_u | merr_wirds_u | m_ap_wirds_u | merr_ap_wirds_u | m_wirds_g | merr_wirds_g | m_ap_wirds_g | merr_ap_wirds_g | m_wirds_r | merr_wirds_r | m_ap_wirds_r | merr_ap_wirds_r | m_wirds_i | merr_wirds_i | m_ap_wirds_i | merr_ap_wirds_i | m_wirds_z | merr_wirds_z | m_ap_wirds_z | merr_ap_wirds_z | m_wirds_j | merr_wirds_j | m_ap_wirds_j | merr_ap_wirds_j | m_wirds_h | merr_wirds_h | m_ap_wirds_h | merr_ap_wirds_h | m_wirds_k | merr_wirds_k | m_ap_wirds_k | merr_ap_wirds_k | f_wirds_u | ferr_wirds_u | flag_wirds_u | f_ap_wirds_u | ferr_ap_wirds_u | f_wirds_g | ferr_wirds_g | flag_wirds_g | f_ap_wirds_g | ferr_ap_wirds_g | f_wirds_r | ferr_wirds_r | flag_wirds_r | f_ap_wirds_r | ferr_ap_wirds_r | f_wirds_i | ferr_wirds_i | flag_wirds_i | f_ap_wirds_i | ferr_ap_wirds_i | f_wirds_z | ferr_wirds_z | flag_wirds_z | f_ap_wirds_z | ferr_ap_wirds_z | f_wirds_j | ferr_wirds_j | flag_wirds_j | f_ap_wirds_j | ferr_ap_wirds_j | f_wirds_h | ferr_wirds_h | flag_wirds_h | f_ap_wirds_h | ferr_ap_wirds_h | f_wirds_k | ferr_wirds_k | flag_wirds_k | f_ap_wirds_k | ferr_ap_wirds_k |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1 | 149.6406419 | 1.7301753 | 13.7393 | 0.0001 | 14.9751 | 0.0001 | 13.1002 | 0.0 | 15.2938 | 0.0 | 12.8992 | 0.0 | 15.1746 | 0.0 | 13.1542 | 0.0 | 15.5602 | 0.0 | 12.3328 | 0.0 | 14.3174 | 0.0 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 11595.2 | 1.06796147302 | False | 3715.01 | 0.342165166512 | 20889.1 | 0.0 | False | 2770.0 | 0.0 | 25137.4 | 0.0 | False | 3091.43 | 0.0 | 19875.6 | 0.0 | False | 2167.3 | 0.0 | 42352.6 | 0.0 | False | 6808.32 | 0.0 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
1 | 2 | 149.828627 | 1.7247336 | 19.895 | 0.0018 | 21.1127 | 0.0014 | 18.1032 | 0.0002 | 19.2616 | 0.0002 | 17.2137 | 0.0001 | 18.3261 | 0.0001 | 16.7427 | 0.0001 | 17.8304 | 0.0001 | 16.4421 | 0.0002 | 17.5265 | 0.0002 | 15.9215 | 0.001 | 16.9939 | 0.0005 | 15.6627 | 0.0004 | 16.6788 | 0.0002 | 15.6748 | 0.0005 | 16.6672 | 0.0003 | 39.9945 | 0.066305254586 | False | 13.0293 | 0.0168005646628 | 208.314 | 0.0383729406167 | False | 71.6737 | 0.0132027751533 | 472.629 | 0.0435307039879 | False | 169.652 | 0.0156255686306 | 729.322 | 0.0671730784234 | False | 267.818 | 0.0246669485932 | 961.966 | 0.177200615872 | False | 354.323 | 0.0652687507682 | 1553.82 | 1.43111823127 | False | 578.682 | 0.266492774244 | 1972.06 | 0.72653349489 | False | 773.535 | 0.14249032829 | 1950.2 | 0.898101716302 | False | 781.843 | 0.216031330638 |
2 | 3 | 149.8262472 | 1.7172161 | 14.66 | 0.0001 | 15.2057 | 0.0001 | 13.6094 | 0.0 | 15.35 | 0.0 | 13.3392 | 0.0 | 15.2236 | 0.0 | 13.5202 | 0.0 | 15.5612 | 0.0 | 12.7953 | 0.0 | 14.3101 | 0.0 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 4965.92 | 0.457378290594 | False | 3004.14 | 0.276691280305 | 13068.9 | 0.0 | False | 2630.27 | 0.0 | 16761.8 | 0.0 | False | 2955.01 | 0.0 | 14188.0 | 0.0 | False | 2165.31 | 0.0 | 27661.8 | 0.0 | False | 6854.25 | 0.0 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
3 | 4 | 149.6261489 | 1.7188406 | 15.51 | 0.0001 | 15.7665 | 0.0001 | 14.3873 | 0.0001 | 15.3027 | 0.0001 | 13.8881 | 0.0001 | 15.1769 | 0.0001 | 13.8746 | 0.0001 | 15.5549 | 0.0001 | 13.3695 | 0.0001 | 14.3361 | 0.0001 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 2269.86 | 0.20906215068 | False | 1792.25 | 0.165072712116 | 6383.81 | 0.587970623747 | False | 2747.39 | 0.253043673001 | 10110.2 | 0.931184459478 | False | 3084.89 | 0.284129031934 | 10236.7 | 0.942834280431 | False | 2177.91 | 0.20059288945 | 16300.5 | 1.50132775307 | False | 6692.06 | 0.616361852735 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
4 | 5 | 149.9132691 | 1.7144459 | 14.3728 | 0.0001 | 15.0035 | 0.0001 | 13.4301 | 0.0 | 15.3215 | 0.0 | 12.9532 | 0.0 | 15.1882 | 0.0 | 13.0106 | 0.0 | 15.547 | 0.0 | 12.4966 | 0.0 | 14.2915 | 0.0 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 6469.64 | 0.595875643194 | False | 3619.09 | 0.333330943249 | 15415.6 | 0.0 | False | 2700.22 | 0.0 | 23917.7 | 0.0 | False | 3052.95 | 0.0 | 22686.1 | 0.0 | False | 2193.81 | 0.0 | 36421.7 | 0.0 | False | 6972.68 | 0.0 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | False | nan | nan |
5 | 6 | 149.9166838 | 1.7108627 | 21.7229 | 0.005 | 22.0077 | 0.0027 | 19.4284 | 0.0005 | 19.6429 | 0.0003 | 18.2359 | 0.0002 | 18.4192 | 0.0002 | 17.4763 | 0.0002 | 17.6439 | 0.0001 | 17.1537 | 0.0003 | 17.3338 | 0.0002 | 16.8013 | 0.0012 | 17.0292 | 0.0006 | 16.5672 | 0.0006 | 16.7762 | 0.0003 | 16.8771 | 0.0009 | 17.0592 | 0.0004 | 7.42745 | 0.0342046610058 | False | 5.71373 | 0.0142088631037 | 61.4667 | 0.0283064691757 | False | 50.4475 | 0.0139391660923 | 184.348 | 0.0339582416927 | False | 155.711 | 0.0286830327241 | 371.091 | 0.068357452983 | False | 318.009 | 0.0292897486361 | 499.482 | 0.138011999661 | False | 423.136 | 0.0779445515946 | 691.003 | 0.763724721037 | False | 560.17 | 0.309561565518 | 857.275 | 0.473747507203 | False | 707.164 | 0.195396656636 | 644.406 | 0.534168106969 | False | 544.904 | 0.200749957003 |
6 | 7 | 149.9790552 | 1.7162439 | 15.7598 | 0.0001 | 15.9526 | 0.0001 | 15.1829 | 0.0 | 15.5756 | 0.0 | 14.9598 | 0.0 | 15.4135 | 0.0 | 15.0653 | 0.0 | 15.6785 | 0.0 | 14.5827 | 0.0 | 14.8264 | 0.0 | 14.2189 | 0.0001 | 14.4236 | 0.0001 | 14.443 | 0.0001 | 14.6306 | 0.0001 | 14.826 | 0.0001 | 14.9653 | 0.0001 | 1803.35 | 0.166094640736 | False | 1509.94 | 0.139070744626 | 3067.89 | 0.0 | False | 2136.78 | 0.0 | 3767.73 | 0.0 | False | 2480.85 | 0.0 | 3418.85 | 0.0 | False | 1943.57 | 0.0 | 5332.37 | 0.0 | False | 4260.3 | 0.0 | 7454.87 | 0.68661859259 | False | 6173.9 | 0.568637391552 | 6064.57 | 0.558567466214 | False | 5102.23 | 0.469932565466 | 4261.87 | 0.392532395199 | False | 3748.69 | 0.345267448574 |
7 | 8 | 149.807708 | 1.7156341 | 20.8242 | 0.0038 | 22.3157 | 0.0035 | 19.6244 | 0.0008 | 21.0039 | 0.0008 | 18.9275 | 0.0006 | 20.1705 | 0.0005 | 18.5236 | 0.0006 | 19.6895 | 0.0004 | 18.3292 | 0.0011 | 19.4644 | 0.0007 | 18.0621 | 0.0061 | 19.0309 | 0.003 | 17.743 | 0.0027 | 18.7592 | 0.0013 | 17.6467 | 0.0028 | 18.6449 | 0.0014 | 16.995 | 0.0594811321207 | False | 4.30249 | 0.0138695750138 | 51.3145 | 0.0378099444788 | False | 14.4026 | 0.0106121966382 | 97.4989 | 0.0538798994967 | False | 31.0313 | 0.0142904373206 | 141.436 | 0.0781604088843 | False | 48.3281 | 0.0178047135705 | 169.169 | 0.171391005279 | False | 59.4621 | 0.0383366113965 | 216.352 | 1.21552959026 | False | 88.6422 | 0.244927396125 | 290.268 | 0.721837137826 | False | 113.847 | 0.136313614348 | 317.19 | 0.818000559229 | False | 126.485 | 0.163096179313 |
8 | 9 | 150.2731697 | 1.7146767 | 20.5961 | 0.0026 | 22.2879 | 0.0031 | 19.1311 | 0.0005 | 20.5427 | 0.0006 | 18.2754 | 0.0003 | 19.5024 | 0.0003 | 17.7904 | 0.0003 | 18.9453 | 0.0002 | 17.525 | 0.0005 | 18.6443 | 0.0004 | 16.9634 | 0.0019 | 18.0901 | 0.0012 | 16.7337 | 0.0012 | 17.7804 | 0.0007 | 16.7139 | 0.0011 | 17.7393 | 0.0007 | 20.9681 | 0.0502121634781 | False | 4.41407 | 0.01260307954 | 80.8276 | 0.0372225040337 | False | 22.0252 | 0.0121715744172 | 177.762 | 0.0491175553179 | False | 57.4169 | 0.0158648799697 | 277.869 | 0.0767780235037 | False | 95.9136 | 0.0176679386641 | 354.813 | 0.163397591677 | False | 126.555 | 0.0466246216092 | 595.168 | 1.04152367567 | False | 210.843 | 0.233032763936 | 735.394 | 0.812787213363 | False | 280.44 | 0.18080646696 | 748.928 | 0.758766825311 | False | 291.259 | 0.187781848945 |
9 | 10 | 150.5890278 | 1.71465 | 23.2844 | 0.0174 | 23.8905 | 0.0117 | 21.4882 | 0.0026 | 22.0811 | 0.0019 | 20.1175 | 0.001 | 20.6588 | 0.0007 | 19.5072 | 0.0008 | 20.0129 | 0.0006 | 19.2027 | 0.0014 | 19.702 | 0.0009 | 18.6783 | 0.0056 | 19.1819 | 0.0034 | 18.2579 | 0.0022 | 18.6917 | 0.0012 | 18.0716 | 0.0021 | 18.4872 | 0.0012 | 1.76295 | 0.0282530006643 | False | 1.00879 | 0.010870810479 | 9.21977 | 0.0220784841076 | False | 5.34023 | 0.00934520539886 | 32.5836 | 0.0300106348732 | False | 19.7915 | 0.0127600813357 | 57.1636 | 0.0421197066316 | False | 35.879 | 0.0198274574359 | 75.6694 | 0.0975716771791 | False | 47.7749 | 0.039602112156 | 122.653 | 0.632620527176 | False | 77.1329 | 0.241543044103 | 180.651 | 0.366048052092 | False | 121.149 | 0.133898842614 | 214.467 | 0.41481537628 | False | 146.258 | 0.161650415976 |