{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# COSMOS master catalogue\n", "## Preparation of UKIRT Infrared Deep Sky Survey / Large Area Survey (UKIDSS/LAS)\n", "\n", "Information about UKIDSS can be found at http://www.ukidss.org/surveys/surveys.html\n", "\n", "The catalogue comes from `dmu0_UKIDSS-LAS`.\n", "\n", "In the catalogue, we keep:\n", "\n", "- The identifier (it's unique in the catalogue);\n", "- The position;\n", "- The stellarity;\n", "- The magnitude for each band in aperture 3 (2 arcsec).\n", "- The hall magnitude is described as the total magnitude.\n", "\n", "J band magnitudes are available in two eopchs. We take the first arbitrarily.\n", "\n", "The magnitudes are “*Vega like*”. The AB offsets are given by Hewett *et al.* (2016):\n", "\n", "| Band | AB offset |\n", "|------|-----------|\n", "| Y | 0.634 |\n", "| J | 0.938 |\n", "| H | 1.379 |\n", "| K | 1.900 |\n", "\n", "Each source is associated with an epoch. These range between 2005 and 2007. We take 2006 for the epoch." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "33f5ec7 (Wed Dec 6 16:56:17 2017 +0000)\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, mag_to_flux" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"las_ra\"\n", "DEC_COL = \"las_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: UnitsWarning: 'RADIANS' did not parse as fits unit: At col 0, Unit 'RADIANS' not supported by the FITS standard. [astropy.units.core]\n" ] } ], "source": [ "#Is the following standard (different names for radec vs mag)?\n", "imported_columns = OrderedDict({\n", " 'SOURCEID': 'las_id',\n", " 'RA': 'las_ra',\n", " 'Dec': 'las_dec',\n", " 'YHALLMAG': 'm_ukidss_y',\n", " 'YHALLMAGERR': 'merr_ukidss_y',\n", " 'YAPERMAG3': 'm_ap_ukidss_y',\n", " 'YAPERMAG3ERR': 'merr_ap_ukidss_y',\n", " 'J_1HALLMAG': 'm_ukidss_j',\n", " 'J_1HALLMAGERR': 'merr_ukidss_j',\n", " 'J_1APERMAG3': 'm_ap_ukidss_j',\n", " 'J_1APERMAG3ERR': 'merr_ap_ukidss_j',\n", " 'HAPERMAG3': 'm_ap_ukidss_h',\n", " 'HAPERMAG3ERR': 'merr_ap_ukidss_h',\n", " 'HHALLMAG': 'm_ukidss_h',\n", " 'HHALLMAGERR': 'merr_ukidss_h',\n", " 'KAPERMAG3': 'm_ap_ukidss_k',\n", " 'KAPERMAG3ERR': 'merr_ap_ukidss_k',\n", " 'KHALLMAG': 'm_ukidss_k',\n", " 'KHALLMAGERR': 'merr_ukidss_k',\n", " 'PSTAR': 'las_stellarity'\n", " })\n", "\n", "catalogue = Table.read(\n", " \"../../dmu0/dmu0_UKIDSS-LAS/data/UKIDSS-LAS_COSMOS.fits\")[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", "\n", "#Epochs between 2005 and 2007. Rough average:\n", "epoch = 2006\n", "\n", "# Clean table metadata\n", "catalogue.meta = None" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.\n", "Check the NumPy 1.11 release notes for more information.\n", " ma.MaskedArray.__setitem__(self, index, value)\n" ] } ], "source": [ "# Adding flux and band-flag columns\n", "for col in catalogue.colnames:\n", " if col.startswith('m_'):\n", " \n", " errcol = \"merr{}\".format(col[1:])\n", " \n", " # LAS uses a huge negative number for missing values\n", " catalogue[col][catalogue[col] < -100] = np.nan\n", " catalogue[errcol][catalogue[errcol] < -100] = np.nan \n", "\n", " # Vega to AB correction\n", " if col.endswith('y'):\n", " catalogue[col] += 0.634\n", " elif col.endswith('j'):\n", " catalogue[col] += 0.938\n", " elif col.endswith('h'):\n", " catalogue[col] += 1.379\n", " elif col.endswith('k'):\n", " catalogue[col] += 1.900\n", " else:\n", " print(\"{} column has wrong band...\".format(col))\n", " \n", " flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))\n", " \n", " # Fluxes are added in µJy\n", " catalogue.add_column(Column(flux * 1.e6, name=\"f{}\".format(col[1:])))\n", " catalogue.add_column(Column(error * 1.e6, name=\"f{}\".format(errcol[1:])))\n", " \n", " # Band-flag column\n", " if \"ap\" not in col:\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag{}\".format(col[1:])))\n", " \n", "# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<Table masked=True length=10>\n", "
idx | las_id | las_ra | las_dec | m_ukidss_y | merr_ukidss_y | m_ap_ukidss_y | merr_ap_ukidss_y | m_ukidss_j | merr_ukidss_j | m_ap_ukidss_j | merr_ap_ukidss_j | m_ap_ukidss_h | merr_ap_ukidss_h | m_ukidss_h | merr_ukidss_h | m_ap_ukidss_k | merr_ap_ukidss_k | m_ukidss_k | merr_ukidss_k | las_stellarity | f_ukidss_y | ferr_ukidss_y | flag_ukidss_y | f_ap_ukidss_y | ferr_ap_ukidss_y | f_ukidss_j | ferr_ukidss_j | flag_ukidss_j | f_ap_ukidss_j | ferr_ap_ukidss_j | f_ap_ukidss_h | ferr_ap_ukidss_h | f_ukidss_h | ferr_ukidss_h | flag_ukidss_h | f_ap_ukidss_k | ferr_ap_ukidss_k | f_ukidss_k | ferr_ukidss_k | flag_ukidss_k |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 433845436807 | 149.943004394 | 3.37550061639 | nan | nan | nan | nan | 20.5811 | 0.184875 | 20.4399 | 0.177104 | nan | nan | nan | nan | nan | nan | nan | nan | 0.9 | nan | nan | False | nan | nan | 21.2605 | 3.62015 | False | 24.2126 | 3.94952 | nan | nan | nan | nan | False | nan | nan | nan | nan | False |
1 | 433845436166 | 149.961687816 | 3.36943768275 | 17.5341 | 0.0163309 | 17.4586 | 0.0126952 | 17.6294 | 0.0181771 | 17.5423 | 0.0138783 | 17.6843 | 0.0220383 | 17.7174 | 0.0328349 | 18.1684 | 0.0276244 | 18.2432 | 0.0375468 | 0.999981 | 351.856 | 5.29238 | False | 377.194 | 4.41043 | 322.286 | 5.39563 | False | 349.21 | 4.46373 | 306.39 | 6.21911 | 297.191 | 8.98767 | False | 196.17 | 4.99115 | 183.114 | 6.33242 | False |
2 | 433845436168 | 149.988292811 | 3.37060250652 | 17.8196 | 0.0211817 | 17.7384 | 0.0154704 | 17.7481 | 0.0211179 | 17.6644 | 0.0152359 | 17.4964 | 0.0186358 | 17.4844 | 0.0284849 | 17.8284 | 0.020604 | 17.8812 | 0.0280092 | 0.999981 | 270.49 | 5.277 | False | 291.488 | 4.15334 | 288.896 | 5.61913 | False | 312.06 | 4.37907 | 364.296 | 6.25286 | 368.321 | 9.66309 | False | 268.301 | 5.09153 | 255.567 | 6.59298 | False |
3 | 433845436187 | 149.951551452 | 3.37635377512 | 18.8597 | 0.120119 | 19.6997 | 0.0767001 | 18.7663 | 0.0801877 | 19.3658 | 0.0669343 | 18.8817 | 0.0638599 | 18.1888 | 0.0947157 | 18.6669 | 0.0426688 | 17.8215 | 0.055345 | 9.52581e-06 | 103.782 | 11.4818 | False | 47.8744 | 3.38201 | 113.1 | 8.35307 | False | 65.1133 | 4.01415 | 101.703 | 5.98188 | 192.52 | 16.7948 | False | 123.946 | 4.87099 | 270.034 | 13.7649 | False |
4 | 433845436930 | 149.988020812 | 3.37863159768 | nan | nan | nan | nan | nan | nan | nan | nan | 19.8766 | 0.156687 | 19.952 | 0.30431 | 19.4762 | 0.0874744 | 19.637 | 0.193198 | 0.00306749 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | 40.6773 | 5.8703 | 37.9477 | 10.636 | False | 58.8201 | 4.73896 | 50.7233 | 9.02579 | False |
5 | 433845436224 | 149.995715204 | 3.3852017212 | 19.95 | 0.173118 | 19.9344 | 0.0933938 | 19.6091 | 0.118857 | 19.5926 | 0.0809342 | 19.3293 | 0.0948812 | 19.6611 | 0.210754 | 19.6989 | 0.106622 | 19.8091 | 0.144176 | 0.999981 | 38.0185 | 6.06195 | False | 38.5678 | 3.31756 | 52.0419 | 5.6971 | False | 52.8394 | 3.93881 | 67.3431 | 5.88503 | 49.61 | 9.62986 | False | 47.9129 | 4.70519 | 43.2868 | 5.74809 | False |
6 | 433845436237 | 149.986283486 | 3.38710769004 | 18.8418 | 0.0556114 | 18.8218 | 0.0360102 | 18.8819 | 0.0579137 | 18.7128 | 0.037063 | 18.6477 | 0.0512846 | 18.7338 | 0.0886796 | 18.8728 | 0.0508002 | 18.9829 | 0.0736365 | 0.999981 | 105.506 | 5.40401 | False | 107.468 | 3.56434 | 101.677 | 5.42348 | False | 118.815 | 4.0559 | 126.16 | 5.95915 | 116.546 | 9.5191 | False | 102.536 | 4.79755 | 92.6502 | 6.28369 | False |
7 | 433845436247 | 149.991241566 | 3.388114693 | 17.0381 | 0.0111581 | 16.9837 | 0.00916234 | 17.139 | 0.0117432 | 17.0438 | 0.00924676 | 17.1768 | 0.0141222 | 17.2129 | 0.0211751 | 17.6132 | 0.017095 | 17.6993 | 0.0239601 | 0.999981 | 555.597 | 5.70986 | False | 584.169 | 4.9297 | 506.314 | 5.47625 | False | 552.702 | 4.70713 | 488.977 | 6.36013 | 472.968 | 9.22428 | False | 327.126 | 5.15061 | 302.186 | 6.66867 | False |
8 | 433845436255 | 149.99970128 | 3.38869980634 | 15.4337 | 0.00403248 | 15.4091 | 0.00366633 | 15.536 | 0.00364612 | 15.4843 | 0.00315254 | 15.5854 | 0.0040656 | 15.6199 | 0.00532505 | 16.017 | 0.00505145 | 16.0825 | 0.00616231 | 0.999981 | 2435.07 | 9.04398 | False | 2490.83 | 8.41108 | 2216.13 | 7.4422 | False | 2324.25 | 6.74867 | 2117.65 | 7.92964 | 2051.42 | 10.0613 | False | 1422.98 | 6.62048 | 1339.69 | 7.60366 | False |
9 | 433845436985 | 149.94668932 | 3.3696568889 | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | nan | 20.1934 | 0.168819 | 19.6578 | 0.192897 | 0.05 | nan | nan | False | nan | nan | nan | nan | False | nan | nan | nan | nan | nan | nan | False | 30.3836 | 4.72428 | 49.7609 | 8.84077 | False |