{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# HATLAS-NGP master catalogue\n", "## Preparation of UKIRT Infrared Deep Sky Survey / Large Area Survey (UKIDSS/LAS)\n", "\n", "Information about UKIDSS can be found at http://www.ukidss.org/surveys/surveys.html\n", "\n", "The catalogue comes from `dmu0_UKIDSS-LAS`.\n", "\n", "In the catalogue, we keep:\n", "\n", "- The identifier (it's unique in the catalogue);\n", "- The position;\n", "- The stellarity;\n", "- The magnitude for each band in aperture 3 (2 arcsec).\n", "- The hall magnitude is described as the total magnitude.\n", "\n", "J band magnitudes are available in two eopchs. We take the first arbitrarily.\n", "\n", "The magnitudes are “*Vega like*”. The AB offsets are given by Hewett *et al.* (2016):\n", "\n", "| Band | AB offset |\n", "|------|-----------|\n", "| Y | 0.634 |\n", "| J | 0.938 |\n", "| H | 1.379 |\n", "| K | 1.900 |\n", "\n", "Each source is associated with an epoch. These range between 2005 and 2007. We take 2006 for the epoch." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, mag_to_flux" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"las_ra\"\n", "DEC_COL = \"las_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "WARNING: UnitsWarning: 'RADIANS' did not parse as fits unit: At col 0, Unit 'RADIANS' not supported by the FITS standard. [astropy.units.core]\n" ] } ], "source": [ "#Is the following standard (different names for radec vs mag)?\n", "imported_columns = OrderedDict({\n", " 'SOURCEID': 'las_id',\n", " 'RA': 'las_ra',\n", " 'Dec': 'las_dec',\n", " 'YHALLMAG': 'm_ukidss_y',\n", " 'YHALLMAGERR': 'merr_ukidss_y',\n", " 'YAPERMAG3': 'm_ap_ukidss_y',\n", " 'YAPERMAG3ERR': 'merr_ap_ukidss_y',\n", " 'J_1HALLMAG': 'm_ukidss_j',\n", " 'J_1HALLMAGERR': 'merr_ukidss_j',\n", " 'J_1APERMAG3': 'm_ap_ukidss_j',\n", " 'J_1APERMAG3ERR': 'merr_ap_ukidss_j',\n", " 'HAPERMAG3': 'm_ap_ukidss_h',\n", " 'HAPERMAG3ERR': 'merr_ap_ukidss_h',\n", " 'HHALLMAG': 'm_ukidss_h',\n", " 'HHALLMAGERR': 'merr_ukidss_h',\n", " 'KAPERMAG3': 'm_ap_ukidss_k',\n", " 'KAPERMAG3ERR': 'merr_ap_ukidss_k',\n", " 'KHALLMAG': 'm_ukidss_k',\n", " 'KHALLMAGERR': 'merr_ukidss_k',\n", " 'PSTAR': 'las_stellarity'\n", " })\n", "\n", "catalogue = Table.read(\n", " \"../../dmu0/dmu0_UKIDSS-LAS/data/UKIDSS-LAS_NGP.fits\")[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", "\n", "#Epochs between 2005 and 2007. Rough average:\n", "epoch = 2006\n", "\n", "# Clean table metadata\n", "catalogue.meta = None" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.\n", "Check the NumPy 1.11 release notes for more information.\n", " ma.MaskedArray.__setitem__(self, index, value)\n" ] } ], "source": [ "# Adding flux and band-flag columns\n", "for col in catalogue.colnames:\n", " if col.startswith('m_'):\n", " \n", " errcol = \"merr{}\".format(col[1:])\n", " \n", " # LAS uses a huge negative number for missing values\n", " catalogue[col][catalogue[col] < -100] = np.nan\n", " catalogue[errcol][catalogue[errcol] < -100] = np.nan \n", "\n", " # Vega to AB correction\n", " if col.endswith('y'):\n", " catalogue[col] += 0.634\n", " elif col.endswith('j'):\n", " catalogue[col] += 0.938\n", " elif col.endswith('h'):\n", " catalogue[col] += 1.379\n", " elif col.endswith('k'):\n", " catalogue[col] += 1.900\n", " else:\n", " print(\"{} column has wrong band...\".format(col))\n", " \n", " flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))\n", " \n", " # Fluxes are added in µJy\n", " catalogue.add_column(Column(flux * 1.e6, name=\"f{}\".format(col[1:])))\n", " catalogue.add_column(Column(error * 1.e6, name=\"f{}\".format(errcol[1:])))\n", " \n", " # Band-flag column\n", " if \"ap\" not in col:\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag{}\".format(col[1:])))\n", " \n", "# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting." ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<Table masked=True length=10>\n", "
idx | las_id | las_ra | las_dec | m_ukidss_y | merr_ukidss_y | m_ap_ukidss_y | merr_ap_ukidss_y | m_ukidss_j | merr_ukidss_j | m_ap_ukidss_j | merr_ap_ukidss_j | m_ap_ukidss_h | merr_ap_ukidss_h | m_ukidss_h | merr_ukidss_h | m_ap_ukidss_k | merr_ap_ukidss_k | m_ukidss_k | merr_ukidss_k | las_stellarity | f_ukidss_y | ferr_ukidss_y | flag_ukidss_y | f_ap_ukidss_y | ferr_ap_ukidss_y | f_ukidss_j | ferr_ukidss_j | flag_ukidss_j | f_ap_ukidss_j | ferr_ap_ukidss_j | f_ap_ukidss_h | ferr_ap_ukidss_h | f_ukidss_h | ferr_ukidss_h | flag_ukidss_h | f_ap_ukidss_k | ferr_ap_ukidss_k | f_ukidss_k | ferr_ukidss_k | flag_ukidss_k |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 433795438053 | 206.360659354 | 29.7855932047 | 20.0984 | 0.207045 | 20.7217 | 0.145656 | nan | nan | nan | nan | 20.0392 | 0.138917 | 19.4186 | 0.21079 | 20.3836 | 0.240432 | 20.1323 | 0.39326 | 0.000171409 | 33.1608 | 6.32362 | False | 18.6769 | 2.50558 | nan | nan | False | nan | nan | 35.0195 | 4.48065 | 62.0241 | 12.0417 | False | 25.5012 | 5.64712 | 32.1436 | 11.6426 | False |
1 | 433795438054 | 206.361234461 | 29.7299660924 | 20.5262 | 0.30814 | 20.8066 | 0.155936 | 20.6763 | 0.179253 | 20.668 | 0.190655 | 20.1888 | 0.157944 | 19.8291 | 0.269121 | nan | nan | nan | nan | 0.000171409 | 22.3632 | 6.34683 | False | 17.2726 | 2.48073 | 19.4751 | 3.2153 | False | 19.6251 | 3.44615 | 30.5113 | 4.43854 | 42.4955 | 10.5334 | False | nan | nan | nan | nan | False |
2 | 433795438055 | 206.361804481 | 29.7276148576 | 20.8654 | 0.270048 | 20.7487 | 0.147968 | 20.5554 | 0.226177 | 20.4512 | 0.156395 | 20.5749 | 0.224459 | 20.4668 | 0.372434 | nan | nan | nan | nan | 0.999657 | 16.3616 | 4.06952 | False | 18.2196 | 2.48303 | 21.77 | 4.53506 | False | 23.9623 | 3.45165 | 21.3817 | 4.42034 | 23.6197 | 8.10214 | False | nan | nan | nan | nan | False |
3 | 433795438057 | 206.363384464 | 29.7827067153 | 20.9654 | 0.370267 | 21.0186 | 0.189817 | nan | nan | nan | nan | 20.4649 | 0.204532 | 20.2374 | 0.296498 | nan | nan | nan | nan | 0.486486 | 14.9227 | 5.08906 | False | 14.2088 | 2.48408 | nan | nan | False | nan | nan | 23.6606 | 4.45722 | 29.1762 | 7.96758 | False | nan | nan | nan | nan | False |
4 | 433795438058 | 206.361917162 | 29.6159844322 | 19.7907 | 0.197032 | 20.5136 | 0.118256 | 19.7558 | 0.122487 | 20.3115 | 0.135587 | 20.0603 | 0.138222 | 19.033 | 0.173318 | 20.1478 | 0.188964 | 19.5717 | 0.250772 | 9.52581e-06 | 44.0284 | 7.98996 | False | 22.6243 | 2.46419 | 45.4652 | 5.12914 | False | 27.2514 | 3.40316 | 34.3456 | 4.37243 | 88.4675 | 14.1222 | False | 31.6883 | 5.5151 | 53.8667 | 12.4416 | False |
5 | 433795438059 | 206.363582148 | 29.7647248542 | 17.9042 | 0.017478 | 17.8518 | 0.0139216 | 17.9594 | 0.0237724 | 17.908 | 0.0168219 | 18.0485 | 0.0235778 | 18.1199 | 0.0367644 | 18.552 | 0.045628 | 18.5362 | 0.0710826 | 0.999981 | 250.209 | 4.02782 | False | 262.6 | 3.36713 | 237.809 | 5.20687 | False | 249.345 | 3.86324 | 219.074 | 4.75741 | 205.139 | 6.94625 | False | 137.79 | 5.79062 | 139.805 | 9.15299 | False |
6 | 433795438060 | 206.362717685 | 29.6396639809 | 19.7809 | 0.113873 | 19.7498 | 0.0607979 | 19.4273 | 0.104103 | 19.4432 | 0.0620997 | 19.1494 | 0.0607773 | 19.0219 | 0.090905 | 18.8882 | 0.0603746 | 18.7441 | 0.10025 | 9.52581e-06 | 44.4271 | 4.65956 | False | 45.7168 | 2.56 | 61.5262 | 5.89925 | False | 60.6343 | 3.46804 | 79.4787 | 4.44905 | 89.3772 | 7.48324 | False | 101.092 | 5.62143 | 115.444 | 10.6594 | False |
7 | 433795438061 | 206.363122774 | 29.6467782186 | 18.6077 | 0.0310503 | 18.5243 | 0.0224318 | 18.4167 | 0.0337989 | 18.331 | 0.0235453 | 18.2468 | 0.0274628 | 18.3729 | 0.0409548 | 18.5049 | 0.0429273 | 18.5114 | 0.0619389 | 0.999981 | 130.9 | 3.74351 | False | 141.348 | 2.92031 | 156.063 | 4.85824 | False | 168.882 | 3.66238 | 182.505 | 4.61632 | 162.49 | 6.12924 | False | 143.889 | 5.68903 | 143.029 | 8.15947 | False |
8 | 433795438062 | 206.363569762 | 29.6581647522 | 19.7249 | 0.127328 | 20.2598 | 0.0948532 | 19.6079 | 0.134669 | 19.8476 | 0.0894695 | 20.0179 | 0.1335 | 19.436 | 0.251983 | 19.7324 | 0.129873 | 19.5996 | 0.333055 | 9.52581e-06 | 46.7796 | 5.48602 | False | 28.5824 | 2.49705 | 52.0988 | 6.46205 | False | 41.7774 | 3.44264 | 35.7143 | 4.39134 | 61.0362 | 14.1656 | False | 46.4561 | 5.55696 | 52.5021 | 16.1053 | False |
9 | 433795438063 | 206.364376219 | 29.7717066453 | 16.2387 | 0.00555804 | 16.1877 | 0.0049949 | 16.1142 | 0.00507694 | 16.0551 | 0.00428316 | 15.8968 | 0.00446262 | 15.9414 | 0.00582707 | 16.2847 | 0.00689185 | 16.3237 | 0.00945186 | 0.999981 | 1160.21 | 5.93928 | False | 1216.01 | 5.59424 | 1301.16 | 6.08429 | False | 1373.95 | 5.42016 | 1589.53 | 6.53334 | 1525.54 | 8.18745 | False | 1112.04 | 7.0588 | 1072.81 | 9.33932 | False |