{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# ELAIS S1 Master List Creation\n", "\n", "## Preparation of VIDEO/VISTA/VIRCAM data\n", "\n", "\n", "The catalogue comes from `dmu0_VISTA_VIDEO-private`.\n", "\n", "There is an old public version of the catalogue but we are using the newer private version in the hope that it will be public by the time we publish the masterlist.\n", "\n", "Filters: Y,J,H,Ks \n", "\n", "In the catalogue, we keep:\n", "\n", "- The identifier (it's unique in the catalogue);\n", "- The position (degrees);\n", "- The stellarity;\n", "- The magnitude for each band in apertude 3, which is 2 arcsec (rs548 presumes same for private catalogue).\n", "- The “auto” magnitude is provided, we presume this is standard Sextractor units etc.\n", "\n", "Yannick said the dates of observation for VIDEO are from 2009/11 to 2016/12. There is a paper from 2012 (Jarvis et al). So will use 2012.\n", "\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "from collections import OrderedDict\n", "import os\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table\n", "import numpy as np\n", "\n", "from herschelhelp_internal.flagging import gaia_flag_column\n", "from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates\n", "from herschelhelp_internal.utils import astrometric_correction, mag_to_flux" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'en_GB'" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import locale\n", "locale.setlocale(locale.LC_ALL, 'en_GB')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import logging\n", "logger = logging.getLogger()\n", "logger.setLevel(logging.INFO)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "OUT_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass\n", "\n", "RA_COL = \"video_ra\"\n", "DEC_COL = \"video_dec\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Column selection" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "imported_columns = OrderedDict({\n", " 'ID': 'video_id', \n", " 'ALPHA_J2000': 'video_ra', \n", " 'DELTA_J2000': 'video_dec', \n", " 'J_CLASS_STAR': 'video_stellarity', \n", " 'Z_MAG_APER_3': 'm_ap_video_z', \n", " 'Z_MAGERR_APER_3': 'merr_ap_video_z', \n", " 'Z_MAG_AUTO': 'm_video_z', \n", " 'Z_MAGERR_AUTO': 'merr_video_z',\n", " 'Z_FLUX_APER_3': 'f_ap_video_z', \n", " 'Z_FLUXERR_APER_3':'ferr_ap_video_z', \n", " 'Z_FLUX_AUTO': 'f_video_z', \n", " 'Z_FLUXERR_AUTO': 'ferr_video_z',\n", " 'Y_MAG_APER_3': 'm_ap_video_y', \n", " 'Y_MAGERR_APER_3': 'merr_ap_video_y', \n", " 'Y_MAG_AUTO': 'm_video_y', \n", " 'Y_MAGERR_AUTO': 'merr_video_y',\n", " 'Y_FLUX_APER_3': 'f_ap_video_y', \n", " 'Y_FLUXERR_APER_3':'ferr_ap_video_y', \n", " 'Y_FLUX_AUTO': 'f_video_y', \n", " 'Y_FLUXERR_AUTO': 'ferr_video_y',\n", " 'J_MAG_APER_3': 'm_ap_video_j', \n", " 'J_MAGERR_APER_3': 'merr_ap_video_j', \n", " 'J_MAG_AUTO': 'm_video_j', \n", " 'J_MAGERR_AUTO': 'merr_video_j',\n", " 'J_FLUX_APER_3': 'f_ap_video_j', \n", " 'J_FLUXERR_APER_3':'ferr_ap_video_j', \n", " 'J_FLUX_AUTO': 'f_video_j', \n", " 'J_FLUXERR_AUTO': 'ferr_video_j',\n", " 'H_MAG_APER_3': 'm_ap_video_h', \n", " 'H_MAGERR_APER_3': 'merr_ap_video_h', \n", " 'H_MAG_AUTO': 'm_video_h', \n", " 'H_MAGERR_AUTO': 'merr_video_h',\n", " 'H_FLUX_APER_3': 'f_ap_video_h', \n", " 'H_FLUXERR_APER_3':'ferr_ap_video_h', \n", " 'H_FLUX_AUTO': 'f_video_h', \n", " 'H_FLUXERR_AUTO': 'ferr_video_h', \n", " 'K_MAG_APER_3': 'm_ap_video_k', \n", " 'K_MAGERR_APER_3': 'merr_ap_video_k', \n", " 'K_MAG_AUTO': 'm_video_k', \n", " 'K_MAGERR_AUTO': 'merr_video_k',\n", " 'K_FLUX_APER_3': 'f_ap_video_k', \n", " 'K_FLUXERR_APER_3':'ferr_ap_video_k', \n", " 'K_FLUX_AUTO': 'f_video_k', \n", " 'K_FLUXERR_AUTO': 'ferr_video_k'\n", "})\n", "\n", "\n", "catalogue = Table.read(\"../../dmu0/dmu0_VISTA-VIDEO-private/data/VIDEO-all_2017-02-12_fullcat_errfix_ELAIS-S1.fits\"\n", " )[list(imported_columns)]\n", "for column in imported_columns:\n", " catalogue[column].name = imported_columns[column]\n", " \n", " \n", "epoch = 2012\n", "\n", "# Clean table metadata\n", "catalogue.meta = None" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Correct wrong values.\n", "# The catalogue contains negative fluxes and 99 in the associated magnitude. These\n", "# values may be considered as upper limits but we don't want this information in the\n", "# master list. Also, some sources may have a very low signal to noise with the error\n", "# on magnitude superior to 99. In all these cases, we set the magnitude, the flux, and\n", "# the associated errors to NaN.\n", "\n", "band_names = [_[8:] for _ in catalogue.colnames if _.startswith(\"merr_ap_\")]\n", "\n", "for band in band_names:\n", " \n", " # Total fluxes and magnitudes\n", " mask = catalogue[\"m_{}\".format(band)] > 90.\n", " mask |= catalogue[\"merr_{}\".format(band)] > 90.\n", " #mask |= catalogue[\"f_{}\".format(band)] < 0.\n", " mask |= np.isclose(catalogue[\"ferr_{}\".format(band)] , 0.)\n", " catalogue[\"m_{}\".format(band)][mask] = np.nan\n", " catalogue[\"merr_{}\".format(band)][mask] = np.nan\n", " catalogue[\"f_{}\".format(band)][mask] = np.nan\n", " catalogue[\"ferr_{}\".format(band)][mask] = np.nan\n", " \n", " # Aperture fluxes and magnitudes\n", " mask = catalogue[\"m_ap_{}\".format(band)] > 90.\n", " mask |= catalogue[\"merr_ap_{}\".format(band)] > 90.\n", " #mask |= catalogue[\"f_ap_{}\".format(band)] < 0.\n", " mask |= np.isclose(catalogue[\"ferr_ap_{}\".format(band)] , 0.)\n", " catalogue[\"m_ap_{}\".format(band)][mask] = np.nan\n", " catalogue[\"merr_ap_{}\".format(band)][mask] = np.nan\n", " catalogue[\"f_ap_{}\".format(band)][mask] = np.nan\n", " catalogue[\"ferr_ap_{}\".format(band)][mask] = np.nan\n", " \n", " # Adding the band-flag column\n", " catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name=\"flag_{}\".format(band)))\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Calculating fluxes from magnitudes due to an issue with some objects\n", "for col in catalogue.colnames:\n", " if col.startswith('m_'):\n", " \n", " errcol = \"merr{}\".format(col[1:]) \n", " \n", " flux_old = catalogue[\"f{}\".format(col[1:])]\n", " error_old = catalogue[\"ferr{}\".format(col[1:])]\n", " \n", " flux_new, error_from_mag = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))\n", " error_new = flux_new * (error_old/flux_old)\n", " \n", " # Fluxes are added in µJy\n", " catalogue[\"f{}\".format(col[1:])] = flux_new * 1.e6\n", " catalogue[\"ferr{}\".format(col[1:])] = error_new * 1.e6\n", " \n" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "Table length=10\n", "
idx | video_id | video_ra | video_dec | video_stellarity | m_ap_video_z | merr_ap_video_z | m_video_z | merr_video_z | f_ap_video_z | ferr_ap_video_z | f_video_z | ferr_video_z | m_ap_video_y | merr_ap_video_y | m_video_y | merr_video_y | f_ap_video_y | ferr_ap_video_y | f_video_y | ferr_video_y | m_ap_video_j | merr_ap_video_j | m_video_j | merr_video_j | f_ap_video_j | ferr_ap_video_j | f_video_j | ferr_video_j | m_ap_video_h | merr_ap_video_h | m_video_h | merr_video_h | f_ap_video_h | ferr_ap_video_h | f_video_h | ferr_video_h | m_ap_video_k | merr_ap_video_k | m_video_k | merr_video_k | f_ap_video_k | ferr_ap_video_k | f_video_k | ferr_video_k | flag_video_z | flag_video_y | flag_video_j | flag_video_h | flag_video_k |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 1000001 | 9.22189563882 | -45.1497035749 | 3.56565e-05 | 22.2937 | 0.0968248 | 22.8411 | 0.150321 | 4.3906 | 0.391562 | 2.6519 | 0.36717 | 23.9577 | 0.826946 | 28.2943 | 37.0292 | 0.948236 | 0.722243 | 0.0174696 | 0.595824 | 22.9415 | 0.172272 | 25.1444 | 1.45016 | 2.41779 | 0.38364 | 0.317872 | 0.424577 | 24.2404 | 1.38406 | nan | nan | 0.730896 | 0.931754 | nan | nan | 20.4389 | 0.0933996 | 20.6395 | 0.0973266 | 24.2348 | 2.08485 | 20.1469 | 1.80605 | False | False | False | False | False |
1 | 1000002 | 9.11960666083 | -45.1492472378 | 0.748559 | 22.6408 | 0.11426 | 23.5344 | 0.240561 | 3.18922 | 0.335636 | 1.4004 | 0.310289 | 22.3925 | 0.0822917 | 22.8425 | 0.131069 | 4.00851 | 0.303829 | 2.64853 | 0.319739 | 22.3976 | 0.110655 | 22.5942 | 0.141997 | 3.99002 | 0.406666 | 3.32916 | 0.435416 | 22.113 | 0.155222 | 22.331 | 0.182456 | 5.18564 | 0.741389 | 4.24246 | 0.712961 | 21.5809 | 0.23158 | 22.2045 | 0.231844 | 8.46491 | 1.80557 | 4.76628 | 1.01781 | False | False | False | False | False |
2 | 1000003 | 9.4920486098 | -45.1496211595 | 0.90358 | 23.7058 | 0.311508 | 23.728 | 0.292208 | 1.19581 | 0.343099 | 1.17166 | 0.315344 | 22.724 | 0.192041 | 22.6693 | 0.1578 | 2.95404 | 0.522519 | 3.10644 | 0.451503 | 22.4071 | 0.145143 | 22.2624 | 0.124894 | 3.95514 | 0.528749 | 4.51904 | 0.519849 | 21.9929 | 0.172855 | 22.1646 | 0.185779 | 5.79212 | 0.922168 | 4.94495 | 0.846151 | 21.2428 | 0.153638 | 21.5024 | 0.156613 | 11.5582 | 1.63561 | 9.09962 | 1.31262 | False | False | False | False | False |
3 | 1000004 | 9.1708830207 | -45.14925548 | 0.932635 | 23.3836 | 0.191344 | 23.3718 | 0.206772 | 1.60904 | 0.283577 | 1.62656 | 0.309779 | 23.1818 | 0.157093 | 22.9818 | 0.16224 | 1.9377 | 0.280371 | 2.32968 | 0.348134 | 22.8777 | 0.16963 | 22.5366 | 0.147733 | 2.56407 | 0.400612 | 3.51054 | 0.477684 | 21.8608 | 0.118206 | 22.0172 | 0.153507 | 6.54181 | 0.712239 | 5.6638 | 0.800804 | 21.4337 | 0.133982 | 21.5287 | 0.136966 | 9.69418 | 1.19632 | 8.88256 | 1.12058 | False | False | False | False | False |
4 | 1000005 | 9.4980769958 | -45.1495933201 | 0.971243 | 23.4184 | 0.241671 | 23.564 | 0.306869 | 1.55828 | 0.346864 | 1.36273 | 0.38517 | 22.3549 | 0.09709 | 22.3143 | 0.110869 | 4.1501 | 0.371127 | 4.30809 | 0.439933 | 21.3685 | 0.0628635 | 21.3795 | 0.0705038 | 10.2946 | 0.59607 | 10.1903 | 0.661744 | 21.2135 | 0.0775846 | 21.2017 | 0.0877574 | 11.8739 | 0.848515 | 12.0035 | 0.970248 | 20.4055 | 0.0778657 | 20.3993 | 0.0776172 | 24.9921 | 1.79242 | 25.1349 | 1.79691 | False | False | False | False | False |
5 | 1000006 | 9.11701662708 | -45.1488084992 | 0.815506 | 22.8845 | 0.0933129 | 22.9591 | 0.124259 | 2.54804 | 0.218997 | 2.37882 | 0.272257 | 22.6567 | 0.0761661 | 22.7346 | 0.119869 | 3.1428 | 0.22048 | 2.92513 | 0.322956 | 22.3401 | 0.0810192 | 22.312 | 0.110797 | 4.20701 | 0.313944 | 4.31735 | 0.440591 | 22.1579 | 0.114592 | 22.422 | 0.197085 | 4.97558 | 0.525158 | 3.90117 | 0.708171 | 21.822 | 0.137795 | 21.7833 | 0.152177 | 6.77971 | 0.860465 | 7.02559 | 0.984742 | False | False | False | False | False |
6 | 1000007 | 9.56032705231 | -45.1491161056 | 0.98233 | 21.4954 | 0.0332294 | 21.4654 | 0.0400053 | 9.15875 | 0.280317 | 9.41538 | 0.346933 | 20.9455 | 0.0220709 | 20.9141 | 0.0280032 | 15.1985 | 0.308966 | 15.6442 | 0.403508 | 20.6847 | 0.0308208 | 20.6677 | 0.0350466 | 19.3245 | 0.548581 | 19.6299 | 0.633659 | 20.2106 | 0.0305616 | 20.2316 | 0.0370524 | 29.9061 | 0.841834 | 29.3319 | 1.00103 | 19.7347 | 0.0232507 | 19.7081 | 0.0275961 | 46.3568 | 0.992751 | 47.5084 | 1.20756 | False | False | False | False | False |
7 | 1000008 | 9.47809268427 | -45.1491167526 | 0.980775 | 21.6627 | 0.0380199 | 20.7934 | 0.0410575 | 7.85068 | 0.274921 | 17.4842 | 0.661194 | 21.281 | 0.02423 | 20.5735 | 0.0266205 | 11.1584 | 0.249026 | 21.409 | 0.524932 | 20.846 | 0.0240912 | 20.1923 | 0.0272273 | 16.6566 | 0.369601 | 30.414 | 0.762725 | 20.1053 | 0.0216434 | 19.6529 | 0.0292092 | 32.9506 | 0.656869 | 49.987 | 1.34483 | 19.8678 | 0.0230953 | 19.4351 | 0.0348331 | 41.0089 | 0.872351 | 61.0902 | 1.95999 | False | False | False | False | False |
8 | 1000009 | 9.088978899 | -45.1484041063 | 0.686791 | 23.7993 | 0.183412 | 23.7662 | 0.249672 | 1.09722 | 0.185358 | 1.1311 | 0.260113 | 23.164 | 0.110417 | 23.4362 | 0.224896 | 1.96961 | 0.200312 | 1.53298 | 0.317548 | 23.861 | 0.309503 | 23.6235 | 0.36742 | 1.03657 | 0.295498 | 1.29009 | 0.436588 | 22.2457 | 0.114278 | 22.304 | 0.173793 | 4.58908 | 0.483034 | 4.34905 | 0.696171 | 21.6323 | 0.102308 | 21.7746 | 0.14659 | 8.07391 | 0.760827 | 7.08219 | 0.95623 | False | False | False | False | False |
9 | 1000010 | 9.843479474 | -45.1482247366 | 0.686377 | 24.3482 | 0.670582 | 24.488 | 0.699839 | 0.661779 | 0.408747 | 0.581844 | 0.375055 | 24.0143 | 0.72212 | 23.913 | 0.402956 | 0.90004 | 0.598634 | 0.988097 | 0.36673 | 22.4418 | 0.125312 | 22.5634 | 0.156071 | 3.8308 | 0.442151 | 3.42483 | 0.492323 | 22.1416 | 0.30306 | 22.0714 | 0.246612 | 5.05059 | 1.40981 | 5.38794 | 1.22385 | 20.9066 | 0.123352 | 21.2709 | 0.143015 | 15.7525 | 1.78972 | 11.2624 | 1.48356 | False | False | False | False | False |