{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Make a full overview of number counts by band by field\n", "\n", "The reviewer requested that we produce number counts for each band on each field" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/Users/rs548/anaconda/envs/herschelhelp_internal/lib/python3.6/site-packages/matplotlib/__init__.py:1405: UserWarning: \n", "This call to matplotlib.use() has no effect because the backend has already\n", "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n", "or matplotlib.backends is imported for the first time.\n", "\n", " warnings.warn(_use_error_msg)\n" ] } ], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib as mpl\n", "mpl.use('pdf')\n", "import matplotlib.pyplot as plt\n", "import matplotlib.gridspec as gridspec\n", "\n", "import numpy as np\n", "#plt.rc('figure', figsize=(10, 6))\n", "from matplotlib_venn import venn3\n", "\n", "import herschelhelp \n", "from herschelhelp.utils import clean_table\n", "\n", "from astropy.table import Table, vstack\n", "\n", "import pyvo as vo\n", "\n", "from pymoc import MOC\n", "\n", "import time\n", "\n", "import yaml\n", "\n", "\n", "\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "#Then we establish the VO connection to our database\n", "service = vo.dal.TAPService(\"https://herschel-vos.phys.sussex.ac.uk/__system__/tap/run/tap\"\n", " )" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "fields = yaml.load(open('../../../dmu2/meta_main.yml', 'r'))['fields']" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "bands = [\n", " 'mmt_g', \n", " 'omegacam_g', \n", " 'suprime_g', \n", " 'megacam_g', \n", " 'wfc_g', \n", " 'gpc1_g', \n", " 'decam_g', \n", " '90prime_g', \n", " 'sdss_g',\n", " 'isaac_k', \n", " 'moircs_k', \n", " 'ukidss_k', \n", " 'newfirm_k', \n", " 'wircs_k', \n", " 'hawki_k',\n", " 'wircam_ks', \n", " 'vista_ks', \n", " 'moircs_ks', \n", " 'omega2000_ks', \n", " 'tifkam_ks'\n", "]\n", "mag_tables = {}\n", "\n", "\n", "for band in bands:\n", " mag_tables[band] = {}\n", " for f in fields:\n", " \n", " mag_tables[band].update({f['name']: None})" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mmt_g\n", "Querying VOX for mmt_g mags on HATLAS-NGP\n", "COMPLETED\n", "omegacam_g\n", "Querying VOX for omegacam_g mags on HATLAS-NGP\n", "COMPLETED\n", "suprime_g\n", "Querying VOX for suprime_g mags on HATLAS-NGP\n", "COMPLETED\n", "megacam_g\n", "Querying VOX for megacam_g mags on HATLAS-NGP\n", "COMPLETED\n", "wfc_g\n", "Querying VOX for wfc_g mags on HATLAS-NGP\n", "COMPLETED\n", "gpc1_g\n", "Querying VOX for gpc1_g mags on HATLAS-NGP\n", "COMPLETED\n", "Band gpc1_g field HATLAS-NGP done in 69 seconds with 3089817 objects\n", "decam_g\n", "Querying VOX for decam_g mags on HATLAS-NGP\n", "COMPLETED\n", "90prime_g\n", "Querying VOX for 90prime_g mags on HATLAS-NGP\n", "COMPLETED\n", "Band 90prime_g field HATLAS-NGP done in 35 seconds with 1099257 objects\n", "sdss_g\n", "Querying VOX for sdss_g mags on HATLAS-NGP\n", "COMPLETED\n", "isaac_k\n", "Querying VOX for isaac_k mags on HATLAS-NGP\n", "COMPLETED\n", "moircs_k\n", "Querying VOX for moircs_k mags on HATLAS-NGP\n", "COMPLETED\n", "ukidss_k\n", "Querying VOX for ukidss_k mags on HATLAS-NGP\n", "COMPLETED\n", "Band ukidss_k field HATLAS-NGP done in 43 seconds with 1778877 objects\n", "newfirm_k\n", "Querying VOX for newfirm_k mags on HATLAS-NGP\n", "COMPLETED\n", "wircs_k\n", "Querying VOX for wircs_k mags on HATLAS-NGP\n", "COMPLETED\n", "hawki_k\n", "Querying VOX for hawki_k mags on HATLAS-NGP\n", "COMPLETED\n", "wircam_ks\n", "Querying VOX for wircam_ks mags on HATLAS-NGP\n", "COMPLETED\n", "vista_ks\n", "Querying VOX for vista_ks mags on HATLAS-NGP\n", "COMPLETED\n", "moircs_ks\n", "Querying VOX for moircs_ks mags on HATLAS-NGP\n", "COMPLETED\n", "omega2000_ks\n", "Querying VOX for omega2000_ks mags on HATLAS-NGP\n", "COMPLETED\n", "tifkam_ks\n", "Querying VOX for tifkam_ks mags on HATLAS-NGP\n", "COMPLETED\n", "Total time: 395 seconds\n" ] } ], "source": [ "start = time.time()\n", "for band in bands:\n", " print(band)\n", " \n", " for f in fields:\n", " \n", " \n", " try:\n", " \n", " mag_tables[band].update({f['name'] : Table.read('./data/{}_{}.fits'.format(band, f['name']))})\n", " print(\"loaded {} from file ({} objects)\".format(band, len(mag_tables[band][f['name']])))\n", " continue\n", " except FileNotFoundError:\n", " print(\"Querying VOX for {} mags on {}\".format(band, f['name']))\n", " \n", " query = \"\"\"\n", " SELECT \n", " m_{}\n", " FROM herschelhelp.main\n", " WHERE herschelhelp.main.m_{} IS NOT NULL\n", " AND herschelhelp.main.field='{}'\"\"\".format(\n", " band, band, \n", " f['name'].replace('Lockman-SWIRE','Lockman SWIRE' ).replace('HATLAS-NGP','NGP' ))\n", "\n", " job = service.submit_job(query, maxrec=100000000)\n", " job.run()\n", " job_url = job.url\n", " job_result = vo.dal.tap.AsyncTAPJob(job_url)\n", " start_time = time.time()\n", " wait = 10.\n", " while job.phase == 'EXECUTING':\n", " #print('Job still running after {} seconds.'.format(round(time.time() - start_time)))\n", " time.sleep(wait) \n", " #wait *=2\n", "\n", " print(job.phase)\n", " result = job_result.fetch_result()\n", " mag_tables[band].update({ f['name']: result.table})\n", " if len(mag_tables[band][f['name']]) != 0:\n", " print(\"Band {} field {} done in {} seconds with {} objects\".format(\n", " band, \n", " f['name'], \n", " round(time.time() - start_time), \n", " len(mag_tables[band][f['name']])\n", " ))\n", " \n", "print(\"Total time: {} seconds\".format(round(time.time() - start)))" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'mmt_g': {'AKARI-NEP':