{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# XMM-13hr master catalogue\n", "\n", "This notebook presents the merge of the various pristine catalogues to produce HELP mater catalogue on XMM-13hr." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]\n", "This notebook was executed on: \n", "2018-05-01 13:42:17.181723\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))\n", "import datetime\n", "print(\"This notebook was executed on: \\n{}\".format(datetime.datetime.now()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "import os\n", "import time\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import Column, Table\n", "import numpy as np\n", "from pymoc import MOC\n", "\n", "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n", "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "TMP_DIR = os.environ.get('TMP_DIR', \"./data_tmp\")\n", "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n", "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n", "\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Reading the prepared pristine catalogues" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "uhs = Table.read(\"{}/UHS.fits\".format(TMP_DIR))\n", "legacy = Table.read(\"{}/LegacySurvey.fits\".format(TMP_DIR))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## II - Merging tables\n", "\n", "We merge UHS and Legacy Survey.\n", "\n", "At every step, we look at the distribution of the distances separating the sources from one catalogue to the other (within a maximum radius) to determine the best cross-matching radius." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### UHS" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue = uhs\n", "master_catalogue['uhs_ra'].name = 'ra'\n", "master_catalogue['uhs_dec'].name = 'dec'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Add Legacy Survey" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAF3CAYAAABnvQURAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUXOV55/HfU0vvrdbSrYWWGiEhCSGBEAixGYONzeaF\nOCFjjJdjhxmZ2Jk4OfbMJJmcZMZzMpM5mTgTG8cEO47tBG9jE2xsyYAxNostgRBCOyAESEItdWvr\nfauuZ/6o262mkdQl6e2+VaXv55w6devet249KqTm1+/73veauwsAAABnLhF3AQAAAKWCYAUAABAI\nwQoAACAQghUAAEAgBCsAAIBACFYAAACBEKwAAAACIVgBAAAEQrACAAAIhGAFAAAQSGqsBmZWIekJ\nSeVR+x+4+1+OanO9pB9JejXa9YC7f/5k562vr/e5c+eeRskAAAAT67nnnjvo7g1jtRszWEnqk/RO\nd+80s7Skp8xsjbuvHdXuSXd/b74Fzp07V+vXr8+3OQAAQGzM7PV82o0ZrDx3l+bO6GU6enDnZgAA\ngFHymmNlZkkz2yipRdKj7r7uOM2uNrNNZrbGzJYErRIAAKAI5BWs3H3Q3S+RNFvSSjNbOqrJBklN\n7n6xpC9JevB45zGzVWa23szWt7a2nkndAAAABeeUrgp096OSHpd086j97e7eGW2vlpQ2s/rjvP8+\nd1/h7isaGsac/wUAAFBUxgxWZtZgZpOj7UpJ75a0Y1SbmWZm0fbK6LyHwpcLAABQuPK5KnCWpG+a\nWVK5wPR9d/+Jmd0tSe5+r6TbJf2+mWUk9Ui6I5r0DgAAcNbI56rATZKWH2f/vSO275F0T9jSAAAA\nigsrrwMAAARCsAIAAAiEYAUAABAIwQoAACAQghUAAEAgBCsAAIBA8lnHCkXu2+t2n/T4nVc0TVAl\nAACUNnqsAAAAAiFYAQAABEKwAgAACIRgBQAAEAjBCgAAIBCCFQAAQCAEKwAAgEAIVgAAAIEQrAAA\nAAIhWAEAAARCsAIAAAiEYAUAABAIwQoAACAQghUAAEAgBCsAAIBACFYAAACBEKwAAAACIVgBAAAE\nQrACAAAIhGAFAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAAgRCsAAAAAiFYAQAABEKwAgAACIRg\nBQAAEAjBCgAAIBCCFQAAQCBjBiszqzCzZ8zsBTPbamb//ThtzMy+aGY7zWyTmV06PuUCAAAUrlQe\nbfokvdPdO80sLekpM1vj7mtHtLlF0oLocYWkr0TPAAAAZ40xe6w8pzN6mY4ePqrZbZK+FbVdK2my\nmc0KWyoAAEBhy2uOlZklzWyjpBZJj7r7ulFNGiXtGfF6b7QPAADgrJFXsHL3QXe/RNJsSSvNbOnp\nfJiZrTKz9Wa2vrW19XROAQAAULBO6apAdz8q6XFJN4869IakOSNez472jX7/fe6+wt1XNDQ0nGqt\nAAAABS2fqwIbzGxytF0p6d2Sdoxq9mNJH4uuDrxSUpu7NwevFgAAoIDlc1XgLEnfNLOkckHs++7+\nEzO7W5Lc/V5JqyXdKmmnpG5JnxinegEAAArWmMHK3TdJWn6c/feO2HZJnw5bGgAAQHFh5XUAAIBA\nCFYAAACBEKwAAAACIVgBAAAEQrACAAAIhGAFAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAAgRCs\nAAAAAiFYAQAABEKwAgAACIRgBQAAEAjBCgAAIBCCFQAAQCAEKwAAgEAIVgAAAIEQrAAAAAIhWAEA\nAARCsAIAAAiEYAUAABAIwQoAACAQghUAAEAgBCsAAIBACFYAAACBEKwAAAACIVgBAAAEQrACAAAI\nhGAFAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAAgRCsAAAAAiFYAQAABEKwAgAACGTMYGVmc8zs\ncTPbZmZbzewzx2lzvZm1mdnG6PEX41MuAABA4Url0SYj6bPuvsHMaiU9Z2aPuvu2Ue2edPf3hi8R\nAACgOIzZY+Xuze6+IdrukLRdUuN4FwYAAFBsTmmOlZnNlbRc0rrjHL7azDaZ2RozWxKgNgAAgKKS\nz1CgJMnMaiT9UNIfuXv7qMMbJDW5e6eZ3SrpQUkLjnOOVZJWSVJTU9NpFw0AAFCI8uqxMrO0cqHq\nfnd/YPRxd293985oe7WktJnVH6fdfe6+wt1XNDQ0nGHpAAAAhSWfqwJN0j9J2u7uXzhBm5lRO5nZ\nyui8h0IWCgAAUOjyGQq8RtJHJW02s43Rvj+T1CRJ7n6vpNsl/b6ZZST1SLrD3X0c6gUAAChYYwYr\nd39Kko3R5h5J94QqCgAAoBix8joAAEAgBCsAAIBACFYAAACBEKwAAAACIVgBAAAEQrACAAAIhGAF\nAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAAgRCsAAAAAiFYAQAABEKwAgAACIRgBQAAEAjBCgAA\nIBCCFQAAQCAEKwAAgEAIVgAAAIEQrAAAAAIhWAEAAARCsAIAAAiEYAUAABAIwQoAACAQghUAAEAg\nBCsAAIBACFYAAACBEKwAAAACIVgBAAAEQrACAAAIhGAFAAAQCMEKAAAgkFTcBeDMfXvd7rhLAAAA\noscKAAAgGIIVAABAIAQrAACAQAhWAAAAgYwZrMxsjpk9bmbbzGyrmX3mOG3MzL5oZjvNbJOZXTo+\n5QIAABSufK4KzEj6rLtvMLNaSc+Z2aPuvm1Em1skLYgeV0j6SvQMAABw1hizx8rdm919Q7TdIWm7\npMZRzW6T9C3PWStpspnNCl4tAABAATulOVZmNlfScknrRh1qlLRnxOu9emv4AgAAKGl5Byszq5H0\nQ0l/5O7tp/NhZrbKzNab2frW1tbTOQUAAEDByitYmVlauVB1v7s/cJwmb0iaM+L17Gjfm7j7fe6+\nwt1XNDQ0nE69AAAABSufqwJN0j9J2u7uXzhBsx9L+lh0deCVktrcvTlgnQAAAAUvn6sCr5H0UUmb\nzWxjtO/PJDVJkrvfK2m1pFsl7ZTULekT4UsFAAAobGMGK3d/SpKN0cYlfTpUUQAAAMWIldcBAAAC\nIVgBAAAEQrACAAAIhGAFAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAAgRCsAAAAAiFYAQAABEKw\nAgAACIRgBQAAEAjBCgAAIBCCFQAAQCAEKwAAgEAIVgAAAIEQrAAAAAIhWAEAAARCsAIAAAiEYAUA\nABAIwQoAACAQghUAAEAgBCsAAIBAUnEXgPh9e93ukx6/84qmCaoEAIDiRo8VAABAIAQrAACAQAhW\nAAAAgRCsAAAAAiFYAQAABEKwAgAACIRgBQAAEAjBCgAAIBCCFQAAQCAEKwAAgEAIVgAAAIEQrAAA\nAAIhWAEAAAQyZrAys6+bWYuZbTnB8evNrM3MNkaPvwhfJgAAQOFL5dHmG5LukfStk7R50t3fG6Qi\nAACAIjVmj5W7PyHp8ATUAgAAUNRCzbG62sw2mdkaM1sS6JwAAABFJZ+hwLFskNTk7p1mdqukByUt\nOF5DM1slaZUkNTU1BfhoAACAwnHGPVbu3u7undH2aklpM6s/Qdv73H2Fu69oaGg4048GAAAoKGcc\nrMxspplZtL0yOuehMz0vAABAsRlzKNDMviPpekn1ZrZX0l9KSkuSu98r6XZJv29mGUk9ku5wdx+3\nigEAAArUmMHK3T80xvF7lFuOAQAA4KzGyusAAACBEKwAAAACIVgBAAAEQrACAAAIhGAFAAAQCMEK\nAAAgEIIVAABAIAQrAACAQAhWAAAAgRCsAAAAAiFYAQAABEKwAgAACIRgBQAAEAjBCgAAIBCCFQAA\nQCAEKwAAgEAIVgAAAIEQrAAAAAIhWAEAAARCsAIAAAgkFXcBmFhHuvq1fX+7djR3qL13QB++4lw1\n1JbHXRYAACWBYHUW6Ogd0Npdh7W9uV3723slSQ215erqy+hrT+3Sf3jbPNUTrgAAOGMEqxKXdde/\nrn1de4/06Nxp1bpl6UwtnjVJ9TXlOtDeq689uUtfe2qX/v2181RfQ7gCAOBMMMeqxK3bdUh7jvTo\nd1fM0aq3z9O1CxqGA9SMSRW669p5Gsy6vvbkLh3s7Iu5WgAAihvBqoQd7e7Xw9sOaOGMGi2bXXfc\nNjNHhatDhCsAAE4bwapEubseemGf3F3vX9YoMzth26Fwlcm6vvrkLnX1ZSawUgAASgfBqkRt3deu\n7fs79K7FMzS1umzM9jMnVegTV5+njt6Mfv3KwQmoEACA0kOwKkG9A4N6aNM+zaqr0NXz6/N+X+OU\nSl14ziT9Ztch9Q4MjmOFAACUJoJVCXp463519mb0geWNSiZOPAR4PNctbFDvQFbPvHp4nKoDAKB0\nEaxKzOuHurTu1cO6ev40zZ5Sdcrvnz2lSuc31OjpnQc1MJgdhwoBAChdBKsS8/DW/aqrTOtdF844\n7XNct6hBHX0Zbdh9JGBlAACUPoJVCWnt6NNrh7p11bxpKk8lT/s88+qrNWdKpZ54qVWDWQ9YIQAA\npY1gVUKee/2wEiYtb5p8RucxM123cLqOdA9o8xtHA1UHAEDpI1iViMGsa8Puo1o0c5JqK9JnfL4L\nZtVqem25fvVSq7L0WgEAkBeCVYl46UCHOvsyWnHulCDnS5jpuoUNOtDep1/saAlyTgAASh3BqkSs\nf/2IastTWjijNtg5L549WVOq0vryL3fKnV4rAADGQrAqAR29A3pxf7uWN00+5XWrTiaZMF27oEHP\n7z6q9a9zhSAAAGMZM1iZ2dfNrMXMtpzguJnZF81sp5ltMrNLw5eJk3l+91FlXbrs3KnBz31p0xRV\nlSX1wIY3gp8bAIBSk0+P1Tck3XyS47dIWhA9Vkn6ypmXhXy5u9a/fkTnTq1SQ2158POXpRK68cIZ\nWr25Wf0ZFgwFAOBkxgxW7v6EpJPd3+Q2Sd/ynLWSJpvZrFAF4uSee/2IDnb2acXcMJPWj+e2SxrV\n1jOgJ15qHbfPAACgFISYY9Uoac+I13ujfZgA31+/R2WphJY21o3bZ7xtQb2mVKX1oxf2jdtnAABQ\nCiZ08rqZrTKz9Wa2vrWV3o8z1dmX0U82NevixrozWml9LOlkQu+5eJYe3bZfXX2ZcfscAACKXYhg\n9YakOSNez472vYW73+fuK9x9RUNDQ4CPPrv9dNM+dfcP6rJAa1edzG2XNKp3IKufbz8w7p8FAECx\nChGsfizpY9HVgVdKanP35gDnxRh+8NxezW+oVtPUqnH/rMuapuicugr9aCPDgQAAnEg+yy18R9Jv\nJC0ys71mdpeZ3W1md0dNVkvaJWmnpK9K+tS4VYthLe29Wv/6Eb1/WaPMwq1ddSKJhOl9l5yjJ15q\n1eGu/nH/PAAAilFqrAbu/qExjrukTwerCHn5+fYWuUs3LZ2hDa9PzI2Sb1vWqH/81S6t3tysj1x5\n7oR8JgAAxYSV14vUI9v2q2lqlRYFvIXNWBbPqtWC6TX6McOBAAAcF8GqCHX0DujXOw/ppiUzJmQY\ncIiZ6bZLztEzrx3WG0d7JuxzAQAoFgSrIvTLF1vVP5jVjUtmTvhnv39Zbomyh1jTCgCAtyBYFaFH\nth3QtOoyXdo0/sssjNY0rUrLmyZzdSAAAMdBsCoyfZlBPb6jRe9aPEPJxMQNA45027JztL25XS8d\n6Ijl8wEAKFQEqyKzdtdhdfZldOOSGbHV8J6Lz1HCpJ9sYrkyAABGIlgVmUe27ldVWVLXnF8fWw0N\nteVaMXeqHtm6P7YaAAAoRASrIpLNuh7ddkDXL2pQRXr87g2Yj5uXzNSO/R167WBXrHUAAFBICFZF\nZOPeo2rp6NONF0781YCjDQ1FPkyvFQAAwwhWReSRrQeUSpjesWh63KVo9pQqLW2cRLACAGAEglUR\neWTbfl01f5rqqtJxlyIpNxy4YfdRHWjvjbsUAAAKAsGqSOxs6dSu1i7deGF8VwOOdlO0QOkj2w7E\nXAkAAIWBYFUkhobc3lVAwer86TWaV1+th7cwHAgAgESwKhqPbDugZbPrNKuuMu5ShpmZblo6U2t3\nHVJb90Dc5QAAEDuCVRFoae/VC3uO6t0F1Fs15KYlM5XJuh7bwXAgAAAEqyLwix0tkqQbFhdesLq4\nsU4zJ1XoZwwHAgBAsCoGP9/eosbJlbpgZm3cpbxFImG6ackMPfFyq3r6B+MuBwCAWBGsClzvwKCe\n2tmqGxZPl1k8N10ey01LZqp3IKtfvdQadykAAMSKYFXgfv3KQfUOZAtyGHDIyvOmanJVmsVCAQBn\nPYJVgfv59hZVlyV15bypcZdyQqlkQu9aPEOPbT+ggcFs3OUAABAbglUBc3f9YnuLrl3QoPJUvDdd\nHstNS2aqvTejtbsOxV0KAACxIVgVsK372rW/vVc3LI7/3oBjuXZBvarKklrD1YEAgLMYwaqAPba9\nRWbSOy4o/GBVkU7qHRdM1yNb92sw63GXAwBALAhWBeyxHQe0fM5k1deUx11KXm5dOksHO/v17GuH\n4y4FAIBYEKwK1IH2Xm3a21bQVwOOdv2iBlWkE1qzuTnuUgAAiAXBqkANrbb+riIKVtXlKV2/cLrW\nbNmvLMOBAICzEMGqQD22/YAaJ1dq4YyauEs5JbdcNFMtHX3asPtI3KUAADDhCFYFKLfa+kG9q4BX\nWz+Rd14wXWWphFZv5upAAMDZh2BVgJ7eWfirrZ9IbUVab1/QoDVbmhkOBACcdQhWBWhotfUrCni1\n9ZO59aKZam7r1Qt7j8ZdCgAAE4pgVWDcXb/YcUBvX1j4q62fyA2LZyidNBYLBQCcdQhWBeaFvW06\n0N5XVFcDjlZXmdbbzq/X6s3Ncmc4EABw9iBYFZg1W5qVSlhRBytJuuWiWdp7pEdb3miPuxQAACYM\nwaqAuLvWbN6vq8+vV11VOu5yzsiNF85QKmFavYXFQgEAZ49U3AXgmG3N7dp9uFufun5+3KW8ybfX\n7T7hsTuvaDru/slVZbpq/jSt2dys/3zToqJbNgIAgNNBj1UB+dmW/UqY9O4Li3sYcMgtS2fptUPd\n2rG/I+5SAACYEASrArJ6c7OunDdN04rkpstjuXHJDCUs9+cCAOBskFewMrObzexFM9tpZn9ynOPX\nm1mbmW2MHn8RvtTS9vKBDr3S2qVbls6Mu5Rg6mvKdfX8ej248Q0WCwUAnBXGDFZmlpT0ZUm3SLpQ\n0ofM7MLjNH3S3S+JHp8PXGfJW715v8ykm5aUTrCSpN+5rFF7Dvfo2dcOx10KAADjLp8eq5WSdrr7\nLnfvl/RdSbeNb1lnnzVbmrXi3CmaPqki7lKCumnJTNWUp/SD5/bGXQoAAOMun2DVKGnPiNd7o32j\nXW1mm8xsjZktCVLdWeLVg13asb9DNy+dFXcpwVWVpfSei2Zp9eZmdfdn4i4HAIBxFWry+gZJTe5+\nsaQvSXrweI3MbJWZrTez9a2trYE+uvitidZ6urmE5leN9DuXzVZX/6B+xi1uAAAlLp9g9YakOSNe\nz472DXP3dnfvjLZXS0qbWf3oE7n7fe6+wt1XNDQ0nEHZpeVnW/Zr2ZzJapxcGXcp4+LyuVPUNLWK\n4UAAQMnLJ1g9K2mBmZ1nZmWS7pD045ENzGymRStAmtnK6LyHQhdbivYc7tamvW0ldTXgaGam2y+b\nrV+/ckh7j3THXQ4AAONmzGDl7hlJfyDpYUnbJX3f3bea2d1mdnfU7HZJW8zsBUlflHSHc/fdvDy8\nNTc8VsrBSpJ++9LctLwHNrwxRksAAIpXXre0iYb3Vo/ad++I7Xsk3RO2tLPD6s3NunDWJJ07rTru\nUsbV7ClVunr+NP3gub36j+88n1vcAABKEiuvx6i5rUcbdh/VrReVdm/VkNsvm63dh7v17GtH4i4F\nAIBxQbCK0dCw2HsvPifmSibGzUtnqrosqR8yiR0AUKIIVjHJZl3fe3aPrpo3TXPrS3sYcEhVWUq3\nXjRLP2VNKwBAiSJYxeQ3uw5p9+Fu3bFyztiNS8jtl81WZ19meNI+AAClhGAVk+8+u0d1lemSuzfg\nWC6fO1VNU6v0vWf3jN0YAIAiQ7CKwZGufj28Zb8+sLxRFelk3OVMqETC9OErmrR212Ft3tsWdzkA\nAARFsIrBvz3/hvoHs/rg5WfXMOCQO69oUm1FSvf+6pW4SwEAICiC1QRzz01aXzZnshbPmhR3ObGo\nrUjrI1eeqzVbmvXqwa64ywEAIBiC1QR7fs9RvXigQ3ecpb1VQz5xzVylkgnd98SuuEsBACAYgtUE\n+94ze1RVltT7lp0da1edyPTaCv3OpbP1ww171dLRG3c5AAAEQbCaQJ19GT20aZ/ee/Es1ZTndTeh\nkrbq7fM0MJjVPz/9WtylAAAQBMFqAv3khX3q7h/UHSub4i6lIJxXX61bls7Uv659XR29A3GXAwDA\nGSNYTaDvPrtHC2fUaPmcyXGXUjDuvm6+Onoz+va63XGXAgDAGSNYTZDtze3auOeoPnh5k8ws7nIK\nxsWzJ+ua86fpn556VX2ZwbjLAQDgjBCsJsg9v9ip6rKkfnt5Y9ylFJy7r5uvlo4+Pfj8G3GXAgDA\nGSFYTYCt+9r0083Nuutt52lKdVnc5RSct51fryXnTNK9v9ql/kw27nIAADhtBKsJ8HePvqRJFSnd\nde28uEspSGamz920SK8e7NJ9T7AaOwCgeBGsxtnzu4/o59tbtOrt81RXmY67nIL1jkXTdetFM/Wl\nX+zU64dYjR0AUJwIVuPsC4++pKnVZfr4NefFXUrB+8v3LVE6mdCfP7hF7h53OQAAnDJWqRxH63Yd\n0pMvH9R/vXVxyS4IOtYyCXdekf+aXTMmVehzNy7Uf3tomx7a1Kz3n+Wr0wMAig89VuPE3fW3j7yk\n6bXl+siV58ZdTtH46FVzdfHsOn3+oW1q62HRUABAcSFYjZOndh7UM68d1h+883xVliXjLqdoJBOm\nv/qti3S4q09/8/COuMsBAOCUEKzGgbvr/zzykhonV+qDl8+Ju5yic9HsOn3sqrm6f91uPb/7SNzl\nAACQN4LVOPjZlv16Yc9R/eEN56s8RW/V6fjsjQs1vbZcf/rAZvUOsCI7AKA4EKwCa27r0Z/922Yt\nnjVJv33p7LjLKVq1FWn9zw9cpBcPdOgz331eg1muEgQAFD6CVUCZwaw+852N6stk9eU7lyud5Os9\nEzcsnqE/f8+FenjrAX3+oa0swQAAKHiluQZATL742Mt65rXD+rsPLtO8hpq4yykJd73tPDUf7dHX\nnnpV50yu1Cevmx93SQAAnBDBKpBf7zyoLz2+U7dfNlsfWM4QYEh/dutiNbf36n+t2aGZdRW67RJu\nZA0AKEwEqwAOdvbpM9/bqHn11fr8bUviLqfkJBKmv/3dZWrt6NPn/t8Laqgt19Xz6+MuCwCAt2AS\n0BnKZl1//L2Nau8Z0D13XqqqMrLqeKhIJ/XVj67Q3GnV+uS3ntPjO1riLgkAgLcgBZwBd9f//tkO\nPfnyQf3VB5Zq8axJcZdU0uqq0vrm763UXd9cr09841l98rp5+tyNi7hIAMBZ7WS3FjuV24ohDILV\naerLDOq//GCTHty4Tx++okl3ruQv70Q4Z3Kl/u1TV+t//GSb/vFXu7T+tSP60oeW65zJlXGXBqAE\nZLOurv6MuvsHo0dGPdF2z8Cg+jJZ9Wey6ssMqj/afu71I8p67pftrLvcJTPJzGQmJcyUMFMqYSpL\nJVSWTCidTKgslVBFOqEPrWzS5Kq0aspTMrO31DTWPVlRWAhWp6Gte0Cr/mW91r16WP/ppkX61PXz\nj/uPAeOjIp3UX33gIl05b5r+9IHNuvWLT+pvf3eZblg8I+7SAEwwd1dfJqvu/kF19WXU1Z9RV1+0\n3ZdRZ/To6suoY2hf77H9nX0ZNR/tjcJSVv2D2TOuKWGSu5TvAjH/8MtXJElJM1WVJVVTkVJdZfpN\nj8lVZaqvKTth+ELhIFidoj2Hu/Xxf35Gew736O/vuIQr1MYw1m9aZ9JN/b5l52hpY50+ff8G3fXN\n9brhgun61Dvm67Jzp572OQGEMZh19QwMqqd/UL0Dg+rLDKqnP6veTO5170A2eh5UbyarvoFR+zO5\n7Z6BQfX251739A+qZyCrnqhHqad/UN0Dg3kvIDzUY1SRTqo8lVB5KvfcOKVy+HVZKqHyVOItPUtl\nyYTSqYTSCVMqmVAqYUolTcmEDfdIDfVODXF3uaSsu7JZKZPN9XANDLr6B3PbvQO5XrGhHrKuvow6\nejNq6xnQ64e61TPqzhPlqYSm1ZSpvqZc9TXlmjGpQjMmlWtadbmSCQJXISBYnYJnXj2sT93/nAYG\nXf9y10pdMW9a3CWd9c6rr9YDn7paX31il77+9Kv6na/8RivPm6pPXT9f1y1s4Dc7IE8Dg9nhnpyO\n3ow6egfe1KvT1ZfR2l2H1Tc0HDaYPdbLk8mqIp0YDjrd/blhstNhktLJhFJJU3oo2CRNsyZXqqos\npanVCVWVpVRVllRlWVLVZanoOanq8tSxR/S6Jno8tGmfUomJnY9pZjJFYSshlSmhqrJTO0d/Jqv2\nngEd7u7Xwc4+HerMPe890qPNe9uGe8VSCVNDbblmTqrQrMmVmlVXoXPqmCIRB4trNesVK1b4+vXr\nY/nsU7V1X5u+8MhLemxHi+ZMrdQ/f3ylzp9eOAuAFvP4e8iJld39GX33mT366pO71NzWqwtnTdJv\nX9qo6xc1aH5DDSELJak/kx0e8uoYHuIaiMJR9DoKSh3D20PtB4bf15dnEBrq0SmPenHKhnp3Usno\ntUVhKBeK0qlcMEpF+1JJUzqRGBWebPh10ox/q3kaGMyqpaNPB9p6daC9V/ujR0dvZrjN7CmVWnLO\nJC05p05LzpmkpY11ml5bznd8GszsOXdfMWY7gtWJ7Wzp0N89+rJ+urlZkypS+uR18/Xxq+equryw\nOvoIVm/Wn8nqwY1v6OtPvaod+zskSY2TK3XdogZdv7BBy5umqL6mjB8siIW7q3cgq86+jLr7h3qD\nBofD0bHnQXX1HwtGXX0ZvdLapf7MsaGzvkxWmTyGwRIm1ZSnlDA7NgyWzg2JVaSSKk/nhsEq0om3\nvC4bMWR0cxZWAAAMLUlEQVRWlkq8aagLhamjd0DNbb1qPtqjdCqhrfva9erBruHj9TVlWnJOnRbP\nmqTFs2p1wcxJmtdQzRXWYwgarMzsZkl/Lykp6Wvu/tejjlt0/FZJ3ZI+7u4bTnbOQg1W+9t69auX\nWvTY9hb9fPsBVaaTuutt5+mua+eprjIdd3nHVczB6kyNFcz2HunWr15q1S9fbNWvdx5UV39uvsKU\nqrQWTK/Vghk1WjC9RnOmVqm+plwNteWaVlOm8lRyIspHgchmc3Nehoa1hoa5hq7+6stk1TeQ2x49\nB2hontDQMFhvdPVYV/+geqKJ1D0DueDU3Z8LS/n+PptOmspSSVUM9RJFoagifWwu0FBQqhiaHxSF\no4p0cng7naQX6Gw19DOysy+j7c3t2vpGm7bsa9fWfe3a2dKhgcHcX8ayZELzp9do4YwazW/IPc6f\nXqO59VX8PIzkG6zG7Hoxs6SkL0t6t6S9kp41sx+7+7YRzW6RtCB6XCHpK9FzQctmXfvaerSrtUu/\nfuWQfvliy3APx4xJ5foP187TJ6+br6nVpzgojoIxe0qVPnzFufrwFecOXxa9vbldL7d06uUDHXro\nhX1qH9FtPmRSRUpTqstUXZZSdXlSVdFzRTp5bIgjmVA6lRvWSFhuhfjcJNZjl1mbcs9Sbu7Ise1j\n+yUN/09vqI0pd76RczQS0cRYMx2bMJvIDZ0kE7ljyURuX2p4/wkeb3rvse1EIndl0vBk3Oi8J6p9\nKCC4fPgqqNxEXVfWj21nsq7BNz1nlRl0DQzmelwGBo+9Hhh+zu3rj7aHjvVlou3oOReCfESbrPoH\nffj48Ouh7czI59z58538fCJmUlU6N+dnMOtvmvBclkqotiKladVlxyZFD/UARceHeojKh4fZciGJ\nycgIpaY8pcvnTtXlc49d3DMwmNWu1i5tb27X9v3t2tHcofWvHdGPNu4bbpMwaVZdpeZMrdScKVWa\nPaVKc6ZWalZdpaZPyv0yWsuVim+Sz5jWSkk73X2XJJnZdyXdJmlksLpN0rc81/211swmm9ksd28O\nXnGeevoH9dKBDrX1DOhoz4DauvvV1jOgI90D2n24W68d7NLrh7uHJ1imEqYVc6foT265QNctbNAF\nM2v5i1IETuWqw7JUQlfNn6ar5h+76MDd1drRp31tvTrY0afWzr7h5xf2HFV/JqtDXf1qbuvN/Q89\nk9Wg58LByEc8A+pnr5EBMTUqNA69TiRGHsv17FSXJZVK5AJLMmlK2aj3DV3tNbQvmbsKLJk0pRK5\nuUBD+4bCdSqZa8/PCxSbdDKhRTNrtWhmrX5Lx65w7+7PaFdrl15p7dQrrV3afahLe4/06ImXW3Wg\nve8t56lIJ3K9/dXl0dIQx5aJmFSRVmVZUlXRozK68GDkFZflqdz2sX+HI/4d25t/YS0G+QSrRkl7\nRrzeq7f2Rh2vTaOk2ILVK62duu3LT79lf1VZUo2TKzW3vlrvuGC65k6r1tz6Kl3UWKfaisIc6sPp\nO51h0mk15ZpWU64LZp7aSvojL60e7skZkbiG49eo9W3e0iY67iPOeex8PrwQoSvX65odtX+4t0i5\nS7yHFi0cOj7oo9qPeM4Of96xHigf8eeJyteyOXXK9WPpTT1yQz1qz+8++qaeu6FeMBvRszfc03a8\n3rXRvW8JGw5ExfQDFihGVWUpLW2s09LGurcc6x0Y1BtHe3SgrVctHX1q7ehTS0du+3BXv4509+u1\nQ11q6xlQe8+AzrAz+C2Gf65Iw6MCMunOlU36b+8vjHv1TugsbDNbJWlV9LLTzF6cyM8fsj2ODx1f\n9ZIOxl1EEeJ7O318d6eH7+308d2dhg/HXcAE+e/RY5ydm0+jfILVG5LmjHg9O9p3qm3k7vdJui+f\nwpA/M1ufz4Q6vBnf2+njuzs9fG+nj+8OxSKfayuflbTAzM4zszJJd0j68ag2P5b0Mcu5UlJbnPOr\nAAAA4jBmj5W7Z8zsDyQ9rNxyC193961mdnd0/F5Jq5VbamGncsstfGL8SgYAAChMec2xcvfVyoWn\nkfvuHbHtkj4dtjScAoZXTw/f2+njuzs9fG+nj+8ORSG2ldcBAABKDevXAwAABEKwKmJmdrOZvWhm\nO83sT+Kup1iY2dfNrMXMtsRdS7Exszlm9riZbTOzrWb2mbhrKgZmVmFmz5jZC9H3NgFXhpcOM0ua\n2fNm9pO4awHGQrAqUiNuNXSLpAslfcjMLoy3qqLxDUk3x11EkcpI+qy7XyjpSkmf5u9dXvokvdPd\nl0m6RNLN0RXUyM9nVJJLEKIUEayK1/Cthty9X9LQrYYwBnd/QtLhuOsoRu7ePHSDdXfvUO5/do0n\nfxc8pzN6mY4eTHDNg5nNlvQeSV+LuxYgHwSr4nWi2wgBE8LM5kpaLmldvJUUh2g4a6OkFkmPujvf\nW37+r6T/LCkbdyFAPghWAE6ZmdVI+qGkP3L39rjrKQbuPujulyh3Z4qVZrY07poKnZm9V1KLuz8X\ndy1AvghWxSuv2wgBoZlZWrlQdb+7PxB3PcXG3Y9KelzM88vHNZLeb2avKTfd4Z1m9q/xlgScHMGq\neOVzqyEgKDMzSf8kabu7fyHueoqFmTWY2eRou1LSuyXtiLeqwufuf+rus919rnI/437h7h+JuSzg\npAhWRcrdM5KGbjW0XdL33X1rvFUVBzP7jqTfSFpkZnvN7K64ayoi10j6qHI9Bxujx61xF1UEZkl6\n3Mw2KfdL0aPuztIBQAli5XUAAIBA6LECAAAIhGAFAAAQCMEKAAAgEIIVAABAIAQrAACAQAhWAAAA\ngRCsAIzJzAajNau2mtkLZvZZM0tEx1aY2RdP8t65ZnbnxFX7ls/uie7RVxDM7INmttPMWMcKKEEE\nKwD56HH3S9x9iXKrht8i6S8lyd3Xu/sfnuS9cyXFEqwir0T36MubmSXHqxh3/56kfz9e5wcQL4IV\ngFPi7i2SVkn6A8u5fqj3xcyuG7Ei+/NmVivpryVdG+3746gX6Ukz2xA9ro7ee72Z/dLMfmBmO8zs\n/ugWOjKzy83s11Fv2TNmVmtmSTP7GzN71sw2mdkn86nfzB40s+ei3rdVI/Z3mtnfmtkLkq46wWcu\nibY3Rp+5IHrvR0bs/8ehYGZmN0d/xhfM7LGA/xkAFKhU3AUAKD7uvisKD9NHHfqcpE+7+9NmViOp\nV9KfSPqcu79XksysStK73b03CibfkbQiev9ySUsk7ZP0tKRrzOwZSd+T9EF3f9bMJknqkXSXpDZ3\nv9zMyiU9bWaPuPurY5T/e+5+OLpn37Nm9kN3PySpWtI6d/9sdP/NHcf5zLsl/b273x+1SZrZYkkf\nlHSNuw+Y2T9I+rCZrZH0VUlvd/dXzWzqKX/RAIoOwQpASE9L+oKZ3S/pAXffG3U6jZSWdI+ZXSJp\nUNLCEceecfe9khTNi5orqU1Ss7s/K0nu3h4dv1HSxWZ2e/TeOkkLJI0VrP7QzD4Qbc+J3nMoquWH\n0f5FJ/jM30j6r2Y2O/rzvWxmN0i6TLmQJkmVklokXSnpiaGg5+6Hx6gLQAkgWAE4ZWY2T7kg0iJp\n8dB+d/9rM/uppFuV60G66Thv/2NJByQtU246Qu+IY30jtgd18p9RJuk/uvvDp1D39ZLeJekqd+82\ns19KqogO97r74Mne7+7fNrN1kt4jaXU0/GiSvunufzrqs96Xb10ASgdzrACcEjNrkHSvpHt81F3c\nzWy+u2929/8t6VlJF0jqkFQ7olmdcr1BWUkflTTWRPEXJc0ys8ujz6g1s5SkhyX9vpmlo/0Lzax6\njHPVSToShaoLlOtVyvszo0C5y92/KOlHki6W9Jik281setR2qpmdK2mtpLeb2XlD+8eoDUAJoMcK\nQD4qo6G5tKSMpH+R9IXjtPsjM3uHpKykrZLWRNuD0aTwb0j6B0k/NLOPSfqZpK6TfbC795vZByV9\nKZoX1aNcr9PXlBsq3BBNcm+V9Ftj/Dl+JuluM9uuXHhae4qf+e8kfdTMBiTtl/Q/o/lafy7pEcst\nQTGg3DyztdHk+Aei/S3KXVEJoITZqF84AaBkmNlcST9x96Uxl/Im0ZDk8IR+AKWDoUAApWxQUp0V\n2AKhyvXaHYm7FgDh0WMFAAAQCD1WAAAAgRCsAAAAAiFYAQAABEKwAgAACIRgBQAAEMj/B5NYRvOg\n1SnYAAAAAElFTkSuQmCC\n", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "nb_merge_dist_plot(\n", " SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n", " SkyCoord(legacy['legacy_ra'], legacy['legacy_dec'])\n", ")" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Given the graph above, we use 0.8 arc-second radius\n", "master_catalogue = merge_catalogues(master_catalogue, legacy, \"legacy_ra\", \"legacy_dec\", radius=0.8*u.arcsec)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Cleaning\n", "\n", "When we merge the catalogues, astropy masks the non-existent values (e.g. when a row comes only from a catalogue and has no counterparts in the other, the columns from the latest are masked for that row). We indicate to use NaN for masked values for floats columns, False for flag columns and -1 for ID columns." ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "collapsed": true }, "outputs": [], "source": [ "for col in master_catalogue.colnames:\n", " if \"m_\" in col or \"merr_\" in col or \"f_\" in col or \"ferr_\" in col or \"stellarity\" in col:\n", " master_catalogue[col].fill_value = np.nan\n", " elif \"flag\" in col:\n", " master_catalogue[col].fill_value = 0\n", " elif \"id\" in col:\n", " master_catalogue[col].fill_value = -1\n", " \n", "master_catalogue = master_catalogue.filled()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/html": [ "Table length=10\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
idxuhs_idradecuhs_stellaritym_ukidss_jmerr_ukidss_jm_ap_ukidss_jmerr_ap_ukidss_jf_ukidss_jferr_ukidss_jflag_ukidss_jf_ap_ukidss_jferr_ap_ukidss_juhs_flag_cleaneduhs_flag_gaiaflag_mergedlegacy_idf_90prime_gferr_90prime_gf_ap_90prime_gferr_ap_90prime_gf_90prime_rferr_90prime_rf_ap_90prime_rferr_ap_90prime_rf_mosaic_zferr_mosaic_zf_ap_mosaic_zferr_ap_mosaic_zlegacy_stellaritym_90prime_gmerr_90prime_gflag_90prime_gm_ap_90prime_gmerr_ap_90prime_gm_90prime_rmerr_90prime_rflag_90prime_rm_ap_90prime_rmerr_ap_90prime_rm_mosaic_zmerr_mosaic_zflag_mosaic_zm_ap_mosaic_zmerr_ap_mosaic_zlegacy_flag_cleanedlegacy_flag_gaia
degdeguJyuJyuJyuJyuJyuJy
0459962597806203.63425709637.61280916480.9938659.524590.00022558410.7740.000403111562555.0116.882False177993.066.085False3False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
1459759173072203.56130245837.85883425130.99386510.47680.0003552610.87240.0004206234041.076.5798False162576.062.9799False3False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
2459794088554203.89148663537.70292115780.99386510.1020.00029417811.1850.000474269330527.089.5556False121901.053.2487False2False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
3459794088563203.89133445137.70229471040.99386511.25890.00050247911.46340.000540217113881.052.7039False94324.546.9319False0False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
4459962597575203.52307239637.63676123030.9938659.191470.00019263811.48390.000556324764563.0135.653False92564.547.4294False3False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
5460022925494203.780812938.30641085580.99386511.94730.00072998911.6950.00061274360405.240.6131False76208.643.0089False2False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
6460022925486203.78122879438.30610503760.99386510.79170.00044559411.88570.000670084175108.071.8656False63930.839.4562False2False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
7459804401241203.52058148938.40846317560.99386512.28180.00085759112.28770.00079378144388.735.0614False44149.732.2778False3False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
8459794088709203.71405576337.77198894880.99386511.72020.00069357512.33840.00082577774460.147.5656False42135.032.0466False2False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
9459804401268203.60620766838.41925436360.99386512.3860.00090791812.39140.00083218240327.933.7232False40128.330.757False3False-1nannannannannannannannannannannannannannannanFalsenannannannanFalsenannannannanFalsenannanFalse0
\n", "\n" ], "text/plain": [ "" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "master_catalogue[:10].show_in_notebook()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## III - Merging flags and stellarity\n", "\n", "Each pristine catalogue contains a flag indicating if the source was associated to a another nearby source that was removed during the cleaning process. We merge these flags in a single one." ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "collapsed": true }, "outputs": [], "source": [ "flag_cleaned_columns = [column for column in master_catalogue.colnames\n", " if 'flag_cleaned' in column]\n", "\n", "flag_column = np.zeros(len(master_catalogue), dtype=bool)\n", "for column in flag_cleaned_columns:\n", " flag_column |= master_catalogue[column]\n", " \n", "master_catalogue.add_column(Column(data=flag_column, name=\"flag_cleaned\"))\n", "master_catalogue.remove_columns(flag_cleaned_columns)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Each pristine catalogue contains a flag indicating the probability of a source being a Gaia object (0: not a Gaia object, 1: possibly, 2: probably, 3: definitely). We merge these flags taking the highest value." ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "collapsed": true }, "outputs": [], "source": [ "flag_gaia_columns = [column for column in master_catalogue.colnames\n", " if 'flag_gaia' in column]\n", "\n", "master_catalogue.add_column(Column(\n", " data=np.max([master_catalogue[column] for column in flag_gaia_columns], axis=0),\n", " name=\"flag_gaia\"\n", "))\n", "master_catalogue.remove_columns(flag_gaia_columns)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Each prisitine catalogue may contain one or several stellarity columns indicating the probability (0 to 1) of each source being a star. We merge these columns taking the highest value. We keep trace of the origin of the stellarity." ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "uhs_stellarity, legacy_stellarity\n" ] } ], "source": [ "stellarity_columns = [column for column in master_catalogue.colnames\n", " if 'stellarity' in column]\n", "\n", "print(\", \".join(stellarity_columns))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# We create an masked array with all the stellarities and get the maximum value, as well as its\n", "# origin. Some sources may not have an associated stellarity.\n", "stellarity_array = np.array([master_catalogue[column] for column in stellarity_columns])\n", "stellarity_array = np.ma.masked_array(stellarity_array, np.isnan(stellarity_array))\n", "\n", "max_stellarity = np.max(stellarity_array, axis=0)\n", "max_stellarity.fill_value = np.nan\n", "\n", "no_stellarity_mask = max_stellarity.mask\n", "\n", "master_catalogue.add_column(Column(data=max_stellarity.filled(), name=\"stellarity\"))\n", "\n", "stellarity_origin = np.full(len(master_catalogue), \"NO_INFORMATION\", dtype=\"S20\")\n", "stellarity_origin[~no_stellarity_mask] = np.array(stellarity_columns)[np.argmax(stellarity_array, axis=0)[~no_stellarity_mask]]\n", "\n", "master_catalogue.add_column(Column(data=stellarity_origin, name=\"stellarity_origin\"))\n", "\n", "master_catalogue.remove_columns(stellarity_columns)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## IV - Adding E(B-V) column" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue.add_column(\n", " ebv(master_catalogue['ra'], master_catalogue['dec'])\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## V - Adding HELP unique identifiers and field columns" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue.add_column(Column(gen_help_id(master_catalogue['ra'], master_catalogue['dec']),\n", " name=\"help_id\"))\n", "master_catalogue.add_column(Column(np.full(len(master_catalogue), \"XMM-13hr\", dtype='" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "nb_merge_dist_plot(\n", " SkyCoord(master_catalogue['ra'], master_catalogue['dec']),\n", " SkyCoord(specz['ra'] * u.deg, specz['dec'] * u.deg)\n", ")" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue = specz_merge(master_catalogue, specz, radius=1. * u.arcsec)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## VII - Choosing between multiple values for the same filter\n", "\n", "There are no duplicate filters" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## VIII.a Wavelength domain coverage\n", "\n", "We add a binary `flag_optnir_obs` indicating that a source was observed in a given wavelength domain:\n", "\n", "- 1 for observation in optical;\n", "- 2 for observation in near-infrared;\n", "- 4 for observation in mid-infrared (IRAC).\n", "\n", "It's an integer binary flag, so a source observed both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.\n", "\n", "*Note 1: The observation flag is based on the creation of multi-order coverage maps from the catalogues, this may not be accurate, especially on the edges of the coverage.*\n", "\n", "*Note 2: Being on the observation coverage does not mean having fluxes in that wavelength domain. For sources observed in one domain but having no flux in it, one must take into consideration de different depths in the catalogue we are using.*" ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "collapsed": true }, "outputs": [], "source": [ "uhs_moc = MOC(filename=\"../../dmu0/dmu0_UHS/data/UHS-DR1_XMM-13hr_MOC.fits\")\n", "legacy_moc = MOC(filename=\"../../dmu0/dmu0_LegacySurvey/data/LegacySurvey-dr4_XMM-13hr_MOC.fits\")" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "collapsed": true }, "outputs": [], "source": [ "was_observed_optical = inMoc(\n", " master_catalogue['ra'], master_catalogue['dec'],\n", " legacy_moc) \n", "\n", "was_observed_nir = inMoc(\n", " master_catalogue['ra'], master_catalogue['dec'],\n", " uhs_moc\n", ")\n", "\n", "was_observed_mir = np.zeros(len(master_catalogue), dtype=bool)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue.add_column(\n", " Column(\n", " 1 * was_observed_optical + 2 * was_observed_nir + 4 * was_observed_mir,\n", " name=\"flag_optnir_obs\")\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## VIII.b Wavelength domain detection\n", "\n", "We add a binary `flag_optnir_det` indicating that a source was detected in a given wavelength domain:\n", "\n", "- 1 for detection in optical;\n", "- 2 for detection in near-infrared;\n", "- 4 for detection in mid-infrared (IRAC).\n", "\n", "It's an integer binary flag, so a source detected both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.\n", "\n", "*Note 1: We use the total flux columns to know if the source has flux, in some catalogues, we may have aperture flux and no total flux.*\n", "\n", "To get rid of artefacts (chip edges, star flares, etc.) we consider that a source is detected in one wavelength domain when it has a flux value in **at least two bands**. That means that good sources will be excluded from this flag when they are on the coverage of only one band." ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# SpARCS is a catalogue of sources detected in r (with fluxes measured at \n", "# this prior position in the other bands). Thus, we are only using the r\n", "# CFHT band.\n", "# Check to use catalogue flags from HSC and PanSTARRS.\n", "nb_optical_flux = (\n", " 1 * ~np.isnan(master_catalogue['f_90prime_g']) +\n", " 1 * ~np.isnan(master_catalogue['f_90prime_r']) +\n", " 1 * ~np.isnan(master_catalogue['f_mosaic_z'])\n", ")\n", "\n", "nb_nir_flux = (\n", " 1 * ~np.isnan(master_catalogue['f_ukidss_j']) \n", ")\n", "\n", "nb_mir_flux = np.zeros(len(master_catalogue), dtype=float)" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "collapsed": true }, "outputs": [], "source": [ "has_optical_flux = nb_optical_flux >= 2\n", "has_nir_flux = nb_nir_flux >= 2\n", "has_mir_flux = nb_mir_flux >= 2\n", "\n", "master_catalogue.add_column(\n", " Column(\n", " 1 * has_optical_flux + 2 * has_nir_flux + 4 * has_mir_flux,\n", " name=\"flag_optnir_det\")\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## IX - Cross-identification table\n", "\n", "We are producing a table associating to each HELP identifier, the identifiers of the sources in the pristine catalogues. This can be used to easily get additional information from them.\n", "\n", "There is no SDSS on XMM-13hr." ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['uhs_id', 'legacy_id', 'help_id', 'specz_id']\n" ] } ], "source": [ "id_names = []\n", "for col in master_catalogue.colnames:\n", " if '_id' in col:\n", " id_names += [col]\n", " if '_intid' in col:\n", " id_names += [col]\n", " \n", "print(id_names)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue[id_names].write(\n", " \"{}/master_list_cross_ident_xmm-13hr{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)\n", "id_names.remove('help_id')\n", "master_catalogue.remove_columns(id_names)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## X - Adding HEALPix index\n", "\n", "We are adding a column with a HEALPix index at order 13 associated with each source." ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue.add_column(Column(\n", " data=coords_to_hpidx(master_catalogue['ra'], master_catalogue['dec'], order=13),\n", " name=\"hp_idx\"\n", "))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## XI - Saving the catalogue" ] }, { "cell_type": "code", "execution_count": 29, "metadata": { "collapsed": true }, "outputs": [], "source": [ "columns = [\"help_id\", \"field\", \"ra\", \"dec\", \"hp_idx\"]\n", "\n", "bands = [column[5:] for column in master_catalogue.colnames if 'f_ap' in column]\n", "for band in bands:\n", " columns += [\"f_ap_{}\".format(band), \"ferr_ap_{}\".format(band),\n", " \"m_ap_{}\".format(band), \"merr_ap_{}\".format(band),\n", " \"f_{}\".format(band), \"ferr_{}\".format(band),\n", " \"m_{}\".format(band), \"merr_{}\".format(band),\n", " \"flag_{}\".format(band)] \n", " \n", "columns += [\"stellarity\", \"stellarity_origin\", \"flag_cleaned\", \"flag_merged\", \"flag_gaia\", \"flag_optnir_obs\", \n", " \"flag_optnir_det\", \"ebv\", 'zspec_association_flag', 'zspec_qual', 'zspec']" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Missing columns: set()\n" ] } ], "source": [ "# We check for columns in the master catalogue that we will not save to disk.\n", "print(\"Missing columns: {}\".format(set(master_catalogue.colnames) - set(columns)))" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": true }, "outputs": [], "source": [ "master_catalogue[columns].write(\"{}/master_catalogue_xmm-13hr{}.fits\".format(OUT_DIR, SUFFIX), overwrite=True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.1" } }, "nbformat": 4, "nbformat_minor": 2 }