{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Lockman-SWIRE check HELP IDs\n", "\n", "Because the photo-zs were computed from an early version of the masterlist, there is some concern that the HELP IDs may have changed. In this notebook we will check for differences and convert the old IDs to the new ones if required." ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "This notebook was run with herschelhelp_internal version: \n", "708e28f (Tue May 8 18:05:21 2018 +0100)\n" ] } ], "source": [ "from herschelhelp_internal import git_version\n", "print(\"This notebook was run with herschelhelp_internal version: \\n{}\".format(git_version()))" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/seaborn/apionly.py:6: UserWarning: As seaborn no longer sets a default style on import, the seaborn.apionly module is deprecated. It will be removed in a future version.\n", " warnings.warn(msg, UserWarning)\n" ] } ], "source": [ "%matplotlib inline\n", "#%config InlineBackend.figure_format = 'svg'\n", "\n", "import matplotlib.pyplot as plt\n", "plt.rc('figure', figsize=(10, 6))\n", "\n", "import os\n", "import time\n", "\n", "from astropy import units as u\n", "from astropy.coordinates import SkyCoord\n", "from astropy.table import join, Column, Table\n", "import numpy as np\n", "from pymoc import MOC\n", "\n", "from collections import Counter\n", "\n", "from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge\n", "from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "\n", "OUT_DIR = os.environ.get('OUT_DIR', \"./data\")\n", "SUFFIX = os.environ.get('SUFFIX', time.strftime(\"_%Y%m%d\"))\n", "\n", "OLD_SUFFIX = \"20170710\"\n", "NEW_SUFFIX = \"20180219\"\n", "\n", "ML_FOLDER = \"../../dmu1/dmu1_ml_Lockman-SWIRE/\"\n", "\n", "try:\n", " os.makedirs(OUT_DIR)\n", "except FileExistsError:\n", " pass" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## I - Reading the prepared pristine catalogues" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "collapsed": true }, "outputs": [], "source": [ "photoz = Table.read(\"./data/master_catalogue_Lockman-SWIRE_20170710_photoz_20170802_r_and_irac1_optimised.fits\")\n", "master_catalogue = Table.read(\"{}data/master_catalogue_lockman-swire_{}.fits\".format(ML_FOLDER, NEW_SUFFIX))[\"help_id\",\"ra\",\"dec\"]\n" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "collapsed": true }, "outputs": [], "source": [ "photoz.meta = None\n", "\n", "#photoz = photoz['help_id', 'z1_median', 'RA', 'DEC']\n", "\n", "photoz['RA'].name = 'photoz_ra'\n", "photoz['DEC'].name = 'photoz_dec'\n", "#photoz['z1_median'].name = 'redshift'\n", "\n", "photoz['z1_median'][photoz['z1_median'] < 0] = np.nan # -99 used for missing values" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "collapsed": true }, "outputs": [], "source": [ "merged_table = join(master_catalogue, photoz, join_type='left')\n", "\n", "# Fill values\n", "merged_table['z1_median'].fill_value = np.nan\n", "merged_table = merged_table.filled()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Master catalogue length: 4366298\n", "Photoz length: 1382785\n", "Merged length: 4366298\n", "Merged photozs length: 797328\n", "Nonnan photozs length: 1377480\n" ] } ], "source": [ "print(\"Master catalogue length: {}\".format(len(master_catalogue)))\n", "print(\"Photoz length: {}\".format(len(photoz)))\n", "print(\"Merged length: {}\".format(len(merged_table)))\n", "print(\"Merged photozs length: {}\".format(np.sum(~np.isnan(merged_table['z1_median']))))\n", "print(\"Nonnan photozs length: {}\".format(np.sum(~np.isnan(photoz['z1_median']))))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Check positions" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/matplotlib/axes/_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n", " warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEKCAYAAADpfBXhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH1VJREFUeJzt3XuUnHWd5/H3p6urr0m6c2kg5EIDgtcJqJHLMq6sOC4qAzuKK95GHZnMeHTU2ZkzR2Zd3OHsOerOWWdGHWUYcQcRLxxl2MigDl4QHLk1ISAhMASEpEkgTbrTSfre1d/9o55OKpXuVHV3VSr99Od1Tp1+6nl+9dS3uHz617/6Pb9HEYGZmaVLXa0LMDOzynO4m5mlkMPdzCyFHO5mZinkcDczSyGHu5lZCjnczcxSyOFuZpZCDnczsxSqr9Ubr1ixIjo7O2v19mZm89KDDz74YkR0lGpXs3Dv7Oykq6urVm9vZjYvSXq2nHYeljEzSyGHu5lZCjnczcxSyOFuZpZCZYe7pIykhyTdNsWxRknflbRN0n2SOitZpJmZzcxMeu6fALZOc+zDQF9EvAT4G+Dzcy3MzMxmr6xwl7QaeBvwtWmaXAbckGx/D7hIkuZenpmZzUa5Pfe/Bf4CmJjm+CpgB0BEjAP9wPLiRpI2SOqS1NXT0zOLcs3MrBwlw13SJcDuiHjwaM2m2HfEzVkj4rqIWB8R6zs6Sl5gZWZms1TOFaoXAJdKeivQBCyR9M2IeF9Bm25gDdAtqR5oA3orXm2FfOu+7VPuf8+5a49xJWZm1VGy5x4RV0XE6ojoBK4AflYU7AAbgQ8k25cnbY7ouZuZ2bEx67VlJF0DdEXERuB64EZJ28j32K+oUH1mZjYLMwr3iLgTuDPZvrpg/zDwzkoWZmZms+crVM3MUsjhbmaWQg53M7MUcribmaWQw93MLIUc7mZmKeRwNzNLIYe7mVkKOdzNzFLI4W5mlkIOdzOzFHK4m5mlkMPdzCyFHO5mZinkcDczSyGHu5lZCjnczcxSqGS4S2qSdL+khyVtkfRXU7T5oKQeSZuTx5XVKdfMzMpRzm32RoA3RsQBSVngl5J+GBH3FrX7bkR8rPIlmpnZTJUM94gI4EDyNJs8oppFmZnZ3JQ15i4pI2kzsBu4IyLum6LZOyQ9Iul7ktZMc54NkrokdfX09MyhbDMzO5qywj0ichFxNrAaOEfSq4qa/ADojIh1wE+AG6Y5z3URsT4i1nd0dMylbjMzO4oZzZaJiL3AncDFRfv3RMRI8vQfgddWpDozM5uVcmbLdEhqT7abgTcBjxe1WVnw9FJgayWLNDOzmSlntsxK4AZJGfK/DG6OiNskXQN0RcRG4OOSLgXGgV7gg9Uq2MzMSitntswjwKun2H91wfZVwFWVLc3MzGbLV6iamaWQw93MLIUc7mZmKeRwNzNLIYe7mVkKOdzNzFLI4W5mlkIOdzOzFHK4m5mlkMPdzCyFHO5mZinkcDczSyGHu5lZCjnczcxSyOFuZpZCDnczsxRyuJuZpVA591BtknS/pIclbZH0V1O0aZT0XUnbJN0nqbMaxZqZWXnK6bmPAG+MiLOAs4GLJZ1X1ObDQF9EvAT4G+DzlS3TzMxmomS4R96B5Gk2eURRs8uAG5Lt7wEXSVLFqjQzsxkpa8xdUkbSZmA3cEdE3FfUZBWwAyAixoF+YPkU59kgqUtSV09Pz9wqNzOzaZUV7hGRi4izgdXAOZJeVdRkql56ce+eiLguItZHxPqOjo6ZV2tmZmWZ0WyZiNgL3AlcXHSoG1gDIKkeaAN6K1CfmZnNQjmzZToktSfbzcCbgMeLmm0EPpBsXw78LCKO6LmbmdmxUV9Gm5XADZIy5H8Z3BwRt0m6BuiKiI3A9cCNkraR77FfUbWKzcyspJLhHhGPAK+eYv/VBdvDwDsrW5qZmc2Wr1A1M0shh7uZWQo53M3MUsjhbmaWQg53M7MUcribmaWQw93MLIUc7mZmKeRwNzNLIYe7mVkKOdzNzFLI4W5mlkIOdzOzFHK4m5mlkMPdzCyFHO5mZinkcDczS6Fy7qG6RtLPJW2VtEXSJ6Zoc6Gkfkmbk8fVU53LzMyOjXLuoToO/FlEbJK0GHhQ0h0R8VhRu7sj4pLKl2hmZjNVsuceEbsiYlOyvR/YCqyqdmFmZjZ7Mxpzl9RJ/mbZ901x+HxJD0v6oaRXTvP6DZK6JHX19PTMuFgzMytP2eEuaRHwfeCTEbGv6PAm4JSIOAv4EnDrVOeIiOsiYn1ErO/o6JhtzWZmVkJZ4S4pSz7Yb4qIW4qPR8S+iDiQbN8OZCWtqGilZmZWtnJmywi4HtgaEV+Yps1JSTsknZOcd08lCzUzs/KVM1vmAuD9wK8lbU72/SWwFiAirgUuBz4iaRwYAq6IiKhCvWZmVoaS4R4RvwRUos2XgS9XqigzM5sbX6FqZpZCDnczsxRyuJuZpZDD3cwshRzuZmYp5HA3M0shh7uZWQo53M3MUsjhbmaWQg53M7MUcribmaWQw93MLIUc7mZmKeRwNzNLIYe7mVkKOdzNzFLI4W5mlkLl3EN1jaSfS9oqaYukT0zRRpK+KGmbpEckvaY65ZqZWTnKuYfqOPBnEbFJ0mLgQUl3RMRjBW3eApyRPM4Fvpr8NDOzGijZc4+IXRGxKdneD2wFVhU1uwz4RuTdC7RLWlnxas3MrCwzGnOX1Am8Griv6NAqYEfB826O/AVgZmbHSNnhLmkR8H3gkxGxr/jwFC+JKc6xQVKXpK6enp6ZVWpmZmUrK9wlZckH+00RccsUTbqBNQXPVwM7ixtFxHURsT4i1nd0dMymXjMzK0M5s2UEXA9sjYgvTNNsI/D7yayZ84D+iNhVwTrNzGwGypktcwHwfuDXkjYn+/4SWAsQEdcCtwNvBbYBg8CHKl+qmZmVq2S4R8QvmXpMvbBNAB+tVFFmZjY3vkLVzCyFHO5mZinkcDczSyGHu5lZCjnczcxSyOFuZpZCDnczsxRyuJuZpZDD3cwshRzuZmYp5HA3M0shh7uZWQo53M3MUsjhbmaWQg53M7MUWpDhfte/9/CVO7eRX4bezCx9yrkTU6rsOTDCzx7fzWhugr7BMZa1NtS6JDOziivnHqpfl7Rb0qPTHL9QUr+kzcnj6sqXWTnX3fU0o7kJALb3DtS4GjOz6ihnWOafgItLtLk7Is5OHtfMvazq6Nk/wg33PMO61W001NexvXew1iWZmVVFOfdQvUtSZ/VLqb5rf/EUY7ngTS8/kYGRcbbvcbibWTpV6gvV8yU9LOmHkl5ZoXNW1MRE8J37t/O761ayYlEja5e1sKt/mJHxXK1LMzOruEqE+ybglIg4C/gScOt0DSVtkNQlqaunp6cCb12+ngMjDIzmeO0pSwFYu6yVALr7ho5pHWZmx8Kcwz0i9kXEgWT7diAracU0ba+LiPURsb6jo2Oubz0jzyZDMGuXt+Z/LmsB8Li7maXSnMNd0kmSlGyfk5xzz1zPW2nP7snPjDklCfXmhgwdixs97m5mqVTyC1VJ3wYuBFZI6gY+A2QBIuJa4HLgI5LGgSHgijgOrw7a3jtIpk6sWtp8cN8py1rYsnMfExHU5X8/mZmlQjmzZd5d4viXgS9XrKIqeXbPICe3N5HNHPpjZe2yFrqe7ePFAyOcsLiphtWZmVXWgll+4NneQU5Z1nrYvtXJEM3OvcO1KMnMrGoWTLhv3zPA2uUth+1b1pJfeqB/cLQWJZmZVc2CCPd9w2P0DY4d/DJ1UkN9Hc3ZDHuHxmpUmZlZdSyIcJ+cEbO2KNwB2luy9DvczSxlFkS4H5rjfmS4tzVn2TvocDezdFkY4Z6s/njK8tYjjrU1u+duZumzIMJ9R+8gy1sbWNR45MzP9pYGhsZyXmPGzFJlQYT7s3sGpxySgXzPHaDfQzNmliILJtyLZ8pMap8Mdw/NmFmKpD7cR8cn2NU/dHDBsGJtLflw93RIM0uT1Id7d98gE8G0PfclTVmEe+5mli6pD/dnkyV9T5lmzD1TJxY31Xs6pJmlSurD/bnkZhyrl04d7jA5HdJLEJhZeqQ+3J/vHyZTJzoWN07bpr2lwT13M0uV1If7rv5hTljcSKZu+vXaJy9kOg6XoTczm5XUh/sL+4Y5qe3oa7W3t2QZnwh6Bzw0Y2bpkPpw39U/xMoS4T55IdOufq/rbmbpUDLcJX1d0m5Jj05zXJK+KGmbpEckvabyZc5ORLCrf5iTljQftd1kuO/cO3QsyjIzq7pyeu7/BFx8lONvAc5IHhuAr869rMrYPzLO4GiuZM+9Pblph8PdzNKiZLhHxF1A71GaXAZ8I/LuBdolraxUgXPxfDLMUmrMvbUhQ32dPCxjZqlRiTH3VcCOgufdyb6amwzrUj13SbQ1Z3nOPXczS4lKhPtUcwynnFMoaYOkLkldPT09FXjro3u+Px/WpXrukB93d8/dzNKiEuHeDawpeL4a2DlVw4i4LiLWR8T6jo6OCrz10U2G9QmLS4d7e0vWY+5mlhqVCPeNwO8ns2bOA/ojYlcFzjtnz/cPs2JRIw31pT9mW3OWF/YNM56bOAaVmZlV15G3Jioi6dvAhcAKSd3AZ4AsQERcC9wOvBXYBgwCH6pWsTO1q3+45Hj7pLbmBiYCXtg/wqr2o0+dNDM73pUM94h4d4njAXy0YhVV0PP9w9PegalYe7Ku+669Qw53M5v3Un2FajlXp046eCGTv1Q1sxRIbbgPjIyzb3i8rJky4KtUzSxdUhvuz+8rb477pKZshsVN9exyuJtZCqQ23F+YvDq1xLoyhVa1N/PcXg/LmNn8l9pw31Xm0gOFVrY1savfPXczm/9SG+6TwzInLZlBuLc3e8zdzFIhteG+q3+I9pYszQ2Zsl+zqr2ZvsExhkZzVazMzKz6Uhvuz/cPz6jXDoe+fN3poRkzm+dSG+7dfTO/GOnkpP0uf6lqZvNcKsM9ItjRO8iaZeVdnTrp5LZ8uLvnbmbzXSrDvW9wjIHR3IzD/cS2RiRfyGRm818qw31H7yAAa5bObFimsT7DikWNHpYxs3kvleG+fTLcZ9hzBzi5rcnDMmY276Uy3Hf0zSHcPdfdzFIgneHeO8Sy1gYWNZZc0fgIK9ua2bl3mPxKxmZm81NKw31wxuPtk05ub2JoLMfewbEKV2VmduykM9z7Zj4NctLqpfnXTY7bm5nNR2WFu6SLJT0haZukT01x/IOSeiRtTh5XVr7U8uQmguf6hmYd7i85oRWAp188UMmyzMyOqXLuoZoB/h74HaAbeEDSxoh4rKjpdyPiY1WocUZ29Q8xPhGsnWW4r13WSqZOPN0zUOHKzMyOnXJ67ucA2yLi6YgYBb4DXFbdsmZvR29+psuapbML94b6OtYsbeapHvfczWz+KifcVwE7Cp53J/uKvUPSI5K+J2lNRaqbhUPTIGd/k+vTOxa5525m81o54a4p9hXPE/wB0BkR64CfADdMeSJpg6QuSV09PT0zq7RMO3oHqdOhRcBm47SOVp5+cYDchKdDmtn8VE64dwOFPfHVwM7CBhGxJyJGkqf/CLx2qhNFxHURsT4i1nd0dMym3pJ29A6ysq2ZbGb2E4FO61jE6PiEL2Yys3mrnAR8ADhD0qmSGoArgI2FDSStLHh6KbC1ciXOzI6+oTkNyUB+WAbwuLuZzVslwz0ixoGPAT8mH9o3R8QWSddIujRp9nFJWyQ9DHwc+GC1Ci5le+/grGfKTDqtIz8d8imPu5vZPFXW9fkRcTtwe9G+qwu2rwKuqmxpMzc8lqNn/8isZ8pMWt7aQFtzlqfdczezeSpVV6j+5sV8T3vt8rmFu6T8l6ruuZvZPJWqcH+key8Av7Wqbc7nOm3FIo+5m9m8lapwf7i7n8VN9XQub53zuU4/oZXd+0fYP+wFxMxs/klVuD/SvZd1q9uoq5tqav7MnLYiP2NmcqjHzGw+SU24D4/leHzXftatbq/I+U4/OGPGQzNmNv+kJty37trH+ERw1uq5j7cDnLI8v4DYv7/gcDez+Sc14f5Idz9AxXruDfV1nL2mnV89taci5zMzO5Zmfh+649TD3XtZsaiRlW1NFTvn689Ywd/99En2Do7S3tJQsfOa2bE1MREMjI4zMJLjwMg4A8njwMg4kljcVM/SlgZOXdFKQ306+rypCfdHuvs5a3Ub0ty/TJ30+jM6+NufPMm/bdvD29atLP0CM6uIiGB4bOJgEB8M5NFxDozkDgvn/M+ifUVBPjiaK+t9MxInLmnkzJMW87rOZSydplP3nnPXVvLjVkUqwn3/8BhP9Rzg0rNOruh5z1rdxuKmeu5+ssfhblbC6PhEUbgeCt3De8tHBnHxvsHRXNmrsmYzoqE+Q2N9XcEjw6LGepa3NuSfZ/PHG5Jjhe0AhsdzHBgeZ1f/MN19g/ziiR5+8UQPL1+5hEvWrZyXf7mnItx//Vw/EbCuQl+mTqrP1HHB6Su4+8kXiYiK/lVgVmu5g0MVpYO4VM95YCTHaG6irPfNSPmQzR4K2MmwXdLUfFhIN2QPD+JDAZ1/3lBfR6YCU58nnZWsf9s3OMoDz/Tyb9te5O9+eoC3vmol6zuXzqsMSEW4P7yjsl+mFnr9mSv40ZbneapngJecsKji5zebicLe8WGP4UNhu3/4yCCeKrCHxsobqhAcFqiN2UM94BWLGlnV3pyEbkEQF4RycW+5fg7LcR8rS1saePMrTmL9Kcu4ZVM3/7z5OX6zZ4C3v2YV9XXHf/2QgnCPCP75oW5etWoJy1or/6fTfzwjv+783U/2ONxtVsZyhwfyQBLAhdv58eExDhSMEx8YPjLER8fL6x1nMzosUCeDd0lzlo7FjQd7vcU954bskcMW2YzmVY+1kpa1NvAHv30qdz7Rw0+2vkD/0BjvO/eUWpdVlnkf7vc+3cu/v3CA/335uqqcf82yFk5d0crdT77Ihy44tSrvYcef3EQcEcYDBb3kw45NE8aTrxuZTSAXhOyy1gZOamuiKQnppuLhjIJeclM2H9p1CzSMq6FO4o0vO4GlLVlu2fQc/3DXU1xy1kpOXFK5mXnVMO/D/Rv3PMPSlmzFv0wtdOFLO7jp3u083XOA0zrcez9eRQRDY7nDQ7YocItD+mAwFz0vd3bF5Jd5TUXDEe3NWU5c3HTkuHJyvOmwUK782LFV3qvXLmVJc5Yb73mWd157DzddeS5r5njviGqa1+G+c+8Q//rYC/zh60+jKZup2vt85MLT+d6D3Xz61ke56cpzF+yfqNUQEYzmJhhIxoEPG644GM7JcMXB7fHk+dgRAV7OBIs6QVNBsDZm62iqz9DaWM/yRQ2H7ZscM56qvQN54Tm9YxEf/u1T+fYD23nHV3/F9R94Hb9V4YkclTKvw/2m+54lInhvleecnrC4ib+4+GX8j1sf5dbNz/F7r15d1fc7nkxM5MN3ZHyC0fEJRsZzBduH9g2PTRz8Em8wGTceHB1nYDR3cDbF4OTMjNEcgwVT3sbLnPJW/GVdUxK0J7U1HxzKmNxXHMSFr6uvW7hjyDZ3a5a1cPMfnc+H/u8DvPMffsVfX34Wv1vFkYPZKivcJV0M/B2QAb4WEZ8rOt4IfIP8jbH3AO+KiGcqW+rhHtrexzfueZaLXn7iMfnT6L3nrOX7D3bzv27byus6l7F6jnd7mk5EsG94nL6BUfYMjNI3MEr/0BhDYzmGx3IMjeYYHs8xNDrB0FiOkbEcQ8ljeCzH0NgEI2P5wBzPTSQ/g/GJ/HYuF0xEEJD/GRAAwcH9cfDn3D5L8fzjhsxkAB+a8jb5pd5hMyqmCGSPI9vx5MwTF/P/PnYBH/nmg/zJtx/i/t/08udvfiltLdlal3ZQyXCXlAH+HvgdoBt4QNLGiHisoNmHgb6IeImkK4DPA++qRsEAv3zyRTbc2MWKRY1cfckrqvU2h6mrE599+2/x9q/8iov+zy/4w9efxocu6GT5osZpXzM5Btw7MErfwBi9g/mw7h0YpW8w/7P3sOdj7B0cLdmTrRNkM3XJQwXb+ef1mTqasnXUNWTISNTV6eDPOuWntklCAAXPk6fJPqHkWH2dyGTyPd76uvz58z9FfV1+O5s5FNINDmNbAFYsauSmK8/jsz/cyg2/eoZ/+fUuPvmmM7js7FW0Ndc+5BUlumeSzgf+Z0T85+T5VQAR8dmCNj9O2twjqR54HuiIo5x8/fr10dXVNeOCf/b4C/zxjZs4raOVb/zBOZwwi2+sv3Xf9in3l3NJ8XN7h/jrHz3OrZt3AtDWnGVVezPZjEBieDSXfHE3dtQxYAEtDflx3paGelobM/mfDRlaGvM/88cyNCczICYD3OO8ZrVVnBWP7dzHZzY+ygPP9NGQqeMNL+3gNWuXcuaJi1i1tJlFjfU0ZTMHhyzbW7Kc3N48q/eW9GBErC/VrpxhmVXAjoLn3cC507WJiHFJ/cBy4MXyyi3fGScs5qKXn8Dn3r6u4n8CTRf6xc45dTmrl7bw9IsD7Dkwwt7BMcYnJojI96hPXNLI2mUtB4cXWgqCurWhnpbGDE3ZjHu2ZinxipOXcPMfnc+vn+vn1od28uMtz3PHYy9M2/6P33A6n3rLy6paUznhPlUCFfdHy2mDpA3AhuTpAUlPlPH+U7r2/bN9JQArqMIvnuNEWj9bWj8XpPezpfVz8d45vv6qz8NVs395WVdRlRPu3cCaguergZ3TtOlOhmXagN7iE0XEdcB15RRWTZK6yvmzZj5K62dL6+eC9H62tH6u+aKcRRIeAM6QdKqkBuAKYGNRm43AB5Lty4GfHW283czMqqtkzz0ZQ/8Y8GPyUyG/HhFbJF0DdEXERuB64EZJ28j32K+oZtFmZnZ0Zc1zj4jbgduL9l1dsD0MvLOypVVVzYeGqiitny2tnwvS+9nS+rnmhZJTIc3MbP6ZHwsTm5nZjCy4cJd0saQnJG2T9Kla11Mpkr4uabekR2tdSyVJWiPp55K2Stoi6RO1rqkSJDVJul/Sw8nn+qta11RpkjKSHpJ0W61rWYgWVLgXLKXwFuAVwLslHZv1C6rvn4CLa11EFYwDfxYRLwfOAz6akn9nI8AbI+Is4GzgYknn1bimSvsEsLXWRSxUCyrcgXOAbRHxdESMAt8BLqtxTRUREXcxxbUF811E7IqITcn2fvJhsaq2Vc1d5B1InmaTR2q+AJO0Gngb8LVa17JQLbRwn2ophXkfFAuFpE7g1cB9ta2kMpJhi83AbuCOiEjF50r8LfAXQHm3obKKW2jhXtYyCXb8kbQI+D7wyYjYV+t6KiEichFxNvmrvs+R9Kpa11QJki4BdkfEg7WuZSFbaOFezlIKdpyRlCUf7DdFxC21rqfSImIvcCfp+c7kAuBSSc+QH/p8o6Rv1rakhWehhXs5SynYcUT5heavB7ZGxBdqXU+lSOqQ1J5sNwNvAh6vbVWVERFXRcTqiOgk///YzyLifTUua8FZUOEeEePA5FIKW4GbI2JLbauqDEnfBu4BXiqpW9KHa11ThVwAvJ98729z8nhrrYuqgJXAzyU9Qr7TcUdEeMqgVYyvUDUzS6EF1XM3M1soHO5mZinkcDczSyGHu5lZCjnczcxSyOFuZpZCDnc7ZiTlknnqW5Klbv+bpLrk2HpJXzzKazslvefYVXvEew8l68AcFyS9K1m22nPjbUoOdzuWhiLi7Ih4JfA7wFuBzwBERFdEfPwor+0EahLuiaeSdWDKliwxXRUR8V3gymqd3+Y/h7vVRETsBjYAH1PehZO9UElvKLga9SFJi4HPAa9P9v1p0pu+W9Km5PEfktdeKOlOSd+T9Likm5IlDJD0Okm/Sv5quF/S4mRlxr+W9ICkRyT9UTn1S7pV0oPJXyEbCvYfkHSNpPuA86d5z1cm25uT9zwjee37Cvb/w+Qvh+QGM5uSc/y0gv8aLM0iwg8/jskDODDFvj7gROBC4LZk3w+AC5LtReRv5H7weLK/BWhKts8AupLtC4F+8ovC1ZFfkuG3gQbgaeB1SbslyXk3AJ9O9jUCXcCpRTV2Ao8W7VuW/GwGHgWWJ88D+K/J9nTv+SXgvQVtmoGXJ587m+z/CvD7QAf5ZapPLXzfgs9621T/rP3wo36GvwvMKm2qZZj/DfiCpJuAWyKiO+l8F8oCX5Z0NpADziw4dn9EdAMk4+Sd5AN/V0Q8ABDJssGS3gysk3R58to28r8sflOi7o9L+r1ke03ymj1JLd9P9r90mve8B/jvyQ0tbomIJyVdBLwWeCD5rM3k13k/D7grIn6TnCN1N2Sx6nC4W81IOo18GO4m33MFICI+J+lfyI/J3yvpTVO8/E+BF4CzyPfQhwuOjRRs58j/dy6mXrtfwJ9ExI9nUPeF5FdxPD8iBiXdCTQlh4cjIldw7iPeMyK+lQzbvA34saQrk7Y3RMRVRe916TR1mx2Vx9ytJiR1ANcCX46IKDp2ekT8OiI+T36Y5GXAfmBxQbM28r3iCfKrRpb68vJx4GRJr0veY7GkevIrhH4kWTMeSWdKai1xrjagLwn2l5HvXZf9nskvtacj4ovkl5xeB/wUuFzSCUnbZZJOIT+s9AZJp07uL1GbGeCeux1bzckwSZb8ja9vBKZao/2Tkv4T+V73Y8APyd+ubVzSw+RvBv4V4PuS3gn8HBg42htHxKikdwFfStZPHyLf+/4a+WGbTckXrz3AfynxOX4E/HGyXO8TwL0zfM93Ae+TNAY8D1wTEb2SPg38azI9dAz4aETcm3xhe0uyfzf5mUZmR+Ulf81KUP7erbdFxHF1G7xkeOjPI+KSWtdixx8Py5iVlgPajreLmMj/9dJX61rs+OSeu5lZCrnnbmaWQg53M7MUcribmaWQw93MLIUc7mZmKfT/ATA1/iE2esjTAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "photoz['photoz_ra'].unit = u.deg\n", "photoz['photoz_dec'].unit = u.deg\n", "nb_merge_dist_plot(\n", " SkyCoord(photoz['photoz_ra'], photoz['photoz_dec']),\n", " SkyCoord(master_catalogue['ra'], master_catalogue['dec'])\n", ")" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def helpid_merge(catalogue, RA_COL, DEC_COL, master_catalogue, radius=0.4*u.arcsec):\n", " \"\"\"Add a column with new HELP_ids to some catalogue.\n", "\n", " This function cross-match a catalogue with the HELP latest masterlist:\n", " \n", " Occasionally some catalogue was produced from an out of date masterlist\n", " leading to subtle differences in help_ids and causing problems with\n", " joins and all means depending on help_ids for matches.\n", " \n", " This code uses a positional cross match to add up to date help_id data.\n", " \n", " Parameters\n", " ----------\n", " catalogue: astropy.table.Table\n", " The table containing the catalogue. It must not contain a 'help_id' \n", " column. \n", " RA_COL: \n", " The name of the ra column in the catalogue\n", " DEC_COL:\n", " The name of the dec column in the catalogue\n", " master_catalogue: astropy.table.Table\n", " The table containing the updated masterlist IDs from the latest HELP \n", " masterlist.\n", " radius: astropy.units.quantity.Quantity\n", " The radius to look for counterparts. When more than one counterpart is\n", " found, the corresponding sources will be flagged.\n", "\n", " Return\n", " ------\n", " astropy.table.Table\n", " The catalogue with current help_id column added.\n", "\n", " \"\"\"\n", " catalogue = catalogue.copy()\n", " cat_coords = SkyCoord(catalogue[RA_COL].data * u.deg,\n", " catalogue[DEC_COL].data * u.deg)\n", " master_coords = SkyCoord(master_catalogue['ra'].data * u.deg,\n", " master_catalogue['dec'].data * u.deg)\n", "\n", " idx_master, idx_cat, d2d, _ = cat_coords.search_around_sky(\n", " master_coords, radius)\n", "\n", " # We sort the three array by increasing d2d\n", " sort_idx = np.argsort(d2d)\n", " idx_cat = idx_cat[sort_idx]\n", " idx_master = idx_master[sort_idx]\n", "\n", " # We want to flag as possible mis-associations the cat ids that may be\n", " # associated to different sources with the given radius.\n", " idx_master_toflag = np.unique(\n", " [item for item, count in Counter(idx_master).items() if count > 1]\n", " )\n", "\n", " # We keep only the first association of a help_id to a source\n", " _, unique_idx = np.unique(idx_master, return_index=True)\n", " idx_cat = idx_cat[unique_idx]\n", " idx_master = idx_master[unique_idx]\n", "\n", " # We add the new help_id columns to the catalogue.\n", " catalogue.add_column(\n", " Column(data=np.full(len(catalogue), '', dtype='Table length=10\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
idxhelp_id_oldphotoz_raphotoz_decidz1_medianz1_minz1_maxz1_areaz2_medianz2_minz2_maxz2_areaza_hbza_eazyzm_eazyzpeak_eazychi_r_eazyl68_eazyu68_eazynfilt_eazyza_atlaszm_atlaszpeak_atlaschi_r_atlasl68_atlasu68_atlasnfilt_atlasza_cosmoszm_cosmoszpeak_cosmoschi_r_cosmosl68_cosmosu68_cosmosnfilt_cosmoschi_r_stellarstellar_typehelp_id
degdeg
0HELP_J105455.339+572548.213163.730578607702557.430059187853251270.21980.18740.24940.766-99.0-99.0-99.0-99.00.227149635921461234.1534.1584.157624.28233754.1074.20954.2154.0674.067222.93475253.7634.25554.1073.8983.898121.41510753.7034.1135110.18145k3iHELP_J105455.339+572548.213
1HELP_J105331.913+573410.713163.382972068702557.569642539853258000.08010.02840.13860.783-99.0-99.0-99.0-99.00.072396868048342582.7512.5032.71780.11556922.5582.81752.8082.6022.61990.27321352.4132.81752.0232.1682.18490.0884027252.0062.35651.44887175g0vHELP_J105331.913+573410.713
2HELP_J105527.800+575424.407163.8658351577025257.9067797838532520490.98780.81141.22070.793-99.0-99.0-99.0-99.00.85535184174114083.6683.573.684540.31323.6063.74253.5713.5663.566738.7619753.4893.64352.0972.1972.087741.8099752.0382.1585108.3082b1iHELP_J105527.800+575424.407
3HELP_J104646.624+601224.028161.694266612702560.2066745188532531480.06960.02530.1150.781-99.0-99.0-99.0-99.00.053294796847041542.9952.9732.97361.849149252.8713.07452.5652.5962.66920.0509888752.5032.82252.5542.6242.7880.2168880752.3343.19954.07228k5vHELP_J104646.624+601224.028
4HELP_J105034.405+593434.787162.643355259702559.5763297068532531510.77120.74750.79520.684-99.0-99.0-99.0-99.00.76852560246164543.0913.1013.10072.787133.0643.13753.1163.1213.121124.794833.0873.15752.1812.2192.189530.62392.1462.237557.492025f0iHELP_J105034.405+593434.787
5HELP_J104700.697+603101.851161.7529050907024860.5171807768532531520.32880.24570.38750.6720.17760.1250.23450.1170.31862077968771770.3071.7382.33910.37009050.3132.41252.0052.2462.13020.346197751.9752.3650.3430.5820.33680.2307389750.3190.367510.98019k4vHELP_J104700.697+603101.851
6HELP_J104700.735+601459.749161.753062776702560.2499304078532549220.22860.0010.5830.798-99.0-99.0-99.0-99.00.148886156443450882.4912.0912.47590.001474335250.2172.72952.7962.5362.55090.0717630752.1082.88152.0232.0412.15110.0879129751.9582.30453.068795wk0iiiHELP_J104700.735+601459.749
7HELP_J104433.838+602158.646161.140992437702560.3662905128532576751.99151.56412.45960.6740.76880.68580.84980.0631.9428624586045820.0130.0130.013419.006830.0050.02252.4182.4612.461234.8920252.3522.58252.4492.4622.461539.022752.392.535512.4638525o5vHELP_J104433.838+602158.646
8HELP_J104606.116+603443.904161.5254834057025260.5788621758532588720.35310.11830.64580.4081.25721.02371.4150.2480.334515577083925930.3351.1450.3350.5433540.3212.352.2982.192.27891.401235251.9772.32250.3310.5940.3352.392334250.3211.336512.6897025k1ivHELP_J104606.116+603443.904
9HELP_J105233.276+571425.534163.138651478702557.24042601785325122040.73610.19811.36480.791.42791.4151.44410.0070.81685282514320320.8170.8040.52781.8998030.4820.86250.5180.6860.68521.85104250.4970.84151.2471.2621.23851.66762951.1621.321517.4939825wf5vHELP_J105233.276+571425.534
\n", "\n" ], "text/plain": [ "" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "photoz_new[:10].show_in_notebook()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Objects not given up to date ids: 347\n", "Objects with the same id before and after: 798933\n" ] } ], "source": [ "photoz_new['help_id'].fill_value = ''\n", "photoz_new = photoz_new.filled()\n", "has_no_new_id = np.sum(photoz_new['help_id'] == '')\n", "has_same_id = np.sum(photoz_new['help_id'] == photoz_new['help_id_old'])\n", "print(\"Objects not given up to date ids: {}\".format(has_no_new_id))\n", "print(\"Objects with the same id before and after: {}\".format(has_same_id))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['help_id',\n", " 'help_id_old',\n", " 'photoz_ra',\n", " 'photoz_dec',\n", " 'id',\n", " 'z1_median',\n", " 'z1_min',\n", " 'z1_max',\n", " 'z1_area',\n", " 'z2_median',\n", " 'z2_min',\n", " 'z2_max',\n", " 'z2_area',\n", " 'za_hb',\n", " 'za_eazy',\n", " 'zm_eazy',\n", " 'zpeak_eazy',\n", " 'chi_r_eazy',\n", " 'l68_eazy',\n", " 'u68_eazy',\n", " 'nfilt_eazy',\n", " 'za_atlas',\n", " 'zm_atlas',\n", " 'zpeak_atlas',\n", " 'chi_r_atlas',\n", " 'l68_atlas',\n", " 'u68_atlas',\n", " 'nfilt_atlas',\n", " 'za_cosmos',\n", " 'zm_cosmos',\n", " 'zpeak_cosmos',\n", " 'chi_r_cosmos',\n", " 'l68_cosmos',\n", " 'u68_cosmos',\n", " 'nfilt_cosmos',\n", " 'chi_r_stellar',\n", " 'stellar_type']" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cols = list(photoz_new.colnames)\n", "cols.remove('help_id')\n", "cols.remove('help_id_old')\n", "cols = ['help_id', 'help_id_old'] + cols\n", "cols" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "Table length=583852\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
help_idhelp_id_old
str27bytes27
HELP_J104622.461+603357.818HELP_J104622.510+603358.173
HELP_J104730.062+603109.289HELP_J104730.140+603108.946
HELP_J105618.837+573323.391HELP_J105618.750+573323.211
HELP_J104550.020+603355.908HELP_J104550.109+603356.145
HELP_J105431.951+600019.291HELP_J105431.952+600019.293
HELP_J104314.651+553713.690HELP_J104314.652+553713.692
HELP_J103016.810+570912.714HELP_J103016.811+570912.716
HELP_J110116.362+584859.411HELP_J110116.362+584859.413
HELP_J103926.939+554659.394HELP_J103926.940+554659.396
HELP_J110320.166+595456.130HELP_J110320.167+595456.132
......
HELP_J105940.483+574644.396HELP_J105940.484+574644.398
HELP_J105407.245+572731.882HELP_J105407.246+572731.884
HELP_J105358.023+571704.684HELP_J105358.024+571704.686
HELP_J105101.507+585342.066HELP_J105101.508+585342.068
HELP_J105046.294+563145.779HELP_J105046.274+563145.803
HELP_J105345.207+573401.349HELP_J105345.208+573401.352
HELP_J105307.987+584825.158HELP_J105307.988+584825.160
HELP_J104823.842+584735.490HELP_J104823.842+584735.492
HELP_J104423.907+584522.265HELP_J104423.911+584522.199
HELP_J103824.511+575223.763HELP_J103824.512+575223.765
" ], "text/plain": [ "\n", " help_id help_id_old \n", " str27 bytes27 \n", "--------------------------- ---------------------------\n", "HELP_J104622.461+603357.818 HELP_J104622.510+603358.173\n", "HELP_J104730.062+603109.289 HELP_J104730.140+603108.946\n", "HELP_J105618.837+573323.391 HELP_J105618.750+573323.211\n", "HELP_J104550.020+603355.908 HELP_J104550.109+603356.145\n", "HELP_J105431.951+600019.291 HELP_J105431.952+600019.293\n", "HELP_J104314.651+553713.690 HELP_J104314.652+553713.692\n", "HELP_J103016.810+570912.714 HELP_J103016.811+570912.716\n", "HELP_J110116.362+584859.411 HELP_J110116.362+584859.413\n", "HELP_J103926.939+554659.394 HELP_J103926.940+554659.396\n", "HELP_J110320.166+595456.130 HELP_J110320.167+595456.132\n", " ... ...\n", "HELP_J105940.483+574644.396 HELP_J105940.484+574644.398\n", "HELP_J105407.245+572731.882 HELP_J105407.246+572731.884\n", "HELP_J105358.023+571704.684 HELP_J105358.024+571704.686\n", "HELP_J105101.507+585342.066 HELP_J105101.508+585342.068\n", "HELP_J105046.294+563145.779 HELP_J105046.274+563145.803\n", "HELP_J105345.207+573401.349 HELP_J105345.208+573401.352\n", "HELP_J105307.987+584825.158 HELP_J105307.988+584825.160\n", "HELP_J104823.842+584735.490 HELP_J104823.842+584735.492\n", "HELP_J104423.907+584522.265 HELP_J104423.911+584522.199\n", "HELP_J103824.511+575223.763 HELP_J103824.512+575223.765" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "photoz_new['help_id', 'help_id_old'][photoz_new['help_id'] != photoz_new['help_id_old']]" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "OK!\n" ] } ], "source": [ "# Check that the HELP Ids are unique\n", "mask = (photoz_new['help_id'] != '')\n", "if len(photoz_new[mask]) != len(np.unique(photoz_new['help_id'][mask])):\n", " print(\"The HELP IDs are not unique!!!\")\n", "else:\n", " print(\"OK!\")" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "collapsed": true }, "outputs": [], "source": [ "#We only include objects with help_ids to ensure joins don't break on empty id objects\n", "photoz_new = photoz_new[mask]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## II - Saving the catalogue" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "photoz_new[cols].write(\"./data/master_catalogue_Lockman-SWIRE_20170710_photoz_20170802_r_and_irac1_optimised_UPDATED_IDs_{}.fits\".format( NEW_SUFFIX)\n", " ,overwrite=True)" ] } ], "metadata": { "kernelspec": { "display_name": "Python (herschelhelp_internal)", "language": "python", "name": "helpint" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }