XMM-LSS master catalogue¶

Preparation of the Subaru/XMM-Newton Deep Survey (SXDS) data¶

The catalogue is in dmu0_SXDS.

In the catalogue, we keep:

  • The position;
  • The stellarity;
  • The aperture magnitude B, V, R, i, z (2 arcsec).
  • The total magnitude B, V, R, i, z (Kron like aperture magnitude).
In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))

This notebook was run with herschelhelp_internal version: 
33f5ec7 (Wed Dec 6 16:56:17 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

from collections import OrderedDict
import os

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table, join, vstack
import numpy as np
#from numpy.core.defchararray import add

from herschelhelp_internal.flagging import  gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates
from herschelhelp_internal.utils import astrometric_correction, mag_to_flux
from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot
In [3]:
OUT_DIR =  os.environ.get('TMP_DIR', "./data_tmp")
try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass

RA_COL = "sxds_ra"
DEC_COL = "sxds_dec"

I - Column selection¶

For each band we have 5 independent and overlapping catalogues. We must first stack the catalogues and remove duplicates then merge the bands together.

In [4]:
#We have to import and combine the H, J and Ks catalogues separately. 
#Fluxes are given in counts sowe compute them fresh from the magnitudes

epoch = 2007 # TODO: check this

bands = ["B", "V", "R", "i", "z"]
cat_numbers = [ 1, 2, 3, 4, 5 ]

def imported_columns(band): 
    return OrderedDict({
        'NUMBER': "sxds_{}_id".format(band.lower()),
        'ra': "sxds_{}_ra".format(band.lower()),
        'dec': "sxds_{}_dec".format(band.lower()),
        'CLASS_STAR_{}'.format(band):  "sxds_{}_stellarity".format(band.lower()),
        'twoApertureMag_{}'.format(band): "m_ap_sxds_{}".format(band.lower()),
        'Err_twoApertureMag_{}'.format(band): "merr_ap_sxds_{}".format(band.lower()),
        'MAG_AUTO_{}'.format(band): "m_sxds_{}".format(band.lower()),
        'Err_MAG_AUTO_{}'.format(band): "merr_sxds_{}".format(band.lower()) 
    })
In [5]:
catalogue_B = vstack([ Table.read("../../dmu0/dmu0_SXDS/data/sxdsB{}_dr1.fits".format(1))[
                                   list(imported_columns("B"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsB{}_dr1.fits".format(2))[
                                   list(imported_columns("B"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsB{}_dr1.fits".format(3))[
                                   list(imported_columns("B"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsB{}_dr1.fits".format(4))[
                                   list(imported_columns("B"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsB{}_dr1.fits".format(5))[
                                   list(imported_columns("B"))]
                     ])
                      

for column in imported_columns("B"):
    catalogue_B[column].name = imported_columns("B")[column]
In [6]:
SORT_COLS = ['merr_ap_sxds_b']
FLAG_NAME = 'sxds_flag_cleaned_b'

nb_orig_sources = len(catalogue_B)

catalogue_B = remove_duplicates(catalogue_B, 'sxds_b_ra', 'sxds_b_dec',  sort_col=SORT_COLS, flag_name=FLAG_NAME)

nb_sources = len(catalogue_B)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue_B[FLAG_NAME])))

The initial catalogue had 940853 sources.
The cleaned catalogue has 908480 sources (32373 removed).
The cleaned catalogue has 32244 sources flagged as having been cleaned
In [7]:
catalogue_V = vstack([ Table.read("../../dmu0/dmu0_SXDS/data/sxdsV{}_dr1.fits".format(1))[
                                   list(imported_columns("V"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsV{}_dr1.fits".format(2))[
                                   list(imported_columns("V"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsV{}_dr1.fits".format(3))[
                                   list(imported_columns("V"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsV{}_dr1.fits".format(4))[
                                   list(imported_columns("V"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsV{}_dr1.fits".format(5))[
                                   list(imported_columns("V"))]
                     ])
                      

for column in imported_columns("V"):
    catalogue_V[column].name = imported_columns("V")[column]
In [8]:
SORT_COLS = ['merr_ap_sxds_v']
FLAG_NAME = 'sxds_flag_cleaned_v'

nb_orig_sources = len(catalogue_V)

catalogue_V = remove_duplicates(catalogue_V, 'sxds_v_ra', 'sxds_v_dec',  sort_col=SORT_COLS, flag_name=FLAG_NAME)

nb_sources = len(catalogue_V)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue_V[FLAG_NAME])))

The initial catalogue had 1002561 sources.
The cleaned catalogue has 970203 sources (32358 removed).
The cleaned catalogue has 32252 sources flagged as having been cleaned
In [9]:
catalogue_R = vstack([ Table.read("../../dmu0/dmu0_SXDS/data/sxdsR{}_dr1.fits".format(1))[
                                   list(imported_columns("R"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsR{}_dr1.fits".format(2))[
                                   list(imported_columns("R"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsR{}_dr1.fits".format(3))[
                                   list(imported_columns("R"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsR{}_dr1.fits".format(4))[
                                   list(imported_columns("R"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsR{}_dr1.fits".format(5))[
                                   list(imported_columns("R"))]
                     ])
                      

for column in imported_columns("R"):
    catalogue_R[column].name = imported_columns("R")[column]
In [10]:
SORT_COLS = ['merr_ap_sxds_r']
FLAG_NAME = 'sxds_flag_cleaned_r'

nb_orig_sources = len(catalogue_R)

catalogue_R = remove_duplicates(catalogue_R, 'sxds_r_ra', 'sxds_r_dec',  sort_col=SORT_COLS, flag_name=FLAG_NAME)

nb_sources = len(catalogue_R)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue_R[FLAG_NAME])))

The initial catalogue had 901094 sources.
The cleaned catalogue has 873890 sources (27204 removed).
The cleaned catalogue has 27129 sources flagged as having been cleaned
In [11]:
catalogue_i = vstack([ Table.read("../../dmu0/dmu0_SXDS/data/sxdsi{}_dr1.fits".format(1))[
                                   list(imported_columns("i"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsi{}_dr1.fits".format(2))[
                                   list(imported_columns("i"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsi{}_dr1.fits".format(3))[
                                   list(imported_columns("i"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsi{}_dr1.fits".format(4))[
                                   list(imported_columns("i"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsi{}_dr1.fits".format(5))[
                                   list(imported_columns("i"))]
                     ])
                      

for column in imported_columns("i"):
    catalogue_i[column].name = imported_columns("i")[column]
In [12]:
SORT_COLS = ['merr_ap_sxds_i']
FLAG_NAME = 'sxds_flag_cleaned_i'

nb_orig_sources = len(catalogue_i)

catalogue_i = remove_duplicates(catalogue_i, 'sxds_i_ra', 'sxds_i_dec',  sort_col=SORT_COLS, flag_name=FLAG_NAME)

nb_sources = len(catalogue_i)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue_i[FLAG_NAME])))

The initial catalogue had 899484 sources.
The cleaned catalogue has 870730 sources (28754 removed).
The cleaned catalogue has 28686 sources flagged as having been cleaned
In [13]:
catalogue_z = vstack([ Table.read("../../dmu0/dmu0_SXDS/data/sxdsz{}_dr1.fits".format(1))[
                                   list(imported_columns("z"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsz{}_dr1.fits".format(2))[
                                   list(imported_columns("z"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsz{}_dr1.fits".format(3))[
                                   list(imported_columns("z"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsz{}_dr1.fits".format(4))[
                                   list(imported_columns("z"))],
                      Table.read("../../dmu0/dmu0_SXDS/data/sxdsz{}_dr1.fits".format(5))[
                                   list(imported_columns("z"))]
                     ])
                      

for column in imported_columns("z"):
    catalogue_z[column].name = imported_columns("z")[column]
In [14]:
SORT_COLS = ['merr_ap_sxds_z']
FLAG_NAME = 'sxds_flag_cleaned_z'

nb_orig_sources = len(catalogue_z)

catalogue_z = remove_duplicates(catalogue_z, 'sxds_z_ra', 'sxds_z_dec',  sort_col=SORT_COLS, flag_name=FLAG_NAME)

nb_sources = len(catalogue_z)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue_z[FLAG_NAME])))

The initial catalogue had 842590 sources.
The cleaned catalogue has 818902 sources (23688 removed).
The cleaned catalogue has 23589 sources flagged as having been cleaned

Merging different bands¶

SXDS has indivdual extractions from each band. We must therefore merge them as if they were individual catalogues (they have different

In [15]:
nb_merge_dist_plot(
    SkyCoord(catalogue_B['sxds_b_ra'], catalogue_B['sxds_b_dec']),
    SkyCoord(catalogue_V['sxds_v_ra'], catalogue_V['sxds_v_dec'])
)
In [16]:
catalogue = catalogue_B
catalogue_B['sxds_b_ra'].name = 'ra'
catalogue_B['sxds_b_dec'].name = 'dec'
# Given the graph above, we use 0.8 arc-second radius
catalogue = merge_catalogues(catalogue, catalogue_V, "sxds_v_ra", "sxds_v_dec", radius=0.8*u.arcsec)
del catalogue_B
del catalogue_V
In [17]:
nb_merge_dist_plot(
    SkyCoord(catalogue['ra'], catalogue['dec']),
    SkyCoord(catalogue_R['sxds_r_ra'], catalogue_R['sxds_r_dec'])
)
In [18]:
catalogue = merge_catalogues(catalogue, catalogue_R, "sxds_r_ra", "sxds_r_dec", radius=0.8*u.arcsec)
del catalogue_R
In [19]:
nb_merge_dist_plot(
    SkyCoord(catalogue['ra'], catalogue['dec']),
    SkyCoord(catalogue_i['sxds_i_ra'], catalogue_i['sxds_i_dec'])
)
In [20]:
catalogue = merge_catalogues(catalogue, catalogue_i, "sxds_i_ra", "sxds_i_dec", radius=0.8*u.arcsec)
del catalogue_i
In [21]:
nb_merge_dist_plot(
    SkyCoord(catalogue['ra'], catalogue['dec']),
    SkyCoord(catalogue_z['sxds_z_ra'], catalogue_z['sxds_z_dec'])
)
In [22]:
catalogue = merge_catalogues(catalogue, catalogue_z, "sxds_z_ra", "sxds_z_dec", radius=0.8*u.arcsec)
del catalogue_z
In [23]:
#rename radec colums
catalogue['ra'].name = 'sxds_ra'
catalogue['dec'].name = 'sxds_dec'

Fill masked values and add fluxes and nans¶

In [24]:
for col in catalogue.colnames:
    if "m_" in col or "merr_" in col or "f_" in col or "ferr_" in col or "stellarity" in col:
        catalogue[col].fill_value = np.nan
    elif "flag" in col:
        catalogue[col].fill_value = 0
    elif "id" in col:
        catalogue[col].fill_value = ""
        
catalogue =  catalogue.filled()
In [25]:
for col in catalogue.colnames:
    if col.startswith('m_'):
        
        errcol = "merr{}".format(col[1:])
        # catalogue_h[col].name = imported_columns_h[col]
        
        # Turn 99.0 and 99.03 to nans
        catalogue[col][catalogue[col] > 90.] = np.nan
        catalogue[errcol][catalogue[errcol] > 90.] = np.nan 
    
        flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))
        
        # Fluxes are added in µJy
        catalogue.add_column(Column(flux * 1.e6, name="f{}".format(col[1:])))
        catalogue.add_column(Column(error * 1.e6, name="f{}".format(errcol[1:])))

        # Band-flag column
        if "ap" not in col:
            catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
        
# Clean table metadata
catalogue.meta = None
    

/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:9: RuntimeWarning: invalid value encountered in greater
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:10: RuntimeWarning: invalid value encountered in greater
In [26]:
catalogue[:10].show_in_notebook()
Out[26]:
<Table length=10>
idxsxds_b_idsxds_rasxds_decsxds_b_stellaritym_ap_sxds_bmerr_ap_sxds_bm_sxds_bmerr_sxds_bsxds_flag_cleaned_bflag_mergedsxds_v_idsxds_v_stellaritym_ap_sxds_vmerr_ap_sxds_vm_sxds_vmerr_sxds_vsxds_flag_cleaned_vsxds_r_idsxds_r_stellaritym_ap_sxds_rmerr_ap_sxds_rm_sxds_rmerr_sxds_rsxds_flag_cleaned_rsxds_i_idsxds_i_stellaritym_ap_sxds_imerr_ap_sxds_im_sxds_imerr_sxds_isxds_flag_cleaned_isxds_z_idsxds_z_stellaritym_ap_sxds_zmerr_ap_sxds_zm_sxds_zmerr_sxds_zsxds_flag_cleaned_zf_ap_sxds_bferr_ap_sxds_bf_sxds_bferr_sxds_bflag_sxds_bf_ap_sxds_vferr_ap_sxds_vf_sxds_vferr_sxds_vflag_sxds_vf_ap_sxds_rferr_ap_sxds_rf_sxds_rferr_sxds_rflag_sxds_rf_ap_sxds_iferr_ap_sxds_if_sxds_iferr_sxds_iflag_sxds_if_ap_sxds_zferr_ap_sxds_zf_sxds_zferr_sxds_zflag_sxds_z
degdeg
0SXDS-BS-08026634.2020416667-5.423702777780.8518.72850.000414.27690.0001FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse117.1116230750.04314551640167067.079876390.650902110975FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
1SXDS-BE-07314034.9161291667-4.782650.8518.71150.000414.86260.0001FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse118.959738270.04382638720124120.595652610.379520884958FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
2SXDS-BE-09411934.9513791667-4.734483333330.8518.7310.000413.96330.0FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse116.8422738170.04304628446759433.655227350.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
3SXDS-BS-05552334.2651083333-5.483836111110.8518.77040.000414.7670.0001FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse112.6782257770.04151219247684499.870338720.414453374494FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
4SXDS-BW-15571133.8873791667-4.795105555560.9618.44660.000417.30570.0002FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse151.8308418330.0559365492897434.2301753640.0799881542977FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
5SXDS-BE-09575134.9587208333-5.208416666670.8518.7450.000413.60140.0FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse115.3453257820.042494788430813165.58010950.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
6SXDS-BC-05358034.2495416667-5.074566666670.8818.81770.000416.10090.0001FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse107.874799120.03974254469831317.164447940.121315328914FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
7SXDS-BW-15106233.8860208333-4.795508333330.7118.51160.000416.94470.0002FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse143.0078903540.0526860538414605.5081597290.111538724982FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
8SXDS-BN-14812634.6976416667-4.393130555560.8518.72450.000413.72380.0FalseFalsenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse117.5438739860.043304763522111761.96817570.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse
9SXDS-BN-01627034.2665791667-4.730994444440.8518.66840.000415.47620.0001FalseTruenannannannannanFalsenannannannannanFalsenannannannannanFalsenannannannannanFalse123.7770135580.04560113700382341.639309160.215672950658FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse

Combine stellarities¶

In [27]:
stellarity_columns = [column for column in catalogue.colnames
                      if 'stellarity' in column]

print(", ".join(stellarity_columns))

sxds_b_stellarity, sxds_v_stellarity, sxds_r_stellarity, sxds_i_stellarity, sxds_z_stellarity
In [28]:
# We create an masked array with all the stellarities and get the maximum value, as well as its
# origin.  Some sources may not have an associated stellarity.
stellarity_array = np.array([catalogue[column] for column in stellarity_columns])
stellarity_array = np.ma.masked_array(stellarity_array, np.isnan(stellarity_array))

max_stellarity = np.max(stellarity_array, axis=0)
max_stellarity.fill_value = np.nan

catalogue.add_column(Column(data=max_stellarity.filled(), name="sxds_stellarity"))

catalogue.remove_columns(stellarity_columns)
In [29]:
catalogue[:10].show_in_notebook()
Out[29]:
<Table length=10>
idxsxds_b_idsxds_rasxds_decm_ap_sxds_bmerr_ap_sxds_bm_sxds_bmerr_sxds_bsxds_flag_cleaned_bflag_mergedsxds_v_idm_ap_sxds_vmerr_ap_sxds_vm_sxds_vmerr_sxds_vsxds_flag_cleaned_vsxds_r_idm_ap_sxds_rmerr_ap_sxds_rm_sxds_rmerr_sxds_rsxds_flag_cleaned_rsxds_i_idm_ap_sxds_imerr_ap_sxds_im_sxds_imerr_sxds_isxds_flag_cleaned_isxds_z_idm_ap_sxds_zmerr_ap_sxds_zm_sxds_zmerr_sxds_zsxds_flag_cleaned_zf_ap_sxds_bferr_ap_sxds_bf_sxds_bferr_sxds_bflag_sxds_bf_ap_sxds_vferr_ap_sxds_vf_sxds_vferr_sxds_vflag_sxds_vf_ap_sxds_rferr_ap_sxds_rf_sxds_rferr_sxds_rflag_sxds_rf_ap_sxds_iferr_ap_sxds_if_sxds_iferr_sxds_iflag_sxds_if_ap_sxds_zferr_ap_sxds_zf_sxds_zferr_sxds_zflag_sxds_zsxds_stellarity
degdeg
0SXDS-BS-08026634.2020416667-5.4237027777818.72850.000414.27690.0001FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse117.1116230750.04314551640167067.079876390.650902110975FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
1SXDS-BE-07314034.9161291667-4.7826518.71150.000414.86260.0001FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse118.959738270.04382638720124120.595652610.379520884958FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
2SXDS-BE-09411934.9513791667-4.7344833333318.7310.000413.96330.0FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse116.8422738170.04304628446759433.655227350.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
3SXDS-BS-05552334.2651083333-5.4838361111118.77040.000414.7670.0001FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse112.6782257770.04151219247684499.870338720.414453374494FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
4SXDS-BW-15571133.8873791667-4.7951055555618.44660.000417.30570.0002FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse151.8308418330.0559365492897434.2301753640.0799881542977FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.96
5SXDS-BE-09575134.9587208333-5.2084166666718.7450.000413.60140.0FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse115.3453257820.042494788430813165.58010950.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
6SXDS-BC-05358034.2495416667-5.0745666666718.81770.000416.10090.0001FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse107.874799120.03974254469831317.164447940.121315328914FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.88
7SXDS-BW-15106233.8860208333-4.7955083333318.51160.000416.94470.0002FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse143.0078903540.0526860538414605.5081597290.111538724982FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.71
8SXDS-BN-14812634.6976416667-4.3931305555618.72450.000413.72380.0FalseFalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse117.5438739860.043304763522111761.96817570.0FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85
9SXDS-BN-01627034.2665791667-4.7309944444418.66840.000415.47620.0001FalseTruenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse123.7770135580.04560113700382341.639309160.215672950658FalsenannannannanFalsenannannannanFalsenannannannanFalsenannannannanFalse0.85

II - Removal of duplicated sources¶

We remove duplicated objects from the input catalogues.

In [30]:
SORT_COLS = ['merr_ap_sxds_b',
            'merr_ap_sxds_v',
            'merr_ap_sxds_r',
            'merr_ap_sxds_i',
            'merr_ap_sxds_z',]
FLAG_NAME = 'sxds_flag_cleaned'

nb_orig_sources = len(catalogue)

catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)

nb_sources = len(catalogue)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))

The initial catalogue had 1522678 sources.
The cleaned catalogue has 1512687 sources (9991 removed).
The cleaned catalogue has 10007 sources flagged as having been cleaned
In [31]:
catalogue['sxds_flag_cleaned'] =  (catalogue['sxds_flag_cleaned'] | 
                                   catalogue['sxds_flag_cleaned_b'] |  
                                   catalogue['sxds_flag_cleaned_v'] | 
                                   catalogue['sxds_flag_cleaned_r'] | 
                                   catalogue['sxds_flag_cleaned_i'] |  
                                   catalogue['sxds_flag_cleaned_z'])
In [32]:
catalogue.remove_columns(['sxds_flag_cleaned_b',
                         'sxds_flag_cleaned_v',
                         'sxds_flag_cleaned_r',
                         'sxds_flag_cleaned_i',
                         'sxds_flag_cleaned_z'])
catalogue['flag_merged'].name = 'sxds_flag_merged'
In [33]:
catalogue[:10].show_in_notebook()
Out[33]:
<Table length=10>
idxsxds_b_idsxds_rasxds_decm_ap_sxds_bmerr_ap_sxds_bm_sxds_bmerr_sxds_bsxds_flag_mergedsxds_v_idm_ap_sxds_vmerr_ap_sxds_vm_sxds_vmerr_sxds_vsxds_r_idm_ap_sxds_rmerr_ap_sxds_rm_sxds_rmerr_sxds_rsxds_i_idm_ap_sxds_imerr_ap_sxds_im_sxds_imerr_sxds_isxds_z_idm_ap_sxds_zmerr_ap_sxds_zm_sxds_zmerr_sxds_zf_ap_sxds_bferr_ap_sxds_bf_sxds_bferr_sxds_bflag_sxds_bf_ap_sxds_vferr_ap_sxds_vf_sxds_vferr_sxds_vflag_sxds_vf_ap_sxds_rferr_ap_sxds_rf_sxds_rferr_sxds_rflag_sxds_rf_ap_sxds_iferr_ap_sxds_if_sxds_iferr_sxds_iflag_sxds_if_ap_sxds_zferr_ap_sxds_zf_sxds_zferr_sxds_zflag_sxds_zsxds_stellaritysxds_flag_cleaned
degdeg
0SXDS-BC-19555034.79535-4.7877305555618.1430.000317.7060.0003FalseSXDS-VC-21008317.81430.000516.86430.0003SXDS-RC-16206317.70570.000316.22580.0002SXDS-iC-15386517.33120.000315.47830.0001SXDS-zC-15181115.7840.000214.6910.0001200.8167804180.0554877270017300.3308878680.0829844910444False271.8191186650.125177330126652.0486498720.180167700132False300.4138838240.08300742367471174.032177280.216264719209False424.1505163140.1171971187262337.114554430.215256205346False1763.59963390.3248670581624826.140932690.444504006732False0.99False
1SXDS-BN-17334734.7773583333-4.3701222222217.94510.000317.24330.0002FalseSXDS-VN-17277817.86150.000516.45210.0002SXDS-RN-15278617.76130.000315.87410.0001SXDS-iN-15333617.40380.000315.45610.0001SXDS-zN-14194415.64270.000314.69320.0002240.9683478420.0665820150748459.9175689150.0847199470551False260.2555501830.119852110044953.1472517710.175576212269False285.4171071320.07886366114021623.155022780.149498102363False396.7162590520.1096167533092385.393326970.219702844624False2008.722758120.5550306094574816.371745410.887208462664False0.96False
2SXDS-BN-15175734.7995416667-4.3933972222217.96310.000316.86110.0002FalseSXDS-VN-15850717.86670.000516.20480.0002SXDS-RN-13900917.74890.000315.72090.0001SXDS-iN-14145317.40340.000315.47850.0001SXDS-zN-13527715.69140.000314.82980.0002237.0063598150.0654872773266653.9732735110.120466328864False259.0120683670.1192794655051196.960999370.220488364323False288.6954904080.07976951191531869.132117150.172153429992False396.8624416490.1096571450532336.684081670.21521655734False1920.613600850.5306851496054246.977889650.782322238319False0.97False
3SXDS-BE-17717535.1538291667-5.2977527777818.03040.000317.56330.0003FalseSXDS-VE-18957517.89770.000517.26410.0004SXDS-RE-15290617.79980.000317.09240.0002SXDS-iE-17303417.48250.000317.00550.0003SXDS-zE-13928316.84820.000416.65070.0004222.7614314630.0615512581658342.5153189320.0946404800993False251.7213197240.115921951677451.1905552450.166224743457False275.4736097750.0761161712856528.4939256230.0973521787902False368.977598570.101952278174572.5323093940.158196523304False661.7896945460.243812333657793.8162774390.292452724321False1.0False
4SXDS-BE-17737535.13665-5.3004861111118.01210.000317.38660.0002FalseSXDS-VE-18558017.9330.000517.03920.0003SXDS-RE-16715717.81120.000316.82320.0002SXDS-iE-16309417.39330.000316.71140.0002SXDS-zE-14011916.50350.000416.27670.0003226.547878710.0625974922039403.0509972090.0742447374311False243.6688423460.112213648803555.034526880.153361707323False272.5963271290.0753211487101677.2047567920.124745726232False400.5714569060.110681983842750.6543402260.138275639504False909.075597360.3349158270221120.263019740.309540111537False1.0False
5SXDS-BW-15993333.8836083333-4.7299611111118.06410.000316.36710.0001FalseSXDS-VW-16303218.02440.000516.14260.0002SXDS-RW-14544017.94710.000415.94210.0001SXDS-iW-13820517.54950.000315.85560.0002SXDS-zW-13597816.02270.000315.37520.0002215.9533771120.05967012323031030.765798650.0949370384938False223.9958643490.1031539076291267.535116180.23348859707False240.5248763190.08861263915291524.614339020.140422169984False346.8965654460.09585106324881651.04914260.304134491484False1415.533005120.3911262235492569.922341970.473397189983False0.99False
6SXDS-BE-17669035.1495791667-5.2465805555618.1260.000317.46910.0002FalseSXDS-VE-19272918.08150.000516.9740.0003SXDS-RE-17260817.99580.000416.63980.0002SXDS-iE-17443817.51060.000316.44630.0002SXDS-zE-16006516.20470.000315.87140.0003203.9858300270.0563633677682373.5596845560.0688122368801False212.5200943270.097869120232589.3862502410.162853439258False229.97466770.0847257986579801.8257515460.147701761815False359.5505852480.0993474981323958.2525861280.176516649611False1197.071248620.330763009481627.196494610.449611007023False1.0False
7SXDS-BW-14634733.8919416667-4.7286777777818.11570.000315.4460.0001FalseSXDS-VW-14887518.060.000515.23880.0001SXDS-RW-13282218.00550.000415.0640.0001SXDS-iW-12637417.59440.000315.04530.0001SXDS-zW-12105015.9120.000314.56450.0002205.9301804250.05690061163722407.686845610.221756153572False216.7704104820.09982646315562913.935939970.268383418293False227.9292177430.08397222704543422.945623160.315264942641False332.8434385620.09196804079193482.410791750.320741887071False1567.47275180.4331087270365422.505485170.998862423747False0.95False
8SXDS-BS-14697934.79925-5.2772194444418.27820.000317.68420.0003TrueSXDS-VS-14298018.10590.000517.1050.0003SXDS-RS-13665018.02710.000416.78880.0002SXDS-iS-14374217.52790.000316.59870.0002SXDS-zS-12232816.29430.000416.05610.0004177.3045987320.0489910711151306.4220403120.0846675386625False207.7973477120.0956942150411522.3961889990.144343401291False223.4395252850.0823181632169699.0045456720.128761395744False353.8669476640.0977770510314832.760275130.153400111644False1102.249727170.4060838064851372.65058660.505703164573False1.0False
9SXDS-BN-13349634.7969-4.4364777777818.10010.000316.91720.0002FalseSXDS-VN-14424818.08050.000516.5570.0003SXDS-RN-13138917.95640.000416.33910.0002SXDS-iN-12883917.59980.000316.27150.0002SXDS-zN-12149516.23420.000415.80350.0003208.9103708430.057724068681621.0406103870.11439990813False212.7159227360.097959302547865.3663429560.239109556952False238.4734309490.08785685874841057.69390080.194834416716False331.1921237080.0915117656361125.641250580.207350781091False1164.984105260.4291960054941732.207840430.478626714161False1.0False

III - Astrometry correction¶

We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.

In [34]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_XMM-LSS.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [35]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)
In [36]:
delta_ra, delta_dec =  astrometric_correction(
    SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
    gaia_coords
)

print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))

RA correction: -0.050656639996304875 arcsec
Dec correction: -0.1055842856297673 arcsec
In [37]:
catalogue[RA_COL] = catalogue[RA_COL] + delta_ra.to(u.deg)
catalogue[DEC_COL] = catalogue[DEC_COL] + delta_dec.to(u.deg)
In [38]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec)

IV - Flagging Gaia objects¶

In [39]:
catalogue.add_column(
    gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [40]:
GAIA_FLAG_NAME = "sxds_flag_gaia"

catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))

2954 sources flagged.
In [41]:
catalogue.add_column(Column(data=np.arange(len(catalogue)), name="sxds_intid"))

V - Saving to disk¶

In [42]:
catalogue.write("{}/SXDS.fits".format(OUT_DIR), overwrite=True)