HDF-N master catalogue¶
Preparation of Hawaii-HDFN data¶
The catalogue comes from dmu0_Hawaii-HDFN.
It contains UBVRIz data.
In the catalogue, we keep:
- The identifier (it's unique in the catalogue);
- The position;
- The stellarity;
- The kron magnitude, there doesn't appear to be aperture magnitudes. This may mean the survey is unusable.
In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'
import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
from collections import OrderedDict
import os
from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table, vstack
import numpy as np
from herschelhelp_internal.flagging import gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates, nb_merge_dist_plot
from herschelhelp_internal.utils import astrometric_correction, flux_to_mag, mag_to_flux
In [3]:
OUT_DIR = os.environ.get('TMP_DIR', "./data_tmp")
try:
os.makedirs(OUT_DIR)
except FileExistsError:
pass
RA_COL = "hawaii_ra"
DEC_COL = "hawaii_dec"
I - Column selection¶
In [4]:
catalogue = Table.read("../../dmu0/dmu0_Hawaii-HDFN/data/R.fits")
catalogue[:10].show_in_notebook()
Out[4]:
In [5]:
imported_columns_old = OrderedDict({
'ID': "hawaii_id",
'RA': "hawaii_ra",
'DEC': "hawaii_dec",
'Uaperflux': "f_ap_mosaic_u",
'dUaperflux': "ferr_ap_mosaic_u",
'Uisoflux': "f_mosaic_u",
'dUisoflux': "ferr_mosaic_u",
'Baperflux': "f_ap_suprime_b",
'dBaperflux': "ferr_ap_suprime_b",
'Bisoflux': "f_suprime_b",
'dBisoflux': "ferr_suprime_b",
'Vaperflux': "f_ap_suprime_v",
'dVaperflux': "ferr_ap_suprime_v",
'Visoflux': "f_suprime_v",
'dVisoflux': "ferr_suprime_v",
'Raperflux': "f_ap_suprime_r",
'dRaperflux': "ferr_ap_suprime_r",
'Risoflux': "f_suprime_r",
'dRisoflux': "ferr_suprime_r",
'Iaperflux': "f_ap_suprime_i",
'dIaperflux': "ferr_ap_suprime_i",
'Iisoflux': "f_suprime_i",
'dIisoflux': "ferr_suprime_i",
'Zaperflux': "f_ap_suprime_z",
'dZaperflux': "ferr_ap_suprime_z",
'Zisoflux': "f_suprime_z",
'dZisoflux': "ferr_suprime_z",
'HKaperflux': "f_ap_quirc_hk",
'dHKaperflux': "ferr_ap_quirc_hk",
'HKisoflux': "f_quirc_hk",
'dHKisoflux': "ferr_quirc_hk"
})
imported_columns = OrderedDict({
'ID': "hawaii_id",
'RA': "hawaii_ra",
'DEC': "hawaii_dec",
'U': "m_ap_mosaic_u",
'dU': "merr_ap_mosaic_u",
'Uiso': "m_mosaic_u",
'dUiso': "merr_mosaic_u",
'B': "m_ap_suprime_b",
'dB': "merr_ap_suprime_b",
'Biso': "m_suprime_b",
'dBiso': "merr_suprime_b",
'V': "m_ap_suprime_v",
'dV': "merr_ap_suprime_v",
'Viso': "m_suprime_v",
'dViso': "merr_suprime_v",
'R': "m_ap_suprime_r",
'dR': "merr_ap_suprime_r",
'Riso': "m_suprime_r",
'dRiso': "merr_suprime_r",
'I': "m_ap_suprime_i",
'dI': "merr_ap_suprime_i",
'Iiso': "m_suprime_i",
'dIiso': "merr_suprime_i",
'Z': "m_ap_suprime_z",
'dZ': "merr_ap_suprime_z",
'Ziso': "m_suprime_z",
'dZiso': "merr_suprime_z",
'HK': "m_ap_quirc_hk",
'dHK': "merr_ap_quirc_hk",
'HKiso': "m_quirc_hk",
'dHKiso': "merr_quirc_hk"
})
catalogue = Table.read("../../dmu0/dmu0_Hawaii-HDFN/data/R.fits")[list(imported_columns)]
for column in imported_columns:
catalogue[column].name = imported_columns[column]
epoch = 2012 #Year of publication
# Clean table metadata
catalogue.meta = None
In [6]:
catalogue_z = Table.read("../../dmu0/dmu0_Hawaii-HDFN/data/Z.fits")[list(imported_columns)]
for column in imported_columns:
catalogue_z[column].name = imported_columns[column]
catalogue_z['hawaii_id'] = catalogue_z['hawaii_id'] + 1000000
# Clean table metadata
catalogue_z.meta = None
In [7]:
catalogue_z[:10].show_in_notebook()
Out[7]:
In [8]:
catalogue[:10].show_in_notebook()
Out[8]:
III - Merging¶
In [9]:
catalogue['hawaii_ra'].unit = u.deg
catalogue['hawaii_dec'].unit = u.deg
catalogue_z['hawaii_ra'].unit = u.deg
catalogue_z['hawaii_dec'].unit = u.deg
nb_merge_dist_plot(
SkyCoord(catalogue['hawaii_ra'], catalogue['hawaii_dec']),
SkyCoord(catalogue_z['hawaii_ra'], catalogue_z['hawaii_dec'])
)
The catalogues apper to contain no cross matches. We therefore simply stack the catalogues. We need to understand why this is the case. Have cross matches already been removed from the Z selected catalogue?
In [10]:
catalogue = vstack([catalogue, catalogue_z])
In [11]:
# Adding flux and band-flag columns
for col in catalogue.colnames:
if col.startswith('m_'):
errcol = "merr{}".format(col[1:])
mask = catalogue[col] < 0.
catalogue[col][mask] = np.nan
catalogue[errcol][mask] = np.nan
flux, error = mag_to_flux(np.array(catalogue[col]) , np.array(catalogue[errcol] ))
# magnitudes are added
catalogue.add_column(Column(flux*1.e6, name="f{}".format(col[1:])))
catalogue.add_column(Column(error*1.e6, name="f{}".format(errcol[1:])))
# Band-flag column
if "ap" not in col:
catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting.
In [12]:
catalogue[:10].show_in_notebook()
Out[12]:
II - Removal of duplicated sources¶
We remove duplicated objects from the input catalogues.
In [13]:
SORT_COLS = ['merr_mosaic_u',
'merr_suprime_b',
'merr_suprime_v',
'merr_suprime_r',
'merr_suprime_i',
'merr_suprime_z',
'merr_quirc_hk']
FLAG_NAME = 'hawaii_flag_cleaned'
nb_orig_sources = len(catalogue)
catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)
nb_sources = len(catalogue)
print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))
III - Astrometry correction¶
We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.
In [14]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_HDF-N.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [15]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL],
gaia_coords.ra, gaia_coords.dec)
In [16]:
delta_ra, delta_dec = astrometric_correction(
SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
gaia_coords
)
print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))
In [17]:
catalogue[RA_COL] = catalogue[RA_COL] + delta_ra.to(u.deg)
catalogue[DEC_COL] = catalogue[DEC_COL] + delta_dec.to(u.deg)
In [18]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL],
gaia_coords.ra, gaia_coords.dec)
IV - Flagging Gaia objects¶
In [19]:
catalogue.add_column(
gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [20]:
GAIA_FLAG_NAME = "hawaii_flag_gaia"
catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))
V - Flagging objects near bright stars¶
VI - Saving to disk¶
In [21]:
len(catalogue)
Out[21]:
In [22]:
catalogue[:10].show_in_notebook()
Out[22]:
In [23]:
catalogue.write("{}/Hawaii.fits".format(OUT_DIR), overwrite=True)