LSSTApplications  20.0.0
LSSTDataManagementBasePackage
read_curated_calibs.py
Go to the documentation of this file.
1 from lsst.meas.algorithms import Defects
2 from lsst.meas.algorithms.simple_curve import Curve
3 from lsst.ip.isr import Linearizer
4 
5 import os
6 import glob
7 import dateutil.parser
8 
9 
10 def read_one_chip(root, chip_name, chip_id):
11  """Read data for a particular sensor from the standard format at a particular root.
12 
13  Parameters
14  ----------
15  root : `str`
16  Path to the top level of the data tree. This is expected to hold directories
17  named after the sensor names. They are expected to be lower case.
18  chip_name : `str`
19  The name of the sensor for which to read data.
20  chip_id : `int`
21  The identifier for the sensor in question.
22 
23  Returns
24  -------
25  `dict`
26  A dictionary of objects constructed from the appropriate factory class.
27  The key is the validity start time as a `datetime` object.
28  """
29  factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer}
30  files = []
31  extensions = (".ecsv", ".yaml")
32  for ext in extensions:
33  files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
34  parts = os.path.split(root)
35  instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
36  data_name = parts[1]
37  if data_name not in factory_map:
38  raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
39  f"Only understand {','.join(k for k in factory_map)}")
40  factory = factory_map[data_name]
41  data_dict = {}
42  for f in files:
43  date_str = os.path.splitext(os.path.basename(f))[0]
44  valid_start = dateutil.parser.parse(date_str)
45  data_dict[valid_start] = factory.readText(f)
46  check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
47  return data_dict, data_name
48 
49 
50 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
51  """Check that the metadata is complete and self consistent
52 
53  Parameters
54  ----------
55  obj : object of same type as the factory
56  Object to retrieve metadata from in order to compare with
57  metadata inferred from the path.
58  valid_start : `datetime`
59  Start of the validity range for data
60  instrument : `str`
61  Name of the instrument in question
62  chip_id : `int`
63  Identifier of the sensor in question
64  filepath : `str`
65  Path of the file read to construct the data
66  data_name : `str`
67  Name of the type of data being read
68 
69  Returns
70  -------
71  None
72 
73  Raises
74  ------
75  ValueError
76  If the metadata from the path and the metadata encoded
77  in the path do not match for any reason.
78  """
79  md = obj.getMetadata()
80  finst = md['INSTRUME']
81  fchip_id = md['DETECTOR']
82  fdata_name = md['OBSTYPE']
83  if not ((finst.lower(), int(fchip_id), fdata_name)
84  == (instrument.lower(), chip_id, data_name)):
85  raise ValueError(f"Path and file metadata do not agree:\n"
86  f"Path metadata: {instrument} {chip_id} {data_name}\n"
87  f"File metadata: {finst} {fchip_id} {fdata_name}\n"
88  f"File read from : %s\n"%(filepath)
89  )
90 
91 
92 def read_all(root, camera):
93  """Read all data from the standard format at a particular root.
94 
95  Parameters
96  ----------
97  root : `str`
98  Path to the top level of the data tree. This is expected to hold directories
99  named after the sensor names. They are expected to be lower case.
100  camera : `lsst.afw.cameraGeom.Camera`
101  The camera that goes with the data being read.
102 
103  Returns
104  -------
105  dict
106  A dictionary of dictionaries of objects constructed with the appropriate factory class.
107  The first key is the sensor name lowered, and the second is the validity
108  start time as a `datetime` object.
109 
110  Notes
111  -----
112  Each leaf object in the constructed dictionary has metadata associated with it.
113  The detector ID may be retrieved from the DETECTOR entry of that metadata.
114  """
115  root = os.path.normpath(root)
116  dirs = os.listdir(root) # assumes all directories contain data
117  dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
118  data_by_chip = {}
119  name_map = {det.getName().lower(): det.getName() for
120  det in camera} # we assume the directories have been lowered
121 
122  if not dirs:
123  raise RuntimeError(f"No data found on path {root}")
124 
125  calib_types = set()
126  for d in dirs:
127  chip_name = os.path.basename(d)
128  chip_id = camera[name_map[chip_name]].getId()
129  data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
130  calib_types.add(calib_type)
131  if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
132  raise ValueError(f'Error mixing calib types: {calib_types}')
133 
134  no_data = all([v == {} for v in data_by_chip.values()])
135  if no_data:
136  raise RuntimeError(f'No data to ingest')
137 
138  return data_by_chip, calib_type
lsst::meas::algorithms.simple_curve
Definition: simple_curve.py:1
lsst::geom::all
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
Definition: CoordinateExpr.h:81
lsst.pipe.tasks.read_curated_calibs.read_all
def read_all(root, camera)
Definition: read_curated_calibs.py:92
lsst::ip::isr
Definition: applyLookupTable.h:34
lsst::meas::algorithms
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations.
Definition: CoaddBoundedField.h:34
set
daf::base::PropertySet * set
Definition: fits.cc:912
lsst.pipe.tasks.read_curated_calibs.check_metadata
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
Definition: read_curated_calibs.py:50
lsst.pipe.tasks.read_curated_calibs.read_one_chip
def read_one_chip(root, chip_name, chip_id)
Definition: read_curated_calibs.py:10