LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
read_curated_calibs.py
Go to the documentation of this file.
2 from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib)
3 
4 import os
5 import glob
6 import dateutil.parser
7 
8 
9 def read_one_chip(root, chip_name, chip_id):
10  """Read data for a particular sensor from the standard format at a particular root.
11 
12  Parameters
13  ----------
14  root : `str`
15  Path to the top level of the data tree. This is expected to hold directories
16  named after the sensor names. They are expected to be lower case.
17  chip_name : `str`
18  The name of the sensor for which to read data.
19  chip_id : `int`
20  The identifier for the sensor in question.
21 
22  Returns
23  -------
24  `dict`
25  A dictionary of objects constructed from the appropriate factory class.
26  The key is the validity start time as a `datetime` object.
27  """
28  factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer,
29  'crosstalk': CrosstalkCalib, 'bfk': BrighterFatterKernel,
30  'photodiode': PhotodiodeCalib, }
31  files = []
32  extensions = (".ecsv", ".yaml")
33  for ext in extensions:
34  files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
35  parts = os.path.split(root)
36  instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
37  data_name = parts[1]
38  if data_name not in factory_map:
39  raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
40  f"Only understand {','.join(k for k in factory_map)}")
41  factory = factory_map[data_name]
42  data_dict = {}
43  for f in files:
44  date_str = os.path.splitext(os.path.basename(f))[0]
45  valid_start = dateutil.parser.parse(date_str)
46  data_dict[valid_start] = factory.readText(f)
47  check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
48  return data_dict, data_name
49 
50 
51 def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
52  """Check that the metadata is complete and self consistent
53 
54  Parameters
55  ----------
56  obj : object of same type as the factory
57  Object to retrieve metadata from in order to compare with
58  metadata inferred from the path.
59  valid_start : `datetime`
60  Start of the validity range for data
61  instrument : `str`
62  Name of the instrument in question
63  chip_id : `int`
64  Identifier of the sensor in question
65  filepath : `str`
66  Path of the file read to construct the data
67  data_name : `str`
68  Name of the type of data being read
69 
70  Returns
71  -------
72  None
73 
74  Raises
75  ------
76  ValueError
77  If the metadata from the path and the metadata encoded
78  in the path do not match for any reason.
79  """
80  md = obj.getMetadata()
81  finst = md['INSTRUME']
82  fchip_id = md['DETECTOR']
83  fdata_name = md['OBSTYPE']
84  if not ((finst.lower(), int(fchip_id), fdata_name.lower())
85  == (instrument.lower(), chip_id, data_name.lower())):
86  raise ValueError(f"Path and file metadata do not agree:\n"
87  f"Path metadata: {instrument} {chip_id} {data_name}\n"
88  f"File metadata: {finst} {fchip_id} {fdata_name}\n"
89  f"File read from : %s\n"%(filepath)
90  )
91 
92 
93 def read_all(root, camera):
94  """Read all data from the standard format at a particular root.
95 
96  Parameters
97  ----------
98  root : `str`
99  Path to the top level of the data tree. This is expected to hold directories
100  named after the sensor names. They are expected to be lower case.
101  camera : `lsst.afw.cameraGeom.Camera`
102  The camera that goes with the data being read.
103 
104  Returns
105  -------
106  dict
107  A dictionary of dictionaries of objects constructed with the appropriate factory class.
108  The first key is the sensor name lowered, and the second is the validity
109  start time as a `datetime` object.
110 
111  Notes
112  -----
113  Each leaf object in the constructed dictionary has metadata associated with it.
114  The detector ID may be retrieved from the DETECTOR entry of that metadata.
115  """
116  root = os.path.normpath(root)
117  dirs = os.listdir(root) # assumes all directories contain data
118  dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
119  data_by_chip = {}
120  name_map = {det.getName().lower(): det.getName() for
121  det in camera} # we assume the directories have been lowered
122 
123  if not dirs:
124  raise RuntimeError(f"No data found on path {root}")
125 
126  calib_types = set()
127  for d in dirs:
128  chip_name = os.path.basename(d)
129  # Give informative error message if the detector name is not known
130  # rather than a simple KeyError
131  if chip_name not in name_map:
132  detectors = [det for det in camera.getNameIter()]
133  max_detectors = 10
134  note_str = "knows"
135  if len(detectors) > max_detectors:
136  # report example subset
137  note_str = "examples"
138  detectors = detectors[:max_detectors]
139  raise RuntimeError(f"Detector {chip_name} not known to supplied camera "
140  f"{camera.getName()} ({note_str}: {','.join(detectors)})")
141  chip_id = camera[name_map[chip_name]].getId()
142  data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
143  calib_types.add(calib_type)
144  if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
145  raise ValueError(f'Error mixing calib types: {calib_types}')
146 
147  no_data = all([v == {} for v in data_by_chip.values()])
148  if no_data:
149  raise RuntimeError("No data to ingest")
150 
151  return data_by_chip, calib_type
daf::base::PropertySet * set
Definition: fits.cc:912
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)
def read_one_chip(root, chip_name, chip_id)