LSST Applications 27.0.0,g0265f82a02+469cd937ee,g02d81e74bb+21ad69e7e1,g1470d8bcf6+cbe83ee85a,g2079a07aa2+e67c6346a6,g212a7c68fe+04a9158687,g2305ad1205+94392ce272,g295015adf3+81dd352a9d,g2bbee38e9b+469cd937ee,g337abbeb29+469cd937ee,g3939d97d7f+72a9f7b576,g487adcacf7+71499e7cba,g50ff169b8f+5929b3527e,g52b1c1532d+a6fc98d2e7,g591dd9f2cf+df404f777f,g5a732f18d5+be83d3ecdb,g64a986408d+21ad69e7e1,g858d7b2824+21ad69e7e1,g8a8a8dda67+a6fc98d2e7,g99cad8db69+f62e5b0af5,g9ddcbc5298+d4bad12328,ga1e77700b3+9c366c4306,ga8c6da7877+71e4819109,gb0e22166c9+25ba2f69a1,gb6a65358fc+469cd937ee,gbb8dafda3b+69d3c0e320,gc07e1c2157+a98bf949bb,gc120e1dc64+615ec43309,gc28159a63d+469cd937ee,gcf0d15dbbd+72a9f7b576,gdaeeff99f8+a38ce5ea23,ge6526c86ff+3a7c1ac5f1,ge79ae78c31+469cd937ee,gee10cc3b42+a6fc98d2e7,gf1cff7945b+21ad69e7e1,gfbcc870c63+9a11dc8c8f
LSST Data Management Base Package
Loading...
Searching...
No Matches
read_curated_calibs.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["read_all"]
23
25from lsst.ip.isr import (Linearizer, CrosstalkCalib, Defects, BrighterFatterKernel, PhotodiodeCalib)
26
27import os
28import glob
29import dateutil.parser
30from deprecated.sphinx import deprecated
31
32
33def read_one_chip(root, chip_name, chip_id):
34 """Read data for a particular sensor from the standard format at a particular root.
35
36 Parameters
37 ----------
38 root : `str`
39 Path to the top level of the data tree. This is expected to hold directories
40 named after the sensor names. They are expected to be lower case.
41 chip_name : `str`
42 The name of the sensor for which to read data.
43 chip_id : `int`
44 The identifier for the sensor in question.
45
46 Returns
47 -------
48 `dict`
49 A dictionary of objects constructed from the appropriate factory class.
50 The key is the validity start time as a `datetime` object.
51 """
52 factory_map = {'qe_curve': Curve, 'defects': Defects, 'linearizer': Linearizer,
53 'crosstalk': CrosstalkCalib, 'bfk': BrighterFatterKernel,
54 'photodiode': PhotodiodeCalib, }
55 files = []
56 extensions = (".ecsv", ".yaml")
57 for ext in extensions:
58 files.extend(glob.glob(os.path.join(root, chip_name, f"*{ext}")))
59 parts = os.path.split(root)
60 instrument = os.path.split(parts[0])[1] # convention is that these reside at <instrument>/<data_name>
61 data_name = parts[1]
62 if data_name not in factory_map:
63 raise ValueError(f"Unknown calibration data type, '{data_name}' found. "
64 f"Only understand {','.join(k for k in factory_map)}")
65 factory = factory_map[data_name]
66 data_dict = {}
67 for f in files:
68 date_str = os.path.splitext(os.path.basename(f))[0]
69 valid_start = dateutil.parser.parse(date_str)
70 data_dict[valid_start] = factory.readText(f)
71 check_metadata(data_dict[valid_start], valid_start, instrument, chip_id, f, data_name)
72 return data_dict, data_name
73
74
75def check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name):
76 """Check that the metadata is complete and self consistent
77
78 Parameters
79 ----------
80 obj : object of same type as the factory
81 Object to retrieve metadata from in order to compare with
82 metadata inferred from the path.
83 valid_start : `datetime`
84 Start of the validity range for data
85 instrument : `str`
86 Name of the instrument in question
87 chip_id : `int`
88 Identifier of the sensor in question
89 filepath : `str`
90 Path of the file read to construct the data
91 data_name : `str`
92 Name of the type of data being read
93
94 Returns
95 -------
96 None
97
98 Raises
99 ------
100 ValueError
101 If the metadata from the path and the metadata encoded
102 in the path do not match for any reason.
103 """
104 md = obj.getMetadata()
105 finst = md['INSTRUME']
106 fchip_id = md['DETECTOR']
107 fdata_name = md['OBSTYPE']
108 if not ((finst.lower(), int(fchip_id), fdata_name.lower())
109 == (instrument.lower(), chip_id, data_name.lower())):
110 raise ValueError(f"Path and file metadata do not agree:\n"
111 f"Path metadata: {instrument} {chip_id} {data_name}\n"
112 f"File metadata: {finst} {fchip_id} {fdata_name}\n"
113 f"File read from : %s\n"%(filepath)
114 )
115
116
117@deprecated(reason="Curated calibration ingest now handled by obs_base Instrument classes."
118 " Will be removed after v25.0.",
119 version="v25.0", category=FutureWarning)
120def read_all(root, camera):
121 """Read all data from the standard format at a particular root.
122
123 Parameters
124 ----------
125 root : `str`
126 Path to the top level of the data tree. This is expected to hold directories
127 named after the sensor names. They are expected to be lower case.
128 camera : `lsst.afw.cameraGeom.Camera`
129 The camera that goes with the data being read.
130
131 Returns
132 -------
133 dict
134 A dictionary of dictionaries of objects constructed with the appropriate factory class.
135 The first key is the sensor name lowered, and the second is the validity
136 start time as a `datetime` object.
137
138 Notes
139 -----
140 Each leaf object in the constructed dictionary has metadata associated with it.
141 The detector ID may be retrieved from the DETECTOR entry of that metadata.
142 """
143 root = os.path.normpath(root)
144 dirs = os.listdir(root) # assumes all directories contain data
145 dirs = [d for d in dirs if os.path.isdir(os.path.join(root, d))]
146 data_by_chip = {}
147 name_map = {det.getName().lower(): det.getName() for
148 det in camera} # we assume the directories have been lowered
149
150 if not dirs:
151 raise RuntimeError(f"No data found on path {root}")
152
153 calib_types = set()
154 for d in dirs:
155 chip_name = os.path.basename(d)
156 # Give informative error message if the detector name is not known
157 # rather than a simple KeyError
158 if chip_name not in name_map:
159 detectors = [det for det in camera.getNameIter()]
160 max_detectors = 10
161 note_str = "knows"
162 if len(detectors) > max_detectors:
163 # report example subset
164 note_str = "examples"
165 detectors = detectors[:max_detectors]
166 raise RuntimeError(f"Detector {chip_name} not known to supplied camera "
167 f"{camera.getName()} ({note_str}: {','.join(detectors)})")
168 chip_id = camera[name_map[chip_name]].getId()
169 data_by_chip[chip_name], calib_type = read_one_chip(root, chip_name, chip_id)
170 calib_types.add(calib_type)
171 if len(calib_types) != 1: # set.add(None) has length 1 so None is OK here.
172 raise ValueError(f'Error mixing calib types: {calib_types}')
173
174 no_data = all([v == {} for v in data_by_chip.values()])
175 if no_data:
176 raise RuntimeError("No data to ingest")
177
178 return data_by_chip, calib_type
daf::base::PropertySet * set
Definition fits.cc:931
read_one_chip(root, chip_name, chip_id)
check_metadata(obj, valid_start, instrument, chip_id, filepath, data_name)