LSST Applications g1cfbe01172+01aa18f939,g20cdd03214+31e6b93548,g28da252d5a+ea8665a95b,g2bbee38e9b+9ec6cc348d,g2bc492864f+9ec6cc348d,g347aa1857d+9ec6cc348d,g3a166c0a6a+9ec6cc348d,g4322eb9e3a+65eff1e020,g461a3dce89+b86e4b8053,g50ff169b8f+f991eae79d,g52b1c1532d+b86e4b8053,g607f77f49a+31e6b93548,g78056777b3+8ae2798781,g858d7b2824+31e6b93548,g8cd86fa7b1+4851e61ca4,g9ddcbc5298+f24b38b85a,ga1e77700b3+3309dba821,gae0086650b+b86e4b8053,gb0e22166c9+6076c0b52b,gbb886bcc26+dccb771098,gbd462c55f0+dc07f8e65d,gc0c51c7ec2+31e6b93548,gc120e1dc64+a417ce3171,gc28159a63d+9ec6cc348d,gc2a6998b7e+f95f64aeae,gcdd4ae20e8+507450c4cd,gcf0d15dbbd+507450c4cd,gd1535ee943+bcf88ba65f,gd598c5cd71+66126f91fb,gdaeeff99f8+006e14e809,gdbce86181e+39d5515b1a,ge3d4d395c2+b12d4d6a95,ge79ae78c31+9ec6cc348d,gf048a9a2f4+d9c36e6b63,gfbcc870c63+ea41c4420b,w.2024.27
LSST Data Management Base Package
Loading...
Searching...
No Matches
calibType.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ["IsrCalib", "IsrProvenance"]
22
23import abc
24import datetime
25import logging
26import os.path
27import warnings
28import yaml
29import numpy as np
30
31from astropy.table import Table
32from astropy.io import fits
33
34from lsst.daf.base import PropertyList
35from lsst.utils.introspection import get_full_type_name
36from lsst.utils import doImport
37
38
39class IsrCalib(abc.ABC):
40 """Generic calibration type.
41
42 Subclasses must implement the toDict, fromDict, toTable, fromTable
43 methods that allow the calibration information to be converted
44 from dictionaries and afw tables. This will allow the calibration
45 to be persisted using the base class read/write methods.
46
47 The validate method is intended to provide a common way to check
48 that the calibration is valid (internally consistent) and
49 appropriate (usable with the intended data). The apply method is
50 intended to allow the calibration to be applied in a consistent
51 manner.
52
53 Parameters
54 ----------
55 camera : `lsst.afw.cameraGeom.Camera`, optional
56 Camera to extract metadata from.
57 detector : `lsst.afw.cameraGeom.Detector`, optional
58 Detector to extract metadata from.
59 log : `logging.Logger`, optional
60 Log for messages.
61 """
62 _OBSTYPE = "generic"
63 _SCHEMA = "NO SCHEMA"
64 _VERSION = 0
65
66 def __init__(self, camera=None, detector=None, log=None, **kwargs):
67 self._instrument = None
68 self._raftName = None
69 self._slotName = None
70 self._detectorName = None
71 self._detectorSerial = None
72 self._detectorId = None
73 self._filter = None
74 self._calibId = None
75 self._seqfile = None
76 self._seqname = None
77 self._seqcksum = None
80 self.calibInfoFromDict(kwargs)
81
82 # Define the required attributes for this calibration. These
83 # are entries that are automatically filled and propagated
84 # from the metadata. The existence of the attribute is
85 # required (they're checked for equivalence), but they do not
86 # necessarily need to have a value (None == None in this
87 # case).
88 self.requiredAttributesrequiredAttributesrequiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"])
89 self.requiredAttributesrequiredAttributesrequiredAttributes.update(["_instrument", "_raftName", "_slotName",
90 "_detectorName", "_detectorSerial", "_detectorId",
91 "_filter", "_calibId", "_seqfile", "_seqname", "_seqcksum",
92 "_metadata"])
93
94 self.log = log if log else logging.getLogger(__name__)
95
96 if detector:
97 self.fromDetector(detector)
98 self.updateMetadata(camera=camera, detector=detector)
99
100 def __str__(self):
101 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
102
103 def __eq__(self, other):
104 """Calibration equivalence.
105
106 Running ``calib.log.setLevel(0)`` enables debug statements to
107 identify problematic fields.
108 """
109 if not isinstance(other, self.__class__):
110 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__)
111 return False
112
113 for attr in self._requiredAttributes:
114 attrSelf = getattr(self, attr)
115 attrOther = getattr(other, attr)
116
117 if isinstance(attrSelf, dict):
118 # Dictionary of arrays.
119 if attrSelf.keys() != attrOther.keys():
120 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
121 return False
122 for key in attrSelf:
123 try:
124 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True):
125 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
126 return False
127 except TypeError:
128 # If it is not something numpy can handle
129 # (it's not a number or array of numbers),
130 # then it needs to have its own equivalence
131 # operator.
132 if attrSelf[key] != attrOther[key]:
133 return False
134 elif isinstance(attrSelf, np.ndarray):
135 # Bare array.
136 if not np.allclose(attrSelf, attrOther, equal_nan=True):
137 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther)
138 return False
139 elif type(attrSelf) != type(attrOther):
140 if set([attrSelf, attrOther]) == set([None, ""]):
141 # Fits converts None to "", but None is not "".
142 continue
143 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
144 attrSelf, attrOther)
145 return False
146 else:
147 if attrSelf != attrOther:
148 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther)
149 return False
150
151 return True
152
153 @property
155 return self._requiredAttributes
156
157 @requiredAttributes.setter
158 def requiredAttributes(self, value):
160
161 def getMetadata(self):
162 """Retrieve metadata associated with this calibration.
163
164 Returns
165 -------
166 meta : `lsst.daf.base.PropertyList`
167 Metadata. The returned `~lsst.daf.base.PropertyList` can be
168 modified by the caller and the changes will be written to
169 external files.
170 """
171 return self._metadata
172
173 def setMetadata(self, metadata):
174 """Store a copy of the supplied metadata with this calibration.
175
176 Parameters
177 ----------
178 metadata : `lsst.daf.base.PropertyList`
179 Metadata to associate with the calibration. Will be copied and
180 overwrite existing metadata.
181 """
182 if metadata is not None:
183 self._metadata.update(metadata)
184
185 # Ensure that we have the obs type required by calibration ingest
186 self._metadata["OBSTYPE"] = self._OBSTYPE
187 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
188 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
189
190 if isinstance(metadata, dict):
191 self.calibInfoFromDict(metadata)
192 elif isinstance(metadata, PropertyList):
193 self.calibInfoFromDict(metadata.toDict())
194
195 def updateMetadata(self, camera=None, detector=None, filterName=None,
196 setCalibId=False, setCalibInfo=False, setDate=False,
197 **kwargs):
198 """Update metadata keywords with new values.
199
200 Parameters
201 ----------
202 camera : `lsst.afw.cameraGeom.Camera`, optional
203 Reference camera to use to set ``_instrument`` field.
204 detector : `lsst.afw.cameraGeom.Detector`, optional
205 Reference detector to use to set ``_detector*`` fields.
206 filterName : `str`, optional
207 Filter name to assign to this calibration.
208 setCalibId : `bool`, optional
209 Construct the ``_calibId`` field from other fields.
210 setCalibInfo : `bool`, optional
211 Set calibration parameters from metadata.
212 setDate : `bool`, optional
213 Ensure the metadata ``CALIBDATE`` fields are set to the current
214 datetime.
215 kwargs : `dict` or `collections.abc.Mapping`, optional
216 Set of ``key=value`` pairs to assign to the metadata.
217 """
218 mdOriginal = self.getMetadata()
219 mdSupplemental = dict()
220
221 for k, v in kwargs.items():
222 if isinstance(v, fits.card.Undefined):
223 kwargs[k] = None
224
225 if setCalibInfo:
226 self.calibInfoFromDict(kwargs)
227
228 if camera:
229 self._instrument = camera.getName()
230
231 if detector:
232 self._detectorName = detector.getName()
233 self._detectorSerial = detector.getSerial()
234 self._detectorId = detector.getId()
235 if "_" in self._detectorName:
236 (self._raftName, self._slotName) = self._detectorName.split("_")
237
238 if filterName:
239 # TOD0 DM-28093: I think this whole comment can go away, if we
240 # always use physicalLabel everywhere in ip_isr.
241 # If set via:
242 # exposure.getInfo().getFilter().getName()
243 # then this will hold the abstract filter.
244 self._filter = filterName
245
246 if setDate:
247 date = datetime.datetime.now()
248 mdSupplemental["CALIBDATE"] = date.isoformat()
249 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat()
250 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat()
251
252 if setCalibId:
253 values = []
254 values.append(f"instrument={self._instrument}") if self._instrument else None
255 values.append(f"raftName={self._raftName}") if self._raftName else None
256 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
257 values.append(f"detector={self._detectorId}") if self._detectorId else None
258 values.append(f"filter={self._filter}") if self._filter else None
259
260 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None))
261 values.append(f"calibDate={calibDate}") if calibDate else None
262
263 self._calibId = " ".join(values)
264
265 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
266 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
267 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
268 self._metadata["DETECTOR"] = self._detectorId
269 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
270 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
271 self._metadata["FILTER"] = self._filter if self._filter else None
272 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
273 self._metadata["SEQFILE"] = self._seqfile if self._seqfile else None
274 self._metadata["SEQNAME"] = self._seqname if self._seqname else None
275 self._metadata["SEQCKSUM"] = self._seqcksum if self._seqcksum else None
276 self._metadata["CALIBCLS"] = get_full_type_name(self)
277
278 mdSupplemental.update(kwargs)
279 mdOriginal.update(mdSupplemental)
280
281 def updateMetadataFromExposures(self, exposures):
282 """Extract and unify metadata information.
283
284 Parameters
285 ----------
286 exposures : `list`
287 Exposures or other calibrations to scan.
288 """
289 # This list of keywords is the set of header entries that
290 # should be checked and propagated. Not having an entry is
291 # not a failure, as they may not be defined for the exposures
292 # being used.
293 keywords = ["SEQNAME", "SEQFILE", "SEQCKSUM", "ODP", "AP0_RC"]
294 metadata = {}
295
296 for exp in exposures:
297 try:
298 expMeta = exp.getMetadata()
299 except AttributeError:
300 continue
301 for key in keywords:
302 if key in expMeta:
303 if key in metadata:
304 if metadata[key] != expMeta[key]:
305 self.log.warning("Metadata mismatch! Have: %s Found %s",
306 metadata[key], expMeta[key])
307 else:
308 metadata[key] = expMeta[key]
309 self.updateMetadata(**metadata)
310
311 def calibInfoFromDict(self, dictionary):
312 """Handle common keywords.
313
314 This isn't an ideal solution, but until all calibrations
315 expect to find everything in the metadata, they still need to
316 search through dictionaries.
317
318 Parameters
319 ----------
320 dictionary : `dict` or `lsst.daf.base.PropertyList`
321 Source for the common keywords.
322
323 Raises
324 ------
325 RuntimeError
326 Raised if the dictionary does not match the expected OBSTYPE.
327 """
328
329 def search(haystack, needles):
330 """Search dictionary 'haystack' for an entry in 'needles'
331 """
332 test = [haystack.get(x) for x in needles]
333 test = set([x for x in test if x is not None])
334 if len(test) == 0:
335 if "metadata" in haystack:
336 return search(haystack["metadata"], needles)
337 else:
338 return None
339 elif len(test) == 1:
340 value = list(test)[0]
341 if value == "":
342 return None
343 else:
344 return value
345 else:
346 raise ValueError(f"Too many values found: {len(test)} {test} {needles}")
347
348 if "metadata" in dictionary:
349 metadata = dictionary["metadata"]
350
351 if self._OBSTYPE != metadata["OBSTYPE"]:
352 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
353 f"found {metadata['OBSTYPE']}")
354
355 self._instrument = search(dictionary, ["INSTRUME", "instrument"])
356 self._raftName = search(dictionary, ["RAFTNAME"])
357 self._slotName = search(dictionary, ["SLOTNAME"])
358 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"])
359 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"])
360 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"])
361 self._filter = search(dictionary, ["FILTER", "filterName"])
362 self._calibId = search(dictionary, ["CALIB_ID"])
363 self._seqfile = search(dictionary, ["SEQFILE"])
364 self._seqname = search(dictionary, ["SEQNAME"])
365 self._seqcksum = search(dictionary, ["SEQCKSUM"])
366
367 @classmethod
368 def determineCalibClass(cls, metadata, message):
369 """Attempt to find calibration class in metadata.
370
371 Parameters
372 ----------
373 metadata : `dict` or `lsst.daf.base.PropertyList`
374 Metadata possibly containing a calibration class entry.
375 message : `str`
376 Message to include in any errors.
377
378 Returns
379 -------
380 calibClass : `object`
381 The class to use to read the file contents. Should be an
382 `lsst.ip.isr.IsrCalib` subclass.
383
384 Raises
385 ------
386 ValueError
387 Raised if the resulting calibClass is the base
388 `lsst.ip.isr.IsrClass` (which does not implement the
389 content methods).
390 """
391 calibClassName = metadata.get("CALIBCLS")
392 calibClass = doImport(calibClassName) if calibClassName is not None else cls
393 if calibClass is IsrCalib:
394 raise ValueError(f"Cannot use base class to read calibration data: {message}")
395 return calibClass
396
397 @classmethod
398 def readText(cls, filename, **kwargs):
399 """Read calibration representation from a yaml/ecsv file.
400
401 Parameters
402 ----------
403 filename : `str`
404 Name of the file containing the calibration definition.
405 kwargs : `dict` or collections.abc.Mapping`, optional
406 Set of key=value pairs to pass to the ``fromDict`` or
407 ``fromTable`` methods.
408
409 Returns
410 -------
411 calib : `~lsst.ip.isr.IsrCalibType`
412 Calibration class.
413
414 Raises
415 ------
416 RuntimeError
417 Raised if the filename does not end in ".ecsv" or ".yaml".
418 """
419 if filename.endswith((".ecsv", ".ECSV")):
420 data = Table.read(filename, format="ascii.ecsv")
421 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV")
422 return calibClass.fromTable([data], **kwargs)
423 elif filename.endswith((".yaml", ".YAML")):
424 with open(filename, "r") as f:
425 data = yaml.load(f, Loader=yaml.CLoader)
426 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML")
427 return calibClass.fromDict(data, **kwargs)
428 else:
429 raise RuntimeError(f"Unknown filename extension: {filename}")
430
431 def writeText(self, filename, format="auto"):
432 """Write the calibration data to a text file.
433
434 Parameters
435 ----------
436 filename : `str`
437 Name of the file to write.
438 format : `str`
439 Format to write the file as. Supported values are:
440 ``"auto"`` : Determine filetype from filename.
441 ``"yaml"`` : Write as yaml.
442 ``"ecsv"`` : Write as ecsv.
443
444 Returns
445 -------
446 used : `str`
447 The name of the file used to write the data. This may
448 differ from the input if the format is explicitly chosen.
449
450 Raises
451 ------
452 RuntimeError
453 Raised if filename does not end in a known extension, or
454 if all information cannot be written.
455
456 Notes
457 -----
458 The file is written to YAML/ECSV format and will include any
459 associated metadata.
460 """
461 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))):
462 outDict = self.toDict()
463 path, ext = os.path.splitext(filename)
464 filename = path + ".yaml"
465 with open(filename, "w") as f:
466 yaml.dump(outDict, f)
467 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))):
468 tableList = self.toTable()
469 if len(tableList) > 1:
470 # ECSV doesn't support multiple tables per file, so we
471 # can only write the first table.
472 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
473
474 table = tableList[0]
475 path, ext = os.path.splitext(filename)
476 filename = path + ".ecsv"
477 table.write(filename, format="ascii.ecsv")
478 else:
479 raise RuntimeError(f"Attempt to write to a file {filename} "
480 "that does not end in '.yaml' or '.ecsv'")
481
482 return filename
483
484 @classmethod
485 def readFits(cls, filename, **kwargs):
486 """Read calibration data from a FITS file.
487
488 Parameters
489 ----------
490 filename : `str`
491 Filename to read data from.
492 kwargs : `dict` or collections.abc.Mapping`, optional
493 Set of key=value pairs to pass to the ``fromTable``
494 method.
495
496 Returns
497 -------
498 calib : `lsst.ip.isr.IsrCalib`
499 Calibration contained within the file.
500 """
501 tableList = []
502 tableList.append(Table.read(filename, hdu=1, mask_invalid=False))
503 extNum = 2 # Fits indices start at 1, we've read one already.
504 keepTrying = True
505
506 while keepTrying:
507 with warnings.catch_warnings():
508 warnings.simplefilter("error")
509 try:
510 newTable = Table.read(filename, hdu=extNum, mask_invalid=False)
511 tableList.append(newTable)
512 extNum += 1
513 except Exception:
514 keepTrying = False
515
516 for table in tableList:
517 for k, v in table.meta.items():
518 if isinstance(v, fits.card.Undefined):
519 table.meta[k] = None
520
521 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits")
522 return calibClass.fromTable(tableList, **kwargs)
523
524 def writeFits(self, filename):
525 """Write calibration data to a FITS file.
526
527 Parameters
528 ----------
529 filename : `str`
530 Filename to write data to.
531
532 Returns
533 -------
534 used : `str`
535 The name of the file used to write the data.
536 """
537 tableList = self.toTable()
538 with warnings.catch_warnings():
539 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
540 astropyList = [fits.table_to_hdu(table) for table in tableList]
541 astropyList.insert(0, fits.PrimaryHDU())
542
543 writer = fits.HDUList(astropyList)
544 writer.writeto(filename, overwrite=True)
545 return filename
546
547 def fromDetector(self, detector):
548 """Modify the calibration parameters to match the supplied detector.
549
550 Parameters
551 ----------
552 detector : `lsst.afw.cameraGeom.Detector`
553 Detector to use to set parameters from.
554
555 Raises
556 ------
557 NotImplementedError
558 Raised if not implemented by a subclass.
559 This needs to be implemented by subclasses for each
560 calibration type.
561 """
562 raise NotImplementedError("Must be implemented by subclass.")
563
564 @classmethod
565 def fromDict(cls, dictionary, **kwargs):
566 """Construct a calibration from a dictionary of properties.
567
568 Must be implemented by the specific calibration subclasses.
569
570 Parameters
571 ----------
572 dictionary : `dict`
573 Dictionary of properties.
574 kwargs : `dict` or collections.abc.Mapping`, optional
575 Set of key=value options.
576
577 Returns
578 -------
579 calib : `lsst.ip.isr.CalibType`
580 Constructed calibration.
581
582 Raises
583 ------
584 NotImplementedError
585 Raised if not implemented.
586 """
587 raise NotImplementedError("Must be implemented by subclass.")
588
589 def toDict(self):
590 """Return a dictionary containing the calibration properties.
591
592 The dictionary should be able to be round-tripped through
593 `fromDict`.
594
595 Returns
596 -------
597 dictionary : `dict`
598 Dictionary of properties.
599
600 Raises
601 ------
602 NotImplementedError
603 Raised if not implemented.
604 """
605 raise NotImplementedError("Must be implemented by subclass.")
606
607 @classmethod
608 def fromTable(cls, tableList, **kwargs):
609 """Construct a calibration from a dictionary of properties.
610
611 Must be implemented by the specific calibration subclasses.
612
613 Parameters
614 ----------
615 tableList : `list` [`lsst.afw.table.Table`]
616 List of tables of properties.
617 kwargs : `dict` or collections.abc.Mapping`, optional
618 Set of key=value options.
619
620 Returns
621 -------
622 calib : `lsst.ip.isr.CalibType`
623 Constructed calibration.
624
625 Raises
626 ------
627 NotImplementedError
628 Raised if not implemented.
629 """
630 raise NotImplementedError("Must be implemented by subclass.")
631
632 def toTable(self):
633 """Return a list of tables containing the calibration properties.
634
635 The table list should be able to be round-tripped through
636 `fromDict`.
637
638 Returns
639 -------
640 tableList : `list` [`lsst.afw.table.Table`]
641 List of tables of properties.
642
643 Raises
644 ------
645 NotImplementedError
646 Raised if not implemented.
647 """
648 raise NotImplementedError("Must be implemented by subclass.")
649
650 def validate(self, other=None):
651 """Validate that this calibration is defined and can be used.
652
653 Parameters
654 ----------
655 other : `object`, optional
656 Thing to validate against.
657
658 Returns
659 -------
660 valid : `bool`
661 Returns true if the calibration is valid and appropriate.
662 """
663 return False
664
665 def apply(self, target):
666 """Method to apply the calibration to the target object.
667
668 Parameters
669 ----------
670 target : `object`
671 Thing to validate against.
672
673 Returns
674 -------
675 valid : `bool`
676 Returns true if the calibration was applied correctly.
677
678 Raises
679 ------
680 NotImplementedError
681 Raised if not implemented.
682 """
683 raise NotImplementedError("Must be implemented by subclass.")
684
685
687 """Class for the provenance of data used to construct calibration.
688
689 Provenance is not really a calibration, but we would like to
690 record this when constructing the calibration, and it provides an
691 example of the base calibration class.
692
693 Parameters
694 ----------
695 instrument : `str`, optional
696 Name of the instrument the data was taken with.
697 calibType : `str`, optional
698 Type of calibration this provenance was generated for.
699 detectorName : `str`, optional
700 Name of the detector this calibration is for.
701 detectorSerial : `str`, optional
702 Identifier for the detector.
703
704 """
705 _OBSTYPE = "IsrProvenance"
706
707 def __init__(self, calibType="unknown",
708 **kwargs):
709 self.calibType = calibType
710 self.dimensions = set()
711 self.dataIdList = list()
712
713 super().__init__(**kwargs)
714
715 self.requiredAttributesrequiredAttributesrequiredAttributes.update(["calibType", "dimensions", "dataIdList"])
716
717 def __str__(self):
718 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
719
720 def __eq__(self, other):
721 return super().__eq__(other)
722
723 def updateMetadata(self, setDate=False, **kwargs):
724 """Update calibration metadata.
725
726 Parameters
727 ----------
728 setDate : `bool`, optional
729 Update the ``CALIBDATE`` fields in the metadata to the current
730 time. Defaults to False.
731 kwargs : `dict` or `collections.abc.Mapping`, optional
732 Other keyword parameters to set in the metadata.
733 """
734 kwargs["calibType"] = self.calibType
735 super().updateMetadata(setDate=setDate, **kwargs)
736
737 def fromDataIds(self, dataIdList):
738 """Update provenance from dataId List.
739
740 Parameters
741 ----------
742 dataIdList : `list` [`lsst.daf.butler.DataId`]
743 List of dataIds used in generating this calibration.
744 """
745 for dataId in dataIdList:
746 for key in dataId:
747 if key not in self.dimensions:
748 self.dimensions.add(key)
749 self.dataIdList.append(dataId)
750
751 @classmethod
752 def fromTable(cls, tableList):
753 """Construct provenance from table list.
754
755 Parameters
756 ----------
757 tableList : `list` [`lsst.afw.table.Table`]
758 List of tables to construct the provenance from.
759
760 Returns
761 -------
762 provenance : `lsst.ip.isr.IsrProvenance`
763 The provenance defined in the tables.
764 """
765 table = tableList[0]
766 metadata = table.meta
767 inDict = dict()
768 inDict["metadata"] = metadata
769 inDict["calibType"] = metadata["calibType"]
770 inDict["dimensions"] = set()
771 inDict["dataIdList"] = list()
772
773 schema = dict()
774 for colName in table.columns:
775 schema[colName.lower()] = colName
776 inDict["dimensions"].add(colName.lower())
777 inDict["dimensions"] = sorted(inDict["dimensions"])
778
779 for row in table:
780 entry = dict()
781 for dim in sorted(inDict["dimensions"]):
782 entry[dim] = row[schema[dim]]
783 inDict["dataIdList"].append(entry)
784
785 return cls.fromDictfromDict(inDict)
786
787 @classmethod
788 def fromDict(cls, dictionary):
789 """Construct provenance from a dictionary.
790
791 Parameters
792 ----------
793 dictionary : `dict`
794 Dictionary of provenance parameters.
795
796 Returns
797 -------
798 provenance : `lsst.ip.isr.IsrProvenance`
799 The provenance defined in the tables.
800 """
801 calib = cls()
802 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]:
803 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
804 f"found {dictionary['metadata']['OBSTYPE']}")
805 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"])
806
807 # These properties should be in the metadata, but occasionally
808 # are found in the dictionary itself. Check both places,
809 # ending with `None` if neither contains the information.
810 calib.calibType = dictionary["calibType"]
811 calib.dimensions = set(dictionary["dimensions"])
812 calib.dataIdList = dictionary["dataIdList"]
813
814 calib.updateMetadata()
815 return calib
816
817 def toDict(self):
818 """Return a dictionary containing the provenance information.
819
820 Returns
821 -------
822 dictionary : `dict`
823 Dictionary of provenance.
824 """
826
827 outDict = {}
828
829 metadata = self.getMetadata()
830 outDict["metadata"] = metadata
831 outDict["detectorName"] = self._detectorName
832 outDict["detectorSerial"] = self._detectorSerial
833 outDict["detectorId"] = self._detectorId
834 outDict["instrument"] = self._instrument
835 outDict["calibType"] = self.calibType
836 outDict["dimensions"] = list(self.dimensions)
837 outDict["dataIdList"] = self.dataIdList
838
839 return outDict
840
841 def toTable(self):
842 """Return a list of tables containing the provenance.
843
844 This seems inefficient and slow, so this may not be the best
845 way to store the data.
846
847 Returns
848 -------
849 tableList : `list` [`lsst.afw.table.Table`]
850 List of tables containing the provenance information
851 """
852 tableList = []
853 self.updateMetadataupdateMetadata(setDate=True, setCalibInfo=True)
854
855 catalog = Table(rows=self.dataIdList,
856 names=self.dimensions)
857 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
858 catalog.meta = filteredMetadata
859 tableList.append(catalog)
860 return tableList
std::vector< SchemaItem< Flag > > * items
Tag types used to declare specialized field types.
Definition misc.h:31
Class for storing ordered metadata with comments.
fromDict(cls, dictionary, **kwargs)
Definition calibType.py:565
writeText(self, filename, format="auto")
Definition calibType.py:431
readText(cls, filename, **kwargs)
Definition calibType.py:398
calibInfoFromDict(self, dictionary)
Definition calibType.py:311
updateMetadataFromExposures(self, exposures)
Definition calibType.py:281
determineCalibClass(cls, metadata, message)
Definition calibType.py:368
__init__(self, camera=None, detector=None, log=None, **kwargs)
Definition calibType.py:66
updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition calibType.py:197
fromTable(cls, tableList, **kwargs)
Definition calibType.py:608
readFits(cls, filename, **kwargs)
Definition calibType.py:485
__init__(self, calibType="unknown", **kwargs)
Definition calibType.py:708
updateMetadata(self, setDate=False, **kwargs)
Definition calibType.py:723