LSST Applications g063fba187b+66a50001ff,g0f08755f38+1a22dc2551,g1653933729+a8ce1bb630,g168dd56ebc+a8ce1bb630,g217e2c1bcf+12e87a5bd8,g246886dfd9+466c7b6c06,g28da252d5a+858b171e04,g2bbee38e9b+404b60ec9b,g2bc492864f+404b60ec9b,g3156d2b45e+6e55a43351,g347aa1857d+404b60ec9b,g35bb328faa+a8ce1bb630,g3a166c0a6a+404b60ec9b,g3e281a1b8c+c5dd892a6c,g414038480c+6b9177ef31,g41af890bb2+70bea58702,g599934f4f4+b8c5400ca5,g781aacb6e4+a8ce1bb630,g7af13505b9+b5b9cefdb8,g80478fca09+c2997882f3,g82479be7b0+8974e6af0f,g858d7b2824+1a22dc2551,g89c8672015+f4add4ffd5,g8f1c07a47a+de51c9b0a5,g9125e01d80+a8ce1bb630,ga5288a1d22+b66f8cf76b,gb58c049af0+d64f4d3760,gc28159a63d+404b60ec9b,gcab2d0539d+66cf1de5d4,gcf0d15dbbd+12cb7e2563,gda6a2b7d83+12cb7e2563,gdaeeff99f8+1711a396fd,ge79ae78c31+404b60ec9b,gef2f8181fd+414189b318,gf0baf85859+c1f95f4921,gf0c06eb49c+1a22dc2551,gfa517265be+1a22dc2551,gfa999e8aa5+17cd334064,v28.0.0.rc2
LSST Data Management Base Package
Loading...
Searching...
No Matches
calibType.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21__all__ = ["IsrCalib", "IsrProvenance"]
22
23import abc
24import datetime
25import logging
26import os.path
27import warnings
28import yaml
29import numpy as np
30
31from astropy.table import Table
32from astropy.io import fits
33
34from lsst.daf.base import PropertyList
35from lsst.utils.introspection import get_full_type_name
36from lsst.utils import doImport
37
38
39class IsrCalib(abc.ABC):
40 """Generic calibration type.
41
42 Subclasses must implement the toDict, fromDict, toTable, fromTable
43 methods that allow the calibration information to be converted
44 from dictionaries and afw tables. This will allow the calibration
45 to be persisted using the base class read/write methods.
46
47 The validate method is intended to provide a common way to check
48 that the calibration is valid (internally consistent) and
49 appropriate (usable with the intended data). The apply method is
50 intended to allow the calibration to be applied in a consistent
51 manner.
52
53 Parameters
54 ----------
55 camera : `lsst.afw.cameraGeom.Camera`, optional
56 Camera to extract metadata from.
57 detector : `lsst.afw.cameraGeom.Detector`, optional
58 Detector to extract metadata from.
59 log : `logging.Logger`, optional
60 Log for messages.
61 """
62 _OBSTYPE = "generic"
63 _SCHEMA = "NO SCHEMA"
64 _VERSION = 0
65
66 def __init__(self, camera=None, detector=None, log=None, **kwargs):
67 self._instrument = None
68 self._raftName = None
69 self._slotName = None
70 self._detectorName = None
71 self._detectorSerial = None
72 self._detectorId = None
73 self._filter = None
74 self._calibId = None
75 self._seqfile = None
76 self._seqname = None
77 self._seqcksum = None
80 self.calibInfoFromDict(kwargs)
81
82 # Define the required attributes for this calibration. These
83 # are entries that are automatically filled and propagated
84 # from the metadata. The existence of the attribute is
85 # required (they're checked for equivalence), but they do not
86 # necessarily need to have a value (None == None in this
87 # case).
88 self.requiredAttributesrequiredAttributesrequiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"])
89 self.requiredAttributesrequiredAttributesrequiredAttributes.update(["_instrument", "_raftName", "_slotName",
90 "_detectorName", "_detectorSerial", "_detectorId",
91 "_filter", "_calibId", "_seqfile", "_seqname", "_seqcksum",
92 "_metadata"])
93
94 self.log = log if log else logging.getLogger(__name__)
95
96 if detector:
97 self.fromDetector(detector)
98 self.updateMetadata(camera=camera, detector=detector)
99
100 def __str__(self):
101 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
102
103 def __eq__(self, other):
104 """Calibration equivalence.
105
106 Running ``calib.log.setLevel(0)`` enables debug statements to
107 identify problematic fields.
108 """
109 if not isinstance(other, self.__class__):
110 self.log.debug("Incorrect class type: %s %s", self.__class__, other.__class__)
111 return False
112
113 for attr in self._requiredAttributes:
114 attrSelf = getattr(self, attr)
115 attrOther = getattr(other, attr)
116
117 if isinstance(attrSelf, dict):
118 # Dictionary of arrays.
119 if attrSelf.keys() != attrOther.keys():
120 self.log.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
121 return False
122 for key in attrSelf:
123 try:
124 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True):
125 self.log.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
126 return False
127 except TypeError:
128 # If it is not something numpy can handle
129 # (it's not a number or array of numbers),
130 # then it needs to have its own equivalence
131 # operator.
132 if np.all(attrSelf[key] != attrOther[key]):
133 return False
134 elif isinstance(attrSelf, np.ndarray):
135 # Bare array.
136 if isinstance(attrSelf[0], (str, np.str_, np.string_)):
137 if not np.all(attrSelf == attrOther):
138 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther)
139 return False
140 else:
141 if not np.allclose(attrSelf, attrOther, equal_nan=True):
142 self.log.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther)
143 return False
144 elif type(attrSelf) is not type(attrOther):
145 if set([attrSelf, attrOther]) == set([None, ""]):
146 # Fits converts None to "", but None is not "".
147 continue
148 self.log.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
149 attrSelf, attrOther)
150 return False
151 else:
152 if attrSelf != attrOther:
153 self.log.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther)
154 return False
155
156 return True
157
158 @property
160 return self._requiredAttributes
161
162 @requiredAttributes.setter
163 def requiredAttributes(self, value):
165
166 # Property accessor associated with getMetadata().
167 @property
168 def metadata(self):
169 return self._metadata
170
171 def getMetadata(self):
172 """Retrieve metadata associated with this calibration.
173
174 Returns
175 -------
176 meta : `lsst.daf.base.PropertyList`
177 Metadata. The returned `~lsst.daf.base.PropertyList` can be
178 modified by the caller and the changes will be written to
179 external files.
180 """
181 return self._metadata
182
183 def setMetadata(self, metadata):
184 """Store a copy of the supplied metadata with this calibration.
185
186 Parameters
187 ----------
188 metadata : `lsst.daf.base.PropertyList`
189 Metadata to associate with the calibration. Will be copied and
190 overwrite existing metadata.
191 """
192 if metadata is not None:
193 self._metadata.update(metadata)
194
195 # Ensure that we have the obs type required by calibration ingest
196 self._metadata["OBSTYPE"] = self._OBSTYPE
197 self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
198 self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION
199
200 if isinstance(metadata, dict):
201 self.calibInfoFromDict(metadata)
202 elif isinstance(metadata, PropertyList):
203 self.calibInfoFromDict(metadata.toDict())
204
205 def updateMetadata(self, camera=None, detector=None, filterName=None,
206 setCalibId=False, setCalibInfo=False, setDate=False,
207 **kwargs):
208 """Update metadata keywords with new values.
209
210 Parameters
211 ----------
212 camera : `lsst.afw.cameraGeom.Camera`, optional
213 Reference camera to use to set ``_instrument`` field.
214 detector : `lsst.afw.cameraGeom.Detector`, optional
215 Reference detector to use to set ``_detector*`` fields.
216 filterName : `str`, optional
217 Filter name to assign to this calibration.
218 setCalibId : `bool`, optional
219 Construct the ``_calibId`` field from other fields.
220 setCalibInfo : `bool`, optional
221 Set calibration parameters from metadata.
222 setDate : `bool`, optional
223 Ensure the metadata ``CALIBDATE`` fields are set to the current
224 datetime.
225 kwargs : `dict` or `collections.abc.Mapping`, optional
226 Set of ``key=value`` pairs to assign to the metadata.
227 """
228 mdOriginal = self.getMetadata()
229 mdSupplemental = dict()
230
231 for k, v in kwargs.items():
232 if isinstance(v, fits.card.Undefined):
233 kwargs[k] = None
234
235 if setCalibInfo:
236 self.calibInfoFromDict(kwargs)
237
238 if camera:
239 self._instrument = camera.getName()
240
241 if detector:
242 self._detectorName = detector.getName()
243 self._detectorSerial = detector.getSerial()
244 self._detectorId = detector.getId()
245 if "_" in self._detectorName:
246 (self._raftName, self._slotName) = self._detectorName.split("_")
247
248 if filterName:
249 # TOD0 DM-28093: I think this whole comment can go away, if we
250 # always use physicalLabel everywhere in ip_isr.
251 # If set via:
252 # exposure.getInfo().getFilter().getName()
253 # then this will hold the abstract filter.
254 self._filter = filterName
255
256 if setDate:
257 date = datetime.datetime.now()
258 mdSupplemental["CALIBDATE"] = date.isoformat()
259 mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat()
260 mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat()
261
262 if setCalibId:
263 values = []
264 values.append(f"instrument={self._instrument}") if self._instrument else None
265 values.append(f"raftName={self._raftName}") if self._raftName else None
266 values.append(f"detectorName={self._detectorName}") if self._detectorName else None
267 values.append(f"detector={self._detectorId}") if self._detectorId else None
268 values.append(f"filter={self._filter}") if self._filter else None
269
270 calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None))
271 values.append(f"calibDate={calibDate}") if calibDate else None
272
273 self._calibId = " ".join(values)
274
275 self._metadata["INSTRUME"] = self._instrument if self._instrument else None
276 self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
277 self._metadata["SLOTNAME"] = self._slotName if self._slotName else None
278 self._metadata["DETECTOR"] = self._detectorId
279 self._metadata["DET_NAME"] = self._detectorName if self._detectorName else None
280 self._metadata["DET_SER"] = self._detectorSerial if self._detectorSerial else None
281 self._metadata["FILTER"] = self._filter if self._filter else None
282 self._metadata["CALIB_ID"] = self._calibId if self._calibId else None
283 self._metadata["SEQFILE"] = self._seqfile if self._seqfile else None
284 self._metadata["SEQNAME"] = self._seqname if self._seqname else None
285 self._metadata["SEQCKSUM"] = self._seqcksum if self._seqcksum else None
286 self._metadata["CALIBCLS"] = get_full_type_name(self)
287
288 mdSupplemental.update(kwargs)
289 mdOriginal.update(mdSupplemental)
290
291 def updateMetadataFromExposures(self, exposures):
292 """Extract and unify metadata information.
293
294 Parameters
295 ----------
296 exposures : `list`
297 Exposures or other calibrations to scan.
298 """
299 # This list of keywords is the set of header entries that
300 # should be checked and propagated. Not having an entry is
301 # not a failure, as they may not be defined for the exposures
302 # being used.
303 keywords = ["SEQNAME", "SEQFILE", "SEQCKSUM", "ODP", "AP0_RC"]
304 metadata = {}
305
306 for exp in exposures:
307 try:
308 expMeta = exp.getMetadata()
309 except AttributeError:
310 continue
311 for key in keywords:
312 if key in expMeta:
313 if key in metadata:
314 if metadata[key] != expMeta[key]:
315 self.log.warning("Metadata mismatch! Have: %s Found %s",
316 metadata[key], expMeta[key])
317 else:
318 metadata[key] = expMeta[key]
319 self.updateMetadata(**metadata)
320
321 def calibInfoFromDict(self, dictionary):
322 """Handle common keywords.
323
324 This isn't an ideal solution, but until all calibrations
325 expect to find everything in the metadata, they still need to
326 search through dictionaries.
327
328 Parameters
329 ----------
330 dictionary : `dict` or `lsst.daf.base.PropertyList`
331 Source for the common keywords.
332
333 Raises
334 ------
335 RuntimeError
336 Raised if the dictionary does not match the expected OBSTYPE.
337 """
338
339 def search(haystack, needles):
340 """Search dictionary 'haystack' for an entry in 'needles'
341 """
342 test = [haystack.get(x) for x in needles]
343 test = set([x for x in test if x is not None])
344 if len(test) == 0:
345 if "metadata" in haystack:
346 return search(haystack["metadata"], needles)
347 else:
348 return None
349 elif len(test) == 1:
350 value = list(test)[0]
351 if value == "":
352 return None
353 else:
354 return value
355 else:
356 raise ValueError(f"Too many values found: {len(test)} {test} {needles}")
357
358 if "metadata" in dictionary:
359 metadata = dictionary["metadata"]
360
361 if self._OBSTYPE != metadata["OBSTYPE"]:
362 raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
363 f"found {metadata['OBSTYPE']}")
364
365 self._instrument = search(dictionary, ["INSTRUME", "instrument"])
366 self._raftName = search(dictionary, ["RAFTNAME"])
367 self._slotName = search(dictionary, ["SLOTNAME"])
368 self._detectorId = search(dictionary, ["DETECTOR", "detectorId"])
369 self._detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"])
370 self._detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"])
371 self._filter = search(dictionary, ["FILTER", "filterName"])
372 self._calibId = search(dictionary, ["CALIB_ID"])
373 self._seqfile = search(dictionary, ["SEQFILE"])
374 self._seqname = search(dictionary, ["SEQNAME"])
375 self._seqcksum = search(dictionary, ["SEQCKSUM"])
376
377 @classmethod
378 def determineCalibClass(cls, metadata, message):
379 """Attempt to find calibration class in metadata.
380
381 Parameters
382 ----------
383 metadata : `dict` or `lsst.daf.base.PropertyList`
384 Metadata possibly containing a calibration class entry.
385 message : `str`
386 Message to include in any errors.
387
388 Returns
389 -------
390 calibClass : `object`
391 The class to use to read the file contents. Should be an
392 `lsst.ip.isr.IsrCalib` subclass.
393
394 Raises
395 ------
396 ValueError
397 Raised if the resulting calibClass is the base
398 `lsst.ip.isr.IsrClass` (which does not implement the
399 content methods).
400 """
401 calibClassName = metadata.get("CALIBCLS")
402 calibClass = doImport(calibClassName) if calibClassName is not None else cls
403 if calibClass is IsrCalib:
404 raise ValueError(f"Cannot use base class to read calibration data: {message}")
405 return calibClass
406
407 @classmethod
408 def readText(cls, filename, **kwargs):
409 """Read calibration representation from a yaml/ecsv file.
410
411 Parameters
412 ----------
413 filename : `str`
414 Name of the file containing the calibration definition.
415 kwargs : `dict` or collections.abc.Mapping`, optional
416 Set of key=value pairs to pass to the ``fromDict`` or
417 ``fromTable`` methods.
418
419 Returns
420 -------
421 calib : `~lsst.ip.isr.IsrCalibType`
422 Calibration class.
423
424 Raises
425 ------
426 RuntimeError
427 Raised if the filename does not end in ".ecsv" or ".yaml".
428 """
429 if filename.endswith((".ecsv", ".ECSV")):
430 data = Table.read(filename, format="ascii.ecsv")
431 calibClass = cls.determineCalibClass(data.meta, "readText/ECSV")
432 return calibClass.fromTable([data], **kwargs)
433 elif filename.endswith((".yaml", ".YAML")):
434 with open(filename, "r") as f:
435 data = yaml.load(f, Loader=yaml.CLoader)
436 calibClass = cls.determineCalibClass(data["metadata"], "readText/YAML")
437 return calibClass.fromDict(data, **kwargs)
438 else:
439 raise RuntimeError(f"Unknown filename extension: {filename}")
440
441 def writeText(self, filename, format="auto"):
442 """Write the calibration data to a text file.
443
444 Parameters
445 ----------
446 filename : `str`
447 Name of the file to write.
448 format : `str`
449 Format to write the file as. Supported values are:
450 ``"auto"`` : Determine filetype from filename.
451 ``"yaml"`` : Write as yaml.
452 ``"ecsv"`` : Write as ecsv.
453
454 Returns
455 -------
456 used : `str`
457 The name of the file used to write the data. This may
458 differ from the input if the format is explicitly chosen.
459
460 Raises
461 ------
462 RuntimeError
463 Raised if filename does not end in a known extension, or
464 if all information cannot be written.
465
466 Notes
467 -----
468 The file is written to YAML/ECSV format and will include any
469 associated metadata.
470 """
471 if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))):
472 outDict = self.toDict()
473 path, ext = os.path.splitext(filename)
474 filename = path + ".yaml"
475 with open(filename, "w") as f:
476 yaml.dump(outDict, f)
477 elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))):
478 tableList = self.toTable()
479 if len(tableList) > 1:
480 # ECSV doesn't support multiple tables per file, so we
481 # can only write the first table.
482 raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
483
484 table = tableList[0]
485 path, ext = os.path.splitext(filename)
486 filename = path + ".ecsv"
487 table.write(filename, format="ascii.ecsv")
488 else:
489 raise RuntimeError(f"Attempt to write to a file {filename} "
490 "that does not end in '.yaml' or '.ecsv'")
491
492 return filename
493
494 @classmethod
495 def readFits(cls, filename, **kwargs):
496 """Read calibration data from a FITS file.
497
498 Parameters
499 ----------
500 filename : `str`
501 Filename to read data from.
502 kwargs : `dict` or collections.abc.Mapping`, optional
503 Set of key=value pairs to pass to the ``fromTable``
504 method.
505
506 Returns
507 -------
508 calib : `lsst.ip.isr.IsrCalib`
509 Calibration contained within the file.
510 """
511 tableList = []
512 tableList.append(Table.read(filename, hdu=1, mask_invalid=False))
513 extNum = 2 # Fits indices start at 1, we've read one already.
514 keepTrying = True
515
516 while keepTrying:
517 with warnings.catch_warnings():
518 warnings.simplefilter("error")
519 try:
520 newTable = Table.read(filename, hdu=extNum, mask_invalid=False)
521 tableList.append(newTable)
522 extNum += 1
523 except Exception:
524 keepTrying = False
525
526 for table in tableList:
527 for k, v in table.meta.items():
528 if isinstance(v, fits.card.Undefined):
529 table.meta[k] = None
530
531 calibClass = cls.determineCalibClass(tableList[0].meta, "readFits")
532 return calibClass.fromTable(tableList, **kwargs)
533
534 def writeFits(self, filename):
535 """Write calibration data to a FITS file.
536
537 Parameters
538 ----------
539 filename : `str`
540 Filename to write data to.
541
542 Returns
543 -------
544 used : `str`
545 The name of the file used to write the data.
546 """
547 tableList = self.toTable()
548 with warnings.catch_warnings():
549 warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
550 astropyList = [fits.table_to_hdu(table) for table in tableList]
551 astropyList.insert(0, fits.PrimaryHDU())
552
553 writer = fits.HDUList(astropyList)
554 writer.writeto(filename, overwrite=True)
555 return filename
556
557 def fromDetector(self, detector):
558 """Modify the calibration parameters to match the supplied detector.
559
560 Parameters
561 ----------
562 detector : `lsst.afw.cameraGeom.Detector`
563 Detector to use to set parameters from.
564
565 Raises
566 ------
567 NotImplementedError
568 Raised if not implemented by a subclass.
569 This needs to be implemented by subclasses for each
570 calibration type.
571 """
572 raise NotImplementedError("Must be implemented by subclass.")
573
574 @classmethod
575 def fromDict(cls, dictionary, **kwargs):
576 """Construct a calibration from a dictionary of properties.
577
578 Must be implemented by the specific calibration subclasses.
579
580 Parameters
581 ----------
582 dictionary : `dict`
583 Dictionary of properties.
584 kwargs : `dict` or collections.abc.Mapping`, optional
585 Set of key=value options.
586
587 Returns
588 -------
589 calib : `lsst.ip.isr.CalibType`
590 Constructed calibration.
591
592 Raises
593 ------
594 NotImplementedError
595 Raised if not implemented.
596 """
597 raise NotImplementedError("Must be implemented by subclass.")
598
599 def toDict(self):
600 """Return a dictionary containing the calibration properties.
601
602 The dictionary should be able to be round-tripped through
603 `fromDict`.
604
605 Returns
606 -------
607 dictionary : `dict`
608 Dictionary of properties.
609
610 Raises
611 ------
612 NotImplementedError
613 Raised if not implemented.
614 """
615 raise NotImplementedError("Must be implemented by subclass.")
616
617 @classmethod
618 def fromTable(cls, tableList, **kwargs):
619 """Construct a calibration from a dictionary of properties.
620
621 Must be implemented by the specific calibration subclasses.
622
623 Parameters
624 ----------
625 tableList : `list` [`lsst.afw.table.Table`]
626 List of tables of properties.
627 kwargs : `dict` or collections.abc.Mapping`, optional
628 Set of key=value options.
629
630 Returns
631 -------
632 calib : `lsst.ip.isr.CalibType`
633 Constructed calibration.
634
635 Raises
636 ------
637 NotImplementedError
638 Raised if not implemented.
639 """
640 raise NotImplementedError("Must be implemented by subclass.")
641
642 def toTable(self):
643 """Return a list of tables containing the calibration properties.
644
645 The table list should be able to be round-tripped through
646 `fromDict`.
647
648 Returns
649 -------
650 tableList : `list` [`lsst.afw.table.Table`]
651 List of tables of properties.
652
653 Raises
654 ------
655 NotImplementedError
656 Raised if not implemented.
657 """
658 raise NotImplementedError("Must be implemented by subclass.")
659
660 def validate(self, other=None):
661 """Validate that this calibration is defined and can be used.
662
663 Parameters
664 ----------
665 other : `object`, optional
666 Thing to validate against.
667
668 Returns
669 -------
670 valid : `bool`
671 Returns true if the calibration is valid and appropriate.
672 """
673 return False
674
675 def apply(self, target):
676 """Method to apply the calibration to the target object.
677
678 Parameters
679 ----------
680 target : `object`
681 Thing to validate against.
682
683 Returns
684 -------
685 valid : `bool`
686 Returns true if the calibration was applied correctly.
687
688 Raises
689 ------
690 NotImplementedError
691 Raised if not implemented.
692 """
693 raise NotImplementedError("Must be implemented by subclass.")
694
695
697 """Class for the provenance of data used to construct calibration.
698
699 Provenance is not really a calibration, but we would like to
700 record this when constructing the calibration, and it provides an
701 example of the base calibration class.
702
703 Parameters
704 ----------
705 instrument : `str`, optional
706 Name of the instrument the data was taken with.
707 calibType : `str`, optional
708 Type of calibration this provenance was generated for.
709 detectorName : `str`, optional
710 Name of the detector this calibration is for.
711 detectorSerial : `str`, optional
712 Identifier for the detector.
713
714 """
715 _OBSTYPE = "IsrProvenance"
716
717 def __init__(self, calibType="unknown",
718 **kwargs):
719 self.calibType = calibType
720 self.dimensions = set()
721 self.dataIdList = list()
722
723 super().__init__(**kwargs)
724
725 self.requiredAttributesrequiredAttributesrequiredAttributes.update(["calibType", "dimensions", "dataIdList"])
726
727 def __str__(self):
728 return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
729
730 def __eq__(self, other):
731 return super().__eq__(other)
732
733 def updateMetadata(self, setDate=False, **kwargs):
734 """Update calibration metadata.
735
736 Parameters
737 ----------
738 setDate : `bool`, optional
739 Update the ``CALIBDATE`` fields in the metadata to the current
740 time. Defaults to False.
741 kwargs : `dict` or `collections.abc.Mapping`, optional
742 Other keyword parameters to set in the metadata.
743 """
744 kwargs["calibType"] = self.calibType
745 super().updateMetadata(setDate=setDate, **kwargs)
746
747 def fromDataIds(self, dataIdList):
748 """Update provenance from dataId List.
749
750 Parameters
751 ----------
752 dataIdList : `list` [`lsst.daf.butler.DataId`]
753 List of dataIds used in generating this calibration.
754 """
755 for dataId in dataIdList:
756 for key in dataId:
757 if key not in self.dimensions:
758 self.dimensions.add(key)
759 self.dataIdList.append(dataId)
760
761 @classmethod
762 def fromTable(cls, tableList):
763 """Construct provenance from table list.
764
765 Parameters
766 ----------
767 tableList : `list` [`lsst.afw.table.Table`]
768 List of tables to construct the provenance from.
769
770 Returns
771 -------
772 provenance : `lsst.ip.isr.IsrProvenance`
773 The provenance defined in the tables.
774 """
775 table = tableList[0]
776 metadata = table.meta
777 inDict = dict()
778 inDict["metadata"] = metadata
779 inDict["calibType"] = metadata["calibType"]
780 inDict["dimensions"] = set()
781 inDict["dataIdList"] = list()
782
783 schema = dict()
784 for colName in table.columns:
785 schema[colName.lower()] = colName
786 inDict["dimensions"].add(colName.lower())
787 inDict["dimensions"] = sorted(inDict["dimensions"])
788
789 for row in table:
790 entry = dict()
791 for dim in sorted(inDict["dimensions"]):
792 entry[dim] = row[schema[dim]]
793 inDict["dataIdList"].append(entry)
794
795 return cls.fromDictfromDict(inDict)
796
797 @classmethod
798 def fromDict(cls, dictionary):
799 """Construct provenance from a dictionary.
800
801 Parameters
802 ----------
803 dictionary : `dict`
804 Dictionary of provenance parameters.
805
806 Returns
807 -------
808 provenance : `lsst.ip.isr.IsrProvenance`
809 The provenance defined in the tables.
810 """
811 calib = cls()
812 if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]:
813 raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
814 f"found {dictionary['metadata']['OBSTYPE']}")
815 calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"])
816
817 # These properties should be in the metadata, but occasionally
818 # are found in the dictionary itself. Check both places,
819 # ending with `None` if neither contains the information.
820 calib.calibType = dictionary["calibType"]
821 calib.dimensions = set(dictionary["dimensions"])
822 calib.dataIdList = dictionary["dataIdList"]
823
824 calib.updateMetadata()
825 return calib
826
827 def toDict(self):
828 """Return a dictionary containing the provenance information.
829
830 Returns
831 -------
832 dictionary : `dict`
833 Dictionary of provenance.
834 """
836
837 outDict = {}
838
839 metadata = self.getMetadata()
840 outDict["metadata"] = metadata
841 outDict["detectorName"] = self._detectorName
842 outDict["detectorSerial"] = self._detectorSerial
843 outDict["detectorId"] = self._detectorId
844 outDict["instrument"] = self._instrument
845 outDict["calibType"] = self.calibType
846 outDict["dimensions"] = list(self.dimensions)
847 outDict["dataIdList"] = self.dataIdList
848
849 return outDict
850
851 def toTable(self):
852 """Return a list of tables containing the provenance.
853
854 This seems inefficient and slow, so this may not be the best
855 way to store the data.
856
857 Returns
858 -------
859 tableList : `list` [`lsst.afw.table.Table`]
860 List of tables containing the provenance information
861 """
862 tableList = []
863 self.updateMetadataupdateMetadata(setDate=True, setCalibInfo=True)
864
865 catalog = Table(rows=self.dataIdList,
866 names=self.dimensions)
867 filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
868 catalog.meta = filteredMetadata
869 tableList.append(catalog)
870 return tableList
std::vector< SchemaItem< Flag > > * items
Tag types used to declare specialized field types.
Definition misc.h:31
Class for storing ordered metadata with comments.
fromDict(cls, dictionary, **kwargs)
Definition calibType.py:575
writeText(self, filename, format="auto")
Definition calibType.py:441
readText(cls, filename, **kwargs)
Definition calibType.py:408
calibInfoFromDict(self, dictionary)
Definition calibType.py:321
updateMetadataFromExposures(self, exposures)
Definition calibType.py:291
determineCalibClass(cls, metadata, message)
Definition calibType.py:378
__init__(self, camera=None, detector=None, log=None, **kwargs)
Definition calibType.py:66
updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition calibType.py:207
fromTable(cls, tableList, **kwargs)
Definition calibType.py:618
readFits(cls, filename, **kwargs)
Definition calibType.py:495
__init__(self, calibType="unknown", **kwargs)
Definition calibType.py:718
updateMetadata(self, setDate=False, **kwargs)
Definition calibType.py:733