LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
calibType.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 import abc
22 import datetime
23 import logging
24 import os.path
25 import warnings
26 import yaml
27 import numpy as np
28 
29 from astropy.table import Table
30 from astropy.io import fits
31 
32 from lsst.daf.base import PropertyList
33 from lsst.utils.introspection import get_full_type_name
34 from lsst.utils import doImport
35 
36 
37 __all__ = ["IsrCalib", "IsrProvenance"]
38 
39 
40 class IsrCalib(abc.ABC):
41  """Generic calibration type.
42 
43  Subclasses must implement the toDict, fromDict, toTable, fromTable
44  methods that allow the calibration information to be converted
45  from dictionaries and afw tables. This will allow the calibration
46  to be persisted using the base class read/write methods.
47 
48  The validate method is intended to provide a common way to check
49  that the calibration is valid (internally consistent) and
50  appropriate (usable with the intended data). The apply method is
51  intended to allow the calibration to be applied in a consistent
52  manner.
53 
54  Parameters
55  ----------
56  camera : `lsst.afw.cameraGeom.Camera`, optional
57  Camera to extract metadata from.
58  detector : `lsst.afw.cameraGeom.Detector`, optional
59  Detector to extract metadata from.
60  log : `logging.Logger`, optional
61  Log for messages.
62  """
63  _OBSTYPE = "generic"
64  _SCHEMA = "NO SCHEMA"
65  _VERSION = 0
66 
67  def __init__(self, camera=None, detector=None, log=None, **kwargs):
68  self._instrument_instrument = None
69  self._raftName_raftName = None
70  self._slotName_slotName = None
71  self._detectorName_detectorName = None
72  self._detectorSerial_detectorSerial = None
73  self._detectorId_detectorId = None
74  self._filter_filter = None
75  self._calibId_calibId = None
76  self._metadata_metadata = PropertyList()
77  self.setMetadatasetMetadata(PropertyList())
78  self.calibInfoFromDictcalibInfoFromDict(kwargs)
79 
80  # Define the required attributes for this calibration.
81  self.requiredAttributesrequiredAttributesrequiredAttributesrequiredAttributes = set(["_OBSTYPE", "_SCHEMA", "_VERSION"])
82  self.requiredAttributesrequiredAttributesrequiredAttributesrequiredAttributes.update(["_instrument", "_raftName", "_slotName",
83  "_detectorName", "_detectorSerial", "_detectorId",
84  "_filter", "_calibId", "_metadata"])
85 
86  self.loglog = log if log else logging.getLogger(__name__)
87 
88  if detector:
89  self.fromDetectorfromDetector(detector)
90  self.updateMetadataupdateMetadata(camera=camera, detector=detector)
91 
92  def __str__(self):
93  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
94 
95  def __eq__(self, other):
96  """Calibration equivalence.
97 
98  Running ``calib.log.setLevel(0)`` enables debug statements to
99  identify problematic fields.
100  """
101  if not isinstance(other, self.__class__):
102  self.loglog.debug("Incorrect class type: %s %s", self.__class__, other.__class__)
103  return False
104 
105  for attr in self._requiredAttributes_requiredAttributes:
106  attrSelf = getattr(self, attr)
107  attrOther = getattr(other, attr)
108 
109  if isinstance(attrSelf, dict):
110  # Dictionary of arrays.
111  if attrSelf.keys() != attrOther.keys():
112  self.loglog.debug("Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
113  return False
114  for key in attrSelf:
115  if not np.allclose(attrSelf[key], attrOther[key], equal_nan=True):
116  self.loglog.debug("Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
117  return False
118  elif isinstance(attrSelf, np.ndarray):
119  # Bare array.
120  if not np.allclose(attrSelf, attrOther, equal_nan=True):
121  self.loglog.debug("Array Failure: %s %s %s", attr, attrSelf, attrOther)
122  return False
123  elif type(attrSelf) != type(attrOther):
124  if set([attrSelf, attrOther]) == set([None, ""]):
125  # Fits converts None to "", but None is not "".
126  continue
127  self.loglog.debug("Type Failure: %s %s %s %s %s", attr, type(attrSelf), type(attrOther),
128  attrSelf, attrOther)
129  return False
130  else:
131  if attrSelf != attrOther:
132  self.loglog.debug("Value Failure: %s %s %s", attr, attrSelf, attrOther)
133  return False
134 
135  return True
136 
137  @property
139  return self._requiredAttributes_requiredAttributes
140 
141  @requiredAttributes.setter
142  def requiredAttributes(self, value):
143  self._requiredAttributes_requiredAttributes = value
144 
145  def getMetadata(self):
146  """Retrieve metadata associated with this calibration.
147 
148  Returns
149  -------
150  meta : `lsst.daf.base.PropertyList`
151  Metadata. The returned `~lsst.daf.base.PropertyList` can be
152  modified by the caller and the changes will be written to
153  external files.
154  """
155  return self._metadata_metadata
156 
157  def setMetadata(self, metadata):
158  """Store a copy of the supplied metadata with this calibration.
159 
160  Parameters
161  ----------
162  metadata : `lsst.daf.base.PropertyList`
163  Metadata to associate with the calibration. Will be copied and
164  overwrite existing metadata.
165  """
166  if metadata is not None:
167  self._metadata_metadata.update(metadata)
168 
169  # Ensure that we have the obs type required by calibration ingest
170  self._metadata_metadata["OBSTYPE"] = self._OBSTYPE_OBSTYPE
171  self._metadata_metadata[self._OBSTYPE_OBSTYPE + "_SCHEMA"] = self._SCHEMA_SCHEMA
172  self._metadata_metadata[self._OBSTYPE_OBSTYPE + "_VERSION"] = self._VERSION_VERSION
173 
174  if isinstance(metadata, dict):
175  self.calibInfoFromDictcalibInfoFromDict(metadata)
176  elif isinstance(metadata, PropertyList):
177  self.calibInfoFromDictcalibInfoFromDict(metadata.toDict())
178 
179  def updateMetadata(self, camera=None, detector=None, filterName=None,
180  setCalibId=False, setCalibInfo=False, setDate=False,
181  **kwargs):
182  """Update metadata keywords with new values.
183 
184  Parameters
185  ----------
186  camera : `lsst.afw.cameraGeom.Camera`, optional
187  Reference camera to use to set _instrument field.
188  detector : `lsst.afw.cameraGeom.Detector`, optional
189  Reference detector to use to set _detector* fields.
190  filterName : `str`, optional
191  Filter name to assign to this calibration.
192  setCalibId : `bool`, optional
193  Construct the _calibId field from other fields.
194  setCalibInfo : `bool`, optional
195  Set calibration parameters from metadata.
196  setDate : `bool`, optional
197  Ensure the metadata CALIBDATE fields are set to the current datetime.
198  kwargs : `dict` or `collections.abc.Mapping`, optional
199  Set of key=value pairs to assign to the metadata.
200  """
201  mdOriginal = self.getMetadatagetMetadata()
202  mdSupplemental = dict()
203 
204  for k, v in kwargs.items():
205  if isinstance(v, fits.card.Undefined):
206  kwargs[k] = None
207 
208  if setCalibInfo:
209  self.calibInfoFromDictcalibInfoFromDict(kwargs)
210 
211  if camera:
212  self._instrument_instrument = camera.getName()
213 
214  if detector:
215  self._detectorName_detectorName = detector.getName()
216  self._detectorSerial_detectorSerial = detector.getSerial()
217  self._detectorId_detectorId = detector.getId()
218  if "_" in self._detectorName_detectorName:
219  (self._raftName_raftName, self._slotName_slotName) = self._detectorName_detectorName.split("_")
220 
221  if filterName:
222  # TOD0 DM-28093: I think this whole comment can go away, if we
223  # always use physicalLabel everywhere in ip_isr.
224  # If set via:
225  # exposure.getInfo().getFilter().getName()
226  # then this will hold the abstract filter.
227  self._filter_filter = filterName
228 
229  if setDate:
230  date = datetime.datetime.now()
231  mdSupplemental["CALIBDATE"] = date.isoformat()
232  mdSupplemental["CALIB_CREATION_DATE"] = date.date().isoformat()
233  mdSupplemental["CALIB_CREATION_TIME"] = date.time().isoformat()
234 
235  if setCalibId:
236  values = []
237  values.append(f"instrument={self._instrument}") if self._instrument_instrument else None
238  values.append(f"raftName={self._raftName}") if self._raftName_raftName else None
239  values.append(f"detectorName={self._detectorName}") if self._detectorName_detectorName else None
240  values.append(f"detector={self._detectorId}") if self._detectorId_detectorId else None
241  values.append(f"filter={self._filter}") if self._filter_filter else None
242 
243  calibDate = mdOriginal.get("CALIBDATE", mdSupplemental.get("CALIBDATE", None))
244  values.append(f"calibDate={calibDate}") if calibDate else None
245 
246  self._calibId_calibId = " ".join(values)
247 
248  self._metadata_metadata["INSTRUME"] = self._instrument_instrument if self._instrument_instrument else None
249  self._metadata_metadata["RAFTNAME"] = self._raftName_raftName if self._raftName_raftName else None
250  self._metadata_metadata["SLOTNAME"] = self._slotName_slotName if self._slotName_slotName else None
251  self._metadata_metadata["DETECTOR"] = self._detectorId_detectorId
252  self._metadata_metadata["DET_NAME"] = self._detectorName_detectorName if self._detectorName_detectorName else None
253  self._metadata_metadata["DET_SER"] = self._detectorSerial_detectorSerial if self._detectorSerial_detectorSerial else None
254  self._metadata_metadata["FILTER"] = self._filter_filter if self._filter_filter else None
255  self._metadata_metadata["CALIB_ID"] = self._calibId_calibId if self._calibId_calibId else None
256  self._metadata_metadata["CALIBCLS"] = get_full_type_name(self)
257 
258  mdSupplemental.update(kwargs)
259  mdOriginal.update(mdSupplemental)
260 
261  def calibInfoFromDict(self, dictionary):
262  """Handle common keywords.
263 
264  This isn't an ideal solution, but until all calibrations
265  expect to find everything in the metadata, they still need to
266  search through dictionaries.
267 
268  Parameters
269  ----------
270  dictionary : `dict` or `lsst.daf.base.PropertyList`
271  Source for the common keywords.
272 
273  Raises
274  ------
275  RuntimeError :
276  Raised if the dictionary does not match the expected OBSTYPE.
277 
278  """
279 
280  def search(haystack, needles):
281  """Search dictionary 'haystack' for an entry in 'needles'
282  """
283  test = [haystack.get(x) for x in needles]
284  test = set([x for x in test if x is not None])
285  if len(test) == 0:
286  if "metadata" in haystack:
287  return search(haystack["metadata"], needles)
288  else:
289  return None
290  elif len(test) == 1:
291  value = list(test)[0]
292  if value == "":
293  return None
294  else:
295  return value
296  else:
297  raise ValueError(f"Too many values found: {len(test)} {test} {needles}")
298 
299  if "metadata" in dictionary:
300  metadata = dictionary["metadata"]
301 
302  if self._OBSTYPE_OBSTYPE != metadata["OBSTYPE"]:
303  raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
304  f"found {metadata['OBSTYPE']}")
305 
306  self._instrument_instrument = search(dictionary, ["INSTRUME", "instrument"])
307  self._raftName_raftName = search(dictionary, ["RAFTNAME"])
308  self._slotName_slotName = search(dictionary, ["SLOTNAME"])
309  self._detectorId_detectorId = search(dictionary, ["DETECTOR", "detectorId"])
310  self._detectorName_detectorName = search(dictionary, ["DET_NAME", "DETECTOR_NAME", "detectorName"])
311  self._detectorSerial_detectorSerial = search(dictionary, ["DET_SER", "DETECTOR_SERIAL", "detectorSerial"])
312  self._filter_filter = search(dictionary, ["FILTER", "filterName"])
313  self._calibId_calibId = search(dictionary, ["CALIB_ID"])
314 
315  @classmethod
316  def determineCalibClass(cls, metadata, message):
317  """Attempt to find calibration class in metadata.
318 
319  Parameters
320  ----------
321  metadata : `dict` or `lsst.daf.base.PropertyList`
322  Metadata possibly containing a calibration class entry.
323  message : `str`
324  Message to include in any errors.
325 
326  Returns
327  -------
328  calibClass : `object`
329  The class to use to read the file contents. Should be an
330  `lsst.ip.isr.IsrCalib` subclass.
331 
332  Raises
333  ------
334  ValueError :
335  Raised if the resulting calibClass is the base
336  `lsst.ip.isr.IsrClass` (which does not implement the
337  content methods).
338  """
339  calibClassName = metadata.get("CALIBCLS")
340  calibClass = doImport(calibClassName) if calibClassName is not None else cls
341  if calibClass is IsrCalib:
342  raise ValueError(f"Cannot use base class to read calibration data: {msg}")
343  return calibClass
344 
345  @classmethod
346  def readText(cls, filename, **kwargs):
347  """Read calibration representation from a yaml/ecsv file.
348 
349  Parameters
350  ----------
351  filename : `str`
352  Name of the file containing the calibration definition.
353  kwargs : `dict` or collections.abc.Mapping`, optional
354  Set of key=value pairs to pass to the ``fromDict`` or
355  ``fromTable`` methods.
356 
357  Returns
358  -------
359  calib : `~lsst.ip.isr.IsrCalibType`
360  Calibration class.
361 
362  Raises
363  ------
364  RuntimeError :
365  Raised if the filename does not end in ".ecsv" or ".yaml".
366  """
367  if filename.endswith((".ecsv", ".ECSV")):
368  data = Table.read(filename, format="ascii.ecsv")
369  calibClass = cls.determineCalibClassdetermineCalibClass(data.meta, "readText/ECSV")
370  return calibClass.fromTable([data], **kwargs)
371  elif filename.endswith((".yaml", ".YAML")):
372  with open(filename, "r") as f:
373  data = yaml.load(f, Loader=yaml.CLoader)
374  calibClass = cls.determineCalibClassdetermineCalibClass(data["metadata"], "readText/YAML")
375  return calibClass.fromDict(data, **kwargs)
376  else:
377  raise RuntimeError(f"Unknown filename extension: {filename}")
378 
379  def writeText(self, filename, format="auto"):
380  """Write the calibration data to a text file.
381 
382  Parameters
383  ----------
384  filename : `str`
385  Name of the file to write.
386  format : `str`
387  Format to write the file as. Supported values are:
388  ``"auto"`` : Determine filetype from filename.
389  ``"yaml"`` : Write as yaml.
390  ``"ecsv"`` : Write as ecsv.
391  Returns
392  -------
393  used : `str`
394  The name of the file used to write the data. This may
395  differ from the input if the format is explicitly chosen.
396 
397  Raises
398  ------
399  RuntimeError :
400  Raised if filename does not end in a known extension, or
401  if all information cannot be written.
402 
403  Notes
404  -----
405  The file is written to YAML/ECSV format and will include any
406  associated metadata.
407  """
408  if format == "yaml" or (format == "auto" and filename.lower().endswith((".yaml", ".YAML"))):
409  outDict = self.toDicttoDict()
410  path, ext = os.path.splitext(filename)
411  filename = path + ".yaml"
412  with open(filename, "w") as f:
413  yaml.dump(outDict, f)
414  elif format == "ecsv" or (format == "auto" and filename.lower().endswith((".ecsv", ".ECSV"))):
415  tableList = self.toTabletoTable()
416  if len(tableList) > 1:
417  # ECSV doesn't support multiple tables per file, so we
418  # can only write the first table.
419  raise RuntimeError(f"Unable to persist {len(tableList)}tables in ECSV format.")
420 
421  table = tableList[0]
422  path, ext = os.path.splitext(filename)
423  filename = path + ".ecsv"
424  table.write(filename, format="ascii.ecsv")
425  else:
426  raise RuntimeError(f"Attempt to write to a file {filename} "
427  "that does not end in '.yaml' or '.ecsv'")
428 
429  return filename
430 
431  @classmethod
432  def readFits(cls, filename, **kwargs):
433  """Read calibration data from a FITS file.
434 
435  Parameters
436  ----------
437  filename : `str`
438  Filename to read data from.
439  kwargs : `dict` or collections.abc.Mapping`, optional
440  Set of key=value pairs to pass to the ``fromTable``
441  method.
442 
443  Returns
444  -------
445  calib : `lsst.ip.isr.IsrCalib`
446  Calibration contained within the file.
447  """
448  tableList = []
449  tableList.append(Table.read(filename, hdu=1))
450  extNum = 2 # Fits indices start at 1, we've read one already.
451  keepTrying = True
452 
453  while keepTrying:
454  with warnings.catch_warnings():
455  warnings.simplefilter("error")
456  try:
457  newTable = Table.read(filename, hdu=extNum)
458  tableList.append(newTable)
459  extNum += 1
460  except Exception:
461  keepTrying = False
462 
463  for table in tableList:
464  for k, v in table.meta.items():
465  if isinstance(v, fits.card.Undefined):
466  table.meta[k] = None
467 
468  calibClass = cls.determineCalibClassdetermineCalibClass(tableList[0].meta, "readFits")
469  return calibClass.fromTable(tableList, **kwargs)
470 
471  def writeFits(self, filename):
472  """Write calibration data to a FITS file.
473 
474  Parameters
475  ----------
476  filename : `str`
477  Filename to write data to.
478 
479  Returns
480  -------
481  used : `str`
482  The name of the file used to write the data.
483 
484  """
485  tableList = self.toTabletoTable()
486  with warnings.catch_warnings():
487  warnings.filterwarnings("ignore", category=Warning, module="astropy.io")
488  astropyList = [fits.table_to_hdu(table) for table in tableList]
489  astropyList.insert(0, fits.PrimaryHDU())
490 
491  writer = fits.HDUList(astropyList)
492  writer.writeto(filename, overwrite=True)
493  return filename
494 
495  def fromDetector(self, detector):
496  """Modify the calibration parameters to match the supplied detector.
497 
498  Parameters
499  ----------
500  detector : `lsst.afw.cameraGeom.Detector`
501  Detector to use to set parameters from.
502 
503  Raises
504  ------
505  NotImplementedError
506  This needs to be implemented by subclasses for each
507  calibration type.
508  """
509  raise NotImplementedError("Must be implemented by subclass.")
510 
511  @classmethod
512  def fromDict(cls, dictionary, **kwargs):
513  """Construct a calibration from a dictionary of properties.
514 
515  Must be implemented by the specific calibration subclasses.
516 
517  Parameters
518  ----------
519  dictionary : `dict`
520  Dictionary of properties.
521  kwargs : `dict` or collections.abc.Mapping`, optional
522  Set of key=value options.
523 
524  Returns
525  ------
526  calib : `lsst.ip.isr.CalibType`
527  Constructed calibration.
528 
529  Raises
530  ------
531  NotImplementedError :
532  Raised if not implemented.
533  """
534  raise NotImplementedError("Must be implemented by subclass.")
535 
536  def toDict(self):
537  """Return a dictionary containing the calibration properties.
538 
539  The dictionary should be able to be round-tripped through
540  `fromDict`.
541 
542  Returns
543  -------
544  dictionary : `dict`
545  Dictionary of properties.
546 
547  Raises
548  ------
549  NotImplementedError :
550  Raised if not implemented.
551  """
552  raise NotImplementedError("Must be implemented by subclass.")
553 
554  @classmethod
555  def fromTable(cls, tableList, **kwargs):
556  """Construct a calibration from a dictionary of properties.
557 
558  Must be implemented by the specific calibration subclasses.
559 
560  Parameters
561  ----------
562  tableList : `list` [`lsst.afw.table.Table`]
563  List of tables of properties.
564  kwargs : `dict` or collections.abc.Mapping`, optional
565  Set of key=value options.
566 
567  Returns
568  ------
569  calib : `lsst.ip.isr.CalibType`
570  Constructed calibration.
571 
572  Raises
573  ------
574  NotImplementedError :
575  Raised if not implemented.
576  """
577  raise NotImplementedError("Must be implemented by subclass.")
578 
579  def toTable(self):
580  """Return a list of tables containing the calibration properties.
581 
582  The table list should be able to be round-tripped through
583  `fromDict`.
584 
585  Returns
586  -------
587  tableList : `list` [`lsst.afw.table.Table`]
588  List of tables of properties.
589 
590  Raises
591  ------
592  NotImplementedError :
593  Raised if not implemented.
594  """
595  raise NotImplementedError("Must be implemented by subclass.")
596 
597  def validate(self, other=None):
598  """Validate that this calibration is defined and can be used.
599 
600  Parameters
601  ----------
602  other : `object`, optional
603  Thing to validate against.
604 
605  Returns
606  -------
607  valid : `bool`
608  Returns true if the calibration is valid and appropriate.
609  """
610  return False
611 
612  def apply(self, target):
613  """Method to apply the calibration to the target object.
614 
615  Parameters
616  ----------
617  target : `object`
618  Thing to validate against.
619 
620  Returns
621  -------
622  valid : `bool`
623  Returns true if the calibration was applied correctly.
624 
625  Raises
626  ------
627  NotImplementedError :
628  Raised if not implemented.
629  """
630  raise NotImplementedError("Must be implemented by subclass.")
631 
632 
634  """Class for the provenance of data used to construct calibration.
635 
636  Provenance is not really a calibration, but we would like to
637  record this when constructing the calibration, and it provides an
638  example of the base calibration class.
639 
640  Parameters
641  ----------
642  instrument : `str`, optional
643  Name of the instrument the data was taken with.
644  calibType : `str`, optional
645  Type of calibration this provenance was generated for.
646  detectorName : `str`, optional
647  Name of the detector this calibration is for.
648  detectorSerial : `str`, optional
649  Identifier for the detector.
650 
651  """
652  _OBSTYPE = "IsrProvenance"
653 
654  def __init__(self, calibType="unknown",
655  **kwargs):
656  self.calibTypecalibType = calibType
657  self.dimensionsdimensions = set()
658  self.dataIdListdataIdList = list()
659 
660  super().__init__(**kwargs)
661 
662  self.requiredAttributesrequiredAttributesrequiredAttributesrequiredAttributes.update(["calibType", "dimensions", "dataIdList"])
663 
664  def __str__(self):
665  return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
666 
667  def __eq__(self, other):
668  return super().__eq__(other)
669 
670  def updateMetadata(self, setDate=False, **kwargs):
671  """Update calibration metadata.
672 
673  Parameters
674  ----------
675  setDate : `bool, optional
676  Update the CALIBDATE fields in the metadata to the current
677  time. Defaults to False.
678  kwargs : `dict` or `collections.abc.Mapping`, optional
679  Other keyword parameters to set in the metadata.
680  """
681  kwargs["calibType"] = self.calibTypecalibType
682  super().updateMetadata(setDate=setDate, **kwargs)
683 
684  def fromDataIds(self, dataIdList):
685  """Update provenance from dataId List.
686 
687  Parameters
688  ----------
689  dataIdList : `list` [`lsst.daf.butler.DataId`]
690  List of dataIds used in generating this calibration.
691  """
692  for dataId in dataIdList:
693  for key in dataId:
694  if key not in self.dimensionsdimensions:
695  self.dimensionsdimensions.add(key)
696  self.dataIdListdataIdList.append(dataId)
697 
698  @classmethod
699  def fromTable(cls, tableList):
700  """Construct provenance from table list.
701 
702  Parameters
703  ----------
704  tableList : `list` [`lsst.afw.table.Table`]
705  List of tables to construct the provenance from.
706 
707  Returns
708  -------
709  provenance : `lsst.ip.isr.IsrProvenance`
710  The provenance defined in the tables.
711  """
712  table = tableList[0]
713  metadata = table.meta
714  inDict = dict()
715  inDict["metadata"] = metadata
716  inDict["calibType"] = metadata["calibType"]
717  inDict["dimensions"] = set()
718  inDict["dataIdList"] = list()
719 
720  schema = dict()
721  for colName in table.columns:
722  schema[colName.lower()] = colName
723  inDict["dimensions"].add(colName.lower())
724  inDict["dimensions"] = sorted(inDict["dimensions"])
725 
726  for row in table:
727  entry = dict()
728  for dim in sorted(inDict["dimensions"]):
729  entry[dim] = row[schema[dim]]
730  inDict["dataIdList"].append(entry)
731 
732  return cls.fromDictfromDictfromDict(inDict)
733 
734  @classmethod
735  def fromDict(cls, dictionary):
736  """Construct provenance from a dictionary.
737 
738  Parameters
739  ----------
740  dictionary : `dict`
741  Dictionary of provenance parameters.
742 
743  Returns
744  -------
745  provenance : `lsst.ip.isr.IsrProvenance`
746  The provenance defined in the tables.
747  """
748  calib = cls()
749  if calib._OBSTYPE != dictionary["metadata"]["OBSTYPE"]:
750  raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
751  f"found {dictionary['metadata']['OBSTYPE']}")
752  calib.updateMetadata(setDate=False, setCalibInfo=True, **dictionary["metadata"])
753 
754  # These properties should be in the metadata, but occasionally
755  # are found in the dictionary itself. Check both places,
756  # ending with `None` if neither contains the information.
757  calib.calibType = dictionary["calibType"]
758  calib.dimensions = set(dictionary["dimensions"])
759  calib.dataIdList = dictionary["dataIdList"]
760 
761  calib.updateMetadata()
762  return calib
763 
764  def toDict(self):
765  """Return a dictionary containing the provenance information.
766 
767  Returns
768  -------
769  dictionary : `dict`
770  Dictionary of provenance.
771  """
772  self.updateMetadataupdateMetadataupdateMetadata()
773 
774  outDict = {}
775 
776  metadata = self.getMetadatagetMetadata()
777  outDict["metadata"] = metadata
778  outDict["detectorName"] = self._detectorName_detectorName
779  outDict["detectorSerial"] = self._detectorSerial_detectorSerial
780  outDict["detectorId"] = self._detectorId_detectorId
781  outDict["instrument"] = self._instrument_instrument
782  outDict["calibType"] = self.calibTypecalibType
783  outDict["dimensions"] = list(self.dimensionsdimensions)
784  outDict["dataIdList"] = self.dataIdListdataIdList
785 
786  return outDict
787 
788  def toTable(self):
789  """Return a list of tables containing the provenance.
790 
791  This seems inefficient and slow, so this may not be the best
792  way to store the data.
793 
794  Returns
795  -------
796  tableList : `list` [`lsst.afw.table.Table`]
797  List of tables containing the provenance information
798 
799  """
800  tableList = []
801  self.updateMetadataupdateMetadataupdateMetadata(setDate=True, setCalibInfo=True)
802 
803  catalog = Table(rows=self.dataIdListdataIdList,
804  names=self.dimensionsdimensions)
805  filteredMetadata = {k: v for k, v in self.getMetadatagetMetadata().toDict().items() if v is not None}
806  catalog.meta = filteredMetadata
807  tableList.append(catalog)
808  return tableList
std::vector< SchemaItem< Flag > > * items
table::Key< int > type
Definition: Detector.cc:163
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
def calibInfoFromDict(self, dictionary)
Definition: calibType.py:261
def fromTable(cls, tableList, **kwargs)
Definition: calibType.py:555
def validate(self, other=None)
Definition: calibType.py:597
def writeText(self, filename, format="auto")
Definition: calibType.py:379
def setMetadata(self, metadata)
Definition: calibType.py:157
def writeFits(self, filename)
Definition: calibType.py:471
def requiredAttributes(self, value)
Definition: calibType.py:142
def readText(cls, filename, **kwargs)
Definition: calibType.py:346
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
Definition: calibType.py:181
def fromDict(cls, dictionary, **kwargs)
Definition: calibType.py:512
def __init__(self, camera=None, detector=None, log=None, **kwargs)
Definition: calibType.py:67
def determineCalibClass(cls, metadata, message)
Definition: calibType.py:316
def fromDetector(self, detector)
Definition: calibType.py:495
def readFits(cls, filename, **kwargs)
Definition: calibType.py:432
def __init__(self, calibType="unknown", **kwargs)
Definition: calibType.py:655
def updateMetadata(self, setDate=False, **kwargs)
Definition: calibType.py:670
def fromDataIds(self, dataIdList)
Definition: calibType.py:684
daf::base::PropertyList * list
Definition: fits.cc:913
daf::base::PropertySet * set
Definition: fits.cc:912
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33