29 from astropy.table
import Table
30 from astropy.io
import fits
33 from lsst.utils.introspection
import get_full_type_name
37 __all__ = [
"IsrCalib",
"IsrProvenance"]
41 """Generic calibration type.
43 Subclasses must implement the toDict, fromDict, toTable, fromTable
44 methods that allow the calibration information to be converted
45 from dictionaries and afw tables. This will allow the calibration
46 to be persisted using the base class read/write methods.
48 The validate method is intended to provide a common way to check
49 that the calibration is valid (internally consistent) and
50 appropriate (usable with the intended data). The apply method is
51 intended to allow the calibration to be applied in a consistent
56 camera : `lsst.afw.cameraGeom.Camera`, optional
57 Camera to extract metadata from.
58 detector : `lsst.afw.cameraGeom.Detector`, optional
59 Detector to extract metadata from.
60 log : `logging.Logger`, optional
67 def __init__(self, camera=None, detector=None, log=None, **kwargs):
83 "_detectorName",
"_detectorSerial",
"_detectorId",
84 "_filter",
"_calibId",
"_metadata"])
86 self.
loglog = log
if log
else logging.getLogger(__name__)
90 self.
updateMetadataupdateMetadata(camera=camera, detector=detector)
93 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
96 """Calibration equivalence.
98 Running ``calib.log.setLevel(0)`` enables debug statements to
99 identify problematic fields.
101 if not isinstance(other, self.__class__):
102 self.
loglog.
debug(
"Incorrect class type: %s %s", self.__class__, other.__class__)
106 attrSelf = getattr(self, attr)
107 attrOther = getattr(other, attr)
109 if isinstance(attrSelf, dict):
111 if attrSelf.keys() != attrOther.keys():
112 self.
loglog.
debug(
"Dict Key Failure: %s %s %s", attr, attrSelf.keys(), attrOther.keys())
115 if not np.allclose(attrSelf[key], attrOther[key], equal_nan=
True):
116 self.
loglog.
debug(
"Array Failure: %s %s %s", key, attrSelf[key], attrOther[key])
118 elif isinstance(attrSelf, np.ndarray):
120 if not np.allclose(attrSelf, attrOther, equal_nan=
True):
121 self.
loglog.
debug(
"Array Failure: %s %s %s", attr, attrSelf, attrOther)
123 elif type(attrSelf) !=
type(attrOther):
124 if set([attrSelf, attrOther]) ==
set([
None,
""]):
127 self.
loglog.
debug(
"Type Failure: %s %s %s %s %s", attr,
type(attrSelf),
type(attrOther),
131 if attrSelf != attrOther:
132 self.
loglog.
debug(
"Value Failure: %s %s %s", attr, attrSelf, attrOther)
141 @requiredAttributes.setter
146 """Retrieve metadata associated with this calibration.
150 meta : `lsst.daf.base.PropertyList`
151 Metadata. The returned `~lsst.daf.base.PropertyList` can be
152 modified by the caller and the changes will be written to
158 """Store a copy of the supplied metadata with this calibration.
162 metadata : `lsst.daf.base.PropertyList`
163 Metadata to associate with the calibration. Will be copied and
164 overwrite existing metadata.
166 if metadata
is not None:
174 if isinstance(metadata, dict):
176 elif isinstance(metadata, PropertyList):
180 setCalibId=False, setCalibInfo=False, setDate=False,
182 """Update metadata keywords with new values.
186 camera : `lsst.afw.cameraGeom.Camera`, optional
187 Reference camera to use to set _instrument field.
188 detector : `lsst.afw.cameraGeom.Detector`, optional
189 Reference detector to use to set _detector* fields.
190 filterName : `str`, optional
191 Filter name to assign to this calibration.
192 setCalibId : `bool`, optional
193 Construct the _calibId field from other fields.
194 setCalibInfo : `bool`, optional
195 Set calibration parameters from metadata.
196 setDate : `bool`, optional
197 Ensure the metadata CALIBDATE fields are set to the current datetime.
198 kwargs : `dict` or `collections.abc.Mapping`, optional
199 Set of key=value pairs to assign to the metadata.
202 mdSupplemental = dict()
204 for k, v
in kwargs.items():
205 if isinstance(v, fits.card.Undefined):
227 self.
_filter_filter = filterName
230 date = datetime.datetime.now()
231 mdSupplemental[
"CALIBDATE"] = date.isoformat()
232 mdSupplemental[
"CALIB_CREATION_DATE"] = date.date().isoformat()
233 mdSupplemental[
"CALIB_CREATION_TIME"] = date.time().isoformat()
237 values.append(f
"instrument={self._instrument}")
if self.
_instrument_instrument
else None
238 values.append(f
"raftName={self._raftName}")
if self.
_raftName_raftName
else None
239 values.append(f
"detectorName={self._detectorName}")
if self.
_detectorName_detectorName
else None
240 values.append(f
"detector={self._detectorId}")
if self.
_detectorId_detectorId
else None
241 values.append(f
"filter={self._filter}")
if self.
_filter_filter
else None
243 calibDate = mdOriginal.get(
"CALIBDATE", mdSupplemental.get(
"CALIBDATE",
None))
244 values.append(f
"calibDate={calibDate}")
if calibDate
else None
246 self.
_calibId_calibId =
" ".join(values)
256 self.
_metadata_metadata[
"CALIBCLS"] = get_full_type_name(self)
258 mdSupplemental.update(kwargs)
259 mdOriginal.update(mdSupplemental)
262 """Handle common keywords.
264 This isn't an ideal solution, but until all calibrations
265 expect to find everything in the metadata, they still need to
266 search through dictionaries.
270 dictionary : `dict` or `lsst.daf.base.PropertyList`
271 Source for the common keywords.
276 Raised if the dictionary does not match the expected OBSTYPE.
280 def search(haystack, needles):
281 """Search dictionary 'haystack' for an entry in 'needles'
283 test = [haystack.get(x)
for x
in needles]
284 test =
set([x
for x
in test
if x
is not None])
286 if "metadata" in haystack:
287 return search(haystack[
"metadata"], needles)
291 value =
list(test)[0]
297 raise ValueError(f
"Too many values found: {len(test)} {test} {needles}")
299 if "metadata" in dictionary:
300 metadata = dictionary[
"metadata"]
302 if self.
_OBSTYPE_OBSTYPE != metadata[
"OBSTYPE"]:
303 raise RuntimeError(f
"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
304 f
"found {metadata['OBSTYPE']}")
306 self.
_instrument_instrument = search(dictionary, [
"INSTRUME",
"instrument"])
307 self.
_raftName_raftName = search(dictionary, [
"RAFTNAME"])
308 self.
_slotName_slotName = search(dictionary, [
"SLOTNAME"])
309 self.
_detectorId_detectorId = search(dictionary, [
"DETECTOR",
"detectorId"])
310 self.
_detectorName_detectorName = search(dictionary, [
"DET_NAME",
"DETECTOR_NAME",
"detectorName"])
311 self.
_detectorSerial_detectorSerial = search(dictionary, [
"DET_SER",
"DETECTOR_SERIAL",
"detectorSerial"])
312 self.
_filter_filter = search(dictionary, [
"FILTER",
"filterName"])
313 self.
_calibId_calibId = search(dictionary, [
"CALIB_ID"])
317 """Attempt to find calibration class in metadata.
321 metadata : `dict` or `lsst.daf.base.PropertyList`
322 Metadata possibly containing a calibration class entry.
324 Message to include in any errors.
328 calibClass : `object`
329 The class to use to read the file contents. Should be an
330 `lsst.ip.isr.IsrCalib` subclass.
335 Raised if the resulting calibClass is the base
336 `lsst.ip.isr.IsrClass` (which does not implement the
339 calibClassName = metadata.get(
"CALIBCLS")
340 calibClass =
doImport(calibClassName)
if calibClassName
is not None else cls
341 if calibClass
is IsrCalib:
342 raise ValueError(f
"Cannot use base class to read calibration data: {msg}")
347 """Read calibration representation from a yaml/ecsv file.
352 Name of the file containing the calibration definition.
353 kwargs : `dict` or collections.abc.Mapping`, optional
354 Set of key=value pairs to pass to the ``fromDict`` or
355 ``fromTable`` methods.
359 calib : `~lsst.ip.isr.IsrCalibType`
365 Raised if the filename does not end in ".ecsv" or ".yaml".
367 if filename.endswith((
".ecsv",
".ECSV")):
368 data = Table.read(filename, format=
"ascii.ecsv")
370 return calibClass.fromTable([data], **kwargs)
371 elif filename.endswith((
".yaml",
".YAML")):
372 with open(filename,
"r")
as f:
373 data = yaml.load(f, Loader=yaml.CLoader)
375 return calibClass.fromDict(data, **kwargs)
377 raise RuntimeError(f
"Unknown filename extension: {filename}")
380 """Write the calibration data to a text file.
385 Name of the file to write.
387 Format to write the file as. Supported values are:
388 ``"auto"`` : Determine filetype from filename.
389 ``"yaml"`` : Write as yaml.
390 ``"ecsv"`` : Write as ecsv.
394 The name of the file used to write the data. This may
395 differ from the input if the format is explicitly chosen.
400 Raised if filename does not end in a known extension, or
401 if all information cannot be written.
405 The file is written to YAML/ECSV format and will include any
408 if format ==
"yaml" or (format ==
"auto" and filename.lower().endswith((
".yaml",
".YAML"))):
409 outDict = self.
toDicttoDict()
410 path, ext = os.path.splitext(filename)
411 filename = path +
".yaml"
412 with open(filename,
"w")
as f:
413 yaml.dump(outDict, f)
414 elif format ==
"ecsv" or (format ==
"auto" and filename.lower().endswith((
".ecsv",
".ECSV"))):
415 tableList = self.
toTabletoTable()
416 if len(tableList) > 1:
419 raise RuntimeError(f
"Unable to persist {len(tableList)}tables in ECSV format.")
422 path, ext = os.path.splitext(filename)
423 filename = path +
".ecsv"
424 table.write(filename, format=
"ascii.ecsv")
426 raise RuntimeError(f
"Attempt to write to a file {filename} "
427 "that does not end in '.yaml' or '.ecsv'")
433 """Read calibration data from a FITS file.
438 Filename to read data from.
439 kwargs : `dict` or collections.abc.Mapping`, optional
440 Set of key=value pairs to pass to the ``fromTable``
445 calib : `lsst.ip.isr.IsrCalib`
446 Calibration contained within the file.
449 tableList.append(Table.read(filename, hdu=1))
454 with warnings.catch_warnings():
455 warnings.simplefilter(
"error")
457 newTable = Table.read(filename, hdu=extNum)
458 tableList.append(newTable)
463 for table
in tableList:
464 for k, v
in table.meta.items():
465 if isinstance(v, fits.card.Undefined):
469 return calibClass.fromTable(tableList, **kwargs)
472 """Write calibration data to a FITS file.
477 Filename to write data to.
482 The name of the file used to write the data.
485 tableList = self.
toTabletoTable()
486 with warnings.catch_warnings():
487 warnings.filterwarnings(
"ignore", category=Warning, module=
"astropy.io")
488 astropyList = [fits.table_to_hdu(table)
for table
in tableList]
489 astropyList.insert(0, fits.PrimaryHDU())
491 writer = fits.HDUList(astropyList)
492 writer.writeto(filename, overwrite=
True)
496 """Modify the calibration parameters to match the supplied detector.
500 detector : `lsst.afw.cameraGeom.Detector`
501 Detector to use to set parameters from.
506 This needs to be implemented by subclasses for each
509 raise NotImplementedError(
"Must be implemented by subclass.")
513 """Construct a calibration from a dictionary of properties.
515 Must be implemented by the specific calibration subclasses.
520 Dictionary of properties.
521 kwargs : `dict` or collections.abc.Mapping`, optional
522 Set of key=value options.
526 calib : `lsst.ip.isr.CalibType`
527 Constructed calibration.
531 NotImplementedError :
532 Raised if not implemented.
534 raise NotImplementedError(
"Must be implemented by subclass.")
537 """Return a dictionary containing the calibration properties.
539 The dictionary should be able to be round-tripped through
545 Dictionary of properties.
549 NotImplementedError :
550 Raised if not implemented.
552 raise NotImplementedError(
"Must be implemented by subclass.")
556 """Construct a calibration from a dictionary of properties.
558 Must be implemented by the specific calibration subclasses.
562 tableList : `list` [`lsst.afw.table.Table`]
563 List of tables of properties.
564 kwargs : `dict` or collections.abc.Mapping`, optional
565 Set of key=value options.
569 calib : `lsst.ip.isr.CalibType`
570 Constructed calibration.
574 NotImplementedError :
575 Raised if not implemented.
577 raise NotImplementedError(
"Must be implemented by subclass.")
580 """Return a list of tables containing the calibration properties.
582 The table list should be able to be round-tripped through
587 tableList : `list` [`lsst.afw.table.Table`]
588 List of tables of properties.
592 NotImplementedError :
593 Raised if not implemented.
595 raise NotImplementedError(
"Must be implemented by subclass.")
598 """Validate that this calibration is defined and can be used.
602 other : `object`, optional
603 Thing to validate against.
608 Returns true if the calibration is valid and appropriate.
613 """Method to apply the calibration to the target object.
618 Thing to validate against.
623 Returns true if the calibration was applied correctly.
627 NotImplementedError :
628 Raised if not implemented.
630 raise NotImplementedError(
"Must be implemented by subclass.")
634 """Class for the provenance of data used to construct calibration.
636 Provenance is not really a calibration, but we would like to
637 record this when constructing the calibration, and it provides an
638 example of the base calibration class.
642 instrument : `str`, optional
643 Name of the instrument the data was taken with.
644 calibType : `str`, optional
645 Type of calibration this provenance was generated for.
646 detectorName : `str`, optional
647 Name of the detector this calibration is for.
648 detectorSerial : `str`, optional
649 Identifier for the detector.
652 _OBSTYPE =
"IsrProvenance"
665 return f
"{self.__class__.__name__}(obstype={self._OBSTYPE}, calibType={self.calibType}, )"
668 return super().
__eq__(other)
671 """Update calibration metadata.
675 setDate : `bool, optional
676 Update the CALIBDATE fields in the metadata to the current
677 time. Defaults to False.
678 kwargs : `dict` or `collections.abc.Mapping`, optional
679 Other keyword parameters to set in the metadata.
681 kwargs[
"calibType"] = self.
calibTypecalibType
685 """Update provenance from dataId List.
689 dataIdList : `list` [`lsst.daf.butler.DataId`]
690 List of dataIds used in generating this calibration.
692 for dataId
in dataIdList:
700 """Construct provenance from table list.
704 tableList : `list` [`lsst.afw.table.Table`]
705 List of tables to construct the provenance from.
709 provenance : `lsst.ip.isr.IsrProvenance`
710 The provenance defined in the tables.
713 metadata = table.meta
715 inDict[
"metadata"] = metadata
716 inDict[
"calibType"] = metadata[
"calibType"]
717 inDict[
"dimensions"] =
set()
718 inDict[
"dataIdList"] =
list()
721 for colName
in table.columns:
722 schema[colName.lower()] = colName
723 inDict[
"dimensions"].add(colName.lower())
724 inDict[
"dimensions"] = sorted(inDict[
"dimensions"])
728 for dim
in sorted(inDict[
"dimensions"]):
729 entry[dim] = row[schema[dim]]
730 inDict[
"dataIdList"].
append(entry)
736 """Construct provenance from a dictionary.
741 Dictionary of provenance parameters.
745 provenance : `lsst.ip.isr.IsrProvenance`
746 The provenance defined in the tables.
749 if calib._OBSTYPE != dictionary[
"metadata"][
"OBSTYPE"]:
750 raise RuntimeError(f
"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
751 f
"found {dictionary['metadata']['OBSTYPE']}")
752 calib.updateMetadata(setDate=
False, setCalibInfo=
True, **dictionary[
"metadata"])
757 calib.calibType = dictionary[
"calibType"]
758 calib.dimensions =
set(dictionary[
"dimensions"])
759 calib.dataIdList = dictionary[
"dataIdList"]
761 calib.updateMetadata()
765 """Return a dictionary containing the provenance information.
770 Dictionary of provenance.
777 outDict[
"metadata"] = metadata
780 outDict[
"detectorId"] = self.
_detectorId_detectorId
781 outDict[
"instrument"] = self.
_instrument_instrument
782 outDict[
"calibType"] = self.
calibTypecalibType
784 outDict[
"dataIdList"] = self.
dataIdListdataIdList
789 """Return a list of tables containing the provenance.
791 This seems inefficient and slow, so this may not be the best
792 way to store the data.
796 tableList : `list` [`lsst.afw.table.Table`]
797 List of tables containing the provenance information
803 catalog = Table(rows=self.
dataIdListdataIdList,
806 catalog.meta = filteredMetadata
807 tableList.append(catalog)
std::vector< SchemaItem< Flag > > * items
Class for storing ordered metadata with comments.
def calibInfoFromDict(self, dictionary)
def fromTable(cls, tableList, **kwargs)
def validate(self, other=None)
def writeText(self, filename, format="auto")
def setMetadata(self, metadata)
def writeFits(self, filename)
def requiredAttributes(self, value)
def readText(cls, filename, **kwargs)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDict(cls, dictionary, **kwargs)
def __init__(self, camera=None, detector=None, log=None, **kwargs)
def determineCalibClass(cls, metadata, message)
def fromDetector(self, detector)
def requiredAttributes(self)
def readFits(cls, filename, **kwargs)
def __init__(self, calibType="unknown", **kwargs)
def fromDict(cls, dictionary)
def updateMetadata(self, setDate=False, **kwargs)
def fromTable(cls, tableList)
def fromDataIds(self, dataIdList)
daf::base::PropertyList * list
daf::base::PropertySet * set
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.