32 from lsst.geom import Box2I, Point2I, Extent2I
33 from .applyLookupTable
import applyLookupTable
35 __all__ = [
"Linearizer",
36 "LinearizeBase",
"LinearizeLookupTable",
"LinearizeSquared",
37 "LinearizeProportional",
"LinearizePolynomial",
"LinearizeNone"]
41 """Parameter set for linearization.
43 These parameters are included in cameraGeom.Amplifier, but
44 should be accessible externally to allow for testing.
48 table : `numpy.array`, optional
49 Lookup table; a 2-dimensional array of floats:
50 - one row for each row index (value of coef[0] in the amplifier)
51 - one column for each image value
52 To avoid copying the table the last index should vary fastest
53 (numpy default "C" order)
54 detector : `lsst.afw.cameraGeom.Detector`
56 override : `bool`, optional
57 Override the parameters defined in the detector/amplifier.
58 log : `lsst.log.Log`, optional
59 Logger to handle messages.
64 Raised if the supplied table is not 2D, or if the table has fewer
65 columns than rows (indicating that the indices are swapped).
68 _OBSTYPE =
"linearizer"
69 """The dataset type name used for this class"""
71 def __init__(self, table=None, detector=None, override=False, log=None):
90 if len(table.shape) != 2:
91 raise RuntimeError(
"table shape = %s; must have two dimensions" % (table.shape,))
92 if table.shape[1] < table.shape[0]:
93 raise RuntimeError(
"table shape = %s; indices are switched" % (table.shape,))
94 self.
tableData = np.array(table, order=
"C")
100 """Apply linearity, setting parameters if necessary.
104 exposure : `lsst.afw.image.Exposure`
109 output : `lsst.pipe.base.Struct`
110 Linearization results:
112 Number of amplifiers considered.
114 Number of amplifiers linearized.
118 """Read linearity parameters from a detector.
122 detector : `lsst.afw.cameraGeom.detector`
123 Input detector with parameters to use.
125 self._detectorName = detector.getName()
126 self._detectorSerial = detector.getSerial()
127 self._detectorId = detector.getId()
128 self.populated =
True
131 for amp
in detector.getAmplifiers():
132 ampName = amp.getName()
133 self.linearityCoeffs[ampName] = amp.getLinearityCoeffs()
134 self.linearityType[ampName] = amp.getLinearityType()
135 self.linearityBBox[ampName] = amp.getBBox()
138 """Read linearity parameters from a dict.
143 Dictionary containing detector and amplifier information.
145 self.
setMetadata(metadata=yamlObject.get(
'metadata',
None))
152 for ampName
in yamlObject[
'amplifiers']:
153 amp = yamlObject[
'amplifiers'][ampName]
154 self.
linearityCoeffs[ampName] = np.array(amp.get(
'linearityCoeffs',
None), dtype=np.float64)
155 self.
linearityType[ampName] = amp.get(
'linearityType',
'None')
159 self.
tableData = yamlObject.get(
'tableData',
None)
166 """Return linearity parameters as a dict.
173 now = datetime.datetime.utcnow()
181 'amplifiers': dict()}
183 outDict[
'amplifiers'][ampName] = {
'linearityType': self.
linearityType[ampName],
187 outDict[
'tableData'] = self.
tableData.tolist()
193 """Read linearity from text file.
198 Name of the file containing the linearity definition.
201 linearity : `~lsst.ip.isr.linearize.Linearizer``
202 Linearity parameters.
205 with open(filename,
'r')
as f:
206 data = yaml.load(f, Loader=yaml.CLoader)
210 """Write the linearity model to a text file.
215 Name of the file to write.
220 The name of the file used to write the data.
225 Raised if filename does not end in ".yaml".
229 The file is written to YAML format and will include any metadata
230 associated with the `Linearity`.
233 if filename.lower().endswith((
".yaml")):
234 with open(filename,
'w')
as f:
235 yaml.dump(outDict, f)
237 raise RuntimeError(f
"Attempt to write to a file {filename} that does not end in '.yaml'")
243 """Read linearity from a FITS file.
247 table : `lsst.afw.table`
248 afwTable read from input file name.
249 tableExtTwo: `lsst.afw.table`, optional
250 afwTable read from second extension of input file name
254 linearity : `~lsst.ip.isr.linearize.Linearizer``
255 Linearity parameters.
259 The method reads a FITS file with 1 or 2 extensions. The metadata is read from the header of
260 extension 1, which must exist. Then the table is loaded, and the ['AMPLIFIER_NAME', 'TYPE',
261 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to
262 set each dictionary by looping over rows.
263 Eextension 2 is then attempted to read in the try block (which only exists for lookup tables).
264 It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row.
266 metadata = table.getMetadata()
267 schema = table.getSchema()
270 linDict[
'metadata'] = metadata
271 linDict[
'detectorId'] = metadata[
'DETECTOR']
272 linDict[
'detectorName'] = metadata[
'DETECTOR_NAME']
274 linDict[
'detectorSerial'] = metadata[
'DETECTOR_SERIAL']
276 linDict[
'detectorSerial'] =
'NOT SET'
277 linDict[
'amplifiers'] = dict()
280 ampNameKey = schema[
'AMPLIFIER_NAME'].asKey()
281 typeKey = schema[
'TYPE'].asKey()
282 coeffsKey = schema[
'COEFFS'].asKey()
283 x0Key = schema[
'BBOX_X0'].asKey()
284 y0Key = schema[
'BBOX_Y0'].asKey()
285 dxKey = schema[
'BBOX_DX'].asKey()
286 dyKey = schema[
'BBOX_DY'].asKey()
289 ampName = record[ampNameKey]
291 ampDict[
'linearityType'] = record[typeKey]
292 ampDict[
'linearityCoeffs'] = record[coeffsKey]
293 ampDict[
'linearityBBox'] =
Box2I(
Point2I(record[x0Key], record[y0Key]),
294 Extent2I(record[dxKey], record[dyKey]))
296 linDict[
'amplifiers'][ampName] = ampDict
298 if tableExtTwo
is not None:
299 lookupValuesKey =
'LOOKUP_VALUES'
300 linDict[
"tableData"] = [record[lookupValuesKey]
for record
in tableExtTwo]
306 """Read linearity from a FITS file.
311 Name of the file containing the linearity definition.
314 linearity : `~lsst.ip.isr.linearize.Linearizer``
315 Linearity parameters.
319 This method and `fromTable` read a FITS file with 1 or 2 extensions. The metadata is read from the
320 header of extension 1, which must exist. Then the table is loaded, and the ['AMPLIFIER_NAME',
321 'TYPE', 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to
322 set each dictionary by looping over rows.
323 Extension 2 is then attempted to read in the try block (which only exists for lookup tables).
324 It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row.
326 table = afwTable.BaseCatalog.readFits(filename)
329 tableExtTwo = afwTable.BaseCatalog.readFits(filename, 2)
335 """Produce linearity catalog
339 metadata : `lsst.daf.base.PropertyList`
344 catalog : `lsst.afw.table.BaseCatalog`
347 metadata[
"LINEARITY_SCHEMA"] =
"Linearity table"
348 metadata[
"LINEARITY_VERSION"] = 1
354 names = schema.addField(
"AMPLIFIER_NAME", type=
"String", size=16, doc=
"linearity amplifier name")
355 types = schema.addField(
"TYPE", type=
"String", size=16, doc=
"linearity type names")
356 coeffs = schema.addField(
"COEFFS", type=
"ArrayD", size=length, doc=
"linearity coefficients")
357 boxX = schema.addField(
"BBOX_X0", type=
"I", doc=
"linearity bbox minimum x")
358 boxY = schema.addField(
"BBOX_Y0", type=
"I", doc=
"linearity bbox minimum y")
359 boxDx = schema.addField(
"BBOX_DX", type=
"I", doc=
"linearity bbox x dimension")
360 boxDy = schema.addField(
"BBOX_DY", type=
"I", doc=
"linearity bbox y dimension")
366 catalog[ii][names] = ampName
368 catalog[ii][coeffs] = np.array(self.
linearityCoeffs[ampName], dtype=float)
371 catalog[ii][boxX], catalog[ii][boxY] = bbox.getMin()
372 catalog[ii][boxDx], catalog[ii][boxDy] = bbox.getDimensions()
373 catalog.setMetadata(metadata)
378 """Produce linearity catalog from table data
382 metadata : `lsst.daf.base.PropertyList`
387 catalog : `lsst.afw.table.BaseCatalog`
393 lut = schema.addField(
"LOOKUP_VALUES", type=
'ArrayF', size=dimensions[1],
394 doc=
"linearity lookup data")
396 catalog.resize(dimensions[0])
398 for ii
in range(dimensions[0]):
399 catalog[ii][lut] = np.array(self.
tableData[ii], dtype=np.float32)
401 metadata[
"LINEARITY_LOOKUP"] =
True
402 catalog.setMetadata(metadata)
407 """Write the linearity model to a FITS file.
412 Name of the file to write.
416 The file is written to YAML format and will include any metadata
417 associated with the `Linearity`.
419 now = datetime.datetime.utcnow()
423 catalog.writeFits(filename)
427 catalog.writeFits(filename,
"a")
432 """Retrieve metadata associated with this `Linearizer`.
436 meta : `lsst.daf.base.PropertyList`
437 Metadata. The returned `~lsst.daf.base.PropertyList` can be
438 modified by the caller and the changes will be written to
444 """Store a copy of the supplied metadata with the `Linearizer`.
448 metadata : `lsst.daf.base.PropertyList`, optional
449 Metadata to associate with the linearizer. Will be copied and
450 overwrite existing metadata. If not supplied the existing
451 metadata will be reset.
461 def updateMetadata(self, date=None, detectorId=None, detectorName=None, instrumentName=None, calibId=None,
463 """Update metadata keywords with new values.
467 date : `datetime.datetime`, optional
468 detectorId : `int`, optional
469 detectorName: `str`, optional
470 instrumentName : `str`, optional
471 calibId: `str`, optional
472 serial: detector serial, `str`, optional
476 mdSupplemental = dict()
479 mdSupplemental[
'CALIBDATE'] = date.isoformat()
480 mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat(),
481 mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat(),
483 mdSupplemental[
'DETECTOR'] = f
"{detectorId}"
485 mdSupplemental[
'DETECTOR_NAME'] = detectorName
487 mdSupplemental[
'INSTRUME'] = instrumentName
489 mdSupplemental[
'CALIB_ID'] = calibId
491 mdSupplemental[
'DETECTOR_SERIAL'] = serial
493 mdOriginal.update(mdSupplemental)
496 """Determine the linearity class to use from the type name.
500 linearityTypeName : str
501 String name of the linearity type that is needed.
505 linearityType : `~lsst.ip.isr.linearize.LinearizeBase`
506 The appropriate linearity class to use. If no matching class
507 is found, `None` is returned.
509 for t
in [LinearizeLookupTable,
512 LinearizeProportional,
514 if t.LinearityType == linearityTypeName:
519 """Validate linearity for a detector/amplifier.
523 detector : `lsst.afw.cameraGeom.Detector`, optional
524 Detector to validate, along with its amplifiers.
525 amplifier : `lsst.afw.cameraGeom.Amplifier`, optional
526 Single amplifier to validate.
531 Raised if there is a mismatch in linearity parameters, and
532 the cameraGeom parameters are not being overridden.
534 amplifiersToCheck = []
537 raise RuntimeError(
"Detector names don't match: %s != %s" %
540 raise RuntimeError(
"Detector IDs don't match: %s != %s" %
543 raise RuntimeError(
"Detector serial numbers don't match: %s != %s" %
546 raise RuntimeError(
"Detector number of amps = %s does not match saved value %s" %
547 (len(detector.getAmplifiers()),
549 amplifiersToCheck.extend(detector.getAmplifiers())
552 amplifiersToCheck.extend(amplifier)
554 for amp
in amplifiersToCheck:
555 ampName = amp.getName()
557 raise RuntimeError(
"Amplifier %s is not in linearity data" %
561 self.
log.
warn(
"Overriding amplifier defined linearityType (%s) for %s",
564 raise RuntimeError(
"Amplifier %s type %s does not match saved value %s" %
565 (ampName, amp.getLinearityType(), self.
linearityType[ampName]))
566 if (amp.getLinearityCoeffs().shape != self.
linearityCoeffs[ampName].shape
or not
567 np.allclose(amp.getLinearityCoeffs(), self.
linearityCoeffs[ampName], equal_nan=
True)):
569 self.
log.
warn(
"Overriding amplifier defined linearityCoeffs (%s) for %s",
572 raise RuntimeError(
"Amplifier %s coeffs %s does not match saved value %s" %
576 """Apply the linearity to an image.
578 If the linearity parameters are populated, use those,
579 otherwise use the values from the detector.
583 image : `~lsst.afw.image.image`
585 detector : `~lsst.afw.cameraGeom.detector`
586 Detector to use for linearity parameters if not already
588 log : `~lsst.log.Log`, optional
589 Log object to use for logging.
604 if linearizer
is not None:
606 success, outOfRange = linearizer()(ampView, **{
'coeffs': self.
linearityCoeffs[ampName],
609 numOutOfRange += outOfRange
612 elif log
is not None:
613 log.warn(
"Amplifier %s did not linearize.",
617 numLinearized=numLinearized,
618 numOutOfRange=numOutOfRange
623 """Abstract base class functor for correcting non-linearity.
625 Subclasses must define __call__ and set class variable
626 LinearityType to a string that will be used for linearity type in
627 the cameraGeom.Amplifier.linearityType field.
629 All linearity corrections should be defined in terms of an
630 additive correction, such that:
632 corrected_value = uncorrected_value + f(uncorrected_value)
638 """Correct non-linearity.
642 image : `lsst.afw.image.Image`
643 Image to be corrected
645 Dictionary of parameter keywords:
647 Coefficient vector (`list` or `numpy.array`).
649 Lookup table data (`numpy.array`).
651 Logger to handle messages (`lsst.log.Log`).
656 If true, a correction was applied successfully.
661 Raised if the linearity type listed in the
662 detector does not match the class type.
667 class LinearizeLookupTable(LinearizeBase):
668 """Correct non-linearity with a persisted lookup table.
670 The lookup table consists of entries such that given
671 "coefficients" c0, c1:
673 for each i,j of image:
675 colInd = int(c1 + uncorrImage[i,j])
676 corrImage[i,j] = uncorrImage[i,j] + table[rowInd, colInd]
678 - c0: row index; used to identify which row of the table to use
679 (typically one per amplifier, though one can have multiple
680 amplifiers use the same table)
681 - c1: column index offset; added to the uncorrected image value
682 before truncation; this supports tables that can handle
683 negative image values; also, if the c1 ends with .5 then
684 the nearest index is used instead of truncating to the
687 LinearityType =
"LookupTable"
690 """Correct for non-linearity.
694 image : `lsst.afw.image.Image`
695 Image to be corrected
697 Dictionary of parameter keywords:
699 Columnation vector (`list` or `numpy.array`).
701 Lookup table data (`numpy.array`).
703 Logger to handle messages (`lsst.log.Log`).
708 If true, a correction was applied successfully.
713 Raised if the requested row index is out of the table
718 rowInd, colIndOffset = kwargs[
'coeffs'][0:2]
719 table = kwargs[
'table']
722 numTableRows = table.shape[0]
724 if rowInd < 0
or rowInd > numTableRows:
725 raise RuntimeError(
"LinearizeLookupTable rowInd=%s not in range[0, %s)" %
726 (rowInd, numTableRows))
727 tableRow = table[rowInd, :]
730 if numOutOfRange > 0
and log
is not None:
731 log.warn(
"%s pixels were out of range of the linearization table",
733 if numOutOfRange < image.getArray().size:
734 return True, numOutOfRange
736 return False, numOutOfRange
740 """Correct non-linearity with a polynomial mode.
742 corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i)
744 where c_i are the linearity coefficients for each amplifier.
745 Lower order coefficients are not included as they duplicate other
746 calibration parameters:
748 A coefficient multiplied by uncorrImage**0 is equivalent to
749 bias level. Irrelevant for correcting non-linearity.
751 A coefficient multiplied by uncorrImage**1 is proportional
752 to the gain. Not necessary for correcting non-linearity.
754 LinearityType =
"Polynomial"
757 """Correct non-linearity.
761 image : `lsst.afw.image.Image`
762 Image to be corrected
764 Dictionary of parameter keywords:
766 Coefficient vector (`list` or `numpy.array`).
767 If the order of the polynomial is n, this list
768 should have a length of n-1 ("k0" and "k1" are
769 not needed for the correction).
771 Logger to handle messages (`lsst.log.Log`).
776 If true, a correction was applied successfully.
778 if not np.any(np.isfinite(kwargs[
'coeffs'])):
780 if not np.any(kwargs[
'coeffs']):
783 ampArray = image.getArray()
784 correction = np.zeros_like(ampArray)
785 for order, coeff
in enumerate(kwargs[
'coeffs'], start=2):
786 correction += coeff * np.power(ampArray, order)
787 ampArray += correction
793 """Correct non-linearity with a squared model.
795 corrImage = uncorrImage + c0*uncorrImage^2
797 where c0 is linearity coefficient 0 for each amplifier.
799 LinearityType =
"Squared"
802 """Correct for non-linearity.
806 image : `lsst.afw.image.Image`
807 Image to be corrected
809 Dictionary of parameter keywords:
811 Coefficient vector (`list` or `numpy.array`).
813 Logger to handle messages (`lsst.log.Log`).
818 If true, a correction was applied successfully.
821 sqCoeff = kwargs[
'coeffs'][0]
823 ampArr = image.getArray()
824 ampArr *= (1 + sqCoeff*ampArr)
831 """Do not correct non-linearity.
833 LinearityType =
"Proportional"
836 """Do not correct for non-linearity.
840 image : `lsst.afw.image.Image`
841 Image to be corrected
843 Dictionary of parameter keywords:
845 Coefficient vector (`list` or `numpy.array`).
847 Logger to handle messages (`lsst.log.Log`).
852 If true, a correction was applied successfully.
858 """Do not correct non-linearity.
860 LinearityType =
"None"
863 """Do not correct for non-linearity.
867 image : `lsst.afw.image.Image`
868 Image to be corrected
870 Dictionary of parameter keywords:
872 Coefficient vector (`list` or `numpy.array`).
874 Logger to handle messages (`lsst.log.Log`).
879 If true, a correction was applied successfully.