32 from lsst.geom import Box2I, Point2I, Extent2I
 
   33 from .applyLookupTable 
import applyLookupTable
 
   35 __all__ = [
"Linearizer",
 
   36            "LinearizeBase", 
"LinearizeLookupTable", 
"LinearizeSquared",
 
   37            "LinearizeProportional", 
"LinearizePolynomial", 
"LinearizeNone"]
 
   41     """Parameter set for linearization. 
   43     These parameters are included in cameraGeom.Amplifier, but 
   44     should be accessible externally to allow for testing. 
   48     table : `numpy.array`, optional 
   49         Lookup table; a 2-dimensional array of floats: 
   50             - one row for each row index (value of coef[0] in the amplifier) 
   51             - one column for each image value 
   52         To avoid copying the table the last index should vary fastest 
   53         (numpy default "C" order) 
   54     detector : `lsst.afw.cameraGeom.Detector` 
   56     override : `bool`, optional 
   57         Override the parameters defined in the detector/amplifier. 
   58     log : `lsst.log.Log`, optional 
   59         Logger to handle messages. 
   64         Raised if the supplied table is not 2D, or if the table has fewer 
   65         columns than rows (indicating that the indices are swapped). 
   68     _OBSTYPE = 
"linearizer" 
   69     """The dataset type name used for this class""" 
   71     def __init__(self, table=None, detector=None, override=False, log=None):
 
   90             if len(table.shape) != 2:
 
   91                 raise RuntimeError(
"table shape = %s; must have two dimensions" % (table.shape,))
 
   92             if table.shape[1] < table.shape[0]:
 
   93                 raise RuntimeError(
"table shape = %s; indices are switched" % (table.shape,))
 
   94             self.
tableData = np.array(table, order=
"C")
 
  100         """Apply linearity, setting parameters if necessary. 
  104         exposure : `lsst.afw.image.Exposure` 
  109         output : `lsst.pipe.base.Struct` 
  110             Linearization results: 
  112                 Number of amplifiers considered. 
  114                 Number of amplifiers linearized. 
  118         """Read linearity parameters from a detector. 
  122         detector : `lsst.afw.cameraGeom.detector` 
  123             Input detector with parameters to use. 
  125         self._detectorName = detector.getName()
 
  126         self._detectorSerial = detector.getSerial()
 
  127         self._detectorId = detector.getId()
 
  128         self.populated = 
True 
  131         for amp 
in detector.getAmplifiers():
 
  132             ampName = amp.getName()
 
  133             self.linearityCoeffs[ampName] = amp.getLinearityCoeffs()
 
  134             self.linearityType[ampName] = amp.getLinearityType()
 
  135             self.linearityBBox[ampName] = amp.getBBox()
 
  138         """Read linearity parameters from a dict. 
  143             Dictionary containing detector and amplifier information. 
  145         self.
setMetadata(metadata=yamlObject.get(
'metadata', 
None))
 
  152         for ampName 
in yamlObject[
'amplifiers']:
 
  153             amp = yamlObject[
'amplifiers'][ampName]
 
  154             self.
linearityCoeffs[ampName] = np.array(amp.get(
'linearityCoeffs', 
None), dtype=np.float64)
 
  155             self.
linearityType[ampName] = amp.get(
'linearityType', 
'None')
 
  159             self.
tableData = yamlObject.get(
'tableData', 
None)
 
  166         """Return linearity parameters as a dict. 
  173         now = datetime.datetime.utcnow()
 
  181                    'amplifiers': dict()}
 
  183             outDict[
'amplifiers'][ampName] = {
'linearityType': self.
linearityType[ampName],
 
  187             outDict[
'tableData'] = self.
tableData.tolist()
 
  193         """Read linearity from text file. 
  198             Name of the file containing the linearity definition. 
  201         linearity : `~lsst.ip.isr.linearize.Linearizer`` 
  202             Linearity parameters. 
  205         with open(filename, 
'r') 
as f:
 
  206             data = yaml.load(f, Loader=yaml.CLoader)
 
  210         """Write the linearity model to a text file. 
  215             Name of the file to write. 
  220             The name of the file used to write the data. 
  225             Raised if filename does not end in ".yaml". 
  229         The file is written to YAML format and will include any metadata 
  230         associated with the `Linearity`. 
  233         if filename.lower().endswith((
".yaml")):
 
  234             with open(filename, 
'w') 
as f:
 
  235                 yaml.dump(outDict, f)
 
  237             raise RuntimeError(f
"Attempt to write to a file {filename} that does not end in '.yaml'")
 
  243         """Read linearity from a FITS file. 
  247         table : `lsst.afw.table` 
  248             afwTable read from input file name. 
  249         tableExtTwo: `lsst.afw.table`, optional 
  250             afwTable read from second extension of input file name 
  254         linearity : `~lsst.ip.isr.linearize.Linearizer`` 
  255             Linearity parameters. 
  259         The method reads a FITS file with 1 or 2 extensions. The metadata is read from the header of 
  260         extension 1, which must exist.  Then the table is loaded, and  the ['AMPLIFIER_NAME', 'TYPE', 
  261         'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to 
  262         set each dictionary by looping over rows. 
  263         Eextension 2 is then attempted to read in the try block (which only exists for lookup tables). 
  264         It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row. 
  266         metadata = table.getMetadata()
 
  267         schema = table.getSchema()
 
  270         linDict[
'metadata'] = metadata
 
  271         linDict[
'detectorId'] = metadata[
'DETECTOR']
 
  272         linDict[
'detectorName'] = metadata[
'DETECTOR_NAME']
 
  274             linDict[
'detectorSerial'] = metadata[
'DETECTOR_SERIAL']
 
  276             linDict[
'detectorSerial'] = 
'NOT SET' 
  277         linDict[
'amplifiers'] = dict()
 
  280         ampNameKey = schema[
'AMPLIFIER_NAME'].asKey()
 
  281         typeKey = schema[
'TYPE'].asKey()
 
  282         coeffsKey = schema[
'COEFFS'].asKey()
 
  283         x0Key = schema[
'BBOX_X0'].asKey()
 
  284         y0Key = schema[
'BBOX_Y0'].asKey()
 
  285         dxKey = schema[
'BBOX_DX'].asKey()
 
  286         dyKey = schema[
'BBOX_DY'].asKey()
 
  289             ampName = record[ampNameKey]
 
  291             ampDict[
'linearityType'] = record[typeKey]
 
  292             ampDict[
'linearityCoeffs'] = record[coeffsKey]
 
  293             ampDict[
'linearityBBox'] = 
Box2I(
Point2I(record[x0Key], record[y0Key]),
 
  294                                              Extent2I(record[dxKey], record[dyKey]))
 
  296             linDict[
'amplifiers'][ampName] = ampDict
 
  298         if tableExtTwo 
is not None:
 
  299             lookupValuesKey = 
'LOOKUP_VALUES' 
  300             linDict[
"tableData"] = [record[lookupValuesKey] 
for record 
in tableExtTwo]
 
  306         """Read linearity from a FITS file. 
  311             Name of the file containing the linearity definition. 
  314         linearity : `~lsst.ip.isr.linearize.Linearizer`` 
  315             Linearity parameters. 
  319         This method and `fromTable` read a FITS file with 1 or 2 extensions. The metadata is read from the 
  320         header of extension 1, which must exist.  Then the table is loaded, and the ['AMPLIFIER_NAME', 
  321         'TYPE', 'COEFFS', 'BBOX_X0', 'BBOX_Y0', 'BBOX_DX', 'BBOX_DY'] columns are read and used to 
  322         set each dictionary by looping over rows. 
  323         Extension 2 is then attempted to read in the try block (which only exists for lookup tables). 
  324         It has a column named 'LOOKUP_VALUES' that contains a vector of the lookup entries in each row. 
  326         table = afwTable.BaseCatalog.readFits(filename)
 
  329             tableExtTwo = afwTable.BaseCatalog.readFits(filename, 2)
 
  335         """Produce linearity catalog 
  339         metadata : `lsst.daf.base.PropertyList` 
  344         catalog : `lsst.afw.table.BaseCatalog` 
  347         metadata[
"LINEARITY_SCHEMA"] = 
"Linearity table" 
  348         metadata[
"LINEARITY_VERSION"] = 1
 
  354         names = schema.addField(
"AMPLIFIER_NAME", type=
"String", size=16, doc=
"linearity amplifier name")
 
  355         types = schema.addField(
"TYPE", type=
"String", size=16, doc=
"linearity type names")
 
  356         coeffs = schema.addField(
"COEFFS", type=
"ArrayD", size=length, doc=
"linearity coefficients")
 
  357         boxX = schema.addField(
"BBOX_X0", type=
"I", doc=
"linearity bbox minimum x")
 
  358         boxY = schema.addField(
"BBOX_Y0", type=
"I", doc=
"linearity bbox minimum y")
 
  359         boxDx = schema.addField(
"BBOX_DX", type=
"I", doc=
"linearity bbox x dimension")
 
  360         boxDy = schema.addField(
"BBOX_DY", type=
"I", doc=
"linearity bbox y dimension")
 
  366             catalog[ii][names] = ampName
 
  368             catalog[ii][coeffs] = np.array(self.
linearityCoeffs[ampName], dtype=float)
 
  371             catalog[ii][boxX], catalog[ii][boxY] = bbox.getMin()
 
  372             catalog[ii][boxDx], catalog[ii][boxDy] = bbox.getDimensions()
 
  373         catalog.setMetadata(metadata)
 
  378         """Produce linearity catalog from table data 
  382         metadata : `lsst.daf.base.PropertyList` 
  387         catalog : `lsst.afw.table.BaseCatalog` 
  393         lut = schema.addField(
"LOOKUP_VALUES", type=
'ArrayF', size=dimensions[1],
 
  394                               doc=
"linearity lookup data")
 
  396         catalog.resize(dimensions[0])
 
  398         for ii 
in range(dimensions[0]):
 
  399             catalog[ii][lut] = np.array(self.
tableData[ii], dtype=np.float32)
 
  401         metadata[
"LINEARITY_LOOKUP"] = 
True 
  402         catalog.setMetadata(metadata)
 
  407         """Write the linearity model to a FITS file. 
  412             Name of the file to write. 
  416         The file is written to YAML format and will include any metadata 
  417         associated with the `Linearity`. 
  419         now = datetime.datetime.utcnow()
 
  423         catalog.writeFits(filename)
 
  427             catalog.writeFits(filename, 
"a")
 
  432         """Retrieve metadata associated with this `Linearizer`. 
  436         meta : `lsst.daf.base.PropertyList` 
  437             Metadata. The returned `~lsst.daf.base.PropertyList` can be 
  438             modified by the caller and the changes will be written to 
  444         """Store a copy of the supplied metadata with the `Linearizer`. 
  448         metadata : `lsst.daf.base.PropertyList`, optional 
  449             Metadata to associate with the linearizer.  Will be copied and 
  450             overwrite existing metadata.  If not supplied the existing 
  451             metadata will be reset. 
  461     def updateMetadata(self, date=None, detectorId=None, detectorName=None, instrumentName=None, calibId=None,
 
  463         """Update metadata keywords with new values. 
  467         date : `datetime.datetime`, optional 
  468         detectorId : `int`, optional 
  469         detectorName: `str`, optional 
  470         instrumentName : `str`, optional 
  471         calibId: `str`, optional 
  472         serial: detector serial, `str`, optional 
  476         mdSupplemental = dict()
 
  479             mdSupplemental[
'CALIBDATE'] = date.isoformat()
 
  480             mdSupplemental[
'CALIB_CREATION_DATE'] = date.date().isoformat(),
 
  481             mdSupplemental[
'CALIB_CREATION_TIME'] = date.time().isoformat(),
 
  483             mdSupplemental[
'DETECTOR'] = f
"{detectorId}" 
  485             mdSupplemental[
'DETECTOR_NAME'] = detectorName
 
  487             mdSupplemental[
'INSTRUME'] = instrumentName
 
  489             mdSupplemental[
'CALIB_ID'] = calibId
 
  491             mdSupplemental[
'DETECTOR_SERIAL'] = serial
 
  493         mdOriginal.update(mdSupplemental)
 
  496         """Determine the linearity class to use from the type name. 
  500         linearityTypeName : str 
  501             String name of the linearity type that is needed. 
  505         linearityType : `~lsst.ip.isr.linearize.LinearizeBase` 
  506             The appropriate linearity class to use.  If no matching class 
  507             is found, `None` is returned. 
  509         for t 
in [LinearizeLookupTable,
 
  512                   LinearizeProportional,
 
  514             if t.LinearityType == linearityTypeName:
 
  519         """Validate linearity for a detector/amplifier. 
  523         detector : `lsst.afw.cameraGeom.Detector`, optional 
  524             Detector to validate, along with its amplifiers. 
  525         amplifier : `lsst.afw.cameraGeom.Amplifier`, optional 
  526             Single amplifier to validate. 
  531             Raised if there is a mismatch in linearity parameters, and 
  532             the cameraGeom parameters are not being overridden. 
  534         amplifiersToCheck = []
 
  537                 raise RuntimeError(
"Detector names don't match: %s != %s" %
 
  540                 raise RuntimeError(
"Detector IDs don't match: %s != %s" %
 
  543                 raise RuntimeError(
"Detector serial numbers don't match: %s != %s" %
 
  546                 raise RuntimeError(
"Detector number of amps = %s does not match saved value %s" %
 
  547                                    (len(detector.getAmplifiers()),
 
  549             amplifiersToCheck.extend(detector.getAmplifiers())
 
  552             amplifiersToCheck.extend(amplifier)
 
  554         for amp 
in amplifiersToCheck:
 
  555             ampName = amp.getName()
 
  557                 raise RuntimeError(
"Amplifier %s is not in linearity data" %
 
  561                     self.
log.
warn(
"Overriding amplifier defined linearityType (%s) for %s",
 
  564                     raise RuntimeError(
"Amplifier %s type %s does not match saved value %s" %
 
  565                                        (ampName, amp.getLinearityType(), self.
linearityType[ampName]))
 
  566             if (amp.getLinearityCoeffs().shape != self.
linearityCoeffs[ampName].shape 
or not 
  567                     np.allclose(amp.getLinearityCoeffs(), self.
linearityCoeffs[ampName], equal_nan=
True)):
 
  569                     self.
log.
warn(
"Overriding amplifier defined linearityCoeffs (%s) for %s",
 
  572                     raise RuntimeError(
"Amplifier %s coeffs %s does not match saved value %s" %
 
  576         """Apply the linearity to an image. 
  578         If the linearity parameters are populated, use those, 
  579         otherwise use the values from the detector. 
  583         image : `~lsst.afw.image.image` 
  585         detector : `~lsst.afw.cameraGeom.detector` 
  586             Detector to use for linearity parameters if not already 
  588         log : `~lsst.log.Log`, optional 
  589             Log object to use for logging. 
  604             if linearizer 
is not None:
 
  606                 success, outOfRange = linearizer()(ampView, **{
'coeffs': self.
linearityCoeffs[ampName],
 
  609                 numOutOfRange += outOfRange
 
  612                 elif log 
is not None:
 
  613                     log.warn(
"Amplifier %s did not linearize.",
 
  617             numLinearized=numLinearized,
 
  618             numOutOfRange=numOutOfRange
 
  623     """Abstract base class functor for correcting non-linearity. 
  625     Subclasses must define __call__ and set class variable 
  626     LinearityType to a string that will be used for linearity type in 
  627     the cameraGeom.Amplifier.linearityType field. 
  629     All linearity corrections should be defined in terms of an 
  630     additive correction, such that: 
  632     corrected_value = uncorrected_value + f(uncorrected_value) 
  638         """Correct non-linearity. 
  642         image : `lsst.afw.image.Image` 
  643             Image to be corrected 
  645             Dictionary of parameter keywords: 
  647                 Coefficient vector (`list` or `numpy.array`). 
  649                 Lookup table data (`numpy.array`). 
  651                 Logger to handle messages (`lsst.log.Log`). 
  656             If true, a correction was applied successfully. 
  661             Raised if the linearity type listed in the 
  662             detector does not match the class type. 
  667 class LinearizeLookupTable(LinearizeBase):
 
  668     """Correct non-linearity with a persisted lookup table. 
  670     The lookup table consists of entries such that given 
  671     "coefficients" c0, c1: 
  673     for each i,j of image: 
  675         colInd = int(c1 + uncorrImage[i,j]) 
  676         corrImage[i,j] = uncorrImage[i,j] + table[rowInd, colInd] 
  678     - c0: row index; used to identify which row of the table to use 
  679             (typically one per amplifier, though one can have multiple 
  680             amplifiers use the same table) 
  681     - c1: column index offset; added to the uncorrected image value 
  682             before truncation; this supports tables that can handle 
  683             negative image values; also, if the c1 ends with .5 then 
  684             the nearest index is used instead of truncating to the 
  687     LinearityType = 
"LookupTable" 
  690         """Correct for non-linearity. 
  694         image : `lsst.afw.image.Image` 
  695             Image to be corrected 
  697             Dictionary of parameter keywords: 
  699                 Columnation vector (`list` or `numpy.array`). 
  701                 Lookup table data (`numpy.array`). 
  703                 Logger to handle messages (`lsst.log.Log`). 
  708             If true, a correction was applied successfully. 
  713             Raised if the requested row index is out of the table 
  718         rowInd, colIndOffset = kwargs[
'coeffs'][0:2]
 
  719         table = kwargs[
'table']
 
  722         numTableRows = table.shape[0]
 
  724         if rowInd < 0 
or rowInd > numTableRows:
 
  725             raise RuntimeError(
"LinearizeLookupTable rowInd=%s not in range[0, %s)" %
 
  726                                (rowInd, numTableRows))
 
  727         tableRow = table[rowInd, :]
 
  730         if numOutOfRange > 0 
and log 
is not None:
 
  731             log.warn(
"%s pixels were out of range of the linearization table",
 
  733         if numOutOfRange < image.getArray().size:
 
  734             return True, numOutOfRange
 
  736             return False, numOutOfRange
 
  740     """Correct non-linearity with a polynomial mode. 
  742     corrImage = uncorrImage + sum_i c_i uncorrImage^(2 + i) 
  744     where c_i are the linearity coefficients for each amplifier. 
  745     Lower order coefficients are not included as they duplicate other 
  746     calibration parameters: 
  748             A coefficient multiplied by uncorrImage**0 is equivalent to 
  749             bias level.  Irrelevant for correcting non-linearity. 
  751             A coefficient multiplied by uncorrImage**1 is proportional 
  752             to the gain.  Not necessary for correcting non-linearity. 
  754     LinearityType = 
"Polynomial" 
  757         """Correct non-linearity. 
  761         image : `lsst.afw.image.Image` 
  762             Image to be corrected 
  764             Dictionary of parameter keywords: 
  766                 Coefficient vector (`list` or `numpy.array`). 
  767                 If the order of the polynomial is n, this list 
  768                 should have a length of n-1 ("k0" and "k1" are 
  769                 not needed for the correction). 
  771                 Logger to handle messages (`lsst.log.Log`). 
  776             If true, a correction was applied successfully. 
  778         if not np.any(np.isfinite(kwargs[
'coeffs'])):
 
  780         if not np.any(kwargs[
'coeffs']):
 
  783         ampArray = image.getArray()
 
  784         correction = np.zeros_like(ampArray)
 
  785         for order, coeff 
in enumerate(kwargs[
'coeffs'], start=2):
 
  786             correction += coeff * np.power(ampArray, order)
 
  787         ampArray += correction
 
  793     """Correct non-linearity with a squared model. 
  795     corrImage = uncorrImage + c0*uncorrImage^2 
  797     where c0 is linearity coefficient 0 for each amplifier. 
  799     LinearityType = 
"Squared" 
  802         """Correct for non-linearity. 
  806         image : `lsst.afw.image.Image` 
  807             Image to be corrected 
  809             Dictionary of parameter keywords: 
  811                 Coefficient vector (`list` or `numpy.array`). 
  813                 Logger to handle messages (`lsst.log.Log`). 
  818             If true, a correction was applied successfully. 
  821         sqCoeff = kwargs[
'coeffs'][0]
 
  823             ampArr = image.getArray()
 
  824             ampArr *= (1 + sqCoeff*ampArr)
 
  831     """Do not correct non-linearity. 
  833     LinearityType = 
"Proportional" 
  836         """Do not correct for non-linearity. 
  840         image : `lsst.afw.image.Image` 
  841             Image to be corrected 
  843             Dictionary of parameter keywords: 
  845                 Coefficient vector (`list` or `numpy.array`). 
  847                 Logger to handle messages (`lsst.log.Log`). 
  852             If true, a correction was applied successfully. 
  858     """Do not correct non-linearity. 
  860     LinearityType = 
"None" 
  863         """Do not correct for non-linearity. 
  867         image : `lsst.afw.image.Image` 
  868             Image to be corrected 
  870             Dictionary of parameter keywords: 
  872                 Coefficient vector (`list` or `numpy.array`). 
  874                 Logger to handle messages (`lsst.log.Log`). 
  879             If true, a correction was applied successfully.