23 Apply intra-detector crosstalk corrections
26 from astropy.table
import Table
30 import lsst.daf.butler
37 __all__ = [
"CrosstalkCalib",
"CrosstalkConfig",
"CrosstalkTask",
42 """Calibration of amp-to-amp crosstalk coefficients.
46 detector : `lsst.afw.cameraGeom.Detector`, optional
47 Detector to use to pull coefficients from.
48 nAmp : `int`, optional
49 Number of amplifiers to initialize.
50 log : `logging.Logger`, optional
51 Log to write messages to.
53 Parameters to pass to parent constructor.
57 The crosstalk attributes stored are:
60 Whether there is crosstalk defined for this detector.
62 Number of amplifiers in this detector.
63 crosstalkShape : `tuple` [`int`, `int`]
64 A tuple containing the shape of the ``coeffs`` matrix. This
65 should be equivalent to (``nAmp``, ``nAmp``).
67 A matrix containing the crosstalk coefficients. coeff[i][j]
68 contains the coefficients to calculate the contribution
69 amplifier_j has on amplifier_i (each row[i] contains the
70 corrections for detector_i).
71 coeffErr : `np.ndarray`, optional
72 A matrix (as defined by ``coeffs``) containing the standard
73 distribution of the crosstalk measurements.
74 coeffNum : `np.ndarray`, optional
75 A matrix containing the number of pixel pairs used to measure
76 the ``coeffs`` and ``coeffErr``.
77 coeffValid : `np.ndarray`, optional
78 A matrix of Boolean values indicating if the coefficient is
79 valid, defined as abs(coeff) > coeffErr / sqrt(coeffNum).
80 interChip : `dict` [`np.ndarray`]
81 A dictionary keyed by detectorName containing ``coeffs``
82 matrices used to correct for inter-chip crosstalk with a
83 source on the detector indicated.
86 _OBSTYPE =
'CROSSTALK'
87 _SCHEMA =
'Gen3 Crosstalk'
90 def __init__(self, detector=None, nAmp=0, **kwargs):
92 self.
nAmpnAmp = nAmp
if nAmp
else 0
98 dtype=int)
if self.
nAmpnAmp
else None
100 dtype=bool)
if self.
nAmpnAmp
else None
105 'coeffErr',
'coeffNum',
'coeffValid',
111 """Update calibration metadata.
113 This calls the base class's method after ensuring the required
114 calibration keywords will be saved.
118 setDate : `bool`, optional
119 Update the CALIBDATE fields in the metadata to the current
120 time. Defaults to False.
122 Other keyword parameters to set in the metadata.
128 kwargs[
'NAMP'] = self.
nAmpnAmp
135 """Set calibration parameters from the detector.
139 detector : `lsst.afw.cameraGeom.Detector`
140 Detector to use to set parameters from.
141 coeffVector : `numpy.array`, optional
142 Use the detector geometry (bounding boxes and flip
143 information), but use ``coeffVector`` instead of the
144 output of ``detector.getCrosstalk()``.
148 calib : `lsst.ip.isr.CrosstalkCalib`
149 The calibration constructed from the detector.
152 if detector.hasCrosstalk()
or coeffVector:
157 self.
nAmpnAmp = len(detector)
160 if coeffVector
is not None:
161 crosstalkCoeffs = coeffVector
163 crosstalkCoeffs = detector.getCrosstalk()
164 if len(crosstalkCoeffs) == 1
and crosstalkCoeffs[0] == 0.0:
169 raise RuntimeError(
"Crosstalk coefficients do not match detector shape. "
170 f
"{self.crosstalkShape} {self.nAmp}")
183 """Construct a calibration from a dictionary of properties.
185 Must be implemented by the specific calibration subclasses.
190 Dictionary of properties.
194 calib : `lsst.ip.isr.CalibType`
195 Constructed calibration.
200 Raised if the supplied dictionary is for a different
205 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
206 raise RuntimeError(f
"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
207 f
"found {dictionary['metadata']['OBSTYPE']}")
209 calib.setMetadata(dictionary[
'metadata'])
211 if 'detectorName' in dictionary:
212 calib._detectorName = dictionary.get(
'detectorName')
213 elif 'DETECTOR_NAME' in dictionary:
214 calib._detectorName = dictionary.get(
'DETECTOR_NAME')
215 elif 'DET_NAME' in dictionary[
'metadata']:
216 calib._detectorName = dictionary[
'metadata'][
'DET_NAME']
218 calib._detectorName =
None
220 if 'detectorSerial' in dictionary:
221 calib._detectorSerial = dictionary.get(
'detectorSerial')
222 elif 'DETECTOR_SERIAL' in dictionary:
223 calib._detectorSerial = dictionary.get(
'DETECTOR_SERIAL')
224 elif 'DET_SER' in dictionary[
'metadata']:
225 calib._detectorSerial = dictionary[
'metadata'][
'DET_SER']
227 calib._detectorSerial =
None
229 if 'detectorId' in dictionary:
230 calib._detectorId = dictionary.get(
'detectorId')
231 elif 'DETECTOR' in dictionary:
232 calib._detectorId = dictionary.get(
'DETECTOR')
233 elif 'DETECTOR' in dictionary[
'metadata']:
234 calib._detectorId = dictionary[
'metadata'][
'DETECTOR']
235 elif calib._detectorSerial:
236 calib._detectorId = calib._detectorSerial
238 calib._detectorId =
None
240 if 'instrument' in dictionary:
241 calib._instrument = dictionary.get(
'instrument')
242 elif 'INSTRUME' in dictionary[
'metadata']:
243 calib._instrument = dictionary[
'metadata'][
'INSTRUME']
245 calib._instrument =
None
247 calib.hasCrosstalk = dictionary.get(
'hasCrosstalk',
248 dictionary[
'metadata'].get(
'HAS_CROSSTALK',
False))
249 if calib.hasCrosstalk:
250 calib.nAmp = dictionary.get(
'nAmp', dictionary[
'metadata'].get(
'NAMP', 0))
251 calib.crosstalkShape = (calib.nAmp, calib.nAmp)
252 calib.coeffs = np.array(dictionary[
'coeffs']).reshape(calib.crosstalkShape)
253 if 'coeffErr' in dictionary:
254 calib.coeffErr = np.array(dictionary[
'coeffErr']).reshape(calib.crosstalkShape)
256 calib.coeffErr = np.zeros_like(calib.coeffs)
257 if 'coeffNum' in dictionary:
258 calib.coeffNum = np.array(dictionary[
'coeffNum']).reshape(calib.crosstalkShape)
260 calib.coeffNum = np.zeros_like(calib.coeffs, dtype=int)
261 if 'coeffValid' in dictionary:
262 calib.coeffValid = np.array(dictionary[
'coeffValid']).reshape(calib.crosstalkShape)
264 calib.coeffValid = np.ones_like(calib.coeffs, dtype=bool)
266 calib.interChip = dictionary.get(
'interChip',
None)
268 for detector
in calib.interChip:
269 coeffVector = calib.interChip[detector]
270 calib.interChip[detector] = np.array(coeffVector).reshape(calib.crosstalkShape)
272 calib.updateMetadata()
276 """Return a dictionary containing the calibration properties.
278 The dictionary should be able to be round-tripped through
284 Dictionary of properties.
290 outDict[
'metadata'] = metadata
293 outDict[
'nAmp'] = self.
nAmpnAmp
296 ctLength = self.
nAmpnAmp*self.
nAmpnAmp
297 outDict[
'coeffs'] = self.
coeffscoeffs.reshape(ctLength).tolist()
299 if self.
coeffErrcoeffErr
is not None:
300 outDict[
'coeffErr'] = self.
coeffErrcoeffErr.reshape(ctLength).tolist()
301 if self.
coeffNumcoeffNum
is not None:
302 outDict[
'coeffNum'] = self.
coeffNumcoeffNum.reshape(ctLength).tolist()
304 outDict[
'coeffValid'] = self.
coeffValidcoeffValid.reshape(ctLength).tolist()
307 outDict[
'interChip'] = dict()
309 outDict[
'interChip'][detector] = self.
interChipinterChip[detector].reshape(ctLength).tolist()
315 """Construct calibration from a list of tables.
317 This method uses the `fromDict` method to create the
318 calibration, after constructing an appropriate dictionary from
323 tableList : `list` [`lsst.afw.table.Table`]
324 List of tables to use to construct the crosstalk
329 calib : `lsst.ip.isr.CrosstalkCalib`
330 The calibration defined in the tables.
333 coeffTable = tableList[0]
335 metadata = coeffTable.meta
337 inDict[
'metadata'] = metadata
338 inDict[
'hasCrosstalk'] = metadata[
'HAS_CROSSTALK']
339 inDict[
'nAmp'] = metadata[
'NAMP']
341 inDict[
'coeffs'] = coeffTable[
'CT_COEFFS']
342 if 'CT_ERRORS' in coeffTable.columns:
343 inDict[
'coeffErr'] = coeffTable[
'CT_ERRORS']
344 if 'CT_COUNTS' in coeffTable.columns:
345 inDict[
'coeffNum'] = coeffTable[
'CT_COUNTS']
346 if 'CT_VALID' in coeffTable.columns:
347 inDict[
'coeffValid'] = coeffTable[
'CT_VALID']
349 if len(tableList) > 1:
350 inDict[
'interChip'] = dict()
351 interChipTable = tableList[1]
352 for record
in interChipTable:
353 inDict[
'interChip'][record[
'IC_SOURCE_DET']] = record[
'IC_COEFFS']
358 """Construct a list of tables containing the information in this calibration.
360 The list of tables should create an identical calibration
361 after being passed to this class's fromTable method.
365 tableList : `list` [`lsst.afw.table.Table`]
366 List of tables containing the crosstalk calibration
372 catalog = Table([{
'CT_COEFFS': self.
coeffscoeffs.reshape(self.
nAmpnAmp*self.
nAmpnAmp),
379 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
380 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
381 catalog.meta = outMeta
382 tableList.append(catalog)
385 interChipTable = Table([{
'IC_SOURCE_DET': sourceDet,
386 'IC_COEFFS': self.
interChipinterChip[sourceDet].reshape(self.
nAmpnAmp*self.
nAmpnAmp)}
388 tableList.append(interChipTable)
394 """Extract the image data from an amp, flipped to match ampTarget.
398 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
399 Image containing the amplifier of interest.
400 amp : `lsst.afw.cameraGeom.Amplifier`
401 Amplifier on image to extract.
402 ampTarget : `lsst.afw.cameraGeom.Amplifier`
403 Target amplifier that the extracted image will be flipped
406 The image is already trimmed.
407 TODO : DM-15409 will resolve this.
411 output : `lsst.afw.image.Image`
412 Image of the amplifier in the desired configuration.
414 X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
415 lsst.afw.cameraGeom.ReadoutCorner.LR:
True,
416 lsst.afw.cameraGeom.ReadoutCorner.UL:
False,
417 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
418 Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
419 lsst.afw.cameraGeom.ReadoutCorner.LR:
False,
420 lsst.afw.cameraGeom.ReadoutCorner.UL:
True,
421 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
423 output = image[amp.getBBox()
if isTrimmed
else amp.getRawDataBBox()]
424 thisAmpCorner = amp.getReadoutCorner()
425 targetAmpCorner = ampTarget.getReadoutCorner()
428 xFlip = X_FLIP[targetAmpCorner] ^ X_FLIP[thisAmpCorner]
429 yFlip = Y_FLIP[targetAmpCorner] ^ Y_FLIP[thisAmpCorner]
434 """Estimate median background in image.
436 Getting a great background model isn't important for crosstalk correction,
437 since the crosstalk is at a low level. The median should be sufficient.
441 mi : `lsst.afw.image.MaskedImage`
442 MaskedImage for which to measure background.
443 badPixels : `list` of `str`
444 Mask planes to ignore.
448 Median background level.
452 stats.setAndMask(mask.getPlaneBitMask(badPixels))
456 badPixels=["BAD"], minPixelToMask=45000,
457 crosstalkStr="CROSSTALK", isTrimmed=False,
458 backgroundMethod="None"):
459 """Subtract the crosstalk from thisExposure, optionally using a different source.
461 We set the mask plane indicated by ``crosstalkStr`` in a target amplifier
462 for pixels in a source amplifier that exceed ``minPixelToMask``. Note that
463 the correction is applied to all pixels in the amplifier, but only those
464 that have a substantial crosstalk are masked with ``crosstalkStr``.
466 The uncorrected image is used as a template for correction. This is good
467 enough if the crosstalk is small (e.g., coefficients < ~ 1e-3), but if it's
468 larger you may want to iterate.
472 thisExposure : `lsst.afw.image.Exposure`
473 Exposure for which to subtract crosstalk.
474 sourceExposure : `lsst.afw.image.Exposure`, optional
475 Exposure to use as the source of the crosstalk. If not set,
476 thisExposure is used as the source (intra-detector crosstalk).
477 crosstalkCoeffs : `numpy.ndarray`, optional.
478 Coefficients to use to correct crosstalk.
479 badPixels : `list` of `str`
480 Mask planes to ignore.
481 minPixelToMask : `float`
482 Minimum pixel value (relative to the background level) in
483 source amplifier for which to set ``crosstalkStr`` mask plane
486 Mask plane name for pixels greatly modified by crosstalk
487 (above minPixelToMask).
489 The image is already trimmed.
490 This should no longer be needed once DM-15409 is resolved.
491 backgroundMethod : `str`
492 Method used to subtract the background. "AMP" uses
493 amplifier-by-amplifier background levels, "DETECTOR" uses full
494 exposure/maskedImage levels. Any other value results in no
495 background subtraction.
497 mi = thisExposure.getMaskedImage()
499 detector = thisExposure.getDetector()
503 numAmps = len(detector)
504 if numAmps != self.
nAmpnAmp:
505 raise RuntimeError(f
"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
506 f
"{numAmps} in {detector.getName()}")
509 source = sourceExposure.getMaskedImage()
510 sourceDetector = sourceExposure.getDetector()
513 sourceDetector = detector
515 if crosstalkCoeffs
is not None:
516 coeffs = crosstalkCoeffs
518 coeffs = self.
coeffscoeffs
519 self.
loglog.
debug(
"CT COEFF: %s", coeffs)
526 backgrounds = [0.0
for amp
in sourceDetector]
527 if backgroundMethod
is None:
529 elif backgroundMethod ==
"AMP":
530 backgrounds = [self.
calculateBackgroundcalculateBackground(source[amp.getBBox()], badPixels)
531 for amp
in sourceDetector]
532 elif backgroundMethod ==
"DETECTOR":
533 backgrounds = [self.
calculateBackgroundcalculateBackground(source, badPixels)
for amp
in sourceDetector]
537 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
540 + thresholdBackground))
541 footprints.setMask(mask, crosstalkStr)
542 crosstalk = mask.getPlaneBitMask(crosstalkStr)
545 subtrahend = source.Factory(source.getBBox())
546 subtrahend.set((0, 0, 0))
548 coeffs = coeffs.transpose()
549 for ii, iAmp
in enumerate(sourceDetector):
550 iImage = subtrahend[iAmp.getBBox()
if isTrimmed
else iAmp.getRawDataBBox()]
551 for jj, jAmp
in enumerate(detector):
552 if coeffs[ii, jj] == 0.0:
554 jImage = self.
extractAmpextractAmp(mi, jAmp, iAmp, isTrimmed)
555 jImage.getMask().getArray()[:] &= crosstalk
556 jImage -= backgrounds[jj]
557 iImage.scaledPlus(coeffs[ii, jj], jImage)
561 mask.clearMaskPlane(crosstalkPlane)
566 """Configuration for intra-detector crosstalk removal."""
569 doc=
"Set crosstalk mask plane for pixels over this value.",
574 doc=
"Name for crosstalk mask plane.",
579 doc=
"Type of background subtraction to use when applying correction.",
582 "None":
"Do no background subtraction.",
583 "AMP":
"Subtract amplifier-by-amplifier background levels.",
584 "DETECTOR":
"Subtract detector level background."
589 doc=
"Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
594 doc=(
"Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
595 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
596 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
597 "vector [corr0 corr1 corr2 ...]^T."),
602 doc=
"Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
607 """Return a 2-D numpy array of crosstalk coefficients in the proper shape.
611 detector : `lsst.afw.cameraGeom.detector`
612 Detector that is to be crosstalk corrected.
616 coeffs : `numpy.ndarray`
617 Crosstalk coefficients that can be used to correct the detector.
622 Raised if no coefficients could be generated from this detector/configuration.
626 if detector
is not None:
628 if coeffs.shape != (nAmp, nAmp):
629 raise RuntimeError(
"Constructed crosstalk coeffients do not match detector shape. "
630 f
"{coeffs.shape} {nAmp}")
632 elif detector
is not None and detector.hasCrosstalk()
is True:
634 return detector.getCrosstalk()
636 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients")
639 """Return a boolean indicating if crosstalk coefficients exist.
643 detector : `lsst.afw.cameraGeom.detector`
644 Detector that is to be crosstalk corrected.
648 hasCrosstalk : `bool`
649 True if this detector/configuration has crosstalk coefficients defined.
653 elif detector
is not None and detector.hasCrosstalk()
is True:
660 """Apply intra-detector crosstalk correction."""
661 ConfigClass = CrosstalkConfig
662 _DefaultName =
'isrCrosstalk'
665 """Placeholder for crosstalk preparation method, e.g., for inter-detector crosstalk.
669 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
670 Butler reference of the detector data to be processed.
671 crosstalk : `~lsst.ip.isr.CrosstalkConfig`
672 Crosstalk calibration that will be used.
676 lsst.obs.decam.crosstalk.DecamCrosstalkTask.prepCrosstalk
680 def run(self, exposure, crosstalk=None,
681 crosstalkSources=None, isTrimmed=False, camera=None):
682 """Apply intra-detector crosstalk correction
686 exposure : `lsst.afw.image.Exposure`
687 Exposure for which to remove crosstalk.
688 crosstalkCalib : `lsst.ip.isr.CrosstalkCalib`, optional
689 External crosstalk calibration to apply. Constructed from
690 detector if not found.
691 crosstalkSources : `defaultdict`, optional
692 Image data for other detectors that are sources of
693 crosstalk in exposure. The keys are expected to be names
694 of the other detectors, with the values containing
695 `lsst.afw.image.Exposure` at the same level of processing
697 The default for intra-detector crosstalk here is None.
698 isTrimmed : `bool`, optional
699 The image is already trimmed.
700 This should no longer be needed once DM-15409 is resolved.
701 camera : `lsst.afw.cameraGeom.Camera`, optional
702 Camera associated with this exposure. Only used for
708 Raised if called for a detector that does not have a
709 crosstalk correction. Also raised if the crosstalkSource
710 is not an expected type.
714 crosstalk = crosstalk.fromDetector(exposure.getDetector(),
715 coeffVector=self.config.crosstalkValues)
716 if not crosstalk.log:
717 crosstalk.log = self.log
718 if not crosstalk.hasCrosstalk:
719 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients.")
722 self.log.
info(
"Applying crosstalk correction.")
723 crosstalk.subtractCrosstalk(exposure, crosstalkCoeffs=crosstalk.coeffs,
724 minPixelToMask=self.config.minPixelToMask,
725 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
726 backgroundMethod=self.config.crosstalkBackgroundMethod)
728 if crosstalk.interChip:
733 sourceNames = [exp.getDetector().
getName()
for exp
in crosstalkSources]
734 elif isinstance(crosstalkSources[0], lsst.daf.butler.DeferredDatasetHandle):
736 detectorList = [source.dataId[
'detector']
for source
in crosstalkSources]
737 sourceNames = [camera[detector].
getName()
for detector
in detectorList]
739 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
740 type(crosstalkSources[0]))
742 for detName
in crosstalk.interChip:
743 if detName
not in sourceNames:
744 self.log.
warning(
"Crosstalk lists %s, not found in sources: %s",
745 detName, sourceNames)
748 interChipCoeffs = crosstalk.interChip[detName]
750 sourceExposure = crosstalkSources[sourceNames.index(detName)]
751 if isinstance(sourceExposure, lsst.daf.butler.DeferredDatasetHandle):
753 sourceExposure = sourceExposure.get()
755 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
756 type(sourceExposure))
758 self.log.
info(
"Correcting detector %s with ctSource %s",
759 exposure.getDetector().
getName(),
760 sourceExposure.getDetector().
getName())
761 crosstalk.subtractCrosstalk(exposure, sourceExposure=sourceExposure,
762 crosstalkCoeffs=interChipCoeffs,
763 minPixelToMask=self.config.minPixelToMask,
764 crosstalkStr=self.config.crosstalkMaskPlane,
766 backgroundMethod=self.config.crosstalkBackgroundMethod)
768 self.log.
warning(
"Crosstalk contains interChip coefficients, but no sources found!")
772 def run(self, exposure, crosstalkSources=None):
773 self.log.
info(
"Not performing any crosstalk correction")
A Threshold is used to pass a threshold value to detection algorithms.
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Pass parameters to a Statistics object.
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDetector(self, detector)
def requiredAttributes(self)
def __init__(self, detector=None, nAmp=0, **kwargs)
def calculateBackground(mi, badPixels=["BAD"])
def fromTable(cls, tableList)
def fromDetector(self, detector, coeffVector=None)
def updateMetadata(self, setDate=False, **kwargs)
def subtractCrosstalk(self, thisExposure, sourceExposure=None, crosstalkCoeffs=None, badPixels=["BAD"], minPixelToMask=45000, crosstalkStr="CROSSTALK", isTrimmed=False, backgroundMethod="None")
def extractAmp(image, amp, ampTarget, isTrimmed=False)
def fromDict(cls, dictionary)
def hasCrosstalk(self, detector=None)
def getCrosstalk(self, detector=None)
def prepCrosstalk(self, dataRef, crosstalk=None)
def run(self, exposure, crosstalk=None, crosstalkSources=None, isTrimmed=False, camera=None)
def run(self, exposure, crosstalkSources=None)
std::string const & getName() const noexcept
Return a filter's name.
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
std::shared_ptr< ImageT > flipImage(ImageT const &inImage, bool flipLR, bool flipTB)
Flip an image left–right and/or top–bottom.