23 Apply intra-detector crosstalk corrections
26 from astropy.table
import Table
30 import lsst.daf.butler
37 __all__ = [
"CrosstalkCalib",
"CrosstalkConfig",
"CrosstalkTask",
42 """Calibration of amp-to-amp crosstalk coefficients.
46 detector : `lsst.afw.cameraGeom.Detector`, optional
47 Detector to use to pull coefficients from.
48 nAmp : `int`, optional
49 Number of amplifiers to initialize.
50 log : `lsst.log.Log`, optional
51 Log to write messages to.
53 Parameters to pass to parent constructor.
57 The crosstalk attributes stored are:
60 Whether there is crosstalk defined for this detector.
62 Number of amplifiers in this detector.
63 crosstalkShape : `tuple` [`int`, `int`]
64 A tuple containing the shape of the ``coeffs`` matrix. This
65 should be equivalent to (``nAmp``, ``nAmp``).
67 A matrix containing the crosstalk coefficients. coeff[i][j]
68 contains the coefficients to calculate the contribution
69 amplifier_j has on amplifier_i (each row[i] contains the
70 corrections for detector_i).
71 coeffErr : `np.ndarray`, optional
72 A matrix (as defined by ``coeffs``) containing the standard
73 distribution of the crosstalk measurements.
74 coeffNum : `np.ndarray`, optional
75 A matrix containing the number of pixel pairs used to measure
76 the ``coeffs`` and ``coeffErr``.
77 coeffValid : `np.ndarray`, optional
78 A matrix of Boolean values indicating if the coefficient is
79 valid, defined as abs(coeff) > coeffErr / sqrt(coeffNum).
80 interChip : `dict` [`np.ndarray`]
81 A dictionary keyed by detectorName containing ``coeffs``
82 matrices used to correct for inter-chip crosstalk with a
83 source on the detector indicated.
86 _OBSTYPE =
'CROSSTALK'
87 _SCHEMA =
'Gen3 Crosstalk'
90 def __init__(self, detector=None, nAmp=0, **kwargs):
92 self.
nAmpnAmp = nAmp
if nAmp
else 0
98 dtype=int)
if self.
nAmpnAmp
else None
100 dtype=bool)
if self.
nAmpnAmp
else None
105 'coeffErr',
'coeffNum',
'coeffValid',
111 """Update calibration metadata.
113 This calls the base class's method after ensuring the required
114 calibration keywords will be saved.
118 setDate : `bool`, optional
119 Update the CALIBDATE fields in the metadata to the current
120 time. Defaults to False.
122 Other keyword parameters to set in the metadata.
128 kwargs[
'NAMP'] = self.
nAmpnAmp
135 """Set calibration parameters from the detector.
139 detector : `lsst.afw.cameraGeom.Detector`
140 Detector to use to set parameters from.
141 coeffVector : `numpy.array`, optional
142 Use the detector geometry (bounding boxes and flip
143 information), but use ``coeffVector`` instead of the
144 output of ``detector.getCrosstalk()``.
148 calib : `lsst.ip.isr.CrosstalkCalib`
149 The calibration constructed from the detector.
152 if detector.hasCrosstalk()
or coeffVector:
156 self.
nAmpnAmp = len(detector)
159 if coeffVector
is not None:
160 crosstalkCoeffs = coeffVector
162 crosstalkCoeffs = detector.getCrosstalk()
163 if len(crosstalkCoeffs) == 1
and crosstalkCoeffs[0] == 0.0:
168 raise RuntimeError(
"Crosstalk coefficients do not match detector shape. "
169 f
"{self.crosstalkShape} {self.nAmp}")
178 """Construct a calibration from a dictionary of properties.
180 Must be implemented by the specific calibration subclasses.
185 Dictionary of properties.
189 calib : `lsst.ip.isr.CalibType`
190 Constructed calibration.
195 Raised if the supplied dictionary is for a different
200 if calib._OBSTYPE != dictionary[
'metadata'][
'OBSTYPE']:
201 raise RuntimeError(f
"Incorrect crosstalk supplied. Expected {calib._OBSTYPE}, "
202 f
"found {dictionary['metadata']['OBSTYPE']}")
204 calib.setMetadata(dictionary[
'metadata'])
206 if 'detectorName' in dictionary:
207 calib._detectorName = dictionary.get(
'detectorName')
208 elif 'DETECTOR_NAME' in dictionary:
209 calib._detectorName = dictionary.get(
'DETECTOR_NAME')
210 elif 'DET_NAME' in dictionary[
'metadata']:
211 calib._detectorName = dictionary[
'metadata'][
'DET_NAME']
213 calib._detectorName =
None
215 if 'detectorSerial' in dictionary:
216 calib._detectorSerial = dictionary.get(
'detectorSerial')
217 elif 'DETECTOR_SERIAL' in dictionary:
218 calib._detectorSerial = dictionary.get(
'DETECTOR_SERIAL')
219 elif 'DET_SER' in dictionary[
'metadata']:
220 calib._detectorSerial = dictionary[
'metadata'][
'DET_SER']
222 calib._detectorSerial =
None
224 if 'detectorId' in dictionary:
225 calib._detectorId = dictionary.get(
'detectorId')
226 elif 'DETECTOR' in dictionary:
227 calib._detectorId = dictionary.get(
'DETECTOR')
228 elif 'DETECTOR' in dictionary[
'metadata']:
229 calib._detectorId = dictionary[
'metadata'][
'DETECTOR']
230 elif calib._detectorSerial:
231 calib._detectorId = calib._detectorSerial
233 calib._detectorId =
None
235 if 'instrument' in dictionary:
236 calib._instrument = dictionary.get(
'instrument')
237 elif 'INSTRUME' in dictionary[
'metadata']:
238 calib._instrument = dictionary[
'metadata'][
'INSTRUME']
240 calib._instrument =
None
242 calib.hasCrosstalk = dictionary.get(
'hasCrosstalk',
243 dictionary[
'metadata'].get(
'HAS_CROSSTALK',
False))
244 if calib.hasCrosstalk:
245 calib.nAmp = dictionary.get(
'nAmp', dictionary[
'metadata'].get(
'NAMP', 0))
246 calib.crosstalkShape = (calib.nAmp, calib.nAmp)
247 calib.coeffs = np.array(dictionary[
'coeffs']).reshape(calib.crosstalkShape)
248 if 'coeffErr' in dictionary:
249 calib.coeffErr = np.array(dictionary[
'coeffErr']).reshape(calib.crosstalkShape)
251 calib.coeffErr = np.zeros_like(calib.coeffs)
252 if 'coeffNum' in dictionary:
253 calib.coeffNum = np.array(dictionary[
'coeffNum']).reshape(calib.crosstalkShape)
255 calib.coeffNum = np.zeros_like(calib.coeffs, dtype=int)
256 if 'coeffValid' in dictionary:
257 calib.coeffValid = np.array(dictionary[
'coeffValid']).reshape(calib.crosstalkShape)
259 calib.coeffValid = np.ones_like(calib.coeffs, dtype=bool)
261 calib.interChip = dictionary.get(
'interChip',
None)
263 for detector
in calib.interChip:
264 coeffVector = calib.interChip[detector]
265 calib.interChip[detector] = np.array(coeffVector).reshape(calib.crosstalkShape)
267 calib.updateMetadata()
271 """Return a dictionary containing the calibration properties.
273 The dictionary should be able to be round-tripped through
279 Dictionary of properties.
285 outDict[
'metadata'] = metadata
288 outDict[
'nAmp'] = self.
nAmpnAmp
291 ctLength = self.
nAmpnAmp*self.
nAmpnAmp
292 outDict[
'coeffs'] = self.
coeffscoeffs.reshape(ctLength).tolist()
294 if self.
coeffErrcoeffErr
is not None:
295 outDict[
'coeffErr'] = self.
coeffErrcoeffErr.reshape(ctLength).tolist()
296 if self.
coeffNumcoeffNum
is not None:
297 outDict[
'coeffNum'] = self.
coeffNumcoeffNum.reshape(ctLength).tolist()
299 outDict[
'coeffValid'] = self.
coeffValidcoeffValid.reshape(ctLength).tolist()
302 outDict[
'interChip'] = dict()
304 outDict[
'interChip'][detector] = self.
interChipinterChip[detector].reshape(ctLength).tolist()
310 """Construct calibration from a list of tables.
312 This method uses the `fromDict` method to create the
313 calibration, after constructing an appropriate dictionary from
318 tableList : `list` [`lsst.afw.table.Table`]
319 List of tables to use to construct the crosstalk
324 calib : `lsst.ip.isr.CrosstalkCalib`
325 The calibration defined in the tables.
328 coeffTable = tableList[0]
330 metadata = coeffTable.meta
332 inDict[
'metadata'] = metadata
333 inDict[
'hasCrosstalk'] = metadata[
'HAS_CROSSTALK']
334 inDict[
'nAmp'] = metadata[
'NAMP']
336 inDict[
'coeffs'] = coeffTable[
'CT_COEFFS']
337 if 'CT_ERRORS' in coeffTable:
338 inDict[
'coeffErr'] = coeffTable[
'CT_ERRORS']
339 if 'CT_COUNTS' in coeffTable:
340 inDict[
'coeffNum'] = coeffTable[
'CT_COUNTS']
341 if 'CT_VALID' in coeffTable:
342 inDict[
'coeffValid'] = coeffTable[
'CT_VALID']
344 if len(tableList) > 1:
345 inDict[
'interChip'] = dict()
346 interChipTable = tableList[1]
347 for record
in interChipTable:
348 inDict[
'interChip'][record[
'IC_SOURCE_DET']] = record[
'IC_COEFFS']
353 """Construct a list of tables containing the information in this calibration.
355 The list of tables should create an identical calibration
356 after being passed to this class's fromTable method.
360 tableList : `list` [`lsst.afw.table.Table`]
361 List of tables containing the crosstalk calibration
367 catalog = Table([{
'CT_COEFFS': self.
coeffscoeffs.reshape(self.
nAmpnAmp*self.
nAmpnAmp),
374 outMeta = {k: v
for k, v
in inMeta.items()
if v
is not None}
375 outMeta.update({k:
"" for k, v
in inMeta.items()
if v
is None})
376 catalog.meta = outMeta
377 tableList.append(catalog)
380 interChipTable = Table([{
'IC_SOURCE_DET': sourceDet,
381 'IC_COEFFS': self.
interChipinterChip[sourceDet].reshape(self.
nAmpnAmp*self.
nAmpnAmp)}
383 tableList.append(interChipTable)
389 """Extract the image data from an amp, flipped to match ampTarget.
393 image : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
394 Image containing the amplifier of interest.
395 amp : `lsst.afw.cameraGeom.Amplifier`
396 Amplifier on image to extract.
397 ampTarget : `lsst.afw.cameraGeom.Amplifier`
398 Target amplifier that the extracted image will be flipped
401 The image is already trimmed.
402 TODO : DM-15409 will resolve this.
406 output : `lsst.afw.image.Image`
407 Image of the amplifier in the desired configuration.
409 X_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
410 lsst.afw.cameraGeom.ReadoutCorner.LR:
True,
411 lsst.afw.cameraGeom.ReadoutCorner.UL:
False,
412 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
413 Y_FLIP = {lsst.afw.cameraGeom.ReadoutCorner.LL:
False,
414 lsst.afw.cameraGeom.ReadoutCorner.LR:
False,
415 lsst.afw.cameraGeom.ReadoutCorner.UL:
True,
416 lsst.afw.cameraGeom.ReadoutCorner.UR:
True}
418 output = image[amp.getBBox()
if isTrimmed
else amp.getRawDataBBox()]
419 thisAmpCorner = amp.getReadoutCorner()
420 targetAmpCorner = ampTarget.getReadoutCorner()
423 xFlip = X_FLIP[targetAmpCorner] ^ X_FLIP[thisAmpCorner]
424 yFlip = Y_FLIP[targetAmpCorner] ^ Y_FLIP[thisAmpCorner]
429 """Estimate median background in image.
431 Getting a great background model isn't important for crosstalk correction,
432 since the crosstalk is at a low level. The median should be sufficient.
436 mi : `lsst.afw.image.MaskedImage`
437 MaskedImage for which to measure background.
438 badPixels : `list` of `str`
439 Mask planes to ignore.
443 Median background level.
447 stats.setAndMask(mask.getPlaneBitMask(badPixels))
451 badPixels=["BAD"], minPixelToMask=45000,
452 crosstalkStr="CROSSTALK", isTrimmed=False,
453 backgroundMethod="None"):
454 """Subtract the crosstalk from thisExposure, optionally using a different source.
456 We set the mask plane indicated by ``crosstalkStr`` in a target amplifier
457 for pixels in a source amplifier that exceed ``minPixelToMask``. Note that
458 the correction is applied to all pixels in the amplifier, but only those
459 that have a substantial crosstalk are masked with ``crosstalkStr``.
461 The uncorrected image is used as a template for correction. This is good
462 enough if the crosstalk is small (e.g., coefficients < ~ 1e-3), but if it's
463 larger you may want to iterate.
467 thisExposure : `lsst.afw.image.Exposure`
468 Exposure for which to subtract crosstalk.
469 sourceExposure : `lsst.afw.image.Exposure`, optional
470 Exposure to use as the source of the crosstalk. If not set,
471 thisExposure is used as the source (intra-detector crosstalk).
472 crosstalkCoeffs : `numpy.ndarray`, optional.
473 Coefficients to use to correct crosstalk.
474 badPixels : `list` of `str`
475 Mask planes to ignore.
476 minPixelToMask : `float`
477 Minimum pixel value (relative to the background level) in
478 source amplifier for which to set ``crosstalkStr`` mask plane
481 Mask plane name for pixels greatly modified by crosstalk
482 (above minPixelToMask).
484 The image is already trimmed.
485 This should no longer be needed once DM-15409 is resolved.
486 backgroundMethod : `str`
487 Method used to subtract the background. "AMP" uses
488 amplifier-by-amplifier background levels, "DETECTOR" uses full
489 exposure/maskedImage levels. Any other value results in no
490 background subtraction.
492 mi = thisExposure.getMaskedImage()
494 detector = thisExposure.getDetector()
498 numAmps = len(detector)
499 if numAmps != self.
nAmpnAmp:
500 raise RuntimeError(f
"Crosstalk built for {self.nAmp} in {self._detectorName}, received "
501 f
"{numAmps} in {detector.getName()}")
504 source = sourceExposure.getMaskedImage()
505 sourceDetector = sourceExposure.getDetector()
508 sourceDetector = detector
510 if crosstalkCoeffs
is not None:
511 coeffs = crosstalkCoeffs
513 coeffs = self.
coeffscoeffs
514 self.
loglog.
debug(
"CT COEFF: %s", coeffs)
521 backgrounds = [0.0
for amp
in sourceDetector]
522 if backgroundMethod
is None:
524 elif backgroundMethod ==
"AMP":
525 backgrounds = [self.
calculateBackgroundcalculateBackground(source[amp.getBBox()], badPixels)
526 for amp
in sourceDetector]
527 elif backgroundMethod ==
"DETECTOR":
528 backgrounds = [self.
calculateBackgroundcalculateBackground(source, badPixels)
for amp
in sourceDetector]
532 crosstalkPlane = mask.addMaskPlane(crosstalkStr)
535 + thresholdBackground))
536 footprints.setMask(mask, crosstalkStr)
537 crosstalk = mask.getPlaneBitMask(crosstalkStr)
540 subtrahend = source.Factory(source.getBBox())
541 subtrahend.set((0, 0, 0))
543 coeffs = coeffs.transpose()
544 for ii, iAmp
in enumerate(sourceDetector):
545 iImage = subtrahend[iAmp.getBBox()
if isTrimmed
else iAmp.getRawDataBBox()]
546 for jj, jAmp
in enumerate(detector):
547 if coeffs[ii, jj] == 0.0:
549 jImage = self.
extractAmpextractAmp(mi, jAmp, iAmp, isTrimmed)
550 jImage.getMask().getArray()[:] &= crosstalk
551 jImage -= backgrounds[jj]
552 iImage.scaledPlus(coeffs[ii, jj], jImage)
556 mask.clearMaskPlane(crosstalkPlane)
561 """Configuration for intra-detector crosstalk removal."""
564 doc=
"Set crosstalk mask plane for pixels over this value.",
569 doc=
"Name for crosstalk mask plane.",
574 doc=
"Type of background subtraction to use when applying correction.",
577 "None":
"Do no background subtraction.",
578 "AMP":
"Subtract amplifier-by-amplifier background levels.",
579 "DETECTOR":
"Subtract detector level background."
584 doc=
"Ignore the detector crosstalk information in favor of CrosstalkConfig values?",
589 doc=(
"Amplifier-indexed crosstalk coefficients to use. This should be arranged as a 1 x nAmp**2 "
590 "list of coefficients, such that when reshaped by crosstalkShape, the result is nAmp x nAmp. "
591 "This matrix should be structured so CT * [amp0 amp1 amp2 ...]^T returns the column "
592 "vector [corr0 corr1 corr2 ...]^T."),
597 doc=
"Shape of the coefficient array. This should be equal to [nAmp, nAmp].",
602 """Return a 2-D numpy array of crosstalk coefficients in the proper shape.
606 detector : `lsst.afw.cameraGeom.detector`
607 Detector that is to be crosstalk corrected.
611 coeffs : `numpy.ndarray`
612 Crosstalk coefficients that can be used to correct the detector.
617 Raised if no coefficients could be generated from this detector/configuration.
621 if detector
is not None:
623 if coeffs.shape != (nAmp, nAmp):
624 raise RuntimeError(
"Constructed crosstalk coeffients do not match detector shape. "
625 f
"{coeffs.shape} {nAmp}")
627 elif detector
is not None and detector.hasCrosstalk()
is True:
629 return detector.getCrosstalk()
631 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients")
634 """Return a boolean indicating if crosstalk coefficients exist.
638 detector : `lsst.afw.cameraGeom.detector`
639 Detector that is to be crosstalk corrected.
643 hasCrosstalk : `bool`
644 True if this detector/configuration has crosstalk coefficients defined.
648 elif detector
is not None and detector.hasCrosstalk()
is True:
655 """Apply intra-detector crosstalk correction."""
656 ConfigClass = CrosstalkConfig
657 _DefaultName =
'isrCrosstalk'
660 """Placeholder for crosstalk preparation method, e.g., for inter-detector crosstalk.
664 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
665 Butler reference of the detector data to be processed.
666 crosstalk : `~lsst.ip.isr.CrosstalkConfig`
667 Crosstalk calibration that will be used.
671 lsst.obs.decam.crosstalk.DecamCrosstalkTask.prepCrosstalk
675 def run(self, exposure, crosstalk=None,
676 crosstalkSources=None, isTrimmed=False, camera=None):
677 """Apply intra-detector crosstalk correction
681 exposure : `lsst.afw.image.Exposure`
682 Exposure for which to remove crosstalk.
683 crosstalkCalib : `lsst.ip.isr.CrosstalkCalib`, optional
684 External crosstalk calibration to apply. Constructed from
685 detector if not found.
686 crosstalkSources : `defaultdict`, optional
687 Image data for other detectors that are sources of
688 crosstalk in exposure. The keys are expected to be names
689 of the other detectors, with the values containing
690 `lsst.afw.image.Exposure` at the same level of processing
692 The default for intra-detector crosstalk here is None.
693 isTrimmed : `bool`, optional
694 The image is already trimmed.
695 This should no longer be needed once DM-15409 is resolved.
696 camera : `lsst.afw.cameraGeom.Camera`, optional
697 Camera associated with this exposure. Only used for
703 Raised if called for a detector that does not have a
704 crosstalk correction. Also raised if the crosstalkSource
705 is not an expected type.
709 crosstalk = crosstalk.fromDetector(exposure.getDetector(),
710 coeffVector=self.config.crosstalkValues)
711 if not crosstalk.log:
712 crosstalk.log = self.log
713 if not crosstalk.hasCrosstalk:
714 raise RuntimeError(
"Attempted to correct crosstalk without crosstalk coefficients.")
717 self.log.
info(
"Applying crosstalk correction.")
718 crosstalk.subtractCrosstalk(exposure, crosstalkCoeffs=crosstalk.coeffs,
719 minPixelToMask=self.config.minPixelToMask,
720 crosstalkStr=self.config.crosstalkMaskPlane, isTrimmed=isTrimmed,
721 backgroundMethod=self.config.crosstalkBackgroundMethod)
723 if crosstalk.interChip:
728 sourceNames = [exp.getDetector().
getName()
for exp
in crosstalkSources]
729 elif isinstance(crosstalkSources[0], lsst.daf.butler.DeferredDatasetHandle):
731 detectorList = [source.dataId[
'detector']
for source
in crosstalkSources]
732 sourceNames = [camera[detector].
getName()
for detector
in detectorList]
734 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
735 type(crosstalkSources[0]))
737 for detName
in crosstalk.interChip:
738 if detName
not in sourceNames:
739 self.log.
warn(
"Crosstalk lists %s, not found in sources: %s",
740 detName, sourceNames)
743 interChipCoeffs = crosstalk.interChip[detName]
745 sourceExposure = crosstalkSources[sourceNames.index(detName)]
746 if isinstance(sourceExposure, lsst.daf.butler.DeferredDatasetHandle):
748 sourceExposure = sourceExposure.get()
750 raise RuntimeError(
"Unknown object passed as crosstalk sources.",
751 type(sourceExposure))
753 self.log.
info(
"Correcting detector %s with ctSource %s",
754 exposure.getDetector().
getName(),
755 sourceExposure.getDetector().
getName())
756 crosstalk.subtractCrosstalk(exposure, sourceExposure=sourceExposure,
757 crosstalkCoeffs=interChipCoeffs,
758 minPixelToMask=self.config.minPixelToMask,
759 crosstalkStr=self.config.crosstalkMaskPlane,
761 backgroundMethod=self.config.crosstalkBackgroundMethod)
763 self.log.
warn(
"Crosstalk contains interChip coefficients, but no sources found!")
767 def run(self, exposure, crosstalkSources=None):
768 self.
loglog.
info(
"Not performing any crosstalk correction")
A Threshold is used to pass a threshold value to detection algorithms.
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Pass parameters to a Statistics object.
def requiredAttributes(self, value)
def updateMetadata(self, camera=None, detector=None, filterName=None, setCalibId=False, setCalibInfo=False, setDate=False, **kwargs)
def fromDetector(self, detector)
def requiredAttributes(self)
def __init__(self, detector=None, nAmp=0, **kwargs)
def calculateBackground(mi, badPixels=["BAD"])
def fromTable(cls, tableList)
def fromDetector(self, detector, coeffVector=None)
def updateMetadata(self, setDate=False, **kwargs)
def subtractCrosstalk(self, thisExposure, sourceExposure=None, crosstalkCoeffs=None, badPixels=["BAD"], minPixelToMask=45000, crosstalkStr="CROSSTALK", isTrimmed=False, backgroundMethod="None")
def extractAmp(image, amp, ampTarget, isTrimmed=False)
def fromDict(cls, dictionary)
def hasCrosstalk(self, detector=None)
def getCrosstalk(self, detector=None)
def prepCrosstalk(self, dataRef, crosstalk=None)
def run(self, exposure, crosstalk=None, crosstalkSources=None, isTrimmed=False, camera=None)
def run(self, exposure, crosstalkSources=None)
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
std::shared_ptr< ImageT > flipImage(ImageT const &inImage, bool flipLR, bool flipTB)
Flip an image left–right and/or top–bottom.