735 Mask SATURATED and SUSPECT pixels and check if any amplifiers
740 badAmpDict : `str` [`bool`]
741 Dictionary of amplifiers, keyed by name, value is True if
742 amplifier is fully masked.
743 ccdExposure : `lsst.afw.image.Exposure`
744 Input exposure to be masked.
745 detector : `lsst.afw.cameraGeom.Detector`
747 defects : `lsst.ip.isr.Defects`
748 List of defects. Used to determine if an entire
750 detectorConfig : `lsst.ip.isr.OverscanDetectorConfig`
751 Per-amplifier configurations.
752 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
753 PTC dataset (used if configured to use PTCTURNOFF).
757 badAmpDict : `str`[`bool`]
758 Dictionary of amplifiers, keyed by name.
760 maskedImage = ccdExposure.getMaskedImage()
762 metadata = ccdExposure.metadata
764 if self.config.doSaturation
and self.config.defaultSaturationSource ==
"PTCTURNOFF" and ptc
is None:
765 raise RuntimeError(
"Must provide ptc if using PTCTURNOFF as saturation source.")
766 if self.config.doSuspect
and self.config.defaultSuspectSource ==
"PTCTURNOFF" and ptc
is None:
767 raise RuntimeError(
"Must provide ptc if using PTCTURNOFF as suspect source.")
770 ampName = amp.getName()
772 ampConfig = detectorConfig.getOverscanAmpConfig(amp)
774 if badAmpDict[ampName]:
780 if self.config.doSaturation:
781 if self.config.defaultSaturationSource ==
"PTCTURNOFF":
782 limits.update({self.config.saturatedMaskName: ptc.ptcTurnoff[amp.getName()]})
783 elif self.config.defaultSaturationSource ==
"CAMERAMODEL":
785 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
786 elif self.config.defaultSaturationSource ==
"NONE":
787 limits.update({self.config.saturatedMaskName: numpy.inf})
790 if math.isfinite(ampConfig.saturation):
791 limits.update({self.config.saturatedMaskName: ampConfig.saturation})
792 metadata[f
"LSST ISR SATURATION LEVEL {ampName}"] = limits[self.config.saturatedMaskName]
794 if self.config.doSuspect:
795 if self.config.defaultSuspectSource ==
"PTCTURNOFF":
796 limits.update({self.config.suspectMaskName: ptc.ptcTurnoff[amp.getName()]})
797 elif self.config.defaultSuspectSource ==
"CAMERAMODEL":
799 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
800 elif self.config.defaultSuspectSource ==
"NONE":
801 limits.update({self.config.suspectMaskName: numpy.inf})
804 if math.isfinite(ampConfig.suspectLevel):
805 limits.update({self.config.suspectMaskName: ampConfig.suspectLevel})
806 metadata[f
"LSST ISR SUSPECT LEVEL {ampName}"] = limits[self.config.suspectMaskName]
808 for maskName, maskThreshold
in limits.items():
809 if not math.isnan(maskThreshold):
810 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
811 toMask = (dataView.image.array >= maskThreshold)
812 dataView.mask.array[toMask] |= dataView.mask.getPlaneBitMask(maskName)
816 maskView =
afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
818 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
819 self.config.suspectMaskName])
820 if numpy.all(maskView.getArray() & maskVal > 0):
821 self.log.warning(
"Amplifier %s is bad (completely SATURATED or SUSPECT)", ampName)
822 badAmpDict[ampName] =
True
823 maskView |= maskView.getPlaneBitMask(
"BAD")
1253 """Apply a brighter fatter correction to the image using the
1254 method defined in Coulton et al. 2019.
1256 Note that this correction requires that the image is in units
1261 ccdExposure : `lsst.afw.image.Exposure`
1262 Exposure to process.
1263 flat : `lsst.afw.image.Exposure`
1264 Flat exposure the same size as ``exp``.
1265 dark : `lsst.afw.image.Exposure`, optional
1266 Dark exposure the same size as ``exp``.
1267 bfKernel : `lsst.ip.isr.BrighterFatterKernel`
1268 The brighter-fatter kernel.
1269 brighterFatterApplyGain : `bool`
1270 Apply the gain to convert the image to electrons?
1272 The gains to use if brighterFatterApplyGain = True.
1276 exp : `lsst.afw.image.Exposure`
1277 The flat and dark corrected exposure.
1279 interpExp = ccdExposure.clone()
1283 isrFunctions.interpolateFromMask(
1284 maskedImage=interpExp.getMaskedImage(),
1285 fwhm=self.config.brighterFatterFwhmForInterpolation,
1286 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1287 maskNameList=list(self.config.brighterFatterMaskListToInterpolate),
1288 useLegacyInterp=self.config.useLegacyInterp,
1290 bfExp = interpExp.clone()
1291 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1292 self.config.brighterFatterMaxIter,
1293 self.config.brighterFatterThreshold,
1294 brighterFatterApplyGain,
1296 bfCorrIters = bfResults[1]
1297 if bfCorrIters == self.config.brighterFatterMaxIter:
1298 self.log.warning(
"Brighter-fatter correction did not converge, final difference %f.",
1301 self.log.info(
"Finished brighter-fatter correction in %d iterations.",
1304 image = ccdExposure.getMaskedImage().getImage()
1305 bfCorr = bfExp.getMaskedImage().getImage()
1306 bfCorr -= interpExp.getMaskedImage().getImage()
1315 self.log.info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1316 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1319 if self.config.brighterFatterMaskGrowSize > 0:
1320 self.log.info(
"Growing masks to account for brighter-fatter kernel convolution.")
1321 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1322 isrFunctions.growMasks(ccdExposure.getMask(),
1323 radius=self.config.brighterFatterMaskGrowSize,
1324 maskNameList=maskPlane,
1325 maskValue=maskPlane)
1327 return ccdExposure, bfCorrIters
1541 def run(self, ccdExposure, *, dnlLUT=None, bias=None, deferredChargeCalib=None, linearizer=None,
1542 ptc=None, crosstalk=None, defects=None, bfKernel=None, bfGains=None, dark=None,
1543 flat=None, camera=None, **kwargs
1546 detector = ccdExposure.getDetector()
1548 overscanDetectorConfig = self.config.overscanCamera.getOverscanDetectorConfig(detector)
1550 if self.config.doBootstrap
and ptc
is not None:
1551 self.log.warning(
"Task configured with doBootstrap=True. Ignoring provided PTC.")
1554 if not self.config.doBootstrap:
1556 raise RuntimeError(
"A PTC must be supplied if config.doBootstrap is False.")
1559 exposureMetadata = ccdExposure.metadata
1560 doRaise = self.config.doRaiseOnCalibMismatch
1561 keywords = self.config.cameraKeywordsToCompare
1562 if not self.config.doBootstrap:
1565 if self.config.doCorrectGains:
1566 raise RuntimeError(
"doCorrectGains is True but no ptc provided.")
1567 if self.config.doDiffNonLinearCorrection:
1569 raise RuntimeError(
"doDiffNonLinearCorrection is True but no dnlLUT provided.")
1571 if self.config.doLinearize:
1572 if linearizer
is None:
1573 raise RuntimeError(
"doLinearize is True but no linearizer provided.")
1575 if self.config.doBias:
1577 raise RuntimeError(
"doBias is True but no bias provided.")
1580 if self.config.doCrosstalk:
1581 if crosstalk
is None:
1582 raise RuntimeError(
"doCrosstalk is True but no crosstalk provided.")
1584 if self.config.doDeferredCharge:
1585 if deferredChargeCalib
is None:
1586 raise RuntimeError(
"doDeferredCharge is True but no deferredChargeCalib provided.")
1591 deferredChargeCalib,
1595 if self.config.doDefect:
1597 raise RuntimeError(
"doDefect is True but no defects provided.")
1599 if self.config.doDark:
1601 raise RuntimeError(
"doDark is True but no dark frame provided.")
1604 if self.config.doBrighterFatter:
1605 if bfKernel
is None:
1606 raise RuntimeError(
"doBrighterFatter is True not no bfKernel provided.")
1608 if self.config.doFlat:
1610 raise RuntimeError(
"doFlat is True but no flat provided.")
1613 if self.config.doSaturation:
1614 if self.config.defaultSaturationSource
in [
"PTCTURNOFF",]:
1617 "doSaturation is True and defaultSaturationSource is "
1618 f
"{self.config.defaultSaturationSource}, but no ptc provided."
1620 if self.config.doSuspect:
1621 if self.config.defaultSuspectSource
in [
"PTCTURNOFF",]:
1624 "doSuspect is True and defaultSuspectSource is "
1625 f
"{self.config.defaultSuspectSource}, but no ptc provided."
1632 exposureMetadata[
"LSST ISR UNITS"] =
"adu"
1633 exposureMetadata[
"LSST ISR CROSSTALK APPLIED"] =
False
1634 exposureMetadata[
"LSST ISR LINEARIZER APPLIED"] =
False
1635 exposureMetadata[
"LSST ISR CTI APPLIED"] =
False
1636 exposureMetadata[
"LSST ISR BIAS APPLIED"] =
False
1637 exposureMetadata[
"LSST ISR DARK APPLIED"] =
False
1638 exposureMetadata[
"LSST ISR BF APPLIED"] =
False
1639 exposureMetadata[
"LSST ISR FLAT APPLIED"] =
False
1640 exposureMetadata[
"LSST ISR DEFECTS APPLIED"] =
False
1642 if self.config.doBootstrap:
1643 self.log.info(
"Configured using doBootstrap=True; using gain of 1.0 (adu units)")
1645 for amp
in detector:
1646 ptc.gain[amp.getName()] = 1.0
1647 ptc.noise[amp.getName()] = 0.0
1649 exposureMetadata[
"LSST ISR BOOTSTRAP"] = self.config.doBootstrap
1656 for amp
in detector:
1657 if not math.isnan(gain := overscanDetectorConfig.getOverscanAmpConfig(amp).gain):
1658 gains[amp.getName()] = gain
1660 "Overriding gain for amp %s with configured value of %.3f.",
1667 self.log.debug(
"Converting exposure to floating point values.")
1677 if self.config.doDiffNonLinearCorrection:
1688 if overscanDetectorConfig.doAnySerialOverscan:
1691 overscanDetectorConfig,
1697 if self.config.doBootstrap:
1699 for amp, serialOverscan
in zip(detector, serialOverscans):
1700 if serialOverscan
is None:
1701 ptc.noise[amp.getName()] = 0.0
1709 ptc.noise[amp.getName()] = serialOverscan.residualSigma * gains[amp.getName()]
1711 serialOverscans = [
None]*len(detector)
1723 overscanDetectorConfig,
1727 if self.config.doCorrectGains:
1730 self.log.info(
"Apply temperature dependence to the gains.")
1731 gains, readNoise = self.
correctGains(ccdExposure, ptc, gains)
1736 if self.config.doApplyGains:
1737 self.log.info(
"Using gain values to convert from adu to electron units.")
1738 isrFunctions.applyGains(ccdExposure, normalizeGains=
False, ptcGains=gains, isTrimmed=
False)
1740 exposureMetadata[
"LSST ISR UNITS"] =
"electron"
1744 for amp
in detector:
1745 ampName = amp.getName()
1746 if (key := f
"LSST ISR SATURATION LEVEL {ampName}")
in exposureMetadata:
1747 exposureMetadata[key] *= gains[ampName]
1748 if (key := f
"LSST ISR SUSPECT LEVEL {ampName}")
in exposureMetadata:
1749 exposureMetadata[key] *= gains[ampName]
1752 metadata = ccdExposure.metadata
1753 metadata[
"LSST ISR READNOISE UNITS"] =
"electron"
1754 for amp
in detector:
1756 metadata[f
"LSST ISR GAIN {amp.getName()}"] = gains[amp.getName()]
1759 noise = ptc.noise[amp.getName()]
1760 metadata[f
"LSST ISR READNOISE {amp.getName()}"] = noise
1764 if self.config.doCrosstalk:
1765 self.log.info(
"Applying crosstalk corrections to full amplifier region.")
1766 if self.config.doBootstrap
and numpy.any(crosstalk.fitGains != 0):
1767 crosstalkGains =
None
1769 crosstalkGains = gains
1772 crosstalk=crosstalk,
1773 gains=crosstalkGains,
1775 badAmpDict=badAmpDict,
1776 ignoreVariance=
True,
1778 ccdExposure.metadata[
"LSST ISR CROSSTALK APPLIED"] =
True
1782 parallelOverscans =
None
1783 if overscanDetectorConfig.doAnyParallelOverscan:
1787 overscanDetectorConfig,
1795 if self.config.doLinearize:
1796 self.log.info(
"Applying linearizer.")
1800 if exposureMetadata[
"LSST ISR UNITS"] ==
"electron":
1801 linearityGains = gains
1803 linearityGains =
None
1804 linearizer.applyLinearity(
1805 image=ccdExposure.image,
1808 gains=linearityGains,
1810 ccdExposure.metadata[
"LSST ISR LINEARIZER APPLIED"] =
True
1815 if self.config.doDeferredCharge:
1816 self.deferredChargeCorrection.run(
1818 deferredChargeCalib,
1821 ccdExposure.metadata[
"LSST ISR CTI APPLIED"] =
True
1825 untrimmedCcdExposure =
None
1826 if self.config.isrStats.doCtiStatistics:
1827 untrimmedCcdExposure = ccdExposure.clone()
1831 if self.config.doAssembleCcd:
1832 self.log.info(
"Assembling CCD from amplifiers.")
1833 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1835 if self.config.expectWcs
and not ccdExposure.getWcs():
1836 self.log.warning(
"No WCS found in input exposure.")
1839 for maskPlane
in self.config.itlDipMaskPlanes:
1840 if maskPlane
not in ccdExposure.mask.getMaskPlaneDict():
1841 self.log.info(
"Adding %s mask plane to image.", maskPlane)
1842 ccdExposure.mask.addMaskPlane(maskPlane)
1844 if self.config.doITLDipMask:
1845 isrFunctions.maskITLDip(
1846 exposure=ccdExposure,
1847 detectorConfig=overscanDetectorConfig,
1849 maskPlaneNames=self.config.itlDipMaskPlanes,
1852 if (self.config.doITLSatSagMask
or self.config.doITLEdgeBleedMask) \
1853 and detector.getPhysicalType() ==
'ITL':
1855 badAmpDict=badAmpDict)
1859 if self.config.doBias:
1860 self.log.info(
"Applying bias correction.")
1863 isrFunctions.biasCorrection(ccdExposure.maskedImage, bias.maskedImage)
1864 ccdExposure.metadata[
"LSST ISR BIAS APPLIED"] =
True
1868 if self.config.doDark:
1869 self.log.info(
"Applying dark subtraction.")
1873 ccdExposure.metadata[
"LSST ISR DARK APPLIED"] =
True
1879 if self.config.doDefect:
1880 self.log.info(
"Applying defect masking.")
1882 ccdExposure.metadata[
"LSST ISR DEFECTS APPLIED"] =
True
1884 self.log.info(
"Adding UNMASKEDNAN mask plane to image.")
1885 ccdExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1886 if self.config.doNanMasking:
1887 self.log.info(
"Masking non-finite (NAN, inf) value pixels.")
1890 if self.config.doWidenSaturationTrails:
1891 self.log.info(
"Widening saturation trails.")
1892 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1896 if self.config.doBrighterFatter:
1897 self.log.info(
"Applying brighter-fatter correction.")
1903 if exposureMetadata[
"LSST ISR UNITS"] ==
"electron":
1904 brighterFatterApplyGain =
False
1906 brighterFatterApplyGain =
True
1908 if brighterFatterApplyGain
and (ptc
is not None)
and (bfGains != gains):
1915 self.log.warning(
"Need to apply gain for brighter-fatter, but the stored"
1916 "gains in the kernel are not the same as the gains stored"
1917 "in the PTC. Using the kernel gains.")
1924 brighterFatterApplyGain,
1928 ccdExposure.metadata[
"LSST ISR BF APPLIED"] =
True
1929 metadata[
"LSST ISR BF ITERS"] = bfCorrIters
1933 if self.config.doVariance:
1940 if self.config.doFlat:
1941 self.log.info(
"Applying flat correction.")
1942 isrFunctions.flatCorrection(
1943 maskedImage=ccdExposure.maskedImage,
1944 flatMaskedImage=flat.maskedImage,
1945 scalingType=self.config.flatScalingType,
1946 userScale=self.config.flatUserScale,
1948 ccdExposure.metadata[
"LSST ISR FLAT APPLIED"] =
True
1954 if self.config.doSaveInterpPixels:
1955 preInterpExp = ccdExposure.clone()
1957 if self.config.doSetBadRegions:
1958 self.log.info(
'Setting values in large contiguous bad regions.')
1961 if self.config.doInterpolate:
1962 self.log.info(
"Interpolating masked pixels.")
1963 isrFunctions.interpolateFromMask(
1964 maskedImage=ccdExposure.getMaskedImage(),
1965 fwhm=self.config.brighterFatterFwhmForInterpolation,
1966 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1967 maskNameList=list(self.config.maskListToInterpolate),
1968 useLegacyInterp=self.config.useLegacyInterp,
1972 if self.config.doAmpOffset:
1973 if self.config.ampOffset.doApplyAmpOffset:
1974 self.log.info(
"Measuring and applying amp offset corrections.")
1976 self.log.info(
"Measuring amp offset corrections only, without applying them.")
1977 self.ampOffset.run(ccdExposure)
1980 if self.config.doStandardStatistics:
1981 metadata = ccdExposure.metadata
1982 for amp
in detector:
1983 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1984 ampName = amp.getName()
1985 metadata[f
"LSST ISR MASK SAT {ampName}"] = isrFunctions.countMaskedPixels(
1986 ampExposure.getMaskedImage(),
1987 [self.config.saturatedMaskName]
1989 metadata[f
"LSST ISR MASK BAD {ampName}"] = isrFunctions.countMaskedPixels(
1990 ampExposure.getMaskedImage(),
1993 metadata[f
"LSST ISR MASK SUSPECT {ampName}"] = isrFunctions.countMaskedPixels(
1994 ampExposure.getMaskedImage(),
1998 afwMath.MEAN | afwMath.MEDIAN | afwMath.STDEVCLIP)
2000 metadata[f
"LSST ISR FINAL MEAN {ampName}"] = qaStats.getValue(afwMath.MEAN)
2001 metadata[f
"LSST ISR FINAL MEDIAN {ampName}"] = qaStats.getValue(afwMath.MEDIAN)
2002 metadata[f
"LSST ISR FINAL STDEV {ampName}"] = qaStats.getValue(afwMath.STDEVCLIP)
2004 k1 = f
"LSST ISR FINAL MEDIAN {ampName}"
2005 k2 = f
"LSST ISR OVERSCAN SERIAL MEDIAN {ampName}"
2006 if overscanDetectorConfig.doAnySerialOverscan
and k1
in metadata
and k2
in metadata:
2007 metadata[f
"LSST ISR LEVEL {ampName}"] = metadata[k1] - metadata[k2]
2009 metadata[f
"LSST ISR LEVEL {ampName}"] = numpy.nan
2012 outputStatistics =
None
2013 if self.config.doCalculateStatistics:
2014 outputStatistics = self.isrStats.run(ccdExposure,
2015 untrimmedInputExposure=untrimmedCcdExposure,
2016 serialOverscanResults=serialOverscans,
2017 parallelOverscanResults=parallelOverscans,
2018 bias=bias, dark=dark, flat=flat,
2019 ptc=ptc, defects=defects).results
2022 outputBin1Exposure =
None
2023 outputBin2Exposure =
None
2024 if self.config.doBinnedExposures:
2025 self.log.info(
"Creating binned exposures.")
2026 outputBin1Exposure = self.binning.run(
2028 binFactor=self.config.binFactor1,
2030 outputBin2Exposure = self.binning.run(
2032 binFactor=self.config.binFactor2,
2035 return pipeBase.Struct(
2036 exposure=ccdExposure,
2038 outputBin1Exposure=outputBin1Exposure,
2039 outputBin2Exposure=outputBin2Exposure,
2041 preInterpExposure=preInterpExp,
2042 outputExposure=ccdExposure,
2043 outputStatistics=outputStatistics,