1205 """Perform instrument signature removal on an exposure.
1207 Steps included in the ISR processing, in order performed, are:
1208 - saturation and suspect pixel masking
1209 - overscan subtraction
1210 - CCD assembly of individual amplifiers
1212 - variance image construction
1213 - linearization of non-linear response
1215 - brighter-fatter correction
1218 - stray light subtraction
1220 - masking of known defects and camera specific features
1221 - vignette calculation
1222 - appending transmission curve and distortion model
1226 ccdExposure : `lsst.afw.image.Exposure`
1227 The raw exposure that is to be run through ISR. The
1228 exposure is modified by this method.
1229 camera : `lsst.afw.cameraGeom.Camera`, optional
1230 The camera geometry for this exposure. Required if ``isGen3`` is
1231 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1232 ``flat`` does not have an associated detector.
1233 bias : `lsst.afw.image.Exposure`, optional
1234 Bias calibration frame.
1235 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1236 Functor for linearization.
1237 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1238 Calibration for crosstalk.
1239 crosstalkSources : `list`, optional
1240 List of possible crosstalk sources.
1241 dark : `lsst.afw.image.Exposure`, optional
1242 Dark calibration frame.
1243 flat : `lsst.afw.image.Exposure`, optional
1244 Flat calibration frame.
1245 bfKernel : `numpy.ndarray`, optional
1246 Brighter-fatter kernel.
1247 bfGains : `dict` of `float`, optional
1248 Gains used to override the detector's nominal gains for the
1249 brighter-fatter correction. A dict keyed by amplifier name for
1250 the detector in question.
1251 defects : `lsst.ip.isr.Defects`, optional
1253 fringes : `lsst.pipe.base.Struct`, optional
1254 Struct containing the fringe correction data, with
1256 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1257 - ``seed``: random seed derived from the ccdExposureId for random
1258 number generator (`uint32`)
1259 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1260 A ``TransmissionCurve`` that represents the throughput of the optics,
1261 to be evaluated in focal-plane coordinates.
1262 filterTransmission : `lsst.afw.image.TransmissionCurve`
1263 A ``TransmissionCurve`` that represents the throughput of the filter
1264 itself, to be evaluated in focal-plane coordinates.
1265 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1266 A ``TransmissionCurve`` that represents the throughput of the sensor
1267 itself, to be evaluated in post-assembly trimmed detector coordinates.
1268 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1269 A ``TransmissionCurve`` that represents the throughput of the
1270 atmosphere, assumed to be spatially constant.
1271 detectorNum : `int`, optional
1272 The integer number for the detector to process.
1273 isGen3 : bool, optional
1274 Flag this call to run() as using the Gen3 butler environment.
1275 strayLightData : `object`, optional
1276 Opaque object containing calibration information for stray-light
1277 correction. If `None`, no correction will be performed.
1278 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1279 Illumination correction image.
1283 result : `lsst.pipe.base.Struct`
1284 Result struct with component:
1285 - ``exposure`` : `afw.image.Exposure`
1286 The fully ISR corrected exposure.
1287 - ``outputExposure`` : `afw.image.Exposure`
1288 An alias for `exposure`
1289 - ``ossThumb`` : `numpy.ndarray`
1290 Thumbnail image of the exposure after overscan subtraction.
1291 - ``flattenedThumb`` : `numpy.ndarray`
1292 Thumbnail image of the exposure after flat-field correction.
1297 Raised if a configuration option is set to True, but the
1298 required calibration data has not been specified.
1302 The current processed exposure can be viewed by setting the
1303 appropriate lsstDebug entries in the `debug.display`
1304 dictionary. The names of these entries correspond to some of
1305 the IsrTaskConfig Boolean options, with the value denoting the
1306 frame to use. The exposure is shown inside the matching
1307 option check and after the processing of that step has
1308 finished. The steps with debug points are:
1319 In addition, setting the "postISRCCD" entry displays the
1320 exposure after all ISR processing has finished.
1328 if detectorNum
is None:
1329 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1331 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1332 bias = self.ensureExposure(bias, camera, detectorNum)
1333 dark = self.ensureExposure(dark, camera, detectorNum)
1334 flat = self.ensureExposure(flat, camera, detectorNum)
1336 if isinstance(ccdExposure, ButlerDataRef):
1337 return self.runDataRef(ccdExposure)
1339 ccd = ccdExposure.getDetector()
1340 filterLabel = ccdExposure.getFilterLabel()
1343 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1344 ccd = [FakeAmp(ccdExposure, self.config)]
1347 if self.config.doBias
and bias
is None:
1348 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1349 if self.doLinearize(ccd)
and linearizer
is None:
1350 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1351 if self.config.doBrighterFatter
and bfKernel
is None:
1352 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1353 if self.config.doDark
and dark
is None:
1354 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1355 if self.config.doFlat
and flat
is None:
1356 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1357 if self.config.doDefect
and defects
is None:
1358 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1359 if (self.config.doFringe
and filterLabel
in self.fringe.config.filters
1360 and fringes.fringes
is None):
1365 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1366 if (self.config.doIlluminationCorrection
and filterLabel
in self.config.illumFilters
1367 and illumMaskedImage
is None):
1368 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1371 if self.config.doConvertIntToFloat:
1372 self.log.
info(
"Converting exposure to floating point values.")
1373 ccdExposure = self.convertIntToFloat(ccdExposure)
1375 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1376 self.log.
info(
"Applying bias correction.")
1377 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1378 trimToFit=self.config.doTrimToMatchCalib)
1379 self.debugView(ccdExposure,
"doBias")
1385 if ccdExposure.getBBox().
contains(amp.getBBox()):
1387 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1389 if self.config.doOverscan
and not badAmp:
1391 overscanResults = self.overscanCorrection(ccdExposure, amp)
1392 self.log.
debug(
"Corrected overscan for amplifier %s.", amp.getName())
1393 if overscanResults
is not None and \
1394 self.config.qa
is not None and self.config.qa.saveStats
is True:
1395 if isinstance(overscanResults.overscanFit, float):
1396 qaMedian = overscanResults.overscanFit
1397 qaStdev = float(
"NaN")
1400 afwMath.MEDIAN | afwMath.STDEVCLIP)
1401 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1402 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1404 self.metadata.
set(f
"FIT MEDIAN {amp.getName()}", qaMedian)
1405 self.metadata.
set(f
"FIT STDEV {amp.getName()}", qaStdev)
1406 self.log.
debug(
" Overscan stats for amplifer %s: %f +/- %f",
1407 amp.getName(), qaMedian, qaStdev)
1411 afwMath.MEDIAN | afwMath.STDEVCLIP)
1412 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1413 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1415 self.metadata.
set(f
"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1416 self.metadata.
set(f
"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1417 self.log.
debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1418 amp.getName(), qaMedianAfter, qaStdevAfter)
1420 ccdExposure.getMetadata().
set(
'OVERSCAN',
"Overscan corrected")
1423 self.log.
warn(
"Amplifier %s is bad.", amp.getName())
1424 overscanResults =
None
1426 overscans.append(overscanResults
if overscanResults
is not None else None)
1428 self.log.
info(
"Skipped OSCAN for %s.", amp.getName())
1430 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1431 self.log.
info(
"Applying crosstalk correction.")
1432 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1433 crosstalkSources=crosstalkSources, camera=camera)
1434 self.debugView(ccdExposure,
"doCrosstalk")
1436 if self.config.doAssembleCcd:
1437 self.log.
info(
"Assembling CCD from amplifiers.")
1438 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1440 if self.config.expectWcs
and not ccdExposure.getWcs():
1441 self.log.
warn(
"No WCS found in input exposure.")
1442 self.debugView(ccdExposure,
"doAssembleCcd")
1445 if self.config.qa.doThumbnailOss:
1446 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1448 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1449 self.log.
info(
"Applying bias correction.")
1450 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1451 trimToFit=self.config.doTrimToMatchCalib)
1452 self.debugView(ccdExposure,
"doBias")
1454 if self.config.doVariance:
1455 for amp, overscanResults
in zip(ccd, overscans):
1456 if ccdExposure.getBBox().
contains(amp.getBBox()):
1457 self.log.
debug(
"Constructing variance map for amplifer %s.", amp.getName())
1458 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1459 if overscanResults
is not None:
1460 self.updateVariance(ampExposure, amp,
1461 overscanImage=overscanResults.overscanImage)
1463 self.updateVariance(ampExposure, amp,
1465 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1467 afwMath.MEDIAN | afwMath.STDEVCLIP)
1468 self.metadata.
set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1469 qaStats.getValue(afwMath.MEDIAN))
1470 self.metadata.
set(f
"ISR VARIANCE {amp.getName()} STDEV",
1471 qaStats.getValue(afwMath.STDEVCLIP))
1472 self.log.
debug(
" Variance stats for amplifer %s: %f +/- %f.",
1473 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1474 qaStats.getValue(afwMath.STDEVCLIP))
1476 if self.doLinearize(ccd):
1477 self.log.
info(
"Applying linearizer.")
1478 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1479 detector=ccd, log=self.log)
1481 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1482 self.log.
info(
"Applying crosstalk correction.")
1483 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1484 crosstalkSources=crosstalkSources, isTrimmed=
True)
1485 self.debugView(ccdExposure,
"doCrosstalk")
1489 if self.config.doDefect:
1490 self.log.
info(
"Masking defects.")
1491 self.maskDefect(ccdExposure, defects)
1493 if self.config.numEdgeSuspect > 0:
1494 self.log.
info(
"Masking edges as SUSPECT.")
1495 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1496 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1498 if self.config.doNanMasking:
1499 self.log.
info(
"Masking non-finite (NAN, inf) value pixels.")
1500 self.maskNan(ccdExposure)
1502 if self.config.doWidenSaturationTrails:
1503 self.log.
info(
"Widening saturation trails.")
1504 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1506 if self.config.doCameraSpecificMasking:
1507 self.log.
info(
"Masking regions for camera specific reasons.")
1508 self.masking.
run(ccdExposure)
1510 if self.config.doBrighterFatter:
1519 interpExp = ccdExposure.clone()
1520 with self.flatContext(interpExp, flat, dark):
1521 isrFunctions.interpolateFromMask(
1522 maskedImage=interpExp.getMaskedImage(),
1523 fwhm=self.config.fwhm,
1524 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1525 maskNameList=
list(self.config.brighterFatterMaskListToInterpolate)
1527 bfExp = interpExp.clone()
1529 self.log.
info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1531 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1532 self.config.brighterFatterMaxIter,
1533 self.config.brighterFatterThreshold,
1534 self.config.brighterFatterApplyGain,
1536 if bfResults[1] == self.config.brighterFatterMaxIter:
1537 self.log.
warn(
"Brighter-fatter correction did not converge, final difference %f.",
1540 self.log.
info(
"Finished brighter-fatter correction in %d iterations.",
1542 image = ccdExposure.getMaskedImage().getImage()
1543 bfCorr = bfExp.getMaskedImage().getImage()
1544 bfCorr -= interpExp.getMaskedImage().getImage()
1553 self.log.
info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1554 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1557 if self.config.brighterFatterMaskGrowSize > 0:
1558 self.log.
info(
"Growing masks to account for brighter-fatter kernel convolution.")
1559 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1560 isrFunctions.growMasks(ccdExposure.getMask(),
1561 radius=self.config.brighterFatterMaskGrowSize,
1562 maskNameList=maskPlane,
1563 maskValue=maskPlane)
1565 self.debugView(ccdExposure,
"doBrighterFatter")
1567 if self.config.doDark:
1568 self.log.
info(
"Applying dark correction.")
1569 self.darkCorrection(ccdExposure, dark)
1570 self.debugView(ccdExposure,
"doDark")
1572 if self.config.doFringe
and not self.config.fringeAfterFlat:
1573 self.log.
info(
"Applying fringe correction before flat.")
1574 self.fringe.
run(ccdExposure, **fringes.getDict())
1575 self.debugView(ccdExposure,
"doFringe")
1577 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1578 self.log.
info(
"Checking strayLight correction.")
1579 self.strayLight.
run(ccdExposure, strayLightData)
1580 self.debugView(ccdExposure,
"doStrayLight")
1582 if self.config.doFlat:
1583 self.log.
info(
"Applying flat correction.")
1584 self.flatCorrection(ccdExposure, flat)
1585 self.debugView(ccdExposure,
"doFlat")
1587 if self.config.doApplyGains:
1588 self.log.
info(
"Applying gain correction instead of flat.")
1589 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1591 if self.config.doFringe
and self.config.fringeAfterFlat:
1592 self.log.
info(
"Applying fringe correction after flat.")
1593 self.fringe.
run(ccdExposure, **fringes.getDict())
1595 if self.config.doVignette:
1596 self.log.
info(
"Constructing Vignette polygon.")
1597 self.vignettePolygon = self.vignette.
run(ccdExposure)
1599 if self.config.vignette.doWriteVignettePolygon:
1600 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1602 if self.config.doAttachTransmissionCurve:
1603 self.log.
info(
"Adding transmission curves.")
1604 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1605 filterTransmission=filterTransmission,
1606 sensorTransmission=sensorTransmission,
1607 atmosphereTransmission=atmosphereTransmission)
1609 flattenedThumb =
None
1610 if self.config.qa.doThumbnailFlattened:
1611 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1613 if self.config.doIlluminationCorrection
and filterLabel
in self.config.illumFilters:
1614 self.log.
info(
"Performing illumination correction.")
1615 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1616 illumMaskedImage, illumScale=self.config.illumScale,
1617 trimToFit=self.config.doTrimToMatchCalib)
1620 if self.config.doSaveInterpPixels:
1621 preInterpExp = ccdExposure.clone()
1636 if self.config.doSetBadRegions:
1637 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1638 if badPixelCount > 0:
1639 self.log.
info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1641 if self.config.doInterpolate:
1642 self.log.
info(
"Interpolating masked pixels.")
1643 isrFunctions.interpolateFromMask(
1644 maskedImage=ccdExposure.getMaskedImage(),
1645 fwhm=self.config.fwhm,
1646 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1647 maskNameList=
list(self.config.maskListToInterpolate)
1650 self.roughZeroPoint(ccdExposure)
1652 if self.config.doMeasureBackground:
1653 self.log.
info(
"Measuring background level.")
1654 self.measureBackground(ccdExposure, self.config.qa)
1656 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1658 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1660 afwMath.MEDIAN | afwMath.STDEVCLIP)
1661 self.metadata.
set(
"ISR BACKGROUND {} MEDIAN".
format(amp.getName()),
1662 qaStats.getValue(afwMath.MEDIAN))
1663 self.metadata.
set(
"ISR BACKGROUND {} STDEV".
format(amp.getName()),
1664 qaStats.getValue(afwMath.STDEVCLIP))
1665 self.log.
debug(
" Background stats for amplifer %s: %f +/- %f",
1666 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1667 qaStats.getValue(afwMath.STDEVCLIP))
1669 self.debugView(ccdExposure,
"postISRCCD")
1671 return pipeBase.Struct(
1672 exposure=ccdExposure,
1674 flattenedThumb=flattenedThumb,
1676 preInterpolatedExposure=preInterpExp,
1677 outputExposure=ccdExposure,
1678 outputOssThumbnail=ossThumb,
1679 outputFlattenedThumbnail=flattenedThumb,
daf::base::PropertyList * list
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)