30import lsst.pipe.base.connectionTypes
as cT
32from contextlib
import contextmanager
33from lsstDebug
import getDebugFrame
40from lsst.utils.timer
import timeMethod
42from .
import isrFunctions
44from .
import linearize
45from .defects
import Defects
47from .assembleCcdTask
import AssembleCcdTask
48from .crosstalk
import CrosstalkTask, CrosstalkCalib
49from .fringe
import FringeTask
50from .isr
import maskNans
51from .masking
import MaskingTask
52from .overscan
import OverscanCorrectionTask
53from .straylight
import StrayLightTask
54from .vignette
import VignetteTask
55from .ampOffset
import AmpOffsetTask
56from lsst.daf.butler
import DimensionGraph
59__all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
73 registry : `lsst.daf.butler.Registry`
74 Butler registry to query.
75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
76 Data id to transform to identify crosstalkSources. The
77 ``detector`` entry will be stripped.
78 collections : `lsst.daf.butler.CollectionSearch`
79 Collections to search through.
83 results : `list` [`lsst.daf.butler.DatasetRef`]
84 List of datasets that match the query that will be used
as
87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument",
"exposure"]))
88 results =
set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
99 dimensions={
"instrument",
"exposure",
"detector"},
100 defaultTemplates={}):
101 ccdExposure = cT.Input(
103 doc=
"Input exposure to process.",
104 storageClass=
"Exposure",
105 dimensions=[
"instrument",
"exposure",
"detector"],
107 camera = cT.PrerequisiteInput(
109 storageClass=
"Camera",
110 doc=
"Input camera to construct complete exposures.",
111 dimensions=[
"instrument"],
115 crosstalk = cT.PrerequisiteInput(
117 doc=
"Input crosstalk object",
118 storageClass=
"CrosstalkCalib",
119 dimensions=[
"instrument",
"detector"],
123 crosstalkSources = cT.PrerequisiteInput(
124 name=
"isrOverscanCorrected",
125 doc=
"Overscan corrected input images.",
126 storageClass=
"Exposure",
127 dimensions=[
"instrument",
"exposure",
"detector"],
130 lookupFunction=crosstalkSourceLookup,
133 bias = cT.PrerequisiteInput(
135 doc=
"Input bias calibration.",
136 storageClass=
"ExposureF",
137 dimensions=[
"instrument",
"detector"],
140 dark = cT.PrerequisiteInput(
142 doc=
"Input dark calibration.",
143 storageClass=
"ExposureF",
144 dimensions=[
"instrument",
"detector"],
147 flat = cT.PrerequisiteInput(
149 doc=
"Input flat calibration.",
150 storageClass=
"ExposureF",
151 dimensions=[
"instrument",
"physical_filter",
"detector"],
154 ptc = cT.PrerequisiteInput(
156 doc=
"Input Photon Transfer Curve dataset",
157 storageClass=
"PhotonTransferCurveDataset",
158 dimensions=[
"instrument",
"detector"],
161 fringes = cT.PrerequisiteInput(
163 doc=
"Input fringe calibration.",
164 storageClass=
"ExposureF",
165 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 strayLightData = cT.PrerequisiteInput(
171 doc=
"Input stray light calibration.",
172 storageClass=
"StrayLightData",
173 dimensions=[
"instrument",
"physical_filter",
"detector"],
178 bfKernel = cT.PrerequisiteInput(
180 doc=
"Input brighter-fatter kernel.",
181 storageClass=
"NumpyArray",
182 dimensions=[
"instrument"],
186 newBFKernel = cT.PrerequisiteInput(
187 name=
'brighterFatterKernel',
188 doc=
"Newer complete kernel + gain solutions.",
189 storageClass=
"BrighterFatterKernel",
190 dimensions=[
"instrument",
"detector"],
194 defects = cT.PrerequisiteInput(
196 doc=
"Input defect tables.",
197 storageClass=
"Defects",
198 dimensions=[
"instrument",
"detector"],
201 linearizer = cT.PrerequisiteInput(
203 storageClass=
"Linearizer",
204 doc=
"Linearity correction calibration.",
205 dimensions=[
"instrument",
"detector"],
209 opticsTransmission = cT.PrerequisiteInput(
210 name=
"transmission_optics",
211 storageClass=
"TransmissionCurve",
212 doc=
"Transmission curve due to the optics.",
213 dimensions=[
"instrument"],
216 filterTransmission = cT.PrerequisiteInput(
217 name=
"transmission_filter",
218 storageClass=
"TransmissionCurve",
219 doc=
"Transmission curve due to the filter.",
220 dimensions=[
"instrument",
"physical_filter"],
223 sensorTransmission = cT.PrerequisiteInput(
224 name=
"transmission_sensor",
225 storageClass=
"TransmissionCurve",
226 doc=
"Transmission curve due to the sensor.",
227 dimensions=[
"instrument",
"detector"],
230 atmosphereTransmission = cT.PrerequisiteInput(
231 name=
"transmission_atmosphere",
232 storageClass=
"TransmissionCurve",
233 doc=
"Transmission curve due to the atmosphere.",
234 dimensions=[
"instrument"],
237 illumMaskedImage = cT.PrerequisiteInput(
239 doc=
"Input illumination correction.",
240 storageClass=
"MaskedImageF",
241 dimensions=[
"instrument",
"physical_filter",
"detector"],
245 outputExposure = cT.Output(
247 doc=
"Output ISR processed exposure.",
248 storageClass=
"Exposure",
249 dimensions=[
"instrument",
"exposure",
"detector"],
251 preInterpExposure = cT.Output(
252 name=
'preInterpISRCCD',
253 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
254 storageClass=
"ExposureF",
255 dimensions=[
"instrument",
"exposure",
"detector"],
257 outputOssThumbnail = cT.Output(
259 doc=
"Output Overscan-subtracted thumbnail image.",
260 storageClass=
"Thumbnail",
261 dimensions=[
"instrument",
"exposure",
"detector"],
263 outputFlattenedThumbnail = cT.Output(
264 name=
"FlattenedThumb",
265 doc=
"Output flat-corrected thumbnail image.",
266 storageClass=
"Thumbnail",
267 dimensions=[
"instrument",
"exposure",
"detector"],
273 if config.doBias
is not True:
274 self.prerequisiteInputs.remove(
"bias")
275 if config.doLinearize
is not True:
276 self.prerequisiteInputs.remove(
"linearizer")
277 if config.doCrosstalk
is not True:
278 self.prerequisiteInputs.remove(
"crosstalkSources")
279 self.prerequisiteInputs.remove(
"crosstalk")
280 if config.doBrighterFatter
is not True:
281 self.prerequisiteInputs.remove(
"bfKernel")
282 self.prerequisiteInputs.remove(
"newBFKernel")
283 if config.doDefect
is not True:
284 self.prerequisiteInputs.remove(
"defects")
285 if config.doDark
is not True:
286 self.prerequisiteInputs.remove(
"dark")
287 if config.doFlat
is not True:
288 self.prerequisiteInputs.remove(
"flat")
289 if config.doFringe
is not True:
290 self.prerequisiteInputs.remove(
"fringes")
291 if config.doStrayLight
is not True:
292 self.prerequisiteInputs.remove(
"strayLightData")
293 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
294 self.prerequisiteInputs.remove(
"ptc")
295 if config.doAttachTransmissionCurve
is not True:
296 self.prerequisiteInputs.remove(
"opticsTransmission")
297 self.prerequisiteInputs.remove(
"filterTransmission")
298 self.prerequisiteInputs.remove(
"sensorTransmission")
299 self.prerequisiteInputs.remove(
"atmosphereTransmission")
301 if config.doUseOpticsTransmission
is not True:
302 self.prerequisiteInputs.remove(
"opticsTransmission")
303 if config.doUseFilterTransmission
is not True:
304 self.prerequisiteInputs.remove(
"filterTransmission")
305 if config.doUseSensorTransmission
is not True:
306 self.prerequisiteInputs.remove(
"sensorTransmission")
307 if config.doUseAtmosphereTransmission
is not True:
308 self.prerequisiteInputs.remove(
"atmosphereTransmission")
309 if config.doIlluminationCorrection
is not True:
310 self.prerequisiteInputs.remove(
"illumMaskedImage")
312 if config.doWrite
is not True:
313 self.outputs.remove(
"outputExposure")
314 self.outputs.remove(
"preInterpExposure")
315 self.outputs.remove(
"outputFlattenedThumbnail")
316 self.outputs.remove(
"outputOssThumbnail")
317 if config.doSaveInterpPixels
is not True:
318 self.outputs.remove(
"preInterpExposure")
319 if config.qa.doThumbnailOss
is not True:
320 self.outputs.remove(
"outputOssThumbnail")
321 if config.qa.doThumbnailFlattened
is not True:
322 self.outputs.remove(
"outputFlattenedThumbnail")
326 pipelineConnections=IsrTaskConnections):
327 """Configuration parameters for IsrTask.
329 Items are grouped in the order
in which they are executed by the task.
331 datasetType = pexConfig.Field(
333 doc="Dataset type for input data; users will typically leave this alone, "
334 "but camera-specific ISR tasks will override it",
338 fallbackFilterName = pexConfig.Field(
340 doc=
"Fallback default filter name for calibrations.",
343 useFallbackDate = pexConfig.Field(
345 doc=
"Pass observation date when using fallback filter.",
348 expectWcs = pexConfig.Field(
351 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
353 fwhm = pexConfig.Field(
355 doc=
"FWHM of PSF in arcseconds.",
358 qa = pexConfig.ConfigField(
360 doc=
"QA related configuration options.",
364 doConvertIntToFloat = pexConfig.Field(
366 doc=
"Convert integer raw images to floating point values?",
371 doSaturation = pexConfig.Field(
373 doc=
"Mask saturated pixels? NB: this is totally independent of the"
374 " interpolation option - this is ONLY setting the bits in the mask."
375 " To have them interpolated make sure doSaturationInterpolation=True",
378 saturatedMaskName = pexConfig.Field(
380 doc=
"Name of mask plane to use in saturation detection and interpolation",
383 saturation = pexConfig.Field(
385 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
386 default=
float(
"NaN"),
388 growSaturationFootprintSize = pexConfig.Field(
390 doc=
"Number of pixels by which to grow the saturation footprints",
395 doSuspect = pexConfig.Field(
397 doc=
"Mask suspect pixels?",
400 suspectMaskName = pexConfig.Field(
402 doc=
"Name of mask plane to use for suspect pixels",
405 numEdgeSuspect = pexConfig.Field(
407 doc=
"Number of edge pixels to be flagged as untrustworthy.",
410 edgeMaskLevel = pexConfig.ChoiceField(
412 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
415 'DETECTOR':
'Mask only the edges of the full detector.',
416 'AMP':
'Mask edges of each amplifier.',
421 doSetBadRegions = pexConfig.Field(
423 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
426 badStatistic = pexConfig.ChoiceField(
428 doc=
"How to estimate the average value for BAD regions.",
431 "MEANCLIP":
"Correct using the (clipped) mean of good data",
432 "MEDIAN":
"Correct using the median of the good data",
437 doOverscan = pexConfig.Field(
439 doc=
"Do overscan subtraction?",
442 overscan = pexConfig.ConfigurableField(
443 target=OverscanCorrectionTask,
444 doc=
"Overscan subtraction task for image segments.",
446 overscanFitType = pexConfig.ChoiceField(
448 doc=
"The method for fitting the overscan bias level.",
451 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
452 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
453 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
454 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
455 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
456 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
457 "MEAN":
"Correct using the mean of the overscan region",
458 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
459 "MEDIAN":
"Correct using the median of the overscan region",
460 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
462 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
463 " This option will no longer be used, and will be removed after v20.")
465 overscanOrder = pexConfig.Field(
467 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
468 "or number of spline knots if overscan fit type is a spline."),
470 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
471 " This option will no longer be used, and will be removed after v20.")
473 overscanNumSigmaClip = pexConfig.Field(
475 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
477 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
478 " This option will no longer be used, and will be removed after v20.")
480 overscanIsInt = pexConfig.Field(
482 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
483 " and overscan.FitType=MEDIAN_PER_ROW.",
485 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
486 " This option will no longer be used, and will be removed after v20.")
490 overscanNumLeadingColumnsToSkip = pexConfig.Field(
492 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
495 overscanNumTrailingColumnsToSkip = pexConfig.Field(
497 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
500 overscanMaxDev = pexConfig.Field(
502 doc=
"Maximum deviation from the median for overscan",
503 default=1000.0, check=
lambda x: x > 0
505 overscanBiasJump = pexConfig.Field(
507 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
510 overscanBiasJumpKeyword = pexConfig.Field(
512 doc=
"Header keyword containing information about devices.",
513 default=
"NO_SUCH_KEY",
515 overscanBiasJumpDevices = pexConfig.ListField(
517 doc=
"List of devices that need piecewise overscan correction.",
520 overscanBiasJumpLocation = pexConfig.Field(
522 doc=
"Location of bias jump along y-axis.",
527 doAssembleCcd = pexConfig.Field(
530 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
532 assembleCcd = pexConfig.ConfigurableField(
533 target=AssembleCcdTask,
534 doc=
"CCD assembly task",
538 doAssembleIsrExposures = pexConfig.Field(
541 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
543 doTrimToMatchCalib = pexConfig.Field(
546 doc=
"Trim raw data to match calibration bounding boxes?"
550 doBias = pexConfig.Field(
552 doc=
"Apply bias frame correction?",
555 biasDataProductName = pexConfig.Field(
557 doc=
"Name of the bias data product",
560 doBiasBeforeOverscan = pexConfig.Field(
562 doc=
"Reverse order of overscan and bias correction.",
567 doVariance = pexConfig.Field(
569 doc=
"Calculate variance?",
572 gain = pexConfig.Field(
574 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
575 default=
float(
"NaN"),
577 readNoise = pexConfig.Field(
579 doc=
"The read noise to use if no Detector is present in the Exposure",
582 doEmpiricalReadNoise = pexConfig.Field(
585 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
587 usePtcReadNoise = pexConfig.Field(
590 doc=
"Use readnoise values from the Photon Transfer Curve?"
592 maskNegativeVariance = pexConfig.Field(
595 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
596 "in the measurement of the overscan at an edge due to the data falling off faster "
597 "than the overscan model can account for it."
599 negativeVarianceMaskName = pexConfig.Field(
602 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
605 doLinearize = pexConfig.Field(
607 doc=
"Correct for nonlinearity of the detector's response?",
612 doCrosstalk = pexConfig.Field(
614 doc=
"Apply intra-CCD crosstalk correction?",
617 doCrosstalkBeforeAssemble = pexConfig.Field(
619 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
622 crosstalk = pexConfig.ConfigurableField(
623 target=CrosstalkTask,
624 doc=
"Intra-CCD crosstalk correction",
628 doDefect = pexConfig.Field(
630 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
633 doNanMasking = pexConfig.Field(
635 doc=
"Mask non-finite (NAN, inf) pixels?",
638 doWidenSaturationTrails = pexConfig.Field(
640 doc=
"Widen bleed trails based on their width?",
645 doBrighterFatter = pexConfig.Field(
648 doc=
"Apply the brighter-fatter correction?"
650 brighterFatterLevel = pexConfig.ChoiceField(
653 doc=
"The level at which to correct for brighter-fatter.",
655 "AMP":
"Every amplifier treated separately.",
656 "DETECTOR":
"One kernel per detector",
659 brighterFatterMaxIter = pexConfig.Field(
662 doc=
"Maximum number of iterations for the brighter-fatter correction"
664 brighterFatterThreshold = pexConfig.Field(
667 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
668 "absolute value of the difference between the current corrected image and the one "
669 "from the previous iteration summed over all the pixels."
671 brighterFatterApplyGain = pexConfig.Field(
674 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
676 brighterFatterMaskListToInterpolate = pexConfig.ListField(
678 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
680 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
682 brighterFatterMaskGrowSize = pexConfig.Field(
685 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
686 "when brighter-fatter correction is applied."
690 doDark = pexConfig.Field(
692 doc=
"Apply dark frame correction?",
695 darkDataProductName = pexConfig.Field(
697 doc=
"Name of the dark data product",
702 doStrayLight = pexConfig.Field(
704 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
707 strayLight = pexConfig.ConfigurableField(
708 target=StrayLightTask,
709 doc=
"y-band stray light correction"
713 doFlat = pexConfig.Field(
715 doc=
"Apply flat field correction?",
718 flatDataProductName = pexConfig.Field(
720 doc=
"Name of the flat data product",
723 flatScalingType = pexConfig.ChoiceField(
725 doc=
"The method for scaling the flat on the fly.",
728 "USER":
"Scale by flatUserScale",
729 "MEAN":
"Scale by the inverse of the mean",
730 "MEDIAN":
"Scale by the inverse of the median",
733 flatUserScale = pexConfig.Field(
735 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
738 doTweakFlat = pexConfig.Field(
740 doc=
"Tweak flats to match observed amplifier ratios?",
746 doApplyGains = pexConfig.Field(
748 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
751 usePtcGains = pexConfig.Field(
753 doc=
"Use the gain values from the Photon Transfer Curve?",
756 normalizeGains = pexConfig.Field(
758 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
763 doFringe = pexConfig.Field(
765 doc=
"Apply fringe correction?",
768 fringe = pexConfig.ConfigurableField(
770 doc=
"Fringe subtraction task",
772 fringeAfterFlat = pexConfig.Field(
774 doc=
"Do fringe subtraction after flat-fielding?",
779 doAmpOffset = pexConfig.Field(
780 doc=
"Calculate and apply amp offset corrections?",
784 ampOffset = pexConfig.ConfigurableField(
785 doc=
"Amp offset correction task.",
786 target=AmpOffsetTask,
790 doMeasureBackground = pexConfig.Field(
792 doc=
"Measure the background level on the reduced image?",
797 doCameraSpecificMasking = pexConfig.Field(
799 doc=
"Mask camera-specific bad regions?",
802 masking = pexConfig.ConfigurableField(
808 doInterpolate = pexConfig.Field(
810 doc=
"Interpolate masked pixels?",
813 doSaturationInterpolation = pexConfig.Field(
815 doc=
"Perform interpolation over pixels masked as saturated?"
816 " NB: This is independent of doSaturation; if that is False this plane"
817 " will likely be blank, resulting in a no-op here.",
820 doNanInterpolation = pexConfig.Field(
822 doc=
"Perform interpolation over pixels masked as NaN?"
823 " NB: This is independent of doNanMasking; if that is False this plane"
824 " will likely be blank, resulting in a no-op here.",
827 doNanInterpAfterFlat = pexConfig.Field(
829 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
830 "also have to interpolate them before flat-fielding."),
833 maskListToInterpolate = pexConfig.ListField(
835 doc=
"List of mask planes that should be interpolated.",
836 default=[
'SAT',
'BAD'],
838 doSaveInterpPixels = pexConfig.Field(
840 doc=
"Save a copy of the pre-interpolated pixel values?",
845 fluxMag0T1 = pexConfig.DictField(
848 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
849 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
852 defaultFluxMag0T1 = pexConfig.Field(
854 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
855 default=pow(10.0, 0.4*28.0)
859 doVignette = pexConfig.Field(
861 doc=(
"Compute and attach the validPolygon defining the unvignetted region to the exposure "
862 "according to vignetting parameters?"),
865 doMaskVignettePolygon = pexConfig.Field(
867 doc=(
"Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
871 vignetteValue = pexConfig.Field(
873 doc=
"Value to replace image array pixels with in the vignetted region? Ignored if None.",
877 vignette = pexConfig.ConfigurableField(
879 doc=
"Vignetting task.",
883 doAttachTransmissionCurve = pexConfig.Field(
886 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
888 doUseOpticsTransmission = pexConfig.Field(
891 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
893 doUseFilterTransmission = pexConfig.Field(
896 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
898 doUseSensorTransmission = pexConfig.Field(
901 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
903 doUseAtmosphereTransmission = pexConfig.Field(
906 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
910 doIlluminationCorrection = pexConfig.Field(
913 doc=
"Perform illumination correction?"
915 illuminationCorrectionDataProductName = pexConfig.Field(
917 doc=
"Name of the illumination correction data product.",
920 illumScale = pexConfig.Field(
922 doc=
"Scale factor for the illumination correction.",
925 illumFilters = pexConfig.ListField(
928 doc=
"Only perform illumination correction for these filters."
933 doWrite = pexConfig.Field(
935 doc=
"Persist postISRCCD?",
942 raise ValueError(
"You may not specify both doFlat and doApplyGains")
944 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
953class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
954 """Apply common instrument signature correction algorithms to a raw frame.
956 The process for correcting imaging data
is very similar
from
957 camera to camera. This task provides a vanilla implementation of
958 doing these corrections, including the ability to turn certain
959 corrections off
if they are
not needed. The inputs to the primary
960 method, `
run()`, are a raw exposure to be corrected
and the
961 calibration data products. The raw input
is a single chip sized
962 mosaic of all amps including overscans
and other non-science
963 pixels. The method `
runDataRef()` identifies
and defines the
964 calibration data products,
and is intended
for use by a
965 `lsst.pipe.base.cmdLineTask.CmdLineTask`
and takes
as input only a
967 subclassed
for different camera, although the most camera specific
968 methods have been split into subtasks that can be redirected
971 The __init__ method sets up the subtasks
for ISR processing, using
977 Positional arguments passed to the Task constructor.
978 None used at this time.
979 kwargs : `dict`, optional
980 Keyword arguments passed on to the Task constructor.
981 None used at this time.
983 ConfigClass = IsrTaskConfig
988 self.makeSubtask(
"assembleCcd")
989 self.makeSubtask(
"crosstalk")
990 self.makeSubtask(
"strayLight")
991 self.makeSubtask(
"fringe")
992 self.makeSubtask(
"masking")
993 self.makeSubtask(
"overscan")
994 self.makeSubtask(
"vignette")
995 self.makeSubtask(
"ampOffset")
998 inputs = butlerQC.get(inputRefs)
1001 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
1002 except Exception
as e:
1003 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
1006 inputs[
'isGen3'] =
True
1008 detector = inputs[
'ccdExposure'].getDetector()
1010 if self.config.doCrosstalk
is True:
1013 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
1014 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
1015 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1017 coeffVector = (self.config.crosstalk.crosstalkValues
1018 if self.config.crosstalk.useConfigCoefficients
else None)
1019 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1020 inputs[
'crosstalk'] = crosstalkCalib
1021 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1022 if 'crosstalkSources' not in inputs:
1023 self.log.
warning(
"No crosstalkSources found for chip with interChip terms!")
1026 if 'linearizer' in inputs:
1027 if isinstance(inputs[
'linearizer'], dict):
1029 linearizer.fromYaml(inputs[
'linearizer'])
1030 self.log.
warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1031 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1035 self.log.
warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1037 linearizer = inputs[
'linearizer']
1038 linearizer.log = self.log
1039 inputs[
'linearizer'] = linearizer
1042 self.log.
warning(
"Constructing linearizer from cameraGeom information.")
1044 if self.config.doDefect
is True:
1045 if "defects" in inputs
and inputs[
'defects']
is not None:
1049 if not isinstance(inputs[
"defects"], Defects):
1050 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1054 if self.config.doBrighterFatter:
1055 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1056 if brighterFatterKernel
is None:
1057 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1059 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1061 detName = detector.getName()
1062 level = brighterFatterKernel.level
1065 inputs[
'bfGains'] = brighterFatterKernel.gain
1066 if self.config.brighterFatterLevel ==
'DETECTOR':
1067 if level ==
'DETECTOR':
1068 if detName
in brighterFatterKernel.detKernels:
1069 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1071 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1072 elif level ==
'AMP':
1073 self.log.
warning(
"Making DETECTOR level kernel from AMP based brighter "
1075 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1076 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1077 elif self.config.brighterFatterLevel ==
'AMP':
1078 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1080 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1081 expId = inputs[
'ccdExposure'].info.id
1082 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1084 assembler=self.assembleCcd
1085 if self.config.doAssembleIsrExposures
else None)
1087 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1089 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1090 if 'strayLightData' not in inputs:
1091 inputs[
'strayLightData'] =
None
1093 outputs = self.
runrun(**inputs)
1094 butlerQC.put(outputs, outputRefs)
1097 """Retrieve necessary frames for instrument signature removal.
1099 Pre-fetching all required ISR data products limits the IO
1100 required by the ISR. Any conflict between the calibration data
1101 available and that needed
for ISR
is also detected prior to
1102 doing processing, allowing it to fail quickly.
1107 Butler reference of the detector data to be processed
1109 The raw exposure that will later be corrected
with the
1110 retrieved calibration data; should
not be modified
in this
1115 result : `lsst.pipe.base.Struct`
1116 Result struct
with components (which may be `
None`):
1118 - ``linearizer``: functor
for linearization
1120 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1123 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1125 - ``fringes``: `lsst.pipe.base.Struct`
with components:
1127 - ``seed``: random seed derived
from the ccdExposureId
for random
1128 number generator (`uint32`).
1130 A ``TransmissionCurve`` that represents the throughput of the
1131 optics, to be evaluated
in focal-plane coordinates.
1133 A ``TransmissionCurve`` that represents the throughput of the
1134 filter itself, to be evaluated
in focal-plane coordinates.
1136 A ``TransmissionCurve`` that represents the throughput of the
1137 sensor itself, to be evaluated
in post-assembly trimmed
1138 detector coordinates.
1140 A ``TransmissionCurve`` that represents the throughput of the
1141 atmosphere, assumed to be spatially constant.
1142 - ``strayLightData`` : `object`
1143 An opaque object containing calibration information
for
1144 stray-light correction. If `
None`, no correction will be
1146 - ``illumMaskedImage`` : illumination correction image
1151 NotImplementedError :
1152 Raised
if a per-amplifier brighter-fatter kernel
is requested by
1156 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1157 dateObs = dateObs.toPython().isoformat()
1158 except RuntimeError:
1159 self.log.
warning(
"Unable to identify dateObs for rawExposure.")
1162 ccd = rawExposure.getDetector()
1163 filterLabel = rawExposure.getFilter()
1164 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1165 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1166 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1167 if self.config.doBias
else None)
1170 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1172 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1173 linearizer.log = self.log
1174 if isinstance(linearizer, numpy.ndarray):
1177 crosstalkCalib =
None
1178 if self.config.doCrosstalk:
1180 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1182 coeffVector = (self.config.crosstalk.crosstalkValues
1183 if self.config.crosstalk.useConfigCoefficients
else None)
1184 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1185 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1186 if self.config.doCrosstalk
else None)
1188 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1189 if self.config.doDark
else None)
1190 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1192 if self.config.doFlat
else None)
1194 brighterFatterKernel =
None
1195 brighterFatterGains =
None
1196 if self.config.doBrighterFatter
is True:
1201 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1202 brighterFatterGains = brighterFatterKernel.gain
1203 self.log.
info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1206 brighterFatterKernel = dataRef.get(
"bfKernel")
1207 self.log.
info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1209 brighterFatterKernel =
None
1210 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1213 if self.config.brighterFatterLevel ==
'DETECTOR':
1214 if brighterFatterKernel.detKernels:
1215 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1217 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1220 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1222 defectList = (dataRef.get(
"defects")
1223 if self.config.doDefect
else None)
1224 expId = rawExposure.info.id
1225 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1226 if self.config.doAssembleIsrExposures
else None)
1227 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1228 else pipeBase.Struct(fringes=
None))
1230 if self.config.doAttachTransmissionCurve:
1231 opticsTransmission = (dataRef.get(
"transmission_optics")
1232 if self.config.doUseOpticsTransmission
else None)
1233 filterTransmission = (dataRef.get(
"transmission_filter")
1234 if self.config.doUseFilterTransmission
else None)
1235 sensorTransmission = (dataRef.get(
"transmission_sensor")
1236 if self.config.doUseSensorTransmission
else None)
1237 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1238 if self.config.doUseAtmosphereTransmission
else None)
1240 opticsTransmission =
None
1241 filterTransmission =
None
1242 sensorTransmission =
None
1243 atmosphereTransmission =
None
1245 if self.config.doStrayLight:
1246 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1248 strayLightData =
None
1251 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1252 if (self.config.doIlluminationCorrection
1253 and physicalFilter
in self.config.illumFilters)
1257 return pipeBase.Struct(bias=biasExposure,
1258 linearizer=linearizer,
1259 crosstalk=crosstalkCalib,
1260 crosstalkSources=crosstalkSources,
1263 bfKernel=brighterFatterKernel,
1264 bfGains=brighterFatterGains,
1266 fringes=fringeStruct,
1267 opticsTransmission=opticsTransmission,
1268 filterTransmission=filterTransmission,
1269 sensorTransmission=sensorTransmission,
1270 atmosphereTransmission=atmosphereTransmission,
1271 strayLightData=strayLightData,
1272 illumMaskedImage=illumMaskedImage
1276 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1277 crosstalk=None, crosstalkSources=None,
1278 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1279 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1280 sensorTransmission=
None, atmosphereTransmission=
None,
1281 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1284 """Perform instrument signature removal on an exposure.
1286 Steps included in the ISR processing,
in order performed, are:
1287 - saturation
and suspect pixel masking
1288 - overscan subtraction
1289 - CCD assembly of individual amplifiers
1291 - variance image construction
1292 - linearization of non-linear response
1294 - brighter-fatter correction
1297 - stray light subtraction
1299 - masking of known defects
and camera specific features
1300 - vignette calculation
1301 - appending transmission curve
and distortion model
1306 The raw exposure that
is to be run through ISR. The
1307 exposure
is modified by this method.
1309 The camera geometry
for this exposure. Required
if
1310 one
or more of ``ccdExposure``, ``bias``, ``dark``,
or
1311 ``flat`` does
not have an associated detector.
1313 Bias calibration frame.
1315 Functor
for linearization.
1317 Calibration
for crosstalk.
1318 crosstalkSources : `list`, optional
1319 List of possible crosstalk sources.
1321 Dark calibration frame.
1323 Flat calibration frame.
1325 Photon transfer curve dataset,
with, e.g., gains
1327 bfKernel : `numpy.ndarray`, optional
1328 Brighter-fatter kernel.
1329 bfGains : `dict` of `float`, optional
1330 Gains used to override the detector
's nominal gains for the
1331 brighter-fatter correction. A dict keyed by amplifier name for
1332 the detector
in question.
1335 fringes : `lsst.pipe.base.Struct`, optional
1336 Struct containing the fringe correction data,
with
1339 - ``seed``: random seed derived
from the ccdExposureId
for random
1340 number generator (`uint32`)
1342 A ``TransmissionCurve`` that represents the throughput of the,
1343 optics, to be evaluated
in focal-plane coordinates.
1345 A ``TransmissionCurve`` that represents the throughput of the
1346 filter itself, to be evaluated
in focal-plane coordinates.
1348 A ``TransmissionCurve`` that represents the throughput of the
1349 sensor itself, to be evaluated
in post-assembly trimmed detector
1352 A ``TransmissionCurve`` that represents the throughput of the
1353 atmosphere, assumed to be spatially constant.
1354 detectorNum : `int`, optional
1355 The integer number
for the detector to process.
1356 isGen3 : bool, optional
1357 Flag this call to
run()
as using the Gen3 butler environment.
1358 strayLightData : `object`, optional
1359 Opaque object containing calibration information
for stray-light
1360 correction. If `
None`, no correction will be performed.
1362 Illumination correction image.
1366 result : `lsst.pipe.base.Struct`
1367 Result struct
with component:
1369 The fully ISR corrected exposure.
1371 An alias
for `exposure`
1372 - ``ossThumb`` : `numpy.ndarray`
1373 Thumbnail image of the exposure after overscan subtraction.
1374 - ``flattenedThumb`` : `numpy.ndarray`
1375 Thumbnail image of the exposure after flat-field correction.
1380 Raised
if a configuration option
is set to
True, but the
1381 required calibration data has
not been specified.
1385 The current processed exposure can be viewed by setting the
1386 appropriate lsstDebug entries
in the `debug.display`
1387 dictionary. The names of these entries correspond to some of
1388 the IsrTaskConfig Boolean options,
with the value denoting the
1389 frame to use. The exposure
is shown inside the matching
1390 option check
and after the processing of that step has
1391 finished. The steps
with debug points are:
1402 In addition, setting the
"postISRCCD" entry displays the
1403 exposure after all ISR processing has finished.
1412 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1413 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1414 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1415 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1417 if isinstance(ccdExposure, ButlerDataRef):
1418 return self.
runDataRefrunDataRef(ccdExposure)
1420 ccd = ccdExposure.getDetector()
1421 filterLabel = ccdExposure.getFilter()
1422 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1425 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1426 ccd = [
FakeAmp(ccdExposure, self.config)]
1429 if self.config.doBias
and bias
is None:
1430 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1431 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1432 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1433 if self.config.doBrighterFatter
and bfKernel
is None:
1434 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1435 if self.config.doDark
and dark
is None:
1436 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1437 if self.config.doFlat
and flat
is None:
1438 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1439 if self.config.doDefect
and defects
is None:
1440 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1441 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1442 and fringes.fringes
is None):
1447 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1448 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1449 and illumMaskedImage
is None):
1450 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1453 if self.config.doConvertIntToFloat:
1454 self.log.
info(
"Converting exposure to floating point values.")
1457 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1458 self.log.
info(
"Applying bias correction.")
1459 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1460 trimToFit=self.config.doTrimToMatchCalib)
1461 self.
debugViewdebugView(ccdExposure,
"doBias")
1468 if ccdExposure.getBBox().
contains(amp.getBBox()):
1471 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1473 if self.config.doOverscan
and not badAmp:
1476 self.log.
debug(
"Corrected overscan for amplifier %s.", amp.getName())
1477 if overscanResults
is not None and \
1478 self.config.qa
is not None and self.config.qa.saveStats
is True:
1479 if isinstance(overscanResults.overscanFit, float):
1480 qaMedian = overscanResults.overscanFit
1481 qaStdev =
float(
"NaN")
1484 afwMath.MEDIAN | afwMath.STDEVCLIP)
1485 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1486 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1488 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = qaMedian
1489 self.metadata[f
"FIT STDEV {amp.getName()}"] = qaStdev
1490 self.log.
debug(
" Overscan stats for amplifer %s: %f +/- %f",
1491 amp.getName(), qaMedian, qaStdev)
1495 afwMath.MEDIAN | afwMath.STDEVCLIP)
1496 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1497 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1499 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1500 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1501 self.log.
debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1502 amp.getName(), qaMedianAfter, qaStdevAfter)
1504 ccdExposure.getMetadata().
set(
'OVERSCAN',
"Overscan corrected")
1507 self.log.
warning(
"Amplifier %s is bad.", amp.getName())
1508 overscanResults =
None
1510 overscans.append(overscanResults
if overscanResults
is not None else None)
1512 self.log.
info(
"Skipped OSCAN for %s.", amp.getName())
1514 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1515 self.log.
info(
"Applying crosstalk correction.")
1516 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1517 crosstalkSources=crosstalkSources, camera=camera)
1518 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1520 if self.config.doAssembleCcd:
1521 self.log.
info(
"Assembling CCD from amplifiers.")
1522 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1524 if self.config.expectWcs
and not ccdExposure.getWcs():
1525 self.log.
warning(
"No WCS found in input exposure.")
1526 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1529 if self.config.qa.doThumbnailOss:
1530 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1532 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1533 self.log.
info(
"Applying bias correction.")
1534 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1535 trimToFit=self.config.doTrimToMatchCalib)
1536 self.
debugViewdebugView(ccdExposure,
"doBias")
1538 if self.config.doVariance:
1539 for amp, overscanResults
in zip(ccd, overscans):
1540 if ccdExposure.getBBox().
contains(amp.getBBox()):
1541 self.log.
debug(
"Constructing variance map for amplifer %s.", amp.getName())
1542 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1543 if overscanResults
is not None:
1545 overscanImage=overscanResults.overscanImage,
1551 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1553 afwMath.MEDIAN | afwMath.STDEVCLIP)
1554 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1555 qaStats.getValue(afwMath.MEDIAN)
1556 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1557 qaStats.getValue(afwMath.STDEVCLIP)
1558 self.log.
debug(
" Variance stats for amplifer %s: %f +/- %f.",
1559 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1560 qaStats.getValue(afwMath.STDEVCLIP))
1561 if self.config.maskNegativeVariance:
1565 self.log.
info(
"Applying linearizer.")
1566 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1567 detector=ccd, log=self.log)
1569 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1570 self.log.
info(
"Applying crosstalk correction.")
1571 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1572 crosstalkSources=crosstalkSources, isTrimmed=
True)
1573 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1578 if self.config.doDefect:
1579 self.log.
info(
"Masking defects.")
1580 self.
maskDefectmaskDefect(ccdExposure, defects)
1582 if self.config.numEdgeSuspect > 0:
1583 self.log.
info(
"Masking edges as SUSPECT.")
1584 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1585 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1587 if self.config.doNanMasking:
1588 self.log.
info(
"Masking non-finite (NAN, inf) value pixels.")
1589 self.
maskNanmaskNan(ccdExposure)
1591 if self.config.doWidenSaturationTrails:
1592 self.log.
info(
"Widening saturation trails.")
1593 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1595 if self.config.doCameraSpecificMasking:
1596 self.log.
info(
"Masking regions for camera specific reasons.")
1597 self.masking.
run(ccdExposure)
1599 if self.config.doBrighterFatter:
1609 interpExp = ccdExposure.clone()
1610 with self.
flatContextflatContext(interpExp, flat, dark):
1611 isrFunctions.interpolateFromMask(
1612 maskedImage=interpExp.getMaskedImage(),
1613 fwhm=self.config.fwhm,
1614 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1615 maskNameList=
list(self.config.brighterFatterMaskListToInterpolate)
1617 bfExp = interpExp.clone()
1619 self.log.
info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1621 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1622 self.config.brighterFatterMaxIter,
1623 self.config.brighterFatterThreshold,
1624 self.config.brighterFatterApplyGain,
1626 if bfResults[1] == self.config.brighterFatterMaxIter:
1627 self.log.
warning(
"Brighter-fatter correction did not converge, final difference %f.",
1630 self.log.
info(
"Finished brighter-fatter correction in %d iterations.",
1632 image = ccdExposure.getMaskedImage().getImage()
1633 bfCorr = bfExp.getMaskedImage().getImage()
1634 bfCorr -= interpExp.getMaskedImage().getImage()
1643 self.log.
info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1644 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1647 if self.config.brighterFatterMaskGrowSize > 0:
1648 self.log.
info(
"Growing masks to account for brighter-fatter kernel convolution.")
1649 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1650 isrFunctions.growMasks(ccdExposure.getMask(),
1651 radius=self.config.brighterFatterMaskGrowSize,
1652 maskNameList=maskPlane,
1653 maskValue=maskPlane)
1655 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1657 if self.config.doDark:
1658 self.log.
info(
"Applying dark correction.")
1660 self.
debugViewdebugView(ccdExposure,
"doDark")
1662 if self.config.doFringe
and not self.config.fringeAfterFlat:
1663 self.log.
info(
"Applying fringe correction before flat.")
1664 self.fringe.
run(ccdExposure, **fringes.getDict())
1665 self.
debugViewdebugView(ccdExposure,
"doFringe")
1667 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1668 self.log.
info(
"Checking strayLight correction.")
1669 self.strayLight.
run(ccdExposure, strayLightData)
1670 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1672 if self.config.doFlat:
1673 self.log.
info(
"Applying flat correction.")
1675 self.
debugViewdebugView(ccdExposure,
"doFlat")
1677 if self.config.doApplyGains:
1678 self.log.
info(
"Applying gain correction instead of flat.")
1679 if self.config.usePtcGains:
1680 self.log.
info(
"Using gains from the Photon Transfer Curve.")
1681 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1684 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1686 if self.config.doFringe
and self.config.fringeAfterFlat:
1687 self.log.
info(
"Applying fringe correction after flat.")
1688 self.fringe.
run(ccdExposure, **fringes.getDict())
1690 if self.config.doVignette:
1691 if self.config.doMaskVignettePolygon:
1692 self.log.
info(
"Constructing, attaching, and masking vignette polygon.")
1694 self.log.
info(
"Constructing and attaching vignette polygon.")
1696 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1697 vignetteValue=self.config.vignetteValue, log=self.log)
1699 if self.config.doAttachTransmissionCurve:
1700 self.log.
info(
"Adding transmission curves.")
1701 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1702 filterTransmission=filterTransmission,
1703 sensorTransmission=sensorTransmission,
1704 atmosphereTransmission=atmosphereTransmission)
1706 flattenedThumb =
None
1707 if self.config.qa.doThumbnailFlattened:
1708 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1710 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1711 self.log.
info(
"Performing illumination correction.")
1712 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1713 illumMaskedImage, illumScale=self.config.illumScale,
1714 trimToFit=self.config.doTrimToMatchCalib)
1717 if self.config.doSaveInterpPixels:
1718 preInterpExp = ccdExposure.clone()
1733 if self.config.doSetBadRegions:
1734 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1735 if badPixelCount > 0:
1736 self.log.
info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1738 if self.config.doInterpolate:
1739 self.log.
info(
"Interpolating masked pixels.")
1740 isrFunctions.interpolateFromMask(
1741 maskedImage=ccdExposure.getMaskedImage(),
1742 fwhm=self.config.fwhm,
1743 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1744 maskNameList=
list(self.config.maskListToInterpolate)
1750 if self.config.doAmpOffset:
1751 self.log.
info(
"Correcting amp offsets.")
1752 self.ampOffset.
run(ccdExposure)
1754 if self.config.doMeasureBackground:
1755 self.log.
info(
"Measuring background level.")
1758 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1760 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1762 afwMath.MEDIAN | afwMath.STDEVCLIP)
1763 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1764 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1765 qaStats.getValue(afwMath.STDEVCLIP)
1766 self.log.
debug(
" Background stats for amplifer %s: %f +/- %f",
1767 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1768 qaStats.getValue(afwMath.STDEVCLIP))
1770 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1772 return pipeBase.Struct(
1773 exposure=ccdExposure,
1775 flattenedThumb=flattenedThumb,
1777 preInterpExposure=preInterpExp,
1778 outputExposure=ccdExposure,
1779 outputOssThumbnail=ossThumb,
1780 outputFlattenedThumbnail=flattenedThumb,
1785 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1787 This method contains the `CmdLineTask` interface to the ISR
1788 processing. All IO is handled here, freeing the `
run()` method
1789 to manage only pixel-level calculations. The steps performed
1791 - Read
in necessary detrending/isr/calibration data.
1792 - Process raw exposure
in `
run()`.
1793 - Persist the ISR-corrected exposure
as "postISRCCD" if
1794 config.doWrite=
True.
1799 DataRef of the detector data to be processed
1803 result : `lsst.pipe.base.Struct`
1804 Result struct
with component:
1806 The fully ISR corrected exposure.
1811 Raised
if a configuration option
is set to
True, but the
1812 required calibration data does
not exist.
1815 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1817 ccdExposure = sensorRef.get(self.config.datasetType)
1819 camera = sensorRef.get(
"camera")
1820 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1822 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1824 if self.config.doWrite:
1825 sensorRef.put(result.exposure,
"postISRCCD")
1826 if result.preInterpExposure
is not None:
1827 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1828 if result.ossThumb
is not None:
1829 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1830 if result.flattenedThumb
is not None:
1831 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1836 """Retrieve a calibration dataset for removing instrument signature.
1842 DataRef of the detector data to find calibration datasets
1845 Type of dataset to retrieve (e.g.
'bias',
'flat', etc).
1846 dateObs : `str`, optional
1847 Date of the observation. Used to correct butler failures
1848 when using fallback filters.
1850 If
True, disable butler proxies to enable error handling
1851 within this routine.
1856 Requested calibration frame.
1861 Raised
if no matching calibration frame can be found.
1864 exp = dataRef.get(datasetType, immediate=immediate)
1865 except Exception
as exc1:
1866 if not self.config.fallbackFilterName:
1867 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1869 if self.config.useFallbackDate
and dateObs:
1870 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1871 dateObs=dateObs, immediate=immediate)
1873 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1874 except Exception
as exc2:
1875 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1876 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1877 self.log.
warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1879 if self.config.doAssembleIsrExposures:
1880 exp = self.assembleCcd.assembleCcd(exp)
1884 """Ensure that the data returned by Butler is a fully constructed exp.
1886 ISR requires exposure-level image data for historical reasons, so
if we
1887 did
not recieve that
from Butler, construct it
from what we have,
1888 modifying the input
in place.
1893 or `lsst.afw.image.ImageF`
1894 The input data structure obtained
from Butler.
1895 camera : `lsst.afw.cameraGeom.camera`, optional
1896 The camera associated
with the image. Used to find the appropriate
1897 detector
if detector
is not already set.
1898 detectorNum : `int`, optional
1899 The detector
in the camera to attach,
if the detector
is not
1905 The re-constructed exposure,
with appropriate detector parameters.
1910 Raised
if the input data cannot be used to construct an exposure.
1912 if isinstance(inputExp, afwImage.DecoratedImageU):
1914 elif isinstance(inputExp, afwImage.ImageF):
1916 elif isinstance(inputExp, afwImage.MaskedImageF):
1920 elif inputExp
is None:
1924 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1927 if inputExp.getDetector()
is None:
1928 if camera
is None or detectorNum
is None:
1929 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1930 'without a detector set.')
1931 inputExp.setDetector(camera[detectorNum])
1936 """Convert exposure image from uint16 to float.
1938 If the exposure does not need to be converted, the input
is
1939 immediately returned. For exposures that are converted to use
1940 floating point pixels, the variance
is set to unity
and the
1946 The raw exposure to be converted.
1951 The input ``exposure``, converted to floating point pixels.
1956 Raised
if the exposure type cannot be converted to float.
1959 if isinstance(exposure, afwImage.ExposureF):
1961 self.log.
debug(
"Exposure already of type float.")
1963 if not hasattr(exposure,
"convertF"):
1964 raise RuntimeError(
"Unable to convert exposure (%s) to float." %
type(exposure))
1966 newexposure = exposure.convertF()
1967 newexposure.variance[:] = 1
1968 newexposure.mask[:] = 0x0
1973 """Identify bad amplifiers, saturated and suspect pixels.
1978 Input exposure to be masked.
1980 Catalog of parameters defining the amplifier on this
1983 List of defects. Used to determine if the entire
1989 If this
is true, the entire amplifier area
is covered by
1990 defects
and unusable.
1993 maskedImage = ccdExposure.getMaskedImage()
2000 if defects
is not None:
2001 badAmp = bool(sum([v.getBBox().
contains(amp.getBBox())
for v
in defects]))
2007 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
2009 maskView = dataView.getMask()
2010 maskView |= maskView.getPlaneBitMask(
"BAD")
2018 if self.config.doSaturation
and not badAmp:
2019 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2020 if self.config.doSuspect
and not badAmp:
2021 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2022 if math.isfinite(self.config.saturation):
2023 limits.update({self.config.saturatedMaskName: self.config.saturation})
2025 for maskName, maskThreshold
in limits.items():
2026 if not math.isnan(maskThreshold):
2027 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2028 isrFunctions.makeThresholdMask(
2029 maskedImage=dataView,
2030 threshold=maskThreshold,
2037 maskView =
afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2039 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2040 self.config.suspectMaskName])
2041 if numpy.all(maskView.getArray() & maskVal > 0):
2043 maskView |= maskView.getPlaneBitMask(
"BAD")
2048 """Apply overscan correction in place.
2050 This method does initial pixel rejection of the overscan
2051 region. The overscan can also be optionally segmented to
2052 allow for discontinuous overscan responses to be fit
2053 separately. The actual overscan subtraction
is performed by
2054 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2055 which
is called here after the amplifier
is preprocessed.
2060 Exposure to have overscan correction performed.
2061 amp : `lsst.afw.cameraGeom.Amplifer`
2062 The amplifier to consider
while correcting the overscan.
2066 overscanResults : `lsst.pipe.base.Struct`
2067 Result struct
with components:
2069 Value
or fit subtracted
from the amplifier image data.
2071 Value
or fit subtracted
from the overscan image data.
2073 Image of the overscan region
with the overscan
2074 correction applied. This quantity
is used to estimate
2075 the amplifier read noise empirically.
2080 Raised
if the ``amp`` does
not contain raw pixel information.
2084 lsst.ip.isr.isrFunctions.overscanCorrection
2086 if amp.getRawHorizontalOverscanBBox().isEmpty():
2087 self.log.
info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2091 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2094 dataBBox = amp.getRawDataBBox()
2095 oscanBBox = amp.getRawHorizontalOverscanBBox()
2099 prescanBBox = amp.getRawPrescanBBox()
2100 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2101 dx0 += self.config.overscanNumLeadingColumnsToSkip
2102 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2104 dx0 += self.config.overscanNumTrailingColumnsToSkip
2105 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2112 if ((self.config.overscanBiasJump
2113 and self.config.overscanBiasJumpLocation)
2114 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2115 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2116 self.config.overscanBiasJumpDevices)):
2117 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2118 yLower = self.config.overscanBiasJumpLocation
2119 yUpper = dataBBox.getHeight() - yLower
2121 yUpper = self.config.overscanBiasJumpLocation
2122 yLower = dataBBox.getHeight() - yUpper
2140 oscanBBox.getHeight())))
2144 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2145 ampImage = ccdExposure.maskedImage[imageBBox]
2146 overscanImage = ccdExposure.maskedImage[overscanBBox]
2148 overscanArray = overscanImage.image.array
2149 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2150 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2151 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2154 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2156 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2159 levelStat = afwMath.MEDIAN
2160 sigmaStat = afwMath.STDEVCLIP
2163 self.config.qa.flatness.nIter)
2164 metadata = ccdExposure.getMetadata()
2165 ampNum = amp.getName()
2167 if isinstance(overscanResults.overscanFit, float):
2168 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2169 metadata[f
"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2172 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2173 metadata[f
"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2175 return overscanResults
2178 """Set the variance plane using the gain and read noise
2180 The read noise is calculated
from the ``overscanImage``
if the
2181 ``doEmpiricalReadNoise`` option
is set
in the configuration; otherwise
2182 the value
from the amplifier data
is used.
2187 Exposure to process.
2188 amp : `lsst.afw.table.AmpInfoRecord`
or `FakeAmp`
2189 Amplifier detector data.
2191 Image of overscan, required only
for empirical read noise.
2193 PTC dataset containing the gains
and read noise.
2199 Raised
if either ``usePtcGains`` of ``usePtcReadNoise``
2200 are ``
True``, but ptcDataset
is not provided.
2202 Raised
if ```doEmpiricalReadNoise``
is ``
True`` but
2203 ``overscanImage``
is ``
None``.
2207 lsst.ip.isr.isrFunctions.updateVariance
2209 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2210 if self.config.usePtcGains:
2211 if ptcDataset
is None:
2212 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2214 gain = ptcDataset.gain[amp.getName()]
2215 self.log.
info(
"Using gain from Photon Transfer Curve.")
2217 gain = amp.getGain()
2219 if math.isnan(gain):
2221 self.log.
warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2224 self.log.
warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2225 amp.getName(), gain, patchedGain)
2228 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2229 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2231 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2233 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2235 self.log.
info(
"Calculated empirical read noise for amp %s: %f.",
2236 amp.getName(), readNoise)
2237 elif self.config.usePtcReadNoise:
2238 if ptcDataset
is None:
2239 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2241 readNoise = ptcDataset.noise[amp.getName()]
2242 self.log.
info(
"Using read noise from Photon Transfer Curve.")
2244 readNoise = amp.getReadNoise()
2246 isrFunctions.updateVariance(
2247 maskedImage=ampExposure.getMaskedImage(),
2249 readNoise=readNoise,
2253 """Identify and mask pixels with negative variance values.
2258 Exposure to process.
2262 lsst.ip.isr.isrFunctions.updateVariance
2264 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2265 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2266 exposure.mask.array[bad] |= maskPlane
2269 """Apply dark correction in place.
2274 Exposure to process.
2276 Dark exposure of the same size as ``exposure``.
2277 invert : `Bool`, optional
2278 If
True, re-add the dark to an already corrected image.
2283 Raised
if either ``exposure``
or ``darkExposure`` do
not
2284 have their dark time defined.
2288 lsst.ip.isr.isrFunctions.darkCorrection
2290 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2291 if math.isnan(expScale):
2292 raise RuntimeError(
"Exposure darktime is NAN.")
2293 if darkExposure.getInfo().getVisitInfo()
is not None \
2294 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2295 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2299 self.log.
warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2302 isrFunctions.darkCorrection(
2303 maskedImage=exposure.getMaskedImage(),
2304 darkMaskedImage=darkExposure.getMaskedImage(),
2306 darkScale=darkScale,
2308 trimToFit=self.config.doTrimToMatchCalib
2312 """Check if linearization is needed for the detector cameraGeom.
2314 Checks config.doLinearize and the linearity type of the first
2320 Detector to get linearity type
from.
2324 doLinearize : `Bool`
2325 If
True, linearization should be performed.
2327 return self.config.doLinearize
and \
2328 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2331 """Apply flat correction in place.
2336 Exposure to process.
2338 Flat exposure of the same size as ``exposure``.
2339 invert : `Bool`, optional
2340 If
True, unflatten an already flattened image.
2344 lsst.ip.isr.isrFunctions.flatCorrection
2346 isrFunctions.flatCorrection(
2347 maskedImage=exposure.getMaskedImage(),
2348 flatMaskedImage=flatExposure.getMaskedImage(),
2349 scalingType=self.config.flatScalingType,
2350 userScale=self.config.flatUserScale,
2352 trimToFit=self.config.doTrimToMatchCalib
2356 """Detect and mask saturated pixels in config.saturatedMaskName.
2361 Exposure to process. Only the amplifier DataSec is processed.
2363 Amplifier detector data.
2367 lsst.ip.isr.isrFunctions.makeThresholdMask
2369 if not math.isnan(amp.getSaturation()):
2370 maskedImage = exposure.getMaskedImage()
2371 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2372 isrFunctions.makeThresholdMask(
2373 maskedImage=dataView,
2374 threshold=amp.getSaturation(),
2376 maskName=self.config.saturatedMaskName,
2380 """Interpolate over saturated pixels, in place.
2382 This method should be called after `saturationDetection`, to
2383 ensure that the saturated pixels have been identified in the
2384 SAT mask. It should also be called after `assembleCcd`, since
2385 saturated regions may cross amplifier boundaries.
2390 Exposure to process.
2394 lsst.ip.isr.isrTask.saturationDetection
2395 lsst.ip.isr.isrFunctions.interpolateFromMask
2397 isrFunctions.interpolateFromMask(
2398 maskedImage=exposure.getMaskedImage(),
2399 fwhm=self.config.fwhm,
2400 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2401 maskNameList=list(self.config.saturatedMaskName),
2405 """Detect and mask suspect pixels in config.suspectMaskName.
2410 Exposure to process. Only the amplifier DataSec is processed.
2412 Amplifier detector data.
2416 lsst.ip.isr.isrFunctions.makeThresholdMask
2420 Suspect pixels are pixels whose value
is greater than
2421 amp.getSuspectLevel(). This
is intended to indicate pixels that may be
2422 affected by unknown systematics;
for example
if non-linearity
2423 corrections above a certain level are unstable then that would be a
2424 useful value
for suspectLevel. A value of `nan` indicates that no such
2425 level exists
and no pixels are to be masked
as suspicious.
2427 suspectLevel = amp.getSuspectLevel()
2428 if math.isnan(suspectLevel):
2431 maskedImage = exposure.getMaskedImage()
2432 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2433 isrFunctions.makeThresholdMask(
2434 maskedImage=dataView,
2435 threshold=suspectLevel,
2437 maskName=self.config.suspectMaskName,
2441 """Mask defects using mask plane "BAD", in place.
2446 Exposure to process.
2449 List of defects to mask.
2453 Call this after CCD assembly, since defects may cross amplifier
2456 maskedImage = exposure.getMaskedImage()
2457 if not isinstance(defectBaseList, Defects):
2459 defectList =
Defects(defectBaseList)
2461 defectList = defectBaseList
2462 defectList.maskPixels(maskedImage, maskName=
"BAD")
2464 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2465 """Mask edge pixels with applicable mask plane.
2470 Exposure to process.
2471 numEdgePixels : `int`, optional
2472 Number of edge pixels to mask.
2473 maskPlane : `str`, optional
2474 Mask plane name to use.
2475 level : `str`, optional
2476 Level at which to mask edges.
2478 maskedImage = exposure.getMaskedImage()
2479 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2481 if numEdgePixels > 0:
2482 if level ==
'DETECTOR':
2483 boxes = [maskedImage.getBBox()]
2484 elif level ==
'AMP':
2485 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2490 subImage = maskedImage[box]
2491 box.grow(-numEdgePixels)
2493 SourceDetectionTask.setEdgeBits(
2499 """Mask and interpolate defects using mask plane "BAD", in place.
2504 Exposure to process.
2507 List of defects to mask
and interpolate.
2511 lsst.ip.isr.isrTask.maskDefect
2513 self.maskDefectmaskDefect(exposure, defectBaseList)
2514 self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2515 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2516 isrFunctions.interpolateFromMask(
2517 maskedImage=exposure.getMaskedImage(),
2518 fwhm=self.config.fwhm,
2519 growSaturatedFootprints=0,
2520 maskNameList=[
"BAD"],
2524 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2529 Exposure to process.
2533 We mask over all non-finite values (NaN, inf), including those
2534 that are masked with other bits (because those may
or may
not be
2535 interpolated over later,
and we want to remove all NaN/infs).
2536 Despite this behaviour, the
"UNMASKEDNAN" mask plane
is used to
2537 preserve the historical name.
2539 maskedImage = exposure.getMaskedImage()
2542 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2543 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2544 numNans =
maskNans(maskedImage, maskVal)
2545 self.metadata[
"NUMNANS"] = numNans
2547 self.log.
warning(
"There were %d unmasked NaNs.", numNans)
2550 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2556 Exposure to process.
2560 lsst.ip.isr.isrTask.maskNan
2563 isrFunctions.interpolateFromMask(
2564 maskedImage=exposure.getMaskedImage(),
2565 fwhm=self.config.fwhm,
2566 growSaturatedFootprints=0,
2567 maskNameList=["UNMASKEDNAN"],
2571 """Measure the image background in subgrids, for quality control.
2576 Exposure to process.
2578 Configuration object containing parameters on which background
2579 statistics and subgrids to use.
2581 if IsrQaConfig
is not None:
2583 IsrQaConfig.flatness.nIter)
2584 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2585 statsControl.setAndMask(maskVal)
2586 maskedImage = exposure.getMaskedImage()
2588 skyLevel = stats.getValue(afwMath.MEDIAN)
2589 skySigma = stats.getValue(afwMath.STDEVCLIP)
2590 self.log.
info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2591 metadata = exposure.getMetadata()
2592 metadata[
"SKYLEVEL"] = skyLevel
2593 metadata[
"SKYSIGMA"] = skySigma
2596 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2597 meshXHalf =
int(IsrQaConfig.flatness.meshX/2.)
2598 meshYHalf =
int(IsrQaConfig.flatness.meshY/2.)
2599 nX =
int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2600 nY =
int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2601 skyLevels = numpy.zeros((nX, nY))
2604 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2606 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2608 xLLC = xc - meshXHalf
2609 yLLC = yc - meshYHalf
2610 xURC = xc + meshXHalf - 1
2611 yURC = yc + meshYHalf - 1
2614 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2618 good = numpy.where(numpy.isfinite(skyLevels))
2619 skyMedian = numpy.median(skyLevels[good])
2620 flatness = (skyLevels[good] - skyMedian) / skyMedian
2621 flatness_rms = numpy.std(flatness)
2622 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2624 self.log.
info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2625 self.log.
info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2626 nX, nY, flatness_pp, flatness_rms)
2628 metadata[
"FLATNESS_PP"] =
float(flatness_pp)
2629 metadata[
"FLATNESS_RMS"] =
float(flatness_rms)
2630 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2631 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2632 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2635 """Set an approximate magnitude zero point for the exposure.
2640 Exposure to process.
2642 filterLabel = exposure.getFilter()
2643 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2645 if physicalFilter
in self.config.fluxMag0T1:
2646 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2648 self.log.
warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2649 fluxMag0 = self.config.defaultFluxMag0T1
2651 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2653 self.log.
warning(
"Non-positive exposure time; skipping rough zero point.")
2656 self.log.
info(
"Setting rough magnitude zero point for filter %s: %f",
2657 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2662 """Context manager that applies and removes flats and darks,
2663 if the task
is configured to apply them.
2668 Exposure to process.
2670 Flat exposure the same size
as ``exp``.
2672 Dark exposure the same size
as ``exp``.
2677 The flat
and dark corrected exposure.
2679 if self.config.doDark
and dark
is not None:
2681 if self.config.doFlat:
2686 if self.config.doFlat:
2688 if self.config.doDark
and dark
is not None:
2692 """Utility function to examine ISR exposure at different stages.
2699 State of processing to view.
2704 display.scale(
'asinh',
'zscale')
2705 display.mtv(exposure)
2706 prompt =
"Press Enter to continue [c]... "
2708 ans = input(prompt).lower()
2709 if ans
in (
"",
"c",):
2714 """A Detector-like object that supports returning gain and saturation level
2716 This is used when the input exposure does
not have a detector.
2721 Exposure to generate a fake amplifier
for.
2722 config : `lsst.ip.isr.isrTaskConfig`
2723 Configuration to apply to the fake amplifier.
2727 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2729 self.
_gain_gain = config.gain
2734 return self.
_bbox_bbox
2737 return self.
_bbox_bbox
2743 return self.
_gain_gain
2756 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2760 """Task to wrap the default IsrTask to allow it to be retargeted.
2762 The standard IsrTask can be called directly from a command line
2763 program, but doing so removes the ability of the task to be
2764 retargeted. As most cameras override some set of the IsrTask
2765 methods, this would remove those data-specific methods
in the
2766 output post-ISR images. This wrapping
class fixes the issue,
2767 allowing identical post-ISR images to be generated by both the
2768 processCcd
and isrTask code.
2770 ConfigClass = RunIsrConfig
2771 _DefaultName = "runIsr"
2775 self.makeSubtask(
"isr")
2782 data reference of the detector data to be processed
2786 result : `pipeBase.Struct`
2787 Result struct with component:
2790 Post-ISR processed exposure.
An immutable representation of a camera.
A representation of a detector in a mosaic camera.
Encapsulate information about a bad portion of a detector.
A class to contain the data, WCS, and other information needed to describe an image of the sky.
A class to represent a 2-dimensional array of pixels.
Represent a 2-dimensional array of bitmask pixels.
A class to manipulate images, masks, and variance as a single object.
A spatially-varying transmission curve as a function of wavelength.
Pass parameters to a Statistics object.
A custom container class for records, based on std::vector.
An integer coordinate rectangle.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def readIsrData(self, dataRef, rawExposure)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
daf::base::PropertyList * list
daf::base::PropertySet * set
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::shared_ptr< PhotoCalib > makePhotoCalibFromCalibZeroPoint(double instFluxMag0, double instFluxMag0Err)
Construct a PhotoCalib from the deprecated Calib-style instFluxMag0/instFluxMag0Err values.
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT > > image, typename std::shared_ptr< Mask< MaskPixelT > > mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT > > variance=Image< VariancePixelT >())
A function to return a MaskedImage of the correct type (cf.
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
A function to return an Exposure of the correct type (cf.
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def getDebugFrame(debugDisplay, name)