32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
43 from .
import isrFunctions
45 from .
import linearize
46 from .defects
import Defects
48 from .assembleCcdTask
import AssembleCcdTask
49 from .crosstalk
import CrosstalkTask, CrosstalkCalib
50 from .fringe
import FringeTask
51 from .isr
import maskNans
52 from .masking
import MaskingTask
53 from .overscan
import OverscanCorrectionTask
54 from .straylight
import StrayLightTask
55 from .vignette
import VignetteTask
56 from lsst.daf.butler
import DimensionGraph
59 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
69 This will be unused until DM-25348 resolves the quantum graph
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
91 results =
list(registry.queryDatasets(datasetType,
92 collections=collections,
100 dimensions={
"instrument",
"exposure",
"detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
104 doc=
"Input exposure to process.",
105 storageClass=
"Exposure",
106 dimensions=[
"instrument",
"exposure",
"detector"],
108 camera = cT.PrerequisiteInput(
110 storageClass=
"Camera",
111 doc=
"Input camera to construct complete exposures.",
112 dimensions=[
"instrument"],
116 crosstalk = cT.PrerequisiteInput(
118 doc=
"Input crosstalk object",
119 storageClass=
"CrosstalkCalib",
120 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
134 bias = cT.PrerequisiteInput(
136 doc=
"Input bias calibration.",
137 storageClass=
"ExposureF",
138 dimensions=[
"instrument",
"detector"],
141 dark = cT.PrerequisiteInput(
143 doc=
"Input dark calibration.",
144 storageClass=
"ExposureF",
145 dimensions=[
"instrument",
"detector"],
148 flat = cT.PrerequisiteInput(
150 doc=
"Input flat calibration.",
151 storageClass=
"ExposureF",
152 dimensions=[
"instrument",
"physical_filter",
"detector"],
155 fringes = cT.PrerequisiteInput(
157 doc=
"Input fringe calibration.",
158 storageClass=
"ExposureF",
159 dimensions=[
"instrument",
"physical_filter",
"detector"],
162 strayLightData = cT.PrerequisiteInput(
164 doc=
"Input stray light calibration.",
165 storageClass=
"StrayLightData",
166 dimensions=[
"instrument",
"physical_filter",
"detector"],
169 bfKernel = cT.PrerequisiteInput(
171 doc=
"Input brighter-fatter kernel.",
172 storageClass=
"NumpyArray",
173 dimensions=[
"instrument"],
176 newBFKernel = cT.PrerequisiteInput(
177 name=
'brighterFatterKernel',
178 doc=
"Newer complete kernel + gain solutions.",
179 storageClass=
"BrighterFatterKernel",
180 dimensions=[
"instrument",
"detector"],
183 defects = cT.PrerequisiteInput(
185 doc=
"Input defect tables.",
186 storageClass=
"Defects",
187 dimensions=[
"instrument",
"detector"],
190 opticsTransmission = cT.PrerequisiteInput(
191 name=
"transmission_optics",
192 storageClass=
"TransmissionCurve",
193 doc=
"Transmission curve due to the optics.",
194 dimensions=[
"instrument"],
197 filterTransmission = cT.PrerequisiteInput(
198 name=
"transmission_filter",
199 storageClass=
"TransmissionCurve",
200 doc=
"Transmission curve due to the filter.",
201 dimensions=[
"instrument",
"physical_filter"],
204 sensorTransmission = cT.PrerequisiteInput(
205 name=
"transmission_sensor",
206 storageClass=
"TransmissionCurve",
207 doc=
"Transmission curve due to the sensor.",
208 dimensions=[
"instrument",
"detector"],
211 atmosphereTransmission = cT.PrerequisiteInput(
212 name=
"transmission_atmosphere",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the atmosphere.",
215 dimensions=[
"instrument"],
218 illumMaskedImage = cT.PrerequisiteInput(
220 doc=
"Input illumination correction.",
221 storageClass=
"MaskedImageF",
222 dimensions=[
"instrument",
"physical_filter",
"detector"],
226 outputExposure = cT.Output(
228 doc=
"Output ISR processed exposure.",
229 storageClass=
"Exposure",
230 dimensions=[
"instrument",
"exposure",
"detector"],
232 preInterpExposure = cT.Output(
233 name=
'preInterpISRCCD',
234 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
235 storageClass=
"ExposureF",
236 dimensions=[
"instrument",
"exposure",
"detector"],
238 outputOssThumbnail = cT.Output(
240 doc=
"Output Overscan-subtracted thumbnail image.",
241 storageClass=
"Thumbnail",
242 dimensions=[
"instrument",
"exposure",
"detector"],
244 outputFlattenedThumbnail = cT.Output(
245 name=
"FlattenedThumb",
246 doc=
"Output flat-corrected thumbnail image.",
247 storageClass=
"Thumbnail",
248 dimensions=[
"instrument",
"exposure",
"detector"],
254 if config.doBias
is not True:
255 self.prerequisiteInputs.discard(
"bias")
256 if config.doLinearize
is not True:
257 self.prerequisiteInputs.discard(
"linearizer")
258 if config.doCrosstalk
is not True:
259 self.inputs.discard(
"crosstalkSources")
260 self.prerequisiteInputs.discard(
"crosstalk")
261 if config.doBrighterFatter
is not True:
262 self.prerequisiteInputs.discard(
"bfKernel")
263 self.prerequisiteInputs.discard(
"newBFKernel")
264 if config.doDefect
is not True:
265 self.prerequisiteInputs.discard(
"defects")
266 if config.doDark
is not True:
267 self.prerequisiteInputs.discard(
"dark")
268 if config.doFlat
is not True:
269 self.prerequisiteInputs.discard(
"flat")
270 if config.doAttachTransmissionCurve
is not True:
271 self.prerequisiteInputs.discard(
"opticsTransmission")
272 self.prerequisiteInputs.discard(
"filterTransmission")
273 self.prerequisiteInputs.discard(
"sensorTransmission")
274 self.prerequisiteInputs.discard(
"atmosphereTransmission")
275 if config.doUseOpticsTransmission
is not True:
276 self.prerequisiteInputs.discard(
"opticsTransmission")
277 if config.doUseFilterTransmission
is not True:
278 self.prerequisiteInputs.discard(
"filterTransmission")
279 if config.doUseSensorTransmission
is not True:
280 self.prerequisiteInputs.discard(
"sensorTransmission")
281 if config.doUseAtmosphereTransmission
is not True:
282 self.prerequisiteInputs.discard(
"atmosphereTransmission")
283 if config.doIlluminationCorrection
is not True:
284 self.prerequisiteInputs.discard(
"illumMaskedImage")
286 if config.doWrite
is not True:
287 self.outputs.discard(
"outputExposure")
288 self.outputs.discard(
"preInterpExposure")
289 self.outputs.discard(
"outputFlattenedThumbnail")
290 self.outputs.discard(
"outputOssThumbnail")
291 if config.doSaveInterpPixels
is not True:
292 self.outputs.discard(
"preInterpExposure")
293 if config.qa.doThumbnailOss
is not True:
294 self.outputs.discard(
"outputOssThumbnail")
295 if config.qa.doThumbnailFlattened
is not True:
296 self.outputs.discard(
"outputFlattenedThumbnail")
300 pipelineConnections=IsrTaskConnections):
301 """Configuration parameters for IsrTask.
303 Items are grouped in the order in which they are executed by the task.
305 datasetType = pexConfig.Field(
307 doc=
"Dataset type for input data; users will typically leave this alone, "
308 "but camera-specific ISR tasks will override it",
312 fallbackFilterName = pexConfig.Field(
314 doc=
"Fallback default filter name for calibrations.",
317 useFallbackDate = pexConfig.Field(
319 doc=
"Pass observation date when using fallback filter.",
322 expectWcs = pexConfig.Field(
325 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
327 fwhm = pexConfig.Field(
329 doc=
"FWHM of PSF in arcseconds.",
332 qa = pexConfig.ConfigField(
334 doc=
"QA related configuration options.",
338 doConvertIntToFloat = pexConfig.Field(
340 doc=
"Convert integer raw images to floating point values?",
345 doSaturation = pexConfig.Field(
347 doc=
"Mask saturated pixels? NB: this is totally independent of the"
348 " interpolation option - this is ONLY setting the bits in the mask."
349 " To have them interpolated make sure doSaturationInterpolation=True",
352 saturatedMaskName = pexConfig.Field(
354 doc=
"Name of mask plane to use in saturation detection and interpolation",
357 saturation = pexConfig.Field(
359 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
360 default=float(
"NaN"),
362 growSaturationFootprintSize = pexConfig.Field(
364 doc=
"Number of pixels by which to grow the saturation footprints",
369 doSuspect = pexConfig.Field(
371 doc=
"Mask suspect pixels?",
374 suspectMaskName = pexConfig.Field(
376 doc=
"Name of mask plane to use for suspect pixels",
379 numEdgeSuspect = pexConfig.Field(
381 doc=
"Number of edge pixels to be flagged as untrustworthy.",
384 edgeMaskLevel = pexConfig.ChoiceField(
386 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
389 'DETECTOR':
'Mask only the edges of the full detector.',
390 'AMP':
'Mask edges of each amplifier.',
395 doSetBadRegions = pexConfig.Field(
397 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
400 badStatistic = pexConfig.ChoiceField(
402 doc=
"How to estimate the average value for BAD regions.",
405 "MEANCLIP":
"Correct using the (clipped) mean of good data",
406 "MEDIAN":
"Correct using the median of the good data",
411 doOverscan = pexConfig.Field(
413 doc=
"Do overscan subtraction?",
416 overscan = pexConfig.ConfigurableField(
417 target=OverscanCorrectionTask,
418 doc=
"Overscan subtraction task for image segments.",
421 overscanFitType = pexConfig.ChoiceField(
423 doc=
"The method for fitting the overscan bias level.",
426 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
427 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
428 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
429 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
430 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
431 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
432 "MEAN":
"Correct using the mean of the overscan region",
433 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
434 "MEDIAN":
"Correct using the median of the overscan region",
435 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
437 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
438 " This option will no longer be used, and will be removed after v20.")
440 overscanOrder = pexConfig.Field(
442 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
443 "or number of spline knots if overscan fit type is a spline."),
445 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
446 " This option will no longer be used, and will be removed after v20.")
448 overscanNumSigmaClip = pexConfig.Field(
450 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
452 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
453 " This option will no longer be used, and will be removed after v20.")
455 overscanIsInt = pexConfig.Field(
457 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
458 " and overscan.FitType=MEDIAN_PER_ROW.",
460 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
461 " This option will no longer be used, and will be removed after v20.")
464 overscanNumLeadingColumnsToSkip = pexConfig.Field(
466 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
469 overscanNumTrailingColumnsToSkip = pexConfig.Field(
471 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
474 overscanMaxDev = pexConfig.Field(
476 doc=
"Maximum deviation from the median for overscan",
477 default=1000.0, check=
lambda x: x > 0
479 overscanBiasJump = pexConfig.Field(
481 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
484 overscanBiasJumpKeyword = pexConfig.Field(
486 doc=
"Header keyword containing information about devices.",
487 default=
"NO_SUCH_KEY",
489 overscanBiasJumpDevices = pexConfig.ListField(
491 doc=
"List of devices that need piecewise overscan correction.",
494 overscanBiasJumpLocation = pexConfig.Field(
496 doc=
"Location of bias jump along y-axis.",
501 doAssembleCcd = pexConfig.Field(
504 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
506 assembleCcd = pexConfig.ConfigurableField(
507 target=AssembleCcdTask,
508 doc=
"CCD assembly task",
512 doAssembleIsrExposures = pexConfig.Field(
515 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
517 doTrimToMatchCalib = pexConfig.Field(
520 doc=
"Trim raw data to match calibration bounding boxes?"
524 doBias = pexConfig.Field(
526 doc=
"Apply bias frame correction?",
529 biasDataProductName = pexConfig.Field(
531 doc=
"Name of the bias data product",
534 doBiasBeforeOverscan = pexConfig.Field(
536 doc=
"Reverse order of overscan and bias correction.",
541 doVariance = pexConfig.Field(
543 doc=
"Calculate variance?",
546 gain = pexConfig.Field(
548 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
549 default=float(
"NaN"),
551 readNoise = pexConfig.Field(
553 doc=
"The read noise to use if no Detector is present in the Exposure",
556 doEmpiricalReadNoise = pexConfig.Field(
559 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
563 doLinearize = pexConfig.Field(
565 doc=
"Correct for nonlinearity of the detector's response?",
570 doCrosstalk = pexConfig.Field(
572 doc=
"Apply intra-CCD crosstalk correction?",
575 doCrosstalkBeforeAssemble = pexConfig.Field(
577 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
580 crosstalk = pexConfig.ConfigurableField(
581 target=CrosstalkTask,
582 doc=
"Intra-CCD crosstalk correction",
586 doDefect = pexConfig.Field(
588 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
591 doNanMasking = pexConfig.Field(
593 doc=
"Mask NAN pixels?",
596 doWidenSaturationTrails = pexConfig.Field(
598 doc=
"Widen bleed trails based on their width?",
603 doBrighterFatter = pexConfig.Field(
606 doc=
"Apply the brighter fatter correction"
608 brighterFatterLevel = pexConfig.ChoiceField(
611 doc=
"The level at which to correct for brighter-fatter.",
613 "AMP":
"Every amplifier treated separately.",
614 "DETECTOR":
"One kernel per detector",
617 brighterFatterMaxIter = pexConfig.Field(
620 doc=
"Maximum number of iterations for the brighter fatter correction"
622 brighterFatterThreshold = pexConfig.Field(
625 doc=
"Threshold used to stop iterating the brighter fatter correction. It is the "
626 " absolute value of the difference between the current corrected image and the one"
627 " from the previous iteration summed over all the pixels."
629 brighterFatterApplyGain = pexConfig.Field(
632 doc=
"Should the gain be applied when applying the brighter fatter correction?"
634 brighterFatterMaskGrowSize = pexConfig.Field(
637 doc=
"Number of pixels to grow the masks listed in config.maskListToInterpolate "
638 " when brighter-fatter correction is applied."
642 doDark = pexConfig.Field(
644 doc=
"Apply dark frame correction?",
647 darkDataProductName = pexConfig.Field(
649 doc=
"Name of the dark data product",
654 doStrayLight = pexConfig.Field(
656 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
659 strayLight = pexConfig.ConfigurableField(
660 target=StrayLightTask,
661 doc=
"y-band stray light correction"
665 doFlat = pexConfig.Field(
667 doc=
"Apply flat field correction?",
670 flatDataProductName = pexConfig.Field(
672 doc=
"Name of the flat data product",
675 flatScalingType = pexConfig.ChoiceField(
677 doc=
"The method for scaling the flat on the fly.",
680 "USER":
"Scale by flatUserScale",
681 "MEAN":
"Scale by the inverse of the mean",
682 "MEDIAN":
"Scale by the inverse of the median",
685 flatUserScale = pexConfig.Field(
687 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
690 doTweakFlat = pexConfig.Field(
692 doc=
"Tweak flats to match observed amplifier ratios?",
697 doApplyGains = pexConfig.Field(
699 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
702 normalizeGains = pexConfig.Field(
704 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
709 doFringe = pexConfig.Field(
711 doc=
"Apply fringe correction?",
714 fringe = pexConfig.ConfigurableField(
716 doc=
"Fringe subtraction task",
718 fringeAfterFlat = pexConfig.Field(
720 doc=
"Do fringe subtraction after flat-fielding?",
725 doMeasureBackground = pexConfig.Field(
727 doc=
"Measure the background level on the reduced image?",
732 doCameraSpecificMasking = pexConfig.Field(
734 doc=
"Mask camera-specific bad regions?",
737 masking = pexConfig.ConfigurableField(
744 doInterpolate = pexConfig.Field(
746 doc=
"Interpolate masked pixels?",
749 doSaturationInterpolation = pexConfig.Field(
751 doc=
"Perform interpolation over pixels masked as saturated?"
752 " NB: This is independent of doSaturation; if that is False this plane"
753 " will likely be blank, resulting in a no-op here.",
756 doNanInterpolation = pexConfig.Field(
758 doc=
"Perform interpolation over pixels masked as NaN?"
759 " NB: This is independent of doNanMasking; if that is False this plane"
760 " will likely be blank, resulting in a no-op here.",
763 doNanInterpAfterFlat = pexConfig.Field(
765 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
766 "also have to interpolate them before flat-fielding."),
769 maskListToInterpolate = pexConfig.ListField(
771 doc=
"List of mask planes that should be interpolated.",
772 default=[
'SAT',
'BAD',
'UNMASKEDNAN'],
774 doSaveInterpPixels = pexConfig.Field(
776 doc=
"Save a copy of the pre-interpolated pixel values?",
781 fluxMag0T1 = pexConfig.DictField(
784 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
785 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
788 defaultFluxMag0T1 = pexConfig.Field(
790 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
791 default=pow(10.0, 0.4*28.0)
795 doVignette = pexConfig.Field(
797 doc=
"Apply vignetting parameters?",
800 vignette = pexConfig.ConfigurableField(
802 doc=
"Vignetting task.",
806 doAttachTransmissionCurve = pexConfig.Field(
809 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
811 doUseOpticsTransmission = pexConfig.Field(
814 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
816 doUseFilterTransmission = pexConfig.Field(
819 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
821 doUseSensorTransmission = pexConfig.Field(
824 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
826 doUseAtmosphereTransmission = pexConfig.Field(
829 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
833 doIlluminationCorrection = pexConfig.Field(
836 doc=
"Perform illumination correction?"
838 illuminationCorrectionDataProductName = pexConfig.Field(
840 doc=
"Name of the illumination correction data product.",
843 illumScale = pexConfig.Field(
845 doc=
"Scale factor for the illumination correction.",
848 illumFilters = pexConfig.ListField(
851 doc=
"Only perform illumination correction for these filters."
855 doWrite = pexConfig.Field(
857 doc=
"Persist postISRCCD?",
864 raise ValueError(
"You may not specify both doFlat and doApplyGains")
866 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
875 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
876 """Apply common instrument signature correction algorithms to a raw frame.
878 The process for correcting imaging data is very similar from
879 camera to camera. This task provides a vanilla implementation of
880 doing these corrections, including the ability to turn certain
881 corrections off if they are not needed. The inputs to the primary
882 method, `run()`, are a raw exposure to be corrected and the
883 calibration data products. The raw input is a single chip sized
884 mosaic of all amps including overscans and other non-science
885 pixels. The method `runDataRef()` identifies and defines the
886 calibration data products, and is intended for use by a
887 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
888 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
889 subclassed for different camera, although the most camera specific
890 methods have been split into subtasks that can be redirected
893 The __init__ method sets up the subtasks for ISR processing, using
894 the defaults from `lsst.ip.isr`.
899 Positional arguments passed to the Task constructor. None used at this time.
900 kwargs : `dict`, optional
901 Keyword arguments passed on to the Task constructor. None used at this time.
903 ConfigClass = IsrTaskConfig
908 self.makeSubtask(
"assembleCcd")
909 self.makeSubtask(
"crosstalk")
910 self.makeSubtask(
"strayLight")
911 self.makeSubtask(
"fringe")
912 self.makeSubtask(
"masking")
913 self.makeSubtask(
"overscan")
914 self.makeSubtask(
"vignette")
917 inputs = butlerQC.get(inputRefs)
920 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
921 except Exception
as e:
922 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
925 inputs[
'isGen3'] =
True
927 detector = inputs[
'ccdExposure'].getDetector()
929 if self.config.doCrosstalk
is True:
932 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
933 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
934 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
936 coeffVector = (self.config.crosstalk.crosstalkValues
937 if self.config.crosstalk.useConfigCoefficients
else None)
938 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
939 inputs[
'crosstalk'] = crosstalkCalib
940 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
941 if 'crosstalkSources' not in inputs:
942 self.log.
warn(
"No crosstalkSources found for chip with interChip terms!")
945 if 'linearizer' in inputs
and isinstance(inputs[
'linearizer'], dict):
947 linearizer.fromYaml(inputs[
'linearizer'])
951 inputs[
'linearizer'] = linearizer
953 if self.config.doDefect
is True:
954 if "defects" in inputs
and inputs[
'defects']
is not None:
957 if not isinstance(inputs[
"defects"], Defects):
958 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
962 if self.config.doBrighterFatter:
963 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
964 if brighterFatterKernel
is None:
965 brighterFatterKernel = inputs.get(
'bfKernel',
None)
967 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
968 detId = detector.getId()
969 inputs[
'bfGains'] = brighterFatterKernel.gain
972 if self.config.brighterFatterLevel ==
'DETECTOR':
973 if brighterFatterKernel.detectorKernel:
974 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernel[detId]
975 elif brighterFatterKernel.detectorKernelFromAmpKernels:
976 inputs[
'bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
978 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
981 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
983 if self.config.doFringe
is True and self.fringe.checkFilter(inputs[
'ccdExposure']):
984 expId = inputs[
'ccdExposure'].
getInfo().getVisitInfo().getExposureId()
985 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
987 assembler=self.assembleCcd
988 if self.config.doAssembleIsrExposures
else None)
990 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
992 if self.config.doStrayLight
is True and self.strayLight.checkFilter(inputs[
'ccdExposure']):
993 if 'strayLightData' not in inputs:
994 inputs[
'strayLightData'] =
None
996 outputs = self.
run(**inputs)
997 butlerQC.put(outputs, outputRefs)
1000 """Retrieve necessary frames for instrument signature removal.
1002 Pre-fetching all required ISR data products limits the IO
1003 required by the ISR. Any conflict between the calibration data
1004 available and that needed for ISR is also detected prior to
1005 doing processing, allowing it to fail quickly.
1009 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1010 Butler reference of the detector data to be processed
1011 rawExposure : `afw.image.Exposure`
1012 The raw exposure that will later be corrected with the
1013 retrieved calibration data; should not be modified in this
1018 result : `lsst.pipe.base.Struct`
1019 Result struct with components (which may be `None`):
1020 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1021 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1022 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1023 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1024 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1025 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1026 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1027 - ``fringes``: `lsst.pipe.base.Struct` with components:
1028 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1029 - ``seed``: random seed derived from the ccdExposureId for random
1030 number generator (`uint32`).
1031 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1032 A ``TransmissionCurve`` that represents the throughput of the optics,
1033 to be evaluated in focal-plane coordinates.
1034 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1035 A ``TransmissionCurve`` that represents the throughput of the filter
1036 itself, to be evaluated in focal-plane coordinates.
1037 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1038 A ``TransmissionCurve`` that represents the throughput of the sensor
1039 itself, to be evaluated in post-assembly trimmed detector coordinates.
1040 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1041 A ``TransmissionCurve`` that represents the throughput of the
1042 atmosphere, assumed to be spatially constant.
1043 - ``strayLightData`` : `object`
1044 An opaque object containing calibration information for
1045 stray-light correction. If `None`, no correction will be
1047 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1051 NotImplementedError :
1052 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1055 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1056 dateObs = dateObs.toPython().isoformat()
1057 except RuntimeError:
1058 self.log.
warn(
"Unable to identify dateObs for rawExposure.")
1061 ccd = rawExposure.getDetector()
1062 filterName =
afwImage.Filter(rawExposure.getFilter().getId()).getName()
1063 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1064 biasExposure = (self.
getIsrExposure(dataRef, self.config.biasDataProductName)
1065 if self.config.doBias
else None)
1067 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1069 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1070 linearizer.log = self.log
1071 if isinstance(linearizer, numpy.ndarray):
1074 crosstalkCalib =
None
1075 if self.config.doCrosstalk:
1077 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1079 coeffVector = (self.config.crosstalk.crosstalkValues
1080 if self.config.crosstalk.useConfigCoefficients
else None)
1081 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1082 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1083 if self.config.doCrosstalk
else None)
1085 darkExposure = (self.
getIsrExposure(dataRef, self.config.darkDataProductName)
1086 if self.config.doDark
else None)
1087 flatExposure = (self.
getIsrExposure(dataRef, self.config.flatDataProductName,
1089 if self.config.doFlat
else None)
1091 brighterFatterKernel =
None
1092 brighterFatterGains =
None
1093 if self.config.doBrighterFatter
is True:
1098 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1099 brighterFatterGains = brighterFatterKernel.gain
1100 self.log.
info(
"New style bright-fatter kernel (brighterFatterKernel) loaded")
1103 brighterFatterKernel = dataRef.get(
"bfKernel")
1104 self.log.
info(
"Old style bright-fatter kernel (np.array) loaded")
1106 brighterFatterKernel =
None
1107 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1110 if self.config.brighterFatterLevel ==
'DETECTOR':
1111 if brighterFatterKernel.detectorKernel:
1112 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1113 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1114 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1116 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1119 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1121 defectList = (dataRef.get(
"defects")
1122 if self.config.doDefect
else None)
1123 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1124 if self.config.doAssembleIsrExposures
else None)
1125 if self.config.doFringe
and self.fringe.checkFilter(rawExposure)
1126 else pipeBase.Struct(fringes=
None))
1128 if self.config.doAttachTransmissionCurve:
1129 opticsTransmission = (dataRef.get(
"transmission_optics")
1130 if self.config.doUseOpticsTransmission
else None)
1131 filterTransmission = (dataRef.get(
"transmission_filter")
1132 if self.config.doUseFilterTransmission
else None)
1133 sensorTransmission = (dataRef.get(
"transmission_sensor")
1134 if self.config.doUseSensorTransmission
else None)
1135 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1136 if self.config.doUseAtmosphereTransmission
else None)
1138 opticsTransmission =
None
1139 filterTransmission =
None
1140 sensorTransmission =
None
1141 atmosphereTransmission =
None
1143 if self.config.doStrayLight:
1144 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1146 strayLightData =
None
1149 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1150 if (self.config.doIlluminationCorrection
1151 and filterName
in self.config.illumFilters)
1155 return pipeBase.Struct(bias=biasExposure,
1156 linearizer=linearizer,
1157 crosstalk=crosstalkCalib,
1158 crosstalkSources=crosstalkSources,
1161 bfKernel=brighterFatterKernel,
1162 bfGains=brighterFatterGains,
1164 fringes=fringeStruct,
1165 opticsTransmission=opticsTransmission,
1166 filterTransmission=filterTransmission,
1167 sensorTransmission=sensorTransmission,
1168 atmosphereTransmission=atmosphereTransmission,
1169 strayLightData=strayLightData,
1170 illumMaskedImage=illumMaskedImage
1173 @pipeBase.timeMethod
1174 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1175 crosstalk=None, crosstalkSources=None,
1176 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1177 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1178 sensorTransmission=
None, atmosphereTransmission=
None,
1179 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1182 """Perform instrument signature removal on an exposure.
1184 Steps included in the ISR processing, in order performed, are:
1185 - saturation and suspect pixel masking
1186 - overscan subtraction
1187 - CCD assembly of individual amplifiers
1189 - variance image construction
1190 - linearization of non-linear response
1192 - brighter-fatter correction
1195 - stray light subtraction
1197 - masking of known defects and camera specific features
1198 - vignette calculation
1199 - appending transmission curve and distortion model
1203 ccdExposure : `lsst.afw.image.Exposure`
1204 The raw exposure that is to be run through ISR. The
1205 exposure is modified by this method.
1206 camera : `lsst.afw.cameraGeom.Camera`, optional
1207 The camera geometry for this exposure. Required if ``isGen3`` is
1208 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1209 ``flat`` does not have an associated detector.
1210 bias : `lsst.afw.image.Exposure`, optional
1211 Bias calibration frame.
1212 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1213 Functor for linearization.
1214 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1215 Calibration for crosstalk.
1216 crosstalkSources : `list`, optional
1217 List of possible crosstalk sources.
1218 dark : `lsst.afw.image.Exposure`, optional
1219 Dark calibration frame.
1220 flat : `lsst.afw.image.Exposure`, optional
1221 Flat calibration frame.
1222 bfKernel : `numpy.ndarray`, optional
1223 Brighter-fatter kernel.
1224 bfGains : `dict` of `float`, optional
1225 Gains used to override the detector's nominal gains for the
1226 brighter-fatter correction. A dict keyed by amplifier name for
1227 the detector in question.
1228 defects : `lsst.ip.isr.Defects`, optional
1230 fringes : `lsst.pipe.base.Struct`, optional
1231 Struct containing the fringe correction data, with
1233 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1234 - ``seed``: random seed derived from the ccdExposureId for random
1235 number generator (`uint32`)
1236 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1237 A ``TransmissionCurve`` that represents the throughput of the optics,
1238 to be evaluated in focal-plane coordinates.
1239 filterTransmission : `lsst.afw.image.TransmissionCurve`
1240 A ``TransmissionCurve`` that represents the throughput of the filter
1241 itself, to be evaluated in focal-plane coordinates.
1242 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1243 A ``TransmissionCurve`` that represents the throughput of the sensor
1244 itself, to be evaluated in post-assembly trimmed detector coordinates.
1245 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1246 A ``TransmissionCurve`` that represents the throughput of the
1247 atmosphere, assumed to be spatially constant.
1248 detectorNum : `int`, optional
1249 The integer number for the detector to process.
1250 isGen3 : bool, optional
1251 Flag this call to run() as using the Gen3 butler environment.
1252 strayLightData : `object`, optional
1253 Opaque object containing calibration information for stray-light
1254 correction. If `None`, no correction will be performed.
1255 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1256 Illumination correction image.
1260 result : `lsst.pipe.base.Struct`
1261 Result struct with component:
1262 - ``exposure`` : `afw.image.Exposure`
1263 The fully ISR corrected exposure.
1264 - ``outputExposure`` : `afw.image.Exposure`
1265 An alias for `exposure`
1266 - ``ossThumb`` : `numpy.ndarray`
1267 Thumbnail image of the exposure after overscan subtraction.
1268 - ``flattenedThumb`` : `numpy.ndarray`
1269 Thumbnail image of the exposure after flat-field correction.
1274 Raised if a configuration option is set to True, but the
1275 required calibration data has not been specified.
1279 The current processed exposure can be viewed by setting the
1280 appropriate lsstDebug entries in the `debug.display`
1281 dictionary. The names of these entries correspond to some of
1282 the IsrTaskConfig Boolean options, with the value denoting the
1283 frame to use. The exposure is shown inside the matching
1284 option check and after the processing of that step has
1285 finished. The steps with debug points are:
1296 In addition, setting the "postISRCCD" entry displays the
1297 exposure after all ISR processing has finished.
1305 if detectorNum
is None:
1306 raise RuntimeError(
"Must supply the detectorNum if running as Gen3.")
1308 ccdExposure = self.
ensureExposure(ccdExposure, camera, detectorNum)
1313 if isinstance(ccdExposure, ButlerDataRef):
1316 ccd = ccdExposure.getDetector()
1317 filterName =
afwImage.Filter(ccdExposure.getFilter().getId()).getName()
1320 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1321 ccd = [
FakeAmp(ccdExposure, self.config)]
1324 if self.config.doBias
and bias
is None:
1325 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1327 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1328 if self.config.doBrighterFatter
and bfKernel
is None:
1329 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1330 if self.config.doDark
and dark
is None:
1331 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1332 if self.config.doFlat
and flat
is None:
1333 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1334 if self.config.doDefect
and defects
is None:
1335 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1336 if (self.config.doFringe
and filterName
in self.fringe.config.filters
1337 and fringes.fringes
is None):
1342 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1343 if (self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters
1344 and illumMaskedImage
is None):
1345 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1348 if self.config.doConvertIntToFloat:
1349 self.log.
info(
"Converting exposure to floating point values.")
1352 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1353 self.log.
info(
"Applying bias correction.")
1354 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1355 trimToFit=self.config.doTrimToMatchCalib)
1362 if ccdExposure.getBBox().
contains(amp.getBBox()):
1366 if self.config.doOverscan
and not badAmp:
1369 self.log.
debug(
"Corrected overscan for amplifier %s.", amp.getName())
1370 if overscanResults
is not None and \
1371 self.config.qa
is not None and self.config.qa.saveStats
is True:
1372 if isinstance(overscanResults.overscanFit, float):
1373 qaMedian = overscanResults.overscanFit
1374 qaStdev = float(
"NaN")
1377 afwMath.MEDIAN | afwMath.STDEVCLIP)
1378 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1379 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1381 self.metadata.
set(f
"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1382 self.metadata.
set(f
"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1383 self.log.
debug(
" Overscan stats for amplifer %s: %f +/- %f",
1384 amp.getName(), qaMedian, qaStdev)
1385 ccdExposure.getMetadata().
set(
'OVERSCAN',
"Overscan corrected")
1388 self.log.
warn(
"Amplifier %s is bad.", amp.getName())
1389 overscanResults =
None
1391 overscans.append(overscanResults
if overscanResults
is not None else None)
1393 self.log.
info(
"Skipped OSCAN for %s.", amp.getName())
1395 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1396 self.log.
info(
"Applying crosstalk correction.")
1397 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1398 crosstalkSources=crosstalkSources)
1399 self.
debugView(ccdExposure,
"doCrosstalk")
1401 if self.config.doAssembleCcd:
1402 self.log.
info(
"Assembling CCD from amplifiers.")
1403 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1405 if self.config.expectWcs
and not ccdExposure.getWcs():
1406 self.log.
warn(
"No WCS found in input exposure.")
1407 self.
debugView(ccdExposure,
"doAssembleCcd")
1410 if self.config.qa.doThumbnailOss:
1411 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1413 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1414 self.log.
info(
"Applying bias correction.")
1415 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1416 trimToFit=self.config.doTrimToMatchCalib)
1419 if self.config.doVariance:
1420 for amp, overscanResults
in zip(ccd, overscans):
1421 if ccdExposure.getBBox().
contains(amp.getBBox()):
1422 self.log.
debug(
"Constructing variance map for amplifer %s.", amp.getName())
1423 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1424 if overscanResults
is not None:
1426 overscanImage=overscanResults.overscanImage)
1430 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1432 afwMath.MEDIAN | afwMath.STDEVCLIP)
1433 self.metadata.
set(f
"ISR VARIANCE {amp.getName()} MEDIAN",
1434 qaStats.getValue(afwMath.MEDIAN))
1435 self.metadata.
set(f
"ISR VARIANCE {amp.getName()} STDEV",
1436 qaStats.getValue(afwMath.STDEVCLIP))
1437 self.log.
debug(
" Variance stats for amplifer %s: %f +/- %f.",
1438 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1439 qaStats.getValue(afwMath.STDEVCLIP))
1442 self.log.
info(
"Applying linearizer.")
1443 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1444 detector=ccd, log=self.log)
1446 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1447 self.log.
info(
"Applying crosstalk correction.")
1448 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1449 crosstalkSources=crosstalkSources, isTrimmed=
True)
1450 self.
debugView(ccdExposure,
"doCrosstalk")
1454 if self.config.doDefect:
1455 self.log.
info(
"Masking defects.")
1458 if self.config.numEdgeSuspect > 0:
1459 self.log.
info(
"Masking edges as SUSPECT.")
1460 self.
maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1461 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1463 if self.config.doNanMasking:
1464 self.log.
info(
"Masking NAN value pixels.")
1467 if self.config.doWidenSaturationTrails:
1468 self.log.
info(
"Widening saturation trails.")
1469 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1471 if self.config.doCameraSpecificMasking:
1472 self.log.
info(
"Masking regions for camera specific reasons.")
1473 self.masking.
run(ccdExposure)
1475 if self.config.doBrighterFatter:
1484 interpExp = ccdExposure.clone()
1486 isrFunctions.interpolateFromMask(
1487 maskedImage=interpExp.getMaskedImage(),
1488 fwhm=self.config.fwhm,
1489 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1490 maskNameList=self.config.maskListToInterpolate
1492 bfExp = interpExp.clone()
1494 self.log.
info(
"Applying brighter fatter correction using kernel type %s / gains %s.",
1496 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1497 self.config.brighterFatterMaxIter,
1498 self.config.brighterFatterThreshold,
1499 self.config.brighterFatterApplyGain,
1501 if bfResults[1] == self.config.brighterFatterMaxIter:
1502 self.log.
warn(
"Brighter fatter correction did not converge, final difference %f.",
1505 self.log.
info(
"Finished brighter fatter correction in %d iterations.",
1507 image = ccdExposure.getMaskedImage().getImage()
1508 bfCorr = bfExp.getMaskedImage().getImage()
1509 bfCorr -= interpExp.getMaskedImage().getImage()
1518 self.log.
info(
"Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1519 self.
maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1522 if self.config.brighterFatterMaskGrowSize > 0:
1523 self.log.
info(
"Growing masks to account for brighter-fatter kernel convolution.")
1524 for maskPlane
in self.config.maskListToInterpolate:
1525 isrFunctions.growMasks(ccdExposure.getMask(),
1526 radius=self.config.brighterFatterMaskGrowSize,
1527 maskNameList=maskPlane,
1528 maskValue=maskPlane)
1530 self.
debugView(ccdExposure,
"doBrighterFatter")
1532 if self.config.doDark:
1533 self.log.
info(
"Applying dark correction.")
1537 if self.config.doFringe
and not self.config.fringeAfterFlat:
1538 self.log.
info(
"Applying fringe correction before flat.")
1539 self.fringe.
run(ccdExposure, **fringes.getDict())
1542 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1543 self.log.
info(
"Checking strayLight correction.")
1544 self.strayLight.
run(ccdExposure, strayLightData)
1545 self.
debugView(ccdExposure,
"doStrayLight")
1547 if self.config.doFlat:
1548 self.log.
info(
"Applying flat correction.")
1552 if self.config.doApplyGains:
1553 self.log.
info(
"Applying gain correction instead of flat.")
1554 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1556 if self.config.doFringe
and self.config.fringeAfterFlat:
1557 self.log.
info(
"Applying fringe correction after flat.")
1558 self.fringe.
run(ccdExposure, **fringes.getDict())
1560 if self.config.doVignette:
1561 self.log.
info(
"Constructing Vignette polygon.")
1564 if self.config.vignette.doWriteVignettePolygon:
1567 if self.config.doAttachTransmissionCurve:
1568 self.log.
info(
"Adding transmission curves.")
1569 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1570 filterTransmission=filterTransmission,
1571 sensorTransmission=sensorTransmission,
1572 atmosphereTransmission=atmosphereTransmission)
1574 flattenedThumb =
None
1575 if self.config.qa.doThumbnailFlattened:
1576 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1578 if self.config.doIlluminationCorrection
and filterName
in self.config.illumFilters:
1579 self.log.
info(
"Performing illumination correction.")
1580 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1581 illumMaskedImage, illumScale=self.config.illumScale,
1582 trimToFit=self.config.doTrimToMatchCalib)
1585 if self.config.doSaveInterpPixels:
1586 preInterpExp = ccdExposure.clone()
1601 if self.config.doSetBadRegions:
1602 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1603 if badPixelCount > 0:
1604 self.log.
info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1606 if self.config.doInterpolate:
1607 self.log.
info(
"Interpolating masked pixels.")
1608 isrFunctions.interpolateFromMask(
1609 maskedImage=ccdExposure.getMaskedImage(),
1610 fwhm=self.config.fwhm,
1611 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1612 maskNameList=
list(self.config.maskListToInterpolate)
1617 if self.config.doMeasureBackground:
1618 self.log.
info(
"Measuring background level.")
1621 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1623 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1625 afwMath.MEDIAN | afwMath.STDEVCLIP)
1626 self.metadata.
set(
"ISR BACKGROUND {} MEDIAN".
format(amp.getName()),
1627 qaStats.getValue(afwMath.MEDIAN))
1628 self.metadata.
set(
"ISR BACKGROUND {} STDEV".
format(amp.getName()),
1629 qaStats.getValue(afwMath.STDEVCLIP))
1630 self.log.
debug(
" Background stats for amplifer %s: %f +/- %f",
1631 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1632 qaStats.getValue(afwMath.STDEVCLIP))
1634 self.
debugView(ccdExposure,
"postISRCCD")
1636 return pipeBase.Struct(
1637 exposure=ccdExposure,
1639 flattenedThumb=flattenedThumb,
1641 preInterpolatedExposure=preInterpExp,
1642 outputExposure=ccdExposure,
1643 outputOssThumbnail=ossThumb,
1644 outputFlattenedThumbnail=flattenedThumb,
1647 @pipeBase.timeMethod
1649 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1651 This method contains the `CmdLineTask` interface to the ISR
1652 processing. All IO is handled here, freeing the `run()` method
1653 to manage only pixel-level calculations. The steps performed
1655 - Read in necessary detrending/isr/calibration data.
1656 - Process raw exposure in `run()`.
1657 - Persist the ISR-corrected exposure as "postISRCCD" if
1658 config.doWrite=True.
1662 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1663 DataRef of the detector data to be processed
1667 result : `lsst.pipe.base.Struct`
1668 Result struct with component:
1669 - ``exposure`` : `afw.image.Exposure`
1670 The fully ISR corrected exposure.
1675 Raised if a configuration option is set to True, but the
1676 required calibration data does not exist.
1679 self.log.
info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1681 ccdExposure = sensorRef.get(self.config.datasetType)
1683 camera = sensorRef.get(
"camera")
1684 isrData = self.
readIsrData(sensorRef, ccdExposure)
1686 result = self.
run(ccdExposure, camera=camera, **isrData.getDict())
1688 if self.config.doWrite:
1689 sensorRef.put(result.exposure,
"postISRCCD")
1690 if result.preInterpolatedExposure
is not None:
1691 sensorRef.put(result.preInterpolatedExposure,
"postISRCCD_uninterpolated")
1692 if result.ossThumb
is not None:
1693 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1694 if result.flattenedThumb
is not None:
1695 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1700 """Retrieve a calibration dataset for removing instrument signature.
1705 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1706 DataRef of the detector data to find calibration datasets
1709 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1710 dateObs : `str`, optional
1711 Date of the observation. Used to correct butler failures
1712 when using fallback filters.
1714 If True, disable butler proxies to enable error handling
1715 within this routine.
1719 exposure : `lsst.afw.image.Exposure`
1720 Requested calibration frame.
1725 Raised if no matching calibration frame can be found.
1728 exp = dataRef.get(datasetType, immediate=immediate)
1729 except Exception
as exc1:
1730 if not self.config.fallbackFilterName:
1731 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1733 if self.config.useFallbackDate
and dateObs:
1734 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1735 dateObs=dateObs, immediate=immediate)
1737 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1738 except Exception
as exc2:
1739 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1740 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1741 self.log.
warn(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1743 if self.config.doAssembleIsrExposures:
1744 exp = self.assembleCcd.assembleCcd(exp)
1748 """Ensure that the data returned by Butler is a fully constructed exposure.
1750 ISR requires exposure-level image data for historical reasons, so if we did
1751 not recieve that from Butler, construct it from what we have, modifying the
1756 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1757 `lsst.afw.image.ImageF`
1758 The input data structure obtained from Butler.
1759 camera : `lsst.afw.cameraGeom.camera`
1760 The camera associated with the image. Used to find the appropriate
1763 The detector this exposure should match.
1767 inputExp : `lsst.afw.image.Exposure`
1768 The re-constructed exposure, with appropriate detector parameters.
1773 Raised if the input data cannot be used to construct an exposure.
1775 if isinstance(inputExp, afwImage.DecoratedImageU):
1777 elif isinstance(inputExp, afwImage.ImageF):
1779 elif isinstance(inputExp, afwImage.MaskedImageF):
1783 elif inputExp
is None:
1787 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1790 if inputExp.getDetector()
is None:
1791 inputExp.setDetector(camera[detectorNum])
1796 """Convert exposure image from uint16 to float.
1798 If the exposure does not need to be converted, the input is
1799 immediately returned. For exposures that are converted to use
1800 floating point pixels, the variance is set to unity and the
1805 exposure : `lsst.afw.image.Exposure`
1806 The raw exposure to be converted.
1810 newexposure : `lsst.afw.image.Exposure`
1811 The input ``exposure``, converted to floating point pixels.
1816 Raised if the exposure type cannot be converted to float.
1819 if isinstance(exposure, afwImage.ExposureF):
1821 self.log.
debug(
"Exposure already of type float.")
1823 if not hasattr(exposure,
"convertF"):
1824 raise RuntimeError(
"Unable to convert exposure (%s) to float." %
type(exposure))
1826 newexposure = exposure.convertF()
1827 newexposure.variance[:] = 1
1828 newexposure.mask[:] = 0x0
1833 """Identify bad amplifiers, saturated and suspect pixels.
1837 ccdExposure : `lsst.afw.image.Exposure`
1838 Input exposure to be masked.
1839 amp : `lsst.afw.table.AmpInfoCatalog`
1840 Catalog of parameters defining the amplifier on this
1842 defects : `lsst.ip.isr.Defects`
1843 List of defects. Used to determine if the entire
1849 If this is true, the entire amplifier area is covered by
1850 defects and unusable.
1853 maskedImage = ccdExposure.getMaskedImage()
1859 if defects
is not None:
1860 badAmp = bool(sum([v.getBBox().
contains(amp.getBBox())
for v
in defects]))
1865 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1867 maskView = dataView.getMask()
1868 maskView |= maskView.getPlaneBitMask(
"BAD")
1875 if self.config.doSaturation
and not badAmp:
1876 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1877 if self.config.doSuspect
and not badAmp:
1878 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1879 if math.isfinite(self.config.saturation):
1880 limits.update({self.config.saturatedMaskName: self.config.saturation})
1882 for maskName, maskThreshold
in limits.items():
1883 if not math.isnan(maskThreshold):
1884 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1885 isrFunctions.makeThresholdMask(
1886 maskedImage=dataView,
1887 threshold=maskThreshold,
1893 maskView =
afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1895 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1896 self.config.suspectMaskName])
1897 if numpy.all(maskView.getArray() & maskVal > 0):
1899 maskView |= maskView.getPlaneBitMask(
"BAD")
1904 """Apply overscan correction in place.
1906 This method does initial pixel rejection of the overscan
1907 region. The overscan can also be optionally segmented to
1908 allow for discontinuous overscan responses to be fit
1909 separately. The actual overscan subtraction is performed by
1910 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1911 which is called here after the amplifier is preprocessed.
1915 ccdExposure : `lsst.afw.image.Exposure`
1916 Exposure to have overscan correction performed.
1917 amp : `lsst.afw.cameraGeom.Amplifer`
1918 The amplifier to consider while correcting the overscan.
1922 overscanResults : `lsst.pipe.base.Struct`
1923 Result struct with components:
1924 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1925 Value or fit subtracted from the amplifier image data.
1926 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1927 Value or fit subtracted from the overscan image data.
1928 - ``overscanImage`` : `lsst.afw.image.Image`
1929 Image of the overscan region with the overscan
1930 correction applied. This quantity is used to estimate
1931 the amplifier read noise empirically.
1936 Raised if the ``amp`` does not contain raw pixel information.
1940 lsst.ip.isr.isrFunctions.overscanCorrection
1942 if amp.getRawHorizontalOverscanBBox().isEmpty():
1943 self.log.
info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
1947 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
1950 dataBBox = amp.getRawDataBBox()
1951 oscanBBox = amp.getRawHorizontalOverscanBBox()
1955 prescanBBox = amp.getRawPrescanBBox()
1956 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
1957 dx0 += self.config.overscanNumLeadingColumnsToSkip
1958 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1960 dx0 += self.config.overscanNumTrailingColumnsToSkip
1961 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1967 if ((self.config.overscanBiasJump
1968 and self.config.overscanBiasJumpLocation)
1969 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1970 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
1971 self.config.overscanBiasJumpDevices)):
1972 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
1973 yLower = self.config.overscanBiasJumpLocation
1974 yUpper = dataBBox.getHeight() - yLower
1976 yUpper = self.config.overscanBiasJumpLocation
1977 yLower = dataBBox.getHeight() - yUpper
1995 oscanBBox.getHeight())))
1998 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
1999 ampImage = ccdExposure.maskedImage[imageBBox]
2000 overscanImage = ccdExposure.maskedImage[overscanBBox]
2002 overscanArray = overscanImage.image.array
2003 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2004 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2005 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2008 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2010 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2013 levelStat = afwMath.MEDIAN
2014 sigmaStat = afwMath.STDEVCLIP
2017 self.config.qa.flatness.nIter)
2018 metadata = ccdExposure.getMetadata()
2019 ampNum = amp.getName()
2021 if isinstance(overscanResults.overscanFit, float):
2022 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2023 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2026 metadata.set(
"ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2027 metadata.set(
"ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2029 return overscanResults
2032 """Set the variance plane using the amplifier gain and read noise
2034 The read noise is calculated from the ``overscanImage`` if the
2035 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2036 the value from the amplifier data is used.
2040 ampExposure : `lsst.afw.image.Exposure`
2041 Exposure to process.
2042 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2043 Amplifier detector data.
2044 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2045 Image of overscan, required only for empirical read noise.
2049 lsst.ip.isr.isrFunctions.updateVariance
2051 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2052 gain = amp.getGain()
2054 if math.isnan(gain):
2056 self.log.
warn(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2059 self.log.
warn(
"Gain for amp %s == %g <= 0; setting to %f.",
2060 amp.getName(), gain, patchedGain)
2063 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2064 self.log.
info(
"Overscan is none for EmpiricalReadNoise.")
2066 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2068 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2070 self.log.
info(
"Calculated empirical read noise for amp %s: %f.",
2071 amp.getName(), readNoise)
2073 readNoise = amp.getReadNoise()
2075 isrFunctions.updateVariance(
2076 maskedImage=ampExposure.getMaskedImage(),
2078 readNoise=readNoise,
2082 """Apply dark correction in place.
2086 exposure : `lsst.afw.image.Exposure`
2087 Exposure to process.
2088 darkExposure : `lsst.afw.image.Exposure`
2089 Dark exposure of the same size as ``exposure``.
2090 invert : `Bool`, optional
2091 If True, re-add the dark to an already corrected image.
2096 Raised if either ``exposure`` or ``darkExposure`` do not
2097 have their dark time defined.
2101 lsst.ip.isr.isrFunctions.darkCorrection
2103 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2104 if math.isnan(expScale):
2105 raise RuntimeError(
"Exposure darktime is NAN.")
2106 if darkExposure.getInfo().getVisitInfo()
is not None \
2107 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2108 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2112 self.log.
warn(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2115 isrFunctions.darkCorrection(
2116 maskedImage=exposure.getMaskedImage(),
2117 darkMaskedImage=darkExposure.getMaskedImage(),
2119 darkScale=darkScale,
2121 trimToFit=self.config.doTrimToMatchCalib
2125 """Check if linearization is needed for the detector cameraGeom.
2127 Checks config.doLinearize and the linearity type of the first
2132 detector : `lsst.afw.cameraGeom.Detector`
2133 Detector to get linearity type from.
2137 doLinearize : `Bool`
2138 If True, linearization should be performed.
2140 return self.config.doLinearize
and \
2141 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2144 """Apply flat correction in place.
2148 exposure : `lsst.afw.image.Exposure`
2149 Exposure to process.
2150 flatExposure : `lsst.afw.image.Exposure`
2151 Flat exposure of the same size as ``exposure``.
2152 invert : `Bool`, optional
2153 If True, unflatten an already flattened image.
2157 lsst.ip.isr.isrFunctions.flatCorrection
2159 isrFunctions.flatCorrection(
2160 maskedImage=exposure.getMaskedImage(),
2161 flatMaskedImage=flatExposure.getMaskedImage(),
2162 scalingType=self.config.flatScalingType,
2163 userScale=self.config.flatUserScale,
2165 trimToFit=self.config.doTrimToMatchCalib
2169 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2173 exposure : `lsst.afw.image.Exposure`
2174 Exposure to process. Only the amplifier DataSec is processed.
2175 amp : `lsst.afw.table.AmpInfoCatalog`
2176 Amplifier detector data.
2180 lsst.ip.isr.isrFunctions.makeThresholdMask
2182 if not math.isnan(amp.getSaturation()):
2183 maskedImage = exposure.getMaskedImage()
2184 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2185 isrFunctions.makeThresholdMask(
2186 maskedImage=dataView,
2187 threshold=amp.getSaturation(),
2189 maskName=self.config.saturatedMaskName,
2193 """Interpolate over saturated pixels, in place.
2195 This method should be called after `saturationDetection`, to
2196 ensure that the saturated pixels have been identified in the
2197 SAT mask. It should also be called after `assembleCcd`, since
2198 saturated regions may cross amplifier boundaries.
2202 exposure : `lsst.afw.image.Exposure`
2203 Exposure to process.
2207 lsst.ip.isr.isrTask.saturationDetection
2208 lsst.ip.isr.isrFunctions.interpolateFromMask
2210 isrFunctions.interpolateFromMask(
2211 maskedImage=exposure.getMaskedImage(),
2212 fwhm=self.config.fwhm,
2213 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2214 maskNameList=
list(self.config.saturatedMaskName),
2218 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2222 exposure : `lsst.afw.image.Exposure`
2223 Exposure to process. Only the amplifier DataSec is processed.
2224 amp : `lsst.afw.table.AmpInfoCatalog`
2225 Amplifier detector data.
2229 lsst.ip.isr.isrFunctions.makeThresholdMask
2233 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2234 This is intended to indicate pixels that may be affected by unknown systematics;
2235 for example if non-linearity corrections above a certain level are unstable
2236 then that would be a useful value for suspectLevel. A value of `nan` indicates
2237 that no such level exists and no pixels are to be masked as suspicious.
2239 suspectLevel = amp.getSuspectLevel()
2240 if math.isnan(suspectLevel):
2243 maskedImage = exposure.getMaskedImage()
2244 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2245 isrFunctions.makeThresholdMask(
2246 maskedImage=dataView,
2247 threshold=suspectLevel,
2249 maskName=self.config.suspectMaskName,
2253 """Mask defects using mask plane "BAD", in place.
2257 exposure : `lsst.afw.image.Exposure`
2258 Exposure to process.
2259 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2260 `lsst.afw.image.DefectBase`.
2261 List of defects to mask.
2265 Call this after CCD assembly, since defects may cross amplifier boundaries.
2267 maskedImage = exposure.getMaskedImage()
2268 if not isinstance(defectBaseList, Defects):
2270 defectList =
Defects(defectBaseList)
2272 defectList = defectBaseList
2273 defectList.maskPixels(maskedImage, maskName=
"BAD")
2275 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2276 """Mask edge pixels with applicable mask plane.
2280 exposure : `lsst.afw.image.Exposure`
2281 Exposure to process.
2282 numEdgePixels : `int`, optional
2283 Number of edge pixels to mask.
2284 maskPlane : `str`, optional
2285 Mask plane name to use.
2286 level : `str`, optional
2287 Level at which to mask edges.
2289 maskedImage = exposure.getMaskedImage()
2290 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2292 if numEdgePixels > 0:
2293 if level ==
'DETECTOR':
2294 boxes = [maskedImage.getBBox()]
2295 elif level ==
'AMP':
2296 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2300 subImage = maskedImage[box]
2301 box.grow(-numEdgePixels)
2303 SourceDetectionTask.setEdgeBits(
2309 """Mask and interpolate defects using mask plane "BAD", in place.
2313 exposure : `lsst.afw.image.Exposure`
2314 Exposure to process.
2315 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2316 `lsst.afw.image.DefectBase`.
2317 List of defects to mask and interpolate.
2321 lsst.ip.isr.isrTask.maskDefect
2324 self.
maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2325 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2326 isrFunctions.interpolateFromMask(
2327 maskedImage=exposure.getMaskedImage(),
2328 fwhm=self.config.fwhm,
2329 growSaturatedFootprints=0,
2330 maskNameList=[
"BAD"],
2334 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2338 exposure : `lsst.afw.image.Exposure`
2339 Exposure to process.
2343 We mask over all NaNs, including those that are masked with
2344 other bits (because those may or may not be interpolated over
2345 later, and we want to remove all NaNs). Despite this
2346 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2347 the historical name.
2349 maskedImage = exposure.getMaskedImage()
2352 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2353 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2354 numNans =
maskNans(maskedImage, maskVal)
2355 self.metadata.
set(
"NUMNANS", numNans)
2357 self.log.
warn(
"There were %d unmasked NaNs.", numNans)
2360 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2364 exposure : `lsst.afw.image.Exposure`
2365 Exposure to process.
2369 lsst.ip.isr.isrTask.maskNan
2372 isrFunctions.interpolateFromMask(
2373 maskedImage=exposure.getMaskedImage(),
2374 fwhm=self.config.fwhm,
2375 growSaturatedFootprints=0,
2376 maskNameList=[
"UNMASKEDNAN"],
2380 """Measure the image background in subgrids, for quality control purposes.
2384 exposure : `lsst.afw.image.Exposure`
2385 Exposure to process.
2386 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2387 Configuration object containing parameters on which background
2388 statistics and subgrids to use.
2390 if IsrQaConfig
is not None:
2392 IsrQaConfig.flatness.nIter)
2393 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2394 statsControl.setAndMask(maskVal)
2395 maskedImage = exposure.getMaskedImage()
2397 skyLevel = stats.getValue(afwMath.MEDIAN)
2398 skySigma = stats.getValue(afwMath.STDEVCLIP)
2399 self.log.
info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2400 metadata = exposure.getMetadata()
2401 metadata.set(
'SKYLEVEL', skyLevel)
2402 metadata.set(
'SKYSIGMA', skySigma)
2405 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2406 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2407 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2408 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2409 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2410 skyLevels = numpy.zeros((nX, nY))
2413 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2415 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2417 xLLC = xc - meshXHalf
2418 yLLC = yc - meshYHalf
2419 xURC = xc + meshXHalf - 1
2420 yURC = yc + meshYHalf - 1
2423 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2427 good = numpy.where(numpy.isfinite(skyLevels))
2428 skyMedian = numpy.median(skyLevels[good])
2429 flatness = (skyLevels[good] - skyMedian) / skyMedian
2430 flatness_rms = numpy.std(flatness)
2431 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2433 self.log.
info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2434 self.log.
info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2435 nX, nY, flatness_pp, flatness_rms)
2437 metadata.set(
'FLATNESS_PP', float(flatness_pp))
2438 metadata.set(
'FLATNESS_RMS', float(flatness_rms))
2439 metadata.set(
'FLATNESS_NGRIDS',
'%dx%d' % (nX, nY))
2440 metadata.set(
'FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2441 metadata.set(
'FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2444 """Set an approximate magnitude zero point for the exposure.
2448 exposure : `lsst.afw.image.Exposure`
2449 Exposure to process.
2452 if filterName
in self.config.fluxMag0T1:
2453 fluxMag0 = self.config.fluxMag0T1[filterName]
2455 self.log.
warn(
"No rough magnitude zero point set for filter %s.", filterName)
2456 fluxMag0 = self.config.defaultFluxMag0T1
2458 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2460 self.log.
warn(
"Non-positive exposure time; skipping rough zero point.")
2463 self.log.
info(
"Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2467 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2471 ccdExposure : `lsst.afw.image.Exposure`
2472 Exposure to process.
2473 fpPolygon : `lsst.afw.geom.Polygon`
2474 Polygon in focal plane coordinates.
2477 ccd = ccdExposure.getDetector()
2478 fpCorners = ccd.getCorners(FOCAL_PLANE)
2479 ccdPolygon =
Polygon(fpCorners)
2482 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2485 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2486 validPolygon =
Polygon(ccdPoints)
2487 ccdExposure.getInfo().setValidPolygon(validPolygon)
2491 """Context manager that applies and removes flats and darks,
2492 if the task is configured to apply them.
2496 exp : `lsst.afw.image.Exposure`
2497 Exposure to process.
2498 flat : `lsst.afw.image.Exposure`
2499 Flat exposure the same size as ``exp``.
2500 dark : `lsst.afw.image.Exposure`, optional
2501 Dark exposure the same size as ``exp``.
2505 exp : `lsst.afw.image.Exposure`
2506 The flat and dark corrected exposure.
2508 if self.config.doDark
and dark
is not None:
2510 if self.config.doFlat:
2515 if self.config.doFlat:
2517 if self.config.doDark
and dark
is not None:
2521 """Utility function to examine ISR exposure at different stages.
2525 exposure : `lsst.afw.image.Exposure`
2528 State of processing to view.
2533 display.scale(
'asinh',
'zscale')
2534 display.mtv(exposure)
2535 prompt =
"Press Enter to continue [c]... "
2537 ans = input(prompt).lower()
2538 if ans
in (
"",
"c",):
2543 """A Detector-like object that supports returning gain and saturation level
2545 This is used when the input exposure does not have a detector.
2549 exposure : `lsst.afw.image.Exposure`
2550 Exposure to generate a fake amplifier for.
2551 config : `lsst.ip.isr.isrTaskConfig`
2552 Configuration to apply to the fake amplifier.
2556 self.
_bbox = exposure.getBBox(afwImage.LOCAL)
2558 self.
_gain = config.gain
2585 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2589 """Task to wrap the default IsrTask to allow it to be retargeted.
2591 The standard IsrTask can be called directly from a command line
2592 program, but doing so removes the ability of the task to be
2593 retargeted. As most cameras override some set of the IsrTask
2594 methods, this would remove those data-specific methods in the
2595 output post-ISR images. This wrapping class fixes the issue,
2596 allowing identical post-ISR images to be generated by both the
2597 processCcd and isrTask code.
2599 ConfigClass = RunIsrConfig
2600 _DefaultName =
"runIsr"
2604 self.makeSubtask(
"isr")
2610 dataRef : `lsst.daf.persistence.ButlerDataRef`
2611 data reference of the detector data to be processed
2615 result : `pipeBase.Struct`
2616 Result struct with component:
2618 - exposure : `lsst.afw.image.Exposure`
2619 Post-ISR processed exposure.