30 import lsst.pipe.base.connectionTypes
as cT
32 from contextlib
import contextmanager
33 from lsstDebug
import getDebugFrame
42 from lsst.utils.timer
import timeMethod
44 from .
import isrFunctions
46 from .
import linearize
47 from .defects
import Defects
49 from .assembleCcdTask
import AssembleCcdTask
50 from .crosstalk
import CrosstalkTask, CrosstalkCalib
51 from .fringe
import FringeTask
52 from .isr
import maskNans
53 from .masking
import MaskingTask
54 from .overscan
import OverscanCorrectionTask
55 from .straylight
import StrayLightTask
56 from .vignette
import VignetteTask
57 from .ampOffset
import AmpOffsetTask
58 from lsst.daf.butler
import DimensionGraph
61 __all__ = [
"IsrTask",
"IsrTaskConfig",
"RunIsrTask",
"RunIsrConfig"]
65 """Lookup function to identify crosstalkSource entries.
67 This should return an empty list under most circumstances. Only
68 when inter-chip crosstalk has been identified should this be
75 registry : `lsst.daf.butler.Registry`
76 Butler registry to query.
77 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
78 Data id to transform to identify crosstalkSources. The
79 ``detector`` entry will be stripped.
80 collections : `lsst.daf.butler.CollectionSearch`
81 Collections to search through.
85 results : `list` [`lsst.daf.butler.DatasetRef`]
86 List of datasets that match the query that will be used as
89 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=[
"instrument",
"exposure"]))
90 results =
set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
97 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records))
for ref
in results]
101 dimensions={
"instrument",
"exposure",
"detector"},
102 defaultTemplates={}):
103 ccdExposure = cT.Input(
105 doc=
"Input exposure to process.",
106 storageClass=
"Exposure",
107 dimensions=[
"instrument",
"exposure",
"detector"],
109 camera = cT.PrerequisiteInput(
111 storageClass=
"Camera",
112 doc=
"Input camera to construct complete exposures.",
113 dimensions=[
"instrument"],
117 crosstalk = cT.PrerequisiteInput(
119 doc=
"Input crosstalk object",
120 storageClass=
"CrosstalkCalib",
121 dimensions=[
"instrument",
"detector"],
125 crosstalkSources = cT.PrerequisiteInput(
126 name=
"isrOverscanCorrected",
127 doc=
"Overscan corrected input images.",
128 storageClass=
"Exposure",
129 dimensions=[
"instrument",
"exposure",
"detector"],
132 lookupFunction=crosstalkSourceLookup,
135 bias = cT.PrerequisiteInput(
137 doc=
"Input bias calibration.",
138 storageClass=
"ExposureF",
139 dimensions=[
"instrument",
"detector"],
142 dark = cT.PrerequisiteInput(
144 doc=
"Input dark calibration.",
145 storageClass=
"ExposureF",
146 dimensions=[
"instrument",
"detector"],
149 flat = cT.PrerequisiteInput(
151 doc=
"Input flat calibration.",
152 storageClass=
"ExposureF",
153 dimensions=[
"instrument",
"physical_filter",
"detector"],
156 ptc = cT.PrerequisiteInput(
158 doc=
"Input Photon Transfer Curve dataset",
159 storageClass=
"PhotonTransferCurveDataset",
160 dimensions=[
"instrument",
"detector"],
163 fringes = cT.PrerequisiteInput(
165 doc=
"Input fringe calibration.",
166 storageClass=
"ExposureF",
167 dimensions=[
"instrument",
"physical_filter",
"detector"],
171 strayLightData = cT.PrerequisiteInput(
173 doc=
"Input stray light calibration.",
174 storageClass=
"StrayLightData",
175 dimensions=[
"instrument",
"physical_filter",
"detector"],
180 bfKernel = cT.PrerequisiteInput(
182 doc=
"Input brighter-fatter kernel.",
183 storageClass=
"NumpyArray",
184 dimensions=[
"instrument"],
188 newBFKernel = cT.PrerequisiteInput(
189 name=
'brighterFatterKernel',
190 doc=
"Newer complete kernel + gain solutions.",
191 storageClass=
"BrighterFatterKernel",
192 dimensions=[
"instrument",
"detector"],
196 defects = cT.PrerequisiteInput(
198 doc=
"Input defect tables.",
199 storageClass=
"Defects",
200 dimensions=[
"instrument",
"detector"],
203 linearizer = cT.PrerequisiteInput(
205 storageClass=
"Linearizer",
206 doc=
"Linearity correction calibration.",
207 dimensions=[
"instrument",
"detector"],
211 opticsTransmission = cT.PrerequisiteInput(
212 name=
"transmission_optics",
213 storageClass=
"TransmissionCurve",
214 doc=
"Transmission curve due to the optics.",
215 dimensions=[
"instrument"],
218 filterTransmission = cT.PrerequisiteInput(
219 name=
"transmission_filter",
220 storageClass=
"TransmissionCurve",
221 doc=
"Transmission curve due to the filter.",
222 dimensions=[
"instrument",
"physical_filter"],
225 sensorTransmission = cT.PrerequisiteInput(
226 name=
"transmission_sensor",
227 storageClass=
"TransmissionCurve",
228 doc=
"Transmission curve due to the sensor.",
229 dimensions=[
"instrument",
"detector"],
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name=
"transmission_atmosphere",
234 storageClass=
"TransmissionCurve",
235 doc=
"Transmission curve due to the atmosphere.",
236 dimensions=[
"instrument"],
239 illumMaskedImage = cT.PrerequisiteInput(
241 doc=
"Input illumination correction.",
242 storageClass=
"MaskedImageF",
243 dimensions=[
"instrument",
"physical_filter",
"detector"],
247 outputExposure = cT.Output(
249 doc=
"Output ISR processed exposure.",
250 storageClass=
"Exposure",
251 dimensions=[
"instrument",
"exposure",
"detector"],
253 preInterpExposure = cT.Output(
254 name=
'preInterpISRCCD',
255 doc=
"Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass=
"ExposureF",
257 dimensions=[
"instrument",
"exposure",
"detector"],
259 outputOssThumbnail = cT.Output(
261 doc=
"Output Overscan-subtracted thumbnail image.",
262 storageClass=
"Thumbnail",
263 dimensions=[
"instrument",
"exposure",
"detector"],
265 outputFlattenedThumbnail = cT.Output(
266 name=
"FlattenedThumb",
267 doc=
"Output flat-corrected thumbnail image.",
268 storageClass=
"Thumbnail",
269 dimensions=[
"instrument",
"exposure",
"detector"],
275 if config.doBias
is not True:
276 self.prerequisiteInputs.discard(
"bias")
277 if config.doLinearize
is not True:
278 self.prerequisiteInputs.discard(
"linearizer")
279 if config.doCrosstalk
is not True:
280 self.prerequisiteInputs.discard(
"crosstalkSources")
281 self.prerequisiteInputs.discard(
"crosstalk")
282 if config.doBrighterFatter
is not True:
283 self.prerequisiteInputs.discard(
"bfKernel")
284 self.prerequisiteInputs.discard(
"newBFKernel")
285 if config.doDefect
is not True:
286 self.prerequisiteInputs.discard(
"defects")
287 if config.doDark
is not True:
288 self.prerequisiteInputs.discard(
"dark")
289 if config.doFlat
is not True:
290 self.prerequisiteInputs.discard(
"flat")
291 if config.doFringe
is not True:
292 self.prerequisiteInputs.discard(
"fringe")
293 if config.doStrayLight
is not True:
294 self.prerequisiteInputs.discard(
"strayLightData")
295 if config.usePtcGains
is not True and config.usePtcReadNoise
is not True:
296 self.prerequisiteInputs.discard(
"ptc")
297 if config.doAttachTransmissionCurve
is not True:
298 self.prerequisiteInputs.discard(
"opticsTransmission")
299 self.prerequisiteInputs.discard(
"filterTransmission")
300 self.prerequisiteInputs.discard(
"sensorTransmission")
301 self.prerequisiteInputs.discard(
"atmosphereTransmission")
302 if config.doUseOpticsTransmission
is not True:
303 self.prerequisiteInputs.discard(
"opticsTransmission")
304 if config.doUseFilterTransmission
is not True:
305 self.prerequisiteInputs.discard(
"filterTransmission")
306 if config.doUseSensorTransmission
is not True:
307 self.prerequisiteInputs.discard(
"sensorTransmission")
308 if config.doUseAtmosphereTransmission
is not True:
309 self.prerequisiteInputs.discard(
"atmosphereTransmission")
310 if config.doIlluminationCorrection
is not True:
311 self.prerequisiteInputs.discard(
"illumMaskedImage")
313 if config.doWrite
is not True:
314 self.outputs.discard(
"outputExposure")
315 self.outputs.discard(
"preInterpExposure")
316 self.outputs.discard(
"outputFlattenedThumbnail")
317 self.outputs.discard(
"outputOssThumbnail")
318 if config.doSaveInterpPixels
is not True:
319 self.outputs.discard(
"preInterpExposure")
320 if config.qa.doThumbnailOss
is not True:
321 self.outputs.discard(
"outputOssThumbnail")
322 if config.qa.doThumbnailFlattened
is not True:
323 self.outputs.discard(
"outputFlattenedThumbnail")
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
332 datasetType = pexConfig.Field(
334 doc=
"Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
339 fallbackFilterName = pexConfig.Field(
341 doc=
"Fallback default filter name for calibrations.",
344 useFallbackDate = pexConfig.Field(
346 doc=
"Pass observation date when using fallback filter.",
349 expectWcs = pexConfig.Field(
352 doc=
"Expect input science images to have a WCS (set False for e.g. spectrographs)."
354 fwhm = pexConfig.Field(
356 doc=
"FWHM of PSF in arcseconds.",
359 qa = pexConfig.ConfigField(
361 doc=
"QA related configuration options.",
365 doConvertIntToFloat = pexConfig.Field(
367 doc=
"Convert integer raw images to floating point values?",
372 doSaturation = pexConfig.Field(
374 doc=
"Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
379 saturatedMaskName = pexConfig.Field(
381 doc=
"Name of mask plane to use in saturation detection and interpolation",
384 saturation = pexConfig.Field(
386 doc=
"The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float(
"NaN"),
389 growSaturationFootprintSize = pexConfig.Field(
391 doc=
"Number of pixels by which to grow the saturation footprints",
396 doSuspect = pexConfig.Field(
398 doc=
"Mask suspect pixels?",
401 suspectMaskName = pexConfig.Field(
403 doc=
"Name of mask plane to use for suspect pixels",
406 numEdgeSuspect = pexConfig.Field(
408 doc=
"Number of edge pixels to be flagged as untrustworthy.",
411 edgeMaskLevel = pexConfig.ChoiceField(
413 doc=
"Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
416 'DETECTOR':
'Mask only the edges of the full detector.',
417 'AMP':
'Mask edges of each amplifier.',
422 doSetBadRegions = pexConfig.Field(
424 doc=
"Should we set the level of all BAD patches of the chip to the chip's average value?",
427 badStatistic = pexConfig.ChoiceField(
429 doc=
"How to estimate the average value for BAD regions.",
432 "MEANCLIP":
"Correct using the (clipped) mean of good data",
433 "MEDIAN":
"Correct using the median of the good data",
438 doOverscan = pexConfig.Field(
440 doc=
"Do overscan subtraction?",
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc=
"Overscan subtraction task for image segments.",
447 overscanFitType = pexConfig.ChoiceField(
449 doc=
"The method for fitting the overscan bias level.",
452 "POLY":
"Fit ordinary polynomial to the longest axis of the overscan region",
453 "CHEB":
"Fit Chebyshev polynomial to the longest axis of the overscan region",
454 "LEG":
"Fit Legendre polynomial to the longest axis of the overscan region",
455 "NATURAL_SPLINE":
"Fit natural spline to the longest axis of the overscan region",
456 "CUBIC_SPLINE":
"Fit cubic spline to the longest axis of the overscan region",
457 "AKIMA_SPLINE":
"Fit Akima spline to the longest axis of the overscan region",
458 "MEAN":
"Correct using the mean of the overscan region",
459 "MEANCLIP":
"Correct using a clipped mean of the overscan region",
460 "MEDIAN":
"Correct using the median of the overscan region",
461 "MEDIAN_PER_ROW":
"Correct using the median per row of the overscan region",
463 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
464 " This option will no longer be used, and will be removed after v20.")
466 overscanOrder = pexConfig.Field(
468 doc=(
"Order of polynomial or to fit if overscan fit type is a polynomial, "
469 "or number of spline knots if overscan fit type is a spline."),
471 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
472 " This option will no longer be used, and will be removed after v20.")
474 overscanNumSigmaClip = pexConfig.Field(
476 doc=
"Rejection threshold (sigma) for collapsing overscan before fit",
478 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
479 " This option will no longer be used, and will be removed after v20.")
481 overscanIsInt = pexConfig.Field(
483 doc=
"Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
484 " and overscan.FitType=MEDIAN_PER_ROW.",
486 deprecated=(
"Please configure overscan via the OverscanCorrectionConfig interface."
487 " This option will no longer be used, and will be removed after v20.")
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
493 doc=
"Number of columns to skip in overscan, i.e. those closest to amplifier",
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
498 doc=
"Number of columns to skip in overscan, i.e. those farthest from amplifier",
501 overscanMaxDev = pexConfig.Field(
503 doc=
"Maximum deviation from the median for overscan",
504 default=1000.0, check=
lambda x: x > 0
506 overscanBiasJump = pexConfig.Field(
508 doc=
"Fit the overscan in a piecewise-fashion to correct for bias jumps?",
511 overscanBiasJumpKeyword = pexConfig.Field(
513 doc=
"Header keyword containing information about devices.",
514 default=
"NO_SUCH_KEY",
516 overscanBiasJumpDevices = pexConfig.ListField(
518 doc=
"List of devices that need piecewise overscan correction.",
521 overscanBiasJumpLocation = pexConfig.Field(
523 doc=
"Location of bias jump along y-axis.",
528 doAssembleCcd = pexConfig.Field(
531 doc=
"Assemble amp-level exposures into a ccd-level exposure?"
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc=
"CCD assembly task",
539 doAssembleIsrExposures = pexConfig.Field(
542 doc=
"Assemble amp-level calibration exposures into ccd-level exposure?"
544 doTrimToMatchCalib = pexConfig.Field(
547 doc=
"Trim raw data to match calibration bounding boxes?"
551 doBias = pexConfig.Field(
553 doc=
"Apply bias frame correction?",
556 biasDataProductName = pexConfig.Field(
558 doc=
"Name of the bias data product",
561 doBiasBeforeOverscan = pexConfig.Field(
563 doc=
"Reverse order of overscan and bias correction.",
568 doVariance = pexConfig.Field(
570 doc=
"Calculate variance?",
573 gain = pexConfig.Field(
575 doc=
"The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float(
"NaN"),
578 readNoise = pexConfig.Field(
580 doc=
"The read noise to use if no Detector is present in the Exposure",
583 doEmpiricalReadNoise = pexConfig.Field(
586 doc=
"Calculate empirical read noise instead of value from AmpInfo data?"
588 usePtcReadNoise = pexConfig.Field(
591 doc=
"Use readnoise values from the Photon Transfer Curve?"
593 maskNegativeVariance = pexConfig.Field(
596 doc=
"Mask pixels that claim a negative variance? This likely indicates a failure "
597 "in the measurement of the overscan at an edge due to the data falling off faster "
598 "than the overscan model can account for it."
600 negativeVarianceMaskName = pexConfig.Field(
603 doc=
"Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
606 doLinearize = pexConfig.Field(
608 doc=
"Correct for nonlinearity of the detector's response?",
613 doCrosstalk = pexConfig.Field(
615 doc=
"Apply intra-CCD crosstalk correction?",
618 doCrosstalkBeforeAssemble = pexConfig.Field(
620 doc=
"Apply crosstalk correction before CCD assembly, and before trimming?",
623 crosstalk = pexConfig.ConfigurableField(
624 target=CrosstalkTask,
625 doc=
"Intra-CCD crosstalk correction",
629 doDefect = pexConfig.Field(
631 doc=
"Apply correction for CCD defects, e.g. hot pixels?",
634 doNanMasking = pexConfig.Field(
636 doc=
"Mask non-finite (NAN, inf) pixels?",
639 doWidenSaturationTrails = pexConfig.Field(
641 doc=
"Widen bleed trails based on their width?",
646 doBrighterFatter = pexConfig.Field(
649 doc=
"Apply the brighter-fatter correction?"
651 brighterFatterLevel = pexConfig.ChoiceField(
654 doc=
"The level at which to correct for brighter-fatter.",
656 "AMP":
"Every amplifier treated separately.",
657 "DETECTOR":
"One kernel per detector",
660 brighterFatterMaxIter = pexConfig.Field(
663 doc=
"Maximum number of iterations for the brighter-fatter correction"
665 brighterFatterThreshold = pexConfig.Field(
668 doc=
"Threshold used to stop iterating the brighter-fatter correction. It is the "
669 "absolute value of the difference between the current corrected image and the one "
670 "from the previous iteration summed over all the pixels."
672 brighterFatterApplyGain = pexConfig.Field(
675 doc=
"Should the gain be applied when applying the brighter-fatter correction?"
677 brighterFatterMaskListToInterpolate = pexConfig.ListField(
679 doc=
"List of mask planes that should be interpolated over when applying the brighter-fatter "
681 default=[
"SAT",
"BAD",
"NO_DATA",
"UNMASKEDNAN"],
683 brighterFatterMaskGrowSize = pexConfig.Field(
686 doc=
"Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
687 "when brighter-fatter correction is applied."
691 doDark = pexConfig.Field(
693 doc=
"Apply dark frame correction?",
696 darkDataProductName = pexConfig.Field(
698 doc=
"Name of the dark data product",
703 doStrayLight = pexConfig.Field(
705 doc=
"Subtract stray light in the y-band (due to encoder LEDs)?",
708 strayLight = pexConfig.ConfigurableField(
709 target=StrayLightTask,
710 doc=
"y-band stray light correction"
714 doFlat = pexConfig.Field(
716 doc=
"Apply flat field correction?",
719 flatDataProductName = pexConfig.Field(
721 doc=
"Name of the flat data product",
724 flatScalingType = pexConfig.ChoiceField(
726 doc=
"The method for scaling the flat on the fly.",
729 "USER":
"Scale by flatUserScale",
730 "MEAN":
"Scale by the inverse of the mean",
731 "MEDIAN":
"Scale by the inverse of the median",
734 flatUserScale = pexConfig.Field(
736 doc=
"If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
739 doTweakFlat = pexConfig.Field(
741 doc=
"Tweak flats to match observed amplifier ratios?",
747 doApplyGains = pexConfig.Field(
749 doc=
"Correct the amplifiers for their gains instead of applying flat correction",
752 usePtcGains = pexConfig.Field(
754 doc=
"Use the gain values from the Photon Transfer Curve?",
757 normalizeGains = pexConfig.Field(
759 doc=
"Normalize all the amplifiers in each CCD to have the same median value.",
764 doFringe = pexConfig.Field(
766 doc=
"Apply fringe correction?",
769 fringe = pexConfig.ConfigurableField(
771 doc=
"Fringe subtraction task",
773 fringeAfterFlat = pexConfig.Field(
775 doc=
"Do fringe subtraction after flat-fielding?",
780 doAmpOffset = pexConfig.Field(
781 doc=
"Calculate and apply amp offset corrections?",
785 ampOffset = pexConfig.ConfigurableField(
786 doc=
"Amp offset correction task.",
787 target=AmpOffsetTask,
791 doMeasureBackground = pexConfig.Field(
793 doc=
"Measure the background level on the reduced image?",
798 doCameraSpecificMasking = pexConfig.Field(
800 doc=
"Mask camera-specific bad regions?",
803 masking = pexConfig.ConfigurableField(
809 doInterpolate = pexConfig.Field(
811 doc=
"Interpolate masked pixels?",
814 doSaturationInterpolation = pexConfig.Field(
816 doc=
"Perform interpolation over pixels masked as saturated?"
817 " NB: This is independent of doSaturation; if that is False this plane"
818 " will likely be blank, resulting in a no-op here.",
821 doNanInterpolation = pexConfig.Field(
823 doc=
"Perform interpolation over pixels masked as NaN?"
824 " NB: This is independent of doNanMasking; if that is False this plane"
825 " will likely be blank, resulting in a no-op here.",
828 doNanInterpAfterFlat = pexConfig.Field(
830 doc=(
"If True, ensure we interpolate NaNs after flat-fielding, even if we "
831 "also have to interpolate them before flat-fielding."),
834 maskListToInterpolate = pexConfig.ListField(
836 doc=
"List of mask planes that should be interpolated.",
837 default=[
'SAT',
'BAD'],
839 doSaveInterpPixels = pexConfig.Field(
841 doc=
"Save a copy of the pre-interpolated pixel values?",
846 fluxMag0T1 = pexConfig.DictField(
849 doc=
"The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
850 default=dict((f, pow(10.0, 0.4*m))
for f, m
in ((
"Unknown", 28.0),
853 defaultFluxMag0T1 = pexConfig.Field(
855 doc=
"Default value for fluxMag0T1 (for an unrecognized filter).",
856 default=pow(10.0, 0.4*28.0)
860 doVignette = pexConfig.Field(
862 doc=
"Apply vignetting parameters?",
865 vignette = pexConfig.ConfigurableField(
867 doc=
"Vignetting task.",
871 doAttachTransmissionCurve = pexConfig.Field(
874 doc=
"Construct and attach a wavelength-dependent throughput curve for this CCD image?"
876 doUseOpticsTransmission = pexConfig.Field(
879 doc=
"Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
881 doUseFilterTransmission = pexConfig.Field(
884 doc=
"Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
886 doUseSensorTransmission = pexConfig.Field(
889 doc=
"Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
891 doUseAtmosphereTransmission = pexConfig.Field(
894 doc=
"Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
898 doIlluminationCorrection = pexConfig.Field(
901 doc=
"Perform illumination correction?"
903 illuminationCorrectionDataProductName = pexConfig.Field(
905 doc=
"Name of the illumination correction data product.",
908 illumScale = pexConfig.Field(
910 doc=
"Scale factor for the illumination correction.",
913 illumFilters = pexConfig.ListField(
916 doc=
"Only perform illumination correction for these filters."
921 doWrite = pexConfig.Field(
923 doc=
"Persist postISRCCD?",
930 raise ValueError(
"You may not specify both doFlat and doApplyGains")
932 raise ValueError(
"You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
941 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
942 """Apply common instrument signature correction algorithms to a raw frame.
944 The process for correcting imaging data is very similar from
945 camera to camera. This task provides a vanilla implementation of
946 doing these corrections, including the ability to turn certain
947 corrections off if they are not needed. The inputs to the primary
948 method, `run()`, are a raw exposure to be corrected and the
949 calibration data products. The raw input is a single chip sized
950 mosaic of all amps including overscans and other non-science
951 pixels. The method `runDataRef()` identifies and defines the
952 calibration data products, and is intended for use by a
953 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
954 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
955 subclassed for different camera, although the most camera specific
956 methods have been split into subtasks that can be redirected
959 The __init__ method sets up the subtasks for ISR processing, using
960 the defaults from `lsst.ip.isr`.
965 Positional arguments passed to the Task constructor.
966 None used at this time.
967 kwargs : `dict`, optional
968 Keyword arguments passed on to the Task constructor.
969 None used at this time.
971 ConfigClass = IsrTaskConfig
976 self.makeSubtask(
"assembleCcd")
977 self.makeSubtask(
"crosstalk")
978 self.makeSubtask(
"strayLight")
979 self.makeSubtask(
"fringe")
980 self.makeSubtask(
"masking")
981 self.makeSubtask(
"overscan")
982 self.makeSubtask(
"vignette")
983 self.makeSubtask(
"ampOffset")
986 inputs = butlerQC.get(inputRefs)
989 inputs[
'detectorNum'] = inputRefs.ccdExposure.dataId[
'detector']
990 except Exception
as e:
991 raise ValueError(
"Failure to find valid detectorNum value for Dataset %s: %s." %
994 inputs[
'isGen3'] =
True
996 detector = inputs[
'ccdExposure'].getDetector()
998 if self.config.doCrosstalk
is True:
1001 if 'crosstalk' in inputs
and inputs[
'crosstalk']
is not None:
1002 if not isinstance(inputs[
'crosstalk'], CrosstalkCalib):
1003 inputs[
'crosstalk'] = CrosstalkCalib.fromTable(inputs[
'crosstalk'])
1005 coeffVector = (self.config.crosstalk.crosstalkValues
1006 if self.config.crosstalk.useConfigCoefficients
else None)
1007 crosstalkCalib =
CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1008 inputs[
'crosstalk'] = crosstalkCalib
1009 if inputs[
'crosstalk'].interChip
and len(inputs[
'crosstalk'].interChip) > 0:
1010 if 'crosstalkSources' not in inputs:
1011 self.log.
warning(
"No crosstalkSources found for chip with interChip terms!")
1014 if 'linearizer' in inputs:
1015 if isinstance(inputs[
'linearizer'], dict):
1017 linearizer.fromYaml(inputs[
'linearizer'])
1018 self.log.
warning(
"Dictionary linearizers will be deprecated in DM-28741.")
1019 elif isinstance(inputs[
'linearizer'], numpy.ndarray):
1023 self.log.
warning(
"Bare lookup table linearizers will be deprecated in DM-28741.")
1025 linearizer = inputs[
'linearizer']
1026 linearizer.log = self.log
1027 inputs[
'linearizer'] = linearizer
1030 self.log.
warning(
"Constructing linearizer from cameraGeom information.")
1032 if self.config.doDefect
is True:
1033 if "defects" in inputs
and inputs[
'defects']
is not None:
1037 if not isinstance(inputs[
"defects"], Defects):
1038 inputs[
"defects"] = Defects.fromTable(inputs[
"defects"])
1042 if self.config.doBrighterFatter:
1043 brighterFatterKernel = inputs.pop(
'newBFKernel',
None)
1044 if brighterFatterKernel
is None:
1045 brighterFatterKernel = inputs.get(
'bfKernel',
None)
1047 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1049 detName = detector.getName()
1050 level = brighterFatterKernel.level
1053 inputs[
'bfGains'] = brighterFatterKernel.gain
1054 if self.config.brighterFatterLevel ==
'DETECTOR':
1055 if level ==
'DETECTOR':
1056 if detName
in brighterFatterKernel.detKernels:
1057 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1059 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1060 elif level ==
'AMP':
1061 self.log.
warning(
"Making DETECTOR level kernel from AMP based brighter "
1063 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1064 inputs[
'bfKernel'] = brighterFatterKernel.detKernels[detName]
1065 elif self.config.brighterFatterLevel ==
'AMP':
1066 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1068 if self.config.doFringe
is True and self.fringe.
checkFilter(inputs[
'ccdExposure']):
1069 expId = inputs[
'ccdExposure'].info.id
1070 inputs[
'fringes'] = self.fringe.loadFringes(inputs[
'fringes'],
1072 assembler=self.assembleCcd
1073 if self.config.doAssembleIsrExposures
else None)
1075 inputs[
'fringes'] = pipeBase.Struct(fringes=
None)
1077 if self.config.doStrayLight
is True and self.strayLight.
checkFilter(inputs[
'ccdExposure']):
1078 if 'strayLightData' not in inputs:
1079 inputs[
'strayLightData'] =
None
1081 outputs = self.
runrun(**inputs)
1082 butlerQC.put(outputs, outputRefs)
1085 """Retrieve necessary frames for instrument signature removal.
1087 Pre-fetching all required ISR data products limits the IO
1088 required by the ISR. Any conflict between the calibration data
1089 available and that needed for ISR is also detected prior to
1090 doing processing, allowing it to fail quickly.
1094 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1095 Butler reference of the detector data to be processed
1096 rawExposure : `afw.image.Exposure`
1097 The raw exposure that will later be corrected with the
1098 retrieved calibration data; should not be modified in this
1103 result : `lsst.pipe.base.Struct`
1104 Result struct with components (which may be `None`):
1105 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1106 - ``linearizer``: functor for linearization
1107 (`ip.isr.linearize.LinearizeBase`)
1108 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1109 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1110 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1111 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1112 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1113 - ``fringes``: `lsst.pipe.base.Struct` with components:
1114 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1115 - ``seed``: random seed derived from the ccdExposureId for random
1116 number generator (`uint32`).
1117 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1118 A ``TransmissionCurve`` that represents the throughput of the
1119 optics, to be evaluated in focal-plane coordinates.
1120 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1121 A ``TransmissionCurve`` that represents the throughput of the
1122 filter itself, to be evaluated in focal-plane coordinates.
1123 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1124 A ``TransmissionCurve`` that represents the throughput of the
1125 sensor itself, to be evaluated in post-assembly trimmed
1126 detector coordinates.
1127 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1128 A ``TransmissionCurve`` that represents the throughput of the
1129 atmosphere, assumed to be spatially constant.
1130 - ``strayLightData`` : `object`
1131 An opaque object containing calibration information for
1132 stray-light correction. If `None`, no correction will be
1134 - ``illumMaskedImage`` : illumination correction image
1135 (`lsst.afw.image.MaskedImage`)
1139 NotImplementedError :
1140 Raised if a per-amplifier brighter-fatter kernel is requested by
1144 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1145 dateObs = dateObs.toPython().isoformat()
1146 except RuntimeError:
1147 self.log.
warning(
"Unable to identify dateObs for rawExposure.")
1150 ccd = rawExposure.getDetector()
1151 filterLabel = rawExposure.getFilterLabel()
1152 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1153 rawExposure.mask.addMaskPlane(
"UNMASKEDNAN")
1154 biasExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1155 if self.config.doBias
else None)
1158 linearizer = (dataRef.get(
"linearizer", immediate=
True)
1160 if linearizer
is not None and not isinstance(linearizer, numpy.ndarray):
1161 linearizer.log = self.log
1162 if isinstance(linearizer, numpy.ndarray):
1165 crosstalkCalib =
None
1166 if self.config.doCrosstalk:
1168 crosstalkCalib = dataRef.get(
"crosstalk", immediate=
True)
1170 coeffVector = (self.config.crosstalk.crosstalkValues
1171 if self.config.crosstalk.useConfigCoefficients
else None)
1172 crosstalkCalib =
CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1173 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1174 if self.config.doCrosstalk
else None)
1176 darkExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1177 if self.config.doDark
else None)
1178 flatExposure = (self.
getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1180 if self.config.doFlat
else None)
1182 brighterFatterKernel =
None
1183 brighterFatterGains =
None
1184 if self.config.doBrighterFatter
is True:
1189 brighterFatterKernel = dataRef.get(
"brighterFatterKernel")
1190 brighterFatterGains = brighterFatterKernel.gain
1191 self.log.
info(
"New style brighter-fatter kernel (brighterFatterKernel) loaded")
1194 brighterFatterKernel = dataRef.get(
"bfKernel")
1195 self.log.
info(
"Old style brighter-fatter kernel (bfKernel) loaded")
1197 brighterFatterKernel =
None
1198 if brighterFatterKernel
is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1201 if self.config.brighterFatterLevel ==
'DETECTOR':
1202 if brighterFatterKernel.detKernels:
1203 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1205 raise RuntimeError(
"Failed to extract kernel from new-style BF kernel.")
1208 raise NotImplementedError(
"Per-amplifier brighter-fatter correction not implemented")
1210 defectList = (dataRef.get(
"defects")
1211 if self.config.doDefect
else None)
1212 expId = rawExposure.info.id
1213 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1214 if self.config.doAssembleIsrExposures
else None)
1215 if self.config.doFringe
and self.fringe.
checkFilter(rawExposure)
1216 else pipeBase.Struct(fringes=
None))
1218 if self.config.doAttachTransmissionCurve:
1219 opticsTransmission = (dataRef.get(
"transmission_optics")
1220 if self.config.doUseOpticsTransmission
else None)
1221 filterTransmission = (dataRef.get(
"transmission_filter")
1222 if self.config.doUseFilterTransmission
else None)
1223 sensorTransmission = (dataRef.get(
"transmission_sensor")
1224 if self.config.doUseSensorTransmission
else None)
1225 atmosphereTransmission = (dataRef.get(
"transmission_atmosphere")
1226 if self.config.doUseAtmosphereTransmission
else None)
1228 opticsTransmission =
None
1229 filterTransmission =
None
1230 sensorTransmission =
None
1231 atmosphereTransmission =
None
1233 if self.config.doStrayLight:
1234 strayLightData = self.strayLight.
readIsrData(dataRef, rawExposure)
1236 strayLightData =
None
1239 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1240 if (self.config.doIlluminationCorrection
1241 and physicalFilter
in self.config.illumFilters)
1245 return pipeBase.Struct(bias=biasExposure,
1246 linearizer=linearizer,
1247 crosstalk=crosstalkCalib,
1248 crosstalkSources=crosstalkSources,
1251 bfKernel=brighterFatterKernel,
1252 bfGains=brighterFatterGains,
1254 fringes=fringeStruct,
1255 opticsTransmission=opticsTransmission,
1256 filterTransmission=filterTransmission,
1257 sensorTransmission=sensorTransmission,
1258 atmosphereTransmission=atmosphereTransmission,
1259 strayLightData=strayLightData,
1260 illumMaskedImage=illumMaskedImage
1264 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1265 crosstalk=None, crosstalkSources=None,
1266 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1267 fringes=pipeBase.Struct(fringes=
None), opticsTransmission=
None, filterTransmission=
None,
1268 sensorTransmission=
None, atmosphereTransmission=
None,
1269 detectorNum=
None, strayLightData=
None, illumMaskedImage=
None,
1272 """Perform instrument signature removal on an exposure.
1274 Steps included in the ISR processing, in order performed, are:
1275 - saturation and suspect pixel masking
1276 - overscan subtraction
1277 - CCD assembly of individual amplifiers
1279 - variance image construction
1280 - linearization of non-linear response
1282 - brighter-fatter correction
1285 - stray light subtraction
1287 - masking of known defects and camera specific features
1288 - vignette calculation
1289 - appending transmission curve and distortion model
1293 ccdExposure : `lsst.afw.image.Exposure`
1294 The raw exposure that is to be run through ISR. The
1295 exposure is modified by this method.
1296 camera : `lsst.afw.cameraGeom.Camera`, optional
1297 The camera geometry for this exposure. Required if
1298 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1299 ``flat`` does not have an associated detector.
1300 bias : `lsst.afw.image.Exposure`, optional
1301 Bias calibration frame.
1302 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1303 Functor for linearization.
1304 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1305 Calibration for crosstalk.
1306 crosstalkSources : `list`, optional
1307 List of possible crosstalk sources.
1308 dark : `lsst.afw.image.Exposure`, optional
1309 Dark calibration frame.
1310 flat : `lsst.afw.image.Exposure`, optional
1311 Flat calibration frame.
1312 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1313 Photon transfer curve dataset, with, e.g., gains
1315 bfKernel : `numpy.ndarray`, optional
1316 Brighter-fatter kernel.
1317 bfGains : `dict` of `float`, optional
1318 Gains used to override the detector's nominal gains for the
1319 brighter-fatter correction. A dict keyed by amplifier name for
1320 the detector in question.
1321 defects : `lsst.ip.isr.Defects`, optional
1323 fringes : `lsst.pipe.base.Struct`, optional
1324 Struct containing the fringe correction data, with
1326 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1327 - ``seed``: random seed derived from the ccdExposureId for random
1328 number generator (`uint32`)
1329 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1330 A ``TransmissionCurve`` that represents the throughput of the,
1331 optics, to be evaluated in focal-plane coordinates.
1332 filterTransmission : `lsst.afw.image.TransmissionCurve`
1333 A ``TransmissionCurve`` that represents the throughput of the
1334 filter itself, to be evaluated in focal-plane coordinates.
1335 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1336 A ``TransmissionCurve`` that represents the throughput of the
1337 sensor itself, to be evaluated in post-assembly trimmed detector
1339 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1340 A ``TransmissionCurve`` that represents the throughput of the
1341 atmosphere, assumed to be spatially constant.
1342 detectorNum : `int`, optional
1343 The integer number for the detector to process.
1344 isGen3 : bool, optional
1345 Flag this call to run() as using the Gen3 butler environment.
1346 strayLightData : `object`, optional
1347 Opaque object containing calibration information for stray-light
1348 correction. If `None`, no correction will be performed.
1349 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1350 Illumination correction image.
1354 result : `lsst.pipe.base.Struct`
1355 Result struct with component:
1356 - ``exposure`` : `afw.image.Exposure`
1357 The fully ISR corrected exposure.
1358 - ``outputExposure`` : `afw.image.Exposure`
1359 An alias for `exposure`
1360 - ``ossThumb`` : `numpy.ndarray`
1361 Thumbnail image of the exposure after overscan subtraction.
1362 - ``flattenedThumb`` : `numpy.ndarray`
1363 Thumbnail image of the exposure after flat-field correction.
1368 Raised if a configuration option is set to True, but the
1369 required calibration data has not been specified.
1373 The current processed exposure can be viewed by setting the
1374 appropriate lsstDebug entries in the `debug.display`
1375 dictionary. The names of these entries correspond to some of
1376 the IsrTaskConfig Boolean options, with the value denoting the
1377 frame to use. The exposure is shown inside the matching
1378 option check and after the processing of that step has
1379 finished. The steps with debug points are:
1390 In addition, setting the "postISRCCD" entry displays the
1391 exposure after all ISR processing has finished.
1400 ccdExposure = self.
ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1401 bias = self.
ensureExposureensureExposure(bias, camera, detectorNum)
1402 dark = self.
ensureExposureensureExposure(dark, camera, detectorNum)
1403 flat = self.
ensureExposureensureExposure(flat, camera, detectorNum)
1405 if isinstance(ccdExposure, ButlerDataRef):
1406 return self.
runDataRefrunDataRef(ccdExposure)
1408 ccd = ccdExposure.getDetector()
1409 filterLabel = ccdExposure.getFilterLabel()
1410 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1413 assert not self.config.doAssembleCcd,
"You need a Detector to run assembleCcd."
1414 ccd = [
FakeAmp(ccdExposure, self.config)]
1417 if self.config.doBias
and bias
is None:
1418 raise RuntimeError(
"Must supply a bias exposure if config.doBias=True.")
1419 if self.
doLinearizedoLinearize(ccd)
and linearizer
is None:
1420 raise RuntimeError(
"Must supply a linearizer if config.doLinearize=True for this detector.")
1421 if self.config.doBrighterFatter
and bfKernel
is None:
1422 raise RuntimeError(
"Must supply a kernel if config.doBrighterFatter=True.")
1423 if self.config.doDark
and dark
is None:
1424 raise RuntimeError(
"Must supply a dark exposure if config.doDark=True.")
1425 if self.config.doFlat
and flat
is None:
1426 raise RuntimeError(
"Must supply a flat exposure if config.doFlat=True.")
1427 if self.config.doDefect
and defects
is None:
1428 raise RuntimeError(
"Must supply defects if config.doDefect=True.")
1429 if (self.config.doFringe
and physicalFilter
in self.fringe.config.filters
1430 and fringes.fringes
is None):
1435 raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct.")
1436 if (self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters
1437 and illumMaskedImage
is None):
1438 raise RuntimeError(
"Must supply an illumcor if config.doIlluminationCorrection=True.")
1441 if self.config.doConvertIntToFloat:
1442 self.log.
info(
"Converting exposure to floating point values.")
1445 if self.config.doBias
and self.config.doBiasBeforeOverscan:
1446 self.log.
info(
"Applying bias correction.")
1447 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1448 trimToFit=self.config.doTrimToMatchCalib)
1449 self.
debugViewdebugView(ccdExposure,
"doBias")
1456 if ccdExposure.getBBox().
contains(amp.getBBox()):
1459 badAmp = self.
maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1461 if self.config.doOverscan
and not badAmp:
1464 self.log.
debug(
"Corrected overscan for amplifier %s.", amp.getName())
1465 if overscanResults
is not None and \
1466 self.config.qa
is not None and self.config.qa.saveStats
is True:
1467 if isinstance(overscanResults.overscanFit, float):
1468 qaMedian = overscanResults.overscanFit
1469 qaStdev = float(
"NaN")
1472 afwMath.MEDIAN | afwMath.STDEVCLIP)
1473 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1474 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1476 self.metadata[f
"FIT MEDIAN {amp.getName()}"] = qaMedian
1477 self.metadata[f
"FIT STDEV {amp.getName()}"] = qaStdev
1478 self.log.
debug(
" Overscan stats for amplifer %s: %f +/- %f",
1479 amp.getName(), qaMedian, qaStdev)
1483 afwMath.MEDIAN | afwMath.STDEVCLIP)
1484 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1485 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1487 self.metadata[f
"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1488 self.metadata[f
"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1489 self.log.
debug(
" Overscan stats for amplifer %s after correction: %f +/- %f",
1490 amp.getName(), qaMedianAfter, qaStdevAfter)
1492 ccdExposure.getMetadata().
set(
'OVERSCAN',
"Overscan corrected")
1495 self.log.
warning(
"Amplifier %s is bad.", amp.getName())
1496 overscanResults =
None
1498 overscans.append(overscanResults
if overscanResults
is not None else None)
1500 self.log.
info(
"Skipped OSCAN for %s.", amp.getName())
1502 if self.config.doCrosstalk
and self.config.doCrosstalkBeforeAssemble:
1503 self.log.
info(
"Applying crosstalk correction.")
1504 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1505 crosstalkSources=crosstalkSources, camera=camera)
1506 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1508 if self.config.doAssembleCcd:
1509 self.log.
info(
"Assembling CCD from amplifiers.")
1510 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1512 if self.config.expectWcs
and not ccdExposure.getWcs():
1513 self.log.
warning(
"No WCS found in input exposure.")
1514 self.
debugViewdebugView(ccdExposure,
"doAssembleCcd")
1517 if self.config.qa.doThumbnailOss:
1518 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1520 if self.config.doBias
and not self.config.doBiasBeforeOverscan:
1521 self.log.
info(
"Applying bias correction.")
1522 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1523 trimToFit=self.config.doTrimToMatchCalib)
1524 self.
debugViewdebugView(ccdExposure,
"doBias")
1526 if self.config.doVariance:
1527 for amp, overscanResults
in zip(ccd, overscans):
1528 if ccdExposure.getBBox().
contains(amp.getBBox()):
1529 self.log.
debug(
"Constructing variance map for amplifer %s.", amp.getName())
1530 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1531 if overscanResults
is not None:
1533 overscanImage=overscanResults.overscanImage,
1539 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1541 afwMath.MEDIAN | afwMath.STDEVCLIP)
1542 self.metadata[f
"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1543 qaStats.getValue(afwMath.MEDIAN)
1544 self.metadata[f
"ISR VARIANCE {amp.getName()} STDEV"] = \
1545 qaStats.getValue(afwMath.STDEVCLIP)
1546 self.log.
debug(
" Variance stats for amplifer %s: %f +/- %f.",
1547 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1548 qaStats.getValue(afwMath.STDEVCLIP))
1549 if self.config.maskNegativeVariance:
1553 self.log.
info(
"Applying linearizer.")
1554 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1555 detector=ccd, log=self.log)
1557 if self.config.doCrosstalk
and not self.config.doCrosstalkBeforeAssemble:
1558 self.log.
info(
"Applying crosstalk correction.")
1559 self.crosstalk.
run(ccdExposure, crosstalk=crosstalk,
1560 crosstalkSources=crosstalkSources, isTrimmed=
True)
1561 self.
debugViewdebugView(ccdExposure,
"doCrosstalk")
1566 if self.config.doDefect:
1567 self.log.
info(
"Masking defects.")
1568 self.
maskDefectmaskDefect(ccdExposure, defects)
1570 if self.config.numEdgeSuspect > 0:
1571 self.log.
info(
"Masking edges as SUSPECT.")
1572 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1573 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
1575 if self.config.doNanMasking:
1576 self.log.
info(
"Masking non-finite (NAN, inf) value pixels.")
1577 self.
maskNanmaskNan(ccdExposure)
1579 if self.config.doWidenSaturationTrails:
1580 self.log.
info(
"Widening saturation trails.")
1581 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1583 if self.config.doCameraSpecificMasking:
1584 self.log.
info(
"Masking regions for camera specific reasons.")
1585 self.masking.
run(ccdExposure)
1587 if self.config.doBrighterFatter:
1597 interpExp = ccdExposure.clone()
1598 with self.
flatContextflatContext(interpExp, flat, dark):
1599 isrFunctions.interpolateFromMask(
1600 maskedImage=interpExp.getMaskedImage(),
1601 fwhm=self.config.fwhm,
1602 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1603 maskNameList=
list(self.config.brighterFatterMaskListToInterpolate)
1605 bfExp = interpExp.clone()
1607 self.log.
info(
"Applying brighter-fatter correction using kernel type %s / gains %s.",
1609 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1610 self.config.brighterFatterMaxIter,
1611 self.config.brighterFatterThreshold,
1612 self.config.brighterFatterApplyGain,
1614 if bfResults[1] == self.config.brighterFatterMaxIter:
1615 self.log.
warning(
"Brighter-fatter correction did not converge, final difference %f.",
1618 self.log.
info(
"Finished brighter-fatter correction in %d iterations.",
1620 image = ccdExposure.getMaskedImage().getImage()
1621 bfCorr = bfExp.getMaskedImage().getImage()
1622 bfCorr -= interpExp.getMaskedImage().getImage()
1631 self.log.
info(
"Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1632 self.
maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1635 if self.config.brighterFatterMaskGrowSize > 0:
1636 self.log.
info(
"Growing masks to account for brighter-fatter kernel convolution.")
1637 for maskPlane
in self.config.brighterFatterMaskListToInterpolate:
1638 isrFunctions.growMasks(ccdExposure.getMask(),
1639 radius=self.config.brighterFatterMaskGrowSize,
1640 maskNameList=maskPlane,
1641 maskValue=maskPlane)
1643 self.
debugViewdebugView(ccdExposure,
"doBrighterFatter")
1645 if self.config.doDark:
1646 self.log.
info(
"Applying dark correction.")
1648 self.
debugViewdebugView(ccdExposure,
"doDark")
1650 if self.config.doFringe
and not self.config.fringeAfterFlat:
1651 self.log.
info(
"Applying fringe correction before flat.")
1652 self.fringe.
run(ccdExposure, **fringes.getDict())
1653 self.
debugViewdebugView(ccdExposure,
"doFringe")
1655 if self.config.doStrayLight
and self.strayLight.check(ccdExposure):
1656 self.log.
info(
"Checking strayLight correction.")
1657 self.strayLight.
run(ccdExposure, strayLightData)
1658 self.
debugViewdebugView(ccdExposure,
"doStrayLight")
1660 if self.config.doFlat:
1661 self.log.
info(
"Applying flat correction.")
1663 self.
debugViewdebugView(ccdExposure,
"doFlat")
1665 if self.config.doApplyGains:
1666 self.log.
info(
"Applying gain correction instead of flat.")
1667 if self.config.usePtcGains:
1668 self.log.
info(
"Using gains from the Photon Transfer Curve.")
1669 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1672 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1674 if self.config.doFringe
and self.config.fringeAfterFlat:
1675 self.log.
info(
"Applying fringe correction after flat.")
1676 self.fringe.
run(ccdExposure, **fringes.getDict())
1678 if self.config.doVignette:
1679 self.log.
info(
"Constructing Vignette polygon.")
1682 if self.config.vignette.doWriteVignettePolygon:
1685 if self.config.doAttachTransmissionCurve:
1686 self.log.
info(
"Adding transmission curves.")
1687 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1688 filterTransmission=filterTransmission,
1689 sensorTransmission=sensorTransmission,
1690 atmosphereTransmission=atmosphereTransmission)
1692 flattenedThumb =
None
1693 if self.config.qa.doThumbnailFlattened:
1694 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1696 if self.config.doIlluminationCorrection
and physicalFilter
in self.config.illumFilters:
1697 self.log.
info(
"Performing illumination correction.")
1698 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1699 illumMaskedImage, illumScale=self.config.illumScale,
1700 trimToFit=self.config.doTrimToMatchCalib)
1703 if self.config.doSaveInterpPixels:
1704 preInterpExp = ccdExposure.clone()
1719 if self.config.doSetBadRegions:
1720 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1721 if badPixelCount > 0:
1722 self.log.
info(
"Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1724 if self.config.doInterpolate:
1725 self.log.
info(
"Interpolating masked pixels.")
1726 isrFunctions.interpolateFromMask(
1727 maskedImage=ccdExposure.getMaskedImage(),
1728 fwhm=self.config.fwhm,
1729 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1730 maskNameList=
list(self.config.maskListToInterpolate)
1736 if self.config.doAmpOffset:
1737 self.log.
info(
"Correcting amp offsets.")
1738 self.ampOffset.
run(ccdExposure)
1740 if self.config.doMeasureBackground:
1741 self.log.
info(
"Measuring background level.")
1744 if self.config.qa
is not None and self.config.qa.saveStats
is True:
1746 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1748 afwMath.MEDIAN | afwMath.STDEVCLIP)
1749 self.metadata[f
"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1750 self.metadata[f
"ISR BACKGROUND {amp.getName()} STDEV"] = \
1751 qaStats.getValue(afwMath.STDEVCLIP)
1752 self.log.
debug(
" Background stats for amplifer %s: %f +/- %f",
1753 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1754 qaStats.getValue(afwMath.STDEVCLIP))
1756 self.
debugViewdebugView(ccdExposure,
"postISRCCD")
1758 return pipeBase.Struct(
1759 exposure=ccdExposure,
1761 flattenedThumb=flattenedThumb,
1763 preInterpExposure=preInterpExp,
1764 outputExposure=ccdExposure,
1765 outputOssThumbnail=ossThumb,
1766 outputFlattenedThumbnail=flattenedThumb,
1771 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1773 This method contains the `CmdLineTask` interface to the ISR
1774 processing. All IO is handled here, freeing the `run()` method
1775 to manage only pixel-level calculations. The steps performed
1777 - Read in necessary detrending/isr/calibration data.
1778 - Process raw exposure in `run()`.
1779 - Persist the ISR-corrected exposure as "postISRCCD" if
1780 config.doWrite=True.
1784 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1785 DataRef of the detector data to be processed
1789 result : `lsst.pipe.base.Struct`
1790 Result struct with component:
1791 - ``exposure`` : `afw.image.Exposure`
1792 The fully ISR corrected exposure.
1797 Raised if a configuration option is set to True, but the
1798 required calibration data does not exist.
1801 self.log.
info(
"Performing ISR on sensor %s.", sensorRef.dataId)
1803 ccdExposure = sensorRef.get(self.config.datasetType)
1805 camera = sensorRef.get(
"camera")
1806 isrData = self.
readIsrDatareadIsrData(sensorRef, ccdExposure)
1808 result = self.
runrun(ccdExposure, camera=camera, **isrData.getDict())
1810 if self.config.doWrite:
1811 sensorRef.put(result.exposure,
"postISRCCD")
1812 if result.preInterpExposure
is not None:
1813 sensorRef.put(result.preInterpExposure,
"postISRCCD_uninterpolated")
1814 if result.ossThumb
is not None:
1815 isrQa.writeThumbnail(sensorRef, result.ossThumb,
"ossThumb")
1816 if result.flattenedThumb
is not None:
1817 isrQa.writeThumbnail(sensorRef, result.flattenedThumb,
"flattenedThumb")
1822 """Retrieve a calibration dataset for removing instrument signature.
1827 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1828 DataRef of the detector data to find calibration datasets
1831 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1832 dateObs : `str`, optional
1833 Date of the observation. Used to correct butler failures
1834 when using fallback filters.
1836 If True, disable butler proxies to enable error handling
1837 within this routine.
1841 exposure : `lsst.afw.image.Exposure`
1842 Requested calibration frame.
1847 Raised if no matching calibration frame can be found.
1850 exp = dataRef.get(datasetType, immediate=immediate)
1851 except Exception
as exc1:
1852 if not self.config.fallbackFilterName:
1853 raise RuntimeError(
"Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1855 if self.config.useFallbackDate
and dateObs:
1856 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1857 dateObs=dateObs, immediate=immediate)
1859 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1860 except Exception
as exc2:
1861 raise RuntimeError(
"Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1862 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1863 self.log.
warning(
"Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1865 if self.config.doAssembleIsrExposures:
1866 exp = self.assembleCcd.assembleCcd(exp)
1870 """Ensure that the data returned by Butler is a fully constructed exp.
1872 ISR requires exposure-level image data for historical reasons, so if we
1873 did not recieve that from Butler, construct it from what we have,
1874 modifying the input in place.
1878 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1879 or `lsst.afw.image.ImageF`
1880 The input data structure obtained from Butler.
1881 camera : `lsst.afw.cameraGeom.camera`, optional
1882 The camera associated with the image. Used to find the appropriate
1883 detector if detector is not already set.
1884 detectorNum : `int`, optional
1885 The detector in the camera to attach, if the detector is not
1890 inputExp : `lsst.afw.image.Exposure`
1891 The re-constructed exposure, with appropriate detector parameters.
1896 Raised if the input data cannot be used to construct an exposure.
1898 if isinstance(inputExp, afwImage.DecoratedImageU):
1900 elif isinstance(inputExp, afwImage.ImageF):
1902 elif isinstance(inputExp, afwImage.MaskedImageF):
1906 elif inputExp
is None:
1910 raise TypeError(
"Input Exposure is not known type in isrTask.ensureExposure: %s." %
1913 if inputExp.getDetector()
is None:
1914 if camera
is None or detectorNum
is None:
1915 raise RuntimeError(
'Must supply both a camera and detector number when using exposures '
1916 'without a detector set.')
1917 inputExp.setDetector(camera[detectorNum])
1922 """Convert exposure image from uint16 to float.
1924 If the exposure does not need to be converted, the input is
1925 immediately returned. For exposures that are converted to use
1926 floating point pixels, the variance is set to unity and the
1931 exposure : `lsst.afw.image.Exposure`
1932 The raw exposure to be converted.
1936 newexposure : `lsst.afw.image.Exposure`
1937 The input ``exposure``, converted to floating point pixels.
1942 Raised if the exposure type cannot be converted to float.
1945 if isinstance(exposure, afwImage.ExposureF):
1947 self.log.
debug(
"Exposure already of type float.")
1949 if not hasattr(exposure,
"convertF"):
1950 raise RuntimeError(
"Unable to convert exposure (%s) to float." %
type(exposure))
1952 newexposure = exposure.convertF()
1953 newexposure.variance[:] = 1
1954 newexposure.mask[:] = 0x0
1959 """Identify bad amplifiers, saturated and suspect pixels.
1963 ccdExposure : `lsst.afw.image.Exposure`
1964 Input exposure to be masked.
1965 amp : `lsst.afw.table.AmpInfoCatalog`
1966 Catalog of parameters defining the amplifier on this
1968 defects : `lsst.ip.isr.Defects`
1969 List of defects. Used to determine if the entire
1975 If this is true, the entire amplifier area is covered by
1976 defects and unusable.
1979 maskedImage = ccdExposure.getMaskedImage()
1986 if defects
is not None:
1987 badAmp = bool(sum([v.getBBox().
contains(amp.getBBox())
for v
in defects]))
1993 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1995 maskView = dataView.getMask()
1996 maskView |= maskView.getPlaneBitMask(
"BAD")
2004 if self.config.doSaturation
and not badAmp:
2005 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2006 if self.config.doSuspect
and not badAmp:
2007 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2008 if math.isfinite(self.config.saturation):
2009 limits.update({self.config.saturatedMaskName: self.config.saturation})
2011 for maskName, maskThreshold
in limits.items():
2012 if not math.isnan(maskThreshold):
2013 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2014 isrFunctions.makeThresholdMask(
2015 maskedImage=dataView,
2016 threshold=maskThreshold,
2023 maskView =
afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2025 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2026 self.config.suspectMaskName])
2027 if numpy.all(maskView.getArray() & maskVal > 0):
2029 maskView |= maskView.getPlaneBitMask(
"BAD")
2034 """Apply overscan correction in place.
2036 This method does initial pixel rejection of the overscan
2037 region. The overscan can also be optionally segmented to
2038 allow for discontinuous overscan responses to be fit
2039 separately. The actual overscan subtraction is performed by
2040 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2041 which is called here after the amplifier is preprocessed.
2045 ccdExposure : `lsst.afw.image.Exposure`
2046 Exposure to have overscan correction performed.
2047 amp : `lsst.afw.cameraGeom.Amplifer`
2048 The amplifier to consider while correcting the overscan.
2052 overscanResults : `lsst.pipe.base.Struct`
2053 Result struct with components:
2054 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2055 Value or fit subtracted from the amplifier image data.
2056 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2057 Value or fit subtracted from the overscan image data.
2058 - ``overscanImage`` : `lsst.afw.image.Image`
2059 Image of the overscan region with the overscan
2060 correction applied. This quantity is used to estimate
2061 the amplifier read noise empirically.
2066 Raised if the ``amp`` does not contain raw pixel information.
2070 lsst.ip.isr.isrFunctions.overscanCorrection
2072 if amp.getRawHorizontalOverscanBBox().isEmpty():
2073 self.log.
info(
"ISR_OSCAN: No overscan region. Not performing overscan correction.")
2077 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2080 dataBBox = amp.getRawDataBBox()
2081 oscanBBox = amp.getRawHorizontalOverscanBBox()
2085 prescanBBox = amp.getRawPrescanBBox()
2086 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()):
2087 dx0 += self.config.overscanNumLeadingColumnsToSkip
2088 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2090 dx0 += self.config.overscanNumTrailingColumnsToSkip
2091 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2098 if ((self.config.overscanBiasJump
2099 and self.config.overscanBiasJumpLocation)
2100 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2101 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword)
in
2102 self.config.overscanBiasJumpDevices)):
2103 if amp.getReadoutCorner()
in (ReadoutCorner.LL, ReadoutCorner.LR):
2104 yLower = self.config.overscanBiasJumpLocation
2105 yUpper = dataBBox.getHeight() - yLower
2107 yUpper = self.config.overscanBiasJumpLocation
2108 yLower = dataBBox.getHeight() - yUpper
2126 oscanBBox.getHeight())))
2130 for imageBBox, overscanBBox
in zip(imageBBoxes, overscanBBoxes):
2131 ampImage = ccdExposure.maskedImage[imageBBox]
2132 overscanImage = ccdExposure.maskedImage[overscanBBox]
2134 overscanArray = overscanImage.image.array
2135 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2136 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2137 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask(
"SAT")
2140 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask(
"SAT"))
2142 overscanResults = self.overscan.
run(ampImage.getImage(), overscanImage, amp)
2145 levelStat = afwMath.MEDIAN
2146 sigmaStat = afwMath.STDEVCLIP
2149 self.config.qa.flatness.nIter)
2150 metadata = ccdExposure.getMetadata()
2151 ampNum = amp.getName()
2153 if isinstance(overscanResults.overscanFit, float):
2154 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2155 metadata[f
"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2158 metadata[f
"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2159 metadata[f
"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2161 return overscanResults
2164 """Set the variance plane using the gain and read noise
2166 The read noise is calculated from the ``overscanImage`` if the
2167 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2168 the value from the amplifier data is used.
2172 ampExposure : `lsst.afw.image.Exposure`
2173 Exposure to process.
2174 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2175 Amplifier detector data.
2176 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2177 Image of overscan, required only for empirical read noise.
2178 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2179 PTC dataset containing the gains and read noise.
2185 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2186 are ``True``, but ptcDataset is not provided.
2188 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2189 ``overscanImage`` is ``None``.
2193 lsst.ip.isr.isrFunctions.updateVariance
2195 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2196 if self.config.usePtcGains:
2197 if ptcDataset
is None:
2198 raise RuntimeError(
"No ptcDataset provided to use PTC gains.")
2200 gain = ptcDataset.gain[amp.getName()]
2201 self.log.
info(
"Using gain from Photon Transfer Curve.")
2203 gain = amp.getGain()
2205 if math.isnan(gain):
2207 self.log.
warning(
"Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2210 self.log.
warning(
"Gain for amp %s == %g <= 0; setting to %f.",
2211 amp.getName(), gain, patchedGain)
2214 if self.config.doEmpiricalReadNoise
and overscanImage
is None:
2215 raise RuntimeError(
"Overscan is none for EmpiricalReadNoise.")
2217 if self.config.doEmpiricalReadNoise
and overscanImage
is not None:
2219 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2221 self.log.
info(
"Calculated empirical read noise for amp %s: %f.",
2222 amp.getName(), readNoise)
2223 elif self.config.usePtcReadNoise:
2224 if ptcDataset
is None:
2225 raise RuntimeError(
"No ptcDataset provided to use PTC readnoise.")
2227 readNoise = ptcDataset.noise[amp.getName()]
2228 self.log.
info(
"Using read noise from Photon Transfer Curve.")
2230 readNoise = amp.getReadNoise()
2232 isrFunctions.updateVariance(
2233 maskedImage=ampExposure.getMaskedImage(),
2235 readNoise=readNoise,
2239 """Identify and mask pixels with negative variance values.
2243 exposure : `lsst.afw.image.Exposure`
2244 Exposure to process.
2248 lsst.ip.isr.isrFunctions.updateVariance
2250 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2251 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2252 exposure.mask.array[bad] |= maskPlane
2255 """Apply dark correction in place.
2259 exposure : `lsst.afw.image.Exposure`
2260 Exposure to process.
2261 darkExposure : `lsst.afw.image.Exposure`
2262 Dark exposure of the same size as ``exposure``.
2263 invert : `Bool`, optional
2264 If True, re-add the dark to an already corrected image.
2269 Raised if either ``exposure`` or ``darkExposure`` do not
2270 have their dark time defined.
2274 lsst.ip.isr.isrFunctions.darkCorrection
2276 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2277 if math.isnan(expScale):
2278 raise RuntimeError(
"Exposure darktime is NAN.")
2279 if darkExposure.getInfo().getVisitInfo()
is not None \
2280 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2281 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2285 self.log.
warning(
"darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2288 isrFunctions.darkCorrection(
2289 maskedImage=exposure.getMaskedImage(),
2290 darkMaskedImage=darkExposure.getMaskedImage(),
2292 darkScale=darkScale,
2294 trimToFit=self.config.doTrimToMatchCalib
2298 """Check if linearization is needed for the detector cameraGeom.
2300 Checks config.doLinearize and the linearity type of the first
2305 detector : `lsst.afw.cameraGeom.Detector`
2306 Detector to get linearity type from.
2310 doLinearize : `Bool`
2311 If True, linearization should be performed.
2313 return self.config.doLinearize
and \
2314 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2317 """Apply flat correction in place.
2321 exposure : `lsst.afw.image.Exposure`
2322 Exposure to process.
2323 flatExposure : `lsst.afw.image.Exposure`
2324 Flat exposure of the same size as ``exposure``.
2325 invert : `Bool`, optional
2326 If True, unflatten an already flattened image.
2330 lsst.ip.isr.isrFunctions.flatCorrection
2332 isrFunctions.flatCorrection(
2333 maskedImage=exposure.getMaskedImage(),
2334 flatMaskedImage=flatExposure.getMaskedImage(),
2335 scalingType=self.config.flatScalingType,
2336 userScale=self.config.flatUserScale,
2338 trimToFit=self.config.doTrimToMatchCalib
2342 """Detect and mask saturated pixels in config.saturatedMaskName.
2346 exposure : `lsst.afw.image.Exposure`
2347 Exposure to process. Only the amplifier DataSec is processed.
2348 amp : `lsst.afw.table.AmpInfoCatalog`
2349 Amplifier detector data.
2353 lsst.ip.isr.isrFunctions.makeThresholdMask
2355 if not math.isnan(amp.getSaturation()):
2356 maskedImage = exposure.getMaskedImage()
2357 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2358 isrFunctions.makeThresholdMask(
2359 maskedImage=dataView,
2360 threshold=amp.getSaturation(),
2362 maskName=self.config.saturatedMaskName,
2366 """Interpolate over saturated pixels, in place.
2368 This method should be called after `saturationDetection`, to
2369 ensure that the saturated pixels have been identified in the
2370 SAT mask. It should also be called after `assembleCcd`, since
2371 saturated regions may cross amplifier boundaries.
2375 exposure : `lsst.afw.image.Exposure`
2376 Exposure to process.
2380 lsst.ip.isr.isrTask.saturationDetection
2381 lsst.ip.isr.isrFunctions.interpolateFromMask
2383 isrFunctions.interpolateFromMask(
2384 maskedImage=exposure.getMaskedImage(),
2385 fwhm=self.config.fwhm,
2386 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2387 maskNameList=
list(self.config.saturatedMaskName),
2391 """Detect and mask suspect pixels in config.suspectMaskName.
2395 exposure : `lsst.afw.image.Exposure`
2396 Exposure to process. Only the amplifier DataSec is processed.
2397 amp : `lsst.afw.table.AmpInfoCatalog`
2398 Amplifier detector data.
2402 lsst.ip.isr.isrFunctions.makeThresholdMask
2406 Suspect pixels are pixels whose value is greater than
2407 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2408 affected by unknown systematics; for example if non-linearity
2409 corrections above a certain level are unstable then that would be a
2410 useful value for suspectLevel. A value of `nan` indicates that no such
2411 level exists and no pixels are to be masked as suspicious.
2413 suspectLevel = amp.getSuspectLevel()
2414 if math.isnan(suspectLevel):
2417 maskedImage = exposure.getMaskedImage()
2418 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2419 isrFunctions.makeThresholdMask(
2420 maskedImage=dataView,
2421 threshold=suspectLevel,
2423 maskName=self.config.suspectMaskName,
2427 """Mask defects using mask plane "BAD", in place.
2431 exposure : `lsst.afw.image.Exposure`
2432 Exposure to process.
2433 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2434 `lsst.afw.image.DefectBase`.
2435 List of defects to mask.
2439 Call this after CCD assembly, since defects may cross amplifier
2442 maskedImage = exposure.getMaskedImage()
2443 if not isinstance(defectBaseList, Defects):
2445 defectList =
Defects(defectBaseList)
2447 defectList = defectBaseList
2448 defectList.maskPixels(maskedImage, maskName=
"BAD")
2450 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2451 """Mask edge pixels with applicable mask plane.
2455 exposure : `lsst.afw.image.Exposure`
2456 Exposure to process.
2457 numEdgePixels : `int`, optional
2458 Number of edge pixels to mask.
2459 maskPlane : `str`, optional
2460 Mask plane name to use.
2461 level : `str`, optional
2462 Level at which to mask edges.
2464 maskedImage = exposure.getMaskedImage()
2465 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2467 if numEdgePixels > 0:
2468 if level ==
'DETECTOR':
2469 boxes = [maskedImage.getBBox()]
2470 elif level ==
'AMP':
2471 boxes = [amp.getBBox()
for amp
in exposure.getDetector()]
2476 subImage = maskedImage[box]
2477 box.grow(-numEdgePixels)
2479 SourceDetectionTask.setEdgeBits(
2485 """Mask and interpolate defects using mask plane "BAD", in place.
2489 exposure : `lsst.afw.image.Exposure`
2490 Exposure to process.
2491 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2492 `lsst.afw.image.DefectBase`.
2493 List of defects to mask and interpolate.
2497 lsst.ip.isr.isrTask.maskDefect
2499 self.
maskDefectmaskDefect(exposure, defectBaseList)
2500 self.
maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2501 maskPlane=
"SUSPECT", level=self.config.edgeMaskLevel)
2502 isrFunctions.interpolateFromMask(
2503 maskedImage=exposure.getMaskedImage(),
2504 fwhm=self.config.fwhm,
2505 growSaturatedFootprints=0,
2506 maskNameList=[
"BAD"],
2510 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2514 exposure : `lsst.afw.image.Exposure`
2515 Exposure to process.
2519 We mask over all non-finite values (NaN, inf), including those
2520 that are masked with other bits (because those may or may not be
2521 interpolated over later, and we want to remove all NaN/infs).
2522 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2523 preserve the historical name.
2525 maskedImage = exposure.getMaskedImage()
2528 maskedImage.getMask().addMaskPlane(
"UNMASKEDNAN")
2529 maskVal = maskedImage.getMask().getPlaneBitMask(
"UNMASKEDNAN")
2530 numNans =
maskNans(maskedImage, maskVal)
2531 self.metadata[
"NUMNANS"] = numNans
2533 self.log.
warning(
"There were %d unmasked NaNs.", numNans)
2536 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2541 exposure : `lsst.afw.image.Exposure`
2542 Exposure to process.
2546 lsst.ip.isr.isrTask.maskNan
2549 isrFunctions.interpolateFromMask(
2550 maskedImage=exposure.getMaskedImage(),
2551 fwhm=self.config.fwhm,
2552 growSaturatedFootprints=0,
2553 maskNameList=[
"UNMASKEDNAN"],
2557 """Measure the image background in subgrids, for quality control.
2561 exposure : `lsst.afw.image.Exposure`
2562 Exposure to process.
2563 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2564 Configuration object containing parameters on which background
2565 statistics and subgrids to use.
2567 if IsrQaConfig
is not None:
2569 IsrQaConfig.flatness.nIter)
2570 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask([
"BAD",
"SAT",
"DETECTED"])
2571 statsControl.setAndMask(maskVal)
2572 maskedImage = exposure.getMaskedImage()
2574 skyLevel = stats.getValue(afwMath.MEDIAN)
2575 skySigma = stats.getValue(afwMath.STDEVCLIP)
2576 self.log.
info(
"Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2577 metadata = exposure.getMetadata()
2578 metadata[
"SKYLEVEL"] = skyLevel
2579 metadata[
"SKYSIGMA"] = skySigma
2582 stat = afwMath.MEANCLIP
if IsrQaConfig.flatness.doClip
else afwMath.MEAN
2583 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2584 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2585 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2586 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2587 skyLevels = numpy.zeros((nX, nY))
2590 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2592 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2594 xLLC = xc - meshXHalf
2595 yLLC = yc - meshYHalf
2596 xURC = xc + meshXHalf - 1
2597 yURC = yc + meshYHalf - 1
2600 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2604 good = numpy.where(numpy.isfinite(skyLevels))
2605 skyMedian = numpy.median(skyLevels[good])
2606 flatness = (skyLevels[good] - skyMedian) / skyMedian
2607 flatness_rms = numpy.std(flatness)
2608 flatness_pp = flatness.max() - flatness.min()
if len(flatness) > 0
else numpy.nan
2610 self.log.
info(
"Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2611 self.log.
info(
"Sky flatness in %dx%d grids - pp: %f rms: %f.",
2612 nX, nY, flatness_pp, flatness_rms)
2614 metadata[
"FLATNESS_PP"] = float(flatness_pp)
2615 metadata[
"FLATNESS_RMS"] = float(flatness_rms)
2616 metadata[
"FLATNESS_NGRIDS"] =
'%dx%d' % (nX, nY)
2617 metadata[
"FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2618 metadata[
"FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2621 """Set an approximate magnitude zero point for the exposure.
2625 exposure : `lsst.afw.image.Exposure`
2626 Exposure to process.
2628 filterLabel = exposure.getFilterLabel()
2629 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2631 if physicalFilter
in self.config.fluxMag0T1:
2632 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2634 self.log.
warning(
"No rough magnitude zero point defined for filter %s.", physicalFilter)
2635 fluxMag0 = self.config.defaultFluxMag0T1
2637 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2639 self.log.
warning(
"Non-positive exposure time; skipping rough zero point.")
2642 self.log.
info(
"Setting rough magnitude zero point for filter %s: %f",
2643 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2647 """Set valid polygon as the intersection of fpPolygon and chip corners.
2651 ccdExposure : `lsst.afw.image.Exposure`
2652 Exposure to process.
2653 fpPolygon : `lsst.afw.geom.Polygon`
2654 Polygon in focal plane coordinates.
2657 ccd = ccdExposure.getDetector()
2658 fpCorners = ccd.getCorners(FOCAL_PLANE)
2659 ccdPolygon =
Polygon(fpCorners)
2662 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2665 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2666 validPolygon =
Polygon(ccdPoints)
2667 ccdExposure.getInfo().setValidPolygon(validPolygon)
2671 """Context manager that applies and removes flats and darks,
2672 if the task is configured to apply them.
2676 exp : `lsst.afw.image.Exposure`
2677 Exposure to process.
2678 flat : `lsst.afw.image.Exposure`
2679 Flat exposure the same size as ``exp``.
2680 dark : `lsst.afw.image.Exposure`, optional
2681 Dark exposure the same size as ``exp``.
2685 exp : `lsst.afw.image.Exposure`
2686 The flat and dark corrected exposure.
2688 if self.config.doDark
and dark
is not None:
2690 if self.config.doFlat:
2695 if self.config.doFlat:
2697 if self.config.doDark
and dark
is not None:
2701 """Utility function to examine ISR exposure at different stages.
2705 exposure : `lsst.afw.image.Exposure`
2708 State of processing to view.
2713 display.scale(
'asinh',
'zscale')
2714 display.mtv(exposure)
2715 prompt =
"Press Enter to continue [c]... "
2717 ans = input(prompt).lower()
2718 if ans
in (
"",
"c",):
2723 """A Detector-like object that supports returning gain and saturation level
2725 This is used when the input exposure does not have a detector.
2729 exposure : `lsst.afw.image.Exposure`
2730 Exposure to generate a fake amplifier for.
2731 config : `lsst.ip.isr.isrTaskConfig`
2732 Configuration to apply to the fake amplifier.
2736 self.
_bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2738 self.
_gain_gain = config.gain
2743 return self.
_bbox_bbox
2746 return self.
_bbox_bbox
2752 return self.
_gain_gain
2765 isr = pexConfig.ConfigurableField(target=IsrTask, doc=
"Instrument signature removal")
2769 """Task to wrap the default IsrTask to allow it to be retargeted.
2771 The standard IsrTask can be called directly from a command line
2772 program, but doing so removes the ability of the task to be
2773 retargeted. As most cameras override some set of the IsrTask
2774 methods, this would remove those data-specific methods in the
2775 output post-ISR images. This wrapping class fixes the issue,
2776 allowing identical post-ISR images to be generated by both the
2777 processCcd and isrTask code.
2779 ConfigClass = RunIsrConfig
2780 _DefaultName =
"runIsr"
2784 self.makeSubtask(
"isr")
2790 dataRef : `lsst.daf.persistence.ButlerDataRef`
2791 data reference of the detector data to be processed
2795 result : `pipeBase.Struct`
2796 Result struct with component:
2798 - exposure : `lsst.afw.image.Exposure`
2799 Post-ISR processed exposure.
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Represent a 2-dimensional array of bitmask pixels.
Pass parameters to a Statistics object.
An integer coordinate rectangle.
def getRawHorizontalOverscanBBox(self)
def getSuspectLevel(self)
_RawHorizontalOverscanBBox
def __init__(self, exposure, config)
doSaturationInterpolation
def __init__(self, *config=None)
def flatCorrection(self, exposure, flatExposure, invert=False)
def maskAndInterpolateNan(self, exposure)
def saturationInterpolation(self, exposure)
def runDataRef(self, sensorRef)
def maskNan(self, exposure)
def maskAmplifier(self, ccdExposure, amp, defects)
def debugView(self, exposure, stepname)
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
def maskNegativeVariance(self, exposure)
def saturationDetection(self, exposure, amp)
def maskDefect(self, exposure, defectBaseList)
def __init__(self, **kwargs)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
def overscanCorrection(self, ccdExposure, amp)
def measureBackground(self, exposure, IsrQaConfig=None)
def roughZeroPoint(self, exposure)
def maskAndInterpolateDefects(self, exposure, defectBaseList)
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
def readIsrData(self, dataRef, rawExposure)
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
def doLinearize(self, detector)
def flatContext(self, exp, flat, dark=None)
def convertIntToFloat(self, exposure)
def suspectDetection(self, exposure, amp)
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
def darkCorrection(self, exposure, darkExposure, invert=False)
def __init__(self, *args, **kwargs)
def runDataRef(self, dataRef)
daf::base::PropertyList * list
daf::base::PropertySet * set
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::shared_ptr< PhotoCalib > makePhotoCalibFromCalibZeroPoint(double instFluxMag0, double instFluxMag0Err)
Construct a PhotoCalib from the deprecated Calib-style instFluxMag0/instFluxMag0Err values.
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
A function to return an Exposure of the correct type (cf.
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
A function to return a MaskedImage of the correct type (cf.
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
def getDebugFrame(debugDisplay, name)