LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.utils.timer import timeMethod
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 from .defects import Defects
48 
49 from .assembleCcdTask import AssembleCcdTask
50 from .crosstalk import CrosstalkTask, CrosstalkCalib
51 from .fringe import FringeTask
52 from .isr import maskNans
53 from .masking import MaskingTask
54 from .overscan import OverscanCorrectionTask
55 from .straylight import StrayLightTask
56 from .vignette import VignetteTask
57 from .ampOffset import AmpOffsetTask
58 from lsst.daf.butler import DimensionGraph
59 
60 
61 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62 
63 
64 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
65  """Lookup function to identify crosstalkSource entries.
66 
67  This should return an empty list under most circumstances. Only
68  when inter-chip crosstalk has been identified should this be
69  populated.
70 
71  Parameters
72  ----------
73  datasetType : `str`
74  Dataset to lookup.
75  registry : `lsst.daf.butler.Registry`
76  Butler registry to query.
77  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
78  Data id to transform to identify crosstalkSources. The
79  ``detector`` entry will be stripped.
80  collections : `lsst.daf.butler.CollectionSearch`
81  Collections to search through.
82 
83  Returns
84  -------
85  results : `list` [`lsst.daf.butler.DatasetRef`]
86  List of datasets that match the query that will be used as
87  crosstalkSources.
88  """
89  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
90  results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
91  findFirst=True))
92  # In some contexts, calling `.expanded()` to expand all data IDs in the
93  # query results can be a lot faster because it vectorizes lookups. But in
94  # this case, expandDataId shouldn't need to hit the database at all in the
95  # steady state, because only the detector record is unknown and those are
96  # cached in the registry.
97  return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
98 
99 
100 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
101  dimensions={"instrument", "exposure", "detector"},
102  defaultTemplates={}):
103  ccdExposure = cT.Input(
104  name="raw",
105  doc="Input exposure to process.",
106  storageClass="Exposure",
107  dimensions=["instrument", "exposure", "detector"],
108  )
109  camera = cT.PrerequisiteInput(
110  name="camera",
111  storageClass="Camera",
112  doc="Input camera to construct complete exposures.",
113  dimensions=["instrument"],
114  isCalibration=True,
115  )
116 
117  crosstalk = cT.PrerequisiteInput(
118  name="crosstalk",
119  doc="Input crosstalk object",
120  storageClass="CrosstalkCalib",
121  dimensions=["instrument", "detector"],
122  isCalibration=True,
123  minimum=0, # can fall back to cameraGeom
124  )
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  minimum=0, # not needed for all instruments, no config to control this
134  )
135  bias = cT.PrerequisiteInput(
136  name="bias",
137  doc="Input bias calibration.",
138  storageClass="ExposureF",
139  dimensions=["instrument", "detector"],
140  isCalibration=True,
141  )
142  dark = cT.PrerequisiteInput(
143  name='dark',
144  doc="Input dark calibration.",
145  storageClass="ExposureF",
146  dimensions=["instrument", "detector"],
147  isCalibration=True,
148  )
149  flat = cT.PrerequisiteInput(
150  name="flat",
151  doc="Input flat calibration.",
152  storageClass="ExposureF",
153  dimensions=["instrument", "physical_filter", "detector"],
154  isCalibration=True,
155  )
156  ptc = cT.PrerequisiteInput(
157  name="ptc",
158  doc="Input Photon Transfer Curve dataset",
159  storageClass="PhotonTransferCurveDataset",
160  dimensions=["instrument", "detector"],
161  isCalibration=True,
162  )
163  fringes = cT.PrerequisiteInput(
164  name="fringe",
165  doc="Input fringe calibration.",
166  storageClass="ExposureF",
167  dimensions=["instrument", "physical_filter", "detector"],
168  isCalibration=True,
169  minimum=0, # only needed for some bands, even when enabled
170  )
171  strayLightData = cT.PrerequisiteInput(
172  name='yBackground',
173  doc="Input stray light calibration.",
174  storageClass="StrayLightData",
175  dimensions=["instrument", "physical_filter", "detector"],
176  deferLoad=True,
177  isCalibration=True,
178  minimum=0, # only needed for some bands, even when enabled
179  )
180  bfKernel = cT.PrerequisiteInput(
181  name='bfKernel',
182  doc="Input brighter-fatter kernel.",
183  storageClass="NumpyArray",
184  dimensions=["instrument"],
185  isCalibration=True,
186  minimum=0, # can use either bfKernel or newBFKernel
187  )
188  newBFKernel = cT.PrerequisiteInput(
189  name='brighterFatterKernel',
190  doc="Newer complete kernel + gain solutions.",
191  storageClass="BrighterFatterKernel",
192  dimensions=["instrument", "detector"],
193  isCalibration=True,
194  minimum=0, # can use either bfKernel or newBFKernel
195  )
196  defects = cT.PrerequisiteInput(
197  name='defects',
198  doc="Input defect tables.",
199  storageClass="Defects",
200  dimensions=["instrument", "detector"],
201  isCalibration=True,
202  )
203  linearizer = cT.PrerequisiteInput(
204  name='linearizer',
205  storageClass="Linearizer",
206  doc="Linearity correction calibration.",
207  dimensions=["instrument", "detector"],
208  isCalibration=True,
209  minimum=0, # can fall back to cameraGeom
210  )
211  opticsTransmission = cT.PrerequisiteInput(
212  name="transmission_optics",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the optics.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  filterTransmission = cT.PrerequisiteInput(
219  name="transmission_filter",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the filter.",
222  dimensions=["instrument", "physical_filter"],
223  isCalibration=True,
224  )
225  sensorTransmission = cT.PrerequisiteInput(
226  name="transmission_sensor",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the sensor.",
229  dimensions=["instrument", "detector"],
230  isCalibration=True,
231  )
232  atmosphereTransmission = cT.PrerequisiteInput(
233  name="transmission_atmosphere",
234  storageClass="TransmissionCurve",
235  doc="Transmission curve due to the atmosphere.",
236  dimensions=["instrument"],
237  isCalibration=True,
238  )
239  illumMaskedImage = cT.PrerequisiteInput(
240  name="illum",
241  doc="Input illumination correction.",
242  storageClass="MaskedImageF",
243  dimensions=["instrument", "physical_filter", "detector"],
244  isCalibration=True,
245  )
246 
247  outputExposure = cT.Output(
248  name='postISRCCD',
249  doc="Output ISR processed exposure.",
250  storageClass="Exposure",
251  dimensions=["instrument", "exposure", "detector"],
252  )
253  preInterpExposure = cT.Output(
254  name='preInterpISRCCD',
255  doc="Output ISR processed exposure, with pixels left uninterpolated.",
256  storageClass="ExposureF",
257  dimensions=["instrument", "exposure", "detector"],
258  )
259  outputOssThumbnail = cT.Output(
260  name="OssThumb",
261  doc="Output Overscan-subtracted thumbnail image.",
262  storageClass="Thumbnail",
263  dimensions=["instrument", "exposure", "detector"],
264  )
265  outputFlattenedThumbnail = cT.Output(
266  name="FlattenedThumb",
267  doc="Output flat-corrected thumbnail image.",
268  storageClass="Thumbnail",
269  dimensions=["instrument", "exposure", "detector"],
270  )
271 
272  def __init__(self, *, config=None):
273  super().__init__(config=config)
274 
275  if config.doBias is not True:
276  self.prerequisiteInputs.discard("bias")
277  if config.doLinearize is not True:
278  self.prerequisiteInputs.discard("linearizer")
279  if config.doCrosstalk is not True:
280  self.prerequisiteInputs.discard("crosstalkSources")
281  self.prerequisiteInputs.discard("crosstalk")
282  if config.doBrighterFatter is not True:
283  self.prerequisiteInputs.discard("bfKernel")
284  self.prerequisiteInputs.discard("newBFKernel")
285  if config.doDefect is not True:
286  self.prerequisiteInputs.discard("defects")
287  if config.doDark is not True:
288  self.prerequisiteInputs.discard("dark")
289  if config.doFlat is not True:
290  self.prerequisiteInputs.discard("flat")
291  if config.doFringe is not True:
292  self.prerequisiteInputs.discard("fringe")
293  if config.doStrayLight is not True:
294  self.prerequisiteInputs.discard("strayLightData")
295  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296  self.prerequisiteInputs.discard("ptc")
297  if config.doAttachTransmissionCurve is not True:
298  self.prerequisiteInputs.discard("opticsTransmission")
299  self.prerequisiteInputs.discard("filterTransmission")
300  self.prerequisiteInputs.discard("sensorTransmission")
301  self.prerequisiteInputs.discard("atmosphereTransmission")
302  if config.doUseOpticsTransmission is not True:
303  self.prerequisiteInputs.discard("opticsTransmission")
304  if config.doUseFilterTransmission is not True:
305  self.prerequisiteInputs.discard("filterTransmission")
306  if config.doUseSensorTransmission is not True:
307  self.prerequisiteInputs.discard("sensorTransmission")
308  if config.doUseAtmosphereTransmission is not True:
309  self.prerequisiteInputs.discard("atmosphereTransmission")
310  if config.doIlluminationCorrection is not True:
311  self.prerequisiteInputs.discard("illumMaskedImage")
312 
313  if config.doWrite is not True:
314  self.outputs.discard("outputExposure")
315  self.outputs.discard("preInterpExposure")
316  self.outputs.discard("outputFlattenedThumbnail")
317  self.outputs.discard("outputOssThumbnail")
318  if config.doSaveInterpPixels is not True:
319  self.outputs.discard("preInterpExposure")
320  if config.qa.doThumbnailOss is not True:
321  self.outputs.discard("outputOssThumbnail")
322  if config.qa.doThumbnailFlattened is not True:
323  self.outputs.discard("outputFlattenedThumbnail")
324 
325 
326 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327  pipelineConnections=IsrTaskConnections):
328  """Configuration parameters for IsrTask.
329 
330  Items are grouped in the order in which they are executed by the task.
331  """
332  datasetType = pexConfig.Field(
333  dtype=str,
334  doc="Dataset type for input data; users will typically leave this alone, "
335  "but camera-specific ISR tasks will override it",
336  default="raw",
337  )
338 
339  fallbackFilterName = pexConfig.Field(
340  dtype=str,
341  doc="Fallback default filter name for calibrations.",
342  optional=True
343  )
344  useFallbackDate = pexConfig.Field(
345  dtype=bool,
346  doc="Pass observation date when using fallback filter.",
347  default=False,
348  )
349  expectWcs = pexConfig.Field(
350  dtype=bool,
351  default=True,
352  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353  )
354  fwhm = pexConfig.Field(
355  dtype=float,
356  doc="FWHM of PSF in arcseconds.",
357  default=1.0,
358  )
359  qa = pexConfig.ConfigField(
360  dtype=isrQa.IsrQaConfig,
361  doc="QA related configuration options.",
362  )
363 
364  # Image conversion configuration
365  doConvertIntToFloat = pexConfig.Field(
366  dtype=bool,
367  doc="Convert integer raw images to floating point values?",
368  default=True,
369  )
370 
371  # Saturated pixel handling.
372  doSaturation = pexConfig.Field(
373  dtype=bool,
374  doc="Mask saturated pixels? NB: this is totally independent of the"
375  " interpolation option - this is ONLY setting the bits in the mask."
376  " To have them interpolated make sure doSaturationInterpolation=True",
377  default=True,
378  )
379  saturatedMaskName = pexConfig.Field(
380  dtype=str,
381  doc="Name of mask plane to use in saturation detection and interpolation",
382  default="SAT",
383  )
384  saturation = pexConfig.Field(
385  dtype=float,
386  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387  default=float("NaN"),
388  )
389  growSaturationFootprintSize = pexConfig.Field(
390  dtype=int,
391  doc="Number of pixels by which to grow the saturation footprints",
392  default=1,
393  )
394 
395  # Suspect pixel handling.
396  doSuspect = pexConfig.Field(
397  dtype=bool,
398  doc="Mask suspect pixels?",
399  default=False,
400  )
401  suspectMaskName = pexConfig.Field(
402  dtype=str,
403  doc="Name of mask plane to use for suspect pixels",
404  default="SUSPECT",
405  )
406  numEdgeSuspect = pexConfig.Field(
407  dtype=int,
408  doc="Number of edge pixels to be flagged as untrustworthy.",
409  default=0,
410  )
411  edgeMaskLevel = pexConfig.ChoiceField(
412  dtype=str,
413  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414  default="DETECTOR",
415  allowed={
416  'DETECTOR': 'Mask only the edges of the full detector.',
417  'AMP': 'Mask edges of each amplifier.',
418  },
419  )
420 
421  # Initial masking options.
422  doSetBadRegions = pexConfig.Field(
423  dtype=bool,
424  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425  default=True,
426  )
427  badStatistic = pexConfig.ChoiceField(
428  dtype=str,
429  doc="How to estimate the average value for BAD regions.",
430  default='MEANCLIP',
431  allowed={
432  "MEANCLIP": "Correct using the (clipped) mean of good data",
433  "MEDIAN": "Correct using the median of the good data",
434  },
435  )
436 
437  # Overscan subtraction configuration.
438  doOverscan = pexConfig.Field(
439  dtype=bool,
440  doc="Do overscan subtraction?",
441  default=True,
442  )
443  overscan = pexConfig.ConfigurableField(
444  target=OverscanCorrectionTask,
445  doc="Overscan subtraction task for image segments.",
446  )
447  overscanFitType = pexConfig.ChoiceField(
448  dtype=str,
449  doc="The method for fitting the overscan bias level.",
450  default='MEDIAN',
451  allowed={
452  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
453  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
454  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
455  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
456  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
457  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
458  "MEAN": "Correct using the mean of the overscan region",
459  "MEANCLIP": "Correct using a clipped mean of the overscan region",
460  "MEDIAN": "Correct using the median of the overscan region",
461  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
462  },
463  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
464  " This option will no longer be used, and will be removed after v20.")
465  )
466  overscanOrder = pexConfig.Field(
467  dtype=int,
468  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
469  "or number of spline knots if overscan fit type is a spline."),
470  default=1,
471  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
472  " This option will no longer be used, and will be removed after v20.")
473  )
474  overscanNumSigmaClip = pexConfig.Field(
475  dtype=float,
476  doc="Rejection threshold (sigma) for collapsing overscan before fit",
477  default=3.0,
478  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
479  " This option will no longer be used, and will be removed after v20.")
480  )
481  overscanIsInt = pexConfig.Field(
482  dtype=bool,
483  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
484  " and overscan.FitType=MEDIAN_PER_ROW.",
485  default=True,
486  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
487  " This option will no longer be used, and will be removed after v20.")
488  )
489  # These options do not get deprecated, as they define how we slice up the
490  # image data.
491  overscanNumLeadingColumnsToSkip = pexConfig.Field(
492  dtype=int,
493  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494  default=0,
495  )
496  overscanNumTrailingColumnsToSkip = pexConfig.Field(
497  dtype=int,
498  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499  default=0,
500  )
501  overscanMaxDev = pexConfig.Field(
502  dtype=float,
503  doc="Maximum deviation from the median for overscan",
504  default=1000.0, check=lambda x: x > 0
505  )
506  overscanBiasJump = pexConfig.Field(
507  dtype=bool,
508  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509  default=False,
510  )
511  overscanBiasJumpKeyword = pexConfig.Field(
512  dtype=str,
513  doc="Header keyword containing information about devices.",
514  default="NO_SUCH_KEY",
515  )
516  overscanBiasJumpDevices = pexConfig.ListField(
517  dtype=str,
518  doc="List of devices that need piecewise overscan correction.",
519  default=(),
520  )
521  overscanBiasJumpLocation = pexConfig.Field(
522  dtype=int,
523  doc="Location of bias jump along y-axis.",
524  default=0,
525  )
526 
527  # Amplifier to CCD assembly configuration
528  doAssembleCcd = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Assemble amp-level exposures into a ccd-level exposure?"
532  )
533  assembleCcd = pexConfig.ConfigurableField(
534  target=AssembleCcdTask,
535  doc="CCD assembly task",
536  )
537 
538  # General calibration configuration.
539  doAssembleIsrExposures = pexConfig.Field(
540  dtype=bool,
541  default=False,
542  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543  )
544  doTrimToMatchCalib = pexConfig.Field(
545  dtype=bool,
546  default=False,
547  doc="Trim raw data to match calibration bounding boxes?"
548  )
549 
550  # Bias subtraction.
551  doBias = pexConfig.Field(
552  dtype=bool,
553  doc="Apply bias frame correction?",
554  default=True,
555  )
556  biasDataProductName = pexConfig.Field(
557  dtype=str,
558  doc="Name of the bias data product",
559  default="bias",
560  )
561  doBiasBeforeOverscan = pexConfig.Field(
562  dtype=bool,
563  doc="Reverse order of overscan and bias correction.",
564  default=False
565  )
566 
567  # Variance construction
568  doVariance = pexConfig.Field(
569  dtype=bool,
570  doc="Calculate variance?",
571  default=True
572  )
573  gain = pexConfig.Field(
574  dtype=float,
575  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576  default=float("NaN"),
577  )
578  readNoise = pexConfig.Field(
579  dtype=float,
580  doc="The read noise to use if no Detector is present in the Exposure",
581  default=0.0,
582  )
583  doEmpiricalReadNoise = pexConfig.Field(
584  dtype=bool,
585  default=False,
586  doc="Calculate empirical read noise instead of value from AmpInfo data?"
587  )
588  usePtcReadNoise = pexConfig.Field(
589  dtype=bool,
590  default=False,
591  doc="Use readnoise values from the Photon Transfer Curve?"
592  )
593  maskNegativeVariance = pexConfig.Field(
594  dtype=bool,
595  default=True,
596  doc="Mask pixels that claim a negative variance? This likely indicates a failure "
597  "in the measurement of the overscan at an edge due to the data falling off faster "
598  "than the overscan model can account for it."
599  )
600  negativeVarianceMaskName = pexConfig.Field(
601  dtype=str,
602  default="BAD",
603  doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
604  )
605  # Linearization.
606  doLinearize = pexConfig.Field(
607  dtype=bool,
608  doc="Correct for nonlinearity of the detector's response?",
609  default=True,
610  )
611 
612  # Crosstalk.
613  doCrosstalk = pexConfig.Field(
614  dtype=bool,
615  doc="Apply intra-CCD crosstalk correction?",
616  default=False,
617  )
618  doCrosstalkBeforeAssemble = pexConfig.Field(
619  dtype=bool,
620  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
621  default=False,
622  )
623  crosstalk = pexConfig.ConfigurableField(
624  target=CrosstalkTask,
625  doc="Intra-CCD crosstalk correction",
626  )
627 
628  # Masking options.
629  doDefect = pexConfig.Field(
630  dtype=bool,
631  doc="Apply correction for CCD defects, e.g. hot pixels?",
632  default=True,
633  )
634  doNanMasking = pexConfig.Field(
635  dtype=bool,
636  doc="Mask non-finite (NAN, inf) pixels?",
637  default=True,
638  )
639  doWidenSaturationTrails = pexConfig.Field(
640  dtype=bool,
641  doc="Widen bleed trails based on their width?",
642  default=True
643  )
644 
645  # Brighter-Fatter correction.
646  doBrighterFatter = pexConfig.Field(
647  dtype=bool,
648  default=False,
649  doc="Apply the brighter-fatter correction?"
650  )
651  brighterFatterLevel = pexConfig.ChoiceField(
652  dtype=str,
653  default="DETECTOR",
654  doc="The level at which to correct for brighter-fatter.",
655  allowed={
656  "AMP": "Every amplifier treated separately.",
657  "DETECTOR": "One kernel per detector",
658  }
659  )
660  brighterFatterMaxIter = pexConfig.Field(
661  dtype=int,
662  default=10,
663  doc="Maximum number of iterations for the brighter-fatter correction"
664  )
665  brighterFatterThreshold = pexConfig.Field(
666  dtype=float,
667  default=1000,
668  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
669  "absolute value of the difference between the current corrected image and the one "
670  "from the previous iteration summed over all the pixels."
671  )
672  brighterFatterApplyGain = pexConfig.Field(
673  dtype=bool,
674  default=True,
675  doc="Should the gain be applied when applying the brighter-fatter correction?"
676  )
677  brighterFatterMaskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
680  "correction.",
681  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
682  )
683  brighterFatterMaskGrowSize = pexConfig.Field(
684  dtype=int,
685  default=0,
686  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
687  "when brighter-fatter correction is applied."
688  )
689 
690  # Dark subtraction.
691  doDark = pexConfig.Field(
692  dtype=bool,
693  doc="Apply dark frame correction?",
694  default=True,
695  )
696  darkDataProductName = pexConfig.Field(
697  dtype=str,
698  doc="Name of the dark data product",
699  default="dark",
700  )
701 
702  # Camera-specific stray light removal.
703  doStrayLight = pexConfig.Field(
704  dtype=bool,
705  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
706  default=False,
707  )
708  strayLight = pexConfig.ConfigurableField(
709  target=StrayLightTask,
710  doc="y-band stray light correction"
711  )
712 
713  # Flat correction.
714  doFlat = pexConfig.Field(
715  dtype=bool,
716  doc="Apply flat field correction?",
717  default=True,
718  )
719  flatDataProductName = pexConfig.Field(
720  dtype=str,
721  doc="Name of the flat data product",
722  default="flat",
723  )
724  flatScalingType = pexConfig.ChoiceField(
725  dtype=str,
726  doc="The method for scaling the flat on the fly.",
727  default='USER',
728  allowed={
729  "USER": "Scale by flatUserScale",
730  "MEAN": "Scale by the inverse of the mean",
731  "MEDIAN": "Scale by the inverse of the median",
732  },
733  )
734  flatUserScale = pexConfig.Field(
735  dtype=float,
736  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
737  default=1.0,
738  )
739  doTweakFlat = pexConfig.Field(
740  dtype=bool,
741  doc="Tweak flats to match observed amplifier ratios?",
742  default=False
743  )
744 
745  # Amplifier normalization based on gains instead of using flats
746  # configuration.
747  doApplyGains = pexConfig.Field(
748  dtype=bool,
749  doc="Correct the amplifiers for their gains instead of applying flat correction",
750  default=False,
751  )
752  usePtcGains = pexConfig.Field(
753  dtype=bool,
754  doc="Use the gain values from the Photon Transfer Curve?",
755  default=False,
756  )
757  normalizeGains = pexConfig.Field(
758  dtype=bool,
759  doc="Normalize all the amplifiers in each CCD to have the same median value.",
760  default=False,
761  )
762 
763  # Fringe correction.
764  doFringe = pexConfig.Field(
765  dtype=bool,
766  doc="Apply fringe correction?",
767  default=True,
768  )
769  fringe = pexConfig.ConfigurableField(
770  target=FringeTask,
771  doc="Fringe subtraction task",
772  )
773  fringeAfterFlat = pexConfig.Field(
774  dtype=bool,
775  doc="Do fringe subtraction after flat-fielding?",
776  default=True,
777  )
778 
779  # Amp offset correction.
780  doAmpOffset = pexConfig.Field(
781  doc="Calculate and apply amp offset corrections?",
782  dtype=bool,
783  default=False,
784  )
785  ampOffset = pexConfig.ConfigurableField(
786  doc="Amp offset correction task.",
787  target=AmpOffsetTask,
788  )
789 
790  # Initial CCD-level background statistics options.
791  doMeasureBackground = pexConfig.Field(
792  dtype=bool,
793  doc="Measure the background level on the reduced image?",
794  default=False,
795  )
796 
797  # Camera-specific masking configuration.
798  doCameraSpecificMasking = pexConfig.Field(
799  dtype=bool,
800  doc="Mask camera-specific bad regions?",
801  default=False,
802  )
803  masking = pexConfig.ConfigurableField(
804  target=MaskingTask,
805  doc="Masking task."
806  )
807 
808  # Interpolation options.
809  doInterpolate = pexConfig.Field(
810  dtype=bool,
811  doc="Interpolate masked pixels?",
812  default=True,
813  )
814  doSaturationInterpolation = pexConfig.Field(
815  dtype=bool,
816  doc="Perform interpolation over pixels masked as saturated?"
817  " NB: This is independent of doSaturation; if that is False this plane"
818  " will likely be blank, resulting in a no-op here.",
819  default=True,
820  )
821  doNanInterpolation = pexConfig.Field(
822  dtype=bool,
823  doc="Perform interpolation over pixels masked as NaN?"
824  " NB: This is independent of doNanMasking; if that is False this plane"
825  " will likely be blank, resulting in a no-op here.",
826  default=True,
827  )
828  doNanInterpAfterFlat = pexConfig.Field(
829  dtype=bool,
830  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
831  "also have to interpolate them before flat-fielding."),
832  default=False,
833  )
834  maskListToInterpolate = pexConfig.ListField(
835  dtype=str,
836  doc="List of mask planes that should be interpolated.",
837  default=['SAT', 'BAD'],
838  )
839  doSaveInterpPixels = pexConfig.Field(
840  dtype=bool,
841  doc="Save a copy of the pre-interpolated pixel values?",
842  default=False,
843  )
844 
845  # Default photometric calibration options.
846  fluxMag0T1 = pexConfig.DictField(
847  keytype=str,
848  itemtype=float,
849  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
850  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
851  ))
852  )
853  defaultFluxMag0T1 = pexConfig.Field(
854  dtype=float,
855  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
856  default=pow(10.0, 0.4*28.0)
857  )
858 
859  # Vignette correction configuration.
860  doVignette = pexConfig.Field(
861  dtype=bool,
862  doc="Apply vignetting parameters?",
863  default=False,
864  )
865  vignette = pexConfig.ConfigurableField(
866  target=VignetteTask,
867  doc="Vignetting task.",
868  )
869 
870  # Transmission curve configuration.
871  doAttachTransmissionCurve = pexConfig.Field(
872  dtype=bool,
873  default=False,
874  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
875  )
876  doUseOpticsTransmission = pexConfig.Field(
877  dtype=bool,
878  default=True,
879  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
880  )
881  doUseFilterTransmission = pexConfig.Field(
882  dtype=bool,
883  default=True,
884  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
885  )
886  doUseSensorTransmission = pexConfig.Field(
887  dtype=bool,
888  default=True,
889  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
890  )
891  doUseAtmosphereTransmission = pexConfig.Field(
892  dtype=bool,
893  default=True,
894  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
895  )
896 
897  # Illumination correction.
898  doIlluminationCorrection = pexConfig.Field(
899  dtype=bool,
900  default=False,
901  doc="Perform illumination correction?"
902  )
903  illuminationCorrectionDataProductName = pexConfig.Field(
904  dtype=str,
905  doc="Name of the illumination correction data product.",
906  default="illumcor",
907  )
908  illumScale = pexConfig.Field(
909  dtype=float,
910  doc="Scale factor for the illumination correction.",
911  default=1.0,
912  )
913  illumFilters = pexConfig.ListField(
914  dtype=str,
915  default=[],
916  doc="Only perform illumination correction for these filters."
917  )
918 
919  # Write the outputs to disk. If ISR is run as a subtask, this may not
920  # be needed.
921  doWrite = pexConfig.Field(
922  dtype=bool,
923  doc="Persist postISRCCD?",
924  default=True,
925  )
926 
927  def validate(self):
928  super().validate()
929  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
930  raise ValueError("You may not specify both doFlat and doApplyGains")
931  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
932  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
933  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
934  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
935  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
936  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
937  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
938  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
939 
940 
941 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
942  """Apply common instrument signature correction algorithms to a raw frame.
943 
944  The process for correcting imaging data is very similar from
945  camera to camera. This task provides a vanilla implementation of
946  doing these corrections, including the ability to turn certain
947  corrections off if they are not needed. The inputs to the primary
948  method, `run()`, are a raw exposure to be corrected and the
949  calibration data products. The raw input is a single chip sized
950  mosaic of all amps including overscans and other non-science
951  pixels. The method `runDataRef()` identifies and defines the
952  calibration data products, and is intended for use by a
953  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
954  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
955  subclassed for different camera, although the most camera specific
956  methods have been split into subtasks that can be redirected
957  appropriately.
958 
959  The __init__ method sets up the subtasks for ISR processing, using
960  the defaults from `lsst.ip.isr`.
961 
962  Parameters
963  ----------
964  args : `list`
965  Positional arguments passed to the Task constructor.
966  None used at this time.
967  kwargs : `dict`, optional
968  Keyword arguments passed on to the Task constructor.
969  None used at this time.
970  """
971  ConfigClass = IsrTaskConfig
972  _DefaultName = "isr"
973 
974  def __init__(self, **kwargs):
975  super().__init__(**kwargs)
976  self.makeSubtask("assembleCcd")
977  self.makeSubtask("crosstalk")
978  self.makeSubtask("strayLight")
979  self.makeSubtask("fringe")
980  self.makeSubtask("masking")
981  self.makeSubtask("overscan")
982  self.makeSubtask("vignette")
983  self.makeSubtask("ampOffset")
984 
985  def runQuantum(self, butlerQC, inputRefs, outputRefs):
986  inputs = butlerQC.get(inputRefs)
987 
988  try:
989  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
990  except Exception as e:
991  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
992  (inputRefs, e))
993 
994  inputs['isGen3'] = True
995 
996  detector = inputs['ccdExposure'].getDetector()
997 
998  if self.config.doCrosstalk is True:
999  # Crosstalk sources need to be defined by the pipeline
1000  # yaml if they exist.
1001  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
1002  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
1003  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
1004  else:
1005  coeffVector = (self.config.crosstalk.crosstalkValues
1006  if self.config.crosstalk.useConfigCoefficients else None)
1007  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1008  inputs['crosstalk'] = crosstalkCalib
1009  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
1010  if 'crosstalkSources' not in inputs:
1011  self.log.warning("No crosstalkSources found for chip with interChip terms!")
1012 
1013  if self.doLinearizedoLinearize(detector) is True:
1014  if 'linearizer' in inputs:
1015  if isinstance(inputs['linearizer'], dict):
1016  linearizer = linearize.Linearizer(detector=detector, log=self.log)
1017  linearizer.fromYaml(inputs['linearizer'])
1018  self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1019  elif isinstance(inputs['linearizer'], numpy.ndarray):
1020  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1021  detector=detector,
1022  log=self.log)
1023  self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1024  else:
1025  linearizer = inputs['linearizer']
1026  linearizer.log = self.log
1027  inputs['linearizer'] = linearizer
1028  else:
1029  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1030  self.log.warning("Constructing linearizer from cameraGeom information.")
1031 
1032  if self.config.doDefect is True:
1033  if "defects" in inputs and inputs['defects'] is not None:
1034  # defects is loaded as a BaseCatalog with columns
1035  # x0, y0, width, height. Masking expects a list of defects
1036  # defined by their bounding box
1037  if not isinstance(inputs["defects"], Defects):
1038  inputs["defects"] = Defects.fromTable(inputs["defects"])
1039 
1040  # Load the correct style of brighter-fatter kernel, and repack
1041  # the information as a numpy array.
1042  if self.config.doBrighterFatter:
1043  brighterFatterKernel = inputs.pop('newBFKernel', None)
1044  if brighterFatterKernel is None:
1045  brighterFatterKernel = inputs.get('bfKernel', None)
1046 
1047  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1048  # This is a ISR calib kernel
1049  detName = detector.getName()
1050  level = brighterFatterKernel.level
1051 
1052  # This is expected to be a dictionary of amp-wise gains.
1053  inputs['bfGains'] = brighterFatterKernel.gain
1054  if self.config.brighterFatterLevel == 'DETECTOR':
1055  if level == 'DETECTOR':
1056  if detName in brighterFatterKernel.detKernels:
1057  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1058  else:
1059  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1060  elif level == 'AMP':
1061  self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1062  "fatter kernels.")
1063  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1064  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1065  elif self.config.brighterFatterLevel == 'AMP':
1066  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1067 
1068  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1069  expId = inputs['ccdExposure'].info.id
1070  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1071  expId=expId,
1072  assembler=self.assembleCcd
1073  if self.config.doAssembleIsrExposures else None)
1074  else:
1075  inputs['fringes'] = pipeBase.Struct(fringes=None)
1076 
1077  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1078  if 'strayLightData' not in inputs:
1079  inputs['strayLightData'] = None
1080 
1081  outputs = self.runrun(**inputs)
1082  butlerQC.put(outputs, outputRefs)
1083 
1084  def readIsrData(self, dataRef, rawExposure):
1085  """Retrieve necessary frames for instrument signature removal.
1086 
1087  Pre-fetching all required ISR data products limits the IO
1088  required by the ISR. Any conflict between the calibration data
1089  available and that needed for ISR is also detected prior to
1090  doing processing, allowing it to fail quickly.
1091 
1092  Parameters
1093  ----------
1094  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1095  Butler reference of the detector data to be processed
1096  rawExposure : `afw.image.Exposure`
1097  The raw exposure that will later be corrected with the
1098  retrieved calibration data; should not be modified in this
1099  method.
1100 
1101  Returns
1102  -------
1103  result : `lsst.pipe.base.Struct`
1104  Result struct with components (which may be `None`):
1105  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1106  - ``linearizer``: functor for linearization
1107  (`ip.isr.linearize.LinearizeBase`)
1108  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1109  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1110  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1111  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1112  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1113  - ``fringes``: `lsst.pipe.base.Struct` with components:
1114  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1115  - ``seed``: random seed derived from the ccdExposureId for random
1116  number generator (`uint32`).
1117  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1118  A ``TransmissionCurve`` that represents the throughput of the
1119  optics, to be evaluated in focal-plane coordinates.
1120  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1121  A ``TransmissionCurve`` that represents the throughput of the
1122  filter itself, to be evaluated in focal-plane coordinates.
1123  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1124  A ``TransmissionCurve`` that represents the throughput of the
1125  sensor itself, to be evaluated in post-assembly trimmed
1126  detector coordinates.
1127  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1128  A ``TransmissionCurve`` that represents the throughput of the
1129  atmosphere, assumed to be spatially constant.
1130  - ``strayLightData`` : `object`
1131  An opaque object containing calibration information for
1132  stray-light correction. If `None`, no correction will be
1133  performed.
1134  - ``illumMaskedImage`` : illumination correction image
1135  (`lsst.afw.image.MaskedImage`)
1136 
1137  Raises
1138  ------
1139  NotImplementedError :
1140  Raised if a per-amplifier brighter-fatter kernel is requested by
1141  the configuration.
1142  """
1143  try:
1144  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1145  dateObs = dateObs.toPython().isoformat()
1146  except RuntimeError:
1147  self.log.warning("Unable to identify dateObs for rawExposure.")
1148  dateObs = None
1149 
1150  ccd = rawExposure.getDetector()
1151  filterLabel = rawExposure.getFilterLabel()
1152  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1153  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1154  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1155  if self.config.doBias else None)
1156  # immediate=True required for functors and linearizers are functors
1157  # see ticket DM-6515
1158  linearizer = (dataRef.get("linearizer", immediate=True)
1159  if self.doLinearizedoLinearize(ccd) else None)
1160  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1161  linearizer.log = self.log
1162  if isinstance(linearizer, numpy.ndarray):
1163  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1164 
1165  crosstalkCalib = None
1166  if self.config.doCrosstalk:
1167  try:
1168  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1169  except NoResults:
1170  coeffVector = (self.config.crosstalk.crosstalkValues
1171  if self.config.crosstalk.useConfigCoefficients else None)
1172  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1173  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1174  if self.config.doCrosstalk else None)
1175 
1176  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1177  if self.config.doDark else None)
1178  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1179  dateObs=dateObs)
1180  if self.config.doFlat else None)
1181 
1182  brighterFatterKernel = None
1183  brighterFatterGains = None
1184  if self.config.doBrighterFatter is True:
1185  try:
1186  # Use the new-style cp_pipe version of the kernel if it exists
1187  # If using a new-style kernel, always use the self-consistent
1188  # gains, i.e. the ones inside the kernel object itself
1189  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1190  brighterFatterGains = brighterFatterKernel.gain
1191  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1192  except NoResults:
1193  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1194  brighterFatterKernel = dataRef.get("bfKernel")
1195  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1196  except NoResults:
1197  brighterFatterKernel = None
1198  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1199  # If the kernel is not an ndarray, it's the cp_pipe version
1200  # so extract the kernel for this detector, or raise an error
1201  if self.config.brighterFatterLevel == 'DETECTOR':
1202  if brighterFatterKernel.detKernels:
1203  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1204  else:
1205  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1206  else:
1207  # TODO DM-15631 for implementing this
1208  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1209 
1210  defectList = (dataRef.get("defects")
1211  if self.config.doDefect else None)
1212  expId = rawExposure.info.id
1213  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1214  if self.config.doAssembleIsrExposures else None)
1215  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1216  else pipeBase.Struct(fringes=None))
1217 
1218  if self.config.doAttachTransmissionCurve:
1219  opticsTransmission = (dataRef.get("transmission_optics")
1220  if self.config.doUseOpticsTransmission else None)
1221  filterTransmission = (dataRef.get("transmission_filter")
1222  if self.config.doUseFilterTransmission else None)
1223  sensorTransmission = (dataRef.get("transmission_sensor")
1224  if self.config.doUseSensorTransmission else None)
1225  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1226  if self.config.doUseAtmosphereTransmission else None)
1227  else:
1228  opticsTransmission = None
1229  filterTransmission = None
1230  sensorTransmission = None
1231  atmosphereTransmission = None
1232 
1233  if self.config.doStrayLight:
1234  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1235  else:
1236  strayLightData = None
1237 
1238  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1239  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1240  if (self.config.doIlluminationCorrection
1241  and physicalFilter in self.config.illumFilters)
1242  else None)
1243 
1244  # Struct should include only kwargs to run()
1245  return pipeBase.Struct(bias=biasExposure,
1246  linearizer=linearizer,
1247  crosstalk=crosstalkCalib,
1248  crosstalkSources=crosstalkSources,
1249  dark=darkExposure,
1250  flat=flatExposure,
1251  bfKernel=brighterFatterKernel,
1252  bfGains=brighterFatterGains,
1253  defects=defectList,
1254  fringes=fringeStruct,
1255  opticsTransmission=opticsTransmission,
1256  filterTransmission=filterTransmission,
1257  sensorTransmission=sensorTransmission,
1258  atmosphereTransmission=atmosphereTransmission,
1259  strayLightData=strayLightData,
1260  illumMaskedImage=illumMaskedImage
1261  )
1262 
1263  @timeMethod
1264  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1265  crosstalk=None, crosstalkSources=None,
1266  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1267  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1268  sensorTransmission=None, atmosphereTransmission=None,
1269  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1270  isGen3=False,
1271  ):
1272  """Perform instrument signature removal on an exposure.
1273 
1274  Steps included in the ISR processing, in order performed, are:
1275  - saturation and suspect pixel masking
1276  - overscan subtraction
1277  - CCD assembly of individual amplifiers
1278  - bias subtraction
1279  - variance image construction
1280  - linearization of non-linear response
1281  - crosstalk masking
1282  - brighter-fatter correction
1283  - dark subtraction
1284  - fringe correction
1285  - stray light subtraction
1286  - flat correction
1287  - masking of known defects and camera specific features
1288  - vignette calculation
1289  - appending transmission curve and distortion model
1290 
1291  Parameters
1292  ----------
1293  ccdExposure : `lsst.afw.image.Exposure`
1294  The raw exposure that is to be run through ISR. The
1295  exposure is modified by this method.
1296  camera : `lsst.afw.cameraGeom.Camera`, optional
1297  The camera geometry for this exposure. Required if
1298  one or more of ``ccdExposure``, ``bias``, ``dark``, or
1299  ``flat`` does not have an associated detector.
1300  bias : `lsst.afw.image.Exposure`, optional
1301  Bias calibration frame.
1302  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1303  Functor for linearization.
1304  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1305  Calibration for crosstalk.
1306  crosstalkSources : `list`, optional
1307  List of possible crosstalk sources.
1308  dark : `lsst.afw.image.Exposure`, optional
1309  Dark calibration frame.
1310  flat : `lsst.afw.image.Exposure`, optional
1311  Flat calibration frame.
1312  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1313  Photon transfer curve dataset, with, e.g., gains
1314  and read noise.
1315  bfKernel : `numpy.ndarray`, optional
1316  Brighter-fatter kernel.
1317  bfGains : `dict` of `float`, optional
1318  Gains used to override the detector's nominal gains for the
1319  brighter-fatter correction. A dict keyed by amplifier name for
1320  the detector in question.
1321  defects : `lsst.ip.isr.Defects`, optional
1322  List of defects.
1323  fringes : `lsst.pipe.base.Struct`, optional
1324  Struct containing the fringe correction data, with
1325  elements:
1326  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1327  - ``seed``: random seed derived from the ccdExposureId for random
1328  number generator (`uint32`)
1329  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1330  A ``TransmissionCurve`` that represents the throughput of the,
1331  optics, to be evaluated in focal-plane coordinates.
1332  filterTransmission : `lsst.afw.image.TransmissionCurve`
1333  A ``TransmissionCurve`` that represents the throughput of the
1334  filter itself, to be evaluated in focal-plane coordinates.
1335  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1336  A ``TransmissionCurve`` that represents the throughput of the
1337  sensor itself, to be evaluated in post-assembly trimmed detector
1338  coordinates.
1339  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1340  A ``TransmissionCurve`` that represents the throughput of the
1341  atmosphere, assumed to be spatially constant.
1342  detectorNum : `int`, optional
1343  The integer number for the detector to process.
1344  isGen3 : bool, optional
1345  Flag this call to run() as using the Gen3 butler environment.
1346  strayLightData : `object`, optional
1347  Opaque object containing calibration information for stray-light
1348  correction. If `None`, no correction will be performed.
1349  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1350  Illumination correction image.
1351 
1352  Returns
1353  -------
1354  result : `lsst.pipe.base.Struct`
1355  Result struct with component:
1356  - ``exposure`` : `afw.image.Exposure`
1357  The fully ISR corrected exposure.
1358  - ``outputExposure`` : `afw.image.Exposure`
1359  An alias for `exposure`
1360  - ``ossThumb`` : `numpy.ndarray`
1361  Thumbnail image of the exposure after overscan subtraction.
1362  - ``flattenedThumb`` : `numpy.ndarray`
1363  Thumbnail image of the exposure after flat-field correction.
1364 
1365  Raises
1366  ------
1367  RuntimeError
1368  Raised if a configuration option is set to True, but the
1369  required calibration data has not been specified.
1370 
1371  Notes
1372  -----
1373  The current processed exposure can be viewed by setting the
1374  appropriate lsstDebug entries in the `debug.display`
1375  dictionary. The names of these entries correspond to some of
1376  the IsrTaskConfig Boolean options, with the value denoting the
1377  frame to use. The exposure is shown inside the matching
1378  option check and after the processing of that step has
1379  finished. The steps with debug points are:
1380 
1381  doAssembleCcd
1382  doBias
1383  doCrosstalk
1384  doBrighterFatter
1385  doDark
1386  doFringe
1387  doStrayLight
1388  doFlat
1389 
1390  In addition, setting the "postISRCCD" entry displays the
1391  exposure after all ISR processing has finished.
1392 
1393  """
1394 
1395  if isGen3 is True:
1396  # Gen3 currently cannot automatically do configuration overrides.
1397  # DM-15257 looks to discuss this issue.
1398  # Configure input exposures;
1399 
1400  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1401  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1402  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1403  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1404  else:
1405  if isinstance(ccdExposure, ButlerDataRef):
1406  return self.runDataRefrunDataRef(ccdExposure)
1407 
1408  ccd = ccdExposure.getDetector()
1409  filterLabel = ccdExposure.getFilterLabel()
1410  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1411 
1412  if not ccd:
1413  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1414  ccd = [FakeAmp(ccdExposure, self.config)]
1415 
1416  # Validate Input
1417  if self.config.doBias and bias is None:
1418  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1419  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1420  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1421  if self.config.doBrighterFatter and bfKernel is None:
1422  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1423  if self.config.doDark and dark is None:
1424  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1425  if self.config.doFlat and flat is None:
1426  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1427  if self.config.doDefect and defects is None:
1428  raise RuntimeError("Must supply defects if config.doDefect=True.")
1429  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1430  and fringes.fringes is None):
1431  # The `fringes` object needs to be a pipeBase.Struct, as
1432  # we use it as a `dict` for the parameters of
1433  # `FringeTask.run()`. The `fringes.fringes` `list` may
1434  # not be `None` if `doFringe=True`. Otherwise, raise.
1435  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1436  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1437  and illumMaskedImage is None):
1438  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1439 
1440  # Begin ISR processing.
1441  if self.config.doConvertIntToFloat:
1442  self.log.info("Converting exposure to floating point values.")
1443  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1444 
1445  if self.config.doBias and self.config.doBiasBeforeOverscan:
1446  self.log.info("Applying bias correction.")
1447  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1448  trimToFit=self.config.doTrimToMatchCalib)
1449  self.debugViewdebugView(ccdExposure, "doBias")
1450 
1451  # Amplifier level processing.
1452  overscans = []
1453  for amp in ccd:
1454  # if ccdExposure is one amp,
1455  # check for coverage to prevent performing ops multiple times
1456  if ccdExposure.getBBox().contains(amp.getBBox()):
1457  # Check for fully masked bad amplifiers,
1458  # and generate masks for SUSPECT and SATURATED values.
1459  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1460 
1461  if self.config.doOverscan and not badAmp:
1462  # Overscan correction on amp-by-amp basis.
1463  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1464  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1465  if overscanResults is not None and \
1466  self.config.qa is not None and self.config.qa.saveStats is True:
1467  if isinstance(overscanResults.overscanFit, float):
1468  qaMedian = overscanResults.overscanFit
1469  qaStdev = float("NaN")
1470  else:
1471  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1472  afwMath.MEDIAN | afwMath.STDEVCLIP)
1473  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1474  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1475 
1476  self.metadata[f"FIT MEDIAN {amp.getName()}"] = qaMedian
1477  self.metadata[f"FIT STDEV {amp.getName()}"] = qaStdev
1478  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1479  amp.getName(), qaMedian, qaStdev)
1480 
1481  # Residuals after overscan correction
1482  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1483  afwMath.MEDIAN | afwMath.STDEVCLIP)
1484  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1485  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1486 
1487  self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1488  self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1489  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1490  amp.getName(), qaMedianAfter, qaStdevAfter)
1491 
1492  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1493  else:
1494  if badAmp:
1495  self.log.warning("Amplifier %s is bad.", amp.getName())
1496  overscanResults = None
1497 
1498  overscans.append(overscanResults if overscanResults is not None else None)
1499  else:
1500  self.log.info("Skipped OSCAN for %s.", amp.getName())
1501 
1502  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1503  self.log.info("Applying crosstalk correction.")
1504  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1505  crosstalkSources=crosstalkSources, camera=camera)
1506  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1507 
1508  if self.config.doAssembleCcd:
1509  self.log.info("Assembling CCD from amplifiers.")
1510  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1511 
1512  if self.config.expectWcs and not ccdExposure.getWcs():
1513  self.log.warning("No WCS found in input exposure.")
1514  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1515 
1516  ossThumb = None
1517  if self.config.qa.doThumbnailOss:
1518  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1519 
1520  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1521  self.log.info("Applying bias correction.")
1522  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1523  trimToFit=self.config.doTrimToMatchCalib)
1524  self.debugViewdebugView(ccdExposure, "doBias")
1525 
1526  if self.config.doVariance:
1527  for amp, overscanResults in zip(ccd, overscans):
1528  if ccdExposure.getBBox().contains(amp.getBBox()):
1529  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1530  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1531  if overscanResults is not None:
1532  self.updateVarianceupdateVariance(ampExposure, amp,
1533  overscanImage=overscanResults.overscanImage,
1534  ptcDataset=ptc)
1535  else:
1536  self.updateVarianceupdateVariance(ampExposure, amp,
1537  overscanImage=None,
1538  ptcDataset=ptc)
1539  if self.config.qa is not None and self.config.qa.saveStats is True:
1540  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1541  afwMath.MEDIAN | afwMath.STDEVCLIP)
1542  self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1543  qaStats.getValue(afwMath.MEDIAN)
1544  self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \
1545  qaStats.getValue(afwMath.STDEVCLIP)
1546  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1547  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1548  qaStats.getValue(afwMath.STDEVCLIP))
1549  if self.config.maskNegativeVariance:
1550  self.maskNegativeVariancemaskNegativeVariance(ccdExposure)
1551 
1552  if self.doLinearizedoLinearize(ccd):
1553  self.log.info("Applying linearizer.")
1554  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1555  detector=ccd, log=self.log)
1556 
1557  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1558  self.log.info("Applying crosstalk correction.")
1559  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1560  crosstalkSources=crosstalkSources, isTrimmed=True)
1561  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1562 
1563  # Masking block. Optionally mask known defects, NAN/inf pixels,
1564  # widen trails, and do anything else the camera needs. Saturated and
1565  # suspect pixels have already been masked.
1566  if self.config.doDefect:
1567  self.log.info("Masking defects.")
1568  self.maskDefectmaskDefect(ccdExposure, defects)
1569 
1570  if self.config.numEdgeSuspect > 0:
1571  self.log.info("Masking edges as SUSPECT.")
1572  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1573  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1574 
1575  if self.config.doNanMasking:
1576  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1577  self.maskNanmaskNan(ccdExposure)
1578 
1579  if self.config.doWidenSaturationTrails:
1580  self.log.info("Widening saturation trails.")
1581  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1582 
1583  if self.config.doCameraSpecificMasking:
1584  self.log.info("Masking regions for camera specific reasons.")
1585  self.masking.run(ccdExposure)
1586 
1587  if self.config.doBrighterFatter:
1588  # We need to apply flats and darks before we can interpolate, and
1589  # we need to interpolate before we do B-F, but we do B-F without
1590  # the flats and darks applied so we can work in units of electrons
1591  # or holes. This context manager applies and then removes the darks
1592  # and flats.
1593  #
1594  # We also do not want to interpolate values here, so operate on
1595  # temporary images so we can apply only the BF-correction and roll
1596  # back the interpolation.
1597  interpExp = ccdExposure.clone()
1598  with self.flatContextflatContext(interpExp, flat, dark):
1599  isrFunctions.interpolateFromMask(
1600  maskedImage=interpExp.getMaskedImage(),
1601  fwhm=self.config.fwhm,
1602  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1603  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1604  )
1605  bfExp = interpExp.clone()
1606 
1607  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1608  type(bfKernel), type(bfGains))
1609  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1610  self.config.brighterFatterMaxIter,
1611  self.config.brighterFatterThreshold,
1612  self.config.brighterFatterApplyGain,
1613  bfGains)
1614  if bfResults[1] == self.config.brighterFatterMaxIter:
1615  self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1616  bfResults[0])
1617  else:
1618  self.log.info("Finished brighter-fatter correction in %d iterations.",
1619  bfResults[1])
1620  image = ccdExposure.getMaskedImage().getImage()
1621  bfCorr = bfExp.getMaskedImage().getImage()
1622  bfCorr -= interpExp.getMaskedImage().getImage()
1623  image += bfCorr
1624 
1625  # Applying the brighter-fatter correction applies a
1626  # convolution to the science image. At the edges this
1627  # convolution may not have sufficient valid pixels to
1628  # produce a valid correction. Mark pixels within the size
1629  # of the brighter-fatter kernel as EDGE to warn of this
1630  # fact.
1631  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1632  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1633  maskPlane="EDGE")
1634 
1635  if self.config.brighterFatterMaskGrowSize > 0:
1636  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1637  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1638  isrFunctions.growMasks(ccdExposure.getMask(),
1639  radius=self.config.brighterFatterMaskGrowSize,
1640  maskNameList=maskPlane,
1641  maskValue=maskPlane)
1642 
1643  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1644 
1645  if self.config.doDark:
1646  self.log.info("Applying dark correction.")
1647  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1648  self.debugViewdebugView(ccdExposure, "doDark")
1649 
1650  if self.config.doFringe and not self.config.fringeAfterFlat:
1651  self.log.info("Applying fringe correction before flat.")
1652  self.fringe.run(ccdExposure, **fringes.getDict())
1653  self.debugViewdebugView(ccdExposure, "doFringe")
1654 
1655  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1656  self.log.info("Checking strayLight correction.")
1657  self.strayLight.run(ccdExposure, strayLightData)
1658  self.debugViewdebugView(ccdExposure, "doStrayLight")
1659 
1660  if self.config.doFlat:
1661  self.log.info("Applying flat correction.")
1662  self.flatCorrectionflatCorrection(ccdExposure, flat)
1663  self.debugViewdebugView(ccdExposure, "doFlat")
1664 
1665  if self.config.doApplyGains:
1666  self.log.info("Applying gain correction instead of flat.")
1667  if self.config.usePtcGains:
1668  self.log.info("Using gains from the Photon Transfer Curve.")
1669  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1670  ptcGains=ptc.gain)
1671  else:
1672  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1673 
1674  if self.config.doFringe and self.config.fringeAfterFlat:
1675  self.log.info("Applying fringe correction after flat.")
1676  self.fringe.run(ccdExposure, **fringes.getDict())
1677 
1678  if self.config.doVignette:
1679  self.log.info("Constructing Vignette polygon.")
1680  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1681 
1682  if self.config.vignette.doWriteVignettePolygon:
1683  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1684 
1685  if self.config.doAttachTransmissionCurve:
1686  self.log.info("Adding transmission curves.")
1687  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1688  filterTransmission=filterTransmission,
1689  sensorTransmission=sensorTransmission,
1690  atmosphereTransmission=atmosphereTransmission)
1691 
1692  flattenedThumb = None
1693  if self.config.qa.doThumbnailFlattened:
1694  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1695 
1696  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1697  self.log.info("Performing illumination correction.")
1698  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1699  illumMaskedImage, illumScale=self.config.illumScale,
1700  trimToFit=self.config.doTrimToMatchCalib)
1701 
1702  preInterpExp = None
1703  if self.config.doSaveInterpPixels:
1704  preInterpExp = ccdExposure.clone()
1705 
1706  # Reset and interpolate bad pixels.
1707  #
1708  # Large contiguous bad regions (which should have the BAD mask
1709  # bit set) should have their values set to the image median.
1710  # This group should include defects and bad amplifiers. As the
1711  # area covered by these defects are large, there's little
1712  # reason to expect that interpolation would provide a more
1713  # useful value.
1714  #
1715  # Smaller defects can be safely interpolated after the larger
1716  # regions have had their pixel values reset. This ensures
1717  # that the remaining defects adjacent to bad amplifiers (as an
1718  # example) do not attempt to interpolate extreme values.
1719  if self.config.doSetBadRegions:
1720  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1721  if badPixelCount > 0:
1722  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1723 
1724  if self.config.doInterpolate:
1725  self.log.info("Interpolating masked pixels.")
1726  isrFunctions.interpolateFromMask(
1727  maskedImage=ccdExposure.getMaskedImage(),
1728  fwhm=self.config.fwhm,
1729  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1730  maskNameList=list(self.config.maskListToInterpolate)
1731  )
1732 
1733  self.roughZeroPointroughZeroPoint(ccdExposure)
1734 
1735  # correct for amp offsets within the CCD
1736  if self.config.doAmpOffset:
1737  self.log.info("Correcting amp offsets.")
1738  self.ampOffset.run(ccdExposure)
1739 
1740  if self.config.doMeasureBackground:
1741  self.log.info("Measuring background level.")
1742  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1743 
1744  if self.config.qa is not None and self.config.qa.saveStats is True:
1745  for amp in ccd:
1746  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1747  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1748  afwMath.MEDIAN | afwMath.STDEVCLIP)
1749  self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1750  self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \
1751  qaStats.getValue(afwMath.STDEVCLIP)
1752  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1753  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1754  qaStats.getValue(afwMath.STDEVCLIP))
1755 
1756  self.debugViewdebugView(ccdExposure, "postISRCCD")
1757 
1758  return pipeBase.Struct(
1759  exposure=ccdExposure,
1760  ossThumb=ossThumb,
1761  flattenedThumb=flattenedThumb,
1762 
1763  preInterpExposure=preInterpExp,
1764  outputExposure=ccdExposure,
1765  outputOssThumbnail=ossThumb,
1766  outputFlattenedThumbnail=flattenedThumb,
1767  )
1768 
1769  @timeMethod
1770  def runDataRef(self, sensorRef):
1771  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1772 
1773  This method contains the `CmdLineTask` interface to the ISR
1774  processing. All IO is handled here, freeing the `run()` method
1775  to manage only pixel-level calculations. The steps performed
1776  are:
1777  - Read in necessary detrending/isr/calibration data.
1778  - Process raw exposure in `run()`.
1779  - Persist the ISR-corrected exposure as "postISRCCD" if
1780  config.doWrite=True.
1781 
1782  Parameters
1783  ----------
1784  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1785  DataRef of the detector data to be processed
1786 
1787  Returns
1788  -------
1789  result : `lsst.pipe.base.Struct`
1790  Result struct with component:
1791  - ``exposure`` : `afw.image.Exposure`
1792  The fully ISR corrected exposure.
1793 
1794  Raises
1795  ------
1796  RuntimeError
1797  Raised if a configuration option is set to True, but the
1798  required calibration data does not exist.
1799 
1800  """
1801  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1802 
1803  ccdExposure = sensorRef.get(self.config.datasetType)
1804 
1805  camera = sensorRef.get("camera")
1806  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1807 
1808  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1809 
1810  if self.config.doWrite:
1811  sensorRef.put(result.exposure, "postISRCCD")
1812  if result.preInterpExposure is not None:
1813  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1814  if result.ossThumb is not None:
1815  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1816  if result.flattenedThumb is not None:
1817  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1818 
1819  return result
1820 
1821  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1822  """Retrieve a calibration dataset for removing instrument signature.
1823 
1824  Parameters
1825  ----------
1826 
1827  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1828  DataRef of the detector data to find calibration datasets
1829  for.
1830  datasetType : `str`
1831  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1832  dateObs : `str`, optional
1833  Date of the observation. Used to correct butler failures
1834  when using fallback filters.
1835  immediate : `Bool`
1836  If True, disable butler proxies to enable error handling
1837  within this routine.
1838 
1839  Returns
1840  -------
1841  exposure : `lsst.afw.image.Exposure`
1842  Requested calibration frame.
1843 
1844  Raises
1845  ------
1846  RuntimeError
1847  Raised if no matching calibration frame can be found.
1848  """
1849  try:
1850  exp = dataRef.get(datasetType, immediate=immediate)
1851  except Exception as exc1:
1852  if not self.config.fallbackFilterName:
1853  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1854  try:
1855  if self.config.useFallbackDate and dateObs:
1856  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1857  dateObs=dateObs, immediate=immediate)
1858  else:
1859  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1860  except Exception as exc2:
1861  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1862  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1863  self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1864 
1865  if self.config.doAssembleIsrExposures:
1866  exp = self.assembleCcd.assembleCcd(exp)
1867  return exp
1868 
1869  def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1870  """Ensure that the data returned by Butler is a fully constructed exp.
1871 
1872  ISR requires exposure-level image data for historical reasons, so if we
1873  did not recieve that from Butler, construct it from what we have,
1874  modifying the input in place.
1875 
1876  Parameters
1877  ----------
1878  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1879  or `lsst.afw.image.ImageF`
1880  The input data structure obtained from Butler.
1881  camera : `lsst.afw.cameraGeom.camera`, optional
1882  The camera associated with the image. Used to find the appropriate
1883  detector if detector is not already set.
1884  detectorNum : `int`, optional
1885  The detector in the camera to attach, if the detector is not
1886  already set.
1887 
1888  Returns
1889  -------
1890  inputExp : `lsst.afw.image.Exposure`
1891  The re-constructed exposure, with appropriate detector parameters.
1892 
1893  Raises
1894  ------
1895  TypeError
1896  Raised if the input data cannot be used to construct an exposure.
1897  """
1898  if isinstance(inputExp, afwImage.DecoratedImageU):
1899  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1900  elif isinstance(inputExp, afwImage.ImageF):
1901  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1902  elif isinstance(inputExp, afwImage.MaskedImageF):
1903  inputExp = afwImage.makeExposure(inputExp)
1904  elif isinstance(inputExp, afwImage.Exposure):
1905  pass
1906  elif inputExp is None:
1907  # Assume this will be caught by the setup if it is a problem.
1908  return inputExp
1909  else:
1910  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1911  (type(inputExp), ))
1912 
1913  if inputExp.getDetector() is None:
1914  if camera is None or detectorNum is None:
1915  raise RuntimeError('Must supply both a camera and detector number when using exposures '
1916  'without a detector set.')
1917  inputExp.setDetector(camera[detectorNum])
1918 
1919  return inputExp
1920 
1921  def convertIntToFloat(self, exposure):
1922  """Convert exposure image from uint16 to float.
1923 
1924  If the exposure does not need to be converted, the input is
1925  immediately returned. For exposures that are converted to use
1926  floating point pixels, the variance is set to unity and the
1927  mask to zero.
1928 
1929  Parameters
1930  ----------
1931  exposure : `lsst.afw.image.Exposure`
1932  The raw exposure to be converted.
1933 
1934  Returns
1935  -------
1936  newexposure : `lsst.afw.image.Exposure`
1937  The input ``exposure``, converted to floating point pixels.
1938 
1939  Raises
1940  ------
1941  RuntimeError
1942  Raised if the exposure type cannot be converted to float.
1943 
1944  """
1945  if isinstance(exposure, afwImage.ExposureF):
1946  # Nothing to be done
1947  self.log.debug("Exposure already of type float.")
1948  return exposure
1949  if not hasattr(exposure, "convertF"):
1950  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1951 
1952  newexposure = exposure.convertF()
1953  newexposure.variance[:] = 1
1954  newexposure.mask[:] = 0x0
1955 
1956  return newexposure
1957 
1958  def maskAmplifier(self, ccdExposure, amp, defects):
1959  """Identify bad amplifiers, saturated and suspect pixels.
1960 
1961  Parameters
1962  ----------
1963  ccdExposure : `lsst.afw.image.Exposure`
1964  Input exposure to be masked.
1965  amp : `lsst.afw.table.AmpInfoCatalog`
1966  Catalog of parameters defining the amplifier on this
1967  exposure to mask.
1968  defects : `lsst.ip.isr.Defects`
1969  List of defects. Used to determine if the entire
1970  amplifier is bad.
1971 
1972  Returns
1973  -------
1974  badAmp : `Bool`
1975  If this is true, the entire amplifier area is covered by
1976  defects and unusable.
1977 
1978  """
1979  maskedImage = ccdExposure.getMaskedImage()
1980 
1981  badAmp = False
1982 
1983  # Check if entire amp region is defined as a defect
1984  # NB: need to use amp.getBBox() for correct comparison with current
1985  # defects definition.
1986  if defects is not None:
1987  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1988 
1989  # In the case of a bad amp, we will set mask to "BAD"
1990  # (here use amp.getRawBBox() for correct association with pixels in
1991  # current ccdExposure).
1992  if badAmp:
1993  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1994  afwImage.PARENT)
1995  maskView = dataView.getMask()
1996  maskView |= maskView.getPlaneBitMask("BAD")
1997  del maskView
1998  return badAmp
1999 
2000  # Mask remaining defects after assembleCcd() to allow for defects that
2001  # cross amplifier boundaries. Saturation and suspect pixels can be
2002  # masked now, though.
2003  limits = dict()
2004  if self.config.doSaturation and not badAmp:
2005  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2006  if self.config.doSuspect and not badAmp:
2007  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2008  if math.isfinite(self.config.saturation):
2009  limits.update({self.config.saturatedMaskName: self.config.saturation})
2010 
2011  for maskName, maskThreshold in limits.items():
2012  if not math.isnan(maskThreshold):
2013  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2014  isrFunctions.makeThresholdMask(
2015  maskedImage=dataView,
2016  threshold=maskThreshold,
2017  growFootprints=0,
2018  maskName=maskName
2019  )
2020 
2021  # Determine if we've fully masked this amplifier with SUSPECT and
2022  # SAT pixels.
2023  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2024  afwImage.PARENT)
2025  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2026  self.config.suspectMaskName])
2027  if numpy.all(maskView.getArray() & maskVal > 0):
2028  badAmp = True
2029  maskView |= maskView.getPlaneBitMask("BAD")
2030 
2031  return badAmp
2032 
2033  def overscanCorrection(self, ccdExposure, amp):
2034  """Apply overscan correction in place.
2035 
2036  This method does initial pixel rejection of the overscan
2037  region. The overscan can also be optionally segmented to
2038  allow for discontinuous overscan responses to be fit
2039  separately. The actual overscan subtraction is performed by
2040  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2041  which is called here after the amplifier is preprocessed.
2042 
2043  Parameters
2044  ----------
2045  ccdExposure : `lsst.afw.image.Exposure`
2046  Exposure to have overscan correction performed.
2047  amp : `lsst.afw.cameraGeom.Amplifer`
2048  The amplifier to consider while correcting the overscan.
2049 
2050  Returns
2051  -------
2052  overscanResults : `lsst.pipe.base.Struct`
2053  Result struct with components:
2054  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2055  Value or fit subtracted from the amplifier image data.
2056  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2057  Value or fit subtracted from the overscan image data.
2058  - ``overscanImage`` : `lsst.afw.image.Image`
2059  Image of the overscan region with the overscan
2060  correction applied. This quantity is used to estimate
2061  the amplifier read noise empirically.
2062 
2063  Raises
2064  ------
2065  RuntimeError
2066  Raised if the ``amp`` does not contain raw pixel information.
2067 
2068  See Also
2069  --------
2070  lsst.ip.isr.isrFunctions.overscanCorrection
2071  """
2072  if amp.getRawHorizontalOverscanBBox().isEmpty():
2073  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2074  return None
2075 
2076  statControl = afwMath.StatisticsControl()
2077  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2078 
2079  # Determine the bounding boxes
2080  dataBBox = amp.getRawDataBBox()
2081  oscanBBox = amp.getRawHorizontalOverscanBBox()
2082  dx0 = 0
2083  dx1 = 0
2084 
2085  prescanBBox = amp.getRawPrescanBBox()
2086  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2087  dx0 += self.config.overscanNumLeadingColumnsToSkip
2088  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2089  else:
2090  dx0 += self.config.overscanNumTrailingColumnsToSkip
2091  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2092 
2093  # Determine if we need to work on subregions of the amplifier
2094  # and overscan.
2095  imageBBoxes = []
2096  overscanBBoxes = []
2097 
2098  if ((self.config.overscanBiasJump
2099  and self.config.overscanBiasJumpLocation)
2100  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2101  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2102  self.config.overscanBiasJumpDevices)):
2103  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2104  yLower = self.config.overscanBiasJumpLocation
2105  yUpper = dataBBox.getHeight() - yLower
2106  else:
2107  yUpper = self.config.overscanBiasJumpLocation
2108  yLower = dataBBox.getHeight() - yUpper
2109 
2110  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2111  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2112  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2113  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2114  yLower)))
2115 
2116  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2117  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2118  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2119  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2120  yUpper)))
2121  else:
2122  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2123  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2124  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2125  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2126  oscanBBox.getHeight())))
2127 
2128  # Perform overscan correction on subregions, ensuring saturated
2129  # pixels are masked.
2130  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2131  ampImage = ccdExposure.maskedImage[imageBBox]
2132  overscanImage = ccdExposure.maskedImage[overscanBBox]
2133 
2134  overscanArray = overscanImage.image.array
2135  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2136  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2137  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2138 
2139  statControl = afwMath.StatisticsControl()
2140  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2141 
2142  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2143 
2144  # Measure average overscan levels and record them in the metadata.
2145  levelStat = afwMath.MEDIAN
2146  sigmaStat = afwMath.STDEVCLIP
2147 
2148  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2149  self.config.qa.flatness.nIter)
2150  metadata = ccdExposure.getMetadata()
2151  ampNum = amp.getName()
2152  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2153  if isinstance(overscanResults.overscanFit, float):
2154  metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2155  metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2156  else:
2157  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2158  metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2159  metadata[f"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2160 
2161  return overscanResults
2162 
2163  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2164  """Set the variance plane using the gain and read noise
2165 
2166  The read noise is calculated from the ``overscanImage`` if the
2167  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2168  the value from the amplifier data is used.
2169 
2170  Parameters
2171  ----------
2172  ampExposure : `lsst.afw.image.Exposure`
2173  Exposure to process.
2174  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2175  Amplifier detector data.
2176  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2177  Image of overscan, required only for empirical read noise.
2178  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2179  PTC dataset containing the gains and read noise.
2180 
2181 
2182  Raises
2183  ------
2184  RuntimeError
2185  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2186  are ``True``, but ptcDataset is not provided.
2187 
2188  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2189  ``overscanImage`` is ``None``.
2190 
2191  See also
2192  --------
2193  lsst.ip.isr.isrFunctions.updateVariance
2194  """
2195  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2196  if self.config.usePtcGains:
2197  if ptcDataset is None:
2198  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2199  else:
2200  gain = ptcDataset.gain[amp.getName()]
2201  self.log.info("Using gain from Photon Transfer Curve.")
2202  else:
2203  gain = amp.getGain()
2204 
2205  if math.isnan(gain):
2206  gain = 1.0
2207  self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2208  elif gain <= 0:
2209  patchedGain = 1.0
2210  self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2211  amp.getName(), gain, patchedGain)
2212  gain = patchedGain
2213 
2214  if self.config.doEmpiricalReadNoise and overscanImage is None:
2215  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2216 
2217  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2218  stats = afwMath.StatisticsControl()
2219  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2220  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2221  self.log.info("Calculated empirical read noise for amp %s: %f.",
2222  amp.getName(), readNoise)
2223  elif self.config.usePtcReadNoise:
2224  if ptcDataset is None:
2225  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2226  else:
2227  readNoise = ptcDataset.noise[amp.getName()]
2228  self.log.info("Using read noise from Photon Transfer Curve.")
2229  else:
2230  readNoise = amp.getReadNoise()
2231 
2232  isrFunctions.updateVariance(
2233  maskedImage=ampExposure.getMaskedImage(),
2234  gain=gain,
2235  readNoise=readNoise,
2236  )
2237 
2238  def maskNegativeVariance(self, exposure):
2239  """Identify and mask pixels with negative variance values.
2240 
2241  Parameters
2242  ----------
2243  exposure : `lsst.afw.image.Exposure`
2244  Exposure to process.
2245 
2246  See Also
2247  --------
2248  lsst.ip.isr.isrFunctions.updateVariance
2249  """
2250  maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2251  bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2252  exposure.mask.array[bad] |= maskPlane
2253 
2254  def darkCorrection(self, exposure, darkExposure, invert=False):
2255  """Apply dark correction in place.
2256 
2257  Parameters
2258  ----------
2259  exposure : `lsst.afw.image.Exposure`
2260  Exposure to process.
2261  darkExposure : `lsst.afw.image.Exposure`
2262  Dark exposure of the same size as ``exposure``.
2263  invert : `Bool`, optional
2264  If True, re-add the dark to an already corrected image.
2265 
2266  Raises
2267  ------
2268  RuntimeError
2269  Raised if either ``exposure`` or ``darkExposure`` do not
2270  have their dark time defined.
2271 
2272  See Also
2273  --------
2274  lsst.ip.isr.isrFunctions.darkCorrection
2275  """
2276  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2277  if math.isnan(expScale):
2278  raise RuntimeError("Exposure darktime is NAN.")
2279  if darkExposure.getInfo().getVisitInfo() is not None \
2280  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2281  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2282  else:
2283  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2284  # so getDarkTime() does not exist.
2285  self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2286  darkScale = 1.0
2287 
2288  isrFunctions.darkCorrection(
2289  maskedImage=exposure.getMaskedImage(),
2290  darkMaskedImage=darkExposure.getMaskedImage(),
2291  expScale=expScale,
2292  darkScale=darkScale,
2293  invert=invert,
2294  trimToFit=self.config.doTrimToMatchCalib
2295  )
2296 
2297  def doLinearize(self, detector):
2298  """Check if linearization is needed for the detector cameraGeom.
2299 
2300  Checks config.doLinearize and the linearity type of the first
2301  amplifier.
2302 
2303  Parameters
2304  ----------
2305  detector : `lsst.afw.cameraGeom.Detector`
2306  Detector to get linearity type from.
2307 
2308  Returns
2309  -------
2310  doLinearize : `Bool`
2311  If True, linearization should be performed.
2312  """
2313  return self.config.doLinearize and \
2314  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2315 
2316  def flatCorrection(self, exposure, flatExposure, invert=False):
2317  """Apply flat correction in place.
2318 
2319  Parameters
2320  ----------
2321  exposure : `lsst.afw.image.Exposure`
2322  Exposure to process.
2323  flatExposure : `lsst.afw.image.Exposure`
2324  Flat exposure of the same size as ``exposure``.
2325  invert : `Bool`, optional
2326  If True, unflatten an already flattened image.
2327 
2328  See Also
2329  --------
2330  lsst.ip.isr.isrFunctions.flatCorrection
2331  """
2332  isrFunctions.flatCorrection(
2333  maskedImage=exposure.getMaskedImage(),
2334  flatMaskedImage=flatExposure.getMaskedImage(),
2335  scalingType=self.config.flatScalingType,
2336  userScale=self.config.flatUserScale,
2337  invert=invert,
2338  trimToFit=self.config.doTrimToMatchCalib
2339  )
2340 
2341  def saturationDetection(self, exposure, amp):
2342  """Detect and mask saturated pixels in config.saturatedMaskName.
2343 
2344  Parameters
2345  ----------
2346  exposure : `lsst.afw.image.Exposure`
2347  Exposure to process. Only the amplifier DataSec is processed.
2348  amp : `lsst.afw.table.AmpInfoCatalog`
2349  Amplifier detector data.
2350 
2351  See Also
2352  --------
2353  lsst.ip.isr.isrFunctions.makeThresholdMask
2354  """
2355  if not math.isnan(amp.getSaturation()):
2356  maskedImage = exposure.getMaskedImage()
2357  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2358  isrFunctions.makeThresholdMask(
2359  maskedImage=dataView,
2360  threshold=amp.getSaturation(),
2361  growFootprints=0,
2362  maskName=self.config.saturatedMaskName,
2363  )
2364 
2365  def saturationInterpolation(self, exposure):
2366  """Interpolate over saturated pixels, in place.
2367 
2368  This method should be called after `saturationDetection`, to
2369  ensure that the saturated pixels have been identified in the
2370  SAT mask. It should also be called after `assembleCcd`, since
2371  saturated regions may cross amplifier boundaries.
2372 
2373  Parameters
2374  ----------
2375  exposure : `lsst.afw.image.Exposure`
2376  Exposure to process.
2377 
2378  See Also
2379  --------
2380  lsst.ip.isr.isrTask.saturationDetection
2381  lsst.ip.isr.isrFunctions.interpolateFromMask
2382  """
2383  isrFunctions.interpolateFromMask(
2384  maskedImage=exposure.getMaskedImage(),
2385  fwhm=self.config.fwhm,
2386  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2387  maskNameList=list(self.config.saturatedMaskName),
2388  )
2389 
2390  def suspectDetection(self, exposure, amp):
2391  """Detect and mask suspect pixels in config.suspectMaskName.
2392 
2393  Parameters
2394  ----------
2395  exposure : `lsst.afw.image.Exposure`
2396  Exposure to process. Only the amplifier DataSec is processed.
2397  amp : `lsst.afw.table.AmpInfoCatalog`
2398  Amplifier detector data.
2399 
2400  See Also
2401  --------
2402  lsst.ip.isr.isrFunctions.makeThresholdMask
2403 
2404  Notes
2405  -----
2406  Suspect pixels are pixels whose value is greater than
2407  amp.getSuspectLevel(). This is intended to indicate pixels that may be
2408  affected by unknown systematics; for example if non-linearity
2409  corrections above a certain level are unstable then that would be a
2410  useful value for suspectLevel. A value of `nan` indicates that no such
2411  level exists and no pixels are to be masked as suspicious.
2412  """
2413  suspectLevel = amp.getSuspectLevel()
2414  if math.isnan(suspectLevel):
2415  return
2416 
2417  maskedImage = exposure.getMaskedImage()
2418  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2419  isrFunctions.makeThresholdMask(
2420  maskedImage=dataView,
2421  threshold=suspectLevel,
2422  growFootprints=0,
2423  maskName=self.config.suspectMaskName,
2424  )
2425 
2426  def maskDefect(self, exposure, defectBaseList):
2427  """Mask defects using mask plane "BAD", in place.
2428 
2429  Parameters
2430  ----------
2431  exposure : `lsst.afw.image.Exposure`
2432  Exposure to process.
2433  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2434  `lsst.afw.image.DefectBase`.
2435  List of defects to mask.
2436 
2437  Notes
2438  -----
2439  Call this after CCD assembly, since defects may cross amplifier
2440  boundaries.
2441  """
2442  maskedImage = exposure.getMaskedImage()
2443  if not isinstance(defectBaseList, Defects):
2444  # Promotes DefectBase to Defect
2445  defectList = Defects(defectBaseList)
2446  else:
2447  defectList = defectBaseList
2448  defectList.maskPixels(maskedImage, maskName="BAD")
2449 
2450  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2451  """Mask edge pixels with applicable mask plane.
2452 
2453  Parameters
2454  ----------
2455  exposure : `lsst.afw.image.Exposure`
2456  Exposure to process.
2457  numEdgePixels : `int`, optional
2458  Number of edge pixels to mask.
2459  maskPlane : `str`, optional
2460  Mask plane name to use.
2461  level : `str`, optional
2462  Level at which to mask edges.
2463  """
2464  maskedImage = exposure.getMaskedImage()
2465  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2466 
2467  if numEdgePixels > 0:
2468  if level == 'DETECTOR':
2469  boxes = [maskedImage.getBBox()]
2470  elif level == 'AMP':
2471  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2472 
2473  for box in boxes:
2474  # This makes a bbox numEdgeSuspect pixels smaller than the
2475  # image on each side
2476  subImage = maskedImage[box]
2477  box.grow(-numEdgePixels)
2478  # Mask pixels outside box
2479  SourceDetectionTask.setEdgeBits(
2480  subImage,
2481  box,
2482  maskBitMask)
2483 
2484  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2485  """Mask and interpolate defects using mask plane "BAD", in place.
2486 
2487  Parameters
2488  ----------
2489  exposure : `lsst.afw.image.Exposure`
2490  Exposure to process.
2491  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2492  `lsst.afw.image.DefectBase`.
2493  List of defects to mask and interpolate.
2494 
2495  See Also
2496  --------
2497  lsst.ip.isr.isrTask.maskDefect
2498  """
2499  self.maskDefectmaskDefect(exposure, defectBaseList)
2500  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2501  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2502  isrFunctions.interpolateFromMask(
2503  maskedImage=exposure.getMaskedImage(),
2504  fwhm=self.config.fwhm,
2505  growSaturatedFootprints=0,
2506  maskNameList=["BAD"],
2507  )
2508 
2509  def maskNan(self, exposure):
2510  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2511 
2512  Parameters
2513  ----------
2514  exposure : `lsst.afw.image.Exposure`
2515  Exposure to process.
2516 
2517  Notes
2518  -----
2519  We mask over all non-finite values (NaN, inf), including those
2520  that are masked with other bits (because those may or may not be
2521  interpolated over later, and we want to remove all NaN/infs).
2522  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2523  preserve the historical name.
2524  """
2525  maskedImage = exposure.getMaskedImage()
2526 
2527  # Find and mask NaNs
2528  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2529  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2530  numNans = maskNans(maskedImage, maskVal)
2531  self.metadata["NUMNANS"] = numNans
2532  if numNans > 0:
2533  self.log.warning("There were %d unmasked NaNs.", numNans)
2534 
2535  def maskAndInterpolateNan(self, exposure):
2536  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2537  in place.
2538 
2539  Parameters
2540  ----------
2541  exposure : `lsst.afw.image.Exposure`
2542  Exposure to process.
2543 
2544  See Also
2545  --------
2546  lsst.ip.isr.isrTask.maskNan
2547  """
2548  self.maskNanmaskNan(exposure)
2549  isrFunctions.interpolateFromMask(
2550  maskedImage=exposure.getMaskedImage(),
2551  fwhm=self.config.fwhm,
2552  growSaturatedFootprints=0,
2553  maskNameList=["UNMASKEDNAN"],
2554  )
2555 
2556  def measureBackground(self, exposure, IsrQaConfig=None):
2557  """Measure the image background in subgrids, for quality control.
2558 
2559  Parameters
2560  ----------
2561  exposure : `lsst.afw.image.Exposure`
2562  Exposure to process.
2563  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2564  Configuration object containing parameters on which background
2565  statistics and subgrids to use.
2566  """
2567  if IsrQaConfig is not None:
2568  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2569  IsrQaConfig.flatness.nIter)
2570  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2571  statsControl.setAndMask(maskVal)
2572  maskedImage = exposure.getMaskedImage()
2573  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2574  skyLevel = stats.getValue(afwMath.MEDIAN)
2575  skySigma = stats.getValue(afwMath.STDEVCLIP)
2576  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2577  metadata = exposure.getMetadata()
2578  metadata["SKYLEVEL"] = skyLevel
2579  metadata["SKYSIGMA"] = skySigma
2580 
2581  # calcluating flatlevel over the subgrids
2582  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2583  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2584  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2585  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2586  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2587  skyLevels = numpy.zeros((nX, nY))
2588 
2589  for j in range(nY):
2590  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2591  for i in range(nX):
2592  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2593 
2594  xLLC = xc - meshXHalf
2595  yLLC = yc - meshYHalf
2596  xURC = xc + meshXHalf - 1
2597  yURC = yc + meshYHalf - 1
2598 
2599  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2600  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2601 
2602  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2603 
2604  good = numpy.where(numpy.isfinite(skyLevels))
2605  skyMedian = numpy.median(skyLevels[good])
2606  flatness = (skyLevels[good] - skyMedian) / skyMedian
2607  flatness_rms = numpy.std(flatness)
2608  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2609 
2610  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2611  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2612  nX, nY, flatness_pp, flatness_rms)
2613 
2614  metadata["FLATNESS_PP"] = float(flatness_pp)
2615  metadata["FLATNESS_RMS"] = float(flatness_rms)
2616  metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY)
2617  metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2618  metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2619 
2620  def roughZeroPoint(self, exposure):
2621  """Set an approximate magnitude zero point for the exposure.
2622 
2623  Parameters
2624  ----------
2625  exposure : `lsst.afw.image.Exposure`
2626  Exposure to process.
2627  """
2628  filterLabel = exposure.getFilterLabel()
2629  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2630 
2631  if physicalFilter in self.config.fluxMag0T1:
2632  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2633  else:
2634  self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2635  fluxMag0 = self.config.defaultFluxMag0T1
2636 
2637  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2638  if not expTime > 0: # handle NaN as well as <= 0
2639  self.log.warning("Non-positive exposure time; skipping rough zero point.")
2640  return
2641 
2642  self.log.info("Setting rough magnitude zero point for filter %s: %f",
2643  physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2644  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2645 
2646  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2647  """Set valid polygon as the intersection of fpPolygon and chip corners.
2648 
2649  Parameters
2650  ----------
2651  ccdExposure : `lsst.afw.image.Exposure`
2652  Exposure to process.
2653  fpPolygon : `lsst.afw.geom.Polygon`
2654  Polygon in focal plane coordinates.
2655  """
2656  # Get ccd corners in focal plane coordinates
2657  ccd = ccdExposure.getDetector()
2658  fpCorners = ccd.getCorners(FOCAL_PLANE)
2659  ccdPolygon = Polygon(fpCorners)
2660 
2661  # Get intersection of ccd corners with fpPolygon
2662  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2663 
2664  # Transform back to pixel positions and build new polygon
2665  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2666  validPolygon = Polygon(ccdPoints)
2667  ccdExposure.getInfo().setValidPolygon(validPolygon)
2668 
2669  @contextmanager
2670  def flatContext(self, exp, flat, dark=None):
2671  """Context manager that applies and removes flats and darks,
2672  if the task is configured to apply them.
2673 
2674  Parameters
2675  ----------
2676  exp : `lsst.afw.image.Exposure`
2677  Exposure to process.
2678  flat : `lsst.afw.image.Exposure`
2679  Flat exposure the same size as ``exp``.
2680  dark : `lsst.afw.image.Exposure`, optional
2681  Dark exposure the same size as ``exp``.
2682 
2683  Yields
2684  ------
2685  exp : `lsst.afw.image.Exposure`
2686  The flat and dark corrected exposure.
2687  """
2688  if self.config.doDark and dark is not None:
2689  self.darkCorrectiondarkCorrection(exp, dark)
2690  if self.config.doFlat:
2691  self.flatCorrectionflatCorrection(exp, flat)
2692  try:
2693  yield exp
2694  finally:
2695  if self.config.doFlat:
2696  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2697  if self.config.doDark and dark is not None:
2698  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2699 
2700  def debugView(self, exposure, stepname):
2701  """Utility function to examine ISR exposure at different stages.
2702 
2703  Parameters
2704  ----------
2705  exposure : `lsst.afw.image.Exposure`
2706  Exposure to view.
2707  stepname : `str`
2708  State of processing to view.
2709  """
2710  frame = getDebugFrame(self._display, stepname)
2711  if frame:
2712  display = getDisplay(frame)
2713  display.scale('asinh', 'zscale')
2714  display.mtv(exposure)
2715  prompt = "Press Enter to continue [c]... "
2716  while True:
2717  ans = input(prompt).lower()
2718  if ans in ("", "c",):
2719  break
2720 
2721 
2723  """A Detector-like object that supports returning gain and saturation level
2724 
2725  This is used when the input exposure does not have a detector.
2726 
2727  Parameters
2728  ----------
2729  exposure : `lsst.afw.image.Exposure`
2730  Exposure to generate a fake amplifier for.
2731  config : `lsst.ip.isr.isrTaskConfig`
2732  Configuration to apply to the fake amplifier.
2733  """
2734 
2735  def __init__(self, exposure, config):
2736  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2737  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2738  self._gain_gain = config.gain
2739  self._readNoise_readNoise = config.readNoise
2740  self._saturation_saturation = config.saturation
2741 
2742  def getBBox(self):
2743  return self._bbox_bbox
2744 
2745  def getRawBBox(self):
2746  return self._bbox_bbox
2747 
2749  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2750 
2751  def getGain(self):
2752  return self._gain_gain
2753 
2754  def getReadNoise(self):
2755  return self._readNoise_readNoise
2756 
2757  def getSaturation(self):
2758  return self._saturation_saturation
2759 
2760  def getSuspectLevel(self):
2761  return float("NaN")
2762 
2763 
2764 class RunIsrConfig(pexConfig.Config):
2765  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2766 
2767 
2768 class RunIsrTask(pipeBase.CmdLineTask):
2769  """Task to wrap the default IsrTask to allow it to be retargeted.
2770 
2771  The standard IsrTask can be called directly from a command line
2772  program, but doing so removes the ability of the task to be
2773  retargeted. As most cameras override some set of the IsrTask
2774  methods, this would remove those data-specific methods in the
2775  output post-ISR images. This wrapping class fixes the issue,
2776  allowing identical post-ISR images to be generated by both the
2777  processCcd and isrTask code.
2778  """
2779  ConfigClass = RunIsrConfig
2780  _DefaultName = "runIsr"
2781 
2782  def __init__(self, *args, **kwargs):
2783  super().__init__(*args, **kwargs)
2784  self.makeSubtask("isr")
2785 
2786  def runDataRef(self, dataRef):
2787  """
2788  Parameters
2789  ----------
2790  dataRef : `lsst.daf.persistence.ButlerDataRef`
2791  data reference of the detector data to be processed
2792 
2793  Returns
2794  -------
2795  result : `pipeBase.Struct`
2796  Result struct with component:
2797 
2798  - exposure : `lsst.afw.image.Exposure`
2799  Post-ISR processed exposure.
2800  """
2801  return self.isr.runDataRef(dataRef)
table::Key< int > type
Definition: Detector.cc:163
Cartesian polygons.
Definition: Polygon.h:59
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Definition: Exposure.h:72
Represent a 2-dimensional array of bitmask pixels.
Definition: Mask.h:77
Pass parameters to a Statistics object.
Definition: Statistics.h:92
An integer coordinate rectangle.
Definition: Box.h:55
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2748
def __init__(self, exposure, config)
Definition: isrTask.py:2735
def __init__(self, *config=None)
Definition: isrTask.py:272
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2316
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2535
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2365
def runDataRef(self, sensorRef)
Definition: isrTask.py:1770
def maskNan(self, exposure)
Definition: isrTask.py:2509
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1958
def debugView(self, exposure, stepname)
Definition: isrTask.py:2700
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
Definition: isrTask.py:1869
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1821
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2238
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2341
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2426
def __init__(self, **kwargs)
Definition: isrTask.py:974
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:985
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2450
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:2033
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2556
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2620
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2484
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2646
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1084
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1271
def doLinearize(self, detector)
Definition: isrTask.py:2297
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2670
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1921
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2390
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2163
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2254
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2782
def runDataRef(self, dataRef)
Definition: isrTask.py:2786
daf::base::PropertyList * list
Definition: fits.cc:913
daf::base::PropertySet * set
Definition: fits.cc:912
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::shared_ptr< PhotoCalib > makePhotoCalibFromCalibZeroPoint(double instFluxMag0, double instFluxMag0Err)
Construct a PhotoCalib from the deprecated Calib-style instFluxMag0/instFluxMag0Err values.
Definition: PhotoCalib.cc:613
std::shared_ptr< Exposure< ImagePixelT, MaskPixelT, VariancePixelT > > makeExposure(MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > &mimage, std::shared_ptr< geom::SkyWcs const > wcs=std::shared_ptr< geom::SkyWcs const >())
A function to return an Exposure of the correct type (cf.
Definition: Exposure.h:462
MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > * makeMaskedImage(typename std::shared_ptr< Image< ImagePixelT >> image, typename std::shared_ptr< Mask< MaskPixelT >> mask=Mask< MaskPixelT >(), typename std::shared_ptr< Image< VariancePixelT >> variance=Image< VariancePixelT >())
A function to return a MaskedImage of the correct type (cf.
Definition: MaskedImage.h:1240
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)
Definition: Statistics.h:359
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:64
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def getDebugFrame(debugDisplay, name)
Definition: lsstDebug.py:95