LSST Applications g070148d5b3+33e5256705,g0d53e28543+25c8b88941,g0da5cf3356+2dd1178308,g1081da9e2a+62d12e78cb,g17e5ecfddb+7e422d6136,g1c76d35bf8+ede3a706f7,g295839609d+225697d880,g2e2c1a68ba+cc1f6f037e,g2ffcdf413f+853cd4dcde,g38293774b4+62d12e78cb,g3b44f30a73+d953f1ac34,g48ccf36440+885b902d19,g4b2f1765b6+7dedbde6d2,g5320a0a9f6+0c5d6105b6,g56b687f8c9+ede3a706f7,g5c4744a4d9+ef6ac23297,g5ffd174ac0+0c5d6105b6,g6075d09f38+66af417445,g667d525e37+2ced63db88,g670421136f+2ced63db88,g71f27ac40c+2ced63db88,g774830318a+463cbe8d1f,g7876bc68e5+1d137996f1,g7985c39107+62d12e78cb,g7fdac2220c+0fd8241c05,g96f01af41f+368e6903a7,g9ca82378b8+2ced63db88,g9d27549199+ef6ac23297,gabe93b2c52+e3573e3735,gb065e2a02a+3dfbe639da,gbc3249ced9+0c5d6105b6,gbec6a3398f+0c5d6105b6,gc9534b9d65+35b9f25267,gd01420fc67+0c5d6105b6,geee7ff78d7+a14128c129,gf63283c776+ede3a706f7,gfed783d017+0c5d6105b6,w.2022.47
LSST Data Management Base Package
Loading...
Searching...
No Matches
dcrAssembleCoadd.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
23
24from math import ceil
25import numpy as np
26from scipy import ndimage
27import lsst.geom as geom
28import lsst.afw.image as afwImage
29import lsst.afw.table as afwTable
30import lsst.coadd.utils as coaddUtils
31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32import lsst.meas.algorithms as measAlg
33from lsst.meas.base import SingleFrameMeasurementTask
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.utils as utils
37from lsst.utils.timer import timeMethod
38from .assembleCoadd import (AssembleCoaddConnections,
39 AssembleCoaddTask,
40 CompareWarpAssembleCoaddConfig,
41 CompareWarpAssembleCoaddTask)
42from .coaddBase import makeSkyInfo
43from .measurePsf import MeasurePsfTask
44
45
47 dimensions=("tract", "patch", "band", "skymap"),
48 defaultTemplates={"inputWarpName": "deep",
49 "inputCoaddName": "deep",
50 "outputCoaddName": "dcr",
51 "warpType": "direct",
52 "warpTypeSuffix": "",
53 "fakesType": ""}):
54 inputWarps = pipeBase.connectionTypes.Input(
55 doc=("Input list of warps to be assembled i.e. stacked."
56 "Note that this will often be different than the inputCoaddName."
57 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
58 name="{inputWarpName}Coadd_{warpType}Warp",
59 storageClass="ExposureF",
60 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
61 deferLoad=True,
62 multiple=True
63 )
64 templateExposure = pipeBase.connectionTypes.Input(
65 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
66 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
67 storageClass="ExposureF",
68 dimensions=("tract", "patch", "skymap", "band"),
69 )
70 dcrCoadds = pipeBase.connectionTypes.Output(
71 doc="Output coadded exposure, produced by stacking input warps",
72 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
73 storageClass="ExposureF",
74 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
75 multiple=True,
76 )
77 dcrNImages = pipeBase.connectionTypes.Output(
78 doc="Output image of number of input images per pixel",
79 name="{outputCoaddName}Coadd_nImage",
80 storageClass="ImageU",
81 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
82 multiple=True,
83 )
84
85 def __init__(self, *, config=None):
86 super().__init__(config=config)
87 if not config.doWrite:
88 self.outputs.remove("dcrCoadds")
89 if not config.doNImage:
90 self.outputs.remove("dcrNImages")
91 # Remove outputs inherited from ``AssembleCoaddConnections`` that are not used
92 self.outputs.remove("coaddExposure")
93 self.outputs.remove("nImage")
94
95
96class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
97 pipelineConnections=DcrAssembleCoaddConnections):
98 dcrNumSubfilters = pexConfig.Field(
99 dtype=int,
100 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
101 default=3,
102 )
103 maxNumIter = pexConfig.Field(
104 dtype=int,
105 optional=True,
106 doc="Maximum number of iterations of forward modeling.",
107 default=None,
108 )
109 minNumIter = pexConfig.Field(
110 dtype=int,
111 optional=True,
112 doc="Minimum number of iterations of forward modeling.",
113 default=None,
114 )
115 convergenceThreshold = pexConfig.Field(
116 dtype=float,
117 doc="Target relative change in convergence between iterations of forward modeling.",
118 default=0.001,
119 )
120 useConvergence = pexConfig.Field(
121 dtype=bool,
122 doc="Use convergence test as a forward modeling end condition?"
123 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
124 default=True,
125 )
126 baseGain = pexConfig.Field(
127 dtype=float,
128 optional=True,
129 doc="Relative weight to give the new solution vs. the last solution when updating the model."
130 "A value of 1.0 gives equal weight to both solutions."
131 "Small values imply slower convergence of the solution, but can "
132 "help prevent overshooting and failures in the fit."
133 "If ``baseGain`` is None, a conservative gain "
134 "will be calculated from the number of subfilters. ",
135 default=None,
136 )
137 useProgressiveGain = pexConfig.Field(
138 dtype=bool,
139 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
140 "When calculating the next gain, we use up to 5 previous gains and convergence values."
141 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
142 default=True,
143 )
144 doAirmassWeight = pexConfig.Field(
145 dtype=bool,
146 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
147 default=False,
148 )
149 modelWeightsWidth = pexConfig.Field(
150 dtype=float,
151 doc="Width of the region around detected sources to include in the DcrModel.",
152 default=3,
153 )
154 useModelWeights = pexConfig.Field(
155 dtype=bool,
156 doc="Width of the region around detected sources to include in the DcrModel.",
157 default=True,
158 )
159 splitSubfilters = pexConfig.Field(
160 dtype=bool,
161 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
162 "Instead of at the midpoint",
163 default=True,
164 )
165 splitThreshold = pexConfig.Field(
166 dtype=float,
167 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
168 "Set to 0 to always split the subfilters.",
169 default=0.1,
170 )
171 regularizeModelIterations = pexConfig.Field(
172 dtype=float,
173 doc="Maximum relative change of the model allowed between iterations."
174 "Set to zero to disable.",
175 default=2.,
176 )
177 regularizeModelFrequency = pexConfig.Field(
178 dtype=float,
179 doc="Maximum relative change of the model allowed between subfilters."
180 "Set to zero to disable.",
181 default=4.,
182 )
183 convergenceMaskPlanes = pexConfig.ListField(
184 dtype=str,
185 default=["DETECTED"],
186 doc="Mask planes to use to calculate convergence."
187 )
188 regularizationWidth = pexConfig.Field(
189 dtype=int,
190 default=2,
191 doc="Minimum radius of a region to include in regularization, in pixels."
192 )
193 imageInterpOrder = pexConfig.Field(
194 dtype=int,
195 doc="The order of the spline interpolation used to shift the image plane.",
196 default=1,
197 )
198 accelerateModel = pexConfig.Field(
199 dtype=float,
200 doc="Factor to amplify the differences between model planes by to speed convergence.",
201 default=3,
202 )
203 doCalculatePsf = pexConfig.Field(
204 dtype=bool,
205 doc="Set to detect stars and recalculate the PSF from the final coadd."
206 "Otherwise the PSF is estimated from a selection of the best input exposures",
207 default=False,
208 )
209 detectPsfSources = pexConfig.ConfigurableField(
210 target=measAlg.SourceDetectionTask,
211 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
212 )
213 measurePsfSources = pexConfig.ConfigurableField(
214 target=SingleFrameMeasurementTask,
215 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
216 )
217 measurePsf = pexConfig.ConfigurableField(
218 target=MeasurePsfTask,
219 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
220 )
221 effectiveWavelength = pexConfig.Field(
222 doc="Effective wavelength of the filter, in nm."
223 "Required if transmission curves aren't used."
224 "Support for using transmission curves is to be added in DM-13668.",
225 dtype=float,
226 )
227 bandwidth = pexConfig.Field(
228 doc="Bandwidth of the physical filter, in nm."
229 "Required if transmission curves aren't used."
230 "Support for using transmission curves is to be added in DM-13668.",
231 dtype=float,
232 )
233
234 def setDefaults(self):
235 CompareWarpAssembleCoaddConfig.setDefaults(self)
236 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
237 self.doNImage = True
238 self.assembleStaticSkyModel.warpType = self.warpType
239 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
240 self.assembleStaticSkyModel.doNImage = False
241 self.assembleStaticSkyModel.doWrite = False
242 self.detectPsfSources.returnOriginalFootprints = False
243 self.detectPsfSources.thresholdPolarity = "positive"
244 # Only use bright sources for PSF measurement
245 self.detectPsfSources.thresholdValue = 50
246 self.detectPsfSources.nSigmaToGrow = 2
247 # A valid star for PSF measurement should at least fill 5x5 pixels
248 self.detectPsfSources.minPixels = 25
249 # Use the variance plane to calculate signal to noise
250 self.detectPsfSources.thresholdType = "pixel_stdev"
251 # The signal to noise limit is good enough, while the flux limit is set
252 # in dimensionless units and may not be appropriate for all data sets.
253 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
254 # Ensure psf candidate size is as large as piff psf size.
255 if (self.doCalculatePsf and self.measurePsf.psfDeterminer.name == "piff"
256 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize):
257 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize
258
259
260class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
261 """Assemble DCR coadded images from a set of warps.
262
263 Attributes
264 ----------
265 bufferSize : `int`
266 The number of pixels to grow each subregion by to allow for DCR.
267
268 Notes
269 -----
270 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
271 Warps (also called coadded temporary exposures), including the effects of
272 Differential Chromatic Refraction (DCR).
273 For full details of the mathematics and algorithm, please see
274 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
275
276 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
277 each subfilter used in the iterative calculation.
278 It begins by dividing the bandpass-defining filter into N equal bandwidth
279 "subfilters", and divides the flux in each pixel from an initial coadd
280 equally into each as a "dcrModel". Because the airmass and parallactic
281 angle of each individual exposure is known, we can calculate the shift
282 relative to the center of the band in each subfilter due to DCR. For each
283 exposure we apply this shift as a linear transformation to the dcrModels
284 and stack the results to produce a DCR-matched exposure. The matched
285 exposures are subtracted from the input exposures to produce a set of
286 residual images, and these residuals are reverse shifted for each
287 exposures' subfilters and stacked. The shifted and stacked residuals are
288 added to the dcrModels to produce a new estimate of the flux in each pixel
289 within each subfilter. The dcrModels are solved for iteratively, which
290 continues until the solution from a new iteration improves by less than
291 a set percentage, or a maximum number of iterations is reached.
292 Two forms of regularization are employed to reduce unphysical results.
293 First, the new solution is averaged with the solution from the previous
294 iteration, which mitigates oscillating solutions where the model
295 overshoots with alternating very high and low values.
296 Second, a common degeneracy when the data have a limited range of airmass or
297 parallactic angle values is for one subfilter to be fit with very low or
298 negative values, while another subfilter is fit with very high values. This
299 typically appears in the form of holes next to sources in one subfilter,
300 and corresponding extended wings in another. Because each subfilter has
301 a narrow bandwidth we assume that physical sources that are above the noise
302 level will not vary in flux by more than a factor of `frequencyClampFactor`
303 between subfilters, and pixels that have flux deviations larger than that
304 factor will have the excess flux distributed evenly among all subfilters.
305 If `splitSubfilters` is set, then each subfilter will be further sub-
306 divided during the forward modeling step (only). This approximates using
307 a higher number of subfilters that may be necessary for high airmass
308 observations, but does not increase the number of free parameters in the
309 fit. This is needed when there are high airmass observations which would
310 otherwise have significant DCR even within a subfilter. Because calculating
311 the shifted images takes most of the time, splitting the subfilters is
312 turned off by way of the `splitThreshold` option for low-airmass
313 observations that do not suffer from DCR within a subfilter.
314 """
315
316 ConfigClass = DcrAssembleCoaddConfig
317 _DefaultName = "dcrAssembleCoadd"
318
319 def __init__(self, *args, **kwargs):
320 super().__init__(*args, **kwargs)
321 if self.config.doCalculatePsf:
322 self.schema = afwTable.SourceTable.makeMinimalSchema()
323 self.makeSubtask("detectPsfSources", schema=self.schema)
324 self.makeSubtask("measurePsfSources", schema=self.schema)
325 self.makeSubtask("measurePsf", schema=self.schema)
326
327 @utils.inheritDoc(pipeBase.PipelineTask)
328 def runQuantum(self, butlerQC, inputRefs, outputRefs):
329 # Docstring to be formatted with info from PipelineTask.runQuantum
330 """
331 Notes
332 -----
333 Assemble a coadd from a set of Warps.
334 """
335 inputData = butlerQC.get(inputRefs)
336
337 # Construct skyInfo expected by run
338 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it
339 skyMap = inputData["skyMap"]
340 outputDataId = butlerQC.quantum.dataId
341
342 inputData['skyInfo'] = makeSkyInfo(skyMap,
343 tractId=outputDataId['tract'],
344 patchId=outputDataId['patch'])
345
346 # Construct list of input Deferred Datasets
347 warpRefList = inputData['inputWarps']
348
349 inputs = self.prepareInputs(warpRefList)
350 self.log.info("Found %d %s", len(inputs.tempExpRefList),
351 self.getTempExpDatasetName(self.warpType))
352 if len(inputs.tempExpRefList) == 0:
353 self.log.warning("No coadd temporary exposures found")
354 return
355
356 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
357 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
358 inputs.weightList, supplementaryData=supplementaryData)
359
360 inputData.setdefault('brightObjectMask', None)
361 for subfilter in range(self.config.dcrNumSubfilters):
362 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
363 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
364 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
365
366 if self.config.doWrite:
367 butlerQC.put(retStruct, outputRefs)
368 return retStruct
369
370 @utils.inheritDoc(AssembleCoaddTask)
371 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
372 """Load the previously-generated template coadd.
373
374 Returns
375 -------
376 templateCoadd : `lsst.pipe.base.Struct`
377 Results as a struct with attributes:
378
379 ``templateCoadd``
380 Coadded exposure (`lsst.afw.image.ExposureF`).
381 """
382 templateCoadd = butlerQC.get(inputRefs.templateExposure)
383
384 return pipeBase.Struct(templateCoadd=templateCoadd)
385
386 def measureCoaddPsf(self, coaddExposure):
387 """Detect sources on the coadd exposure and measure the final PSF.
388
389 Parameters
390 ----------
391 coaddExposure : `lsst.afw.image.Exposure`
392 The final coadded exposure.
393 """
394 table = afwTable.SourceTable.make(self.schema)
395 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
396 coaddSources = detResults.sources
397 self.measurePsfSources.run(
398 measCat=coaddSources,
399 exposure=coaddExposure
400 )
401 # Measure the PSF on the stacked subfilter coadds if possible.
402 # We should already have a decent estimate of the coadd PSF, however,
403 # so in case of any errors simply log them as a warning and use the
404 # default PSF.
405 try:
406 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
407 except Exception as e:
408 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e)
409 else:
410 coaddExposure.setPsf(psfResults.psf)
411
412 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
413 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
414
415 Sets the property ``bufferSize``.
416
417 Parameters
418 ----------
419 templateCoadd : `lsst.afw.image.ExposureF`
420 The initial coadd exposure before accounting for DCR.
421 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
422 The data references to the input warped exposures.
423 weightList : `list` of `float`
424 The weight to give each input exposure in the coadd.
425 Will be modified in place if ``doAirmassWeight`` is set.
426
427 Returns
428 -------
429 dcrModels : `lsst.pipe.tasks.DcrModel`
430 Best fit model of the true sky after correcting chromatic effects.
431
432 Raises
433 ------
434 NotImplementedError
435 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
436 """
437 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
438 filterLabel = templateCoadd.getFilter()
439 dcrShifts = []
440 airmassDict = {}
441 angleDict = {}
442 psfSizeDict = {}
443 for visitNum, warpExpRef in enumerate(warpRefList):
444 visitInfo = warpExpRef.get(component="visitInfo")
445 psf = warpExpRef.get(component="psf")
446 visit = warpExpRef.dataId["visit"]
447 # Just need a rough estimate; average positions are fine
448 psfAvgPos = psf.getAveragePosition()
449 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
450 airmass = visitInfo.getBoresightAirmass()
451 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
452 airmassDict[visit] = airmass
453 angleDict[visit] = parallacticAngle
454 psfSizeDict[visit] = psfSize
455 if self.config.doAirmassWeight:
456 weightList[visitNum] *= airmass
457 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
458 self.config.effectiveWavelength,
459 self.config.bandwidth,
460 self.config.dcrNumSubfilters))))
461 self.log.info("Selected airmasses:\n%s", airmassDict)
462 self.log.info("Selected parallactic angles:\n%s", angleDict)
463 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
464 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
465 try:
466 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
467 except Exception as e:
468 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e)
469 else:
470 psf = templateCoadd.getPsf()
471 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
472 self.config.dcrNumSubfilters,
473 effectiveWavelength=self.config.effectiveWavelength,
474 bandwidth=self.config.bandwidth,
475 wcs=templateCoadd.getWcs(),
476 filterLabel=filterLabel,
477 psf=psf)
478 return dcrModels
479
480 @timeMethod
481 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
482 supplementaryData=None):
483 r"""Assemble the coadd.
484
485 Requires additional inputs Struct ``supplementaryData`` to contain a
486 ``templateCoadd`` that serves as the model of the static sky.
487
488 Find artifacts and apply them to the warps' masks creating a list of
489 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
490 Then pass these alternative masks to the base class's assemble method.
491
492 Divide the ``templateCoadd`` evenly between each subfilter of a
493 ``DcrModel`` as the starting best estimate of the true wavelength-
494 dependent sky. Forward model the ``DcrModel`` using the known
495 chromatic effects in each subfilter and calculate a convergence metric
496 based on how well the modeled template matches the input warps. If
497 the convergence has not yet reached the desired threshold, then shift
498 and stack the residual images to build a new ``DcrModel``. Apply
499 conditioning to prevent oscillating solutions between iterations or
500 between subfilters.
501
502 Once the ``DcrModel`` reaches convergence or the maximum number of
503 iterations has been reached, fill the metadata for each subfilter
504 image and make them proper ``coaddExposure``\ s.
505
506 Parameters
507 ----------
508 skyInfo : `lsst.pipe.base.Struct`
509 Patch geometry information, from getSkyInfo
510 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
511 The data references to the input warped exposures.
512 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
513 The image scalars correct for the zero point of the exposures.
514 weightList : `list` of `float`
515 The weight to give each input exposure in the coadd
516 supplementaryData : `lsst.pipe.base.Struct`
517 Result struct returned by ``_makeSupplementaryData`` with attributes:
518
519 ``templateCoadd``
520 Coadded exposure (`lsst.afw.image.Exposure`).
521
522 Returns
523 -------
524 result : `lsst.pipe.base.Struct`
525 Results as a struct with attributes:
526
527 ``coaddExposure``
528 Coadded exposure (`lsst.afw.image.Exposure`).
529 ``nImage``
530 Exposure count image (`lsst.afw.image.ImageU`).
531 ``dcrCoadds``
532 `list` of coadded exposures for each subfilter.
533 ``dcrNImages``
534 `list` of exposure count images for each subfilter.
535 """
536 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
537 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
538 templateCoadd = supplementaryData.templateCoadd
539 baseMask = templateCoadd.mask.clone()
540 # The variance plane is for each subfilter
541 # and should be proportionately lower than the full-band image
542 baseVariance = templateCoadd.variance.clone()
543 baseVariance /= self.config.dcrNumSubfilters
544 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
545 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
546 templateCoadd.setMask(baseMask)
547 badMaskPlanes = self.config.badMaskPlanes[:]
548 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
549 # This is because pixels in observations that are significantly affect by DCR
550 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
551 # but those are necessary to constrain the DCR model.
552 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
553
554 stats = self.prepareStats(mask=badPixelMask)
555 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
556 if self.config.doNImage:
557 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
558 spanSetMaskList, stats.ctrl)
559 nImage = afwImage.ImageU(skyInfo.bbox)
560 # Note that this nImage will be a factor of dcrNumSubfilters higher than
561 # the nImage returned by assembleCoadd for most pixels. This is because each
562 # subfilter may have a different nImage, and fractional values are not allowed.
563 for dcrNImage in dcrNImages:
564 nImage += dcrNImage
565 else:
566 dcrNImages = None
567
568 subregionSize = geom.Extent2I(*self.config.subregionSize)
569 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
570 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
571 subIter = 0
572 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
573 modelIter = 0
574 subIter += 1
575 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
576 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
577 dcrBBox = geom.Box2I(subBBox)
578 dcrBBox.grow(self.bufferSize)
579 dcrBBox.clip(dcrModels.bbox)
580 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
581 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
582 imageScalerList, spanSetMaskList)
583 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
584 warpRefList, weightList, stats.ctrl)
585 self.log.info("Initial convergence : %s", convergenceMetric)
586 convergenceList = [convergenceMetric]
587 gainList = []
588 convergenceCheck = 1.
589 refImage = templateCoadd.image
590 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
591 gain = self.calculateGain(convergenceList, gainList)
592 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
593 stats.ctrl, convergenceMetric, gain,
594 modelWeights, refImage, dcrWeights)
595 if self.config.useConvergence:
596 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
597 warpRefList, weightList, stats.ctrl)
598 if convergenceMetric == 0:
599 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
600 "most likely due to there being no valid data in the region.",
601 skyInfo.patchInfo.getIndex(), subIter)
602 break
603 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
604 if (convergenceCheck < 0) & (modelIter > minNumIter):
605 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum "
606 "iterations or desired convergence improvement of %s."
607 " Divergence: %s",
608 skyInfo.patchInfo.getIndex(), subIter,
609 self.config.convergenceThreshold, convergenceCheck)
610 break
611 convergenceList.append(convergenceMetric)
612 if modelIter > maxNumIter:
613 if self.config.useConvergence:
614 self.log.warning("Coadd patch %s subregion %s reached maximum iterations "
615 "before reaching desired convergence improvement of %s."
616 " Final convergence improvement: %s",
617 skyInfo.patchInfo.getIndex(), subIter,
618 self.config.convergenceThreshold, convergenceCheck)
619 break
620
621 if self.config.useConvergence:
622 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
623 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
624 modelIter += 1
625 else:
626 if self.config.useConvergence:
627 self.log.info("Coadd patch %s subregion %s finished with "
628 "convergence metric %s after %s iterations",
629 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
630 else:
631 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
632 skyInfo.patchInfo.getIndex(), subIter, modelIter)
633 if self.config.useConvergence and convergenceMetric > 0:
634 self.log.info("Final convergence improvement was %.4f%% overall",
635 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
636
637 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
638 calibration=self.scaleZeroPoint.getPhotoCalib(),
639 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
640 mask=baseMask,
641 variance=baseVariance)
642 coaddExposure = self.stackCoadd(dcrCoadds)
643 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
644 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
645
646 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
647 """Calculate the number of exposures contributing to each subfilter.
648
649 Parameters
650 ----------
651 dcrModels : `lsst.pipe.tasks.DcrModel`
652 Best fit model of the true sky after correcting chromatic effects.
653 bbox : `lsst.geom.box.Box2I`
654 Bounding box of the patch to coadd.
655 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
656 The data references to the input warped exposures.
657 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None`
658 Each element of the `dict` contains the new mask plane name
659 (e.g. "CLIPPED and/or "NO_DATA") as the key,
660 and the list of SpanSets to apply to the mask.
662 Statistics control object for coadd
663
664 Returns
665 -------
666 dcrNImages : `list` of `lsst.afw.image.ImageU`
667 List of exposure count images for each subfilter.
668 dcrWeights : `list` of `lsst.afw.image.ImageF`
669 Per-pixel weights for each subfilter.
670 Equal to 1/(number of unmasked images contributing to each pixel).
671 """
672 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
673 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
674 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
675 exposure = warpExpRef.get(parameters={'bbox': bbox})
676 visitInfo = exposure.getInfo().getVisitInfo()
677 wcs = exposure.getInfo().getWcs()
678 mask = exposure.mask
679 if altMaskSpans is not None:
680 self.applyAltMaskPlanes(mask, altMaskSpans)
681 weightImage = np.zeros_like(exposure.image.array)
682 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
683 # The weights must be shifted in exactly the same way as the residuals,
684 # because they will be used as the denominator in the weighted average of residuals.
685 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
686 dcrModels.effectiveWavelength, dcrModels.bandwidth)
687 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
688 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
689 dcrWeight.array += shiftedWeights
690 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
691 weightsThreshold = 1.
692 goodPix = dcrWeights[0].array > weightsThreshold
693 for weights in dcrWeights[1:]:
694 goodPix = (weights.array > weightsThreshold) & goodPix
695 for subfilter in range(self.config.dcrNumSubfilters):
696 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
697 dcrWeights[subfilter].array[~goodPix] = 0.
698 dcrNImages[subfilter].array[~goodPix] = 0
699 return (dcrNImages, dcrWeights)
700
701 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
702 statsCtrl, convergenceMetric,
703 gain, modelWeights, refImage, dcrWeights):
704 """Assemble the DCR coadd for a sub-region.
705
706 Build a DCR-matched template for each input exposure, then shift the
707 residuals according to the DCR in each subfilter.
708 Stack the shifted residuals and apply them as a correction to the
709 solution from the previous iteration.
710 Restrict the new model solutions from varying by more than a factor of
711 `modelClampFactor` from the last solution, and additionally restrict the
712 individual subfilter models from varying by more than a factor of
713 `frequencyClampFactor` from their average.
714 Finally, mitigate potentially oscillating solutions by averaging the new
715 solution with the solution from the previous iteration, weighted by
716 their convergence metric.
717
718 Parameters
719 ----------
720 dcrModels : `lsst.pipe.tasks.DcrModel`
721 Best fit model of the true sky after correcting chromatic effects.
722 subExposures : `dict` of `lsst.afw.image.ExposureF`
723 The pre-loaded exposures for the current subregion.
724 bbox : `lsst.geom.box.Box2I`
725 Bounding box of the subregion to coadd.
726 dcrBBox : `lsst.geom.box.Box2I`
727 Sub-region of the coadd which includes a buffer to allow for DCR.
728 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
729 The data references to the input warped exposures.
731 Statistics control object for coadd.
732 convergenceMetric : `float`
733 Quality of fit metric for the matched templates of the input images.
734 gain : `float`, optional
735 Relative weight to give the new solution when updating the model.
736 modelWeights : `numpy.ndarray` or `float`
737 A 2D array of weight values that tapers smoothly to zero away from detected sources.
738 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
739 refImage : `lsst.afw.image.Image`
740 A reference image used to supply the default pixel values.
741 dcrWeights : `list` of `lsst.afw.image.Image`
742 Per-pixel weights for each subfilter.
743 Equal to 1/(number of unmasked images contributing to each pixel).
744 """
745 residualGeneratorList = []
746
747 for warpExpRef in warpRefList:
748 visit = warpExpRef.dataId["visit"]
749 exposure = subExposures[visit]
750 visitInfo = exposure.getInfo().getVisitInfo()
751 wcs = exposure.getInfo().getWcs()
752 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
753 bbox=exposure.getBBox(),
754 order=self.config.imageInterpOrder,
755 splitSubfilters=self.config.splitSubfilters,
756 splitThreshold=self.config.splitThreshold,
757 amplifyModel=self.config.accelerateModel)
758 residual = exposure.image.array - templateImage.array
759 # Note that the variance plane here is used to store weights, not the actual variance
760 residual *= exposure.variance.array
761 # The residuals are stored as a list of generators.
762 # This allows the residual for a given subfilter and exposure to be created
763 # on the fly, instead of needing to store them all in memory.
764 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
765 dcrModels.effectiveWavelength,
766 dcrModels.bandwidth))
767
768 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
769 gain=gain,
770 modelWeights=modelWeights,
771 refImage=refImage,
772 dcrWeights=dcrWeights)
773 dcrModels.assign(dcrSubModelOut, bbox)
774
775 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
776 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
777
778 Parameters
779 ----------
780 residual : `numpy.ndarray`
781 The residual masked image for one exposure,
782 after subtracting the matched template.
783 visitInfo : `lsst.afw.image.VisitInfo`
784 Metadata for the exposure.
786 Coordinate system definition (wcs) for the exposure.
787
788 Yields
789 ------
790 residualImage : `numpy.ndarray`
791 The residual image for the next subfilter, shifted for DCR.
792 """
793 if self.config.imageInterpOrder > 1:
794 # Pre-calculate the spline-filtered residual image, so that step can be
795 # skipped in the shift calculation in `applyDcr`.
796 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
797 else:
798 # No need to prefilter if order=1 (it will also raise an error)
799 filteredResidual = residual
800 # Note that `splitSubfilters` is always turned off in the reverse direction.
801 # This option introduces additional blurring if applied to the residuals.
802 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
803 splitSubfilters=False)
804 for dcr in dcrShift:
805 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
806 doPrefilter=False, order=self.config.imageInterpOrder)
807
808 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
809 gain, modelWeights, refImage, dcrWeights):
810 """Calculate a new DcrModel from a set of image residuals.
811
812 Parameters
813 ----------
814 dcrModels : `lsst.pipe.tasks.DcrModel`
815 Current model of the true sky after correcting chromatic effects.
816 residualGeneratorList : `generator` of `numpy.ndarray`
817 The residual image for the next subfilter, shifted for DCR.
818 dcrBBox : `lsst.geom.box.Box2I`
819 Sub-region of the coadd which includes a buffer to allow for DCR.
821 Statistics control object for coadd.
822 gain : `float`
823 Relative weight to give the new solution when updating the model.
824 modelWeights : `numpy.ndarray` or `float`
825 A 2D array of weight values that tapers smoothly to zero away from detected sources.
826 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
827 refImage : `lsst.afw.image.Image`
828 A reference image used to supply the default pixel values.
829 dcrWeights : `list` of `lsst.afw.image.Image`
830 Per-pixel weights for each subfilter.
831 Equal to 1/(number of unmasked images contributing to each pixel).
832
833 Returns
834 -------
835 dcrModel : `lsst.pipe.tasks.DcrModel`
836 New model of the true sky after correcting chromatic effects.
837 """
838 newModelImages = []
839 for subfilter, model in enumerate(dcrModels):
840 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
841 residual = np.sum(residualsList, axis=0)
842 residual *= dcrWeights[subfilter][dcrBBox].array
843 # `MaskedImage`s only support in-place addition, so rename for readability
844 newModel = model[dcrBBox].clone()
845 newModel.array += residual
846 # Catch any invalid values
847 badPixels = ~np.isfinite(newModel.array)
848 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
849 if self.config.regularizeModelIterations > 0:
850 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
851 self.config.regularizeModelIterations,
852 self.config.regularizationWidth)
853 newModelImages.append(newModel)
854 if self.config.regularizeModelFrequency > 0:
855 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
856 self.config.regularizeModelFrequency,
857 self.config.regularizationWidth)
858 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
859 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
860 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
861 dcrModels.bandwidth, dcrModels.psf,
862 dcrModels.mask, dcrModels.variance)
863
864 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
865 """Calculate a quality of fit metric for the matched templates.
866
867 Parameters
868 ----------
869 dcrModels : `lsst.pipe.tasks.DcrModel`
870 Best fit model of the true sky after correcting chromatic effects.
871 subExposures : `dict` of `lsst.afw.image.ExposureF`
872 The pre-loaded exposures for the current subregion.
873 bbox : `lsst.geom.box.Box2I`
874 Sub-region to coadd.
875 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
876 The data references to the input warped exposures.
877 weightList : `list` of `float`
878 The weight to give each input exposure in the coadd.
880 Statistics control object for coadd.
881
882 Returns
883 -------
884 convergenceMetric : `float`
885 Quality of fit metric for all input exposures, within the sub-region.
886 """
887 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
888 nSigma = 3.
889 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
890 bufferSize=self.bufferSize)
891 if np.max(significanceImage) == 0:
892 significanceImage += 1.
893 weight = 0
894 metric = 0.
895 metricList = {}
896 for warpExpRef, expWeight in zip(warpRefList, weightList):
897 visit = warpExpRef.dataId["visit"]
898 exposure = subExposures[visit][bbox]
899 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
900 metric += singleMetric
901 metricList[visit] = singleMetric
902 weight += 1.
903 self.log.info("Individual metrics:\n%s", metricList)
904 return 1.0 if weight == 0.0 else metric/weight
905
906 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
907 """Calculate a quality of fit metric for a single matched template.
908
909 Parameters
910 ----------
911 dcrModels : `lsst.pipe.tasks.DcrModel`
912 Best fit model of the true sky after correcting chromatic effects.
913 exposure : `lsst.afw.image.ExposureF`
914 The input warped exposure to evaluate.
915 significanceImage : `numpy.ndarray`
916 Array of weights for each pixel corresponding to its significance
917 for the convergence calculation.
919 Statistics control object for coadd.
920
921 Returns
922 -------
923 convergenceMetric : `float`
924 Quality of fit metric for one exposure, within the sub-region.
925 """
926 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
927 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
928 bbox=exposure.getBBox(),
929 order=self.config.imageInterpOrder,
930 splitSubfilters=self.config.splitSubfilters,
931 splitThreshold=self.config.splitThreshold,
932 amplifyModel=self.config.accelerateModel)
933 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
934 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
935
936 finitePixels = np.isfinite(diffVals)
937 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
938 convergeMaskPixels = exposure.mask.array & convergeMask > 0
939 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
940 if np.sum(usePixels) == 0:
941 metric = 0.
942 else:
943 diffUse = diffVals[usePixels]
944 refUse = refVals[usePixels]
945 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
946 return metric
947
948 def stackCoadd(self, dcrCoadds):
949 """Add a list of sub-band coadds together.
950
951 Parameters
952 ----------
953 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
954 A list of coadd exposures, each exposure containing
955 the model for one subfilter.
956
957 Returns
958 -------
959 coaddExposure : `lsst.afw.image.ExposureF`
960 A single coadd exposure that is the sum of the sub-bands.
961 """
962 coaddExposure = dcrCoadds[0].clone()
963 for coadd in dcrCoadds[1:]:
964 coaddExposure.maskedImage += coadd.maskedImage
965 return coaddExposure
966
967 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
968 mask=None, variance=None):
969 """Create a list of coadd exposures from a list of masked images.
970
971 Parameters
972 ----------
973 dcrModels : `lsst.pipe.tasks.DcrModel`
974 Best fit model of the true sky after correcting chromatic effects.
975 skyInfo : `lsst.pipe.base.Struct`
976 Patch geometry information, from getSkyInfo.
977 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
978 The data references to the input warped exposures.
979 weightList : `list` of `float`
980 The weight to give each input exposure in the coadd.
981 calibration : `lsst.afw.Image.PhotoCalib`, optional
982 Scale factor to set the photometric calibration of an exposure.
983 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
984 A record of the observations that are included in the coadd.
985 mask : `lsst.afw.image.Mask`, optional
986 Optional mask to override the values in the final coadd.
987 variance : `lsst.afw.image.Image`, optional
988 Optional variance plane to override the values in the final coadd.
989
990 Returns
991 -------
992 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
993 A list of coadd exposures, each exposure containing
994 the model for one subfilter.
995 """
996 dcrCoadds = []
997 refModel = dcrModels.getReferenceImage()
998 for model in dcrModels:
999 if self.config.accelerateModel > 1:
1000 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1001 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1002 if calibration is not None:
1003 coaddExposure.setPhotoCalib(calibration)
1004 if coaddInputs is not None:
1005 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1006 # Set the metadata for the coadd, including PSF and aperture corrections.
1007 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1008 # Overwrite the PSF
1009 coaddExposure.setPsf(dcrModels.psf)
1010 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1011 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1012 maskedImage.image = model
1013 maskedImage.mask = dcrModels.mask
1014 maskedImage.variance = dcrModels.variance
1015 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1016 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1017 if mask is not None:
1018 coaddExposure.setMask(mask)
1019 if variance is not None:
1020 coaddExposure.setVariance(variance)
1021 dcrCoadds.append(coaddExposure)
1022 return dcrCoadds
1023
1024 def calculateGain(self, convergenceList, gainList):
1025 """Calculate the gain to use for the current iteration.
1026
1027 After calculating a new DcrModel, each value is averaged with the
1028 value in the corresponding pixel from the previous iteration. This
1029 reduces oscillating solutions that iterative techniques are plagued by,
1030 and speeds convergence. By far the biggest changes to the model
1031 happen in the first couple iterations, so we can also use a more
1032 aggressive gain later when the model is changing slowly.
1033
1034 Parameters
1035 ----------
1036 convergenceList : `list` of `float`
1037 The quality of fit metric from each previous iteration.
1038 gainList : `list` of `float`
1039 The gains used in each previous iteration: appended with the new
1040 gain value.
1041 Gains are numbers between ``self.config.baseGain`` and 1.
1042
1043 Returns
1044 -------
1045 gain : `float`
1046 Relative weight to give the new solution when updating the model.
1047 A value of 1.0 gives equal weight to both solutions.
1048
1049 Raises
1050 ------
1051 ValueError
1052 If ``len(convergenceList) != len(gainList)+1``.
1053 """
1054 nIter = len(convergenceList)
1055 if nIter != len(gainList) + 1:
1056 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1057 % (len(convergenceList), len(gainList)))
1058
1059 if self.config.baseGain is None:
1060 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1061 # The more subfilters being modeled, the lower the gain should be.
1062 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1063 else:
1064 baseGain = self.config.baseGain
1065
1066 if self.config.useProgressiveGain and nIter > 2:
1067 # To calculate the best gain to use, compare the past gains that have been used
1068 # with the resulting convergences to estimate the best gain to use.
1069 # Algorithmically, this is a Kalman filter.
1070 # If forward modeling proceeds perfectly, the convergence metric should
1071 # asymptotically approach a final value.
1072 # We can estimate that value from the measured changes in convergence
1073 # weighted by the gains used in each previous iteration.
1074 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1075 for i in range(nIter - 1)]
1076 # The convergence metric is strictly positive, so if the estimated final convergence is
1077 # less than zero, force it to zero.
1078 estFinalConv = np.array(estFinalConv)
1079 estFinalConv[estFinalConv < 0] = 0
1080 # Because the estimate may slowly change over time, only use the most recent measurements.
1081 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1082 lastGain = gainList[-1]
1083 lastConv = convergenceList[-2]
1084 newConv = convergenceList[-1]
1085 # The predicted convergence is the value we would get if the new model calculated
1086 # in the previous iteration was perfect. Recall that the updated model that is
1087 # actually used is the gain-weighted average of the new and old model,
1088 # so the convergence would be similarly weighted.
1089 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1090 # If the measured and predicted convergence are very close, that indicates
1091 # that our forward model is accurate and we can use a more aggressive gain
1092 # If the measured convergence is significantly worse (or better!) than predicted,
1093 # that indicates that the model is not converging as expected and
1094 # we should use a more conservative gain.
1095 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1096 newGain = 1 - abs(delta)
1097 # Average the gains to prevent oscillating solutions.
1098 newGain = (newGain + lastGain)/2.
1099 gain = max(baseGain, newGain)
1100 else:
1101 gain = baseGain
1102 gainList.append(gain)
1103 return gain
1104
1105 def calculateModelWeights(self, dcrModels, dcrBBox):
1106 """Build an array that smoothly tapers to 0 away from detected sources.
1107
1108 Parameters
1109 ----------
1110 dcrModels : `lsst.pipe.tasks.DcrModel`
1111 Best fit model of the true sky after correcting chromatic effects.
1112 dcrBBox : `lsst.geom.box.Box2I`
1113 Sub-region of the coadd which includes a buffer to allow for DCR.
1114
1115 Returns
1116 -------
1117 weights : `numpy.ndarray` or `float`
1118 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1119 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1120
1121 Raises
1122 ------
1123 ValueError
1124 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1125 """
1126 if not self.config.useModelWeights:
1127 return 1.0
1128 if self.config.modelWeightsWidth < 0:
1129 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1130 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1131 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1132 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1133 weights[convergeMaskPixels] = 1.
1134 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth)
1135 weights /= np.max(weights)
1136 return weights
1137
1138 def applyModelWeights(self, modelImages, refImage, modelWeights):
1139 """Smoothly replace model pixel values with those from a
1140 reference at locations away from detected sources.
1141
1142 Parameters
1143 ----------
1144 modelImages : `list` of `lsst.afw.image.Image`
1145 The new DCR model images from the current iteration.
1146 The values will be modified in place.
1147 refImage : `lsst.afw.image.MaskedImage`
1148 A reference image used to supply the default pixel values.
1149 modelWeights : `numpy.ndarray` or `float`
1150 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1151 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1152 """
1153 if self.config.useModelWeights:
1154 for model in modelImages:
1155 model.array *= modelWeights
1156 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1157
1158 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1159 """Pre-load sub-regions of a list of exposures.
1160
1161 Parameters
1162 ----------
1163 bbox : `lsst.geom.box.Box2I`
1164 Sub-region to coadd.
1166 Statistics control object for coadd.
1167 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1168 The data references to the input warped exposures.
1169 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1170 The image scalars correct for the zero point of the exposures.
1171 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None`
1172 Each element is dict with keys = mask plane name to add the spans to.
1173
1174 Returns
1175 -------
1176 subExposures : `dict`
1177 The `dict` keys are the visit IDs,
1178 and the values are `lsst.afw.image.ExposureF`
1179 The pre-loaded exposures for the current subregion.
1180 The variance plane contains weights, and not the variance
1181 """
1182 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1183 subExposures = {}
1184 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1185 exposure = warpExpRef.get(parameters={'bbox': bbox})
1186 visit = warpExpRef.dataId["visit"]
1187 if altMaskSpans is not None:
1188 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1189 imageScaler.scaleMaskedImage(exposure.maskedImage)
1190 # Note that the variance plane here is used to store weights, not the actual variance
1191 exposure.variance.array[:, :] = 0.
1192 # Set the weight of unmasked pixels to 1.
1193 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1194 # Set the image value of masked pixels to zero.
1195 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1196 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1197 subExposures[visit] = exposure
1198 return subExposures
1199
1200 def selectCoaddPsf(self, templateCoadd, warpRefList):
1201 """Compute the PSF of the coadd from the exposures with the best seeing.
1202
1203 Parameters
1204 ----------
1205 templateCoadd : `lsst.afw.image.ExposureF`
1206 The initial coadd exposure before accounting for DCR.
1207 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1208 The data references to the input warped exposures.
1209
1210 Returns
1211 -------
1213 The average PSF of the input exposures with the best seeing.
1214 """
1215 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1216 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1217 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1218 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1219 templatePsf = templateCoadd.getPsf()
1220 # Just need a rough estimate; average positions are fine
1221 templateAvgPos = templatePsf.getAveragePosition()
1222 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm
1223 psfSizes = np.zeros(len(ccds))
1224 ccdVisits = np.array(ccds["visit"])
1225 for warpExpRef in warpRefList:
1226 psf = warpExpRef.get(component="psf")
1227 visit = warpExpRef.dataId["visit"]
1228 psfAvgPos = psf.getAveragePosition()
1229 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
1230 psfSizes[ccdVisits == visit] = psfSize
1231 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1232 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1233 # of the mean or median FWHM of the input exposures.
1234 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1235 goodPsfs = psfSizes <= sizeThreshold
1236 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1237 self.config.coaddPsf.makeControl())
1238 return psf
int min
int max
A 2-dimensional celestial WCS that transform pixels to ICRS RA/Dec, using the LSST standard for pixel...
Definition: SkyWcs.h:117
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Definition: Exposure.h:72
A class to represent a 2-dimensional array of pixels.
Definition: Image.h:51
Represent a 2-dimensional array of bitmask pixels.
Definition: Mask.h:77
A class to manipulate images, masks, and variance as a single object.
Definition: MaskedImage.h:74
Information about a single exposure of an imaging camera.
Definition: VisitInfo.h:68
Pass parameters to a Statistics object.
Definition: Statistics.h:83
An integer coordinate rectangle.
Definition: Box.h:55
CoaddPsf is the Psf derived to be used for non-PSF-matched Coadd images.
Definition: CoaddPsf.h:58
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth)