LSSTApplications  18.0.0+106,18.0.0+50,19.0.0,19.0.0+1,19.0.0+10,19.0.0+11,19.0.0+13,19.0.0+17,19.0.0+2,19.0.0-1-g20d9b18+6,19.0.0-1-g425ff20,19.0.0-1-g5549ca4,19.0.0-1-g580fafe+6,19.0.0-1-g6fe20d0+1,19.0.0-1-g7011481+9,19.0.0-1-g8c57eb9+6,19.0.0-1-gb5175dc+11,19.0.0-1-gdc0e4a7+9,19.0.0-1-ge272bc4+6,19.0.0-1-ge3aa853,19.0.0-10-g448f008b,19.0.0-12-g6990b2c,19.0.0-2-g0d9f9cd+11,19.0.0-2-g3d9e4fb2+11,19.0.0-2-g5037de4,19.0.0-2-gb96a1c4+3,19.0.0-2-gd955cfd+15,19.0.0-3-g2d13df8,19.0.0-3-g6f3c7dc,19.0.0-4-g725f80e+11,19.0.0-4-ga671dab3b+1,19.0.0-4-gad373c5+3,19.0.0-5-ga2acb9c+2,19.0.0-5-gfe96e6c+2,w.2020.01
LSSTDataManagementBasePackage
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.meas.algorithms as measAlg
32 from lsst.meas.base import SingleFrameMeasurementTask
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf import MeasurePsfTask
37 
38 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
39 
40 
42  dcrNumSubfilters = pexConfig.Field(
43  dtype=int,
44  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
45  default=3,
46  )
47  maxNumIter = pexConfig.Field(
48  dtype=int,
49  optional=True,
50  doc="Maximum number of iterations of forward modeling.",
51  default=None,
52  )
53  minNumIter = pexConfig.Field(
54  dtype=int,
55  optional=True,
56  doc="Minimum number of iterations of forward modeling.",
57  default=None,
58  )
59  convergenceThreshold = pexConfig.Field(
60  dtype=float,
61  doc="Target relative change in convergence between iterations of forward modeling.",
62  default=0.001,
63  )
64  useConvergence = pexConfig.Field(
65  dtype=bool,
66  doc="Use convergence test as a forward modeling end condition?"
67  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
68  default=True,
69  )
70  baseGain = pexConfig.Field(
71  dtype=float,
72  optional=True,
73  doc="Relative weight to give the new solution vs. the last solution when updating the model."
74  "A value of 1.0 gives equal weight to both solutions."
75  "Small values imply slower convergence of the solution, but can "
76  "help prevent overshooting and failures in the fit."
77  "If ``baseGain`` is None, a conservative gain "
78  "will be calculated from the number of subfilters. ",
79  default=None,
80  )
81  useProgressiveGain = pexConfig.Field(
82  dtype=bool,
83  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
84  "When calculating the next gain, we use up to 5 previous gains and convergence values."
85  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
86  default=True,
87  )
88  doAirmassWeight = pexConfig.Field(
89  dtype=bool,
90  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
91  default=False,
92  )
93  modelWeightsWidth = pexConfig.Field(
94  dtype=float,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=3,
97  )
98  useModelWeights = pexConfig.Field(
99  dtype=bool,
100  doc="Width of the region around detected sources to include in the DcrModel.",
101  default=True,
102  )
103  splitSubfilters = pexConfig.Field(
104  dtype=bool,
105  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
106  "Instead of at the midpoint",
107  default=True,
108  )
109  splitThreshold = pexConfig.Field(
110  dtype=float,
111  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
112  "Set to 0 to always split the subfilters.",
113  default=0.1,
114  )
115  regularizeModelIterations = pexConfig.Field(
116  dtype=float,
117  doc="Maximum relative change of the model allowed between iterations."
118  "Set to zero to disable.",
119  default=2.,
120  )
121  regularizeModelFrequency = pexConfig.Field(
122  dtype=float,
123  doc="Maximum relative change of the model allowed between subfilters."
124  "Set to zero to disable.",
125  default=4.,
126  )
127  convergenceMaskPlanes = pexConfig.ListField(
128  dtype=str,
129  default=["DETECTED"],
130  doc="Mask planes to use to calculate convergence."
131  )
132  regularizationWidth = pexConfig.Field(
133  dtype=int,
134  default=2,
135  doc="Minimum radius of a region to include in regularization, in pixels."
136  )
137  imageInterpOrder = pexConfig.Field(
138  dtype=int,
139  doc="The order of the spline interpolation used to shift the image plane.",
140  default=3,
141  )
142  accelerateModel = pexConfig.Field(
143  dtype=float,
144  doc="Factor to amplify the differences between model planes by to speed convergence.",
145  default=3,
146  )
147  doCalculatePsf = pexConfig.Field(
148  dtype=bool,
149  doc="Set to detect stars and recalculate the PSF from the final coadd."
150  "Otherwise the PSF is estimated from a selection of the best input exposures",
151  default=False,
152  )
153  detectPsfSources = pexConfig.ConfigurableField(
154  target=measAlg.SourceDetectionTask,
155  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
156  )
157  measurePsfSources = pexConfig.ConfigurableField(
158  target=SingleFrameMeasurementTask,
159  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
160  )
161  measurePsf = pexConfig.ConfigurableField(
162  target=MeasurePsfTask,
163  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
164  )
165 
166  def setDefaults(self):
167  CompareWarpAssembleCoaddConfig.setDefaults(self)
168  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
169  self.doNImage = True
170  self.warpType = "direct"
171  self.assembleStaticSkyModel.warpType = self.warpType
172  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
173  self.assembleStaticSkyModel.doNImage = False
174  self.assembleStaticSkyModel.doWrite = False
175  self.detectPsfSources.returnOriginalFootprints = False
176  self.detectPsfSources.thresholdPolarity = "positive"
177  # Only use bright sources for PSF measurement
178  self.detectPsfSources.thresholdValue = 50
179  self.detectPsfSources.nSigmaToGrow = 2
180  # A valid star for PSF measurement should at least fill 5x5 pixels
181  self.detectPsfSources.minPixels = 25
182  # Use the variance plane to calculate signal to noise
183  self.detectPsfSources.thresholdType = "pixel_stdev"
184  # The signal to noise limit is good enough, while the flux limit is set
185  # in dimensionless units and may not be appropriate for all data sets.
186  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
187 
188 
190  """Assemble DCR coadded images from a set of warps.
191 
192  Attributes
193  ----------
194  bufferSize : `int`
195  The number of pixels to grow each subregion by to allow for DCR.
196 
197  Notes
198  -----
199  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
200  Warps (also called coadded temporary exposures), including the effects of
201  Differential Chromatic Refraction (DCR).
202  For full details of the mathematics and algorithm, please see
203  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
204 
205  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
206  each subfilter used in the iterative calculation.
207  It begins by dividing the bandpass-defining filter into N equal bandwidth
208  "subfilters", and divides the flux in each pixel from an initial coadd
209  equally into each as a "dcrModel". Because the airmass and parallactic
210  angle of each individual exposure is known, we can calculate the shift
211  relative to the center of the band in each subfilter due to DCR. For each
212  exposure we apply this shift as a linear transformation to the dcrModels
213  and stack the results to produce a DCR-matched exposure. The matched
214  exposures are subtracted from the input exposures to produce a set of
215  residual images, and these residuals are reverse shifted for each
216  exposures' subfilters and stacked. The shifted and stacked residuals are
217  added to the dcrModels to produce a new estimate of the flux in each pixel
218  within each subfilter. The dcrModels are solved for iteratively, which
219  continues until the solution from a new iteration improves by less than
220  a set percentage, or a maximum number of iterations is reached.
221  Two forms of regularization are employed to reduce unphysical results.
222  First, the new solution is averaged with the solution from the previous
223  iteration, which mitigates oscillating solutions where the model
224  overshoots with alternating very high and low values.
225  Second, a common degeneracy when the data have a limited range of airmass or
226  parallactic angle values is for one subfilter to be fit with very low or
227  negative values, while another subfilter is fit with very high values. This
228  typically appears in the form of holes next to sources in one subfilter,
229  and corresponding extended wings in another. Because each subfilter has
230  a narrow bandwidth we assume that physical sources that are above the noise
231  level will not vary in flux by more than a factor of `frequencyClampFactor`
232  between subfilters, and pixels that have flux deviations larger than that
233  factor will have the excess flux distributed evenly among all subfilters.
234  If `splitSubfilters` is set, then each subfilter will be further sub-
235  divided during the forward modeling step (only). This approximates using
236  a higher number of subfilters that may be necessary for high airmass
237  observations, but does not increase the number of free parameters in the
238  fit. This is needed when there are high airmass observations which would
239  otherwise have significant DCR even within a subfilter. Because calculating
240  the shifted images takes most of the time, splitting the subfilters is
241  turned off by way of the `splitThreshold` option for low-airmass
242  observations that do not suffer from DCR within a subfilter.
243  """
244 
245  ConfigClass = DcrAssembleCoaddConfig
246  _DefaultName = "dcrAssembleCoadd"
247 
248  def __init__(self, *args, **kwargs):
249  super().__init__(*args, **kwargs)
250  if self.config.doCalculatePsf:
251  self.schema = afwTable.SourceTable.makeMinimalSchema()
252  self.makeSubtask("detectPsfSources", schema=self.schema)
253  self.makeSubtask("measurePsfSources", schema=self.schema)
254  self.makeSubtask("measurePsf", schema=self.schema)
255 
256  @pipeBase.timeMethod
257  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258  """Assemble a coadd from a set of warps.
259 
260  Coadd a set of Warps. Compute weights to be applied to each Warp and
261  find scalings to match the photometric zeropoint to a reference Warp.
262  Assemble the Warps using run method.
263  Forward model chromatic effects across multiple subfilters,
264  and subtract from the input Warps to build sets of residuals.
265  Use the residuals to construct a new ``DcrModel`` for each subfilter,
266  and iterate until the model converges.
267  Interpolate over NaNs and optionally write the coadd to disk.
268  Return the coadded exposure.
269 
270  Parameters
271  ----------
272  dataRef : `lsst.daf.persistence.ButlerDataRef`
273  Data reference defining the patch for coaddition and the
274  reference Warp
275  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
276  List of data references to warps. Data to be coadded will be
277  selected from this list based on overlap with the patch defined by
278  the data reference.
279 
280  Returns
281  -------
282  results : `lsst.pipe.base.Struct`
283  The Struct contains the following fields:
284 
285  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
286  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
287  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
288  - ``dcrNImages``: `list` of exposure count images for each subfilter
289  """
290  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
291  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
292 
293  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294  warpRefList=warpRefList)
295  if results is None:
296  skyInfo = self.getSkyInfo(dataRef)
297  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
298  skyInfo.patchInfo.getIndex())
299  return
300  if self.config.doCalculatePsf:
301  self.measureCoaddPsf(results.coaddExposure)
302  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
303  for subfilter in range(self.config.dcrNumSubfilters):
304  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
305  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
306  self.processResults(results.dcrCoadds[subfilter],
307  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
308  if self.config.doWrite:
309  self.log.info("Persisting dcrCoadd")
310  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
311  numSubfilters=self.config.dcrNumSubfilters)
312  if self.config.doNImage and results.dcrNImages is not None:
313  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
314  numSubfilters=self.config.dcrNumSubfilters)
315 
316  return results
317 
318  def measureCoaddPsf(self, coaddExposure):
319  """Detect sources on the coadd exposure and measure the final PSF.
320 
321  Parameters
322  ----------
323  coaddExposure : `lsst.afw.image.Exposure`
324  The final coadded exposure.
325  """
326  table = afwTable.SourceTable.make(self.schema)
327  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
328  coaddSources = detResults.sources
329  self.measurePsfSources.run(
330  measCat=coaddSources,
331  exposure=coaddExposure
332  )
333  # Measure the PSF on the stacked subfilter coadds if possible.
334  # We should already have a decent estimate of the coadd PSF, however,
335  # so in case of any errors simply log them as a warning and use the
336  # default PSF.
337  try:
338  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
339  except Exception as e:
340  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
341  else:
342  coaddExposure.setPsf(psfResults.psf)
343 
344  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
345  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
346 
347  Sets the property ``bufferSize``.
348 
349  Parameters
350  ----------
351  templateCoadd : `lsst.afw.image.ExposureF`
352  The initial coadd exposure before accounting for DCR.
353  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
354  The data references to the input warped exposures.
355  weightList : `list` of `float`
356  The weight to give each input exposure in the coadd
357  Will be modified in place if ``doAirmassWeight`` is set.
358 
359  Returns
360  -------
361  dcrModels : `lsst.pipe.tasks.DcrModel`
362  Best fit model of the true sky after correcting chromatic effects.
363 
364  Raises
365  ------
366  NotImplementedError
367  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
368  """
369  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
370  filterInfo = templateCoadd.getFilter()
371  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
372  raise NotImplementedError("No minimum/maximum wavelength information found"
373  " in the filter definition! Please add lambdaMin and lambdaMax"
374  " to the Mapper class in your obs package.")
375  tempExpName = self.getTempExpDatasetName(self.warpType)
376  dcrShifts = []
377  airmassDict = {}
378  angleDict = {}
379  psfSizeDict = {}
380  for visitNum, warpExpRef in enumerate(warpRefList):
381  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
382  visit = warpExpRef.dataId["visit"]
383  psf = warpExpRef.get(tempExpName).getPsf()
384  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
385  airmass = visitInfo.getBoresightAirmass()
386  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
387  airmassDict[visit] = airmass
388  angleDict[visit] = parallacticAngle
389  psfSizeDict[visit] = psfSize
390  if self.config.doAirmassWeight:
391  weightList[visitNum] *= airmass
392  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
393  filterInfo, self.config.dcrNumSubfilters))))
394  self.log.info("Selected airmasses:\n%s", airmassDict)
395  self.log.info("Selected parallactic angles:\n%s", angleDict)
396  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
397  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
398  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
399  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
400  self.config.dcrNumSubfilters,
401  filterInfo=filterInfo,
402  psf=psf)
403  return dcrModels
404 
405  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
406  supplementaryData=None):
407  """Assemble the coadd.
408 
409  Requires additional inputs Struct ``supplementaryData`` to contain a
410  ``templateCoadd`` that serves as the model of the static sky.
411 
412  Find artifacts and apply them to the warps' masks creating a list of
413  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
414  Then pass these alternative masks to the base class's assemble method.
415 
416  Divide the ``templateCoadd`` evenly between each subfilter of a
417  ``DcrModel`` as the starting best estimate of the true wavelength-
418  dependent sky. Forward model the ``DcrModel`` using the known
419  chromatic effects in each subfilter and calculate a convergence metric
420  based on how well the modeled template matches the input warps. If
421  the convergence has not yet reached the desired threshold, then shift
422  and stack the residual images to build a new ``DcrModel``. Apply
423  conditioning to prevent oscillating solutions between iterations or
424  between subfilters.
425 
426  Once the ``DcrModel`` reaches convergence or the maximum number of
427  iterations has been reached, fill the metadata for each subfilter
428  image and make them proper ``coaddExposure``s.
429 
430  Parameters
431  ----------
432  skyInfo : `lsst.pipe.base.Struct`
433  Patch geometry information, from getSkyInfo
434  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
435  The data references to the input warped exposures.
436  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
437  The image scalars correct for the zero point of the exposures.
438  weightList : `list` of `float`
439  The weight to give each input exposure in the coadd
440  supplementaryData : `lsst.pipe.base.Struct`
441  Result struct returned by ``makeSupplementaryData`` with components:
442 
443  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
444 
445  Returns
446  -------
447  result : `lsst.pipe.base.Struct`
448  Result struct with components:
449 
450  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
451  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
452  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
453  - ``dcrNImages``: `list` of exposure count images for each subfilter
454  """
455  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
456  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
457  templateCoadd = supplementaryData.templateCoadd
458  baseMask = templateCoadd.mask.clone()
459  # The variance plane is for each subfilter
460  # and should be proportionately lower than the full-band image
461  baseVariance = templateCoadd.variance.clone()
462  baseVariance /= self.config.dcrNumSubfilters
463  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
464  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
465  templateCoadd.setMask(baseMask)
466  badMaskPlanes = self.config.badMaskPlanes[:]
467  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
468  # This is because pixels in observations that are significantly affect by DCR
469  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
470  # but those are necessary to constrain the DCR model.
471  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
472 
473  stats = self.prepareStats(mask=badPixelMask)
474  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
475  if self.config.doNImage:
476  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
477  spanSetMaskList, stats.ctrl)
478  nImage = afwImage.ImageU(skyInfo.bbox)
479  # Note that this nImage will be a factor of dcrNumSubfilters higher than
480  # the nImage returned by assembleCoadd for most pixels. This is because each
481  # subfilter may have a different nImage, and fractional values are not allowed.
482  for dcrNImage in dcrNImages:
483  nImage += dcrNImage
484  else:
485  dcrNImages = None
486 
487  subregionSize = geom.Extent2I(*self.config.subregionSize)
488  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
489  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
490  subIter = 0
491  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
492  modelIter = 0
493  subIter += 1
494  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
495  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
496  dcrBBox = geom.Box2I(subBBox)
497  dcrBBox.grow(self.bufferSize)
498  dcrBBox.clip(dcrModels.bbox)
499  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
500  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
501  imageScalerList, spanSetMaskList)
502  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
503  warpRefList, weightList, stats.ctrl)
504  self.log.info("Initial convergence : %s", convergenceMetric)
505  convergenceList = [convergenceMetric]
506  gainList = []
507  convergenceCheck = 1.
508  refImage = templateCoadd.image
509  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
510  gain = self.calculateGain(convergenceList, gainList)
511  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
512  stats.ctrl, convergenceMetric, gain,
513  modelWeights, refImage, dcrWeights)
514  if self.config.useConvergence:
515  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
516  warpRefList, weightList, stats.ctrl)
517  if convergenceMetric == 0:
518  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
519  "most likely due to there being no valid data in the region.",
520  skyInfo.patchInfo.getIndex(), subIter)
521  break
522  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
523  if (convergenceCheck < 0) & (modelIter > minNumIter):
524  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
525  "iterations or desired convergence improvement of %s."
526  " Divergence: %s",
527  skyInfo.patchInfo.getIndex(), subIter,
528  self.config.convergenceThreshold, convergenceCheck)
529  break
530  convergenceList.append(convergenceMetric)
531  if modelIter > maxNumIter:
532  if self.config.useConvergence:
533  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
534  "before reaching desired convergence improvement of %s."
535  " Final convergence improvement: %s",
536  skyInfo.patchInfo.getIndex(), subIter,
537  self.config.convergenceThreshold, convergenceCheck)
538  break
539 
540  if self.config.useConvergence:
541  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
542  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
543  modelIter += 1
544  else:
545  if self.config.useConvergence:
546  self.log.info("Coadd patch %s subregion %s finished with "
547  "convergence metric %s after %s iterations",
548  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
549  else:
550  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
551  skyInfo.patchInfo.getIndex(), subIter, modelIter)
552  if self.config.useConvergence and convergenceMetric > 0:
553  self.log.info("Final convergence improvement was %.4f%% overall",
554  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
555 
556  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
557  calibration=self.scaleZeroPoint.getPhotoCalib(),
558  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
559  mask=baseMask,
560  variance=baseVariance)
561  coaddExposure = self.stackCoadd(dcrCoadds)
562  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
563  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
564 
565  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
566  """Calculate the number of exposures contributing to each subfilter.
567 
568  Parameters
569  ----------
570  dcrModels : `lsst.pipe.tasks.DcrModel`
571  Best fit model of the true sky after correcting chromatic effects.
572  bbox : `lsst.geom.box.Box2I`
573  Bounding box of the patch to coadd.
574  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
575  The data references to the input warped exposures.
576  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
577  Each element of the `dict` contains the new mask plane name
578  (e.g. "CLIPPED and/or "NO_DATA") as the key,
579  and the list of SpanSets to apply to the mask.
580  statsCtrl : `lsst.afw.math.StatisticsControl`
581  Statistics control object for coadd
582 
583  Returns
584  -------
585  dcrNImages : `list` of `lsst.afw.image.ImageU`
586  List of exposure count images for each subfilter
587  dcrWeights : `list` of `lsst.afw.image.ImageF`
588  Per-pixel weights for each subfilter.
589  Equal to 1/(number of unmasked images contributing to each pixel).
590  """
591  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
592  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
593  tempExpName = self.getTempExpDatasetName(self.warpType)
594  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
595  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
596  visitInfo = exposure.getInfo().getVisitInfo()
597  wcs = exposure.getInfo().getWcs()
598  mask = exposure.mask
599  if altMaskSpans is not None:
600  self.applyAltMaskPlanes(mask, altMaskSpans)
601  weightImage = np.zeros_like(exposure.image.array)
602  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
603  # The weights must be shifted in exactly the same way as the residuals,
604  # because they will be used as the denominator in the weighted average of residuals.
605  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
606  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
607  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
608  dcrWeight.array += shiftedWeights
609  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
610  weightsThreshold = 1.
611  goodPix = dcrWeights[0].array > weightsThreshold
612  for weights in dcrWeights[1:]:
613  goodPix = (weights.array > weightsThreshold) & goodPix
614  for subfilter in range(self.config.dcrNumSubfilters):
615  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
616  dcrWeights[subfilter].array[~goodPix] = 0.
617  dcrNImages[subfilter].array[~goodPix] = 0
618  return (dcrNImages, dcrWeights)
619 
620  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
621  statsCtrl, convergenceMetric,
622  gain, modelWeights, refImage, dcrWeights):
623  """Assemble the DCR coadd for a sub-region.
624 
625  Build a DCR-matched template for each input exposure, then shift the
626  residuals according to the DCR in each subfilter.
627  Stack the shifted residuals and apply them as a correction to the
628  solution from the previous iteration.
629  Restrict the new model solutions from varying by more than a factor of
630  `modelClampFactor` from the last solution, and additionally restrict the
631  individual subfilter models from varying by more than a factor of
632  `frequencyClampFactor` from their average.
633  Finally, mitigate potentially oscillating solutions by averaging the new
634  solution with the solution from the previous iteration, weighted by
635  their convergence metric.
636 
637  Parameters
638  ----------
639  dcrModels : `lsst.pipe.tasks.DcrModel`
640  Best fit model of the true sky after correcting chromatic effects.
641  subExposures : `dict` of `lsst.afw.image.ExposureF`
642  The pre-loaded exposures for the current subregion.
643  bbox : `lsst.geom.box.Box2I`
644  Bounding box of the subregion to coadd.
645  dcrBBox : `lsst.geom.box.Box2I`
646  Sub-region of the coadd which includes a buffer to allow for DCR.
647  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
648  The data references to the input warped exposures.
649  statsCtrl : `lsst.afw.math.StatisticsControl`
650  Statistics control object for coadd
651  convergenceMetric : `float`
652  Quality of fit metric for the matched templates of the input images.
653  gain : `float`, optional
654  Relative weight to give the new solution when updating the model.
655  modelWeights : `numpy.ndarray` or `float`
656  A 2D array of weight values that tapers smoothly to zero away from detected sources.
657  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
658  refImage : `lsst.afw.image.Image`
659  A reference image used to supply the default pixel values.
660  dcrWeights : `list` of `lsst.afw.image.Image`
661  Per-pixel weights for each subfilter.
662  Equal to 1/(number of unmasked images contributing to each pixel).
663  """
664  residualGeneratorList = []
665 
666  for warpExpRef in warpRefList:
667  exposure = subExposures[warpExpRef.dataId["visit"]]
668  visitInfo = exposure.getInfo().getVisitInfo()
669  wcs = exposure.getInfo().getWcs()
670  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
671  order=self.config.imageInterpOrder,
672  splitSubfilters=self.config.splitSubfilters,
673  splitThreshold=self.config.splitThreshold,
674  amplifyModel=self.config.accelerateModel)
675  residual = exposure.image.array - templateImage.array
676  # Note that the variance plane here is used to store weights, not the actual variance
677  residual *= exposure.variance.array
678  # The residuals are stored as a list of generators.
679  # This allows the residual for a given subfilter and exposure to be created
680  # on the fly, instead of needing to store them all in memory.
681  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
682 
683  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
684  gain=gain,
685  modelWeights=modelWeights,
686  refImage=refImage,
687  dcrWeights=dcrWeights)
688  dcrModels.assign(dcrSubModelOut, bbox)
689 
690  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
691  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
692 
693  Parameters
694  ----------
695  residual : `numpy.ndarray`
696  The residual masked image for one exposure,
697  after subtracting the matched template
698  visitInfo : `lsst.afw.image.VisitInfo`
699  Metadata for the exposure.
700  wcs : `lsst.afw.geom.SkyWcs`
701  Coordinate system definition (wcs) for the exposure.
702  filterInfo : `lsst.afw.image.Filter`
703  The filter definition, set in the current instruments' obs package.
704  Required for any calculation of DCR, including making matched templates.
705 
706  Yields
707  ------
708  residualImage : `numpy.ndarray`
709  The residual image for the next subfilter, shifted for DCR.
710  """
711  # Pre-calculate the spline-filtered residual image, so that step can be
712  # skipped in the shift calculation in `applyDcr`.
713  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
714  # Note that `splitSubfilters` is always turned off in the reverse direction.
715  # This option introduces additional blurring if applied to the residuals.
716  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
717  splitSubfilters=False)
718  for dcr in dcrShift:
719  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
720  doPrefilter=False, order=self.config.imageInterpOrder)
721 
722  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
723  gain, modelWeights, refImage, dcrWeights):
724  """Calculate a new DcrModel from a set of image residuals.
725 
726  Parameters
727  ----------
728  dcrModels : `lsst.pipe.tasks.DcrModel`
729  Current model of the true sky after correcting chromatic effects.
730  residualGeneratorList : `generator` of `numpy.ndarray`
731  The residual image for the next subfilter, shifted for DCR.
732  dcrBBox : `lsst.geom.box.Box2I`
733  Sub-region of the coadd which includes a buffer to allow for DCR.
734  statsCtrl : `lsst.afw.math.StatisticsControl`
735  Statistics control object for coadd
736  gain : `float`
737  Relative weight to give the new solution when updating the model.
738  modelWeights : `numpy.ndarray` or `float`
739  A 2D array of weight values that tapers smoothly to zero away from detected sources.
740  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
741  refImage : `lsst.afw.image.Image`
742  A reference image used to supply the default pixel values.
743  dcrWeights : `list` of `lsst.afw.image.Image`
744  Per-pixel weights for each subfilter.
745  Equal to 1/(number of unmasked images contributing to each pixel).
746 
747  Returns
748  -------
749  dcrModel : `lsst.pipe.tasks.DcrModel`
750  New model of the true sky after correcting chromatic effects.
751  """
752  newModelImages = []
753  for subfilter, model in enumerate(dcrModels):
754  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
755  residual = np.sum(residualsList, axis=0)
756  residual *= dcrWeights[subfilter][dcrBBox].array
757  # `MaskedImage`s only support in-place addition, so rename for readability
758  newModel = model[dcrBBox].clone()
759  newModel.array += residual
760  # Catch any invalid values
761  badPixels = ~np.isfinite(newModel.array)
762  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
763  if self.config.regularizeModelIterations > 0:
764  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
765  self.config.regularizeModelIterations,
766  self.config.regularizationWidth)
767  newModelImages.append(newModel)
768  if self.config.regularizeModelFrequency > 0:
769  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
770  self.config.regularizeModelFrequency,
771  self.config.regularizationWidth)
772  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
773  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
774  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
775  dcrModels.mask, dcrModels.variance)
776 
777  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
778  """Calculate a quality of fit metric for the matched templates.
779 
780  Parameters
781  ----------
782  dcrModels : `lsst.pipe.tasks.DcrModel`
783  Best fit model of the true sky after correcting chromatic effects.
784  subExposures : `dict` of `lsst.afw.image.ExposureF`
785  The pre-loaded exposures for the current subregion.
786  bbox : `lsst.geom.box.Box2I`
787  Sub-region to coadd
788  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
789  The data references to the input warped exposures.
790  weightList : `list` of `float`
791  The weight to give each input exposure in the coadd
792  statsCtrl : `lsst.afw.math.StatisticsControl`
793  Statistics control object for coadd
794 
795  Returns
796  -------
797  convergenceMetric : `float`
798  Quality of fit metric for all input exposures, within the sub-region
799  """
800  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
801  nSigma = 3.
802  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
803  bufferSize=self.bufferSize)
804  if np.max(significanceImage) == 0:
805  significanceImage += 1.
806  weight = 0
807  metric = 0.
808  metricList = {}
809  for warpExpRef, expWeight in zip(warpRefList, weightList):
810  exposure = subExposures[warpExpRef.dataId["visit"]][bbox]
811  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
812  metric += singleMetric
813  metricList[warpExpRef.dataId["visit"]] = singleMetric
814  weight += 1.
815  self.log.info("Individual metrics:\n%s", metricList)
816  return 1.0 if weight == 0.0 else metric/weight
817 
818  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
819  """Calculate a quality of fit metric for a single matched template.
820 
821  Parameters
822  ----------
823  dcrModels : `lsst.pipe.tasks.DcrModel`
824  Best fit model of the true sky after correcting chromatic effects.
825  exposure : `lsst.afw.image.ExposureF`
826  The input warped exposure to evaluate.
827  significanceImage : `numpy.ndarray`
828  Array of weights for each pixel corresponding to its significance
829  for the convergence calculation.
830  statsCtrl : `lsst.afw.math.StatisticsControl`
831  Statistics control object for coadd
832 
833  Returns
834  -------
835  convergenceMetric : `float`
836  Quality of fit metric for one exposure, within the sub-region.
837  """
838  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
839  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
840  order=self.config.imageInterpOrder,
841  splitSubfilters=self.config.splitSubfilters,
842  splitThreshold=self.config.splitThreshold,
843  amplifyModel=self.config.accelerateModel)
844  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
845  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
846 
847  finitePixels = np.isfinite(diffVals)
848  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
849  convergeMaskPixels = exposure.mask.array & convergeMask > 0
850  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
851  if np.sum(usePixels) == 0:
852  metric = 0.
853  else:
854  diffUse = diffVals[usePixels]
855  refUse = refVals[usePixels]
856  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
857  return metric
858 
859  def stackCoadd(self, dcrCoadds):
860  """Add a list of sub-band coadds together.
861 
862  Parameters
863  ----------
864  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
865  A list of coadd exposures, each exposure containing
866  the model for one subfilter.
867 
868  Returns
869  -------
870  coaddExposure : `lsst.afw.image.ExposureF`
871  A single coadd exposure that is the sum of the sub-bands.
872  """
873  coaddExposure = dcrCoadds[0].clone()
874  for coadd in dcrCoadds[1:]:
875  coaddExposure.maskedImage += coadd.maskedImage
876  return coaddExposure
877 
878  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
879  mask=None, variance=None):
880  """Create a list of coadd exposures from a list of masked images.
881 
882  Parameters
883  ----------
884  dcrModels : `lsst.pipe.tasks.DcrModel`
885  Best fit model of the true sky after correcting chromatic effects.
886  skyInfo : `lsst.pipe.base.Struct`
887  Patch geometry information, from getSkyInfo
888  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
889  The data references to the input warped exposures.
890  weightList : `list` of `float`
891  The weight to give each input exposure in the coadd
892  calibration : `lsst.afw.Image.PhotoCalib`, optional
893  Scale factor to set the photometric calibration of an exposure.
894  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
895  A record of the observations that are included in the coadd.
896  mask : `lsst.afw.image.Mask`, optional
897  Optional mask to override the values in the final coadd.
898  variance : `lsst.afw.image.Image`, optional
899  Optional variance plane to override the values in the final coadd.
900 
901  Returns
902  -------
903  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
904  A list of coadd exposures, each exposure containing
905  the model for one subfilter.
906  """
907  dcrCoadds = []
908  refModel = dcrModels.getReferenceImage()
909  for model in dcrModels:
910  if self.config.accelerateModel > 1:
911  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
912  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
913  if calibration is not None:
914  coaddExposure.setPhotoCalib(calibration)
915  if coaddInputs is not None:
916  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
917  # Set the metadata for the coadd, including PSF and aperture corrections.
918  self.assembleMetadata(coaddExposure, warpRefList, weightList)
919  # Overwrite the PSF
920  coaddExposure.setPsf(dcrModels.psf)
921  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
922  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
923  maskedImage.image = model
924  maskedImage.mask = dcrModels.mask
925  maskedImage.variance = dcrModels.variance
926  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
927  if mask is not None:
928  coaddExposure.setMask(mask)
929  if variance is not None:
930  coaddExposure.setVariance(variance)
931  dcrCoadds.append(coaddExposure)
932  return dcrCoadds
933 
934  def calculateGain(self, convergenceList, gainList):
935  """Calculate the gain to use for the current iteration.
936 
937  After calculating a new DcrModel, each value is averaged with the
938  value in the corresponding pixel from the previous iteration. This
939  reduces oscillating solutions that iterative techniques are plagued by,
940  and speeds convergence. By far the biggest changes to the model
941  happen in the first couple iterations, so we can also use a more
942  aggressive gain later when the model is changing slowly.
943 
944  Parameters
945  ----------
946  convergenceList : `list` of `float`
947  The quality of fit metric from each previous iteration.
948  gainList : `list` of `float`
949  The gains used in each previous iteration: appended with the new
950  gain value.
951  Gains are numbers between ``self.config.baseGain`` and 1.
952 
953  Returns
954  -------
955  gain : `float`
956  Relative weight to give the new solution when updating the model.
957  A value of 1.0 gives equal weight to both solutions.
958 
959  Raises
960  ------
961  ValueError
962  If ``len(convergenceList) != len(gainList)+1``.
963  """
964  nIter = len(convergenceList)
965  if nIter != len(gainList) + 1:
966  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
967  % (len(convergenceList), len(gainList)))
968 
969  if self.config.baseGain is None:
970  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
971  # The more subfilters being modeled, the lower the gain should be.
972  baseGain = 1./(self.config.dcrNumSubfilters - 1)
973  else:
974  baseGain = self.config.baseGain
975 
976  if self.config.useProgressiveGain and nIter > 2:
977  # To calculate the best gain to use, compare the past gains that have been used
978  # with the resulting convergences to estimate the best gain to use.
979  # Algorithmically, this is a Kalman filter.
980  # If forward modeling proceeds perfectly, the convergence metric should
981  # asymptotically approach a final value.
982  # We can estimate that value from the measured changes in convergence
983  # weighted by the gains used in each previous iteration.
984  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
985  for i in range(nIter - 1)]
986  # The convergence metric is strictly positive, so if the estimated final convergence is
987  # less than zero, force it to zero.
988  estFinalConv = np.array(estFinalConv)
989  estFinalConv[estFinalConv < 0] = 0
990  # Because the estimate may slowly change over time, only use the most recent measurements.
991  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
992  lastGain = gainList[-1]
993  lastConv = convergenceList[-2]
994  newConv = convergenceList[-1]
995  # The predicted convergence is the value we would get if the new model calculated
996  # in the previous iteration was perfect. Recall that the updated model that is
997  # actually used is the gain-weighted average of the new and old model,
998  # so the convergence would be similarly weighted.
999  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1000  # If the measured and predicted convergence are very close, that indicates
1001  # that our forward model is accurate and we can use a more aggressive gain
1002  # If the measured convergence is significantly worse (or better!) than predicted,
1003  # that indicates that the model is not converging as expected and
1004  # we should use a more conservative gain.
1005  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1006  newGain = 1 - abs(delta)
1007  # Average the gains to prevent oscillating solutions.
1008  newGain = (newGain + lastGain)/2.
1009  gain = max(baseGain, newGain)
1010  else:
1011  gain = baseGain
1012  gainList.append(gain)
1013  return gain
1014 
1015  def calculateModelWeights(self, dcrModels, dcrBBox):
1016  """Build an array that smoothly tapers to 0 away from detected sources.
1017 
1018  Parameters
1019  ----------
1020  dcrModels : `lsst.pipe.tasks.DcrModel`
1021  Best fit model of the true sky after correcting chromatic effects.
1022  dcrBBox : `lsst.geom.box.Box2I`
1023  Sub-region of the coadd which includes a buffer to allow for DCR.
1024 
1025  Returns
1026  -------
1027  weights : `numpy.ndarray` or `float`
1028  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1029  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1030 
1031  Raises
1032  ------
1033  ValueError
1034  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1035  """
1036  if not self.config.useModelWeights:
1037  return 1.0
1038  if self.config.modelWeightsWidth < 0:
1039  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1040  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1041  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1042  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1043  weights[convergeMaskPixels] = 1.
1044  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1045  weights /= np.max(weights)
1046  return weights
1047 
1048  def applyModelWeights(self, modelImages, refImage, modelWeights):
1049  """Smoothly replace model pixel values with those from a
1050  reference at locations away from detected sources.
1051 
1052  Parameters
1053  ----------
1054  modelImages : `list` of `lsst.afw.image.Image`
1055  The new DCR model images from the current iteration.
1056  The values will be modified in place.
1057  refImage : `lsst.afw.image.MaskedImage`
1058  A reference image used to supply the default pixel values.
1059  modelWeights : `numpy.ndarray` or `float`
1060  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1061  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1062  """
1063  if self.config.useModelWeights:
1064  for model in modelImages:
1065  model.array *= modelWeights
1066  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1067 
1068  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1069  """Pre-load sub-regions of a list of exposures.
1070 
1071  Parameters
1072  ----------
1073  bbox : `lsst.geom.box.Box2I`
1074  Sub-region to coadd
1075  statsCtrl : `lsst.afw.math.StatisticsControl`
1076  Statistics control object for coadd
1077  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1078  The data references to the input warped exposures.
1079  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1080  The image scalars correct for the zero point of the exposures.
1081  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1082  Each element is dict with keys = mask plane name to add the spans to
1083 
1084  Returns
1085  -------
1086  subExposures : `dict`
1087  The `dict` keys are the visit IDs,
1088  and the values are `lsst.afw.image.ExposureF`
1089  The pre-loaded exposures for the current subregion.
1090  The variance plane contains weights, and not the variance
1091  """
1092  tempExpName = self.getTempExpDatasetName(self.warpType)
1093  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1094  subExposures = {}
1095  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1096  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1097  if altMaskSpans is not None:
1098  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1099  imageScaler.scaleMaskedImage(exposure.maskedImage)
1100  # Note that the variance plane here is used to store weights, not the actual variance
1101  exposure.variance.array[:, :] = 0.
1102  # Set the weight of unmasked pixels to 1.
1103  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1104  # Set the image value of masked pixels to zero.
1105  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1106  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1107  subExposures[warpExpRef.dataId["visit"]] = exposure
1108  return subExposures
1109 
1110  def selectCoaddPsf(self, templateCoadd, warpRefList):
1111  """Compute the PSF of the coadd from the exposures with the best seeing.
1112 
1113  Parameters
1114  ----------
1115  templateCoadd : `lsst.afw.image.ExposureF`
1116  The initial coadd exposure before accounting for DCR.
1117  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1118  The data references to the input warped exposures.
1119 
1120  Returns
1121  -------
1122  psf : `lsst.meas.algorithms.CoaddPsf`
1123  The average PSF of the input exposures with the best seeing.
1124  """
1125  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1126  tempExpName = self.getTempExpDatasetName(self.warpType)
1127  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1128  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1129  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1130  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1131  psfSizes = np.zeros(len(ccds))
1132  ccdVisits = np.array(ccds["visit"])
1133  for warpExpRef in warpRefList:
1134  psf = warpExpRef.get(tempExpName).getPsf()
1135  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1136  visit = warpExpRef.dataId["visit"]
1137  psfSizes[ccdVisits == visit] = psfSize
1138  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1139  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1140  # of the mean or median FWHM of the input exposures.
1141  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1142  goodPsfs = psfSizes <= sizeThreshold
1143  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1144  self.config.coaddPsf.makeControl())
1145  return psf
Extent< int, N > ceil(Extent< double, N > const &input) noexcept
Return the component-wise ceil (round towards more positive).
Definition: Extent.cc:118
Angle abs(Angle const &a)
Definition: Angle.h:106
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def applyDcr(image, dcr, useInverse=False, splitSubfilters=False, splitThreshold=0., doPrefilter=True, order=3)
Definition: dcrModel.py:585
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
int min
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
int max
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters, splitSubfilters=False)
Definition: dcrModel.py:649
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
An integer coordinate rectangle.
Definition: Box.h:55
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)