LSSTApplications  17.0+11,17.0+34,17.0+56,17.0+57,17.0+59,17.0+7,17.0-1-g377950a+33,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+28,17.0.1-1-g55520dc+49,17.0.1-1-g5f4ed7e+52,17.0.1-1-g6dd7d69+17,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+55,17.0.1-1-gfc65f5f+50,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+1,17.0.1-11-ge9de802+16,17.0.1-16-ga14f7d5c+4,17.0.1-17-gc79d625+1,17.0.1-17-gdae4c4a+8,17.0.1-2-g26618f5+29,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-20-g2ca2f74+6,17.0.1-23-gf3eadeb7+1,17.0.1-3-g7e86b59+39,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+40,17.0.1-30-g596af8797,17.0.1-4-g59d126d+4,17.0.1-4-gc69c472+5,17.0.1-6-g5afd9b9+4,17.0.1-7-g35889ee+1,17.0.1-7-gc7c8782+18,17.0.1-9-gc4bbfb2+3,w.2019.22
LSSTDataManagementBasePackage
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.meas.algorithms as measAlg
32 from lsst.meas.base import SingleFrameMeasurementTask
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf import MeasurePsfTask
37 from .multiBand import DetectCoaddSourcesTask
38 from .multiBandUtils import _makeMakeIdFactory
39 
40 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
41 
42 
44  dcrNumSubfilters = pexConfig.Field(
45  dtype=int,
46  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
47  default=3,
48  )
49  maxNumIter = pexConfig.Field(
50  dtype=int,
51  optional=True,
52  doc="Maximum number of iterations of forward modeling.",
53  default=None,
54  )
55  minNumIter = pexConfig.Field(
56  dtype=int,
57  optional=True,
58  doc="Minimum number of iterations of forward modeling.",
59  default=None,
60  )
61  convergenceThreshold = pexConfig.Field(
62  dtype=float,
63  doc="Target relative change in convergence between iterations of forward modeling.",
64  default=0.001,
65  )
66  useConvergence = pexConfig.Field(
67  dtype=bool,
68  doc="Use convergence test as a forward modeling end condition?"
69  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
70  default=True,
71  )
72  baseGain = pexConfig.Field(
73  dtype=float,
74  optional=True,
75  doc="Relative weight to give the new solution vs. the last solution when updating the model."
76  "A value of 1.0 gives equal weight to both solutions."
77  "Small values imply slower convergence of the solution, but can "
78  "help prevent overshooting and failures in the fit."
79  "If ``baseGain`` is None, a conservative gain "
80  "will be calculated from the number of subfilters. ",
81  default=None,
82  )
83  useProgressiveGain = pexConfig.Field(
84  dtype=bool,
85  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
86  "When calculating the next gain, we use up to 5 previous gains and convergence values."
87  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
88  default=True,
89  )
90  doAirmassWeight = pexConfig.Field(
91  dtype=bool,
92  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
93  default=False,
94  )
95  modelWeightsWidth = pexConfig.Field(
96  dtype=float,
97  doc="Width of the region around detected sources to include in the DcrModel.",
98  default=3,
99  )
100  useModelWeights = pexConfig.Field(
101  dtype=bool,
102  doc="Width of the region around detected sources to include in the DcrModel.",
103  default=True,
104  )
105  splitSubfilters = pexConfig.Field(
106  dtype=bool,
107  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
108  "Instead of at the midpoint",
109  default=True,
110  )
111  regularizeModelIterations = pexConfig.Field(
112  dtype=float,
113  doc="Maximum relative change of the model allowed between iterations."
114  "Set to zero to disable.",
115  default=2.,
116  )
117  regularizeModelFrequency = pexConfig.Field(
118  dtype=float,
119  doc="Maximum relative change of the model allowed between subfilters."
120  "Set to zero to disable.",
121  default=4.,
122  )
123  convergenceMaskPlanes = pexConfig.ListField(
124  dtype=str,
125  default=["DETECTED"],
126  doc="Mask planes to use to calculate convergence."
127  )
128  regularizationWidth = pexConfig.Field(
129  dtype=int,
130  default=2,
131  doc="Minimum radius of a region to include in regularization, in pixels."
132  )
133  imageInterpOrder = pexConfig.Field(
134  dtype=int,
135  doc="The order of the spline interpolation used to shift the image plane.",
136  default=3,
137  )
138  accelerateModel = pexConfig.Field(
139  dtype=float,
140  doc="Factor to amplify the differences between model planes by to speed convergence.",
141  default=3,
142  )
143  doCalculatePsf = pexConfig.Field(
144  dtype=bool,
145  doc="Set to detect stars and recalculate the PSF from the final coadd."
146  "Otherwise the PSF is estimated from a selection of the best input exposures",
147  default=True,
148  )
149  detection = pexConfig.ConfigurableField(
150  target=DetectCoaddSourcesTask,
151  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
152  )
153  measurement = pexConfig.ConfigurableField(
154  target=SingleFrameMeasurementTask,
155  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
156  )
157  measurePsf = pexConfig.ConfigurableField(
158  target=MeasurePsfTask,
159  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
160  )
161 
162  def setDefaults(self):
163  CompareWarpAssembleCoaddConfig.setDefaults(self)
164  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
165  self.doNImage = True
166  self.warpType = 'direct'
167  self.assembleStaticSkyModel.warpType = self.warpType
168  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
169  self.assembleStaticSkyModel.doNImage = False
170  self.assembleStaticSkyModel.doWrite = False
171  self.detection.detection.thresholdValue = 10.0
172 
173 
175  """Assemble DCR coadded images from a set of warps.
176 
177  Attributes
178  ----------
179  bufferSize : `int`
180  The number of pixels to grow each subregion by to allow for DCR.
181 
182  Notes
183  -----
184  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
185  Warps (also called coadded temporary exposures), including the effects of
186  Differential Chromatic Refraction (DCR).
187  For full details of the mathematics and algorithm, please see
188  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
189 
190  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
191  each subfilter used in the iterative calculation.
192  It begins by dividing the bandpass-defining filter into N equal bandwidth
193  "subfilters", and divides the flux in each pixel from an initial coadd
194  equally into each as a "dcrModel". Because the airmass and parallactic
195  angle of each individual exposure is known, we can calculate the shift
196  relative to the center of the band in each subfilter due to DCR. For each
197  exposure we apply this shift as a linear transformation to the dcrModels
198  and stack the results to produce a DCR-matched exposure. The matched
199  exposures are subtracted from the input exposures to produce a set of
200  residual images, and these residuals are reverse shifted for each
201  exposures' subfilters and stacked. The shifted and stacked residuals are
202  added to the dcrModels to produce a new estimate of the flux in each pixel
203  within each subfilter. The dcrModels are solved for iteratively, which
204  continues until the solution from a new iteration improves by less than
205  a set percentage, or a maximum number of iterations is reached.
206  Two forms of regularization are employed to reduce unphysical results.
207  First, the new solution is averaged with the solution from the previous
208  iteration, which mitigates oscillating solutions where the model
209  overshoots with alternating very high and low values.
210  Second, a common degeneracy when the data have a limited range of airmass or
211  parallactic angle values is for one subfilter to be fit with very low or
212  negative values, while another subfilter is fit with very high values. This
213  typically appears in the form of holes next to sources in one subfilter,
214  and corresponding extended wings in another. Because each subfilter has
215  a narrow bandwidth we assume that physical sources that are above the noise
216  level will not vary in flux by more than a factor of `frequencyClampFactor`
217  between subfilters, and pixels that have flux deviations larger than that
218  factor will have the excess flux distributed evenly among all subfilters.
219  """
220 
221  ConfigClass = DcrAssembleCoaddConfig
222  _DefaultName = "dcrAssembleCoadd"
223  makeIdFactory = _makeMakeIdFactory("CoaddId")
224 
225  def __init__(self, *args, **kwargs):
226  super().__init__(*args, **kwargs)
227  if self.config.doCalculatePsf:
228  schema = afwTable.SourceTable.makeMinimalSchema()
229  self.makeSubtask("detection", schema=schema)
230  self.makeSubtask('measurement', schema=schema)
231  self.makeSubtask("measurePsf", schema=schema)
232 
233  @pipeBase.timeMethod
234  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
235  """Assemble a coadd from a set of warps.
236 
237  Coadd a set of Warps. Compute weights to be applied to each Warp and
238  find scalings to match the photometric zeropoint to a reference Warp.
239  Assemble the Warps using run method.
240  Forward model chromatic effects across multiple subfilters,
241  and subtract from the input Warps to build sets of residuals.
242  Use the residuals to construct a new ``DcrModel`` for each subfilter,
243  and iterate until the model converges.
244  Interpolate over NaNs and optionally write the coadd to disk.
245  Return the coadded exposure.
246 
247  Parameters
248  ----------
249  dataRef : `lsst.daf.persistence.ButlerDataRef`
250  Data reference defining the patch for coaddition and the
251  reference Warp
252  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
253  List of data references to warps. Data to be coadded will be
254  selected from this list based on overlap with the patch defined by
255  the data reference.
256 
257  Returns
258  -------
259  results : `lsst.pipe.base.Struct`
260  The Struct contains the following fields:
261 
262  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
263  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
264  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
265  - ``dcrNImages``: `list` of exposure count images for each subfilter
266  """
267  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
268  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
269 
270  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
271  warpRefList=warpRefList)
272  if results is None:
273  skyInfo = self.getSkyInfo(dataRef)
274  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
275  skyInfo.patchInfo.getIndex())
276  return
277  for subfilter in range(self.config.dcrNumSubfilters):
278  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
279  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
280  AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
281  if self.config.doWrite:
282  self.log.info("Persisting dcrCoadd")
283  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
284  numSubfilters=self.config.dcrNumSubfilters)
285  if self.config.doNImage and results.dcrNImages is not None:
286  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
287  numSubfilters=self.config.dcrNumSubfilters)
288 
289  return results
290 
291  def processResults(self, coaddExposure, dataRef):
292  """Interpolate over missing data and mask bright stars.
293 
294  Also detect sources on the coadd exposure and measure the final PSF,
295  if ``doCalculatePsf`` is set.
296 
297  Parameters
298  ----------
299  coaddExposure : `lsst.afw.image.Exposure`
300  The final coadded exposure.
301  dataRef : `lsst.daf.persistence.ButlerDataRef`
302  Data reference defining the patch for coaddition and the
303  reference Warp
304  """
305  super().processResults(coaddExposure, dataRef)
306 
307  if self.config.doCalculatePsf:
308  idFactory = self.makeIdFactory(dataRef)
309  expId = dataRef.get("dcrCoaddId")
310  detResults = self.detection.run(coaddExposure, idFactory, expId)
311  coaddSources = detResults.outputSources
312  self.measurement.run(
313  measCat=coaddSources,
314  exposure=coaddExposure,
315  exposureId=expId
316  )
317  psfResults = self.measurePsf.run(coaddExposure, coaddSources, expId=expId)
318  coaddExposure.setPsf(psfResults.psf)
319 
320  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
321  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
322 
323  Sets the property ``bufferSize``.
324 
325  Parameters
326  ----------
327  templateCoadd : `lsst.afw.image.ExposureF`
328  The initial coadd exposure before accounting for DCR.
329  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
330  The data references to the input warped exposures.
331  weightList : `list` of `float`
332  The weight to give each input exposure in the coadd
333  Will be modified in place if ``doAirmassWeight`` is set.
334 
335  Returns
336  -------
337  dcrModels : `lsst.pipe.tasks.DcrModel`
338  Best fit model of the true sky after correcting chromatic effects.
339 
340  Raises
341  ------
342  NotImplementedError
343  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
344  """
345  filterInfo = templateCoadd.getFilter()
346  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
347  raise NotImplementedError("No minimum/maximum wavelength information found"
348  " in the filter definition! Please add lambdaMin and lambdaMax"
349  " to the Mapper class in your obs package.")
350  tempExpName = self.getTempExpDatasetName(self.warpType)
351  dcrShifts = []
352  airmassDict = {}
353  angleDict = {}
354  for visitNum, warpExpRef in enumerate(warpRefList):
355  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
356  visit = warpExpRef.dataId["visit"]
357  airmass = visitInfo.getBoresightAirmass()
358  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
359  airmassDict[visit] = airmass
360  angleDict[visit] = parallacticAngle
361  if self.config.doAirmassWeight:
362  weightList[visitNum] *= airmass
363  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
364  filterInfo, self.config.dcrNumSubfilters))))
365  self.log.info("Selected airmasses:\n%s", airmassDict)
366  self.log.info("Selected parallactic angles:\n%s", angleDict)
367  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
368  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
369  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
370  self.config.dcrNumSubfilters,
371  filterInfo=filterInfo,
372  psf=psf)
373  return dcrModels
374 
375  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
376  supplementaryData=None):
377  """Assemble the coadd.
378 
379  Requires additional inputs Struct ``supplementaryData`` to contain a
380  ``templateCoadd`` that serves as the model of the static sky.
381 
382  Find artifacts and apply them to the warps' masks creating a list of
383  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
384  Then pass these alternative masks to the base class's assemble method.
385 
386  Divide the ``templateCoadd`` evenly between each subfilter of a
387  ``DcrModel`` as the starting best estimate of the true wavelength-
388  dependent sky. Forward model the ``DcrModel`` using the known
389  chromatic effects in each subfilter and calculate a convergence metric
390  based on how well the modeled template matches the input warps. If
391  the convergence has not yet reached the desired threshold, then shift
392  and stack the residual images to build a new ``DcrModel``. Apply
393  conditioning to prevent oscillating solutions between iterations or
394  between subfilters.
395 
396  Once the ``DcrModel`` reaches convergence or the maximum number of
397  iterations has been reached, fill the metadata for each subfilter
398  image and make them proper ``coaddExposure``s.
399 
400  Parameters
401  ----------
402  skyInfo : `lsst.pipe.base.Struct`
403  Patch geometry information, from getSkyInfo
404  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
405  The data references to the input warped exposures.
406  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
407  The image scalars correct for the zero point of the exposures.
408  weightList : `list` of `float`
409  The weight to give each input exposure in the coadd
410  supplementaryData : `lsst.pipe.base.Struct`
411  Result struct returned by ``makeSupplementaryData`` with components:
412 
413  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
414 
415  Returns
416  -------
417  result : `lsst.pipe.base.Struct`
418  Result struct with components:
419 
420  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
421  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
422  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
423  - ``dcrNImages``: `list` of exposure count images for each subfilter
424  """
425  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
426  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
427  templateCoadd = supplementaryData.templateCoadd
428  baseMask = templateCoadd.mask.clone()
429  # The variance plane is for each subfilter
430  # and should be proportionately lower than the full-band image
431  baseVariance = templateCoadd.variance.clone()
432  baseVariance /= self.config.dcrNumSubfilters
433  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
434  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
435  templateCoadd.setMask(baseMask)
436  badMaskPlanes = self.config.badMaskPlanes[:]
437  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
438  # This is because pixels in observations that are significantly affect by DCR
439  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
440  # but those are necessary to constrain the DCR model.
441  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
442 
443  stats = self.prepareStats(mask=badPixelMask)
444  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
445  if self.config.doNImage:
446  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
447  spanSetMaskList, stats.ctrl)
448  nImage = afwImage.ImageU(skyInfo.bbox)
449  # Note that this nImage will be a factor of dcrNumSubfilters higher than
450  # the nImage returned by assembleCoadd for most pixels. This is because each
451  # subfilter may have a different nImage, and fractional values are not allowed.
452  for dcrNImage in dcrNImages:
453  nImage += dcrNImage
454  else:
455  dcrNImages = None
456 
457  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
458  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
459  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
460  subIter = 0
461  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
462  modelIter = 0
463  subIter += 1
464  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
465  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
466  dcrBBox = afwGeom.Box2I(subBBox)
467  dcrBBox.grow(self.bufferSize)
468  dcrBBox.clip(dcrModels.bbox)
469  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
470  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
471  imageScalerList, spanSetMaskList)
472  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
473  warpRefList, weightList, stats.ctrl)
474  self.log.info("Initial convergence : %s", convergenceMetric)
475  convergenceList = [convergenceMetric]
476  gainList = []
477  convergenceCheck = 1.
478  refImage = templateCoadd.image
479  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
480  gain = self.calculateGain(convergenceList, gainList)
481  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
482  stats.ctrl, convergenceMetric, baseMask, gain,
483  modelWeights, refImage, dcrWeights)
484  if self.config.useConvergence:
485  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
486  warpRefList, weightList, stats.ctrl)
487  if convergenceMetric == 0:
488  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
489  "most likely due to there being no valid data in the region.",
490  skyInfo.patchInfo.getIndex(), subIter)
491  break
492  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
493  if (convergenceCheck < 0) & (modelIter > minNumIter):
494  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
495  "iterations or desired convergence improvement of %s."
496  " Divergence: %s",
497  skyInfo.patchInfo.getIndex(), subIter,
498  self.config.convergenceThreshold, convergenceCheck)
499  break
500  convergenceList.append(convergenceMetric)
501  if modelIter > maxNumIter:
502  if self.config.useConvergence:
503  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
504  "before reaching desired convergence improvement of %s."
505  " Final convergence improvement: %s",
506  skyInfo.patchInfo.getIndex(), subIter,
507  self.config.convergenceThreshold, convergenceCheck)
508  break
509 
510  if self.config.useConvergence:
511  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
512  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
513  modelIter += 1
514  else:
515  if self.config.useConvergence:
516  self.log.info("Coadd patch %s subregion %s finished with "
517  "convergence metric %s after %s iterations",
518  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
519  else:
520  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
521  skyInfo.patchInfo.getIndex(), subIter, modelIter)
522  if self.config.useConvergence and convergenceMetric > 0:
523  self.log.info("Final convergence improvement was %.4f%% overall",
524  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
525 
526  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
527  calibration=self.scaleZeroPoint.getPhotoCalib(),
528  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
529  mask=baseMask,
530  variance=baseVariance)
531  coaddExposure = self.stackCoadd(dcrCoadds)
532  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
533  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
534 
535  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
536  """Calculate the number of exposures contributing to each subfilter.
537 
538  Parameters
539  ----------
540  dcrModels : `lsst.pipe.tasks.DcrModel`
541  Best fit model of the true sky after correcting chromatic effects.
542  bbox : `lsst.afw.geom.box.Box2I`
543  Bounding box of the patch to coadd.
544  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
545  The data references to the input warped exposures.
546  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
547  Each element of the `dict` contains the new mask plane name
548  (e.g. "CLIPPED and/or "NO_DATA") as the key,
549  and the list of SpanSets to apply to the mask.
550  statsCtrl : `lsst.afw.math.StatisticsControl`
551  Statistics control object for coadd
552 
553  Returns
554  -------
555  dcrNImages : `list` of `lsst.afw.image.ImageU`
556  List of exposure count images for each subfilter
557  dcrWeights : `list` of `lsst.afw.image.ImageF`
558  Per-pixel weights for each subfilter.
559  Equal to 1/(number of unmasked images contributing to each pixel).
560  """
561  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
562  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
563  tempExpName = self.getTempExpDatasetName(self.warpType)
564  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
565  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
566  visitInfo = exposure.getInfo().getVisitInfo()
567  wcs = exposure.getInfo().getWcs()
568  mask = exposure.mask
569  if altMaskSpans is not None:
570  self.applyAltMaskPlanes(mask, altMaskSpans)
571  weightImage = np.zeros_like(exposure.image.array)
572  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
573  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
574  for dcr, dcrNImage, dcrWeight in zip(dcrShift, dcrNImages, dcrWeights):
575  # Note that we use the same interpolation for the weights and the images.
576  shiftedWeights = applyDcr(weightImage, dcr, useInverse=True,
577  order=self.config.imageInterpOrder)
578  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
579  dcrWeight.array += shiftedWeights
580  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
581  weightsThreshold = 1.
582  goodPix = dcrWeights[0].array > weightsThreshold
583  for weights in dcrWeights[1:]:
584  goodPix = (weights.array > weightsThreshold) & goodPix
585  for subfilter in range(self.config.dcrNumSubfilters):
586  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
587  dcrWeights[subfilter].array[~goodPix] = 0.
588  dcrNImages[subfilter].array[~goodPix] = 0
589  return (dcrNImages, dcrWeights)
590 
591  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
592  statsCtrl, convergenceMetric,
593  baseMask, gain, modelWeights, refImage, dcrWeights):
594  """Assemble the DCR coadd for a sub-region.
595 
596  Build a DCR-matched template for each input exposure, then shift the
597  residuals according to the DCR in each subfilter.
598  Stack the shifted residuals and apply them as a correction to the
599  solution from the previous iteration.
600  Restrict the new model solutions from varying by more than a factor of
601  `modelClampFactor` from the last solution, and additionally restrict the
602  individual subfilter models from varying by more than a factor of
603  `frequencyClampFactor` from their average.
604  Finally, mitigate potentially oscillating solutions by averaging the new
605  solution with the solution from the previous iteration, weighted by
606  their convergence metric.
607 
608  Parameters
609  ----------
610  dcrModels : `lsst.pipe.tasks.DcrModel`
611  Best fit model of the true sky after correcting chromatic effects.
612  subExposures : `dict` of `lsst.afw.image.ExposureF`
613  The pre-loaded exposures for the current subregion.
614  bbox : `lsst.afw.geom.box.Box2I`
615  Bounding box of the subregion to coadd.
616  dcrBBox : `lsst.afw.geom.box.Box2I`
617  Sub-region of the coadd which includes a buffer to allow for DCR.
618  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
619  The data references to the input warped exposures.
620  statsCtrl : `lsst.afw.math.StatisticsControl`
621  Statistics control object for coadd
622  convergenceMetric : `float`
623  Quality of fit metric for the matched templates of the input images.
624  baseMask : `lsst.afw.image.Mask`
625  Mask of the initial template coadd.
626  gain : `float`, optional
627  Relative weight to give the new solution when updating the model.
628  modelWeights : `numpy.ndarray` or `float`
629  A 2D array of weight values that tapers smoothly to zero away from detected sources.
630  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
631  refImage : `lsst.afw.image.Image`
632  A reference image used to supply the default pixel values.
633  dcrWeights : `list` of `lsst.afw.image.Image`
634  Per-pixel weights for each subfilter.
635  Equal to 1/(number of unmasked images contributing to each pixel).
636  """
637  residualGeneratorList = []
638 
639  for warpExpRef in warpRefList:
640  exposure = subExposures[warpExpRef.dataId["visit"]]
641  visitInfo = exposure.getInfo().getVisitInfo()
642  wcs = exposure.getInfo().getWcs()
643  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
644  order=self.config.imageInterpOrder,
645  splitSubfilters=self.config.splitSubfilters,
646  amplifyModel=self.config.accelerateModel)
647  residual = exposure.image.array - templateImage.array
648  # Note that the variance plane here is used to store weights, not the actual variance
649  residual *= exposure.variance.array
650  # The residuals are stored as a list of generators.
651  # This allows the residual for a given subfilter and exposure to be created
652  # on the fly, instead of needing to store them all in memory.
653  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
654 
655  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
656  mask=baseMask, gain=gain,
657  modelWeights=modelWeights,
658  refImage=refImage,
659  dcrWeights=dcrWeights)
660  dcrModels.assign(dcrSubModelOut, bbox)
661 
662  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
663  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
664 
665  Parameters
666  ----------
667  residual : `numpy.ndarray`
668  The residual masked image for one exposure,
669  after subtracting the matched template
670  visitInfo : `lsst.afw.image.VisitInfo`
671  Metadata for the exposure.
672  wcs : `lsst.afw.geom.SkyWcs`
673  Coordinate system definition (wcs) for the exposure.
674  filterInfo : `lsst.afw.image.Filter`
675  The filter definition, set in the current instruments' obs package.
676  Required for any calculation of DCR, including making matched templates.
677 
678  Yields
679  ------
680  residualImage : `numpy.ndarray`
681  The residual image for the next subfilter, shifted for DCR.
682  """
683  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
684  for dcr in dcrShift:
685  yield applyDcr(residual, dcr, useInverse=True, order=self.config.imageInterpOrder)
686 
687  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
688  mask, gain, modelWeights, refImage, dcrWeights):
689  """Calculate a new DcrModel from a set of image residuals.
690 
691  Parameters
692  ----------
693  dcrModels : `lsst.pipe.tasks.DcrModel`
694  Current model of the true sky after correcting chromatic effects.
695  residualGeneratorList : `generator` of `numpy.ndarray`
696  The residual image for the next subfilter, shifted for DCR.
697  dcrBBox : `lsst.afw.geom.box.Box2I`
698  Sub-region of the coadd which includes a buffer to allow for DCR.
699  statsCtrl : `lsst.afw.math.StatisticsControl`
700  Statistics control object for coadd
701  mask : `lsst.afw.image.Mask`
702  Mask to use for each new model image.
703  gain : `float`
704  Relative weight to give the new solution when updating the model.
705  modelWeights : `numpy.ndarray` or `float`
706  A 2D array of weight values that tapers smoothly to zero away from detected sources.
707  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
708  refImage : `lsst.afw.image.Image`
709  A reference image used to supply the default pixel values.
710  dcrWeights : `list` of `lsst.afw.image.Image`
711  Per-pixel weights for each subfilter.
712  Equal to 1/(number of unmasked images contributing to each pixel).
713 
714  Returns
715  -------
716  dcrModel : `lsst.pipe.tasks.DcrModel`
717  New model of the true sky after correcting chromatic effects.
718  """
719  newModelImages = []
720  for subfilter, model in enumerate(dcrModels):
721  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
722  residual = np.sum(residualsList, axis=0)
723  residual *= dcrWeights[subfilter][dcrBBox].array
724  # `MaskedImage`s only support in-place addition, so rename for readability
725  newModel = model[dcrBBox].clone()
726  newModel.array += residual
727  # Catch any invalid values
728  badPixels = ~np.isfinite(newModel.array)
729  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
730  if self.config.regularizeModelIterations > 0:
731  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
732  self.config.regularizeModelIterations,
733  self.config.regularizationWidth)
734  newModelImages.append(newModel)
735  if self.config.regularizeModelFrequency > 0:
736  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
737  self.config.regularizeModelFrequency,
738  self.config.regularizationWidth,
739  mask=mask)
740  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
741  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
742  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
743  dcrModels.mask, dcrModels.variance)
744 
745  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
746  """Calculate a quality of fit metric for the matched templates.
747 
748  Parameters
749  ----------
750  dcrModels : `lsst.pipe.tasks.DcrModel`
751  Best fit model of the true sky after correcting chromatic effects.
752  subExposures : `dict` of `lsst.afw.image.ExposureF`
753  The pre-loaded exposures for the current subregion.
754  bbox : `lsst.afw.geom.box.Box2I`
755  Sub-region to coadd
756  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
757  The data references to the input warped exposures.
758  weightList : `list` of `float`
759  The weight to give each input exposure in the coadd
760  statsCtrl : `lsst.afw.math.StatisticsControl`
761  Statistics control object for coadd
762 
763  Returns
764  -------
765  convergenceMetric : `float`
766  Quality of fit metric for all input exposures, within the sub-region
767  """
768  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
769  nSigma = 3.
770  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
771  bufferSize=self.bufferSize)
772  if np.max(significanceImage) == 0:
773  significanceImage += 1.
774  weight = 0
775  metric = 0.
776  metricList = {}
777  for warpExpRef, expWeight in zip(warpRefList, weightList):
778  exposure = subExposures[warpExpRef.dataId["visit"]][bbox]
779  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
780  metric += singleMetric
781  metricList[warpExpRef.dataId["visit"]] = singleMetric
782  weight += 1.
783  self.log.info("Individual metrics:\n%s", metricList)
784  return 1.0 if weight == 0.0 else metric/weight
785 
786  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
787  """Calculate a quality of fit metric for a single matched template.
788 
789  Parameters
790  ----------
791  dcrModels : `lsst.pipe.tasks.DcrModel`
792  Best fit model of the true sky after correcting chromatic effects.
793  exposure : `lsst.afw.image.ExposureF`
794  The input warped exposure to evaluate.
795  significanceImage : `numpy.ndarray`
796  Array of weights for each pixel corresponding to its significance
797  for the convergence calculation.
798  statsCtrl : `lsst.afw.math.StatisticsControl`
799  Statistics control object for coadd
800 
801  Returns
802  -------
803  convergenceMetric : `float`
804  Quality of fit metric for one exposure, within the sub-region.
805  """
806  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
807  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
808  order=self.config.imageInterpOrder,
809  splitSubfilters=self.config.splitSubfilters,
810  amplifyModel=self.config.accelerateModel)
811  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
812  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
813 
814  finitePixels = np.isfinite(diffVals)
815  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
816  convergeMaskPixels = exposure.mask.array & convergeMask > 0
817  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
818  if np.sum(usePixels) == 0:
819  metric = 0.
820  else:
821  diffUse = diffVals[usePixels]
822  refUse = refVals[usePixels]
823  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
824  return metric
825 
826  def stackCoadd(self, dcrCoadds):
827  """Add a list of sub-band coadds together.
828 
829  Parameters
830  ----------
831  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
832  A list of coadd exposures, each exposure containing
833  the model for one subfilter.
834 
835  Returns
836  -------
837  coaddExposure : `lsst.afw.image.ExposureF`
838  A single coadd exposure that is the sum of the sub-bands.
839  """
840  coaddExposure = dcrCoadds[0].clone()
841  for coadd in dcrCoadds[1:]:
842  coaddExposure.maskedImage += coadd.maskedImage
843  return coaddExposure
844 
845  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
846  mask=None, variance=None):
847  """Create a list of coadd exposures from a list of masked images.
848 
849  Parameters
850  ----------
851  dcrModels : `lsst.pipe.tasks.DcrModel`
852  Best fit model of the true sky after correcting chromatic effects.
853  skyInfo : `lsst.pipe.base.Struct`
854  Patch geometry information, from getSkyInfo
855  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
856  The data references to the input warped exposures.
857  weightList : `list` of `float`
858  The weight to give each input exposure in the coadd
859  calibration : `lsst.afw.Image.PhotoCalib`, optional
860  Scale factor to set the photometric calibration of an exposure.
861  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
862  A record of the observations that are included in the coadd.
863  mask : `lsst.afw.image.Mask`, optional
864  Optional mask to override the values in the final coadd.
865  variance : `lsst.afw.image.Image`, optional
866  Optional variance plane to override the values in the final coadd.
867 
868  Returns
869  -------
870  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
871  A list of coadd exposures, each exposure containing
872  the model for one subfilter.
873  """
874  dcrCoadds = []
875  refModel = dcrModels.getReferenceImage()
876  for model in dcrModels:
877  if self.config.accelerateModel > 1:
878  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
879  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
880  if calibration is not None:
881  coaddExposure.setPhotoCalib(calibration)
882  if coaddInputs is not None:
883  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
884  # Set the metadata for the coadd, including PSF and aperture corrections.
885  self.assembleMetadata(coaddExposure, warpRefList, weightList)
886  # Overwrite the PSF
887  coaddExposure.setPsf(dcrModels.psf)
888  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
889  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
890  maskedImage.image = model
891  maskedImage.mask = dcrModels.mask
892  maskedImage.variance = dcrModels.variance
893  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
894  if mask is not None:
895  coaddExposure.setMask(mask)
896  if variance is not None:
897  coaddExposure.setVariance(variance)
898  dcrCoadds.append(coaddExposure)
899  return dcrCoadds
900 
901  def calculateGain(self, convergenceList, gainList):
902  """Calculate the gain to use for the current iteration.
903 
904  After calculating a new DcrModel, each value is averaged with the
905  value in the corresponding pixel from the previous iteration. This
906  reduces oscillating solutions that iterative techniques are plagued by,
907  and speeds convergence. By far the biggest changes to the model
908  happen in the first couple iterations, so we can also use a more
909  aggressive gain later when the model is changing slowly.
910 
911  Parameters
912  ----------
913  convergenceList : `list` of `float`
914  The quality of fit metric from each previous iteration.
915  gainList : `list` of `float`
916  The gains used in each previous iteration: appended with the new
917  gain value.
918  Gains are numbers between ``self.config.baseGain`` and 1.
919 
920  Returns
921  -------
922  gain : `float`
923  Relative weight to give the new solution when updating the model.
924  A value of 1.0 gives equal weight to both solutions.
925 
926  Raises
927  ------
928  ValueError
929  If ``len(convergenceList) != len(gainList)+1``.
930  """
931  nIter = len(convergenceList)
932  if nIter != len(gainList) + 1:
933  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
934  % (len(convergenceList), len(gainList)))
935 
936  if self.config.baseGain is None:
937  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
938  # The more subfilters being modeled, the lower the gain should be.
939  baseGain = 1./(self.config.dcrNumSubfilters - 1)
940  else:
941  baseGain = self.config.baseGain
942 
943  if self.config.useProgressiveGain and nIter > 2:
944  # To calculate the best gain to use, compare the past gains that have been used
945  # with the resulting convergences to estimate the best gain to use.
946  # Algorithmically, this is a Kalman filter.
947  # If forward modeling proceeds perfectly, the convergence metric should
948  # asymptotically approach a final value.
949  # We can estimate that value from the measured changes in convergence
950  # weighted by the gains used in each previous iteration.
951  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
952  for i in range(nIter - 1)]
953  # The convergence metric is strictly positive, so if the estimated final convergence is
954  # less than zero, force it to zero.
955  estFinalConv = np.array(estFinalConv)
956  estFinalConv[estFinalConv < 0] = 0
957  # Because the estimate may slowly change over time, only use the most recent measurements.
958  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
959  lastGain = gainList[-1]
960  lastConv = convergenceList[-2]
961  newConv = convergenceList[-1]
962  # The predicted convergence is the value we would get if the new model calculated
963  # in the previous iteration was perfect. Recall that the updated model that is
964  # actually used is the gain-weighted average of the new and old model,
965  # so the convergence would be similarly weighted.
966  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
967  # If the measured and predicted convergence are very close, that indicates
968  # that our forward model is accurate and we can use a more aggressive gain
969  # If the measured convergence is significantly worse (or better!) than predicted,
970  # that indicates that the model is not converging as expected and
971  # we should use a more conservative gain.
972  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
973  newGain = 1 - abs(delta)
974  # Average the gains to prevent oscillating solutions.
975  newGain = (newGain + lastGain)/2.
976  gain = max(baseGain, newGain)
977  else:
978  gain = baseGain
979  gainList.append(gain)
980  return gain
981 
982  def calculateModelWeights(self, dcrModels, dcrBBox):
983  """Build an array that smoothly tapers to 0 away from detected sources.
984 
985  Parameters
986  ----------
987  dcrModels : `lsst.pipe.tasks.DcrModel`
988  Best fit model of the true sky after correcting chromatic effects.
989  dcrBBox : `lsst.afw.geom.box.Box2I`
990  Sub-region of the coadd which includes a buffer to allow for DCR.
991 
992  Returns
993  -------
994  weights : `numpy.ndarray` or `float`
995  A 2D array of weight values that tapers smoothly to zero away from detected sources.
996  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
997 
998  Raises
999  ------
1000  ValueError
1001  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1002  """
1003  if not self.config.useModelWeights:
1004  return 1.0
1005  if self.config.modelWeightsWidth < 0:
1006  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1007  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1008  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1009  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1010  weights[convergeMaskPixels] = 1.
1011  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1012  weights /= np.max(weights)
1013  return weights
1014 
1015  def applyModelWeights(self, modelImages, refImage, modelWeights):
1016  """Smoothly replace model pixel values with those from a
1017  reference at locations away from detected sources.
1018 
1019  Parameters
1020  ----------
1021  modelImages : `list` of `lsst.afw.image.Image`
1022  The new DCR model images from the current iteration.
1023  The values will be modified in place.
1024  refImage : `lsst.afw.image.MaskedImage`
1025  A reference image used to supply the default pixel values.
1026  modelWeights : `numpy.ndarray` or `float`
1027  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1028  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1029  """
1030  if self.config.useModelWeights:
1031  for model in modelImages:
1032  model.array *= modelWeights
1033  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1034 
1035  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1036  """Pre-load sub-regions of a list of exposures.
1037 
1038  Parameters
1039  ----------
1040  bbox : `lsst.afw.geom.box.Box2I`
1041  Sub-region to coadd
1042  statsCtrl : `lsst.afw.math.StatisticsControl`
1043  Statistics control object for coadd
1044  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1045  The data references to the input warped exposures.
1046  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1047  The image scalars correct for the zero point of the exposures.
1048  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1049  Each element is dict with keys = mask plane name to add the spans to
1050 
1051  Returns
1052  -------
1053  subExposures : `dict`
1054  The `dict` keys are the visit IDs,
1055  and the values are `lsst.afw.image.ExposureF`
1056  The pre-loaded exposures for the current subregion.
1057  The variance plane contains weights, and not the variance
1058  """
1059  tempExpName = self.getTempExpDatasetName(self.warpType)
1060  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1061  subExposures = {}
1062  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1063  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1064  if altMaskSpans is not None:
1065  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1066  imageScaler.scaleMaskedImage(exposure.maskedImage)
1067  # Note that the variance plane here is used to store weights, not the actual variance
1068  exposure.variance.array[:, :] = 0.
1069  # Set the weight of unmasked pixels to 1.
1070  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1071  # Set the image value of masked pixels to zero.
1072  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1073  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1074  subExposures[warpExpRef.dataId["visit"]] = exposure
1075  return subExposures
1076 
1077  def selectCoaddPsf(self, templateCoadd, warpRefList):
1078  """Compute the PSF of the coadd from the exposures with the best seeing.
1079 
1080  Parameters
1081  ----------
1082  templateCoadd : `lsst.afw.image.ExposureF`
1083  The initial coadd exposure before accounting for DCR.
1084  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1085  The data references to the input warped exposures.
1086 
1087  Returns
1088  -------
1089  psf : `lsst.meas.algorithms.CoaddPsf`
1090  The average PSF of the input exposures with the best seeing.
1091  """
1092  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1093  tempExpName = self.getTempExpDatasetName(self.warpType)
1094  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1095  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1096  psfSizeList = []
1097  for visitNum, warpExpRef in enumerate(warpRefList):
1098  psf = warpExpRef.get(tempExpName).getPsf()
1099  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1100  psfSizeList.append(psfSize)
1101  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1102  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1103  # of the mean or median FWHM of the input exposures.
1104  sizeThreshold = min(np.median(psfSizeList), psfRefSize)
1105  goodVisits = np.array(psfSizeList) <= sizeThreshold
1106  psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1107  self.config.coaddPsf.makeControl())
1108  return psf
Extent< int, N > ceil(Extent< double, N > const &input) noexcept
Return the component-wise ceil (round towards more positive).
Definition: Extent.cc:118
Angle abs(Angle const &a)
Definition: Angle.h:106
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
int min
def applyDcr(image, dcr, useInverse=False, splitSubfilters=False, kwargs)
Definition: dcrModel.py:582
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
int max
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, baseMask, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters, splitSubfilters=False)
Definition: dcrModel.py:629
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, mask, gain, modelWeights, refImage, dcrWeights)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
An integer coordinate rectangle.
Definition: Box.h:54
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)