22__all__ = [
"DcrAssembleCoaddConnections",
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
26from scipy
import ndimage
37from lsst.utils.timer
import timeMethod
38from .assembleCoadd
import (AssembleCoaddConnections,
40 CompareWarpAssembleCoaddConfig,
41 CompareWarpAssembleCoaddTask)
42from .coaddBase
import makeSkyInfo
43from .measurePsf
import MeasurePsfTask
47 dimensions=(
"tract",
"patch",
"band",
"skymap"),
48 defaultTemplates={
"inputWarpName":
"deep",
49 "inputCoaddName":
"deep",
50 "outputCoaddName":
"dcr",
54 inputWarps = pipeBase.connectionTypes.Input(
55 doc=(
"Input list of warps to be assembled i.e. stacked."
56 "Note that this will often be different than the inputCoaddName."
57 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
58 name=
"{inputWarpName}Coadd_{warpType}Warp",
59 storageClass=
"ExposureF",
60 dimensions=(
"tract",
"patch",
"skymap",
"visit",
"instrument"),
64 templateExposure = pipeBase.connectionTypes.Input(
65 doc=
"Input coadded exposure, produced by previous call to AssembleCoadd",
66 name=
"{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
67 storageClass=
"ExposureF",
68 dimensions=(
"tract",
"patch",
"skymap",
"band"),
70 dcrCoadds = pipeBase.connectionTypes.Output(
71 doc=
"Output coadded exposure, produced by stacking input warps",
72 name=
"{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
73 storageClass=
"ExposureF",
74 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
77 dcrNImages = pipeBase.connectionTypes.Output(
78 doc=
"Output image of number of input images per pixel",
79 name=
"{outputCoaddName}Coadd_nImage",
80 storageClass=
"ImageU",
81 dimensions=(
"tract",
"patch",
"skymap",
"band",
"subfilter"),
85 def __init__(self, *, config=None):
86 super().__init__(config=config)
87 if not config.doWrite:
88 self.outputs.remove(
"dcrCoadds")
89 if not config.doNImage:
90 self.outputs.remove(
"dcrNImages")
92 self.outputs.remove(
"coaddExposure")
93 self.outputs.remove(
"nImage")
97 pipelineConnections=DcrAssembleCoaddConnections):
98 dcrNumSubfilters = pexConfig.Field(
100 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
103 maxNumIter = pexConfig.Field(
106 doc=
"Maximum number of iterations of forward modeling.",
109 minNumIter = pexConfig.Field(
112 doc=
"Minimum number of iterations of forward modeling.",
115 convergenceThreshold = pexConfig.Field(
117 doc=
"Target relative change in convergence between iterations of forward modeling.",
120 useConvergence = pexConfig.Field(
122 doc=
"Use convergence test as a forward modeling end condition?"
123 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
126 baseGain = pexConfig.Field(
129 doc=
"Relative weight to give the new solution vs. the last solution when updating the model."
130 "A value of 1.0 gives equal weight to both solutions."
131 "Small values imply slower convergence of the solution, but can "
132 "help prevent overshooting and failures in the fit."
133 "If ``baseGain`` is None, a conservative gain "
134 "will be calculated from the number of subfilters. ",
137 useProgressiveGain = pexConfig.Field(
139 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
140 "When calculating the next gain, we use up to 5 previous gains and convergence values."
141 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
144 doAirmassWeight = pexConfig.Field(
146 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
149 modelWeightsWidth = pexConfig.Field(
151 doc=
"Width of the region around detected sources to include in the DcrModel.",
154 useModelWeights = pexConfig.Field(
156 doc=
"Width of the region around detected sources to include in the DcrModel.",
159 splitSubfilters = pexConfig.Field(
161 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter."
162 "Instead of at the midpoint",
165 splitThreshold = pexConfig.Field(
167 doc=
"Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
168 "Set to 0 to always split the subfilters.",
171 regularizeModelIterations = pexConfig.Field(
173 doc=
"Maximum relative change of the model allowed between iterations."
174 "Set to zero to disable.",
177 regularizeModelFrequency = pexConfig.Field(
179 doc=
"Maximum relative change of the model allowed between subfilters."
180 "Set to zero to disable.",
183 convergenceMaskPlanes = pexConfig.ListField(
185 default=[
"DETECTED"],
186 doc=
"Mask planes to use to calculate convergence."
188 regularizationWidth = pexConfig.Field(
191 doc=
"Minimum radius of a region to include in regularization, in pixels."
193 imageInterpOrder = pexConfig.Field(
195 doc=
"The order of the spline interpolation used to shift the image plane.",
198 accelerateModel = pexConfig.Field(
200 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
203 doCalculatePsf = pexConfig.Field(
205 doc=
"Set to detect stars and recalculate the PSF from the final coadd."
206 "Otherwise the PSF is estimated from a selection of the best input exposures",
209 detectPsfSources = pexConfig.ConfigurableField(
210 target=measAlg.SourceDetectionTask,
211 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
213 measurePsfSources = pexConfig.ConfigurableField(
214 target=SingleFrameMeasurementTask,
215 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
217 measurePsf = pexConfig.ConfigurableField(
218 target=MeasurePsfTask,
219 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
221 effectiveWavelength = pexConfig.Field(
222 doc=
"Effective wavelength of the filter, in nm."
223 "Required if transmission curves aren't used."
224 "Support for using transmission curves is to be added in DM-13668.",
227 bandwidth = pexConfig.Field(
228 doc=
"Bandwidth of the physical filter, in nm."
229 "Required if transmission curves aren't used."
230 "Support for using transmission curves is to be added in DM-13668.",
234 def setDefaults(self):
235 CompareWarpAssembleCoaddConfig.setDefaults(self)
236 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
238 self.assembleStaticSkyModel.warpType = self.warpType
240 self.assembleStaticSkyModel.doNImage =
False
241 self.assembleStaticSkyModel.doWrite =
False
242 self.detectPsfSources.returnOriginalFootprints =
False
243 self.detectPsfSources.thresholdPolarity =
"positive"
245 self.detectPsfSources.thresholdValue = 50
246 self.detectPsfSources.nSigmaToGrow = 2
248 self.detectPsfSources.minPixels = 25
250 self.detectPsfSources.thresholdType =
"pixel_stdev"
253 self.measurePsf.starSelector[
"objectSize"].doFluxLimit =
False
255 if (self.doCalculatePsf
and self.measurePsf.psfDeterminer.name ==
"piff"
256 and self.psfDeterminer[
"piff"].kernelSize > self.makePsfCandidates.kernelSize):
257 self.makePsfCandidates.kernelSize = self.psfDeterminer[
"piff"].kernelSize
261 """Assemble DCR coadded images from a set of warps.
266 The number of pixels to grow each subregion by to allow for DCR.
270 As
with AssembleCoaddTask, we want to assemble a coadded image
from a set of
271 Warps (also called coadded temporary exposures), including the effects of
272 Differential Chromatic Refraction (DCR).
273 For full details of the mathematics
and algorithm, please see
274 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
276 This Task produces a DCR-corrected deepCoadd,
as well
as a dcrCoadd
for
277 each subfilter used
in the iterative calculation.
278 It begins by dividing the bandpass-defining filter into N equal bandwidth
279 "subfilters",
and divides the flux
in each pixel
from an initial coadd
280 equally into each
as a
"dcrModel". Because the airmass
and parallactic
281 angle of each individual exposure
is known, we can calculate the shift
282 relative to the center of the band
in each subfilter due to DCR. For each
283 exposure we apply this shift
as a linear transformation to the dcrModels
284 and stack the results to produce a DCR-matched exposure. The matched
285 exposures are subtracted
from the input exposures to produce a set of
286 residual images,
and these residuals are reverse shifted
for each
287 exposures
' subfilters and stacked. The shifted and stacked residuals are
288 added to the dcrModels to produce a new estimate of the flux in each pixel
289 within each subfilter. The dcrModels are solved
for iteratively, which
290 continues until the solution
from a new iteration improves by less than
291 a set percentage,
or a maximum number of iterations
is reached.
292 Two forms of regularization are employed to reduce unphysical results.
293 First, the new solution
is averaged
with the solution
from the previous
294 iteration, which mitigates oscillating solutions where the model
295 overshoots
with alternating very high
and low values.
296 Second, a common degeneracy when the data have a limited range of airmass
or
297 parallactic angle values
is for one subfilter to be fit
with very low
or
298 negative values,
while another subfilter
is fit
with very high values. This
299 typically appears
in the form of holes next to sources
in one subfilter,
300 and corresponding extended wings
in another. Because each subfilter has
301 a narrow bandwidth we assume that physical sources that are above the noise
302 level will
not vary
in flux by more than a factor of `frequencyClampFactor`
303 between subfilters,
and pixels that have flux deviations larger than that
304 factor will have the excess flux distributed evenly among all subfilters.
305 If `splitSubfilters`
is set, then each subfilter will be further sub-
306 divided during the forward modeling step (only). This approximates using
307 a higher number of subfilters that may be necessary
for high airmass
308 observations, but does
not increase the number of free parameters
in the
309 fit. This
is needed when there are high airmass observations which would
310 otherwise have significant DCR even within a subfilter. Because calculating
311 the shifted images takes most of the time, splitting the subfilters
is
312 turned off by way of the `splitThreshold` option
for low-airmass
313 observations that do
not suffer
from DCR within a subfilter.
316 ConfigClass = DcrAssembleCoaddConfig
317 _DefaultName = "dcrAssembleCoadd"
319 def __init__(self, *args, **kwargs):
320 super().__init__(*args, **kwargs)
321 if self.config.doCalculatePsf:
322 self.schema = afwTable.SourceTable.makeMinimalSchema()
323 self.makeSubtask(
"detectPsfSources", schema=self.schema)
324 self.makeSubtask(
"measurePsfSources", schema=self.schema)
325 self.makeSubtask(
"measurePsf", schema=self.schema)
327 @utils.inheritDoc(pipeBase.PipelineTask)
328 def runQuantum(self, butlerQC, inputRefs, outputRefs):
333 Assemble a coadd from a set of Warps.
335 inputData = butlerQC.get(inputRefs)
339 skyMap = inputData[
"skyMap"]
340 outputDataId = butlerQC.quantum.dataId
342 inputData[
'skyInfo'] = makeSkyInfo(skyMap,
343 tractId=outputDataId[
'tract'],
344 patchId=outputDataId[
'patch'])
347 warpRefList = inputData[
'inputWarps']
349 inputs = self.prepareInputs(warpRefList)
350 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
351 self.getTempExpDatasetName(self.warpType))
352 if len(inputs.tempExpRefList) == 0:
353 self.log.warning(
"No coadd temporary exposures found")
356 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
357 retStruct = self.run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
358 inputs.weightList, supplementaryData=supplementaryData)
360 inputData.setdefault(
'brightObjectMask',
None)
361 for subfilter
in range(self.config.dcrNumSubfilters):
363 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
364 self.processResults(retStruct.dcrCoadds[subfilter], inputData[
'brightObjectMask'], outputDataId)
366 if self.config.doWrite:
367 butlerQC.put(retStruct, outputRefs)
370 @utils.inheritDoc(AssembleCoaddTask)
371 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
372 """Load the previously-generated template coadd.
376 templateCoadd : `lsst.pipe.base.Struct`
377 Results as a struct
with attributes:
380 Coadded exposure (`lsst.afw.image.ExposureF`).
382 templateCoadd = butlerQC.get(inputRefs.templateExposure)
384 return pipeBase.Struct(templateCoadd=templateCoadd)
386 def measureCoaddPsf(self, coaddExposure):
387 """Detect sources on the coadd exposure and measure the final PSF.
392 The final coadded exposure.
394 table = afwTable.SourceTable.make(self.schema)
395 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
396 coaddSources = detResults.sources
397 self.measurePsfSources.run(
398 measCat=coaddSources,
399 exposure=coaddExposure
406 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
407 except Exception
as e:
408 self.log.warning(
"Unable to calculate PSF, using default coadd PSF: %s", e)
410 coaddExposure.setPsf(psfResults.psf)
412 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
413 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
415 Sets the property ``bufferSize``.
419 templateCoadd : `lsst.afw.image.ExposureF`
420 The initial coadd exposure before accounting for DCR.
421 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
422 The data references to the input warped exposures.
423 weightList : `list` of `float`
424 The weight to give each input exposure
in the coadd.
425 Will be modified
in place
if ``doAirmassWeight``
is set.
429 dcrModels : `lsst.pipe.tasks.DcrModel`
430 Best fit model of the true sky after correcting chromatic effects.
435 If ``lambdaMin``
is missing
from the Mapper
class of the obs package being used.
437 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
438 filterLabel = templateCoadd.getFilter()
443 for visitNum, warpExpRef
in enumerate(warpRefList):
444 visitInfo = warpExpRef.get(component=
"visitInfo")
445 psf = warpExpRef.get(component=
"psf")
446 visit = warpExpRef.dataId[
"visit"]
448 psfAvgPos = psf.getAveragePosition()
449 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
450 airmass = visitInfo.getBoresightAirmass()
451 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
452 airmassDict[visit] = airmass
453 angleDict[visit] = parallacticAngle
454 psfSizeDict[visit] = psfSize
455 if self.config.doAirmassWeight:
456 weightList[visitNum] *= airmass
457 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
458 self.config.effectiveWavelength,
459 self.config.bandwidth,
460 self.config.dcrNumSubfilters))))
461 self.log.info(
"Selected airmasses:\n%s", airmassDict)
462 self.log.info(
"Selected parallactic angles:\n%s", angleDict)
463 self.log.info(
"Selected PSF sizes:\n%s", psfSizeDict)
464 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
466 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
467 except Exception
as e:
468 self.log.warning(
"Unable to calculate restricted PSF, using default coadd PSF: %s", e)
470 psf = templateCoadd.getPsf()
471 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
472 self.config.dcrNumSubfilters,
473 effectiveWavelength=self.config.effectiveWavelength,
474 bandwidth=self.config.bandwidth,
475 wcs=templateCoadd.getWcs(),
476 filterLabel=filterLabel,
481 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
482 supplementaryData=None):
483 r"""Assemble the coadd.
485 Requires additional inputs Struct ``supplementaryData`` to contain a
486 ``templateCoadd`` that serves as the model of the static sky.
488 Find artifacts
and apply them to the warps
' masks creating a list of
489 alternative masks with a new
"CLIPPED" plane
and updated
"NO_DATA" plane
490 Then
pass these alternative masks to the base
class's assemble method.
492 Divide the ``templateCoadd`` evenly between each subfilter of a
493 ``DcrModel`` as the starting best estimate of the true wavelength-
494 dependent sky. Forward model the ``DcrModel`` using the known
495 chromatic effects
in each subfilter
and calculate a convergence metric
496 based on how well the modeled template matches the input warps. If
497 the convergence has
not yet reached the desired threshold, then shift
498 and stack the residual images to build a new ``DcrModel``. Apply
499 conditioning to prevent oscillating solutions between iterations
or
502 Once the ``DcrModel`` reaches convergence
or the maximum number of
503 iterations has been reached, fill the metadata
for each subfilter
504 image
and make them proper ``coaddExposure``\ s.
508 skyInfo : `lsst.pipe.base.Struct`
509 Patch geometry information,
from getSkyInfo
510 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
511 The data references to the input warped exposures.
512 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
513 The image scalars correct
for the zero point of the exposures.
514 weightList : `list` of `float`
515 The weight to give each input exposure
in the coadd
516 supplementaryData : `lsst.pipe.base.Struct`
517 Result struct returned by ``_makeSupplementaryData``
with attributes:
524 result : `lsst.pipe.base.Struct`
525 Results
as a struct
with attributes:
530 Exposure count image (`lsst.afw.image.ImageU`).
532 `list` of coadded exposures
for each subfilter.
534 `list` of exposure count images
for each subfilter.
536 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
537 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
538 templateCoadd = supplementaryData.templateCoadd
539 baseMask = templateCoadd.mask.clone()
542 baseVariance = templateCoadd.variance.clone()
543 baseVariance /= self.config.dcrNumSubfilters
544 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
546 templateCoadd.setMask(baseMask)
547 badMaskPlanes = self.config.badMaskPlanes[:]
552 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
554 stats = self.prepareStats(mask=badPixelMask)
555 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
556 if self.config.doNImage:
557 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
558 spanSetMaskList, stats.ctrl)
559 nImage = afwImage.ImageU(skyInfo.bbox)
563 for dcrNImage
in dcrNImages:
569 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
570 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
572 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
575 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
576 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
578 dcrBBox.grow(self.bufferSize)
579 dcrBBox.clip(dcrModels.bbox)
580 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
581 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
582 imageScalerList, spanSetMaskList)
583 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
584 warpRefList, weightList, stats.ctrl)
585 self.log.info(
"Initial convergence : %s", convergenceMetric)
586 convergenceList = [convergenceMetric]
588 convergenceCheck = 1.
589 refImage = templateCoadd.image
590 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
591 gain = self.calculateGain(convergenceList, gainList)
592 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
593 stats.ctrl, convergenceMetric, gain,
594 modelWeights, refImage, dcrWeights)
595 if self.config.useConvergence:
596 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
597 warpRefList, weightList, stats.ctrl)
598 if convergenceMetric == 0:
599 self.log.warning(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is "
600 "most likely due to there being no valid data in the region.",
601 skyInfo.patchInfo.getIndex(), subIter)
603 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
604 if (convergenceCheck < 0) & (modelIter > minNumIter):
605 self.log.warning(
"Coadd patch %s subregion %s diverged before reaching maximum "
606 "iterations or desired convergence improvement of %s."
608 skyInfo.patchInfo.getIndex(), subIter,
609 self.config.convergenceThreshold, convergenceCheck)
611 convergenceList.append(convergenceMetric)
612 if modelIter > maxNumIter:
613 if self.config.useConvergence:
614 self.log.warning(
"Coadd patch %s subregion %s reached maximum iterations "
615 "before reaching desired convergence improvement of %s."
616 " Final convergence improvement: %s",
617 skyInfo.patchInfo.getIndex(), subIter,
618 self.config.convergenceThreshold, convergenceCheck)
621 if self.config.useConvergence:
622 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
623 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
626 if self.config.useConvergence:
627 self.log.info(
"Coadd patch %s subregion %s finished with "
628 "convergence metric %s after %s iterations",
629 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
631 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
632 skyInfo.patchInfo.getIndex(), subIter, modelIter)
633 if self.config.useConvergence
and convergenceMetric > 0:
634 self.log.info(
"Final convergence improvement was %.4f%% overall",
635 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
637 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
638 calibration=self.scaleZeroPoint.getPhotoCalib(),
639 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
641 variance=baseVariance)
642 coaddExposure = self.stackCoadd(dcrCoadds)
643 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
644 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
646 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
647 """Calculate the number of exposures contributing to each subfilter.
651 dcrModels : `lsst.pipe.tasks.DcrModel`
652 Best fit model of the true sky after correcting chromatic effects.
653 bbox : `lsst.geom.box.Box2I`
654 Bounding box of the patch to coadd.
655 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
656 The data references to the input warped exposures.
657 spanSetMaskList : `list` of `dict` containing spanSet lists, or `
None`
658 Each element of the `dict` contains the new mask plane name
659 (e.g.
"CLIPPED and/or "NO_DATA
") as the key,
660 and the list of SpanSets to apply to the mask.
662 Statistics control object
for coadd
666 dcrNImages : `list` of `lsst.afw.image.ImageU`
667 List of exposure count images
for each subfilter.
668 dcrWeights : `list` of `lsst.afw.image.ImageF`
669 Per-pixel weights
for each subfilter.
670 Equal to 1/(number of unmasked images contributing to each pixel).
672 dcrNImages = [afwImage.ImageU(bbox) for subfilter
in range(self.config.dcrNumSubfilters)]
673 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
674 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
675 exposure = warpExpRef.get(parameters={
'bbox': bbox})
676 visitInfo = exposure.getInfo().getVisitInfo()
677 wcs = exposure.getInfo().getWcs()
679 if altMaskSpans
is not None:
680 self.applyAltMaskPlanes(mask, altMaskSpans)
681 weightImage = np.zeros_like(exposure.image.array)
682 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
685 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
686 dcrModels.effectiveWavelength, dcrModels.bandwidth)
687 for shiftedWeights, dcrNImage, dcrWeight
in zip(weightsGenerator, dcrNImages, dcrWeights):
688 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
689 dcrWeight.array += shiftedWeights
691 weightsThreshold = 1.
692 goodPix = dcrWeights[0].array > weightsThreshold
693 for weights
in dcrWeights[1:]:
694 goodPix = (weights.array > weightsThreshold) & goodPix
695 for subfilter
in range(self.config.dcrNumSubfilters):
696 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
697 dcrWeights[subfilter].array[~goodPix] = 0.
698 dcrNImages[subfilter].array[~goodPix] = 0
699 return (dcrNImages, dcrWeights)
702 statsCtrl, convergenceMetric,
703 gain, modelWeights, refImage, dcrWeights):
704 """Assemble the DCR coadd for a sub-region.
706 Build a DCR-matched template for each input exposure, then shift the
707 residuals according to the DCR
in each subfilter.
708 Stack the shifted residuals
and apply them
as a correction to the
709 solution
from the previous iteration.
710 Restrict the new model solutions
from varying by more than a factor of
711 `modelClampFactor`
from the last solution,
and additionally restrict the
712 individual subfilter models
from varying by more than a factor of
713 `frequencyClampFactor`
from their average.
714 Finally, mitigate potentially oscillating solutions by averaging the new
715 solution
with the solution
from the previous iteration, weighted by
716 their convergence metric.
720 dcrModels : `lsst.pipe.tasks.DcrModel`
721 Best fit model of the true sky after correcting chromatic effects.
722 subExposures : `dict` of `lsst.afw.image.ExposureF`
723 The pre-loaded exposures
for the current subregion.
724 bbox : `lsst.geom.box.Box2I`
725 Bounding box of the subregion to coadd.
726 dcrBBox : `lsst.geom.box.Box2I`
727 Sub-region of the coadd which includes a buffer to allow
for DCR.
728 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
729 The data references to the input warped exposures.
731 Statistics control object
for coadd.
732 convergenceMetric : `float`
733 Quality of fit metric
for the matched templates of the input images.
734 gain : `float`, optional
735 Relative weight to give the new solution when updating the model.
736 modelWeights : `numpy.ndarray`
or `float`
737 A 2D array of weight values that tapers smoothly to zero away
from detected sources.
738 Set to a placeholder value of 1.0
if ``self.config.useModelWeights``
is False.
740 A reference image used to supply the default pixel values.
742 Per-pixel weights
for each subfilter.
743 Equal to 1/(number of unmasked images contributing to each pixel).
745 residualGeneratorList = []
747 for warpExpRef
in warpRefList:
748 visit = warpExpRef.dataId[
"visit"]
749 exposure = subExposures[visit]
750 visitInfo = exposure.getInfo().getVisitInfo()
751 wcs = exposure.getInfo().getWcs()
752 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
753 bbox=exposure.getBBox(),
754 order=self.config.imageInterpOrder,
755 splitSubfilters=self.config.splitSubfilters,
756 splitThreshold=self.config.splitThreshold,
757 amplifyModel=self.config.accelerateModel)
758 residual = exposure.image.array - templateImage.array
760 residual *= exposure.variance.array
764 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
765 dcrModels.effectiveWavelength,
766 dcrModels.bandwidth))
768 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
770 modelWeights=modelWeights,
772 dcrWeights=dcrWeights)
773 dcrModels.assign(dcrSubModelOut, bbox)
775 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
776 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
780 residual : `numpy.ndarray`
781 The residual masked image for one exposure,
782 after subtracting the matched template.
784 Metadata
for the exposure.
786 Coordinate system definition (wcs)
for the exposure.
790 residualImage : `numpy.ndarray`
791 The residual image
for the next subfilter, shifted
for DCR.
793 if self.config.imageInterpOrder > 1:
796 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
799 filteredResidual = residual
802 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
803 splitSubfilters=
False)
805 yield applyDcr(filteredResidual, dcr, useInverse=
True, splitSubfilters=
False,
806 doPrefilter=
False, order=self.config.imageInterpOrder)
809 gain, modelWeights, refImage, dcrWeights):
810 """Calculate a new DcrModel from a set of image residuals.
814 dcrModels : `lsst.pipe.tasks.DcrModel`
815 Current model of the true sky after correcting chromatic effects.
816 residualGeneratorList : `generator` of `numpy.ndarray`
817 The residual image for the next subfilter, shifted
for DCR.
818 dcrBBox : `lsst.geom.box.Box2I`
819 Sub-region of the coadd which includes a buffer to allow
for DCR.
821 Statistics control object
for coadd.
823 Relative weight to give the new solution when updating the model.
824 modelWeights : `numpy.ndarray`
or `float`
825 A 2D array of weight values that tapers smoothly to zero away
from detected sources.
826 Set to a placeholder value of 1.0
if ``self.config.useModelWeights``
is False.
828 A reference image used to supply the default pixel values.
830 Per-pixel weights
for each subfilter.
831 Equal to 1/(number of unmasked images contributing to each pixel).
835 dcrModel : `lsst.pipe.tasks.DcrModel`
836 New model of the true sky after correcting chromatic effects.
839 for subfilter, model
in enumerate(dcrModels):
840 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
841 residual = np.sum(residualsList, axis=0)
842 residual *= dcrWeights[subfilter][dcrBBox].array
844 newModel = model[dcrBBox].clone()
845 newModel.array += residual
847 badPixels = ~np.isfinite(newModel.array)
848 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
849 if self.config.regularizeModelIterations > 0:
850 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
851 self.config.regularizeModelIterations,
852 self.config.regularizationWidth)
853 newModelImages.append(newModel)
854 if self.config.regularizeModelFrequency > 0:
855 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
856 self.config.regularizeModelFrequency,
857 self.config.regularizationWidth)
858 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
859 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
860 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
861 dcrModels.bandwidth, dcrModels.psf,
862 dcrModels.mask, dcrModels.variance)
865 """Calculate a quality of fit metric for the matched templates.
869 dcrModels : `lsst.pipe.tasks.DcrModel`
870 Best fit model of the true sky after correcting chromatic effects.
871 subExposures : `dict` of `lsst.afw.image.ExposureF`
872 The pre-loaded exposures for the current subregion.
873 bbox : `lsst.geom.box.Box2I`
875 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
876 The data references to the input warped exposures.
877 weightList : `list` of `float`
878 The weight to give each input exposure
in the coadd.
880 Statistics control object
for coadd.
884 convergenceMetric : `float`
885 Quality of fit metric
for all input exposures, within the sub-region.
887 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
889 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
890 bufferSize=self.bufferSize)
891 if np.max(significanceImage) == 0:
892 significanceImage += 1.
896 for warpExpRef, expWeight
in zip(warpRefList, weightList):
897 visit = warpExpRef.dataId[
"visit"]
898 exposure = subExposures[visit][bbox]
899 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
900 metric += singleMetric
901 metricList[visit] = singleMetric
903 self.log.info(
"Individual metrics:\n%s", metricList)
904 return 1.0
if weight == 0.0
else metric/weight
907 """Calculate a quality of fit metric for a single matched template.
911 dcrModels : `lsst.pipe.tasks.DcrModel`
912 Best fit model of the true sky after correcting chromatic effects.
913 exposure : `lsst.afw.image.ExposureF`
914 The input warped exposure to evaluate.
915 significanceImage : `numpy.ndarray`
916 Array of weights for each pixel corresponding to its significance
917 for the convergence calculation.
919 Statistics control object
for coadd.
923 convergenceMetric : `float`
924 Quality of fit metric
for one exposure, within the sub-region.
926 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
927 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
928 bbox=exposure.getBBox(),
929 order=self.config.imageInterpOrder,
930 splitSubfilters=self.config.splitSubfilters,
931 splitThreshold=self.config.splitThreshold,
932 amplifyModel=self.config.accelerateModel)
933 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
934 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
936 finitePixels = np.isfinite(diffVals)
937 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
938 convergeMaskPixels = exposure.mask.array & convergeMask > 0
939 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
940 if np.sum(usePixels) == 0:
943 diffUse = diffVals[usePixels]
944 refUse = refVals[usePixels]
945 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
949 """Add a list of sub-band coadds together.
953 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
954 A list of coadd exposures, each exposure containing
955 the model for one subfilter.
959 coaddExposure : `lsst.afw.image.ExposureF`
960 A single coadd exposure that
is the sum of the sub-bands.
962 coaddExposure = dcrCoadds[0].clone()
963 for coadd
in dcrCoadds[1:]:
964 coaddExposure.maskedImage += coadd.maskedImage
967 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
968 mask=None, variance=None):
969 """Create a list of coadd exposures from a list of masked images.
973 dcrModels : `lsst.pipe.tasks.DcrModel`
974 Best fit model of the true sky after correcting chromatic effects.
975 skyInfo : `lsst.pipe.base.Struct`
976 Patch geometry information, from getSkyInfo.
977 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
978 The data references to the input warped exposures.
979 weightList : `list` of `float`
980 The weight to give each input exposure
in the coadd.
981 calibration : `lsst.afw.Image.PhotoCalib`, optional
982 Scale factor to set the photometric calibration of an exposure.
983 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
984 A record of the observations that are included
in the coadd.
986 Optional mask to override the values
in the final coadd.
988 Optional variance plane to override the values
in the final coadd.
992 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
993 A list of coadd exposures, each exposure containing
994 the model
for one subfilter.
997 refModel = dcrModels.getReferenceImage()
998 for model
in dcrModels:
999 if self.config.accelerateModel > 1:
1000 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1001 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1002 if calibration
is not None:
1003 coaddExposure.setPhotoCalib(calibration)
1004 if coaddInputs
is not None:
1005 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1007 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1009 coaddExposure.setPsf(dcrModels.psf)
1011 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1012 maskedImage.image = model
1013 maskedImage.mask = dcrModels.mask
1014 maskedImage.variance = dcrModels.variance
1015 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1016 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1017 if mask
is not None:
1018 coaddExposure.setMask(mask)
1019 if variance
is not None:
1020 coaddExposure.setVariance(variance)
1021 dcrCoadds.append(coaddExposure)
1025 """Calculate the gain to use for the current iteration.
1027 After calculating a new DcrModel, each value is averaged
with the
1028 value
in the corresponding pixel
from the previous iteration. This
1029 reduces oscillating solutions that iterative techniques are plagued by,
1030 and speeds convergence. By far the biggest changes to the model
1031 happen
in the first couple iterations, so we can also use a more
1032 aggressive gain later when the model
is changing slowly.
1036 convergenceList : `list` of `float`
1037 The quality of fit metric
from each previous iteration.
1038 gainList : `list` of `float`
1039 The gains used
in each previous iteration: appended
with the new
1041 Gains are numbers between ``self.config.baseGain``
and 1.
1046 Relative weight to give the new solution when updating the model.
1047 A value of 1.0 gives equal weight to both solutions.
1052 If ``len(convergenceList) != len(gainList)+1``.
1054 nIter = len(convergenceList)
1055 if nIter != len(gainList) + 1:
1056 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)."
1057 % (len(convergenceList), len(gainList)))
1059 if self.config.baseGain
is None:
1062 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1064 baseGain = self.config.baseGain
1066 if self.config.useProgressiveGain
and nIter > 2:
1074 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1075 for i
in range(nIter - 1)]
1078 estFinalConv = np.array(estFinalConv)
1079 estFinalConv[estFinalConv < 0] = 0
1081 estFinalConv = np.median(estFinalConv[
max(nIter - 5, 0):])
1082 lastGain = gainList[-1]
1083 lastConv = convergenceList[-2]
1084 newConv = convergenceList[-1]
1089 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1095 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1096 newGain = 1 - abs(delta)
1098 newGain = (newGain + lastGain)/2.
1099 gain =
max(baseGain, newGain)
1102 gainList.append(gain)
1106 """Build an array that smoothly tapers to 0 away from detected sources.
1110 dcrModels : `lsst.pipe.tasks.DcrModel`
1111 Best fit model of the true sky after correcting chromatic effects.
1112 dcrBBox : `lsst.geom.box.Box2I`
1113 Sub-region of the coadd which includes a buffer to allow for DCR.
1117 weights : `numpy.ndarray`
or `float`
1118 A 2D array of weight values that tapers smoothly to zero away
from detected sources.
1119 Set to a placeholder value of 1.0
if ``self.config.useModelWeights``
is False.
1124 If ``useModelWeights``
is set
and ``modelWeightsWidth``
is negative.
1126 if not self.config.useModelWeights:
1128 if self.config.modelWeightsWidth < 0:
1129 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1130 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1131 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1132 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1133 weights[convergeMaskPixels] = 1.
1134 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth)
1135 weights /= np.max(weights)
1139 """Smoothly replace model pixel values with those from a
1140 reference at locations away from detected sources.
1145 The new DCR model images
from the current iteration.
1146 The values will be modified
in place.
1148 A reference image used to supply the default pixel values.
1149 modelWeights : `numpy.ndarray`
or `float`
1150 A 2D array of weight values that tapers smoothly to zero away
from detected sources.
1151 Set to a placeholder value of 1.0
if ``self.config.useModelWeights``
is False.
1153 if self.config.useModelWeights:
1154 for model
in modelImages:
1155 model.array *= modelWeights
1156 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1159 """Pre-load sub-regions of a list of exposures.
1163 bbox : `lsst.geom.box.Box2I`
1164 Sub-region to coadd.
1166 Statistics control object for coadd.
1167 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1168 The data references to the input warped exposures.
1169 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1170 The image scalars correct
for the zero point of the exposures.
1171 spanSetMaskList : `list` of `dict` containing spanSet lists,
or `
None`
1172 Each element
is dict
with keys = mask plane name to add the spans to.
1176 subExposures : `dict`
1177 The `dict` keys are the visit IDs,
1178 and the values are `lsst.afw.image.ExposureF`
1179 The pre-loaded exposures
for the current subregion.
1180 The variance plane contains weights,
and not the variance
1182 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1184 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1185 exposure = warpExpRef.get(parameters={
'bbox': bbox})
1186 visit = warpExpRef.dataId[
"visit"]
1187 if altMaskSpans
is not None:
1188 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1189 imageScaler.scaleMaskedImage(exposure.maskedImage)
1191 exposure.variance.array[:, :] = 0.
1193 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1196 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1197 subExposures[visit] = exposure
1201 """Compute the PSF of the coadd from the exposures with the best seeing.
1205 templateCoadd : `lsst.afw.image.ExposureF`
1206 The initial coadd exposure before accounting for DCR.
1207 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1208 The data references to the input warped exposures.
1213 The average PSF of the input exposures
with the best seeing.
1215 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1218 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1219 templatePsf = templateCoadd.getPsf()
1221 templateAvgPos = templatePsf.getAveragePosition()
1222 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm
1223 psfSizes = np.zeros(len(ccds))
1224 ccdVisits = np.array(ccds[
"visit"])
1225 for warpExpRef
in warpRefList:
1226 psf = warpExpRef.get(component=
"psf")
1227 visit = warpExpRef.dataId[
"visit"]
1228 psfAvgPos = psf.getAveragePosition()
1229 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
1230 psfSizes[ccdVisits == visit] = psfSize
1234 sizeThreshold =
min(np.median(psfSizes), psfRefSize)
1235 goodPsfs = psfSizes <= sizeThreshold
1236 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1237 self.config.coaddPsf.makeControl())
A 2-dimensional celestial WCS that transform pixels to ICRS RA/Dec, using the LSST standard for pixel...
A class to contain the data, WCS, and other information needed to describe an image of the sky.
A class to represent a 2-dimensional array of pixels.
Represent a 2-dimensional array of bitmask pixels.
A class to manipulate images, masks, and variance as a single object.
Information about a single exposure of an imaging camera.
Pass parameters to a Statistics object.
An integer coordinate rectangle.
CoaddPsf is the Psf derived to be used for non-PSF-matched Coadd images.
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def stackCoadd(self, dcrCoadds)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth)