25 from scipy
import ndimage
33 import lsst.pex.config
as pexConfig
35 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf
import MeasurePsfTask
38 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
42 dcrNumSubfilters = pexConfig.Field(
44 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
47 maxNumIter = pexConfig.Field(
50 doc=
"Maximum number of iterations of forward modeling.",
53 minNumIter = pexConfig.Field(
56 doc=
"Minimum number of iterations of forward modeling.",
59 convergenceThreshold = pexConfig.Field(
61 doc=
"Target relative change in convergence between iterations of forward modeling.",
64 useConvergence = pexConfig.Field(
66 doc=
"Use convergence test as a forward modeling end condition?" 67 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
70 baseGain = pexConfig.Field(
73 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 74 "A value of 1.0 gives equal weight to both solutions." 75 "Small values imply slower convergence of the solution, but can " 76 "help prevent overshooting and failures in the fit." 77 "If ``baseGain`` is None, a conservative gain " 78 "will be calculated from the number of subfilters. ",
81 useProgressiveGain = pexConfig.Field(
83 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 84 "When calculating the next gain, we use up to 5 previous gains and convergence values." 85 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
88 doAirmassWeight = pexConfig.Field(
90 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
93 modelWeightsWidth = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 useModelWeights = pexConfig.Field(
100 doc=
"Width of the region around detected sources to include in the DcrModel.",
103 splitSubfilters = pexConfig.Field(
105 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 106 "Instead of at the midpoint",
109 splitThreshold = pexConfig.Field(
111 doc=
"Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 112 "Set to 0 to always split the subfilters.",
115 regularizeModelIterations = pexConfig.Field(
117 doc=
"Maximum relative change of the model allowed between iterations." 118 "Set to zero to disable.",
121 regularizeModelFrequency = pexConfig.Field(
123 doc=
"Maximum relative change of the model allowed between subfilters." 124 "Set to zero to disable.",
127 convergenceMaskPlanes = pexConfig.ListField(
129 default=[
"DETECTED"],
130 doc=
"Mask planes to use to calculate convergence." 132 regularizationWidth = pexConfig.Field(
135 doc=
"Minimum radius of a region to include in regularization, in pixels." 137 imageInterpOrder = pexConfig.Field(
139 doc=
"The order of the spline interpolation used to shift the image plane.",
142 accelerateModel = pexConfig.Field(
144 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
147 doCalculatePsf = pexConfig.Field(
149 doc=
"Set to detect stars and recalculate the PSF from the final coadd." 150 "Otherwise the PSF is estimated from a selection of the best input exposures",
153 detectPsfSources = pexConfig.ConfigurableField(
154 target=measAlg.SourceDetectionTask,
155 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
157 measurePsfSources = pexConfig.ConfigurableField(
158 target=SingleFrameMeasurementTask,
159 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 161 measurePsf = pexConfig.ConfigurableField(
162 target=MeasurePsfTask,
163 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
167 CompareWarpAssembleCoaddConfig.setDefaults(self)
186 self.
measurePsf.starSelector[
"objectSize"].doFluxLimit =
False 190 """Assemble DCR coadded images from a set of warps. 195 The number of pixels to grow each subregion by to allow for DCR. 199 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 200 Warps (also called coadded temporary exposures), including the effects of 201 Differential Chromatic Refraction (DCR). 202 For full details of the mathematics and algorithm, please see 203 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 205 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 206 each subfilter used in the iterative calculation. 207 It begins by dividing the bandpass-defining filter into N equal bandwidth 208 "subfilters", and divides the flux in each pixel from an initial coadd 209 equally into each as a "dcrModel". Because the airmass and parallactic 210 angle of each individual exposure is known, we can calculate the shift 211 relative to the center of the band in each subfilter due to DCR. For each 212 exposure we apply this shift as a linear transformation to the dcrModels 213 and stack the results to produce a DCR-matched exposure. The matched 214 exposures are subtracted from the input exposures to produce a set of 215 residual images, and these residuals are reverse shifted for each 216 exposures' subfilters and stacked. The shifted and stacked residuals are 217 added to the dcrModels to produce a new estimate of the flux in each pixel 218 within each subfilter. The dcrModels are solved for iteratively, which 219 continues until the solution from a new iteration improves by less than 220 a set percentage, or a maximum number of iterations is reached. 221 Two forms of regularization are employed to reduce unphysical results. 222 First, the new solution is averaged with the solution from the previous 223 iteration, which mitigates oscillating solutions where the model 224 overshoots with alternating very high and low values. 225 Second, a common degeneracy when the data have a limited range of airmass or 226 parallactic angle values is for one subfilter to be fit with very low or 227 negative values, while another subfilter is fit with very high values. This 228 typically appears in the form of holes next to sources in one subfilter, 229 and corresponding extended wings in another. Because each subfilter has 230 a narrow bandwidth we assume that physical sources that are above the noise 231 level will not vary in flux by more than a factor of `frequencyClampFactor` 232 between subfilters, and pixels that have flux deviations larger than that 233 factor will have the excess flux distributed evenly among all subfilters. 234 If `splitSubfilters` is set, then each subfilter will be further sub- 235 divided during the forward modeling step (only). This approximates using 236 a higher number of subfilters that may be necessary for high airmass 237 observations, but does not increase the number of free parameters in the 238 fit. This is needed when there are high airmass observations which would 239 otherwise have significant DCR even within a subfilter. Because calculating 240 the shifted images takes most of the time, splitting the subfilters is 241 turned off by way of the `splitThreshold` option for low-airmass 242 observations that do not suffer from DCR within a subfilter. 245 ConfigClass = DcrAssembleCoaddConfig
246 _DefaultName =
"dcrAssembleCoadd" 250 if self.config.doCalculatePsf:
251 self.
schema = afwTable.SourceTable.makeMinimalSchema()
252 self.makeSubtask(
"detectPsfSources", schema=self.
schema)
253 self.makeSubtask(
"measurePsfSources", schema=self.
schema)
254 self.makeSubtask(
"measurePsf", schema=self.
schema)
257 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258 """Assemble a coadd from a set of warps. 260 Coadd a set of Warps. Compute weights to be applied to each Warp and 261 find scalings to match the photometric zeropoint to a reference Warp. 262 Assemble the Warps using run method. 263 Forward model chromatic effects across multiple subfilters, 264 and subtract from the input Warps to build sets of residuals. 265 Use the residuals to construct a new ``DcrModel`` for each subfilter, 266 and iterate until the model converges. 267 Interpolate over NaNs and optionally write the coadd to disk. 268 Return the coadded exposure. 272 dataRef : `lsst.daf.persistence.ButlerDataRef` 273 Data reference defining the patch for coaddition and the 275 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 276 List of data references to warps. Data to be coadded will be 277 selected from this list based on overlap with the patch defined by 282 results : `lsst.pipe.base.Struct` 283 The Struct contains the following fields: 285 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 286 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 287 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 288 - ``dcrNImages``: `list` of exposure count images for each subfilter 290 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
291 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
293 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294 warpRefList=warpRefList)
296 skyInfo = self.getSkyInfo(dataRef)
297 self.log.
warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
298 skyInfo.patchInfo.getIndex())
300 if self.config.doCalculatePsf:
302 brightObjects = self.readBrightObjectMasks(dataRef)
if self.config.doMaskBrightObjects
else None 303 for subfilter
in range(self.config.dcrNumSubfilters):
305 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
306 self.processResults(results.dcrCoadds[subfilter],
307 brightObjectMasks=brightObjects, dataId=dataRef.dataId)
308 if self.config.doWrite:
309 self.log.
info(
"Persisting dcrCoadd")
310 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
311 numSubfilters=self.config.dcrNumSubfilters)
312 if self.config.doNImage
and results.dcrNImages
is not None:
313 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
314 numSubfilters=self.config.dcrNumSubfilters)
319 """Detect sources on the coadd exposure and measure the final PSF. 323 coaddExposure : `lsst.afw.image.Exposure` 324 The final coadded exposure. 326 table = afwTable.SourceTable.make(self.
schema)
327 detResults = self.detectPsfSources.
run(table, coaddExposure, clearMask=
False)
328 coaddSources = detResults.sources
329 self.measurePsfSources.
run(
330 measCat=coaddSources,
331 exposure=coaddExposure
338 psfResults = self.measurePsf.
run(coaddExposure, coaddSources)
339 except Exception
as e:
340 self.log.
warn(
"Unable to calculate PSF, using default coadd PSF: %s" % e)
342 coaddExposure.setPsf(psfResults.psf)
345 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 347 Sets the property ``bufferSize``. 351 templateCoadd : `lsst.afw.image.ExposureF` 352 The initial coadd exposure before accounting for DCR. 353 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 354 The data references to the input warped exposures. 355 weightList : `list` of `float` 356 The weight to give each input exposure in the coadd 357 Will be modified in place if ``doAirmassWeight`` is set. 361 dcrModels : `lsst.pipe.tasks.DcrModel` 362 Best fit model of the true sky after correcting chromatic effects. 367 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 369 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
370 filterInfo = templateCoadd.getFilter()
371 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
372 raise NotImplementedError(
"No minimum/maximum wavelength information found" 373 " in the filter definition! Please add lambdaMin and lambdaMax" 374 " to the Mapper class in your obs package.")
375 tempExpName = self.getTempExpDatasetName(self.warpType)
380 for visitNum, warpExpRef
in enumerate(warpRefList):
381 visitInfo = warpExpRef.get(tempExpName +
"_visitInfo")
382 visit = warpExpRef.dataId[
"visit"]
383 psf = warpExpRef.get(tempExpName).getPsf()
384 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
385 airmass = visitInfo.getBoresightAirmass()
386 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
387 airmassDict[visit] = airmass
388 angleDict[visit] = parallacticAngle
389 psfSizeDict[visit] = psfSize
390 if self.config.doAirmassWeight:
391 weightList[visitNum] *= airmass
392 dcrShifts.append(np.max(np.abs(
calculateDcr(visitInfo, templateCoadd.getWcs(),
393 filterInfo, self.config.dcrNumSubfilters))))
394 self.log.
info(
"Selected airmasses:\n%s", airmassDict)
395 self.log.
info(
"Selected parallactic angles:\n%s", angleDict)
396 self.log.
info(
"Selected PSF sizes:\n%s", psfSizeDict)
399 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
400 self.config.dcrNumSubfilters,
401 filterInfo=filterInfo,
405 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
406 supplementaryData=None):
407 """Assemble the coadd. 409 Requires additional inputs Struct ``supplementaryData`` to contain a 410 ``templateCoadd`` that serves as the model of the static sky. 412 Find artifacts and apply them to the warps' masks creating a list of 413 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 414 Then pass these alternative masks to the base class's assemble method. 416 Divide the ``templateCoadd`` evenly between each subfilter of a 417 ``DcrModel`` as the starting best estimate of the true wavelength- 418 dependent sky. Forward model the ``DcrModel`` using the known 419 chromatic effects in each subfilter and calculate a convergence metric 420 based on how well the modeled template matches the input warps. If 421 the convergence has not yet reached the desired threshold, then shift 422 and stack the residual images to build a new ``DcrModel``. Apply 423 conditioning to prevent oscillating solutions between iterations or 426 Once the ``DcrModel`` reaches convergence or the maximum number of 427 iterations has been reached, fill the metadata for each subfilter 428 image and make them proper ``coaddExposure``s. 432 skyInfo : `lsst.pipe.base.Struct` 433 Patch geometry information, from getSkyInfo 434 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 435 The data references to the input warped exposures. 436 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 437 The image scalars correct for the zero point of the exposures. 438 weightList : `list` of `float` 439 The weight to give each input exposure in the coadd 440 supplementaryData : `lsst.pipe.base.Struct` 441 Result struct returned by ``makeSupplementaryData`` with components: 443 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 447 result : `lsst.pipe.base.Struct` 448 Result struct with components: 450 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 451 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 452 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 453 - ``dcrNImages``: `list` of exposure count images for each subfilter 455 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
456 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
457 templateCoadd = supplementaryData.templateCoadd
458 baseMask = templateCoadd.mask.clone()
461 baseVariance = templateCoadd.variance.clone()
462 baseVariance /= self.config.dcrNumSubfilters
463 spanSetMaskList = self.
findArtifacts(templateCoadd, warpRefList, imageScalerList)
465 templateCoadd.setMask(baseMask)
466 badMaskPlanes = self.config.badMaskPlanes[:]
471 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
473 stats = self.prepareStats(mask=badPixelMask)
475 if self.config.doNImage:
476 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
477 spanSetMaskList, stats.ctrl)
478 nImage = afwImage.ImageU(skyInfo.bbox)
482 for dcrNImage
in dcrNImages:
488 nSubregions = (
ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
489 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
491 for subBBox
in self._subBBoxIter(skyInfo.bbox, subregionSize):
494 self.log.
info(
"Computing coadd over patch %s subregion %s of %s: %s",
495 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
498 dcrBBox.clip(dcrModels.bbox)
501 imageScalerList, spanSetMaskList)
503 warpRefList, weightList, stats.ctrl)
504 self.log.
info(
"Initial convergence : %s", convergenceMetric)
505 convergenceList = [convergenceMetric]
507 convergenceCheck = 1.
508 refImage = templateCoadd.image
509 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
512 stats.ctrl, convergenceMetric, gain,
513 modelWeights, refImage, dcrWeights)
514 if self.config.useConvergence:
516 warpRefList, weightList, stats.ctrl)
517 if convergenceMetric == 0:
518 self.log.
warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 519 "most likely due to there being no valid data in the region.",
520 skyInfo.patchInfo.getIndex(), subIter)
522 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
523 if (convergenceCheck < 0) & (modelIter > minNumIter):
524 self.log.
warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 525 "iterations or desired convergence improvement of %s." 527 skyInfo.patchInfo.getIndex(), subIter,
528 self.config.convergenceThreshold, convergenceCheck)
530 convergenceList.append(convergenceMetric)
531 if modelIter > maxNumIter:
532 if self.config.useConvergence:
533 self.log.
warn(
"Coadd patch %s subregion %s reached maximum iterations " 534 "before reaching desired convergence improvement of %s." 535 " Final convergence improvement: %s",
536 skyInfo.patchInfo.getIndex(), subIter,
537 self.config.convergenceThreshold, convergenceCheck)
540 if self.config.useConvergence:
541 self.log.
info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
542 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
545 if self.config.useConvergence:
546 self.log.
info(
"Coadd patch %s subregion %s finished with " 547 "convergence metric %s after %s iterations",
548 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
550 self.log.
info(
"Coadd patch %s subregion %s finished after %s iterations",
551 skyInfo.patchInfo.getIndex(), subIter, modelIter)
552 if self.config.useConvergence
and convergenceMetric > 0:
553 self.log.
info(
"Final convergence improvement was %.4f%% overall",
554 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
556 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
557 calibration=self.scaleZeroPoint.getPhotoCalib(),
558 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
560 variance=baseVariance)
562 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
563 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
566 """Calculate the number of exposures contributing to each subfilter. 570 dcrModels : `lsst.pipe.tasks.DcrModel` 571 Best fit model of the true sky after correcting chromatic effects. 572 bbox : `lsst.geom.box.Box2I` 573 Bounding box of the patch to coadd. 574 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 575 The data references to the input warped exposures. 576 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 577 Each element of the `dict` contains the new mask plane name 578 (e.g. "CLIPPED and/or "NO_DATA") as the key, 579 and the list of SpanSets to apply to the mask. 580 statsCtrl : `lsst.afw.math.StatisticsControl` 581 Statistics control object for coadd 585 dcrNImages : `list` of `lsst.afw.image.ImageU` 586 List of exposure count images for each subfilter 587 dcrWeights : `list` of `lsst.afw.image.ImageF` 588 Per-pixel weights for each subfilter. 589 Equal to 1/(number of unmasked images contributing to each pixel). 591 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
592 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
593 tempExpName = self.getTempExpDatasetName(self.warpType)
594 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
595 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
596 visitInfo = exposure.getInfo().getVisitInfo()
597 wcs = exposure.getInfo().getWcs()
599 if altMaskSpans
is not None:
600 self.applyAltMaskPlanes(mask, altMaskSpans)
601 weightImage = np.zeros_like(exposure.image.array)
602 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
605 weightsGenerator = self.
dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
606 for shiftedWeights, dcrNImage, dcrWeight
in zip(weightsGenerator, dcrNImages, dcrWeights):
607 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
608 dcrWeight.array += shiftedWeights
610 weightsThreshold = 1.
611 goodPix = dcrWeights[0].array > weightsThreshold
612 for weights
in dcrWeights[1:]:
613 goodPix = (weights.array > weightsThreshold) & goodPix
614 for subfilter
in range(self.config.dcrNumSubfilters):
615 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
616 dcrWeights[subfilter].array[~goodPix] = 0.
617 dcrNImages[subfilter].array[~goodPix] = 0
618 return (dcrNImages, dcrWeights)
621 statsCtrl, convergenceMetric,
622 gain, modelWeights, refImage, dcrWeights):
623 """Assemble the DCR coadd for a sub-region. 625 Build a DCR-matched template for each input exposure, then shift the 626 residuals according to the DCR in each subfilter. 627 Stack the shifted residuals and apply them as a correction to the 628 solution from the previous iteration. 629 Restrict the new model solutions from varying by more than a factor of 630 `modelClampFactor` from the last solution, and additionally restrict the 631 individual subfilter models from varying by more than a factor of 632 `frequencyClampFactor` from their average. 633 Finally, mitigate potentially oscillating solutions by averaging the new 634 solution with the solution from the previous iteration, weighted by 635 their convergence metric. 639 dcrModels : `lsst.pipe.tasks.DcrModel` 640 Best fit model of the true sky after correcting chromatic effects. 641 subExposures : `dict` of `lsst.afw.image.ExposureF` 642 The pre-loaded exposures for the current subregion. 643 bbox : `lsst.geom.box.Box2I` 644 Bounding box of the subregion to coadd. 645 dcrBBox : `lsst.geom.box.Box2I` 646 Sub-region of the coadd which includes a buffer to allow for DCR. 647 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 648 The data references to the input warped exposures. 649 statsCtrl : `lsst.afw.math.StatisticsControl` 650 Statistics control object for coadd 651 convergenceMetric : `float` 652 Quality of fit metric for the matched templates of the input images. 653 gain : `float`, optional 654 Relative weight to give the new solution when updating the model. 655 modelWeights : `numpy.ndarray` or `float` 656 A 2D array of weight values that tapers smoothly to zero away from detected sources. 657 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 658 refImage : `lsst.afw.image.Image` 659 A reference image used to supply the default pixel values. 660 dcrWeights : `list` of `lsst.afw.image.Image` 661 Per-pixel weights for each subfilter. 662 Equal to 1/(number of unmasked images contributing to each pixel). 664 residualGeneratorList = []
666 for warpExpRef
in warpRefList:
667 exposure = subExposures[warpExpRef.dataId[
"visit"]]
668 visitInfo = exposure.getInfo().getVisitInfo()
669 wcs = exposure.getInfo().getWcs()
670 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
671 order=self.config.imageInterpOrder,
672 splitSubfilters=self.config.splitSubfilters,
673 splitThreshold=self.config.splitThreshold,
674 amplifyModel=self.config.accelerateModel)
675 residual = exposure.image.array - templateImage.array
677 residual *= exposure.variance.array
681 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
683 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
685 modelWeights=modelWeights,
687 dcrWeights=dcrWeights)
688 dcrModels.assign(dcrSubModelOut, bbox)
691 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 695 residual : `numpy.ndarray` 696 The residual masked image for one exposure, 697 after subtracting the matched template 698 visitInfo : `lsst.afw.image.VisitInfo` 699 Metadata for the exposure. 700 wcs : `lsst.afw.geom.SkyWcs` 701 Coordinate system definition (wcs) for the exposure. 702 filterInfo : `lsst.afw.image.Filter` 703 The filter definition, set in the current instruments' obs package. 704 Required for any calculation of DCR, including making matched templates. 708 residualImage : `numpy.ndarray` 709 The residual image for the next subfilter, shifted for DCR. 713 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
716 dcrShift =
calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
717 splitSubfilters=
False)
719 yield applyDcr(filteredResidual, dcr, useInverse=
True, splitSubfilters=
False,
720 doPrefilter=
False, order=self.config.imageInterpOrder)
723 gain, modelWeights, refImage, dcrWeights):
724 """Calculate a new DcrModel from a set of image residuals. 728 dcrModels : `lsst.pipe.tasks.DcrModel` 729 Current model of the true sky after correcting chromatic effects. 730 residualGeneratorList : `generator` of `numpy.ndarray` 731 The residual image for the next subfilter, shifted for DCR. 732 dcrBBox : `lsst.geom.box.Box2I` 733 Sub-region of the coadd which includes a buffer to allow for DCR. 734 statsCtrl : `lsst.afw.math.StatisticsControl` 735 Statistics control object for coadd 737 Relative weight to give the new solution when updating the model. 738 modelWeights : `numpy.ndarray` or `float` 739 A 2D array of weight values that tapers smoothly to zero away from detected sources. 740 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 741 refImage : `lsst.afw.image.Image` 742 A reference image used to supply the default pixel values. 743 dcrWeights : `list` of `lsst.afw.image.Image` 744 Per-pixel weights for each subfilter. 745 Equal to 1/(number of unmasked images contributing to each pixel). 749 dcrModel : `lsst.pipe.tasks.DcrModel` 750 New model of the true sky after correcting chromatic effects. 753 for subfilter, model
in enumerate(dcrModels):
754 residualsList = [
next(residualGenerator)
for residualGenerator
in residualGeneratorList]
755 residual = np.sum(residualsList, axis=0)
756 residual *= dcrWeights[subfilter][dcrBBox].array
758 newModel = model[dcrBBox].
clone()
759 newModel.array += residual
761 badPixels = ~np.isfinite(newModel.array)
762 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
763 if self.config.regularizeModelIterations > 0:
764 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
765 self.config.regularizeModelIterations,
766 self.config.regularizationWidth)
767 newModelImages.append(newModel)
768 if self.config.regularizeModelFrequency > 0:
769 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
770 self.config.regularizeModelFrequency,
771 self.config.regularizationWidth)
772 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
774 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
775 dcrModels.mask, dcrModels.variance)
778 """Calculate a quality of fit metric for the matched templates. 782 dcrModels : `lsst.pipe.tasks.DcrModel` 783 Best fit model of the true sky after correcting chromatic effects. 784 subExposures : `dict` of `lsst.afw.image.ExposureF` 785 The pre-loaded exposures for the current subregion. 786 bbox : `lsst.geom.box.Box2I` 788 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 789 The data references to the input warped exposures. 790 weightList : `list` of `float` 791 The weight to give each input exposure in the coadd 792 statsCtrl : `lsst.afw.math.StatisticsControl` 793 Statistics control object for coadd 797 convergenceMetric : `float` 798 Quality of fit metric for all input exposures, within the sub-region 800 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
802 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
804 if np.max(significanceImage) == 0:
805 significanceImage += 1.
809 for warpExpRef, expWeight
in zip(warpRefList, weightList):
810 exposure = subExposures[warpExpRef.dataId[
"visit"]][bbox]
812 metric += singleMetric
813 metricList[warpExpRef.dataId[
"visit"]] = singleMetric
815 self.log.
info(
"Individual metrics:\n%s", metricList)
816 return 1.0
if weight == 0.0
else metric/weight
819 """Calculate a quality of fit metric for a single matched template. 823 dcrModels : `lsst.pipe.tasks.DcrModel` 824 Best fit model of the true sky after correcting chromatic effects. 825 exposure : `lsst.afw.image.ExposureF` 826 The input warped exposure to evaluate. 827 significanceImage : `numpy.ndarray` 828 Array of weights for each pixel corresponding to its significance 829 for the convergence calculation. 830 statsCtrl : `lsst.afw.math.StatisticsControl` 831 Statistics control object for coadd 835 convergenceMetric : `float` 836 Quality of fit metric for one exposure, within the sub-region. 838 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
839 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
840 order=self.config.imageInterpOrder,
841 splitSubfilters=self.config.splitSubfilters,
842 splitThreshold=self.config.splitThreshold,
843 amplifyModel=self.config.accelerateModel)
844 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
845 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
847 finitePixels = np.isfinite(diffVals)
848 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
849 convergeMaskPixels = exposure.mask.array & convergeMask > 0
850 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
851 if np.sum(usePixels) == 0:
854 diffUse = diffVals[usePixels]
855 refUse = refVals[usePixels]
856 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
860 """Add a list of sub-band coadds together. 864 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 865 A list of coadd exposures, each exposure containing 866 the model for one subfilter. 870 coaddExposure : `lsst.afw.image.ExposureF` 871 A single coadd exposure that is the sum of the sub-bands. 873 coaddExposure = dcrCoadds[0].
clone()
874 for coadd
in dcrCoadds[1:]:
875 coaddExposure.maskedImage += coadd.maskedImage
878 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
879 mask=None, variance=None):
880 """Create a list of coadd exposures from a list of masked images. 884 dcrModels : `lsst.pipe.tasks.DcrModel` 885 Best fit model of the true sky after correcting chromatic effects. 886 skyInfo : `lsst.pipe.base.Struct` 887 Patch geometry information, from getSkyInfo 888 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 889 The data references to the input warped exposures. 890 weightList : `list` of `float` 891 The weight to give each input exposure in the coadd 892 calibration : `lsst.afw.Image.PhotoCalib`, optional 893 Scale factor to set the photometric calibration of an exposure. 894 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 895 A record of the observations that are included in the coadd. 896 mask : `lsst.afw.image.Mask`, optional 897 Optional mask to override the values in the final coadd. 898 variance : `lsst.afw.image.Image`, optional 899 Optional variance plane to override the values in the final coadd. 903 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 904 A list of coadd exposures, each exposure containing 905 the model for one subfilter. 908 refModel = dcrModels.getReferenceImage()
909 for model
in dcrModels:
910 if self.config.accelerateModel > 1:
911 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
912 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
913 if calibration
is not None:
914 coaddExposure.setPhotoCalib(calibration)
915 if coaddInputs
is not None:
916 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
918 self.assembleMetadata(coaddExposure, warpRefList, weightList)
920 coaddExposure.setPsf(dcrModels.psf)
922 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
923 maskedImage.image = model
924 maskedImage.mask = dcrModels.mask
925 maskedImage.variance = dcrModels.variance
926 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
928 coaddExposure.setMask(mask)
929 if variance
is not None:
930 coaddExposure.setVariance(variance)
931 dcrCoadds.append(coaddExposure)
935 """Calculate the gain to use for the current iteration. 937 After calculating a new DcrModel, each value is averaged with the 938 value in the corresponding pixel from the previous iteration. This 939 reduces oscillating solutions that iterative techniques are plagued by, 940 and speeds convergence. By far the biggest changes to the model 941 happen in the first couple iterations, so we can also use a more 942 aggressive gain later when the model is changing slowly. 946 convergenceList : `list` of `float` 947 The quality of fit metric from each previous iteration. 948 gainList : `list` of `float` 949 The gains used in each previous iteration: appended with the new 951 Gains are numbers between ``self.config.baseGain`` and 1. 956 Relative weight to give the new solution when updating the model. 957 A value of 1.0 gives equal weight to both solutions. 962 If ``len(convergenceList) != len(gainList)+1``. 964 nIter = len(convergenceList)
965 if nIter != len(gainList) + 1:
966 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 967 % (len(convergenceList), len(gainList)))
969 if self.config.baseGain
is None:
972 baseGain = 1./(self.config.dcrNumSubfilters - 1)
974 baseGain = self.config.baseGain
976 if self.config.useProgressiveGain
and nIter > 2:
984 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
985 for i
in range(nIter - 1)]
988 estFinalConv = np.array(estFinalConv)
989 estFinalConv[estFinalConv < 0] = 0
991 estFinalConv = np.median(estFinalConv[
max(nIter - 5, 0):])
992 lastGain = gainList[-1]
993 lastConv = convergenceList[-2]
994 newConv = convergenceList[-1]
999 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1005 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1006 newGain = 1 -
abs(delta)
1008 newGain = (newGain + lastGain)/2.
1009 gain =
max(baseGain, newGain)
1012 gainList.append(gain)
1016 """Build an array that smoothly tapers to 0 away from detected sources. 1020 dcrModels : `lsst.pipe.tasks.DcrModel` 1021 Best fit model of the true sky after correcting chromatic effects. 1022 dcrBBox : `lsst.geom.box.Box2I` 1023 Sub-region of the coadd which includes a buffer to allow for DCR. 1027 weights : `numpy.ndarray` or `float` 1028 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1029 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1034 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 1036 if not self.config.useModelWeights:
1038 if self.config.modelWeightsWidth < 0:
1039 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1040 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1041 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1042 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1043 weights[convergeMaskPixels] = 1.
1044 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1045 weights /= np.max(weights)
1049 """Smoothly replace model pixel values with those from a 1050 reference at locations away from detected sources. 1054 modelImages : `list` of `lsst.afw.image.Image` 1055 The new DCR model images from the current iteration. 1056 The values will be modified in place. 1057 refImage : `lsst.afw.image.MaskedImage` 1058 A reference image used to supply the default pixel values. 1059 modelWeights : `numpy.ndarray` or `float` 1060 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1061 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1063 if self.config.useModelWeights:
1064 for model
in modelImages:
1065 model.array *= modelWeights
1066 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1069 """Pre-load sub-regions of a list of exposures. 1073 bbox : `lsst.geom.box.Box2I` 1075 statsCtrl : `lsst.afw.math.StatisticsControl` 1076 Statistics control object for coadd 1077 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1078 The data references to the input warped exposures. 1079 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 1080 The image scalars correct for the zero point of the exposures. 1081 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 1082 Each element is dict with keys = mask plane name to add the spans to 1086 subExposures : `dict` 1087 The `dict` keys are the visit IDs, 1088 and the values are `lsst.afw.image.ExposureF` 1089 The pre-loaded exposures for the current subregion. 1090 The variance plane contains weights, and not the variance 1092 tempExpName = self.getTempExpDatasetName(self.warpType)
1093 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1095 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1096 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
1097 if altMaskSpans
is not None:
1098 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1099 imageScaler.scaleMaskedImage(exposure.maskedImage)
1101 exposure.variance.array[:, :] = 0.
1103 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1106 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1107 subExposures[warpExpRef.dataId[
"visit"]] = exposure
1111 """Compute the PSF of the coadd from the exposures with the best seeing. 1115 templateCoadd : `lsst.afw.image.ExposureF` 1116 The initial coadd exposure before accounting for DCR. 1117 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1118 The data references to the input warped exposures. 1122 psf : `lsst.meas.algorithms.CoaddPsf` 1123 The average PSF of the input exposures with the best seeing. 1125 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1126 tempExpName = self.getTempExpDatasetName(self.warpType)
1129 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1130 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1131 psfSizes = np.zeros(len(ccds))
1132 ccdVisits = np.array(ccds[
"visit"])
1133 for warpExpRef
in warpRefList:
1134 psf = warpExpRef.get(tempExpName).getPsf()
1135 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1136 visit = warpExpRef.dataId[
"visit"]
1137 psfSizes[ccdVisits == visit] = psfSize
1141 sizeThreshold =
min(np.median(psfSizes), psfRefSize)
1142 goodPsfs = psfSizes <= sizeThreshold
1143 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1144 self.config.coaddPsf.makeControl())
def __init__(self, args, kwargs)
Extent< int, N > ceil(Extent< double, N > const &input) noexcept
Return the component-wise ceil (round towards more positive).
Angle abs(Angle const &a)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def applyDcr(image, dcr, useInverse=False, splitSubfilters=False, splitThreshold=0., doPrefilter=True, order=3)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
def measureCoaddPsf(self, coaddExposure)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters, splitSubfilters=False)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
An integer coordinate rectangle.
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)