25 from scipy
import ndimage
35 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf
import MeasurePsfTask
37 from .multiBand
import DetectCoaddSourcesTask
38 from .multiBandUtils
import _makeMakeIdFactory
40 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
44 dcrNumSubfilters = pexConfig.Field(
46 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
49 maxNumIter = pexConfig.Field(
52 doc=
"Maximum number of iterations of forward modeling.",
55 minNumIter = pexConfig.Field(
58 doc=
"Minimum number of iterations of forward modeling.",
61 convergenceThreshold = pexConfig.Field(
63 doc=
"Target relative change in convergence between iterations of forward modeling.",
66 useConvergence = pexConfig.Field(
68 doc=
"Use convergence test as a forward modeling end condition?" 69 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
72 baseGain = pexConfig.Field(
75 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 76 "A value of 1.0 gives equal weight to both solutions." 77 "Small values imply slower convergence of the solution, but can " 78 "help prevent overshooting and failures in the fit." 79 "If ``baseGain`` is None, a conservative gain " 80 "will be calculated from the number of subfilters. ",
83 useProgressiveGain = pexConfig.Field(
85 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 86 "When calculating the next gain, we use up to 5 previous gains and convergence values." 87 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
90 doAirmassWeight = pexConfig.Field(
92 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
95 modelWeightsWidth = pexConfig.Field(
97 doc=
"Width of the region around detected sources to include in the DcrModel.",
100 useModelWeights = pexConfig.Field(
102 doc=
"Width of the region around detected sources to include in the DcrModel.",
105 splitSubfilters = pexConfig.Field(
107 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 108 "Instead of at the midpoint",
111 regularizeModelIterations = pexConfig.Field(
113 doc=
"Maximum relative change of the model allowed between iterations." 114 "Set to zero to disable.",
117 regularizeModelFrequency = pexConfig.Field(
119 doc=
"Maximum relative change of the model allowed between subfilters." 120 "Set to zero to disable.",
123 convergenceMaskPlanes = pexConfig.ListField(
125 default=[
"DETECTED"],
126 doc=
"Mask planes to use to calculate convergence." 128 regularizationWidth = pexConfig.Field(
131 doc=
"Minimum radius of a region to include in regularization, in pixels." 133 imageInterpOrder = pexConfig.Field(
135 doc=
"The order of the spline interpolation used to shift the image plane.",
138 accelerateModel = pexConfig.Field(
140 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
143 doCalculatePsf = pexConfig.Field(
145 doc=
"Set to detect stars and recalculate the PSF from the final coadd." 146 "Otherwise the PSF is estimated from a selection of the best input exposures",
149 detection = pexConfig.ConfigurableField(
150 target=DetectCoaddSourcesTask,
151 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
153 measurement = pexConfig.ConfigurableField(
154 target=SingleFrameMeasurementTask,
155 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 157 measurePsf = pexConfig.ConfigurableField(
158 target=MeasurePsfTask,
159 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
163 CompareWarpAssembleCoaddConfig.setDefaults(self)
171 self.
detection.detection.thresholdValue = 10.0
175 """Assemble DCR coadded images from a set of warps. 180 The number of pixels to grow each subregion by to allow for DCR. 184 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 185 Warps (also called coadded temporary exposures), including the effects of 186 Differential Chromatic Refraction (DCR). 187 For full details of the mathematics and algorithm, please see 188 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 190 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 191 each subfilter used in the iterative calculation. 192 It begins by dividing the bandpass-defining filter into N equal bandwidth 193 "subfilters", and divides the flux in each pixel from an initial coadd 194 equally into each as a "dcrModel". Because the airmass and parallactic 195 angle of each individual exposure is known, we can calculate the shift 196 relative to the center of the band in each subfilter due to DCR. For each 197 exposure we apply this shift as a linear transformation to the dcrModels 198 and stack the results to produce a DCR-matched exposure. The matched 199 exposures are subtracted from the input exposures to produce a set of 200 residual images, and these residuals are reverse shifted for each 201 exposures' subfilters and stacked. The shifted and stacked residuals are 202 added to the dcrModels to produce a new estimate of the flux in each pixel 203 within each subfilter. The dcrModels are solved for iteratively, which 204 continues until the solution from a new iteration improves by less than 205 a set percentage, or a maximum number of iterations is reached. 206 Two forms of regularization are employed to reduce unphysical results. 207 First, the new solution is averaged with the solution from the previous 208 iteration, which mitigates oscillating solutions where the model 209 overshoots with alternating very high and low values. 210 Second, a common degeneracy when the data have a limited range of airmass or 211 parallactic angle values is for one subfilter to be fit with very low or 212 negative values, while another subfilter is fit with very high values. This 213 typically appears in the form of holes next to sources in one subfilter, 214 and corresponding extended wings in another. Because each subfilter has 215 a narrow bandwidth we assume that physical sources that are above the noise 216 level will not vary in flux by more than a factor of `frequencyClampFactor` 217 between subfilters, and pixels that have flux deviations larger than that 218 factor will have the excess flux distributed evenly among all subfilters. 221 ConfigClass = DcrAssembleCoaddConfig
222 _DefaultName =
"dcrAssembleCoadd" 223 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
227 if self.config.doCalculatePsf:
228 schema = afwTable.SourceTable.makeMinimalSchema()
229 self.makeSubtask(
"detection", schema=schema)
230 self.makeSubtask(
'measurement', schema=schema)
231 self.makeSubtask(
"measurePsf", schema=schema)
234 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
235 """Assemble a coadd from a set of warps. 237 Coadd a set of Warps. Compute weights to be applied to each Warp and 238 find scalings to match the photometric zeropoint to a reference Warp. 239 Assemble the Warps using run method. 240 Forward model chromatic effects across multiple subfilters, 241 and subtract from the input Warps to build sets of residuals. 242 Use the residuals to construct a new ``DcrModel`` for each subfilter, 243 and iterate until the model converges. 244 Interpolate over NaNs and optionally write the coadd to disk. 245 Return the coadded exposure. 249 dataRef : `lsst.daf.persistence.ButlerDataRef` 250 Data reference defining the patch for coaddition and the 252 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 253 List of data references to warps. Data to be coadded will be 254 selected from this list based on overlap with the patch defined by 259 results : `lsst.pipe.base.Struct` 260 The Struct contains the following fields: 262 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 263 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 264 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 265 - ``dcrNImages``: `list` of exposure count images for each subfilter 267 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
268 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
270 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
271 warpRefList=warpRefList)
274 self.log.
warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
275 skyInfo.patchInfo.getIndex())
277 for subfilter
in range(self.config.dcrNumSubfilters):
279 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
280 AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
281 if self.config.doWrite:
282 self.log.
info(
"Persisting dcrCoadd")
283 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
284 numSubfilters=self.config.dcrNumSubfilters)
285 if self.config.doNImage
and results.dcrNImages
is not None:
286 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
287 numSubfilters=self.config.dcrNumSubfilters)
292 """Interpolate over missing data and mask bright stars. 294 Also detect sources on the coadd exposure and measure the final PSF, 295 if ``doCalculatePsf`` is set. 299 coaddExposure : `lsst.afw.image.Exposure` 300 The final coadded exposure. 301 dataRef : `lsst.daf.persistence.ButlerDataRef` 302 Data reference defining the patch for coaddition and the 307 if self.config.doCalculatePsf:
309 expId = dataRef.get(
"dcrCoaddId")
310 detResults = self.detection.
run(coaddExposure, idFactory, expId)
311 coaddSources = detResults.outputSources
312 self.measurement.
run(
313 measCat=coaddSources,
314 exposure=coaddExposure,
317 psfResults = self.measurePsf.
run(coaddExposure, coaddSources, expId=expId)
318 coaddExposure.setPsf(psfResults.psf)
321 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 323 Sets the property ``bufferSize``. 327 templateCoadd : `lsst.afw.image.ExposureF` 328 The initial coadd exposure before accounting for DCR. 329 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 330 The data references to the input warped exposures. 331 weightList : `list` of `float` 332 The weight to give each input exposure in the coadd 333 Will be modified in place if ``doAirmassWeight`` is set. 337 dcrModels : `lsst.pipe.tasks.DcrModel` 338 Best fit model of the true sky after correcting chromatic effects. 343 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 345 filterInfo = templateCoadd.getFilter()
346 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
347 raise NotImplementedError(
"No minimum/maximum wavelength information found" 348 " in the filter definition! Please add lambdaMin and lambdaMax" 349 " to the Mapper class in your obs package.")
354 for visitNum, warpExpRef
in enumerate(warpRefList):
355 visitInfo = warpExpRef.get(tempExpName +
"_visitInfo")
356 visit = warpExpRef.dataId[
"visit"]
357 airmass = visitInfo.getBoresightAirmass()
358 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
359 airmassDict[visit] = airmass
360 angleDict[visit] = parallacticAngle
361 if self.config.doAirmassWeight:
362 weightList[visitNum] *= airmass
363 dcrShifts.append(np.max(np.abs(
calculateDcr(visitInfo, templateCoadd.getWcs(),
364 filterInfo, self.config.dcrNumSubfilters))))
365 self.log.
info(
"Selected airmasses:\n%s", airmassDict)
366 self.log.
info(
"Selected parallactic angles:\n%s", angleDict)
369 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
370 self.config.dcrNumSubfilters,
371 filterInfo=filterInfo,
375 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
376 supplementaryData=None):
377 """Assemble the coadd. 379 Requires additional inputs Struct ``supplementaryData`` to contain a 380 ``templateCoadd`` that serves as the model of the static sky. 382 Find artifacts and apply them to the warps' masks creating a list of 383 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 384 Then pass these alternative masks to the base class's assemble method. 386 Divide the ``templateCoadd`` evenly between each subfilter of a 387 ``DcrModel`` as the starting best estimate of the true wavelength- 388 dependent sky. Forward model the ``DcrModel`` using the known 389 chromatic effects in each subfilter and calculate a convergence metric 390 based on how well the modeled template matches the input warps. If 391 the convergence has not yet reached the desired threshold, then shift 392 and stack the residual images to build a new ``DcrModel``. Apply 393 conditioning to prevent oscillating solutions between iterations or 396 Once the ``DcrModel`` reaches convergence or the maximum number of 397 iterations has been reached, fill the metadata for each subfilter 398 image and make them proper ``coaddExposure``s. 402 skyInfo : `lsst.pipe.base.Struct` 403 Patch geometry information, from getSkyInfo 404 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 405 The data references to the input warped exposures. 406 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 407 The image scalars correct for the zero point of the exposures. 408 weightList : `list` of `float` 409 The weight to give each input exposure in the coadd 410 supplementaryData : `lsst.pipe.base.Struct` 411 Result struct returned by ``makeSupplementaryData`` with components: 413 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 417 result : `lsst.pipe.base.Struct` 418 Result struct with components: 420 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 421 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 422 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 423 - ``dcrNImages``: `list` of exposure count images for each subfilter 425 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
426 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
427 templateCoadd = supplementaryData.templateCoadd
428 baseMask = templateCoadd.mask.clone()
431 baseVariance = templateCoadd.variance.clone()
432 baseVariance /= self.config.dcrNumSubfilters
433 spanSetMaskList = self.
findArtifacts(templateCoadd, warpRefList, imageScalerList)
435 templateCoadd.setMask(baseMask)
436 badMaskPlanes = self.config.badMaskPlanes[:]
441 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
445 if self.config.doNImage:
446 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
447 spanSetMaskList, stats.ctrl)
448 nImage = afwImage.ImageU(skyInfo.bbox)
452 for dcrNImage
in dcrNImages:
458 nSubregions = (
ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
459 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
461 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
464 self.log.
info(
"Computing coadd over patch %s subregion %s of %s: %s",
465 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
468 dcrBBox.clip(dcrModels.bbox)
471 imageScalerList, spanSetMaskList)
473 warpRefList, weightList, stats.ctrl)
474 self.log.
info(
"Initial convergence : %s", convergenceMetric)
475 convergenceList = [convergenceMetric]
477 convergenceCheck = 1.
478 refImage = templateCoadd.image
479 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
482 stats.ctrl, convergenceMetric, baseMask, gain,
483 modelWeights, refImage, dcrWeights)
484 if self.config.useConvergence:
486 warpRefList, weightList, stats.ctrl)
487 if convergenceMetric == 0:
488 self.log.
warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 489 "most likely due to there being no valid data in the region.",
490 skyInfo.patchInfo.getIndex(), subIter)
492 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
493 if (convergenceCheck < 0) & (modelIter > minNumIter):
494 self.log.
warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 495 "iterations or desired convergence improvement of %s." 497 skyInfo.patchInfo.getIndex(), subIter,
498 self.config.convergenceThreshold, convergenceCheck)
500 convergenceList.append(convergenceMetric)
501 if modelIter > maxNumIter:
502 if self.config.useConvergence:
503 self.log.
warn(
"Coadd patch %s subregion %s reached maximum iterations " 504 "before reaching desired convergence improvement of %s." 505 " Final convergence improvement: %s",
506 skyInfo.patchInfo.getIndex(), subIter,
507 self.config.convergenceThreshold, convergenceCheck)
510 if self.config.useConvergence:
511 self.log.
info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
512 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
515 if self.config.useConvergence:
516 self.log.
info(
"Coadd patch %s subregion %s finished with " 517 "convergence metric %s after %s iterations",
518 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
520 self.log.
info(
"Coadd patch %s subregion %s finished after %s iterations",
521 skyInfo.patchInfo.getIndex(), subIter, modelIter)
522 if self.config.useConvergence
and convergenceMetric > 0:
523 self.log.
info(
"Final convergence improvement was %.4f%% overall",
524 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
526 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
527 calibration=self.scaleZeroPoint.getPhotoCalib(),
528 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
530 variance=baseVariance)
532 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
533 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
536 """Calculate the number of exposures contributing to each subfilter. 540 dcrModels : `lsst.pipe.tasks.DcrModel` 541 Best fit model of the true sky after correcting chromatic effects. 542 bbox : `lsst.afw.geom.box.Box2I` 543 Bounding box of the patch to coadd. 544 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 545 The data references to the input warped exposures. 546 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 547 Each element of the `dict` contains the new mask plane name 548 (e.g. "CLIPPED and/or "NO_DATA") as the key, 549 and the list of SpanSets to apply to the mask. 550 statsCtrl : `lsst.afw.math.StatisticsControl` 551 Statistics control object for coadd 555 dcrNImages : `list` of `lsst.afw.image.ImageU` 556 List of exposure count images for each subfilter 557 dcrWeights : `list` of `lsst.afw.image.ImageF` 558 Per-pixel weights for each subfilter. 559 Equal to 1/(number of unmasked images contributing to each pixel). 561 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
562 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
564 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
565 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
566 visitInfo = exposure.getInfo().getVisitInfo()
567 wcs = exposure.getInfo().getWcs()
569 if altMaskSpans
is not None:
571 weightImage = np.zeros_like(exposure.image.array)
572 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
573 dcrShift =
calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
574 for dcr, dcrNImage, dcrWeight
in zip(dcrShift, dcrNImages, dcrWeights):
576 shiftedWeights =
applyDcr(weightImage, dcr, useInverse=
True,
577 order=self.config.imageInterpOrder)
578 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
579 dcrWeight.array += shiftedWeights
581 weightsThreshold = 1.
582 goodPix = dcrWeights[0].array > weightsThreshold
583 for weights
in dcrWeights[1:]:
584 goodPix = (weights.array > weightsThreshold) & goodPix
585 for subfilter
in range(self.config.dcrNumSubfilters):
586 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
587 dcrWeights[subfilter].array[~goodPix] = 0.
588 dcrNImages[subfilter].array[~goodPix] = 0
589 return (dcrNImages, dcrWeights)
592 statsCtrl, convergenceMetric,
593 baseMask, gain, modelWeights, refImage, dcrWeights):
594 """Assemble the DCR coadd for a sub-region. 596 Build a DCR-matched template for each input exposure, then shift the 597 residuals according to the DCR in each subfilter. 598 Stack the shifted residuals and apply them as a correction to the 599 solution from the previous iteration. 600 Restrict the new model solutions from varying by more than a factor of 601 `modelClampFactor` from the last solution, and additionally restrict the 602 individual subfilter models from varying by more than a factor of 603 `frequencyClampFactor` from their average. 604 Finally, mitigate potentially oscillating solutions by averaging the new 605 solution with the solution from the previous iteration, weighted by 606 their convergence metric. 610 dcrModels : `lsst.pipe.tasks.DcrModel` 611 Best fit model of the true sky after correcting chromatic effects. 612 subExposures : `dict` of `lsst.afw.image.ExposureF` 613 The pre-loaded exposures for the current subregion. 614 bbox : `lsst.afw.geom.box.Box2I` 615 Bounding box of the subregion to coadd. 616 dcrBBox : `lsst.afw.geom.box.Box2I` 617 Sub-region of the coadd which includes a buffer to allow for DCR. 618 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 619 The data references to the input warped exposures. 620 statsCtrl : `lsst.afw.math.StatisticsControl` 621 Statistics control object for coadd 622 convergenceMetric : `float` 623 Quality of fit metric for the matched templates of the input images. 624 baseMask : `lsst.afw.image.Mask` 625 Mask of the initial template coadd. 626 gain : `float`, optional 627 Relative weight to give the new solution when updating the model. 628 modelWeights : `numpy.ndarray` or `float` 629 A 2D array of weight values that tapers smoothly to zero away from detected sources. 630 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 631 refImage : `lsst.afw.image.Image` 632 A reference image used to supply the default pixel values. 633 dcrWeights : `list` of `lsst.afw.image.Image` 634 Per-pixel weights for each subfilter. 635 Equal to 1/(number of unmasked images contributing to each pixel). 637 residualGeneratorList = []
639 for warpExpRef
in warpRefList:
640 exposure = subExposures[warpExpRef.dataId[
"visit"]]
641 visitInfo = exposure.getInfo().getVisitInfo()
642 wcs = exposure.getInfo().getWcs()
643 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
644 order=self.config.imageInterpOrder,
645 splitSubfilters=self.config.splitSubfilters,
646 amplifyModel=self.config.accelerateModel)
647 residual = exposure.image.array - templateImage.array
649 residual *= exposure.variance.array
653 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
655 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
656 mask=baseMask, gain=gain,
657 modelWeights=modelWeights,
659 dcrWeights=dcrWeights)
660 dcrModels.assign(dcrSubModelOut, bbox)
663 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 667 residual : `numpy.ndarray` 668 The residual masked image for one exposure, 669 after subtracting the matched template 670 visitInfo : `lsst.afw.image.VisitInfo` 671 Metadata for the exposure. 672 wcs : `lsst.afw.geom.SkyWcs` 673 Coordinate system definition (wcs) for the exposure. 674 filterInfo : `lsst.afw.image.Filter` 675 The filter definition, set in the current instruments' obs package. 676 Required for any calculation of DCR, including making matched templates. 680 residualImage : `numpy.ndarray` 681 The residual image for the next subfilter, shifted for DCR. 683 dcrShift =
calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
685 yield applyDcr(residual, dcr, useInverse=
True, order=self.config.imageInterpOrder)
688 mask, gain, modelWeights, refImage, dcrWeights):
689 """Calculate a new DcrModel from a set of image residuals. 693 dcrModels : `lsst.pipe.tasks.DcrModel` 694 Current model of the true sky after correcting chromatic effects. 695 residualGeneratorList : `generator` of `numpy.ndarray` 696 The residual image for the next subfilter, shifted for DCR. 697 dcrBBox : `lsst.afw.geom.box.Box2I` 698 Sub-region of the coadd which includes a buffer to allow for DCR. 699 statsCtrl : `lsst.afw.math.StatisticsControl` 700 Statistics control object for coadd 701 mask : `lsst.afw.image.Mask` 702 Mask to use for each new model image. 704 Relative weight to give the new solution when updating the model. 705 modelWeights : `numpy.ndarray` or `float` 706 A 2D array of weight values that tapers smoothly to zero away from detected sources. 707 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 708 refImage : `lsst.afw.image.Image` 709 A reference image used to supply the default pixel values. 710 dcrWeights : `list` of `lsst.afw.image.Image` 711 Per-pixel weights for each subfilter. 712 Equal to 1/(number of unmasked images contributing to each pixel). 716 dcrModel : `lsst.pipe.tasks.DcrModel` 717 New model of the true sky after correcting chromatic effects. 720 for subfilter, model
in enumerate(dcrModels):
721 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
722 residual = np.sum(residualsList, axis=0)
723 residual *= dcrWeights[subfilter][dcrBBox].array
725 newModel = model[dcrBBox].
clone()
726 newModel.array += residual
728 badPixels = ~np.isfinite(newModel.array)
729 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
730 if self.config.regularizeModelIterations > 0:
731 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
732 self.config.regularizeModelIterations,
733 self.config.regularizationWidth)
734 newModelImages.append(newModel)
735 if self.config.regularizeModelFrequency > 0:
736 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
737 self.config.regularizeModelFrequency,
738 self.config.regularizationWidth,
740 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
742 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
743 dcrModels.mask, dcrModels.variance)
746 """Calculate a quality of fit metric for the matched templates. 750 dcrModels : `lsst.pipe.tasks.DcrModel` 751 Best fit model of the true sky after correcting chromatic effects. 752 subExposures : `dict` of `lsst.afw.image.ExposureF` 753 The pre-loaded exposures for the current subregion. 754 bbox : `lsst.afw.geom.box.Box2I` 756 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 757 The data references to the input warped exposures. 758 weightList : `list` of `float` 759 The weight to give each input exposure in the coadd 760 statsCtrl : `lsst.afw.math.StatisticsControl` 761 Statistics control object for coadd 765 convergenceMetric : `float` 766 Quality of fit metric for all input exposures, within the sub-region 768 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
770 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
772 if np.max(significanceImage) == 0:
773 significanceImage += 1.
777 for warpExpRef, expWeight
in zip(warpRefList, weightList):
778 exposure = subExposures[warpExpRef.dataId[
"visit"]][bbox]
780 metric += singleMetric
781 metricList[warpExpRef.dataId[
"visit"]] = singleMetric
783 self.log.
info(
"Individual metrics:\n%s", metricList)
784 return 1.0
if weight == 0.0
else metric/weight
787 """Calculate a quality of fit metric for a single matched template. 791 dcrModels : `lsst.pipe.tasks.DcrModel` 792 Best fit model of the true sky after correcting chromatic effects. 793 exposure : `lsst.afw.image.ExposureF` 794 The input warped exposure to evaluate. 795 significanceImage : `numpy.ndarray` 796 Array of weights for each pixel corresponding to its significance 797 for the convergence calculation. 798 statsCtrl : `lsst.afw.math.StatisticsControl` 799 Statistics control object for coadd 803 convergenceMetric : `float` 804 Quality of fit metric for one exposure, within the sub-region. 806 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
807 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
808 order=self.config.imageInterpOrder,
809 splitSubfilters=self.config.splitSubfilters,
810 amplifyModel=self.config.accelerateModel)
811 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
812 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
814 finitePixels = np.isfinite(diffVals)
815 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
816 convergeMaskPixels = exposure.mask.array & convergeMask > 0
817 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
818 if np.sum(usePixels) == 0:
821 diffUse = diffVals[usePixels]
822 refUse = refVals[usePixels]
823 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
827 """Add a list of sub-band coadds together. 831 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 832 A list of coadd exposures, each exposure containing 833 the model for one subfilter. 837 coaddExposure : `lsst.afw.image.ExposureF` 838 A single coadd exposure that is the sum of the sub-bands. 840 coaddExposure = dcrCoadds[0].
clone()
841 for coadd
in dcrCoadds[1:]:
842 coaddExposure.maskedImage += coadd.maskedImage
845 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
846 mask=None, variance=None):
847 """Create a list of coadd exposures from a list of masked images. 851 dcrModels : `lsst.pipe.tasks.DcrModel` 852 Best fit model of the true sky after correcting chromatic effects. 853 skyInfo : `lsst.pipe.base.Struct` 854 Patch geometry information, from getSkyInfo 855 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 856 The data references to the input warped exposures. 857 weightList : `list` of `float` 858 The weight to give each input exposure in the coadd 859 calibration : `lsst.afw.Image.PhotoCalib`, optional 860 Scale factor to set the photometric calibration of an exposure. 861 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 862 A record of the observations that are included in the coadd. 863 mask : `lsst.afw.image.Mask`, optional 864 Optional mask to override the values in the final coadd. 865 variance : `lsst.afw.image.Image`, optional 866 Optional variance plane to override the values in the final coadd. 870 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 871 A list of coadd exposures, each exposure containing 872 the model for one subfilter. 875 refModel = dcrModels.getReferenceImage()
876 for model
in dcrModels:
877 if self.config.accelerateModel > 1:
878 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
879 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
880 if calibration
is not None:
881 coaddExposure.setPhotoCalib(calibration)
882 if coaddInputs
is not None:
883 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
887 coaddExposure.setPsf(dcrModels.psf)
889 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
890 maskedImage.image = model
891 maskedImage.mask = dcrModels.mask
892 maskedImage.variance = dcrModels.variance
893 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
895 coaddExposure.setMask(mask)
896 if variance
is not None:
897 coaddExposure.setVariance(variance)
898 dcrCoadds.append(coaddExposure)
902 """Calculate the gain to use for the current iteration. 904 After calculating a new DcrModel, each value is averaged with the 905 value in the corresponding pixel from the previous iteration. This 906 reduces oscillating solutions that iterative techniques are plagued by, 907 and speeds convergence. By far the biggest changes to the model 908 happen in the first couple iterations, so we can also use a more 909 aggressive gain later when the model is changing slowly. 913 convergenceList : `list` of `float` 914 The quality of fit metric from each previous iteration. 915 gainList : `list` of `float` 916 The gains used in each previous iteration: appended with the new 918 Gains are numbers between ``self.config.baseGain`` and 1. 923 Relative weight to give the new solution when updating the model. 924 A value of 1.0 gives equal weight to both solutions. 929 If ``len(convergenceList) != len(gainList)+1``. 931 nIter = len(convergenceList)
932 if nIter != len(gainList) + 1:
933 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 934 % (len(convergenceList), len(gainList)))
936 if self.config.baseGain
is None:
939 baseGain = 1./(self.config.dcrNumSubfilters - 1)
941 baseGain = self.config.baseGain
943 if self.config.useProgressiveGain
and nIter > 2:
951 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
952 for i
in range(nIter - 1)]
955 estFinalConv = np.array(estFinalConv)
956 estFinalConv[estFinalConv < 0] = 0
958 estFinalConv = np.median(estFinalConv[
max(nIter - 5, 0):])
959 lastGain = gainList[-1]
960 lastConv = convergenceList[-2]
961 newConv = convergenceList[-1]
966 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
972 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
973 newGain = 1 -
abs(delta)
975 newGain = (newGain + lastGain)/2.
976 gain =
max(baseGain, newGain)
979 gainList.append(gain)
983 """Build an array that smoothly tapers to 0 away from detected sources. 987 dcrModels : `lsst.pipe.tasks.DcrModel` 988 Best fit model of the true sky after correcting chromatic effects. 989 dcrBBox : `lsst.afw.geom.box.Box2I` 990 Sub-region of the coadd which includes a buffer to allow for DCR. 994 weights : `numpy.ndarray` or `float` 995 A 2D array of weight values that tapers smoothly to zero away from detected sources. 996 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1001 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 1003 if not self.config.useModelWeights:
1005 if self.config.modelWeightsWidth < 0:
1006 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1007 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1008 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1009 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1010 weights[convergeMaskPixels] = 1.
1011 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1012 weights /= np.max(weights)
1016 """Smoothly replace model pixel values with those from a 1017 reference at locations away from detected sources. 1021 modelImages : `list` of `lsst.afw.image.Image` 1022 The new DCR model images from the current iteration. 1023 The values will be modified in place. 1024 refImage : `lsst.afw.image.MaskedImage` 1025 A reference image used to supply the default pixel values. 1026 modelWeights : `numpy.ndarray` or `float` 1027 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1028 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1030 if self.config.useModelWeights:
1031 for model
in modelImages:
1032 model.array *= modelWeights
1033 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1036 """Pre-load sub-regions of a list of exposures. 1040 bbox : `lsst.afw.geom.box.Box2I` 1042 statsCtrl : `lsst.afw.math.StatisticsControl` 1043 Statistics control object for coadd 1044 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1045 The data references to the input warped exposures. 1046 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 1047 The image scalars correct for the zero point of the exposures. 1048 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 1049 Each element is dict with keys = mask plane name to add the spans to 1053 subExposures : `dict` 1054 The `dict` keys are the visit IDs, 1055 and the values are `lsst.afw.image.ExposureF` 1056 The pre-loaded exposures for the current subregion. 1057 The variance plane contains weights, and not the variance 1060 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1062 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1063 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
1064 if altMaskSpans
is not None:
1066 imageScaler.scaleMaskedImage(exposure.maskedImage)
1068 exposure.variance.array[:, :] = 0.
1070 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1073 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1074 subExposures[warpExpRef.dataId[
"visit"]] = exposure
1078 """Compute the PSF of the coadd from the exposures with the best seeing. 1082 templateCoadd : `lsst.afw.image.ExposureF` 1083 The initial coadd exposure before accounting for DCR. 1084 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1085 The data references to the input warped exposures. 1089 psf : `lsst.meas.algorithms.CoaddPsf` 1090 The average PSF of the input exposures with the best seeing. 1092 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1094 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1095 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1097 for visitNum, warpExpRef
in enumerate(warpRefList):
1098 psf = warpExpRef.get(tempExpName).getPsf()
1099 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1100 psfSizeList.append(psfSize)
1104 sizeThreshold =
min(np.median(psfSizeList), psfRefSize)
1105 goodVisits = np.array(psfSizeList) <= sizeThreshold
1106 psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1107 self.config.coaddPsf.makeControl())
def __init__(self, args, kwargs)
Extent< int, N > ceil(Extent< double, N > const &input) noexcept
Return the component-wise ceil (round towards more positive).
Angle abs(Angle const &a)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
void setCoaddEdgeBits(lsst::afw::image::Mask< lsst::afw::image::MaskPixel > &coaddMask, lsst::afw::image::Image< WeightPixelT > const &weightMap)
set edge bits of coadd mask based on weight map
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Fit spatial kernel using approximate fluxes for candidates, and solving a linear system of equations...
def applyDcr(image, dcr, useInverse=False, splitSubfilters=False, kwargs)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def processResults(self, coaddExposure, dataRef)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, baseMask, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters, splitSubfilters=False)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, mask, gain, modelWeights, refImage, dcrWeights)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
def _subBBoxIter(bbox, subregionSize)
An integer coordinate rectangle.
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)