22__all__ = [
'SourceDeblendConfig',
'SourceDeblendTask']
35from lsst.utils.timer
import timeMethod
40 edgeHandling = pexConfig.ChoiceField(
41 doc=
'What to do when a peak to be deblended is close to the edge of the image',
42 dtype=str, default=
'ramp',
44 'clip':
'Clip the template at the edge AND the mirror of the edge.',
45 'ramp':
'Ramp down flux at the image edge by the PSF',
46 'noclip':
'Ignore the edge when building the symmetric template.',
50 strayFluxToPointSources = pexConfig.ChoiceField(
51 doc=
'When the deblender should attribute stray flux to point sources',
52 dtype=str, default=
'necessary',
54 'necessary':
'When there is not an extended object in the footprint',
56 'never': (
'Never; stray flux will not be attributed to any deblended child '
57 'if the deblender thinks all peaks look like point sources'),
61 assignStrayFlux = pexConfig.Field(dtype=bool, default=
True,
62 doc=
'Assign stray flux (not claimed by any child in the deblender) '
63 'to deblend children.')
65 strayFluxRule = pexConfig.ChoiceField(
66 doc=
'How to split flux among peaks',
67 dtype=str, default=
'trim',
69 'r-to-peak':
'~ 1/(1+R^2) to the peak',
70 'r-to-footprint': (
'~ 1/(1+R^2) to the closest pixel in the footprint. '
71 'CAUTION: this can be computationally expensive on large footprints!'),
72 'nearest-footprint': (
'Assign 100% to the nearest footprint (using L-1 norm aka '
73 'Manhattan distance)'),
74 'trim': (
'Shrink the parent footprint to pixels that are not assigned to children')
78 clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
79 doc=(
'When splitting stray flux, clip fractions below '
80 'this value to zero.'))
81 psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
82 doc=(
'Chi-squared per DOF cut for deciding a source is '
83 'a PSF during deblending (un-shifted PSF model)'))
84 psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=
False,
85 doc=(
'Chi-squared per DOF cut for deciding a source is '
86 'PSF during deblending (shifted PSF model)'))
87 psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=
False,
88 doc=(
'Chi-squared per DOF cut for deciding a source is '
89 'a PSF during deblending (shifted PSF model #2)'))
90 maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
91 doc=(
"Only deblend the brightest maxNumberOfPeaks peaks in the parent"
92 " (<= 0: unlimited)"))
93 maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
94 doc=(
"Maximum area for footprints before they are ignored as large; "
95 "non-positive means no threshold applied"))
96 maxFootprintSize = pexConfig.Field(dtype=int, default=0,
97 doc=(
"Maximum linear dimension for footprints before they are ignored "
98 "as large; non-positive means no threshold applied"))
99 minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
100 doc=(
"Minimum axis ratio for footprints before they are ignored "
101 "as large; non-positive means no threshold applied"))
102 notDeblendedMask = pexConfig.Field(dtype=str, default=
"NOT_DEBLENDED", optional=
True,
103 doc=
"Mask name for footprints not deblended, or None")
105 tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=
True,
106 doc=(
'Footprints smaller in width or height than this value '
107 'will be ignored; minimum of 2 due to PSF gradient '
110 propagateAllPeaks = pexConfig.Field(dtype=bool, default=
False,
111 doc=(
'Guarantee that all peaks produce a child source.'))
112 catchFailures = pexConfig.Field(
115 doc=(
"If True, catch exceptions thrown by the deblender, log them, "
116 "and set a flag on the parent, instead of letting them propagate up."))
117 maskPlanes = pexConfig.ListField(dtype=str, default=[
"SAT",
"INTRP",
"NO_DATA"],
118 doc=
"Mask planes to ignore when performing statistics")
119 maskLimits = pexConfig.DictField(
123 doc=(
"Mask planes with the corresponding limit on the fraction of masked pixels. "
124 "Sources violating this limit will not be deblended."),
126 weightTemplates = pexConfig.Field(
127 dtype=bool, default=
False,
128 doc=(
"If true, a least-squares fit of the templates will be done to the "
129 "full image. The templates will be re-weighted based on this fit."))
130 removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=
False,
131 doc=(
"Try to remove similar templates?"))
132 maxTempDotProd = pexConfig.Field(
133 dtype=float, default=0.5,
134 doc=(
"If the dot product between two templates is larger than this value, we consider them to be "
135 "describing the same object (i.e. they are degenerate). If one of the objects has been "
136 "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
138 medianSmoothTemplate = pexConfig.Field(dtype=bool, default=
True,
139 doc=
"Apply a smoothing filter to all of the template images")
148 useCiLimits = pexConfig.Field(
149 dtype=bool, default=
False,
150 doc=
"Limit the number of sources deblended for CI to prevent long build times")
151 ciDeblendChildRange = pexConfig.ListField(
152 dtype=int, default=[2, 10],
153 doc=
"Only deblend parent Footprints with a number of peaks in the (inclusive) range indicated."
154 "If `useCiLimits==False` then this parameter is ignored.")
155 ciNumParentsToDeblend = pexConfig.Field(
156 dtype=int, default=10,
157 doc=
"Only use the first `ciNumParentsToDeblend` parent footprints with a total peak count "
158 "within `ciDebledChildRange`. "
159 "If `useCiLimits==False` then this parameter is ignored.")
163 """Split blended sources into individual sources.
165 This task has no return value; it only modifies the SourceCatalog
in-place.
167 ConfigClass = SourceDeblendConfig
168 _DefaultName = "sourceDeblend"
170 def __init__(self, schema, peakSchema=None, **kwargs):
171 """Create the task, adding necessary fields to the given schema.
176 Schema object for measurement fields; will be modified
in-place.
177 peakSchema : `lsst.afw.table.peakSchema`
178 Schema of Footprint Peaks that will be passed to the deblender.
179 Any fields beyond the PeakTable minimal schema will be transferred
180 to the main source Schema. If
None, no fields will be transferred
183 Additional keyword arguments passed to ~lsst.pipe.base.task
185 pipeBase.Task.__init__(self, **kwargs)
188 if item.field.getName().startswith(
"merge_footprint")]
189 peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
190 if peakSchema
is None:
196 for item
in peakSchema:
197 if item.key
not in peakMinimalSchema:
203 schema.addField(item.field)
204 assert schema == self.
peakSchemaMapper.getOutputSchema(),
"Logic bug mapping schemas"
208 self.
nChildKey = schema.addField(
'deblend_nChild', type=np.int32,
209 doc=
'Number of children this object has (defaults to 0)')
210 self.
psfKey = schema.addField(
'deblend_deblendedAsPsf', type=
'Flag',
211 doc=
'Deblender thought this source looked like a PSF')
212 self.
psfCenterKey = afwTable.Point2DKey.addFields(schema,
'deblend_psfCenter',
213 'If deblended-as-psf, the PSF centroid',
"pixel")
214 self.
psfFluxKey = schema.addField(
'deblend_psf_instFlux', type=
'D',
215 doc=
'If deblended-as-psf, the instrumental PSF flux', units=
'count')
217 doc=
'Source had too many peaks; '
218 'only the brightest were included')
219 self.
tooBigKey = schema.addField(
'deblend_parentTooBig', type=
'Flag',
220 doc=
'Parent footprint covered too many pixels')
221 self.
maskedKey = schema.addField(
'deblend_masked', type=
'Flag',
222 doc=
'Parent footprint was predominantly masked')
224 if self.config.catchFailures:
226 doc=
"Deblending failed on source")
229 doc=
"Deblender skipped this source")
232 'deblend_rampedTemplate', type=
'Flag',
233 doc=(
'This source was near an image edge and the deblender used '
234 '"ramp" edge-handling.'))
237 'deblend_patchedTemplate', type=
'Flag',
238 doc=(
'This source was near an image edge and the deblender used '
239 '"patched" edge-handling.'))
242 'deblend_hasStrayFlux', type=
'Flag',
243 doc=(
'This source was assigned some stray flux'))
245 self.log.trace(
'Added keys to schema: %s',
", ".join(str(x)
for x
in (
248 self.
peakCenter = afwTable.Point2IKey.addFields(schema, name=
"deblend_peak_center",
249 doc=
"Center used to apply constraints in scarlet",
251 self.
peakIdKey = schema.addField(
"deblend_peakId", type=np.int32,
252 doc=
"ID of the peak in the parent footprint. "
253 "This is not unique, but the combination of 'parent'"
254 "and 'peakId' should be for all child sources. "
255 "Top level blends with no parents have 'peakId=0'")
256 self.
nPeaksKey = schema.addField(
"deblend_nPeaks", type=np.int32,
257 doc=
"Number of initial peaks in the blend. "
258 "This includes peaks that may have been culled "
259 "during deblending or failed to deblend")
261 doc=
"Same as deblend_n_peaks, but the number of peaks "
262 "in the parent footprint")
265 def run(self, exposure, sources):
266 """Get the PSF from the provided exposure and then run deblend.
271 Exposure to be processed
273 SourceCatalog containing sources detected on this exposure.
275 psf = exposure.getPsf()
276 assert sources.getSchema() == self.
schema
277 self.
deblend(exposure, sources, psf)
280 return psf.computeShape(position).getDeterminantRadius() * 2.35
289 Exposure to be processed
291 SourceCatalog containing sources detected on this exposure
293 Point source function
300 if self.config.useCiLimits:
301 self.log.info(f
"Using CI catalog limits, "
302 f
"the original number of sources to deblend was {len(srcs)}.")
305 minChildren, maxChildren = self.config.ciDeblendChildRange
306 nPeaks = np.array([len(src.getFootprint().peaks)
for src
in srcs])
307 childrenInRange = np.where((nPeaks >= minChildren) & (nPeaks <= maxChildren))[0]
308 if len(childrenInRange) < self.config.ciNumParentsToDeblend:
309 raise ValueError(
"Fewer than ciNumParentsToDeblend children were contained in the range "
310 "indicated by ciDeblendChildRange. Adjust this range to include more "
314 parents = nPeaks == 1
315 children = np.zeros((len(srcs),), dtype=bool)
316 children[childrenInRange[:self.config.ciNumParentsToDeblend]] =
True
317 srcs = srcs[parents | children]
320 idFactory = srcs.getIdFactory()
321 maxId = np.max(srcs[
"id"])
322 idFactory.notify(maxId)
324 self.log.info(
"Deblending %d sources", len(srcs))
329 mi = exposure.getMaskedImage()
331 statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
333 sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
334 self.log.trace(
'sigma1: %g', sigma1)
338 for i, src
in enumerate(srcs):
341 fp = src.getFootprint()
354 self.log.warning(
'Parent %i: skipping large footprint (area: %i)',
355 int(src.getId()), int(fp.getArea()))
357 if self.
isMasked(fp, exposure.getMaskedImage().getMask()):
360 self.log.warning(
'Parent %i: skipping masked footprint (area: %i)',
361 int(src.getId()), int(fp.getArea()))
365 center = fp.getCentroid()
368 if not (psf_fwhm > 0):
369 if self.config.catchFailures:
370 self.log.warning(
"Unable to deblend source %d: because PSF FWHM=%f is invalid.",
371 src.getId(), psf_fwhm)
375 raise ValueError(f
"PSF at {center} has an invalid FWHM value of {psf_fwhm}")
377 self.log.trace(
'Parent %i: deblending %i peaks', int(src.getId()), len(pks))
383 src.set(self.
tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
387 fp, mi, psf, psf_fwhm, sigma1=sigma1,
388 psfChisqCut1=self.config.psfChisq1,
389 psfChisqCut2=self.config.psfChisq2,
390 psfChisqCut2b=self.config.psfChisq2b,
391 maxNumberOfPeaks=self.config.maxNumberOfPeaks,
392 strayFluxToPointSources=self.config.strayFluxToPointSources,
393 assignStrayFlux=self.config.assignStrayFlux,
394 strayFluxAssignment=self.config.strayFluxRule,
395 rampFluxAtEdge=(self.config.edgeHandling ==
'ramp'),
396 patchEdges=(self.config.edgeHandling ==
'noclip'),
397 tinyFootprintSize=self.config.tinyFootprintSize,
398 clipStrayFluxFraction=self.config.clipStrayFluxFraction,
399 weightTemplates=self.config.weightTemplates,
400 removeDegenerateTemplates=self.config.removeDegenerateTemplates,
401 maxTempDotProd=self.config.maxTempDotProd,
402 medianSmoothTemplate=self.config.medianSmoothTemplate
404 if self.config.catchFailures:
406 except Exception
as e:
407 if self.config.catchFailures:
408 self.log.warning(
"Unable to deblend source %d: %s", src.getId(), e)
411 traceback.print_exc()
418 for j, peak
in enumerate(res.deblendedParents[0].peaks):
419 heavy = peak.getFluxPortion()
420 if heavy
is None or peak.skip:
422 if not self.config.propagateAllPeaks:
427 self.log.trace(
"Peak at (%i,%i) failed. Using minimal default info for child.",
428 pks[j].getIx(), pks[j].getIy())
432 peakList = foot.getPeaks()
434 peakList.append(peak.peak)
435 zeroMimg = afwImage.MaskedImageF(foot.getBBox())
437 if peak.deblendedAsPsf:
438 if peak.psfFitFlux
is None:
439 peak.psfFitFlux = 0.0
440 if peak.psfFitCenter
is None:
441 peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
443 assert len(heavy.getPeaks()) == 1
446 child = srcs.addNew()
449 child.set(key, src.get(key))
451 child.setParent(src.getId())
452 child.setFootprint(heavy)
453 child.set(self.
psfKey, peak.deblendedAsPsf)
455 if peak.deblendedAsPsf:
456 (cx, cy) = peak.psfFitCenter
468 child.set(self.
peakIdKey, pks[j].getId())
482 spans = src.getFootprint().spans
484 spans = spans.union(child.getFootprint().spans)
485 src.getFootprint().setSpans(spans)
493 self.log.info(
'Deblended: of %i sources, %i were deblended, creating %i children, total %i sources',
494 n0, nparents, n1-n0, n1)
499 def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
503 """Returns whether a Footprint is large
505 'Large' is defined by thresholds on the area, size
and axis ratio.
506 These may be disabled independently by configuring them to be non-positive.
508 This
is principally intended to get rid of satellite streaks, which the
509 deblender
or other downstream processing can have trouble dealing
with
510 (e.g., multiple large HeavyFootprints can chew up memory).
512 if self.config.maxFootprintArea > 0
and footprint.getArea() > self.config.maxFootprintArea:
514 if self.config.maxFootprintSize > 0:
515 bbox = footprint.getBBox()
516 if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
518 if self.config.minFootprintAxisRatio > 0:
519 axes = afwEll.Axes(footprint.getShape())
520 if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
525 """Returns whether the footprint violates the mask limits
527 size = float(footprint.getArea())
528 for maskName, limit
in self.config.maskLimits.items():
529 maskVal = mask.getPlaneBitMask(maskName)
530 unmaskedSpan = footprint.spans.intersectNot(mask, maskVal)
531 if (size - unmaskedSpan.getArea())/size > limit:
536 """Indicate that the parent source is not being deblended
538 We set the appropriate flags and mask.
543 The source to flag
as skipped
547 fp = source.getFootprint()
549 if self.config.notDeblendedMask:
550 mask.addMaskPlane(self.config.notDeblendedMask)
551 fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
555 centerX = int(bbox.getMinX()+bbox.getWidth()/2)
556 centerY = int(bbox.getMinY()+bbox.getHeight()/2)
562 source.set(self.
nPeaksKey, len(fp.peaks))
A polymorphic base class for representing an image's Point Spread Function.
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Represent a 2-dimensional array of bitmask pixels.
Pass parameters to a Statistics object.
Defines the fields and offsets for a table.
A mapping between the keys of two Schemas, used to copy data between them.
Record class that contains measurements made on a single exposure.
isLargeFootprint(self, footprint)
isMasked(self, footprint, mask)
skipParent(self, source, mask)
preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
deblend(self, exposure, srcs, psf)
__init__(self, schema, peakSchema=None, **kwargs)
addSchemaKeys(self, schema)
postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
_getPsfFwhm(self, psf, position)
deblendPatchedTemplateKey
HeavyFootprint< ImagePixelT, MaskPixelT, VariancePixelT > makeHeavyFootprint(Footprint const &foot, lsst::afw::image::MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > const &img, HeavyFootprintCtrl const *ctrl=nullptr)
Create a HeavyFootprint with footprint defined by the given Footprint and pixel values from the given...
Statistics makeStatistics(lsst::afw::image::Image< Pixel > const &img, lsst::afw::image::Mask< image::MaskPixel > const &msk, int const flags, StatisticsControl const &sctrl=StatisticsControl())
Handle a watered-down front-end to the constructor (no variance)