LSSTApplications  17.0+124,17.0+14,17.0+73,18.0.0+37,18.0.0+80,18.0.0-4-g68ffd23+4,18.1.0-1-g0001055+12,18.1.0-1-g03d53ef+5,18.1.0-1-g1349e88+55,18.1.0-1-g2505f39+44,18.1.0-1-g5315e5e+4,18.1.0-1-g5e4b7ea+14,18.1.0-1-g7e8fceb+4,18.1.0-1-g85f8cd4+48,18.1.0-1-g8ff0b9f+4,18.1.0-1-ga2c679d+1,18.1.0-1-gd55f500+35,18.1.0-10-gb58edde+2,18.1.0-11-g0997b02+4,18.1.0-13-gfe4edf0b+12,18.1.0-14-g259bd21+21,18.1.0-19-gdb69f3f+2,18.1.0-2-g5f9922c+24,18.1.0-2-gd3b74e5+11,18.1.0-2-gfbf3545+32,18.1.0-26-g728bddb4+5,18.1.0-27-g6ff7ca9+2,18.1.0-3-g52aa583+25,18.1.0-3-g8ea57af+9,18.1.0-3-gb69f684+42,18.1.0-3-gfcaddf3+6,18.1.0-32-gd8786685a,18.1.0-4-gf3f9b77+6,18.1.0-5-g1dd662b+2,18.1.0-5-g6dbcb01+41,18.1.0-6-gae77429+3,18.1.0-7-g9d75d83+9,18.1.0-7-gae09a6d+30,18.1.0-9-gc381ef5+4,w.2019.45
LSSTDataManagementBasePackage
deblend.py
Go to the documentation of this file.
1 # This file is part of meas_deblender.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy as np
24 import time
25 
26 import scarlet
27 
28 import lsst.log
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 import lsst.afw.math as afwMath
32 import lsst.afw.geom as afwGeom
33 import lsst.geom as geom
34 import lsst.afw.geom.ellipses as afwEll
35 import lsst.afw.image as afwImage
36 import lsst.afw.detection as afwDet
37 import lsst.afw.table as afwTable
38 
39 logger = lsst.log.Log.getLogger("meas.deblender.deblend")
40 
41 __all__ = 'SourceDeblendConfig', 'SourceDeblendTask', 'MultibandDeblendConfig', 'MultibandDeblendTask'
42 
43 
44 class SourceDeblendConfig(pexConfig.Config):
45 
46  edgeHandling = pexConfig.ChoiceField(
47  doc='What to do when a peak to be deblended is close to the edge of the image',
48  dtype=str, default='ramp',
49  allowed={
50  'clip': 'Clip the template at the edge AND the mirror of the edge.',
51  'ramp': 'Ramp down flux at the image edge by the PSF',
52  'noclip': 'Ignore the edge when building the symmetric template.',
53  }
54  )
55 
56  strayFluxToPointSources = pexConfig.ChoiceField(
57  doc='When the deblender should attribute stray flux to point sources',
58  dtype=str, default='necessary',
59  allowed={
60  'necessary': 'When there is not an extended object in the footprint',
61  'always': 'Always',
62  'never': ('Never; stray flux will not be attributed to any deblended child '
63  'if the deblender thinks all peaks look like point sources'),
64  }
65  )
66 
67  assignStrayFlux = pexConfig.Field(dtype=bool, default=True,
68  doc='Assign stray flux (not claimed by any child in the deblender) '
69  'to deblend children.')
70 
71  strayFluxRule = pexConfig.ChoiceField(
72  doc='How to split flux among peaks',
73  dtype=str, default='trim',
74  allowed={
75  'r-to-peak': '~ 1/(1+R^2) to the peak',
76  'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. '
77  'CAUTION: this can be computationally expensive on large footprints!'),
78  'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka '
79  'Manhattan distance)'),
80  'trim': ('Shrink the parent footprint to pixels that are not assigned to children')
81  }
82  )
83 
84  clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
85  doc=('When splitting stray flux, clip fractions below '
86  'this value to zero.'))
87  psfChisq1 = pexConfig.Field(dtype=float, default=1.5, optional=False,
88  doc=('Chi-squared per DOF cut for deciding a source is '
89  'a PSF during deblending (un-shifted PSF model)'))
90  psfChisq2 = pexConfig.Field(dtype=float, default=1.5, optional=False,
91  doc=('Chi-squared per DOF cut for deciding a source is '
92  'PSF during deblending (shifted PSF model)'))
93  psfChisq2b = pexConfig.Field(dtype=float, default=1.5, optional=False,
94  doc=('Chi-squared per DOF cut for deciding a source is '
95  'a PSF during deblending (shifted PSF model #2)'))
96  maxNumberOfPeaks = pexConfig.Field(dtype=int, default=0,
97  doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
98  " (<= 0: unlimited)"))
99  maxFootprintArea = pexConfig.Field(dtype=int, default=1000000,
100  doc=("Maximum area for footprints before they are ignored as large; "
101  "non-positive means no threshold applied"))
102  maxFootprintSize = pexConfig.Field(dtype=int, default=0,
103  doc=("Maximum linear dimension for footprints before they are ignored "
104  "as large; non-positive means no threshold applied"))
105  minFootprintAxisRatio = pexConfig.Field(dtype=float, default=0.0,
106  doc=("Minimum axis ratio for footprints before they are ignored "
107  "as large; non-positive means no threshold applied"))
108  notDeblendedMask = pexConfig.Field(dtype=str, default="NOT_DEBLENDED", optional=True,
109  doc="Mask name for footprints not deblended, or None")
110 
111  tinyFootprintSize = pexConfig.RangeField(dtype=int, default=2, min=2, inclusiveMin=True,
112  doc=('Footprints smaller in width or height than this value '
113  'will be ignored; minimum of 2 due to PSF gradient '
114  'calculation.'))
115 
116  propagateAllPeaks = pexConfig.Field(dtype=bool, default=False,
117  doc=('Guarantee that all peaks produce a child source.'))
118  catchFailures = pexConfig.Field(
119  dtype=bool, default=False,
120  doc=("If True, catch exceptions thrown by the deblender, log them, "
121  "and set a flag on the parent, instead of letting them propagate up"))
122  maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
123  doc="Mask planes to ignore when performing statistics")
124  maskLimits = pexConfig.DictField(
125  keytype=str,
126  itemtype=float,
127  default={},
128  doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
129  "Sources violating this limit will not be deblended."),
130  )
131  weightTemplates = pexConfig.Field(
132  dtype=bool, default=False,
133  doc=("If true, a least-squares fit of the templates will be done to the "
134  "full image. The templates will be re-weighted based on this fit."))
135  removeDegenerateTemplates = pexConfig.Field(dtype=bool, default=False,
136  doc=("Try to remove similar templates?"))
137  maxTempDotProd = pexConfig.Field(
138  dtype=float, default=0.5,
139  doc=("If the dot product between two templates is larger than this value, we consider them to be "
140  "describing the same object (i.e. they are degenerate). If one of the objects has been "
141  "labeled as a PSF it will be removed, otherwise the template with the lowest value will "
142  "be removed."))
143  medianSmoothTemplate = pexConfig.Field(dtype=bool, default=True,
144  doc="Apply a smoothing filter to all of the template images")
145 
146 
152 
153 
154 class SourceDeblendTask(pipeBase.Task):
155  """!
156  \anchor SourceDeblendTask_
157 
158  \brief Split blended sources into individual sources.
159 
160  This task has no return value; it only modifies the SourceCatalog in-place.
161  """
162  ConfigClass = SourceDeblendConfig
163  _DefaultName = "sourceDeblend"
164 
165  def __init__(self, schema, peakSchema=None, **kwargs):
166  """!
167  Create the task, adding necessary fields to the given schema.
168 
169  @param[in,out] schema Schema object for measurement fields; will be modified in-place.
170  @param[in] peakSchema Schema of Footprint Peaks that will be passed to the deblender.
171  Any fields beyond the PeakTable minimal schema will be transferred
172  to the main source Schema. If None, no fields will be transferred
173  from the Peaks.
174  @param[in] **kwargs Passed to Task.__init__.
175  """
176  pipeBase.Task.__init__(self, **kwargs)
177  self.schema = schema
178  self.toCopyFromParent = [item.key for item in self.schema
179  if item.field.getName().startswith("merge_footprint")]
180  peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
181  if peakSchema is None:
182  # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
183  # to simplify downstream code
184  self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
185  else:
186  self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
187  for item in peakSchema:
188  if item.key not in peakMinimalSchema:
189  self.peakSchemaMapper.addMapping(item.key, item.field)
190  # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
191  # updating this Schema in place. That's probably a design flaw, but in the meantime,
192  # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
193  # by adding the same fields to both.
194  schema.addField(item.field)
195  assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
196  self.addSchemaKeys(schema)
197 
198  def addSchemaKeys(self, schema):
199  self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
200  doc='Number of children this object has (defaults to 0)')
201  self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
202  doc='Deblender thought this source looked like a PSF')
203  self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter',
204  'If deblended-as-psf, the PSF centroid', "pixel")
205  self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D',
206  doc='If deblended-as-psf, the instrumental PSF flux', units='count')
207  self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
208  doc='Source had too many peaks; '
209  'only the brightest were included')
210  self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
211  doc='Parent footprint covered too many pixels')
212  self.maskedKey = schema.addField('deblend_masked', type='Flag',
213  doc='Parent footprint was predominantly masked')
214 
215  if self.config.catchFailures:
216  self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
217  doc="Deblending failed on source")
218 
219  self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
220  doc="Deblender skipped this source")
221 
222  self.deblendRampedTemplateKey = schema.addField(
223  'deblend_rampedTemplate', type='Flag',
224  doc=('This source was near an image edge and the deblender used '
225  '"ramp" edge-handling.'))
226 
227  self.deblendPatchedTemplateKey = schema.addField(
228  'deblend_patchedTemplate', type='Flag',
229  doc=('This source was near an image edge and the deblender used '
230  '"patched" edge-handling.'))
231 
232  self.hasStrayFluxKey = schema.addField(
233  'deblend_hasStrayFlux', type='Flag',
234  doc=('This source was assigned some stray flux'))
235 
236  self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in (
237  self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey,
238  self.tooManyPeaksKey, self.tooBigKey)))
239 
240  @pipeBase.timeMethod
241  def run(self, exposure, sources):
242  """!
243  Get the psf from the provided exposure and then run deblend().
244 
245  @param[in] exposure Exposure to process
246  @param[in,out] sources SourceCatalog containing sources detected on this exposure.
247 
248  @return None
249  """
250  psf = exposure.getPsf()
251  assert sources.getSchema() == self.schema
252  self.deblend(exposure, sources, psf)
253 
254  def _getPsfFwhm(self, psf, bbox):
255  # It should be easier to get a PSF's fwhm;
256  # https://dev.lsstcorp.org/trac/ticket/3030
257  return psf.computeShape().getDeterminantRadius() * 2.35
258 
259  @pipeBase.timeMethod
260  def deblend(self, exposure, srcs, psf):
261  """!
262  Deblend.
263 
264  @param[in] exposure Exposure to process
265  @param[in,out] srcs SourceCatalog containing sources detected on this exposure.
266  @param[in] psf PSF
267 
268  @return None
269  """
270  self.log.info("Deblending %d sources" % len(srcs))
271 
272  from lsst.meas.deblender.baseline import deblend
273 
274  # find the median stdev in the image...
275  mi = exposure.getMaskedImage()
276  statsCtrl = afwMath.StatisticsControl()
277  statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
278  stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
279  sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
280  self.log.trace('sigma1: %g', sigma1)
281 
282  n0 = len(srcs)
283  nparents = 0
284  for i, src in enumerate(srcs):
285  # t0 = time.clock()
286 
287  fp = src.getFootprint()
288  pks = fp.getPeaks()
289 
290  # Since we use the first peak for the parent object, we should propagate its flags
291  # to the parent source.
292  src.assign(pks[0], self.peakSchemaMapper)
293 
294  if len(pks) < 2:
295  continue
296 
297  if self.isLargeFootprint(fp):
298  src.set(self.tooBigKey, True)
299  self.skipParent(src, mi.getMask())
300  self.log.warn('Parent %i: skipping large footprint (area: %i)',
301  int(src.getId()), int(fp.getArea()))
302  continue
303  if self.isMasked(fp, exposure.getMaskedImage().getMask()):
304  src.set(self.maskedKey, True)
305  self.skipParent(src, mi.getMask())
306  self.log.warn('Parent %i: skipping masked footprint (area: %i)',
307  int(src.getId()), int(fp.getArea()))
308  continue
309 
310  nparents += 1
311  bb = fp.getBBox()
312  psf_fwhm = self._getPsfFwhm(psf, bb)
313 
314  self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(pks))
315 
316  self.preSingleDeblendHook(exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
317  npre = len(srcs)
318 
319  # This should really be set in deblend, but deblend doesn't have access to the src
320  src.set(self.tooManyPeaksKey, len(fp.getPeaks()) > self.config.maxNumberOfPeaks)
321 
322  try:
323  res = deblend(
324  fp, mi, psf, psf_fwhm, sigma1=sigma1,
325  psfChisqCut1=self.config.psfChisq1,
326  psfChisqCut2=self.config.psfChisq2,
327  psfChisqCut2b=self.config.psfChisq2b,
328  maxNumberOfPeaks=self.config.maxNumberOfPeaks,
329  strayFluxToPointSources=self.config.strayFluxToPointSources,
330  assignStrayFlux=self.config.assignStrayFlux,
331  strayFluxAssignment=self.config.strayFluxRule,
332  rampFluxAtEdge=(self.config.edgeHandling == 'ramp'),
333  patchEdges=(self.config.edgeHandling == 'noclip'),
334  tinyFootprintSize=self.config.tinyFootprintSize,
335  clipStrayFluxFraction=self.config.clipStrayFluxFraction,
336  weightTemplates=self.config.weightTemplates,
337  removeDegenerateTemplates=self.config.removeDegenerateTemplates,
338  maxTempDotProd=self.config.maxTempDotProd,
339  medianSmoothTemplate=self.config.medianSmoothTemplate
340  )
341  if self.config.catchFailures:
342  src.set(self.deblendFailedKey, False)
343  except Exception as e:
344  if self.config.catchFailures:
345  self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
346  src.set(self.deblendFailedKey, True)
347  import traceback
348  traceback.print_exc()
349  continue
350  else:
351  raise
352 
353  kids = []
354  nchild = 0
355  for j, peak in enumerate(res.deblendedParents[0].peaks):
356  heavy = peak.getFluxPortion()
357  if heavy is None or peak.skip:
358  src.set(self.deblendSkippedKey, True)
359  if not self.config.propagateAllPeaks:
360  # Don't care
361  continue
362  # We need to preserve the peak: make sure we have enough info to create a minimal
363  # child src
364  self.log.trace("Peak at (%i,%i) failed. Using minimal default info for child.",
365  pks[j].getIx(), pks[j].getIy())
366  if heavy is None:
367  # copy the full footprint and strip out extra peaks
368  foot = afwDet.Footprint(src.getFootprint())
369  peakList = foot.getPeaks()
370  peakList.clear()
371  peakList.append(peak.peak)
372  zeroMimg = afwImage.MaskedImageF(foot.getBBox())
373  heavy = afwDet.makeHeavyFootprint(foot, zeroMimg)
374  if peak.deblendedAsPsf:
375  if peak.psfFitFlux is None:
376  peak.psfFitFlux = 0.0
377  if peak.psfFitCenter is None:
378  peak.psfFitCenter = (peak.peak.getIx(), peak.peak.getIy())
379 
380  assert(len(heavy.getPeaks()) == 1)
381 
382  src.set(self.deblendSkippedKey, False)
383  child = srcs.addNew()
384  nchild += 1
385  for key in self.toCopyFromParent:
386  child.set(key, src.get(key))
387  child.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
388  child.setParent(src.getId())
389  child.setFootprint(heavy)
390  child.set(self.psfKey, peak.deblendedAsPsf)
391  child.set(self.hasStrayFluxKey, peak.strayFlux is not None)
392  if peak.deblendedAsPsf:
393  (cx, cy) = peak.psfFitCenter
394  child.set(self.psfCenterKey, geom.Point2D(cx, cy))
395  child.set(self.psfFluxKey, peak.psfFitFlux)
396  child.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
397  child.set(self.deblendPatchedTemplateKey, peak.patched)
398  kids.append(child)
399 
400  # Child footprints may extend beyond the full extent of their parent's which
401  # results in a failure of the replace-by-noise code to reinstate these pixels
402  # to their original values. The following updates the parent footprint
403  # in-place to ensure it contains the full union of itself and all of its
404  # children's footprints.
405  spans = src.getFootprint().spans
406  for child in kids:
407  spans = spans.union(child.getFootprint().spans)
408  src.getFootprint().setSpans(spans)
409 
410  src.set(self.nChildKey, nchild)
411 
412  self.postSingleDeblendHook(exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
413  # print('Deblending parent id', src.getId(), 'took', time.clock() - t0)
414 
415  n1 = len(srcs)
416  self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
417  % (n0, nparents, n1-n0, n1))
418 
419  def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1):
420  pass
421 
422  def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res):
423  pass
424 
425  def isLargeFootprint(self, footprint):
426  """Returns whether a Footprint is large
427 
428  'Large' is defined by thresholds on the area, size and axis ratio.
429  These may be disabled independently by configuring them to be non-positive.
430 
431  This is principally intended to get rid of satellite streaks, which the
432  deblender or other downstream processing can have trouble dealing with
433  (e.g., multiple large HeavyFootprints can chew up memory).
434  """
435  if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
436  return True
437  if self.config.maxFootprintSize > 0:
438  bbox = footprint.getBBox()
439  if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
440  return True
441  if self.config.minFootprintAxisRatio > 0:
442  axes = afwEll.Axes(footprint.getShape())
443  if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
444  return True
445  return False
446 
447  def isMasked(self, footprint, mask):
448  """Returns whether the footprint violates the mask limits"""
449  size = float(footprint.getArea())
450  for maskName, limit in self.config.maskLimits.items():
451  maskVal = mask.getPlaneBitMask(maskName)
452  unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels
453  if (size - unmaskedSpan.getArea())/size > limit:
454  return True
455  return False
456 
457  def skipParent(self, source, mask):
458  """Indicate that the parent source is not being deblended
459 
460  We set the appropriate flags and mask.
461 
462  @param source The source to flag as skipped
463  @param mask The mask to update
464  """
465  fp = source.getFootprint()
466  source.set(self.deblendSkippedKey, True)
467  source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all
468  if self.config.notDeblendedMask:
469  mask.addMaskPlane(self.config.notDeblendedMask)
470  fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
471 
472 
473 class MultibandDeblendConfig(pexConfig.Config):
474  """MultibandDeblendConfig
475 
476  Configuration for the multiband deblender.
477  The parameters are organized by the parameter types, which are
478  - Stopping Criteria: Used to determine if the fit has converged
479  - Position Fitting Criteria: Used to fit the positions of the peaks
480  - Constraints: Used to apply constraints to the peaks and their components
481  - Other: Parameters that don't fit into the above categories
482  """
483  # Stopping Criteria
484  maxIter = pexConfig.Field(dtype=int, default=200,
485  doc=("Maximum number of iterations to deblend a single parent"))
486  relativeError = pexConfig.Field(dtype=float, default=1e-3,
487  doc=("Relative error to use when determining stopping criteria"))
488 
489  # Blend Configuration options
490  minTranslation = pexConfig.Field(dtype=float, default=1e-3,
491  doc=("A peak must be updated by at least 'minTranslation' (pixels)"
492  "or no update is performed."
493  "This field is ignored if fitPositions is False."))
494  refinementSkip = pexConfig.Field(dtype=int, default=10,
495  doc=("If fitPositions is True, the positions and box sizes are"
496  "updated on every 'refinementSkip' iterations."))
497  translationMethod = pexConfig.Field(dtype=str, default="default",
498  doc=("Method to use for fitting translations."
499  "Currently 'default' is the only available option,"
500  "which performs a linear fit, but it is possible that we"
501  "will use galsim or some other method as a future option"))
502  edgeFluxThresh = pexConfig.Field(dtype=float, default=1.0,
503  doc=("Boxes are resized when the flux at an edge is "
504  "> edgeFluxThresh * background RMS"))
505  exactLipschitz = pexConfig.Field(dtype=bool, default=False,
506  doc=("Calculate exact Lipschitz constant in every step"
507  "(True) or only calculate the approximate"
508  "Lipschitz constant with significant changes in A,S"
509  "(False)"))
510  stepSlack = pexConfig.Field(dtype=float, default=0.2,
511  doc=("A fractional measure of how much a value (like the exactLipschitz)"
512  "can change before it needs to be recalculated."
513  "This must be between 0 and 1."))
514 
515  # Constraints
516  constraints = pexConfig.Field(dtype=str, default="1,+,S,M",
517  doc=("List of constraints to use for each object"
518  "(order does not matter)"
519  "Current options are all used by default:\n"
520  "S: symmetry\n"
521  "M: monotonicity\n"
522  "1: normalized SED to unity"
523  "+: non-negative morphology"))
524  symmetryThresh = pexConfig.Field(dtype=float, default=1.0,
525  doc=("Strictness of symmetry, from"
526  "0 (no symmetry enforced) to"
527  "1 (perfect symmetry required)."
528  "If 'S' is not in `constraints`, this argument is ignored"))
529  l0Thresh = pexConfig.Field(dtype=float, default=np.nan,
530  doc=("L0 threshold. NaN results in no L0 penalty."))
531  l1Thresh = pexConfig.Field(dtype=float, default=np.nan,
532  doc=("L1 threshold. NaN results in no L1 penalty."))
533  tvxThresh = pexConfig.Field(dtype=float, default=np.nan,
534  doc=("Threshold for TV (total variation) constraint in the x-direction."
535  "NaN results in no TVx penalty."))
536  tvyThresh = pexConfig.Field(dtype=float, default=np.nan,
537  doc=("Threshold for TV (total variation) constraint in the y-direction."
538  "NaN results in no TVy penalty."))
539 
540  # Other scarlet paremeters
541  useWeights = pexConfig.Field(dtype=bool, default=False, doc="Use inverse variance as deblender weights")
542  bgScale = pexConfig.Field(
543  dtype=float, default=0.5,
544  doc=("Fraction of background RMS level to use as a"
545  "cutoff for defining the background of the image"
546  "This is used to initialize the model for each source"
547  "and to set the size of the bounding box for each source"
548  "every `refinementSkip` iteration."))
549  usePsfConvolution = pexConfig.Field(
550  dtype=bool, default=True,
551  doc=("Whether or not to convolve the morphology with the"
552  "PSF in each band or use the same morphology in all bands"))
553  saveTemplates = pexConfig.Field(
554  dtype=bool, default=True,
555  doc="Whether or not to save the SEDs and templates")
556  processSingles = pexConfig.Field(
557  dtype=bool, default=False,
558  doc="Whether or not to process isolated sources in the deblender")
559  badMask = pexConfig.Field(
560  dtype=str, default="BAD,CR,NO_DATA,SAT,SUSPECT",
561  doc="Whether or not to process isolated sources in the deblender")
562  # Old deblender parameters used in this implementation (some of which might be removed later)
563 
564  maxNumberOfPeaks = pexConfig.Field(
565  dtype=int, default=0,
566  doc=("Only deblend the brightest maxNumberOfPeaks peaks in the parent"
567  " (<= 0: unlimited)"))
568  maxFootprintArea = pexConfig.Field(
569  dtype=int, default=1000000,
570  doc=("Maximum area for footprints before they are ignored as large; "
571  "non-positive means no threshold applied"))
572  maxFootprintSize = pexConfig.Field(
573  dtype=int, default=0,
574  doc=("Maximum linear dimension for footprints before they are ignored "
575  "as large; non-positive means no threshold applied"))
576  minFootprintAxisRatio = pexConfig.Field(
577  dtype=float, default=0.0,
578  doc=("Minimum axis ratio for footprints before they are ignored "
579  "as large; non-positive means no threshold applied"))
580  notDeblendedMask = pexConfig.Field(
581  dtype=str, default="NOT_DEBLENDED", optional=True,
582  doc="Mask name for footprints not deblended, or None")
583 
584  tinyFootprintSize = pexConfig.RangeField(
585  dtype=int, default=2, min=2, inclusiveMin=True,
586  doc=('Footprints smaller in width or height than this value will '
587  'be ignored; minimum of 2 due to PSF gradient calculation.'))
588  catchFailures = pexConfig.Field(
589  dtype=bool, default=False,
590  doc=("If True, catch exceptions thrown by the deblender, log them, "
591  "and set a flag on the parent, instead of letting them propagate up"))
592  propagateAllPeaks = pexConfig.Field(dtype=bool, default=False,
593  doc=('Guarantee that all peaks produce a child source.'))
594  maskPlanes = pexConfig.ListField(dtype=str, default=["SAT", "INTRP", "NO_DATA"],
595  doc="Mask planes to ignore when performing statistics")
596  maskLimits = pexConfig.DictField(
597  keytype=str,
598  itemtype=float,
599  default={},
600  doc=("Mask planes with the corresponding limit on the fraction of masked pixels. "
601  "Sources violating this limit will not be deblended."),
602  )
603 
604  edgeHandling = pexConfig.ChoiceField(
605  doc='What to do when a peak to be deblended is close to the edge of the image',
606  dtype=str, default='noclip',
607  allowed={
608  'clip': 'Clip the template at the edge AND the mirror of the edge.',
609  'ramp': 'Ramp down flux at the image edge by the PSF',
610  'noclip': 'Ignore the edge when building the symmetric template.',
611  }
612  )
613 
614  medianSmoothTemplate = pexConfig.Field(dtype=bool, default=False,
615  doc="Apply a smoothing filter to all of the template images")
616  medianFilterHalfsize = pexConfig.Field(dtype=float, default=2,
617  doc=('Half size of the median smoothing filter'))
618  clipFootprintToNonzero = pexConfig.Field(dtype=bool, default=False,
619  doc=("Clip non-zero spans in the footprints"))
620 
621  conserveFlux = pexConfig.Field(dtype=bool, default=True,
622  doc=("Reapportion flux to the footprints so that flux is conserved"))
623  weightTemplates = pexConfig.Field(
624  dtype=bool, default=False,
625  doc=("If true, a least-squares fit of the templates will be done to the "
626  "full image. The templates will be re-weighted based on this fit."))
627  strayFluxToPointSources = pexConfig.ChoiceField(
628  doc='When the deblender should attribute stray flux to point sources',
629  dtype=str, default='necessary',
630  allowed={
631  'necessary': 'When there is not an extended object in the footprint',
632  'always': 'Always',
633  'never': ('Never; stray flux will not be attributed to any deblended child '
634  'if the deblender thinks all peaks look like point sources'),
635  }
636  )
637 
638  assignStrayFlux = pexConfig.Field(dtype=bool, default=True,
639  doc='Assign stray flux (not claimed by any child in the deblender) '
640  'to deblend children.')
641 
642  strayFluxRule = pexConfig.ChoiceField(
643  doc='How to split flux among peaks',
644  dtype=str, default='trim',
645  allowed={
646  'r-to-peak': '~ 1/(1+R^2) to the peak',
647  'r-to-footprint': ('~ 1/(1+R^2) to the closest pixel in the footprint. '
648  'CAUTION: this can be computationally expensive on large footprints!'),
649  'nearest-footprint': ('Assign 100% to the nearest footprint (using L-1 norm aka '
650  'Manhattan distance)'),
651  'trim': ('Shrink the parent footprint to pixels that are not assigned to children')
652  }
653  )
654 
655  clipStrayFluxFraction = pexConfig.Field(dtype=float, default=0.001,
656  doc=('When splitting stray flux, clip fractions below '
657  'this value to zero.'))
658  getTemplateSum = pexConfig.Field(dtype=bool, default=False,
659  doc=("As part of the flux calculation, the sum of the templates is"
660  "calculated. If 'getTemplateSum==True' then the sum of the"
661  "templates is stored in the result (a 'PerFootprint')."))
662 
663 
664 class MultibandDeblendTask(pipeBase.Task):
665  """MultibandDeblendTask
666 
667  Split blended sources into individual sources.
668 
669  This task has no return value; it only modifies the SourceCatalog in-place.
670  """
671  ConfigClass = MultibandDeblendConfig
672  _DefaultName = "multibandDeblend"
673 
674  def __init__(self, schema, peakSchema=None, **kwargs):
675  """Create the task, adding necessary fields to the given schema.
676 
677  Parameters
678  ----------
679  schema: `lsst.afw.table.schema.schema.Schema`
680  Schema object for measurement fields; will be modified in-place.
681  peakSchema: `lsst.afw.table.schema.schema.Schema`
682  Schema of Footprint Peaks that will be passed to the deblender.
683  Any fields beyond the PeakTable minimal schema will be transferred
684  to the main source Schema. If None, no fields will be transferred
685  from the Peaks.
686  filters: list of str
687  Names of the filters used for the eposures. This is needed to store the SED as a field
688  **kwargs
689  Passed to Task.__init__.
690  """
691  from lsst.meas.deblender import plugins
692 
693  pipeBase.Task.__init__(self, **kwargs)
694  if not self.config.conserveFlux and not self.config.saveTemplates:
695  raise ValueError("Either `conserveFlux` or `saveTemplates` must be True")
696 
697  peakMinimalSchema = afwDet.PeakTable.makeMinimalSchema()
698  if peakSchema is None:
699  # In this case, the peakSchemaMapper will transfer nothing, but we'll still have one
700  # to simplify downstream code
701  self.peakSchemaMapper = afwTable.SchemaMapper(peakMinimalSchema, schema)
702  else:
703  self.peakSchemaMapper = afwTable.SchemaMapper(peakSchema, schema)
704  for item in peakSchema:
705  if item.key not in peakMinimalSchema:
706  self.peakSchemaMapper.addMapping(item.key, item.field)
707  # Because SchemaMapper makes a copy of the output schema you give its ctor, it isn't
708  # updating this Schema in place. That's probably a design flaw, but in the meantime,
709  # we'll keep that schema in sync with the peakSchemaMapper.getOutputSchema() manually,
710  # by adding the same fields to both.
711  schema.addField(item.field)
712  assert schema == self.peakSchemaMapper.getOutputSchema(), "Logic bug mapping schemas"
713  self._addSchemaKeys(schema)
714  self.schema = schema
715 
716  # Create the plugins for multiband deblending using the Config options
717 
718  # Basic deblender configuration
719  config = scarlet.config.Config(
720  center_min_dist=self.config.minTranslation,
721  edge_flux_thresh=self.config.edgeFluxThresh,
722  exact_lipschitz=self.config.exactLipschitz,
723  refine_skip=self.config.refinementSkip,
724  slack=self.config.stepSlack,
725  )
726  if self.config.translationMethod != "default":
727  err = "Currently the only supported translationMethod is 'default', you entered '{0}'"
728  raise NotImplementedError(err.format(self.config.translationMethod))
729 
730  # If the default constraints are not used, set the constraints for
731  # all of the sources
732  constraints = None
733  _constraints = self.config.constraints.split(",")
734  if (sorted(_constraints) != ['+', '1', 'M', 'S'] or
735  ~np.isnan(self.config.l0Thresh) or
736  ~np.isnan(self.config.l1Thresh)):
737  constraintDict = {
738  "+": scarlet.constraint.PositivityConstraint,
739  "1": scarlet.constraint.SimpleConstraint,
740  "M": scarlet.constraint.DirectMonotonicityConstraint(use_nearest=False),
741  "S": scarlet.constraint.DirectSymmetryConstraint(sigma=self.config.symmetryThresh)
742  }
743  for c in _constraints:
744  if constraints is None:
745  constraints = [constraintDict[c]]
746  else:
747  constraints += [constraintDict[c]]
748  if constraints is None:
749  constraints = scarlet.constraint.MinimalConstraint()
750  if ~np.isnan(self.config.l0Thresh):
751  constraints += [scarlet.constraint.L0Constraint(self.config.l0Thresh)]
752  if ~np.isnan(self.config.l1Thresh):
753  constraints += [scarlet.constraint.L1Constraint(self.config.l1Thresh)]
754  if ~np.isnan(self.config.tvxThresh):
755  constraints += [scarlet.constraint.TVxConstraint(self.config.tvxThresh)]
756  if ~np.isnan(self.config.tvyThresh):
757  constraints += [scarlet.constraint.TVyConstraint(self.config.tvyThresh)]
758 
759  multiband_plugin = plugins.DeblenderPlugin(
760  plugins.buildMultibandTemplates,
761  useWeights=self.config.useWeights,
762  usePsf=self.config.usePsfConvolution,
763  constraints=constraints,
764  config=config,
765  maxIter=self.config.maxIter,
766  bgScale=self.config.bgScale,
767  relativeError=self.config.relativeError,
768  badMask=self.config.badMask.split(","),
769  )
770  self.plugins = [multiband_plugin]
771 
772  # Plugins from the old deblender for post-template processing
773  # (see lsst.meas_deblender.baseline.deblend)
774  if self.config.edgeHandling == 'ramp':
775  self.plugins.append(plugins.DeblenderPlugin(plugins.rampFluxAtEdge, patchEdges=False))
776  if self.config.medianSmoothTemplate:
778  plugins.medianSmoothTemplates,
779  medianFilterHalfsize=self.config.medianFilterHalfsize))
780  if self.config.clipFootprintToNonzero:
781  self.plugins.append(plugins.DeblenderPlugin(plugins.clipFootprintsToNonzero))
782  if self.config.conserveFlux:
783  if self.config.weightTemplates:
784  self.plugins.append(plugins.DeblenderPlugin(plugins.weightTemplates))
786  plugins.apportionFlux,
787  clipStrayFluxFraction=self.config.clipStrayFluxFraction,
788  assignStrayFlux=self.config.assignStrayFlux,
789  strayFluxAssignment=self.config.strayFluxRule,
790  strayFluxToPointSources=self.config.strayFluxToPointSources,
791  getTemplateSum=self.config.getTemplateSum))
792 
793  def _addSchemaKeys(self, schema):
794  """Add deblender specific keys to the schema
795  """
796  self.runtimeKey = schema.addField('runtime', type=np.float32, doc='runtime in ms')
797  # Keys from old Deblender that might be kept in the new deblender
798  self.nChildKey = schema.addField('deblend_nChild', type=np.int32,
799  doc='Number of children this object has (defaults to 0)')
800  self.psfKey = schema.addField('deblend_deblendedAsPsf', type='Flag',
801  doc='Deblender thought this source looked like a PSF')
802  self.tooManyPeaksKey = schema.addField('deblend_tooManyPeaks', type='Flag',
803  doc='Source had too many peaks; '
804  'only the brightest were included')
805  self.tooBigKey = schema.addField('deblend_parentTooBig', type='Flag',
806  doc='Parent footprint covered too many pixels')
807  self.maskedKey = schema.addField('deblend_masked', type='Flag',
808  doc='Parent footprint was predominantly masked')
809  self.deblendFailedKey = schema.addField('deblend_failed', type='Flag',
810  doc="Deblending failed on source")
811 
812  self.deblendSkippedKey = schema.addField('deblend_skipped', type='Flag',
813  doc="Deblender skipped this source")
814 
815  # Keys from the old Deblender that some measruement tasks require.
816  # TODO: Remove these whem the old deblender is removed
817  self.psfCenterKey = afwTable.Point2DKey.addFields(schema, 'deblend_psfCenter',
818  'If deblended-as-psf, the PSF centroid', "pixel")
819  self.psfFluxKey = schema.addField('deblend_psf_instFlux', type='D',
820  doc='If deblended-as-psf, the instrumental PSF flux', units='count')
821  self.deblendRampedTemplateKey = schema.addField(
822  'deblend_rampedTemplate', type='Flag',
823  doc=('This source was near an image edge and the deblender used '
824  '"ramp" edge-handling.'))
825 
826  self.deblendPatchedTemplateKey = schema.addField(
827  'deblend_patchedTemplate', type='Flag',
828  doc=('This source was near an image edge and the deblender used '
829  '"patched" edge-handling.'))
830 
831  self.hasStrayFluxKey = schema.addField(
832  'deblend_hasStrayFlux', type='Flag',
833  doc=('This source was assigned some stray flux'))
834 
835  self.log.trace('Added keys to schema: %s', ", ".join(str(x) for x in (
836  self.nChildKey, self.psfKey, self.psfCenterKey, self.psfFluxKey,
837  self.tooManyPeaksKey, self.tooBigKey)))
838 
839  @pipeBase.timeMethod
840  def run(self, mExposure, mergedSources):
841  """Get the psf from each exposure and then run deblend().
842 
843  Parameters
844  ----------
845  mExposure: `MultibandExposure`
846  The exposures should be co-added images of the same
847  shape and region of the sky.
848  mergedSources: `SourceCatalog`
849  The merged `SourceCatalog` that contains parent footprints
850  to (potentially) deblend.
851 
852  Returns
853  -------
854  fluxCatalogs: dict or None
855  Keys are the names of the filters and the values are
856  `lsst.afw.table.source.source.SourceCatalog`'s.
857  These are the flux-conserved catalogs with heavy footprints with
858  the image data weighted by the multiband templates.
859  If `self.config.conserveFlux` is `False`, then this item will be None
860  templateCatalogs: dict or None
861  Keys are the names of the filters and the values are
862  `lsst.afw.table.source.source.SourceCatalog`'s.
863  These are catalogs with heavy footprints that are the templates
864  created by the multiband templates.
865  If `self.config.saveTemplates` is `False`, then this item will be None
866  """
867  psfs = {f: mExposure[f].getPsf() for f in mExposure.filters}
868  return self.deblend(mExposure, mergedSources, psfs)
869 
870  def _getPsfFwhm(self, psf, bbox):
871  return psf.computeShape().getDeterminantRadius() * 2.35
872 
873  def _addChild(self, parentId, peak, sources, heavy):
874  """Add a child to a catalog
875 
876  This creates a new child in the source catalog,
877  assigning it a parent id, adding a footprint,
878  and setting all appropriate flags based on the
879  deblender result.
880  """
881  assert len(heavy.getPeaks()) == 1
882  src = sources.addNew()
883  src.assign(heavy.getPeaks()[0], self.peakSchemaMapper)
884  src.setParent(parentId)
885  src.setFootprint(heavy)
886  src.set(self.psfKey, peak.deblendedAsPsf)
887  src.set(self.hasStrayFluxKey, peak.strayFlux is not None)
888  src.set(self.deblendRampedTemplateKey, peak.hasRampedTemplate)
889  src.set(self.deblendPatchedTemplateKey, peak.patched)
890  src.set(self.runtimeKey, 0)
891  return src
892 
893  @pipeBase.timeMethod
894  def deblend(self, mExposure, sources, psfs):
895  """Deblend a data cube of multiband images
896 
897  Parameters
898  ----------
899  mExposure: `MultibandExposure`
900  The exposures should be co-added images of the same
901  shape and region of the sky.
902  sources: `SourceCatalog`
903  The merged `SourceCatalog` that contains parent footprints
904  to (potentially) deblend.
905  psfs: dict
906  Keys are the names of the filters
907  (should be the same as `mExposure.filters`)
908  and the values are the PSFs in each band.
909 
910  Returns
911  -------
912  fluxCatalogs: dict or None
913  Keys are the names of the filters and the values are
914  `lsst.afw.table.source.source.SourceCatalog`'s.
915  These are the flux-conserved catalogs with heavy footprints with
916  the image data weighted by the multiband templates.
917  If `self.config.conserveFlux` is `False`, then this item will be None
918  templateCatalogs: dict or None
919  Keys are the names of the filters and the values are
920  `lsst.afw.table.source.source.SourceCatalog`'s.
921  These are catalogs with heavy footprints that are the templates
922  created by the multiband templates.
923  If `self.config.saveTemplates` is `False`, then this item will be None
924  """
925  from lsst.meas.deblender.baseline import newDeblend
926 
927  if tuple(psfs.keys()) != mExposure.filters:
928  msg = "PSF keys must be the same as mExposure.filters ({0}), got {1}"
929  raise ValueError(msg.format(mExposure.filters, psfs.keys()))
930 
931  filters = mExposure.filters
932  mMaskedImage = afwImage.MultibandMaskedImage(filters=mExposure.filters, image=mExposure.image,
933  mask=mExposure.mask, variance=mExposure.variance)
934  self.log.info("Deblending {0} sources in {1} exposures".format(len(sources), len(mExposure)))
935 
936  # find the median stdev in each image
937  sigmas = {}
938  for f in filters:
939  exposure = mExposure[f]
940  mi = exposure.getMaskedImage()
941  statsCtrl = afwMath.StatisticsControl()
942  statsCtrl.setAndMask(mi.getMask().getPlaneBitMask(self.config.maskPlanes))
943  stats = afwMath.makeStatistics(mi.getVariance(), mi.getMask(), afwMath.MEDIAN, statsCtrl)
944  sigma1 = math.sqrt(stats.getValue(afwMath.MEDIAN))
945  self.log.trace('Exposure {0}, sigma1: {1}'.format(f, sigma1))
946  sigmas[f] = sigma1
947 
948  # Create the output catalogs
949  if self.config.conserveFlux:
950  fluxCatalogs = {}
951  for f in filters:
952  _catalog = afwTable.SourceCatalog(sources.table.clone())
953  _catalog.extend(sources)
954  fluxCatalogs[f] = _catalog
955  else:
956  fluxCatalogs = None
957  if self.config.saveTemplates:
958  templateCatalogs = {}
959  for f in filters:
960  _catalog = afwTable.SourceCatalog(sources.table.clone())
961  _catalog.extend(sources)
962  templateCatalogs[f] = _catalog
963  else:
964  templateCatalogs = None
965 
966  n0 = len(sources)
967  nparents = 0
968  for pk, src in enumerate(sources):
969  foot = src.getFootprint()
970  logger.info("id: {0}".format(src["id"]))
971  peaks = foot.getPeaks()
972 
973  # Since we use the first peak for the parent object, we should propagate its flags
974  # to the parent source.
975  src.assign(peaks[0], self.peakSchemaMapper)
976 
977  # Block of Skipping conditions
978  if len(peaks) < 2 and not self.config.processSingles:
979  for f in filters:
980  if self.config.saveTemplates:
981  templateCatalogs[f][pk].set(self.runtimeKey, 0)
982  if self.config.conserveFlux:
983  fluxCatalogs[f][pk].set(self.runtimeKey, 0)
984  continue
985  if self.isLargeFootprint(foot):
986  src.set(self.tooBigKey, True)
987  self.skipParent(src, [mi.getMask() for mi in mMaskedImage])
988  self.log.warn('Parent %i: skipping large footprint (area: %i)',
989  int(src.getId()), int(foot.getArea()))
990  continue
991  if self.isMasked(foot, exposure.getMaskedImage().getMask()):
992  src.set(self.maskedKey, True)
993  self.skipParent(src, mi.getMask())
994  self.log.trace('Parent %i: skipping masked footprint (area: %i)',
995  int(src.getId()), int(foot.getArea()))
996  continue
997  if len(peaks) > self.config.maxNumberOfPeaks:
998  src.set(self.tooManyPeaksKey, True)
999  msg = 'Parent {0}: Too many peaks, using the first {1} peaks'
1000  self.log.trace(msg.format(int(src.getId()), self.config.maxNumberOfPeaks))
1001 
1002  nparents += 1
1003  bbox = foot.getBBox()
1004  psf_fwhms = {f: self._getPsfFwhm(psf, bbox) for f, psf in psfs.items()}
1005  self.log.trace('Parent %i: deblending %i peaks', int(src.getId()), len(peaks))
1006  self.preSingleDeblendHook(mExposure.singles, sources, pk, foot, psfs, psf_fwhms, sigmas)
1007  npre = len(sources)
1008  # Run the deblender
1009  try:
1010  t0 = time.time()
1011  # Build the parameter lists with the same ordering
1012  images = mMaskedImage[:, bbox]
1013  psf_list = [psfs[f] for f in filters]
1014  fwhm_list = [psf_fwhms[f] for f in filters]
1015  avgNoise = [sigmas[f] for f in filters]
1016 
1017  result = newDeblend(debPlugins=self.plugins,
1018  footprint=foot,
1019  mMaskedImage=images,
1020  psfs=psf_list,
1021  psfFwhms=fwhm_list,
1022  avgNoise=avgNoise,
1023  maxNumberOfPeaks=self.config.maxNumberOfPeaks)
1024  tf = time.time()
1025  runtime = (tf-t0)*1000
1026  if result.failed:
1027  src.set(self.deblendFailedKey, False)
1028  src.set(self.runtimeKey, 0)
1029  continue
1030  except Exception as e:
1031  if self.config.catchFailures:
1032  self.log.warn("Unable to deblend source %d: %s" % (src.getId(), e))
1033  src.set(self.deblendFailedKey, True)
1034  src.set(self.runtimeKey, 0)
1035  import traceback
1036  traceback.print_exc()
1037  continue
1038  else:
1039  raise
1040 
1041  # Add the merged source as a parent in the catalog for each band
1042  templateParents = {}
1043  fluxParents = {}
1044  parentId = src.getId()
1045  for f in filters:
1046  if self.config.saveTemplates:
1047  templateParents[f] = templateCatalogs[f][pk]
1048  templateParents[f].set(self.runtimeKey, runtime)
1049  if self.config.conserveFlux:
1050  fluxParents[f] = fluxCatalogs[f][pk]
1051  fluxParents[f].set(self.runtimeKey, runtime)
1052 
1053  # Add each source to the catalogs in each band
1054  templateSpans = {f: afwGeom.SpanSet() for f in filters}
1055  fluxSpans = {f: afwGeom.SpanSet() for f in filters}
1056  nchild = 0
1057  for j, multiPeak in enumerate(result.peaks):
1058  heavy = {f: peak.getFluxPortion() for f, peak in multiPeak.deblendedPeaks.items()}
1059  no_flux = all([v is None for v in heavy.values()])
1060  skip_peak = all([peak.skip for peak in multiPeak.deblendedPeaks.values()])
1061  if no_flux or skip_peak:
1062  src.set(self.deblendSkippedKey, True)
1063  if not self.config.propagateAllPeaks:
1064  # We don't care
1065  continue
1066  # We need to preserve the peak: make sure we have enough info to create a minimal
1067  # child src
1068  msg = "Peak at {0} failed deblending. Using minimal default info for child."
1069  self.log.trace(msg.format(multiPeak.x, multiPeak.y))
1070 
1071  # copy the full footprint and strip out extra peaks
1072  pfoot = afwDet.Footprint(foot)
1073  peakList = pfoot.getPeaks()
1074  peakList.clear()
1075  pfoot.addPeak(multiPeak.x, multiPeak.y, 0)
1076  zeroMimg = afwImage.MaskedImageF(pfoot.getBBox())
1077  for f in filters:
1078  heavy[f] = afwDet.makeHeavyFootprint(pfoot, zeroMimg)
1079  else:
1080  src.set(self.deblendSkippedKey, False)
1081 
1082  # Add the peak to the source catalog in each band
1083  for f in filters:
1084  if len(heavy[f].getPeaks()) != 1:
1085  err = "Heavy footprint should have a single peak, got {0}"
1086  raise ValueError(err.format(len(heavy[f].getPeaks())))
1087  peak = multiPeak.deblendedPeaks[f]
1088  if self.config.saveTemplates:
1089  cat = templateCatalogs[f]
1090  tfoot = peak.templateFootprint
1091  timg = afwImage.MaskedImageF(peak.templateImage)
1092  tHeavy = afwDet.makeHeavyFootprint(tfoot, timg)
1093  child = self._addChild(parentId, peak, cat, tHeavy)
1094  if parentId == 0:
1095  child.setId(src.getId())
1096  child.set(self.runtimeKey, runtime)
1097  else:
1098  templateSpans[f] = templateSpans[f].union(tHeavy.getSpans())
1099  if self.config.conserveFlux:
1100  cat = fluxCatalogs[f]
1101  child = self._addChild(parentId, peak, cat, heavy[f])
1102  if parentId == 0:
1103  child.setId(src.getId())
1104  child.set(self.runtimeKey, runtime)
1105  else:
1106  fluxSpans[f] = fluxSpans[f].union(heavy[f].getSpans())
1107  nchild += 1
1108 
1109  # Child footprints may extend beyond the full extent of their parent's which
1110  # results in a failure of the replace-by-noise code to reinstate these pixels
1111  # to their original values. The following updates the parent footprint
1112  # in-place to ensure it contains the full union of itself and all of its
1113  # children's footprints.
1114  for f in filters:
1115  if self.config.saveTemplates:
1116  templateParents[f].set(self.nChildKey, nchild)
1117  templateParents[f].getFootprint().setSpans(templateSpans[f])
1118  if self.config.conserveFlux:
1119  fluxParents[f].set(self.nChildKey, nchild)
1120  fluxParents[f].getFootprint().setSpans(fluxSpans[f])
1121 
1122  self.postSingleDeblendHook(exposure, fluxCatalogs, templateCatalogs,
1123  pk, npre, foot, psfs, psf_fwhms, sigmas, result)
1124 
1125  if fluxCatalogs is not None:
1126  n1 = len(list(fluxCatalogs.values())[0])
1127  else:
1128  n1 = len(list(templateCatalogs.values())[0])
1129  self.log.info('Deblended: of %i sources, %i were deblended, creating %i children, total %i sources'
1130  % (n0, nparents, n1-n0, n1))
1131  return fluxCatalogs, templateCatalogs
1132 
1133  def preSingleDeblendHook(self, exposures, sources, pk, fp, psfs, psf_fwhms, sigmas):
1134  pass
1135 
1136  def postSingleDeblendHook(self, exposures, fluxCatalogs, templateCatalogs,
1137  pk, npre, fp, psfs, psf_fwhms, sigmas, result):
1138  pass
1139 
1140  def isLargeFootprint(self, footprint):
1141  """Returns whether a Footprint is large
1142 
1143  'Large' is defined by thresholds on the area, size and axis ratio.
1144  These may be disabled independently by configuring them to be non-positive.
1145 
1146  This is principally intended to get rid of satellite streaks, which the
1147  deblender or other downstream processing can have trouble dealing with
1148  (e.g., multiple large HeavyFootprints can chew up memory).
1149  """
1150  if self.config.maxFootprintArea > 0 and footprint.getArea() > self.config.maxFootprintArea:
1151  return True
1152  if self.config.maxFootprintSize > 0:
1153  bbox = footprint.getBBox()
1154  if max(bbox.getWidth(), bbox.getHeight()) > self.config.maxFootprintSize:
1155  return True
1156  if self.config.minFootprintAxisRatio > 0:
1157  axes = afwEll.Axes(footprint.getShape())
1158  if axes.getB() < self.config.minFootprintAxisRatio*axes.getA():
1159  return True
1160  return False
1161 
1162  def isMasked(self, footprint, mask):
1163  """Returns whether the footprint violates the mask limits"""
1164  size = float(footprint.getArea())
1165  for maskName, limit in self.config.maskLimits.items():
1166  maskVal = mask.getPlaneBitMask(maskName)
1167  unmaskedSpan = footprint.spans.intersectNot(mask, maskVal) # spanset of unmasked pixels
1168  if (size - unmaskedSpan.getArea())/size > limit:
1169  return True
1170  return False
1171 
1172  def skipParent(self, source, masks):
1173  """Indicate that the parent source is not being deblended
1174 
1175  We set the appropriate flags and masks for each exposure.
1176 
1177  Parameters
1178  ----------
1179  source: `lsst.afw.table.source.source.SourceRecord`
1180  The source to flag as skipped
1181  masks: list of `lsst.afw.image.MaskX`
1182  The mask in each band to update with the non-detection
1183  """
1184  fp = source.getFootprint()
1185  source.set(self.deblendSkippedKey, True)
1186  source.set(self.nChildKey, len(fp.getPeaks())) # It would have this many if we deblended them all
1187  if self.config.notDeblendedMask:
1188  for mask in masks:
1189  mask.addMaskPlane(self.config.notDeblendedMask)
1190  fp.spans.setMask(mask, mask.getPlaneBitMask(self.config.notDeblendedMask))
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
def newDeblend(debPlugins, footprint, mMaskedImage, psfs, psfFwhms, log=None, verbose=False, avgNoise=None, maxNumberOfPeaks=0)
Definition: baseline.py:675
def _addChild(self, parentId, peak, sources, heavy)
Definition: deblend.py:873
def postSingleDeblendHook(self, exposures, fluxCatalogs, templateCatalogs, pk, npre, fp, psfs, psf_fwhms, sigmas, result)
Definition: deblend.py:1137
def postSingleDeblendHook(self, exposure, srcs, i, npre, kids, fp, psf, psf_fwhm, sigma1, res)
Definition: deblend.py:422
A compact representation of a collection of pixels.
Definition: SpanSet.h:77
A mapping between the keys of two Schemas, used to copy data between them.
Definition: SchemaMapper.h:21
def preSingleDeblendHook(self, exposure, srcs, i, fp, psf, psf_fwhm, sigma1)
Definition: deblend.py:419
def run(self, exposure, sources)
Get the psf from the provided exposure and then run deblend().
Definition: deblend.py:241
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
Definition: functional.cc:33
daf::base::PropertySet * set
Definition: fits.cc:902
Definition: Log.h:706
Statistics makeStatistics(lsst::afw::math::MaskedVector< EntryT > const &mv, std::vector< WeightPixel > const &vweights, int const flags, StatisticsControl const &sctrl=StatisticsControl())
The makeStatistics() overload to handle lsst::afw::math::MaskedVector<>
Definition: Statistics.h:520
def __init__(self, schema, peakSchema=None, kwargs)
Definition: deblend.py:674
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
Pass parameters to a Statistics object.
Definition: Statistics.h:93
def preSingleDeblendHook(self, exposures, sources, pk, fp, psfs, psf_fwhms, sigmas)
Definition: deblend.py:1133
int max
Class to describe the properties of a detected object from an image.
Definition: Footprint.h:62
Split blended sources into individual sources.
Definition: deblend.py:154
def deblend(self, exposure, srcs, psf)
Deblend.
Definition: deblend.py:260
static Log getLogger(Log const &logger)
Definition: Log.h:760
def __init__(self, schema, peakSchema=None, kwargs)
Create the task, adding necessary fields to the given schema.
Definition: deblend.py:165
def deblend(self, mExposure, sources, psfs)
Definition: deblend.py:894
def isMasked(self, footprint, mask)
Definition: deblend.py:447
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...
daf::base::PropertyList * list
Definition: fits.cc:903
def run(self, mExposure, mergedSources)
Definition: deblend.py:840
HeavyFootprint< ImagePixelT, MaskPixelT, VariancePixelT > makeHeavyFootprint(Footprint const &foot, lsst::afw::image::MaskedImage< ImagePixelT, MaskPixelT, VariancePixelT > const &img, HeavyFootprintCtrl const *ctrl=NULL)
Create a HeavyFootprint with footprint defined by the given Footprint and pixel values from the given...