LSST Applications g063fba187b+66a50001ff,g0f08755f38+1a22dc2551,g1653933729+a8ce1bb630,g168dd56ebc+a8ce1bb630,g217e2c1bcf+12e87a5bd8,g246886dfd9+466c7b6c06,g28da252d5a+858b171e04,g2bbee38e9b+404b60ec9b,g2bc492864f+404b60ec9b,g3156d2b45e+6e55a43351,g347aa1857d+404b60ec9b,g35bb328faa+a8ce1bb630,g3a166c0a6a+404b60ec9b,g3e281a1b8c+c5dd892a6c,g414038480c+6b9177ef31,g41af890bb2+70bea58702,g599934f4f4+b8c5400ca5,g781aacb6e4+a8ce1bb630,g7af13505b9+b5b9cefdb8,g80478fca09+c2997882f3,g82479be7b0+8974e6af0f,g858d7b2824+1a22dc2551,g89c8672015+f4add4ffd5,g8f1c07a47a+de51c9b0a5,g9125e01d80+a8ce1bb630,ga5288a1d22+b66f8cf76b,gb58c049af0+d64f4d3760,gc28159a63d+404b60ec9b,gcab2d0539d+66cf1de5d4,gcf0d15dbbd+12cb7e2563,gda6a2b7d83+12cb7e2563,gdaeeff99f8+1711a396fd,ge79ae78c31+404b60ec9b,gef2f8181fd+414189b318,gf0baf85859+c1f95f4921,gf0c06eb49c+1a22dc2551,gfa517265be+1a22dc2551,gfa999e8aa5+17cd334064,v28.0.0.rc2
LSST Data Management Base Package
Loading...
Searching...
No Matches
baseMeasurement.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""Base measurement task, which subclassed by the single frame and forced
23measurement tasks.
24"""
25import warnings
26
27import lsst.pipe.base
28import lsst.pex.config
29
30from .pluginRegistry import PluginMap
31from ._measBaseLib import FatalAlgorithmError, MeasurementError
32from lsst.afw.detection import InvalidPsfError
33from .pluginsBase import BasePluginConfig, BasePlugin
34from .noiseReplacer import NoiseReplacerConfig
35
36__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
37 "BaseMeasurementConfig", "BaseMeasurementTask")
38
39# Exceptions that the measurement tasks should always propagate up to their
40# callers
41FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
42
43
45 """Base config class for all measurement plugins.
46
47 Notes
48 -----
49 Most derived classes will want to override `setDefaults` in order to
50 customize the default `executionOrder`.
51
52 A derived class whose corresponding Plugin class implements a do `measureN`
53 method should additionally add a bool `doMeasureN` field to replace the
54 bool class attribute defined here.
55 """
56
57 doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
58 doc="whether to run this plugin in single-object mode")
59
60 doMeasureN = False # replace this class attribute with a Field if measureN-capable
61
62
64 """Base class for all measurement plugins.
65
66 Notes
67 -----
68 This is class is a placeholder for future behavior which will be shared
69 only between measurement plugins and is implemented for symmetry with the
70 measurement base plugin configuration class
71 """
72
73 pass
74
75
77 """Assign named plugins to measurement slots.
78
79 Slot configuration which assigns a particular named plugin to each of a set
80 of slots. Each slot allows a type of measurement to be fetched from the
81 `lsst.afw.table.SourceTable` without knowing which algorithm was used to
82 produced the data.
83
84 Notes
85 -----
86 The default algorithm for each slot must be registered, even if the default
87 is not used.
88 """
89
91 centroid = Field(dtype=str, default="base_SdssCentroid", optional=True,
92 doc="the name of the centroiding algorithm used to set source x,y")
93 shape = Field(dtype=str, default="base_SdssShape", optional=True,
94 doc="the name of the algorithm used to set source moments parameters")
95 psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True,
96 doc="the name of the algorithm used to set PSF moments parameters")
97 apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
98 doc="the name of the algorithm used to set the source aperture instFlux slot")
99 modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
100 doc="the name of the algorithm used to set the source model instFlux slot")
101 psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True,
102 doc="the name of the algorithm used to set the source psf instFlux slot")
103 gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
104 doc="the name of the algorithm used to set the source Gaussian instFlux slot")
105 calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
106 doc="the name of the instFlux measurement algorithm used for calibration")
107
108 def setupSchema(self, schema):
109 """Set up a slots in a schema following configuration directives.
110
111 Parameters
112 ----------
113 schema : `lsst.afw.table.Schema`
114 The schema in which slots will be set up.
115
116 Notes
117 -----
118 This is defined in this configuration class to support use in unit
119 tests without needing to construct an `lsst.pipe.base.Task` object.
120 """
121 aliases = schema.getAliasMap()
122 if self.centroid is not None:
123 aliases.set("slot_Centroid", self.centroid)
124 if self.shape is not None:
125 aliases.set("slot_Shape", self.shape)
126 if self.psfShape is not None:
127 aliases.set("slot_PsfShape", self.psfShape)
128 if self.apFlux is not None:
129 aliases.set("slot_ApFlux", self.apFlux)
130 if self.modelFlux is not None:
131 aliases.set("slot_ModelFlux", self.modelFlux)
132 if self.psfFlux is not None:
133 aliases.set("slot_PsfFlux", self.psfFlux)
134 if self.gaussianFlux is not None:
135 aliases.set("slot_GaussianFlux", self.gaussianFlux)
136 if self.calibFlux is not None:
137 aliases.set("slot_CalibFlux", self.calibFlux)
138
139
141 """Base configuration for all measurement driver tasks.
142
143 Parameters
144 ----------
145 ignoreSlotPluginChecks : `bool`, optional
146 Do not check that all slots have an associated plugin to run when
147 validating this config. This is primarily for tests that were written
148 before we made Tasks always call `config.validate()` on init.
149 DEPRECATED DM-35949: this is a temporary workaround while we better
150 define how config/schema validation works for measurement tasks.
151
152 Examples
153 --------
154 Subclasses should define the 'plugins' and 'undeblended' registries, e.g.
155
156 .. code-block:: py
157
158 plugins = PluginBaseClass.registry.makeField(
159 multi=True,
160 default=[],
161 doc="Plugins to be run and their configuration"
162 )
163 undeblended = PluginBaseClass.registry.makeField(
164 multi=True,
165 default=[],
166 doc="Plugins to run on undeblended image"
167 )
168
169 where ``PluginBaseClass`` is the appropriate base class of the plugin
170 (e.g., `SingleFramePlugin` or `ForcedPlugin`).
171 """
172 def __new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs):
173 instance = super().__new__(cls, *args, **kwargs)
174 if ignoreSlotPluginChecks:
175 msg = ("ignoreSlotPluginChecks is deprecated and should only be used in tests."
176 " No removal date has been set; see DM-35949.")
177 warnings.warn(msg, category=FutureWarning, stacklevel=2)
178 object.__setattr__(instance, "_ignoreSlotPluginChecks", ignoreSlotPluginChecks)
179 return instance
180
182 dtype=SourceSlotConfig,
183 doc="Mapping from algorithms to special aliases in Source."
184 )
185
186 doReplaceWithNoise = lsst.pex.config.Field(
187 dtype=bool, default=True, optional=False,
188 doc='When measuring, replace other detected footprints with noise?')
189
191 dtype=NoiseReplacerConfig,
192 doc="configuration that sets how to replace neighboring sources with noise"
193 )
194 undeblendedPrefix = lsst.pex.config.Field(
195 dtype=str, default="undeblended_",
196 doc="Prefix to give undeblended plugins"
197 )
198
199 def validate(self):
200 super().validate()
201 if self._ignoreSlotPluginChecks:
202 return
203 if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
205 self.__class__.slots,
206 self,
207 "source centroid slot algorithm is not being run."
208 )
209 if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
211 self.__class__.slots,
212 self,
213 "source shape slot algorithm '%s' is not being run." % self.slots.shape
214 )
215 for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
216 self.slots.gaussianFlux, self.slots.calibFlux):
217 if slot is not None:
218 for name in self.plugins.names:
219 if len(name) <= len(slot) and name == slot[:len(name)]:
220 break
221 else:
223 self.__class__.slots,
224 self,
225 f"Source instFlux algorithm '{slot}' is not being run, required from "
226 f"non-None slots in: {self.slots}."
227 )
228
229
230class BaseMeasurementTask(lsst.pipe.base.Task):
231 """Ultimate base class for all measurement tasks.
232
233 Parameters
234 ----------
235 algMetadata : `lsst.daf.base.PropertyList` or `None`
236 Will be modified in-place to contain metadata about the plugins being
237 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
238 created.
239 **kwds
240 Additional arguments passed to `lsst.pipe.base.Task.__init__`.
241
242 Notes
243 -----
244 This base class for `SingleFrameMeasurementTask` and
245 `ForcedMeasurementTask` mostly exists to share code between the two, and
246 generally should not be used directly.
247 """
248
249 ConfigClass = BaseMeasurementConfig
250 _DefaultName = "measurement"
251
252 plugins = None
253 """Plugins to be invoked (`PluginMap`).
254
255 Initially empty, this will be populated as plugins are initialized. It
256 should be considered read-only.
257 """
258
259 algMetadata = None
260 """Metadata about active plugins (`lsst.daf.base.PropertyList`).
261
262 Contains additional information about active plugins to be saved with
263 the output catalog. Will be filled by subclasses.
264 """
265
266 def __init__(self, algMetadata=None, **kwds):
267 super(BaseMeasurementTask, self).__init__(**kwds)
268 self.plugins = PluginMap()
270 if algMetadata is None:
271 algMetadata = lsst.daf.base.PropertyList()
272 self.algMetadata = algMetadata
273
274 def initializePlugins(self, **kwds):
275 """Initialize plugins (and slots) according to configuration.
276
277 Parameters
278 ----------
279 **kwds
280 Keyword arguments forwarded directly to plugin constructors.
281
282 Notes
283 -----
284 Derived class constructors should call this method to fill the
285 `plugins` attribute and add corresponding output fields and slot
286 aliases to the output schema.
287
288 In addition to the attributes added by `BaseMeasurementTask.__init__`,
289 a ``schema``` attribute holding the output schema must be present
290 before this method is called.
291
292 Keyword arguments are forwarded directly to plugin constructors,
293 allowing derived classes to use plugins with different signatures.
294 """
295 # Make a place at the beginning for the centroid plugin to run first
296 # (because it's an OrderedDict, adding an empty element in advance
297 # means it will get run first when it's reassigned to the actual
298 # Plugin).
299 if self.config.slots.centroid is not None:
300 self.plugins[self.config.slots.centroid] = None
301 # Init the plugins, sorted by execution order. At the same time add to
302 # the schema
303 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
304 # Pass logName to the plugin if the plugin is marked as using it
305 # The task will use this name to log plugin errors, regardless.
306 if getattr(PluginClass, "hasLogName", False):
307 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
308 logName=self.log.getChild(name).name, **kwds)
309 else:
310 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
311
312 # In rare circumstances (usually tests), the centroid slot not be
313 # coming from an algorithm, which means we'll have added something we
314 # don't want to the plugins map, and we should remove it.
315 if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
316 del self.plugins[self.config.slots.centroid]
317 # Initialize the plugins to run on the undeblended image
318 for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
319 undeblendedName = self.config.undeblendedPrefix + name
320 if getattr(PluginClass, "hasLogName", False):
321 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
322 metadata=self.algMetadata,
323 logName=self.log.getChild(undeblendedName).name,
324 **kwds)
325 else:
326 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
327 metadata=self.algMetadata, **kwds)
328
329 if "schemaMapper" in kwds:
330 schema = kwds["schemaMapper"].editOutputSchema()
331 else:
332 schema = kwds["schema"]
333
334 invalidPsfName = "base_InvalidPsf_flag"
335 if invalidPsfName in schema:
336 self.keyInvalidPsf = schema.find(invalidPsfName).key
337 else:
338 self.keyInvalidPsf = schema.addField(
339 invalidPsfName,
340 type="Flag",
341 doc="Invalid PSF at this location.",
342 )
343
344 def callMeasure(self, measRecord, *args, **kwds):
345 """Call ``measure`` on all plugins and consistently handle exceptions.
346
347 Parameters
348 ----------
349 measRecord : `lsst.afw.table.SourceRecord`
350 The record corresponding to the object being measured. Will be
351 updated in-place with the results of measurement.
352 *args
353 Positional arguments forwarded to ``plugin.measure``
354 **kwds
355 Keyword arguments. Two are handled locally:
356
357 beginOrder : `int`
358 Beginning execution order (inclusive). Measurements with
359 ``executionOrder`` < ``beginOrder`` are not executed. `None`
360 for no limit.
361
362 endOrder : `int`
363 Ending execution order (exclusive). Measurements with
364 ``executionOrder`` >= ``endOrder`` are not executed. `None`
365 for no limit.
366
367 Others are forwarded to ``plugin.measure()``.
368
369 Notes
370 -----
371 This method can be used with plugins that have different signatures;
372 the only requirement is that ``measRecord`` be the first argument.
373 Subsequent positional arguments and keyword arguments are forwarded
374 directly to the plugin.
375
376 This method should be considered "protected": it is intended for use by
377 derived classes, not users.
378 """
379 beginOrder = kwds.pop("beginOrder", None)
380 endOrder = kwds.pop("endOrder", None)
381 for plugin in self.plugins.iter():
382 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
383 continue
384 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
385 break
386 self.doMeasurement(plugin, measRecord, *args, **kwds)
387
388 def doMeasurement(self, plugin, measRecord, *args, **kwds):
389 """Call ``measure`` on the specified plugin.
390
391 Exceptions are handled in a consistent way.
392
393 Parameters
394 ----------
395 plugin : subclass of `BasePlugin`
396 Plugin that will be executed.
397 measRecord : `lsst.afw.table.SourceRecord`
398 The record corresponding to the object being measured. Will be
399 updated in-place with the results of measurement.
400 *args
401 Positional arguments forwarded to ``plugin.measure()``.
402 **kwds
403 Keyword arguments forwarded to ``plugin.measure()``.
404
405 Notes
406 -----
407 This method can be used with plugins that have different signatures;
408 the only requirement is that ``plugin`` and ``measRecord`` be the first
409 two arguments. Subsequent positional arguments and keyword arguments
410 are forwarded directly to the plugin.
411
412 This method should be considered "protected": it is intended for use by
413 derived classes, not users.
414 """
415 try:
416 plugin.measure(measRecord, *args, **kwds)
417 except FATAL_EXCEPTIONS:
418 raise
419 except MeasurementError as error:
420 self.log.getChild(plugin.name).debug(
421 "MeasurementError in %s.measure on record %s: %s",
422 plugin.name, measRecord.getId(), error)
423 plugin.fail(measRecord, error)
424 except InvalidPsfError as error:
425 self.log.getChild(plugin.name).debug(
426 "InvalidPsfError in %s.measure on record %s: %s",
427 plugin.name, measRecord.getId(), error)
428 measRecord.set(self.keyInvalidPsf, True)
429 plugin.fail(measRecord)
430 except Exception as error:
431 self.log.getChild(plugin.name).warning(
432 "Exception in %s.measure on record %s: %s",
433 plugin.name, measRecord.getId(), error)
434 plugin.fail(measRecord)
435
436 def callMeasureN(self, measCat, *args, **kwds):
437 """Call ``measureN`` on all plugins and consistently handle exceptions.
438
439 Parameters
440 ----------
441 measCat : `lsst.afw.table.SourceCatalog`
442 Catalog containing only the records for the source family to be
443 measured, and where outputs should be written.
444 *args
445 Positional arguments forwarded to ``plugin.measure()``
446 **kwds
447 Keyword arguments. Two are handled locally:
448
449 beginOrder:
450 Beginning execution order (inclusive): Measurements with
451 ``executionOrder`` < ``beginOrder`` are not executed. `None`
452 for no limit.
453 endOrder:
454 Ending execution order (exclusive): measurements with
455 ``executionOrder`` >= ``endOrder`` are not executed. `None` for
456 no ``limit``.
457
458 Others are are forwarded to ``plugin.measure()``.
459
460 Notes
461 -----
462 This method can be used with plugins that have different signatures;
463 the only requirement is that ``measRecord`` be the first argument.
464 Subsequent positional arguments and keyword arguments are forwarded
465 directly to the plugin.
466
467 This method should be considered "protected": it is intended for use by
468 derived classes, not users.
469 """
470 beginOrder = kwds.pop("beginOrder", None)
471 endOrder = kwds.pop("endOrder", None)
472 for plugin in self.plugins.iterN():
473 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
474 continue
475 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
476 break
477 self.doMeasurementN(plugin, measCat, *args, **kwds)
478
479 def doMeasurementN(self, plugin, measCat, *args, **kwds):
480 """Call ``measureN`` on the specified plugin.
481
482 Exceptions are handled in a consistent way.
483
484 Parameters
485 ----------
486 plugin : subclass of `BasePlugin`
487 Plugin that will be executed.
488 measCat : `lsst.afw.table.SourceCatalog`
489 Catalog containing only the records for the source family to be
490 measured, and where outputs should be written.
491 *args
492 Positional arguments forwarded to ``plugin.measureN()``.
493 **kwds
494 Keyword arguments forwarded to ``plugin.measureN()``.
495
496 Notes
497 -----
498 This method can be used with plugins that have different signatures;
499 the only requirement is that the ``plugin`` and ``measCat`` be the
500 first two arguments. Subsequent positional arguments and keyword
501 arguments are forwarded directly to the plugin.
502
503 This method should be considered "protected": it is intended for use by
504 derived classes, not users.
505 """
506 try:
507 plugin.measureN(measCat, *args, **kwds)
508 except FATAL_EXCEPTIONS:
509 raise
510
511 except MeasurementError as error:
512 self.log.getChild(plugin.name).debug(
513 "MeasurementError in %s.measureN on records %s-%s: %s",
514 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
515 for measRecord in measCat:
516 plugin.fail(measRecord, error)
517 except InvalidPsfError as error:
518 self.log.getChild(plugin.name).debug(
519 "InvalidPsfError in %s.measureN on records %s-%s: %s",
520 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
521 for measRecord in measCat:
522 measRecord.set(self.keyInvalidPsf, True)
523 plugin.fail(measRecord, error)
524 except Exception as error:
525 self.log.getChild(plugin.name).warning(
526 "Exception in %s.measureN on records %s-%s: %s",
527 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
528 for measRecord in measCat:
529 plugin.fail(measRecord)
530
531 @staticmethod
533 """Get a set of footprints from a catalog, keyed by id.
534
535 Parameters
536 ----------
537 catalog : `lsst.afw.table.SourceCatalog`
538 Catalog with `lsst.afw.detection.Footprint`s attached.
539
540 Returns
541 -------
542 footprints : `dict` [`int`: (`int`, `lsst.afw.detection.Footprint`)]
543 Dictionary of footprint, keyed by id number, with a tuple of
544 the parent id and footprint.
545 """
546 return {measRecord.getId(): (measRecord.getParent(), measRecord.getFootprint())
547 for measRecord in catalog}
Class for storing ordered metadata with comments.
__new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs)
doMeasurementN(self, plugin, measCat, *args, **kwds)
doMeasurement(self, plugin, measRecord, *args, **kwds)