LSST Applications g04e9c324dd+8c5ae1fdc5,g134cb467dc+1b3060144d,g18429d2f64+f642bf4753,g199a45376c+0ba108daf9,g1fd858c14a+2dcf163641,g262e1987ae+7b8c96d2ca,g29ae962dfc+3bd6ecb08a,g2cef7863aa+aef1011c0b,g35bb328faa+8c5ae1fdc5,g3fd5ace14f+53e1a9e7c5,g4595892280+fef73a337f,g47891489e3+2efcf17695,g4d44eb3520+642b70b07e,g53246c7159+8c5ae1fdc5,g67b6fd64d1+2efcf17695,g67fd3c3899+b70e05ef52,g74acd417e5+317eb4c7d4,g786e29fd12+668abc6043,g87389fa792+8856018cbb,g89139ef638+2efcf17695,g8d7436a09f+3be3c13596,g8ea07a8fe4+9f5ccc88ac,g90f42f885a+a4e7b16d9b,g97be763408+ad77d7208f,g9dd6db0277+b70e05ef52,ga681d05dcb+a3f46e7fff,gabf8522325+735880ea63,gac2eed3f23+2efcf17695,gb89ab40317+2efcf17695,gbf99507273+8c5ae1fdc5,gd8ff7fe66e+b70e05ef52,gdab6d2f7ff+317eb4c7d4,gdc713202bf+b70e05ef52,gdfd2d52018+b10e285e0f,ge365c994fd+310e8507c4,ge410e46f29+2efcf17695,geaed405ab2+562b3308c0,gffca2db377+8c5ae1fdc5,w.2025.35
LSST Data Management Base Package
Loading...
Searching...
No Matches
baseMeasurement.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""Base measurement task, which subclassed by the single frame and forced
23measurement tasks.
24"""
25import warnings
26
27import lsst.pipe.base
28import lsst.pex.config
29
30from .pluginRegistry import PluginMap
31from ._measBaseLib import FatalAlgorithmError, MeasurementError
32from lsst.afw.detection import InvalidPsfError
33from .pluginsBase import BasePluginConfig, BasePlugin
34from .noiseReplacer import NoiseReplacerConfig
35
36__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
37 "BaseMeasurementConfig", "BaseMeasurementTask")
38
39# Exceptions that the measurement tasks should always propagate up to their
40# callers
41FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
42
43
45 """Base config class for all measurement plugins.
46
47 Notes
48 -----
49 Most derived classes will want to override `setDefaults` in order to
50 customize the default `executionOrder`.
51
52 A derived class whose corresponding Plugin class implements a do `measureN`
53 method should additionally add a bool `doMeasureN` field to replace the
54 bool class attribute defined here.
55 """
56
57 doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
58 doc="whether to run this plugin in single-object mode")
59
60 doMeasureN = False # replace this class attribute with a Field if measureN-capable
61
62
64 """Base class for all measurement plugins.
65
66 Notes
67 -----
68 This is class is a placeholder for future behavior which will be shared
69 only between measurement plugins and is implemented for symmetry with the
70 measurement base plugin configuration class
71 """
72
73 pass
74
75
77 """Assign named plugins to measurement slots.
78
79 Slot configuration which assigns a particular named plugin to each of a set
80 of slots. Each slot allows a type of measurement to be fetched from the
81 `lsst.afw.table.SourceTable` without knowing which algorithm was used to
82 produced the data.
83
84 Notes
85 -----
86 The default algorithm for each slot must be registered, even if the default
87 is not used.
88 """
89
91 centroid = Field(dtype=str, default="base_SdssCentroid", optional=True,
92 doc="the name of the centroiding algorithm used to set source x,y")
93 shape = Field(dtype=str, default="base_SdssShape", optional=True,
94 doc="the name of the algorithm used to set source moments parameters")
95 psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True,
96 doc="the name of the algorithm used to set PSF moments parameters")
97 apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
98 doc="the name of the algorithm used to set the source aperture instFlux slot")
99 modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
100 doc="the name of the algorithm used to set the source model instFlux slot")
101 psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True,
102 doc="the name of the algorithm used to set the source psf instFlux slot")
103 gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
104 doc="the name of the algorithm used to set the source Gaussian instFlux slot")
105 calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
106 doc="the name of the instFlux measurement algorithm used for calibration")
107
108 def setupSchema(self, schema):
109 """Set up a slots in a schema following configuration directives.
110
111 Parameters
112 ----------
113 schema : `lsst.afw.table.Schema`
114 The schema in which slots will be set up.
115
116 Notes
117 -----
118 This is defined in this configuration class to support use in unit
119 tests without needing to construct an `lsst.pipe.base.Task` object.
120 """
121 aliases = schema.getAliasMap()
122 if self.centroid is not None:
123 aliases.set("slot_Centroid", self.centroid)
124 if self.shape is not None:
125 aliases.set("slot_Shape", self.shape)
126 if self.psfShape is not None:
127 aliases.set("slot_PsfShape", self.psfShape)
128 if self.apFlux is not None:
129 aliases.set("slot_ApFlux", self.apFlux)
130 if self.modelFlux is not None:
131 aliases.set("slot_ModelFlux", self.modelFlux)
132 if self.psfFlux is not None:
133 aliases.set("slot_PsfFlux", self.psfFlux)
134 if self.gaussianFlux is not None:
135 aliases.set("slot_GaussianFlux", self.gaussianFlux)
136 if self.calibFlux is not None:
137 aliases.set("slot_CalibFlux", self.calibFlux)
138
139
141 """Base configuration for all measurement driver tasks."""
143 dtype=SourceSlotConfig,
144 doc="Mapping from algorithms to special column aliases."
145 )
146
147 def validate(self):
148 super().validate()
149 if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
151 self.__class__.slots,
152 self,
153 "source centroid slot algorithm is not being run."
154 )
155 if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
157 self.__class__.slots,
158 self,
159 "source shape slot algorithm '%s' is not being run." % self.slots.shape
160 )
161 for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
162 self.slots.gaussianFlux, self.slots.calibFlux):
163 if slot is not None:
164 for name in self.plugins.names:
165 if len(name) <= len(slot) and name == slot[:len(name)]:
166 break
167 else:
169 self.__class__.slots,
170 self,
171 f"Source instFlux algorithm '{slot}' is not being run, required from "
172 f"non-None slots in: {self.slots}."
173 )
174
175
177 """Base configuration for all measurement driver tasks except
178 SimpleForcedMeasurementTask.
179
180 Parameters
181 ----------
182 ignoreSlotPluginChecks : `bool`, optional
183 Do not check that all slots have an associated plugin to run when
184 validating this config. This is primarily for tests that were written
185 before we made Tasks always call `config.validate()` on init.
186 DEPRECATED DM-35949: this is a temporary workaround while we better
187 define how config/schema validation works for measurement tasks.
188
189 Examples
190 --------
191 Subclasses should define the 'plugins' and 'undeblended' registries, e.g.
192
193 .. code-block:: py
194
195 plugins = PluginBaseClass.registry.makeField(
196 multi=True,
197 default=[],
198 doc="Plugins to be run and their configuration"
199 )
200 undeblended = PluginBaseClass.registry.makeField(
201 multi=True,
202 default=[],
203 doc="Plugins to run on undeblended image"
204 )
205
206 where ``PluginBaseClass`` is the appropriate base class of the plugin
207 (e.g., `SingleFramePlugin` or `ForcedPlugin`).
208 """
209 def __new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs):
210 instance = super().__new__(cls, *args, **kwargs)
211 if ignoreSlotPluginChecks:
212 msg = ("ignoreSlotPluginChecks is deprecated and should only be used in tests."
213 " No removal date has been set; see DM-35949.")
214 warnings.warn(msg, category=FutureWarning, stacklevel=2)
215 object.__setattr__(instance, "_ignoreSlotPluginChecks", ignoreSlotPluginChecks)
216 return instance
217
218 doReplaceWithNoise = lsst.pex.config.Field(
219 dtype=bool, default=True, optional=False,
220 doc='When measuring, replace other detected footprints with noise?')
221
223 dtype=NoiseReplacerConfig,
224 doc="configuration that sets how to replace neighboring sources with noise"
225 )
226 undeblendedPrefix = lsst.pex.config.Field(
227 dtype=str, default="undeblended_",
228 doc="Prefix to give undeblended plugins"
229 )
230
231 def validate(self):
233 return
234 super().validate()
235
236
237class SimpleBaseMeasurementTask(lsst.pipe.base.Task):
238 """Ultimate base class for all measurement tasks.
239
240 Parameters
241 ----------
242 algMetadata : `lsst.daf.base.PropertyList` or `None`
243 Will be modified in-place to contain metadata about the plugins being
244 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
245 created.
246 **kwds
247 Additional arguments passed to `lsst.pipe.base.Task.__init__`.
248
249 Notes
250 -----
251 This base class was created after `BaseMeasurementTask` already existed
252 to add a common base class for `SimpleForcedMeasurementTask`,
253 `SingleFrameMeasurementTask`, and `ForcedMeasurementTask` without
254 breaking downstream code. It is not intended to be used directly,
255 but rather to be subclassed by those tasks.
256 """
257
258 ConfigClass = SimpleBaseMeasurementConfig
259 _DefaultName = "measurement"
260
261 plugins = None
262 """Plugins to be invoked (`PluginMap`).
263
264 Initially empty, this will be populated as plugins are initialized. It
265 should be considered read-only.
266 """
267
268 algMetadata = None
269 """Metadata about active plugins (`lsst.daf.base.PropertyList`).
270
271 Contains additional information about active plugins to be saved with
272 the output catalog. Will be filled by subclasses.
273 """
274
275 def __init__(self, algMetadata=None, **kwds):
276 super().__init__(**kwds)
277 self.plugins = PluginMap()
278 if algMetadata is None:
279 algMetadata = lsst.daf.base.PropertyList()
280 self.algMetadata = algMetadata
281
282 def initializePlugins(self, **kwds):
283 """Initialize plugins (and slots) according to configuration.
284
285 Parameters
286 ----------
287 **kwds
288 Keyword arguments forwarded directly to plugin constructors.
289
290 Notes
291 -----
292 Derived class constructors should call this method to fill the
293 `plugins` attribute and add corresponding output fields and slot
294 aliases to the output schema.
295
296 In addition to the attributes added by `BaseMeasurementTask.__init__`,
297 a ``schema``` attribute holding the output schema must be present
298 before this method is called.
299
300 Keyword arguments are forwarded directly to plugin constructors,
301 allowing derived classes to use plugins with different signatures.
302 """
303 # Make a place at the beginning for the centroid plugin to run first
304 # (because it's an OrderedDict, adding an empty element in advance
305 # means it will get run first when it's reassigned to the actual
306 # Plugin).
307 if self.config.slots.centroid is not None:
308 self.plugins[self.config.slots.centroid] = None
309 # Init the plugins, sorted by execution order. At the same time add to
310 # the schema
311 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
312 # Pass logName to the plugin if the plugin is marked as using it
313 # The task will use this name to log plugin errors, regardless.
314 if getattr(PluginClass, "hasLogName", False):
315 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
316 logName=self.log.getChild(name).name, **kwds)
317 else:
318 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
319
320 # In rare circumstances (usually tests), the centroid slot not be
321 # coming from an algorithm, which means we'll have added something we
322 # don't want to the plugins map, and we should remove it.
323 if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
324 del self.plugins[self.config.slots.centroid]
325
326 def addInvalidPsfFlag(self, schema):
327 invalidPsfName = "base_InvalidPsf_flag"
328 if invalidPsfName in schema:
329 self.keyInvalidPsf = schema.find(invalidPsfName).key
330 else:
331 self.keyInvalidPsf = schema.addField(
332 invalidPsfName,
333 type="Flag",
334 doc="Invalid PSF at this location.",
335 )
336
337 def callMeasure(self, measRecord, *args, **kwds):
338 """Call ``measure`` on all plugins and consistently handle exceptions.
339
340 Parameters
341 ----------
342 measRecord : `lsst.afw.table.SourceRecord`
343 The record corresponding to the object being measured. Will be
344 updated in-place with the results of measurement.
345 *args
346 Positional arguments forwarded to ``plugin.measure``
347 **kwds
348 Keyword arguments. Two are handled locally:
349
350 beginOrder : `int`
351 Beginning execution order (inclusive). Measurements with
352 ``executionOrder`` < ``beginOrder`` are not executed. `None`
353 for no limit.
354
355 endOrder : `int`
356 Ending execution order (exclusive). Measurements with
357 ``executionOrder`` >= ``endOrder`` are not executed. `None`
358 for no limit.
359
360 Others are forwarded to ``plugin.measure()``.
361
362 Notes
363 -----
364 This method can be used with plugins that have different signatures;
365 the only requirement is that ``measRecord`` be the first argument.
366 Subsequent positional arguments and keyword arguments are forwarded
367 directly to the plugin.
368
369 This method should be considered "protected": it is intended for use by
370 derived classes, not users.
371 """
372 beginOrder = kwds.pop("beginOrder", None)
373 endOrder = kwds.pop("endOrder", None)
374 for plugin in self.plugins.iter():
375 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
376 continue
377 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
378 break
379 self.doMeasurement(plugin, measRecord, *args, **kwds)
380
381 def doMeasurement(self, plugin, measRecord, *args, **kwds):
382 """Call ``measure`` on the specified plugin.
383
384 Exceptions are handled in a consistent way.
385
386 Parameters
387 ----------
388 plugin : subclass of `BasePlugin`
389 Plugin that will be executed.
390 measRecord : `lsst.afw.table.SourceRecord`
391 The record corresponding to the object being measured. Will be
392 updated in-place with the results of measurement.
393 *args
394 Positional arguments forwarded to ``plugin.measure()``.
395 **kwds
396 Keyword arguments forwarded to ``plugin.measure()``.
397
398 Notes
399 -----
400 This method can be used with plugins that have different signatures;
401 the only requirement is that ``plugin`` and ``measRecord`` be the first
402 two arguments. Subsequent positional arguments and keyword arguments
403 are forwarded directly to the plugin.
404
405 This method should be considered "protected": it is intended for use by
406 derived classes, not users.
407 """
408 try:
409 plugin.measure(measRecord, *args, **kwds)
410 except FATAL_EXCEPTIONS:
411 raise
412 except MeasurementError as error:
413 self.log.getChild(plugin.name).debug(
414 "MeasurementError in %s.measure on record %s: %s",
415 plugin.name, measRecord.getId(), error)
416 plugin.fail(measRecord, error)
417 except InvalidPsfError as error:
418 self.log.getChild(plugin.name).debug(
419 "InvalidPsfError in %s.measure on record %s: %s",
420 plugin.name, measRecord.getId(), error)
421 measRecord.set(self.keyInvalidPsf, True)
422 plugin.fail(measRecord)
423 except Exception as error:
424 self.log.getChild(plugin.name).warning(
425 "Exception in %s.measure on record %s: %s",
426 plugin.name, measRecord.getId(), error)
427 plugin.fail(measRecord)
428
429
431 """Ultimate base class for all measurement tasks
432 other than SimpleForcedMeasurementTask.
433
434 Parameters
435 ----------
436 algMetadata : `lsst.daf.base.PropertyList` or `None`
437 Will be modified in-place to contain metadata about the plugins being
438 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
439 created.
440 **kwds
441 Additional arguments passed to `lsst.pipe.base.Task.__init__`.
442
443 Notes
444 -----
445 This base class for `SingleFrameMeasurementTask` and
446 `ForcedMeasurementTask` mostly exists to share code between the two, and
447 generally should not be used directly.
448 """
449
450 ConfigClass = BaseMeasurementConfig
451
452 def __init__(self, algMetadata=None, **kwds):
453 super().__init__(algMetadata=algMetadata, **kwds)
455
456 def initializePlugins(self, **kwds):
457 # Docstring inherited.
458 super().initializePlugins(**kwds)
459 for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
460 undeblendedName = self.config.undeblendedPrefix + name
461 if getattr(PluginClass, "hasLogName", False):
462 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
463 metadata=self.algMetadata,
464 logName=self.log.getChild(undeblendedName).name,
465 **kwds)
466 else:
467 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
468 metadata=self.algMetadata, **kwds)
469
470 def callMeasureN(self, measCat, *args, **kwds):
471 """Call ``measureN`` on all plugins and consistently handle exceptions.
472
473 Parameters
474 ----------
475 measCat : `lsst.afw.table.SourceCatalog`
476 Catalog containing only the records for the source family to be
477 measured, and where outputs should be written.
478 *args
479 Positional arguments forwarded to ``plugin.measure()``
480 **kwds
481 Keyword arguments. Two are handled locally:
482
483 beginOrder:
484 Beginning execution order (inclusive): Measurements with
485 ``executionOrder`` < ``beginOrder`` are not executed. `None`
486 for no limit.
487 endOrder:
488 Ending execution order (exclusive): measurements with
489 ``executionOrder`` >= ``endOrder`` are not executed. `None` for
490 no ``limit``.
491
492 Others are are forwarded to ``plugin.measure()``.
493
494 Notes
495 -----
496 This method can be used with plugins that have different signatures;
497 the only requirement is that ``measRecord`` be the first argument.
498 Subsequent positional arguments and keyword arguments are forwarded
499 directly to the plugin.
500
501 This method should be considered "protected": it is intended for use by
502 derived classes, not users.
503 """
504 beginOrder = kwds.pop("beginOrder", None)
505 endOrder = kwds.pop("endOrder", None)
506 for plugin in self.plugins.iterN():
507 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
508 continue
509 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
510 break
511 self.doMeasurementN(plugin, measCat, *args, **kwds)
512
513 def doMeasurementN(self, plugin, measCat, *args, **kwds):
514 """Call ``measureN`` on the specified plugin.
515
516 Exceptions are handled in a consistent way.
517
518 Parameters
519 ----------
520 plugin : subclass of `BasePlugin`
521 Plugin that will be executed.
522 measCat : `lsst.afw.table.SourceCatalog`
523 Catalog containing only the records for the source family to be
524 measured, and where outputs should be written.
525 *args
526 Positional arguments forwarded to ``plugin.measureN()``.
527 **kwds
528 Keyword arguments forwarded to ``plugin.measureN()``.
529
530 Notes
531 -----
532 This method can be used with plugins that have different signatures;
533 the only requirement is that the ``plugin`` and ``measCat`` be the
534 first two arguments. Subsequent positional arguments and keyword
535 arguments are forwarded directly to the plugin.
536
537 This method should be considered "protected": it is intended for use by
538 derived classes, not users.
539 """
540 try:
541 plugin.measureN(measCat, *args, **kwds)
542 except FATAL_EXCEPTIONS:
543 raise
544
545 except MeasurementError as error:
546 self.log.getChild(plugin.name).debug(
547 "MeasurementError in %s.measureN on records %s-%s: %s",
548 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
549 for measRecord in measCat:
550 plugin.fail(measRecord, error)
551 except InvalidPsfError as error:
552 self.log.getChild(plugin.name).debug(
553 "InvalidPsfError in %s.measureN on records %s-%s: %s",
554 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
555 for measRecord in measCat:
556 measRecord.set(self.keyInvalidPsf, True)
557 plugin.fail(measRecord, error)
558 except Exception as error:
559 self.log.getChild(plugin.name).warning(
560 "Exception in %s.measureN on records %s-%s: %s",
561 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
562 for measRecord in measCat:
563 plugin.fail(measRecord)
564
565 @staticmethod
567 """Get a set of footprints from a catalog, keyed by id.
568
569 Parameters
570 ----------
571 catalog : `lsst.afw.table.SourceCatalog`
572 Catalog with `lsst.afw.detection.Footprint`s attached.
573
574 Returns
575 -------
576 footprints : `dict` [`int`: (`int`, `lsst.afw.detection.Footprint`)]
577 Dictionary of footprint, keyed by id number, with a tuple of
578 the parent id and footprint.
579 """
580 return {measRecord.getId(): (measRecord.getParent(), measRecord.getFootprint())
581 for measRecord in catalog}
Class for storing ordered metadata with comments.
__new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs)
doMeasurementN(self, plugin, measCat, *args, **kwds)
doMeasurement(self, plugin, measRecord, *args, **kwds)