LSST Applications g04c3c9f7ca+2075667efa,g1e125bf412+5f448d5fcf,g2079a07aa2+3e9fd84d81,g2305ad1205+b635cf1488,g2bbee38e9b+6c6beb4891,g337abbeb29+6c6beb4891,g33d1c0ed96+6c6beb4891,g3a166c0a6a+6c6beb4891,g50ff169b8f+96c6868917,g52b1c1532d+585e252eca,g591dd9f2cf+42f171e1e6,g5c3423f6d4+d536b04327,g607f77f49a+d536b04327,g6f43f06aed+ca1339dc19,g858d7b2824+d536b04327,g8ee334c5b4+d7f9608c2f,g9963eaa53e+b3dc1655d3,g998f4353bf+d536b04327,g99cad8db69+8ef2408349,g9ddcbc5298+9a081db1e4,ga1e77700b3+2cbb763275,gadfd92a7e4+aec2f3b930,gae0086650b+585e252eca,gb0e22166c9+0e73c8378f,gb3b7280ab2+cb5fdb229e,gbb8dafda3b+a327199e22,gc120e1dc64+88074880ea,gc28159a63d+6c6beb4891,gcdd4ae20e8+bd241b2308,gcde1bda545+903e937d91,gcf0d15dbbd+bd241b2308,gdaeeff99f8+f9a426f77a,gddc38dedce+585e252eca,ge79ae78c31+6c6beb4891,gfbcc870c63+b310236976,w.2024.23
LSST Data Management Base Package
Loading...
Searching...
No Matches
baseMeasurement.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22"""Base measurement task, which subclassed by the single frame and forced
23measurement tasks.
24"""
25import warnings
26
27import lsst.pipe.base
28import lsst.pex.config
29
30from .pluginRegistry import PluginMap
31from ._measBaseLib import FatalAlgorithmError, MeasurementError
32from lsst.afw.detection import InvalidPsfError
33from .pluginsBase import BasePluginConfig, BasePlugin
34from .noiseReplacer import NoiseReplacerConfig
35
36__all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
37 "BaseMeasurementConfig", "BaseMeasurementTask")
38
39# Exceptions that the measurement tasks should always propagate up to their
40# callers
41FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
42
43
45 """Base config class for all measurement plugins.
46
47 Notes
48 -----
49 Most derived classes will want to override `setDefaults` in order to
50 customize the default `executionOrder`.
51
52 A derived class whose corresponding Plugin class implements a do `measureN`
53 method should additionally add a bool `doMeasureN` field to replace the
54 bool class attribute defined here.
55 """
56
57 doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
58 doc="whether to run this plugin in single-object mode")
59
60 doMeasureN = False # replace this class attribute with a Field if measureN-capable
61
62
64 """Base class for all measurement plugins.
65
66 Notes
67 -----
68 This is class is a placeholder for future behavior which will be shared
69 only between measurement plugins and is implemented for symmetry with the
70 measurement base plugin configuration class
71 """
72
73 pass
74
75
77 """Assign named plugins to measurement slots.
78
79 Slot configuration which assigns a particular named plugin to each of a set
80 of slots. Each slot allows a type of measurement to be fetched from the
81 `lsst.afw.table.SourceTable` without knowing which algorithm was used to
82 produced the data.
83
84 Notes
85 -----
86 The default algorithm for each slot must be registered, even if the default
87 is not used.
88 """
89
91 centroid = Field(dtype=str, default="base_SdssCentroid", optional=True,
92 doc="the name of the centroiding algorithm used to set source x,y")
93 shape = Field(dtype=str, default="base_SdssShape", optional=True,
94 doc="the name of the algorithm used to set source moments parameters")
95 psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True,
96 doc="the name of the algorithm used to set PSF moments parameters")
97 apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
98 doc="the name of the algorithm used to set the source aperture instFlux slot")
99 modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
100 doc="the name of the algorithm used to set the source model instFlux slot")
101 psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True,
102 doc="the name of the algorithm used to set the source psf instFlux slot")
103 gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
104 doc="the name of the algorithm used to set the source Gaussian instFlux slot")
105 calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
106 doc="the name of the instFlux measurement algorithm used for calibration")
107
108 def setupSchema(self, schema):
109 """Set up a slots in a schema following configuration directives.
110
111 Parameters
112 ----------
113 schema : `lsst.afw.table.Schema`
114 The schema in which slots will be set up.
115
116 Notes
117 -----
118 This is defined in this configuration class to support use in unit
119 tests without needing to construct an `lsst.pipe.base.Task` object.
120 """
121 aliases = schema.getAliasMap()
122 if self.centroid is not None:
123 aliases.set("slot_Centroid", self.centroid)
124 if self.shape is not None:
125 aliases.set("slot_Shape", self.shape)
126 if self.psfShape is not None:
127 aliases.set("slot_PsfShape", self.psfShape)
128 if self.apFlux is not None:
129 aliases.set("slot_ApFlux", self.apFlux)
130 if self.modelFlux is not None:
131 aliases.set("slot_ModelFlux", self.modelFlux)
132 if self.psfFlux is not None:
133 aliases.set("slot_PsfFlux", self.psfFlux)
134 if self.gaussianFlux is not None:
135 aliases.set("slot_GaussianFlux", self.gaussianFlux)
136 if self.calibFlux is not None:
137 aliases.set("slot_CalibFlux", self.calibFlux)
138
139
141 """Base configuration for all measurement driver tasks.
142
143 Parameters
144 ----------
145 ignoreSlotPluginChecks : `bool`, optional
146 Do not check that all slots have an associated plugin to run when
147 validating this config. This is primarily for tests that were written
148 before we made Tasks always call `config.validate()` on init.
149 DEPRECATED DM-35949: this is a temporary workaround while we better
150 define how config/schema validation works for measurement tasks.
151
152 Examples
153 --------
154 Subclasses should define the 'plugins' and 'undeblended' registries, e.g.
155
156 .. code-block:: py
157
158 plugins = PluginBaseClass.registry.makeField(
159 multi=True,
160 default=[],
161 doc="Plugins to be run and their configuration"
162 )
163 undeblended = PluginBaseClass.registry.makeField(
164 multi=True,
165 default=[],
166 doc="Plugins to run on undeblended image"
167 )
168
169 where ``PluginBaseClass`` is the appropriate base class of the plugin
170 (e.g., `SingleFramePlugin` or `ForcedPlugin`).
171 """
172 def __new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs):
173 instance = super().__new__(cls, *args, **kwargs)
174 if ignoreSlotPluginChecks:
175 msg = ("ignoreSlotPluginChecks is deprecated and should only be used in tests."
176 " No removal date has been set; see DM-35949.")
177 warnings.warn(msg, category=FutureWarning, stacklevel=2)
178 object.__setattr__(instance, "_ignoreSlotPluginChecks", ignoreSlotPluginChecks)
179 return instance
180
182 dtype=SourceSlotConfig,
183 doc="Mapping from algorithms to special aliases in Source."
184 )
185
186 doReplaceWithNoise = lsst.pex.config.Field(
187 dtype=bool, default=True, optional=False,
188 doc='When measuring, replace other detected footprints with noise?')
189
191 dtype=NoiseReplacerConfig,
192 doc="configuration that sets how to replace neighboring sources with noise"
193 )
194 undeblendedPrefix = lsst.pex.config.Field(
195 dtype=str, default="undeblended_",
196 doc="Prefix to give undeblended plugins"
197 )
198
199 def validate(self):
200 super().validate()
201 if self._ignoreSlotPluginChecks:
202 return
203 if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
204 raise ValueError("source centroid slot algorithm is not being run.")
205 if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
206 raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape)
207 for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
208 self.slots.gaussianFlux, self.slots.calibFlux):
209 if slot is not None:
210 for name in self.plugins.names:
211 if len(name) <= len(slot) and name == slot[:len(name)]:
212 break
213 else:
214 raise ValueError("source instFlux slot algorithm '%s' is not being run." % slot)
215
216
217class BaseMeasurementTask(lsst.pipe.base.Task):
218 """Ultimate base class for all measurement tasks.
219
220 Parameters
221 ----------
222 algMetadata : `lsst.daf.base.PropertyList` or `None`
223 Will be modified in-place to contain metadata about the plugins being
224 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
225 created.
226 **kwds
227 Additional arguments passed to `lsst.pipe.base.Task.__init__`.
228
229 Notes
230 -----
231 This base class for `SingleFrameMeasurementTask` and
232 `ForcedMeasurementTask` mostly exists to share code between the two, and
233 generally should not be used directly.
234 """
235
236 ConfigClass = BaseMeasurementConfig
237 _DefaultName = "measurement"
238
239 plugins = None
240 """Plugins to be invoked (`PluginMap`).
241
242 Initially empty, this will be populated as plugins are initialized. It
243 should be considered read-only.
244 """
245
246 algMetadata = None
247 """Metadata about active plugins (`lsst.daf.base.PropertyList`).
248
249 Contains additional information about active plugins to be saved with
250 the output catalog. Will be filled by subclasses.
251 """
252
253 def __init__(self, algMetadata=None, **kwds):
254 super(BaseMeasurementTask, self).__init__(**kwds)
255 self.plugins = PluginMap()
257 if algMetadata is None:
258 algMetadata = lsst.daf.base.PropertyList()
259 self.algMetadata = algMetadata
260
261 def initializePlugins(self, **kwds):
262 """Initialize plugins (and slots) according to configuration.
263
264 Parameters
265 ----------
266 **kwds
267 Keyword arguments forwarded directly to plugin constructors.
268
269 Notes
270 -----
271 Derived class constructors should call this method to fill the
272 `plugins` attribute and add corresponding output fields and slot
273 aliases to the output schema.
274
275 In addition to the attributes added by `BaseMeasurementTask.__init__`,
276 a ``schema``` attribute holding the output schema must be present
277 before this method is called.
278
279 Keyword arguments are forwarded directly to plugin constructors,
280 allowing derived classes to use plugins with different signatures.
281 """
282 # Make a place at the beginning for the centroid plugin to run first
283 # (because it's an OrderedDict, adding an empty element in advance
284 # means it will get run first when it's reassigned to the actual
285 # Plugin).
286 if self.config.slots.centroid is not None:
287 self.plugins[self.config.slots.centroid] = None
288 # Init the plugins, sorted by execution order. At the same time add to
289 # the schema
290 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
291 # Pass logName to the plugin if the plugin is marked as using it
292 # The task will use this name to log plugin errors, regardless.
293 if getattr(PluginClass, "hasLogName", False):
294 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
295 logName=self.log.getChild(name).name, **kwds)
296 else:
297 self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
298
299 # In rare circumstances (usually tests), the centroid slot not be
300 # coming from an algorithm, which means we'll have added something we
301 # don't want to the plugins map, and we should remove it.
302 if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
303 del self.plugins[self.config.slots.centroid]
304 # Initialize the plugins to run on the undeblended image
305 for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
306 undeblendedName = self.config.undeblendedPrefix + name
307 if getattr(PluginClass, "hasLogName", False):
308 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
309 metadata=self.algMetadata,
310 logName=self.log.getChild(undeblendedName).name,
311 **kwds)
312 else:
313 self.undeblendedPlugins[name] = PluginClass(config, undeblendedName,
314 metadata=self.algMetadata, **kwds)
315
316 if "schemaMapper" in kwds:
317 schema = kwds["schemaMapper"].editOutputSchema()
318 else:
319 schema = kwds["schema"]
320
321 invalidPsfName = "base_InvalidPsf_flag"
322 if invalidPsfName in schema:
323 self.keyInvalidPsf = schema.find(invalidPsfName).key
324 else:
325 self.keyInvalidPsf = schema.addField(
326 invalidPsfName,
327 type="Flag",
328 doc="Invalid PSF at this location.",
329 )
330
331 def callMeasure(self, measRecord, *args, **kwds):
332 """Call ``measure`` on all plugins and consistently handle exceptions.
333
334 Parameters
335 ----------
336 measRecord : `lsst.afw.table.SourceRecord`
337 The record corresponding to the object being measured. Will be
338 updated in-place with the results of measurement.
339 *args
340 Positional arguments forwarded to ``plugin.measure``
341 **kwds
342 Keyword arguments. Two are handled locally:
343
344 beginOrder : `int`
345 Beginning execution order (inclusive). Measurements with
346 ``executionOrder`` < ``beginOrder`` are not executed. `None`
347 for no limit.
348
349 endOrder : `int`
350 Ending execution order (exclusive). Measurements with
351 ``executionOrder`` >= ``endOrder`` are not executed. `None`
352 for no limit.
353
354 Others are forwarded to ``plugin.measure()``.
355
356 Notes
357 -----
358 This method can be used with plugins that have different signatures;
359 the only requirement is that ``measRecord`` be the first argument.
360 Subsequent positional arguments and keyword arguments are forwarded
361 directly to the plugin.
362
363 This method should be considered "protected": it is intended for use by
364 derived classes, not users.
365 """
366 beginOrder = kwds.pop("beginOrder", None)
367 endOrder = kwds.pop("endOrder", None)
368 for plugin in self.plugins.iter():
369 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
370 continue
371 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
372 break
373 self.doMeasurement(plugin, measRecord, *args, **kwds)
374
375 def doMeasurement(self, plugin, measRecord, *args, **kwds):
376 """Call ``measure`` on the specified plugin.
377
378 Exceptions are handled in a consistent way.
379
380 Parameters
381 ----------
382 plugin : subclass of `BasePlugin`
383 Plugin that will be executed.
384 measRecord : `lsst.afw.table.SourceRecord`
385 The record corresponding to the object being measured. Will be
386 updated in-place with the results of measurement.
387 *args
388 Positional arguments forwarded to ``plugin.measure()``.
389 **kwds
390 Keyword arguments forwarded to ``plugin.measure()``.
391
392 Notes
393 -----
394 This method can be used with plugins that have different signatures;
395 the only requirement is that ``plugin`` and ``measRecord`` be the first
396 two arguments. Subsequent positional arguments and keyword arguments
397 are forwarded directly to the plugin.
398
399 This method should be considered "protected": it is intended for use by
400 derived classes, not users.
401 """
402 try:
403 plugin.measure(measRecord, *args, **kwds)
404 except FATAL_EXCEPTIONS:
405 raise
406 except MeasurementError as error:
407 self.log.getChild(plugin.name).debug(
408 "MeasurementError in %s.measure on record %s: %s",
409 plugin.name, measRecord.getId(), error)
410 plugin.fail(measRecord, error)
411 except InvalidPsfError as error:
412 self.log.getChild(plugin.name).debug(
413 "InvalidPsfError in %s.measure on record %s: %s",
414 plugin.name, measRecord.getId(), error)
415 measRecord.set(self.keyInvalidPsf, True)
416 plugin.fail(measRecord)
417 except Exception as error:
418 self.log.getChild(plugin.name).warning(
419 "Exception in %s.measure on record %s: %s",
420 plugin.name, measRecord.getId(), error)
421 plugin.fail(measRecord)
422
423 def callMeasureN(self, measCat, *args, **kwds):
424 """Call ``measureN`` on all plugins and consistently handle exceptions.
425
426 Parameters
427 ----------
428 measCat : `lsst.afw.table.SourceCatalog`
429 Catalog containing only the records for the source family to be
430 measured, and where outputs should be written.
431 *args
432 Positional arguments forwarded to ``plugin.measure()``
433 **kwds
434 Keyword arguments. Two are handled locally:
435
436 beginOrder:
437 Beginning execution order (inclusive): Measurements with
438 ``executionOrder`` < ``beginOrder`` are not executed. `None`
439 for no limit.
440 endOrder:
441 Ending execution order (exclusive): measurements with
442 ``executionOrder`` >= ``endOrder`` are not executed. `None` for
443 no ``limit``.
444
445 Others are are forwarded to ``plugin.measure()``.
446
447 Notes
448 -----
449 This method can be used with plugins that have different signatures;
450 the only requirement is that ``measRecord`` be the first argument.
451 Subsequent positional arguments and keyword arguments are forwarded
452 directly to the plugin.
453
454 This method should be considered "protected": it is intended for use by
455 derived classes, not users.
456 """
457 beginOrder = kwds.pop("beginOrder", None)
458 endOrder = kwds.pop("endOrder", None)
459 for plugin in self.plugins.iterN():
460 if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
461 continue
462 if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
463 break
464 self.doMeasurementN(plugin, measCat, *args, **kwds)
465
466 def doMeasurementN(self, plugin, measCat, *args, **kwds):
467 """Call ``measureN`` on the specified plugin.
468
469 Exceptions are handled in a consistent way.
470
471 Parameters
472 ----------
473 plugin : subclass of `BasePlugin`
474 Plugin that will be executed.
475 measCat : `lsst.afw.table.SourceCatalog`
476 Catalog containing only the records for the source family to be
477 measured, and where outputs should be written.
478 *args
479 Positional arguments forwarded to ``plugin.measureN()``.
480 **kwds
481 Keyword arguments forwarded to ``plugin.measureN()``.
482
483 Notes
484 -----
485 This method can be used with plugins that have different signatures;
486 the only requirement is that the ``plugin`` and ``measCat`` be the
487 first two arguments. Subsequent positional arguments and keyword
488 arguments are forwarded directly to the plugin.
489
490 This method should be considered "protected": it is intended for use by
491 derived classes, not users.
492 """
493 try:
494 plugin.measureN(measCat, *args, **kwds)
495 except FATAL_EXCEPTIONS:
496 raise
497
498 except MeasurementError as error:
499 self.log.getChild(plugin.name).debug(
500 "MeasurementError in %s.measureN on records %s-%s: %s",
501 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
502 for measRecord in measCat:
503 plugin.fail(measRecord, error)
504 except InvalidPsfError as error:
505 self.log.getChild(plugin.name).debug(
506 "InvalidPsfError in %s.measureN on records %s-%s: %s",
507 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
508 for measRecord in measCat:
509 measRecord.set(self.keyInvalidPsf, True)
510 plugin.fail(measRecord, error)
511 except Exception as error:
512 self.log.getChild(plugin.name).warning(
513 "Exception in %s.measureN on records %s-%s: %s",
514 plugin.name, measCat[0].getId(), measCat[-1].getId(), error)
515 for measRecord in measCat:
516 plugin.fail(measRecord)
517
518 @staticmethod
520 """Get a set of footprints from a catalog, keyed by id.
521
522 Parameters
523 ----------
524 catalog : `lsst.afw.table.SourceCatalog`
525 Catalog with `lsst.afw.detection.Footprint`s attached.
526
527 Returns
528 -------
529 footprints : `dict` [`int`: (`int`, `lsst.afw.detection.Footprint`)]
530 Dictionary of footprint, keyed by id number, with a tuple of
531 the parent id and footprint.
532 """
533 return {measRecord.getId(): (measRecord.getParent(), measRecord.getFootprint())
534 for measRecord in catalog}
Class for storing ordered metadata with comments.
__new__(cls, *args, ignoreSlotPluginChecks=False, **kwargs)
doMeasurementN(self, plugin, measCat, *args, **kwds)
doMeasurement(self, plugin, measRecord, *args, **kwds)