LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
catalogCalculation.py
Go to the documentation of this file.
1 # This file is part of meas_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 from collections import namedtuple
23 
24 import lsst.pipe.base
25 import lsst.pex.config
26 import lsst.daf.base
27 
28 from .pluginsBase import BasePlugin, BasePluginConfig
29 from .pluginRegistry import PluginRegistry, PluginMap
30 from . import FatalAlgorithmError, MeasurementError
31 
32 # Exceptions that the measurement tasks should always propagate up to their
33 # callers
34 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
35 
36 __all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig",
37  "CatalogCalculationTask")
38 
39 
41  """Default configuration class for catalog calcuation plugins.
42  """
43  pass
44 
45 
47  """Base class for catalog calculation plugins.
48 
49  Parameters
50  ----------
51  config : `CatalogCalculationPlugin.ConfigClass`
52  Plugin configuration.
53  name : `str`
54  The string the plugin was registered with.
55  schema : `lsst.afw.table.Schema`
56  The source schema, New fields should be added here to
57  hold output produced by this plugin.
58  metadata : `lsst.daf.base.PropertySet`
59  Plugin metadata that will be attached to the output catalog
60  """
61 
62  ConfigClass = CatalogCalculationPluginConfig # documentation inherited
63 
64  registry = PluginRegistry(CatalogCalculationPluginConfig)
65  """List of available plugins (`lsst.meas.base.PluginRegistry`).
66  """
67 
68  plugType = 'single'
69  """Does the plugin operate on a single source or the whole catalog (`str`)?
70 
71  If the plugin operates on a single source at a time, this should be set to
72  ``"single"``; if it expects the whoe catalog, to ``"multi"``. If the
73  plugin is of type ``"multi"``, the `fail` method must be implemented to
74  accept the whole catalog. If the plugin is of type ``"single"``, `fail`
75  should accept a single source record.
76  """
77 
78  def __init__(self, config, name, schema, metadata):
79  BasePlugin.__init__(self, config, name)
80 
81  @classmethod
83  r"""Used to set the relative order of plugin execution.
84 
85  The values returned by `getExecutionOrder` are compared across all
86  plugins, and smaller numbers run first.
87 
88  Notes
89  -----
90  `CatalogCalculationPlugin`\s must run with
91  `BasePlugin.DEFAULT_CATALOGCALCULATION` or higher.
92 
93  All plugins must implement this method with an appropriate run level
94  """
95  raise NotImplementedError()
96 
97  def calculate(self, cat, **kwargs):
98  """Perform the calculation specified by this plugin.
99 
100  This method can either be used to operate on a single catalog record
101  or a whole catalog, populating it with the output defined by this
102  plugin.
103 
104  Note that results may be added to catalog records as new columns, or
105  may result in changes to existing values.
106 
107  Parameters
108  ----------
109  cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord`
110  May either be a `~lsst.afw.table.SourceCatalog` or a single
111  `~lsst.afw.table.SourceRecord`, depending on the plugin type. Will
112  be updated in place to contain the results of plugin execution.
113  **kwargs
114  Any additional keyword arguments that may be passed to the plugin.
115  """
116  raise NotImplementedError()
117 
118 
119 class CCContext:
120  """Handle errors that are thrown by catalog calculation plugins.
121 
122  This is a context manager.
123 
124  Parameters
125  ----------
126  plugin : `CatalogCalculationPlugin`
127  The plugin that is to be run.
128  cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord`
129  May either be a `~lsst.afw.table.SourceCatalog` or a single
130  `~lsst.afw.table.SourceRecord`, depending on the plugin type.
131  log : `lsst.log.Log`
132  A logger. Generally, this should be the logger of the object in which
133  the context manager is being used.
134  """
135  def __init__(self, plugin, cat, log):
136  self.plugin = plugin
137  self.cat = cat
138  self.log = log
139 
140  def __enter__(self):
141  return
142 
143  def __exit__(self, exc_type, exc_value, traceback):
144  if exc_type is None:
145  return True
146  if exc_type in FATAL_EXCEPTIONS:
147  raise exc_value
148  elif exc_type is MeasurementError:
149  self.plugin.fail(self.cat, exc_value)
150  else:
151  self.log.warn("Error in {}.calculate: {}".format(self.plugin.name, exc_value))
152  return True
153 
154 
156  """Config class for the catalog calculation driver task.
157 
158  Specifies which plugins will execute when the `CatalogCalculationTask`
159  associated with this configuration is run.
160  """
161 
162  plugins = CatalogCalculationPlugin.registry.makeField(
163  multi=True,
164  default=["base_ClassificationExtendedness",
165  "base_FootprintArea"],
166  doc="Plugins to be run and their configuration")
167 
168 
170  """Run plugins which operate on a catalog of sources.
171 
172  This task facilitates running plugins which will operate on a source
173  catalog. These plugins may do things such as classifying an object based
174  on source record entries inserted during a measurement task.
175 
176  Parameters
177  ----------
178  plugMetaData : `lsst.daf.base.PropertyList` or `None`
179  Will be modified in-place to contain metadata about the plugins being
180  run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
181  created.
182  **kwargs
183  Additional arguments passed to the superclass constructor.
184 
185  Notes
186  -----
187  Plugins may either take an entire catalog to work on at a time, or work on
188  individual records.
189  """
190  ConfigClass = CatalogCalculationConfig
191  _DefaultName = "catalogCalculation"
192 
193  def __init__(self, schema, plugMetadata=None, **kwargs):
194  lsst.pipe.base.Task.__init__(self, **kwargs)
195  self.schema = schema
196  if plugMetadata is None:
197  plugMetadata = lsst.daf.base.PropertyList()
198  self.plugMetadata = plugMetadata
200 
201  self.initializePlugins()
202 
203  def initializePlugins(self):
204  """Initialize the plugins according to the configuration.
205  """
206 
207  pluginType = namedtuple('pluginType', 'single multi')
208  self.executionDict = {}
209  # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that
210  # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level,
211  # the plugins are sorted into either single record, or multi record groups to later be run
212  # appropriately
213  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
214  if executionOrder not in self.executionDict:
215  self.executionDict[executionOrder] = pluginType(single=[], multi=[])
216  if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION:
217  plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata)
218  self.plugins[name] = plug
219  if plug.plugType == 'single':
220  self.executionDict[executionOrder].single.append(plug)
221  elif plug.plugType == 'multi':
222  self.executionDict[executionOrder].multi.append(plug)
223  else:
224  errorTuple = (PluginClass, PluginClass.getExecutionOrder(),
225  BasePlugin.DEFAULT_CATALOGCALCULATION)
226  raise ValueError("{} has an execution order less than the minimum for an catalogCalculation "
227  "plugin. Value {} : Minimum {}".format(*errorTuple))
228 
229  @lsst.pipe.base.timeMethod
230  def run(self, measCat):
231  """The entry point for the catalog calculation task.
232 
233  Parameters
234  ----------
235  meascat : `lsst.afw.table.SourceCatalog`
236  Catalog for measurement.
237  """
238  self.callCompute(measCat)
239 
240  def callCompute(self, catalog):
241  """Run each of the plugins on the catalog.
242 
243  Parameters
244  ----------
245  catalog : `lsst.afw.table.SourceCatalog`
246  The catalog on which the plugins will operate.
247  """
248  for runlevel in sorted(self.executionDict):
249  # Run all of the plugins which take a whole catalog first
250  for plug in self.executionDict[runlevel].multi:
251  with CCContext(plug, catalog, self.log):
252  plug.calculate(catalog)
253  # Run all the plugins which take single catalog entries
254  for measRecord in catalog:
255  for plug in self.executionDict[runlevel].single:
256  with CCContext(plug, measRecord, self.log):
257  plug.calculate(measRecord)
lsst::log.log.logContinued.warn
def warn(fmt, *args)
Definition: logContinued.py:205
lsst::meas::base.catalogCalculation.CCContext.plugin
plugin
Definition: catalogCalculation.py:136
lsst::meas::base.catalogCalculation.CatalogCalculationPlugin.calculate
def calculate(self, cat, **kwargs)
Definition: catalogCalculation.py:97
lsst::meas::base.catalogCalculation.CatalogCalculationPlugin.__init__
def __init__(self, config, name, schema, metadata)
Definition: catalogCalculation.py:78
lsst::meas::base.catalogCalculation.CatalogCalculationTask.plugMetadata
plugMetadata
Definition: catalogCalculation.py:198
lsst::meas::base.catalogCalculation.CatalogCalculationPluginConfig
Definition: catalogCalculation.py:40
lsst::meas::base.catalogCalculation.CatalogCalculationPlugin
Definition: catalogCalculation.py:46
lsst::meas::base.catalogCalculation.CCContext.log
log
Definition: catalogCalculation.py:138
lsst::meas::base.catalogCalculation.CCContext.__exit__
def __exit__(self, exc_type, exc_value, traceback)
Definition: catalogCalculation.py:143
lsst::meas::base.catalogCalculation.CatalogCalculationTask.schema
schema
Definition: catalogCalculation.py:195
lsst::meas::base.catalogCalculation.CatalogCalculationTask.callCompute
def callCompute(self, catalog)
Definition: catalogCalculation.py:240
lsst::daf::base::PropertyList
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
lsst::meas::base.catalogCalculation.CCContext
Definition: catalogCalculation.py:119
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
lsst::meas::base.catalogCalculation.CatalogCalculationTask.plugins
plugins
Definition: catalogCalculation.py:199
lsst::meas::base.pluginRegistry.PluginMap
Definition: pluginRegistry.py:190
lsst::meas::base.pluginsBase.BasePluginConfig
Definition: pluginsBase.py:30
lsst::meas::base.pluginRegistry.PluginRegistry
Definition: pluginRegistry.py:76
lsst::meas::base.catalogCalculation.CatalogCalculationTask
Definition: catalogCalculation.py:169
lsst::meas::base.catalogCalculation.CCContext.__init__
def __init__(self, plugin, cat, log)
Definition: catalogCalculation.py:135
lsst.pex.config
Definition: __init__.py:1
lsst::meas::base.catalogCalculation.CatalogCalculationTask.run
def run(self, measCat)
Definition: catalogCalculation.py:230
lsst.pipe.base.task.Task.config
config
Definition: task.py:162
lsst::meas::base.catalogCalculation.CatalogCalculationPlugin.getExecutionOrder
def getExecutionOrder(cls)
Definition: catalogCalculation.py:82
lsst.pipe.base.task.Task.log
log
Definition: task.py:161
lsst::meas::base.catalogCalculation.CCContext.__enter__
def __enter__(self)
Definition: catalogCalculation.py:140
lsst::daf::base
Definition: Utils.h:47
lsst.pipe.base.task.Task
Definition: task.py:47
lsst::meas::base.catalogCalculation.CatalogCalculationTask.__init__
def __init__(self, schema, plugMetadata=None, **kwargs)
Definition: catalogCalculation.py:193
lsst::meas::base.catalogCalculation.CatalogCalculationTask.executionDict
executionDict
Definition: catalogCalculation.py:208
lsst.pex.config.config.Config
Definition: config.py:736
lsst::meas::base.catalogCalculation.CCContext.cat
cat
Definition: catalogCalculation.py:137
lsst::meas::base.catalogCalculation.CatalogCalculationConfig
Definition: catalogCalculation.py:155
lsst::meas::base.pluginsBase.BasePlugin
Definition: pluginsBase.py:42
lsst.pipe.base
Definition: __init__.py:1
lsst::meas::base.catalogCalculation.CatalogCalculationTask.initializePlugins
def initializePlugins(self)
Definition: catalogCalculation.py:203