22 """Module defining config classes for PipelineTask. 25 __all__ = [
"InputDatasetConfig",
"InputDatasetField",
26 "OutputDatasetConfig",
"OutputDatasetField",
27 "InitInputDatasetConfig",
"InitInputDatasetField",
28 "InitOutputDatasetConfig",
"InitOutputDatasetField",
29 "ResourceConfig",
"QuantumConfig",
"PipelineTaskConfig"]
34 from textwrap
import dedent, indent
49 PIPELINETASK_CONFIG_TEMPLATE_DICT = {}
52 def _makeDatasetField(name, dtype):
53 """ Function to make callables which produce ConfigField objects 55 This is factory function which produces factory functions. The factories 56 returned by this function are used to simplify the process of creating 57 `lsst.pex.config.ConfigField` which have dtypes derived from either 58 `_DatasetTypeConfig`, or `_GlobalDatasetTypeConfig`. These functions can 59 then be used in a mannor similar to other `~lsst.pex.config.ConfigField` 62 Below is a flow diagram to explain the use of this function visually, 63 where arrows indicate processing flow. 65 Make a ConfigField factory: 66 _makeDatasetField() -> return wrappedFunc -> assign to variable corresponding 69 Use a ConfigField factory: 70 name() -> factory() -> return pexConfig instance 74 FooField = _makeDatasetField("FooField", FooConfig) 75 fooFieldInstance = FooField("An example Foo ConfigField", 83 The name to use as the final output `~lsst.pex.config.Field` 85 dtype : Configuration Object 86 This is the python type to set as the dtype in the ConfigField 92 Python callable function which can be used to produce instances of 93 `~lsst.pex.config.ConfigField` of type dtype. 98 Possibly raises a TypeError if attempting to create a factory function 99 from an incompatible type 102 def factory(**kwargs):
103 """ This is the innermost function in the closure, and does the work 104 of actually producing the ConfigField 111 return pexConfig.ConfigField(doc=kwargs[
'doc'],
114 **{k: v
for k, v
in kwargs.items()
115 if k
not in (
'doc',
'check')}),
116 check=kwargs[
'check'])
121 if issubclass(dtype, _GlobalDatasetTypeConfig):
124 def wrappedFunc(*, doc, storageClass, check=None, name="", nameTemplate=''):
125 return factory(**{k: v
for k, v
in locals().
items()
if k !=
'factory'})
130 elif issubclass(dtype, _DatasetTypeConfig):
132 def wrappedFunc(*, doc, dimensions, storageClass, name="", scalar=False, check=None, nameTemplate='',
134 return factory(**{k: v
for k, v
in locals().
items()
if k !=
'factory'})
138 dimensions : iterable of `str` 139 Iterable of Dimensions for this `~lsst.daf.butler.DatasetType` 140 scalar : `bool`, optional 141 If set to True then only a single dataset is expected on input or 142 produced on output. In that case list of objects/DataIds will be 143 unpacked before calling task methods, returned data is expected 144 to contain single objects as well. 145 nameTemplate : `str`, optional 146 Template for the `name` field which is specified as a python formattable 147 string. The template is formatted during the configuration of a Config 148 class with a user defined string. Defaults to empty string, in which 149 case no formatting is done. 151 Indicates runQuantum will not load the data from the butler, and that 152 the task intends to do the loading itself. Defaults to False 157 extraFields =
", dimensions, scalar, nameTemplate, manualLoad" 161 raise TypeError(f
"Cannot create a factory for dtype {dtype}")
164 docstring = f
""" Factory function to create `~lsst.pex.config.Config` class instances 165 of `{dtype.__name__}` 167 This function servers as syntactic sugar for creating 168 `~lsst.pex.config.ConfigField` which are `{dtype.__name__}`. The naming of 169 this function violates the normal convention of a lowercase first letter 170 in the function name, as this function is intended to sit in the same 171 place as `~lsst.pex.config.ConfigField` classes, and consistency in 172 declaration syntax is important. 174 The input arguments for this class are a combination of the arguments for 175 `~lsst.pex.config.ConfigField` and `{dtype.__name__}`. The arguments 176 doc and check come from `~lsst.pex.config.ConfigField`, while name{extraFields} 177 and storageClass come from `{dtype.__name__}`. 182 Documentation string for the `{dtype.__name__}` 184 Name of the `~lsst.daf.butler.DatasetType` in the returned 185 `{dtype.__name__}`{indent(dedent(extraDoc), " " * 4)} 187 Name of the `~lsst.daf.butler.StorageClass` in the `{dtype.__name__}` 189 A callable to be called with the field value that returns 190 False if the value is invalid. 194 result : `~lsst.pex.config.ConfigField` 195 Instance of a `~lsst.pex.config.ConfigField` with `InputDatasetConfig` as a dtype 198 wrappedFunc.__name__ = name
201 wrappedFunc.__doc__ = dedent(docstring)
206 """Configuration class which defines PipelineTask quanta dimensions. 208 In addition to a list of dataUnit names this also includes optional list of 209 SQL statements to be executed against Registry database. Exact meaning and 210 format of SQL will be determined at later point. 212 dimensions = pexConfig.ListField(dtype=str,
213 doc=
"list of Dimensions which define quantum")
214 sql = pexConfig.ListField(dtype=str,
215 doc=
"sequence of SQL statements",
220 """Intermediate base class for dataset type configuration in PipelineTask. 222 name = pexConfig.Field(dtype=str,
223 doc=
"name of the DatasetType")
224 storageClass = pexConfig.Field(dtype=str,
225 doc=
"name of the StorageClass")
226 nameTemplate = pexConfig.Field(dtype=str,
229 doc=(
"Templated name of string, used to set name " 230 "field according to a shared substring when " 231 "`~PipelineTaskConfig.formatTemplateNames` " 236 """Configuration class which defines dataset type used by PipelineTask. 238 Consists of DatasetType name, list of Dimension names and StorageCass name. 239 PipelineTasks typically define one or more input and output datasets. This 240 class should not be used directly, instead one of `InputDatasetConfig` or 241 `OutputDatasetConfig` should be used in PipelineTask config. 243 dimensions = pexConfig.ListField(dtype=str,
244 doc=
"list of Dimensions for this DatasetType")
245 scalar = pexConfig.Field(dtype=bool,
248 doc=(
"If set to True then only a single dataset is expected " 249 "on input or produced on output. In that case list of " 250 "objects/DataIds will be unpacked before calling task " 251 "methods, returned data is expected to contain single " 253 manualLoad = pexConfig.Field(dtype=bool,
256 doc=(
"If this is set to True, the class intends to load " 257 "the data associated with this Configurable Field " 258 "manually, and runQuantum should not load it. Should " 259 "not be set by configuration override"))
266 class OutputDatasetConfig(_DatasetTypeConfig):
271 """Configuration class which defines dataset types used in PipelineTask 274 Consists of DatasetType name and StorageCass name, with a read-only 275 ``dimensions`` property that returns an empty tuple, enforcing the 276 constraint that datasets used in initialization are not associated with 277 any Dimensions. This class should not be used directly, instead one of 278 `InitInputDatasetConfig` or `InitOutputDatasetConfig` should be used in 283 """Dimensions associated with this DatasetType (always empty).""" 291 class InitOutputDatasetConfig(_GlobalDatasetTypeConfig):
296 """Configuration for resource requirements. 298 This configuration class will be used by some activators to estimate 299 resource use by pipeline. Additionally some tasks could use it to adjust 300 their resource use (e.g. reduce the number of threads). 302 For some resources their limit can be estimated by corresponding task, 303 in that case task could set the field value. For many fields defined in 304 this class their associated resource used by a task will depend on the 305 size of the data and is not known in advance. For these resources their 306 value will be configured through overrides based on some external 309 minMemoryMB = pexConfig.Field(dtype=int, default=
None, optional=
True,
310 doc=
"Minimal memory needed by task, can be None if estimate is unknown.")
311 minNumCores = pexConfig.Field(dtype=int, default=1,
312 doc=
"Minimal number of cores needed by task.")
316 """Base class for all PipelineTask configurations. 318 This class defines fields that must be defined for every 319 `~lsst.pipe.base.PipelineTask`. It will be used as a base class for all 320 `~lsst.pipe.base.PipelineTask` configurations instead of 323 quantum = pexConfig.ConfigField(dtype=QuantumConfig,
324 doc=
"configuration for PipelineTask quantum")
329 storedParamsDict = PIPELINETASK_CONFIG_TEMPLATE_DICT.setdefault(
id(self), {})
330 storedParamsDict.update(templateParamsDict)
331 for key, value
in self.items():
332 if isinstance(value, _BaseDatasetTypeConfig)
and value.nameTemplate !=
'':
333 value.name = value.nameTemplate.format(**storedParamsDict)
336 InputDatasetField = _makeDatasetField(
"InputDatasetField", InputDatasetConfig)
337 OutputDatasetField = _makeDatasetField(
"OutputDatasetField", OutputDatasetConfig)
338 InitInputDatasetField = _makeDatasetField(
"InitInputDatasetField", InitInputDatasetConfig)
339 InitOutputDatasetField = _makeDatasetField(
"InitOutputDatasetField", InitOutputDatasetConfig)
def formatTemplateNames(self, templateParamsDict)
std::vector< SchemaItem< Flag > > * items