LSST Applications g2079a07aa2+86d27d4dc4,g2305ad1205+a659bff248,g2bbee38e9b+3c60f8fe34,g337abbeb29+3c60f8fe34,g33d1c0ed96+3c60f8fe34,g3502564af9+d77d6d1350,g3a166c0a6a+3c60f8fe34,g487adcacf7+25d9892218,g4be5004598+d77d6d1350,g50ff169b8f+96c6868917,g52b1c1532d+585e252eca,g591dd9f2cf+4d81263f9a,g5cd07815a0+980d2b1c3b,g607f77f49a+d77d6d1350,g858d7b2824+d77d6d1350,g88963caddf+83e433e629,g99cad8db69+a4d3c48eeb,g9ddcbc5298+9a081db1e4,ga1e77700b3+bcf1af89ad,ga57fefb910+9a39d7b2d7,gae0086650b+585e252eca,gb065fddaf9+4f9fd82a2c,gb0e22166c9+60f28cb32d,gb363559e06+d84b1d3d07,gb3b7280ab2+4563d032e1,gb4b16eec92+babe958938,gba4ed39666+c2a2e4ac27,gbb8dafda3b+ed6854b564,gc120e1dc64+b72d212f87,gc28159a63d+3c60f8fe34,gc3e9b769f7+921dbcd359,gcf0d15dbbd+9a39d7b2d7,gdaeeff99f8+f9a426f77a,gddc38dedce+585e252eca,ge79ae78c31+3c60f8fe34,w.2024.21
LSST Data Management Base Package
Loading...
Searching...
No Matches
config.py
Go to the documentation of this file.
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
28
29__all__ = (
30 "Config",
31 "ConfigMeta",
32 "Field",
33 "FieldValidationError",
34 "UnexpectedProxyUsageError",
35 "FieldTypeVar",
36)
37
38import copy
39import importlib
40import io
41import math
42import os
43import re
44import shutil
45import sys
46import tempfile
47import warnings
48from collections.abc import Mapping
49from typing import Any, ForwardRef, Generic, TypeVar, cast, overload
50
51try:
52 from types import GenericAlias
53except ImportError:
54 # cover python 3.8 usage
55 GenericAlias = type(Mapping[int, int])
56
57# if YAML is not available that's fine and we simply don't register
58# the yaml representer since we know it won't be used.
59try:
60 import yaml
61except ImportError:
62 yaml = None
63
64from .callStack import getCallStack, getStackFrame
65from .comparison import compareConfigs, compareScalars, getComparisonName
66
67if yaml:
68 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
69
70 try:
71 # CLoader is not always available
72 from yaml import CLoader
73
74 YamlLoaders += (CLoader,)
75 except ImportError:
76 pass
77else:
78 YamlLoaders = ()
79 doImport = None
80
81
82class _PexConfigGenericAlias(GenericAlias):
83 """A Subclass of python's GenericAlias used in defining and instantiating
84 Generics.
85
86 This class differs from `types.GenericAlias` in that it calls a method
87 named _parseTypingArgs defined on Fields. This method gives Field and its
88 subclasses an opportunity to transform type parameters into class key word
89 arguments. Code authors do not need to implement any returns of this object
90 directly, and instead only need implement _parseTypingArgs, if a Field
91 subclass differs from the base class implementation.
92
93 This class is intended to be an implementation detail, returned from a
94 Field's `__class_getitem__` method.
95 """
96
97 def __call__(self, *args: Any, **kwds: Any) -> Any:
98 origin_kwargs = self._parseTypingArgs(self.__args__, kwds)
99 return super().__call__(*args, **{**kwds, **origin_kwargs})
100
101
102FieldTypeVar = TypeVar("FieldTypeVar")
103
104
106 """Exception raised when a proxy class is used in a context that suggests
107 it should have already been converted to the thing it proxies.
108 """
109
110
111def _joinNamePath(prefix=None, name=None, index=None):
112 """Generate nested configuration names."""
113 if not prefix and not name:
114 raise ValueError("Invalid name: cannot be None")
115 elif not name:
116 name = prefix
117 elif prefix and name:
118 name = prefix + "." + name
119
120 if index is not None:
121 return f"{name}[{index!r}]"
122 else:
123 return name
124
125
126def _autocast(x, dtype):
127 """Cast a value to a type, if appropriate.
128
129 Parameters
130 ----------
131 x : object
132 A value.
133 dtype : tpye
134 Data type, such as `float`, `int`, or `str`.
135
136 Returns
137 -------
138 values : object
139 If appropriate, the returned value is ``x`` cast to the given type
140 ``dtype``. If the cast cannot be performed the original value of
141 ``x`` is returned.
142 """
143 if dtype == float and isinstance(x, int):
144 return float(x)
145 return x
146
147
148def _typeStr(x):
149 """Generate a fully-qualified type name.
150
151 Returns
152 -------
153 `str`
154 Fully-qualified type name.
155
156 Notes
157 -----
158 This function is used primarily for writing config files to be executed
159 later upon with the 'load' function.
160 """
161 if hasattr(x, "__module__") and hasattr(x, "__name__"):
162 xtype = x
163 else:
164 xtype = type(x)
165 if xtype.__module__ == "builtins":
166 return xtype.__name__
167 else:
168 return f"{xtype.__module__}.{xtype.__name__}"
169
170
171if yaml:
172
173 def _yaml_config_representer(dumper, data):
174 """Represent a Config object in a form suitable for YAML.
175
176 Stores the serialized stream as a scalar block string.
177 """
178 stream = io.StringIO()
179 data.saveToStream(stream)
180 config_py = stream.getvalue()
181
182 # Strip multiple newlines from the end of the config
183 # This simplifies the YAML to use | and not |+
184 config_py = config_py.rstrip() + "\n"
185
186 # Trailing spaces force pyyaml to use non-block form.
187 # Remove the trailing spaces so it has no choice
188 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
189
190 # Store the Python as a simple scalar
191 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
192
193 def _yaml_config_constructor(loader, node):
194 """Construct a config from YAML."""
195 config_py = loader.construct_scalar(node)
196 return Config._fromPython(config_py)
197
198 # Register a generic constructor for Config and all subclasses
199 # Need to register for all the loaders we would like to use
200 for loader in YamlLoaders:
201 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
202
203
204class ConfigMeta(type):
205 """A metaclass for `lsst.pex.config.Config`.
206
207 Parameters
208 ----------
209 name : `str`
210 Name to use for class.
211 bases : `~collections.abc.Iterable`
212 Base classes.
213 dict_ : `dict`
214 Additional parameters.
215
216 Notes
217 -----
218 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
219 class attributes as a class attribute called ``_fields``, and adds
220 the name of each field as an instance variable of the field itself (so you
221 don't have to pass the name of the field to the field constructor).
222 """
223
224 def __init__(cls, name, bases, dict_):
225 type.__init__(cls, name, bases, dict_)
226 cls._fields = {}
227 cls._source = getStackFrame()
228
229 def getFields(classtype):
230 fields = {}
231 bases = list(classtype.__bases__)
232 bases.reverse()
233 for b in bases:
234 fields.update(getFields(b))
235
236 for k, v in classtype.__dict__.items():
237 if isinstance(v, Field):
238 fields[k] = v
239 return fields
240
241 fields = getFields(cls)
242 for k, v in fields.items():
243 setattr(cls, k, copy.deepcopy(v))
244
245 def __setattr__(cls, name, value):
246 if isinstance(value, Field):
247 value.name = name
248 cls._fields[name] = value
249 type.__setattr__(cls, name, value)
250
251
252class FieldValidationError(ValueError):
253 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
254 particular ``~lsst.pex.config.Config``.
255
256 Parameters
257 ----------
258 field : `lsst.pex.config.Field`
259 The field that was not valid.
260 config : `lsst.pex.config.Config`
261 The config containing the invalid field.
262 msg : `str`
263 Text describing why the field was not valid.
264 """
265
266 def __init__(self, field, config, msg):
267 self.fieldType = type(field)
268 """Type of the `~lsst.pex.config.Field` that incurred the error.
269 """
270
271 self.fieldName = field.name
272 """Name of the `~lsst.pex.config.Field` instance that incurred the
273 error (`str`).
274
275 See also
276 --------
277 lsst.pex.config.Field.name
278 """
279
280 self.fullname = _joinNamePath(config._name, field.name)
281 """Fully-qualified name of the `~lsst.pex.config.Field` instance
282 (`str`).
283 """
284
285 self.history = config.history.setdefault(field.name, [])
286 """Full history of all changes to the `~lsst.pex.config.Field`
287 instance.
288 """
289
290 self.fieldSource = field.source
291 """File and line number of the `~lsst.pex.config.Field` definition.
292 """
293
294 self.configSource = config._source
295 error = (
296 "%s '%s' failed validation: %s\n"
297 "For more information see the Field definition at:\n%s"
298 " and the Config definition at:\n%s"
299 % (
300 self.fieldType.__name__,
301 self.fullname,
302 msg,
303 self.fieldSource.format(),
304 self.configSource.format(),
305 )
306 )
307 super().__init__(error)
308
309
310class Field(Generic[FieldTypeVar]):
311 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
312 `complex`, `bool`, and `str` data types.
313
314 Parameters
315 ----------
316 doc : `str`
317 A description of the field for users.
318 dtype : type, optional
319 The field's data type. ``Field`` only supports basic data types:
320 `int`, `float`, `complex`, `bool`, and `str`. See
321 `Field.supportedTypes`. Optional if supplied as a typing argument to
322 the class.
323 default : object, optional
324 The field's default value.
325 check : callable, optional
326 A callable that is called with the field's value. This callable should
327 return `False` if the value is invalid. More complex inter-field
328 validation can be written as part of the
329 `lsst.pex.config.Config.validate` method.
330 optional : `bool`, optional
331 This sets whether the field is considered optional, and therefore
332 doesn't need to be set by the user. When `False`,
333 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
334 deprecated : None or `str`, optional
335 A description of why this Field is deprecated, including removal date.
336 If not None, the string is appended to the docstring for this Field.
337
338 Raises
339 ------
340 ValueError
341 Raised when the ``dtype`` parameter is not one of the supported types
342 (see `Field.supportedTypes`).
343
344 See Also
345 --------
346 ChoiceField
347 ConfigChoiceField
348 ConfigDictField
349 ConfigField
350 ConfigurableField
351 DictField
352 ListField
353 RangeField
354 RegistryField
355
356 Notes
357 -----
358 ``Field`` instances (including those of any subclass of ``Field``) are used
359 as class attributes of `~lsst.pex.config.Config` subclasses (see the
360 example, below). ``Field`` attributes work like the `property` attributes
361 of classes that implement custom setters and getters. `Field` attributes
362 belong to the class, but operate on the instance. Formally speaking,
363 `Field` attributes are `descriptors
364 <https://docs.python.org/3/howto/descriptor.html>`_.
365
366 When you access a `Field` attribute on a `Config` instance, you don't
367 get the `Field` instance itself. Instead, you get the value of that field,
368 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
369 container type (like a `lsst.pex.config.List`) depending on the field's
370 type. See the example, below.
371
372 Fields can be annotated with a type similar to other python classes (python
373 specification `here <https://peps.python.org/pep-0484/#generics>`_ ).
374 See the name field in the Config example below for an example of this.
375 Unlike most other uses in python, this has an effect at type checking *and*
376 runtime. If the type is specified with a class annotation, it will be used
377 as the value of the ``dtype`` in the ``Field`` and there is no need to
378 specify it as an argument during instantiation.
379
380 There are Some notes on dtype through type annotation syntax. Type
381 annotation syntax supports supplying the argument as a string of a type
382 name. i.e. "float", but this cannot be used to resolve circular references.
383 Type annotation syntax can be used on an identifier in addition to Class
384 assignment i.e. ``variable: Field[str] = Config.someField`` vs
385 ``someField = Field[str](doc="some doc"). However, this syntax is only
386 useful for annotating the type of the identifier (i.e. variable in previous
387 example) and does nothing for assigning the dtype of the ``Field``.
388
389 Examples
390 --------
391 Instances of ``Field`` should be used as class attributes of
392 `lsst.pex.config.Config` subclasses:
393
394 >>> from lsst.pex.config import Config, Field
395 >>> class Example(Config):
396 ... myInt = Field("An integer field.", int, default=0)
397 ... name = Field[str](doc="A string Field")
398 ...
399 >>> print(config.myInt)
400 0
401 >>> config.myInt = 5
402 >>> print(config.myInt)
403 5
404 """
405
406 name: str
407 """Identifier (variable name) used to refer to a Field within a Config
408 Class.
409 """
410
411 supportedTypes = {str, bool, float, int, complex}
412 """Supported data types for field values (`set` of types).
413 """
414
415 @staticmethod
417 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
418 ) -> Mapping[str, Any]:
419 """Parse type annotations into keyword constructor arguments.
420
421 This is a special private method that interprets type arguments (i.e.
422 Field[str]) into keyword arguments to be passed on to the constructor.
423
424 Subclasses of Field can implement this method to customize how they
425 handle turning type parameters into keyword arguments (see DictField
426 for an example)
427
428 Parameters
429 ----------
430 params : `tuple` of `type` or `tuple` of str
431 Parameters passed to the type annotation. These will either be
432 types or strings. Strings are to interpreted as forward references
433 and will be treated as such.
434 kwds : `MutableMapping` with keys of `str` and values of `Any`
435 These are the user supplied keywords that are to be passed to the
436 Field constructor.
437
438 Returns
439 -------
440 kwds : `MutableMapping` with keys of `str` and values of `Any`
441 The mapping of keywords that will be passed onto the constructor
442 of the Field. Should be filled in with any information gleaned
443 from the input parameters.
444
445 Raises
446 ------
447 ValueError
448 Raised if params is of incorrect length.
449 Raised if a forward reference could not be resolved
450 Raised if there is a conflict between params and values in kwds
451 """
452 if len(params) > 1:
453 raise ValueError("Only single type parameters are supported")
454 unpackedParams = params[0]
455 if isinstance(unpackedParams, str):
456 _typ = ForwardRef(unpackedParams)
457 # type ignore below because typeshed seems to be wrong. It
458 # indicates there are only 2 args, as it was in python 3.8, but
459 # 3.9+ takes 3 args. Attempt in old style and new style to
460 # work with both.
461 try:
462 result = _typ._evaluate(globals(), locals(), set()) # type: ignore
463 except TypeError:
464 # python 3.8 path
465 result = _typ._evaluate(globals(), locals())
466 if result is None:
467 raise ValueError("Could not deduce type from input")
468 unpackedParams = cast(type, result)
469 if "dtype" in kwds and kwds["dtype"] != unpackedParams:
470 raise ValueError("Conflicting definition for dtype")
471 elif "dtype" not in kwds:
472 kwds = {**kwds, **{"dtype": unpackedParams}}
473 return kwds
474
475 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef):
476 return _PexConfigGenericAlias(cls, params)
477
478 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None):
479 if dtype is None:
480 raise ValueError(
481 "dtype must either be supplied as an argument or as a type argument to the class"
482 )
483 if dtype not in self.supportedTypes:
484 raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
485
486 source = getStackFrame()
487 self._setup(
488 doc=doc,
489 dtype=dtype,
490 default=default,
491 check=check,
492 optional=optional,
493 source=source,
494 deprecated=deprecated,
495 )
496
497 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
498 """Set attributes, usually during initialization."""
499 self.dtype = dtype
500 """Data type for the field.
501 """
502
503 if not doc:
504 raise ValueError("Docstring is empty.")
505
506 # append the deprecation message to the docstring.
507 if deprecated is not None:
508 doc = f"{doc} Deprecated: {deprecated}"
509 self.doc = doc
510 """A description of the field (`str`).
511 """
512
513 self.deprecated = deprecated
514 """If not None, a description of why this field is deprecated (`str`).
515 """
516
517 self.__doc__ = f"{doc} (`{dtype.__name__}`"
518 if optional or default is not None:
519 self.__doc__ += f", default ``{default!r}``"
520 self.__doc__ += ")"
521
522 self.default = default
523 """Default value for this field.
524 """
525
526 self.check = check
527 """A user-defined function that validates the value of the field.
528 """
529
530 self.optional = optional
531 """Flag that determines if the field is required to be set (`bool`).
532
533 When `False`, `lsst.pex.config.Config.validate` will fail if the
534 field's value is `None`.
535 """
536
537 self.source = source
538 """The stack frame where this field is defined (`list` of
539 `~lsst.pex.config.callStack.StackFrame`).
540 """
541
542 def rename(self, instance):
543 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
544 only).
545
546 Parameters
547 ----------
548 instance : `lsst.pex.config.Config`
549 The config instance that contains this field.
550
551 Notes
552 -----
553 This method is invoked by the `lsst.pex.config.Config` object that
554 contains this field and should not be called directly.
555
556 Renaming is only relevant for `~lsst.pex.config.Field` instances that
557 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should
558 rename each subconfig with the full field name as generated by
559 `lsst.pex.config.config._joinNamePath`.
560 """
561 pass
562
563 def validate(self, instance):
564 """Validate the field (for internal use only).
565
566 Parameters
567 ----------
568 instance : `lsst.pex.config.Config`
569 The config instance that contains this field.
570
571 Raises
572 ------
573 lsst.pex.config.FieldValidationError
574 Raised if verification fails.
575
576 Notes
577 -----
578 This method provides basic validation:
579
580 - Ensures that the value is not `None` if the field is not optional.
581 - Ensures type correctness.
582 - Ensures that the user-provided ``check`` function is valid.
583
584 Most `~lsst.pex.config.Field` subclasses should call
585 `lsst.pex.config.Field.validate` if they re-implement
586 `~lsst.pex.config.Field.validate`.
587 """
588 value = self.__get__(instance)
589 if not self.optional and value is None:
590 raise FieldValidationError(self, instance, "Required value cannot be None")
591
592 def freeze(self, instance):
593 """Make this field read-only (for internal use only).
594
595 Parameters
596 ----------
597 instance : `lsst.pex.config.Config`
598 The config instance that contains this field.
599
600 Notes
601 -----
602 Freezing is only relevant for fields that hold subconfigs. Fields which
603 hold subconfigs should freeze each subconfig.
604
605 **Subclasses should implement this method.**
606 """
607 pass
608
609 def _validateValue(self, value):
610 """Validate a value.
611
612 Parameters
613 ----------
614 value : object
615 The value being validated.
616
617 Raises
618 ------
619 TypeError
620 Raised if the value's type is incompatible with the field's
621 ``dtype``.
622 ValueError
623 Raised if the value is rejected by the ``check`` method.
624 """
625 if value is None:
626 return
627
628 if not isinstance(value, self.dtype):
629 msg = "Value {} is of incorrect type {}. Expected type {}".format(
630 value,
631 _typeStr(value),
632 _typeStr(self.dtype),
633 )
634 raise TypeError(msg)
635 if self.check is not None and not self.check(value):
636 msg = "Value %s is not a valid value" % str(value)
637 raise ValueError(msg)
638
639 def _collectImports(self, instance, imports):
640 """Call the _collectImports method on all config
641 objects the field may own, and union them with the supplied imports
642 set.
643
644 Parameters
645 ----------
646 instance : instance or subclass of `lsst.pex.config.Config`
647 A config object that has this field defined on it
648 imports : `set`
649 Set of python modules that need imported after persistence
650 """
651 pass
652
653 def save(self, outfile, instance):
654 """Save this field to a file (for internal use only).
655
656 Parameters
657 ----------
658 outfile : file-like object
659 A writeable field handle.
660 instance : `~lsst.pex.config.Config`
661 The `~lsst.pex.config.Config` instance that contains this field.
662
663 Notes
664 -----
665 This method is invoked by the `~lsst.pex.config.Config` object that
666 contains this field and should not be called directly.
667
668 The output consists of the documentation string
669 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
670 line is formatted as an assignment: ``{fullname}={value}``.
671
672 This output can be executed with Python.
673 """
674 value = self.__get__(instance)
675 fullname = _joinNamePath(instance._name, self.name)
676
677 if self.deprecated and value == self.default:
678 return
679
680 # write full documentation string as comment lines
681 # (i.e. first character is #)
682 doc = "# " + str(self.doc).replace("\n", "\n# ")
683 if isinstance(value, float) and not math.isfinite(value):
684 # non-finite numbers need special care
685 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n")
686 else:
687 outfile.write(f"{doc}\n{fullname}={value!r}\n\n")
688
689 def toDict(self, instance):
690 """Convert the field value so that it can be set as the value of an
691 item in a `dict` (for internal use only).
692
693 Parameters
694 ----------
695 instance : `~lsst.pex.config.Config`
696 The `~lsst.pex.config.Config` that contains this field.
697
698 Returns
699 -------
700 value : object
701 The field's value. See *Notes*.
702
703 Notes
704 -----
705 This method invoked by the owning `~lsst.pex.config.Config` object and
706 should not be called directly.
707
708 Simple values are passed through. Complex data structures must be
709 manipulated. For example, a `~lsst.pex.config.Field` holding a
710 subconfig should, instead of the subconfig object, return a `dict`
711 where the keys are the field names in the subconfig, and the values are
712 the field values in the subconfig.
713 """
714 return self.__get____get____get__(instance)
715
716 @overload
718 self, instance: None, owner: Any = None, at: Any = None, label: str = "default"
719 ) -> Field[FieldTypeVar]:
720 ...
721
722 @overload
724 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default"
725 ) -> FieldTypeVar:
726 ...
727
728 def __get__(self, instance, owner=None, at=None, label="default"):
729 """Define how attribute access should occur on the Config instance
730 This is invoked by the owning config object and should not be called
731 directly.
732
733 When the field attribute is accessed on a Config class object, it
734 returns the field object itself in order to allow inspection of
735 Config classes.
736
737 When the field attribute is access on a config instance, the actual
738 value described by the field (and held by the Config instance) is
739 returned.
740 """
741 if instance is None:
742 return self
743 else:
744 # try statements are almost free in python if they succeed
745 try:
746 return instance._storage[self.namename]
747 except AttributeError:
748 if not isinstance(instance, Config):
749 return self
750 else:
751 raise AttributeError(
752 f"Config {instance} is missing _storage attribute, likely incorrectly initialized"
753 )
754
756 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment"
757 ) -> None:
758 """Set an attribute on the config instance.
759
760 Parameters
761 ----------
762 instance : `lsst.pex.config.Config`
763 The config instance that contains this field.
764 value : obj
765 Value to set on this field.
766 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\
767 optional
768 The call stack (created by
769 `lsst.pex.config.callStack.getCallStack`).
770 label : `str`, optional
771 Event label for the history.
772
773 Notes
774 -----
775 This method is invoked by the owning `lsst.pex.config.Config` object
776 and should not be called directly.
777
778 Derived `~lsst.pex.config.Field` classes may need to override the
779 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
780 should follow the following rules:
781
782 - Do not allow modification of frozen configs.
783 - Validate the new value **before** modifying the field. Except if the
784 new value is `None`. `None` is special and no attempt should be made
785 to validate it until `lsst.pex.config.Config.validate` is called.
786 - Do not modify the `~lsst.pex.config.Config` instance to contain
787 invalid values.
788 - If the field is modified, update the history of the
789 `lsst.pex.config.field.Field` to reflect the changes.
790
791 In order to decrease the need to implement this method in derived
792 `~lsst.pex.config.Field` types, value validation is performed in the
793 `lsst.pex.config.Field._validateValue`. If only the validation step
794 differs in the derived `~lsst.pex.config.Field`, it is simpler to
795 implement `lsst.pex.config.Field._validateValue` than to reimplement
796 ``__set__``. More complicated behavior, however, may require
797 reimplementation.
798 """
799 if instance._frozen:
800 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
801
802 history = instance._history.setdefault(self.namename, [])
803 if value is not None:
804 value = _autocast(value, self.dtype)
805 try:
806 self._validateValue(value)
807 except BaseException as e:
808 raise FieldValidationError(self, instance, str(e))
809
810 instance._storage[self.namename] = value
811 if at is None:
812 at = getCallStack()
813 history.append((value, at, label))
814
815 def __delete__(self, instance, at=None, label="deletion"):
816 """Delete an attribute from a `lsst.pex.config.Config` instance.
817
818 Parameters
819 ----------
820 instance : `lsst.pex.config.Config`
821 The config instance that contains this field.
822 at : `list` of `lsst.pex.config.callStack.StackFrame`
823 The call stack (created by
824 `lsst.pex.config.callStack.getCallStack`).
825 label : `str`, optional
826 Event label for the history.
827
828 Notes
829 -----
830 This is invoked by the owning `~lsst.pex.config.Config` object and
831 should not be called directly.
832 """
833 if at is None:
834 at = getCallStack()
835 self.__set__(instance, None, at=at, label=label)
836
837 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
838 """Compare a field (named `Field.name`) in two
839 `~lsst.pex.config.Config` instances for equality.
840
841 Parameters
842 ----------
843 instance1 : `lsst.pex.config.Config`
844 Left-hand side `Config` instance to compare.
845 instance2 : `lsst.pex.config.Config`
846 Right-hand side `Config` instance to compare.
847 shortcut : `bool`, optional
848 **Unused.**
849 rtol : `float`, optional
850 Relative tolerance for floating point comparisons.
851 atol : `float`, optional
852 Absolute tolerance for floating point comparisons.
853 output : callable, optional
854 A callable that takes a string, used (possibly repeatedly) to
855 report inequalities.
856
857 Notes
858 -----
859 This method must be overridden by more complex `Field` subclasses.
860
861 See Also
862 --------
863 lsst.pex.config.compareScalars
864 """
865 v1 = getattr(instance1, self.namename)
866 v2 = getattr(instance2, self.namename)
867 name = getComparisonName(
868 _joinNamePath(instance1._name, self.namename), _joinNamePath(instance2._name, self.namename)
869 )
870 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
871
872
874 """Importer (for `sys.meta_path`) that records which modules are being
875 imported.
876
877 *This class does not do any importing itself.*
878
879 Examples
880 --------
881 Use this class as a context manager to ensure it is properly uninstalled
882 when done:
883
884 >>> with RecordingImporter() as importer:
885 ... # import stuff
886 ... import numpy as np
887 ... print("Imported: " + importer.getModules())
888 """
889
890 def __init__(self):
891 self._modules = set()
892
893 def __enter__(self):
894 self.origMetaPath = sys.meta_path
895 sys.meta_path = [self] + sys.meta_path # type: ignore
896 return self
897
898 def __exit__(self, *args):
899 self.uninstall()
900 return False # Don't suppress exceptions
901
902 def uninstall(self):
903 """Uninstall the importer."""
904 sys.meta_path = self.origMetaPath
905
906 def find_spec(self, fullname, path, target=None):
907 """Find a module.
908
909 Called as part of the ``import`` chain of events.
910
911 Parameters
912 ----------
913 fullname : `str`
914 Name of module.
915 path : `list` [`str`]
916 Search path. Unused.
917 target : `~typing.Any`, optional
918 Unused.
919 """
920 self._modules.add(fullname)
921 # Return None because we don't do any importing.
922 return None
923
924 def getModules(self):
925 """Get the set of modules that were imported.
926
927 Returns
928 -------
929 modules : `set` of `str`
930 Set of imported module names.
931 """
932 return self._modules
933
934
935# type ignore because type checker thinks ConfigMeta is Generic when it is not
936class Config(metaclass=ConfigMeta): # type: ignore
937 """Base class for configuration (*config*) objects.
938
939 Notes
940 -----
941 A ``Config`` object will usually have several `~lsst.pex.config.Field`
942 instances as class attributes. These are used to define most of the base
943 class behavior.
944
945 ``Config`` implements a mapping API that provides many `dict`-like methods,
946 such as `keys`, `values`, and `items`. ``Config`` instances also support
947 the ``in`` operator to test if a field is in the config. Unlike a `dict`,
948 ``Config`` classes are not subscriptable. Instead, access individual
949 fields as attributes of the configuration instance.
950
951 Examples
952 --------
953 Config classes are subclasses of ``Config`` that have
954 `~lsst.pex.config.Field` instances (or instances of
955 `~lsst.pex.config.Field` subclasses) as class attributes:
956
957 >>> from lsst.pex.config import Config, Field, ListField
958 >>> class DemoConfig(Config):
959 ... intField = Field(doc="An integer field", dtype=int, default=42)
960 ... listField = ListField(doc="List of favorite beverages.", dtype=str,
961 ... default=['coffee', 'green tea', 'water'])
962 ...
963 >>> config = DemoConfig()
964
965 Configs support many `dict`-like APIs:
966
967 >>> config.keys()
968 ['intField', 'listField']
969 >>> 'intField' in config
970 True
971
972 Individual fields can be accessed as attributes of the configuration:
973
974 >>> config.intField
975 42
976 >>> config.listField.append('earl grey tea')
977 >>> print(config.listField)
978 ['coffee', 'green tea', 'water', 'earl grey tea']
979 """
980
981 _storage: dict[str, Any]
982 _fields: dict[str, Field]
983 _history: dict[str, list[Any]]
984 _imports: set[Any]
985
986 def __iter__(self):
987 """Iterate over fields."""
988 return self._fields_fields.__iter__()
989
990 def keys(self):
991 """Get field names.
992
993 Returns
994 -------
995 names : `~collections.abc.KeysView`
996 List of `lsst.pex.config.Field` names.
997 """
998 return self._storage.keys()
999
1000 def values(self):
1001 """Get field values.
1002
1003 Returns
1004 -------
1005 values : `~collections.abc.ValuesView`
1006 Iterator of field values.
1007 """
1008 return self._storage.values()
1009
1010 def items(self):
1011 """Get configurations as ``(field name, field value)`` pairs.
1012
1013 Returns
1014 -------
1015 items : `~collections.abc.ItemsView`
1016 Iterator of tuples for each configuration. Tuple items are:
1017
1018 0. Field name.
1019 1. Field value.
1020 """
1021 return self._storage.items()
1022
1023 def __contains__(self, name):
1024 """Return `True` if the specified field exists in this config.
1025
1026 Parameters
1027 ----------
1028 name : `str`
1029 Field name to test for.
1030
1031 Returns
1032 -------
1033 in : `bool`
1034 `True` if the specified field exists in the config.
1035 """
1036 return self._storage.__contains__(name)
1037
1038 def __new__(cls, *args, **kw):
1039 """Allocate a new `lsst.pex.config.Config` object.
1040
1041 In order to ensure that all Config object are always in a proper state
1042 when handed to users or to derived `~lsst.pex.config.Config` classes,
1043 some attributes are handled at allocation time rather than at
1044 initialization.
1045
1046 This ensures that even if a derived `~lsst.pex.config.Config` class
1047 implements ``__init__``, its author does not need to be concerned about
1048 when or even the base ``Config.__init__`` should be called.
1049 """
1050 name = kw.pop("__name", None)
1051 at = kw.pop("__at", getCallStack())
1052 # remove __label and ignore it
1053 kw.pop("__label", "default")
1054
1055 instance = object.__new__(cls)
1056 instance._frozen = False
1057 instance._name = name
1058 instance._storage = {}
1059 instance._history = {}
1060 instance._imports = set()
1061 # load up defaults
1062 for field in instance._fields.values():
1063 instance._history[field.name] = []
1064 field.__set__(instance, field.default, at=at + [field.source], label="default")
1065 # set custom default-overrides
1066 instance.setDefaults()
1067 # set constructor overrides
1068 instance.update(__at=at, **kw)
1069 return instance
1070
1071 def __reduce__(self):
1072 """Reduction for pickling (function with arguments to reproduce).
1073
1074 We need to condense and reconstitute the `~lsst.pex.config.Config`,
1075 since it may contain lambdas (as the ``check`` elements) that cannot
1076 be pickled.
1077 """
1078 # The stream must be in characters to match the API but pickle
1079 # requires bytes
1080 stream = io.StringIO()
1081 self.saveToStream(stream)
1082 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
1083
1084 def setDefaults(self):
1085 """Subclass hook for computing defaults.
1086
1087 Notes
1088 -----
1089 Derived `~lsst.pex.config.Config` classes that must compute defaults
1090 rather than using the `~lsst.pex.config.Field` instances's defaults
1091 should do so here. To correctly use inherited defaults,
1092 implementations of ``setDefaults`` must call their base class's
1093 ``setDefaults``.
1094 """
1095 pass
1096
1097 def update(self, **kw):
1098 """Update values of fields specified by the keyword arguments.
1099
1100 Parameters
1101 ----------
1102 **kw
1103 Keywords are configuration field names. Values are configuration
1104 field values.
1105
1106 Notes
1107 -----
1108 The ``__at`` and ``__label`` keyword arguments are special internal
1109 keywords. They are used to strip out any internal steps from the
1110 history tracebacks of the config. Do not modify these keywords to
1111 subvert a `~lsst.pex.config.Config` instance's history.
1112
1113 Examples
1114 --------
1115 This is a config with three fields:
1116
1117 >>> from lsst.pex.config import Config, Field
1118 >>> class DemoConfig(Config):
1119 ... fieldA = Field(doc='Field A', dtype=int, default=42)
1120 ... fieldB = Field(doc='Field B', dtype=bool, default=True)
1121 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
1122 ...
1123 >>> config = DemoConfig()
1124
1125 These are the default values of each field:
1126
1127 >>> for name, value in config.iteritems():
1128 ... print(f"{name}: {value}")
1129 ...
1130 fieldA: 42
1131 fieldB: True
1132 fieldC: 'Hello world'
1133
1134 Using this method to update ``fieldA`` and ``fieldC``:
1135
1136 >>> config.update(fieldA=13, fieldC='Updated!')
1137
1138 Now the values of each field are:
1139
1140 >>> for name, value in config.iteritems():
1141 ... print(f"{name}: {value}")
1142 ...
1143 fieldA: 13
1144 fieldB: True
1145 fieldC: 'Updated!'
1146 """
1147 at = kw.pop("__at", getCallStack())
1148 label = kw.pop("__label", "update")
1149
1150 for name, value in kw.items():
1151 try:
1152 field = self._fields[name]
1153 field.__set__(self, value, at=at, label=label)
1154 except KeyError:
1155 raise KeyError(f"No field of name {name} exists in config type {_typeStr(self)}")
1156
1157 def load(self, filename, root="config"):
1158 """Modify this config in place by executing the Python code in a
1159 configuration file.
1160
1161 Parameters
1162 ----------
1163 filename : `str`
1164 Name of the configuration file. A configuration file is Python
1165 module.
1166 root : `str`, optional
1167 Name of the variable in file that refers to the config being
1168 overridden.
1169
1170 For example, the value of root is ``"config"`` and the file
1171 contains::
1172
1173 config.myField = 5
1174
1175 Then this config's field ``myField`` is set to ``5``.
1176
1177 See Also
1178 --------
1179 lsst.pex.config.Config.loadFromStream
1180 lsst.pex.config.Config.loadFromString
1181 lsst.pex.config.Config.save
1182 lsst.pex.config.Config.saveToStream
1183 lsst.pex.config.Config.saveToString
1184 """
1185 with open(filename) as f:
1186 code = compile(f.read(), filename=filename, mode="exec")
1187 self.loadFromString(code, root=root, filename=filename)
1188
1189 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None):
1190 """Modify this Config in place by executing the Python code in the
1191 provided stream.
1192
1193 Parameters
1194 ----------
1195 stream : file-like object, `str`, `bytes`, or `~types.CodeType`
1196 Stream containing configuration override code. If this is a
1197 code object, it should be compiled with ``mode="exec"``.
1198 root : `str`, optional
1199 Name of the variable in file that refers to the config being
1200 overridden.
1201
1202 For example, the value of root is ``"config"`` and the file
1203 contains::
1204
1205 config.myField = 5
1206
1207 Then this config's field ``myField`` is set to ``5``.
1208 filename : `str`, optional
1209 Name of the configuration file, or `None` if unknown or contained
1210 in the stream. Used for error reporting.
1211 extraLocals : `dict` of `str` to `object`, optional
1212 Any extra variables to include in local scope when loading.
1213
1214 Notes
1215 -----
1216 For backwards compatibility reasons, this method accepts strings, bytes
1217 and code objects as well as file-like objects. New code should use
1218 `loadFromString` instead for most of these types.
1219
1220 See Also
1221 --------
1222 lsst.pex.config.Config.load
1223 lsst.pex.config.Config.loadFromString
1224 lsst.pex.config.Config.save
1225 lsst.pex.config.Config.saveToStream
1226 lsst.pex.config.Config.saveToString
1227 """
1228 if hasattr(stream, "read"):
1229 if filename is None:
1230 filename = getattr(stream, "name", "?")
1231 code = compile(stream.read(), filename=filename, mode="exec")
1232 else:
1233 code = stream
1234 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1235
1236 def loadFromString(self, code, root="config", filename=None, extraLocals=None):
1237 """Modify this Config in place by executing the Python code in the
1238 provided string.
1239
1240 Parameters
1241 ----------
1242 code : `str`, `bytes`, or `~types.CodeType`
1243 Stream containing configuration override code.
1244 root : `str`, optional
1245 Name of the variable in file that refers to the config being
1246 overridden.
1247
1248 For example, the value of root is ``"config"`` and the file
1249 contains::
1250
1251 config.myField = 5
1252
1253 Then this config's field ``myField`` is set to ``5``.
1254 filename : `str`, optional
1255 Name of the configuration file, or `None` if unknown or contained
1256 in the stream. Used for error reporting.
1257 extraLocals : `dict` of `str` to `object`, optional
1258 Any extra variables to include in local scope when loading.
1259
1260 Raises
1261 ------
1262 ValueError
1263 Raised if a key in extraLocals is the same value as the value of
1264 the root argument.
1265
1266 See Also
1267 --------
1268 lsst.pex.config.Config.load
1269 lsst.pex.config.Config.loadFromStream
1270 lsst.pex.config.Config.save
1271 lsst.pex.config.Config.saveToStream
1272 lsst.pex.config.Config.saveToString
1273 """
1274 if filename is None:
1275 # try to determine the file name; a compiled string
1276 # has attribute "co_filename",
1277 filename = getattr(code, "co_filename", "?")
1278 with RecordingImporter() as importer:
1279 globals = {"__file__": filename}
1280 local = {root: self}
1281 if extraLocals is not None:
1282 # verify the value of root was not passed as extra local args
1283 if root in extraLocals:
1284 raise ValueError(
1285 f"{root} is reserved and cannot be used as a variable name in extraLocals"
1286 )
1287 local.update(extraLocals)
1288 exec(code, globals, local)
1289
1290 self._imports_imports.update(importer.getModules())
1291
1292 def save(self, filename, root="config"):
1293 """Save a Python script to the named file, which, when loaded,
1294 reproduces this config.
1295
1296 Parameters
1297 ----------
1298 filename : `str`
1299 Desination filename of this configuration.
1300 root : `str`, optional
1301 Name to use for the root config variable. The same value must be
1302 used when loading (see `lsst.pex.config.Config.load`).
1303
1304 See Also
1305 --------
1306 lsst.pex.config.Config.saveToStream
1307 lsst.pex.config.Config.saveToString
1308 lsst.pex.config.Config.load
1309 lsst.pex.config.Config.loadFromStream
1310 lsst.pex.config.Config.loadFromString
1311 """
1312 d = os.path.dirname(filename)
1313 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1314 self.saveToStream(outfile, root)
1315 # tempfile is hardcoded to create files with mode '0600'
1316 # for an explantion of these antics see:
1317 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1318 umask = os.umask(0o077)
1319 os.umask(umask)
1320 os.chmod(outfile.name, (~umask & 0o666))
1321 # chmod before the move so we get quasi-atomic behavior if the
1322 # source and dest. are on the same filesystem.
1323 # os.rename may not work across filesystems
1324 shutil.move(outfile.name, filename)
1325
1326 def saveToString(self, skipImports=False):
1327 """Return the Python script form of this configuration as an executable
1328 string.
1329
1330 Parameters
1331 ----------
1332 skipImports : `bool`, optional
1333 If `True` then do not include ``import`` statements in output,
1334 this is to support human-oriented output from ``pipetask`` where
1335 additional clutter is not useful.
1336
1337 Returns
1338 -------
1339 code : `str`
1340 A code string readable by `loadFromString`.
1341
1342 See Also
1343 --------
1344 lsst.pex.config.Config.save
1345 lsst.pex.config.Config.saveToStream
1346 lsst.pex.config.Config.load
1347 lsst.pex.config.Config.loadFromStream
1348 lsst.pex.config.Config.loadFromString
1349 """
1350 buffer = io.StringIO()
1351 self.saveToStream(buffer, skipImports=skipImports)
1352 return buffer.getvalue()
1353
1354 def saveToStream(self, outfile, root="config", skipImports=False):
1355 """Save a configuration file to a stream, which, when loaded,
1356 reproduces this config.
1357
1358 Parameters
1359 ----------
1360 outfile : file-like object
1361 Destination file object write the config into. Accepts strings not
1362 bytes.
1363 root : `str`, optional
1364 Name to use for the root config variable. The same value must be
1365 used when loading (see `lsst.pex.config.Config.load`).
1366 skipImports : `bool`, optional
1367 If `True` then do not include ``import`` statements in output,
1368 this is to support human-oriented output from ``pipetask`` where
1369 additional clutter is not useful.
1370
1371 See Also
1372 --------
1373 lsst.pex.config.Config.save
1374 lsst.pex.config.Config.saveToString
1375 lsst.pex.config.Config.load
1376 lsst.pex.config.Config.loadFromStream
1377 lsst.pex.config.Config.loadFromString
1378 """
1379 tmp = self._name
1380 self._rename(root)
1381 try:
1382 if not skipImports:
1383 self._collectImports()
1384 # Remove self from the set, as it is handled explicitly below
1385 self._imports_imports.remove(self.__module__)
1386 configType = type(self)
1387 typeString = _typeStr(configType)
1388 outfile.write(f"import {configType.__module__}\n")
1389 outfile.write(
1390 f"assert type({root})=={typeString}, 'config is of type %s.%s instead of "
1391 f"{typeString}' % (type({root}).__module__, type({root}).__name__)\n"
1392 )
1393 for imp in sorted(self._imports_imports):
1394 if imp in sys.modules and sys.modules[imp] is not None:
1395 outfile.write(f"import {imp}\n")
1396 self._save(outfile)
1397 finally:
1398 self._rename(tmp)
1399
1400 def freeze(self):
1401 """Make this config, and all subconfigs, read-only."""
1402 self._frozen = True
1403 for field in self._fields_fields.values():
1404 field.freeze(self)
1405
1406 def _save(self, outfile):
1407 """Save this config to an open stream object.
1408
1409 Parameters
1410 ----------
1411 outfile : file-like object
1412 Destination file object write the config into. Accepts strings not
1413 bytes.
1414 """
1415 for field in self._fields_fields.values():
1416 field.save(outfile, self)
1417
1419 """Add module containing self to the list of things to import and
1420 then loops over all the fields in the config calling a corresponding
1421 collect method.
1422
1423 The field method will call _collectImports on any
1424 configs it may own and return the set of things to import. This
1425 returned set will be merged with the set of imports for this config
1426 class.
1427 """
1428 self._imports_imports.add(self.__module__)
1429 for name, field in self._fields_fields.items():
1430 field._collectImports(self, self._imports_imports)
1431
1432 def toDict(self):
1433 """Make a dictionary of field names and their values.
1434
1435 Returns
1436 -------
1437 dict_ : `dict`
1438 Dictionary with keys that are `~lsst.pex.config.Field` names.
1439 Values are `~lsst.pex.config.Field` values.
1440
1441 See Also
1442 --------
1443 lsst.pex.config.Field.toDict
1444
1445 Notes
1446 -----
1447 This method uses the `~lsst.pex.config.Field.toDict` method of
1448 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1449 implement a ``toDict`` method for *this* method to work.
1450 """
1451 dict_ = {}
1452 for name, field in self._fields_fields.items():
1453 dict_[name] = field.toDict(self)
1454 return dict_
1455
1456 def names(self):
1457 """Get all the field names in the config, recursively.
1458
1459 Returns
1460 -------
1461 names : `list` of `str`
1462 Field names.
1463 """
1464 #
1465 # Rather than sort out the recursion all over again use the
1466 # pre-existing saveToStream()
1467 #
1468 with io.StringIO() as strFd:
1469 self.saveToStream(strFd, "config")
1470 contents = strFd.getvalue()
1471 strFd.close()
1472 #
1473 # Pull the names out of the dumped config
1474 #
1475 keys = []
1476 for line in contents.split("\n"):
1477 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1478 continue
1479
1480 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1481 if mat:
1482 keys.append(mat.group(1))
1483
1484 return keys
1485
1486 def _rename(self, name):
1487 """Rename this config object in its parent `~lsst.pex.config.Config`.
1488
1489 Parameters
1490 ----------
1491 name : `str`
1492 New name for this config in its parent `~lsst.pex.config.Config`.
1493
1494 Notes
1495 -----
1496 This method uses the `~lsst.pex.config.Field.rename` method of
1497 individual `lsst.pex.config.Field` instances.
1498 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1499 method for *this* method to work.
1500
1501 See Also
1502 --------
1503 lsst.pex.config.Field.rename
1504 """
1505 self._name = name
1506 for field in self._fields_fields.values():
1507 field.rename(self)
1508
1509 def validate(self):
1510 """Validate the Config, raising an exception if invalid.
1511
1512 Raises
1513 ------
1514 lsst.pex.config.FieldValidationError
1515 Raised if verification fails.
1516
1517 Notes
1518 -----
1519 The base class implementation performs type checks on all fields by
1520 calling their `~lsst.pex.config.Field.validate` methods.
1521
1522 Complex single-field validation can be defined by deriving new Field
1523 types. For convenience, some derived `lsst.pex.config.Field`-types
1524 (`~lsst.pex.config.ConfigField` and
1525 `~lsst.pex.config.ConfigChoiceField`) are defined in
1526 ``lsst.pex.config`` that handle recursing into subconfigs.
1527
1528 Inter-field relationships should only be checked in derived
1529 `~lsst.pex.config.Config` classes after calling this method, and base
1530 validation is complete.
1531 """
1532 for field in self._fields_fields.values():
1533 field.validate(self)
1534
1535 def formatHistory(self, name, **kwargs):
1536 """Format a configuration field's history to a human-readable format.
1537
1538 Parameters
1539 ----------
1540 name : `str`
1541 Name of a `~lsst.pex.config.Field` in this config.
1542 **kwargs
1543 Keyword arguments passed to `lsst.pex.config.history.format`.
1544
1545 Returns
1546 -------
1547 history : `str`
1548 A string containing the formatted history.
1549
1550 See Also
1551 --------
1552 lsst.pex.config.history.format
1553 """
1554 import lsst.pex.config.history as pexHist
1555
1556 return pexHist.format(self, name, **kwargs)
1557
1558 history = property(lambda x: x._history)
1559 """Read-only history.
1560 """
1561
1562 def __setattr__(self, attr, value, at=None, label="assignment"):
1563 """Set an attribute (such as a field's value).
1564
1565 Notes
1566 -----
1567 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1568 locked such that no additional attributes nor properties may be added
1569 to them dynamically.
1570
1571 Although this is not the standard Python behavior, it helps to protect
1572 users from accidentally mispelling a field name, or trying to set a
1573 non-existent field.
1574 """
1575 if attr in self._fields_fields:
1576 if self._fields_fields[attr].deprecated is not None:
1577 fullname = _joinNamePath(self._name, self._fields_fields[attr].name)
1578 warnings.warn(
1579 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1580 FutureWarning,
1581 stacklevel=2,
1582 )
1583 if at is None:
1584 at = getCallStack()
1585 # This allows Field descriptors to work.
1586 self._fields_fields[attr].__set__(self, value, at=at, label=label)
1587 elif hasattr(getattr(self.__class__, attr, None), "__set__"):
1588 # This allows properties and other non-Field descriptors to work.
1589 return object.__setattr__(self, attr, value)
1590 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"):
1591 # This allows specific private attributes to work.
1592 self.__dict__[attr] = value
1593 else:
1594 # We throw everything else.
1595 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
1596
1597 def __delattr__(self, attr, at=None, label="deletion"):
1598 if attr in self._fields_fields:
1599 if at is None:
1600 at = getCallStack()
1601 self._fields_fields[attr].__delete__(self, at=at, label=label)
1602 else:
1603 object.__delattr__(self, attr)
1604
1605 def __eq__(self, other):
1606 if type(other) is type(self):
1607 for name in self._fields_fields:
1608 thisValue = getattr(self, name)
1609 otherValue = getattr(other, name)
1610 if isinstance(thisValue, float) and math.isnan(thisValue):
1611 if not math.isnan(otherValue):
1612 return False
1613 elif thisValue != otherValue:
1614 return False
1615 return True
1616 return False
1617
1618 def __ne__(self, other):
1619 return not self.__eq__(other)
1620
1621 def __str__(self):
1622 return str(self.toDict())
1623
1624 def __repr__(self):
1625 return "{}({})".format(
1626 _typeStr(self),
1627 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None),
1628 )
1629
1630 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None):
1631 """Compare this configuration to another `~lsst.pex.config.Config` for
1632 equality.
1633
1634 Parameters
1635 ----------
1636 other : `lsst.pex.config.Config`
1637 Other `~lsst.pex.config.Config` object to compare against this
1638 config.
1639 shortcut : `bool`, optional
1640 If `True`, return as soon as an inequality is found. Default is
1641 `True`.
1642 rtol : `float`, optional
1643 Relative tolerance for floating point comparisons.
1644 atol : `float`, optional
1645 Absolute tolerance for floating point comparisons.
1646 output : callable, optional
1647 A callable that takes a string, used (possibly repeatedly) to
1648 report inequalities.
1649
1650 Returns
1651 -------
1652 isEqual : `bool`
1653 `True` when the two `lsst.pex.config.Config` instances are equal.
1654 `False` if there is an inequality.
1655
1656 See Also
1657 --------
1658 lsst.pex.config.compareConfigs
1659
1660 Notes
1661 -----
1662 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1663 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1664 are not considered by this method.
1665
1666 Floating point comparisons are performed by `numpy.allclose`.
1667 """
1668 name1 = self._name if self._name is not None else "config"
1669 name2 = other._name if other._name is not None else "config"
1670 name = getComparisonName(name1, name2)
1671 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output)
1672
1673 @classmethod
1674 def __init_subclass__(cls, **kwargs):
1675 """Run initialization for every subclass.
1676
1677 Specifically registers the subclass with a YAML representer
1678 and YAML constructor (if pyyaml is available)
1679 """
1680 super().__init_subclass__(**kwargs)
1681
1682 if not yaml:
1683 return
1684
1685 yaml.add_representer(cls, _yaml_config_representer)
1686
1687 @classmethod
1688 def _fromPython(cls, config_py):
1689 """Instantiate a `Config`-subclass from serialized Python form.
1690
1691 Parameters
1692 ----------
1693 config_py : `str`
1694 A serialized form of the Config as created by
1695 `Config.saveToStream`.
1696
1697 Returns
1698 -------
1699 config : `Config`
1700 Reconstructed `Config` instant.
1701 """
1702 cls = _classFromPython(config_py)
1703 return unreduceConfig(cls, config_py)
1704
1705
1706def _classFromPython(config_py):
1707 """Return the Config subclass required by this Config serialization.
1708
1709 Parameters
1710 ----------
1711 config_py : `str`
1712 A serialized form of the Config as created by
1713 `Config.saveToStream`.
1714
1715 Returns
1716 -------
1717 cls : `type`
1718 The `Config` subclass associated with this config.
1719 """
1720 # standard serialization has the form:
1721 # import config.class
1722 # assert type(config)==config.class.Config, ...
1723 # We want to parse these two lines so we can get the class itself
1724
1725 # Do a single regex to avoid large string copies when splitting a
1726 # large config into separate lines.
1727 matches = re.search(r"^import ([\w.]+)\nassert .*==(.*?),", config_py)
1728
1729 if not matches:
1730 first_line, second_line, _ = config_py.split("\n", 2)
1731 raise ValueError(
1732 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}"
1733 )
1734
1735 module_name = matches.group(1)
1736 module = importlib.import_module(module_name)
1737
1738 # Second line
1739 full_name = matches.group(2)
1740
1741 # Remove the module name from the full name
1742 if not full_name.startswith(module_name):
1743 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1744
1745 # if module name is a.b.c and full name is a.b.c.d.E then
1746 # we need to remove a.b.c. and iterate over the remainder
1747 # The +1 is for the extra dot after a.b.c
1748 remainder = full_name[len(module_name) + 1 :]
1749 components = remainder.split(".")
1750 pytype = module
1751 for component in components:
1752 pytype = getattr(pytype, component)
1753 return pytype
1754
1755
1756def unreduceConfig(cls_, stream):
1757 """Create a `~lsst.pex.config.Config` from a stream.
1758
1759 Parameters
1760 ----------
1761 cls_ : `lsst.pex.config.Config`-type
1762 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1763 with configurations in the ``stream``.
1764 stream : file-like object, `str`, or `~types.CodeType`
1765 Stream containing configuration override code.
1766
1767 Returns
1768 -------
1769 config : `lsst.pex.config.Config`
1770 Config instance.
1771
1772 See Also
1773 --------
1774 lsst.pex.config.Config.loadFromStream
1775 """
1776 config = cls_()
1777 config.loadFromStream(stream)
1778 return config
std::vector< SchemaItem< Flag > > * items
Any __call__(self, *Any args, **Any kwds)
Definition config.py:97
saveToStream(self, outfile, root="config", skipImports=False)
Definition config.py:1354
__setattr__(self, attr, value, at=None, label="assignment")
Definition config.py:1562
loadFromStream(self, stream, root="config", filename=None, extraLocals=None)
Definition config.py:1189
__new__(cls, *args, **kw)
Definition config.py:1038
save(self, filename, root="config")
Definition config.py:1292
_fromPython(cls, config_py)
Definition config.py:1688
loadFromString(self, code, root="config", filename=None, extraLocals=None)
Definition config.py:1236
__delattr__(self, attr, at=None, label="deletion")
Definition config.py:1597
saveToString(self, skipImports=False)
Definition config.py:1326
formatHistory(self, name, **kwargs)
Definition config.py:1535
__init_subclass__(cls, **kwargs)
Definition config.py:1674
load(self, filename, root="config")
Definition config.py:1157
compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None)
Definition config.py:1630
__setattr__(cls, name, value)
Definition config.py:245
__init__(cls, name, bases, dict_)
Definition config.py:224
Mapping[str, Any] _parseTypingArgs(tuple[type,...]|tuple[str,...] params, Mapping[str, Any] kwds)
Definition config.py:418
save(self, outfile, instance)
Definition config.py:653
_collectImports(self, instance, imports)
Definition config.py:639
__get__(self, instance, owner=None, at=None, label="default")
Definition config.py:728
FieldTypeVar __get__(self, Config instance, Any owner=None, Any at=None, str label="default")
Definition config.py:725
__delete__(self, instance, at=None, label="deletion")
Definition config.py:815
Field[FieldTypeVar] __get__(self, None instance, Any owner=None, Any at=None, str label="default")
Definition config.py:719
rename(self, instance)
Definition config.py:542
_compare(self, instance1, instance2, shortcut, rtol, atol, output)
Definition config.py:837
toDict(self, instance)
Definition config.py:689
_validateValue(self, value)
Definition config.py:609
None __set__(self, Config instance, FieldTypeVar|None value, Any at=None, str label="assignment")
Definition config.py:757
__class_getitem__(cls, tuple[type,...]|type|ForwardRef params)
Definition config.py:475
freeze(self, instance)
Definition config.py:592
__init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None)
Definition config.py:478
_setup(self, doc, dtype, default, check, optional, source, deprecated)
Definition config.py:497
__init__(self, field, config, msg)
Definition config.py:266
find_spec(self, fullname, path, target=None)
Definition config.py:906
daf::base::PropertySet * set
Definition fits.cc:931
_autocast(x, dtype)
Definition config.py:126
_yaml_config_representer(dumper, data)
Definition config.py:173
_classFromPython(config_py)
Definition config.py:1706
_joinNamePath(prefix=None, name=None, index=None)
Definition config.py:111
unreduceConfig(cls_, stream)
Definition config.py:1756