LSST Applications g1653933729+a8ce1bb630,g171997e3ee+76e92115be,g1a997c3884+a8ce1bb630,g28da252d5a+4013ff5194,g2bbee38e9b+d6d0f9f6ae,g2bc492864f+d6d0f9f6ae,g2cdde0e794+ccb74358b7,g3156d2b45e+07302053f8,g347aa1857d+d6d0f9f6ae,g35bb328faa+a8ce1bb630,g3a166c0a6a+d6d0f9f6ae,g3e281a1b8c+130fae20e3,g4005a62e65+17cd334064,g414038480c+c9f68e2a12,g41af890bb2+3e1f62f438,g4e1a3235cc+9db7d56ad0,g7b55021d1b+7b623691d5,g80478fca09+334cc8d2bd,g82479be7b0+b568f6b267,g858d7b2824+37b39d8760,g9125e01d80+a8ce1bb630,ga5288a1d22+11cb34fefe,gae0086650b+a8ce1bb630,gb4ec7eb0ab+37b39d8760,gb58c049af0+d64f4d3760,gc081298178+284e133171,gc28159a63d+d6d0f9f6ae,gcf0d15dbbd+00fe2e0b07,gd6b7c0dfd1+8b62435e69,gda3e153d99+37b39d8760,gda6a2b7d83+00fe2e0b07,gdaeeff99f8+1711a396fd,gdd5a9049c5+23f3b3239a,ge2409df99d+5d9f551a54,ge33fd446bb+37b39d8760,ge79ae78c31+d6d0f9f6ae,gf0baf85859+2dea8344a2,gf5289d68f6+3777f3df5e,w.2024.41
LSST Data Management Base Package
Loading...
Searching...
No Matches
config.py
Go to the documentation of this file.
1# This file is part of pex_config.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
28
29__all__ = (
30 "Config",
31 "ConfigMeta",
32 "Field",
33 "FieldValidationError",
34 "UnexpectedProxyUsageError",
35 "FieldTypeVar",
36)
37
38import copy
39import importlib
40import io
41import math
42import os
43import re
44import shutil
45import sys
46import tempfile
47import warnings
48from collections.abc import Mapping
49from types import GenericAlias
50from typing import Any, ForwardRef, Generic, TypeVar, cast, overload
51
52# if YAML is not available that's fine and we simply don't register
53# the yaml representer since we know it won't be used.
54try:
55 import yaml
56except ImportError:
57 yaml = None
58
59from .callStack import getCallStack, getStackFrame
60from .comparison import compareConfigs, compareScalars, getComparisonName
61
62if yaml:
63 YamlLoaders: tuple[Any, ...] = (yaml.Loader, yaml.FullLoader, yaml.SafeLoader, yaml.UnsafeLoader)
64
65 try:
66 # CLoader is not always available
67 from yaml import CLoader
68
69 YamlLoaders += (CLoader,)
70 except ImportError:
71 pass
72else:
73 YamlLoaders = ()
74 doImport = None
75
76
77class _PexConfigGenericAlias(GenericAlias):
78 """A Subclass of python's GenericAlias used in defining and instantiating
79 Generics.
80
81 This class differs from `types.GenericAlias` in that it calls a method
82 named _parseTypingArgs defined on Fields. This method gives Field and its
83 subclasses an opportunity to transform type parameters into class key word
84 arguments. Code authors do not need to implement any returns of this object
85 directly, and instead only need implement _parseTypingArgs, if a Field
86 subclass differs from the base class implementation.
87
88 This class is intended to be an implementation detail, returned from a
89 Field's `__class_getitem__` method.
90 """
91
92 def __call__(self, *args: Any, **kwds: Any) -> Any:
93 origin_kwargs = self._parseTypingArgs(self.__args__, kwds)
94 return super().__call__(*args, **{**kwds, **origin_kwargs})
95
96
97FieldTypeVar = TypeVar("FieldTypeVar")
98
99
101 """Exception raised when a proxy class is used in a context that suggests
102 it should have already been converted to the thing it proxies.
103 """
104
105
106def _joinNamePath(prefix=None, name=None, index=None):
107 """Generate nested configuration names."""
108 if not prefix and not name:
109 raise ValueError("Invalid name: cannot be None")
110 elif not name:
111 name = prefix
112 elif prefix and name:
113 name = prefix + "." + name
114
115 if index is not None:
116 return f"{name}[{index!r}]"
117 else:
118 return name
119
120
121def _autocast(x, dtype):
122 """Cast a value to a type, if appropriate.
123
124 Parameters
125 ----------
126 x : object
127 A value.
128 dtype : type
129 Data type, such as `float`, `int`, or `str`.
130
131 Returns
132 -------
133 values : object
134 If appropriate, the returned value is ``x`` cast to the given type
135 ``dtype``. If the cast cannot be performed the original value of
136 ``x`` is returned.
137 """
138 if dtype is float and isinstance(x, int):
139 return float(x)
140 return x
141
142
143def _typeStr(x):
144 """Generate a fully-qualified type name.
145
146 Returns
147 -------
148 `str`
149 Fully-qualified type name.
150
151 Notes
152 -----
153 This function is used primarily for writing config files to be executed
154 later upon with the 'load' function.
155 """
156 if hasattr(x, "__module__") and hasattr(x, "__name__"):
157 xtype = x
158 else:
159 xtype = type(x)
160 if xtype.__module__ == "builtins":
161 return xtype.__name__
162 else:
163 return f"{xtype.__module__}.{xtype.__name__}"
164
165
166if yaml:
167
168 def _yaml_config_representer(dumper, data):
169 """Represent a Config object in a form suitable for YAML.
170
171 Stores the serialized stream as a scalar block string.
172 """
173 stream = io.StringIO()
174 data.saveToStream(stream)
175 config_py = stream.getvalue()
176
177 # Strip multiple newlines from the end of the config
178 # This simplifies the YAML to use | and not |+
179 config_py = config_py.rstrip() + "\n"
180
181 # Trailing spaces force pyyaml to use non-block form.
182 # Remove the trailing spaces so it has no choice
183 config_py = re.sub(r"\s+$", "\n", config_py, flags=re.MULTILINE)
184
185 # Store the Python as a simple scalar
186 return dumper.represent_scalar("lsst.pex.config.Config", config_py, style="|")
187
188 def _yaml_config_constructor(loader, node):
189 """Construct a config from YAML."""
190 config_py = loader.construct_scalar(node)
191 return Config._fromPython(config_py)
192
193 # Register a generic constructor for Config and all subclasses
194 # Need to register for all the loaders we would like to use
195 for loader in YamlLoaders:
196 yaml.add_constructor("lsst.pex.config.Config", _yaml_config_constructor, Loader=loader)
197
198
199class ConfigMeta(type):
200 """A metaclass for `lsst.pex.config.Config`.
201
202 Parameters
203 ----------
204 name : `str`
205 Name to use for class.
206 bases : `~collections.abc.Iterable`
207 Base classes.
208 dict_ : `dict`
209 Additional parameters.
210
211 Notes
212 -----
213 ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
214 class attributes as a class attribute called ``_fields``, and adds
215 the name of each field as an instance variable of the field itself (so you
216 don't have to pass the name of the field to the field constructor).
217 """
218
219 def __init__(cls, name, bases, dict_):
220 type.__init__(cls, name, bases, dict_)
221 cls._fields = {}
222 cls._source = getStackFrame()
223
224 def getFields(classtype):
225 fields = {}
226 bases = list(classtype.__bases__)
227 bases.reverse()
228 for b in bases:
229 fields.update(getFields(b))
230
231 for k, v in classtype.__dict__.items():
232 if isinstance(v, Field):
233 fields[k] = v
234 return fields
235
236 fields = getFields(cls)
237 for k, v in fields.items():
238 setattr(cls, k, copy.deepcopy(v))
239
240 def __setattr__(cls, name, value):
241 if isinstance(value, Field):
242 value.name = name
243 cls._fields[name] = value
244 type.__setattr__(cls, name, value)
245
246
247class FieldValidationError(ValueError):
248 """Raised when a ``~lsst.pex.config.Field`` is not valid in a
249 particular ``~lsst.pex.config.Config``.
250
251 Parameters
252 ----------
253 field : `lsst.pex.config.Field`
254 The field that was not valid.
255 config : `lsst.pex.config.Config`
256 The config containing the invalid field.
257 msg : `str`
258 Text describing why the field was not valid.
259 """
260
261 def __init__(self, field, config, msg):
262 self.fieldType = type(field)
263 """Type of the `~lsst.pex.config.Field` that incurred the error.
264 """
265
266 self.fieldName = field.name
267 """Name of the `~lsst.pex.config.Field` instance that incurred the
268 error (`str`).
269
270 See also
271 --------
272 lsst.pex.config.Field.name
273 """
274
275 self.fullname = _joinNamePath(config._name, field.name)
276 """Fully-qualified name of the `~lsst.pex.config.Field` instance
277 (`str`).
278 """
279
280 self.history = config.history.setdefault(field.name, [])
281 """Full history of all changes to the `~lsst.pex.config.Field`
282 instance.
283 """
284
285 self.fieldSource = field.source
286 """File and line number of the `~lsst.pex.config.Field` definition.
287 """
288
289 self.configSource = config._source
290 error = (
291 f"{self.fieldType.__name__} '{self.fullname}' failed validation: {msg}\n"
292 f"For more information see the Field definition at:\n{self.fieldSource.format()}"
293 f" and the Config definition at:\n{self.configSource.format()}"
294 )
295 super().__init__(error)
296
297
298class Field(Generic[FieldTypeVar]):
299 """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
300 `complex`, `bool`, and `str` data types.
301
302 Parameters
303 ----------
304 doc : `str`
305 A description of the field for users.
306 dtype : type, optional
307 The field's data type. ``Field`` only supports basic data types:
308 `int`, `float`, `complex`, `bool`, and `str`. See
309 `Field.supportedTypes`. Optional if supplied as a typing argument to
310 the class.
311 default : object, optional
312 The field's default value.
313 check : callable, optional
314 A callable that is called with the field's value. This callable should
315 return `False` if the value is invalid. More complex inter-field
316 validation can be written as part of the
317 `lsst.pex.config.Config.validate` method.
318 optional : `bool`, optional
319 This sets whether the field is considered optional, and therefore
320 doesn't need to be set by the user. When `False`,
321 `lsst.pex.config.Config.validate` fails if the field's value is `None`.
322 deprecated : None or `str`, optional
323 A description of why this Field is deprecated, including removal date.
324 If not None, the string is appended to the docstring for this Field.
325
326 Raises
327 ------
328 ValueError
329 Raised when the ``dtype`` parameter is not one of the supported types
330 (see `Field.supportedTypes`).
331
332 See Also
333 --------
334 ChoiceField
335 ConfigChoiceField
336 ConfigDictField
337 ConfigField
338 ConfigurableField
339 DictField
340 ListField
341 RangeField
342 RegistryField
343
344 Notes
345 -----
346 ``Field`` instances (including those of any subclass of ``Field``) are used
347 as class attributes of `~lsst.pex.config.Config` subclasses (see the
348 example, below). ``Field`` attributes work like the `property` attributes
349 of classes that implement custom setters and getters. `Field` attributes
350 belong to the class, but operate on the instance. Formally speaking,
351 `Field` attributes are `descriptors
352 <https://docs.python.org/3/howto/descriptor.html>`_.
353
354 When you access a `Field` attribute on a `Config` instance, you don't
355 get the `Field` instance itself. Instead, you get the value of that field,
356 which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
357 container type (like a `lsst.pex.config.List`) depending on the field's
358 type. See the example, below.
359
360 Fields can be annotated with a type similar to other python classes (python
361 specification `here <https://peps.python.org/pep-0484/#generics>`_ ).
362 See the name field in the Config example below for an example of this.
363 Unlike most other uses in python, this has an effect at type checking *and*
364 runtime. If the type is specified with a class annotation, it will be used
365 as the value of the ``dtype`` in the ``Field`` and there is no need to
366 specify it as an argument during instantiation.
367
368 There are Some notes on dtype through type annotation syntax. Type
369 annotation syntax supports supplying the argument as a string of a type
370 name. i.e. "float", but this cannot be used to resolve circular references.
371 Type annotation syntax can be used on an identifier in addition to Class
372 assignment i.e. ``variable: Field[str] = Config.someField`` vs
373 ``someField = Field[str](doc="some doc"). However, this syntax is only
374 useful for annotating the type of the identifier (i.e. variable in previous
375 example) and does nothing for assigning the dtype of the ``Field``.
376
377 Examples
378 --------
379 Instances of ``Field`` should be used as class attributes of
380 `lsst.pex.config.Config` subclasses:
381
382 >>> from lsst.pex.config import Config, Field
383 >>> class Example(Config):
384 ... myInt = Field("An integer field.", int, default=0)
385 ... name = Field[str](doc="A string Field")
386 ...
387 >>> print(config.myInt)
388 0
389 >>> config.myInt = 5
390 >>> print(config.myInt)
391 5
392 """
393
394 name: str
395 """Identifier (variable name) used to refer to a Field within a Config
396 Class.
397 """
398
399 supportedTypes = {str, bool, float, int, complex}
400 """Supported data types for field values (`set` of types).
401 """
402
403 @staticmethod
405 params: tuple[type, ...] | tuple[str, ...], kwds: Mapping[str, Any]
406 ) -> Mapping[str, Any]:
407 """Parse type annotations into keyword constructor arguments.
408
409 This is a special private method that interprets type arguments (i.e.
410 Field[str]) into keyword arguments to be passed on to the constructor.
411
412 Subclasses of Field can implement this method to customize how they
413 handle turning type parameters into keyword arguments (see DictField
414 for an example)
415
416 Parameters
417 ----------
418 params : `tuple` of `type` or `tuple` of str
419 Parameters passed to the type annotation. These will either be
420 types or strings. Strings are to interpreted as forward references
421 and will be treated as such.
422 kwds : `MutableMapping` with keys of `str` and values of `Any`
423 These are the user supplied keywords that are to be passed to the
424 Field constructor.
425
426 Returns
427 -------
428 kwds : `MutableMapping` with keys of `str` and values of `Any`
429 The mapping of keywords that will be passed onto the constructor
430 of the Field. Should be filled in with any information gleaned
431 from the input parameters.
432
433 Raises
434 ------
435 ValueError
436 Raised if params is of incorrect length.
437 Raised if a forward reference could not be resolved
438 Raised if there is a conflict between params and values in kwds
439 """
440 if len(params) > 1:
441 raise ValueError("Only single type parameters are supported")
442 unpackedParams = params[0]
443 if isinstance(unpackedParams, str):
444 _typ = ForwardRef(unpackedParams)
445 # type ignore below because typeshed seems to be wrong. It
446 # indicates there are only 2 args, as it was in python 3.8, but
447 # 3.9+ takes 3 args.
448 result = _typ._evaluate(globals(), locals(), recursive_guard=set()) # type: ignore
449 if result is None:
450 raise ValueError("Could not deduce type from input")
451 unpackedParams = cast(type, result)
452 if "dtype" in kwds and kwds["dtype"] != unpackedParams:
453 raise ValueError("Conflicting definition for dtype")
454 elif "dtype" not in kwds:
455 kwds = {**kwds, **{"dtype": unpackedParams}}
456 return kwds
457
458 def __class_getitem__(cls, params: tuple[type, ...] | type | ForwardRef):
459 return _PexConfigGenericAlias(cls, params)
460
461 def __init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None):
462 if dtype is None:
463 raise ValueError(
464 "dtype must either be supplied as an argument or as a type argument to the class"
465 )
466 if dtype not in self.supportedTypes:
467 raise ValueError(f"Unsupported Field dtype {_typeStr(dtype)}")
468
469 source = getStackFrame()
470 self._setup(
471 doc=doc,
472 dtype=dtype,
473 default=default,
474 check=check,
475 optional=optional,
476 source=source,
477 deprecated=deprecated,
478 )
479
480 def _setup(self, doc, dtype, default, check, optional, source, deprecated):
481 """Set attributes, usually during initialization."""
482 self.dtype = dtype
483 """Data type for the field.
484 """
485
486 if not doc:
487 raise ValueError("Docstring is empty.")
488
489 # append the deprecation message to the docstring.
490 if deprecated is not None:
491 doc = f"{doc} Deprecated: {deprecated}"
492 self.doc = doc
493 """A description of the field (`str`).
494 """
495
496 self.deprecated = deprecated
497 """If not None, a description of why this field is deprecated (`str`).
498 """
499
500 self.__doc__ = f"{doc} (`{dtype.__name__}`"
501 if optional or default is not None:
502 self.__doc__ += f", default ``{default!r}``"
503 self.__doc__ += ")"
504
505 self.default = default
506 """Default value for this field.
507 """
508
509 self.check = check
510 """A user-defined function that validates the value of the field.
511 """
512
513 self.optional = optional
514 """Flag that determines if the field is required to be set (`bool`).
515
516 When `False`, `lsst.pex.config.Config.validate` will fail if the
517 field's value is `None`.
518 """
519
520 self.source = source
521 """The stack frame where this field is defined (`list` of
522 `~lsst.pex.config.callStack.StackFrame`).
523 """
524
525 def rename(self, instance):
526 r"""Rename the field in a `~lsst.pex.config.Config` (for internal use
527 only).
528
529 Parameters
530 ----------
531 instance : `lsst.pex.config.Config`
532 The config instance that contains this field.
533
534 Notes
535 -----
536 This method is invoked by the `lsst.pex.config.Config` object that
537 contains this field and should not be called directly.
538
539 Renaming is only relevant for `~lsst.pex.config.Field` instances that
540 hold subconfigs. `~lsst.pex.config.Field`\s that hold subconfigs should
541 rename each subconfig with the full field name as generated by
542 `lsst.pex.config.config._joinNamePath`.
543 """
544 pass
545
546 def validate(self, instance):
547 """Validate the field (for internal use only).
548
549 Parameters
550 ----------
551 instance : `lsst.pex.config.Config`
552 The config instance that contains this field.
553
554 Raises
555 ------
556 lsst.pex.config.FieldValidationError
557 Raised if verification fails.
558
559 Notes
560 -----
561 This method provides basic validation:
562
563 - Ensures that the value is not `None` if the field is not optional.
564 - Ensures type correctness.
565 - Ensures that the user-provided ``check`` function is valid.
566
567 Most `~lsst.pex.config.Field` subclasses should call
568 `lsst.pex.config.Field.validate` if they re-implement
569 `~lsst.pex.config.Field.validate`.
570 """
571 value = self.__get__(instance)
572 if not self.optional and value is None:
573 raise FieldValidationError(self, instance, "Required value cannot be None")
574
575 def freeze(self, instance):
576 """Make this field read-only (for internal use only).
577
578 Parameters
579 ----------
580 instance : `lsst.pex.config.Config`
581 The config instance that contains this field.
582
583 Notes
584 -----
585 Freezing is only relevant for fields that hold subconfigs. Fields which
586 hold subconfigs should freeze each subconfig.
587
588 **Subclasses should implement this method.**
589 """
590 pass
591
592 def _validateValue(self, value):
593 """Validate a value.
594
595 Parameters
596 ----------
597 value : object
598 The value being validated.
599
600 Raises
601 ------
602 TypeError
603 Raised if the value's type is incompatible with the field's
604 ``dtype``.
605 ValueError
606 Raised if the value is rejected by the ``check`` method.
607 """
608 if value is None:
609 return
610
611 if not isinstance(value, self.dtype):
612 msg = (
613 f"Value {value} is of incorrect type {_typeStr(value)}. Expected type {_typeStr(self.dtype)}"
614 )
615 raise TypeError(msg)
616 if self.check is not None and not self.check(value):
617 msg = f"Value {value} is not a valid value"
618 raise ValueError(msg)
619
620 def _collectImports(self, instance, imports):
621 """Call the _collectImports method on all config
622 objects the field may own, and union them with the supplied imports
623 set.
624
625 Parameters
626 ----------
627 instance : instance or subclass of `lsst.pex.config.Config`
628 A config object that has this field defined on it
629 imports : `set`
630 Set of python modules that need imported after persistence
631 """
632 pass
633
634 def save(self, outfile, instance):
635 """Save this field to a file (for internal use only).
636
637 Parameters
638 ----------
639 outfile : file-like object
640 A writeable field handle.
641 instance : `~lsst.pex.config.Config`
642 The `~lsst.pex.config.Config` instance that contains this field.
643
644 Notes
645 -----
646 This method is invoked by the `~lsst.pex.config.Config` object that
647 contains this field and should not be called directly.
648
649 The output consists of the documentation string
650 (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
651 line is formatted as an assignment: ``{fullname}={value}``.
652
653 This output can be executed with Python.
654 """
655 value = self.__get__(instance)
656 fullname = _joinNamePath(instance._name, self.name)
657
658 if self.deprecated and value == self.default:
659 return
660
661 # write full documentation string as comment lines
662 # (i.e. first character is #)
663 doc = "# " + str(self.doc).replace("\n", "\n# ")
664 if isinstance(value, float) and not math.isfinite(value):
665 # non-finite numbers need special care
666 outfile.write(f"{doc}\n{fullname}=float('{value!r}')\n\n")
667 else:
668 outfile.write(f"{doc}\n{fullname}={value!r}\n\n")
669
670 def toDict(self, instance):
671 """Convert the field value so that it can be set as the value of an
672 item in a `dict` (for internal use only).
673
674 Parameters
675 ----------
676 instance : `~lsst.pex.config.Config`
677 The `~lsst.pex.config.Config` that contains this field.
678
679 Returns
680 -------
681 value : object
682 The field's value. See *Notes*.
683
684 Notes
685 -----
686 This method invoked by the owning `~lsst.pex.config.Config` object and
687 should not be called directly.
688
689 Simple values are passed through. Complex data structures must be
690 manipulated. For example, a `~lsst.pex.config.Field` holding a
691 subconfig should, instead of the subconfig object, return a `dict`
692 where the keys are the field names in the subconfig, and the values are
693 the field values in the subconfig.
694 """
695 return self.__get____get____get__(instance)
696
697 @overload
699 self, instance: None, owner: Any = None, at: Any = None, label: str = "default"
700 ) -> Field[FieldTypeVar]: ...
701
702 @overload
704 self, instance: Config, owner: Any = None, at: Any = None, label: str = "default"
705 ) -> FieldTypeVar: ...
706
707 def __get__(self, instance, owner=None, at=None, label="default"):
708 """Define how attribute access should occur on the Config instance
709 This is invoked by the owning config object and should not be called
710 directly.
711
712 When the field attribute is accessed on a Config class object, it
713 returns the field object itself in order to allow inspection of
714 Config classes.
715
716 When the field attribute is access on a config instance, the actual
717 value described by the field (and held by the Config instance) is
718 returned.
719 """
720 if instance is None:
721 return self
722 else:
723 # try statements are almost free in python if they succeed
724 try:
725 return instance._storage[self.namename]
726 except AttributeError:
727 if not isinstance(instance, Config):
728 return self
729 else:
730 raise AttributeError(
731 f"Config {instance} is missing _storage attribute, likely incorrectly initialized"
732 )
733
735 self, instance: Config, value: FieldTypeVar | None, at: Any = None, label: str = "assignment"
736 ) -> None:
737 """Set an attribute on the config instance.
738
739 Parameters
740 ----------
741 instance : `lsst.pex.config.Config`
742 The config instance that contains this field.
743 value : obj
744 Value to set on this field.
745 at : `list` of `~lsst.pex.config.callStack.StackFrame` or `None`,\
746 optional
747 The call stack (created by
748 `lsst.pex.config.callStack.getCallStack`).
749 label : `str`, optional
750 Event label for the history.
751
752 Notes
753 -----
754 This method is invoked by the owning `lsst.pex.config.Config` object
755 and should not be called directly.
756
757 Derived `~lsst.pex.config.Field` classes may need to override the
758 behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
759 should follow the following rules:
760
761 - Do not allow modification of frozen configs.
762 - Validate the new value **before** modifying the field. Except if the
763 new value is `None`. `None` is special and no attempt should be made
764 to validate it until `lsst.pex.config.Config.validate` is called.
765 - Do not modify the `~lsst.pex.config.Config` instance to contain
766 invalid values.
767 - If the field is modified, update the history of the
768 `lsst.pex.config.field.Field` to reflect the changes.
769
770 In order to decrease the need to implement this method in derived
771 `~lsst.pex.config.Field` types, value validation is performed in the
772 `lsst.pex.config.Field._validateValue`. If only the validation step
773 differs in the derived `~lsst.pex.config.Field`, it is simpler to
774 implement `lsst.pex.config.Field._validateValue` than to reimplement
775 ``__set__``. More complicated behavior, however, may require
776 reimplementation.
777 """
778 if instance._frozen:
779 raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
780
781 history = instance._history.setdefault(self.namename, [])
782 if value is not None:
783 value = _autocast(value, self.dtype)
784 try:
785 self._validateValue(value)
786 except BaseException as e:
787 raise FieldValidationError(self, instance, str(e))
788
789 instance._storage[self.namename] = value
790 if at is None:
791 at = getCallStack()
792 history.append((value, at, label))
793
794 def __delete__(self, instance, at=None, label="deletion"):
795 """Delete an attribute from a `lsst.pex.config.Config` instance.
796
797 Parameters
798 ----------
799 instance : `lsst.pex.config.Config`
800 The config instance that contains this field.
801 at : `list` of `lsst.pex.config.callStack.StackFrame`
802 The call stack (created by
803 `lsst.pex.config.callStack.getCallStack`).
804 label : `str`, optional
805 Event label for the history.
806
807 Notes
808 -----
809 This is invoked by the owning `~lsst.pex.config.Config` object and
810 should not be called directly.
811 """
812 if at is None:
813 at = getCallStack()
814 self.__set__(instance, None, at=at, label=label)
815
816 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
817 """Compare a field (named `Field.name`) in two
818 `~lsst.pex.config.Config` instances for equality.
819
820 Parameters
821 ----------
822 instance1 : `lsst.pex.config.Config`
823 Left-hand side `Config` instance to compare.
824 instance2 : `lsst.pex.config.Config`
825 Right-hand side `Config` instance to compare.
826 shortcut : `bool`, optional
827 **Unused.**
828 rtol : `float`, optional
829 Relative tolerance for floating point comparisons.
830 atol : `float`, optional
831 Absolute tolerance for floating point comparisons.
832 output : callable, optional
833 A callable that takes a string, used (possibly repeatedly) to
834 report inequalities.
835
836 Notes
837 -----
838 This method must be overridden by more complex `Field` subclasses.
839
840 See Also
841 --------
842 lsst.pex.config.compareScalars
843 """
844 v1 = getattr(instance1, self.namename)
845 v2 = getattr(instance2, self.namename)
846 name = getComparisonName(
847 _joinNamePath(instance1._name, self.namename), _joinNamePath(instance2._name, self.namename)
848 )
849 return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
850
851
853 """Importer (for `sys.meta_path`) that records which modules are being
854 imported.
855
856 *This class does not do any importing itself.*
857
858 Examples
859 --------
860 Use this class as a context manager to ensure it is properly uninstalled
861 when done:
862
863 >>> with RecordingImporter() as importer:
864 ... # import stuff
865 ... import numpy as np
866 ... print("Imported: " + importer.getModules())
867 """
868
869 def __init__(self):
870 self._modules = set()
871
872 def __enter__(self):
873 self.origMetaPath = sys.meta_path
874 sys.meta_path = [self] + sys.meta_path # type: ignore
875 return self
876
877 def __exit__(self, *args):
878 self.uninstall()
879 return False # Don't suppress exceptions
880
881 def uninstall(self):
882 """Uninstall the importer."""
883 sys.meta_path = self.origMetaPath
884
885 def find_spec(self, fullname, path, target=None):
886 """Find a module.
887
888 Called as part of the ``import`` chain of events.
889
890 Parameters
891 ----------
892 fullname : `str`
893 Name of module.
894 path : `list` [`str`]
895 Search path. Unused.
896 target : `~typing.Any`, optional
897 Unused.
898 """
899 self._modules.add(fullname)
900 # Return None because we don't do any importing.
901 return None
902
903 def getModules(self):
904 """Get the set of modules that were imported.
905
906 Returns
907 -------
908 modules : `set` of `str`
909 Set of imported module names.
910 """
911 return self._modules
912
913
914# type ignore because type checker thinks ConfigMeta is Generic when it is not
915class Config(metaclass=ConfigMeta): # type: ignore
916 """Base class for configuration (*config*) objects.
917
918 Notes
919 -----
920 A ``Config`` object will usually have several `~lsst.pex.config.Field`
921 instances as class attributes. These are used to define most of the base
922 class behavior.
923
924 ``Config`` implements a mapping API that provides many `dict`-like methods,
925 such as `keys`, `values`, and `items`. ``Config`` instances also support
926 the ``in`` operator to test if a field is in the config. Unlike a `dict`,
927 ``Config`` classes are not subscriptable. Instead, access individual
928 fields as attributes of the configuration instance.
929
930 Examples
931 --------
932 Config classes are subclasses of ``Config`` that have
933 `~lsst.pex.config.Field` instances (or instances of
934 `~lsst.pex.config.Field` subclasses) as class attributes:
935
936 >>> from lsst.pex.config import Config, Field, ListField
937 >>> class DemoConfig(Config):
938 ... intField = Field(doc="An integer field", dtype=int, default=42)
939 ... listField = ListField(doc="List of favorite beverages.", dtype=str,
940 ... default=['coffee', 'green tea', 'water'])
941 ...
942 >>> config = DemoConfig()
943
944 Configs support many `dict`-like APIs:
945
946 >>> config.keys()
947 ['intField', 'listField']
948 >>> 'intField' in config
949 True
950
951 Individual fields can be accessed as attributes of the configuration:
952
953 >>> config.intField
954 42
955 >>> config.listField.append('earl grey tea')
956 >>> print(config.listField)
957 ['coffee', 'green tea', 'water', 'earl grey tea']
958 """
959
960 _storage: dict[str, Any]
961 _fields: dict[str, Field]
962 _history: dict[str, list[Any]]
963 _imports: set[Any]
964
965 def __iter__(self):
966 """Iterate over fields."""
967 return self._fields_fields.__iter__()
968
969 def keys(self):
970 """Get field names.
971
972 Returns
973 -------
974 names : `~collections.abc.KeysView`
975 List of `lsst.pex.config.Field` names.
976 """
977 return self._storage.keys()
978
979 def values(self):
980 """Get field values.
981
982 Returns
983 -------
984 values : `~collections.abc.ValuesView`
985 Iterator of field values.
986 """
987 return self._storage.values()
988
989 def items(self):
990 """Get configurations as ``(field name, field value)`` pairs.
991
992 Returns
993 -------
994 items : `~collections.abc.ItemsView`
995 Iterator of tuples for each configuration. Tuple items are:
996
997 0. Field name.
998 1. Field value.
999 """
1000 return self._storage.items()
1001
1002 def __contains__(self, name):
1003 """Return `True` if the specified field exists in this config.
1004
1005 Parameters
1006 ----------
1007 name : `str`
1008 Field name to test for.
1009
1010 Returns
1011 -------
1012 in : `bool`
1013 `True` if the specified field exists in the config.
1014 """
1015 return self._storage.__contains__(name)
1016
1017 def __new__(cls, *args, **kw):
1018 """Allocate a new `lsst.pex.config.Config` object.
1019
1020 In order to ensure that all Config object are always in a proper state
1021 when handed to users or to derived `~lsst.pex.config.Config` classes,
1022 some attributes are handled at allocation time rather than at
1023 initialization.
1024
1025 This ensures that even if a derived `~lsst.pex.config.Config` class
1026 implements ``__init__``, its author does not need to be concerned about
1027 when or even the base ``Config.__init__`` should be called.
1028 """
1029 name = kw.pop("__name", None)
1030 at = kw.pop("__at", getCallStack())
1031 # remove __label and ignore it
1032 kw.pop("__label", "default")
1033
1034 instance = object.__new__(cls)
1035 instance._frozen = False
1036 instance._name = name
1037 instance._storage = {}
1038 instance._history = {}
1039 instance._imports = set()
1040 # load up defaults
1041 for field in instance._fields.values():
1042 instance._history[field.name] = []
1043 field.__set__(instance, field.default, at=at + [field.source], label="default")
1044 # set custom default-overrides
1045 instance.setDefaults()
1046 # set constructor overrides
1047 instance.update(__at=at, **kw)
1048 return instance
1049
1050 def __reduce__(self):
1051 """Reduction for pickling (function with arguments to reproduce).
1052
1053 We need to condense and reconstitute the `~lsst.pex.config.Config`,
1054 since it may contain lambdas (as the ``check`` elements) that cannot
1055 be pickled.
1056 """
1057 # The stream must be in characters to match the API but pickle
1058 # requires bytes
1059 stream = io.StringIO()
1060 self.saveToStream(stream)
1061 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
1062
1063 def setDefaults(self):
1064 """Subclass hook for computing defaults.
1065
1066 Notes
1067 -----
1068 Derived `~lsst.pex.config.Config` classes that must compute defaults
1069 rather than using the `~lsst.pex.config.Field` instances's defaults
1070 should do so here. To correctly use inherited defaults,
1071 implementations of ``setDefaults`` must call their base class's
1072 ``setDefaults``.
1073 """
1074 pass
1075
1076 def update(self, **kw):
1077 """Update values of fields specified by the keyword arguments.
1078
1079 Parameters
1080 ----------
1081 **kw
1082 Keywords are configuration field names. Values are configuration
1083 field values.
1084
1085 Notes
1086 -----
1087 The ``__at`` and ``__label`` keyword arguments are special internal
1088 keywords. They are used to strip out any internal steps from the
1089 history tracebacks of the config. Do not modify these keywords to
1090 subvert a `~lsst.pex.config.Config` instance's history.
1091
1092 Examples
1093 --------
1094 This is a config with three fields:
1095
1096 >>> from lsst.pex.config import Config, Field
1097 >>> class DemoConfig(Config):
1098 ... fieldA = Field(doc='Field A', dtype=int, default=42)
1099 ... fieldB = Field(doc='Field B', dtype=bool, default=True)
1100 ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
1101 ...
1102 >>> config = DemoConfig()
1103
1104 These are the default values of each field:
1105
1106 >>> for name, value in config.iteritems():
1107 ... print(f"{name}: {value}")
1108 ...
1109 fieldA: 42
1110 fieldB: True
1111 fieldC: 'Hello world'
1112
1113 Using this method to update ``fieldA`` and ``fieldC``:
1114
1115 >>> config.update(fieldA=13, fieldC='Updated!')
1116
1117 Now the values of each field are:
1118
1119 >>> for name, value in config.iteritems():
1120 ... print(f"{name}: {value}")
1121 ...
1122 fieldA: 13
1123 fieldB: True
1124 fieldC: 'Updated!'
1125 """
1126 at = kw.pop("__at", getCallStack())
1127 label = kw.pop("__label", "update")
1128
1129 for name, value in kw.items():
1130 try:
1131 field = self._fields[name]
1132 field.__set__(self, value, at=at, label=label)
1133 except KeyError:
1134 raise KeyError(f"No field of name {name} exists in config type {_typeStr(self)}")
1135
1136 def load(self, filename, root="config"):
1137 """Modify this config in place by executing the Python code in a
1138 configuration file.
1139
1140 Parameters
1141 ----------
1142 filename : `str`
1143 Name of the configuration file. A configuration file is Python
1144 module.
1145 root : `str`, optional
1146 Name of the variable in file that refers to the config being
1147 overridden.
1148
1149 For example, the value of root is ``"config"`` and the file
1150 contains::
1151
1152 config.myField = 5
1153
1154 Then this config's field ``myField`` is set to ``5``.
1155
1156 See Also
1157 --------
1158 lsst.pex.config.Config.loadFromStream
1159 lsst.pex.config.Config.loadFromString
1160 lsst.pex.config.Config.save
1161 lsst.pex.config.Config.saveToStream
1162 lsst.pex.config.Config.saveToString
1163 """
1164 with open(filename) as f:
1165 code = compile(f.read(), filename=filename, mode="exec")
1166 self.loadFromString(code, root=root, filename=filename)
1167
1168 def loadFromStream(self, stream, root="config", filename=None, extraLocals=None):
1169 """Modify this Config in place by executing the Python code in the
1170 provided stream.
1171
1172 Parameters
1173 ----------
1174 stream : file-like object, `str`, `bytes`, or `~types.CodeType`
1175 Stream containing configuration override code. If this is a
1176 code object, it should be compiled with ``mode="exec"``.
1177 root : `str`, optional
1178 Name of the variable in file that refers to the config being
1179 overridden.
1180
1181 For example, the value of root is ``"config"`` and the file
1182 contains::
1183
1184 config.myField = 5
1185
1186 Then this config's field ``myField`` is set to ``5``.
1187 filename : `str`, optional
1188 Name of the configuration file, or `None` if unknown or contained
1189 in the stream. Used for error reporting.
1190 extraLocals : `dict` of `str` to `object`, optional
1191 Any extra variables to include in local scope when loading.
1192
1193 Notes
1194 -----
1195 For backwards compatibility reasons, this method accepts strings, bytes
1196 and code objects as well as file-like objects. New code should use
1197 `loadFromString` instead for most of these types.
1198
1199 See Also
1200 --------
1201 lsst.pex.config.Config.load
1202 lsst.pex.config.Config.loadFromString
1203 lsst.pex.config.Config.save
1204 lsst.pex.config.Config.saveToStream
1205 lsst.pex.config.Config.saveToString
1206 """
1207 if hasattr(stream, "read"):
1208 if filename is None:
1209 filename = getattr(stream, "name", "?")
1210 code = compile(stream.read(), filename=filename, mode="exec")
1211 else:
1212 code = stream
1213 self.loadFromString(code, root=root, filename=filename, extraLocals=extraLocals)
1214
1215 def loadFromString(self, code, root="config", filename=None, extraLocals=None):
1216 """Modify this Config in place by executing the Python code in the
1217 provided string.
1218
1219 Parameters
1220 ----------
1221 code : `str`, `bytes`, or `~types.CodeType`
1222 Stream containing configuration override code.
1223 root : `str`, optional
1224 Name of the variable in file that refers to the config being
1225 overridden.
1226
1227 For example, the value of root is ``"config"`` and the file
1228 contains::
1229
1230 config.myField = 5
1231
1232 Then this config's field ``myField`` is set to ``5``.
1233 filename : `str`, optional
1234 Name of the configuration file, or `None` if unknown or contained
1235 in the stream. Used for error reporting.
1236 extraLocals : `dict` of `str` to `object`, optional
1237 Any extra variables to include in local scope when loading.
1238
1239 Raises
1240 ------
1241 ValueError
1242 Raised if a key in extraLocals is the same value as the value of
1243 the root argument.
1244
1245 See Also
1246 --------
1247 lsst.pex.config.Config.load
1248 lsst.pex.config.Config.loadFromStream
1249 lsst.pex.config.Config.save
1250 lsst.pex.config.Config.saveToStream
1251 lsst.pex.config.Config.saveToString
1252 """
1253 if filename is None:
1254 # try to determine the file name; a compiled string
1255 # has attribute "co_filename",
1256 filename = getattr(code, "co_filename", "?")
1257 with RecordingImporter() as importer:
1258 globals = {"__file__": filename}
1259 local = {root: self}
1260 if extraLocals is not None:
1261 # verify the value of root was not passed as extra local args
1262 if root in extraLocals:
1263 raise ValueError(
1264 f"{root} is reserved and cannot be used as a variable name in extraLocals"
1265 )
1266 local.update(extraLocals)
1267 exec(code, globals, local)
1268
1269 self._imports_imports.update(importer.getModules())
1270
1271 def save(self, filename, root="config"):
1272 """Save a Python script to the named file, which, when loaded,
1273 reproduces this config.
1274
1275 Parameters
1276 ----------
1277 filename : `str`
1278 Desination filename of this configuration.
1279 root : `str`, optional
1280 Name to use for the root config variable. The same value must be
1281 used when loading (see `lsst.pex.config.Config.load`).
1282
1283 See Also
1284 --------
1285 lsst.pex.config.Config.saveToStream
1286 lsst.pex.config.Config.saveToString
1287 lsst.pex.config.Config.load
1288 lsst.pex.config.Config.loadFromStream
1289 lsst.pex.config.Config.loadFromString
1290 """
1291 d = os.path.dirname(filename)
1292 with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1293 self.saveToStream(outfile, root)
1294 # tempfile is hardcoded to create files with mode '0600'
1295 # for an explantion of these antics see:
1296 # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1297 umask = os.umask(0o077)
1298 os.umask(umask)
1299 os.chmod(outfile.name, (~umask & 0o666))
1300 # chmod before the move so we get quasi-atomic behavior if the
1301 # source and dest. are on the same filesystem.
1302 # os.rename may not work across filesystems
1303 shutil.move(outfile.name, filename)
1304
1305 def saveToString(self, skipImports=False):
1306 """Return the Python script form of this configuration as an executable
1307 string.
1308
1309 Parameters
1310 ----------
1311 skipImports : `bool`, optional
1312 If `True` then do not include ``import`` statements in output,
1313 this is to support human-oriented output from ``pipetask`` where
1314 additional clutter is not useful.
1315
1316 Returns
1317 -------
1318 code : `str`
1319 A code string readable by `loadFromString`.
1320
1321 See Also
1322 --------
1323 lsst.pex.config.Config.save
1324 lsst.pex.config.Config.saveToStream
1325 lsst.pex.config.Config.load
1326 lsst.pex.config.Config.loadFromStream
1327 lsst.pex.config.Config.loadFromString
1328 """
1329 buffer = io.StringIO()
1330 self.saveToStream(buffer, skipImports=skipImports)
1331 return buffer.getvalue()
1332
1333 def saveToStream(self, outfile, root="config", skipImports=False):
1334 """Save a configuration file to a stream, which, when loaded,
1335 reproduces this config.
1336
1337 Parameters
1338 ----------
1339 outfile : file-like object
1340 Destination file object write the config into. Accepts strings not
1341 bytes.
1342 root : `str`, optional
1343 Name to use for the root config variable. The same value must be
1344 used when loading (see `lsst.pex.config.Config.load`).
1345 skipImports : `bool`, optional
1346 If `True` then do not include ``import`` statements in output,
1347 this is to support human-oriented output from ``pipetask`` where
1348 additional clutter is not useful.
1349
1350 See Also
1351 --------
1352 lsst.pex.config.Config.save
1353 lsst.pex.config.Config.saveToString
1354 lsst.pex.config.Config.load
1355 lsst.pex.config.Config.loadFromStream
1356 lsst.pex.config.Config.loadFromString
1357 """
1358 tmp = self._name
1359 self._rename(root)
1360 try:
1361 if not skipImports:
1362 self._collectImports()
1363 # Remove self from the set, as it is handled explicitly below
1364 self._imports_imports.remove(self.__module__)
1365 configType = type(self)
1366 typeString = _typeStr(configType)
1367 outfile.write(f"import {configType.__module__}\n")
1368 # We are required to write this on a single line because
1369 # of later regex matching, rather than adopting black style
1370 # formatting.
1371 outfile.write(
1372 f'assert type({root}) is {typeString}, f"config is of type '
1373 f'{{type({root}).__module__}}.{{type({root}).__name__}} instead of {typeString}"\n\n'
1374 )
1375 for imp in sorted(self._imports_imports):
1376 if imp in sys.modules and sys.modules[imp] is not None:
1377 outfile.write(f"import {imp}\n")
1378 self._save(outfile)
1379 finally:
1380 self._rename(tmp)
1381
1382 def freeze(self):
1383 """Make this config, and all subconfigs, read-only."""
1384 self._frozen = True
1385 for field in self._fields_fields.values():
1386 field.freeze(self)
1387
1388 def _save(self, outfile):
1389 """Save this config to an open stream object.
1390
1391 Parameters
1392 ----------
1393 outfile : file-like object
1394 Destination file object write the config into. Accepts strings not
1395 bytes.
1396 """
1397 for field in self._fields_fields.values():
1398 field.save(outfile, self)
1399
1401 """Add module containing self to the list of things to import and
1402 then loops over all the fields in the config calling a corresponding
1403 collect method.
1404
1405 The field method will call _collectImports on any
1406 configs it may own and return the set of things to import. This
1407 returned set will be merged with the set of imports for this config
1408 class.
1409 """
1410 self._imports_imports.add(self.__module__)
1411 for name, field in self._fields_fields.items():
1412 field._collectImports(self, self._imports_imports)
1413
1414 def toDict(self):
1415 """Make a dictionary of field names and their values.
1416
1417 Returns
1418 -------
1419 dict_ : `dict`
1420 Dictionary with keys that are `~lsst.pex.config.Field` names.
1421 Values are `~lsst.pex.config.Field` values.
1422
1423 See Also
1424 --------
1425 lsst.pex.config.Field.toDict
1426
1427 Notes
1428 -----
1429 This method uses the `~lsst.pex.config.Field.toDict` method of
1430 individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1431 implement a ``toDict`` method for *this* method to work.
1432 """
1433 dict_ = {}
1434 for name, field in self._fields_fields.items():
1435 dict_[name] = field.toDict(self)
1436 return dict_
1437
1438 def names(self):
1439 """Get all the field names in the config, recursively.
1440
1441 Returns
1442 -------
1443 names : `list` of `str`
1444 Field names.
1445 """
1446 #
1447 # Rather than sort out the recursion all over again use the
1448 # pre-existing saveToStream()
1449 #
1450 with io.StringIO() as strFd:
1451 self.saveToStream(strFd, "config")
1452 contents = strFd.getvalue()
1453 strFd.close()
1454 #
1455 # Pull the names out of the dumped config
1456 #
1457 keys = []
1458 for line in contents.split("\n"):
1459 if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1460 continue
1461
1462 mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1463 if mat:
1464 keys.append(mat.group(1))
1465
1466 return keys
1467
1468 def _rename(self, name):
1469 """Rename this config object in its parent `~lsst.pex.config.Config`.
1470
1471 Parameters
1472 ----------
1473 name : `str`
1474 New name for this config in its parent `~lsst.pex.config.Config`.
1475
1476 Notes
1477 -----
1478 This method uses the `~lsst.pex.config.Field.rename` method of
1479 individual `lsst.pex.config.Field` instances.
1480 `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1481 method for *this* method to work.
1482
1483 See Also
1484 --------
1485 lsst.pex.config.Field.rename
1486 """
1487 self._name = name
1488 for field in self._fields_fields.values():
1489 field.rename(self)
1490
1491 def validate(self):
1492 """Validate the Config, raising an exception if invalid.
1493
1494 Raises
1495 ------
1496 lsst.pex.config.FieldValidationError
1497 Raised if verification fails.
1498
1499 Notes
1500 -----
1501 The base class implementation performs type checks on all fields by
1502 calling their `~lsst.pex.config.Field.validate` methods.
1503
1504 Complex single-field validation can be defined by deriving new Field
1505 types. For convenience, some derived `lsst.pex.config.Field`-types
1506 (`~lsst.pex.config.ConfigField` and
1507 `~lsst.pex.config.ConfigChoiceField`) are defined in
1508 ``lsst.pex.config`` that handle recursing into subconfigs.
1509
1510 Inter-field relationships should only be checked in derived
1511 `~lsst.pex.config.Config` classes after calling this method, and base
1512 validation is complete.
1513 """
1514 for field in self._fields_fields.values():
1515 field.validate(self)
1516
1517 def formatHistory(self, name, **kwargs):
1518 """Format a configuration field's history to a human-readable format.
1519
1520 Parameters
1521 ----------
1522 name : `str`
1523 Name of a `~lsst.pex.config.Field` in this config.
1524 **kwargs
1525 Keyword arguments passed to `lsst.pex.config.history.format`.
1526
1527 Returns
1528 -------
1529 history : `str`
1530 A string containing the formatted history.
1531
1532 See Also
1533 --------
1534 lsst.pex.config.history.format
1535 """
1536 import lsst.pex.config.history as pexHist
1537
1538 return pexHist.format(self, name, **kwargs)
1539
1540 history = property(lambda x: x._history)
1541 """Read-only history.
1542 """
1543
1544 def __setattr__(self, attr, value, at=None, label="assignment"):
1545 """Set an attribute (such as a field's value).
1546
1547 Notes
1548 -----
1549 Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1550 locked such that no additional attributes nor properties may be added
1551 to them dynamically.
1552
1553 Although this is not the standard Python behavior, it helps to protect
1554 users from accidentally mispelling a field name, or trying to set a
1555 non-existent field.
1556 """
1557 if attr in self._fields_fields:
1558 if self._fields_fields[attr].deprecated is not None:
1559 fullname = _joinNamePath(self._name, self._fields_fields[attr].name)
1560 warnings.warn(
1561 f"Config field {fullname} is deprecated: {self._fields[attr].deprecated}",
1562 FutureWarning,
1563 stacklevel=2,
1564 )
1565 if at is None:
1566 at = getCallStack()
1567 # This allows Field descriptors to work.
1568 self._fields_fields[attr].__set__(self, value, at=at, label=label)
1569 elif hasattr(getattr(self.__class__, attr, None), "__set__"):
1570 # This allows properties and other non-Field descriptors to work.
1571 return object.__setattr__(self, attr, value)
1572 elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"):
1573 # This allows specific private attributes to work.
1574 self.__dict__[attr] = value
1575 else:
1576 # We throw everything else.
1577 raise AttributeError(f"{_typeStr(self)} has no attribute {attr}")
1578
1579 def __delattr__(self, attr, at=None, label="deletion"):
1580 if attr in self._fields_fields:
1581 if at is None:
1582 at = getCallStack()
1583 self._fields_fields[attr].__delete__(self, at=at, label=label)
1584 else:
1585 object.__delattr__(self, attr)
1586
1587 def __eq__(self, other):
1588 if type(other) is type(self):
1589 for name in self._fields_fields:
1590 thisValue = getattr(self, name)
1591 otherValue = getattr(other, name)
1592 if isinstance(thisValue, float) and math.isnan(thisValue):
1593 if not math.isnan(otherValue):
1594 return False
1595 elif thisValue != otherValue:
1596 return False
1597 return True
1598 return False
1599
1600 def __ne__(self, other):
1601 return not self.__eq__(other)
1602
1603 def __str__(self):
1604 return str(self.toDict())
1605
1606 def __repr__(self):
1607 return "{}({})".format(
1608 _typeStr(self),
1609 ", ".join(f"{k}={v!r}" for k, v in self.toDict().items() if v is not None),
1610 )
1611
1612 def compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None):
1613 """Compare this configuration to another `~lsst.pex.config.Config` for
1614 equality.
1615
1616 Parameters
1617 ----------
1618 other : `lsst.pex.config.Config`
1619 Other `~lsst.pex.config.Config` object to compare against this
1620 config.
1621 shortcut : `bool`, optional
1622 If `True`, return as soon as an inequality is found. Default is
1623 `True`.
1624 rtol : `float`, optional
1625 Relative tolerance for floating point comparisons.
1626 atol : `float`, optional
1627 Absolute tolerance for floating point comparisons.
1628 output : callable, optional
1629 A callable that takes a string, used (possibly repeatedly) to
1630 report inequalities.
1631
1632 Returns
1633 -------
1634 isEqual : `bool`
1635 `True` when the two `lsst.pex.config.Config` instances are equal.
1636 `False` if there is an inequality.
1637
1638 See Also
1639 --------
1640 lsst.pex.config.compareConfigs
1641
1642 Notes
1643 -----
1644 Unselected targets of `~lsst.pex.config.RegistryField` fields and
1645 unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1646 are not considered by this method.
1647
1648 Floating point comparisons are performed by `numpy.allclose`.
1649 """
1650 name1 = self._name if self._name is not None else "config"
1651 name2 = other._name if other._name is not None else "config"
1652 name = getComparisonName(name1, name2)
1653 return compareConfigs(name, self, other, shortcut=shortcut, rtol=rtol, atol=atol, output=output)
1654
1655 @classmethod
1656 def __init_subclass__(cls, **kwargs):
1657 """Run initialization for every subclass.
1658
1659 Specifically registers the subclass with a YAML representer
1660 and YAML constructor (if pyyaml is available)
1661 """
1662 super().__init_subclass__(**kwargs)
1663
1664 if not yaml:
1665 return
1666
1667 yaml.add_representer(cls, _yaml_config_representer)
1668
1669 @classmethod
1670 def _fromPython(cls, config_py):
1671 """Instantiate a `Config`-subclass from serialized Python form.
1672
1673 Parameters
1674 ----------
1675 config_py : `str`
1676 A serialized form of the Config as created by
1677 `Config.saveToStream`.
1678
1679 Returns
1680 -------
1681 config : `Config`
1682 Reconstructed `Config` instant.
1683 """
1684 cls = _classFromPython(config_py)
1685 return unreduceConfig(cls, config_py)
1686
1687
1688def _classFromPython(config_py):
1689 """Return the Config subclass required by this Config serialization.
1690
1691 Parameters
1692 ----------
1693 config_py : `str`
1694 A serialized form of the Config as created by
1695 `Config.saveToStream`.
1696
1697 Returns
1698 -------
1699 cls : `type`
1700 The `Config` subclass associated with this config.
1701 """
1702 # standard serialization has the form:
1703 # import config.class
1704 # assert type(config) is config.class.Config, ...
1705 # Older files use "type(config)==" instead.
1706 # We want to parse these two lines so we can get the class itself
1707
1708 # Do a single regex to avoid large string copies when splitting a
1709 # large config into separate lines.
1710 # The assert regex cannot be greedy because the assert error string
1711 # can include both "," and " is ".
1712 matches = re.search(r"^import ([\w.]+)\nassert type\‍(\S+\‍)(?:\s*==\s*| is )(.*?),", config_py)
1713
1714 if not matches:
1715 first_line, second_line, _ = config_py.split("\n", 2)
1716 raise ValueError(
1717 f"First two lines did not match expected form. Got:\n - {first_line}\n - {second_line}"
1718 )
1719
1720 module_name = matches.group(1)
1721 module = importlib.import_module(module_name)
1722
1723 # Second line
1724 full_name = matches.group(2)
1725
1726 # Remove the module name from the full name
1727 if not full_name.startswith(module_name):
1728 raise ValueError(f"Module name ({module_name}) inconsistent with full name ({full_name})")
1729
1730 # if module name is a.b.c and full name is a.b.c.d.E then
1731 # we need to remove a.b.c. and iterate over the remainder
1732 # The +1 is for the extra dot after a.b.c
1733 remainder = full_name[len(module_name) + 1 :]
1734 components = remainder.split(".")
1735 pytype = module
1736 for component in components:
1737 pytype = getattr(pytype, component)
1738 return pytype
1739
1740
1741def unreduceConfig(cls_, stream):
1742 """Create a `~lsst.pex.config.Config` from a stream.
1743
1744 Parameters
1745 ----------
1746 cls_ : `lsst.pex.config.Config`-type
1747 A `lsst.pex.config.Config` type (not an instance) that is instantiated
1748 with configurations in the ``stream``.
1749 stream : file-like object, `str`, or `~types.CodeType`
1750 Stream containing configuration override code.
1751
1752 Returns
1753 -------
1754 config : `lsst.pex.config.Config`
1755 Config instance.
1756
1757 See Also
1758 --------
1759 lsst.pex.config.Config.loadFromStream
1760 """
1761 config = cls_()
1762 config.loadFromStream(stream)
1763 return config
std::vector< SchemaItem< Flag > > * items
Any __call__(self, *Any args, **Any kwds)
Definition config.py:92
saveToStream(self, outfile, root="config", skipImports=False)
Definition config.py:1333
__setattr__(self, attr, value, at=None, label="assignment")
Definition config.py:1544
loadFromStream(self, stream, root="config", filename=None, extraLocals=None)
Definition config.py:1168
__new__(cls, *args, **kw)
Definition config.py:1017
save(self, filename, root="config")
Definition config.py:1271
_fromPython(cls, config_py)
Definition config.py:1670
loadFromString(self, code, root="config", filename=None, extraLocals=None)
Definition config.py:1215
__delattr__(self, attr, at=None, label="deletion")
Definition config.py:1579
saveToString(self, skipImports=False)
Definition config.py:1305
formatHistory(self, name, **kwargs)
Definition config.py:1517
__init_subclass__(cls, **kwargs)
Definition config.py:1656
load(self, filename, root="config")
Definition config.py:1136
compare(self, other, shortcut=True, rtol=1e-8, atol=1e-8, output=None)
Definition config.py:1612
__setattr__(cls, name, value)
Definition config.py:240
__init__(cls, name, bases, dict_)
Definition config.py:219
Mapping[str, Any] _parseTypingArgs(tuple[type,...]|tuple[str,...] params, Mapping[str, Any] kwds)
Definition config.py:406
save(self, outfile, instance)
Definition config.py:634
_collectImports(self, instance, imports)
Definition config.py:620
__get__(self, instance, owner=None, at=None, label="default")
Definition config.py:707
FieldTypeVar __get__(self, Config instance, Any owner=None, Any at=None, str label="default")
Definition config.py:705
__delete__(self, instance, at=None, label="deletion")
Definition config.py:794
Field[FieldTypeVar] __get__(self, None instance, Any owner=None, Any at=None, str label="default")
Definition config.py:700
rename(self, instance)
Definition config.py:525
_compare(self, instance1, instance2, shortcut, rtol, atol, output)
Definition config.py:816
toDict(self, instance)
Definition config.py:670
_validateValue(self, value)
Definition config.py:592
None __set__(self, Config instance, FieldTypeVar|None value, Any at=None, str label="assignment")
Definition config.py:736
__class_getitem__(cls, tuple[type,...]|type|ForwardRef params)
Definition config.py:458
freeze(self, instance)
Definition config.py:575
__init__(self, doc, dtype=None, default=None, check=None, optional=False, deprecated=None)
Definition config.py:461
_setup(self, doc, dtype, default, check, optional, source, deprecated)
Definition config.py:480
__init__(self, field, config, msg)
Definition config.py:261
find_spec(self, fullname, path, target=None)
Definition config.py:885
_autocast(x, dtype)
Definition config.py:121
_yaml_config_representer(dumper, data)
Definition config.py:168
_classFromPython(config_py)
Definition config.py:1688
_joinNamePath(prefix=None, name=None, index=None)
Definition config.py:106
unreduceConfig(cls_, stream)
Definition config.py:1741