22 __all__ = (
"Config",
"Field",
"FieldValidationError")
33 from .comparison
import getComparisonName, compareScalars, compareConfigs
34 from .callStack
import getStackFrame, getCallStack
37 def _joinNamePath(prefix=None, name=None, index=None):
39 Utility function for generating nested configuration names 41 if not prefix
and not name:
42 raise ValueError(
"Invalid name: cannot be None")
46 name = prefix +
"." + name
49 return "%s[%r]" % (name, index)
54 def _autocast(x, dtype):
56 If appropriate perform type casting of value x to type dtype, 57 otherwise return the original value x 59 if dtype == float
and isinstance(x, int):
66 Utility function to generate a fully qualified type name. 68 This is used primarily in writing config files to be 69 executed later upon 'load'. 71 if hasattr(x,
'__module__')
and hasattr(x,
'__name__'):
75 if (sys.version_info.major <= 2
and xtype.__module__ ==
'__builtin__')
or xtype.__module__ ==
'builtins':
78 return "%s.%s" % (xtype.__module__, xtype.__name__)
82 """A metaclass for Config 84 Adds a dictionary containing all Field class attributes 85 as a class attribute called '_fields', and adds the name of each field as 86 an instance variable of the field itself (so you don't have to pass the 87 name of the field to the field constructor). 90 type.__init__(cls, name, bases, dict_)
94 def getFields(classtype):
96 bases =
list(classtype.__bases__)
99 fields.update(getFields(b))
101 for k, v
in classtype.__dict__.items():
102 if isinstance(v, Field):
106 fields = getFields(cls)
107 for k, v
in fields.items():
108 setattr(cls, k, copy.deepcopy(v))
111 if isinstance(value, Field):
114 type.__setattr__(cls, name, value)
119 Custom exception class which holds additional information useful to 120 debuggin Config errors: 121 - fieldType: type of the Field which incurred the error 122 - fieldName: name of the Field which incurred the error 123 - fullname: fully qualified name of the Field instance 124 - history: full history of all changes to the Field instance 125 - fieldSource: file and line number of the Field definition 130 self.
fullname = _joinNamePath(config._name, field.name)
131 self.
history = config.history.setdefault(field.name, [])
134 error =
"%s '%s' failed validation: %s\n"\
135 "For more information read the Field definition at:\n%s"\
136 "And the Config definition at:\n%s" % \
139 ValueError.__init__(self, error)
143 """A field in a a Config. 145 Instances of Field should be class attributes of Config subclasses: 146 Field only supports basic data types (int, float, complex, bool, str) 148 class Example(Config): 149 myInt = Field(int, "an integer field!", default=0) 153 supportedTypes =
set((str, bool, float, int, complex))
155 def __init__(self, doc, dtype, default=None, check=None, optional=False):
156 """Initialize a Field. 158 dtype ------ Data type for the field. 159 doc -------- Documentation for the field. 160 default ---- A default value for the field. 161 check ------ A callable to be called with the field value that returns 162 False if the value is invalid. More complex inter-field 163 validation can be written as part of Config validate() 164 method; this will be ignored if set to None. 165 optional --- When False, Config validate() will fail if value is None 168 raise ValueError(
"Unsupported Field dtype %s" % _typeStr(dtype))
171 self.
_setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
173 def _setup(self, doc, dtype, default, check, optional, source):
175 Convenience function provided to simplify initialization of derived 180 self.
__doc__ = doc+
" (`"+dtype.__name__+
"`, default "+
'``{0!r}``'.
format(default)+
")" 188 Rename an instance of this field, not the field itself. 189 This is invoked by the owning config object and should not be called 192 Only useful for fields which hold sub-configs. 193 Fields which hold subconfigs should rename each sub-config with 194 the full field name as generated by _joinNamePath 200 Base validation for any field. 201 Ensures that non-optional fields are not None. 202 Ensures type correctness 203 Ensures that user-provided check function is valid 204 Most derived Field types should call Field.validate if they choose 205 to re-implement validate 207 value = self.__get__(instance)
208 if not self.optional
and value
is None:
209 raise FieldValidationError(self, instance,
"Required value cannot be None")
213 Make this field read-only. 214 Only important for fields which hold sub-configs. 215 Fields which hold subconfigs should freeze each sub-config. 219 def _validateValue(self, value):
221 Validate a value that is not None 223 This is called from __set__ 224 This is not part of the Field API. However, simple derived field types 225 may benefit from implementing _validateValue 230 if not isinstance(value, self.dtype):
231 msg =
"Value %s is of incorrect type %s. Expected type %s" % \
232 (value, _typeStr(value), _typeStr(self.dtype))
234 if self.check
is not None and not self.check(value):
235 msg =
"Value %s is not a valid value" %
str(value)
236 raise ValueError(msg)
238 def save(self, outfile, instance):
240 Saves an instance of this field to file. 241 This is invoked by the owning config object, and should not be called 244 outfile ---- an open output stream. 247 fullname = _joinNamePath(instance._name, self.name)
250 doc =
"# " +
str(self.
doc).replace(
"\n",
"\n# ")
251 if isinstance(value, float)
and (math.isinf(value)
or math.isnan(value)):
253 outfile.write(
u"{}\n{}=float('{!r}')\n\n".
format(doc, fullname, value))
255 outfile.write(
u"{}\n{}={!r}\n\n".
format(doc, fullname, value))
259 Convert the field value so that it can be set as the value of an item 261 This is invoked by the owning config object and should not be called 264 Simple values are passed through. Complex data structures must be 265 manipulated. For example, a field holding a sub-config should, instead 266 of the subconfig object, return a dict where the keys are the field 267 names in the subconfig, and the values are the field values in the 272 def __get__(self, instance, owner=None, at=None, label="default"):
274 Define how attribute access should occur on the Config instance 275 This is invoked by the owning config object and should not be called 278 When the field attribute is accessed on a Config class object, it 279 returns the field object itself in order to allow inspection of 282 When the field attribute is access on a config instance, the actual 283 value described by the field (and held by the Config instance) is 286 if instance
is None or not isinstance(instance, Config):
289 return instance._storage[self.name]
291 def __set__(self, instance, value, at=None, label='assignment'):
293 Describe how attribute setting should occur on the config instance. 294 This is invoked by the owning config object and should not be called 297 Derived Field classes may need to override the behavior. When overriding 298 __set__, Field authors should follow the following rules: 299 * Do not allow modification of frozen configs 300 * Validate the new value *BEFORE* modifying the field. Except if the 301 new value is None. None is special and no attempt should be made to 302 validate it until Config.validate is called. 303 * Do not modify the Config instance to contain invalid values. 304 * If the field is modified, update the history of the field to reflect the 307 In order to decrease the need to implement this method in derived Field 308 types, value validation is performed in the method _validateValue. If 309 only the validation step differs in the derived Field, it is simpler to 310 implement _validateValue than to re-implement __set__. More complicated 311 behavior, however, may require a reimplementation. 316 history = instance._history.setdefault(self.name, [])
317 if value
is not None:
318 value = _autocast(value, self.
dtype)
321 except BaseException
as e:
324 instance._storage[self.name] = value
327 history.append((value, at, label))
331 Describe how attribute deletion should occur on the Config instance. 332 This is invoked by the owning config object and should not be called 337 self.
__set__(instance,
None, at=at, label=label)
339 def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
340 """Helper function for Config.compare; used to compare two fields for equality. 342 Must be overridden by more complex field types. 344 @param[in] instance1 LHS Config instance to compare. 345 @param[in] instance2 RHS Config instance to compare. 346 @param[in] shortcut If True, return as soon as an inequality is found. 347 @param[in] rtol Relative tolerance for floating point comparisons. 348 @param[in] atol Absolute tolerance for floating point comparisons. 349 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 350 to report inequalities. 352 Floating point comparisons are performed by numpy.allclose; refer to that for details. 354 v1 = getattr(instance1, self.name)
355 v2 = getattr(instance2, self.name)
357 _joinNamePath(instance1._name, self.name),
358 _joinNamePath(instance2._name, self.name)
364 """An Importer (for sys.meta_path) that records which modules are being imported. 366 Objects also act as Context Managers, so you can: 367 with RecordingImporter() as importer: 369 print("Imported: " + importer.getModules()) 370 This ensures it is properly uninstalled when done. 372 This class makes no effort to do any importing itself. 375 """Create and install the Importer""" 381 sys.meta_path = [self] + sys.meta_path
389 """Uninstall the Importer""" 393 """Called as part of the 'import' chain of events. 395 We return None because we don't do any importing. 401 """Return the set of modules that were imported.""" 406 """Base class for control objects. 408 A Config object will usually have several Field instances as class 409 attributes; these are used to define most of the base class behavior. 410 Simple derived class should be able to be defined simply by setting those 413 Config also emulates a dict of field name: field value 417 """!Iterate over fields 422 """!Return the list of field names 427 """!Return the list of field values 432 """!Return the list of (field name, field value) pairs 437 """!Iterate over (field name, field value) pairs 439 return iter(self._storage.
items())
442 """!Iterate over field values 444 return iter(self.storage.
values())
447 """!Iterate over field names 449 return iter(self.storage.
keys())
452 """!Return True if the specified field exists in this config 454 @param[in] name field name to test for 459 """!Allocate a new Config object. 461 In order to ensure that all Config object are always in a proper 462 state when handed to users or to derived Config classes, some 463 attributes are handled at allocation time rather than at initialization 465 This ensures that even if a derived Config class implements __init__, 466 the author does not need to be concerned about when or even if he 467 should call the base Config.__init__ 469 name = kw.pop(
"__name",
None)
472 kw.pop(
"__label",
"default")
474 instance = object.__new__(cls)
475 instance._frozen =
False 476 instance._name = name
477 instance._storage = {}
478 instance._history = {}
479 instance._imports =
set()
481 for field
in instance._fields.values():
482 instance._history[field.name] = []
483 field.__set__(instance, field.default, at=at + [field.source], label=
"default")
485 instance.setDefaults()
487 instance.update(__at=at, **kw)
491 """Reduction for pickling (function with arguments to reproduce). 493 We need to condense and reconstitute the Config, since it may contain lambdas 494 (as the 'check' elements) that cannot be pickled. 497 stream = io.StringIO()
499 return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
503 Derived config classes that must compute defaults rather than using the 504 Field defaults should do so here. 505 To correctly use inherited defaults, implementations of setDefaults() 506 must call their base class' setDefaults() 511 """!Update values specified by the keyword arguments 513 @warning The '__at' and '__label' keyword arguments are special internal 514 keywords. They are used to strip out any internal steps from the 515 history tracebacks of the config. Modifying these keywords allows users 516 to lie about a Config's history. Please do not do so! 519 label = kw.pop(
"__label",
"update")
521 for name, value
in kw.items():
523 field = self._fields[name]
524 field.__set__(self, value, at=at, label=label)
526 raise KeyError(
"No field of name %s exists in config type %s" % (name, _typeStr(self)))
528 def load(self, filename, root="config"):
529 """!Modify this config in place by executing the Python code in the named file. 531 @param[in] filename name of file containing config override code 532 @param[in] root name of variable in file that refers to the config being overridden 534 For example: if the value of root is "config" and the file contains this text: 535 "config.myField = 5" then this config's field "myField" is set to 5. 537 @deprecated For purposes of backwards compatibility, older config files that use 538 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 539 This feature will be removed at some point. 541 with open(filename,
"r") as f: 542 code = compile(f.read(), filename=filename, mode="exec")
546 """!Modify this config in place by executing the python code in the provided stream. 548 @param[in] stream open file object, string or compiled string containing config override code 549 @param[in] root name of variable in stream that refers to the config being overridden 550 @param[in] filename name of config override file, or None if unknown or contained 551 in the stream; used for error reporting 553 For example: if the value of root is "config" and the stream contains this text: 554 "config.myField = 5" then this config's field "myField" is set to 5. 556 @deprecated For purposes of backwards compatibility, older config files that use 557 root="root" instead of root="config" will be loaded with a warning printed to sys.stderr. 558 This feature will be removed at some point. 563 exec(stream, {}, local)
564 except NameError
as e:
565 if root ==
"config" and "root" in e.args[0]:
569 filename = getattr(stream,
"co_filename",
None)
571 filename = getattr(stream,
"name",
"?")
572 print(f
"Config override file {filename!r}" 573 " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
574 local = {
"root": self}
575 exec(stream, {}, local)
579 self._imports.
update(importer.getModules())
581 def save(self, filename, root="config"):
582 """!Save a python script to the named file, which, when loaded, reproduces this Config 584 @param[in] filename name of file to which to write the config 585 @param[in] root name to use for the root config variable; the same value must be used when loading 587 d = os.path.dirname(filename)
588 with tempfile.NamedTemporaryFile(mode=
"w", delete=
False, dir=d)
as outfile:
593 umask = os.umask(0o077)
595 os.chmod(outfile.name, (~umask & 0o666))
599 shutil.move(outfile.name, filename)
602 """!Save a python script to a stream, which, when loaded, reproduces this Config 604 @param outfile [inout] open file object to which to write the config. Accepts strings not bytes. 605 @param root [in] name to use for the root config variable; the same value must be used when loading 610 configType =
type(self)
611 typeString = _typeStr(configType)
612 outfile.write(
u"import {}\n".
format(configType.__module__))
613 outfile.write(
u"assert type({})=={}, 'config is of type %s.%s ".
format(root, typeString))
614 outfile.write(
u"instead of {}' % (type({}).__module__, type({}).__name__)\n".
format(typeString,
622 """!Make this Config and all sub-configs read-only 628 def _save(self, outfile):
629 """!Save this Config to an open stream object 631 for imp
in self._imports:
632 if imp
in sys.modules
and sys.modules[imp]
is not None:
633 outfile.write(
u"import {}\n".
format(imp))
635 field.save(outfile, self)
638 """!Return a dict of field name: value 640 Correct behavior is dependent on proper implementation of Field.toDict. If implementing a new 641 Field type, you may need to implement your own toDict method. 645 dict_[name] = field.toDict(self)
649 """!Return all the keys in a config recursively 655 with io.StringIO()
as strFd:
657 contents = strFd.getvalue()
663 for line
in contents.split(
"\n"):
664 if re.search(
r"^((assert|import)\s+|\s*$|#)", line):
667 mat = re.search(
r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
669 keys.append(mat.group(1))
673 def _rename(self, name):
674 """!Rename this Config object in its parent config 676 @param[in] name new name for this config in its parent config 678 Correct behavior is dependent on proper implementation of Field.rename. If implementing a new 679 Field type, you may need to implement your own rename method. 686 """!Validate the Config; raise an exception if invalid 688 The base class implementation performs type checks on all fields by 689 calling Field.validate(). 691 Complex single-field validation can be defined by deriving new Field 692 types. As syntactic sugar, some derived Field types are defined in 693 this module which handle recursing into sub-configs 694 (ConfigField, ConfigChoiceField) 696 Inter-field relationships should only be checked in derived Config 697 classes after calling this method, and base validation is complete 703 """!Format the specified config field's history to a more human-readable format 705 @param[in] name name of field whose history is wanted 706 @param[in] kwargs keyword arguments for lsst.pex.config.history.format 707 @return a string containing the formatted history 710 return pexHist.format(self, name, **kwargs)
713 Read-only history property 715 history = property(
lambda x: x._history)
718 """!Regulate which attributes can be set 720 Unlike normal python objects, Config objects are locked such 721 that no additional attributes nor properties may be added to them 724 Although this is not the standard Python behavior, it helps to 725 protect users from accidentally mispelling a field name, or 726 trying to set a non-existent field. 732 self.
_fields[attr].__set__(self, value, at=at, label=label)
733 elif hasattr(getattr(self.__class__, attr,
None),
'__set__'):
735 return object.__setattr__(self, attr, value)
736 elif attr
in self.__dict__
or attr
in (
"_name",
"_history",
"_storage",
"_frozen",
"_imports"):
738 self.__dict__[attr] = value
741 raise AttributeError(
"%s has no attribute %s" % (_typeStr(self), attr))
747 self.
_fields[attr].__delete__(self, at=at, label=label)
749 object.__delattr__(self, attr)
754 thisValue = getattr(self, name)
755 otherValue = getattr(other, name)
756 if isinstance(thisValue, float)
and math.isnan(thisValue):
757 if not math.isnan(otherValue):
759 elif thisValue != otherValue:
765 return not self.
__eq__(other)
773 ", ".join(
"%s=%r" % (k, v)
for k, v
in self.
toDict().
items()
if v
is not None)
776 def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
777 """!Compare two Configs for equality; return True if equal 779 If the Configs contain RegistryFields or ConfigChoiceFields, unselected Configs 780 will not be compared. 782 @param[in] other Config object to compare with self. 783 @param[in] shortcut If True, return as soon as an inequality is found. 784 @param[in] rtol Relative tolerance for floating point comparisons. 785 @param[in] atol Absolute tolerance for floating point comparisons. 786 @param[in] output If not None, a callable that takes a string, used (possibly repeatedly) 787 to report inequalities. 789 Floating point comparisons are performed by numpy.allclose; refer to that for details. 791 name1 = self.
_name if self.
_name is not None else "config" 792 name2 = other._name
if other._name
is not None else "config" 795 rtol=rtol, atol=atol, output=output)
800 config.loadFromStream(stream)
def toDict(self, instance)
def formatHistory(self, name, kwargs)
Format the specified config field's history to a more human-readable format.
def load(self, filename, root="config")
Modify this config in place by executing the Python code in the named file.
def _rename(self, name)
Rename this Config object in its parent config.
def __init__(self, doc, dtype, default=None, check=None, optional=False)
def compareConfigs(name, c1, c2, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
def unreduceConfig(cls, stream)
def _save(self, outfile)
Save this Config to an open stream object.
def __init__(self, field, config, msg)
def saveToStream(self, outfile, root="config")
Save a python script to a stream, which, when loaded, reproduces this Config.
def __delattr__(self, attr, at=None, label="deletion")
daf::base::PropertySet * set
def __get__(self, instance, owner=None, at=None, label="default")
def save(self, filename, root="config")
Save a python script to the named file, which, when loaded, reproduces this Config.
def iteritems(self)
Iterate over (field name, field value) pairs.
def _setup(self, doc, dtype, default, check, optional, source)
def __iter__(self)
Iterate over fields.
def values(self)
Return the list of field values.
def getStackFrame(relative=0)
def __contains__(self, name)
Return True if the specified field exists in this config.
def save(self, outfile, instance)
def _validateValue(self, value)
def validate(self, instance)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def iterkeys(self)
Iterate over field names.
def __new__(cls, args, kw)
Allocate a new Config object.
def validate(self)
Validate the Config; raise an exception if invalid.
def __setattr__(self, attr, value, at=None, label="assignment")
Regulate which attributes can be set.
def keys(self)
Return the list of field names.
def freeze(self, instance)
def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None)
def names(self)
Return all the keys in a config recursively.
def itervalues(self)
Iterate over field values.
def update(self, kw)
Update values specified by the keyword arguments.
def __set__(self, instance, value, at=None, label='assignment')
def items(self)
Return the list of (field name, field value) pairs.
def loadFromStream(self, stream, root="config", filename=None)
Modify this config in place by executing the python code in the provided stream.
def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Compare two Configs for equality; return True if equal.
def toDict(self)
Return a dict of field name: value.
daf::base::PropertyList * list
def getComparisonName(name1, name2)
def rename(self, instance)
def __delete__(self, instance, at=None, label='deletion')
def find_module(self, fullname, path=None)
def freeze(self)
Make this Config and all sub-configs read-only.