LSSTApplications  17.0,17.0+1,17.0+11,17.0+12,17.0+15,17.0+6,17.0-1-g377950a+5,17.0.1-1-g0d345a5+4,17.0.1-1-g444bd44+3,17.0.1-1-g46e6382+4,17.0.1-1-g4d4fbc4,17.0.1-1-g703d48b+1,17.0.1-1-g9deacb5+3,17.0.1-1-gaef33af,17.0.1-1-gea52513+3,17.0.1-1-gf4e0155+4,17.0.1-1-gfc65f5f+3,17.0.1-1-gfc6fb1f,17.0.1-2-g0ce9737+4,17.0.1-2-g2a2f1b99+4,17.0.1-2-gd73ec07+4,17.0.1-3-gb71a564+4,17.0.1-3-gc20ba7d+5,17.0.1-3-gcbbb95d,17.0.1-4-g41c8d5dc0+1,17.0.1-4-gfa71e81+2,17.0.1-5-gb7d1e01+3,17.0.1-5-gf0ac6446+5,17.0.1-7-g69836a1+3,w.2019.12
LSSTDataManagementBasePackage
config.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 __all__ = ("Config", "Field", "FieldValidationError")
23 
24 import io
25 import os
26 import re
27 import sys
28 import math
29 import copy
30 import tempfile
31 import shutil
32 
33 from .comparison import getComparisonName, compareScalars, compareConfigs
34 from .callStack import getStackFrame, getCallStack
35 
36 
37 def _joinNamePath(prefix=None, name=None, index=None):
38  """Generate nested configuration names.
39  """
40  if not prefix and not name:
41  raise ValueError("Invalid name: cannot be None")
42  elif not name:
43  name = prefix
44  elif prefix and name:
45  name = prefix + "." + name
46 
47  if index is not None:
48  return "%s[%r]" % (name, index)
49  else:
50  return name
51 
52 
53 def _autocast(x, dtype):
54  """Cast a value to a type, if appropriate.
55 
56  Parameters
57  ----------
58  x : object
59  A value.
60  dtype : tpye
61  Data type, such as `float`, `int`, or `str`.
62 
63  Returns
64  -------
65  values : object
66  If appropriate, the returned value is ``x`` cast to the given type
67  ``dtype``. If the cast cannot be performed the original value of
68  ``x`` is returned.
69  """
70  if dtype == float and isinstance(x, int):
71  return float(x)
72  return x
73 
74 
75 def _typeStr(x):
76  """Generate a fully-qualified type name.
77 
78  Returns
79  -------
80  `str`
81  Fully-qualified type name.
82 
83  Notes
84  -----
85  This function is used primarily for writing config files to be executed
86  later upon with the 'load' function.
87  """
88  if hasattr(x, '__module__') and hasattr(x, '__name__'):
89  xtype = x
90  else:
91  xtype = type(x)
92  if (sys.version_info.major <= 2 and xtype.__module__ == '__builtin__') or xtype.__module__ == 'builtins':
93  return xtype.__name__
94  else:
95  return "%s.%s" % (xtype.__module__, xtype.__name__)
96 
97 
99  """A metaclass for `lsst.pex.config.Config`.
100 
101  Notes
102  -----
103  ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
104  class attributes as a class attribute called ``_fields``, and adds
105  the name of each field as an instance variable of the field itself (so you
106  don't have to pass the name of the field to the field constructor).
107  """
108 
109  def __init__(cls, name, bases, dict_):
110  type.__init__(cls, name, bases, dict_)
111  cls._fields = {}
112  cls._source = getStackFrame()
113 
114  def getFields(classtype):
115  fields = {}
116  bases = list(classtype.__bases__)
117  bases.reverse()
118  for b in bases:
119  fields.update(getFields(b))
120 
121  for k, v in classtype.__dict__.items():
122  if isinstance(v, Field):
123  fields[k] = v
124  return fields
125 
126  fields = getFields(cls)
127  for k, v in fields.items():
128  setattr(cls, k, copy.deepcopy(v))
129 
130  def __setattr__(cls, name, value):
131  if isinstance(value, Field):
132  value.name = name
133  cls._fields[name] = value
134  type.__setattr__(cls, name, value)
135 
136 
137 class FieldValidationError(ValueError):
138  """An exception that holds additional information, as attributes,
139  for debugging `lsst.pex.config.Config` errors.
140  """
141 
142  def __init__(self, field, config, msg):
143  self.fieldType = type(field)
144  """Type of the `~lsst.pex.config.Field` that incurred the error.
145  """
146 
147  self.fieldName = field.name
148  """Name of the `~lsst.pex.config.Field` instance that incurred the
149  error (`str`).
150 
151  See also
152  --------
153  lsst.pex.config.Field.name
154  """
155 
156  self.fullname = _joinNamePath(config._name, field.name)
157  """Fully-qualified name of the `~lsst.pex.config.Field` instance
158  (`str`).
159  """
160 
161  self.history = config.history.setdefault(field.name, [])
162  """Full history of all changes to the `~lsst.pex.config.Field`
163  instance.
164  """
165 
166  self.fieldSource = field.source
167  """File and line number of the `~lsst.pex.config.Field` definition.
168  """
169 
170  self.configSource = config._source
171  error = "%s '%s' failed validation: %s\n"\
172  "For more information read the Field definition at:\n%s"\
173  "And the Config definition at:\n%s" % \
174  (self.fieldType.__name__, self.fullname, msg,
175  self.fieldSource.format(), self.configSource.format())
176  ValueError.__init__(self, error)
177 
178 
179 class Field:
180  """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
181  `complex`, `bool`, and `str` data types.
182 
183  Parameters
184  ----------
185  doc : `str`
186  A description of the field for users.
187  dtype : type
188  The field's data type. ``Field`` only supports basic data types:
189  `int`, `float`, `complex`, `bool`, and `str`. See
190  `Field.supportedTypes`.
191  default : object, optional
192  The field's default value.
193  check : callable, optional
194  A callable that is called with the field's value. This callable should
195  return `False` if the value is invalid. More complex inter-field
196  validation can be written as part of the
197  `lsst.pex.config.Config.validate` method.
198  optional : `bool`, optional
199  This sets whether the field is considered optional, and therefore
200  doesn't need to be set by the user. When `False`,
201  `lsst.pex.config.Config.validate` fails if the field's value is `None`.
202 
203  Raises
204  ------
205  ValueError
206  Raised when the ``dtype`` parameter is not one of the supported types
207  (see `Field.supportedTypes`).
208 
209  See also
210  --------
211  ChoiceField
212  ConfigChoiceField
213  ConfigDictField
214  ConfigField
215  ConfigurableField
216  DictField
217  ListField
218  RangeField
219  RegistryField
220 
221  Notes
222  -----
223  ``Field`` instances (including those of any subclass of ``Field``) are used
224  as class attributes of `~lsst.pex.config.Config` subclasses (see the
225  example, below). ``Field`` attributes work like the `property` attributes
226  of classes that implement custom setters and getters. `Field` attributes
227  belong to the class, but operate on the instance. Formally speaking,
228  `Field` attributes are `descriptors
229  <https://docs.python.org/3/howto/descriptor.html>`_.
230 
231  When you access a `Field` attribute on a `Config` instance, you don't
232  get the `Field` instance itself. Instead, you get the value of that field,
233  which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
234  container type (like a `lsst.pex.config.List`) depending on the field's
235  type. See the example, below.
236 
237  Examples
238  --------
239  Instances of ``Field`` should be used as class attributes of
240  `lsst.pex.config.Config` subclasses:
241 
242  >>> from lsst.pex.config import Config, Field
243  >>> class Example(Config):
244  ... myInt = Field("An integer field.", int, default=0)
245  ...
246  >>> print(config.myInt)
247  0
248  >>> config.myInt = 5
249  >>> print(config.myInt)
250  5
251  """
252 
253  supportedTypes = set((str, bool, float, int, complex))
254  """Supported data types for field values (`set` of types).
255  """
256 
257  def __init__(self, doc, dtype, default=None, check=None, optional=False):
258  if dtype not in self.supportedTypes:
259  raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
260 
261  source = getStackFrame()
262  self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
263 
264  def _setup(self, doc, dtype, default, check, optional, source):
265  """Set attributes, usually during initialization.
266  """
267  self.dtype = dtype
268  """Data type for the field.
269  """
270 
271  self.doc = doc
272  """A description of the field (`str`).
273  """
274 
275  self.__doc__ = f"{doc} (`{dtype.__name__}`"
276  if optional or default is not None:
277  self.__doc__ += f", default ``{default!r}``"
278  self.__doc__ += ")"
279 
280  self.default = default
281  """Default value for this field.
282  """
283 
284  self.check = check
285  """A user-defined function that validates the value of the field.
286  """
287 
288  self.optional = optional
289  """Flag that determines if the field is required to be set (`bool`).
290 
291  When `False`, `lsst.pex.config.Config.validate` will fail if the
292  field's value is `None`.
293  """
294 
295  self.source = source
296  """The stack frame where this field is defined (`list` of
297  `lsst.pex.config.callStack.StackFrame`).
298  """
299 
300  def rename(self, instance):
301  """Rename the field in a `~lsst.pex.config.Config` (for internal use
302  only).
303 
304  Parameters
305  ----------
306  instance : `lsst.pex.config.Config`
307  The config instance that contains this field.
308 
309  Notes
310  -----
311  This method is invoked by the `lsst.pex.config.Config` object that
312  contains this field and should not be called directly.
313 
314  Renaming is only relevant for `~lsst.pex.config.Field` instances that
315  hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should
316  rename each subconfig with the full field name as generated by
317  `lsst.pex.config.config._joinNamePath`.
318  """
319  pass
320 
321  def validate(self, instance):
322  """Validate the field (for internal use only).
323 
324  Parameters
325  ----------
326  instance : `lsst.pex.config.Config`
327  The config instance that contains this field.
328 
329  Raises
330  ------
331  lsst.pex.config.FieldValidationError
332  Raised if verification fails.
333 
334  Notes
335  -----
336  This method provides basic validation:
337 
338  - Ensures that the value is not `None` if the field is not optional.
339  - Ensures type correctness.
340  - Ensures that the user-provided ``check`` function is valid.
341 
342  Most `~lsst.pex.config.Field` subclasses should call
343  `lsst.pex.config.field.Field.validate` if they re-implement
344  `~lsst.pex.config.field.Field.validate`.
345  """
346  value = self.__get__(instance)
347  if not self.optional and value is None:
348  raise FieldValidationError(self, instance, "Required value cannot be None")
349 
350  def freeze(self, instance):
351  """Make this field read-only (for internal use only).
352 
353  Parameters
354  ----------
355  instance : `lsst.pex.config.Config`
356  The config instance that contains this field.
357 
358  Notes
359  -----
360  Freezing is only relevant for fields that hold subconfigs. Fields which
361  hold subconfigs should freeze each subconfig.
362 
363  **Subclasses should implement this method.**
364  """
365  pass
366 
367  def _validateValue(self, value):
368  """Validate a value.
369 
370  Parameters
371  ----------
372  value : object
373  The value being validated.
374 
375  Raises
376  ------
377  TypeError
378  Raised if the value's type is incompatible with the field's
379  ``dtype``.
380  ValueError
381  Raised if the value is rejected by the ``check`` method.
382  """
383  if value is None:
384  return
385 
386  if not isinstance(value, self.dtype):
387  msg = "Value %s is of incorrect type %s. Expected type %s" % \
388  (value, _typeStr(value), _typeStr(self.dtype))
389  raise TypeError(msg)
390  if self.check is not None and not self.check(value):
391  msg = "Value %s is not a valid value" % str(value)
392  raise ValueError(msg)
393 
394  def _collectImports(self, instance, imports):
395  """This function should call the _collectImports method on all config
396  objects the field may own, and union them with the supplied imports
397  set.
398 
399  Parameters
400  ----------
401  instance : instance or subclass of `lsst.pex.config.Config`
402  A config object that has this field defined on it
403  imports : `set`
404  Set of python modules that need imported after persistence
405  """
406  pass
407 
408  def save(self, outfile, instance):
409  """Save this field to a file (for internal use only).
410 
411  Parameters
412  ----------
413  outfile : file-like object
414  A writeable field handle.
415  instance : `Config`
416  The `Config` instance that contains this field.
417 
418  Notes
419  -----
420  This method is invoked by the `~lsst.pex.config.Config` object that
421  contains this field and should not be called directly.
422 
423  The output consists of the documentation string
424  (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
425  line is formatted as an assignment: ``{fullname}={value}``.
426 
427  This output can be executed with Python.
428  """
429  value = self.__get__(instance)
430  fullname = _joinNamePath(instance._name, self.name)
431 
432  # write full documentation string as comment lines (i.e. first character is #)
433  doc = "# " + str(self.doc).replace("\n", "\n# ")
434  if isinstance(value, float) and (math.isinf(value) or math.isnan(value)):
435  # non-finite numbers need special care
436  outfile.write(u"{}\n{}=float('{!r}')\n\n".format(doc, fullname, value))
437  else:
438  outfile.write(u"{}\n{}={!r}\n\n".format(doc, fullname, value))
439 
440  def toDict(self, instance):
441  """Convert the field value so that it can be set as the value of an
442  item in a `dict` (for internal use only).
443 
444  Parameters
445  ----------
446  instance : `Config`
447  The `Config` that contains this field.
448 
449  Returns
450  -------
451  value : object
452  The field's value. See *Notes*.
453 
454  Notes
455  -----
456  This method invoked by the owning `~lsst.pex.config.Config` object and
457  should not be called directly.
458 
459  Simple values are passed through. Complex data structures must be
460  manipulated. For example, a `~lsst.pex.config.Field` holding a
461  subconfig should, instead of the subconfig object, return a `dict`
462  where the keys are the field names in the subconfig, and the values are
463  the field values in the subconfig.
464  """
465  return self.__get__(instance)
466 
467  def __get__(self, instance, owner=None, at=None, label="default"):
468  """Define how attribute access should occur on the Config instance
469  This is invoked by the owning config object and should not be called
470  directly
471 
472  When the field attribute is accessed on a Config class object, it
473  returns the field object itself in order to allow inspection of
474  Config classes.
475 
476  When the field attribute is access on a config instance, the actual
477  value described by the field (and held by the Config instance) is
478  returned.
479  """
480  if instance is None or not isinstance(instance, Config):
481  return self
482  else:
483  return instance._storage[self.name]
484 
485  def __set__(self, instance, value, at=None, label='assignment'):
486  """Set an attribute on the config instance.
487 
488  Parameters
489  ----------
490  instance : `lsst.pex.config.Config`
491  The config instance that contains this field.
492  value : obj
493  Value to set on this field.
494  at : `list` of `lsst.pex.config.callStack.StackFrame`
495  The call stack (created by
496  `lsst.pex.config.callStack.getCallStack`).
497  label : `str`, optional
498  Event label for the history.
499 
500  Notes
501  -----
502  This method is invoked by the owning `lsst.pex.config.Config` object
503  and should not be called directly.
504 
505  Derived `~lsst.pex.config.Field` classes may need to override the
506  behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
507  should follow the following rules:
508 
509  - Do not allow modification of frozen configs.
510  - Validate the new value **before** modifying the field. Except if the
511  new value is `None`. `None` is special and no attempt should be made
512  to validate it until `lsst.pex.config.Config.validate` is called.
513  - Do not modify the `~lsst.pex.config.Config` instance to contain
514  invalid values.
515  - If the field is modified, update the history of the
516  `lsst.pex.config.field.Field` to reflect the changes.
517 
518  In order to decrease the need to implement this method in derived
519  `~lsst.pex.config.Field` types, value validation is performed in the
520  `lsst.pex.config.Field._validateValue`. If only the validation step
521  differs in the derived `~lsst.pex.config.Field`, it is simpler to
522  implement `lsst.pex.config.Field._validateValue` than to reimplement
523  ``__set__``. More complicated behavior, however, may require
524  reimplementation.
525  """
526  if instance._frozen:
527  raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
528 
529  history = instance._history.setdefault(self.name, [])
530  if value is not None:
531  value = _autocast(value, self.dtype)
532  try:
533  self._validateValue(value)
534  except BaseException as e:
535  raise FieldValidationError(self, instance, str(e))
536 
537  instance._storage[self.name] = value
538  if at is None:
539  at = getCallStack()
540  history.append((value, at, label))
541 
542  def __delete__(self, instance, at=None, label='deletion'):
543  """Delete an attribute from a `lsst.pex.config.Config` instance.
544 
545  Parameters
546  ----------
547  instance : `lsst.pex.config.Config`
548  The config instance that contains this field.
549  at : `list` of `lsst.pex.config.callStack.StackFrame`
550  The call stack (created by
551  `lsst.pex.config.callStack.getCallStack`).
552  label : `str`, optional
553  Event label for the history.
554 
555  Notes
556  -----
557  This is invoked by the owning `~lsst.pex.config.Config` object and
558  should not be called directly.
559  """
560  if at is None:
561  at = getCallStack()
562  self.__set__(instance, None, at=at, label=label)
563 
564  def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
565  """Compare a field (named `Field.name`) in two
566  `~lsst.pex.config.Config` instances for equality.
567 
568  Parameters
569  ----------
570  instance1 : `lsst.pex.config.Config`
571  Left-hand side `Config` instance to compare.
572  instance2 : `lsst.pex.config.Config`
573  Right-hand side `Config` instance to compare.
574  shortcut : `bool`, optional
575  **Unused.**
576  rtol : `float`, optional
577  Relative tolerance for floating point comparisons.
578  atol : `float`, optional
579  Absolute tolerance for floating point comparisons.
580  output : callable, optional
581  A callable that takes a string, used (possibly repeatedly) to
582  report inequalities.
583 
584  Notes
585  -----
586  This method must be overridden by more complex `Field` subclasses.
587 
588  See also
589  --------
590  lsst.pex.config.compareScalars
591  """
592  v1 = getattr(instance1, self.name)
593  v2 = getattr(instance2, self.name)
594  name = getComparisonName(
595  _joinNamePath(instance1._name, self.name),
596  _joinNamePath(instance2._name, self.name)
597  )
598  return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
599 
600 
602  """Importer (for `sys.meta_path`) that records which modules are being
603  imported.
604 
605  *This class does not do any importing itself.*
606 
607  Examples
608  --------
609  Use this class as a context manager to ensure it is properly uninstalled
610  when done:
611 
612  >>> with RecordingImporter() as importer:
613  ... # import stuff
614  ... import numpy as np
615  ... print("Imported: " + importer.getModules())
616  """
617 
618  def __init__(self):
619  self._modules = set()
620 
621  def __enter__(self):
622  self.origMetaPath = sys.meta_path
623  sys.meta_path = [self] + sys.meta_path
624  return self
625 
626  def __exit__(self, *args):
627  self.uninstall()
628  return False # Don't suppress exceptions
629 
630  def uninstall(self):
631  """Uninstall the importer.
632  """
633  sys.meta_path = self.origMetaPath
634 
635  def find_module(self, fullname, path=None):
636  """Called as part of the ``import`` chain of events.
637  """
638  self._modules.add(fullname)
639  # Return None because we don't do any importing.
640  return None
641 
642  def getModules(self):
643  """Get the set of modules that were imported.
644 
645  Returns
646  -------
647  modules : `set` of `str`
648  Set of imported module names.
649  """
650  return self._modules
651 
652 
653 class Config(metaclass=ConfigMeta):
654  """Base class for configuration (*config*) objects.
655 
656  Notes
657  -----
658  A ``Config`` object will usually have several `~lsst.pex.config.Field`
659  instances as class attributes. These are used to define most of the base
660  class behavior.
661 
662  ``Config`` implements a mapping API that provides many `dict`-like methods,
663  such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and
664  `itervalues`. ``Config`` instances also support the ``in`` operator to
665  test if a field is in the config. Unlike a `dict`, ``Config`` classes are
666  not subscriptable. Instead, access individual fields as attributes of the
667  configuration instance.
668 
669  Examples
670  --------
671  Config classes are subclasses of ``Config`` that have
672  `~lsst.pex.config.Field` instances (or instances of
673  `~lsst.pex.config.Field` subclasses) as class attributes:
674 
675  >>> from lsst.pex.config import Config, Field, ListField
676  >>> class DemoConfig(Config):
677  ... intField = Field(doc="An integer field", dtype=int, default=42)
678  ... listField = ListField(doc="List of favorite beverages.", dtype=str,
679  ... default=['coffee', 'green tea', 'water'])
680  ...
681  >>> config = DemoConfig()
682 
683  Configs support many `dict`-like APIs:
684 
685  >>> config.keys()
686  ['intField', 'listField']
687  >>> 'intField' in config
688  True
689 
690  Individual fields can be accessed as attributes of the configuration:
691 
692  >>> config.intField
693  42
694  >>> config.listField.append('earl grey tea')
695  >>> print(config.listField)
696  ['coffee', 'green tea', 'water', 'earl grey tea']
697  """
698 
699  def __iter__(self):
700  """Iterate over fields.
701  """
702  return self._fields.__iter__()
703 
704  def keys(self):
705  """Get field names.
706 
707  Returns
708  -------
709  names : `list`
710  List of `lsst.pex.config.Field` names.
711 
712  See also
713  --------
714  lsst.pex.config.Config.iterkeys
715  """
716  return list(self._storage.keys())
717 
718  def values(self):
719  """Get field values.
720 
721  Returns
722  -------
723  values : `list`
724  List of field values.
725 
726  See also
727  --------
728  lsst.pex.config.Config.itervalues
729  """
730  return list(self._storage.values())
731 
732  def items(self):
733  """Get configurations as ``(field name, field value)`` pairs.
734 
735  Returns
736  -------
737  items : `list`
738  List of tuples for each configuration. Tuple items are:
739 
740  0. Field name.
741  1. Field value.
742 
743  See also
744  --------
745  lsst.pex.config.Config.iteritems
746  """
747  return list(self._storage.items())
748 
749  def iteritems(self):
750  """Iterate over (field name, field value) pairs.
751 
752  Yields
753  ------
754  item : `tuple`
755  Tuple items are:
756 
757  0. Field name.
758  1. Field value.
759 
760  See also
761  --------
762  lsst.pex.config.Config.items
763  """
764  return iter(self._storage.items())
765 
766  def itervalues(self):
767  """Iterate over field values.
768 
769  Yields
770  ------
771  value : obj
772  A field value.
773 
774  See also
775  --------
776  lsst.pex.config.Config.values
777  """
778  return iter(self.storage.values())
779 
780  def iterkeys(self):
781  """Iterate over field names
782 
783  Yields
784  ------
785  key : `str`
786  A field's key (attribute name).
787 
788  See also
789  --------
790  lsst.pex.config.Config.values
791  """
792  return iter(self.storage.keys())
793 
794  def __contains__(self, name):
795  """!Return True if the specified field exists in this config
796 
797  @param[in] name field name to test for
798  """
799  return self._storage.__contains__(name)
800 
801  def __new__(cls, *args, **kw):
802  """Allocate a new `lsst.pex.config.Config` object.
803 
804  In order to ensure that all Config object are always in a proper state
805  when handed to users or to derived `~lsst.pex.config.Config` classes,
806  some attributes are handled at allocation time rather than at
807  initialization.
808 
809  This ensures that even if a derived `~lsst.pex.config.Config` class
810  implements ``__init__``, its author does not need to be concerned about
811  when or even the base ``Config.__init__`` should be called.
812  """
813  name = kw.pop("__name", None)
814  at = kw.pop("__at", getCallStack())
815  # remove __label and ignore it
816  kw.pop("__label", "default")
817 
818  instance = object.__new__(cls)
819  instance._frozen = False
820  instance._name = name
821  instance._storage = {}
822  instance._history = {}
823  instance._imports = set()
824  # load up defaults
825  for field in instance._fields.values():
826  instance._history[field.name] = []
827  field.__set__(instance, field.default, at=at + [field.source], label="default")
828  # set custom default-overides
829  instance.setDefaults()
830  # set constructor overides
831  instance.update(__at=at, **kw)
832  return instance
833 
834  def __reduce__(self):
835  """Reduction for pickling (function with arguments to reproduce).
836 
837  We need to condense and reconstitute the `~lsst.pex.config.Config`,
838  since it may contain lambdas (as the ``check`` elements) that cannot
839  be pickled.
840  """
841  # The stream must be in characters to match the API but pickle requires bytes
842  stream = io.StringIO()
843  self.saveToStream(stream)
844  return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
845 
846  def setDefaults(self):
847  """Subclass hook for computing defaults.
848 
849  Notes
850  -----
851  Derived `~lsst.pex.config.Config` classes that must compute defaults
852  rather than using the `~lsst.pex.config.Field` instances's defaults
853  should do so here. To correctly use inherited defaults,
854  implementations of ``setDefaults`` must call their base class's
855  ``setDefaults``.
856  """
857  pass
858 
859  def update(self, **kw):
860  """Update values of fields specified by the keyword arguments.
861 
862  Parameters
863  ----------
864  kw
865  Keywords are configuration field names. Values are configuration
866  field values.
867 
868  Notes
869  -----
870  The ``__at`` and ``__label`` keyword arguments are special internal
871  keywords. They are used to strip out any internal steps from the
872  history tracebacks of the config. Do not modify these keywords to
873  subvert a `~lsst.pex.config.Config` instance's history.
874 
875  Examples
876  --------
877  This is a config with three fields:
878 
879  >>> from lsst.pex.config import Config, Field
880  >>> class DemoConfig(Config):
881  ... fieldA = Field(doc='Field A', dtype=int, default=42)
882  ... fieldB = Field(doc='Field B', dtype=bool, default=True)
883  ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
884  ...
885  >>> config = DemoConfig()
886 
887  These are the default values of each field:
888 
889  >>> for name, value in config.iteritems():
890  ... print(f"{name}: {value}")
891  ...
892  fieldA: 42
893  fieldB: True
894  fieldC: 'Hello world'
895 
896  Using this method to update ``fieldA`` and ``fieldC``:
897 
898  >>> config.update(fieldA=13, fieldC='Updated!')
899 
900  Now the values of each field are:
901 
902  >>> for name, value in config.iteritems():
903  ... print(f"{name}: {value}")
904  ...
905  fieldA: 13
906  fieldB: True
907  fieldC: 'Updated!'
908  """
909  at = kw.pop("__at", getCallStack())
910  label = kw.pop("__label", "update")
911 
912  for name, value in kw.items():
913  try:
914  field = self._fields[name]
915  field.__set__(self, value, at=at, label=label)
916  except KeyError:
917  raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
918 
919  def load(self, filename, root="config"):
920  """Modify this config in place by executing the Python code in a
921  configuration file.
922 
923  Parameters
924  ----------
925  filename : `str`
926  Name of the configuration file. A configuration file is Python
927  module.
928  root : `str`, optional
929  Name of the variable in file that refers to the config being
930  overridden.
931 
932  For example, the value of root is ``"config"`` and the file
933  contains::
934 
935  config.myField = 5
936 
937  Then this config's field ``myField`` is set to ``5``.
938 
939  **Deprecated:** For backwards compatibility, older config files
940  that use ``root="root"`` instead of ``root="config"`` will be
941  loaded with a warning printed to `sys.stderr`. This feature will be
942  removed at some point.
943 
944  See also
945  --------
946  lsst.pex.config.Config.loadFromStream
947  lsst.pex.config.Config.save
948  lsst.pex.config.Config.saveFromStream
949  """
950  with open(filename, "r") as f:
951  code = compile(f.read(), filename=filename, mode="exec")
952  self.loadFromStream(stream=code, root=root)
953 
954  def loadFromStream(self, stream, root="config", filename=None):
955  """Modify this Config in place by executing the Python code in the
956  provided stream.
957 
958  Parameters
959  ----------
960  stream : file-like object, `str`, or compiled string
961  Stream containing configuration override code.
962  root : `str`, optional
963  Name of the variable in file that refers to the config being
964  overridden.
965 
966  For example, the value of root is ``"config"`` and the file
967  contains::
968 
969  config.myField = 5
970 
971  Then this config's field ``myField`` is set to ``5``.
972 
973  **Deprecated:** For backwards compatibility, older config files
974  that use ``root="root"`` instead of ``root="config"`` will be
975  loaded with a warning printed to `sys.stderr`. This feature will be
976  removed at some point.
977  filename : `str`, optional
978  Name of the configuration file, or `None` if unknown or contained
979  in the stream. Used for error reporting.
980 
981  See also
982  --------
983  lsst.pex.config.Config.load
984  lsst.pex.config.Config.save
985  lsst.pex.config.Config.saveFromStream
986  """
987  with RecordingImporter() as importer:
988  try:
989  local = {root: self}
990  exec(stream, {}, local)
991  except NameError as e:
992  if root == "config" and "root" in e.args[0]:
993  if filename is None:
994  # try to determine the file name; a compiled string has attribute "co_filename",
995  # an open file has attribute "name", else give up
996  filename = getattr(stream, "co_filename", None)
997  if filename is None:
998  filename = getattr(stream, "name", "?")
999  print(f"Config override file {filename!r}"
1000  " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
1001  local = {"root": self}
1002  exec(stream, {}, local)
1003  else:
1004  raise
1005 
1006  self._imports.update(importer.getModules())
1007 
1008  def save(self, filename, root="config"):
1009  """Save a Python script to the named file, which, when loaded,
1010  reproduces this config.
1011 
1012  Parameters
1013  ----------
1014  filename : `str`
1015  Desination filename of this configuration.
1016  root : `str`, optional
1017  Name to use for the root config variable. The same value must be
1018  used when loading (see `lsst.pex.config.Config.load`).
1019 
1020  See also
1021  --------
1022  lsst.pex.config.Config.saveToStream
1023  lsst.pex.config.Config.load
1024  lsst.pex.config.Config.loadFromStream
1025  """
1026  d = os.path.dirname(filename)
1027  with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1028  self.saveToStream(outfile, root)
1029  # tempfile is hardcoded to create files with mode '0600'
1030  # for an explantion of these antics see:
1031  # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1032  umask = os.umask(0o077)
1033  os.umask(umask)
1034  os.chmod(outfile.name, (~umask & 0o666))
1035  # chmod before the move so we get quasi-atomic behavior if the
1036  # source and dest. are on the same filesystem.
1037  # os.rename may not work across filesystems
1038  shutil.move(outfile.name, filename)
1039 
1040  def saveToStream(self, outfile, root="config"):
1041  """Save a configuration file to a stream, which, when loaded,
1042  reproduces this config.
1043 
1044  Parameters
1045  ----------
1046  outfile : file-like object
1047  Destination file object write the config into. Accepts strings not
1048  bytes.
1049  root
1050  Name to use for the root config variable. The same value must be
1051  used when loading (see `lsst.pex.config.Config.load`).
1052 
1053  See also
1054  --------
1055  lsst.pex.config.Config.save
1056  lsst.pex.config.Config.load
1057  lsst.pex.config.Config.loadFromStream
1058  """
1059  tmp = self._name
1060  self._rename(root)
1061  try:
1062  self._collectImports()
1063  # Remove self from the set, as it is handled explicitly below
1064  self._imports.remove(self.__module__)
1065  configType = type(self)
1066  typeString = _typeStr(configType)
1067  outfile.write(u"import {}\n".format(configType.__module__))
1068  outfile.write(u"assert type({})=={}, 'config is of type %s.%s ".format(root, typeString))
1069  outfile.write(u"instead of {}' % (type({}).__module__, type({}).__name__)\n".format(typeString,
1070  root,
1071  root))
1072  for imp in self._imports:
1073  if imp in sys.modules and sys.modules[imp] is not None:
1074  outfile.write(u"import {}\n".format(imp))
1075  self._save(outfile)
1076  finally:
1077  self._rename(tmp)
1078 
1079  def freeze(self):
1080  """Make this config, and all subconfigs, read-only.
1081  """
1082  self._frozen = True
1083  for field in self._fields.values():
1084  field.freeze(self)
1085 
1086  def _save(self, outfile):
1087  """Save this config to an open stream object.
1088 
1089  Parameters
1090  ----------
1091  outfile : file-like object
1092  Destination file object write the config into. Accepts strings not
1093  bytes.
1094  """
1095  for field in self._fields.values():
1096  field.save(outfile, self)
1097 
1098  def _collectImports(self):
1099  """Adds module containing self to the list of things to import and
1100  then loops over all the fields in the config calling a corresponding
1101  collect method. The field method will call _collectImports on any configs
1102  it may own and return the set of things to import. This returned set
1103  will be merged with the set of imports for this config class.
1104  """
1105  self._imports.add(self.__module__)
1106  for name, field in self._fields.items():
1107  field._collectImports(self, self._imports)
1108 
1109  def toDict(self):
1110  """Make a dictionary of field names and their values.
1111 
1112  Returns
1113  -------
1114  dict_ : `dict`
1115  Dictionary with keys that are `~lsst.pex.config.Field` names.
1116  Values are `~lsst.pex.config.Field` values.
1117 
1118  See also
1119  --------
1120  lsst.pex.config.Field.toDict
1121 
1122  Notes
1123  -----
1124  This method uses the `~lsst.pex.config.Field.toDict` method of
1125  individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1126  implement a ``toDict`` method for *this* method to work.
1127  """
1128  dict_ = {}
1129  for name, field in self._fields.items():
1130  dict_[name] = field.toDict(self)
1131  return dict_
1132 
1133  def names(self):
1134  """Get all the field names in the config, recursively.
1135 
1136  Returns
1137  -------
1138  names : `list` of `str`
1139  Field names.
1140  """
1141  #
1142  # Rather than sort out the recursion all over again use the
1143  # pre-existing saveToStream()
1144  #
1145  with io.StringIO() as strFd:
1146  self.saveToStream(strFd, "config")
1147  contents = strFd.getvalue()
1148  strFd.close()
1149  #
1150  # Pull the names out of the dumped config
1151  #
1152  keys = []
1153  for line in contents.split("\n"):
1154  if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1155  continue
1156 
1157  mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1158  if mat:
1159  keys.append(mat.group(1))
1160 
1161  return keys
1162 
1163  def _rename(self, name):
1164  """Rename this config object in its parent `~lsst.pex.config.Config`.
1165 
1166  Parameters
1167  ----------
1168  name : `str`
1169  New name for this config in its parent `~lsst.pex.config.Config`.
1170 
1171  Notes
1172  -----
1173  This method uses the `~lsst.pex.config.Field.rename` method of
1174  individual `lsst.pex.config.Field` instances.
1175  `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1176  method for *this* method to work.
1177 
1178  See also
1179  --------
1180  lsst.pex.config.Field.rename
1181  """
1182  self._name = name
1183  for field in self._fields.values():
1184  field.rename(self)
1185 
1186  def validate(self):
1187  """Validate the Config, raising an exception if invalid.
1188 
1189  Raises
1190  ------
1191  lsst.pex.config.FieldValidationError
1192  Raised if verification fails.
1193 
1194  Notes
1195  -----
1196  The base class implementation performs type checks on all fields by
1197  calling their `~lsst.pex.config.Field.validate` methods.
1198 
1199  Complex single-field validation can be defined by deriving new Field
1200  types. For convenience, some derived `lsst.pex.config.Field`-types
1201  (`~lsst.pex.config.ConfigField` and
1202  `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config`
1203  that handle recursing into subconfigs.
1204 
1205  Inter-field relationships should only be checked in derived
1206  `~lsst.pex.config.Config` classes after calling this method, and base
1207  validation is complete.
1208  """
1209  for field in self._fields.values():
1210  field.validate(self)
1211 
1212  def formatHistory(self, name, **kwargs):
1213  """Format a configuration field's history to a human-readable format.
1214 
1215  Parameters
1216  ----------
1217  name : `str`
1218  Name of a `~lsst.pex.config.Field` in this config.
1219  kwargs
1220  Keyword arguments passed to `lsst.pex.config.history.format`.
1221 
1222  Returns
1223  -------
1224  history : `str`
1225  A string containing the formatted history.
1226 
1227  See also
1228  --------
1229  lsst.pex.config.history.format
1230  """
1231  import lsst.pex.config.history as pexHist
1232  return pexHist.format(self, name, **kwargs)
1233 
1234  history = property(lambda x: x._history)
1235  """Read-only history.
1236  """
1237 
1238  def __setattr__(self, attr, value, at=None, label="assignment"):
1239  """Set an attribute (such as a field's value).
1240 
1241  Notes
1242  -----
1243  Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1244  locked such that no additional attributes nor properties may be added
1245  to them dynamically.
1246 
1247  Although this is not the standard Python behavior, it helps to protect
1248  users from accidentally mispelling a field name, or trying to set a
1249  non-existent field.
1250  """
1251  if attr in self._fields:
1252  if at is None:
1253  at = getCallStack()
1254  # This allows Field descriptors to work.
1255  self._fields[attr].__set__(self, value, at=at, label=label)
1256  elif hasattr(getattr(self.__class__, attr, None), '__set__'):
1257  # This allows properties and other non-Field descriptors to work.
1258  return object.__setattr__(self, attr, value)
1259  elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"):
1260  # This allows specific private attributes to work.
1261  self.__dict__[attr] = value
1262  else:
1263  # We throw everything else.
1264  raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr))
1265 
1266  def __delattr__(self, attr, at=None, label="deletion"):
1267  if attr in self._fields:
1268  if at is None:
1269  at = getCallStack()
1270  self._fields[attr].__delete__(self, at=at, label=label)
1271  else:
1272  object.__delattr__(self, attr)
1273 
1274  def __eq__(self, other):
1275  if type(other) == type(self):
1276  for name in self._fields:
1277  thisValue = getattr(self, name)
1278  otherValue = getattr(other, name)
1279  if isinstance(thisValue, float) and math.isnan(thisValue):
1280  if not math.isnan(otherValue):
1281  return False
1282  elif thisValue != otherValue:
1283  return False
1284  return True
1285  return False
1286 
1287  def __ne__(self, other):
1288  return not self.__eq__(other)
1289 
1290  def __str__(self):
1291  return str(self.toDict())
1292 
1293  def __repr__(self):
1294  return "%s(%s)" % (
1295  _typeStr(self),
1296  ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None)
1297  )
1298 
1299  def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
1300  """Compare this configuration to another `~lsst.pex.config.Config` for
1301  equality.
1302 
1303  Parameters
1304  ----------
1305  other : `lsst.pex.config.Config`
1306  Other `~lsst.pex.config.Config` object to compare against this
1307  config.
1308  shortcut : `bool`, optional
1309  If `True`, return as soon as an inequality is found. Default is
1310  `True`.
1311  rtol : `float`, optional
1312  Relative tolerance for floating point comparisons.
1313  atol : `float`, optional
1314  Absolute tolerance for floating point comparisons.
1315  output : callable, optional
1316  A callable that takes a string, used (possibly repeatedly) to
1317  report inequalities.
1318 
1319  Returns
1320  -------
1321  isEqual : `bool`
1322  `True` when the two `lsst.pex.config.Config` instances are equal.
1323  `False` if there is an inequality.
1324 
1325  See also
1326  --------
1327  lsst.pex.config.compareConfigs
1328 
1329  Notes
1330  -----
1331  Unselected targets of `~lsst.pex.config.RegistryField` fields and
1332  unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1333  are not considered by this method.
1334 
1335  Floating point comparisons are performed by `numpy.allclose`.
1336  """
1337  name1 = self._name if self._name is not None else "config"
1338  name2 = other._name if other._name is not None else "config"
1339  name = getComparisonName(name1, name2)
1340  return compareConfigs(name, self, other, shortcut=shortcut,
1341  rtol=rtol, atol=atol, output=output)
1342 
1343 
1344 def unreduceConfig(cls, stream):
1345  """Create a `~lsst.pex.config.Config` from a stream.
1346 
1347  Parameters
1348  ----------
1349  cls : `lsst.pex.config.Config`-type
1350  A `lsst.pex.config.Config` type (not an instance) that is instantiated
1351  with configurations in the ``stream``.
1352  stream : file-like object, `str`, or compiled string
1353  Stream containing configuration override code.
1354 
1355  Returns
1356  -------
1357  config : `lsst.pex.config.Config`
1358  Config instance.
1359 
1360  See also
1361  --------
1362  lsst.pex.config.Config.loadFromStream
1363  """
1364  config = cls()
1365  config.loadFromStream(stream)
1366  return config
def toDict(self, instance)
Definition: config.py:440
def __eq__(self, other)
Definition: config.py:1274
def formatHistory(self, name, kwargs)
Definition: config.py:1212
def load(self, filename, root="config")
Definition: config.py:919
def _rename(self, name)
Definition: config.py:1163
def __init__(self, doc, dtype, default=None, check=None, optional=False)
Definition: config.py:257
def compareConfigs(name, c1, c2, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Definition: comparison.py:105
def unreduceConfig(cls, stream)
Definition: config.py:1344
def _save(self, outfile)
Definition: config.py:1086
def __init__(self, field, config, msg)
Definition: config.py:142
def saveToStream(self, outfile, root="config")
Definition: config.py:1040
def __delattr__(self, attr, at=None, label="deletion")
Definition: config.py:1266
daf::base::PropertySet * set
Definition: fits.cc:832
def getCallStack(skip=0)
Definition: callStack.py:169
def __setattr__(cls, name, value)
Definition: config.py:130
def __get__(self, instance, owner=None, at=None, label="default")
Definition: config.py:467
def save(self, filename, root="config")
Definition: config.py:1008
def __ne__(self, other)
Definition: config.py:1287
def _setup(self, doc, dtype, default, check, optional, source)
Definition: config.py:264
def getStackFrame(relative=0)
Definition: callStack.py:52
def __contains__(self, name)
Return True if the specified field exists in this config.
Definition: config.py:794
table::Key< int > type
Definition: Detector.cc:164
def save(self, outfile, instance)
Definition: config.py:408
def _validateValue(self, value)
Definition: config.py:367
def validate(self, instance)
Definition: config.py:321
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def __new__(cls, args, kw)
Definition: config.py:801
def __init__(cls, name, bases, dict_)
Definition: config.py:109
def __setattr__(self, attr, value, at=None, label="assignment")
Definition: config.py:1238
def freeze(self, instance)
Definition: config.py:350
def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None)
Definition: comparison.py:56
def __set__(self, instance, value, at=None, label='assignment')
Definition: config.py:485
def loadFromStream(self, stream, root="config", filename=None)
Definition: config.py:954
def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Definition: config.py:1299
daf::base::PropertyList * list
Definition: fits.cc:833
def getComparisonName(name1, name2)
Definition: comparison.py:34
def rename(self, instance)
Definition: config.py:300
def __delete__(self, instance, at=None, label='deletion')
Definition: config.py:542
def find_module(self, fullname, path=None)
Definition: config.py:635