LSSTApplications  16.0-10-g9d3e444,16.0-10-gb7df26b+3,16.0-10-gbd9fc95,16.0-11-g09ed895+3,16.0-11-g12e47bd+11,16.0-12-g5c924a4+21,16.0-12-g71e5ef5+10,16.0-15-g7af1f30+2,16.0-15-gdd5ca33+2,16.0-16-gf0259e2+1,16.0-18-gcf94535+13,16.0-19-g283fd30+1,16.0-19-g9d290d5+13,16.0-2-g0febb12+22,16.0-2-g9d5294e+76,16.0-20-g42ca1a7,16.0-20-g8d11721+2,16.0-23-g28ad22d+2,16.0-25-g723f23e8+2,16.0-29-g44d5b4c,16.0-32-ga392cb8+2,16.0-32-gc794025+1,16.0-4-g18f3627+20,16.0-4-g5f3a788+21,16.0-4-g6d309fa,16.0-4-ga3eb747+11,16.0-4-gabf74b7+42,16.0-4-gb13d127+7,16.0-5-g27fb78a+20,16.0-5-g6a53317+47,16.0-5-gb3f8a4b+101,16.0-6-g9321be7+5,16.0-6-gcbc7b31+58,16.0-6-gf49912c+44,16.0-65-g12857137,16.0-76-g802307b6d+2,16.0-8-g21fd5fe+45,16.0-8-g3a9f023+27,16.0-8-gc11f1cf+11,master-g5605b1aa4d+2,w.2019.05
LSSTDataManagementBasePackage
config.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2015 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 __all__ = ("Config", "Field", "FieldValidationError")
23 
24 import io
25 import os
26 import re
27 import sys
28 import math
29 import copy
30 import tempfile
31 import shutil
32 
33 from .comparison import getComparisonName, compareScalars, compareConfigs
34 from .callStack import getStackFrame, getCallStack
35 
36 
37 def _joinNamePath(prefix=None, name=None, index=None):
38  """Generate nested configuration names.
39  """
40  if not prefix and not name:
41  raise ValueError("Invalid name: cannot be None")
42  elif not name:
43  name = prefix
44  elif prefix and name:
45  name = prefix + "." + name
46 
47  if index is not None:
48  return "%s[%r]" % (name, index)
49  else:
50  return name
51 
52 
53 def _autocast(x, dtype):
54  """Cast a value to a type, if appropriate.
55 
56  Parameters
57  ----------
58  x : object
59  A value.
60  dtype : tpye
61  Data type, such as `float`, `int`, or `str`.
62 
63  Returns
64  -------
65  values : object
66  If appropriate, the returned value is ``x`` cast to the given type
67  ``dtype``. If the cast cannot be performed the original value of
68  ``x`` is returned.
69  """
70  if dtype == float and isinstance(x, int):
71  return float(x)
72  return x
73 
74 
75 def _typeStr(x):
76  """Generate a fully-qualified type name.
77 
78  Returns
79  -------
80  `str`
81  Fully-qualified type name.
82 
83  Notes
84  -----
85  This function is used primarily for writing config files to be executed
86  later upon with the 'load' function.
87  """
88  if hasattr(x, '__module__') and hasattr(x, '__name__'):
89  xtype = x
90  else:
91  xtype = type(x)
92  if (sys.version_info.major <= 2 and xtype.__module__ == '__builtin__') or xtype.__module__ == 'builtins':
93  return xtype.__name__
94  else:
95  return "%s.%s" % (xtype.__module__, xtype.__name__)
96 
97 
99  """A metaclass for `lsst.pex.config.Config`.
100 
101  Notes
102  -----
103  ``ConfigMeta`` adds a dictionary containing all `~lsst.pex.config.Field`
104  class attributes as a class attribute called ``_fields``, and adds
105  the name of each field as an instance variable of the field itself (so you
106  don't have to pass the name of the field to the field constructor).
107  """
108 
109  def __init__(cls, name, bases, dict_):
110  type.__init__(cls, name, bases, dict_)
111  cls._fields = {}
112  cls._source = getStackFrame()
113 
114  def getFields(classtype):
115  fields = {}
116  bases = list(classtype.__bases__)
117  bases.reverse()
118  for b in bases:
119  fields.update(getFields(b))
120 
121  for k, v in classtype.__dict__.items():
122  if isinstance(v, Field):
123  fields[k] = v
124  return fields
125 
126  fields = getFields(cls)
127  for k, v in fields.items():
128  setattr(cls, k, copy.deepcopy(v))
129 
130  def __setattr__(cls, name, value):
131  if isinstance(value, Field):
132  value.name = name
133  cls._fields[name] = value
134  type.__setattr__(cls, name, value)
135 
136 
137 class FieldValidationError(ValueError):
138  """An exception that holds additional information, as attributes,
139  for debugging `lsst.pex.config.Config` errors.
140  """
141 
142  def __init__(self, field, config, msg):
143  self.fieldType = type(field)
144  """Type of the `~lsst.pex.config.Field` that incurred the error.
145  """
146 
147  self.fieldName = field.name
148  """Name of the `~lsst.pex.config.Field` instance that incurred the
149  error (`str`).
150 
151  See also
152  --------
153  lsst.pex.config.Field.name
154  """
155 
156  self.fullname = _joinNamePath(config._name, field.name)
157  """Fully-qualified name of the `~lsst.pex.config.Field` instance
158  (`str`).
159  """
160 
161  self.history = config.history.setdefault(field.name, [])
162  """Full history of all changes to the `~lsst.pex.config.Field`
163  instance.
164  """
165 
166  self.fieldSource = field.source
167  """File and line number of the `~lsst.pex.config.Field` definition.
168  """
169 
170  self.configSource = config._source
171  error = "%s '%s' failed validation: %s\n"\
172  "For more information read the Field definition at:\n%s"\
173  "And the Config definition at:\n%s" % \
174  (self.fieldType.__name__, self.fullname, msg,
175  self.fieldSource.format(), self.configSource.format())
176  ValueError.__init__(self, error)
177 
178 
179 class Field:
180  """A field in a `~lsst.pex.config.Config` that supports `int`, `float`,
181  `complex`, `bool`, and `str` data types.
182 
183  Parameters
184  ----------
185  doc : `str`
186  A description of the field for users.
187  dtype : type
188  The field's data type. ``Field`` only supports basic data types:
189  `int`, `float`, `complex`, `bool`, and `str`. See
190  `Field.supportedTypes`.
191  default : object, optional
192  The field's default value.
193  check : callable, optional
194  A callable that is called with the field's value. This callable should
195  return `False` if the value is invalid. More complex inter-field
196  validation can be written as part of the
197  `lsst.pex.config.Config.validate` method.
198  optional : `bool`, optional
199  This sets whether the field is considered optional, and therefore
200  doesn't need to be set by the user. When `False`,
201  `lsst.pex.config.Config.validate` fails if the field's value is `None`.
202 
203  Raises
204  ------
205  ValueError
206  Raised when the ``dtype`` parameter is not one of the supported types
207  (see `Field.supportedTypes`).
208 
209  See also
210  --------
211  ChoiceField
212  ConfigChoiceField
213  ConfigDictField
214  ConfigField
215  ConfigurableField
216  DictField
217  ListField
218  RangeField
219  RegistryField
220 
221  Notes
222  -----
223  ``Field`` instances (including those of any subclass of ``Field``) are used
224  as class attributes of `~lsst.pex.config.Config` subclasses (see the
225  example, below). ``Field`` attributes work like the `property` attributes
226  of classes that implement custom setters and getters. `Field` attributes
227  belong to the class, but operate on the instance. Formally speaking,
228  `Field` attributes are `descriptors
229  <https://docs.python.org/3/howto/descriptor.html>`_.
230 
231  When you access a `Field` attribute on a `Config` instance, you don't
232  get the `Field` instance itself. Instead, you get the value of that field,
233  which might be a simple type (`int`, `float`, `str`, `bool`) or a custom
234  container type (like a `lsst.pex.config.List`) depending on the field's
235  type. See the example, below.
236 
237  Examples
238  --------
239  Instances of ``Field`` should be used as class attributes of
240  `lsst.pex.config.Config` subclasses:
241 
242  >>> from lsst.pex.config import Config, Field
243  >>> class Example(Config):
244  ... myInt = Field("An integer field.", int, default=0)
245  ...
246  >>> print(config.myInt)
247  0
248  >>> config.myInt = 5
249  >>> print(config.myInt)
250  5
251  """
252 
253  supportedTypes = set((str, bool, float, int, complex))
254  """Supported data types for field values (`set` of types).
255  """
256 
257  def __init__(self, doc, dtype, default=None, check=None, optional=False):
258  if dtype not in self.supportedTypes:
259  raise ValueError("Unsupported Field dtype %s" % _typeStr(dtype))
260 
261  source = getStackFrame()
262  self._setup(doc=doc, dtype=dtype, default=default, check=check, optional=optional, source=source)
263 
264  def _setup(self, doc, dtype, default, check, optional, source):
265  """Set attributes, usually during initialization.
266  """
267  self.dtype = dtype
268  """Data type for the field.
269  """
270 
271  self.doc = doc
272  """A description of the field (`str`).
273  """
274 
275  self.__doc__ = f"{doc} (`{dtype.__name__}`"
276  if optional or default is not None:
277  self.__doc__ += f", default ``{default!r}``"
278  self.__doc__ += ")"
279 
280  self.default = default
281  """Default value for this field.
282  """
283 
284  self.check = check
285  """A user-defined function that validates the value of the field.
286  """
287 
288  self.optional = optional
289  """Flag that determines if the field is required to be set (`bool`).
290 
291  When `False`, `lsst.pex.config.Config.validate` will fail if the
292  field's value is `None`.
293  """
294 
295  self.source = source
296  """The stack frame where this field is defined (`list` of
297  `lsst.pex.config.callStack.StackFrame`).
298  """
299 
300  def rename(self, instance):
301  """Rename the field in a `~lsst.pex.config.Config` (for internal use
302  only).
303 
304  Parameters
305  ----------
306  instance : `lsst.pex.config.Config`
307  The config instance that contains this field.
308 
309  Notes
310  -----
311  This method is invoked by the `lsst.pex.config.Config` object that
312  contains this field and should not be called directly.
313 
314  Renaming is only relevant for `~lsst.pex.config.Field` instances that
315  hold subconfigs. `~lsst.pex.config.Fields` that hold subconfigs should
316  rename each subconfig with the full field name as generated by
317  `lsst.pex.config.config._joinNamePath`.
318  """
319  pass
320 
321  def validate(self, instance):
322  """Validate the field (for internal use only).
323 
324  Parameters
325  ----------
326  instance : `lsst.pex.config.Config`
327  The config instance that contains this field.
328 
329  Raises
330  ------
331  lsst.pex.config.FieldValidationError
332  Raised if verification fails.
333 
334  Notes
335  -----
336  This method provides basic validation:
337 
338  - Ensures that the value is not `None` if the field is not optional.
339  - Ensures type correctness.
340  - Ensures that the user-provided ``check`` function is valid.
341 
342  Most `~lsst.pex.config.Field` subclasses should call
343  `lsst.pex.config.field.Field.validate` if they re-implement
344  `~lsst.pex.config.field.Field.validate`.
345  """
346  value = self.__get__(instance)
347  if not self.optional and value is None:
348  raise FieldValidationError(self, instance, "Required value cannot be None")
349 
350  def freeze(self, instance):
351  """Make this field read-only (for internal use only).
352 
353  Parameters
354  ----------
355  instance : `lsst.pex.config.Config`
356  The config instance that contains this field.
357 
358  Notes
359  -----
360  Freezing is only relevant for fields that hold subconfigs. Fields which
361  hold subconfigs should freeze each subconfig.
362 
363  **Subclasses should implement this method.**
364  """
365  pass
366 
367  def _validateValue(self, value):
368  """Validate a value.
369 
370  Parameters
371  ----------
372  value : object
373  The value being validated.
374 
375  Raises
376  ------
377  TypeError
378  Raised if the value's type is incompatible with the field's
379  ``dtype``.
380  ValueError
381  Raised if the value is rejected by the ``check`` method.
382  """
383  if value is None:
384  return
385 
386  if not isinstance(value, self.dtype):
387  msg = "Value %s is of incorrect type %s. Expected type %s" % \
388  (value, _typeStr(value), _typeStr(self.dtype))
389  raise TypeError(msg)
390  if self.check is not None and not self.check(value):
391  msg = "Value %s is not a valid value" % str(value)
392  raise ValueError(msg)
393 
394  def save(self, outfile, instance):
395  """Save this field to a file (for internal use only).
396 
397  Parameters
398  ----------
399  outfile : file-like object
400  A writeable field handle.
401  instance : `Config`
402  The `Config` instance that contains this field.
403 
404  Notes
405  -----
406  This method is invoked by the `~lsst.pex.config.Config` object that
407  contains this field and should not be called directly.
408 
409  The output consists of the documentation string
410  (`lsst.pex.config.Field.doc`) formatted as a Python comment. The second
411  line is formatted as an assignment: ``{fullname}={value}``.
412 
413  This output can be executed with Python.
414  """
415  value = self.__get__(instance)
416  fullname = _joinNamePath(instance._name, self.name)
417 
418  # write full documentation string as comment lines (i.e. first character is #)
419  doc = "# " + str(self.doc).replace("\n", "\n# ")
420  if isinstance(value, float) and (math.isinf(value) or math.isnan(value)):
421  # non-finite numbers need special care
422  outfile.write(u"{}\n{}=float('{!r}')\n\n".format(doc, fullname, value))
423  else:
424  outfile.write(u"{}\n{}={!r}\n\n".format(doc, fullname, value))
425 
426  def toDict(self, instance):
427  """Convert the field value so that it can be set as the value of an
428  item in a `dict` (for internal use only).
429 
430  Parameters
431  ----------
432  instance : `Config`
433  The `Config` that contains this field.
434 
435  Returns
436  -------
437  value : object
438  The field's value. See *Notes*.
439 
440  Notes
441  -----
442  This method invoked by the owning `~lsst.pex.config.Config` object and
443  should not be called directly.
444 
445  Simple values are passed through. Complex data structures must be
446  manipulated. For example, a `~lsst.pex.config.Field` holding a
447  subconfig should, instead of the subconfig object, return a `dict`
448  where the keys are the field names in the subconfig, and the values are
449  the field values in the subconfig.
450  """
451  return self.__get__(instance)
452 
453  def __get__(self, instance, owner=None, at=None, label="default"):
454  """Define how attribute access should occur on the Config instance
455  This is invoked by the owning config object and should not be called
456  directly
457 
458  When the field attribute is accessed on a Config class object, it
459  returns the field object itself in order to allow inspection of
460  Config classes.
461 
462  When the field attribute is access on a config instance, the actual
463  value described by the field (and held by the Config instance) is
464  returned.
465  """
466  if instance is None or not isinstance(instance, Config):
467  return self
468  else:
469  return instance._storage[self.name]
470 
471  def __set__(self, instance, value, at=None, label='assignment'):
472  """Set an attribute on the config instance.
473 
474  Parameters
475  ----------
476  instance : `lsst.pex.config.Config`
477  The config instance that contains this field.
478  value : obj
479  Value to set on this field.
480  at : `list` of `lsst.pex.config.callStack.StackFrame`
481  The call stack (created by
482  `lsst.pex.config.callStack.getCallStack`).
483  label : `str`, optional
484  Event label for the history.
485 
486  Notes
487  -----
488  This method is invoked by the owning `lsst.pex.config.Config` object
489  and should not be called directly.
490 
491  Derived `~lsst.pex.config.Field` classes may need to override the
492  behavior. When overriding ``__set__``, `~lsst.pex.config.Field` authors
493  should follow the following rules:
494 
495  - Do not allow modification of frozen configs.
496  - Validate the new value **before** modifying the field. Except if the
497  new value is `None`. `None` is special and no attempt should be made
498  to validate it until `lsst.pex.config.Config.validate` is called.
499  - Do not modify the `~lsst.pex.config.Config` instance to contain
500  invalid values.
501  - If the field is modified, update the history of the
502  `lsst.pex.config.field.Field` to reflect the changes.
503 
504  In order to decrease the need to implement this method in derived
505  `~lsst.pex.config.Field` types, value validation is performed in the
506  `lsst.pex.config.Field._validateValue`. If only the validation step
507  differs in the derived `~lsst.pex.config.Field`, it is simpler to
508  implement `lsst.pex.config.Field._validateValue` than to reimplement
509  ``__set__``. More complicated behavior, however, may require
510  reimplementation.
511  """
512  if instance._frozen:
513  raise FieldValidationError(self, instance, "Cannot modify a frozen Config")
514 
515  history = instance._history.setdefault(self.name, [])
516  if value is not None:
517  value = _autocast(value, self.dtype)
518  try:
519  self._validateValue(value)
520  except BaseException as e:
521  raise FieldValidationError(self, instance, str(e))
522 
523  instance._storage[self.name] = value
524  if at is None:
525  at = getCallStack()
526  history.append((value, at, label))
527 
528  def __delete__(self, instance, at=None, label='deletion'):
529  """Delete an attribute from a `lsst.pex.config.Config` instance.
530 
531  Parameters
532  ----------
533  instance : `lsst.pex.config.Config`
534  The config instance that contains this field.
535  at : `list` of `lsst.pex.config.callStack.StackFrame`
536  The call stack (created by
537  `lsst.pex.config.callStack.getCallStack`).
538  label : `str`, optional
539  Event label for the history.
540 
541  Notes
542  -----
543  This is invoked by the owning `~lsst.pex.config.Config` object and
544  should not be called directly.
545  """
546  if at is None:
547  at = getCallStack()
548  self.__set__(instance, None, at=at, label=label)
549 
550  def _compare(self, instance1, instance2, shortcut, rtol, atol, output):
551  """Compare a field (named `Field.name`) in two
552  `~lsst.pex.config.Config` instances for equality.
553 
554  Parameters
555  ----------
556  instance1 : `lsst.pex.config.Config`
557  Left-hand side `Config` instance to compare.
558  instance2 : `lsst.pex.config.Config`
559  Right-hand side `Config` instance to compare.
560  shortcut : `bool`, optional
561  **Unused.**
562  rtol : `float`, optional
563  Relative tolerance for floating point comparisons.
564  atol : `float`, optional
565  Absolute tolerance for floating point comparisons.
566  output : callable, optional
567  A callable that takes a string, used (possibly repeatedly) to
568  report inequalities.
569 
570  Notes
571  -----
572  This method must be overridden by more complex `Field` subclasses.
573 
574  See also
575  --------
576  lsst.pex.config.compareScalars
577  """
578  v1 = getattr(instance1, self.name)
579  v2 = getattr(instance2, self.name)
580  name = getComparisonName(
581  _joinNamePath(instance1._name, self.name),
582  _joinNamePath(instance2._name, self.name)
583  )
584  return compareScalars(name, v1, v2, dtype=self.dtype, rtol=rtol, atol=atol, output=output)
585 
586 
588  """Importer (for `sys.meta_path`) that records which modules are being
589  imported.
590 
591  *This class does not do any importing itself.*
592 
593  Examples
594  --------
595  Use this class as a context manager to ensure it is properly uninstalled
596  when done:
597 
598  >>> with RecordingImporter() as importer:
599  ... # import stuff
600  ... import numpy as np
601  ... print("Imported: " + importer.getModules())
602  """
603 
604  def __init__(self):
605  self._modules = set()
606 
607  def __enter__(self):
608  self.origMetaPath = sys.meta_path
609  sys.meta_path = [self] + sys.meta_path
610  return self
611 
612  def __exit__(self, *args):
613  self.uninstall()
614  return False # Don't suppress exceptions
615 
616  def uninstall(self):
617  """Uninstall the importer.
618  """
619  sys.meta_path = self.origMetaPath
620 
621  def find_module(self, fullname, path=None):
622  """Called as part of the ``import`` chain of events.
623  """
624  self._modules.add(fullname)
625  # Return None because we don't do any importing.
626  return None
627 
628  def getModules(self):
629  """Get the set of modules that were imported.
630 
631  Returns
632  -------
633  modules : `set` of `str`
634  Set of imported module names.
635  """
636  return self._modules
637 
638 
639 class Config(metaclass=ConfigMeta):
640  """Base class for configuration (*config*) objects.
641 
642  Notes
643  -----
644  A ``Config`` object will usually have several `~lsst.pex.config.Field`
645  instances as class attributes. These are used to define most of the base
646  class behavior.
647 
648  ``Config`` implements a mapping API that provides many `dict`-like methods,
649  such as `keys`, `values`, `items`, `iteritems`, `iterkeys`, and
650  `itervalues`. ``Config`` instances also support the ``in`` operator to
651  test if a field is in the config. Unlike a `dict`, ``Config`` classes are
652  not subscriptable. Instead, access individual fields as attributes of the
653  configuration instance.
654 
655  Examples
656  --------
657  Config classes are subclasses of ``Config`` that have
658  `~lsst.pex.config.Field` instances (or instances of
659  `~lsst.pex.config.Field` subclasses) as class attributes:
660 
661  >>> from lsst.pex.config import Config, Field, ListField
662  >>> class DemoConfig(Config):
663  ... intField = Field(doc="An integer field", dtype=int, default=42)
664  ... listField = ListField(doc="List of favorite beverages.", dtype=str,
665  ... default=['coffee', 'green tea', 'water'])
666  ...
667  >>> config = DemoConfig()
668 
669  Configs support many `dict`-like APIs:
670 
671  >>> config.keys()
672  ['intField', 'listField']
673  >>> 'intField' in config
674  True
675 
676  Individual fields can be accessed as attributes of the configuration:
677 
678  >>> config.intField
679  42
680  >>> config.listField.append('earl grey tea')
681  >>> print(config.listField)
682  ['coffee', 'green tea', 'water', 'earl grey tea']
683  """
684 
685  def __iter__(self):
686  """Iterate over fields.
687  """
688  return self._fields.__iter__()
689 
690  def keys(self):
691  """Get field names.
692 
693  Returns
694  -------
695  names : `list`
696  List of `lsst.pex.config.Field` names.
697 
698  See also
699  --------
700  lsst.pex.config.Config.iterkeys
701  """
702  return list(self._storage.keys())
703 
704  def values(self):
705  """Get field values.
706 
707  Returns
708  -------
709  values : `list`
710  List of field values.
711 
712  See also
713  --------
714  lsst.pex.config.Config.itervalues
715  """
716  return list(self._storage.values())
717 
718  def items(self):
719  """Get configurations as ``(field name, field value)`` pairs.
720 
721  Returns
722  -------
723  items : `list`
724  List of tuples for each configuration. Tuple items are:
725 
726  0. Field name.
727  1. Field value.
728 
729  See also
730  --------
731  lsst.pex.config.Config.iteritems
732  """
733  return list(self._storage.items())
734 
735  def iteritems(self):
736  """Iterate over (field name, field value) pairs.
737 
738  Yields
739  ------
740  item : `tuple`
741  Tuple items are:
742 
743  0. Field name.
744  1. Field value.
745 
746  See also
747  --------
748  lsst.pex.config.Config.items
749  """
750  return iter(self._storage.items())
751 
752  def itervalues(self):
753  """Iterate over field values.
754 
755  Yields
756  ------
757  value : obj
758  A field value.
759 
760  See also
761  --------
762  lsst.pex.config.Config.values
763  """
764  return iter(self.storage.values())
765 
766  def iterkeys(self):
767  """Iterate over field names
768 
769  Yields
770  ------
771  key : `str`
772  A field's key (attribute name).
773 
774  See also
775  --------
776  lsst.pex.config.Config.values
777  """
778  return iter(self.storage.keys())
779 
780  def __contains__(self, name):
781  """!Return True if the specified field exists in this config
782 
783  @param[in] name field name to test for
784  """
785  return self._storage.__contains__(name)
786 
787  def __new__(cls, *args, **kw):
788  """Allocate a new `lsst.pex.config.Config` object.
789 
790  In order to ensure that all Config object are always in a proper state
791  when handed to users or to derived `~lsst.pex.config.Config` classes,
792  some attributes are handled at allocation time rather than at
793  initialization.
794 
795  This ensures that even if a derived `~lsst.pex.config.Config` class
796  implements ``__init__``, its author does not need to be concerned about
797  when or even the base ``Config.__init__`` should be called.
798  """
799  name = kw.pop("__name", None)
800  at = kw.pop("__at", getCallStack())
801  # remove __label and ignore it
802  kw.pop("__label", "default")
803 
804  instance = object.__new__(cls)
805  instance._frozen = False
806  instance._name = name
807  instance._storage = {}
808  instance._history = {}
809  instance._imports = set()
810  # load up defaults
811  for field in instance._fields.values():
812  instance._history[field.name] = []
813  field.__set__(instance, field.default, at=at + [field.source], label="default")
814  # set custom default-overides
815  instance.setDefaults()
816  # set constructor overides
817  instance.update(__at=at, **kw)
818  return instance
819 
820  def __reduce__(self):
821  """Reduction for pickling (function with arguments to reproduce).
822 
823  We need to condense and reconstitute the `~lsst.pex.config.Config`,
824  since it may contain lambdas (as the ``check`` elements) that cannot
825  be pickled.
826  """
827  # The stream must be in characters to match the API but pickle requires bytes
828  stream = io.StringIO()
829  self.saveToStream(stream)
830  return (unreduceConfig, (self.__class__, stream.getvalue().encode()))
831 
832  def setDefaults(self):
833  """Subclass hook for computing defaults.
834 
835  Notes
836  -----
837  Derived `~lsst.pex.config.Config` classes that must compute defaults
838  rather than using the `~lsst.pex.config.Field` instances's defaults
839  should do so here. To correctly use inherited defaults,
840  implementations of ``setDefaults`` must call their base class's
841  ``setDefaults``.
842  """
843  pass
844 
845  def update(self, **kw):
846  """Update values of fields specified by the keyword arguments.
847 
848  Parameters
849  ----------
850  kw
851  Keywords are configuration field names. Values are configuration
852  field values.
853 
854  Notes
855  -----
856  The ``__at`` and ``__label`` keyword arguments are special internal
857  keywords. They are used to strip out any internal steps from the
858  history tracebacks of the config. Do not modify these keywords to
859  subvert a `~lsst.pex.config.Config` instance's history.
860 
861  Examples
862  --------
863  This is a config with three fields:
864 
865  >>> from lsst.pex.config import Config, Field
866  >>> class DemoConfig(Config):
867  ... fieldA = Field(doc='Field A', dtype=int, default=42)
868  ... fieldB = Field(doc='Field B', dtype=bool, default=True)
869  ... fieldC = Field(doc='Field C', dtype=str, default='Hello world')
870  ...
871  >>> config = DemoConfig()
872 
873  These are the default values of each field:
874 
875  >>> for name, value in config.iteritems():
876  ... print(f"{name}: {value}")
877  ...
878  fieldA: 42
879  fieldB: True
880  fieldC: 'Hello world'
881 
882  Using this method to update ``fieldA`` and ``fieldC``:
883 
884  >>> config.update(fieldA=13, fieldC='Updated!')
885 
886  Now the values of each field are:
887 
888  >>> for name, value in config.iteritems():
889  ... print(f"{name}: {value}")
890  ...
891  fieldA: 13
892  fieldB: True
893  fieldC: 'Updated!'
894  """
895  at = kw.pop("__at", getCallStack())
896  label = kw.pop("__label", "update")
897 
898  for name, value in kw.items():
899  try:
900  field = self._fields[name]
901  field.__set__(self, value, at=at, label=label)
902  except KeyError:
903  raise KeyError("No field of name %s exists in config type %s" % (name, _typeStr(self)))
904 
905  def load(self, filename, root="config"):
906  """Modify this config in place by executing the Python code in a
907  configuration file.
908 
909  Parameters
910  ----------
911  filename : `str`
912  Name of the configuration file. A configuration file is Python
913  module.
914  root : `str`, optional
915  Name of the variable in file that refers to the config being
916  overridden.
917 
918  For example, the value of root is ``"config"`` and the file
919  contains::
920 
921  config.myField = 5
922 
923  Then this config's field ``myField`` is set to ``5``.
924 
925  **Deprecated:** For backwards compatibility, older config files
926  that use ``root="root"`` instead of ``root="config"`` will be
927  loaded with a warning printed to `sys.stderr`. This feature will be
928  removed at some point.
929 
930  See also
931  --------
932  lsst.pex.config.Config.loadFromStream
933  lsst.pex.config.Config.save
934  lsst.pex.config.Config.saveFromStream
935  """
936  with open(filename, "r") as f:
937  code = compile(f.read(), filename=filename, mode="exec")
938  self.loadFromStream(stream=code, root=root)
939 
940  def loadFromStream(self, stream, root="config", filename=None):
941  """Modify this Config in place by executing the Python code in the
942  provided stream.
943 
944  Parameters
945  ----------
946  stream : file-like object, `str`, or compiled string
947  Stream containing configuration override code.
948  root : `str`, optional
949  Name of the variable in file that refers to the config being
950  overridden.
951 
952  For example, the value of root is ``"config"`` and the file
953  contains::
954 
955  config.myField = 5
956 
957  Then this config's field ``myField`` is set to ``5``.
958 
959  **Deprecated:** For backwards compatibility, older config files
960  that use ``root="root"`` instead of ``root="config"`` will be
961  loaded with a warning printed to `sys.stderr`. This feature will be
962  removed at some point.
963  filename : `str`, optional
964  Name of the configuration file, or `None` if unknown or contained
965  in the stream. Used for error reporting.
966 
967  See also
968  --------
969  lsst.pex.config.Config.load
970  lsst.pex.config.Config.save
971  lsst.pex.config.Config.saveFromStream
972  """
973  with RecordingImporter() as importer:
974  try:
975  local = {root: self}
976  exec(stream, {}, local)
977  except NameError as e:
978  if root == "config" and "root" in e.args[0]:
979  if filename is None:
980  # try to determine the file name; a compiled string has attribute "co_filename",
981  # an open file has attribute "name", else give up
982  filename = getattr(stream, "co_filename", None)
983  if filename is None:
984  filename = getattr(stream, "name", "?")
985  print(f"Config override file {filename!r}"
986  " appears to use 'root' instead of 'config'; trying with 'root'", file=sys.stderr)
987  local = {"root": self}
988  exec(stream, {}, local)
989  else:
990  raise
991 
992  self._imports.update(importer.getModules())
993 
994  def save(self, filename, root="config"):
995  """Save a Python script to the named file, which, when loaded,
996  reproduces this config.
997 
998  Parameters
999  ----------
1000  filename : `str`
1001  Desination filename of this configuration.
1002  root : `str`, optional
1003  Name to use for the root config variable. The same value must be
1004  used when loading (see `lsst.pex.config.Config.load`).
1005 
1006  See also
1007  --------
1008  lsst.pex.config.Config.saveToStream
1009  lsst.pex.config.Config.load
1010  lsst.pex.config.Config.loadFromStream
1011  """
1012  d = os.path.dirname(filename)
1013  with tempfile.NamedTemporaryFile(mode="w", delete=False, dir=d) as outfile:
1014  self.saveToStream(outfile, root)
1015  # tempfile is hardcoded to create files with mode '0600'
1016  # for an explantion of these antics see:
1017  # https://stackoverflow.com/questions/10291131/how-to-use-os-umask-in-python
1018  umask = os.umask(0o077)
1019  os.umask(umask)
1020  os.chmod(outfile.name, (~umask & 0o666))
1021  # chmod before the move so we get quasi-atomic behavior if the
1022  # source and dest. are on the same filesystem.
1023  # os.rename may not work across filesystems
1024  shutil.move(outfile.name, filename)
1025 
1026  def saveToStream(self, outfile, root="config"):
1027  """Save a configuration file to a stream, which, when loaded,
1028  reproduces this config.
1029 
1030  Parameters
1031  ----------
1032  outfile : file-like object
1033  Destination file object write the config into. Accepts strings not
1034  bytes.
1035  root
1036  Name to use for the root config variable. The same value must be
1037  used when loading (see `lsst.pex.config.Config.load`).
1038 
1039  See also
1040  --------
1041  lsst.pex.config.Config.save
1042  lsst.pex.config.Config.load
1043  lsst.pex.config.Config.loadFromStream
1044  """
1045  tmp = self._name
1046  self._rename(root)
1047  try:
1048  configType = type(self)
1049  typeString = _typeStr(configType)
1050  outfile.write(u"import {}\n".format(configType.__module__))
1051  outfile.write(u"assert type({})=={}, 'config is of type %s.%s ".format(root, typeString))
1052  outfile.write(u"instead of {}' % (type({}).__module__, type({}).__name__)\n".format(typeString,
1053  root,
1054  root))
1055  self._save(outfile)
1056  finally:
1057  self._rename(tmp)
1058 
1059  def freeze(self):
1060  """Make this config, and all subconfigs, read-only.
1061  """
1062  self._frozen = True
1063  for field in self._fields.values():
1064  field.freeze(self)
1065 
1066  def _save(self, outfile):
1067  """Save this config to an open stream object.
1068 
1069  Parameters
1070  ----------
1071  outfile : file-like object
1072  Destination file object write the config into. Accepts strings not
1073  bytes.
1074  """
1075  for imp in self._imports:
1076  if imp in sys.modules and sys.modules[imp] is not None:
1077  outfile.write(u"import {}\n".format(imp))
1078  for field in self._fields.values():
1079  field.save(outfile, self)
1080 
1081  def toDict(self):
1082  """Make a dictionary of field names and their values.
1083 
1084  Returns
1085  -------
1086  dict_ : `dict`
1087  Dictionary with keys that are `~lsst.pex.config.Field` names.
1088  Values are `~lsst.pex.config.Field` values.
1089 
1090  See also
1091  --------
1092  lsst.pex.config.Field.toDict
1093 
1094  Notes
1095  -----
1096  This method uses the `~lsst.pex.config.Field.toDict` method of
1097  individual fields. Subclasses of `~lsst.pex.config.Field` may need to
1098  implement a ``toDict`` method for *this* method to work.
1099  """
1100  dict_ = {}
1101  for name, field in self._fields.items():
1102  dict_[name] = field.toDict(self)
1103  return dict_
1104 
1105  def names(self):
1106  """Get all the field names in the config, recursively.
1107 
1108  Returns
1109  -------
1110  names : `list` of `str`
1111  Field names.
1112  """
1113  #
1114  # Rather than sort out the recursion all over again use the
1115  # pre-existing saveToStream()
1116  #
1117  with io.StringIO() as strFd:
1118  self.saveToStream(strFd, "config")
1119  contents = strFd.getvalue()
1120  strFd.close()
1121  #
1122  # Pull the names out of the dumped config
1123  #
1124  keys = []
1125  for line in contents.split("\n"):
1126  if re.search(r"^((assert|import)\s+|\s*$|#)", line):
1127  continue
1128 
1129  mat = re.search(r"^(?:config\.)?([^=]+)\s*=\s*.*", line)
1130  if mat:
1131  keys.append(mat.group(1))
1132 
1133  return keys
1134 
1135  def _rename(self, name):
1136  """Rename this config object in its parent `~lsst.pex.config.Config`.
1137 
1138  Parameters
1139  ----------
1140  name : `str`
1141  New name for this config in its parent `~lsst.pex.config.Config`.
1142 
1143  Notes
1144  -----
1145  This method uses the `~lsst.pex.config.Field.rename` method of
1146  individual `lsst.pex.config.Field` instances.
1147  `lsst.pex.config.Field` subclasses may need to implement a ``rename``
1148  method for *this* method to work.
1149 
1150  See also
1151  --------
1152  lsst.pex.config.Field.rename
1153  """
1154  self._name = name
1155  for field in self._fields.values():
1156  field.rename(self)
1157 
1158  def validate(self):
1159  """Validate the Config, raising an exception if invalid.
1160 
1161  Raises
1162  ------
1163  lsst.pex.config.FieldValidationError
1164  Raised if verification fails.
1165 
1166  Notes
1167  -----
1168  The base class implementation performs type checks on all fields by
1169  calling their `~lsst.pex.config.Field.validate` methods.
1170 
1171  Complex single-field validation can be defined by deriving new Field
1172  types. For convenience, some derived `lsst.pex.config.Field`-types
1173  (`~lsst.pex.config.ConfigField` and
1174  `~lsst.pex.config.ConfigChoiceField`) are defined in `lsst.pex.config`
1175  that handle recursing into subconfigs.
1176 
1177  Inter-field relationships should only be checked in derived
1178  `~lsst.pex.config.Config` classes after calling this method, and base
1179  validation is complete.
1180  """
1181  for field in self._fields.values():
1182  field.validate(self)
1183 
1184  def formatHistory(self, name, **kwargs):
1185  """Format a configuration field's history to a human-readable format.
1186 
1187  Parameters
1188  ----------
1189  name : `str`
1190  Name of a `~lsst.pex.config.Field` in this config.
1191  kwargs
1192  Keyword arguments passed to `lsst.pex.config.history.format`.
1193 
1194  Returns
1195  -------
1196  history : `str`
1197  A string containing the formatted history.
1198 
1199  See also
1200  --------
1201  lsst.pex.config.history.format
1202  """
1203  import lsst.pex.config.history as pexHist
1204  return pexHist.format(self, name, **kwargs)
1205 
1206  history = property(lambda x: x._history)
1207  """Read-only history.
1208  """
1209 
1210  def __setattr__(self, attr, value, at=None, label="assignment"):
1211  """Set an attribute (such as a field's value).
1212 
1213  Notes
1214  -----
1215  Unlike normal Python objects, `~lsst.pex.config.Config` objects are
1216  locked such that no additional attributes nor properties may be added
1217  to them dynamically.
1218 
1219  Although this is not the standard Python behavior, it helps to protect
1220  users from accidentally mispelling a field name, or trying to set a
1221  non-existent field.
1222  """
1223  if attr in self._fields:
1224  if at is None:
1225  at = getCallStack()
1226  # This allows Field descriptors to work.
1227  self._fields[attr].__set__(self, value, at=at, label=label)
1228  elif hasattr(getattr(self.__class__, attr, None), '__set__'):
1229  # This allows properties and other non-Field descriptors to work.
1230  return object.__setattr__(self, attr, value)
1231  elif attr in self.__dict__ or attr in ("_name", "_history", "_storage", "_frozen", "_imports"):
1232  # This allows specific private attributes to work.
1233  self.__dict__[attr] = value
1234  else:
1235  # We throw everything else.
1236  raise AttributeError("%s has no attribute %s" % (_typeStr(self), attr))
1237 
1238  def __delattr__(self, attr, at=None, label="deletion"):
1239  if attr in self._fields:
1240  if at is None:
1241  at = getCallStack()
1242  self._fields[attr].__delete__(self, at=at, label=label)
1243  else:
1244  object.__delattr__(self, attr)
1245 
1246  def __eq__(self, other):
1247  if type(other) == type(self):
1248  for name in self._fields:
1249  thisValue = getattr(self, name)
1250  otherValue = getattr(other, name)
1251  if isinstance(thisValue, float) and math.isnan(thisValue):
1252  if not math.isnan(otherValue):
1253  return False
1254  elif thisValue != otherValue:
1255  return False
1256  return True
1257  return False
1258 
1259  def __ne__(self, other):
1260  return not self.__eq__(other)
1261 
1262  def __str__(self):
1263  return str(self.toDict())
1264 
1265  def __repr__(self):
1266  return "%s(%s)" % (
1267  _typeStr(self),
1268  ", ".join("%s=%r" % (k, v) for k, v in self.toDict().items() if v is not None)
1269  )
1270 
1271  def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None):
1272  """Compare this configuration to another `~lsst.pex.config.Config` for
1273  equality.
1274 
1275  Parameters
1276  ----------
1277  other : `lsst.pex.config.Config`
1278  Other `~lsst.pex.config.Config` object to compare against this
1279  config.
1280  shortcut : `bool`, optional
1281  If `True`, return as soon as an inequality is found. Default is
1282  `True`.
1283  rtol : `float`, optional
1284  Relative tolerance for floating point comparisons.
1285  atol : `float`, optional
1286  Absolute tolerance for floating point comparisons.
1287  output : callable, optional
1288  A callable that takes a string, used (possibly repeatedly) to
1289  report inequalities.
1290 
1291  Returns
1292  -------
1293  isEqual : `bool`
1294  `True` when the two `lsst.pex.config.Config` instances are equal.
1295  `False` if there is an inequality.
1296 
1297  See also
1298  --------
1299  lsst.pex.config.compareConfigs
1300 
1301  Notes
1302  -----
1303  Unselected targets of `~lsst.pex.config.RegistryField` fields and
1304  unselected choices of `~lsst.pex.config.ConfigChoiceField` fields
1305  are not considered by this method.
1306 
1307  Floating point comparisons are performed by `numpy.allclose`.
1308  """
1309  name1 = self._name if self._name is not None else "config"
1310  name2 = other._name if other._name is not None else "config"
1311  name = getComparisonName(name1, name2)
1312  return compareConfigs(name, self, other, shortcut=shortcut,
1313  rtol=rtol, atol=atol, output=output)
1314 
1315 
1316 def unreduceConfig(cls, stream):
1317  """Create a `~lsst.pex.config.Config` from a stream.
1318 
1319  Parameters
1320  ----------
1321  cls : `lsst.pex.config.Config`-type
1322  A `lsst.pex.config.Config` type (not an instance) that is instantiated
1323  with configurations in the ``stream``.
1324  stream : file-like object, `str`, or compiled string
1325  Stream containing configuration override code.
1326 
1327  Returns
1328  -------
1329  config : `lsst.pex.config.Config`
1330  Config instance.
1331 
1332  See also
1333  --------
1334  lsst.pex.config.Config.loadFromStream
1335  """
1336  config = cls()
1337  config.loadFromStream(stream)
1338  return config
def toDict(self, instance)
Definition: config.py:426
def __eq__(self, other)
Definition: config.py:1246
def formatHistory(self, name, kwargs)
Definition: config.py:1184
def load(self, filename, root="config")
Definition: config.py:905
def _rename(self, name)
Definition: config.py:1135
def __init__(self, doc, dtype, default=None, check=None, optional=False)
Definition: config.py:257
def compareConfigs(name, c1, c2, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Definition: comparison.py:105
def unreduceConfig(cls, stream)
Definition: config.py:1316
def _save(self, outfile)
Definition: config.py:1066
def __init__(self, field, config, msg)
Definition: config.py:142
def saveToStream(self, outfile, root="config")
Definition: config.py:1026
def __delattr__(self, attr, at=None, label="deletion")
Definition: config.py:1238
daf::base::PropertySet * set
Definition: fits.cc:832
def getCallStack(skip=0)
Definition: callStack.py:169
def __setattr__(cls, name, value)
Definition: config.py:130
def __get__(self, instance, owner=None, at=None, label="default")
Definition: config.py:453
def save(self, filename, root="config")
Definition: config.py:994
def __ne__(self, other)
Definition: config.py:1259
def _setup(self, doc, dtype, default, check, optional, source)
Definition: config.py:264
def getStackFrame(relative=0)
Definition: callStack.py:52
def __contains__(self, name)
Return True if the specified field exists in this config.
Definition: config.py:780
table::Key< int > type
Definition: Detector.cc:164
def save(self, outfile, instance)
Definition: config.py:394
def _validateValue(self, value)
Definition: config.py:367
def validate(self, instance)
Definition: config.py:321
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def __new__(cls, args, kw)
Definition: config.py:787
def __init__(cls, name, bases, dict_)
Definition: config.py:109
def __setattr__(self, attr, value, at=None, label="assignment")
Definition: config.py:1210
def freeze(self, instance)
Definition: config.py:350
def compareScalars(name, v1, v2, output, rtol=1E-8, atol=1E-8, dtype=None)
Definition: comparison.py:56
def __set__(self, instance, value, at=None, label='assignment')
Definition: config.py:471
def loadFromStream(self, stream, root="config", filename=None)
Definition: config.py:940
def compare(self, other, shortcut=True, rtol=1E-8, atol=1E-8, output=None)
Definition: config.py:1271
daf::base::PropertyList * list
Definition: fits.cc:833
def getComparisonName(name1, name2)
Definition: comparison.py:34
def rename(self, instance)
Definition: config.py:300
def __delete__(self, instance, at=None, label='deletion')
Definition: config.py:528
def find_module(self, fullname, path=None)
Definition: config.py:621