LSST Applications  21.0.0+75b29a8a7f,21.0.0+e70536a077,21.0.0-1-ga51b5d4+62c747d40b,21.0.0-10-gbfb87ad6+3307648ee3,21.0.0-15-gedb9d5423+47cba9fc36,21.0.0-2-g103fe59+fdf0863a2a,21.0.0-2-g1367e85+d38a93257c,21.0.0-2-g45278ab+e70536a077,21.0.0-2-g5242d73+d38a93257c,21.0.0-2-g7f82c8f+e682ffb718,21.0.0-2-g8dde007+d179fbfa6a,21.0.0-2-g8f08a60+9402881886,21.0.0-2-ga326454+e682ffb718,21.0.0-2-ga63a54e+08647d4b1b,21.0.0-2-gde069b7+26c92b3210,21.0.0-2-gecfae73+0445ed2f95,21.0.0-2-gfc62afb+d38a93257c,21.0.0-27-gbbd0d29+ae871e0f33,21.0.0-28-g5fc5e037+feb0e9397b,21.0.0-3-g21c7a62+f4b9c0ff5c,21.0.0-3-g357aad2+57b0bddf0b,21.0.0-3-g4be5c26+d38a93257c,21.0.0-3-g65f322c+3f454acf5d,21.0.0-3-g7d9da8d+75b29a8a7f,21.0.0-3-gaa929c8+9e4ef6332c,21.0.0-3-ge02ed75+4b120a55c4,21.0.0-4-g3300ddd+e70536a077,21.0.0-4-g591bb35+4b120a55c4,21.0.0-4-gc004bbf+4911b9cd27,21.0.0-4-gccdca77+f94adcd104,21.0.0-4-ge8fba5a+2b3a696ff9,21.0.0-5-gb155db7+2c5429117a,21.0.0-5-gdf36809+637e4641ee,21.0.0-6-g00874e7+c9fd7f7160,21.0.0-6-g4e60332+4b120a55c4,21.0.0-7-gc8ca178+40eb9cf840,21.0.0-8-gfbe0b4b+9e4ef6332c,21.0.0-9-g2fd488a+d83b7cd606,w.2021.05
LSST Data Management Base Package
packages.py
Go to the documentation of this file.
1 # This file is part of base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """
22 Determine which packages are being used in the system and their versions
23 """
24 import os
25 import sys
26 import hashlib
27 import importlib
28 import subprocess
29 import logging
30 import pickle as pickle
31 import re
32 import yaml
33 from collections.abc import Mapping
34 from functools import lru_cache
35 
36 from .versions import getRuntimeVersions
37 
38 log = logging.getLogger(__name__)
39 
40 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages",
41  "getCondaPackages", "Packages"]
42 
43 
44 # Packages used at build-time (e.g., header-only)
45 BUILDTIME = set(["boost", "eigen", "tmv"])
46 
47 # Python modules to attempt to load so we can try to get the version
48 # We do this because the version only appears to be available from python,
49 # but we use the library
50 PYTHON = set(["galsim"])
51 
52 # Packages that don't seem to have a mechanism for reporting the runtime
53 # version. We need to guess the version from the environment
54 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
55 
56 
58  """Determine the version of a python module.
59 
60  Parameters
61  ----------
62  module : `module`
63  Module for which to get version.
64 
65  Returns
66  -------
67  version : `str`
68 
69  Raises
70  ------
71  AttributeError
72  Raised if __version__ attribute is not set.
73 
74  Notes
75  -----
76  We supplement the version with information from the
77  ``__dependency_versions__`` (a specific variable set by LSST's
78  `~lsst.sconsUtils` at build time) only for packages that are typically
79  used only at build-time.
80  """
81  version = module.__version__
82  if hasattr(module, "__dependency_versions__"):
83  # Add build-time dependencies
84  deps = module.__dependency_versions__
85  buildtime = BUILDTIME & set(deps.keys())
86  if buildtime:
87  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
88  for pkg in sorted(buildtime))
89  return str(version)
90 
91 
93  """Get imported python packages and their versions.
94 
95  Returns
96  -------
97  packages : `dict`
98  Keys (type `str`) are package names; values (type `str`) are their
99  versions.
100 
101  Notes
102  -----
103  We wade through `sys.modules` and attempt to determine the version for each
104  module. Note, therefore, that we can only report on modules that have
105  *already* been imported.
106 
107  We don't include any module for which we cannot determine a version.
108  """
109  # Attempt to import libraries that only report their version in python
110  for module in PYTHON:
111  try:
112  importlib.import_module(module)
113  except Exception:
114  pass # It's not available, so don't care
115 
116  packages = {"python": sys.version}
117  # Not iterating with sys.modules.iteritems() because it's not atomic and
118  # subject to race conditions
119  moduleNames = list(sys.modules.keys())
120  for name in moduleNames:
121  module = sys.modules[name]
122  try:
123  ver = getVersionFromPythonModule(module)
124  except Exception:
125  continue # Can't get a version from it, don't care
126 
127  # Remove "foo.bar.version" in favor of "foo.bar"
128  # This prevents duplication when the __init__.py includes
129  # "from .version import *"
130  for ending in (".version", "._version"):
131  if name.endswith(ending):
132  name = name[:-len(ending)]
133  if name in packages:
134  assert ver == packages[name]
135  elif name in packages:
136  assert ver == packages[name]
137 
138  # Use LSST package names instead of python module names
139  # This matches the names we get from the environment (i.e., EUPS)
140  # so we can clobber these build-time versions if the environment
141  # reveals that we're not using the packages as-built.
142  if "lsst" in name:
143  name = name.replace("lsst.", "").replace(".", "_")
144 
145  packages[name] = ver
146 
147  return packages
148 
149 
150 _eups = None # Singleton Eups object
151 
152 
153 @lru_cache(maxsize=1)
155  """Get products and their versions from the environment.
156 
157  Returns
158  -------
159  packages : `dict`
160  Keys (type `str`) are product names; values (type `str`) are their
161  versions.
162 
163  Notes
164  -----
165  We use EUPS to determine the version of certain products (those that don't
166  provide a means to determine the version any other way) and to check if
167  uninstalled packages are being used. We only report the product/version
168  for these packages.
169  """
170  try:
171  from eups import Eups
172  from eups.Product import Product
173  except ImportError:
174  log.warning("Unable to import eups, so cannot determine package versions from environment")
175  return {}
176 
177  # Cache eups object since creating it can take a while
178  global _eups
179  if not _eups:
180  _eups = Eups()
181  products = _eups.findProducts(tags=["setup"])
182 
183  # Get versions for things we can't determine via runtime mechanisms
184  # XXX Should we just grab everything we can, rather than just a
185  # predetermined set?
186  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
187 
188  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the
189  # version name indicates uninstalled code, so the version could be
190  # different than what's being reported by the runtime environment (because
191  # we don't tend to run "scons" every time we update some python file,
192  # and even if we did sconsUtils probably doesn't check to see if the repo
193  # is clean).
194  for prod in products:
195  if not prod.version.startswith(Product.LocalVersionPrefix):
196  continue
197  ver = prod.version
198 
199  gitDir = os.path.join(prod.dir, ".git")
200  if os.path.exists(gitDir):
201  # get the git revision and an indication if the working copy is
202  # clean
203  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
204  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
205  "--patch"]
206  try:
207  rev = subprocess.check_output(revCmd).decode().strip()
208  diff = subprocess.check_output(diffCmd)
209  except Exception:
210  ver += "@GIT_ERROR"
211  else:
212  ver += "@" + rev
213  if diff:
214  ver += "+" + hashlib.md5(diff).hexdigest()
215  else:
216  ver += "@NO_GIT"
217 
218  packages[prod.name] = ver
219  return packages
220 
221 
222 @lru_cache(maxsize=1)
224  """Get products and their versions from the conda environment.
225 
226  Returns
227  -------
228  packages : `dict`
229  Keys (type `str`) are product names; values (type `str`) are their
230  versions.
231 
232  Notes
233  -----
234  Returns empty result if a conda environment is not in use or can not
235  be queried.
236  """
237 
238  try:
239  import json
240  from conda.cli.python_api import Commands, run_command
241  except ImportError:
242  return {}
243 
244  # Get the installed package list
245  versions_json = run_command(Commands.LIST, "--json")
246  packages = {pkg["name"]: pkg["version"] for pkg in json.loads(versions_json[0])}
247 
248  # Try to work out the conda environment name and include it as a fake
249  # package. The "obvious" way of running "conda info --json" does give
250  # access to the active_prefix but takes about 2 seconds to run.
251  # The equivalent to the code above would be:
252  # info_json = run_command(Commands.INFO, "--json")
253  # As a comporomise look for the env name in the path to the python
254  # executable
255  match = re.search(r"/envs/(.*?)/bin/", sys.executable)
256  if match:
257  packages["conda_env"] = match.group(1)
258 
259  return packages
260 
261 
262 class Packages:
263  """A table of packages and their versions.
264 
265  There are a few different types of packages, and their versions are
266  collected in different ways:
267 
268  1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
269  interrogating the dynamic library
270  2. Python modules (e.g., afw, numpy; galsim is also in this group even
271  though we only use it through the library, because no version
272  information is currently provided through the library): we get their
273  version from the ``__version__`` module variable. Note that this means
274  that we're only aware of modules that have already been imported.
275  3. Other packages provide no run-time accessible version information (e.g.,
276  astrometry_net): we get their version from interrogating the
277  environment. Currently, that means EUPS; if EUPS is replaced or dropped
278  then we'll need to consider an alternative means of getting this version
279  information.
280  4. Local versions of packages (a non-installed EUPS package, selected with
281  ``setup -r /path/to/package``): we identify these through the
282  environment (EUPS again) and use as a version the path supplemented with
283  the ``git`` SHA and, if the git repo isn't clean, an MD5 of the diff.
284 
285  These package versions are collected and stored in a Packages object, which
286  provides useful comparison and persistence features.
287 
288  Example usage:
289 
290  .. code-block:: python
291 
292  from lsst.base import Packages
293  pkgs = Packages.fromSystem()
294  print("Current packages:", pkgs)
295  old = Packages.read("/path/to/packages.pickle")
296  print("Old packages:", old)
297  print("Missing packages compared to before:", pkgs.missing(old))
298  print("Extra packages compared to before:", pkgs.extra(old))
299  print("Different packages: ", pkgs.difference(old))
300  old.update(pkgs) # Include any new packages in the old
301  old.write("/path/to/packages.pickle")
302 
303  Parameters
304  ----------
305  packages : `dict`
306  A mapping {package: version} where both keys and values are type `str`.
307 
308  Notes
309  -----
310  This is essentially a wrapper around a dict with some conveniences.
311  """
312 
313  formats = {".pkl": "pickle",
314  ".pickle": "pickle",
315  ".yaml": "yaml"}
316 
317  def __init__(self, packages):
318  assert isinstance(packages, Mapping)
319  self._packages_packages = packages
320  self._names_names = set(packages.keys())
321 
322  @classmethod
323  def fromSystem(cls):
324  """Construct a `Packages` by examining the system.
325 
326  Determine packages by examining python's `sys.modules`, runtime
327  libraries and EUPS.
328 
329  Returns
330  -------
331  packages : `Packages`
332  """
333  packages = {}
334  packages.update(getPythonPackages())
335  packages.update(getCondaPackages())
336  packages.update(getRuntimeVersions())
337  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
338  return cls(packages)
339 
340  @classmethod
341  def fromBytes(cls, data, format):
342  """Construct the object from a byte representation.
343 
344  Parameters
345  ----------
346  data : `bytes`
347  The serialized form of this object in bytes.
348  format : `str`
349  The format of those bytes. Can be ``yaml`` or ``pickle``.
350  """
351  if format == "pickle":
352  new = pickle.loads(data)
353  elif format == "yaml":
354  new = yaml.load(data, Loader=yaml.SafeLoader)
355  else:
356  raise ValueError(f"Unexpected serialization format given: {format}")
357  if not isinstance(new, cls):
358  raise TypeError(f"Extracted object of class '{type(new)}' but expected '{cls}'")
359  return new
360 
361  @classmethod
362  def read(cls, filename):
363  """Read packages from filename.
364 
365  Parameters
366  ----------
367  filename : `str`
368  Filename from which to read. The format is determined from the
369  file extension. Currently support ``.pickle``, ``.pkl``
370  and ``.yaml``.
371 
372  Returns
373  -------
374  packages : `Packages`
375  """
376  _, ext = os.path.splitext(filename)
377  if ext not in cls.formatsformats:
378  raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
379  with open(filename, "rb") as ff:
380  # We assume that these classes are tiny so there is no
381  # substantive memory impact by reading the entire file up front
382  data = ff.read()
383  return cls.fromBytesfromBytes(data, cls.formatsformats[ext])
384 
385  def toBytes(self, format):
386  """Convert the object to a serialized bytes form using the
387  specified format.
388 
389  Parameters
390  ----------
391  format : `str`
392  Format to use when serializing. Can be ``yaml`` or ``pickle``.
393 
394  Returns
395  -------
396  data : `bytes`
397  Byte string representing the serialized object.
398  """
399  if format == "pickle":
400  return pickle.dumps(self)
401  elif format == "yaml":
402  return yaml.dump(self).encode("utf-8")
403  else:
404  raise ValueError(f"Unexpected serialization format requested: {format}")
405 
406  def write(self, filename):
407  """Write to file.
408 
409  Parameters
410  ----------
411  filename : `str`
412  Filename to which to write. The format of the data file
413  is determined from the file extension. Currently supports
414  ``.pickle`` and ``.yaml``
415  """
416  _, ext = os.path.splitext(filename)
417  if ext not in self.formatsformats:
418  raise ValueError(f"Format from {ext} extension in file {filename} not recognized")
419  with open(filename, "wb") as ff:
420  # Assumes that the bytes serialization of this object is
421  # relatively small.
422  ff.write(self.toBytestoBytes(self.formatsformats[ext]))
423 
424  def __len__(self):
425  return len(self._packages_packages)
426 
427  def __str__(self):
428  ss = "%s({\n" % self.__class__.__name__
429  # Sort alphabetically by module name, for convenience in reading
430  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages_packages[prod])) for
431  prod in sorted(self._names_names))
432  ss += ",\n})"
433  return ss
434 
435  def __repr__(self):
436  return "%s(%s)" % (self.__class__.__name__, repr(self._packages_packages))
437 
438  def __contains__(self, pkg):
439  return pkg in self._packages_packages
440 
441  def __iter__(self):
442  return iter(self._packages_packages)
443 
444  def __eq__(self, other):
445  if not isinstance(other, type(self)):
446  return False
447 
448  return self._packages_packages == other._packages
449 
450  def update(self, other):
451  """Update packages with contents of another set of packages.
452 
453  Parameters
454  ----------
455  other : `Packages`
456  Other packages to merge with self.
457 
458  Notes
459  -----
460  No check is made to see if we're clobbering anything.
461  """
462  self._packages_packages.update(other._packages)
463  self._names_names.update(other._names)
464 
465  def extra(self, other):
466  """Get packages in self but not in another `Packages` object.
467 
468  Parameters
469  ----------
470  other : `Packages`
471  Other packages to compare against.
472 
473  Returns
474  -------
475  extra : `dict`
476  Extra packages. Keys (type `str`) are package names; values
477  (type `str`) are their versions.
478  """
479  return {pkg: self._packages_packages[pkg] for pkg in self._names_names - other._names}
480 
481  def missing(self, other):
482  """Get packages in another `Packages` object but missing from self.
483 
484  Parameters
485  ----------
486  other : `Packages`
487  Other packages to compare against.
488 
489  Returns
490  -------
491  missing : `dict`
492  Missing packages. Keys (type `str`) are package names; values
493  (type `str`) are their versions.
494  """
495  return {pkg: other._packages[pkg] for pkg in other._names - self._names_names}
496 
497  def difference(self, other):
498  """Get packages in symmetric difference of self and another `Packages`
499  object.
500 
501  Parameters
502  ----------
503  other : `Packages`
504  Other packages to compare against.
505 
506  Returns
507  -------
508  difference : `dict`
509  Packages in symmetric difference. Keys (type `str`) are package
510  names; values (type `str`) are their versions.
511  """
512  return {pkg: (self._packages_packages[pkg], other._packages[pkg]) for
513  pkg in self._names_names & other._names if self._packages_packages[pkg] != other._packages[pkg]}
514 
515 
516 # Register YAML representers
517 
518 def pkg_representer(dumper, data):
519  """Represent Packages as a simple dict"""
520  return dumper.represent_mapping("lsst.base.Packages", data._packages,
521  flow_style=None)
522 
523 
524 yaml.add_representer(Packages, pkg_representer)
525 
526 
527 def pkg_constructor(loader, node):
528  yield Packages(loader.construct_mapping(node, deep=True))
529 
530 
531 for loader in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
532  yaml.add_constructor("lsst.base.Packages", pkg_constructor, Loader=loader)
def fromBytes(cls, data, format)
Definition: packages.py:341
def __contains__(self, pkg)
Definition: packages.py:438
def read(cls, filename)
Definition: packages.py:362
def missing(self, other)
Definition: packages.py:481
def write(self, filename)
Definition: packages.py:406
def difference(self, other)
Definition: packages.py:497
def extra(self, other)
Definition: packages.py:465
def __init__(self, packages)
Definition: packages.py:317
def update(self, other)
Definition: packages.py:450
def toBytes(self, format)
Definition: packages.py:385
def __eq__(self, other)
Definition: packages.py:444
def getEnvironmentPackages()
Definition: packages.py:154
def getCondaPackages()
Definition: packages.py:223
def getPythonPackages()
Definition: packages.py:92
def pkg_representer(dumper, data)
Definition: packages.py:518
def getVersionFromPythonModule(module)
Definition: packages.py:57
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
table::Key< int > type
Definition: Detector.cc:163
daf::base::PropertyList * list
Definition: fits.cc:913
daf::base::PropertySet * set
Definition: fits.cc:912
bool strip
Definition: fits.cc:911