LSSTApplications  20.0.0
LSSTDataManagementBasePackage
packages.py
Go to the documentation of this file.
1 # This file is part of base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 """
22 Determine which packages are being used in the system and their versions
23 """
24 import os
25 import sys
26 import hashlib
27 import importlib
28 import subprocess
29 import logging
30 import pickle as pickle
31 from collections.abc import Mapping
32 
33 from .versions import getRuntimeVersions
34 
35 log = logging.getLogger(__name__)
36 
37 __all__ = ["getVersionFromPythonModule", "getPythonPackages", "getEnvironmentPackages", "Packages"]
38 
39 
40 # Packages used at build-time (e.g., header-only)
41 BUILDTIME = set(["boost", "eigen", "tmv"])
42 
43 # Python modules to attempt to load so we can try to get the version
44 # We do this because the version only appears to be available from python, but we use the library
45 PYTHON = set(["galsim"])
46 
47 # Packages that don't seem to have a mechanism for reporting the runtime version
48 # We need to guess the version from the environment
49 ENVIRONMENT = set(["astrometry_net", "astrometry_net_data", "minuit2", "xpa"])
50 
51 
53  """Determine the version of a python module.
54 
55  Parameters
56  ----------
57  module : `module`
58  Module for which to get version.
59 
60  Returns
61  -------
62  version : `str`
63 
64  Raises
65  ------
66  AttributeError
67  Raised if __version__ attribute is not set.
68 
69  Notes
70  -----
71  We supplement the version with information from the
72  ``__dependency_versions__`` (a specific variable set by LSST's
73  `~lsst.sconsUtils` at build time) only for packages that are typically
74  used only at build-time.
75  """
76  version = module.__version__
77  if hasattr(module, "__dependency_versions__"):
78  # Add build-time dependencies
79  deps = module.__dependency_versions__
80  buildtime = BUILDTIME & set(deps.keys())
81  if buildtime:
82  version += " with " + " ".join("%s=%s" % (pkg, deps[pkg])
83  for pkg in sorted(buildtime))
84  return version
85 
86 
88  """Get imported python packages and their versions.
89 
90  Returns
91  -------
92  packages : `dict`
93  Keys (type `str`) are package names; values (type `str`) are their
94  versions.
95 
96  Notes
97  -----
98  We wade through `sys.modules` and attempt to determine the version for each
99  module. Note, therefore, that we can only report on modules that have
100  *already* been imported.
101 
102  We don't include any module for which we cannot determine a version.
103  """
104  # Attempt to import libraries that only report their version in python
105  for module in PYTHON:
106  try:
107  importlib.import_module(module)
108  except Exception:
109  pass # It's not available, so don't care
110 
111  packages = {"python": sys.version}
112  # Not iterating with sys.modules.iteritems() because it's not atomic and subject to race conditions
113  moduleNames = list(sys.modules.keys())
114  for name in moduleNames:
115  module = sys.modules[name]
116  try:
117  ver = getVersionFromPythonModule(module)
118  except Exception:
119  continue # Can't get a version from it, don't care
120 
121  # Remove "foo.bar.version" in favor of "foo.bar"
122  # This prevents duplication when the __init__.py includes "from .version import *"
123  for ending in (".version", "._version"):
124  if name.endswith(ending):
125  name = name[:-len(ending)]
126  if name in packages:
127  assert ver == packages[name]
128  elif name in packages:
129  assert ver == packages[name]
130 
131  # Use LSST package names instead of python module names
132  # This matches the names we get from the environment (i.e., EUPS) so we can clobber these build-time
133  # versions if the environment reveals that we're not using the packages as-built.
134  if "lsst" in name:
135  name = name.replace("lsst.", "").replace(".", "_")
136 
137  packages[name] = ver
138 
139  return packages
140 
141 
142 _eups = None # Singleton Eups object
143 
144 
146  """Get products and their versions from the environment.
147 
148  Returns
149  -------
150  packages : `dict`
151  Keys (type `str`) are product names; values (type `str`) are their
152  versions.
153 
154  Notes
155  -----
156  We use EUPS to determine the version of certain products (those that don't
157  provide a means to determine the version any other way) and to check if
158  uninstalled packages are being used. We only report the product/version
159  for these packages.
160  """
161  try:
162  from eups import Eups
163  from eups.Product import Product
164  except ImportError:
165  log.warning("Unable to import eups, so cannot determine package versions from environment")
166  return {}
167 
168  # Cache eups object since creating it can take a while
169  global _eups
170  if not _eups:
171  _eups = Eups()
172  products = _eups.findProducts(tags=["setup"])
173 
174  # Get versions for things we can't determine via runtime mechanisms
175  # XXX Should we just grab everything we can, rather than just a predetermined set?
176  packages = {prod.name: prod.version for prod in products if prod in ENVIRONMENT}
177 
178  # The string 'LOCAL:' (the value of Product.LocalVersionPrefix) in the version name indicates uninstalled
179  # code, so the version could be different than what's being reported by the runtime environment (because
180  # we don't tend to run "scons" every time we update some python file, and even if we did sconsUtils
181  # probably doesn't check to see if the repo is clean).
182  for prod in products:
183  if not prod.version.startswith(Product.LocalVersionPrefix):
184  continue
185  ver = prod.version
186 
187  gitDir = os.path.join(prod.dir, ".git")
188  if os.path.exists(gitDir):
189  # get the git revision and an indication if the working copy is clean
190  revCmd = ["git", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "rev-parse", "HEAD"]
191  diffCmd = ["git", "--no-pager", "--git-dir=" + gitDir, "--work-tree=" + prod.dir, "diff",
192  "--patch"]
193  try:
194  rev = subprocess.check_output(revCmd).decode().strip()
195  diff = subprocess.check_output(diffCmd)
196  except Exception:
197  ver += "@GIT_ERROR"
198  else:
199  ver += "@" + rev
200  if diff:
201  ver += "+" + hashlib.md5(diff).hexdigest()
202  else:
203  ver += "@NO_GIT"
204 
205  packages[prod.name] = ver
206  return packages
207 
208 
209 class Packages:
210  """A table of packages and their versions.
211 
212  There are a few different types of packages, and their versions are collected
213  in different ways:
214 
215  1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
216  interrogating the dynamic library
217  2. Python modules (e.g., afw, numpy; galsim is also in this group even though
218  we only use it through the library, because no version information is
219  currently provided through the library): we get their version from the
220  ``__version__`` module variable. Note that this means that we're only aware
221  of modules that have already been imported.
222  3. Other packages provide no run-time accessible version information (e.g.,
223  astrometry_net): we get their version from interrogating the environment.
224  Currently, that means EUPS; if EUPS is replaced or dropped then we'll need
225  to consider an alternative means of getting this version information.
226  4. Local versions of packages (a non-installed EUPS package, selected with
227  ``setup -r /path/to/package``): we identify these through the environment
228  (EUPS again) and use as a version the path supplemented with the ``git``
229  SHA and, if the git repo isn't clean, an MD5 of the diff.
230 
231  These package versions are collected and stored in a Packages object, which
232  provides useful comparison and persistence features.
233 
234  Example usage:
235 
236  .. code-block:: python
237 
238  from lsst.base import Packages
239  pkgs = Packages.fromSystem()
240  print("Current packages:", pkgs)
241  old = Packages.read("/path/to/packages.pickle")
242  print("Old packages:", old)
243  print("Missing packages compared to before:", pkgs.missing(old))
244  print("Extra packages compared to before:", pkgs.extra(old))
245  print("Different packages: ", pkgs.difference(old))
246  old.update(pkgs) # Include any new packages in the old
247  old.write("/path/to/packages.pickle")
248 
249  Parameters
250  ----------
251  packages : `dict`
252  A mapping {package: version} where both keys and values are type `str`.
253 
254  Notes
255  -----
256  This is essentially a wrapper around a dict with some conveniences.
257  """
258 
259  def __init__(self, packages):
260  assert isinstance(packages, Mapping)
261  self._packages = packages
262  self._names = set(packages.keys())
263 
264  @classmethod
265  def fromSystem(cls):
266  """Construct a `Packages` by examining the system.
267 
268  Determine packages by examining python's `sys.modules`, runtime
269  libraries and EUPS.
270 
271  Returns
272  -------
273  packages : `Packages`
274  """
275  packages = {}
276  packages.update(getPythonPackages())
277  packages.update(getRuntimeVersions())
278  packages.update(getEnvironmentPackages()) # Should be last, to override products with LOCAL versions
279  return cls(packages)
280 
281  @classmethod
282  def read(cls, filename):
283  """Read packages from filename.
284 
285  Parameters
286  ----------
287  filename : `str`
288  Filename from which to read.
289 
290  Returns
291  -------
292  packages : `Packages`
293  """
294  with open(filename, "rb") as ff:
295  return pickle.load(ff)
296 
297  def write(self, filename):
298  """Write to file.
299 
300  Parameters
301  ----------
302  filename : `str`
303  Filename to which to write.
304  """
305  with open(filename, "wb") as ff:
306  pickle.dump(self, ff)
307 
308  def __len__(self):
309  return len(self._packages)
310 
311  def __str__(self):
312  ss = "%s({\n" % self.__class__.__name__
313  # Sort alphabetically by module name, for convenience in reading
314  ss += ",\n".join("%s: %s" % (repr(prod), repr(self._packages[prod])) for
315  prod in sorted(self._names))
316  ss += ",\n})"
317  return ss
318 
319  def __repr__(self):
320  return "%s(%s)" % (self.__class__.__name__, repr(self._packages))
321 
322  def __contains__(self, pkg):
323  return pkg in self._packages
324 
325  def __iter__(self):
326  return iter(self._packages)
327 
328  def update(self, other):
329  """Update packages with contents of another set of packages.
330 
331  Parameters
332  ----------
333  other : `Packages`
334  Other packages to merge with self.
335 
336  Notes
337  -----
338  No check is made to see if we're clobbering anything.
339  """
340  self._packages.update(other._packages)
341  self._names.update(other._names)
342 
343  def extra(self, other):
344  """Get packages in self but not in another `Packages` object.
345 
346  Parameters
347  ----------
348  other : `Packages`
349  Other packages to compare against.
350 
351  Returns
352  -------
353  extra : `dict`
354  Extra packages. Keys (type `str`) are package names; values
355  (type `str`) are their versions.
356  """
357  return {pkg: self._packages[pkg] for pkg in self._names - other._names}
358 
359  def missing(self, other):
360  """Get packages in another `Packages` object but missing from self.
361 
362  Parameters
363  ----------
364  other : `Packages`
365  Other packages to compare against.
366 
367  Returns
368  -------
369  missing : `dict`
370  Missing packages. Keys (type `str`) are package names; values
371  (type `str`) are their versions.
372  """
373  return {pkg: other._packages[pkg] for pkg in other._names - self._names}
374 
375  def difference(self, other):
376  """Get packages in symmetric difference of self and another `Packages`
377  object.
378 
379  Parameters
380  ----------
381  other : `Packages`
382  Other packages to compare against.
383 
384  Returns
385  -------
386  difference : `dict`
387  Packages in symmetric difference. Keys (type `str`) are package
388  names; values (type `str`) are their versions.
389  """
390  return {pkg: (self._packages[pkg], other._packages[pkg]) for
391  pkg in self._names & other._names if self._packages[pkg] != other._packages[pkg]}
base.packages.Packages.fromSystem
def fromSystem(cls)
Definition: packages.py:265
base.packages.Packages.__str__
def __str__(self)
Definition: packages.py:311
base.packages.Packages._names
_names
Definition: packages.py:262
base.packages.getPythonPackages
def getPythonPackages()
Definition: packages.py:87
base.packages.Packages.update
def update(self, other)
Definition: packages.py:328
base.packages.Packages.missing
def missing(self, other)
Definition: packages.py:359
strip
bool strip
Definition: fits.cc:911
base.packages.Packages.__contains__
def __contains__(self, pkg)
Definition: packages.py:322
lsst::base::getRuntimeVersions
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
Definition: versions.cc:54
base.packages.Packages.__repr__
def __repr__(self)
Definition: packages.py:319
lsst::afw::geom.transform.transformContinued.cls
cls
Definition: transformContinued.py:33
base.packages.Packages.__len__
def __len__(self)
Definition: packages.py:308
base.packages.Packages
Definition: packages.py:209
base.packages.Packages.extra
def extra(self, other)
Definition: packages.py:343
base.packages.getVersionFromPythonModule
def getVersionFromPythonModule(module)
Definition: packages.py:52
list
daf::base::PropertyList * list
Definition: fits.cc:913
base.packages.Packages.__init__
def __init__(self, packages)
Definition: packages.py:259
base.packages.getEnvironmentPackages
def getEnvironmentPackages()
Definition: packages.py:145
base.packages.Packages.read
def read(cls, filename)
Definition: packages.py:282
base.packages.Packages._packages
_packages
Definition: packages.py:261
base.packages.Packages.difference
def difference(self, other)
Definition: packages.py:375
set
daf::base::PropertySet * set
Definition: fits.cc:912
astshim.fitsChanContinued.iter
def iter(self)
Definition: fitsChanContinued.py:88
base.packages.Packages.__iter__
def __iter__(self)
Definition: packages.py:325
base.packages.Packages.write
def write(self, filename)
Definition: packages.py:297