22 Determine which packages are being used in the system and their versions
30 import pickle
as pickle
34 from functools
import lru_cache
36 from .versions
import getRuntimeVersions
38 log = logging.getLogger(__name__)
40 __all__ = [
"getVersionFromPythonModule",
"getPythonPackages",
"getEnvironmentPackages",
41 "getCondaPackages",
"Packages"]
45 BUILDTIME =
set([
"boost",
"eigen",
"tmv"])
50 PYTHON =
set([
"galsim"])
54 ENVIRONMENT =
set([
"astrometry_net",
"astrometry_net_data",
"minuit2",
"xpa"])
58 """Determine the version of a python module.
63 Module for which to get version.
72 Raised if __version__ attribute is not set.
76 We supplement the version with information from the
77 ``__dependency_versions__`` (a specific variable set by LSST's
78 `~lsst.sconsUtils` at build time) only for packages that are typically
79 used only at build-time.
81 version = module.__version__
82 if hasattr(module,
"__dependency_versions__"):
84 deps = module.__dependency_versions__
85 buildtime = BUILDTIME &
set(deps.keys())
87 version +=
" with " +
" ".join(
"%s=%s" % (pkg, deps[pkg])
88 for pkg
in sorted(buildtime))
93 """Get imported python packages and their versions.
98 Keys (type `str`) are package names; values (type `str`) are their
103 We wade through `sys.modules` and attempt to determine the version for each
104 module. Note, therefore, that we can only report on modules that have
105 *already* been imported.
107 We don't include any module for which we cannot determine a version.
110 for module
in PYTHON:
112 importlib.import_module(module)
116 packages = {
"python": sys.version}
119 moduleNames =
list(sys.modules.keys())
120 for name
in moduleNames:
121 module = sys.modules[name]
130 for ending
in (
".version",
"._version"):
131 if name.endswith(ending):
132 name = name[:-len(ending)]
134 assert ver == packages[name]
135 elif name
in packages:
136 assert ver == packages[name]
143 name = name.replace(
"lsst.",
"").replace(
".",
"_")
153 @lru_cache(maxsize=1)
155 """Get products and their versions from the environment.
160 Keys (type `str`) are product names; values (type `str`) are their
165 We use EUPS to determine the version of certain products (those that don't
166 provide a means to determine the version any other way) and to check if
167 uninstalled packages are being used. We only report the product/version
171 from eups
import Eups
172 from eups.Product
import Product
174 log.warning(
"Unable to import eups, so cannot determine package versions from environment")
181 products = _eups.findProducts(tags=[
"setup"])
186 packages = {prod.name: prod.version
for prod
in products
if prod
in ENVIRONMENT}
194 for prod
in products:
195 if not prod.version.startswith(Product.LocalVersionPrefix):
199 gitDir = os.path.join(prod.dir,
".git")
200 if os.path.exists(gitDir):
203 revCmd = [
"git",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"rev-parse",
"HEAD"]
204 diffCmd = [
"git",
"--no-pager",
"--git-dir=" + gitDir,
"--work-tree=" + prod.dir,
"diff",
207 rev = subprocess.check_output(revCmd).decode().
strip()
208 diff = subprocess.check_output(diffCmd)
214 ver +=
"+" + hashlib.md5(diff).hexdigest()
218 packages[prod.name] = ver
222 @lru_cache(maxsize=1)
224 """Get products and their versions from the conda environment.
229 Keys (type `str`) are product names; values (type `str`) are their
234 Returns empty result if a conda environment is not in use or can not
240 from conda.cli.python_api
import Commands, run_command
245 versions_json = run_command(Commands.LIST,
"--json")
246 packages = {pkg[
"name"]: pkg[
"version"]
for pkg
in json.loads(versions_json[0])}
255 match = re.search(
r"/envs/(.*?)/bin/", sys.executable)
257 packages[
"conda_env"] = match.group(1)
263 """A table of packages and their versions.
265 There are a few different types of packages, and their versions are
266 collected in different ways:
268 1. Run-time libraries (e.g., cfitsio, fftw): we get their version from
269 interrogating the dynamic library
270 2. Python modules (e.g., afw, numpy; galsim is also in this group even
271 though we only use it through the library, because no version
272 information is currently provided through the library): we get their
273 version from the ``__version__`` module variable. Note that this means
274 that we're only aware of modules that have already been imported.
275 3. Other packages provide no run-time accessible version information (e.g.,
276 astrometry_net): we get their version from interrogating the
277 environment. Currently, that means EUPS; if EUPS is replaced or dropped
278 then we'll need to consider an alternative means of getting this version
280 4. Local versions of packages (a non-installed EUPS package, selected with
281 ``setup -r /path/to/package``): we identify these through the
282 environment (EUPS again) and use as a version the path supplemented with
283 the ``git`` SHA and, if the git repo isn't clean, an MD5 of the diff.
285 These package versions are collected and stored in a Packages object, which
286 provides useful comparison and persistence features.
290 .. code-block:: python
292 from lsst.base import Packages
293 pkgs = Packages.fromSystem()
294 print("Current packages:", pkgs)
295 old = Packages.read("/path/to/packages.pickle")
296 print("Old packages:", old)
297 print("Missing packages compared to before:", pkgs.missing(old))
298 print("Extra packages compared to before:", pkgs.extra(old))
299 print("Different packages: ", pkgs.difference(old))
300 old.update(pkgs) # Include any new packages in the old
301 old.write("/path/to/packages.pickle")
306 A mapping {package: version} where both keys and values are type `str`.
310 This is essentially a wrapper around a dict with some conveniences.
313 formats = {
".pkl":
"pickle",
318 assert isinstance(packages, Mapping)
324 """Construct a `Packages` by examining the system.
326 Determine packages by examining python's `sys.modules`, runtime
331 packages : `Packages`
342 """Construct the object from a byte representation.
347 The serialized form of this object in bytes.
349 The format of those bytes. Can be ``yaml`` or ``pickle``.
351 if format ==
"pickle":
352 new = pickle.loads(data)
353 elif format ==
"yaml":
354 new = yaml.load(data, Loader=yaml.SafeLoader)
356 raise ValueError(f
"Unexpected serialization format given: {format}")
357 if not isinstance(new, cls):
358 raise TypeError(f
"Extracted object of class '{type(new)}' but expected '{cls}'")
363 """Read packages from filename.
368 Filename from which to read. The format is determined from the
369 file extension. Currently support ``.pickle``, ``.pkl``
374 packages : `Packages`
376 _, ext = os.path.splitext(filename)
377 if ext
not in cls.
formatsformats:
378 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
379 with open(filename,
"rb")
as ff:
386 """Convert the object to a serialized bytes form using the
392 Format to use when serializing. Can be ``yaml`` or ``pickle``.
397 Byte string representing the serialized object.
399 if format ==
"pickle":
400 return pickle.dumps(self)
401 elif format ==
"yaml":
402 return yaml.dump(self).
encode(
"utf-8")
404 raise ValueError(f
"Unexpected serialization format requested: {format}")
412 Filename to which to write. The format of the data file
413 is determined from the file extension. Currently supports
414 ``.pickle`` and ``.yaml``
416 _, ext = os.path.splitext(filename)
417 if ext
not in self.
formatsformats:
418 raise ValueError(f
"Format from {ext} extension in file {filename} not recognized")
419 with open(filename,
"wb")
as ff:
428 ss =
"%s({\n" % self.__class__.__name__
430 ss +=
",\n".join(
"%s: %s" % (repr(prod), repr(self.
_packages_packages[prod]))
for
431 prod
in sorted(self.
_names_names))
436 return "%s(%s)" % (self.__class__.__name__, repr(self.
_packages_packages))
445 if not isinstance(other,
type(self)):
448 return self.
_packages_packages == other._packages
451 """Update packages with contents of another set of packages.
456 Other packages to merge with self.
460 No check is made to see if we're clobbering anything.
466 """Get packages in self but not in another `Packages` object.
471 Other packages to compare against.
476 Extra packages. Keys (type `str`) are package names; values
477 (type `str`) are their versions.
479 return {pkg: self.
_packages_packages[pkg]
for pkg
in self.
_names_names - other._names}
482 """Get packages in another `Packages` object but missing from self.
487 Other packages to compare against.
492 Missing packages. Keys (type `str`) are package names; values
493 (type `str`) are their versions.
495 return {pkg: other._packages[pkg]
for pkg
in other._names - self.
_names_names}
498 """Get packages in symmetric difference of self and another `Packages`
504 Other packages to compare against.
509 Packages in symmetric difference. Keys (type `str`) are package
510 names; values (type `str`) are their versions.
512 return {pkg: (self.
_packages_packages[pkg], other._packages[pkg])
for
513 pkg
in self.
_names_names & other._names
if self.
_packages_packages[pkg] != other._packages[pkg]}
519 """Represent Packages as a simple dict"""
520 return dumper.represent_mapping(
"lsst.base.Packages", data._packages,
524 yaml.add_representer(Packages, pkg_representer)
528 yield Packages(loader.construct_mapping(node, deep=
True))
531 for loader
in (yaml.Loader, yaml.CLoader, yaml.UnsafeLoader, yaml.SafeLoader, yaml.FullLoader):
532 yaml.add_constructor(
"lsst.base.Packages", pkg_constructor, Loader=loader)
def fromBytes(cls, data, format)
def __contains__(self, pkg)
def write(self, filename)
def difference(self, other)
def __init__(self, packages)
def toBytes(self, format)
daf::base::PropertyList * list
daf::base::PropertySet * set
def getEnvironmentPackages()
def pkg_representer(dumper, data)
def getVersionFromPythonModule(module)
std::map< std::string, std::string > getRuntimeVersions()
Return version strings for dependencies.
pybind11::bytes encode(Region const &self)
Encode a Region as a pybind11 bytes object.