LSSTApplications  10.0-2-g4f67435,11.0.rc2+1,11.0.rc2+12,11.0.rc2+3,11.0.rc2+4,11.0.rc2+5,11.0.rc2+6,11.0.rc2+7,11.0.rc2+8
LSSTDataManagementBasePackage
installation.py
Go to the documentation of this file.
1 ##
2 # @file installation.py
3 #
4 # Builders and path setup for installation targets.
5 ##
6 
7 from __future__ import absolute_import, division, print_function
8 import os.path
9 import glob
10 import re
11 import sys
12 import shutil
13 
14 import SCons.Script
15 from SCons.Script.SConscript import SConsEnvironment
16 
17 from .vcs import svn
18 from .vcs import hg
19 from .vcs import git
20 
21 from . import state
22 from .utils import memberOf
23 
24 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
25 
26 ##
27 # @brief return a path to use as the installation directory for a product
28 # @param pathFormat the format string to process
29 # @param env the scons environment
30 ##
31 def makeProductPath(env, pathFormat):
32  pathFormat = re.sub(r"%(\w)", r"%(\1)s", pathFormat)
33 
34  eupsPath = os.environ['PWD']
35  if 'eupsPath' in env and env['eupsPath']:
36  eupsPath = env['eupsPath']
37 
38  return pathFormat % { "P": eupsPath,
39  "f": env['eupsFlavor'],
40  "p": env['eupsProduct'],
41  "v": env['version'],
42  "c": os.environ['PWD'] }
43 
44 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
45 
46 ## @brief Set a version ID from env, or a version control ID string ($name$ or $HeadURL$)
47 def determineVersion(env, versionString):
48  version = "unknown"
49  if 'version' in env:
50  version = env['version']
51  elif not versionString:
52  version = "unknown"
53  elif re.search(r"^[$]Name:\s+", versionString):
54  # CVS. Extract the tagname
55  version = re.search(r"^[$]Name:\s+([^ $]*)", versionString).group(1)
56  if version == "":
57  version = "cvs"
58  elif re.search(r"^[$]HeadURL:\s+", versionString):
59  # SVN. Guess the tagname from the last part of the directory
60  HeadURL = re.search(r"^[$]HeadURL:\s+(.*)", versionString).group(1)
61  HeadURL = os.path.split(HeadURL)[0]
62  version = svn.guessVersionName(HeadURL)
63  elif versionString.lower() in ("hg", "mercurial"):
64  # Mercurial (hg).
65  version = hg.guessVersionName()
66  elif versionString.lower() in ("git",):
67  # git.
68  version = git.guessVersionName()
69  return version.replace("/", "_")
70 
71 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
72 
73 ## @brief Return a unique fingerprint for a version (e.g. an SHA1); return None if unavailable
74 def getFingerprint(versionString):
75  if versionString.lower() in ("hg", "mercurial"):
76  fingerprint, modified = hg.guessFingerprint()
77  elif versionString.lower() in ("git",):
78  fingerprint, modified = git.guessFingerprint()
79  else:
80  fingerprint, modified = None, False
81 
82  if fingerprint and modified:
83  fingerprint += " *"
84 
85  return fingerprint
86 
87 ## @brief Set a prefix based on the EUPS_PATH, the product name, and a versionString from cvs or svn.
88 def setPrefix(env, versionString, eupsProductPath=None):
89  try:
90  env['version'] = determineVersion(env, versionString)
91  except RuntimeError as err:
92  env['version'] = "unknown"
93  if (env.installing or env.declaring) and not env['force']:
94  state.log.fail(
95  "%s\nFound problem with version number; update or specify force=True to proceed"
96  % err
97  )
98 
99  if state.env['no_eups']:
100  if 'prefix' in env and env['prefix']:
101  return env['prefix']
102  else:
103  return "/usr/local"
104 
105  if eupsProductPath:
106  eupsPrefix = makeProductPath(env, eupsProductPath)
107  elif 'eupsPath' in env and env['eupsPath']:
108  eupsPrefix = env['eupsPath']
109  else:
110  state.log.fail("Unable to determine eupsPrefix from eupsProductPath or eupsPath")
111  flavor = env['eupsFlavor']
112  if not re.search("/" + flavor + "$", eupsPrefix):
113  eupsPrefix = os.path.join(eupsPrefix, flavor)
114  prodPath = env['eupsProduct']
115  if 'eupsProductPath' in env and env['eupsProductPath']:
116  prodPath = env['eupsProductPath']
117  eupsPrefix = os.path.join(eupsPrefix, prodPath, env["version"])
118  else:
119  eupsPrefix = None
120  if 'prefix' in env:
121  if env['version'] != "unknown" and eupsPrefix and eupsPrefix != env['prefix']:
122  state.log.warn("Ignoring prefix %s from EUPS_PATH" % eupsPrefix)
123  return makeProductPath(env, env['prefix'])
124  elif 'eupsPath' in env and env['eupsPath']:
125  prefix = eupsPrefix
126  else:
127  prefix = "/usr/local"
128  return prefix
129 
130 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
131 
132 ##
133 # Create current and declare targets for products. products
134 # may be a list of (product, version) tuples. If product is None
135 # it's taken to be self['eupsProduct']; if version is None it's
136 # taken to be self['version'].
137 ##
138 @memberOf(SConsEnvironment)
139 def Declare(self, products=None):
140 
141  if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS and not self.GetOption("silent"):
142  state.log.warn("'scons undeclare' is deprecated; please use 'scons declare -c' instead")
143 
144  acts = []
145  if \
146  "declare" in SCons.Script.COMMAND_LINE_TARGETS or \
147  "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or \
148  ("install" in SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean")) or \
149  "current" in SCons.Script.COMMAND_LINE_TARGETS:
150  current = []; declare = []; undeclare = []
151 
152  if not products:
153  products = [None]
154 
155  for prod in products:
156  if not prod or isinstance(prod, str): # i.e. no version
157  product = prod
158 
159  if 'version' in self:
160  version = self['version']
161  else:
162  version = None
163  else:
164  product, version = prod
165 
166  if not product:
167  product = self['eupsProduct']
168 
169  if "EUPS_DIR" in os.environ:
170  self['ENV']['PATH'] += os.pathsep + "%s/bin" % (os.environ["EUPS_DIR"])
171  self["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
172  if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or self.GetOption("clean"):
173  if version:
174  command = "eups undeclare --flavor %s %s %s" % \
175  (self['eupsFlavor'], product, version)
176  if ("current" in SCons.Script.COMMAND_LINE_TARGETS
177  and not "declare" in SCons.Script.COMMAND_LINE_TARGETS):
178  command += " --current"
179 
180  if self.GetOption("clean"):
181  self.Execute(command)
182  else:
183  undeclare += [command]
184  else:
185  state.log.warn("I don't know your version; not undeclaring to eups")
186  else:
187  command = "eups declare --force --flavor %s --root %s" % \
188  (self['eupsFlavor'], self['prefix'])
189 
190  if 'eupsPath' in self:
191  command += " -Z %s" % self['eupsPath']
192 
193  if version:
194  command += " %s %s" % (product, version)
195 
196  current += [command + " --current"]
197 
198  if self.GetOption("tag"):
199  command += " --tag=%s" % self.GetOption("tag")
200 
201  declare += [command]
202 
203  if current:
204  acts += self.Command("current", "", action=current)
205  if declare:
206  if "current" in SCons.Script.COMMAND_LINE_TARGETS:
207  acts += self.Command("declare", "", action="") # current will declare it for us
208  else:
209  acts += self.Command("declare", "", action=declare)
210  if undeclare:
211  acts += self.Command("undeclare", "", action=undeclare)
212 
213  return acts
214 
215 #=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
216 
217 ##
218 # @brief SCons Action callable to recursively install a directory.
219 #
220 # This is separate from the InstallDir function to allow the directory-walking
221 # to happen when installation is actually invoked, rather than when the SConscripts
222 # are parsed. This still does not ensure that all necessary files are built as
223 # prerequisites to installing, but if one explicitly marks the install targets
224 # as dependent on the build targets, that should be enough.
225 ##
226 class DirectoryInstaller(object):
227 
228  def __init__(self, ignoreRegex, recursive):
229  self.ignoreRegex = re.compile(ignoreRegex)
230  self.recursive = recursive
231 
232  def __call__(self, target, source, env):
233  prefix = os.path.abspath(os.path.join(target[0].abspath, ".."))
234  destpath = os.path.join(target[0].abspath)
235  if not os.path.isdir(destpath):
236  state.log.info("Creating directory %s" % destpath)
237  os.makedirs(destpath)
238  for root, dirnames, filenames in os.walk(source[0].path):
239  if not self.recursive:
240  dirnames[:] = []
241  else:
242  dirnames[:] = [d for d in dirnames if d != ".svn"] # ignore .svn tree
243  for dirname in dirnames:
244  destpath = os.path.join(prefix, root, dirname)
245  if not os.path.isdir(destpath):
246  state.log.info("Creating directory %s" % destpath)
247  os.makedirs(destpath)
248  for filename in filenames:
249  if self.ignoreRegex.search(filename):
250  continue
251  destpath = os.path.join(prefix, root)
252  srcpath = os.path.join(root, filename)
253  state.log.info("Copying %s to %s" % (srcpath, destpath))
254  shutil.copy(srcpath, destpath)
255  return 0
256 
257 
258 ##
259 # Install the directory dir into prefix, (along with all its descendents if recursive is True).
260 # Omit files and directories that match ignoreRegex
261 ##
262 @memberOf(SConsEnvironment)
263 def InstallDir(self, prefix, dir, ignoreRegex=r"(~$|\.pyc$|\.os?$)", recursive=True):
264  if not self.installing:
265  return []
266  result = self.Command(target=os.path.join(self.Dir(prefix).abspath, dir), source=dir,
267  action=DirectoryInstaller(ignoreRegex, recursive))
268  self.AlwaysBuild(result)
269  return result
270 
271 #=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
272 
273 ##
274 # Install a ups directory, setting absolute versions as appropriate
275 # (unless you're installing from the trunk, in which case no versions
276 # are expanded). Any build/table files present in "./ups" are automatically
277 # added to files.
278 #
279 # If presetup is provided, it's expected to be a dictionary with keys
280 # product names and values the version that should be installed into
281 # the table files, overriding eups expandtable's usual behaviour. E.g.
282 # env.InstallEups(os.path.join(env['prefix'], "ups"), presetup={"sconsUtils" : env['version']})
283 ##
284 @memberOf(SConsEnvironment)
285 def InstallEups(env, dest, files=[], presetup=""):
286 
287  acts = []
288  if not env.installing:
289  return acts
290 
291  if env.GetOption("clean"):
292  state.log.warn("Removing" + dest)
293  shutil.rmtree(dest, ignore_errors=True)
294  else:
295  presetupStr = []
296  for p in presetup:
297  presetupStr += ["--product %s=%s" % (p, presetup[p])]
298  presetup = " ".join(presetupStr)
299 
300  env = env.Clone(ENV = os.environ)
301  #
302  # Add any build/table/cfg files to the desired files
303  #
304  files = [str(f) for f in files] # in case the user used Glob not glob.glob
305  files += glob.glob(os.path.join("ups", "*.build")) + glob.glob(os.path.join("ups","*.table")) \
306  + glob.glob(os.path.join("ups", "*.cfg")) \
307  + glob.glob(os.path.join("ups", "eupspkg*"))
308  files = list(set(files)) # remove duplicates
309 
310  buildFiles = [f for f in files if re.search(r"\.build$", f)]
311  build_obj = env.Install(dest, buildFiles)
312  acts += build_obj
313 
314  tableFiles = [f for f in files if re.search(r"\.table$", f)]
315  table_obj = env.Install(dest, tableFiles)
316  acts += table_obj
317 
318  eupspkgFiles = [f for f in files if re.search(r"^eupspkg", f)]
319  eupspkg_obj = env.Install(dest, eupspkgFiles)
320  acts += eupspkg_obj
321 
322  miscFiles = [f for f in files if not re.search(r"\.(build|table)$", f)]
323  misc_obj = env.Install(dest, miscFiles)
324  acts += misc_obj
325 
326  try:
327  import eups.lock
328 
329  path = eups.Eups.setEupsPath()
330  if path:
331  locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH)
332  env["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
333  except ImportError:
334  state.log.warn("Unable to import eups; not locking")
335 
336  eupsTargets = []
337 
338  for i in build_obj:
339  env.AlwaysBuild(i)
340 
341  cmd = "eups expandbuild -i --version %s " % env['version']
342  if 'baseversion' in env:
343  cmd += " --repoversion %s " % env['baseversion']
344  cmd += str(i)
345  eupsTargets.extend(env.AddPostAction(build_obj, env.Action("%s" %(cmd), cmd)))
346 
347  for i in table_obj:
348  env.AlwaysBuild(i)
349 
350  cmd = "eups expandtable -i -W '^(?!LOCAL:)' " # version doesn't start "LOCAL:"
351  if presetup:
352  cmd += presetup + " "
353  cmd += str(i)
354 
355  act = env.Command("table", "", env.Action("%s" %(cmd), cmd))
356  eupsTargets.extend(act)
357  acts += act
358  env.Depends(act, i)
359 
360  # By declaring that all the Eups operations create a file called "eups" as a side-effect,
361  # even though they don't, SCons knows it can't run them in parallel (it thinks of the
362  # side-effect file as something like a log, and knows you shouldn't be appending to it
363  # in parallel). When Eups locking is working, we may be able to remove this.
364  env.SideEffect("eups", eupsTargets)
365 
366  return acts
367 
368 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
369 
370 ## @brief Install directories in the usual LSST way, handling "ups" specially.
371 @memberOf(SConsEnvironment)
372 def InstallLSST(self, prefix, dirs, ignoreRegex=None):
373  results = []
374  for d in dirs:
375  # if eups is disabled, the .build & .table files will not be "expanded"
376  if d == "ups" and not state.env['no_eups']:
377  t = self.InstallEups(os.path.join(prefix, "ups"))
378  else:
379  t = self.InstallDir(prefix, d, ignoreRegex=ignoreRegex)
380  self.Depends(t, d)
381  results.extend(t)
382  self.Alias("install", t)
383  self.Clean("install", prefix)
384  return results
def Declare
Create current and declare targets for products.
def getFingerprint
Return a unique fingerprint for a version (e.g.
Definition: installation.py:74
def memberOf
A Python decorator that injects functions into a class.
Definition: utils.py:85
def makeProductPath
return a path to use as the installation directory for a product
Definition: installation.py:31
def InstallLSST
Install directories in the usual LSST way, handling "ups" specially.
def determineVersion
Set a version ID from env, or a version control ID string ($name$ or $HeadURL$)
Definition: installation.py:47
def InstallEups
Install a ups directory, setting absolute versions as appropriate (unless you're installing from the ...
SCons Action callable to recursively install a directory.
def setPrefix
Set a prefix based on the EUPS_PATH, the product name, and a versionString from cvs or svn...
Definition: installation.py:88
def InstallDir
Install the directory dir into prefix, (along with all its descendents if recursive is True)...