LSSTApplications  16.0-10-g0ee56ad+4,16.0-11-ga33d1f2+4,16.0-12-g3ef5c14+2,16.0-12-g71e5ef5+17,16.0-12-gbdf3636+2,16.0-13-g118c103+2,16.0-13-g8f68b0a+2,16.0-15-gbf5c1cb+3,16.0-16-gfd17674+2,16.0-17-g7c01f5c+2,16.0-18-g0a50484,16.0-20-ga20f992+7,16.0-21-g0e05fd4+5,16.0-21-g15e2d33+3,16.0-22-g62d8060+3,16.0-22-g847a80f+3,16.0-25-gf00d9b8,16.0-28-g3990c221+3,16.0-3-gf928089+2,16.0-32-g88a4f23+4,16.0-34-gd7987ad+2,16.0-37-gc7333cb+1,16.0-4-g10fc685+1,16.0-4-g18f3627+25,16.0-4-g5f3a788+25,16.0-5-gaf5c3d7+3,16.0-5-gcc1f4bb,16.0-6-g3b92700+3,16.0-6-g4412fcd+2,16.0-6-g7235603+3,16.0-69-g2562ce1b+1,16.0-7-g0913a87,16.0-8-g14ebd58+3,16.0-8-g2df868b,16.0-8-g4cec79c+5,16.0-8-gadf6c7a,16.0-82-g59ec2a54a,16.0-9-g5400cdc+1,16.0-9-ge6233d7+4,master-g2880f2d8cf+2,v17.0.rc1
LSSTDataManagementBasePackage
installation.py
Go to the documentation of this file.
1 
6 
7 import os.path
8 import glob
9 import re
10 import shutil
11 
12 import SCons.Script
13 from SCons.Script.SConscript import SConsEnvironment
14 
15 from .vcs import svn
16 from .vcs import hg
17 from .vcs import git
18 
19 from . import state
20 from .utils import memberOf
21 
22 
23 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
24 
25 
30 def makeProductPath(env, pathFormat):
31  pathFormat = re.sub(r"%(\w)", r"%(\1)s", pathFormat)
32 
33  eupsPath = os.environ['PWD']
34  if 'eupsPath' in env and env['eupsPath']:
35  eupsPath = env['eupsPath']
36 
37  return pathFormat % {"P": eupsPath,
38  "f": env['eupsFlavor'],
39  "p": env['eupsProduct'],
40  "v": env['version'],
41  "c": os.environ['PWD']}
42 
43 
44 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
45 
46 # @brief Set a version ID from env, or a version control ID string ($name$ or $HeadURL$)
47 def determineVersion(env, versionString):
48  version = "unknown"
49  if 'version' in env:
50  version = env['version']
51  elif not versionString:
52  version = "unknown"
53  elif re.search(r"^[$]Name:\s+", versionString):
54  # CVS. Extract the tagname
55  version = re.search(r"^[$]Name:\s+([^ $]*)", versionString).group(1)
56  if version == "":
57  version = "cvs"
58  elif re.search(r"^[$]HeadURL:\s+", versionString):
59  # SVN. Guess the tagname from the last part of the directory
60  HeadURL = re.search(r"^[$]HeadURL:\s+(.*)", versionString).group(1)
61  HeadURL = os.path.split(HeadURL)[0]
62  version = svn.guessVersionName(HeadURL)
63  elif versionString.lower() in ("hg", "mercurial"):
64  # Mercurial (hg).
65  version = hg.guessVersionName()
66  elif versionString.lower() in ("git",):
67  # git.
68  version = git.guessVersionName()
69  return version.replace("/", "_")
70 
71 
72 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
73 
74 # @brief Return a unique fingerprint for a version (e.g. an SHA1); return None if unavailable
75 def getFingerprint(versionString):
76  if versionString.lower() in ("hg", "mercurial"):
77  fingerprint, modified = hg.guessFingerprint()
78  elif versionString.lower() in ("git",):
79  fingerprint, modified = git.guessFingerprint()
80  else:
81  fingerprint, modified = None, False
82 
83  if fingerprint and modified:
84  fingerprint += " *"
85 
86  return fingerprint
87 
88 
89 # @brief Set a prefix based on the EUPS_PATH, the product name, and a versionString from cvs or svn.
90 def setPrefix(env, versionString, eupsProductPath=None):
91  try:
92  env['version'] = determineVersion(env, versionString)
93  except RuntimeError as err:
94  env['version'] = "unknown"
95  if (env.installing or env.declaring) and not env['force']:
96  state.log.fail(
97  "%s\nFound problem with version number; update or specify force=True to proceed"
98  % err
99  )
100 
101  if state.env['no_eups']:
102  if 'prefix' in env and env['prefix']:
103  return env['prefix']
104  else:
105  return "/usr/local"
106 
107  if eupsProductPath:
108  eupsPrefix = makeProductPath(env, eupsProductPath)
109  elif 'eupsPath' in env and env['eupsPath']:
110  eupsPrefix = env['eupsPath']
111  else:
112  state.log.fail("Unable to determine eupsPrefix from eupsProductPath or eupsPath")
113  flavor = env['eupsFlavor']
114  if not re.search("/" + flavor + "$", eupsPrefix):
115  eupsPrefix = os.path.join(eupsPrefix, flavor)
116  prodPath = env['eupsProduct']
117  if 'eupsProductPath' in env and env['eupsProductPath']:
118  prodPath = env['eupsProductPath']
119  eupsPrefix = os.path.join(eupsPrefix, prodPath, env["version"])
120  else:
121  eupsPrefix = None
122  if 'prefix' in env:
123  if env['version'] != "unknown" and eupsPrefix and eupsPrefix != env['prefix']:
124  state.log.warn("Ignoring prefix %s from EUPS_PATH" % eupsPrefix)
125  return makeProductPath(env, env['prefix'])
126  elif 'eupsPath' in env and env['eupsPath']:
127  prefix = eupsPrefix
128  else:
129  prefix = "/usr/local"
130  return prefix
131 
132 
133 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
134 
135 
141 @memberOf(SConsEnvironment)
142 def Declare(self, products=None):
143 
144  if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS and not self.GetOption("silent"):
145  state.log.warn("'scons undeclare' is deprecated; please use 'scons declare -c' instead")
146 
147  acts = []
148  if ("declare" in SCons.Script.COMMAND_LINE_TARGETS or
149  "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or
150  ("install" in SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean")) or
151  "current" in SCons.Script.COMMAND_LINE_TARGETS):
152  current = []
153  declare = []
154  undeclare = []
155 
156  if not products:
157  products = [None]
158 
159  for prod in products:
160  if not prod or isinstance(prod, str): # i.e. no version
161  product = prod
162 
163  if 'version' in self:
164  version = self['version']
165  else:
166  version = None
167  else:
168  product, version = prod
169 
170  if not product:
171  product = self['eupsProduct']
172 
173  if "EUPS_DIR" in os.environ:
174  self['ENV']['PATH'] += os.pathsep + "%s/bin" % (os.environ["EUPS_DIR"])
175  self["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
176  if "undeclare" in SCons.Script.COMMAND_LINE_TARGETS or self.GetOption("clean"):
177  if version:
178  command = "eups undeclare --flavor %s %s %s" % \
179  (self['eupsFlavor'], product, version)
180  if ("current" in SCons.Script.COMMAND_LINE_TARGETS and
181  "declare" not in SCons.Script.COMMAND_LINE_TARGETS):
182  command += " --current"
183 
184  if self.GetOption("clean"):
185  self.Execute(command)
186  else:
187  undeclare += [command]
188  else:
189  state.log.warn("I don't know your version; not undeclaring to eups")
190  else:
191  command = "eups declare --force --flavor %s --root %s" % \
192  (self['eupsFlavor'], self['prefix'])
193 
194  if 'eupsPath' in self:
195  command += " -Z %s" % self['eupsPath']
196 
197  if version:
198  command += " %s %s" % (product, version)
199 
200  current += [command + " --current"]
201 
202  if self.GetOption("tag"):
203  command += " --tag=%s" % self.GetOption("tag")
204 
205  declare += [command]
206 
207  if current:
208  acts += self.Command("current", "", action=current)
209  if declare:
210  if "current" in SCons.Script.COMMAND_LINE_TARGETS:
211  acts += self.Command("declare", "", action="") # current will declare it for us
212  else:
213  acts += self.Command("declare", "", action=declare)
214  if undeclare:
215  acts += self.Command("undeclare", "", action=undeclare)
216 
217  return acts
218 
219 
220 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
221 
222 
232 
233  def __init__(self, ignoreRegex, recursive):
234  self.ignoreRegex = re.compile(ignoreRegex)
235  self.recursive = recursive
236 
237  def __call__(self, target, source, env):
238  prefix = os.path.abspath(os.path.join(target[0].abspath, ".."))
239  destpath = os.path.join(target[0].abspath)
240  if not os.path.isdir(destpath):
241  state.log.info("Creating directory %s" % destpath)
242  os.makedirs(destpath)
243  for root, dirnames, filenames in os.walk(source[0].path):
244  if not self.recursive:
245  dirnames[:] = []
246  else:
247  dirnames[:] = [d for d in dirnames if d != ".svn"] # ignore .svn tree
248  for dirname in dirnames:
249  destpath = os.path.join(prefix, root, dirname)
250  if not os.path.isdir(destpath):
251  state.log.info("Creating directory %s" % destpath)
252  os.makedirs(destpath)
253  for filename in filenames:
254  if self.ignoreRegex.search(filename):
255  continue
256  destpath = os.path.join(prefix, root)
257  srcpath = os.path.join(root, filename)
258  state.log.info("Copying %s to %s" % (srcpath, destpath))
259  shutil.copy(srcpath, destpath)
260  return 0
261 
262 
263 
267 @memberOf(SConsEnvironment)
268 def InstallDir(self, prefix, dir, ignoreRegex=r"(~$|\.pyc$|\.os?$)", recursive=True):
269  if not self.installing:
270  return []
271  result = self.Command(target=os.path.join(self.Dir(prefix).abspath, dir), source=dir,
272  action=DirectoryInstaller(ignoreRegex, recursive))
273  self.AlwaysBuild(result)
274  return result
275 
276 
277 # =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
278 
279 
290 @memberOf(SConsEnvironment)
291 def InstallEups(env, dest, files=[], presetup=""):
292 
293  acts = []
294  if not env.installing:
295  return acts
296 
297  if env.GetOption("clean"):
298  state.log.warn("Removing" + dest)
299  shutil.rmtree(dest, ignore_errors=True)
300  else:
301  presetupStr = []
302  for p in presetup:
303  presetupStr += ["--product %s=%s" % (p, presetup[p])]
304  presetup = " ".join(presetupStr)
305 
306  env = env.Clone(ENV=os.environ)
307  #
308  # Add any build/table/cfg files to the desired files
309  #
310  files = [str(f) for f in files] # in case the user used Glob not glob.glob
311  files += glob.glob(os.path.join("ups", "*.build")) + glob.glob(os.path.join("ups", "*.table")) \
312  + glob.glob(os.path.join("ups", "*.cfg")) \
313  + glob.glob(os.path.join("ups", "eupspkg*"))
314  files = list(set(files)) # remove duplicates
315 
316  buildFiles = [f for f in files if re.search(r"\.build$", f)]
317  build_obj = env.Install(dest, buildFiles)
318  acts += build_obj
319 
320  tableFiles = [f for f in files if re.search(r"\.table$", f)]
321  table_obj = env.Install(dest, tableFiles)
322  acts += table_obj
323 
324  eupspkgFiles = [f for f in files if re.search(r"^eupspkg", f)]
325  eupspkg_obj = env.Install(dest, eupspkgFiles)
326  acts += eupspkg_obj
327 
328  miscFiles = [f for f in files if not re.search(r"\.(build|table)$", f)]
329  misc_obj = env.Install(dest, miscFiles)
330  acts += misc_obj
331 
332  try:
333  import eups.lock
334 
335  path = eups.Eups.setEupsPath()
336  if path:
337  locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH) # noqa F841 keep locks active
338  env["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
339  except ImportError:
340  state.log.warn("Unable to import eups; not locking")
341 
342  eupsTargets = []
343 
344  for i in build_obj:
345  env.AlwaysBuild(i)
346 
347  cmd = "eups expandbuild -i --version %s " % env['version']
348  if 'baseversion' in env:
349  cmd += " --repoversion %s " % env['baseversion']
350  cmd += str(i)
351  eupsTargets.extend(env.AddPostAction(build_obj, env.Action("%s" % (cmd), cmd)))
352 
353  for i in table_obj:
354  env.AlwaysBuild(i)
355 
356  cmd = "eups expandtable -i -W '^(?!LOCAL:)' " # version doesn't start "LOCAL:"
357  if presetup:
358  cmd += presetup + " "
359  cmd += str(i)
360 
361  act = env.Command("table", "", env.Action("%s" % (cmd), cmd))
362  eupsTargets.extend(act)
363  acts += act
364  env.Depends(act, i)
365 
366  # By declaring that all the Eups operations create a file called "eups" as a side-effect,
367  # even though they don't, SCons knows it can't run them in parallel (it thinks of the
368  # side-effect file as something like a log, and knows you shouldn't be appending to it
369  # in parallel). When Eups locking is working, we may be able to remove this.
370  env.SideEffect("eups", eupsTargets)
371 
372  return acts
373 
374 
375 # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
376 
377 # @brief Install directories in the usual LSST way, handling "ups" specially.
378 @memberOf(SConsEnvironment)
379 def InstallLSST(self, prefix, dirs, ignoreRegex=None):
380  results = []
381  for d in dirs:
382  # if eups is disabled, the .build & .table files will not be "expanded"
383  if d == "ups" and not state.env['no_eups']:
384  t = self.InstallEups(os.path.join(prefix, "ups"))
385  else:
386  t = self.InstallDir(prefix, d, ignoreRegex=ignoreRegex)
387  self.Depends(t, d)
388  results.extend(t)
389  self.Alias("install", t)
390  self.Clean("install", prefix)
391  return results
def makeProductPath(env, pathFormat)
return a path to use as the installation directory for a product
Definition: installation.py:30
daf::base::PropertySet * set
Definition: fits.cc:832
def Declare(self, products=None)
Create current and declare targets for products.
def setPrefix(env, versionString, eupsProductPath=None)
Definition: installation.py:90
def InstallLSST(self, prefix, dirs, ignoreRegex=None)
def __call__(self, target, source, env)
def determineVersion(env, versionString)
Definition: installation.py:47
def InstallEups(env, dest, files=[], presetup="")
Install a ups directory, setting absolute versions as appropriate (unless you're installing from the ...
def InstallDir(self, prefix, dir, ignoreRegex=r"(~$|\yc$|\s?$)", recursive=True)
Install the directory dir into prefix, (along with all its descendents if recursive is True)...
SCons Action callable to recursively install a directory.
table::Key< table::Array< int > > group
Definition: PsfexPsf.cc:359
daf::base::PropertyList * list
Definition: fits.cc:833
def memberOf(cls, name=None)
A Python decorator that injects functions into a class.
Definition: utils.py:185
def __init__(self, ignoreRegex, recursive)
def getFingerprint(versionString)
Definition: installation.py:75