LSSTApplications  10.0-2-g4f67435,11.0.rc2+1,11.0.rc2+12,11.0.rc2+3,11.0.rc2+4,11.0.rc2+5,11.0.rc2+6,11.0.rc2+7,11.0.rc2+8
LSSTDataManagementBasePackage
builders.py
Go to the documentation of this file.
1 #
2 # @file builders.py
3 #
4 # Extra builders and methods to be injected into the SConsEnvironment class.
5 ##
6 
7 from __future__ import absolute_import, division, print_function
8 import os
9 import re
10 import fnmatch
11 
12 import SCons.Script
13 from SCons.Script.SConscript import SConsEnvironment
14 
15 from .utils import memberOf
16 from .installation import determineVersion, getFingerprint
17 from . import state
18 
19 ## @brief Like SharedLibrary, but don't insist that all symbols are resolved
20 @memberOf(SConsEnvironment)
21 def SharedLibraryIncomplete(self, target, source, **keywords):
22  myenv = self.Clone()
23  if myenv['PLATFORM'] == 'darwin':
24  myenv['SHLINKFLAGS'] += ["-undefined", "suppress", "-flat_namespace", "-headerpad_max_install_names"]
25  return myenv.SharedLibrary(target, source, **keywords)
26 
27 ## @brief Like LoadableModule, but don't insist that all symbols are resolved, and set
28 ## some SWIG-specific flags.
29 @memberOf(SConsEnvironment)
30 def SwigLoadableModule(self, target, source, **keywords):
31  myenv = self.Clone()
32  if myenv['PLATFORM'] == 'darwin':
33  myenv.Append(LDMODULEFLAGS = ["-undefined", "suppress", "-flat_namespace", "-headerpad_max_install_names"])
34  #
35  # Swig-generated .cc files cast pointers to long longs and back,
36  # which is illegal. This flag tells g++ about the sin
37  #
38  try:
39  if myenv.whichCc == "gcc":
40  myenv.Append(CCFLAGS = ["-fno-strict-aliasing",])
41  except AttributeError:
42  pass
43  return myenv.LoadableModule(target, source, **keywords)
44 
45 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
46 
47 ##
48 # @brief Prepare the list of files to be passed to a SharedLibrary constructor
49 #
50 # In particular, ensure that any files listed in env.NoOptFiles (set by the command line option
51 # noOptFile="file1 file2") are built without optimisation and files listed in env.optFiles are
52 # built with optimisation
53 #
54 # The usage pattern in an SConscript file is:
55 # ccFiles = env.SourcesForSharedLibrary(Glob("../src/*/*.cc"))
56 # env.SharedLibrary('afw', ccFiles, LIBS=env.getLibs("self")))
57 #
58 # This is automatically used by scripts.BasicSConscript.lib().
59 ##
60 @memberOf(SConsEnvironment)
61 def SourcesForSharedLibrary(self, files):
62 
63  files = [SCons.Script.File(file) for file in files]
64 
65  if not (self.get("optFiles") or self.get("noOptFiles")):
66  files.sort()
67  return files
68 
69  if self.get("optFiles"):
70  optFiles = self["optFiles"].replace(".", r"\.") # it'll be used in an RE
71  optFiles = SCons.Script.Split(optFiles.replace(",", " "))
72  optFilesRe = "/(%s)$" % "|".join(optFiles)
73  else:
74  optFilesRe = None
75 
76  if self.get("noOptFiles"):
77  noOptFiles = self["noOptFiles"].replace(".", r"\.") # it'll be used in an RE
78  noOptFiles = SCons.Script.Split(noOptFiles.replace(",", " "))
79  noOptFilesRe = "/(%s)$" % "|".join(noOptFiles)
80  else:
81  noOptFilesRe = None
82 
83  if self.get("opt"):
84  opt = int(self["opt"])
85  else:
86  opt = 0
87 
88  if opt == 0:
89  opt = 3
90 
91  CCFLAGS_OPT = re.sub(r"-O(\d|s)\s*", "-O%d " % opt, " ".join(self["CCFLAGS"]))
92  CCFLAGS_NOOPT = re.sub(r"-O(\d|s)\s*", "-O0 ", " ".join(self["CCFLAGS"])) # remove -O flags from CCFLAGS
93 
94  sources = []
95  for ccFile in files:
96  if optFilesRe and re.search(optFilesRe, ccFile.abspath):
97  self.SharedObject(ccFile, CCFLAGS=CCFLAGS_OPT)
98  ccFile = os.path.splitext(ccFile.abspath)[0] + self["SHOBJSUFFIX"]
99  elif noOptFilesRe and re.search(noOptFilesRe, ccFile.abspath):
100  self.SharedObject(ccFile, CCFLAGS=CCFLAGS_NOOPT)
101  ccFile = os.path.splitext(ccFile.abspath)[0] + self["SHOBJSUFFIX"]
102 
103  sources.append(ccFile)
104 
105  sources.sort()
106  return sources
107 
108 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
109 
110 ##
111 # @brief Return a list of files that need to be scanned for tags, starting at directory root
112 #
113 # These tags are for advanced Emacs users, and should not be confused with SVN tags or Doxygen tags.
114 #
115 # Files are chosen if they match fileRegex; toplevel directories in list ignoreDirs are ignored
116 # This routine won't do anything unless you specified a "TAGS" target
117 ##
118 def filesToTag(root=None, fileRegex=None, ignoreDirs=None):
119  if root is None: root = "."
120  if fileRegex is None: fileRegex = r"^[a-zA-Z0-9_].*\.(cc|h(pp)?|py)$"
121  if ignoreDirs is None: ignoreDirs = ["examples", "tests"]
122 
123  if "TAGS" not in SCons.Script.COMMAND_LINE_TARGETS:
124  return []
125 
126  files = []
127  for dirpath, dirnames, filenames in os.walk(root):
128  if dirpath == ".":
129  dirnames[:] = [d for d in dirnames if not re.search(r"^(%s)$" % "|".join(ignoreDirs), d)]
130 
131  dirnames[:] = [d for d in dirnames if not re.search(r"^(\.svn)$", d)] # ignore .svn tree
132  #
133  # List of possible files to tag, but there's some cleanup required for machine-generated files
134  #
135  candidates = [f for f in filenames if re.search(fileRegex, f)]
136  #
137  # Remove files generated by swig
138  #
139  for swigFile in [f for f in filenames if re.search(r"\.i$", f)]:
140  name = os.path.splitext(swigFile)[0]
141  candidates = [f for f in candidates if not re.search(r"%s(_wrap\.cc?|\.py)$" % name, f)]
142 
143  files += [os.path.join(dirpath, f) for f in candidates]
144 
145  return files
146 
147 ##
148 # @brief Build Emacs tags (see man etags for more information).
149 #
150 # Files are chosen if they match fileRegex; toplevel directories in list ignoreDirs are ignored
151 # This routine won't do anything unless you specified a "TAGS" target
152 ##
153 @memberOf(SConsEnvironment)
154 def BuildETags(env, root=None, fileRegex=None, ignoreDirs=None):
155  toTag = filesToTag(root, fileRegex, ignoreDirs)
156  if toTag:
157  return env.Command("TAGS", toTag, "etags -o $TARGET $SOURCES")
158 
159 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
160 
161 ##
162 # @brief Remove files matching the argument list starting at dir
163 # when scons is invoked with -c/--clean and no explicit targets are listed
164 #
165 # E.g. CleanTree(r"*~ core")
166 #
167 # If recurse is True, recursively descend the file system; if
168 # verbose is True, print each filename after deleting it
169 ##
170 @memberOf(SConsEnvironment)
171 def CleanTree(self, files, dir=".", recurse=True, verbose=False):
172  #
173  # Generate command that we may want to execute
174  #
175  files_expr = ""
176  for file in SCons.Script.Split(files):
177  if files_expr:
178  files_expr += " -o "
179 
180  files_expr += "-name %s" % re.sub(r"(^|[^\\])([[*])", r"\1\\\2",file) # quote unquoted * and []
181  #
182  # don't use xargs --- who knows what needs quoting?
183  #
184  action = "find %s" % dir
185  action += r" \( -name .svn -prune -o -name \* \) "
186  if not recurse:
187  action += " ! -name . -prune"
188 
189  file_action = "rm -f"
190 
191  action += r" \( %s \) -exec %s {} \;" % \
192  (files_expr, file_action)
193 
194  if verbose:
195  action += " -print"
196  #
197  # Clean up scons files --- users want to be able to say scons -c and get a clean copy
198  # We can't delete .sconsign.dblite if we use "scons clean" instead of "scons --clean",
199  # so the former is no longer supported.
200  #
201  action += " ; rm -rf .sconf_temp .sconsign.dblite .sconsign.tmp config.log"
202  #
203  # Do we actually want to clean up? We don't if the command is e.g. "scons -c install"
204  #
205  if "clean" in SCons.Script.COMMAND_LINE_TARGETS:
206  state.log.fail("'scons clean' is no longer supported; please use 'scons --clean'.")
207  elif not SCons.Script.COMMAND_LINE_TARGETS and self.GetOption("clean"):
208  self.Execute(self.Action([action]))
209 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
210 
211 ## @brief Return a product's PRODUCT_DIR, or None
212 @memberOf(SConsEnvironment)
213 def ProductDir(env, product):
214  from . import eupsForScons
215  global _productDirs
216  try:
217  _productDirs
218  except:
219  try:
220  _productDirs = eupsForScons.productDir(eupsenv=eupsForScons.getEups())
221  except TypeError: # old version of eups (pre r18588)
222  _productDirs = None
223  if _productDirs:
224  pdir = _productDirs.get(product)
225  else:
226  pdir = eupsForScons.productDir(product)
227  if pdir == "none":
228  pdir = None
229  return pdir
230 
231 #-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
232 
233 ##
234 # @brief A callable to be used as an SCons Action to run Doxygen.
235 #
236 # This should only be used by the env.Doxygen pseudo-builder method.
237 #
238 class DoxygenBuilder(object):
239 
240  def __init__(self, **kw):
241  self.__dict__.update(kw)
242  self.results = []
243  self.sources = []
244  self.targets = []
245  self.useTags = list(SCons.Script.File(item).abspath for item in self.useTags)
246  self.inputs = list(SCons.Script.Entry(item).abspath for item in self.inputs)
247  self.excludes = list(SCons.Script.Entry(item).abspath for item in self.excludes)
248  self.outputPaths = list(SCons.Script.Dir(item) for item in self.outputs)
249 
250  def __call__(self, env, config):
251  self.findSources()
252  self.findTargets()
253  inConfigNode = SCons.Script.File(config)
254  outConfigName, ext = os.path.splitext(inConfigNode.abspath)
255  outConfigNode = SCons.Script.File(outConfigName)
256  if self.makeTag:
257  tagNode = SCons.Script.File(self.makeTag)
258  self.makeTag = tagNode.abspath
259  self.targets.append(tagNode)
260  config = env.Command(target=outConfigNode, source=inConfigNode if os.path.exists(config) else None,
261  action=self.buildConfig)
262  env.AlwaysBuild(config)
263  doc = env.Command(target=self.targets, source=self.sources,
264  action="doxygen %s" % outConfigNode.abspath)
265  for path in self.outputPaths:
266  env.Clean(doc, path)
267  env.Depends(doc, config)
268  self.results.extend(config)
269  self.results.extend(doc)
270  return self.results
271 
272  def findSources(self):
273  for path in self.inputs:
274  if os.path.isdir(path):
275  for root, dirs, files in os.walk(path):
276  if os.path.abspath(root) in self.excludes:
277  dirs[:] = []
278  continue
279  if not self.recursive:
280  dirs[:] = []
281  else:
282  toKeep = []
283  for relDir in dirs:
284  if relDir.startswith("."):
285  continue
286  absDir = os.path.abspath(os.path.join(root, relDir))
287  if absDir not in self.excludes:
288  toKeep.append(relDir)
289  dirs[:] = toKeep
290  if self.excludeSwig:
291  for relFile in files:
292  base, ext = os.path.splitext(relFile)
293  if ext == ".i":
294  self.excludes.append(os.path.join(root, base + ".py"))
295  self.excludes.append(os.path.join(root, base + "_wrap.cc"))
296  for relFile in files:
297  absFile = os.path.abspath(os.path.join(root, relFile))
298  if absFile in self.excludes:
299  continue
300  for pattern in self.patterns:
301  if fnmatch.fnmatch(relFile, pattern):
302  self.sources.append(SCons.Script.File(absFile))
303  break
304  elif os.path.isfile(path):
305  self.sources.append(SCons.Script.File(path))
306 
307  def findTargets(self):
308  for item in self.outputs:
309  self.targets.append(SCons.Script.Dir(item))
310 
311  def buildConfig(self, target, source, env):
312  outConfigFile = open(target[0].abspath, "w")
313  for tagPath in self.useTags:
314  docDir, tagFile = os.path.split(tagPath)
315  htmlDir = os.path.join(docDir, "html")
316  outConfigFile.write('TAGFILES += "%s=%s"\n' % (tagPath, htmlDir))
317  self.sources.append(SCons.Script.Dir(docDir))
318  docPaths = []
319  incFiles = []
320  for incPath in self.includes:
321  docDir, incFile = os.path.split(incPath)
322  docPaths.append('"%s"' % docDir)
323  incFiles.append('"%s"' % incFile)
324  self.sources.append(SCons.Script.File(incPath))
325  if docPaths:
326  outConfigFile.write('@INCLUDE_PATH = %s\n' % " ".join(docPaths))
327  for incFile in incFiles:
328  outConfigFile.write('@INCLUDE = %s\n' % incFile)
329  if self.projectName is not None:
330  outConfigFile.write("PROJECT_NAME = %s\n" % self.projectName)
331  if self.projectNumber is not None:
332  outConfigFile.write("PROJECT_NUMBER = %s\n" % self.projectNumber)
333  outConfigFile.write("INPUT = %s\n" % " ".join(self.inputs))
334  outConfigFile.write("EXCLUDE = %s\n" % " ".join(self.excludes))
335  outConfigFile.write("FILE_PATTERNS = %s\n" % " ".join(self.patterns))
336  outConfigFile.write("RECURSIVE = YES\n" if self.recursive else "RECURSIVE = NO\n")
337  allOutputs = set(("html", "latex", "man", "rtf", "xml"))
338  for output, path in zip(self.outputs, self.outputPaths):
339  try:
340  allOutputs.remove(output.lower())
341  except:
342  state.log.fail("Unknown Doxygen output format '%s'." % output)
343  state.log.finish()
344  outConfigFile.write("GENERATE_%s = YES\n" % output.upper())
345  outConfigFile.write("%s_OUTPUT = %s\n" % (output.upper(), path.abspath))
346  for output in allOutputs:
347  outConfigFile.write("GENERATE_%s = NO\n" % output.upper())
348  if self.makeTag is not None:
349  outConfigFile.write("GENERATE_TAGFILE = %s\n" % self.makeTag)
350  #
351  # Append the local overrides (usually doxygen.conf.in)
352  #
353  if len(source) > 0:
354  with open(source[0].abspath, "r") as inConfigFile:
355  outConfigFile.write(inConfigFile.read())
356 
357  outConfigFile.close()
358 
359 ##
360 # @brief Generate a Doxygen config file and run Doxygen on it.
361 #
362 # Rather than parse a complete Doxygen config file for SCons sources
363 # and targets, this Doxygen builder builds a Doxygen config file,
364 # adding INPUT, FILE_PATTERNS, RECUSRIVE, EXCLUDE, XX_OUTPUT and
365 # GENERATE_XX options (and possibly others) to an existing
366 # proto-config file. Generated settings will override those in
367 # the proto-config file.
368 #
369 # @param config A Doxygen config file, usually with the
370 # extension .conf.in; a new file with the .in
371 # removed will be generated and passed to
372 # Doxygen. Settings in the original config
373 # file will be overridden by those generated
374 # by this method.
375 # @param inputs A sequence of folders or files to be passed
376 # as the INPUT setting for Doxygen. This list
377 # will be turned into absolute paths by SCons,
378 # so the "#folder" syntax will work.
379 # Otherwise, the list is passed in as-is, but
380 # the builder will also examine those
381 # directories to find which source files the
382 # Doxygen output actually depends on.
383 # @param patterns A sequence of glob patterns for the
384 # FILE_PATTERNS Doxygen setting. This will be
385 # passed directly to Doxygen, but it is also
386 # used to determine which source files should
387 # be considered dependencies.
388 # @param recursive Whether the inputs should be searched
389 # recursively (used for the Doxygen RECURSIVE
390 # setting).
391 # @param outputs A sequence of output formats which will also
392 # be used as output directories.
393 # @param exclude A sequence of folders or files (not globs)
394 # to be ignored by Doxygen (the Doxygen
395 # EXCLUDE setting). Hidden directories are
396 # automatically ignored.
397 # @param includes A sequence of Doxygen config files to
398 # include. These will automatically be
399 # separated into paths and files to fill in
400 # the \@INCLUDE_PATH and \@INCLUDE settings.
401 # @param useTags A sequence of Doxygen tag files to use. It
402 # will be assumed that the html directory for
403 # each tag file is in an "html" subdirectory
404 # in the same directory as the tag file.
405 # @param makeTag A string indicating the name of a tag file
406 # to be generated.
407 # @param projectName Sets the Doxygen PROJECT_NAME setting.
408 # @param projectNumber Sets the Doxygen PROJECT_NUMBER setting.
409 # @param excludeSwig If True (default), looks for SWIG .i files
410 # in the input directories and adds Python
411 # and C++ files generated by SWIG to the
412 # list of files to exclude. For this to work,
413 # the SWIG-generated filenames must be the
414 # default ones ("module.i" generates "module.py"
415 # and "moduleLib_wrap.cc").
416 #
417 # @note When building documentation from a clean source tree,
418 # generated source files (like headers generated with M4)
419 # will not be included among the dependencies, because
420 # they aren't present when we walk the input folders.
421 # The workaround is just to build the docs after building
422 # the source.
423 ##
424 @memberOf(SConsEnvironment)
425 def Doxygen(self, config, **kw):
426  inputs = [d for d in ["#doc", "#include", "#python", "#src"]
427  if os.path.exists(SCons.Script.Entry(d).abspath)]
428  defaults = {
429  "inputs": inputs,
430  "recursive": True,
431  "patterns": ["*.h", "*.cc", "*.py", "*.dox"],
432  "outputs": ["html",],
433  "excludes": [],
434  "includes": [],
435  "useTags": [],
436  "makeTag": None,
437  "projectName": None,
438  "projectNumber": None,
439  "excludeSwig": True
440  }
441  for k in defaults:
442  if kw.get(k) is None:
443  kw[k] = defaults[k]
444  builder = DoxygenBuilder(**kw)
445  return builder(self, config)
446 
447 @memberOf(SConsEnvironment)
448 def VersionModule(self, filename, versionString=None):
449  if versionString is None:
450  for n in ("git", "hg", "svn",):
451  if os.path.isdir(".%s" % n):
452  versionString = n
453 
454  if not versionString:
455  versionString = "git"
456 
457  def calcMd5(filename):
458  try:
459  import hashlib
460  md5 = hashlib.md5("\n".join(open(filename).readlines())).hexdigest()
461  except IOError:
462  md5 = None
463 
464  return md5
465 
466  oldMd5 = calcMd5(filename)
467 
468  def makeVersionModule(target, source, env):
469  try:
470  version = determineVersion(state.env, versionString)
471  except RuntimeError:
472  version = "unknown"
473  parts = version.split("+")
474 
475  names = []
476  with open(target[0].abspath, "w") as outFile:
477  outFile.write("#--------- This file is automatically generated by LSST's sconsUtils ---------#\n")
478 
479  what = "__version__"
480  outFile.write("%s = '%s'\n" % (what, version))
481  names.append(what)
482 
483  what = "__repo_version__"
484  outFile.write("%s = '%s'\n" % (what, parts[0]))
485  names.append(what)
486 
487  what = "__repo_version__"
488  outFile.write("%s = '%s'\n" % (what, parts[0]))
489  names.append(what)
490 
491  what = "__fingerprint__"
492  outFile.write("%s = '%s'\n" % (what, getFingerprint(versionString)))
493  names.append(what)
494 
495  try:
496  info = tuple(int(v) for v in parts[0].split("."))
497  what = "__version_info__"
498  names.append(what)
499  outFile.write("%s = %r\n" % (what, info))
500  except ValueError:
501  pass
502 
503  if len(parts) > 1:
504  try:
505  what = "__rebuild_version__"
506  outFile.write("%s = %s\n" % (what, int(parts[1])))
507  names.append(what)
508  except ValueError:
509  pass
510 
511  what = "__dependency_versions__"
512  names.append(what)
513  outFile.write("%s = {\n" % (what))
514  for name, mod in env.dependencies.packages.items():
515  if mod is None:
516  outFile.write(" '%s': None,\n" % name)
517  elif hasattr(mod.config, "version"):
518  outFile.write(" '%s': '%s',\n" % (name, mod.config.version))
519  else:
520  outFile.write(" '%s': 'unknown',\n" % name)
521  outFile.write("}\n")
522 
523  outFile.write("__all__ = %r\n" % (tuple(names),))
524 
525  if calcMd5(target[0].abspath) != oldMd5: # only print if something's changed
526  state.log.info("makeVersionModule([\"%s\"], [])" % str(target[0]))
527 
528  result = self.Command(filename, [], self.Action(makeVersionModule, strfunction=lambda *args: None))
529 
530  self.AlwaysBuild(result)
531  return result
def getFingerprint
Return a unique fingerprint for a version (e.g.
Definition: installation.py:74
def memberOf
A Python decorator that injects functions into a class.
Definition: utils.py:85
def filesToTag
Return a list of files that need to be scanned for tags, starting at directory root.
Definition: builders.py:118
def CleanTree
Remove files matching the argument list starting at dir when scons is invoked with -c/–clean and no e...
Definition: builders.py:171
def ProductDir
Return a product's PRODUCT_DIR, or None.
Definition: builders.py:213
def determineVersion
Set a version ID from env, or a version control ID string ($name$ or $HeadURL$)
Definition: installation.py:47
def Doxygen
Generate a Doxygen config file and run Doxygen on it.
Definition: builders.py:425
def SharedLibraryIncomplete
Like SharedLibrary, but don't insist that all symbols are resolved.
Definition: builders.py:21
def BuildETags
Build Emacs tags (see man etags for more information).
Definition: builders.py:154
def SwigLoadableModule
Like LoadableModule, but don't insist that all symbols are resolved, and set some SWIG-specific flags...
Definition: builders.py:30
def SourcesForSharedLibrary
Prepare the list of files to be passed to a SharedLibrary constructor.
Definition: builders.py:61
A callable to be used as an SCons Action to run Doxygen.
Definition: builders.py:238