LSSTApplications  10.0-2-g4f67435,11.0.rc2+1,11.0.rc2+12,11.0.rc2+3,11.0.rc2+4,11.0.rc2+5,11.0.rc2+6,11.0.rc2+7,11.0.rc2+8
LSSTDataManagementBasePackage
scripts.py
Go to the documentation of this file.
1 ##
2 # @file scripts.py
3 #
4 # Convenience functions to do the work of standard LSST SConstruct/SConscript files.
5 #
6 # @defgroup sconsUtilsScripts Convenience functions for SConstruct/SConscript files
7 # @{
8 ##
9 from __future__ import absolute_import, division, print_function
10 import os.path
11 from SCons.Script import *
12 from distutils.spawn import find_executable
13 
14 from . import dependencies
15 from . import builders
16 from . import installation
17 from . import state
18 from . import tests
19 
20 def _getFileBase(node):
21  name, ext = os.path.splitext(os.path.basename(str(node)))
22  return name
23 
24 ##
25 # @brief A scope-only class for SConstruct-replacement convenience functions.
26 #
27 # The boilerplate for a standard LSST SConstruct file is replaced by two static methods:
28 # initialize() and finish(). The former configures dependencies, sets up package-dependent
29 # environment variables, and calls any SConscript files found in subdirectories, while the
30 # latter sets up installation paths, default targets, and explicit dependencies.
31 #
32 # Calling BasicSConstruct as a function invokes its __new__ method, which calls both
33 # initialize() and finish(), and should be used when the SConstruct file doesn't need to
34 # do anything other than what they provide.
35 ##
36 class BasicSConstruct(object):
37 
38  _initializing = False
39 
40  ##
41  # @brief Convenience function to replace standard SConstruct boilerplate.
42  #
43  # This is a shortcut for
44  # @code
45  # BasicSConstruct.initialize(...)
46  # BasicSConstruct.finalize(...)
47  # @endcode
48  #
49  # This returns the sconsUtils.env Environment object rather than
50  # a BasicSConstruct instance (which would be useless).
51  ##
52  def __new__(cls, packageName, versionString=None, eupsProduct=None, eupsProductPath=None, cleanExt=None,
53  defaultTargets=("lib", "python", "tests", "examples", "doc"),
54  subDirList=None, ignoreRegex=None,
55  versionModuleName="python/lsst/%s/version.py", noCfgFile=False):
56  cls.initialize(packageName, versionString, eupsProduct, eupsProductPath, cleanExt,
57  versionModuleName, noCfgFile=noCfgFile)
58  cls.finish(defaultTargets, subDirList, ignoreRegex)
59  return state.env
60 
61  ##
62  # @brief Convenience function to replace standard SConstruct boilerplate (step 1).
63  #
64  # This function:
65  # - Calls all SConscript files found in subdirectories.
66  # - Configures dependencies.
67  # - Sets how the --clean option works.
68  #
69  # @param packageName Name of the package being built; must correspond to a .cfg file in ups/.
70  # @param versionString Version-control system string to be parsed for version information
71  # ($HeadURL$ for SVN). Defaults to "git" if not set or None.
72  # @param eupsProduct Name of the EUPS product being built. Defaults to and is almost always
73  # the name of the package.
74  # @param eupsProductPath An alternate directory where the package should be installed.
75  # @param cleanExt Whitespace delimited sequence of globs for files to remove with --clean.
76  # @param versionModuleName If non-None, builds a version.py module as this file; '%s' is replaced with
77  # the name of the package.
78  # @param noCfgFile If True, this package has no .cfg file
79  #
80  # @returns an SCons Environment object (which is also available as lsst.sconsUtils.env).
81  ##
82  @classmethod
83  def initialize(cls, packageName, versionString=None, eupsProduct=None, eupsProductPath=None,
84  cleanExt=None, versionModuleName="python/lsst/%s/version.py", noCfgFile=False):
85  if cls._initializing:
86  state.log.fail("Recursion detected; an SConscript file should not call BasicSConstruct.")
87  cls._initializing = True
88  if cleanExt is None:
89  cleanExt = r"*~ core *.so *.os *.o *.pyc *.pkgc"
90  dependencies.configure(packageName, versionString, eupsProduct, eupsProductPath, noCfgFile)
91  state.env.BuildETags()
92  state.env.CleanTree(cleanExt)
93  if versionModuleName is not None:
94  try:
95  versionModuleName = versionModuleName % "/".join(packageName.split("_"))
96  except TypeError:
97  pass
98  state.targets["version"] = state.env.VersionModule(versionModuleName)
99  for root, dirs, files in os.walk("."):
100  if "SConstruct" in files and root != ".":
101  dirs[:] = []
102  continue
103  dirs[:] = [d for d in dirs if (not d.startswith('.'))]
104  dirs.sort() # happy coincidence that include < libs < python < tests
105  if "SConscript" in files:
106  state.log.info("Using Sconscript at %s/SConscript" % root)
107  SCons.Script.SConscript(os.path.join(root, "SConscript"))
108  cls._initializing = False
109  return state.env
110 
111  ##
112  # @brief Convenience function to replace standard SConstruct boilerplate (step 2).
113  #
114  # This function:
115  # - Sets up installation paths.
116  # - Tells SCons to only do MD5 checks when timestamps have changed.
117  # - Sets the "include", "lib", "python", and "tests" targets as the defaults
118  # to be built when scons is run with no target arguments.
119  #
120  # @param subDirList An explicit list of subdirectories that should be installed. By default,
121  # all non-hidden subdirectories will be installed.
122  # @param defaultTargets A sequence of targets (see state.targets) that should be built when
123  # scons is run with no arguments.
124  # @param ignoreRegex Regular expression that matches files that should not be installed.
125  #
126  # @returns an SCons Environment object (which is also available as lsst.sconsUtils.env).
127  ##
128  @staticmethod
129  def finish(defaultTargets=("lib", "python", "tests", "examples", "doc"),
130  subDirList=None, ignoreRegex=None):
131  if ignoreRegex is None:
132  ignoreRegex = r"(~$|\.pyc$|^\.svn$|\.o|\.os$)"
133  if subDirList is None:
134  subDirList = []
135  for path in os.listdir("."):
136  if os.path.isdir(path) and not path.startswith("."):
137  subDirList.append(path)
138  install = state.env.InstallLSST(state.env["prefix"],
139  [subDir for subDir in subDirList],
140  ignoreRegex=ignoreRegex)
141  for name, target in state.targets.items():
142  state.env.Requires(install, target)
143  state.env.Alias(name, target)
144  state.env.Requires(state.targets["python"], state.targets["version"])
145  declarer = state.env.Declare()
146  state.env.Requires(declarer, install) # Ensure declaration fires after installation available
147  state.env.Default([t for t in defaultTargets if os.path.exists(t)])
148  if "version" in state.targets:
149  state.env.Default(state.targets["version"])
150  state.env.Requires(state.targets["tests"], state.targets["version"])
151  state.env.Decider("MD5-timestamp") # if timestamps haven't changed, don't do MD5 checks
152  #
153  # Check if any of the tests failed by looking for *.failed files.
154  # Perform this test just before scons exits
155  #
156  # N.b. the test is written in sh not python as then we can use @ to suppress output
157  #
158  if "tests" in [str(t) for t in BUILD_TARGETS]:
159  testsDir = os.path.join(os.getcwd(), "tests", ".tests")
160  checkTestStatus_command = state.env.Command('checkTestStatus', [], """
161  @ if [ -d %s ]; then \
162  nfail=`find %s -name \*.failed | wc -l | sed -e 's/ //g'`; \
163  if [ $$nfail -gt 0 ]; then \
164  echo "$$nfail tests failed" >&2; exit 1; \
165  fi; \
166  fi; \
167  """ % (testsDir, testsDir))
168 
169  state.env.Depends(checkTestStatus_command, BUILD_TARGETS) # this is why the check runs last
170  BUILD_TARGETS.extend(checkTestStatus_command)
171  state.env.AlwaysBuild(checkTestStatus_command)
172 
173 ##
174 # @brief A scope-only class for SConscript-replacement convenience functions.
175 #
176 # All methods of BasicSConscript are static. All of these functions update the state.targets
177 # dictionary of targets used to set default targets and fix build dependencies; if you build anything
178 # without using BasicSConscript methods, be sure to manually it to the state.targets dict.
179 ##
180 class BasicSConscript(object):
181 
182  ##
183  # @brief Convenience function to replace standard lib/SConscript boilerplate.
184  #
185  # With no arguments, this will build a shared library with the same name as the package.
186  # This uses env.SourcesForSharedLibrary to support the optFiles/noOptFiles command-line variables.
187  #
188  # @param libName Name of the shared libray to be built (defaults to env["packageName"]).
189  # @param src Source to compile into the library. Defaults to a 4-directory deep glob
190  # of all *.cc files in \#src.
191  # @param libs Libraries to link against, either as a string argument to be passed to
192  # env.getLibs() or a sequence of actual libraries to pass in.
193  ##
194  @staticmethod
195  def lib(libName=None, src=None, libs="self"):
196  if libName is None:
197  libName = state.env["packageName"]
198  if src is None:
199  src = Glob("#src/*.cc") + Glob("#src/*/*.cc") + Glob("#src/*/*/*.cc") + Glob("#src/*/*/*/*.cc")
200  src = state.env.SourcesForSharedLibrary(src)
201  if isinstance(libs, basestring):
202  libs = state.env.getLibs(libs)
203  elif libs is None:
204  libs = []
205  result = state.env.SharedLibrary(libName, src, LIBS=libs)
206  state.targets["lib"].extend(result)
207  return result
208 
209  ##
210  # @brief Convenience function to replace standard python/*/SConscript boilerplate.
211  #
212  # With no arguments, this will build a SWIG module with the name determined according
213  # to our current pseudo-convention: last part of env["packageName"], split by underscores,
214  # with "Lib" appended to the end.
215  #
216  # @param swigNameList Sequence of SWIG modules to be built (does not include the file extensions).
217  # @param libs Libraries to link against, either as a string argument to be passed to
218  # env.getLibs() or a sequence of actual libraries to pass in.
219  # @param swigSrc A dictionary of additional source files that go into the modules. Each
220  # key should be an entry in swigNameList, and each value should be a list
221  # of additional C++ source files not generated by SWIG.
222  ##
223  @staticmethod
224  def python(swigNameList=None, libs="main python", swigSrc=None):
225  if swigNameList is None:
226  swigNameList = [state.env["packageName"].split("_")[-1] + "Lib"]
227  swigFileList = [File(name + ".i") for name in swigNameList]
228  if swigSrc is None:
229  swigSrc = {}
230  for name, node in zip(swigNameList, swigFileList):
231  swigSrc.setdefault(name, []).append(node)
232  if isinstance(libs, basestring):
233  libs = state.env.getLibs(libs)
234  elif libs is None:
235  libs = []
236  result = []
237  for name, src in swigSrc.items():
238  result.extend(state.env.SwigLoadableModule("_" + name, src, LIBS=libs))
239  state.targets["python"].extend(result)
240  return result
241 
242  ##
243  # @brief Convenience function to replace standard doc/SConscript boilerplate.
244  #
245  # With no arguments, this will generate a Doxygen config file and run Doxygen
246  # with env.Doxygen(), using the projectName and projectNumber from
247  # env["packageName"] and env["version"], respectively.
248  #
249  # This essentially just forwards all arguments (which should be passed as
250  # keyword arguments) to env.Doxygen().
251  ##
252  @staticmethod
253  def doc(config="doxygen.conf.in", projectName=None, projectNumber=None, **kw):
254  if not find_executable("doxygen"):
255  state.log.warn("doxygen executable not found; skipping documentation build.")
256  return []
257  if projectName is None:
258  projectName = ".".join(["lsst"] + state.env["packageName"].split("_"))
259  if projectNumber is None:
260  projectNumber = state.env["version"]
261  result = state.env.Doxygen(
262  config, projectName=projectName, projectNumber=projectNumber,
263  includes=state.env.doxygen["includes"],
264  useTags=state.env.doxygen["tags"],
265  makeTag=(state.env["packageName"] + ".tag"),
266  **kw
267  )
268  state.targets["doc"].extend(result)
269  return result
270 
271  ##
272  # @brief Convenience function to replace standard tests/SConscript boilerplate.
273  #
274  # With no arguments, will attempt to figure out which files should be run as tests
275  # and which are support code (like SWIG modules).
276  #
277  # Python tests will be marked as dependent on the entire \#python directory and
278  # any SWIG modules built in the tests directory. This should ensure tests are always
279  # run when their results might have changed, but may result in them being re-run more often
280  # than necessary.
281  #
282  # @param pyList A sequence of Python tests to run (including .py extensions).
283  # Defaults to a *.py glob of the tests directory, minus any
284  # files corresponding to the SWIG modules in swigFileList.
285  # @param ccList A sequence of C++ unit tests to run (including .cc extensions).
286  # Defaults to a *.cc glob of the tests directory, minus any
287  # files that end with *_wrap.cc and files present in swigSrc.
288  # @param swigNameList A sequence of SWIG modules to build (NOT including .i extensions).
289  # @param swigSrc Additional source files to be compiled into SWIG modules, as a
290  # dictionary; each key must be an entry in swigNameList, and each
291  # value a list of additional source files.
292  # @param ignoreList List of ignored tests to be passed to tests.Control (note that
293  # ignored tests will be built, but not run).
294  # @param nobuildList List of tests that should not even be built.
295  # @param args A dictionary of program arguments for tests, passed directly
296  # to tests.Control.
297  ##
298  @staticmethod
299  def tests(pyList=None, ccList=None, swigNameList=None, swigSrc=None,
300  ignoreList=None, noBuildList=None,
301  args=None):
302  if noBuildList is None:
303  noBuildList = []
304  if swigNameList is None:
305  swigFileList = Glob("*.i")
306  swigNameList = [_getFileBase(node) for node in swigFileList]
307  else:
308  swigFileList = [File(name + ".i") for name in swigNameList]
309  if swigSrc is None:
310  swigSrc = {}
311  allSwigSrc = set()
312  for name, node in zip(swigNameList, swigFileList):
313  src = swigSrc.setdefault(name, [])
314  allSwigSrc.update(str(element) for element in src)
315  src.append(node)
316  if pyList is None:
317  pyList = [node for node in Glob("*.py")
318  if _getFileBase(node) not in swigNameList
319  and os.path.basename(str(node)) not in noBuildList]
320  if ccList is None:
321  ccList = [node for node in Glob("*.cc")
322  if (not str(node).endswith("_wrap.cc")) and str(node) not in allSwigSrc
323  and os.path.basename(str(node)) not in noBuildList]
324  if ignoreList is None:
325  ignoreList = []
326  s = lambda l: [str(i) for i in l]
327  state.log.info("SWIG modules for tests: %s" % s(swigFileList))
328  state.log.info("Python tests: %s" % s(pyList))
329  state.log.info("C++ tests: %s" % s(ccList))
330  state.log.info("Files that will not be built: %s" % noBuildList)
331  state.log.info("Ignored tests: %s" % ignoreList)
332  control = tests.Control(state.env, ignoreList=ignoreList, args=args, verbose=True)
333  for ccTest in ccList:
334  state.env.Program(ccTest, LIBS=state.env.getLibs("main test"))
335  swigMods = []
336  for name, src in swigSrc.items():
337  swigMods.extend(
338  state.env.SwigLoadableModule("_" + name, src, LIBS=state.env.getLibs("main python"))
339  )
340  ccList = [control.run(str(node)) for node in ccList]
341  pyList = [control.run(str(node)) for node in pyList]
342  for pyTest in pyList:
343  state.env.Depends(pyTest, swigMods)
344  state.env.Depends(pyTest, state.targets["python"])
345  result = ccList + pyList
346  state.targets["tests"].extend(result)
347  return result
348 
349  ##
350  # @brief Convenience function to replace standard examples/SConscript boilerplate.
351  #
352  # @param ccList A sequence of C++ examples to build (including .cc extensions).
353  # Defaults to a *.cc glob of the examples directory, minus any
354  # files that end with *_wrap.cc and files present in swigSrc.
355  # @param swigNameList A sequence of SWIG modules to build (NOT including .i extensions).
356  # @param swigSrc Additional source files to be compiled into SWIG modules, as a
357  # dictionary; each key must be an entry in swigNameList, and each
358  # value a list of additional source files.
359  ##
360  @staticmethod
361  def examples(ccList=None, swigNameList=None, swigSrc=None):
362  if swigNameList is None:
363  swigFileList = Glob("*.i")
364  swigNameList = [_getFileBase(node) for node in swigFileList]
365  else:
366  swigFileList = [File(name) for name in swigNameList]
367  if swigSrc is None:
368  swigSrc = {}
369  allSwigSrc = set()
370  for name, node in zip(swigNameList, swigFileList):
371  src = swigSrc.setdefault(name, [])
372  allSwigSrc.update(str(element) for element in src)
373  src.append(node)
374  if ccList is None:
375  ccList = [node for node in Glob("*.cc")
376  if (not str(node).endswith("_wrap.cc")) and str(node) not in allSwigSrc]
377  state.log.info("SWIG modules for examples: %s" % swigFileList)
378  state.log.info("C++ examples: %s" % ccList)
379  results = []
380  for src in ccList:
381  results.extend(state.env.Program(src, LIBS=state.env.getLibs("main")))
382  swigMods = []
383  for name, src in swigSrc.items():
384  results.extend(
385  state.env.SwigLoadableModule("_" + name, src, LIBS=state.env.getLibs("main python"))
386  )
387  for result in results:
388  state.env.Depends(result, state.targets["lib"])
389  state.targets["examples"].extend(results)
390  return results
391 
392 ## @}
def lib
Convenience function to replace standard lib/SConscript boilerplate.
Definition: scripts.py:195
def initialize
Convenience function to replace standard SConstruct boilerplate (step 1).
Definition: scripts.py:84
def finish
Convenience function to replace standard SConstruct boilerplate (step 2).
Definition: scripts.py:130
def doc
Convenience function to replace standard doc/SConscript boilerplate.
Definition: scripts.py:253
A scope-only class for SConstruct-replacement convenience functions.
Definition: scripts.py:36
A scope-only class for SConscript-replacement convenience functions.
Definition: scripts.py:180
def examples
Convenience function to replace standard examples/SConscript boilerplate.
Definition: scripts.py:361
def __new__
Convenience function to replace standard SConstruct boilerplate.
Definition: scripts.py:55
A class to control unit tests.
Definition: tests.py:17
def tests
Convenience function to replace standard tests/SConscript boilerplate.
Definition: scripts.py:301
def python
Convenience function to replace standard python/*/SConscript boilerplate.
Definition: scripts.py:224