LSSTApplications  10.0-2-g4f67435,11.0.rc2+1,11.0.rc2+12,11.0.rc2+3,11.0.rc2+4,11.0.rc2+5,11.0.rc2+6,11.0.rc2+7,11.0.rc2+8
LSSTDataManagementBasePackage
state.py
Go to the documentation of this file.
1 ##
2 # @file state.py
3 #
4 # This module acts like a singleton, holding all global state for sconsUtils.
5 # This includes the primary Environment object (state.env), the message log (state.log),
6 # the command-line variables object (state.opts), and a dictionary of command-line targets
7 # used to setup aliases, default targets, and dependencies (state.targets). All four of
8 # these variables are aliased to the main lsst.sconsUtils scope, so there should be no
9 # need for users to deal with the state module directly.
10 #
11 # These are all initialized when the module is imported, but may be modified by other code
12 # (particularly dependencies.configure()).
13 ##
14 
15 from __future__ import absolute_import, division, print_function
16 import os
17 import re
18 
19 import SCons.Script
20 import SCons.Conftest
21 from . import eupsForScons
22 
23 SCons.Script.EnsureSConsVersion(2, 1, 0)
24 
25 ##
26 # @brief A dictionary of SCons aliases and targets.
27 #
28 # These are used to setup aliases, default targets, and dependencies by BasicSConstruct.finish().
29 # While one can still use env.Alias to setup aliases (and should for "install"), putting targets
30 # here will generally provide better build-time dependency handling (like ensuring everything
31 # is built before we try to install, and making sure SCons doesn't rebuild the world before
32 # installing).
33 #
34 # Users can add additional keys to the dictionary if desired.
35 #
36 # Targets should be added by calling extend() or using += on the dict values, to keep the lists of
37 # targets from turning into lists-of-lists.
38 ##
39 targets = {"doc": [], "tests": [], "lib": [], "python": [], "examples": [], "include": [], "version": []}
40 
41 ## @cond INTERNAL
42 
43 env = None
44 log = None
45 opts = None
46 
47 
48 def _initOptions():
49  SCons.Script.AddOption('--checkDependencies', dest='checkDependencies',
50  action='store_true', default=False,
51  help="Verify dependencies with autoconf-style tests.")
52  SCons.Script.AddOption('--filterWarn', dest='filterWarn', action='store_true', default=False,
53  help="Filter out a class of warnings deemed irrelevant"),
54  SCons.Script.AddOption('--force', dest='force', action='store_true', default=False,
55  help="Set to force possibly dangerous behaviours")
56  SCons.Script.AddOption('--linkFarmDir', dest='linkFarmDir', action='store', default=None,
57  help="The directory of symbolic links needed to build and use the package")
58  SCons.Script.AddOption('--prefix', dest='prefix', action='store', default=False,
59  help="Specify the install destination")
60  SCons.Script.AddOption('--setenv', dest='setenv', action='store_true', default=False,
61  help="Treat arguments such as Foo=bar as defining construction variables")
62  SCons.Script.AddOption('--tag', dest='tag', action='store', default=None,
63  help="Declare product with this eups tag")
64  SCons.Script.AddOption('--verbose', dest='verbose', action='store_true', default=False,
65  help="Print additional messages for debugging.")
66  SCons.Script.AddOption('--traceback', dest='traceback', action='store_true', default=False,
67  help="Print full exception tracebacks when errors occur.")
68  SCons.Script.AddOption('--no-eups', dest='no_eups', action='store_true', default=False,
69  help="Do not use EUPS for configuration")
70 
71 def _initLog():
72  from . import utils
73  global log
74  log = utils.Log()
75 
76 def _initVariables():
77  files = []
78  if "optfile" in SCons.Script.ARGUMENTS:
79  configfile = SCons.Script.ARGUMENTS["optfile"]
80  if configfile not in files:
81  files.append(configfile)
82  for file in files:
83  if not os.path.isfile(file):
84  log.warn("Warning: Will ignore non-existent options file, %s" % file)
85  if "optfile" not in SCons.Script.ARGUMENTS:
86  files.append("buildOpts.py")
87  global opts
88  opts = SCons.Script.Variables(files)
89  opts.AddVariables(
90  ('archflags', 'Extra architecture specification to add to CC/LINK flags (e.g. -m32)', ''),
91  ('cc', 'Choose the compiler to use', ''),
92  SCons.Script.BoolVariable('debug', 'Set to enable debugging flags (use --debug)', True),
93  ('eupsdb', 'Specify which element of EUPS_PATH should be used', None),
94  ('flavor', 'Set the build flavor', None),
95  SCons.Script.BoolVariable('force', 'Set to force possibly dangerous behaviours', False),
96  ('optfile', 'Specify a file to read default options from', None),
97  ('prefix', 'Specify the install destination', None),
98  SCons.Script.EnumVariable('opt', 'Set the optimisation level', 0,
99  allowed_values=('0', '1', '2', '3')),
100  SCons.Script.EnumVariable('profile', 'Compile/link for profiler', 0,
101  allowed_values=('0', '1', 'pg', 'gcov')),
102  ('version', 'Specify the version to declare', None),
103  ('baseversion', 'Specify the current base version', None),
104  ('optFiles', "Specify a list of files that SHOULD be optimized", None),
105  ('noOptFiles', "Specify a list of files that should NOT be optimized", None),
106  )
107 
108 def _initEnvironment():
109  """Construction and basic setup of the state.env variable."""
110 
111  ourEnv = {}
112  for key in ('EUPS_DIR', 'EUPS_PATH', 'EUPS_SHELL', 'PATH' ,'DYLD_LIBRARY_PATH', 'LD_LIBRARY_PATH',
113  'SHELL', 'TMPDIR', 'TEMP', 'TMP', 'EUPS_LOCK_PID', 'XPA_PORT'):
114  if key in os.environ:
115  ourEnv[key] = os.environ[key]
116 
117  # Find and propagate EUPS environment variables.
118  cfgPath = []
119  for k in os.environ:
120  m = re.search(r"^(?P<name>\w+)_DIR(?P<extra>_EXTRA)?$", k)
121  if not m: continue
122  cfgPath.append(os.path.join(os.environ[k], "ups"))
123  if m.group("extra"):
124  cfgPath.append(os.environ[k])
125  else:
126  cfgPath.append(os.path.join(os.environ[k], "ups"))
127  p = m.group("name")
128  varname = eupsForScons.utils.setupEnvNameFor(p)
129  if varname in os.environ:
130  ourEnv[varname] = os.environ[varname]
131  ourEnv[k] = os.environ[k]
132 
133  # add <build root>/ups directory to the configuration search path
134  # this allows the .cfg file for the package being built to be found without
135  # requiring <product name>_DIR to be in the env
136  cfgPath.append(os.path.join(SCons.Script.Dir('#').abspath, 'ups'))
137 
138  # Recursively walk LSST_CFG_PATH
139  for root in os.environ.get("LSST_CFG_PATH", "").split(":"):
140  for base, dirs, files in os.walk(root):
141  dirs = [d for d in dirs if not d.startswith(".")]
142  cfgPath.insert(0, base)
143  #
144  # Add any values marked as export=FOO=XXX[,GOO=YYY] to ourEnv
145  #
146  exportVal = SCons.Script.ARGUMENTS.pop("export", None)
147  if exportVal:
148  for kv in exportVal.split(','):
149  k, v = kv.split('=')
150  ourEnv[k] = v
151  global env
152  sconsUtilsPath, thisFile = os.path.split(__file__)
153  toolPath = os.path.join(sconsUtilsPath, "tools")
154  env = SCons.Script.Environment(
155  ENV=ourEnv,
156  variables=opts,
157  toolpath=[toolPath],
158  tools=["default", "cuda"]
159  )
160  env.cfgPath = cfgPath
161  #
162  # We don't want "lib" inserted at the beginning of loadable module names;
163  # we'll import them under their given names.
164  #
165  env['LDMODULEPREFIX'] = ""
166  if env['PLATFORM'] == 'darwin':
167  env['LDMODULESUFFIX'] = ".so"
168  if not re.search(r"-install_name", str(env['SHLINKFLAGS'])):
169  env.Append(SHLINKFLAGS = ["-Wl,-install_name", "-Wl,${TARGET.file}"])
170  if not re.search(r"-headerpad_max_install_names", str(env['SHLINKFLAGS'])):
171  env.Append(SHLINKFLAGS = ["-Wl,-headerpad_max_install_names"])
172  #
173  # Remove valid options from the arguments
174  #
175  # SCons Variables do not behave like dicts
176  for opt in opts.keys():
177  try:
178  del SCons.Script.ARGUMENTS[opt]
179  except KeyError:
180  pass
181  #
182  # Process those arguments
183  #
184  for k in ("force", "prefix"): # these may now be set as options instead of variables
185  if SCons.Script.GetOption(k):
186  env[k] = SCons.Script.GetOption(k)
187 
188  if env['debug']:
189  env.Append(CCFLAGS = ['-g'])
190 
191  #
192  # determine if EUPS is present
193  #
194 
195  # --no-eups overides probing
196  # XXX is it possible to test python snippets as a scons action?
197  if SCons.Script.GetOption("no_eups"):
198  env['no_eups'] = True
199  else:
200  env['no_eups'] = not eupsForScons.haveEups()
201 
202  if not env.GetOption("no_progress"):
203  if env['no_eups']:
204  log.info('EUPS integration: disabled')
205  else:
206  log.info('EUPS integration: enabled')
207 
208  #
209  # Find the eups path, replace 'flavor' in favor of 'PLATFORM' if needed.
210  #
211  eupsPath = None
212  try:
213  db = env['eupsdb']
214  if 'EUPS_PATH' not in os.environ:
215  raise RuntimeError("You can't use eupsdb=XXX without an EUPS_PATH set")
216  eupsPath = None
217  for d in os.environ['EUPS_PATH'].split(':'):
218  if re.search(r"/%s$|^%s/|/%s/" % (db, db, db), d):
219  eupsPath = d
220  break
221  if not eupsPath:
222  raise RuntimeError("I cannot find DB \"%s\" in $EUPS_PATH" % db)
223  except KeyError:
224  if 'EUPS_PATH' in os.environ:
225  eupsPath = os.environ['EUPS_PATH'].split(':')[0]
226  env['eupsPath'] = eupsPath
227  try:
228  env['PLATFORM'] = env['flavor']
229  del env['flavor']
230  except KeyError:
231  pass
232  #
233  # Check arguments
234  #
235  errorStr = ""
236  #
237  # Process otherwise unknown arguments. If setenv is true,
238  # set construction variables; otherwise generate an error
239  #
240  if SCons.Script.GetOption("setenv"):
241  for key in SCons.Script.ARGUMENTS:
242  env[key] = SCons.Script.Split(SCons.Script.ARGUMENTS[key])
243  else:
244  for key in SCons.Script.ARGUMENTS:
245  errorStr += " %s=%s" % (key, SCons.Script.ARGUMENTS[key])
246  if errorStr:
247  log.fail("Unprocessed arguments:%s" % errorStr)
248  #
249  # We need a binary name, not just "Posix"
250  #
251  env['eupsFlavor'] = eupsForScons.flavor()
252 
253 def _configureCommon():
254  """Configuration checks for the compiler, platform, and standard libraries."""
255  #
256  # Is the C compiler really gcc/g++?
257  #
258  def ClassifyCc(context):
259  """Return a pair of string identifying the compiler in use
260 
261  @return (compiler, version) as a pair of strings, or ("unknown", "unknown") if unknown
262  """
263  versionNameList = (
264  (r"gcc(?:\-.+)? +\(.+\) +([0-9.a-zA-Z]+)", "gcc"),
265  (r"LLVM +version +([0-9.a-zA-Z]+) ", "clang"), # clang on Mac
266  (r"clang +version +([0-9.a-zA-Z]+) ", "clang"), # clang on linux
267  (r"\(ICC\) +([0-9.a-zA-Z]+) ", "icc"),
268  )
269 
270  context.Message("Checking who built the CC compiler...")
271  result = context.TryAction(SCons.Script.Action(r"$CC --version > $TARGET"))
272  ccVersDumpOK, ccVersDump = result[0:2]
273  if ccVersDumpOK:
274  for reStr, compilerName in versionNameList:
275  match = re.search(reStr, ccVersDump)
276  if match:
277  compilerVersion = match.groups()[0]
278  return (compilerName, compilerVersion)
279  return ("unknown", "unknown")
280 
281  if env.GetOption("clean") or env.GetOption("no_exec") or env.GetOption("help") :
282  env.whichCc = "unknown" # who cares? We're cleaning/not execing, not building
283  else:
284  if env['cc'] != '':
285  CC = CXX = None
286  if re.search(r"^gcc(-\d+(\.\d+)*)?( |$)", env['cc']):
287  CC = env['cc']
288  CXX = re.sub(r"^gcc", "g++", CC)
289  elif re.search(r"^icc( |$)", env['cc']):
290  CC = env['cc']
291  CXX = re.sub(r"^icc", "icpc", CC)
292  elif re.search(r"^clang( |$)", env['cc']):
293  CC = env['cc']
294  CXX = re.sub(r"^clang", "clang++", CC)
295  elif re.search(r"^cc( |$)", env['cc']):
296  CC = env['cc']
297  CXX = re.sub(r"^cc", "c++", CC)
298  else:
299  log.fail("Unrecognised compiler:%s" % env['cc'])
300  env0 = SCons.Script.Environment()
301  if CC and env['CC'] == env0['CC']:
302  env['CC'] = CC
303  if CC and env['CXX'] == env0['CXX']:
304  env['CXX'] = CXX
305  conf = env.Configure(custom_tests = {'ClassifyCc' : ClassifyCc,})
306  env.whichCc, env.ccVersion = conf.ClassifyCc()
307  if not env.GetOption("no_progress"):
308  log.info("CC is %s version %s" % (env.whichCc, env.ccVersion))
309  conf.Finish()
310  #
311  # Compiler flags, including CCFLAGS for C and C++ and CXXFLAGS for C++ only
312  #
313  ARCHFLAGS = os.environ.get("ARCHFLAGS", env.get('archflags'))
314  if ARCHFLAGS:
315  env.Append(CCFLAGS = ARCHFLAGS.split())
316  env.Append(LINKFLAGS = ARCHFLAGS.split())
317  # We'll add warning and optimisation options last
318  if env['profile'] == '1' or env['profile'] == "pg":
319  env.Append(CCFLAGS = ['-pg'])
320  env.Append(LINKFLAGS = ['-pg'])
321  elif env['profile'] == 'gcov':
322  env.Append(CCFLAGS = '--coverage')
323  env.Append(LINKFLAGS = '--coverage')
324 
325  #
326  # Enable C++11 support (and C99 support for gcc)
327  #
328  if not (env.GetOption("clean") or env.GetOption("help") or env.GetOption("no_exec")):
329  if not env.GetOption("no_progress"):
330  log.info("Checking for C++11 support")
331  conf = env.Configure()
332  for cpp11Arg in ("-std=%s" % (val,) for val in ("c++11", "c++0x")):
333  conf.env = env.Clone()
334  conf.env.Append(CXXFLAGS = cpp11Arg)
335  if conf.CheckCXX():
336  env.Append(CXXFLAGS = cpp11Arg)
337  if not env.GetOption("no_progress"):
338  log.info("C++11 supported with %r" % (cpp11Arg,))
339  break
340  else:
341  log.fail("C++11 extensions could not be enabled for compiler %r" % env.whichCc)
342  conf.Finish()
343 
344  #
345  # Is C++'s TR1 available? If not, use e.g. #include "lsst/tr1/foo.h"
346  #
347  # NOTE: previously this was only checked when none of --clean, --help, and --noexec,
348  # but that was causing "no" to be cached and used on later runs.
349  if not (env.GetOption("clean") or env.GetOption("help") or env.GetOption("no_exec")):
350  conf = env.Configure()
351  env.Append(CCFLAGS = ['-DLSST_HAVE_TR1=%d' % int(conf.CheckCXXHeader("tr1/unordered_map"))])
352  conf.Finish()
353  #
354  # Byte order
355  #
356  import socket
357  if socket.htons(1) != 1:
358  env.Append(CCFLAGS = ['-DLSST_LITTLE_ENDIAN=1'])
359  #
360  # If we're linking to libraries that themselves linked to
361  # shareable libraries we need to do something special.
362  #
363  if (re.search(r"^(Linux|Linux64)$", env["eupsFlavor"]) and "LD_LIBRARY_PATH" in os.environ):
364  env.Append(LINKFLAGS = ["-Wl,-rpath-link"])
365  env.Append(LINKFLAGS = ["-Wl,%s" % os.environ["LD_LIBRARY_PATH"]])
366  #
367  # Set the optimization level.
368  #
369  if env['opt']:
370  env["CCFLAGS"] = [o for o in env["CCFLAGS"] if not re.search(r"^-O(\d|s)$", o)]
371  env.MergeFlags('-O%d' % int(env['opt']))
372  #
373  # Set compiler-specific warning flags.
374  #
375  if env.whichCc == "clang":
376  env.Append(CCFLAGS = ['-Wall'])
377  env["CCFLAGS"] = [o for o in env["CCFLAGS"] if not re.search(r"^-mno-fused-madd$", o)]
378 
379  ignoreWarnings = {
380  "unused-function" : 'boost::regex has functions in anon namespaces in headers',
381  }
382  filterWarnings = {
383  "attributes" : "clang pretends to be g++, but complains about g++ attributes such as flatten",
384  "char-subscripts" : 'seems innocous enough, and is used by boost',
385  "constant-logical-operand" : "Used by eigen 2.0.15. Should get this fixed",
386  "format-security" : "format string is not a string literal",
387  "mismatched-tags" : "mixed class and struct. Used by gcc 4.2 RTL and eigen 2.0.15",
388  "parentheses" : "equality comparison with extraneous parentheses",
389  "shorten-64-to-32" : "implicit conversion loses integer precision",
390  "self-assign" : "x = x",
391  "unknown-pragmas" : "unknown pragma ignored",
392  "deprecated-register" : "register is deprecated",
393  }
394  for k in ignoreWarnings:
395  env.Append(CCFLAGS = ["-Wno-%s" % k])
396  if env.GetOption('filterWarn'):
397  for k in filterWarnings:
398  env.Append(CCFLAGS = ["-Wno-%s" % k])
399  elif env.whichCc == "gcc":
400  env.Append(CCFLAGS = ['-Wall'])
401  env.Append(CCFLAGS = ["-Wno-unknown-pragmas"]) # we don't want complaints about icc/clang pragmas
402  elif env.whichCc == "icc":
403  env.Append(CCFLAGS = ['-Wall'])
404  filterWarnings = {
405  21 : 'type qualifiers are meaningless in this declaration',
406  68 : 'integer conversion resulted in a change of sign',
407  111 : 'statement is unreachable',
408  191 : 'type qualifier is meaningless on cast type',
409  193 : 'zero used for undefined preprocessing identifier "SYMB"',
410  279 : 'controlling expression is constant',
411  304 : 'access control not specified ("public" by default)', # comes from boost
412  383 : 'value copied to temporary, reference to temporary used',
413  #424 : 'Extra ";" ignored',
414  444 : 'destructor for base class "CLASS" is not virtual',
415  981 : 'operands are evaluated in unspecified order',
416  1418 : 'external function definition with no prior declaration',
417  1419 : 'external declaration in primary source file',
418  1572 : 'floating-point equality and inequality comparisons are unreliable',
419  1720 : 'function "FUNC" has no corresponding member operator delete (to be called if an exception is thrown during initialization of an allocated object)',
420  2259 : 'non-pointer conversion from "int" to "float" may lose significant bits',
421  }
422  if env.GetOption('filterWarn'):
423  env.Append(CCFLAGS = ["-wd%s" % (",".join([str(k) for k in filterWarnings]))])
424  # Workaround intel bug; cf. RHL's intel bug report 580167
425  env.Append(LINKFLAGS = ["-Wl,-no_compact_unwind", "-wd,11015"])
426 
427 def _saveState():
428  """Save state such as optimization level used. The scons mailing lists were unable to tell
429  RHL how to get this back from .sconsign.dblite
430  """
431 
432  if env.GetOption("clean"):
433  return
434 
435  import ConfigParser
436 
437  config = ConfigParser.ConfigParser()
438  config.add_section('Build')
439  config.set('Build', 'cc', env.whichCc)
440  if env['opt']:
441  config.set('Build', 'opt', env['opt'])
442 
443  try:
444  confFile = os.path.join(env.Dir(env["CONFIGUREDIR"]).abspath, "build.cfg")
445  with open(confFile, 'wb') as configfile:
446  config.write(configfile)
447  except Exception as e:
448  log.warn("Unexpected exception in _saveState: %s" % e)
449 
450 _initOptions()
451 _initLog()
452 _initVariables()
453 _initEnvironment()
454 _configureCommon()
455 _saveState()
456 
457 ## @endcond
A dead-simple logger for all messages.
Definition: utils.py:20