scripts.py

Go to the documentation of this file.

import os.path
import re
import pipes
from stat import ST_MODE
from SCons.Script import SConscript, File, Dir, Glob, BUILD_TARGETS
from distutils.spawn import find_executable

from . import dependencies
from . import state
from . import tests
from . import utils

DEFAULT_TARGETS = ("lib", "python", "shebang", "tests", "examples", "doc")


def _getFileBase(node):
    name, ext = os.path.splitext(os.path.basename(str(node)))
    return name



class BasicSConstruct:

    _initializing = False

    
    def __new__(cls, packageName, versionString=None, eupsProduct=None, eupsProductPath=None, cleanExt=None,
                defaultTargets=DEFAULT_TARGETS,
                subDirList=None, ignoreRegex=None,
                versionModuleName="python/lsst/%s/version.py", noCfgFile=False,
                sconscriptOrder=None, disableCc=False):
        cls.initialize(packageName, versionString, eupsProduct, eupsProductPath, cleanExt,
                       versionModuleName, noCfgFile=noCfgFile, sconscriptOrder=sconscriptOrder,
                       disableCc=disableCc)
        cls.finish(defaultTargets, subDirList, ignoreRegex)
        return state.env

    
    @classmethod
    def initialize(cls, packageName, versionString=None, eupsProduct=None, eupsProductPath=None,
                   cleanExt=None, versionModuleName="python/lsst/%s/version.py", noCfgFile=False,
                   sconscriptOrder=None, disableCc=False):
        if not disableCc:
            state._configureCommon()
            state._saveState()
        if cls._initializing:
            state.log.fail("Recursion detected; an SConscript file should not call BasicSConstruct.")
        cls._initializing = True
        dependencies.configure(packageName, versionString, eupsProduct, eupsProductPath, noCfgFile)
        state.env.BuildETags()
        if cleanExt is None:
            cleanExt = r"*~ core core.[1-9]* *.so *.os *.o *.pyc *.pkgc"
        state.env.CleanTree(cleanExt, ".cache __pycache__ .pytest_cache")
        if versionModuleName is not None:
            try:
                versionModuleName = versionModuleName % "/".join(packageName.split("_"))
            except TypeError:
                pass
            state.targets["version"] = state.env.VersionModule(versionModuleName)
        scripts = []
        for root, dirs, files in os.walk("."):
            if "SConstruct" in files and root != ".":
                dirs[:] = []
                continue
            dirs[:] = [d for d in dirs if not d.startswith('.')]
            dirs.sort()  # os.walk order is not specified, but we want builds to be deterministic
            if "SConscript" in files:
                scripts.append(os.path.join(root, "SConscript"))
        if sconscriptOrder is None:
            sconscriptOrder = DEFAULT_TARGETS

        # directory for shebang target is bin.src
        sconscriptOrder = [t if t != "shebang" else "bin.src" for t in sconscriptOrder]

        def key(path):
            for i, item in enumerate(sconscriptOrder):
                if path.lstrip("./").startswith(item):
                    return i
            return len(sconscriptOrder)
        scripts.sort(key=key)
        for script in scripts:
            state.log.info("Using SConscript at %s" % script)
            SConscript(script)
        cls._initializing = False
        return state.env

    
    @staticmethod
    def finish(defaultTargets=DEFAULT_TARGETS,
               subDirList=None, ignoreRegex=None):
        if ignoreRegex is None:
            ignoreRegex = r"(~$|\.pyc$|^\.svn$|\.o|\.os$)"
        if subDirList is None:
            subDirList = []
            for path in os.listdir("."):
                if os.path.isdir(path) and not path.startswith("."):
                    subDirList.append(path)
        install = state.env.InstallLSST(state.env["prefix"],
                                        [subDir for subDir in subDirList],
                                        ignoreRegex=ignoreRegex)
        for name, target in state.targets.items():
            state.env.Requires(install, target)
            state.env.Alias(name, target)
        state.env.Requires(state.targets["python"], state.targets["version"])
        declarer = state.env.Declare()
        state.env.Requires(declarer, install)  # Ensure declaration fires after installation available

        # shebang should be in the list if bin.src exists but the location matters
        # so we can not append it afterwards.
        state.env.Default([t for t in defaultTargets
                           if os.path.exists(t) or (t == "shebang" and os.path.exists("bin.src"))])
        if "version" in state.targets:
            state.env.Default(state.targets["version"])
        state.env.Requires(state.targets["tests"], state.targets["version"])
        state.env.Decider("MD5-timestamp")  # if timestamps haven't changed, don't do MD5 checks
        #
        # Check if any of the tests failed by looking for *.failed files.
        # Perform this test just before scons exits
        #
        # N.b. the test is written in sh not python as then we can use @ to suppress output
        #
        if "tests" in [str(t) for t in BUILD_TARGETS]:
            testsDir = pipes.quote(os.path.join(os.getcwd(), "tests", ".tests"))
            checkTestStatus_command = state.env.Command('checkTestStatus', [], """
                @ if [ -d {0} ]; then \
                      nfail=`find {0} -name "*.failed" | wc -l | sed -e 's/ //g'`; \
                      if [ $$nfail -gt 0 ]; then \
                          echo "Failed test output:" >&2; \
                          for f in `find {0} -name "*.failed"`; do \
                              case "$$f" in \
                              *.xml.failed) \
                                echo "Global pytest output is in $$f" >&2; \
                                ;; \
                              *.failed) \
                                cat $$f >&2; \
                                ;; \
                              esac; \
                          done; \
                          echo "The following tests failed:" >&2;\
                          find {0} -name "*.failed" >&2; \
                          echo "$$nfail tests failed" >&2; exit 1; \
                      fi; \
                  fi; \
            """.format(testsDir))

            state.env.Depends(checkTestStatus_command, BUILD_TARGETS)  # this is why the check runs last
            BUILD_TARGETS.extend(checkTestStatus_command)
            state.env.AlwaysBuild(checkTestStatus_command)



class BasicSConscript:

    
    @staticmethod
    def lib(libName=None, src=None, libs="self", noBuildList=None):
        if libName is None:
            libName = state.env["packageName"]
        if src is None:
            src = Glob("#src/*.cc") + Glob("#src/*/*.cc") + Glob("#src/*/*/*.cc") + Glob("#src/*/*/*/*.cc")
        if noBuildList is not None:
            src = [node for node in src if os.path.basename(str(node)) not in noBuildList]
        src = state.env.SourcesForSharedLibrary(src)
        if isinstance(libs, str):
            libs = state.env.getLibs(libs)
        elif libs is None:
            libs = []
        result = state.env.SharedLibrary(libName, src, LIBS=libs)
        state.targets["lib"].extend(result)
        return result

    
    @staticmethod
    def shebang(src=None):
        # check if Python is called on the first line with this expression
        # This comes from distutils copy_scripts
        FIRST_LINE_RE = re.compile(r'^#!.*python[0-9.]*([ \t].*)?$')
        doRewrite = utils.needShebangRewrite()

        def rewrite_shebang(target, source, env):
            """Copy source to target, rewriting the shebang"""
            # Currently just use this python
            usepython = utils.whichPython()
            for targ, src in zip(target, source):
                with open(str(src), "r") as srcfd:
                    with open(str(targ), "w") as outfd:
                        first_line = srcfd.readline()
                        # Always match the first line so we can warn people
                        # if an attempt is being made to rewrite a file that should
                        # not be rewritten
                        match = FIRST_LINE_RE.match(first_line)
                        if match and doRewrite:
                            post_interp = match.group(1) or ''
                            # Paths can be long so ensure that flake8 won't complain
                            outfd.write("#!{}{}  # noqa\n".format(usepython, post_interp))
                        else:
                            if not match:
                                state.log.warn("Could not rewrite shebang of {}. Please check"
                                               " file or move it to bin directory.".format(str(src)))
                            outfd.write(first_line)
                        for line in srcfd.readlines():
                            outfd.write(line)
                # Ensure the bin/ file is executable
                oldmode = os.stat(str(targ))[ST_MODE] & 0o7777
                newmode = (oldmode | 0o555) & 0o7777
                if newmode != oldmode:
                    state.log.info("changing mode of {} from {} to {}".format(
                                   str(targ), oldmode, newmode))
                    os.chmod(str(targ), newmode)

        if src is None:
            src = Glob("#bin.src/*")
        for s in src:
            filename = str(s)
            # Do not try to rewrite files starting with non-letters
            if filename != "SConscript" and re.match("[A-Za-z]", filename):
                result = state.env.Command(target=os.path.join(Dir("#bin").abspath, filename),
                                           source=s, action=rewrite_shebang)
                state.targets["shebang"].extend(result)

    
    @staticmethod
    def python(module=None, src=None, extra=(), libs="main python"):
        if module is None:
            module = "_" + state.env["packageName"].split("_")[-1]
        if src is None:
            src = state.env.Glob("*.cc")
        src.extend(extra)
        if isinstance(libs, str):
            libs = state.env.getLibs(libs)
        elif libs is None:
            libs = []
        result = state.env.Pybind11LoadableModule(module, src, LIBS=libs)
        state.targets["python"].append(result)
        return result

    
    @staticmethod
    def pybind11(nameList=[], libs="main python", extraSrc=None, addUnderscore=True):
        srcList = extraSrc
        if srcList is None:
            srcList = dict([(name, []) for name in nameList])
        for name in nameList:
            srcList[name].append(name + ".cc")
        if isinstance(libs, str):
            libs = state.env.getLibs(libs)
        elif libs is None:
            libs = []
        result = []
        for name in nameList:
            # TODO remove this block and the `addUnderscore` argument and always use pyLibName = name;
            # but we can't do that until all our pybind11 SConscript files have been converted
            if addUnderscore:
                if name.startswith("_"):
                    pyLibName = name
                else:
                    pyLibName = "_" + name
            else:
                pyLibName = name
            result.extend(state.env.Pybind11LoadableModule(pyLibName, srcList[name], LIBS=libs))
        state.targets["python"].extend(result)
        return result

    
    @staticmethod
    def doc(config="doxygen.conf.in", projectName=None, projectNumber=None, **kw):
        if not find_executable("doxygen"):
            state.log.warn("doxygen executable not found; skipping documentation build.")
            return []
        if projectName is None:
            projectName = ".".join(["lsst"] + state.env["packageName"].split("_"))
        if projectNumber is None:
            projectNumber = state.env["version"]
        result = state.env.Doxygen(
            config, projectName=projectName, projectNumber=projectNumber,
            includes=state.env.doxygen["includes"],
            useTags=state.env.doxygen["tags"],
            makeTag=(state.env["packageName"] + ".tag"),
            **kw
        )
        state.targets["doc"].extend(result)
        return result

    
    @staticmethod
    def tests(pyList=None, ccList=None, swigNameList=None, swigSrc=None,
              ignoreList=None, noBuildList=None, pySingles=None,
              args=None):
        if noBuildList is None:
            noBuildList = []
        if pySingles is None:
            pySingles = []
        if swigNameList is None:
            swigFileList = Glob("*.i")
            swigNameList = [_getFileBase(node) for node in swigFileList]
        else:
            swigFileList = [File(name + ".i") for name in swigNameList]
        if swigSrc is None:
            swigSrc = {}
        allSwigSrc = set()
        for name, node in zip(swigNameList, swigFileList):
            src = swigSrc.setdefault(name, [])
            allSwigSrc.update(str(element) for element in src)
            src.append(node)
        if pyList is None:
            pyList = [node for node in Glob("*.py")
                      if _getFileBase(node) not in swigNameList and
                      os.path.basename(str(node)) not in noBuildList]
            # if we got no matches, reset to None so we do not enabled
            # auto test detection in pytest
            if not pyList:
                pyList = None
        if ccList is None:
            ccList = [node for node in Glob("*.cc")
                      if (not str(node).endswith("_wrap.cc")) and str(node) not in allSwigSrc and
                      os.path.basename(str(node)) not in noBuildList]
        if ignoreList is None:
            ignoreList = []

        def s(l):
            if l is None:
                return ['None']
            return [str(i) for i in l]

        state.log.info("SWIG modules for tests: %s" % s(swigFileList))
        state.log.info("Python tests: %s" % s(pyList))
        state.log.info("C++ tests: %s" % s(ccList))
        state.log.info("Files that will not be built: %s" % noBuildList)
        state.log.info("Ignored tests: %s" % ignoreList)
        control = tests.Control(state.env, ignoreList=ignoreList, args=args, verbose=True)
        for ccTest in ccList:
            state.env.Program(ccTest, LIBS=state.env.getLibs("main test"))
        swigMods = []
        for name, src in swigSrc.items():
            swigMods.extend(
                state.env.SwigLoadableModule("_" + name, src, LIBS=state.env.getLibs("main python"))
            )

        # Warn about insisting that a test in pySingles starts with test_ and
        # therefore might be automatically discovered by pytest. These files
        # should not be discovered automatically.
        for node in pySingles:
            if str(node).startswith("test_"):
                state.log.warn("Warning: {} should be run independently but"
                               " can be automatically discovered".format(node))

        # Ensure that python tests listed in pySingles are not included in pyList.
        if pyList is not None:
            pyList = [str(node) for node in pyList if str(node) not in pySingles]

        ccList = [control.run(str(node)) for node in ccList]
        pySingles = [control.run(str(node)) for node in pySingles]

        # If we tried to discover .py files and found none, do not then
        # try to use auto test discovery.
        if pyList is not None:
            pyList = [control.runPythonTests(pyList)]
        else:
            pyList = []
        pyList.extend(pySingles)
        for pyTest in pyList:
            state.env.Depends(pyTest, ccList)
            state.env.Depends(pyTest, swigMods)
            state.env.Depends(pyTest, state.targets["python"])
            state.env.Depends(pyTest, state.targets["shebang"])
        result = ccList + pyList
        state.targets["tests"].extend(result)
        return result

    
    @staticmethod
    def examples(ccList=None, swigNameList=None, swigSrc=None):
        if swigNameList is None:
            swigFileList = Glob("*.i")
            swigNameList = [_getFileBase(node) for node in swigFileList]
        else:
            swigFileList = [File(name) for name in swigNameList]
        if swigSrc is None:
            swigSrc = {}
        allSwigSrc = set()
        for name, node in zip(swigNameList, swigFileList):
            src = swigSrc.setdefault(name, [])
            allSwigSrc.update(str(element) for element in src)
            src.append(node)
        if ccList is None:
            ccList = [node for node in Glob("*.cc")
                      if (not str(node).endswith("_wrap.cc")) and str(node) not in allSwigSrc]
        state.log.info("SWIG modules for examples: %s" % swigFileList)
        state.log.info("C++ examples: %s" % ccList)
        results = []
        for src in ccList:
            results.extend(state.env.Program(src, LIBS=state.env.getLibs("main")))
        for name, src in swigSrc.items():
            results.extend(
                state.env.SwigLoadableModule("_" + name, src, LIBS=state.env.getLibs("main python"))
            )
        for result in results:
            state.env.Depends(result, state.targets["lib"])
        state.targets["examples"].extend(results)
        return results