lsst.ctrl.pool.parallel Namespace Reference

Classes
class	Batch
class	BatchArgumentParser
class	BatchCmdLineTask
class	BatchParallelTask
class	BatchPoolTask
class	BatchTaskRunner
class	PbsBatch
class	SlurmBatch
class	SmpBatch

Functions
def	shQuote (arg)
def	shCommandFromArgs (args)
def	processStats ()
def	printProcessStats ()
def	exportEnv ()

Variables
int	UMASK = 0o002
dictionary	BATCH_TYPES

Function Documentation

exportEnv()

def lsst.ctrl.pool.parallel.exportEnv

(

)

Generate bash script to regenerate the current environment

Definition at line 409 of file parallel.py.

def exportEnv():
    """Generate bash script to regenerate the current environment"""
    output = ""
    for key, val in os.environ.items():
        if key in ("DISPLAY",):
            continue
        if val.startswith("() {"):
            # This is a function.
            # "Two parentheses, a single space, and a brace"
            # is exactly the same criterion as bash uses.
 
            # From 2014-09-25, the function name is prefixed by 'BASH_FUNC_'
            # and suffixed by '()', which we have to remove.
            if key.startswith("BASH_FUNC_") and key.endswith("()"):
                key = key[10:-2]
 
            output += "{key} {val}\nexport -f {key}\n".format(key=key, val=val)
        else:
            # This is a variable.
            output += "export {key}='{val}'\n".format(key=key, val=val.replace("'", "'\"'\"'"))
    return output
 
 

printProcessStats()

def lsst.ctrl.pool.parallel.printProcessStats

(

)

Print the process statistics to the log

Definition at line 61 of file parallel.py.

def printProcessStats():
    """Print the process statistics to the log"""
    from lsst.log import Log
    log = Log.getDefaultLogger()
    log.info("Process stats for %s: %s" % (NODE, processStats()))
 
 

processStats()

def lsst.ctrl.pool.parallel.processStats

(

)

Collect Linux-specific process statistics

Parses the /proc/self/status file (N.B. Linux-specific!) into a dict
which is returned.

Definition at line 47 of file parallel.py.

def processStats():
    """Collect Linux-specific process statistics
 
    Parses the /proc/self/status file (N.B. Linux-specific!) into a dict
    which is returned.
    """
    result = {}
    with open("/proc/self/status") as f:
        for line in f:
            key, _, value = line.partition(":")
            result[key] = value.strip()
    return result
 
 

shCommandFromArgs()

def lsst.ctrl.pool.parallel.shCommandFromArgs

(

args

)

Convert a list of shell arguments to a shell command-line

Definition at line 42 of file parallel.py.

def shCommandFromArgs(args):
    """Convert a list of shell arguments to a shell command-line"""
    return ' '.join([shQuote(a) for a in args])
 
 

shQuote()

def lsst.ctrl.pool.parallel.shQuote

(

arg

)

Quote the argument for the shell.

>>> quote('\t')
'\\\t'
>>> quote('foo bar')
'foo\\ bar'

Definition at line 27 of file parallel.py.

def shQuote(arg):
    r"""Quote the argument for the shell.
 
    >>> quote('\t')
    '\\\t'
    >>> quote('foo bar')
    'foo\\ bar'
    """
    # This is the logic emacs uses
    if arg:
        return _quote_pos.sub('\\\\', arg).replace('\n', "'\n'")
    else:
        return "''"
 
 

Variable Documentation

BATCH_TYPES

dictionary lsst.ctrl.pool.parallel.BATCH_TYPES

Initial value:

=  {'none': None,
               'None': None,
               'pbs': PbsBatch,
               'slurm': SlurmBatch,
               'smp': SmpBatch,
               }

Definition at line 290 of file parallel.py.

UMASK

int lsst.ctrl.pool.parallel.UMASK = 0o002

Definition at line 20 of file parallel.py.