LSSTApplications  17.0+50,17.0+84,17.0+9,18.0.0+14,18.0.0+2,18.0.0+30,18.0.0+4,18.0.0+9,18.0.0-2-ge43143a+4,18.1.0-1-g0001055,18.1.0-1-g0896a44+6,18.1.0-1-g1349e88+4,18.1.0-1-g2505f39+3,18.1.0-1-g380d4d4+4,18.1.0-1-g5e4b7ea,18.1.0-1-g85f8cd4+3,18.1.0-1-g9a6769a+2,18.1.0-1-ga1a4c1a+2,18.1.0-1-gc037db8,18.1.0-1-gd55f500+1,18.1.0-1-ge10677a+3,18.1.0-10-g73b8679e+7,18.1.0-11-g311e899+3,18.1.0-12-g0d73a3591,18.1.0-12-gc95f69a+3,18.1.0-2-g000ad9a+3,18.1.0-2-g31c43f9+3,18.1.0-2-g9c63283+4,18.1.0-2-gdf0b915+4,18.1.0-2-gf03bb23,18.1.0-3-g2e29e3d+6,18.1.0-3-g52aa583+2,18.1.0-3-g9cb968e+3,18.1.0-4-gd2e8982+6,18.1.0-5-g510c42a+3,18.1.0-5-gaeab27e+4,18.1.0-6-gdda7f3e+6,18.1.0-7-g89824ecc+4,w.2019.32
LSSTDataManagementBasePackage
Classes | Functions | Variables
lsst.ctrl.pool.parallel Namespace Reference

Classes

class  Batch
 
class  BatchArgumentParser
 
class  BatchCmdLineTask
 
class  BatchParallelTask
 
class  BatchPoolTask
 
class  BatchTaskRunner
 
class  PbsBatch
 
class  SlurmBatch
 
class  SmpBatch
 

Functions

def shQuote (arg)
 
def shCommandFromArgs (args)
 
def processStats ()
 
def printProcessStats ()
 
def exportEnv ()
 

Variables

int UMASK = 0o002
 
dictionary BATCH_TYPES
 

Function Documentation

◆ exportEnv()

def lsst.ctrl.pool.parallel.exportEnv ( )
Generate bash script to regenerate the current environment

Definition at line 407 of file parallel.py.

407 def exportEnv():
408  """Generate bash script to regenerate the current environment"""
409  output = ""
410  for key, val in os.environ.items():
411  if key in ("DISPLAY",):
412  continue
413  if val.startswith("() {"):
414  # This is a function.
415  # "Two parentheses, a single space, and a brace"
416  # is exactly the same criterion as bash uses.
417 
418  # From 2014-09-25, the function name is prefixed by 'BASH_FUNC_'
419  # and suffixed by '()', which we have to remove.
420  if key.startswith("BASH_FUNC_") and key.endswith("()"):
421  key = key[10:-2]
422 
423  output += "{key} {val}\nexport -f {key}\n".format(key=key, val=val)
424  else:
425  # This is a variable.
426  output += "export {key}='{val}'\n".format(key=key, val=val.replace("'", "'\"'\"'"))
427  return output
428 
429 
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168

◆ printProcessStats()

def lsst.ctrl.pool.parallel.printProcessStats ( )
Print the process statistics to the log

Definition at line 61 of file parallel.py.

61 def printProcessStats():
62  """Print the process statistics to the log"""
63  from lsst.log import Log
64  log = Log.getDefaultLogger()
65  log.info("Process stats for %s: %s" % (NODE, processStats()))
66 
67 
Definition: Log.h:691

◆ processStats()

def lsst.ctrl.pool.parallel.processStats ( )
Collect Linux-specific process statistics

Parses the /proc/self/status file (N.B. Linux-specific!) into a dict
which is returned.

Definition at line 47 of file parallel.py.

47 def processStats():
48  """Collect Linux-specific process statistics
49 
50  Parses the /proc/self/status file (N.B. Linux-specific!) into a dict
51  which is returned.
52  """
53  result = {}
54  with open("/proc/self/status") as f:
55  for line in f:
56  key, _, value = line.partition(":")
57  result[key] = value.strip()
58  return result
59 
60 

◆ shCommandFromArgs()

def lsst.ctrl.pool.parallel.shCommandFromArgs (   args)
Convert a list of shell arguments to a shell command-line

Definition at line 42 of file parallel.py.

42 def shCommandFromArgs(args):
43  """Convert a list of shell arguments to a shell command-line"""
44  return ' '.join([shQuote(a) for a in args])
45 
46 
def shCommandFromArgs(args)
Definition: parallel.py:42

◆ shQuote()

def lsst.ctrl.pool.parallel.shQuote (   arg)
Quote the argument for the shell.

>>> quote('\t')
'\\\t'
>>> quote('foo bar')
'foo\\ bar'

Definition at line 27 of file parallel.py.

27 def shQuote(arg):
28  r"""Quote the argument for the shell.
29 
30  >>> quote('\t')
31  '\\\t'
32  >>> quote('foo bar')
33  'foo\\ bar'
34  """
35  # This is the logic emacs uses
36  if arg:
37  return _quote_pos.sub('\\\\', arg).replace('\n', "'\n'")
38  else:
39  return "''"
40 
41 

Variable Documentation

◆ BATCH_TYPES

dictionary lsst.ctrl.pool.parallel.BATCH_TYPES
Initial value:
1 = {'none': None,
2  'None': None,
3  'pbs': PbsBatch,
4  'slurm': SlurmBatch,
5  'smp': SmpBatch,
6  }

Definition at line 290 of file parallel.py.

◆ UMASK

int lsst.ctrl.pool.parallel.UMASK = 0o002

Definition at line 20 of file parallel.py.