lsst.ctrl.pool.parallel.BatchPoolTask Class Reference

Inheritance diagram for lsst.ctrl.pool.parallel.BatchPoolTask:

Public Member Functions
def	parseAndRun (cls, *args, **kwargs)
def	parseAndSubmit (cls, args=None, **kwargs)
def	batchWallTime (cls, time, parsedCmd, numCores)
	Return walltime request for batch job. More...
def	batchCommand (cls, args)
	Return command to run CmdLineTask. More...
def	logOperation (self, operation, catch=False, trace=True)
	Provide a context manager for logging an operation. More...

Detailed Description

Starts a BatchCmdLineTask with an MPI process pool

Use this subclass of BatchCmdLineTask if you want to use the Pool directly.

Definition at line 527 of file parallel.py.

Member Function Documentation

batchCommand()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.batchCommand (   cls,   args  )

inherited

Return command to run CmdLineTask.

    @param cls: Class
    @param args: Parsed batch job arguments (from BatchArgumentParser)

Definition at line 476 of file parallel.py.

    def batchCommand(cls, args):
        """!Return command to run CmdLineTask
 
        @param cls: Class
        @param args: Parsed batch job arguments (from BatchArgumentParser)
        """
        job = args.job if args.job is not None else "job"
        module = cls.__module__
        script = ("import os; os.umask(%#05o); " +
                  "import lsst.base; lsst.base.disableImplicitThreading(); " +
                  "import lsst.ctrl.pool.log; lsst.ctrl.pool.log.jobLog(\"%s\"); ") % (UMASK, job)
 
        if args.batchStats:
            script += ("import lsst.ctrl.pool.parallel; import atexit; " +
                       "atexit.register(lsst.ctrl.pool.parallel.printProcessStats); ")
 
        script += "import %s; %s.%s.parseAndRun();" % (module, module, cls.__name__)
 
        profilePre = "import cProfile; import os; cProfile.run(\"\"\""
        profilePost = "\"\"\", filename=\"profile-" + job + "-%s-%d.dat\" % (os.uname()[1], os.getpid()))"
 
        return ("python -c '" + (profilePre if args.batchProfile else "") + script +
                (profilePost if args.batchProfile else "") + "' " + shCommandFromArgs(args.leftover) +
                " --noExit")
 

batchWallTime()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.batchWallTime (   cls,   time,   parsedCmd,   numCores  )

inherited

Return walltime request for batch job.

    Subclasses should override if the walltime should be calculated
    differently (e.g., addition of some serial time).

    @param cls: Class
    @param time: Requested time per iteration
    @param parsedCmd: Results of argument parsing
    @param numCores: Number of cores

Reimplemented in lsst.pipe.drivers.multiBandDriver.MultiBandDriverTask, lsst.pipe.drivers.visualizeVisit.VisualizeVisitTask, lsst.pipe.drivers.skyCorrection.SkyCorrectionTask, lsst.pipe.drivers.ingestDriver.PoolIngestTask, lsst.pipe.drivers.constructCalibs.CalibTask, lsst.pipe.drivers.coaddDriver.CoaddDriverTask, and lsst.ctrl.pool.test.demoTask.DemoTask.

Definition at line 461 of file parallel.py.

    def batchWallTime(cls, time, parsedCmd, numCores):
        """!Return walltime request for batch job
 
        Subclasses should override if the walltime should be calculated
        differently (e.g., addition of some serial time).
 
        @param cls: Class
        @param time: Requested time per iteration
        @param parsedCmd: Results of argument parsing
        @param numCores: Number of cores
        """
        numTargets = len(cls.RunnerClass.getTargetList(parsedCmd))
        return time*numTargets/float(numCores)
 

logOperation()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.logOperation (   self,   operation,   catch = False,   trace = True  )

inherited

Provide a context manager for logging an operation.

    @param operation: description of operation (string)
    @param catch: Catch all exceptions?
    @param trace: Log a traceback of caught exception?

    Note that if 'catch' is True, all exceptions are swallowed, but there may
    be other side-effects such as undefined variables.

Definition at line 502 of file parallel.py.

    def logOperation(self, operation, catch=False, trace=True):
        """!Provide a context manager for logging an operation
 
        @param operation: description of operation (string)
        @param catch: Catch all exceptions?
        @param trace: Log a traceback of caught exception?
 
        Note that if 'catch' is True, all exceptions are swallowed, but there may
        be other side-effects such as undefined variables.
        """
        self.log.info("%s: Start %s" % (NODE, operation))
        try:
            yield
        except Exception:
            if catch:
                cls, e, _ = sys.exc_info()
                self.log.warn("%s: Caught %s while %s: %s" % (NODE, cls.__name__, operation, e))
                if trace:
                    self.log.info("%s: Traceback:\n%s" % (NODE, traceback.format_exc()))
                return
            raise
        finally:
            self.log.info("%s: Finished %s" % (NODE, operation))
 
 

parseAndRun()

def lsst.ctrl.pool.parallel.BatchPoolTask.parseAndRun	(		cls,
		*	args,
		**	kwargs
	)

Run with a MPI process pool

Definition at line 534 of file parallel.py.

    def parseAndRun(cls, *args, **kwargs):
        """Run with a MPI process pool"""
        pool = startPool()
        super(BatchPoolTask, cls).parseAndRun(*args, **kwargs)
        pool.exit()
 
 

parseAndSubmit()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.parseAndSubmit (   cls,   args = None, **  kwargs  )

inherited

Definition at line 435 of file parallel.py.

    def parseAndSubmit(cls, args=None, **kwargs):
        taskParser = cls._makeArgumentParser(doBatch=True, add_help=False)
        batchParser = BatchArgumentParser(parent=taskParser)
        batchArgs = batchParser.parse_args(config=cls.ConfigClass(), args=args, override=cls.applyOverrides,
                                           **kwargs)
 
        if not cls.RunnerClass(cls, batchArgs.parent).precall(batchArgs.parent):  # Write config, schema
            taskParser.error("Error in task preparation")
 
        setBatchType(batchArgs.batch)
 
        if batchArgs.batch is None:     # don't use a batch system
            sys.argv = [sys.argv[0]] + batchArgs.leftover  # Remove all batch arguments
 
            return cls.parseAndRun()
        else:
            if batchArgs.walltime > 0:
                walltime = batchArgs.walltime
            else:
                numCores = batchArgs.cores if batchArgs.cores > 0 else batchArgs.nodes*batchArgs.procs
                walltime = cls.batchWallTime(batchArgs.time, batchArgs.parent, numCores)
 
            command = cls.batchCommand(batchArgs)
            batchArgs.batch.run(command, walltime=walltime)
 

---

The documentation for this class was generated from the following file:

• /j/snowflake/release/lsstsw/stack/lsst-scipipe-0.7.0/Linux64/ctrl_pool/22.0.1-2-g92698f7+dcf3732eb2/python/lsst/ctrl/pool/parallel.py