lsst.pipe.drivers.constructCalibs.FlatTask Class Reference

Inheritance diagram for lsst.pipe.drivers.constructCalibs.FlatTask:

Public Member Functions
def	applyOverrides (cls, config)
def	__init__ (self, *args, **kwargs)
def	processResult (self, exposure)
def	scale (self, ccdIdLists, data)
def	batchWallTime (cls, time, parsedCmd, numCores)
	Return walltime request for batch job. More...
def	runDataRef (self, expRefList, butler, calibId)
	Construct a calib from a list of exposure references. More...
def	getOutputId (self, expRefList, calibId)
	Generate the data identifier for the output calib. More...
def	getMjd (self, butler, dataId, timescale=dafBase.DateTime.UTC)
def	getFilter (self, butler, dataId)
def	addMissingKeys (self, dataId, butler, missingKeys=None, calibName=None)
def	updateMetadata (self, calibImage, exposureTime, darkTime=None, **kwargs)
	Update the metadata from the VisitInfo. More...
def	scatterProcess (self, pool, ccdIdLists)
	Scatter the processing among the nodes. More...
def	process (self, cache, ccdId, outputName="postISRCCD", **kwargs)
	Process a CCD, specified by a data identifier. More...
def	processSingle (self, dataRef)
def	processWrite (self, dataRef, exposure, outputName="postISRCCD")
	Write the processed CCD. More...
def	scatterCombine (self, pool, outputId, ccdIdLists, scales)
	Scatter the combination of exposures across multiple nodes. More...
def	getFullyQualifiedOutputId (self, ccdName, butler, outputId)
def	combine (self, cache, struct, outputId)
	Combine multiple exposures of a particular CCD and write the output. More...
def	calculateOutputHeaderFromRaws (self, butler, calib, dataIdList, outputId)
	Calculate the output header from the raw headers. More...
def	recordCalibInputs (self, butler, calib, dataIdList, outputId)
	Record metadata including the inputs and creation details. More...
def	interpolateNans (self, image)
def	write (self, butler, exposure, dataId)
	Write the final combined calib. More...
def	makeCameraImage (self, camera, dataId, calibs)
	Create and write an image of the entire camera. More...
def	checkCcdIdLists (self, ccdIdLists)
def	parseAndRun (cls, *args, **kwargs)
def	parseAndSubmit (cls, args=None, **kwargs)
def	batchCommand (cls, args)
	Return command to run CmdLineTask. More...
def	logOperation (self, operation, catch=False, trace=True)
	Provide a context manager for logging an operation. More...

Static Public Attributes
	ConfigClass = FlatConfig
string	calibName = "flat"
	RunnerClass = CalibTaskRunner
	filterName = None
float	exposureTime = 1.0

Detailed Description

Flat construction

The principal change from the base class involves gathering the background
values from each image and using them to determine the scalings for the final
combination.

Definition at line 1018 of file constructCalibs.py.

Constructor & Destructor Documentation

__init__()

def lsst.pipe.drivers.constructCalibs.FlatTask.__init__	(		self,
		*	args,
		**	kwargs
	)

Constructor

Reimplemented from lsst.pipe.drivers.constructCalibs.CalibTask.

Definition at line 1035 of file constructCalibs.py.

    def __init__(self, *args, **kwargs):
        CalibTask.__init__(self, *args, **kwargs)
        self.makeSubtask("stats")
 

Member Function Documentation

addMissingKeys()

def lsst.pipe.drivers.constructCalibs.CalibTask.addMissingKeys (   self,   dataId,   butler,   missingKeys = None,   calibName = None  )

inherited

Definition at line 549 of file constructCalibs.py.

    def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None):
        if calibName is None:
            calibName = self.calibName
 
        if missingKeys is None:
            missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
 
        for k in missingKeys:
            try:
                v = butler.queryMetadata('raw', [k], dataId)  # n.b. --id refers to 'raw'
            except Exception:
                continue
 
            if len(v) == 0:         # failed to lookup value
                continue
 
            if len(v) == 1:
                dataId[k] = v[0]
            else:
                raise RuntimeError("No unique lookup for %s: %s" % (k, v))
 

applyOverrides()

def lsst.pipe.drivers.constructCalibs.FlatTask.applyOverrides	(		cls,
			config
	)

Overrides for flat construction

Definition at line 1030 of file constructCalibs.py.

    def applyOverrides(cls, config):
        """Overrides for flat construction"""
        config.isr.doFlat = False
        config.isr.doFringe = False
 

batchCommand()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.batchCommand (   cls,   args  )

inherited

Return command to run CmdLineTask.

    @param cls: Class
    @param args: Parsed batch job arguments (from BatchArgumentParser)

Definition at line 476 of file parallel.py.

    def batchCommand(cls, args):
        """!Return command to run CmdLineTask
 
        @param cls: Class
        @param args: Parsed batch job arguments (from BatchArgumentParser)
        """
        job = args.job if args.job is not None else "job"
        module = cls.__module__
        script = ("import os; os.umask(%#05o); " +
                  "import lsst.base; lsst.base.disableImplicitThreading(); " +
                  "import lsst.ctrl.pool.log; lsst.ctrl.pool.log.jobLog(\"%s\"); ") % (UMASK, job)
 
        if args.batchStats:
            script += ("import lsst.ctrl.pool.parallel; import atexit; " +
                       "atexit.register(lsst.ctrl.pool.parallel.printProcessStats); ")
 
        script += "import %s; %s.%s.parseAndRun();" % (module, module, cls.__name__)
 
        profilePre = "import cProfile; import os; cProfile.run(\"\"\""
        profilePost = "\"\"\", filename=\"profile-" + job + "-%s-%d.dat\" % (os.uname()[1], os.getpid()))"
 
        return ("python -c '" + (profilePre if args.batchProfile else "") + script +
                (profilePost if args.batchProfile else "") + "' " + shCommandFromArgs(args.leftover) +
                " --noExit")
 

batchWallTime()

def lsst.pipe.drivers.constructCalibs.CalibTask.batchWallTime (   cls,   time,   parsedCmd,   numCores  )

inherited

Return walltime request for batch job.

    Subclasses should override if the walltime should be calculated
    differently (e.g., addition of some serial time).

    @param cls: Class
    @param time: Requested time per iteration
    @param parsedCmd: Results of argument parsing
    @param numCores: Number of cores

Reimplemented from lsst.ctrl.pool.parallel.BatchCmdLineTask.

Definition at line 410 of file constructCalibs.py.

    def batchWallTime(cls, time, parsedCmd, numCores):
        numCcds = len(parsedCmd.butler.get("camera"))
        numExps = len(cls.RunnerClass.getTargetList(
            parsedCmd)[0]['expRefList'])
        numCycles = int(numCcds/float(numCores) + 0.5)
        return time*numExps*numCycles
 

calculateOutputHeaderFromRaws()

def lsst.pipe.drivers.constructCalibs.CalibTask.calculateOutputHeaderFromRaws (   self,   butler,   calib,   dataIdList,   outputId  )

inherited

Calculate the output header from the raw headers.

    This metadata will go into the output FITS header. It will include all
    headers that are identical in all inputs.

    @param butler  Data butler
    @param calib  Combined calib exposure.
    @param dataIdList  List of data identifiers for calibration inputs
    @param outputId  Data identifier for output

Definition at line 766 of file constructCalibs.py.

    def calculateOutputHeaderFromRaws(self, butler, calib, dataIdList, outputId):
        """!Calculate the output header from the raw headers.
 
        This metadata will go into the output FITS header. It will include all
        headers that are identical in all inputs.
 
        @param butler  Data butler
        @param calib  Combined calib exposure.
        @param dataIdList  List of data identifiers for calibration inputs
        @param outputId  Data identifier for output
        """
        header = calib.getMetadata()
 
        rawmd = [butler.get("raw_md", dataId) for dataId in dataIdList if
                 dataId is not None]
 
        merged = merge_headers(rawmd, mode="drop")
 
        # Place merged set into the PropertyList if a value is not
        # present already
        # Comments are not present in the merged version so copy them across
        for k, v in merged.items():
            if k not in header:
                comment = rawmd[0].getComment(k) if k in rawmd[0] else None
                header.set(k, v, comment=comment)
 
        # Create an observation group so we can add some standard headers
        # independent of the form in the input files.
        # Use try block in case we are dealing with unexpected data headers
        try:
            group = ObservationGroup(rawmd, pedantic=False)
        except Exception:
            group = None
 
        comments = {"TIMESYS": "Time scale for all dates",
                    "DATE-OBS": "Start date of earliest input observation",
                    "MJD-OBS": "[d] Start MJD of earliest input observation",
                    "DATE-END": "End date of oldest input observation",
                    "MJD-END": "[d] End MJD of oldest input observation",
                    "MJD-AVG": "[d] MJD midpoint of all input observations",
                    "DATE-AVG": "Midpoint date of all input observations"}
 
        if group is not None:
            oldest, newest = group.extremes()
            dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
        else:
            # Fall back to setting a DATE-OBS from the calibDate
            dateCards = {"DATE-OBS": "{}T00:00:00.00".format(outputId[self.config.dateCalib])}
            comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
 
        for k, v in dateCards.items():
            header.set(k, v, comment=comments.get(k, None))
 

checkCcdIdLists()

def lsst.pipe.drivers.constructCalibs.CalibTask.checkCcdIdLists (   self,   ccdIdLists  )

inherited

Check that the list of CCD dataIds is consistent

@param ccdIdLists  Dict of data identifier lists for each CCD name
@return Number of exposures, number of CCDs

Definition at line 886 of file constructCalibs.py.

    def checkCcdIdLists(self, ccdIdLists):
        """Check that the list of CCD dataIds is consistent
 
        @param ccdIdLists  Dict of data identifier lists for each CCD name
        @return Number of exposures, number of CCDs
        """
        visitIdLists = collections.defaultdict(list)
        for ccdName in ccdIdLists:
            for dataId in ccdIdLists[ccdName]:
                visitName = dictToTuple(dataId, self.config.visitKeys)
                visitIdLists[visitName].append(dataId)
 
        numExps = set(len(expList) for expList in ccdIdLists.values())
        numCcds = set(len(ccdList) for ccdList in visitIdLists.values())
 
        if len(numExps) != 1 or len(numCcds) != 1:
            # Presumably a visit somewhere doesn't have the full complement available.
            # Dump the information so the user can figure it out.
            self.log.warn("Number of visits for each CCD: %s",
                          {ccdName: len(ccdIdLists[ccdName]) for ccdName in ccdIdLists})
            self.log.warn("Number of CCDs for each visit: %s",
                          {vv: len(visitIdLists[vv]) for vv in visitIdLists})
            raise RuntimeError("Inconsistent number of exposures/CCDs")
 
        return numExps.pop(), numCcds.pop()
 
 

combine()

def lsst.pipe.drivers.constructCalibs.CalibTask.combine (   self,   cache,   struct,   outputId  )

inherited

Combine multiple exposures of a particular CCD and write the output.

    Only the slave nodes execute this method.

    @param cache  Process pool cache
    @param struct  Parameters for the combination, which has the following components:
        * ccdName     Name tuple for CCD
        * ccdIdList   List of data identifiers for combination
        * scales      Scales to apply (expScales are scalings for each exposure,
                           ccdScale is final scale for combined image)
    @param outputId    Data identifier for combined image (exposure part only)
    @return binned calib image

Reimplemented in lsst.pipe.drivers.constructCalibs.SkyTask.

Definition at line 726 of file constructCalibs.py.

    def combine(self, cache, struct, outputId):
        """!Combine multiple exposures of a particular CCD and write the output
 
        Only the slave nodes execute this method.
 
        @param cache  Process pool cache
        @param struct  Parameters for the combination, which has the following components:
            * ccdName     Name tuple for CCD
            * ccdIdList   List of data identifiers for combination
            * scales      Scales to apply (expScales are scalings for each exposure,
                               ccdScale is final scale for combined image)
        @param outputId    Data identifier for combined image (exposure part only)
        @return binned calib image
        """
        outputId = self.getFullyQualifiedOutputId(struct.ccdName, cache.butler, outputId)
        dataRefList = [getDataRef(cache.butler, dataId) if dataId is not None else None for
                       dataId in struct.ccdIdList]
        self.log.info("Combining %s on %s" % (outputId, NODE))
        calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
                                     finalScale=struct.scales.ccdScale)
 
        if not hasattr(calib, "getMetadata"):
            if hasattr(calib, "getVariance"):
                calib = afwImage.makeExposure(calib)
            else:
                calib = afwImage.DecoratedImageF(calib.getImage())  # n.b. hardwires "F" for the output type
 
        self.calculateOutputHeaderFromRaws(cache.butler, calib, struct.ccdIdList, outputId)
 
        self.updateMetadata(calib, self.exposureTime)
 
        self.recordCalibInputs(cache.butler, calib,
                               struct.ccdIdList, outputId)
 
        self.interpolateNans(calib)
 
        self.write(cache.butler, calib, outputId)
 
        return afwMath.binImage(calib.getImage(), self.config.binning)
 

getFilter()

def lsst.pipe.drivers.constructCalibs.CalibTask.getFilter (   self,   butler,   dataId  )

inherited

Determine the filter from a data identifier

Definition at line 544 of file constructCalibs.py.

    def getFilter(self, butler, dataId):
        """Determine the filter from a data identifier"""
        filt = butler.queryMetadata('raw', [self.config.filter], dataId)[0]
        return filt
 

getFullyQualifiedOutputId()

def lsst.pipe.drivers.constructCalibs.CalibTask.getFullyQualifiedOutputId (   self,   ccdName,   butler,   outputId  )

inherited

Get fully-qualified output data identifier

We may need to look up keys that aren't in the output dataId.

@param ccdName  Name tuple for CCD
@param butler  Data butler
@param outputId  Data identifier for combined image (exposure part only)
@return fully-qualified output dataId

Definition at line 710 of file constructCalibs.py.

    def getFullyQualifiedOutputId(self, ccdName, butler, outputId):
        """Get fully-qualified output data identifier
 
        We may need to look up keys that aren't in the output dataId.
 
        @param ccdName  Name tuple for CCD
        @param butler  Data butler
        @param outputId  Data identifier for combined image (exposure part only)
        @return fully-qualified output dataId
        """
        fullOutputId = {k: ccdName[i] for i, k in enumerate(self.config.ccdKeys)}
        fullOutputId.update(outputId)
        self.addMissingKeys(fullOutputId, butler)
        fullOutputId.update(outputId)  # must be after the call to queryMetadata in 'addMissingKeys'
        return fullOutputId
 

getMjd()

def lsst.pipe.drivers.constructCalibs.CalibTask.getMjd (   self,   butler,   dataId,   timescale = dafBase.DateTime.UTC  )

inherited

Determine the Modified Julian Date (MJD; in TAI) from a data identifier

Definition at line 531 of file constructCalibs.py.

    def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
        """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
        if self.config.dateObs in dataId:
            dateObs = dataId[self.config.dateObs]
        else:
            dateObs = butler.queryMetadata('raw', [self.config.dateObs], dataId)[0]
        if "T" not in dateObs:
            dateObs = dateObs + "T12:00:00.0Z"
        elif not dateObs.endswith("Z"):
            dateObs += "Z"
 
        return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
 

getOutputId()

def lsst.pipe.drivers.constructCalibs.CalibTask.getOutputId (   self,   expRefList,   calibId  )

inherited

Generate the data identifier for the output calib.

    The mean date and the common filter are included, using keywords
    from the configuration.  The CCD-specific part is not included
    in the data identifier.

    @param expRefList  List of data references at exposure level
    @param calibId  Data identifier elements for the calib provided by the user
    @return data identifier

Definition at line 496 of file constructCalibs.py.

    def getOutputId(self, expRefList, calibId):
        """!Generate the data identifier for the output calib
 
        The mean date and the common filter are included, using keywords
        from the configuration.  The CCD-specific part is not included
        in the data identifier.
 
        @param expRefList  List of data references at exposure level
        @param calibId  Data identifier elements for the calib provided by the user
        @return data identifier
        """
        midTime = 0
        filterName = None
        for expRef in expRefList:
            butler = expRef.getButler()
            dataId = expRef.dataId
 
            midTime += self.getMjd(butler, dataId)
            thisFilter = self.getFilter(
                butler, dataId) if self.filterName is None else self.filterName
            if filterName is None:
                filterName = thisFilter
            elif filterName != thisFilter:
                raise RuntimeError("Filter mismatch for %s: %s vs %s" % (
                    dataId, thisFilter, filterName))
 
        midTime /= len(expRefList)
        date = str(dafBase.DateTime(
            midTime, dafBase.DateTime.MJD).toPython().date())
 
        outputId = {self.config.filter: filterName,
                    self.config.dateCalib: date}
        outputId.update(calibId)
        return outputId
 

interpolateNans()

def lsst.pipe.drivers.constructCalibs.CalibTask.interpolateNans (   self,   image  )

inherited

Interpolate over NANs in the combined image

NANs can result from masked areas on the CCD.  We don't want them getting
into our science images, so we replace them with the median of the image.

Definition at line 847 of file constructCalibs.py.

    def interpolateNans(self, image):
        """Interpolate over NANs in the combined image
 
        NANs can result from masked areas on the CCD.  We don't want them getting
        into our science images, so we replace them with the median of the image.
        """
        if hasattr(image, "getMaskedImage"):  # Deal with Exposure vs Image
            self.interpolateNans(image.getMaskedImage().getVariance())
            image = image.getMaskedImage().getImage()
        if hasattr(image, "getImage"):  # Deal with DecoratedImage or MaskedImage vs Image
            image = image.getImage()
        array = image.getArray()
        bad = np.isnan(array)
        array[bad] = np.median(array[np.logical_not(bad)])
 

logOperation()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.logOperation (   self,   operation,   catch = False,   trace = True  )

inherited

Provide a context manager for logging an operation.

    @param operation: description of operation (string)
    @param catch: Catch all exceptions?
    @param trace: Log a traceback of caught exception?

    Note that if 'catch' is True, all exceptions are swallowed, but there may
    be other side-effects such as undefined variables.

Definition at line 502 of file parallel.py.

    def logOperation(self, operation, catch=False, trace=True):
        """!Provide a context manager for logging an operation
 
        @param operation: description of operation (string)
        @param catch: Catch all exceptions?
        @param trace: Log a traceback of caught exception?
 
        Note that if 'catch' is True, all exceptions are swallowed, but there may
        be other side-effects such as undefined variables.
        """
        self.log.info("%s: Start %s" % (NODE, operation))
        try:
            yield
        except Exception:
            if catch:
                cls, e, _ = sys.exc_info()
                self.log.warn("%s: Caught %s while %s: %s" % (NODE, cls.__name__, operation, e))
                if trace:
                    self.log.info("%s: Traceback:\n%s" % (NODE, traceback.format_exc()))
                return
            raise
        finally:
            self.log.info("%s: Finished %s" % (NODE, operation))
 
 

makeCameraImage()

def lsst.pipe.drivers.constructCalibs.CalibTask.makeCameraImage (   self,   camera,   dataId,   calibs  )

inherited

Create and write an image of the entire camera.

    This is useful for judging the quality or getting an overview of
    the features of the calib.

    @param camera  Camera object
    @param dataId  Data identifier for output
    @param calibs  Dict mapping CCD detector ID to calib image

Definition at line 874 of file constructCalibs.py.

    def makeCameraImage(self, camera, dataId, calibs):
        """!Create and write an image of the entire camera
 
        This is useful for judging the quality or getting an overview of
        the features of the calib.
 
        @param camera  Camera object
        @param dataId  Data identifier for output
        @param calibs  Dict mapping CCD detector ID to calib image
        """
        return makeCameraImage(camera, calibs, self.config.binning)
 

parseAndRun()

def lsst.ctrl.pool.parallel.BatchPoolTask.parseAndRun (   cls, *  args, **  kwargs  )

inherited

Run with a MPI process pool

Definition at line 534 of file parallel.py.

    def parseAndRun(cls, *args, **kwargs):
        """Run with a MPI process pool"""
        pool = startPool()
        super(BatchPoolTask, cls).parseAndRun(*args, **kwargs)
        pool.exit()
 
 

parseAndSubmit()

def lsst.ctrl.pool.parallel.BatchCmdLineTask.parseAndSubmit (   cls,   args = None, **  kwargs  )

inherited

Definition at line 435 of file parallel.py.

    def parseAndSubmit(cls, args=None, **kwargs):
        taskParser = cls._makeArgumentParser(doBatch=True, add_help=False)
        batchParser = BatchArgumentParser(parent=taskParser)
        batchArgs = batchParser.parse_args(config=cls.ConfigClass(), args=args, override=cls.applyOverrides,
                                           **kwargs)
 
        if not cls.RunnerClass(cls, batchArgs.parent).precall(batchArgs.parent):  # Write config, schema
            taskParser.error("Error in task preparation")
 
        setBatchType(batchArgs.batch)
 
        if batchArgs.batch is None:     # don't use a batch system
            sys.argv = [sys.argv[0]] + batchArgs.leftover  # Remove all batch arguments
 
            return cls.parseAndRun()
        else:
            if batchArgs.walltime > 0:
                walltime = batchArgs.walltime
            else:
                numCores = batchArgs.cores if batchArgs.cores > 0 else batchArgs.nodes*batchArgs.procs
                walltime = cls.batchWallTime(batchArgs.time, batchArgs.parent, numCores)
 
            command = cls.batchCommand(batchArgs)
            batchArgs.batch.run(command, walltime=walltime)
 

process()

def lsst.pipe.drivers.constructCalibs.CalibTask.process (   self,   cache,   ccdId,   outputName = "postISRCCD", **  kwargs  )

inherited

Process a CCD, specified by a data identifier.

    After processing, optionally returns a result (produced by
    the 'processResult' method) calculated from the processed
    exposure.  These results will be gathered by the master node,
    and is a means for coordinated scaling of all CCDs for flats,
    etc.

    Only slave nodes execute this method.

    @param cache  Process pool cache
    @param ccdId  Data identifier for CCD
    @param outputName  Output dataset name for butler
    @return result from 'processResult'

Definition at line 602 of file constructCalibs.py.

    def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
        """!Process a CCD, specified by a data identifier
 
        After processing, optionally returns a result (produced by
        the 'processResult' method) calculated from the processed
        exposure.  These results will be gathered by the master node,
        and is a means for coordinated scaling of all CCDs for flats,
        etc.
 
        Only slave nodes execute this method.
 
        @param cache  Process pool cache
        @param ccdId  Data identifier for CCD
        @param outputName  Output dataset name for butler
        @return result from 'processResult'
        """
        if ccdId is None:
            self.log.warn("Null identifier received on %s" % NODE)
            return None
        sensorRef = getDataRef(cache.butler, ccdId)
        if self.config.clobber or not sensorRef.datasetExists(outputName):
            self.log.info("Processing %s on %s" % (ccdId, NODE))
            try:
                exposure = self.processSingle(sensorRef, **kwargs)
            except Exception as e:
                self.log.warn("Unable to process %s: %s" % (ccdId, e))
                raise
                return None
            self.processWrite(sensorRef, exposure)
        else:
            self.log.info(
                "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
            exposure = sensorRef.get(outputName)
        return self.processResult(exposure)
 

processResult()

def lsst.pipe.drivers.constructCalibs.FlatTask.processResult	(		self,
			exposure
	)

Extract processing results from a processed exposure

This method generates what is gathered by the master node.
This can be a background measurement or similar for scaling
flat-fields.  It must be picklable!

Only slave nodes execute this method.

Reimplemented from lsst.pipe.drivers.constructCalibs.CalibTask.

Definition at line 1039 of file constructCalibs.py.

    def processResult(self, exposure):
        return self.stats.run(exposure)
 

processSingle()

def lsst.pipe.drivers.constructCalibs.CalibTask.processSingle (   self,   dataRef  )

inherited

Process a single CCD, specified by a data reference

Generally, this simply means doing ISR.

Only slave nodes execute this method.

Reimplemented in lsst.pipe.drivers.constructCalibs.FringeTask, and lsst.pipe.drivers.constructCalibs.DarkTask.

Definition at line 637 of file constructCalibs.py.

    def processSingle(self, dataRef):
        """Process a single CCD, specified by a data reference
 
        Generally, this simply means doing ISR.
 
        Only slave nodes execute this method.
        """
        return self.isr.runDataRef(dataRef).exposure
 

processWrite()

def lsst.pipe.drivers.constructCalibs.CalibTask.processWrite (   self,   dataRef,   exposure,   outputName = "postISRCCD"  )

inherited

Write the processed CCD.

    We need to write these out because we can't hold them all in
    memory at once.

    Only slave nodes execute this method.

    @param dataRef     Data reference
    @param exposure    CCD exposure to write
    @param outputName  Output dataset name for butler.

Definition at line 646 of file constructCalibs.py.

    def processWrite(self, dataRef, exposure, outputName="postISRCCD"):
        """!Write the processed CCD
 
        We need to write these out because we can't hold them all in
        memory at once.
 
        Only slave nodes execute this method.
 
        @param dataRef     Data reference
        @param exposure    CCD exposure to write
        @param outputName  Output dataset name for butler.
        """
        dataRef.put(exposure, outputName)
 

recordCalibInputs()

def lsst.pipe.drivers.constructCalibs.CalibTask.recordCalibInputs (   self,   butler,   calib,   dataIdList,   outputId  )

inherited

Record metadata including the inputs and creation details.

    This metadata will go into the FITS header.

    @param butler  Data butler
    @param calib  Combined calib exposure.
    @param dataIdList  List of data identifiers for calibration inputs
    @param outputId  Data identifier for output

Definition at line 819 of file constructCalibs.py.

    def recordCalibInputs(self, butler, calib, dataIdList, outputId):
        """!Record metadata including the inputs and creation details
 
        This metadata will go into the FITS header.
 
        @param butler  Data butler
        @param calib  Combined calib exposure.
        @param dataIdList  List of data identifiers for calibration inputs
        @param outputId  Data identifier for output
        """
        header = calib.getMetadata()
        header.set("OBSTYPE", self.calibName)  # Used by ingestCalibs.py
 
        # date, time, host, and root
        now = time.localtime()
        header.set("CALIB_CREATION_DATE", time.strftime("%Y-%m-%d", now))
        header.set("CALIB_CREATION_TIME", time.strftime("%X %Z", now))
 
        # Inputs
        visits = [str(dictToTuple(dataId, self.config.visitKeys)) for dataId in dataIdList if
                  dataId is not None]
        for i, v in enumerate(sorted(set(visits))):
            header.set("CALIB_INPUT_%d" % (i,), v)
 
        header.set("CALIB_ID", " ".join("%s=%s" % (key, value)
                                        for key, value in outputId.items()))
        checksum(calib, header)
 

runDataRef()

def lsst.pipe.drivers.constructCalibs.CalibTask.runDataRef (   self,   expRefList,   butler,   calibId  )

inherited

Construct a calib from a list of exposure references.

    This is the entry point, called by the TaskRunner.__call__

    Only the master node executes this method.

    @param expRefList  List of data references at the exposure level
    @param butler      Data butler
    @param calibId   Identifier dict for calib

Definition at line 422 of file constructCalibs.py.

    def runDataRef(self, expRefList, butler, calibId):
        """!Construct a calib from a list of exposure references
 
        This is the entry point, called by the TaskRunner.__call__
 
        Only the master node executes this method.
 
        @param expRefList  List of data references at the exposure level
        @param butler      Data butler
        @param calibId   Identifier dict for calib
        """
        if len(expRefList) < 1:
            raise RuntimeError("No valid input data")
 
        for expRef in expRefList:
            self.addMissingKeys(expRef.dataId, butler, self.config.ccdKeys, 'raw')
 
        outputId = self.getOutputId(expRefList, calibId)
        ccdIdLists = getCcdIdListFromExposures(
            expRefList, level="sensor", ccdKeys=self.config.ccdKeys)
        self.checkCcdIdLists(ccdIdLists)
 
        # Ensure we can generate filenames for each output
        outputIdItemList = list(outputId.items())
        for ccdName in ccdIdLists:
            dataId = dict([(k, ccdName[i]) for i, k in enumerate(self.config.ccdKeys)])
            dataId.update(outputIdItemList)
            self.addMissingKeys(dataId, butler)
            dataId.update(outputIdItemList)
 
            try:
                butler.get(self.calibName + "_filename", dataId)
            except Exception as e:
                raise RuntimeError(
                    "Unable to determine output filename \"%s_filename\" from %s: %s" %
                    (self.calibName, dataId, e))
 
        processPool = Pool("process")
        processPool.storeSet(butler=butler)
 
        # Scatter: process CCDs independently
        data = self.scatterProcess(processPool, ccdIdLists)
 
        # Gather: determine scalings
        scales = self.scale(ccdIdLists, data)
 
        combinePool = Pool("combine")
        combinePool.storeSet(butler=butler)
 
        # Scatter: combine
        calibs = self.scatterCombine(combinePool, outputId, ccdIdLists, scales)
 
        if self.config.doCameraImage:
            camera = butler.get("camera")
            # Convert indexing of calibs from "ccdName" to detector ID (as used by makeImageFromCamera)
            calibs = {butler.get("postISRCCD_detector",
                                 dict(zip(self.config.ccdKeys, ccdName))).getId(): calibs[ccdName]
                      for ccdName in ccdIdLists}
 
            try:
                cameraImage = self.makeCameraImage(camera, outputId, calibs)
                butler.put(cameraImage, self.calibName + "_camera", dataId)
            except Exception as exc:
                self.log.warn("Unable to create camera image: %s" % (exc,))
 
        return Struct(
            outputId=outputId,
            ccdIdLists=ccdIdLists,
            scales=scales,
            calibs=calibs,
            processPool=processPool,
            combinePool=combinePool,
        )
 

scale()

def lsst.pipe.drivers.constructCalibs.FlatTask.scale	(		self,
			ccdIdLists,
			data
	)

Determine the scalings for the final combination

We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
of one CCD to all the others in an exposure, and E_j is the scaling
of the exposure.  We convert everything to logarithms so we can work
with a linear system.  We determine the C_i and E_j from B_ij by iteration,
under the additional constraint that the average CCD scale is unity.

This algorithm comes from Eugene Magnier and Pan-STARRS.

Reimplemented from lsst.pipe.drivers.constructCalibs.CalibTask.

Definition at line 1042 of file constructCalibs.py.

    def scale(self, ccdIdLists, data):
        """Determine the scalings for the final combination
 
        We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
        of one CCD to all the others in an exposure, and E_j is the scaling
        of the exposure.  We convert everything to logarithms so we can work
        with a linear system.  We determine the C_i and E_j from B_ij by iteration,
        under the additional constraint that the average CCD scale is unity.
 
        This algorithm comes from Eugene Magnier and Pan-STARRS.
        """
        assert len(ccdIdLists.values()) > 0, "No successful CCDs"
        lengths = set([len(expList) for expList in ccdIdLists.values()])
        assert len(lengths) == 1, "Number of successful exposures for each CCD differs"
        assert tuple(lengths)[0] > 0, "No successful exposures"
        # Format background measurements into a matrix
        indices = dict((name, i) for i, name in enumerate(ccdIdLists))
        bgMatrix = np.array([[0.0] * len(expList) for expList in ccdIdLists.values()])
        for name in ccdIdLists:
            i = indices[name]
            bgMatrix[i] = [d if d is not None else np.nan for d in data[name]]
 
        numpyPrint = np.get_printoptions()
        np.set_printoptions(threshold=np.inf)
        self.log.info("Input backgrounds: %s" % bgMatrix)
 
        # Flat-field scaling
        numCcds = len(ccdIdLists)
        numExps = bgMatrix.shape[1]
        # log(Background) for each exposure/component
        bgMatrix = np.log(bgMatrix)
        bgMatrix = np.ma.masked_array(bgMatrix, ~np.isfinite(bgMatrix))
        # Initial guess at log(scale) for each component
        compScales = np.zeros(numCcds)
        expScales = np.array([(bgMatrix[:, i0] - compScales).mean() for i0 in range(numExps)])
 
        for iterate in range(self.config.iterations):
            compScales = np.array([(bgMatrix[i1, :] - expScales).mean() for i1 in range(numCcds)])
            bad = np.isnan(compScales)
            if np.any(bad):
                # Bad CCDs: just set them to the mean scale
                compScales[bad] = compScales[~bad].mean()
            expScales = np.array([(bgMatrix[:, i2] - compScales).mean() for i2 in range(numExps)])
 
            avgScale = np.average(np.exp(compScales))
            compScales -= np.log(avgScale)
            self.log.debug("Iteration %d exposure scales: %s", iterate, np.exp(expScales))
            self.log.debug("Iteration %d component scales: %s", iterate, np.exp(compScales))
 
        expScales = np.array([(bgMatrix[:, i3] - compScales).mean() for i3 in range(numExps)])
 
        if np.any(np.isnan(expScales)):
            raise RuntimeError("Bad exposure scales: %s --> %s" % (bgMatrix, expScales))
 
        expScales = np.exp(expScales)
        compScales = np.exp(compScales)
 
        self.log.info("Exposure scales: %s" % expScales)
        self.log.info("Component relative scaling: %s" % compScales)
        np.set_printoptions(**numpyPrint)
 
        return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
                    for ccdName in ccdIdLists)
 
 

scatterCombine()

def lsst.pipe.drivers.constructCalibs.CalibTask.scatterCombine (   self,   pool,   outputId,   ccdIdLists,   scales  )

inherited

Scatter the combination of exposures across multiple nodes.

    In this case, we can only scatter across as many nodes as
    there are CCDs.

    Only the master node executes this method.

    @param pool  Process pool
    @param outputId  Output identifier (exposure part only)
    @param ccdIdLists  Dict of data identifier lists for each CCD name
    @param scales  Dict of structs with scales, for each CCD name
    @param dict of binned images

Definition at line 690 of file constructCalibs.py.

    def scatterCombine(self, pool, outputId, ccdIdLists, scales):
        """!Scatter the combination of exposures across multiple nodes
 
        In this case, we can only scatter across as many nodes as
        there are CCDs.
 
        Only the master node executes this method.
 
        @param pool  Process pool
        @param outputId  Output identifier (exposure part only)
        @param ccdIdLists  Dict of data identifier lists for each CCD name
        @param scales  Dict of structs with scales, for each CCD name
        @param dict of binned images
        """
        self.log.info("Scatter combination")
        data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName]) for
                ccdName in ccdIdLists]
        images = pool.map(self.combine, data, outputId)
        return dict(zip(ccdIdLists.keys(), images))
 

scatterProcess()

def lsst.pipe.drivers.constructCalibs.CalibTask.scatterProcess (   self,   pool,   ccdIdLists  )

inherited

Scatter the processing among the nodes.

    We scatter each CCD independently (exposures aren't grouped together),
    to make full use of all available processors. This necessitates piecing
    everything back together in the same format as ccdIdLists afterwards.

    Only the master node executes this method.

    @param pool  Process pool
    @param ccdIdLists  Dict of data identifier lists for each CCD name
    @return Dict of lists of returned data for each CCD name

Reimplemented in lsst.pipe.drivers.constructCalibs.SkyTask.

Definition at line 586 of file constructCalibs.py.

    def scatterProcess(self, pool, ccdIdLists):
        """!Scatter the processing among the nodes
 
        We scatter each CCD independently (exposures aren't grouped together),
        to make full use of all available processors. This necessitates piecing
        everything back together in the same format as ccdIdLists afterwards.
 
        Only the master node executes this method.
 
        @param pool  Process pool
        @param ccdIdLists  Dict of data identifier lists for each CCD name
        @return Dict of lists of returned data for each CCD name
        """
        self.log.info("Scatter processing")
        return mapToMatrix(pool, self.process, ccdIdLists)
 

updateMetadata()

def lsst.pipe.drivers.constructCalibs.CalibTask.updateMetadata (   self,   calibImage,   exposureTime,   darkTime = None, **  kwargs  )

inherited

Update the metadata from the VisitInfo.

    @param calibImage       The image whose metadata is to be set
    @param exposureTime     The exposure time for the image
    @param darkTime         The time since the last read (default: exposureTime)

Definition at line 570 of file constructCalibs.py.

    def updateMetadata(self, calibImage, exposureTime, darkTime=None, **kwargs):
        """!Update the metadata from the VisitInfo
 
        @param calibImage       The image whose metadata is to be set
        @param exposureTime     The exposure time for the image
        @param darkTime         The time since the last read (default: exposureTime)
        """
 
        if darkTime is None:
            darkTime = exposureTime     # avoid warning messages when using calibration products
 
        visitInfo = afwImage.VisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
        md = calibImage.getMetadata()
 
        afwImage.setVisitInfoMetadata(md, visitInfo)
 

write()

def lsst.pipe.drivers.constructCalibs.CalibTask.write (   self,   butler,   exposure,   dataId  )

inherited

Write the final combined calib.

    Only the slave nodes execute this method

    @param butler  Data butler
    @param exposure  CCD exposure to write
    @param dataId  Data identifier for output

Definition at line 862 of file constructCalibs.py.

    def write(self, butler, exposure, dataId):
        """!Write the final combined calib
 
        Only the slave nodes execute this method
 
        @param butler  Data butler
        @param exposure  CCD exposure to write
        @param dataId  Data identifier for output
        """
        self.log.info("Writing %s on %s" % (dataId, NODE))
        butler.put(exposure, self.calibName, dataId)
 

Member Data Documentation

calibName

string lsst.pipe.drivers.constructCalibs.FlatTask.calibName = "flat"

static

Definition at line 1027 of file constructCalibs.py.

ConfigClass

lsst.pipe.drivers.constructCalibs.FlatTask.ConfigClass = FlatConfig

static

Definition at line 1025 of file constructCalibs.py.

exposureTime

float lsst.pipe.drivers.constructCalibs.CalibTask.exposureTime = 1.0

staticinherited

Definition at line 401 of file constructCalibs.py.

filterName

lsst.pipe.drivers.constructCalibs.CalibTask.filterName = None

staticinherited

Definition at line 399 of file constructCalibs.py.

RunnerClass

lsst.pipe.drivers.constructCalibs.CalibTask.RunnerClass = CalibTaskRunner

staticinherited

Definition at line 398 of file constructCalibs.py.

---

The documentation for this class was generated from the following file:

• /j/snowflake/release/lsstsw/stack/lsst-scipipe-0.7.0/Linux64/pipe_drivers/22.0.0+a28f4c53b1/python/lsst/pipe/drivers/constructCalibs.py