LSSTApplications: lsst.pipe.tasks.ingest.IngestTask Class Reference

Inheritance diagram for lsst.pipe.tasks.ingest.IngestTask:

Public Member Functions
def	__init__

def	parseAndRun

def	ingest

def	isBadFile

def	isBadId

def	run

Static Public Attributes
	ConfigClass = IngestConfig

	ArgumentParser = IngestArgumentParser

Static Private Attributes
string	_DefaultName = "ingest"

Detailed Description

Task that will ingest images into the data repository

Definition at line 335 of file ingest.py.

Constructor & Destructor Documentation

def lsst.pipe.tasks.ingest.IngestTask.__init__	(	self,
		args,
		kwargs
	)

Definition at line 341 of file ingest.py.

 
     def __init__(self, *args, **kwargs):
         super(IngestTask, self).__init__(*args, **kwargs)
         self.makeSubtask("parse")
         self.makeSubtask("register")

lsst.pipe.tasks.ingest.IngestTask.__init__

def __init__

Definition: ingest.py:341

Member Function Documentation

def lsst.pipe.tasks.ingest.IngestTask.ingest	(	self,
		infile,
		outfile,
		mode = `"move"`,
		dryrun = `False`
	)

Ingest a file into the image repository.

@param infile  Name of input file
@param outfile Name of output file (file in repository)
@param mode    Mode of ingest (copy/link/move/skip)
@param dryrun  Only report what would occur?
@param Success boolean

Definition at line 355 of file ingest.py.

 
     def ingest(self, infile, outfile, mode="move", dryrun=False):
         """Ingest a file into the image repository.
 
         @param infile  Name of input file
         @param outfile Name of output file (file in repository)
         @param mode    Mode of ingest (copy/link/move/skip)
         @param dryrun  Only report what would occur?
         @param Success boolean
         """
         if mode == "skip":
             return True
         if dryrun:
             self.log.info("Would %s from %s to %s" % (mode, infile, outfile))
             return True
         try:
             outdir = os.path.dirname(outfile)
             if not os.path.isdir(outdir):
                 try:
                     os.makedirs(outdir)
                 except:
                     # Silently ignore mkdir failures due to race conditions
                     if not os.path.isdir(outdir):
                         raise
             if self.config.clobber and os.path.lexists(outfile):
                 os.unlink(outfile)
             if mode == "copy":
                 assertCanCopy(infile, outfile)
                 shutil.copyfile(infile, outfile)
             elif mode == "link":
                 os.symlink(os.path.abspath(infile), outfile)
             elif mode == "move":
                 assertCanCopy(infile, outfile)
                 os.rename(infile, outfile)
             else:
                 raise AssertionError("Unknown mode: %s" % mode)
             print "%s --<%s>--> %s" % (infile, mode, outfile)
         except Exception, e:
             self.log.warn("Failed to %s %s to %s: %s" % (mode, infile, outfile, e))
             if not self.config.allowError:
                 raise
             return False
         return True

lsst.pipe.tasks.ingest.assertCanCopy

def assertCanCopy

Definition: ingest.py:446

lsst.pipe.tasks.ingest.IngestTask.ingest

def ingest

Definition: ingest.py:355

def lsst.pipe.tasks.ingest.IngestTask.isBadFile	(	self,
		filename,
		badFileList
	)

Return whether the file qualifies as bad

We match against the list of bad file patterns.

Definition at line 398 of file ingest.py.

 
     def isBadFile(self, filename, badFileList):
         """Return whether the file qualifies as bad
 
         We match against the list of bad file patterns.
         """
         filename = os.path.basename(filename)
         if not badFileList:
             return False
         for badFile in badFileList:
             if fnmatch(filename, badFile):
                 return True
         return False

lsst.pipe.tasks.ingest.IngestTask.isBadFile

def isBadFile

Definition: ingest.py:398

def lsst.pipe.tasks.ingest.IngestTask.isBadId	(	self,
		info,
		badIdList
	)

Return whether the file information qualifies as bad

We match against the list of bad data identifiers.

Definition at line 411 of file ingest.py.

 
     def isBadId(self, info, badIdList):
         """Return whether the file information qualifies as bad
 
         We match against the list of bad data identifiers.
         """
         if not badIdList:
             return False
         for badId in badIdList:
             if all(info[key] == value for key, value in badId.iteritems()):
                 return True
         return False

lsst::afw::geom::all

bool all(CoordinateExpr< N > const &expr)

Return true if all elements are true.

Definition: CoordinateExpr.h:80

lsst.pipe.tasks.ingest.IngestTask.isBadId

def isBadId

Definition: ingest.py:411

def lsst.pipe.tasks.ingest.IngestTask.parseAndRun ( cls )

Parse the command-line arguments and run the Task

Definition at line 347 of file ingest.py.

 
     def parseAndRun(cls):
         """Parse the command-line arguments and run the Task"""
         config = cls.ConfigClass()
         parser = cls.ArgumentParser("ingest")
         args = parser.parse_args(config)
         task = cls(config=args.config)
         task.run(args)

lsst.pipe.tasks.ingest.IngestTask.parseAndRun

def parseAndRun

Definition: ingest.py:347

def lsst.pipe.tasks.ingest.IngestTask.run	(	self,
		args
	)

Ingest all specified files and add them to the registry

Definition at line 423 of file ingest.py.

 
     def run(self, args):
         """Ingest all specified files and add them to the registry"""
         filenameList = sum([glob(filename) for filename in args.files], [])
         context = self.register.openRegistry(args.butler, create=args.create, dryrun=args.dryrun)
         with context as registry:
             for infile in filenameList:
                 if self.isBadFile(infile, args.badFile):
                     self.log.info("Skipping declared bad file %s" % infile)
                     continue
                 fileInfo, hduInfoList = self.parse.getInfo(infile)
                 if self.isBadId(fileInfo, args.badId.idList):
                     self.log.info("Skipping declared bad file %s: %s" % (infile, fileInfo))
                     continue
                 if self.register.check(registry, fileInfo):
                     self.log.warn("%s: already ingested: %s" % (infile, fileInfo))
                 outfile = self.parse.getDestination(args.butler, fileInfo, infile)
                 ingested = self.ingest(infile, outfile, mode=args.mode, dryrun=args.dryrun)
                 if not ingested:
                     continue
                 for info in hduInfoList:
                     self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
             self.register.addVisits(registry, dryrun=args.dryrun)