LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
ingest.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import os
23 import shutil
24 import sqlite3
25 import sys
26 from fnmatch import fnmatch
27 from glob import glob
28 from contextlib import contextmanager
29 
30 from astro_metadata_translator import fix_header
31 from lsst.pex.config import Config, Field, DictField, ListField, ConfigurableField
32 from lsst.afw.fits import readMetadata
33 from lsst.pipe.base import Task, InputOnlyArgumentParser
34 from lsst.afw.fits import DEFAULT_HDU
35 
36 
37 class IngestArgumentParser(InputOnlyArgumentParser):
38  """Argument parser to support ingesting images into the image repository"""
39 
40  def __init__(self, *args, **kwargs):
41  super(IngestArgumentParser, self).__init__(*args, **kwargs)
42  self.add_argument("-n", "--dry-run", dest="dryrun", action="store_true", default=False,
43  help="Don't perform any action?")
44  self.add_argument("--mode", choices=["move", "copy", "link", "skip"], default="link",
45  help="Mode of delivering the files to their destination")
46  self.add_argument("--create", action="store_true", help="Create new registry (clobber old)?")
47  self.add_argument("--ignore-ingested", dest="ignoreIngested", action="store_true",
48  help="Don't register files that have already been registered")
49  self.add_id_argument("--badId", "raw", "Data identifier for bad data", doMakeDataRefList=False)
50  self.add_argument("--badFile", nargs="*", default=[],
51  help="Names of bad files (no path; wildcards allowed)")
52  self.add_argument("files", nargs="+", help="Names of file")
53 
54 
56  """Configuration for ParseTask"""
57  translation = DictField(keytype=str, itemtype=str, default={},
58  doc="Translation table for property --> header")
59  translators = DictField(keytype=str, itemtype=str, default={},
60  doc="Properties and name of translator method")
61  defaults = DictField(keytype=str, itemtype=str, default={},
62  doc="Default values if header is not present")
63  hdu = Field(dtype=int, default=DEFAULT_HDU, doc="HDU to read for metadata")
64  extnames = ListField(dtype=str, default=[], doc="Extension names to search for")
65 
66 
67 class ParseTask(Task):
68  """Task that will parse the filename and/or its contents to get the required information
69  for putting the file in the correct location and populating the registry."""
70  ConfigClass = ParseConfig
71  translator_class = None
72  """Metadata translation support (astro_metadata_translator.MetadataTranslator).
73 
74  Notes
75  -----
76  The default of `None` will attempt to guess the correct translator,
77  but specifying one (e.g., in obs-package specific ingest) will be faster.
78  """
79 
80  def getInfo(self, filename):
81  """Get information about the image from the filename and its contents
82 
83  Here, we open the image and parse the header, but one could also look at the filename itself
84  and derive information from that, or set values from the configuration.
85 
86  Parameters
87  ----------
88  filename : `str`
89  Name of file to inspect
90 
91  Returns
92  -------
93  phuInfo : `dict`
94  File properties
95  infoList : `list`
96  List of file properties for each extension
97  """
98  md = readMetadata(filename, self.config.hdu)
99  fix_header(md, translator_class=self.translator_classtranslator_class)
100  phuInfo = self.getInfoFromMetadatagetInfoFromMetadata(md)
101  if len(self.config.extnames) == 0:
102  # No extensions to worry about
103  return phuInfo, [phuInfo]
104  # Look in the provided extensions
105  extnames = set(self.config.extnames)
106  extnum = 0
107  infoList = []
108  while len(extnames) > 0:
109  extnum += 1
110  try:
111  md = readMetadata(filename, extnum)
112  fix_header(md, translator_class=self.translator_classtranslator_class)
113  except Exception as e:
114  self.log.warning("Error reading %s extensions %s: %s", filename, extnames, e)
115  break
116  ext = self.getExtensionNamegetExtensionName(md)
117  if ext in extnames:
118  hduInfo = self.getInfoFromMetadatagetInfoFromMetadata(md, info=phuInfo.copy())
119  # We need the HDU number when registering MEF files.
120  hduInfo["hdu"] = extnum
121  infoList.append(hduInfo)
122  extnames.discard(ext)
123  return phuInfo, infoList
124 
125  @staticmethod
127  """ Get the name of a FITS extension.
128 
129  Parameters
130  ----------
131  md : `lsst.daf.base.PropertySet`
132  FITS header metadata.
133 
134  Returns
135  -------
136  result : `str` or `None`
137  The string from the EXTNAME header card if it exists. None otherwise.
138  """
139  try:
140  ext = md["EXTNAME"]
141  except KeyError:
142  ext = None
143  return ext
144 
145  def getInfoFromMetadata(self, md, info=None):
146  """Attempt to pull the desired information out of the header
147 
148  This is done through two mechanisms:
149  * translation: a property is set directly from the relevant header keyword
150  * translator: a property is set with the result of calling a method
151 
152  The translator methods receive the header metadata and should return the
153  appropriate value, or None if the value cannot be determined.
154 
155  @param md FITS header
156  @param info File properties, to be supplemented
157  @return info
158  """
159  if info is None:
160  info = {}
161  for p, h in self.config.translation.items():
162  value = md.get(h, None)
163  if value is not None:
164  if isinstance(value, str):
165  value = value.strip()
166  info[p] = value
167  elif p in self.config.defaults:
168  info[p] = self.config.defaults[p]
169  else:
170  self.log.warning("Unable to find value for %s (derived from %s)", p, h)
171  for p, t in self.config.translators.items():
172  func = getattr(self, t)
173  try:
174  value = func(md)
175  except Exception as e:
176  self.log.warning("%s failed to translate %s: %s", t, p, e)
177  value = None
178  if value is not None:
179  info[p] = value
180  return info
181 
182  def translate_date(self, md):
183  """Convert a full DATE-OBS to a mere date
184 
185  Besides being an example of a translator, this is also generally useful.
186  It will only be used if listed as a translator in the configuration.
187  """
188  date = md.getScalar("DATE-OBS").strip()
189  c = date.find("T")
190  if c > 0:
191  date = date[:c]
192  return date
193 
194  def translate_filter(self, md):
195  """Translate a full filter description into a mere filter name
196 
197  Besides being an example of a translator, this is also generally useful.
198  It will only be used if listed as a translator in the configuration.
199  """
200  filterName = md.getScalar("FILTER").strip()
201  filterName = filterName.strip()
202  c = filterName.find(" ")
203  if c > 0:
204  filterName = filterName[:c]
205  return filterName
206 
207  def getDestination(self, butler, info, filename):
208  """Get destination for the file
209 
210  @param butler Data butler
211  @param info File properties, used as dataId for the butler
212  @param filename Input filename
213  @return Destination filename
214  """
215  raw = butler.get("raw_filename", info)[0]
216  # Ensure filename is devoid of cfitsio directions about HDUs
217  c = raw.find("[")
218  if c > 0:
219  raw = raw[:c]
220  return raw
221 
222 
224  """Configuration for the RegisterTask"""
225  table = Field(dtype=str, default="raw", doc="Name of table")
226  columns = DictField(keytype=str, itemtype=str, doc="List of columns for raw table, with their types",
227  itemCheck=lambda x: x in ("text", "int", "double"),
228  default={'object': 'text',
229  'visit': 'int',
230  'ccd': 'int',
231  'filter': 'text',
232  'date': 'text',
233  'taiObs': 'text',
234  'expTime': 'double',
235  },
236  )
237  unique = ListField(dtype=str, doc="List of columns to be declared unique for the table",
238  default=["visit", "ccd"])
239  visit = ListField(dtype=str, default=["visit", "object", "date", "filter"],
240  doc="List of columns for raw_visit table")
241  ignore = Field(dtype=bool, default=False, doc="Ignore duplicates in the table?")
242  permissions = Field(dtype=int, default=0o664, doc="Permissions mode for registry; 0o664 = rw-rw-r--")
243 
244 
246  """Context manager to provide a registry
247  """
248 
249  def __init__(self, registryName, createTableFunc, forceCreateTables, permissions):
250  """Construct a context manager
251 
252  @param registryName: Name of registry file
253  @param createTableFunc: Function to create tables
254  @param forceCreateTables: Force the (re-)creation of tables?
255  @param permissions: Permissions to set on database file
256  """
257  self.connconn = sqlite3.connect(registryName)
258  os.chmod(registryName, permissions)
259  createTableFunc(self.connconn, forceCreateTables=forceCreateTables)
260 
261  def __enter__(self):
262  """Provide the 'as' value"""
263  return self.connconn
264 
265  def __exit__(self, excType, excValue, traceback):
266  self.connconn.commit()
267  self.connconn.close()
268  return False # Don't suppress any exceptions
269 
270 
271 @contextmanager
273  """A context manager that doesn't provide any context
274 
275  Useful for dry runs where we don't want to actually do anything real.
276  """
277  yield
278 
279 
280 class RegisterTask(Task):
281  """Task that will generate the registry for the Mapper"""
282  ConfigClass = RegisterConfig
283  placeHolder = '?' # Placeholder for parameter substitution; this value suitable for sqlite3
284  typemap = {'text': str, 'int': int, 'double': float} # Mapping database type --> python type
285 
286  def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"):
287  """Open the registry and return the connection handle.
288 
289  @param directory Directory in which the registry file will be placed
290  @param create Clobber any existing registry and create a new one?
291  @param dryrun Don't do anything permanent?
292  @param name Filename of the registry
293  @return Database connection
294  """
295  if dryrun:
296  return fakeContext()
297 
298  registryName = os.path.join(directory, name)
299  context = RegistryContext(registryName, self.createTablecreateTable, create, self.config.permissions)
300  return context
301 
302  def createTable(self, conn, table=None, forceCreateTables=False):
303  """Create the registry tables
304 
305  One table (typically 'raw') contains information on all files, and the
306  other (typically 'raw_visit') contains information on all visits.
307 
308  @param conn Database connection
309  @param table Name of table to create in database
310  """
311  cursor = conn.cursor()
312  if table is None:
313  table = self.config.table
314  cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % table
315  cursor.execute(cmd)
316  if cursor.fetchone() and not forceCreateTables: # Assume if we get an answer the table exists
317  self.log.info('Table "%s" exists. Skipping creation', table)
318  return
319  else:
320  cmd = "drop table if exists %s" % table
321  cursor.execute(cmd)
322  cmd = "drop table if exists %s_visit" % table
323  cursor.execute(cmd)
324 
325  cmd = "create table %s (id integer primary key autoincrement, " % table
326  cmd += ",".join([("%s %s" % (col, colType)) for col, colType in self.config.columns.items()])
327  if len(self.config.unique) > 0:
328  cmd += ", unique(" + ",".join(self.config.unique) + ")"
329  cmd += ")"
330  cursor.execute(cmd)
331 
332  cmd = "create table %s_visit (" % table
333  cmd += ",".join([("%s %s" % (col, self.config.columns[col])) for col in self.config.visit])
334  cmd += ", unique(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
335  cmd += ")"
336  cursor.execute(cmd)
337 
338  conn.commit()
339 
340  def check(self, conn, info, table=None):
341  """Check for the presence of a row already
342 
343  Not sure this is required, given the 'ignore' configuration option.
344  """
345  if table is None:
346  table = self.config.table
347  if self.config.ignore or len(self.config.unique) == 0:
348  return False # Our entry could already be there, but we don't care
349  cursor = conn.cursor()
350  sql = "SELECT COUNT(*) FROM %s WHERE " % table
351  sql += " AND ".join(["%s = %s" % (col, self.placeHolderplaceHolder) for col in self.config.unique])
352  values = [self.typemaptypemap[self.config.columns[col]](info[col]) for col in self.config.unique]
353 
354  cursor.execute(sql, values)
355  if cursor.fetchone()[0] > 0:
356  return True
357  return False
358 
359  def addRow(self, conn, info, dryrun=False, create=False, table=None):
360  """Add a row to the file table (typically 'raw').
361 
362  @param conn Database connection
363  @param info File properties to add to database
364  @param table Name of table in database
365  """
366  with conn:
367  if table is None:
368  table = self.config.table
369  ignoreClause = ""
370  if self.config.ignore:
371  ignoreClause = " OR IGNORE"
372  sql = "INSERT%s INTO %s (%s) VALUES (" % (ignoreClause, table, ",".join(self.config.columns))
373  sql += ",".join([self.placeHolderplaceHolder] * len(self.config.columns)) + ")"
374  values = [self.typemaptypemap[tt](info[col]) for col, tt in self.config.columns.items()]
375 
376  if dryrun:
377  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
378  else:
379  conn.cursor().execute(sql, values)
380 
381  sql = "INSERT OR IGNORE INTO %s_visit VALUES (" % table
382  sql += ",".join([self.placeHolderplaceHolder] * len(self.config.visit)) + ")"
383  values = [self.typemaptypemap[self.config.columns[col]](info[col]) for col in self.config.visit]
384 
385  if dryrun:
386  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
387  else:
388  conn.cursor().execute(sql, values)
389 
390 
392  """Configuration for IngestTask"""
393  parse = ConfigurableField(target=ParseTask, doc="File parsing")
394  register = ConfigurableField(target=RegisterTask, doc="Registry entry")
395  allowError = Field(dtype=bool, default=False, doc="Allow error in ingestion?")
396  clobber = Field(dtype=bool, default=False, doc="Clobber existing file?")
397 
398 
399 class IngestError(RuntimeError):
400  def __init__(self, message, pathname, position):
401  super().__init__(message)
402  self.pathnamepathname = pathname
403  self.positionposition = position
404 
405 
406 class IngestTask(Task):
407  """Task that will ingest images into the data repository"""
408  ConfigClass = IngestConfig
409  ArgumentParser = IngestArgumentParser
410  _DefaultName = "ingest"
411 
412  def __init__(self, *args, **kwargs):
413  super(IngestTask, self).__init__(*args, **kwargs)
414  self.makeSubtask("parse")
415  self.makeSubtask("register")
416 
417  @classmethod
418  def _parse(cls):
419  """Parse the command-line arguments and return them along with a Task
420  instance."""
421  config = cls.ConfigClassConfigClass()
422  parser = cls.ArgumentParserArgumentParser(name=cls._DefaultName_DefaultName)
423  args = parser.parse_args(config)
424  task = cls(config=args.config)
425  return task, args
426 
427  @classmethod
428  def parseAndRun(cls):
429  """Parse the command-line arguments and run the Task."""
430  task, args = cls._parse_parse()
431  task.run(args)
432 
433  @classmethod
434  def prepareTask(cls, root=None, dryrun=False, mode="move", create=False,
435  ignoreIngested=False):
436  """Prepare for running the task repeatedly with `ingestFiles`.
437 
438  Saves the parsed arguments, including the Butler and log, as a
439  private instance variable.
440 
441  Parameters
442  ----------
443  root : `str`, optional
444  Repository root pathname. If None, run the Task using the
445  command line arguments, ignoring all other arguments below.
446  dryrun : `bool`, optional
447  If True, don't perform any action; log what would have happened.
448  mode : `str`, optional
449  How files are delivered to their destination. Default is "move",
450  unlike the command-line default of "link".
451  create : `bool`, optional
452  If True, create a new registry, clobbering any old one present.
453  ignoreIngested : `bool`, optional
454  If True, do not complain if the file is already present in the
455  registry (and do nothing else).
456 
457  Returns
458  -------
459  task : `IngestTask`
460  If `root` was provided, the IngestTask instance
461  """
462  sys.argv = ["IngestTask"]
463  sys.argv.append(root)
464  if dryrun:
465  sys.argv.append("--dry-run")
466  sys.argv.append("--mode")
467  sys.argv.append(mode)
468  if create:
469  sys.argv.append("--create")
470  if ignoreIngested:
471  sys.argv.append("--ignore-ingested")
472  sys.argv.append("__fakefile__") # needed for parsing, not used
473 
474  task, args = cls._parse_parse()
475  task._args = args
476  return task
477 
478  def ingest(self, infile, outfile, mode="move", dryrun=False):
479  """Ingest a file into the image repository.
480 
481  @param infile Name of input file
482  @param outfile Name of output file (file in repository)
483  @param mode Mode of ingest (copy/link/move/skip)
484  @param dryrun Only report what would occur?
485  @param Success boolean
486  """
487  if mode == "skip":
488  return True
489  if dryrun:
490  self.log.info("Would %s from %s to %s", mode, infile, outfile)
491  return True
492  try:
493  outdir = os.path.dirname(outfile)
494  if not os.path.isdir(outdir):
495  try:
496  os.makedirs(outdir)
497  except OSError as exc:
498  # Silently ignore mkdir failures due to race conditions
499  if not os.path.isdir(outdir):
500  raise RuntimeError(f"Failed to create directory {outdir}") from exc
501  if os.path.lexists(outfile):
502  if self.config.clobber:
503  os.unlink(outfile)
504  else:
505  raise RuntimeError("File %s already exists; consider --config clobber=True" % outfile)
506 
507  if mode == "copy":
508  assertCanCopy(infile, outfile)
509  shutil.copyfile(infile, outfile)
510  elif mode == "link":
511  if os.path.exists(outfile):
512  if os.path.samefile(infile, outfile):
513  self.log.debug("Already linked %s to %s: ignoring", infile, outfile)
514  else:
515  self.log.warning("%s already has a file at the target location (%s): ignoring "
516  "(set clobber=True to overwrite)", infile, outfile)
517  return False
518  os.symlink(os.path.abspath(infile), outfile)
519  elif mode == "move":
520  assertCanCopy(infile, outfile)
521  shutil.move(infile, outfile)
522  else:
523  raise AssertionError("Unknown mode: %s" % mode)
524  self.log.info("%s --<%s>--> %s", infile, mode, outfile)
525  except Exception as e:
526  self.log.warning("Failed to %s %s to %s: %s", mode, infile, outfile, e)
527  if not self.config.allowError:
528  raise RuntimeError(f"Failed to {mode} {infile} to {outfile}") from e
529  return False
530  return True
531 
532  def isBadFile(self, filename, badFileList):
533  """Return whether the file qualifies as bad
534 
535  We match against the list of bad file patterns.
536  """
537  filename = os.path.basename(filename)
538  if not badFileList:
539  return False
540  for badFile in badFileList:
541  if fnmatch(filename, badFile):
542  return True
543  return False
544 
545  def isBadId(self, info, badIdList):
546  """Return whether the file information qualifies as bad
547 
548  We match against the list of bad data identifiers.
549  """
550  if not badIdList:
551  return False
552  for badId in badIdList:
553  if all(info[key] == value for key, value in badId.items()):
554  return True
555  return False
556 
557  def expandFiles(self, fileNameList):
558  """!Expand a set of filenames and globs, returning a list of filenames
559 
560  @param fileNameList A list of files and glob patterns
561 
562  N.b. globs obey Posix semantics, so a pattern that matches nothing is returned unchanged
563  """
564  filenameList = []
565  for globPattern in fileNameList:
566  files = glob(globPattern)
567 
568  if not files: # posix behaviour is to return pattern unchanged
569  self.log.warning("%s doesn't match any file", globPattern)
570  continue
571 
572  filenameList.extend(files)
573 
574  return filenameList
575 
576  def runFile(self, infile, registry, args, pos):
577  """!Examine and ingest a single file
578 
579  @param infile: File to process
580  @param registry: Registry into which to insert Butler metadata, or None
581  @param args: Parsed command-line arguments
582  @param pos: Position number of this file in the input list
583  @return parsed information from FITS HDUs if registry is None; or None
584  """
585  if self.isBadFileisBadFile(infile, args.badFile):
586  self.log.info("Skipping declared bad file %s", infile)
587  return None
588  try:
589  fileInfo, hduInfoList = self.parse.getInfo(infile)
590  except Exception as e:
591  if not self.config.allowError:
592  raise RuntimeError(f"Error parsing {infile}") from e
593  self.log.warning("Error parsing %s (%s); skipping", infile, e)
594  return None
595  if self.isBadIdisBadId(fileInfo, args.badId.idList):
596  self.log.info("Skipping declared bad file %s: %s", infile, fileInfo)
597  return None
598  if registry is not None and self.register.check(registry, fileInfo):
599  if args.ignoreIngested:
600  return None
601  self.log.warning("%s: already ingested: %s", infile, fileInfo)
602  outfile = self.parse.getDestination(args.butler, fileInfo, infile)
603  if not self.ingestingest(infile, outfile, mode=args.mode, dryrun=args.dryrun):
604  return None
605  if hduInfoList is None:
606  return None
607  if registry is None:
608  return hduInfoList
609  for info in hduInfoList:
610  try:
611  self.register.addRow(registry, info, dryrun=args.dryrun, create=args.create)
612  except Exception as exc:
613  raise IngestError(f"Failed to register file {infile}", infile, pos) from exc
614  return None # No further registration should be performed
615 
616  def run(self, args):
617  """Ingest all specified files and add them to the registry"""
618  filenameList = self.expandFilesexpandFiles(args.files)
619  root = args.input
620  context = self.register.openRegistry(root, create=args.create, dryrun=args.dryrun)
621  with context as registry:
622  for pos in range(len(filenameList)):
623  infile = filenameList[pos]
624  try:
625  self.runFilerunFile(infile, registry, args, pos)
626  except Exception as exc:
627  self.log.warning("Failed to ingest file %s: %s", infile, exc)
628  if not self.config.allowError:
629  raise IngestError(f"Failed to ingest file {infile}", infile, pos) from exc
630  continue
631 
632  def ingestFiles(self, fileList):
633  """Ingest specified file or list of files and add them to the registry.
634 
635  This method can only be called if `prepareTask` was used.
636 
637  Parameters
638  ----------
639  fileList : `str` or `list` [`str`]
640  Pathname or list of pathnames of files to ingest.
641  """
642  if not hasattr(self, "_args"):
643  raise RuntimeError("Task not created with prepareTask")
644  if isinstance(fileList, str):
645  fileList = [fileList]
646  self._args.files = fileList
647  self.runrun(self._args)
648 
649 
650 def assertCanCopy(fromPath, toPath):
651  """Can I copy a file? Raise an exception is space constraints not met.
652 
653  @param fromPath Path from which the file is being copied
654  @param toPath Path to which the file is being copied
655  """
656  req = os.stat(fromPath).st_size
657  st = os.statvfs(os.path.dirname(toPath))
658  avail = st.f_bavail * st.f_frsize
659  if avail < req:
660  raise RuntimeError("Insufficient space: %d vs %d" % (req, avail))
def __init__(self, *args, **kwargs)
Definition: ingest.py:40
def __init__(self, message, pathname, position)
Definition: ingest.py:400
def expandFiles(self, fileNameList)
Expand a set of filenames and globs, returning a list of filenames.
Definition: ingest.py:557
def runFile(self, infile, registry, args, pos)
Examine and ingest a single file.
Definition: ingest.py:576
def ingestFiles(self, fileList)
Definition: ingest.py:632
def __init__(self, *args, **kwargs)
Definition: ingest.py:412
def prepareTask(cls, root=None, dryrun=False, mode="move", create=False, ignoreIngested=False)
Definition: ingest.py:435
def isBadFile(self, filename, badFileList)
Definition: ingest.py:532
def isBadId(self, info, badIdList)
Definition: ingest.py:545
def ingest(self, infile, outfile, mode="move", dryrun=False)
Definition: ingest.py:478
def getInfoFromMetadata(self, md, info=None)
Definition: ingest.py:145
def translate_date(self, md)
Definition: ingest.py:182
def getDestination(self, butler, info, filename)
Definition: ingest.py:207
def getInfo(self, filename)
Definition: ingest.py:80
def translate_filter(self, md)
Definition: ingest.py:194
def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3")
Definition: ingest.py:286
def check(self, conn, info, table=None)
Definition: ingest.py:340
def addRow(self, conn, info, dryrun=False, create=False, table=None)
Definition: ingest.py:359
def createTable(self, conn, table=None, forceCreateTables=False)
Definition: ingest.py:302
def __exit__(self, excType, excValue, traceback)
Definition: ingest.py:265
def __init__(self, registryName, createTableFunc, forceCreateTables, permissions)
Definition: ingest.py:249
daf::base::PropertySet * set
Definition: fits.cc:912
bool strip
Definition: fits.cc:911
std::shared_ptr< daf::base::PropertyList > readMetadata(std::string const &fileName, int hdu=DEFAULT_HDU, bool strip=false)
Read FITS header.
Definition: fits.cc:1657
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
def assertCanCopy(fromPath, toPath)
Definition: ingest.py:650