LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
Public Member Functions | Static Public Attributes | List of all members
lsst.pipe.tasks.ingest.RegisterTask Class Reference
Inheritance diagram for lsst.pipe.tasks.ingest.RegisterTask:
lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask

Public Member Functions

def openRegistry (self, directory, create=False, dryrun=False, name="registry.sqlite3")
 
def createTable (self, conn, table=None, forceCreateTables=False)
 
def check (self, conn, info, table=None)
 
def addRow (self, conn, info, dryrun=False, create=False, table=None)
 

Static Public Attributes

 ConfigClass = RegisterConfig
 
string placeHolder = '?'
 
dictionary typemap = {'text': str, 'int': int, 'double': float}
 

Detailed Description

Task that will generate the registry for the Mapper

Definition at line 280 of file ingest.py.

Member Function Documentation

◆ addRow()

def lsst.pipe.tasks.ingest.RegisterTask.addRow (   self,
  conn,
  info,
  dryrun = False,
  create = False,
  table = None 
)
Add a row to the file table (typically 'raw').

@param conn    Database connection
@param info    File properties to add to database
@param table   Name of table in database

Definition at line 359 of file ingest.py.

359  def addRow(self, conn, info, dryrun=False, create=False, table=None):
360  """Add a row to the file table (typically 'raw').
361 
362  @param conn Database connection
363  @param info File properties to add to database
364  @param table Name of table in database
365  """
366  with conn:
367  if table is None:
368  table = self.config.table
369  ignoreClause = ""
370  if self.config.ignore:
371  ignoreClause = " OR IGNORE"
372  sql = "INSERT%s INTO %s (%s) VALUES (" % (ignoreClause, table, ",".join(self.config.columns))
373  sql += ",".join([self.placeHolder] * len(self.config.columns)) + ")"
374  values = [self.typemap[tt](info[col]) for col, tt in self.config.columns.items()]
375 
376  if dryrun:
377  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
378  else:
379  conn.cursor().execute(sql, values)
380 
381  sql = "INSERT OR IGNORE INTO %s_visit VALUES (" % table
382  sql += ",".join([self.placeHolder] * len(self.config.visit)) + ")"
383  values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.visit]
384 
385  if dryrun:
386  print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
387  else:
388  conn.cursor().execute(sql, values)
389 
390 

◆ check()

def lsst.pipe.tasks.ingest.RegisterTask.check (   self,
  conn,
  info,
  table = None 
)
Check for the presence of a row already

Not sure this is required, given the 'ignore' configuration option.

Definition at line 340 of file ingest.py.

340  def check(self, conn, info, table=None):
341  """Check for the presence of a row already
342 
343  Not sure this is required, given the 'ignore' configuration option.
344  """
345  if table is None:
346  table = self.config.table
347  if self.config.ignore or len(self.config.unique) == 0:
348  return False # Our entry could already be there, but we don't care
349  cursor = conn.cursor()
350  sql = "SELECT COUNT(*) FROM %s WHERE " % table
351  sql += " AND ".join(["%s = %s" % (col, self.placeHolder) for col in self.config.unique])
352  values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.unique]
353 
354  cursor.execute(sql, values)
355  if cursor.fetchone()[0] > 0:
356  return True
357  return False
358 

◆ createTable()

def lsst.pipe.tasks.ingest.RegisterTask.createTable (   self,
  conn,
  table = None,
  forceCreateTables = False 
)
Create the registry tables

One table (typically 'raw') contains information on all files, and the
other (typically 'raw_visit') contains information on all visits.

@param conn    Database connection
@param table   Name of table to create in database

Definition at line 302 of file ingest.py.

302  def createTable(self, conn, table=None, forceCreateTables=False):
303  """Create the registry tables
304 
305  One table (typically 'raw') contains information on all files, and the
306  other (typically 'raw_visit') contains information on all visits.
307 
308  @param conn Database connection
309  @param table Name of table to create in database
310  """
311  cursor = conn.cursor()
312  if table is None:
313  table = self.config.table
314  cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % table
315  cursor.execute(cmd)
316  if cursor.fetchone() and not forceCreateTables: # Assume if we get an answer the table exists
317  self.log.info('Table "%s" exists. Skipping creation', table)
318  return
319  else:
320  cmd = "drop table if exists %s" % table
321  cursor.execute(cmd)
322  cmd = "drop table if exists %s_visit" % table
323  cursor.execute(cmd)
324 
325  cmd = "create table %s (id integer primary key autoincrement, " % table
326  cmd += ",".join([("%s %s" % (col, colType)) for col, colType in self.config.columns.items()])
327  if len(self.config.unique) > 0:
328  cmd += ", unique(" + ",".join(self.config.unique) + ")"
329  cmd += ")"
330  cursor.execute(cmd)
331 
332  cmd = "create table %s_visit (" % table
333  cmd += ",".join([("%s %s" % (col, self.config.columns[col])) for col in self.config.visit])
334  cmd += ", unique(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
335  cmd += ")"
336  cursor.execute(cmd)
337 
338  conn.commit()
339 
daf::base::PropertySet * set
Definition: fits.cc:912

◆ openRegistry()

def lsst.pipe.tasks.ingest.RegisterTask.openRegistry (   self,
  directory,
  create = False,
  dryrun = False,
  name = "registry.sqlite3" 
)
Open the registry and return the connection handle.

@param directory  Directory in which the registry file will be placed
@param create  Clobber any existing registry and create a new one?
@param dryrun  Don't do anything permanent?
@param name    Filename of the registry
@return Database connection

Reimplemented in lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.

Definition at line 286 of file ingest.py.

286  def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"):
287  """Open the registry and return the connection handle.
288 
289  @param directory Directory in which the registry file will be placed
290  @param create Clobber any existing registry and create a new one?
291  @param dryrun Don't do anything permanent?
292  @param name Filename of the registry
293  @return Database connection
294  """
295  if dryrun:
296  return fakeContext()
297 
298  registryName = os.path.join(directory, name)
299  context = RegistryContext(registryName, self.createTable, create, self.config.permissions)
300  return context
301 

Member Data Documentation

◆ ConfigClass

lsst.pipe.tasks.ingest.RegisterTask.ConfigClass = RegisterConfig
static

Definition at line 282 of file ingest.py.

◆ placeHolder

string lsst.pipe.tasks.ingest.RegisterTask.placeHolder = '?'
static

Definition at line 283 of file ingest.py.

◆ typemap

dictionary lsst.pipe.tasks.ingest.RegisterTask.typemap = {'text': str, 'int': int, 'double': float}
static

Definition at line 284 of file ingest.py.


The documentation for this class was generated from the following file: