lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask Class Reference

Inheritance diagram for lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask:

Public Member Functions
def	openRegistry (self, directory, create=False, dryrun=False)
def	createTable (self, conn, table=None)
def	openRegistry (self, directory, create=False, dryrun=False, name="registry.sqlite3")
def	createTable (self, conn, table=None, forceCreateTables=False)
def	check (self, conn, info, table=None)
def	addRow (self, conn, info, dryrun=False, create=False, table=None)

Static Public Attributes
string	placeHolder = "%s"
	ConfigClass = RegisterConfig
dictionary	typemap = {'text': str, 'int': int, 'double': float}

Detailed Description

Definition at line 49 of file ingestPgsql.py.

Member Function Documentation

addRow()

def lsst.pipe.tasks.ingest.RegisterTask.addRow (   self,   conn,   info,   dryrun = False,   create = False,   table = None  )

inherited

Add a row to the file table (typically 'raw').

@param conn    Database connection
@param info    File properties to add to database
@param table   Name of table in database

Definition at line 359 of file ingest.py.

    def addRow(self, conn, info, dryrun=False, create=False, table=None):
        """Add a row to the file table (typically 'raw').
 
        @param conn    Database connection
        @param info    File properties to add to database
        @param table   Name of table in database
        """
        with conn:
            if table is None:
                table = self.config.table
            ignoreClause = ""
            if self.config.ignore:
                ignoreClause = " OR IGNORE"
            sql = "INSERT%s INTO %s (%s) VALUES (" % (ignoreClause, table, ",".join(self.config.columns))
            sql += ",".join([self.placeHolder] * len(self.config.columns)) + ")"
            values = [self.typemap[tt](info[col]) for col, tt in self.config.columns.items()]
 
            if dryrun:
                print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
            else:
                conn.cursor().execute(sql, values)
 
            sql = "INSERT OR IGNORE INTO %s_visit VALUES (" % table
            sql += ",".join([self.placeHolder] * len(self.config.visit)) + ")"
            values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.visit]
 
            if dryrun:
                print("Would execute: '%s' with %s" % (sql, ",".join([str(value) for value in values])))
            else:
                conn.cursor().execute(sql, values)
 
 

check()

def lsst.pipe.tasks.ingest.RegisterTask.check (   self,   conn,   info,   table = None  )

inherited

Check for the presence of a row already

Not sure this is required, given the 'ignore' configuration option.

Definition at line 340 of file ingest.py.

    def check(self, conn, info, table=None):
        """Check for the presence of a row already
 
        Not sure this is required, given the 'ignore' configuration option.
        """
        if table is None:
            table = self.config.table
        if self.config.ignore or len(self.config.unique) == 0:
            return False  # Our entry could already be there, but we don't care
        cursor = conn.cursor()
        sql = "SELECT COUNT(*) FROM %s WHERE " % table
        sql += " AND ".join(["%s = %s" % (col, self.placeHolder) for col in self.config.unique])
        values = [self.typemap[self.config.columns[col]](info[col]) for col in self.config.unique]
 
        cursor.execute(sql, values)
        if cursor.fetchone()[0] > 0:
            return True
        return False
 

createTable() [1/2]

def lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask.createTable	(		self,
			conn,
			table = None
	)

Create the registry tables

One table (typically 'raw') contains information on all files, and the
other (typically 'raw_visit') contains information on all visits.

This method is required because there's a slightly different syntax
compared to SQLite (FLOAT instead of DOUBLE, SERIAL instead of
AUTOINCREMENT).

@param conn    Database connection
@param table   Name of table to create in database

Definition at line 65 of file ingestPgsql.py.

    def createTable(self, conn, table=None):
        """Create the registry tables
 
        One table (typically 'raw') contains information on all files, and the
        other (typically 'raw_visit') contains information on all visits.
 
        This method is required because there's a slightly different syntax
        compared to SQLite (FLOAT instead of DOUBLE, SERIAL instead of
        AUTOINCREMENT).
 
        @param conn    Database connection
        @param table   Name of table to create in database
        """
        if table is None:
            table = self.config.table
 
        typeMap = {'int': 'INT',
                   'double': 'FLOAT',  # Defaults to double precision
                   }
 
        cur = conn.cursor()
        cmd = "CREATE TABLE %s (id SERIAL NOT NULL PRIMARY KEY, " % table
        cmd += ",".join(["%s %s" % (col, typeMap.get(colType.lower(), 'text')) for
                         col, colType in self.config.columns.items()])
        if len(self.config.unique) > 0:
            cmd += ", UNIQUE(" + ",".join(self.config.unique) + ")"
        cmd += ")"
        cur.execute(cmd)
 
        cmd = "CREATE TABLE %s_visit (" % self.config.table
        cmd += ",".join(["%s %s" % (col, typeMap.get(self.config.columns[col].lower(), 'TEXT')) for
                         col in self.config.visit])
        cmd += ", UNIQUE(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
        cmd += ")"
        cur.execute(cmd)
        del cur
        conn.commit()
 
 

createTable() [2/2]

def lsst.pipe.tasks.ingest.RegisterTask.createTable (   self,   conn,   table = None,   forceCreateTables = False  )

inherited

Create the registry tables

One table (typically 'raw') contains information on all files, and the
other (typically 'raw_visit') contains information on all visits.

@param conn    Database connection
@param table   Name of table to create in database

Definition at line 302 of file ingest.py.

    def createTable(self, conn, table=None, forceCreateTables=False):
        """Create the registry tables
 
        One table (typically 'raw') contains information on all files, and the
        other (typically 'raw_visit') contains information on all visits.
 
        @param conn    Database connection
        @param table   Name of table to create in database
        """
        cursor = conn.cursor()
        if table is None:
            table = self.config.table
        cmd = "SELECT name FROM sqlite_master WHERE type='table' AND name='%s'" % table
        cursor.execute(cmd)
        if cursor.fetchone() and not forceCreateTables:  # Assume if we get an answer the table exists
            self.log.info('Table "%s" exists.  Skipping creation', table)
            return
        else:
            cmd = "drop table if exists %s" % table
            cursor.execute(cmd)
            cmd = "drop table if exists %s_visit" % table
            cursor.execute(cmd)
 
        cmd = "create table %s (id integer primary key autoincrement, " % table
        cmd += ",".join([("%s %s" % (col, colType)) for col, colType in self.config.columns.items()])
        if len(self.config.unique) > 0:
            cmd += ", unique(" + ",".join(self.config.unique) + ")"
        cmd += ")"
        cursor.execute(cmd)
 
        cmd = "create table %s_visit (" % table
        cmd += ",".join([("%s %s" % (col, self.config.columns[col])) for col in self.config.visit])
        cmd += ", unique(" + ",".join(set(self.config.visit).intersection(set(self.config.unique))) + ")"
        cmd += ")"
        cursor.execute(cmd)
 
        conn.commit()
 

openRegistry() [1/2]

def lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask.openRegistry	(		self,
			directory,
			create = False,
			dryrun = False
	)

Open the registry and return the connection handle.

@param directory  Directory in which the registry file will be placed
@param create  Clobber any existing registry and create a new one?
@param dryrun  Don't do anything permanent?
@return Database connection

Definition at line 52 of file ingestPgsql.py.

    def openRegistry(self, directory, create=False, dryrun=False):
        """Open the registry and return the connection handle.
 
        @param directory  Directory in which the registry file will be placed
        @param create  Clobber any existing registry and create a new one?
        @param dryrun  Don't do anything permanent?
        @return Database connection
        """
        if dryrun:
            return fakeContext()
        registryName = os.path.join(directory, "registry.pgsql")
        return PgsqlRegistryContext(registryName, self.createTable, create)
 

openRegistry() [2/2]

def lsst.pipe.tasks.ingest.RegisterTask.openRegistry (   self,   directory,   create = False,   dryrun = False,   name = "registry.sqlite3"  )

inherited

Open the registry and return the connection handle.

@param directory  Directory in which the registry file will be placed
@param create  Clobber any existing registry and create a new one?
@param dryrun  Don't do anything permanent?
@param name    Filename of the registry
@return Database connection

Reimplemented in lsst.pipe.tasks.ingestCalibs.CalibsRegisterTask.

Definition at line 286 of file ingest.py.

    def openRegistry(self, directory, create=False, dryrun=False, name="registry.sqlite3"):
        """Open the registry and return the connection handle.
 
        @param directory  Directory in which the registry file will be placed
        @param create  Clobber any existing registry and create a new one?
        @param dryrun  Don't do anything permanent?
        @param name    Filename of the registry
        @return Database connection
        """
        if dryrun:
            return fakeContext()
 
        registryName = os.path.join(directory, name)
        context = RegistryContext(registryName, self.createTable, create, self.config.permissions)
        return context
 

Member Data Documentation

ConfigClass

lsst.pipe.tasks.ingest.RegisterTask.ConfigClass = RegisterConfig

staticinherited

Definition at line 282 of file ingest.py.

placeHolder

string lsst.pipe.tasks.ingestPgsql.PgsqlRegisterTask.placeHolder = "%s"

static

Definition at line 50 of file ingestPgsql.py.

typemap

dictionary lsst.pipe.tasks.ingest.RegisterTask.typemap = {'text': str, 'int': int, 'double': float}

staticinherited

Definition at line 284 of file ingest.py.

---

The documentation for this class was generated from the following file:

• /j/snowflake/release/lsstsw/stack/lsst-scipipe-0.7.0/Linux64/pipe_tasks/21.0.0-147-g0e635eb1+1acddb5be5/python/lsst/pipe/tasks/ingestPgsql.py