LSSTApplications  19.0.0,19.0.0+1,19.0.0+10,19.0.0+13,19.0.0+3,19.0.0+5,19.0.0+9,tickets.DM-22703-ga158cbef15,w.2019.51
LSSTDataManagementBasePackage
Public Member Functions | Public Attributes | List of all members
lsst.dax.ppdb.ppdbSchema.PpdbSchema Class Reference
Inheritance diagram for lsst.dax.ppdb.ppdbSchema.PpdbSchema:

Public Member Functions

def __init__ (self, engine, dia_object_index, dia_object_nightly, schema_file, extra_schema_file=None, column_map=None, afw_schemas=None, prefix="")
 
def makeSchema (self, drop=False, mysql_engine='InnoDB', oracle_tablespace=None, oracle_iot=False)
 
def getAfwSchema (self, table_name, columns=None)
 
def getAfwColumns (self, table_name)
 
def getColumnMap (self, table_name)
 

Public Attributes

 objects
 
 objects_nightly
 
 objects_last
 
 sources
 
 forcedSources
 
 visits
 

Detailed Description

Class for management of PPDB schema.

Attributes
----------
objects : `sqlalchemy.Table`
    DiaObject table instance
objects_nightly : `sqlalchemy.Table`
    DiaObjectNightly table instance, may be None
objects_last : `sqlalchemy.Table`
    DiaObjectLast table instance, may be None
sources : `sqlalchemy.Table`
    DiaSource table instance
forcedSources : `sqlalchemy.Table`
    DiaForcedSource table instance
visits : `sqlalchemy.Table`
    PpdbProtoVisits table instance

Parameters
----------
engine : `sqlalchemy.engine.Engine`
    SQLAlchemy engine instance
dia_object_index : `str`
    Indexing mode for DiaObject table, see `PpdbConfig.dia_object_index`
    for details.
dia_object_nightly : `bool`
    If `True` then create per-night DiaObject table as well.
schema_file : `str`
    Name of the YAML schema file.
extra_schema_file : `str`, optional
    Name of the YAML schema file with extra column definitions.
column_map : `str`, optional
    Name of the YAML file with column mappings.
afw_schemas : `dict`, optional
    Dictionary with table name for a key and `afw.table.Schema`
    for a value. Columns in schema will be added to standard PPDB
    schema (only if standard schema does not have matching column).
prefix : `str`, optional
    Prefix to add to all scheam elements.

Definition at line 161 of file ppdbSchema.py.

Constructor & Destructor Documentation

◆ __init__()

def lsst.dax.ppdb.ppdbSchema.PpdbSchema.__init__ (   self,
  engine,
  dia_object_index,
  dia_object_nightly,
  schema_file,
  extra_schema_file = None,
  column_map = None,
  afw_schemas = None,
  prefix = "" 
)

Definition at line 220 of file ppdbSchema.py.

220  afw_schemas=None, prefix=""):
221 
222  self._engine = engine
223  self._dia_object_index = dia_object_index
224  self._dia_object_nightly = dia_object_nightly
225  self._prefix = prefix
226 
227  self._metadata = MetaData(self._engine)
228 
229  self.objects = None
230  self.objects_nightly = None
231  self.objects_last = None
232  self.sources = None
233  self.forcedSources = None
234  self.visits = None
235 
236  if column_map:
237  _LOG.debug("Reading column map file %s", column_map)
238  with open(column_map) as yaml_stream:
239  # maps cat column name to afw column name
240  self._column_map = yaml.load(yaml_stream, Loader=yaml.SafeLoader)
241  _LOG.debug("column map: %s", self._column_map)
242  else:
243  _LOG.debug("No column map file is given, initialize to empty")
244  self._column_map = {}
245  self._column_map_reverse = {}
246  for table, cmap in self._column_map.items():
247  # maps afw column name to cat column name
248  self._column_map_reverse[table] = {v: k for k, v in cmap.items()}
249  _LOG.debug("reverse column map: %s", self._column_map_reverse)
250 
251  # build complete table schema
252  self._schemas = self._buildSchemas(schema_file, extra_schema_file,
253  afw_schemas)
254 
255  # map cat column types to alchemy
256  self._type_map = dict(DOUBLE=self._getDoubleType(),
257  FLOAT=sqlalchemy.types.Float,
258  DATETIME=sqlalchemy.types.TIMESTAMP,
259  BIGINT=sqlalchemy.types.BigInteger,
260  INTEGER=sqlalchemy.types.Integer,
261  INT=sqlalchemy.types.Integer,
262  TINYINT=sqlalchemy.types.Integer,
263  BLOB=sqlalchemy.types.LargeBinary,
264  CHAR=sqlalchemy.types.CHAR,
265  BOOL=sqlalchemy.types.Boolean)
266 
267  # generate schema for all tables, must be called last
268  self._makeTables()
269 
items
std::vector< SchemaItem< Flag > > * items
Definition: BaseColumnView.cc:142

Member Function Documentation

◆ getAfwColumns()

def lsst.dax.ppdb.ppdbSchema.PpdbSchema.getAfwColumns (   self,
  table_name 
) 
Returns mapping of afw column names to Column definitions.

Parameters
----------
table_name : `str`
    One of known PPDB table names.

Returns
-------
column_map : `dict`
    Mapping of afw column names to `ColumnDef` instances.

Definition at line 435 of file ppdbSchema.py.

435  def getAfwColumns(self, table_name):
436  """Returns mapping of afw column names to Column definitions.
437 
438  Parameters
439  ----------
440  table_name : `str`
441  One of known PPDB table names.
442 
443  Returns
444  -------
445  column_map : `dict`
446  Mapping of afw column names to `ColumnDef` instances.
447  """
448  table = self._schemas[table_name]
449  col_map = self._column_map.get(table_name, {})
450 
451  cmap = {}
452  for column in table.columns:
453  afw_name = col_map.get(column.name, column.name)
454  cmap[afw_name] = column
455  return cmap
456 

◆ getAfwSchema()

def lsst.dax.ppdb.ppdbSchema.PpdbSchema.getAfwSchema (   self,
  table_name,
  columns = None 
) 
Return afw schema for given table.

Parameters
----------
table_name : `str`
    One of known PPDB table names.
columns : `list` of `str`, optional
    Include only given table columns in schema, by default all columns
    are included.

Returns
-------
schema : `lsst.afw.table.Schema`
column_map : `dict`
    Mapping of the table/result column names into schema key.

Definition at line 367 of file ppdbSchema.py.

367  def getAfwSchema(self, table_name, columns=None):
368  """Return afw schema for given table.
369 
370  Parameters
371  ----------
372  table_name : `str`
373  One of known PPDB table names.
374  columns : `list` of `str`, optional
375  Include only given table columns in schema, by default all columns
376  are included.
377 
378  Returns
379  -------
380  schema : `lsst.afw.table.Schema`
381  column_map : `dict`
382  Mapping of the table/result column names into schema key.
383  """
384 
385  table = self._schemas[table_name]
386  col_map = self._column_map.get(table_name, {})
387 
388  # make a schema
389  col2afw = {}
390  schema = afwTable.SourceTable.makeMinimalSchema()
391  for column in table.columns:
392  if columns and column.name not in columns:
393  continue
394  afw_col = col_map.get(column.name, column.name)
395  if afw_col in schema.getNames():
396  # Continue if the column is already in the minimal schema.
397  key = schema.find(afw_col).getKey()
398  elif column.type in ("DOUBLE", "FLOAT") and column.unit == "deg":
399  #
400  # NOTE: degree to radian conversion is not supported (yet)
401  #
402  # angles in afw are radians and have special "Angle" type
403  key = schema.addField(afw_col,
404  type="Angle",
405  doc=column.description or "",
406  units="rad")
407  elif column.type == "BLOB":
408  # No BLOB support for now
409  key = None
410  else:
411  units = column.unit or ""
412  # some units in schema are not recognized by afw but we do not care
413  if self._afw_type_map_reverse[column.type] == 'String':
414  key = schema.addField(afw_col,
415  type=self._afw_type_map_reverse[column.type],
416  doc=column.description or "",
417  units=units,
418  parse_strict="silent",
419  size=10)
420  elif units == "deg":
421  key = schema.addField(afw_col,
422  type='Angle',
423  doc=column.description or "",
424  parse_strict="silent")
425  else:
426  key = schema.addField(afw_col,
427  type=self._afw_type_map_reverse[column.type],
428  doc=column.description or "",
429  units=units,
430  parse_strict="silent")
431  col2afw[column.name] = key
432 
433  return schema, col2afw
434 

◆ getColumnMap()

def lsst.dax.ppdb.ppdbSchema.PpdbSchema.getColumnMap (   self,
  table_name 
) 
Returns mapping of column names to Column definitions.

Parameters
----------
table_name : `str`
    One of known PPDB table names.

Returns
-------
column_map : `dict`
    Mapping of column names to `ColumnDef` instances.

Definition at line 457 of file ppdbSchema.py.

457  def getColumnMap(self, table_name):
458  """Returns mapping of column names to Column definitions.
459 
460  Parameters
461  ----------
462  table_name : `str`
463  One of known PPDB table names.
464 
465  Returns
466  -------
467  column_map : `dict`
468  Mapping of column names to `ColumnDef` instances.
469  """
470  table = self._schemas[table_name]
471  cmap = {column.name: column for column in table.columns}
472  return cmap
473 

◆ makeSchema()

def lsst.dax.ppdb.ppdbSchema.PpdbSchema.makeSchema (   self,
  drop = False,
  mysql_engine = 'InnoDB',
  oracle_tablespace = None,
  oracle_iot = False 
) 
Create or re-create all tables.

Parameters
----------
drop : `bool`, optional
    If True then drop tables before creating new ones.
mysql_engine : `str`, optional
    MySQL engine type to use for new tables.
oracle_tablespace : `str`, optional
    Name of Oracle tablespace, only useful with oracle
oracle_iot : `bool`, optional
    Make Index-organized DiaObjectLast table.

Definition at line 337 of file ppdbSchema.py.

337  def makeSchema(self, drop=False, mysql_engine='InnoDB', oracle_tablespace=None, oracle_iot=False):
338  """Create or re-create all tables.
339 
340  Parameters
341  ----------
342  drop : `bool`, optional
343  If True then drop tables before creating new ones.
344  mysql_engine : `str`, optional
345  MySQL engine type to use for new tables.
346  oracle_tablespace : `str`, optional
347  Name of Oracle tablespace, only useful with oracle
348  oracle_iot : `bool`, optional
349  Make Index-organized DiaObjectLast table.
350  """
351 
352  # re-make table schema for all needed tables with possibly different options
353  _LOG.debug("clear metadata")
354  self._metadata.clear()
355  _LOG.debug("re-do schema mysql_engine=%r oracle_tablespace=%r",
356  mysql_engine, oracle_tablespace)
357  self._makeTables(mysql_engine=mysql_engine, oracle_tablespace=oracle_tablespace,
358  oracle_iot=oracle_iot)
359 
360  # create all tables (optionally drop first)
361  if drop:
362  _LOG.info('dropping all tables')
363  self._metadata.drop_all()
364  _LOG.info('creating all tables')
365  self._metadata.create_all()
366 

Member Data Documentation

◆ forcedSources

lsst.dax.ppdb.ppdbSchema.PpdbSchema.forcedSources

Definition at line 233 of file ppdbSchema.py.

◆ objects

lsst.dax.ppdb.ppdbSchema.PpdbSchema.objects

Definition at line 229 of file ppdbSchema.py.

◆ objects_last

lsst.dax.ppdb.ppdbSchema.PpdbSchema.objects_last

Definition at line 231 of file ppdbSchema.py.

◆ objects_nightly

lsst.dax.ppdb.ppdbSchema.PpdbSchema.objects_nightly

Definition at line 230 of file ppdbSchema.py.

◆ sources

lsst.dax.ppdb.ppdbSchema.PpdbSchema.sources

Definition at line 232 of file ppdbSchema.py.

◆ visits

lsst.dax.ppdb.ppdbSchema.PpdbSchema.visits

Definition at line 234 of file ppdbSchema.py.

The documentation for this class was generated from the following file:
Generated on Mon Dec 23 2019 19:04:22 for LSSTApplications by   doxygen 1.8.13