Class for database storage with data loading from TSV files. More...

#include <DbTsvStorage.h>

Inheritance diagram for lsst::daf::persistence::DbTsvStorage:

Public Types
typedef boost::shared_ptr < DbTsvStorage >	Ptr

Public Types inherited from lsst::daf::persistence::DbStorage
typedef boost::shared_ptr < DbStorage >	Ptr

Public Types inherited from lsst::daf::persistence::Storage
typedef boost::shared_ptr < Storage >	Ptr

typedef std::vector< Ptr >	List

Public Types inherited from lsst::daf::base::Citizen
enum	{ magicSentinel = 0xdeadbeef }

typedef unsigned long	memId
	Type of the block's ID. More...

typedef memId(*	memNewCallback )(const memId cid)
	A function used to register a callback. More...

typedef memId(*	memCallback )(const Citizen *ptr)

Public Member Functions
	DbTsvStorage (void)

	~DbTsvStorage (void)

virtual void	setPolicy (lsst::pex::policy::Policy::Ptr policy)

virtual void	setPersistLocation (LogicalLocation const &location)

virtual void	setRetrieveLocation (LogicalLocation const &location)

virtual void	startTransaction (void)

virtual void	endTransaction (void)

virtual void	createTableFromTemplate (std::string const &tableName, std::string const &templateName, bool mayAlreadyExist=false)

virtual void	dropTable (std::string const &tableName)

virtual void	truncateTable (std::string const &tableName)

virtual void	setTableForInsert (std::string const &tableName)

template<typename T >
void	setColumn (std::string const &columnName, T const &value)

virtual void	setColumnToNull (std::string const &columnName)

virtual void	insertRow (void)

template<typename T >
void	outParam (std::string const &columnName, T *location, bool isExpr=false)

template<typename T >
void	condParam (std::string const &paramName, T const &value)

template<typename T >
T const &	getColumnByPos (int pos)

template<>
void	setColumn (std::string const &columnName, char const &value)

template<>
void	setColumn (std::string const &columnName, double const &value)

template<>
void	setColumn (std::string const &columnName, float const &value)

template<>
void	setColumn (std::string const &columnName, DateTime const &value)

Public Member Functions inherited from lsst::daf::persistence::DbStorage
	DbStorage (void)

	~DbStorage (void)

virtual void	executeSql (std::string const &sqlStatement)

template<typename T >
void	setColumn (std::string const &columnName, T const &value)

virtual void	setTableForQuery (std::string const &tableName, bool isExpr=false)

virtual void	setTableListForQuery (std::vector< std::string > const &tableNameList)

virtual void	outColumn (std::string const &columnName, bool isExpr=false)

template<typename T >
void	outParam (std::string const &columnName, T *location, bool isExpr=false)

template<typename T >
void	condParam (std::string const &paramName, T const &value)

virtual void	orderBy (std::string const &expression)

virtual void	groupBy (std::string const &expression)

virtual void	setQueryWhere (std::string const &whereClause)

virtual void	query (void)

virtual bool	next (void)

template<typename T >
T const &	getColumnByPos (int pos)

bool	columnIsNull (int pos)

virtual void	finishQuery (void)

Public Member Functions inherited from lsst::daf::persistence::Storage
virtual	~Storage (void)

Public Member Functions inherited from lsst::daf::base::Citizen
	Citizen (const std::type_info &)

	Citizen (Citizen const &)

	~Citizen ()

Citizen &	operator= (Citizen const &)

std::string	repr () const
	Return a string representation of a Citizen. More...

void	markPersistent (void)
	Mark a Citizen as persistent and not destroyed until process end. More...

memId	getId () const
	Return the Citizen's ID. More...

Private Member Functions
int	_getColumnIndex (std::string const &columnName)

Private Attributes
bool	_persisting

bool	_saveTemp
	Do not delete temporary TSV file if true. More...

std::string	_tempPath
	Directory pathname for temporary TSV file. More...

char *	_fileName
	Full pathname for temporary TSV file. More...

std::string	_location
	Database location URL. More...

std::string	_tableName

std::map< std::string, int >	_colMap
	Map from column names to positions. More...

std::ostringstream	_convertStream
	Stream to convert to text. More...

std::vector< std::string >	_rowBuffer

boost::scoped_ptr< std::ofstream >	_osp
	Output TSV stream. More...

Additional Inherited Members
Static Public Member Functions inherited from lsst::daf::persistence::Storage
static Ptr	createInstance (std::string const &name, LogicalLocation const &location, bool persist, lsst::pex::policy::Policy::Ptr policy)

Static Public Member Functions inherited from lsst::daf::base::Citizen
static bool	hasBeenCorrupted ()
	Check all allocated blocks for corruption. More...

static memId	getNextMemId ()
	Return the memId of the next object to be allocated. More...

static int	init ()
	Called once when the memory system is being initialised. More...

static int	census (int, memId startingMemId=0)
	How many active Citizens are there? More...

static void	census (std::ostream &stream, memId startingMemId=0)
	Print a list of all active Citizens to stream, sorted by ID. More...

static const std::vector < const Citizen * > *	census ()
	Return a (newly allocated) std::vector of active Citizens sorted by ID. More...

static memId	setNewCallbackId (memId id)
	Call the NewCallback when block is allocated. More...

static memId	setDeleteCallbackId (memId id)
	Call the current DeleteCallback when block is deleted. More...

static memNewCallback	setNewCallback (memNewCallback func)
	Set the NewCallback function. More...

static memCallback	setDeleteCallback (memCallback func)
	Set the DeleteCallback function. More...

static memCallback	setCorruptionCallback (memCallback func)
	Set the CorruptionCallback function. More...

Protected Member Functions inherited from lsst::daf::persistence::DbStorage
	DbStorage (std::type_info const &type)

Protected Member Functions inherited from lsst::daf::persistence::Storage
	Storage (std::type_info const &type)

void	verifyPathName (std::string const &pathName)

Detailed Description

Class for database storage with data loading from TSV files.

Subclass of DbStorage, overriding persistence methods.

Persists data to a database using TSV files as an intermediary for performance. Provides methods for writing rows to a table and retrieving rows from a query.

Definition at line 66 of file DbTsvStorage.h.

Member Typedef Documentation

typedef boost::shared_ptr<DbTsvStorage> lsst::daf::persistence::DbTsvStorage::Ptr

Definition at line 68 of file DbTsvStorage.h.

Constructor & Destructor Documentation

lsst::daf::persistence::DbTsvStorage::DbTsvStorage ( void )

Constructor.

Definition at line 67 of file DbTsvStorage.cc.

67 : _saveTemp(false) {

68 }

lsst::daf::persistence::DbTsvStorage::_saveTemp

bool _saveTemp

Do not delete temporary TSV file if true.

Definition: DbTsvStorage.h:101

lsst::daf::persistence::DbTsvStorage::~DbTsvStorage ( void )

Minimal destructor.

Definition at line 72 of file DbTsvStorage.cc.

72 {

73 }

Member Function Documentation

int lsst::daf::persistence::DbTsvStorage::_getColumnIndex ( std::string const & columnName )

private

Get the index of a given column. Create a new entry in the row buffer if the column hasn't already been seen. May modify the row buffer, so do not call this inside "_rowBuffer[]".

Parameters

[in] columnName Name of the column

Returns: Index of the column in the row buffer

Definition at line 252 of file DbTsvStorage.cc.

                                                              {
     std::map<std::string, int>::iterator i = _colMap.find(columnName);
     if (i == _colMap.end()) {
         _colMap.insert(std::pair<std::string, int>(columnName,
                                                    _rowBuffer.size()));
         _rowBuffer.push_back(std::string());
         return _rowBuffer.size() - 1;
     }
     else {
         return i->second;
     }
 }

template<typename T >

void lsst::daf::persistence::DbTsvStorage::condParam	(	std::string const &	paramName,
		T const &	value
	)

Bind a value to a WHERE condition parameter.

Parameters

[in]	paramName	Name of the parameter (prefixed by ":" in the WHERE clause)
[in]	value	Value to be bound to the parameter.

Definition at line 368 of file DbTsvStorage.cc.

                                                                        {
     DbStorage::condParam<T>(paramName, value);
 }

void lsst::daf::persistence::DbTsvStorage::createTableFromTemplate	(	std::string const &	tableName,
		std::string const &	templateName,
		bool	mayAlreadyExist = `false`
	)

virtual

Create a new table from an existing template table.

Parameters

[in]	tableName	Name of the new table
[in]	templateName	Name of the existing template table
[in]	mayAlreadyExist	False (default) if the table must not be present

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 183 of file DbTsvStorage.cc.

                                                               {
     if (_persisting) {
         DbStorage dbs;
         dbs.setPersistLocation(LogicalLocation(_location));
         dbs.startTransaction();
         dbs.createTableFromTemplate(tableName, templateName, mayAlreadyExist);
         dbs.endTransaction();
     }
     else {
         DbStorage::createTableFromTemplate(tableName, templateName,
                                            mayAlreadyExist);
     }
 }

void lsst::daf::persistence::DbTsvStorage::dropTable ( std::string const & tableName )

virtual

Drop a table.

Parameters

[in] tableName Name of the table to drop

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 202 of file DbTsvStorage.cc.

                                                        {
     if (_persisting) {
         DbStorage dbs;
         dbs.setPersistLocation(LogicalLocation(_location));
         dbs.startTransaction();
         dbs.dropTable(tableName);
         dbs.endTransaction();
     }
     else {
         DbStorage::dropTable(tableName);
     }
 }

void lsst::daf::persistence::DbTsvStorage::endTransaction ( void )

virtual

End a transaction.

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 114 of file DbTsvStorage.cc.

                                       {
     if (!_persisting) {
         DbStorage::endTransaction();
         return;
     }
 
     // close stream
     _osp->close();
 
     MYSQL* db = mysql_init(0);
     if (db == 0) {
         throw LSST_EXCEPT(lsst::pex::exceptions::RuntimeError, "Unable to allocate MySQL connection");
     }
     DbStorageLocation dbLoc(_location);
     unsigned int port = strtoul(dbLoc.getPort().c_str(), 0, 10);
     if (mysql_real_connect(db,
                            dbLoc.getHostname().c_str(),
                            dbLoc.getUsername().c_str(),
                            dbLoc.getPassword().c_str(),
                            dbLoc.getDbName().c_str(),
                            port, 0,
                            CLIENT_COMPRESS | CLIENT_LOCAL_FILES) == 0) {
         throw LSST_EXCEPT(lsst::pex::exceptions::RuntimeError,
             "Unable to connect to MySQL database: " + _location);
     }
     if (mysql_options(db, MYSQL_OPT_LOCAL_INFILE, 0) != 0) {
         throw LSST_EXCEPT(lsst::pex::exceptions::RuntimeError,
             std::string("Unable to set LOCAL INFILE option - * ") +
             mysql_error(db));
     }
 
     std::string query = "LOAD DATA LOCAL INFILE";
     query += " '";
     query += _fileName;
     query += "'";
     query += " REPLACE";
     query += " INTO TABLE `" + _tableName;
     query += "` (";
     for (std::map<std::string, int>::const_iterator it = _colMap.begin();
          it != _colMap.end(); ++it) {
         _rowBuffer[it->second] = it->first;
     }
     for (std::vector<std::string>::const_iterator it = _rowBuffer.begin();
          it != _rowBuffer.end(); ++it) {
         if (it != _rowBuffer.begin()) query += " ,";
         query += *it;
     }
     query += ")";
 
     if (mysql_query(db, query.c_str()) != 0) {
         mysql_close(db);
         throw LSST_EXCEPT(lsst::pex::exceptions::RuntimeError,
             "Unable to load data into database table: " + _tableName +
             "- * " + mysql_error(db));
     }
     mysql_close(db);
 
     // unlink file
     if (!_saveTemp) {
         unlink(_fileName);
     }
     delete[] _fileName;
 }

template<typename T >

T const & lsst::daf::persistence::DbTsvStorage::getColumnByPos ( int pos )

Get the value of a column of the query result row by position.

Parameters

[in] pos Position of the column (starts at 0)

Definition at line 376 of file DbTsvStorage.cc.

                                              {
     return DbStorage::getColumnByPos<T>(pos);
 }

void lsst::daf::persistence::DbTsvStorage::insertRow ( void )

virtual

Insert the row. Row values must have been set with setColumn() calls.

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 334 of file DbTsvStorage.cc.

                                  {
     // Output row to stream
     for (std::vector<std::string>::const_iterator i = _rowBuffer.begin();
          i != _rowBuffer.end(); ++i) {
         if (i != _rowBuffer.begin()) *_osp << '\t';
         *_osp << *i;
     }
     // \todo Optimization: if columns all outputted, just put out endl
     *_osp << std::endl;
 }

template<typename T >

void lsst::daf::persistence::DbTsvStorage::outParam	(	std::string const &	columnName,
		T *	location,
		bool	isExpr = `false`
	)

Request a column in the query output and bind a destination location.

Parameters

[in]	columnName	Name of the column
[in]	location	Pointer to the destination
[in]	isExpr	True if the name is actually an expression

The order of outParam() calls is the order of appearance in the output row. Use either outColumn() or outParam() but not both.

Definition at line 357 of file DbTsvStorage.cc.

                                          {
     DbStorage::outParam<T>(columnName, location, isExpr);
 }

template<typename T >

void lsst::daf::persistence::DbTsvStorage::setColumn	(	std::string const &	columnName,
		T const &	value
	)

Set the value to insert in a given column.

Parameters

[in]	columnName	Name of the column
[in]	value	Value to set in the column

Definition at line 270 of file DbTsvStorage.cc.

                                                                         {
     int colIndex = _getColumnIndex(columnName);
 
     // set value in row buffer
     // \todo quote value as appropriate
     _convertStream.str(std::string());
     _convertStream << value;
     _rowBuffer[colIndex] = _convertStream.str();
 
     // \todo Optimization: if next column, output now, plus any others saved
 }

template<>

void lsst::daf::persistence::DbTsvStorage::setColumn	(	std::string const &	columnName,
		char const &	value
	)

Definition at line 284 of file DbTsvStorage.cc.

                                                 {
     int colIndex = _getColumnIndex(columnName);
     _convertStream.str(std::string());
     _convertStream << static_cast<int>(value);
     _rowBuffer[colIndex] = _convertStream.str();
 }

template<>

void lsst::daf::persistence::DbTsvStorage::setColumn	(	std::string const &	columnName,
		double const &	value
	)

Definition at line 294 of file DbTsvStorage.cc.

                                                   {
     int colIndex = _getColumnIndex(columnName);
     _convertStream.str(std::string());
     _convertStream << std::setprecision(17) << value;
     _rowBuffer[colIndex] = _convertStream.str();
 }

template<>

void lsst::daf::persistence::DbTsvStorage::setColumn	(	std::string const &	columnName,
		float const &	value
	)

Definition at line 303 of file DbTsvStorage.cc.

                                                  {
     int colIndex = _getColumnIndex(columnName);
     _convertStream.str(std::string());
     _convertStream << std::setprecision(9) << value;
     _rowBuffer[colIndex] = _convertStream.str();
 }

template<>

void lsst::daf::persistence::DbTsvStorage::setColumn	(	std::string const &	columnName,
		DateTime const &	value
	)

Definition at line 313 of file DbTsvStorage.cc.

                                                     {
     int colIndex = _getColumnIndex(columnName);
     _convertStream.str(std::string());
     struct tm t = value.gmtime();
     char buf[20];
     strftime(buf, sizeof(buf), "%F %T", &t);
     _rowBuffer[colIndex] = std::string(buf);
 }

void lsst::daf::persistence::DbTsvStorage::setColumnToNull ( std::string const & columnName )

virtual

Set a given column to NULL.

Parameters

[in] columnName Name of the column

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 326 of file DbTsvStorage.cc.

                                                               {
     int colIndex = _getColumnIndex(columnName);
     _rowBuffer[colIndex] = "\\N";
 }

void lsst::daf::persistence::DbTsvStorage::setPersistLocation ( LogicalLocation const & location )

virtual

Set the database location to persist to.

Parameters

[in] location Database connection string to insert to.

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 91 of file DbTsvStorage.cc.

                                                                      {
     _persisting = true;
     _location = location.locString();
     // Set the timezone for any DATE/TIME/DATETIME fields.
     setenv("TZ", "UTC", 1);
 }

void lsst::daf::persistence::DbTsvStorage::setPolicy ( lsst::pex::policy::Policy::Ptr policy )

virtual

Allow a policy to be used to configure the DbTsvStorage.

Parameters

[in] policy

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 78 of file DbTsvStorage.cc.

                                                           {
     _tempPath = "/tmp";
     if (policy && policy->exists("TempPath")) {
         _tempPath = policy->getString("TempPath");
     }
     if (policy && policy->exists("SaveTemp") && policy->getBool("SaveTemp")) {
         _saveTemp = true;
     }
 }

void lsst::daf::persistence::DbTsvStorage::setRetrieveLocation ( LogicalLocation const & location )

virtual

Set the database location to retrieve from.

Parameters

[in] location Database connection string to retrieve from.

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 101 of file DbTsvStorage.cc.

                                                                       {
     _persisting = false;
     DbStorage::setRetrieveLocation(location);
 }

void lsst::daf::persistence::DbTsvStorage::setTableForInsert ( std::string const & tableName )

virtual

Set the table to insert rows into.

Parameters

[in] tableName Name of the table

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 234 of file DbTsvStorage.cc.

                                                                {
     _tableName = tableName;
     std::string templ = _tempPath + "/" + tableName + ".XXXXXX";
     _fileName = new char[templ.size() + 1];
     strncpy(_fileName, templ.c_str(), templ.size());
     _fileName[templ.size()] = '\0';
     int fd = mkstemp(_fileName);
     // \todo check for errors
     close(fd);
     _osp.reset(new std::ofstream(_fileName));
 }

void lsst::daf::persistence::DbTsvStorage::startTransaction ( void )

virtual

Start a transaction.

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 108 of file DbTsvStorage.cc.

                                         {
     if (!_persisting) DbStorage::startTransaction();
 }

void lsst::daf::persistence::DbTsvStorage::truncateTable ( std::string const & tableName )

virtual

Truncate a table.

Parameters

[in] tableName Name of the table to truncate

Reimplemented from lsst::daf::persistence::DbStorage.

Definition at line 218 of file DbTsvStorage.cc.

                                                            {
     if (_persisting) {
         DbStorage dbs;
         dbs.setPersistLocation(LogicalLocation(_location));
         dbs.startTransaction();
         dbs.truncateTable(tableName);
         dbs.endTransaction();
     }
     else {
         DbStorage::truncateTable(tableName);
     }
 }

Member Data Documentation

std::map<std::string, int> lsst::daf::persistence::DbTsvStorage::_colMap

private

Map from column names to positions.

Definition at line 106 of file DbTsvStorage.h.

std::ostringstream lsst::daf::persistence::DbTsvStorage::_convertStream

private

Stream to convert to text.

Definition at line 107 of file DbTsvStorage.h.

char* lsst::daf::persistence::DbTsvStorage::_fileName

private

Full pathname for temporary TSV file.

Definition at line 103 of file DbTsvStorage.h.

std::string lsst::daf::persistence::DbTsvStorage::_location

private

Database location URL.

Definition at line 104 of file DbTsvStorage.h.

boost::scoped_ptr<std::ofstream> lsst::daf::persistence::DbTsvStorage::_osp

private

Output TSV stream.

Definition at line 109 of file DbTsvStorage.h.

bool lsst::daf::persistence::DbTsvStorage::_persisting

private

Definition at line 100 of file DbTsvStorage.h.

std::vector<std::string> lsst::daf::persistence::DbTsvStorage::_rowBuffer

private

Definition at line 108 of file DbTsvStorage.h.

bool lsst::daf::persistence::DbTsvStorage::_saveTemp

private

Do not delete temporary TSV file if true.

Definition at line 101 of file DbTsvStorage.h.

std::string lsst::daf::persistence::DbTsvStorage::_tableName

private

Definition at line 105 of file DbTsvStorage.h.

std::string lsst::daf::persistence::DbTsvStorage::_tempPath

private

Directory pathname for temporary TSV file.

Definition at line 102 of file DbTsvStorage.h.

The documentation for this class was generated from the following files:

/home/lsstsw/stack/Linux64/daf_persistence/11.0-1-g7418c06/include/lsst/daf/persistence/DbTsvStorage.h
/home/lsstsw/stack/Linux64/daf_persistence/11.0-1-g7418c06/src/DbTsvStorage.cc

Public Types

Public Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Typedef Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation