LSSTApplications  20.0.0
LSSTDataManagementBasePackage
OutputArchive.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include <typeinfo>
4 #include <vector>
5 #include <map>
6 #include <memory>
7 
8 #include "boost/format.hpp"
9 
10 #include "lsst/pex/exceptions.h"
15 #include "lsst/afw/fits.h"
16 
17 namespace lsst {
18 namespace afw {
19 namespace table {
20 namespace io {
21 
22 namespace {
23 
24 ArchiveIndexSchema const &indexKeys = ArchiveIndexSchema::get();
25 
26 // we don't need sorting, but you can't use weak_ptrs as keys in an
27 // unordered_map
30 
31 typedef Map::value_type MapItem;
32 
33 } // namespace
34 
35 // ----- OutputArchive::Impl --------------------------------------------------------------------------------
36 
38 public:
40  int catArchive = 1;
41  CatalogVector::iterator iter = _catalogs.begin();
43  for (; iter != _catalogs.end(); ++iter, ++catArchive) {
44  if (iter->getSchema().compare(schema, flags) == flags) {
45  break;
46  }
47  }
48  if (iter == _catalogs.end()) {
50  }
51  if (!iter->getTable()->getMetadata()) {
53  iter->getTable()->setMetadata(metadata);
54  metadata->set("EXTTYPE", "ARCHIVE_DATA");
55  metadata->set("AR_CATN", catArchive, "# of this catalog relative to the start of this archive");
56  }
57  return BaseCatalog(iter->getTable());
58  }
59 
61  auto indexRecord = _index.addNew();
62  indexRecord->set(indexKeys.id, id);
63  indexRecord->set(indexKeys.name, name);
64  indexRecord->set(indexKeys.module, module);
65  return indexRecord;
66  }
67 
68  void saveEmpty(int id, std::string const &name, std::string const &module) {
69  auto indexRecord = addIndexRecord(id, name, module);
70  indexRecord->set(indexKeys.nRows, 0);
71  indexRecord->set(indexKeys.catPersistable, ArchiveIndexSchema::NO_CATALOGS_SAVED);
72  indexRecord->set(indexKeys.row0, ArchiveIndexSchema::NO_CATALOGS_SAVED);
73  indexRecord->set(indexKeys.catArchive, ArchiveIndexSchema::NO_CATALOGS_SAVED);
74  }
75 
76  void saveCatalog(BaseCatalog const &catalog, int id, std::string const &name, std::string const &module,
77  int catPersistable) {
78  auto indexRecord = addIndexRecord(id, name, module);
79  indexRecord->set(indexKeys.catPersistable, catPersistable);
80  indexRecord->set(indexKeys.nRows, catalog.size());
81  int catArchive = 1;
82  CatalogVector::iterator iter = _catalogs.begin();
83  for (; iter != _catalogs.end(); ++iter, ++catArchive) {
84  if (iter->getTable() == catalog.getTable()) {
85  break;
86  }
87  }
88  if (iter == _catalogs.end()) {
90  "All catalogs passed to saveCatalog must be created by makeCatalog");
91  }
92  // Add the name of the class to the header so anyone looking at it can
93  // tell what's stored there. But we don't want to add it multiple times.
94  try {
95  auto names = iter->getTable()->getMetadata()->getArray<std::string>("AR_NAME");
96  if (std::find(names.begin(), names.end(), name) == names.end()) {
97  iter->getTable()->getMetadata()->add("AR_NAME", name, "Class name for objects stored here");
98  }
99  } catch (pex::exceptions::NotFoundError &) {
100  iter->getTable()->getMetadata()->add("AR_NAME", name, "Class name for objects stored here");
101  }
102  indexRecord->set(indexKeys.row0, iter->size());
103  indexRecord->set(indexKeys.catArchive, catArchive);
104  iter->insert(iter->end(), catalog.begin(), catalog.end(), false);
105  }
106 
107  int put(Persistable const *obj, std::shared_ptr<Impl> const &self, bool permissive) {
108  if (!obj) return 0;
109  if (permissive && !obj->isPersistable()) return 0;
110  int const currentId = _nextId;
111  ++_nextId;
112  OutputArchiveHandle handle(currentId, obj->getPersistenceName(), obj->getPythonModule(), self);
113  obj->write(handle);
114  return currentId;
115  }
116 
117  int put(std::shared_ptr<Persistable const> obj, std::shared_ptr<Impl> const &self, bool permissive) {
118  if (!obj) return 0;
119  if (permissive && !obj->isPersistable()) return 0;
120  MapItem item(obj, _nextId);
121  std::pair<Map::iterator, bool> r = _map.insert(item);
122  if (r.second) {
123  // We've never seen this object before. Save it.
124  return put(obj.get(), self, permissive);
125  } else {
126  // We had already saved this object, and insert returned an iterator
127  // to the ID we used before; return that.
128  return r.first->second;
129  }
130  }
131 
132  void writeFits(fits::Fits &fitsfile) {
133  _index.getTable()->getMetadata()->set("AR_NCAT", int(_catalogs.size() + 1),
134  "# of catalogs in this archive, including the index");
135  _index.writeFits(fitsfile);
136  int n = 1;
137  for (CatalogVector::const_iterator iter = _catalogs.begin(); iter != _catalogs.end(); ++iter, ++n) {
138  iter->writeFits(fitsfile);
139  }
140  }
141 
144  metadata->set("EXTTYPE", "ARCHIVE_INDEX");
145  metadata->set("AR_CATN", 0, "# of this catalog relative to the start of this archive");
146  _index.getTable()->setMetadata(metadata);
147  }
148 
149  int _nextId;
150  Map _map;
153 };
154 
155 // ----- OutputArchive --------------------------------------------------------------------------------------
156 
158 
160 // Delegate to copy constructor for backward compatibility
162 
164  _impl = other._impl;
165  return *this;
166 }
167 // Delegate to copy assignment for backward compatibility
169 
171 
172 int OutputArchive::put(Persistable const *obj, bool permissive) {
173  if (!_impl.unique()) { // copy on write
174  std::shared_ptr<Impl> tmp(new Impl(*_impl));
175  _impl.swap(tmp);
176  }
177  return _impl->put(obj, _impl, permissive);
178 }
179 
181  if (!_impl.unique()) { // copy on write
182  std::shared_ptr<Impl> tmp(new Impl(*_impl));
183  _impl.swap(tmp);
184  }
185  return _impl->put(std::move(obj), _impl, permissive);
186 }
187 
188 BaseCatalog const &OutputArchive::getIndexCatalog() const { return _impl->_index; }
189 
191  if (n == 0) return _impl->_index;
192  if (std::size_t(n) > _impl->_catalogs.size() || n < 0) {
193  throw LSST_EXCEPT(
195  (boost::format("Catalog number %d is out of range [0,%d]") % n % _impl->_catalogs.size())
196  .str());
197  }
198  return _impl->_catalogs[n - 1];
199 }
200 
201 int OutputArchive::countCatalogs() const { return _impl->_catalogs.size() + 1; }
202 
203 void OutputArchive::writeFits(fits::Fits &fitsfile) const { _impl->writeFits(fitsfile); }
204 
205 // ----- OutputArchiveHandle ------------------------------------------------------------------------------
206 
207 BaseCatalog OutputArchiveHandle::makeCatalog(Schema const &schema) { return _impl->makeCatalog(schema); }
208 
209 void OutputArchiveHandle::saveEmpty() { _impl->saveEmpty(_id, _name, _module); }
210 
212  _impl->saveCatalog(catalog, _id, _name, _module, _catPersistable);
213  ++_catPersistable;
214 }
215 
216 int OutputArchiveHandle::put(Persistable const *obj, bool permissive) {
217  // Handle doesn't worry about copy-on-write, because Handles should only exist
218  // while an OutputArchive::put() call is active.
219  return _impl->put(obj, _impl, permissive);
220 }
221 
223  // Handle doesn't worry about copy-on-write, because Handles should only exist
224  // while an OutputArchive::put() call is active.
225  return _impl->put(std::move(obj), _impl, permissive);
226 }
227 
230  : _id(id), _catPersistable(0), _name(name), _module(module), _impl(impl) {}
231 
233 } // namespace io
234 } // namespace table
235 } // namespace afw
236 } // namespace lsst
schema
table::Schema schema
Definition: Amplifier.cc:115
lsst::afw::table::CatalogT::end
iterator end()
Definition: Catalog.h:397
lsst::afw::table::io::OutputArchive::Impl::makeCatalog
BaseCatalog makeCatalog(Schema const &schema)
Definition: OutputArchive.cc:39
lsst::afw::table::io::OutputArchive::Impl::put
int put(std::shared_ptr< Persistable const > obj, std::shared_ptr< Impl > const &self, bool permissive)
Definition: OutputArchive.cc:117
lsst::afw::table::io::OutputArchive::Impl::_map
Map _map
Definition: OutputArchive.cc:150
lsst::afw::table::io::Persistable::getPythonModule
virtual std::string getPythonModule() const
Return the fully-qualified Python module that should be imported to guarantee that its factory is reg...
Definition: Persistable.cc:36
std::string
STL class.
std::shared_ptr
STL class.
lsst::afw::table::io::OutputArchive::operator=
OutputArchive & operator=(OutputArchive const &other)
Assign from another OutputArchive. Saved objects are not deep-copied.
Definition: OutputArchive.cc:163
lsst::afw::table::io::OutputArchive::Impl
Definition: OutputArchive.cc:37
lsst::afw::table::io::ArchiveIndexSchema::NO_CATALOGS_SAVED
static constexpr int const NO_CATALOGS_SAVED
Special value used for catArchive, catPersistable, and row0 when an object with no state is saved.
Definition: ArchiveIndexSchema.h:52
std::move
T move(T... args)
ArchiveIndexSchema.h
lsst::afw::table::io::OutputArchive::~OutputArchive
~OutputArchive()
std::pair
lsst::afw::table::io::OutputArchive::getCatalog
BaseCatalog const & getCatalog(int n) const
Return the nth catalog. Catalog 0 is always the index catalog.
Definition: OutputArchive.cc:190
lsst::afw::table::io::OutputArchive::put
int put(std::shared_ptr< Persistable const > obj, bool permissive=false)
Save an object to the archive and return a unique ID that can be used to retrieve it from an InputArc...
Definition: OutputArchive.cc:180
lsst::afw::table::io::OutputArchiveHandle
An object passed to Persistable::write to allow it to persist itself.
Definition: OutputArchive.h:118
lsst::afw::fits::Fits
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
std::find
T find(T... args)
std::vector::size
T size(T... args)
lsst::pex::exceptions::NotFoundError
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
lsst::afw::table::io::OutputArchiveHandle::saveCatalog
void saveCatalog(BaseCatalog const &catalog)
Save a catalog in the archive.
Definition: OutputArchive.cc:211
pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
fits.h
lsst::afw
Definition: imageAlgorithm.dox:1
std::shared_ptr::get
T get(T... args)
lsst::afw::table::io::OutputArchive::Impl::_catalogs
CatalogVector _catalogs
Definition: OutputArchive.cc:152
std::owner_less
lsst::daf::base::PropertyList
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
lsst::afw::table::io::OutputArchiveHandle::~OutputArchiveHandle
~OutputArchiveHandle()
Definition: OutputArchive.cc:232
lsst::afw::table::io::ArchiveIndexSchema::get
static ArchiveIndexSchema const & get()
Return the singleton instance.
Definition: ArchiveIndexSchema.cc:14
lsst::afw::table::Schema
Defines the fields and offsets for a table.
Definition: Schema.h:50
CatalogVector.h
lsst::afw::geom.transform.transformContinued.name
string name
Definition: transformContinued.py:32
lsst::afw::table::io::Persistable::isPersistable
virtual bool isPersistable() const noexcept
Return true if this particular object can be persisted using afw::table::io.
Definition: Persistable.h:102
lsst::afw::table::Schema::EQUAL_KEYS
@ EQUAL_KEYS
Keys have the same types offsets, and sizes.
Definition: Schema.h:65
lsst::afw::table::io::OutputArchive::Impl::saveEmpty
void saveEmpty(int id, std::string const &name, std::string const &module)
Definition: OutputArchive.cc:68
lsst::afw::table::io::OutputArchiveHandle::makeCatalog
BaseCatalog makeCatalog(Schema const &schema)
Return a new, empty catalog with the given schema.
Definition: OutputArchive.cc:207
lsst::afw::table::io::Persistable::getPersistenceName
virtual std::string getPersistenceName() const
Return the unique name used to persist this object and look up its factory.
Definition: Persistable.cc:34
lsst::afw::table::CatalogT::size
size_type size() const
Return the number of elements in the catalog.
Definition: Catalog.h:408
lsst::afw::table::io::OutputArchive::Impl::_index
BaseCatalog _index
Definition: OutputArchive.cc:151
id
table::Key< int > id
Definition: Detector.cc:162
lsst::afw::table::io::OutputArchive::writeFits
void writeFits(fits::Fits &fitsfile) const
Write the archive to an already-open FITS object.
Definition: OutputArchive.cc:203
lsst::pex::exceptions::LengthError
Reports attempts to exceed implementation-defined length limits for some classes.
Definition: Runtime.h:76
lsst::afw::table::io::ArchiveIndexSchema
Schema for the index catalog that specifies where objects are stored in the data catalogs.
Definition: ArchiveIndexSchema.h:35
other
ItemVariant const * other
Definition: Schema.cc:56
lsst::afw::table::CatalogT::getTable
std::shared_ptr< Table > getTable() const
Return the table associated with the catalog.
Definition: Catalog.h:114
lsst::pex::exceptions::LogicError
Reports errors in the logical structure of the program.
Definition: Runtime.h:46
lsst::afw::table::io::CatalogVector
A vector of catalogs used by Persistable.
Definition: CatalogVector.h:29
lsst::afw::table::io::OutputArchiveHandle::saveEmpty
void saveEmpty()
Indicate that the object being persisted has no state, and hence will never call makeCatalog() or sav...
Definition: OutputArchive.cc:209
lsst::afw::table::io::Persistable::write
virtual void write(OutputArchiveHandle &handle) const
Write the object to one or more catalogs.
Definition: Persistable.cc:38
lsst::afw::table::io::OutputArchive
A multi-catalog archive object used to save table::io::Persistable objects.
Definition: OutputArchive.h:34
std::map
STL class.
lsst
A base class for image defects.
Definition: imageAlgorithm.dox:1
LSST_EXCEPT
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
lsst::afw::table::io::OutputArchive::Impl::Impl
Impl()
Definition: OutputArchive.cc:142
lsst::afw::table::CatalogT::writeFits
void writeFits(std::string const &filename, std::string const &mode="w", int flags=0) const
Write a FITS binary table to a regular file.
Definition: Catalog.h:306
lsst::afw::table::io::Persistable
A base class for objects that can be persisted via afw::table::io Archive classes.
Definition: Persistable.h:74
lsst::afw::table::BaseCatalog
CatalogT< BaseRecord > BaseCatalog
Definition: fwd.h:71
std::vector::begin
T begin(T... args)
Persistable.h
lsst::afw::table::io::OutputArchive::Impl::addIndexRecord
std::shared_ptr< BaseRecord > addIndexRecord(int id, std::string const &name, std::string const &module)
Definition: OutputArchive.cc:60
std::vector::insert
T insert(T... args)
lsst::afw::table::io::OutputArchive::Impl::_nextId
int _nextId
Definition: OutputArchive.cc:149
lsst::afw::table::io::OutputArchive::Impl::put
int put(Persistable const *obj, std::shared_ptr< Impl > const &self, bool permissive)
Definition: OutputArchive.cc:107
std::size_t
lsst::afw::table::io::OutputArchiveHandle::put
int put(Persistable const *obj, bool permissive=false)
Save an object to the archive and return a unique ID that can be used to retrieve it from an InputArc...
Definition: OutputArchive.cc:216
std::vector::end
T end(T... args)
lsst::afw::table::CatalogT::addNew
std::shared_ptr< RecordT > addNew()
Create a new record, add it to the end of the catalog, and return a pointer to it.
Definition: Catalog.h:485
lsst::afw::table::Schema::EQUAL_NAMES
@ EQUAL_NAMES
Fields have the same names (ordered).
Definition: Schema.h:66
lsst::afw::table::io::OutputArchiveHandle::OutputArchiveHandle
OutputArchiveHandle(const OutputArchiveHandle &)=delete
lsst::afw::table::CatalogT::begin
iterator begin()
Iterator access.
Definition: Catalog.h:396
lsst::afw::table::io::OutputArchive::OutputArchive
OutputArchive()
Construct an empty OutputArchive containing no objects.
Definition: OutputArchive.cc:157
lsst::afw::table::CatalogT< BaseRecord >
astshim.fitsChanContinued.iter
def iter(self)
Definition: fitsChanContinued.py:88
lsst::afw::table::io::OutputArchive::Impl::saveCatalog
void saveCatalog(BaseCatalog const &catalog, int id, std::string const &name, std::string const &module, int catPersistable)
Definition: OutputArchive.cc:76
lsst::meas::modelfit.psf.psfContinued.module
module
Definition: psfContinued.py:42
lsst::afw::table::io::OutputArchive::Impl::writeFits
void writeFits(fits::Fits &fitsfile)
Definition: OutputArchive.cc:132
OutputArchive.h
exceptions.h
lsst::afw::table::io::OutputArchive::countCatalogs
int countCatalogs() const
Return the total number of catalogs, including the index.
Definition: OutputArchive.cc:201
lsst::afw::table::io::OutputArchive::getIndexCatalog
BaseCatalog const & getIndexCatalog() const
Return the index catalog that specifies where objects are stored in the data catalogs.
Definition: OutputArchive.cc:188