LSST Applications  21.0.0-147-g0e635eb1+1acddb5be5,22.0.0+052faf71bd,22.0.0+1ea9a8b2b2,22.0.0+6312710a6c,22.0.0+729191ecac,22.0.0+7589c3a021,22.0.0+9f079a9461,22.0.1-1-g7d6de66+b8044ec9de,22.0.1-1-g87000a6+536b1ee016,22.0.1-1-g8e32f31+6312710a6c,22.0.1-10-gd060f87+016f7cdc03,22.0.1-12-g9c3108e+df145f6f68,22.0.1-16-g314fa6d+c825727ab8,22.0.1-19-g93a5c75+d23f2fb6d8,22.0.1-19-gb93eaa13+aab3ef7709,22.0.1-2-g8ef0a89+b8044ec9de,22.0.1-2-g92698f7+9f079a9461,22.0.1-2-ga9b0f51+052faf71bd,22.0.1-2-gac51dbf+052faf71bd,22.0.1-2-gb66926d+6312710a6c,22.0.1-2-gcb770ba+09e3807989,22.0.1-20-g32debb5+b8044ec9de,22.0.1-23-gc2439a9a+fb0756638e,22.0.1-3-g496fd5d+09117f784f,22.0.1-3-g59f966b+1e6ba2c031,22.0.1-3-g849a1b8+f8b568069f,22.0.1-3-gaaec9c0+c5c846a8b1,22.0.1-32-g5ddfab5d3+60ce4897b0,22.0.1-4-g037fbe1+64e601228d,22.0.1-4-g8623105+b8044ec9de,22.0.1-5-g096abc9+d18c45d440,22.0.1-5-g15c806e+57f5c03693,22.0.1-7-gba73697+57f5c03693,master-g6e05de7fdc+c1283a92b8,master-g72cdda8301+729191ecac,w.2021.39
LSST Data Management Base Package
OutputArchive.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include <typeinfo>
4 #include <vector>
5 #include <map>
6 #include <memory>
7 
8 #include "boost/format.hpp"
9 
10 #include "lsst/pex/exceptions.h"
15 #include "lsst/afw/fits.h"
16 
17 namespace lsst {
18 namespace afw {
19 namespace table {
20 namespace io {
21 
22 namespace {
23 
24 ArchiveIndexSchema const &indexKeys = ArchiveIndexSchema::get();
25 
26 // we don't need sorting, but you can't use weak_ptrs as keys in an
27 // unordered_map
30 
31 using MapItem = Map::value_type;
32 
33 } // namespace
34 
35 // ----- OutputArchive::Impl --------------------------------------------------------------------------------
36 
38 public:
40  int catArchive = 1;
41  CatalogVector::iterator iter = _catalogs.begin();
43  for (; iter != _catalogs.end(); ++iter, ++catArchive) {
44  if (iter->getSchema().compare(schema, flags) == flags) {
45  break;
46  }
47  }
48  if (iter == _catalogs.end()) {
50  }
51  if (!iter->getTable()->getMetadata()) {
53  iter->getTable()->setMetadata(metadata);
54  metadata->set("EXTTYPE", "ARCHIVE_DATA");
55  metadata->set("AR_CATN", catArchive, "# of this catalog relative to the start of this archive");
56  }
57  return BaseCatalog(iter->getTable());
58  }
59 
61  auto indexRecord = _index.addNew();
62  indexRecord->set(indexKeys.id, id);
63  indexRecord->set(indexKeys.name, name);
64  indexRecord->set(indexKeys.module, module);
65  return indexRecord;
66  }
67 
68  void saveEmpty(int id, std::string const &name, std::string const &module) {
69  auto indexRecord = addIndexRecord(id, name, module);
70  indexRecord->set(indexKeys.nRows, 0);
71  indexRecord->set(indexKeys.catPersistable, ArchiveIndexSchema::NO_CATALOGS_SAVED);
72  indexRecord->set(indexKeys.row0, ArchiveIndexSchema::NO_CATALOGS_SAVED);
73  indexRecord->set(indexKeys.catArchive, ArchiveIndexSchema::NO_CATALOGS_SAVED);
74  }
75 
76  void saveCatalog(BaseCatalog const &catalog, int id, std::string const &name, std::string const &module,
77  int catPersistable) {
78  auto indexRecord = addIndexRecord(id, name, module);
79  indexRecord->set(indexKeys.catPersistable, catPersistable);
80  indexRecord->set(indexKeys.nRows, catalog.size());
81  int catArchive = 1;
82  CatalogVector::iterator iter = _catalogs.begin();
83  for (; iter != _catalogs.end(); ++iter, ++catArchive) {
84  if (iter->getTable() == catalog.getTable()) {
85  break;
86  }
87  }
88  if (iter == _catalogs.end()) {
90  "All catalogs passed to saveCatalog must be created by makeCatalog");
91  }
92  // Add the name of the class to the header so anyone looking at it can
93  // tell what's stored there. But we don't want to add it multiple times.
94  try {
95  auto names = iter->getTable()->getMetadata()->getArray<std::string>("AR_NAME");
96  if (std::find(names.begin(), names.end(), name) == names.end()) {
97  iter->getTable()->getMetadata()->add("AR_NAME", name, "Class name for objects stored here");
98  }
99  } catch (pex::exceptions::NotFoundError &) {
100  iter->getTable()->getMetadata()->add("AR_NAME", name, "Class name for objects stored here");
101  }
102  // Also add an EXTNAME. The most recent AR_NAME given will be used.
103  iter->getTable()->getMetadata()->set("EXTNAME", name);
104  indexRecord->set(indexKeys.row0, iter->size());
105  indexRecord->set(indexKeys.catArchive, catArchive);
106  iter->insert(iter->end(), catalog.begin(), catalog.end(), false);
107  }
108 
109  int put(Persistable const *obj, std::shared_ptr<Impl> const &self, bool permissive) {
110  if (!obj) return 0;
111  if (permissive && !obj->isPersistable()) return 0;
112  int const currentId = _nextId;
113  ++_nextId;
114  OutputArchiveHandle handle(currentId, obj->getPersistenceName(), obj->getPythonModule(), self);
115  obj->write(handle);
116  return currentId;
117  }
118 
119  int put(std::shared_ptr<Persistable const> obj, std::shared_ptr<Impl> const &self, bool permissive) {
120  if (!obj) return 0;
121  if (permissive && !obj->isPersistable()) return 0;
122  MapItem item(obj, _nextId);
123  std::pair<Map::iterator, bool> r = _map.insert(item);
124  if (r.second) {
125  // We've never seen this object before. Save it.
126  return put(obj.get(), self, permissive);
127  } else {
128  // We had already saved this object, and insert returned an iterator
129  // to the ID we used before; return that.
130  return r.first->second;
131  }
132  }
133 
134  void writeFits(fits::Fits &fitsfile) {
135  _index.getTable()->getMetadata()->set("AR_NCAT", int(_catalogs.size() + 1),
136  "# of catalogs in this archive, including the index");
137  _index.writeFits(fitsfile);
138  int n = 1;
139  for (CatalogVector::const_iterator iter = _catalogs.begin(); iter != _catalogs.end(); ++iter, ++n) {
140  iter->writeFits(fitsfile);
141  }
142  }
143 
146  metadata->set("EXTTYPE", "ARCHIVE_INDEX");
147  metadata->set("EXTNAME", "ARCHIVE_INDEX");
148  metadata->set("AR_CATN", 0, "# of this catalog relative to the start of this archive");
149  _index.getTable()->setMetadata(metadata);
150  }
151 
152  int _nextId{1};
153  Map _map;
156 };
157 
158 // ----- OutputArchive --------------------------------------------------------------------------------------
159 
161 
162 OutputArchive::OutputArchive(OutputArchive const &other) = default;
163 // Delegate to copy constructor for backward compatibility
165 
166 OutputArchive &OutputArchive::operator=(OutputArchive const &other) = default;
167 // Delegate to copy assignment for backward compatibility
168 OutputArchive &OutputArchive::operator=(OutputArchive &&other) { return *this = other; }
169 
171 
172 int OutputArchive::put(Persistable const *obj, bool permissive) {
173  if (!_impl.unique()) { // copy on write
174  std::shared_ptr<Impl> tmp(new Impl(*_impl));
175  _impl.swap(tmp);
176  }
177  return _impl->put(obj, _impl, permissive);
178 }
179 
181  if (!_impl.unique()) { // copy on write
182  std::shared_ptr<Impl> tmp(new Impl(*_impl));
183  _impl.swap(tmp);
184  }
185  return _impl->put(std::move(obj), _impl, permissive);
186 }
187 
188 BaseCatalog const &OutputArchive::getIndexCatalog() const { return _impl->_index; }
189 
191  if (n == 0) return _impl->_index;
192  if (std::size_t(n) > _impl->_catalogs.size() || n < 0) {
193  throw LSST_EXCEPT(
195  (boost::format("Catalog number %d is out of range [0,%d]") % n % _impl->_catalogs.size())
196  .str());
197  }
198  return _impl->_catalogs[n - 1];
199 }
200 
201 std::size_t OutputArchive::countCatalogs() const { return _impl->_catalogs.size() + 1; }
202 
203 void OutputArchive::writeFits(fits::Fits &fitsfile) const { _impl->writeFits(fitsfile); }
204 
205 // ----- OutputArchiveHandle ------------------------------------------------------------------------------
206 
207 BaseCatalog OutputArchiveHandle::makeCatalog(Schema const &schema) { return _impl->makeCatalog(schema); }
208 
209 void OutputArchiveHandle::saveEmpty() { _impl->saveEmpty(_id, _name, _module); }
210 
212  _impl->saveCatalog(catalog, _id, _name, _module, _catPersistable);
213  ++_catPersistable;
214 }
215 
216 int OutputArchiveHandle::put(Persistable const *obj, bool permissive) {
217  // Handle doesn't worry about copy-on-write, because Handles should only exist
218  // while an OutputArchive::put() call is active.
219  return _impl->put(obj, _impl, permissive);
220 }
221 
223  // Handle doesn't worry about copy-on-write, because Handles should only exist
224  // while an OutputArchive::put() call is active.
225  return _impl->put(std::move(obj), _impl, permissive);
226 }
227 
230  : _id(id), _catPersistable(0), _name(name), _module(module), _impl(impl) {}
231 
233 } // namespace io
234 } // namespace table
235 } // namespace afw
236 } // namespace lsst
table::Key< std::string > name
Definition: Amplifier.cc:116
table::Key< int > id
Definition: Detector.cc:162
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
table::Schema schema
Definition: python.h:134
T begin(T... args)
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
std::shared_ptr< RecordT > addNew()
Create a new record, add it to the end of the catalog, and return a pointer to it.
Definition: Catalog.h:490
size_type size() const
Return the number of elements in the catalog.
Definition: Catalog.h:413
iterator begin()
Iterator access.
Definition: Catalog.h:401
std::shared_ptr< Table > getTable() const
Return the table associated with the catalog.
Definition: Catalog.h:115
void writeFits(std::string const &filename, std::string const &mode="w", int flags=0) const
Write a FITS binary table to a regular file.
Definition: Catalog.h:311
Defines the fields and offsets for a table.
Definition: Schema.h:51
@ EQUAL_NAMES
Fields have the same names (ordered).
Definition: Schema.h:67
@ EQUAL_KEYS
Keys have the same types offsets, and sizes.
Definition: Schema.h:66
A vector of catalogs used by Persistable.
Definition: CatalogVector.h:29
int put(Persistable const *obj, std::shared_ptr< Impl > const &self, bool permissive)
BaseCatalog makeCatalog(Schema const &schema)
int put(std::shared_ptr< Persistable const > obj, std::shared_ptr< Impl > const &self, bool permissive)
void writeFits(fits::Fits &fitsfile)
void saveEmpty(int id, std::string const &name, std::string const &module)
void saveCatalog(BaseCatalog const &catalog, int id, std::string const &name, std::string const &module, int catPersistable)
std::shared_ptr< BaseRecord > addIndexRecord(int id, std::string const &name, std::string const &module)
An object passed to Persistable::write to allow it to persist itself.
void saveCatalog(BaseCatalog const &catalog)
Save a catalog in the archive.
OutputArchiveHandle(const OutputArchiveHandle &)=delete
void saveEmpty()
Indicate that the object being persisted has no state, and hence will never call makeCatalog() or sav...
BaseCatalog makeCatalog(Schema const &schema)
Return a new, empty catalog with the given schema.
int put(Persistable const *obj, bool permissive=false)
Save an object to the archive and return a unique ID that can be used to retrieve it from an InputArc...
A multi-catalog archive object used to save table::io::Persistable objects.
Definition: OutputArchive.h:34
std::size_t countCatalogs() const
Return the total number of catalogs, including the index.
int put(std::shared_ptr< Persistable const > obj, bool permissive=false)
Save an object to the archive and return a unique ID that can be used to retrieve it from an InputArc...
void writeFits(fits::Fits &fitsfile) const
Write the archive to an already-open FITS object.
BaseCatalog const & getCatalog(int n) const
Return the nth catalog. Catalog 0 is always the index catalog.
OutputArchive & operator=(OutputArchive const &other)
Assign from another OutputArchive. Saved objects are not deep-copied.
BaseCatalog const & getIndexCatalog() const
Return the index catalog that specifies where objects are stored in the data catalogs.
OutputArchive()
Construct an empty OutputArchive containing no objects.
A base class for objects that can be persisted via afw::table::io Archive classes.
Definition: Persistable.h:74
virtual bool isPersistable() const noexcept
Return true if this particular object can be persisted using afw::table::io.
Definition: Persistable.h:102
virtual std::string getPythonModule() const
Return the fully-qualified Python module that should be imported to guarantee that its factory is reg...
Definition: Persistable.cc:36
virtual std::string getPersistenceName() const
Return the unique name used to persist this object and look up its factory.
Definition: Persistable.cc:34
virtual void write(OutputArchiveHandle &handle) const
Write the object to one or more catalogs.
Definition: Persistable.cc:38
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
Reports attempts to exceed implementation-defined length limits for some classes.
Definition: Runtime.h:76
Reports errors in the logical structure of the program.
Definition: Runtime.h:46
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
T end(T... args)
T find(T... args)
T get(T... args)
T insert(T... args)
T move(T... args)
CatalogT< BaseRecord > BaseCatalog
Definition: fwd.h:72
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
A base class for image defects.
T size(T... args)
Schema for the index catalog that specifies where objects are stored in the data catalogs.
static constexpr int const NO_CATALOGS_SAVED
Special value used for catArchive, catPersistable, and row0 when an object with no state is saved.
static ArchiveIndexSchema const & get()
Return the singleton instance.
T swap(T... args)
T unique(T... args)