LSSTApplications  17.0+11,17.0+34,17.0+56,17.0+57,17.0+59,17.0+7,17.0-1-g377950a+33,17.0.1-1-g114240f+2,17.0.1-1-g4d4fbc4+28,17.0.1-1-g55520dc+49,17.0.1-1-g5f4ed7e+52,17.0.1-1-g6dd7d69+17,17.0.1-1-g8de6c91+11,17.0.1-1-gb9095d2+7,17.0.1-1-ge9fec5e+5,17.0.1-1-gf4e0155+55,17.0.1-1-gfc65f5f+50,17.0.1-1-gfc6fb1f+20,17.0.1-10-g87f9f3f+1,17.0.1-11-ge9de802+16,17.0.1-16-ga14f7d5c+4,17.0.1-17-gc79d625+1,17.0.1-17-gdae4c4a+8,17.0.1-2-g26618f5+29,17.0.1-2-g54f2ebc+9,17.0.1-2-gf403422+1,17.0.1-20-g2ca2f74+6,17.0.1-23-gf3eadeb7+1,17.0.1-3-g7e86b59+39,17.0.1-3-gb5ca14a,17.0.1-3-gd08d533+40,17.0.1-30-g596af8797,17.0.1-4-g59d126d+4,17.0.1-4-gc69c472+5,17.0.1-6-g5afd9b9+4,17.0.1-7-g35889ee+1,17.0.1-7-gc7c8782+18,17.0.1-9-gc4bbfb2+3,w.2019.22
LSSTDataManagementBasePackage
InputArchive.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include "boost/format.hpp"
4 
5 #include "lsst/pex/exceptions.h"
10 #include "lsst/afw/fits.h"
11 
12 namespace lsst {
13 namespace afw {
14 namespace table {
15 namespace io {
16 
17 namespace {
18 
19 ArchiveIndexSchema const& indexKeys = ArchiveIndexSchema::get();
20 
21 // Functor to sort records by ID and then by catPersistable
22 struct IndexSortCompare {
23  bool operator()(BaseRecord const& a, BaseRecord const& b) const {
24  if (a.get(indexKeys.id) < b.get(indexKeys.id)) {
25  return true;
26  }
27  if (a.get(indexKeys.id) == b.get(indexKeys.id)) {
28  return a.get(indexKeys.catPersistable) < b.get(indexKeys.catPersistable);
29  }
30  return false;
31  }
32 };
33 
34 } // namespace
35 
36 // ----- InputArchive::Impl ---------------------------------------------------------------------------------
37 
39 public:
42  if (id == 0) return empty;
43  std::pair<Map::iterator, bool> r = _map.insert(std::make_pair(id, empty));
44  if (r.second) {
45  // insertion successful means we haven't reassembled this object yet; do that now.
46  CatalogVector factoryArgs;
47  // iterate over records in index with this ID; we know they're sorted by ID and then
48  // by catPersistable, so we can just append to factoryArgs.
51  for (BaseCatalog::iterator indexIter = _index.find(id, indexKeys.id);
52  indexIter != _index.end() && indexIter->get(indexKeys.id) == id; ++indexIter) {
53  if (name.empty()) {
54  name = indexIter->get(indexKeys.name);
55  } else if (name != indexIter->get(indexKeys.name)) {
56  throw LSST_EXCEPT(
58  (boost::format("Inconsistent name in index for ID %d; got '%s', expected '%s'") %
59  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.name) % name)
60  .str());
61  }
62  if (module.empty()) {
63  module = indexIter->get(indexKeys.module);
64  } else if (module != indexIter->get(indexKeys.module)) {
65  throw LSST_EXCEPT(
68  "Inconsistent module in index for ID %d; got '%s', expected '%s'") %
69  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.module) % module)
70  .str());
71  }
72  int catArchive = indexIter->get(indexKeys.catArchive);
73  if (catArchive == ArchiveIndexSchema::NO_CATALOGS_SAVED) {
74  break; // object was written with saveEmpty, and hence no catalogs.
75  }
76  std::size_t catN = catArchive - 1;
77  if (catN >= _catalogs.size()) {
78  throw LSST_EXCEPT(
81  "Invalid catalog number in index for ID %d; got '%d', max is '%d'") %
82  indexIter->get(indexKeys.id) % catN % _catalogs.size())
83  .str());
84  }
85  BaseCatalog& fullCatalog = _catalogs[catN];
86  std::size_t i1 = indexIter->get(indexKeys.row0);
87  std::size_t i2 = i1 + indexIter->get(indexKeys.nRows);
88  if (i2 > fullCatalog.size()) {
90  (boost::format("Index and data catalogs do not agree for ID %d; "
91  "catalog %d has %d rows, not %d") %
92  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.catArchive) %
93  fullCatalog.size() % i2)
94  .str());
95  }
96  factoryArgs.push_back(BaseCatalog(fullCatalog.getTable(), fullCatalog.begin() + i1,
97  fullCatalog.begin() + i2));
98  }
99  try {
100  PersistableFactory const& factory = PersistableFactory::lookup(name, module);
101  r.first->second = factory.read(self, factoryArgs);
102  } catch (pex::exceptions::Exception& err) {
103  LSST_EXCEPT_ADD(err,
104  (boost::format("loading object with id=%d, name='%s'") % id % name).str());
105  throw;
106  }
107  // If we're loading the object for the first time, and we've failed, we should have already
108  // thrown an exception, and we assert that here.
109  assert(r.first->second);
110  } else if (!r.first->second) {
111  // If we'd already tried and failed to load this object before - but we'd caught the exception
112  // previously (because the calling code didn't consider that to be a fatal error) - we'll
113  // just throw an exception again. While we can't know exactly what was thrown before,
114  // it's most likely it was a NotFoundError because a needed extension package was not setup.
115  // And conveniently it's appropriate to throw that here too, since now the problem is that
116  // the object should have been loaded into the cache and it wasn't found there.
118  (boost::format("Not trying to reload object with id=%d; a previous attempt to "
119  "load it already failed.") %
120  id)
121  .str());
122  }
123  return r.first->second;
124  }
125 
126  Map const& getAll(InputArchive const& self) {
127  int id = 0;
128  for (BaseCatalog::iterator indexIter = _index.begin(); indexIter != _index.end(); ++indexIter) {
129  if (indexIter->get(indexKeys.id) != id) {
130  id = indexIter->get(indexKeys.id);
131  get(id, self);
132  }
133  }
134  return _map;
135  }
136 
137  Impl() : _index(ArchiveIndexSchema::get().schema) {}
138 
139  Impl(BaseCatalog const& index, CatalogVector const& catalogs) : _index(index), _catalogs(catalogs) {
140  if (index.getSchema() != indexKeys.schema) {
141  throw LSST_EXCEPT(pex::exceptions::RuntimeError, "Incorrect schema for index catalog");
142  }
143  _map.insert(std::make_pair(0, std::shared_ptr<Persistable>()));
144  _index.sort(IndexSortCompare());
145  }
146 
147  // No copying
148  Impl(const Impl&) = delete;
149  Impl& operator=(const Impl&) = delete;
150 
151  // No moving
152  Impl(Impl&&) = delete;
153  Impl& operator=(Impl&&) = delete;
154 
158 };
159 
160 // ----- InputArchive ---------------------------------------------------------------------------------------
161 
162 InputArchive::InputArchive() : _impl(new Impl()) {}
163 
165 
167  : _impl(new Impl(index, catalogs)) {}
168 
169 InputArchive::InputArchive(InputArchive const& other) : _impl(other._impl) {}
170 // Delegate to copy constructor for backwards compatibility
172 
174  _impl = other._impl;
175  return *this;
176 }
177 // Delegate to copy assignment for backwards compatibility
179 
180 InputArchive::~InputArchive() = default;
181 
182 std::shared_ptr<Persistable> InputArchive::get(int id) const { return _impl->get(id, *this); }
183 
184 InputArchive::Map const& InputArchive::getAll() const { return _impl->getAll(*this); }
185 
187  BaseCatalog index = BaseCatalog::readFits(fitsfile);
188  std::shared_ptr<daf::base::PropertyList> metadata = index.getTable()->popMetadata();
189  assert(metadata); // BaseCatalog::readFits should always read metadata, even if there's nothing there
190  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_INDEX") {
191  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
192  boost::format("Wrong value for archive index EXTTYPE: '%s'") %
193  metadata->get<std::string>("EXTTYPE"));
194  }
195  int nCatalogs = metadata->get<int>("AR_NCAT");
196  CatalogVector catalogs;
197  catalogs.reserve(nCatalogs);
198  for (int n = 1; n < nCatalogs; ++n) {
199  fitsfile.setHdu(1, true); // increment HDU by one
200  catalogs.push_back(BaseCatalog::readFits(fitsfile));
201  metadata = catalogs.back().getTable()->popMetadata();
202  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_DATA") {
203  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
204  boost::format("Wrong value for archive data EXTTYPE: '%s'") %
205  metadata->get<std::string>("EXTTYPE"));
206  }
207  if (metadata->get<int>("AR_CATN") != n) {
208  throw LSST_FITS_EXCEPT(
209  fits::FitsError, fitsfile,
210  boost::format("Incorrect order for archive catalogs: AR_CATN=%d found at position %d") %
211  metadata->get<int>("AR_CATN") % n);
212  }
213  }
214  std::shared_ptr<Impl> impl(new Impl(index, catalogs));
215  return InputArchive(impl);
216 }
217 } // namespace io
218 } // namespace table
219 } // namespace afw
220 } // namespace lsst
T empty(T... args)
Schema for the index catalog that specifies where objects are stored in the data catalogs.
CatalogT< BaseRecord > BaseCatalog
Definition: fwd.h:71
A base class for factory classes used to reconstruct objects from records.
Definition: Persistable.h:228
table::Key< int > b
std::shared_ptr< Table > getTable() const
Return the table associated with the catalog.
Definition: Catalog.h:114
table::Key< int > a
Map const & getAll() const
Load and return all objects in the archive.
Provides consistent interface for LSST exceptions.
Definition: Exception.h:107
table::Key< int > id
Definition: Detector.cc:166
Schema getSchema() const
Return the schema associated with the catalog&#39;s table.
Definition: Catalog.h:117
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
static PersistableFactory const & lookup(std::string const &name, std::string const &module="")
Return the factory that has been registered with the given name.
Definition: Persistable.cc:76
STL class.
static CatalogT readFits(std::string const &filename, int hdu=fits::DEFAULT_HDU, int flags=0)
Read a FITS binary table from a regular file.
Definition: Catalog.h:343
virtual std::shared_ptr< Persistable > read(InputArchive const &archive, CatalogVector const &catalogs) const =0
Construct a new object from the given InputArchive and vector of catalogs.
An exception thrown when problems are found when reading or writing FITS files.
Definition: fits.h:36
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
T push_back(T... args)
static constexpr int const NO_CATALOGS_SAVED
Special value used for catArchive, catPersistable, and row0 when an object with no state is saved...
A base class for image defects.
InputArchive & operator=(InputArchive const &other)
Assignment. Does not deep-copy loaded Persistables.
Iterator class for CatalogT.
Definition: Catalog.h:38
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
InputArchive()
Construct an empty InputArchive that contains no objects.
T make_pair(T... args)
table::Schema schema
Definition: Camera.cc:161
void setHdu(int hdu, bool relative=false)
Set the current HDU.
Definition: fits.cc:494
Map const & getAll(InputArchive const &self)
T get(T... args)
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
A vector of catalogs used by Persistable.
Definition: CatalogVector.h:29
static InputArchive readFits(fits::Fits &fitsfile)
Read an object from an already open FITS object.
Impl(BaseCatalog const &index, CatalogVector const &catalogs)
ItemVariant const * other
Definition: Schema.cc:56
A multi-catalog archive object used to load table::io::Persistable objects.
Definition: InputArchive.h:31
static ArchiveIndexSchema const & get()
Return the singleton instance.
#define LSST_FITS_EXCEPT(type, fitsObj,...)
A FITS-related replacement for LSST_EXCEPT that takes an additional Fits object and uses makeErrorMes...
Definition: fits.h:105
#define LSST_EXCEPT_ADD(e, m)
Add the current location and a message to an existing exception before rethrowing it...
Definition: Exception.h:54
An exception thrown when an InputArchive&#39;s contents do not make sense.
Definition: Persistable.h:39
std::shared_ptr< Persistable > get(int id) const
Load the Persistable with the given ID and return it.
T reserve(T... args)
Reports errors that are due to events beyond the control of the program.
Definition: Runtime.h:104