LSST Applications  21.0.0-147-g0e635eb1+1acddb5be5,22.0.0+052faf71bd,22.0.0+1ea9a8b2b2,22.0.0+6312710a6c,22.0.0+729191ecac,22.0.0+7589c3a021,22.0.0+9f079a9461,22.0.1-1-g7d6de66+b8044ec9de,22.0.1-1-g87000a6+536b1ee016,22.0.1-1-g8e32f31+6312710a6c,22.0.1-10-gd060f87+016f7cdc03,22.0.1-12-g9c3108e+df145f6f68,22.0.1-16-g314fa6d+c825727ab8,22.0.1-19-g93a5c75+d23f2fb6d8,22.0.1-19-gb93eaa13+aab3ef7709,22.0.1-2-g8ef0a89+b8044ec9de,22.0.1-2-g92698f7+9f079a9461,22.0.1-2-ga9b0f51+052faf71bd,22.0.1-2-gac51dbf+052faf71bd,22.0.1-2-gb66926d+6312710a6c,22.0.1-2-gcb770ba+09e3807989,22.0.1-20-g32debb5+b8044ec9de,22.0.1-23-gc2439a9a+fb0756638e,22.0.1-3-g496fd5d+09117f784f,22.0.1-3-g59f966b+1e6ba2c031,22.0.1-3-g849a1b8+f8b568069f,22.0.1-3-gaaec9c0+c5c846a8b1,22.0.1-32-g5ddfab5d3+60ce4897b0,22.0.1-4-g037fbe1+64e601228d,22.0.1-4-g8623105+b8044ec9de,22.0.1-5-g096abc9+d18c45d440,22.0.1-5-g15c806e+57f5c03693,22.0.1-7-gba73697+57f5c03693,master-g6e05de7fdc+c1283a92b8,master-g72cdda8301+729191ecac,w.2021.39
LSST Data Management Base Package
InputArchive.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include "boost/format.hpp"
4 
5 #include "lsst/pex/exceptions.h"
10 #include "lsst/afw/fits.h"
11 
12 namespace lsst {
13 namespace afw {
14 namespace table {
15 namespace io {
16 
17 namespace {
18 
19 ArchiveIndexSchema const& indexKeys = ArchiveIndexSchema::get();
20 
21 // Functor to sort records by ID and then by catPersistable
22 struct IndexSortCompare {
23  bool operator()(BaseRecord const& a, BaseRecord const& b) const {
24  if (a.get(indexKeys.id) < b.get(indexKeys.id)) {
25  return true;
26  }
27  if (a.get(indexKeys.id) == b.get(indexKeys.id)) {
28  return a.get(indexKeys.catPersistable) < b.get(indexKeys.catPersistable);
29  }
30  return false;
31  }
32 };
33 
34 } // namespace
35 
36 // ----- InputArchive::Impl ---------------------------------------------------------------------------------
37 
39 public:
42  if (id == 0) return empty;
44  if (r.second) {
45  // insertion successful means we haven't reassembled this object yet; do that now.
46  CatalogVector factoryArgs;
47  // iterate over records in index with this ID; we know they're sorted by ID and then
48  // by catPersistable, so we can just append to factoryArgs.
51  for (BaseCatalog::iterator indexIter = _index.find(id, indexKeys.id);
52  indexIter != _index.end() && indexIter->get(indexKeys.id) == id; ++indexIter) {
53  if (name.empty()) {
54  name = indexIter->get(indexKeys.name);
55  } else if (name != indexIter->get(indexKeys.name)) {
56  throw LSST_EXCEPT(
58  (boost::format("Inconsistent name in index for ID %d; got '%s', expected '%s'") %
59  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.name) % name)
60  .str());
61  }
62  if (module.empty()) {
63  module = indexIter->get(indexKeys.module);
64  } else if (module != indexIter->get(indexKeys.module)) {
65  throw LSST_EXCEPT(
68  "Inconsistent module in index for ID %d; got '%s', expected '%s'") %
69  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.module) % module)
70  .str());
71  }
72  int catArchive = indexIter->get(indexKeys.catArchive);
73  if (catArchive == ArchiveIndexSchema::NO_CATALOGS_SAVED) {
74  break; // object was written with saveEmpty, and hence no catalogs.
75  }
76  std::size_t catN = catArchive - 1;
77  if (catN >= _catalogs.size()) {
78  throw LSST_EXCEPT(
81  "Invalid catalog number in index for ID %d; got '%d', max is '%d'") %
82  indexIter->get(indexKeys.id) % catN % _catalogs.size())
83  .str());
84  }
85  BaseCatalog& fullCatalog = _catalogs[catN];
86  std::size_t i1 = indexIter->get(indexKeys.row0);
87  std::size_t i2 = i1 + indexIter->get(indexKeys.nRows);
88  if (i2 > fullCatalog.size()) {
90  (boost::format("Index and data catalogs do not agree for ID %d; "
91  "catalog %d has %d rows, not %d") %
92  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.catArchive) %
93  fullCatalog.size() % i2)
94  .str());
95  }
96  factoryArgs.push_back(BaseCatalog(fullCatalog.getTable(), fullCatalog.begin() + i1,
97  fullCatalog.begin() + i2));
98  }
99  try {
101  r.first->second = factory.read(self, factoryArgs);
102  } catch (pex::exceptions::Exception& err) {
103  LSST_EXCEPT_ADD(err,
104  (boost::format("loading object with id=%d, name='%s'") % id % name).str());
105  throw;
106  }
107  // If we're loading the object for the first time, and we've failed, we should have already
108  // thrown an exception, and we assert that here.
109  assert(r.first->second);
110  } else if (!r.first->second) {
111  // If we'd already tried and failed to load this object before - but we'd caught the exception
112  // previously (because the calling code didn't consider that to be a fatal error) - we'll
113  // just throw an exception again. While we can't know exactly what was thrown before,
114  // it's most likely it was a NotFoundError because a needed extension package was not setup.
115  // And conveniently it's appropriate to throw that here too, since now the problem is that
116  // the object should have been loaded into the cache and it wasn't found there.
118  (boost::format("Not trying to reload object with id=%d; a previous attempt to "
119  "load it already failed.") %
120  id)
121  .str());
122  }
123  return r.first->second;
124  }
125 
126  Map const& getAll(InputArchive const& self) {
127  int id = 0;
128  for (BaseCatalog::iterator indexIter = _index.begin(); indexIter != _index.end(); ++indexIter) {
129  if (indexIter->get(indexKeys.id) != id) {
130  id = indexIter->get(indexKeys.id);
131  get(id, self);
132  }
133  }
134  return _map;
135  }
136 
138 
139  Impl(BaseCatalog const& index, CatalogVector const& catalogs) : _index(index), _catalogs(catalogs) {
140  if (index.getSchema() != indexKeys.schema) {
141  throw LSST_EXCEPT(pex::exceptions::RuntimeError, "Incorrect schema for index catalog");
142  }
144  _index.sort(IndexSortCompare());
145  }
146 
147  // No copying
148  Impl(const Impl&) = delete;
149  Impl& operator=(const Impl&) = delete;
150 
151  // No moving
152  Impl(Impl&&) = delete;
153  Impl& operator=(Impl&&) = delete;
154 
158 };
159 
160 // ----- InputArchive ---------------------------------------------------------------------------------------
161 
162 InputArchive::InputArchive() : _impl(new Impl()) {}
163 
165 
167  : _impl(new Impl(index, catalogs)) {}
168 
169 InputArchive::InputArchive(InputArchive const& other) = default;
170 // Delegate to copy constructor for backwards compatibility
172 
173 InputArchive& InputArchive::operator=(InputArchive const& other) = default;
174 // Delegate to copy assignment for backwards compatibility
175 InputArchive& InputArchive::operator=(InputArchive&& other) { return *this = other; }
176 
177 InputArchive::~InputArchive() = default;
178 
179 std::shared_ptr<Persistable> InputArchive::get(int id) const { return _impl->get(id, *this); }
180 
181 InputArchive::Map const& InputArchive::getAll() const { return _impl->getAll(*this); }
182 
184  BaseCatalog index = BaseCatalog::readFits(fitsfile);
185  std::shared_ptr<daf::base::PropertyList> metadata = index.getTable()->popMetadata();
186  assert(metadata); // BaseCatalog::readFits should always read metadata, even if there's nothing there
187  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_INDEX") {
188  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
189  boost::format("Wrong value for archive index EXTTYPE: '%s'") %
190  metadata->get<std::string>("EXTTYPE"));
191  }
192  int nCatalogs = metadata->get<int>("AR_NCAT");
193  CatalogVector catalogs;
194  catalogs.reserve(nCatalogs);
195  for (int n = 1; n < nCatalogs; ++n) {
196  fitsfile.setHdu(1, true); // increment HDU by one
197  catalogs.push_back(BaseCatalog::readFits(fitsfile));
198  metadata = catalogs.back().getTable()->popMetadata();
199  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_DATA") {
200  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
201  boost::format("Wrong value for archive data EXTTYPE: '%s'") %
202  metadata->get<std::string>("EXTTYPE"));
203  }
204  if (metadata->get<int>("AR_CATN") != n) {
205  throw LSST_FITS_EXCEPT(
206  fits::FitsError, fitsfile,
207  boost::format("Incorrect order for archive catalogs: AR_CATN=%d found at position %d") %
208  metadata->get<int>("AR_CATN") % n);
209  }
210  }
211  std::shared_ptr<Impl> impl(new Impl(index, catalogs));
212  return InputArchive(impl);
213 }
214 } // namespace io
215 } // namespace table
216 } // namespace afw
217 } // namespace lsst
table::Key< std::string > name
Definition: Amplifier.cc:116
#define LSST_EXCEPT_ADD(e, m)
Add the current location and a message to an existing exception before rethrowing it.
Definition: Exception.h:54
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
table::Key< int > b
table::Key< int > a
table::Schema schema
Definition: python.h:134
An exception thrown when problems are found when reading or writing FITS files.
Definition: fits.h:36
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
void setHdu(int hdu, bool relative=false)
Set the current HDU.
Definition: fits.cc:513
Iterator class for CatalogT.
Definition: Catalog.h:40
static CatalogT readFits(std::string const &filename, int hdu=fits::DEFAULT_HDU, int flags=0)
Read a FITS binary table from a regular file.
Definition: Catalog.h:348
size_type size() const
Return the number of elements in the catalog.
Definition: Catalog.h:413
iterator find(typename Field< T >::Value const &value, Key< T > const &key)
Return an iterator to the record with the given value.
Definition: Catalog.h:763
iterator begin()
Iterator access.
Definition: Catalog.h:401
void sort(Key< T > const &key)
Sort the catalog in-place by the field with the given key.
Definition: Catalog.h:756
std::shared_ptr< Table > getTable() const
Return the table associated with the catalog.
Definition: Catalog.h:115
Schema getSchema() const
Return the schema associated with the catalog's table.
Definition: Catalog.h:118
A vector of catalogs used by Persistable.
Definition: CatalogVector.h:29
Impl(BaseCatalog const &index, CatalogVector const &catalogs)
Impl & operator=(const Impl &)=delete
std::shared_ptr< Persistable > get(int id, InputArchive const &self)
Definition: InputArchive.cc:40
Map const & getAll(InputArchive const &self)
A multi-catalog archive object used to load table::io::Persistable objects.
Definition: InputArchive.h:31
static InputArchive readFits(fits::Fits &fitsfile)
Read an object from an already open FITS object.
InputArchive & operator=(InputArchive const &other)
Assignment. Does not deep-copy loaded Persistables.
Map const & getAll() const
Load and return all objects in the archive.
std::shared_ptr< Persistable > get(int id) const
Load the Persistable with the given ID and return it.
InputArchive()
Construct an empty InputArchive that contains no objects.
An exception thrown when an InputArchive's contents do not make sense.
Definition: Persistable.h:39
A base class for factory classes used to reconstruct objects from records.
Definition: Persistable.h:228
static PersistableFactory const & lookup(std::string const &name, std::string const &module="")
Return the factory that has been registered with the given name.
Definition: Persistable.cc:76
virtual std::shared_ptr< Persistable > read(InputArchive const &archive, CatalogVector const &catalogs) const =0
Construct a new object from the given InputArchive and vector of catalogs.
Provides consistent interface for LSST exceptions.
Definition: Exception.h:107
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
Reports errors that are due to events beyond the control of the program.
Definition: Runtime.h:104
#define LSST_FITS_EXCEPT(type, fitsObj,...)
A FITS-related replacement for LSST_EXCEPT that takes an additional Fits object and uses makeErrorMes...
Definition: fits.h:105
T get(T... args)
T insert(T... args)
T make_pair(T... args)
CatalogT< BaseRecord > BaseCatalog
Definition: fwd.h:72
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
A base class for image defects.
T push_back(T... args)
T reserve(T... args)
T size(T... args)
Schema for the index catalog that specifies where objects are stored in the data catalogs.
static constexpr int const NO_CATALOGS_SAVED
Special value used for catArchive, catPersistable, and row0 when an object with no state is saved.
static ArchiveIndexSchema const & get()
Return the singleton instance.