LSSTApplications  19.0.0-14-gb0260a2+b89439f59d,20.0.0+126303c00d,20.0.0+2f3d0e5c40,20.0.0+36ef800059,20.0.0+6ab788e194,20.0.0+a788ff1d83,20.0.0+b545961afb,20.0.0+bebc1f60e8,20.0.0+e2e26847c2,20.0.0+ec00f03e9c,20.0.0-1-g10df615+b92a01c08d,20.0.0-1-g253301a+36ef800059,20.0.0-1-g2b7511a+bebc1f60e8,20.0.0-1-g4d801e7+7c74587066,20.0.0-1-g5b95a8c+3e2d406de1,20.0.0-1-g660595b+f45b7d88f4,20.0.0-1-gc96f8cb+665e8551f1,20.0.0-1-gd1c87d7+85c46248f3,20.0.0-1-gedffbd8+ca2c461684,20.0.0-14-g1ce627f+450400e286,20.0.0-16-g111fe95+665e8551f1,20.0.0-16-g233ea98+b7ef15a6f8,20.0.0-17-ga9337b4+235318a901,20.0.0-2-g4dae9ad+665e8551f1,20.0.0-2-g7818986+85c46248f3,20.0.0-2-gec03fae+ff10c6d78d,20.0.0-25-g38d2b7ae+6870ac0cf6,20.0.0-3-g4cc78c6+63636aeed8,20.0.0-3-g6a8623c+450400e286,20.0.0-3-g750bffe+50c7740d2d,20.0.0-4-gfea843c+f45b7d88f4,20.0.0-48-gd64a390+cf2ddfd0ca,20.0.0-5-g357b56b+f45b7d88f4,20.0.0-5-gfcebe35+806271b712,20.0.0-7-gc818c6e6+7d435f9f99,20.0.0-7-gcda7bf1+1884c7ddc6,20.0.0-9-g61a2a9a3d+14f89e4eca,w.2020.39
LSSTDataManagementBasePackage
InputArchive.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include "boost/format.hpp"
4 
5 #include "lsst/pex/exceptions.h"
10 #include "lsst/afw/fits.h"
11 
12 namespace lsst {
13 namespace afw {
14 namespace table {
15 namespace io {
16 
17 namespace {
18 
19 ArchiveIndexSchema const& indexKeys = ArchiveIndexSchema::get();
20 
21 // Functor to sort records by ID and then by catPersistable
22 struct IndexSortCompare {
23  bool operator()(BaseRecord const& a, BaseRecord const& b) const {
24  if (a.get(indexKeys.id) < b.get(indexKeys.id)) {
25  return true;
26  }
27  if (a.get(indexKeys.id) == b.get(indexKeys.id)) {
28  return a.get(indexKeys.catPersistable) < b.get(indexKeys.catPersistable);
29  }
30  return false;
31  }
32 };
33 
34 } // namespace
35 
36 // ----- InputArchive::Impl ---------------------------------------------------------------------------------
37 
39 public:
42  if (id == 0) return empty;
44  if (r.second) {
45  // insertion successful means we haven't reassembled this object yet; do that now.
46  CatalogVector factoryArgs;
47  // iterate over records in index with this ID; we know they're sorted by ID and then
48  // by catPersistable, so we can just append to factoryArgs.
51  for (BaseCatalog::iterator indexIter = _index.find(id, indexKeys.id);
52  indexIter != _index.end() && indexIter->get(indexKeys.id) == id; ++indexIter) {
53  if (name.empty()) {
54  name = indexIter->get(indexKeys.name);
55  } else if (name != indexIter->get(indexKeys.name)) {
56  throw LSST_EXCEPT(
58  (boost::format("Inconsistent name in index for ID %d; got '%s', expected '%s'") %
59  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.name) % name)
60  .str());
61  }
62  if (module.empty()) {
63  module = indexIter->get(indexKeys.module);
64  } else if (module != indexIter->get(indexKeys.module)) {
65  throw LSST_EXCEPT(
68  "Inconsistent module in index for ID %d; got '%s', expected '%s'") %
69  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.module) % module)
70  .str());
71  }
72  int catArchive = indexIter->get(indexKeys.catArchive);
73  if (catArchive == ArchiveIndexSchema::NO_CATALOGS_SAVED) {
74  break; // object was written with saveEmpty, and hence no catalogs.
75  }
76  std::size_t catN = catArchive - 1;
77  if (catN >= _catalogs.size()) {
78  throw LSST_EXCEPT(
81  "Invalid catalog number in index for ID %d; got '%d', max is '%d'") %
82  indexIter->get(indexKeys.id) % catN % _catalogs.size())
83  .str());
84  }
85  BaseCatalog& fullCatalog = _catalogs[catN];
86  std::size_t i1 = indexIter->get(indexKeys.row0);
87  std::size_t i2 = i1 + indexIter->get(indexKeys.nRows);
88  if (i2 > fullCatalog.size()) {
90  (boost::format("Index and data catalogs do not agree for ID %d; "
91  "catalog %d has %d rows, not %d") %
92  indexIter->get(indexKeys.id) % indexIter->get(indexKeys.catArchive) %
93  fullCatalog.size() % i2)
94  .str());
95  }
96  factoryArgs.push_back(BaseCatalog(fullCatalog.getTable(), fullCatalog.begin() + i1,
97  fullCatalog.begin() + i2));
98  }
99  try {
101  r.first->second = factory.read(self, factoryArgs);
102  } catch (pex::exceptions::Exception& err) {
103  LSST_EXCEPT_ADD(err,
104  (boost::format("loading object with id=%d, name='%s'") % id % name).str());
105  throw;
106  }
107  // If we're loading the object for the first time, and we've failed, we should have already
108  // thrown an exception, and we assert that here.
109  assert(r.first->second);
110  } else if (!r.first->second) {
111  // If we'd already tried and failed to load this object before - but we'd caught the exception
112  // previously (because the calling code didn't consider that to be a fatal error) - we'll
113  // just throw an exception again. While we can't know exactly what was thrown before,
114  // it's most likely it was a NotFoundError because a needed extension package was not setup.
115  // And conveniently it's appropriate to throw that here too, since now the problem is that
116  // the object should have been loaded into the cache and it wasn't found there.
118  (boost::format("Not trying to reload object with id=%d; a previous attempt to "
119  "load it already failed.") %
120  id)
121  .str());
122  }
123  return r.first->second;
124  }
125 
126  Map const& getAll(InputArchive const& self) {
127  int id = 0;
128  for (BaseCatalog::iterator indexIter = _index.begin(); indexIter != _index.end(); ++indexIter) {
129  if (indexIter->get(indexKeys.id) != id) {
130  id = indexIter->get(indexKeys.id);
131  get(id, self);
132  }
133  }
134  return _map;
135  }
136 
138 
139  Impl(BaseCatalog const& index, CatalogVector const& catalogs) : _index(index), _catalogs(catalogs) {
140  if (index.getSchema() != indexKeys.schema) {
141  throw LSST_EXCEPT(pex::exceptions::RuntimeError, "Incorrect schema for index catalog");
142  }
144  _index.sort(IndexSortCompare());
145  }
146 
147  // No copying
148  Impl(const Impl&) = delete;
149  Impl& operator=(const Impl&) = delete;
150 
151  // No moving
152  Impl(Impl&&) = delete;
153  Impl& operator=(Impl&&) = delete;
154 
158 };
159 
160 // ----- InputArchive ---------------------------------------------------------------------------------------
161 
162 InputArchive::InputArchive() : _impl(new Impl()) {}
163 
165 
167  : _impl(new Impl(index, catalogs)) {}
168 
170 // Delegate to copy constructor for backwards compatibility
172 
174  _impl = other._impl;
175  return *this;
176 }
177 // Delegate to copy assignment for backwards compatibility
179 
180 InputArchive::~InputArchive() = default;
181 
182 std::shared_ptr<Persistable> InputArchive::get(int id) const { return _impl->get(id, *this); }
183 
184 InputArchive::Map const& InputArchive::getAll() const { return _impl->getAll(*this); }
185 
187  BaseCatalog index = BaseCatalog::readFits(fitsfile);
188  std::shared_ptr<daf::base::PropertyList> metadata = index.getTable()->popMetadata();
189  assert(metadata); // BaseCatalog::readFits should always read metadata, even if there's nothing there
190  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_INDEX") {
191  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
192  boost::format("Wrong value for archive index EXTTYPE: '%s'") %
193  metadata->get<std::string>("EXTTYPE"));
194  }
195  int nCatalogs = metadata->get<int>("AR_NCAT");
196  CatalogVector catalogs;
197  catalogs.reserve(nCatalogs);
198  for (int n = 1; n < nCatalogs; ++n) {
199  fitsfile.setHdu(1, true); // increment HDU by one
200  catalogs.push_back(BaseCatalog::readFits(fitsfile));
201  metadata = catalogs.back().getTable()->popMetadata();
202  if (metadata->get<std::string>("EXTTYPE") != "ARCHIVE_DATA") {
203  throw LSST_FITS_EXCEPT(fits::FitsError, fitsfile,
204  boost::format("Wrong value for archive data EXTTYPE: '%s'") %
205  metadata->get<std::string>("EXTTYPE"));
206  }
207  if (metadata->get<int>("AR_CATN") != n) {
208  throw LSST_FITS_EXCEPT(
209  fits::FitsError, fitsfile,
210  boost::format("Incorrect order for archive catalogs: AR_CATN=%d found at position %d") %
211  metadata->get<int>("AR_CATN") % n);
212  }
213  }
214  std::shared_ptr<Impl> impl(new Impl(index, catalogs));
215  return InputArchive(impl);
216 }
217 } // namespace io
218 } // namespace table
219 } // namespace afw
220 } // namespace lsst
schema
table::Schema schema
Definition: Amplifier.cc:115
lsst::afw::table::CatalogT::end
iterator end()
Definition: Catalog.h:397
lsst::afw::table::io::PersistableFactory::lookup
static PersistableFactory const & lookup(std::string const &name, std::string const &module="")
Return the factory that has been registered with the given name.
Definition: Persistable.cc:76
std::string
STL class.
std::shared_ptr< Persistable >
lsst::afw::table::io::ArchiveIndexSchema::NO_CATALOGS_SAVED
static constexpr int const NO_CATALOGS_SAVED
Special value used for catArchive, catPersistable, and row0 when an object with no state is saved.
Definition: ArchiveIndexSchema.h:52
ArchiveIndexSchema.h
lsst::afw::fits::FitsError
An exception thrown when problems are found when reading or writing FITS files.
Definition: fits.h:36
std::pair
std::vector::reserve
T reserve(T... args)
lsst::afw::fits::Fits
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
std::vector::size
T size(T... args)
lsst.pex::exceptions::NotFoundError
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
LSST_EXCEPT_ADD
#define LSST_EXCEPT_ADD(e, m)
Add the current location and a message to an existing exception before rethrowing it.
Definition: Exception.h:54
fits.h
lsst::afw
Definition: imageAlgorithm.dox:1
lsst::afw::table::io::InputArchive
A multi-catalog archive object used to load table::io::Persistable objects.
Definition: InputArchive.h:31
lsst::afw::table::io::InputArchive::Impl
Definition: InputArchive.cc:38
lsst::afw::table::io::InputArchive::Impl::_catalogs
CatalogVector _catalogs
Definition: InputArchive.cc:157
std::shared_ptr::get
T get(T... args)
lsst::afw::table::io::ArchiveIndexSchema::get
static ArchiveIndexSchema const & get()
Return the singleton instance.
Definition: ArchiveIndexSchema.cc:14
lsst::afw::table::io::InputArchive::getAll
Map const & getAll() const
Load and return all objects in the archive.
Definition: InputArchive.cc:184
CatalogVector.h
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
lsst::afw::table::io::InputArchive::Impl::getAll
Map const & getAll(InputArchive const &self)
Definition: InputArchive.cc:126
lsst::afw::geom.transform.transformContinued.name
string name
Definition: transformContinued.py:32
std::vector::push_back
T push_back(T... args)
lsst::afw::table::CatalogT::size
size_type size() const
Return the number of elements in the catalog.
Definition: Catalog.h:408
LSST_FITS_EXCEPT
#define LSST_FITS_EXCEPT(type, fitsObj,...)
A FITS-related replacement for LSST_EXCEPT that takes an additional Fits object and uses makeErrorMes...
Definition: fits.h:105
lsst::afw::table::io::InputArchive::~InputArchive
~InputArchive()
lsst::afw::table::io::InputArchive::Impl::Impl
Impl()
Definition: InputArchive.cc:137
lsst::afw::table::io::InputArchive::Impl::Impl
Impl(BaseCatalog const &index, CatalogVector const &catalogs)
Definition: InputArchive.cc:139
lsst::afw::table::io::ArchiveIndexSchema
Schema for the index catalog that specifies where objects are stored in the data catalogs.
Definition: ArchiveIndexSchema.h:35
lsst::afw::table::io::InputArchive::operator=
InputArchive & operator=(InputArchive const &other)
Assignment. Does not deep-copy loaded Persistables.
Definition: InputArchive.cc:173
other
ItemVariant const * other
Definition: Schema.cc:56
lsst::afw::table::CatalogT::find
iterator find(typename Field< T >::Value const &value, Key< T > const &key)
Return an iterator to the record with the given value.
Definition: Catalog.h:756
lsst::afw::table::CatalogT::getTable
std::shared_ptr< Table > getTable() const
Return the table associated with the catalog.
Definition: Catalog.h:114
lsst::afw::fits::Fits::setHdu
void setHdu(int hdu, bool relative=false)
Set the current HDU.
Definition: fits.cc:513
lsst::afw::table::io::CatalogVector
A vector of catalogs used by Persistable.
Definition: CatalogVector.h:29
lsst::afw::table::CatalogT::getSchema
Schema getSchema() const
Return the schema associated with the catalog's table.
Definition: Catalog.h:117
lsst::afw::table::io::InputArchive::InputArchive
InputArchive()
Construct an empty InputArchive that contains no objects.
Definition: InputArchive.cc:162
lsst::afw::table::io::PersistableFactory::read
virtual std::shared_ptr< Persistable > read(InputArchive const &archive, CatalogVector const &catalogs) const =0
Construct a new object from the given InputArchive and vector of catalogs.
lsst::afw::table::CatalogIterator
Iterator class for CatalogT.
Definition: Catalog.h:39
lsst::afw::table::io::PersistableFactory
A base class for factory classes used to reconstruct objects from records.
Definition: Persistable.h:228
std::map< int, std::shared_ptr< Persistable > >
lsst::afw::table::CatalogT::sort
void sort(Key< T > const &key)
Sort the catalog in-place by the field with the given key.
Definition: Catalog.h:749
lsst::afw::table::CatalogT< BaseRecord >::readFits
static CatalogT readFits(std::string const &filename, int hdu=fits::DEFAULT_HDU, int flags=0)
Read a FITS binary table from a regular file.
Definition: Catalog.h:343
lsst::afw::table::io::InputArchive::Impl::get
std::shared_ptr< Persistable > get(int id, InputArchive const &self)
Definition: InputArchive.cc:40
b
table::Key< int > b
Definition: TransmissionCurve.cc:467
lsst
A base class for image defects.
Definition: imageAlgorithm.dox:1
LSST_EXCEPT
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
lsst::afw::table::io::InputArchive::Impl::operator=
Impl & operator=(Impl &&)=delete
exceptions.h
lsst::afw::table::BaseCatalog
CatalogT< BaseRecord > BaseCatalog
Definition: fwd.h:71
lsst::afw::table::io::MalformedArchiveError
An exception thrown when an InputArchive's contents do not make sense.
Definition: Persistable.h:39
lsst::afw::table::io::InputArchive::Impl::Impl
Impl(const Impl &)=delete
a
table::Key< int > a
Definition: TransmissionCurve.cc:466
std::map::insert
T insert(T... args)
lsst::afw::table::io::InputArchive::readFits
static InputArchive readFits(fits::Fits &fitsfile)
Read an object from an already open FITS object.
Definition: InputArchive.cc:186
lsst::afw::table::io::InputArchive::Impl::_index
BaseCatalog _index
Definition: InputArchive.cc:156
InputArchive.h
std::size_t
std::make_pair
T make_pair(T... args)
lsst.pex::exceptions::Exception
Provides consistent interface for LSST exceptions.
Definition: Exception.h:107
lsst::afw::table::io::InputArchive::Impl::Impl
Impl(Impl &&)=delete
lsst::afw::table::CatalogT::begin
iterator begin()
Iterator access.
Definition: Catalog.h:396
lsst::afw::table::io::InputArchive::Impl::_map
Map _map
Definition: InputArchive.cc:155
lsst::afw::table::CatalogT< BaseRecord >
lsst::afw::table::io::InputArchive::Impl::operator=
Impl & operator=(const Impl &)=delete
lsst::meas::modelfit.psf.psfContinued.module
module
Definition: psfContinued.py:42
lsst.pex::exceptions::RuntimeError
Reports errors that are due to events beyond the control of the program.
Definition: Runtime.h:104
Persistable.h
lsst::afw::table::io::InputArchive::get
std::shared_ptr< Persistable > get(int id) const
Load the Persistable with the given ID and return it.
Definition: InputArchive.cc:182