LSST Applications  21.0.0+3c14b91618,21.0.0+9f51b1e3f7,21.0.0-1-ga51b5d4+6691386486,21.0.0-10-g2408eff+1a328412bf,21.0.0-10-g560fb7b+52fd22d7b4,21.0.0-10-g8d1d15d+2f9043cae0,21.0.0-10-gcf60f90+d15de71c48,21.0.0-11-g25eff31+d43066e4ef,21.0.0-15-g490e301a+a676f0d5cf,21.0.0-2-g103fe59+4758c8ef83,21.0.0-2-g1367e85+a9f57e981a,21.0.0-2-g45278ab+9f51b1e3f7,21.0.0-2-g5242d73+a9f57e981a,21.0.0-2-g7f82c8f+1bcc828e4f,21.0.0-2-g8f08a60+e6fd6d9ff9,21.0.0-2-ga326454+1bcc828e4f,21.0.0-2-gde069b7+66c51b65da,21.0.0-2-gecfae73+251b9830c3,21.0.0-2-gfc62afb+a9f57e981a,21.0.0-20-g09baf175d+e1e7d1c708,21.0.0-3-g357aad2+854c3902c3,21.0.0-3-g4be5c26+a9f57e981a,21.0.0-3-g65f322c+feaa1990e9,21.0.0-3-g7d9da8d+3c14b91618,21.0.0-3-ge02ed75+bda8df9b93,21.0.0-4-g591bb35+bda8df9b93,21.0.0-4-g65b4814+52fd22d7b4,21.0.0-4-g88306b8+656365ce3f,21.0.0-4-gccdca77+86bf7a300d,21.0.0-4-ge8a399c+b99e86088e,21.0.0-5-gd00fb1e+6a0dc09319,21.0.0-51-gd3b42663+bf9f0d1c8b,21.0.0-6-g2d4f3f3+9f51b1e3f7,21.0.0-7-g04766d7+ee2ae02087,21.0.0-7-g98eecf7+3609eddee2,21.0.0-7-gde99fe0+bda8df9b93,21.0.0-8-gd70ce6f+79e45e76e4,master-gac4afde19b+bda8df9b93,w.2021.10
LSST Data Management Base Package
Schema.cc
Go to the documentation of this file.
1 #include <list>
2 #include <memory>
3 #include <stdexcept>
4 #include <type_traits>
5 
6 #include "boost/mpl/and.hpp"
7 #include "boost/mpl/bool.hpp"
8 #include "boost/iterator/transform_iterator.hpp"
9 #include "boost/iterator/filter_iterator.hpp"
10 #include "boost/preprocessor/seq/for_each.hpp"
11 #include "boost/preprocessor/tuple/to_seq.hpp"
12 
13 #include "lsst/utils/hashCombine.h"
14 #include "lsst/afw/table/Schema.h"
18 #include "lsst/afw/fits.h"
19 
20 namespace lsst {
21 namespace afw {
22 namespace table {
23 
24 //-----------------------------------------------------------------------------------------------------------
25 //----- Miscellaneous Utilities -----------------------------------------------------------------------------
26 //-----------------------------------------------------------------------------------------------------------
27 
28 namespace {
29 
30 inline char getDelimiter() { return '_'; }
31 
32 // Concatenate two strings with a single-character delimiter between them
33 std::string join(std::string const &a, std::string const &b, char delimiter) {
34  std::string full;
35  full.reserve(a.size() + b.size() + 1);
36  full += a;
37  full.push_back(delimiter);
38  full += b;
39  return full;
40 }
41 
42 // Functor to compare two ItemVariants for Key equality.
43 class ItemFunctors {
44  typedef detail::SchemaImpl::ItemVariant ItemVariant;
45 
46  // Compares keys - must be initialized with one ItemVariant and passed the other.
47  struct KeyHelper : public boost::static_visitor<bool> {
48  explicit KeyHelper(ItemVariant const *other_) : other(other_) {}
49 
50  template <typename T>
51  bool operator()(SchemaItem<T> const &a) const {
52  SchemaItem<T> const *b = boost::get<SchemaItem<T> >(other);
53  return (b) && a.key == b->key;
54  }
55 
56  ItemVariant const *other;
57  };
58 
59  // Extracts field name from an ItemVariant
60  struct NameHelper : public boost::static_visitor<std::string const &> {
61  template <typename T>
62  std::string const &operator()(SchemaItem<T> const &a) const {
63  return a.field.getName();
64  }
65  };
66 
67  // Extracts field doc from an ItemVariant
68  struct DocHelper : public boost::static_visitor<std::string const &> {
69  template <typename T>
70  std::string const &operator()(SchemaItem<T> const &a) const {
71  return a.field.getDoc();
72  }
73  };
74 
75  // Extracts field units from an ItemVariant
76  struct UnitsHelper : public boost::static_visitor<std::string const &> {
77  template <typename T>
78  std::string const &operator()(SchemaItem<T> const &a) const {
79  return a.field.getUnits();
80  }
81  };
82 
83 public:
84  static bool compareKeys(ItemVariant const &a, ItemVariant const &b) {
85  return boost::apply_visitor(KeyHelper(&b), a);
86  }
87 
88  static bool compareNames(ItemVariant const &a, ItemVariant const &b) {
89  return boost::apply_visitor(NameHelper(), a) == boost::apply_visitor(NameHelper(), b);
90  }
91 
92  static bool compareDocs(ItemVariant const &a, ItemVariant const &b) {
93  return boost::apply_visitor(DocHelper(), a) == boost::apply_visitor(DocHelper(), b);
94  }
95 
96  static bool compareUnits(ItemVariant const &a, ItemVariant const &b) {
97  return boost::apply_visitor(UnitsHelper(), a) == boost::apply_visitor(UnitsHelper(), b);
98  }
99 };
100 
101 } // namespace
102 
103 //-----------------------------------------------------------------------------------------------------------
104 //----- SchemaImpl implementation ---------------------------------------------------------------------------
105 //-----------------------------------------------------------------------------------------------------------
106 
107 namespace detail {
108 
109 //----- Finding a SchemaItem by field name ------------------------------------------------------------------
110 
111 // This is easier to understand if you start reading from the bottom of this section, with
112 // SchemaImpl::find(std::string const &), then work your way up.
113 
114 namespace {
115 
116 // Given a SchemaItem for a regular field, look for a subfield with the given name.
117 // Return the index of the subfield (>= 0) on success, -1 on failure.
118 template <typename T>
119 inline int findNamedSubfield(
120  SchemaItem<T> const &item, std::string const &name, char delimiter,
121  boost::mpl::true_ * // whether a match is possible based on the type of T; computed by caller
122 ) {
123  if (name.size() <= item.field.getName().size()) return -1;
124 
125  if ( // compare invocation is equivalent to "name.startswith(item.field.getName())" in Python
126  name.compare(0, item.field.getName().size(), item.field.getName()) == 0 &&
127  name[item.field.getName().size()] == delimiter) {
128  int const position = item.field.getName().size() + 1;
129  int const size = name.size() - position;
130  int const nElements = item.field.getElementCount();
131  for (int i = 0; i < nElements; ++i) {
132  if (name.compare(position, size, Key<T>::subfields[i]) == 0) {
133  return i;
134  }
135  }
136  }
137  return -1;
138 }
139 
140 // This is an overload of findNamedSubfield that always fails; it's called when we
141 // know from the type of the field and subfield that they're incompatible.
142 template <typename T>
143 inline int findNamedSubfield(
144  SchemaItem<T> const &item, std::string const &name, char delimiter,
145  boost::mpl::false_ * // whether a match is possible based on the type of T; computed by caller
146 ) {
147  return -1;
148 }
149 
150 // Given a SchemaItem and a subfield index, make a new SchemaItem that corresponds to that
151 // subfield and put it in the result smart pointer.
152 template <typename T, typename U>
153 inline void makeSubfieldItem(
154  SchemaItem<T> const &item, int index, char delimiter, std::unique_ptr<SchemaItem<U> > &result,
155  boost::mpl::true_ * // whether a match is possible based on the types of T and U; computed by caller
156 ) {
157  result.reset(new SchemaItem<U>(detail::Access::extractElement(item.key, index),
158  Field<U>(join(item.field.getName(), Key<T>::subfields[index], delimiter),
159  item.field.getDoc(), item.field.getUnits())));
160 }
161 
162 // An overload of makeSubfieldItem that always fails because we know T and U aren't compatible.
163 template <typename T, typename U>
164 inline void makeSubfieldItem(
165  SchemaItem<T> const &item, int index, char delimiter, std::unique_ptr<SchemaItem<U> > &result,
166  boost::mpl::false_ * // whether a match is possible based on the types of T and U; computed by caller
167 ) {}
168 
169 // This is a Variant visitation functor used to extract subfield items by name.
170 // For example, if we have a Point field "a", if we search the Schema for "a.x",
171 // we want to return a SchemaItem that makes it look like "a.x" is a full-fledged
172 // field in its own right.
173 template <typename U>
174 struct ExtractItemByName : public boost::static_visitor<> {
175  explicit ExtractItemByName(std::string const &name_, char delimiter_)
176  : delimiter(delimiter_), name(name_) {}
177 
178  template <typename T>
179  void operator()(SchemaItem<T> const &item) const {
180  // We want to find out if 'item' has a subfield whose fully-qualified name matches our
181  // name data member. But we also know that the subfield needs to have type U, and that
182  // the field needs to have named subfields.
183  // This typedef is boost::mpl::true_ if all the above is true, and boost::mpl::false_ otherwise.
184  typedef typename boost::mpl::and_<std::is_same<U, typename Field<T>::Element>,
185  boost::mpl::bool_<KeyBase<T>::HAS_NAMED_SUBFIELDS> >::type
186  IsMatchPossible;
187  // We use that type to dispatch one of the two overloads of findNamedSubfield.
188  int n = findNamedSubfield(item, name, delimiter, (IsMatchPossible *)0);
189  // If we have a match, we call another overloaded template to make the subfield.
190  if (n >= 0) makeSubfieldItem(item, n, delimiter, result, (IsMatchPossible *)0);
191  }
192 
193  char delimiter;
194  std::string name; // name we're looking for
195  mutable std::unique_ptr<SchemaItem<U> > result; // where we put the result to signal that we're done
196 };
197 
198 } // namespace
199 
200 // Here's the driver for the find-by-name algorithm.
201 template <typename T>
203  NameMap::const_iterator i = _names.lower_bound(name);
204  if (i != _names.end()) {
205  if (i->first == name) {
206  // got an exact match; we're done if it has the right type, and dead if it doesn't.
207  try {
208  return boost::get<SchemaItem<T> const>(_items[i->second]);
209  } catch (boost::bad_get &err) {
211  (boost::format("Field '%s' does not have the given type.") % name).str());
212  }
213  }
214  }
215  // We didn't get an exact match, but we might be searching for "a.x/a_x" and "a" might be a point field.
216  // Because the names are sorted, we know we overshot it, so we work backwards.
217  ExtractItemByName<T> extractor(name, getDelimiter());
218  while (i != _names.begin()) {
219  --i;
220  boost::apply_visitor(extractor, _items[i->second]); // see if the current item is a match
221  if (extractor.result) return *extractor.result;
222  }
224  (boost::format("Field or subfield with name '%s' not found with type '%s'.") % name %
226  .str());
227 }
228 
229 //----- Finding a SchemaItem by key -------------------------------------------------------------------------
230 
231 // This is easier to understand if you start reading from the bottom of this section, with
232 // SchemaImpl::find(Key<T> const &), then work your way up.
233 
234 namespace {
235 
236 // Given a SchemaItem for a regular field, look for a subfield with the given Key
237 // Return the index of the subfield (>= 0) on success, -1 on failure.
238 template <typename T, typename U>
239 inline int findKeySubfield(
240  SchemaItem<T> const &item, Key<U> const &key,
241  boost::mpl::true_ * // whether a match is possible based on the types of T and U; computed by caller
242 ) {
243  int n = (key.getOffset() - item.key.getOffset()) / sizeof(U);
244  if (n >= 0 && n < item.key.getElementCount()) {
245  return n;
246  }
247  return -1;
248 }
249 
250 // This is an overload of findKeySubfield that always fails; it's called when we
251 // know from the type of the field and subfield that they're incompatible.
252 template <typename T, typename U>
253 inline int findKeySubfield(
254  SchemaItem<T> const &item, Key<U> const &key,
255  boost::mpl::false_ * // whether a match is possible based on the types of T and U; computed by caller
256 ) {
257  return -1;
258 }
259 
260 // This is a Variant visitation functor used to extract subfield items by key.
261 template <typename U>
262 struct ExtractItemByKey : public boost::static_visitor<> {
263  explicit ExtractItemByKey(Key<U> const &key_, char delimiter_) : delimiter(delimiter_), key(key_) {}
264 
265  template <typename T>
266  void operator()(SchemaItem<T> const &item) const {
267  // We want to find out if 'item' has a subfield whose matches our key data member.
268  // But we also know that the subfield needs to have type U.
269  // This typedef is boost::mpl::true_ if the above is true, and boost::mpl::false_ otherwise.
270  typedef typename boost::mpl::and_<std::is_same<U, typename Field<T>::Element>,
271  boost::mpl::bool_<KeyBase<T>::HAS_NAMED_SUBFIELDS> >::type
272  IsMatchPossible;
273  // We use that type to dispatch one of the two overloads of findKeySubfield.
274  int n = findKeySubfield(item, key, (IsMatchPossible *)0);
275  // If we have a match, we call another overloaded template to make the subfield.
276  // (this is the same makeSubfieldItem used in ExtractItemByName, so it's defined up there)
277  if (n >= 0) makeSubfieldItem(item, n, delimiter, result, (IsMatchPossible *)0);
278  }
279 
280  char delimiter;
281  Key<U> key;
283 };
284 
285 } // namespace
286 
287 // Here's the driver for the find-by-key algorithm. It's pretty similar to the find-by-name algorithm.
288 template <typename T>
290  OffsetMap::const_iterator i = _offsets.lower_bound(key.getOffset());
291  if (i != _offsets.end()) {
292  if (i->first == key.getOffset()) {
293  try {
294  return boost::get<SchemaItem<T> const>(_items[i->second]);
295  } catch (boost::bad_get &err) {
296  // just swallow the exception; this might be a subfield key that points to the beginning.
297  }
298  }
299  // We didn't get an exact match, but we might be searching for a subfield.
300  // Because the offsets are sorted, we know we overshot it, so we work backwards.
301  ExtractItemByKey<T> extractor(key, getDelimiter());
302  while (i != _offsets.begin()) {
303  --i;
304  boost::apply_visitor(extractor, _items[i->second]);
305  if (extractor.result) return *extractor.result;
306  }
307  }
309  (boost::format("Field or subfield with offset %d not found with type '%s'.") %
310  key.getOffset() % Field<T>::getTypeString())
311  .str());
312 }
313 
314 // We handle Flag fields separately when searching for keys, because their keys aren't like the others.
316  FlagMap::const_iterator i = _flags.lower_bound(std::make_pair(key.getOffset(), key.getBit()));
317  if (i != _flags.end()) {
318  if (i->first.first == key.getOffset() && i->first.second == key.getBit()) {
319  try {
320  return boost::get<SchemaItem<Flag> const>(_items[i->second]);
321  } catch (boost::bad_get &err) {
323  (boost::format("Flag field with offset %d and bit %d not found.") %
324  key.getOffset() % key.getBit())
325  .str());
326  }
327  }
328  }
329  // Flag keys are never subfields, so we require an exact match.
331  (boost::format("Flag field with offset %d and bit %d not found.") % key.getOffset() %
332  key.getBit())
333  .str());
334 }
335 
336 //----- Replacing an existing SchemaItem --------------------------------------------------------------------
337 
338 // This is easier to understand if you start reading from the bottom of this section, with
339 // SchemaImpl::replaceField, then work your way up.
340 
341 namespace {
342 
343 // Find an exact SchemaItem by key ('exact' means no subfields, unlike the find member function above)
344 // Return the index into the item container.
345 template <typename T>
346 inline int findKey(SchemaImpl::OffsetMap const &offsets, SchemaImpl::FlagMap const &flags, Key<T> const &key,
347  bool throwIfMissing = true) {
348  SchemaImpl::OffsetMap::const_iterator i = offsets.find(key.getOffset());
349  if (i == offsets.end()) {
350  if (throwIfMissing) {
352  (boost::format("Key of type %s with offset %d not found in Schema") %
353  Field<T>::getTypeString() % key.getOffset())
354  .str());
355  } else {
356  return -1;
357  }
358  }
359  return i->second;
360 }
361 
362 // Like the above, but special-cased for Flag
363 inline int findKey(SchemaImpl::OffsetMap const &offsets, SchemaImpl::FlagMap const &flags,
364  Key<Flag> const &key, bool throwIfMissing = true) {
365  SchemaImpl::FlagMap::const_iterator i = flags.find(std::make_pair(key.getOffset(), key.getBit()));
366  if (i == flags.end()) {
367  if (throwIfMissing) {
368  throw LSST_EXCEPT(
369  pex::exceptions::NotFoundError,
370  (boost::format("Key of type Flag with offset %d and bit %d not found in Schema") %
371  key.getOffset() % key.getBit())
372  .str());
373  } else {
374  return -1;
375  }
376  }
377  return i->second;
378 }
379 
380 } // namespace
381 
382 template <typename T>
384  NameMap::iterator j = _names.find(field.getName());
385  SchemaItem<T> *item = 0;
386  if (j != _names.end()) {
387  // The field name is already present in the Schema; see if it's the one we're replacing.
388  // If we can get the old item with this, we don't need to update the name map at all.
389  item = boost::get<SchemaItem<T> >(&_items[j->second]);
390  if (!item || key != item->key) {
391  throw LSST_EXCEPT(
393  (boost::format("Field with name '%s' already present in schema with a different key.") %
394  field.getName())
395  .str());
396  }
397  }
398  if (!item) { // Need to find the original item by key, since it's a new name.
399  int index = findKey(_offsets, _flags, key);
400  item = boost::get<SchemaItem<T> >(&_items[index]);
401  if (!item) {
403  (boost::format("Incorrect key type '%s'.") % key).str());
404  }
405  j = _names.find(item->field.getName());
406  _names.insert(j, std::pair<std::string, int>(field.getName(), j->second));
407  _names.erase(j);
408  }
409  item->field = field;
410 }
411 
412 //----- Other SchemaImpl things -----------------------------------------------------------------------------
413 
414 template <typename T>
415 int SchemaImpl::contains(SchemaItem<T> const &item, int flags) const {
416  if (!(flags & Schema::EQUAL_KEYS)) {
418  "Can only check whether item is in schema if flags & EQUAL_KEYS");
419  }
420  SchemaItem<T> const *cmpItem = 0;
421  int index = findKey(_offsets, _flags, item.key, false);
422  if (index >= 0) {
423  cmpItem = boost::get<SchemaItem<T> >(&_items[index]);
424  if (!cmpItem) {
425  if ((flags & Schema::EQUAL_NAMES) && cmpItem->field.getName() != item.field.getName()) {
426  flags &= ~Schema::EQUAL_NAMES;
427  }
428  if ((flags & Schema::EQUAL_DOCS) && cmpItem->field.getDoc() != item.field.getDoc()) {
429  flags &= ~Schema::EQUAL_DOCS;
430  }
431  if ((flags & Schema::EQUAL_UNITS) && cmpItem->field.getUnits() != item.field.getUnits()) {
432  flags &= ~Schema::EQUAL_UNITS;
433  }
434  }
435  } else {
436  flags = 0;
437  }
438  return flags;
439 }
440 
443  if (topOnly) {
444  for (NameMap::const_iterator i = _names.begin(); i != _names.end(); ++i) {
445  std::size_t sep = i->first.find(getDelimiter());
446  if (sep == std::string::npos) {
447  result.insert(result.end(), i->first);
448  } else {
449  result.insert(result.end(), i->first.substr(0, sep));
450  }
451  }
452  } else {
453  for (NameMap::const_iterator i = _names.begin(); i != _names.end(); ++i) {
454  result.insert(result.end(), i->first);
455  }
456  }
457  return result;
458 }
459 
462  if (topOnly) {
463  for (NameMap::const_iterator i = _names.lower_bound(prefix); i != _names.end(); ++i) {
464  if (i->first.compare(0, prefix.size(), prefix) != 0) break;
465  std::size_t sep = i->first.find(getDelimiter(), prefix.size() + 1);
466  if (sep == std::string::npos) {
467  result.insert(result.end(),
468  i->first.substr(prefix.size() + 1, i->first.size() - prefix.size()));
469  } else {
470  result.insert(result.end(), i->first.substr(prefix.size() + 1, sep - prefix.size() - 1));
471  }
472  }
473  } else {
474  for (NameMap::const_iterator i = _names.lower_bound(prefix); i != _names.end(); ++i) {
475  if (i->first.compare(0, prefix.size(), prefix) != 0) break;
476  result.insert(result.end(),
477  i->first.substr(prefix.size() + 1, i->first.size() - prefix.size() - 1));
478  }
479  }
480  return result;
481 }
482 
483 template <typename T>
484 Key<Array<T> > SchemaImpl::addField(Field<Array<T> > const &field, bool doReplace) {
485  if (field.isVariableLength()) {
486  // Variable-length array: allocate space for one ndarray
487  return addFieldImpl(sizeof(ndarray::Array<T, 1, 1>), 1, field, doReplace);
488  }
489  // Fixed-length array: allocate space for getElementCount() elements of type T
490  return addFieldImpl(sizeof(typename Field<T>::Element), field.getElementCount(), field, doReplace);
491 }
492 
494  if (field.isVariableLength()) {
495  // Variable-length string: allocate space for one std::string
496  return addFieldImpl(sizeof(std::string), 1, field, doReplace);
497  }
498  // Fixed-length string: allocate space for getElementCount() chars
499  return addFieldImpl(sizeof(typename Field<std::string>::Element), field.getElementCount(), field,
500  doReplace);
501 }
502 
503 template <typename T>
504 Key<T> SchemaImpl::addField(Field<T> const &field, bool doReplace) {
505  return addFieldImpl(sizeof(typename Field<T>::Element), field.getElementCount(), field, doReplace);
506 }
507 
509  static int const ELEMENT_SIZE = sizeof(Field<Flag>::Element);
511  _names.insert(std::pair<std::string, int>(field.getName(), _items.size()));
512  if (!result.second) {
513  if (doReplace) {
514  SchemaItem<Flag> *item = boost::get<SchemaItem<Flag> >(&_items[result.first->second]);
515  if (!item) {
516  throw LSST_EXCEPT(
518  (boost::format("Cannot replace field with name '%s' because types differ.") %
519  field.getName())
520  .str());
521  }
522  if (item->field.getElementCount() != field.getElementCount()) {
523  throw LSST_EXCEPT(
525  (boost::format("Cannot replace field with name '%s' because sizes differ.") %
526  field.getName())
527  .str());
528  }
529  item->field = field;
530  return item->key;
531  } else {
532  throw LSST_EXCEPT(
534  (boost::format("Field with name '%s' already present in schema.") % field.getName())
535  .str());
536  }
537  } else {
538  if (_lastFlagField < 0 || _lastFlagBit >= ELEMENT_SIZE * 8) {
539  int padding = ELEMENT_SIZE - _recordSize % ELEMENT_SIZE;
540  if (padding != ELEMENT_SIZE) {
541  _recordSize += padding;
542  }
543  _lastFlagField = _recordSize;
544  _lastFlagBit = 0;
545  _recordSize += field.getElementCount() * ELEMENT_SIZE;
546  }
547  SchemaItem<Flag> item(detail::Access::makeKey(_lastFlagField, _lastFlagBit), field);
548  ++_lastFlagBit;
549  _flags.insert(std::pair<std::pair<int, int>, int>(
550  std::make_pair(item.key.getOffset(), item.key.getBit()), _items.size()));
551  _items.push_back(item);
552  return item.key;
553  }
554 }
555 
556 template <typename T>
557 Key<T> SchemaImpl::addFieldImpl(int elementSize, int elementCount, Field<T> const &field, bool doReplace) {
559  _names.insert(std::pair<std::string, int>(field.getName(), _items.size()));
560  if (!result.second) {
561  if (doReplace) {
562  SchemaItem<T> *item = boost::get<SchemaItem<T> >(&_items[result.first->second]);
563  if (!item) {
564  throw LSST_EXCEPT(
566  (boost::format("Cannot replace field with name '%s' because types differ.") %
567  field.getName())
568  .str());
569  }
570  // n.b. we don't use elementCount here because we *do* want variable length arrays (for
571  // which we set elementCount == 1, but field->getElementCount() == -1) to compare as different
572  // from fixed-length arrays with a single element.
573  if (item->field.getElementCount() != field.getElementCount()) {
574  throw LSST_EXCEPT(
576  (boost::format("Cannot replace field with name '%s' because sizes differ.") %
577  field.getName())
578  .str());
579  }
580  item->field = field;
581  return item->key;
582  } else {
583  throw LSST_EXCEPT(
585  (boost::format("Field with name '%s' already present in schema.") % field.getName())
586  .str());
587  }
588  } else {
589  int padding = elementSize - _recordSize % elementSize;
590  if (padding != elementSize) {
591  _recordSize += padding;
592  }
593  SchemaItem<T> item(detail::Access::makeKey(field, _recordSize), field);
594  _recordSize += elementCount * elementSize;
595  _offsets.insert(std::pair<int, int>(item.key.getOffset(), _items.size()));
596  _items.push_back(item);
597  return item.key;
598  }
599 }
600 
601 } // namespace detail
602 
603 //-----------------------------------------------------------------------------------------------------------
604 //----- Schema implementation -------------------------------------------------------------------------------
605 //-----------------------------------------------------------------------------------------------------------
606 
607 int const Schema::VERSION;
608 
609 Schema::Schema() : _impl(std::make_shared<Impl>()), _aliases(std::make_shared<AliasMap>()) {}
610 
611 Schema::Schema(Schema const &other) : _impl(other._impl), _aliases(other._aliases) {}
612 // Delegate to copy constructor for backwards compatibility
614 
615 Schema &Schema::operator=(Schema const &) = default;
616 Schema &Schema::operator=(Schema &&) = default;
617 Schema::~Schema() = default;
618 
619 Schema Schema::readFits(std::string const &filename, int hdu) {
621  fp.setHdu(hdu);
622  return readFits(fp);
623 }
624 
627  fp.setHdu(hdu);
628  return readFits(fp);
629 }
630 
633  fitsfile.readMetadata(header, false);
634  return fromFitsMetadata(header);
635 }
636 
638  return io::FitsSchemaInputMapper(header, stripMetadata).finalize();
639 }
640 
642  // delegate to utility funcs at top of this file
643  return afw::table::join(a, b, getDelimiter());
644 }
645 
646 void Schema::_edit() {
647  if (!_impl.unique()) {
648  std::shared_ptr<Impl> data(std::make_shared<Impl>(*_impl));
649  _impl.swap(data);
650  }
651 }
652 
653 std::set<std::string> Schema::getNames(bool topOnly) const { return _impl->getNames(topOnly); }
654 
655 template <typename T>
657  std::string tmp(name);
658  _aliases->_apply(tmp);
659  return _impl->find<T>(tmp);
660 }
661 
662 template <typename T>
664  return _impl->find(key);
665 }
666 
667 template <typename T>
668 Key<T> Schema::addField(Field<T> const &field, bool doReplace) {
669  _edit();
670  return _impl->addField(field, doReplace);
671 }
672 
673 template <typename T>
675  _edit();
676  _impl->replaceField(key, field);
677 }
678 
679 int Schema::contains(Schema const &other, int flags) const {
680  if (_impl == other._impl) return flags;
681  if (_impl->getItems().size() < other._impl->getItems().size()) return 0;
682  int result = flags;
683  if (result & EQUAL_FIELDS) {
684  for (Impl::ItemContainer::const_iterator i1 = _impl->getItems().begin(),
685  i2 = other._impl->getItems().begin();
686  i2 != other._impl->getItems().end(); ++i1, ++i2) {
687  if ((result & EQUAL_KEYS) && !ItemFunctors::compareKeys(*i1, *i2)) result &= ~EQUAL_KEYS;
688  if ((result & EQUAL_NAMES) && !ItemFunctors::compareNames(*i1, *i2)) result &= ~EQUAL_NAMES;
689  if ((result & EQUAL_DOCS) && !ItemFunctors::compareDocs(*i1, *i2)) result &= ~EQUAL_DOCS;
690  if ((result & EQUAL_UNITS) && !ItemFunctors::compareUnits(*i1, *i2)) result &= ~EQUAL_UNITS;
691  if (!result) break;
692  }
693  }
694  if ((result & EQUAL_ALIASES) && !getAliasMap()->contains(*other.getAliasMap())) result &= ~EQUAL_ALIASES;
695  return result;
696 }
697 
698 int Schema::compare(Schema const &other, int flags) const {
699  int result = contains(other, flags);
700  if (_impl->getItems().size() != other._impl->getItems().size()) {
701  result &= ~EQUAL_FIELDS;
702  }
703  if (getAliasMap()->size() != other.getAliasMap()->size()) {
704  result &= ~EQUAL_ALIASES;
705  }
706  return result;
707 }
708 
709 std::size_t Schema::hash_value() const noexcept {
710  // Completely arbitrary seed
711  std::size_t result = 17;
712  auto hasher = [&result](auto const &item) { result = utils::hashCombine(result, item.key); };
713  forEach(hasher);
714  return result;
715 }
716 
717 template <typename T>
718 int Schema::contains(SchemaItem<T> const &item, int flags) const {
719  return _impl->contains(item, flags);
720 }
721 
723  if (!aliases) {
724  aliases = std::make_shared<AliasMap>();
725  }
726  _aliases = aliases;
727 }
728 
729 void Schema::disconnectAliases() { _aliases = std::make_shared<AliasMap>(*_aliases); }
730 
731 //----- Stringification -------------------------------------------------------------------------------------
732 
733 namespace {
734 
735 // Schema::forEach functor used for stringificationx
736 struct Stream {
737  typedef void result_type;
738 
739  template <typename T>
740  void operator()(SchemaItem<T> const &item) const {
741  *os << " (" << item.field << ", " << item.key << "),\n";
742  }
743 
744  explicit Stream(std::ostream *os_) : os(os_) {}
745 
747 };
748 
749 } // namespace
750 
752  os << "Schema(\n";
753  schema.forEach(Stream(&os));
754  for (auto iter = schema.getAliasMap()->begin(); iter != schema.getAliasMap()->end(); ++iter) {
755  os << " '" << iter->first << "'->'" << iter->second << "'\n";
756  }
757  return os << ")\n";
758 }
759 
760 //-----------------------------------------------------------------------------------------------------------
761 //----- SubSchema implementation ----------------------------------------------------------------------------
762 //-----------------------------------------------------------------------------------------------------------
763 
765  // delegate to utility funcs at top of this file
766  return afw::table::join(a, b, getDelimiter());
767 }
768 
769 SubSchema::SubSchema(std::shared_ptr<Impl> impl, std::shared_ptr<AliasMap> aliases, std::string const &name)
770  : _impl(impl), _aliases(aliases), _name(name) {}
771 
772 template <typename T>
774  return _impl->find<T>(_aliases->apply(join(_name, name)));
775 }
776 
778  return SubSchema(_impl, _aliases, join(_name, name));
779 }
780 
781 std::set<std::string> SubSchema::getNames(bool topOnly) const { return _impl->getNames(topOnly, _name); }
782 
783 //-----------------------------------------------------------------------------------------------------------
784 //----- Explicit instantiation ------------------------------------------------------------------------------
785 //-----------------------------------------------------------------------------------------------------------
786 
787 // Note: by instantiating the public functions below, we also instantiate a lot of the private
788 // implementation functions. If you move some of those to a different source file, you'll need
789 // more explicit instantiation.
790 
791 #define INSTANTIATE_LAYOUT(r, data, elem) \
792  template Key<elem> Schema::addField(Field<elem> const &, bool); \
793  template SchemaItem<elem> Schema::find(std::string const &) const; \
794  template SchemaItem<elem> Schema::find(Key<elem> const &) const; \
795  template SchemaItem<elem> detail::SchemaImpl::find(std::string const &name) const; \
796  template int Schema::contains(SchemaItem<elem> const &, int) const; \
797  template void Schema::replaceField(Key<elem> const &, Field<elem> const &); \
798  template SchemaItem<elem> SubSchema::find(std::string const &) const;
799 
801  BOOST_PP_TUPLE_TO_SEQ(AFW_TABLE_FIELD_TYPE_N, AFW_TABLE_FIELD_TYPE_TUPLE))
802 } // namespace table
803 } // namespace afw
804 } // namespace lsst
char * data
Definition: BaseRecord.cc:62
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
char delimiter
Definition: Schema.cc:193
ItemVariant const * other
Definition: Schema.cc:56
std::string name
Definition: Schema.cc:194
#define INSTANTIATE_LAYOUT(r, data, elem)
Definition: Schema.cc:791
Key< U > key
Definition: Schema.cc:281
std::ostream * os
Definition: Schema.cc:746
std::unique_ptr< SchemaItem< U > > result
Definition: Schema.cc:195
std::string prefix
Definition: SchemaMapper.cc:79
table::Schema schema
Definition: python.h:134
T begin(T... args)
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
void readMetadata(daf::base::PropertySet &metadata, bool strip=false)
Read a FITS header into a PropertySet or PropertyList.
Definition: fits.cc:1087
Lifetime-management for memory that goes into FITS memory files.
Definition: fits.h:121
Mapping class that holds aliases for a Schema.
Definition: AliasMap.h:36
Tag types used to declare specialized field types.
Definition: misc.h:32
Key specialization for Flag.
Definition: Flag.h:94
int getOffset() const
Return the offset in bytes of the integer element that holds this field's bit.
Definition: Flag.h:126
int getBit() const
The index of this field's bit within the integer it shares with other Flag fields.
Definition: Flag.h:129
A class used as a handle to a particular field in a table.
Definition: Key.h:53
Defines the fields and offsets for a table.
Definition: Schema.h:50
static int const VERSION
Definition: Schema.h:56
Schema()
Construct an empty Schema.
Definition: Schema.cc:609
void disconnectAliases()
Sever the connection between this schema and any others with which it shares aliases.
Definition: Schema.cc:729
Schema & operator=(Schema const &other)
std::string join(std::string const &a, std::string const &b) const
Join strings using the field delimiter appropriate for this Schema.
Definition: Schema.cc:641
void setAliasMap(std::shared_ptr< AliasMap > aliases)
Set the alias map.
Definition: Schema.cc:722
std::set< std::string > getNames(bool topOnly=false) const
Return a set of field names in the schema.
Definition: Schema.cc:653
static Schema fromFitsMetadata(daf::base::PropertyList &header, bool stripMetadata=true)
Construct from reading a FITS header.
Definition: Schema.cc:637
Key< T > addField(Field< T > const &field, bool doReplace=false)
Add a new field to the Schema, and return the associated Key.
Definition: Schema.cc:668
void forEach(F &&func) const
Apply a functor to each SchemaItem in the Schema.
Definition: Schema.h:212
static Schema readFits(std::string const &filename, int hdu=fits::DEFAULT_HDU)
Construct from reading a FITS file.
Definition: Schema.cc:619
std::size_t hash_value() const noexcept
Return a hash of this object.
Definition: Schema.cc:709
int contains(Schema const &other, int flags=EQUAL_KEYS) const
Test whether the given schema is a subset of this.
Definition: Schema.cc:679
int compare(Schema const &other, int flags=EQUAL_KEYS) const
Do a detailed equality comparison of two schemas.
Definition: Schema.cc:698
void replaceField(Key< T > const &key, Field< T > const &field)
Replace the Field (name/description) for an existing Key.
Definition: Schema.cc:674
SchemaItem< T > find(std::string const &name) const
Find a SchemaItem in the Schema by name.
Definition: Schema.cc:656
@ EQUAL_DOCS
Fields have the same documentation (ordered).
Definition: Schema.h:67
@ EQUAL_NAMES
Fields have the same names (ordered).
Definition: Schema.h:66
@ EQUAL_UNITS
Fields have the same units (ordered).
Definition: Schema.h:68
@ EQUAL_KEYS
Keys have the same types offsets, and sizes.
Definition: Schema.h:65
@ EQUAL_FIELDS
Fields are identical (but aliases may not be).
Definition: Schema.h:69
@ EQUAL_ALIASES
Schemas have identical AliasMaps.
Definition: Schema.h:70
std::shared_ptr< AliasMap > getAliasMap() const
Return the map of aliases.
Definition: Schema.h:269
A proxy type for name lookups in a Schema.
Definition: Schema.h:357
std::string join(std::string const &a, std::string const &b) const
Join strings using the field delimiter appropriate for this Schema.
Definition: Schema.cc:764
SubSchema operator[](std::string const &name) const
Return a nested proxy.
Definition: Schema.cc:777
std::set< std::string > getNames(bool topOnly=false) const
Return a set of nested names that start with the SubSchema's prefix.
Definition: Schema.cc:781
SchemaItem< T > find(std::string const &name) const
Find a nested SchemaItem by name.
Definition: Schema.cc:773
static Key< typename Key< T >::Element > extractElement(KeyBase< T > const &kb, int n)
Definition: Access.h:35
static Key< T > makeKey(int offset)
Definition: Access.h:65
A private implementation class to hide the messy details of Schema.
Definition: SchemaImpl.h:48
std::map< std::pair< int, int >, int > FlagMap
A map from Flag field offset/bit pairs to position in the vector, so we can do Flag field lookups.
Definition: SchemaImpl.h:72
std::set< std::string > getNames(bool topOnly) const
Return a set of field names (used to implement Schema::getNames).
Definition: Schema.cc:441
Key< T > addField(Field< T > const &field, bool doReplace=false)
Add a field to the schema (used to implement Schema::addField).
Definition: Schema.cc:504
int contains(SchemaItem< T > const &item, int flags) const
Definition: Schema.cc:415
void replaceField(Key< T > const &key, Field< T > const &field)
Replace the Field in an existing SchemaItem without changing the Key.
Definition: Schema.cc:383
SchemaItem< T > find(std::string const &name) const
Find an item by name (used to implement Schema::find).
Definition: Schema.cc:202
boost::make_variant_over< ItemTypes >::type ItemVariant
A Boost.Variant type that can hold any one of the allowed SchemaItem types.
Definition: SchemaImpl.h:64
std::map< int, int > OffsetMap
A map from standard field offsets to position in the vector, so we can do field lookups.
Definition: SchemaImpl.h:70
A class that describes a mapping from a FITS binary table to an afw::table Schema.
Schema finalize()
Map any remaining items into regular Schema items, and return the final Schema.
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
Reports invalid arguments.
Definition: Runtime.h:66
Reports errors in the logical structure of the program.
Definition: Runtime.h:46
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
Reports errors from accepting an object of an unexpected or inappropriate type.
Definition: Runtime.h:167
T compare(T... args)
T end(T... args)
T erase(T... args)
T find(T... args)
T insert(T... args)
T lower_bound(T... args)
T make_pair(T... args)
std::ostream & operator<<(std::ostream &os, BaseRecord const &record)
Definition: BaseRecord.cc:159
BOOST_PP_SEQ_FOR_EACH(INSTANTIATE_COLUMNVIEW_SCALAR, _, BOOST_PP_TUPLE_TO_SEQ(AFW_TABLE_SCALAR_FIELD_TYPE_N, AFW_TABLE_SCALAR_FIELD_TYPE_TUPLE)) BOOST_PP_SEQ_FOR_EACH(INSTANTIATE_COLUMNVIEW_ARRAY
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
std::size_t hashCombine(std::size_t seed) noexcept
Combine hashes.
Definition: hashCombine.h:35
A base class for image defects.
STL namespace.
T push_back(T... args)
T reserve(T... args)
T size(T... args)
table::Key< int > type
Definition: Detector.cc:163
table::Key< int > field
Definition: ApCorrMap.cc:77
table::Key< int > b
table::Key< int > a
int getElementCount() const
Return the number of subfield elements (always one for scalars).
Definition: Flag.h:38
A description of a field in a table.
Definition: Field.h:24
FieldBase< T >::Element Element
Type used to store field data in the table (a field may have multiple elements).
Definition: Field.h:26
A simple pair-like struct for mapping a Field (name and description) with a Key (used for actual data...
Definition: SchemaImpl.h:25
#define AFW_TABLE_FIELD_TYPE_N
Definition: types.h:39
#define AFW_TABLE_FIELD_TYPE_TUPLE
Definition: types.h:44