LSST Applications  21.0.0-172-gfb10e10a+18fedfabac,22.0.0+297cba6710,22.0.0+80564b0ff1,22.0.0+8d77f4f51a,22.0.0+a28f4c53b1,22.0.0+dcf3732eb2,22.0.1-1-g7d6de66+2a20fdde0d,22.0.1-1-g8e32f31+297cba6710,22.0.1-1-geca5380+7fa3b7d9b6,22.0.1-12-g44dc1dc+2a20fdde0d,22.0.1-15-g6a90155+515f58c32b,22.0.1-16-g9282f48+790f5f2caa,22.0.1-2-g92698f7+dcf3732eb2,22.0.1-2-ga9b0f51+7fa3b7d9b6,22.0.1-2-gd1925c9+bf4f0e694f,22.0.1-24-g1ad7a390+a9625a72a8,22.0.1-25-g5bf6245+3ad8ecd50b,22.0.1-25-gb120d7b+8b5510f75f,22.0.1-27-g97737f7+2a20fdde0d,22.0.1-32-gf62ce7b1+aa4237961e,22.0.1-4-g0b3f228+2a20fdde0d,22.0.1-4-g243d05b+871c1b8305,22.0.1-4-g3a563be+32dcf1063f,22.0.1-4-g44f2e3d+9e4ab0f4fa,22.0.1-42-gca6935d93+ba5e5ca3eb,22.0.1-5-g15c806e+85460ae5f3,22.0.1-5-g58711c4+611d128589,22.0.1-5-g75bb458+99c117b92f,22.0.1-6-g1c63a23+7fa3b7d9b6,22.0.1-6-g50866e6+84ff5a128b,22.0.1-6-g8d3140d+720564cf76,22.0.1-6-gd805d02+cc5644f571,22.0.1-8-ge5750ce+85460ae5f3,master-g6e05de7fdc+babf819c66,master-g99da0e417a+8d77f4f51a,w.2021.48
LSST Data Management Base Package
FitsSchemaInputMapper.cc
Go to the documentation of this file.
1 // -*- lsst-c++ -*-
2 
3 #include <array>
4 #include <cmath>
5 #include <cstdint>
6 #include <string>
7 #include <algorithm>
8 #include <cctype>
9 #include <regex>
10 
11 #include "boost/multi_index_container.hpp"
12 #include "boost/multi_index/sequenced_index.hpp"
13 #include "boost/multi_index/ordered_index.hpp"
14 #include "boost/multi_index/hashed_index.hpp"
15 #include "boost/multi_index/member.hpp"
16 
17 #include "lsst/log/Log.h"
18 #include "lsst/geom.h"
21 
22 namespace lsst {
23 namespace afw {
24 namespace table {
25 namespace io {
26 
27 namespace {
28 
29 // A quirk of Boost.MultiIndex (which we use for our container of FitsSchemaItems)
30 // that you have to use a special functor (like this one) to set data members
31 // in a container with set indices (because setting those values might require
32 // the element to be moved to a different place in the set). Check out
33 // the Boost.MultiIndex docs for more information.
34 template <std::string FitsSchemaItem::*Member>
35 struct SetFitsSchemaString {
36  void operator()(FitsSchemaItem &item) { item.*Member = _v; }
37  explicit SetFitsSchemaString(std::string const &v) : _v(v) {}
38 
39 private:
40  std::string const &_v;
41 };
42 
43 } // namespace
44 
46 public:
47  // A container class (based on Boost.MultiIndex) that provides three sort orders,
48  // on column number, flag bit, and name (ttype). This allows us to insert fields into the
49  // schema in the correct order, regardless of which order they appear in the
50  // FITS header.
51  using InputContainer = boost::multi_index_container<FitsSchemaItem, boost::multi_index::indexed_by<boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::column>>, boost::multi_index::ordered_non_unique<boost::multi_index::member<FitsSchemaItem, int, &FitsSchemaItem::bit>>, boost::multi_index::hashed_unique<boost::multi_index::member<FitsSchemaItem, std::string, &FitsSchemaItem::ttype>>, boost::multi_index::sequenced<>>>;
52 
53  // Typedefs for the special functors used to set data members.
54  using SetTTYPE = SetFitsSchemaString<&FitsSchemaItem::ttype>;
55  using SetTFORM = SetFitsSchemaString<&FitsSchemaItem::tform>;
56  using SetTCCLS = SetFitsSchemaString<&FitsSchemaItem::tccls>;
57  using SetTUNIT = SetFitsSchemaString<&FitsSchemaItem::tunit>;
58  using SetDoc = SetFitsSchemaString<&FitsSchemaItem::doc>;
59 
60  // Typedefs for the different indices.
65 
66  // Getters for the different indices.
67  ByColumn &byColumn() { return inputs.get<0>(); }
68  ByBit &byBit() { return inputs.get<1>(); }
69  ByName &byName() { return inputs.get<2>(); }
70  AsList &asList() { return inputs.get<3>(); }
71 
72  Impl() {}
73 
74  int version{0};
76  int flagColumn{0};
77  int archiveHdu{-1};
85 };
86 
87 std::size_t FitsSchemaInputMapper::PREPPED_ROWS_FACTOR = 1 << 15; // determined empirically; see DM-19461.
88 
90  : _impl(std::make_shared<Impl>()) {
91  // Set the table version. If AFW_TABLE_VERSION tag exists, use that
92  // If not, set to 0 if it has an AFW_TYPE, Schema default otherwise (DM-590)
93  if (!metadata.exists("AFW_TYPE")) {
95  }
96  _impl->version = metadata.get("AFW_TABLE_VERSION", _impl->version);
97  _impl->type = metadata.get("AFW_TYPE", _impl->type);
98  if (stripMetadata) {
99  metadata.remove("AFW_TABLE_VERSION");
100  }
101  if (stripMetadata) {
102  metadata.remove("AFW_TYPE");
103  }
104 
105  // Find a key that indicates an Archive stored on other HDUs
106  _impl->archiveHdu = metadata.get("AR_HDU", -1);
107  if (_impl->archiveHdu > 0) {
108  --_impl->archiveHdu; // AR_HDU is 1-indexed for historical reasons (RFC-304; see Source.cc)
109  if (stripMetadata) {
110  metadata.remove("AR_HDU");
111  }
112  }
113 
114  // Read aliases, stored as header entries with key 'ALIAS'
115  try {
116  std::vector<std::string> rawAliases = metadata.getArray<std::string>("ALIAS");
117  for (auto const &rawAliase : rawAliases) {
118  std::size_t pos = rawAliase.find_first_of(':');
119  if (pos == std::string::npos) {
121  (boost::format("Malformed alias definition: '%s'") % rawAliase).str());
122  }
123  _impl->schema.getAliasMap()->set(rawAliase.substr(0, pos), rawAliase.substr(pos + 1, std::string::npos));
124  }
125  if (stripMetadata) {
126  metadata.remove("ALIAS");
127  }
128  } catch (pex::exceptions::NotFoundError &) {
129  // if there are no aliases, just move on
130  }
131 
132  if (_impl->version == 0) {
133  // Read slots saved using an old mechanism in as aliases, since the new slot mechanism delegates
134  // slot definition to the AliasMap.
135  static std::array<std::pair<std::string, std::string>, 7> oldSlotKeys = {
136  {std::make_pair("PSF_FLUX", "slot_PsfFlux"), std::make_pair("AP_FLUX", "slot_ApFlux"),
137  std::make_pair("INST_FLUX", "slot_GaussianFlux"),
138  std::make_pair("MODEL_FLUX", "slot_ModelFlux"),
139  std::make_pair("CALIB_FLUX", "slot_CalibFlux"), std::make_pair("CENTROID", "slot_Centroid"),
140  std::make_pair("SHAPE", "slot_Shape")}};
141  for (auto const &oldSlotKey : oldSlotKeys) {
142  std::string target = metadata.get(oldSlotKey.first + "_SLOT", std::string(""));
143  if (!target.empty()) {
144  _impl->schema.getAliasMap()->set(oldSlotKey.second, target);
145  if (stripMetadata) {
146  metadata.remove(oldSlotKey.first);
147  metadata.remove(oldSlotKey.first + "_ERR_SLOT");
148  metadata.remove(oldSlotKey.first + "_FLAG_SLOT");
149  }
150  }
151  }
152  }
153 
154  // Read the rest of the header into the intermediate inputs container.
155  std::vector<std::string> keyList = metadata.getOrderedNames();
156  for (auto const &key : keyList) {
157  if (key.compare(0, 5, "TTYPE") == 0) {
158  int column = std::stoi(key.substr(5)) - 1;
159  auto iter = _impl->byColumn().lower_bound(column);
160  if (iter == _impl->byColumn().end() || iter->column != column) {
161  iter = _impl->byColumn().insert(iter, FitsSchemaItem(column, -1));
162  }
163  std::string v = metadata.get<std::string>(key);
164  _impl->byColumn().modify(iter, Impl::SetTTYPE(v));
165  if (iter->doc.empty()) { // don't overwrite if already set with TDOCn
166  _impl->byColumn().modify(iter, Impl::SetDoc(metadata.getComment(key)));
167  }
168  if (stripMetadata) {
169  metadata.remove(key);
170  }
171  } else if (key.compare(0, 5, "TFLAG") == 0) {
172  int bit = std::stoi(key.substr(5)) - 1;
173  auto iter = _impl->byBit().lower_bound(bit);
174  if (iter == _impl->byBit().end() || iter->bit != bit) {
175  iter = _impl->byBit().insert(iter, FitsSchemaItem(-1, bit));
176  }
177  std::string v = metadata.get<std::string>(key);
178  _impl->byBit().modify(iter, Impl::SetTTYPE(v));
179  if (iter->doc.empty()) { // don't overwrite if already set with TFDOCn
180  _impl->byBit().modify(iter, Impl::SetDoc(metadata.getComment(key)));
181  }
182  if (stripMetadata) {
183  metadata.remove(key);
184  }
185  } else if (key.compare(0, 4, "TDOC") == 0) {
186  int column = std::stoi(key.substr(4)) - 1;
187  auto iter = _impl->byColumn().lower_bound(column);
188  if (iter == _impl->byColumn().end() || iter->column != column) {
189  iter = _impl->byColumn().insert(iter, FitsSchemaItem(column, -1));
190  }
191  _impl->byColumn().modify(iter, Impl::SetDoc(metadata.get<std::string>(key)));
192  if (stripMetadata) {
193  metadata.remove(key);
194  }
195  } else if (key.compare(0, 5, "TFDOC") == 0) {
196  int bit = std::stoi(key.substr(5)) - 1;
197  auto iter = _impl->byBit().lower_bound(bit);
198  if (iter == _impl->byBit().end() || iter->bit != bit) {
199  iter = _impl->byBit().insert(iter, FitsSchemaItem(-1, bit));
200  }
201  _impl->byBit().modify(iter, Impl::SetDoc(metadata.get<std::string>(key)));
202  if (stripMetadata) {
203  metadata.remove(key);
204  }
205  } else if (key.compare(0, 5, "TUNIT") == 0) {
206  int column = std::stoi(key.substr(5)) - 1;
207  auto iter = _impl->byColumn().lower_bound(column);
208  if (iter == _impl->byColumn().end() || iter->column != column) {
209  iter = _impl->byColumn().insert(iter, FitsSchemaItem(column, -1));
210  }
211  _impl->byColumn().modify(iter, Impl::SetTUNIT(metadata.get<std::string>(key)));
212  if (stripMetadata) {
213  metadata.remove(key);
214  }
215  } else if (key.compare(0, 5, "TCCLS") == 0) {
216  int column = std::stoi(key.substr(5)) - 1;
217  auto iter = _impl->byColumn().lower_bound(column);
218  if (iter == _impl->byColumn().end() || iter->column != column) {
219  iter = _impl->byColumn().insert(iter, FitsSchemaItem(column, -1));
220  }
221  _impl->byColumn().modify(iter, Impl::SetTCCLS(metadata.get<std::string>(key)));
222  if (stripMetadata) {
223  metadata.remove(key);
224  }
225  } else if (key.compare(0, 5, "TFORM") == 0) {
226  int column = std::stoi(key.substr(5)) - 1;
227  auto iter = _impl->byColumn().lower_bound(column);
228  if (iter == _impl->byColumn().end() || iter->column != column) {
229  iter = _impl->byColumn().insert(iter, FitsSchemaItem(column, -1));
230  }
231  _impl->byColumn().modify(iter, Impl::SetTFORM(metadata.get<std::string>(key)));
232  if (stripMetadata) {
233  metadata.remove(key);
234  }
235  } else if (key.compare(0, 5, "TZERO") == 0) {
236  if (stripMetadata) {
237  metadata.remove(key);
238  }
239  } else if (key.compare(0, 5, "TSCAL") == 0) {
240  if (stripMetadata) {
241  metadata.remove(key);
242  }
243  } else if (key.compare(0, 5, "TNULL") == 0) {
244  if (stripMetadata) {
245  metadata.remove(key);
246  }
247  } else if (key.compare(0, 5, "TDISP") == 0) {
248  if (stripMetadata) {
249  metadata.remove(key);
250  }
251  }
252  }
253 
254  // Find the column used to store flags, and setup the flag-handling data members from it.
255  _impl->flagColumn = metadata.get("FLAGCOL", 0);
256  if (_impl->flagColumn > 0) {
257  if (stripMetadata) {
258  metadata.remove("FLAGCOL");
259  }
260  --_impl->flagColumn; // switch from 1-indexed to 0-indexed
261  auto iter = _impl->byColumn().find(_impl->flagColumn);
262  if (iter == _impl->byColumn().end()) {
263  throw LSST_EXCEPT(
265  (boost::format("Column for flag data not found; FLAGCOL=%d") % _impl->flagColumn).str());
266  }
267  // Regex to unpack a FITS TFORM value for a bit array column (TFORM code 'X'). The number
268  // that precedes the code is the size of the array; the number that follows it (if present)
269  // is ignored.
270  static std::regex const regex("(\\d+)?X\\(?(\\d)*\\)?");
271  std::smatch m;
272  if (!std::regex_match(iter->tform, m, regex)) {
273  throw LSST_EXCEPT(
275  (boost::format("Invalid TFORM key for flags column: '%s'") % iter->tform).str());
276  }
277  int nFlags = 1;
278  if (m[1].matched) {
279  nFlags = std::stoi(m[1].str());
280  }
281  _impl->flagKeys.resize(nFlags);
282  _impl->flagWorkspace.reset(new bool[nFlags]);
283  // Delete the flag column from the input list so we don't interpret it as a
284  // regular field.
285  _impl->byColumn().erase(iter);
286  }
287 }
288 
294 
296 
298  int oldHdu = fits.getHdu();
299  if (_impl->archiveHdu < 0) _impl->archiveHdu = oldHdu + 1;
300  try {
301  fits.setHdu(_impl->archiveHdu);
303  fits.setHdu(oldHdu);
304  return true;
305  } catch (afw::fits::FitsError &) {
306  fits.status = 0;
307  fits.setHdu(oldHdu);
308  _impl->archiveHdu = -1;
309  return false;
310  }
311 }
312 
313 bool FitsSchemaInputMapper::hasArchive() const { return static_cast<bool>(_impl->archive); }
314 
316  auto iter = _impl->byName().find(ttype);
317  if (iter == _impl->byName().end()) {
318  return nullptr;
319  }
320  return &(*iter);
321 }
322 
323 FitsSchemaItem const *FitsSchemaInputMapper::find(int column) const {
324  auto iter = _impl->byColumn().lower_bound(column);
325  if (iter == _impl->byColumn().end() || iter->column != column) {
326  return nullptr;
327  }
328  return &(*iter);
329 }
330 
332  auto iter = _impl->byColumn().lower_bound(item->column);
333  assert(iter != _impl->byColumn().end() && iter->column == item->column);
334  _impl->byColumn().erase(iter);
335 }
336 
338  auto iter = _impl->byName().find(ttype);
339  if (iter != _impl->byName().end() && iter->ttype == ttype) {
340  _impl->byName().erase(iter);
341  }
342 }
343 
345  auto iter = _impl->byColumn().lower_bound(column);
346  if (iter != _impl->byColumn().end() && iter->column == column) {
347  _impl->byColumn().erase(iter);
348  }
349 }
350 
351 void erase(int column);
352 
354  _impl->readers.push_back(std::move(reader));
355 }
356 
357 namespace {
358 
359 template <typename T>
360 class StandardReader : public FitsColumnReader {
361 public:
363  FieldBase<T> const &base = FieldBase<T>()) {
364  return std::unique_ptr<FitsColumnReader>(new StandardReader(schema, item, base));
365  }
366 
367  StandardReader(Schema &schema, FitsSchemaItem const &item, FieldBase<T> const &base)
368  : _column(item.column), _key(schema.addField<T>(item.ttype, item.doc, item.tunit, base)),
369  _cache(), _cacheFirstRow(0)
370  {}
371 
372  void prepRead(std::size_t firstRow, std::size_t nRows, fits::Fits & fits) override {
373  // We only prep and cache scalar-valued columns, not array-valued
374  // columns, as apparently the order CFITSIO reads array-valued columns
375  // is not the order we want.
376  if (_key.getElementCount() == 1u) {
377  std::size_t nElements = nRows*_key.getElementCount();
378  _cache.resize(nElements);
379  _cacheFirstRow = firstRow;
380  fits.readTableArray(firstRow, _column, nElements, &_cache.front());
381  }
382  }
383 
384  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
385  std::shared_ptr<InputArchive> const &archive) const override {
386  if (_cache.empty()) {
387  fits.readTableArray(row, _column, _key.getElementCount(), record.getElement(_key));
388  } else {
389  assert(row >= _cacheFirstRow);
390  std::size_t offset = row - _cacheFirstRow;
391  assert(offset < _cache.size());
392  std::copy_n(_cache.begin() + offset, _key.getElementCount(), record.getElement(_key));
393  }
394  }
395 
396 private:
397  int _column;
398  Key<T> _key;
400  std::size_t _cacheFirstRow;
401  std::size_t _nRowsToPrep;
402 };
403 
404 class AngleReader : public FitsColumnReader {
405 public:
407  Schema &schema, FitsSchemaItem const &item,
408  FieldBase<lsst::geom::Angle> const &base = FieldBase<lsst::geom::Angle>()) {
409  return std::unique_ptr<FitsColumnReader>(new AngleReader(schema, item, base));
410  }
411 
412  AngleReader(Schema &schema, FitsSchemaItem const &item, FieldBase<lsst::geom::Angle> const &base)
413  : _column(item.column), _key(schema.addField<lsst::geom::Angle>(item.ttype, item.doc, "", base)) {
414  // We require an LSST-specific key in the headers before parsing a column
415  // as Angle at all, so we don't need to worry about other units or other
416  // spellings of radians. We do continue to support no units for backwards
417  // compatibility.
418  if (!item.tunit.empty() && item.tunit != "rad") {
419  throw LSST_EXCEPT(afw::fits::FitsError,
420  "Angle fields must be persisted in radians (TUNIT='rad').");
421  }
422  }
423 
424  void prepRead(std::size_t firstRow, std::size_t nRows, fits::Fits & fits) override {
425  assert(_key.getElementCount() == 1u);
426  _cache.resize(nRows);
427  _cacheFirstRow = firstRow;
428  fits.readTableArray(firstRow, _column, nRows, &_cache.front());
429  }
430 
431  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
432  std::shared_ptr<InputArchive> const &archive) const override {
433  if (_cache.empty()) {
434  double tmp = 0;
435  fits.readTableScalar(row, _column, tmp);
436  record.set(_key, tmp * lsst::geom::radians);
437  } else {
438  assert(row >= _cacheFirstRow);
439  std::size_t offset = row - _cacheFirstRow;
440  assert(offset < _cache.size());
441  record.set(_key, _cache[offset] * lsst::geom::radians);
442  }
443  }
444 
445 private:
446  int _column;
447  Key<lsst::geom::Angle> _key;
448  std::vector<double> _cache;
449  std::size_t _cacheFirstRow;
450 };
451 
452 class StringReader : public FitsColumnReader {
453 public:
454  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item, int size) {
455  return std::unique_ptr<FitsColumnReader>(new StringReader(schema, item, size));
456  }
457 
458  StringReader(Schema &schema, FitsSchemaItem const &item, int size)
459  : _column(item.column),
460  _key(schema.addField<std::string>(item.ttype, item.doc, item.tunit, size)),
461  _isVariableLength(size == 0) {}
462 
463  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
464  std::shared_ptr<InputArchive> const &archive) const override {
465  std::string s;
466  fits.readTableScalar(row, _column, s, _isVariableLength);
467  record.set(_key, s);
468  }
469 
470 private:
471  int _column;
472  Key<std::string> _key;
473  bool _isVariableLength;
474 };
475 
476 template <typename T>
477 class VariableLengthArrayReader : public FitsColumnReader {
478 public:
479  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item) {
480  return std::unique_ptr<FitsColumnReader>(new VariableLengthArrayReader(schema, item));
481  }
482 
483  VariableLengthArrayReader(Schema &schema, FitsSchemaItem const &item)
484  : _column(item.column), _key(schema.addField<Array<T>>(item.ttype, item.doc, item.tunit, 0)) {}
485 
486  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
487  std::shared_ptr<InputArchive> const &archive) const override {
488  int size = fits.getTableArraySize(row, _column);
489  ndarray::Array<T, 1, 1> array = ndarray::allocate(size);
490  fits.readTableArray(row, _column, size, array.getData());
491  record.set(_key, array);
492  }
493 
494 private:
495  int _column;
496  Key<Array<T>> _key;
497 };
498 
499 // Read a 2-element FITS array column as separate x and y Schema fields (hence converting
500 // from the old Point compound field to the new PointKey FunctorKey).
501 template <typename T>
502 class PointConversionReader : public FitsColumnReader {
503 public:
504  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item) {
505  return std::unique_ptr<FitsColumnReader>(new PointConversionReader(schema, item));
506  }
507 
508  PointConversionReader(Schema &schema, FitsSchemaItem const &item)
509  : _column(item.column), _key(PointKey<T>::addFields(schema, item.ttype, item.doc, item.tunit)) {}
510 
511  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
512  std::shared_ptr<InputArchive> const &archive) const override {
513  std::array<T, 2> buffer;
514  fits.readTableArray(row, _column, 2, buffer.data());
515  record.set(_key, lsst::geom::Point<T, 2>(buffer[0], buffer[1]));
516  }
517 
518 private:
519  int _column;
520  PointKey<T> _key;
521 };
522 
523 // Read a 2-element FITS array column as separate ra and dec Schema fields (hence converting
524 // from the old Coord compound field to the new CoordKey FunctorKey).
525 class CoordConversionReader : public FitsColumnReader {
526 public:
527  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item) {
528  return std::unique_ptr<FitsColumnReader>(new CoordConversionReader(schema, item));
529  }
530 
531  CoordConversionReader(Schema &schema, FitsSchemaItem const &item)
532  : _column(item.column), _key(CoordKey::addFields(schema, item.ttype, item.doc)) {}
533 
534  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
535  std::shared_ptr<InputArchive> const &archive) const override {
537  fits.readTableArray(row, _column, 2, buffer.data());
538  record.set(_key, lsst::geom::SpherePoint(buffer[0], buffer[1]));
539  }
540 
541 private:
542  int _column;
543  CoordKey _key;
544 };
545 
546 // Read a 3-element FITS array column as separate xx, yy, and xy Schema fields (hence converting
547 // from the old Moments compound field to the new QuadrupoleKey FunctorKey).
548 class MomentsConversionReader : public FitsColumnReader {
549 public:
550  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item) {
551  return std::unique_ptr<FitsColumnReader>(new MomentsConversionReader(schema, item));
552  }
553 
554  MomentsConversionReader(Schema &schema, FitsSchemaItem const &item)
555  : _column(item.column),
556  _key(QuadrupoleKey::addFields(schema, item.ttype, item.doc, CoordinateType::PIXEL)) {}
557 
558  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
559  std::shared_ptr<InputArchive> const &archive) const override {
560  std::array<double, 3> buffer;
561  fits.readTableArray(row, _column, 3, buffer.data());
562  record.set(_key, geom::ellipses::Quadrupole(buffer[0], buffer[1], buffer[2], false));
563  }
564 
565 private:
566  int _column;
567  QuadrupoleKey _key;
568 };
569 
570 // Read a FITS array column representing a packed symmetric matrix into
571 // Schema fields for each element (hence converting from the old Covariance
572 // compound field to the new CovarianceMatrixKey FunctorKey).
573 template <typename T, int N>
574 class CovarianceConversionReader : public FitsColumnReader {
575 public:
576  static std::string guessUnits(std::string const &oldUnits) {
577  static std::regex const regex("(.*)(\\^(\\d+))?");
578  std::smatch m;
579  if (!std::regex_match(oldUnits, m, regex)) {
580  int oldPower = std::stoi(m[2]);
581  int newPower = std::sqrt(oldPower);
582  return std::to_string(newPower);
583  }
584  return oldUnits;
585  }
586 
587  static std::unique_ptr<FitsColumnReader> make(Schema &schema, FitsSchemaItem const &item,
588  std::vector<std::string> const &names) {
589  return std::unique_ptr<FitsColumnReader>(new CovarianceConversionReader(schema, item, names));
590  }
591 
592  CovarianceConversionReader(Schema &schema, FitsSchemaItem const &item,
593  std::vector<std::string> const &names)
594  : _column(item.column),
595  _size(names.size()),
596  _key(CovarianceMatrixKey<T, N>::addFields(schema, item.ttype, names, guessUnits(item.tunit))),
597  _buffer(new T[detail::computeCovariancePackedSize(names.size())]) {}
598 
599  void readCell(BaseRecord &record, std::size_t row, afw::fits::Fits &fits,
600  std::shared_ptr<InputArchive> const &archive) const override {
601  fits.readTableArray(row, _column, detail::computeCovariancePackedSize(_size), _buffer.get());
602  for (int i = 0; i < _size; ++i) {
603  for (int j = i; j < _size; ++j) {
604  _key.setElement(record, i, j, _buffer[detail::indexCovariance(i, j)]);
605  }
606  }
607  }
608 
609 private:
610  int _column;
611  int _size;
612  CovarianceMatrixKey<T, N> _key;
613  std::unique_ptr<T[]> _buffer;
614 };
615 
616 std::unique_ptr<FitsColumnReader> makeColumnReader(Schema &schema, FitsSchemaItem const &item) {
617  // Regex to unpack a FITS TFORM value. The first number is the size of the array (1 if not present),
618  // followed by an alpha code indicating the type (preceded by P or Q for variable size array).
619  // The last number is ignored.
620  static std::regex const regex("(\\d+)?([PQ])?([A-Z])\\(?(\\d)*\\)?");
621  // start by parsing the format; this tells the element type of the field and the number of elements
622  std::smatch m;
623  if (!std::regex_match(item.tform, m, regex)) {
625  }
626  int size = 1;
627  if (m[1].matched) {
628  size = std::stoi(m[1].str());
629  }
630  char code = m[3].str()[0];
631  if (m[2].matched) {
632  // P or Q presence indicates a variable-length array, which we can get by just setting the
633  // size to zero and letting the rest of the logic run its course.
634  size = 0;
635  }
636  // switch code over FITS codes that correspond to different element types
637  switch (code) {
638  case 'B': // 8-bit unsigned integers -- can only be scalars or Arrays
639  if (size == 1) {
640  if (item.tccls == "Array") {
641  return StandardReader<Array<std::uint8_t>>::make(schema, item, size);
642  }
643  return StandardReader<std::uint8_t>::make(schema, item);
644  }
645  if (size == 0) {
646  return VariableLengthArrayReader<std::uint8_t>::make(schema, item);
647  }
648  return StandardReader<Array<std::uint8_t>>::make(schema, item, size);
649 
650  case 'I': // 16-bit integers - can only be scalars or Arrays (we assume they're unsigned, since
651  // that's all we ever write, and CFITSIO will complain later if they aren't)
652  if (size == 1) {
653  if (item.tccls == "Array") {
654  return StandardReader<Array<std::uint16_t>>::make(schema, item, size);
655  }
656  return StandardReader<std::uint16_t>::make(schema, item);
657  }
658  if (size == 0) {
659  return VariableLengthArrayReader<std::uint16_t>::make(schema, item);
660  }
661  return StandardReader<Array<std::uint16_t>>::make(schema, item, size);
662  case 'J': // 32-bit integers - can only be scalars, Point fields, or Arrays
663  if (size == 0) {
664  return VariableLengthArrayReader<std::int32_t>::make(schema, item);
665  }
666  if (item.tccls == "Point") {
667  return PointConversionReader<std::int32_t>::make(schema, item);
668  }
669  if (size > 1 || item.tccls == "Array") {
670  return StandardReader<Array<std::int32_t>>::make(schema, item, size);
671  }
672  return StandardReader<std::int32_t>::make(schema, item);
673  case 'K': // 64-bit integers - can only be scalars.
674  if (size == 1) {
675  return StandardReader<std::int64_t>::make(schema, item);
676  }
677  case 'E': // floats
678  if (size == 0) {
679  return VariableLengthArrayReader<float>::make(schema, item);
680  }
681  if (size == 1) {
682  if (item.tccls == "Array") {
683  return StandardReader<Array<float>>::make(schema, item, 1);
684  }
685  // Just use scalars for Covariances of size 1, since that results in more
686  // natural field names (essentially never happens anyway).
687  return StandardReader<float>::make(schema, item);
688  }
689  if (size == 3 && item.tccls == "Covariance(Point)") {
690  std::vector<std::string> names = {"x", "y"};
691  return CovarianceConversionReader<float, 2>::make(schema, item, names);
692  }
693  if (size == 6 && item.tccls == "Covariance(Moments)") {
694  std::vector<std::string> names = {"xx", "yy", "xy"};
695  return CovarianceConversionReader<float, 3>::make(schema, item, names);
696  }
697  if (item.tccls == "Covariance") {
698  double v = 0.5 * (std::sqrt(1 + 8 * size) - 1);
699  std::size_t n = std::lround(v);
700  if (n * (n + 1) != size * 2) {
701  throw LSST_EXCEPT(afw::fits::FitsError, "Covariance field has invalid size.");
702  }
703  std::vector<std::string> names(n);
704  for (std::size_t i = 0; i < n; ++i) {
705  names[i] = std::to_string(i);
706  }
707  return CovarianceConversionReader<float, Eigen::Dynamic>::make(schema, item, names);
708  }
709  return StandardReader<Array<float>>::make(schema, item, size);
710  case 'D': // doubles
711  if (size == 0) {
712  return VariableLengthArrayReader<double>::make(schema, item);
713  }
714  if (size == 1) {
715  if (item.tccls == "Angle") {
716  return AngleReader::make(schema, item);
717  }
718  if (item.tccls == "Array") {
719  return StandardReader<Array<double>>::make(schema, item, 1);
720  }
721  return StandardReader<double>::make(schema, item);
722  }
723  if (size == 2) {
724  if (item.tccls == "Point") {
725  return PointConversionReader<double>::make(schema, item);
726  }
727  if (item.tccls == "Coord") {
728  return CoordConversionReader::make(schema, item);
729  }
730  }
731  if (size == 3 && item.tccls == "Moments") {
732  return MomentsConversionReader::make(schema, item);
733  }
734  return StandardReader<Array<double>>::make(schema, item, size);
735  case 'A': // strings
736  // StringReader can read both fixed-length and variable-length (size=0) strings
737  return StringReader::make(schema, item, size);
738  default:
740  }
741 }
742 
743 bool endswith(std::string const &s, std::string const &suffix) {
744  return s.size() >= suffix.size() && s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
745 }
746 
747 bool isInstFlux(FitsSchemaItem const & item) {
748  // helper lambda to make reading the real logic easier
749  auto includes = [](std::string const & s, char const * target) {
750  return s.find(target) != std::string::npos;
751  };
752  if (!includes(item.ttype, "flux")) return false;
753  if (includes(item.ttype, "modelfit_CModel") && item.tunit.empty()) {
754  // CModel flux fields were written with no units prior to DM-16068,
755  // but should have been "count".
756  return true;
757  }
758  // transform units to lowercase.
759  std::string units(item.tunit);
760  std::transform(units.begin(), units.end(), units.begin(), [](char c) { return std::tolower(c); } );
761  return includes(units, "count") || includes(units, "dn") || includes (units, "adu");
762 }
763 
764 // Replace 'from' with 'to' in 'full', returning the result.
765 std::string replace(std::string full, std::string const & from, std::string const & to) {
766  return full.replace(full.find(from), from.size(), to);
767 }
768 
769 } // namespace
770 
772  if (_impl->version == 0) {
773  AliasMap &aliases = *_impl->schema.getAliasMap();
774  for (auto iter = _impl->asList().begin(); iter != _impl->asList().end(); ++iter) {
775  std::size_t flagPos = iter->ttype.find("flags");
776  if (flagPos != std::string::npos) {
777  // We want to create aliases that resolve "(.*)_flag" to "$1_flags"; old schemas will have
778  // the latter, but new conventions (including slots) expect the former.
779  // But we can't do that, because adding that alias directly results in a cycle in the
780  // aliases (since aliases do partial matches, and keep trying until there are no matches,
781  // we'd have "(.*)_flag" resolve to "$1_flagssssssssssssss...").
782  // Instead, we *rename* from "flags" to "flag", then create the reverse alias.
783  std::string ttype = iter->ttype;
784  std::string prefix = iter->ttype.substr(0, flagPos);
785  ttype.replace(flagPos, 5, "flag");
786  _impl->asList().modify(iter, Impl::SetTTYPE(ttype));
787  // Note that we're not aliasing the full field, just the first part - if we have multiple
788  // flag fields, one alias should be sufficient for all of them (because of partial matching).
789  // Of course, we'll try to recreate that alias every time we handle another flag field with
790  // the same prefix, but AliasMap know hows to handle that no-op set.
791  aliases.set(prefix + "flags", prefix + "flag");
792  } else if (isInstFlux(*iter)) {
793  // Create an alias that resolves "X_instFlux" to "X" or "X_instFluxErr" to "X_err".
794  if (endswith(iter->ttype, "_err")) {
795  aliases.set(replace(iter->ttype, "_err", "_instFluxErr"), iter->ttype);
796  } else {
797  aliases.set(iter->ttype + "_instFlux", iter->ttype);
798  }
799  } else if (endswith(iter->ttype, "_err")) {
800  // Create aliases that resolve "(.*)_(.*)Err" and "(.*)_(.*)_(.*)_Cov" to
801  // "$1_err_$2Err" and "$1_err_$2_$3_Cov", to make centroid and shape uncertainties
802  // available under the new conventions. We don't have to create aliases for the
803  // centroid and shape values themselves, as those will automatically be correct
804  // after the PointConversionReader and MomentsConversionReader do their work.
805  if (iter->tccls == "Covariance(Point)") {
806  aliases.set(replace(iter->ttype, "_err", "_yErr"), iter->ttype + "_yErr");
807  aliases.set(replace(iter->ttype, "_err", "_xErr"), iter->ttype + "_xErr");
808  aliases.set(replace(iter->ttype, "_err", "_x_y_Cov"), iter->ttype + "_x_y_Cov");
809  } else if (iter->tccls == "Covariance(Moments)") {
810  aliases.set(replace(iter->ttype, "_err", "_xxErr"), iter->ttype + "_xxErr");
811  aliases.set(replace(iter->ttype, "_err", "_yyErr"), iter->ttype + "_yyErr");
812  aliases.set(replace(iter->ttype, "_err", "_xyErr"), iter->ttype + "_xyErr");
813  aliases.set(replace(iter->ttype, "_err", "_xx_yy_Cov"), iter->ttype + "_xx_yy_Cov");
814  aliases.set(replace(iter->ttype, "_err", "_xx_xy_Cov"), iter->ttype + "_xx_xy_Cov");
815  aliases.set(replace(iter->ttype, "_err", "_yy_xy_Cov"), iter->ttype + "_yy_xy_Cov");
816  }
817  }
818  }
819  } else if (_impl->version < 3) {
820  // Version == 1 tables use Sigma when we should use Err (see RFC-333) and had no fields
821  // that should have been named Sigma. So provide aliases xErr -> xSigma.
822  // Version <= 2 tables used _flux when we should use _instFlux (see RFC-322).
823  AliasMap &aliases = *_impl->schema.getAliasMap();
824  for (auto iter = _impl->asList().begin(); iter != _impl->asList().end(); ++iter) {
825  std::string name = iter->ttype;
826  if (_impl->version < 2 && endswith(name, "Sigma")) {
827  name = replace(std::move(name), "Sigma", "Err");
828  }
829  if (_impl->version < 3 && isInstFlux(*iter)) {
830  name = replace(std::move(name), "flux", "instFlux");
831  }
832  if (name != iter->ttype) {
833  aliases.set(name, iter->ttype);
834  }
835  }
836  }
837  for (auto iter = _impl->asList().begin(); iter != _impl->asList().end(); ++iter) {
838  if (iter->bit < 0) { // not a Flag column
839  std::unique_ptr<FitsColumnReader> reader = makeColumnReader(_impl->schema, *iter);
840  if (reader) {
841  _impl->readers.push_back(std::move(reader));
842  } else {
843  LOGLS_WARN("lsst.afw.FitsSchemaInputMapper", "Format " << iter->tform << " for column "
844  << iter->ttype
845  << " not supported; skipping.");
846  }
847  } else { // is a Flag column
848  if (static_cast<std::size_t>(iter->bit) >= _impl->flagKeys.size()) {
850  (boost::format("Flag field '%s' is is in bit %d (0-indexed) of only %d") %
851  iter->ttype % iter->bit % _impl->flagKeys.size())
852  .str());
853  }
854  _impl->flagKeys[iter->bit] = _impl->schema.addField<Flag>(iter->ttype, iter->doc);
855  }
856  }
857  _impl->asList().clear();
858  if (_impl->schema.getRecordSize() <= 0) {
859  throw LSST_EXCEPT(
861  (boost::format("Non-positive record size: %d; file is corrupt or invalid.") %
862  _impl->schema.getRecordSize()).str()
863  );
864  }
866  return _impl->schema;
867 }
868 
870  if (!_impl->flagKeys.empty()) {
871  fits.readTableArray<bool>(row, _impl->flagColumn, _impl->flagKeys.size(), _impl->flagWorkspace.get());
872  for (std::size_t bit = 0; bit < _impl->flagKeys.size(); ++bit) {
873  record.set(_impl->flagKeys[bit], _impl->flagWorkspace[bit]);
874  }
875  }
876  if (_impl->nRowsToPrep != 1 && row % _impl->nRowsToPrep == 0) {
877  // Give readers a chance to read and cache up to nRowsToPrep rows-
878  // worth of values.
879  std::size_t size = std::min(_impl->nRowsToPrep, fits.countRows() - row);
880  for (auto const &reader : _impl->readers) {
881  reader->prepRead(row, size, fits);
882  }
883  }
884  for (auto const & reader : _impl->readers) {
885  reader->readCell(record, row, fits, _impl->archive);
886  }
887 }
888 } // namespace io
889 } // namespace table
890 } // namespace afw
891 } // namespace lsst
table::Key< std::string > name
Definition: Amplifier.cc:116
Key< Flag > const & target
table::Key< int > type
Definition: Detector.cc:163
#define LSST_EXCEPT(type,...)
Create an exception with a given type.
Definition: Exception.h:48
Fits * fits
Definition: FitsWriter.cc:90
int nFlags
Definition: FitsWriter.cc:91
LSST DM logging module built on log4cxx.
#define LOGLS_WARN(logger, message)
Log a warn-level message using an iostream-based interface.
Definition: Log.h:659
std::string prefix
Definition: SchemaMapper.cc:72
int m
Definition: SpanSet.cc:48
table::Key< int > from
table::Key< int > to
table::Schema schema
Definition: python.h:134
An exception thrown when problems are found when reading or writing FITS files.
Definition: fits.h:36
A simple struct that combines the two arguments that must be passed to most cfitsio routines and cont...
Definition: fits.h:297
Mapping class that holds aliases for a Schema.
Definition: AliasMap.h:36
void set(std::string const &alias, std::string const &target)
Add an alias to the schema or replace an existing one.
Definition: AliasMap.cc:82
Base class for all records.
Definition: BaseRecord.h:31
void set(Key< T > const &key, U const &value)
Set value of a field for the given key.
Definition: BaseRecord.h:164
Defines the fields and offsets for a table.
Definition: Schema.h:51
static int const VERSION
Definition: Schema.h:57
Key< T > addField(Field< T > const &field, bool doReplace=false)
Add a new field to the Schema, and return the associated Key.
Definition: Schema.cc:479
std::size_t getRecordSize() const
Return the raw size of a record in bytes.
Definition: Schema.h:149
std::shared_ptr< AliasMap > getAliasMap() const
Return the map of aliases.
Definition: Schema.h:279
Polymorphic reader interface used to read different kinds of objects from one or more FITS binary tab...
boost::multi_index_container< FitsSchemaItem, boost::multi_index::indexed_by< boost::multi_index::ordered_non_unique< boost::multi_index::member< FitsSchemaItem, int, &FitsSchemaItem::column > >, boost::multi_index::ordered_non_unique< boost::multi_index::member< FitsSchemaItem, int, &FitsSchemaItem::bit > >, boost::multi_index::hashed_unique< boost::multi_index::member< FitsSchemaItem, std::string, &FitsSchemaItem::ttype > >, boost::multi_index::sequenced<> >> InputContainer
std::vector< std::unique_ptr< FitsColumnReader > > readers
SetFitsSchemaString<&FitsSchemaItem::tccls > SetTCCLS
SetFitsSchemaString<&FitsSchemaItem::ttype > SetTTYPE
SetFitsSchemaString<&FitsSchemaItem::doc > SetDoc
SetFitsSchemaString<&FitsSchemaItem::tform > SetTFORM
SetFitsSchemaString<&FitsSchemaItem::tunit > SetTUNIT
A class that describes a mapping from a FITS binary table to an afw::table Schema.
FitsSchemaInputMapper & operator=(FitsSchemaInputMapper const &)
static std::size_t PREPPED_ROWS_FACTOR
When processing each column, divide this number by the record size (in bytes) and ask CFITSIO to read...
bool readArchive(afw::fits::Fits &fits)
Set the Archive by reading from the HDU specified by the AR_HDU header entry.
Schema finalize()
Map any remaining items into regular Schema items, and return the final Schema.
void customize(std::unique_ptr< FitsColumnReader > reader)
Customize a mapping by providing a FitsColumnReader instance that will be invoked by readRecords().
void readRecord(BaseRecord &record, afw::fits::Fits &fits, std::size_t row)
Fill a record from a FITS binary table row.
FitsSchemaInputMapper(daf::base::PropertyList &metadata, bool stripMetadata)
Construct a mapper from a PropertyList of FITS header values, stripping recognized keys if desired.
void erase(Item const *item)
Remove the given item (which should have been retrieved via find()) from the mapping,...
bool hasArchive() const
Return true if the mapper has an InputArchive.
Item const * find(std::string const &ttype) const
Find an item with the given column name (ttype), returning nullptr if no such column exists.
void setArchive(std::shared_ptr< InputArchive > archive)
Set the Archive to an externally-provided one, overriding any that may have been read.
A multi-catalog archive object used to load table::io::Persistable objects.
Definition: InputArchive.h:31
static InputArchive readFits(fits::Fits &fitsfile)
Read an object from an already open FITS object.
Class for storing ordered metadata with comments.
Definition: PropertyList.h:68
T get(std::string const &name) const
Get the last value for a property name (possibly hierarchical).
Definition: PropertyList.cc:62
std::string const & getComment(std::string const &name) const
Get the comment for a string property name (possibly hierarchical).
Definition: PropertyList.cc:78
std::vector< T > getArray(std::string const &name) const
Get the vector of values for a property name (possibly hierarchical).
Definition: PropertyList.cc:74
std::vector< std::string > getOrderedNames() const
Get the list of property names, in the order they were added.
Definition: PropertyList.cc:82
virtual void remove(std::string const &name)
Remove all values for a property name (possibly hierarchical).
bool exists(std::string const &name) const
Determine if a name (possibly hierarchical) exists.
A coordinate class intended to represent absolute positions (2-d specialization).
Definition: Point.h:211
Point in an unspecified spherical coordinate system.
Definition: SpherePoint.h:57
Reports attempts to exceed implementation-defined length limits for some classes.
Definition: Runtime.h:76
Reports attempts to access elements using an invalid key.
Definition: Runtime.h:151
T compare(T... args)
T copy_n(T... args)
T data(T... args)
T find(T... args)
T get(T... args)
T includes(T... args)
T make_pair(T... args)
T max(T... args)
T min(T... args)
T move(T... args)
std::size_t computeCovariancePackedSize(std::size_t size)
Defines the packed size of a covariance matrices.
Definition: FieldBase.h:33
std::size_t indexCovariance(std::size_t i, std::size_t j)
Defines the ordering of packed covariance matrices.
Definition: FieldBase.h:30
void erase(int column)
lsst::geom::Angle Angle
Definition: misc.h:33
CoordinateType
Enum used to set units for geometric FunctorKeys.
Definition: aggregates.h:277
constexpr AngleUnit radians
constant with units of radians
Definition: Angle.h:108
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
A base class for image defects.
STL namespace.
T regex_match(T... args)
T replace(T... args)
T reset(T... args)
T lround(T... args)
T size(T... args)
T sqrt(T... args)
int row
Definition: CR.cc:145
table::Key< table::Array< int > > _size
Definition: PsfexPsf.cc:364
T stoi(T... args)
Field base class default implementation (used for numeric scalars and lsst::geom::Angle).
Definition: FieldBase.h:41
A structure that describes a field as a collection of related strings read from the FITS header.
T to_string(T... args)
T transform(T... args)