LSST Applications  21.0.0+75b29a8a7f,21.0.0+e70536a077,21.0.0-1-ga51b5d4+62c747d40b,21.0.0-10-gbfb87ad6+3307648ee3,21.0.0-15-gedb9d5423+47cba9fc36,21.0.0-2-g103fe59+fdf0863a2a,21.0.0-2-g1367e85+d38a93257c,21.0.0-2-g45278ab+e70536a077,21.0.0-2-g5242d73+d38a93257c,21.0.0-2-g7f82c8f+e682ffb718,21.0.0-2-g8dde007+d179fbfa6a,21.0.0-2-g8f08a60+9402881886,21.0.0-2-ga326454+e682ffb718,21.0.0-2-ga63a54e+08647d4b1b,21.0.0-2-gde069b7+26c92b3210,21.0.0-2-gecfae73+0445ed2f95,21.0.0-2-gfc62afb+d38a93257c,21.0.0-27-gbbd0d29+ae871e0f33,21.0.0-28-g5fc5e037+feb0e9397b,21.0.0-3-g21c7a62+f4b9c0ff5c,21.0.0-3-g357aad2+57b0bddf0b,21.0.0-3-g4be5c26+d38a93257c,21.0.0-3-g65f322c+3f454acf5d,21.0.0-3-g7d9da8d+75b29a8a7f,21.0.0-3-gaa929c8+9e4ef6332c,21.0.0-3-ge02ed75+4b120a55c4,21.0.0-4-g3300ddd+e70536a077,21.0.0-4-g591bb35+4b120a55c4,21.0.0-4-gc004bbf+4911b9cd27,21.0.0-4-gccdca77+f94adcd104,21.0.0-4-ge8fba5a+2b3a696ff9,21.0.0-5-gb155db7+2c5429117a,21.0.0-5-gdf36809+637e4641ee,21.0.0-6-g00874e7+c9fd7f7160,21.0.0-6-g4e60332+4b120a55c4,21.0.0-7-gc8ca178+40eb9cf840,21.0.0-8-gfbe0b4b+9e4ef6332c,21.0.0-9-g2fd488a+d83b7cd606,w.2021.05
LSST Data Management Base Package
mapper_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import abc
23 import os
24 import inspect
25 
26 import lsst.afw.geom
27 import lsst.utils.tests
29 import lsst.afw.image
30 import collections
31 
32 __all__ = ["MapperTests"]
33 
34 
35 class MapperTests(metaclass=abc.ABCMeta):
36  """
37  Generic tests of obs_* package mapper functionality.
38 
39  In the subclasses's setUp():
40  * Call setUp_mapper() to fill in required parameters.
41  """
42 
43  def setUp_mapper(self,
44  output=None,
45  path_to_raw=None,
46  keys=None,
47  query_format=None,
48  queryMetadata=None,
49  metadata_output_path=None,
50  map_python_type=None,
51  map_python_std_type=None,
52  map_cpp_type=None,
53  map_storage_name=None,
54  raw_filename=None,
55  default_level=None,
56  raw_levels=None,
57  test_config_metadata=True,
58  ):
59  """
60  Set up the necessary variables for mapper tests.
61 
62  Parameters
63  ----------
64 
65  output : `str`
66  full path to output repository (can be the same as data_dir input
67  repository)
68  path_to_raw : `str`
69  full path to the raw file referenced by dataIds['raw']
70  keys : `set`
71  dictionary keys that this mapper should contain
72  query_format : `list`
73  format list for the results portion of queryMetadata
74  queryMetadata : `tuple` of (`dict`, `tuple`)
75  dataIds and the results of calling them in queryMetadata
76  metadata_output_path : `str`
77  path to metadata output associated with dataIds['raw']
78  map_python_type : `type`
79  full python type specification returned by the mapper for
80  dataIds['raw']
81  map_python_std_type : `type`
82  full python type specification returned by the mapper for
83  dataIds['raw'] after standardization
84  map_cpp_type : `str`
85  C++ type specification returned by the mapper for dataIds['raw']
86  map_storage_name : `str`
87  butler name for the storage type dataIds['raw']
88  raw_filename : `str`
89  Name of the raw files returned by the mapper for dataIds['raw']
90  default_level : `str`
91  value returned from mapper.getDefaultLevel
92  raw_levels : `tuple` of (`str`, `set` of `str`)
93  (level, expect) level and expected mapper return for
94  mapper.getKeys('raw', level)
95  test_config_metadata : `bool`
96  Test persisted config and metadata? These tests may not be
97  appropriate for test stand data. Defaults to True.
98  """
99  fields = ['output',
100  'path_to_raw',
101  'keys',
102  'query_format',
103  'queryMetadata',
104  'metadata_output_path',
105  'map_python_type',
106  'map_python_std_type',
107  'map_cpp_type',
108  'map_storage_name',
109  'raw_filename',
110  'default_level',
111  'raw_levels',
112  'test_config_metadata',
113  ]
114  MapperData = collections.namedtuple("MapperData", fields)
115  self.mapper_datamapper_data = MapperData(output=output,
116  path_to_raw=path_to_raw,
117  keys=keys,
118  query_format=query_format,
119  queryMetadata=queryMetadata,
120  metadata_output_path=metadata_output_path,
121  map_python_type=map_python_type,
122  map_python_std_type=map_python_std_type,
123  map_cpp_type=map_cpp_type,
124  map_storage_name=map_storage_name,
125  raw_filename=raw_filename,
126  default_level=default_level,
127  raw_levels=raw_levels,
128  test_config_metadata=test_config_metadata,
129  )
130 
132  if not self.mapper_datamapper_data.test_config_metadata:
133  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
134  dataId = self.dataIds['raw']
135  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
136  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
137  self.assertEqual(butlerLocation.getCppType(), "Config")
138  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
139  processCcd_path = os.path.join("config", "processCcd.py")
140  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
141  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
142  for k, v in dataId.items():
143  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
144  msg="Failed for key={}".format(k))
145 
147  if not self.mapper_datamapper_data.test_config_metadata:
148  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
149  dataId = self.dataIds['raw']
150  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
151  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
152  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
153  self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
154  self.assertEqual(butlerLocation.getLocations(), [self.mapper_datamapper_data.metadata_output_path])
155  for k, v in dataId.items():
156  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
157  msg="Failed for key={}".format(k))
158 
159  def test_keys(self):
160  self.assertEqual(set(self.mapper.keys()), self.mapper_datamapper_data.keys)
161 
163  if not self.mapper_datamapper_data.test_config_metadata:
164  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
165  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
166  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
167 
168  def test_get_keys_raw(self):
169  for level, expect in self.mapper_datamapper_data.raw_levels:
170  result = self.mapper.getKeys("raw", level)
171  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
172 
174  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_datamapper_data.default_level)
175 
176  def _test_map(self, butlerLocation, dataId):
177  self.assertEqual(butlerLocation.getPythonType(), self.mapper_datamapper_data.map_python_type)
178  self.assertEqual(butlerLocation.getCppType(), self.mapper_datamapper_data.map_cpp_type)
179  self.assertEqual(butlerLocation.getStorageName(), self.mapper_datamapper_data.map_storage_name)
180  locationList = butlerLocation.getLocations()
181  self.assertEqual(len(locationList), 1)
182  fileName = os.path.basename(locationList[0])
183  self.assertEqual(fileName, self.mapper_datamapper_data.raw_filename)
184  for k, v in dataId.items():
185  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
186  msg="Failed for key={}".format(k))
187 
188  def test_map(self):
189  dataId = self.dataIds['raw']
190  location = self.mapper.map_raw(dataId)
191  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
192  self._test_map_test_map(location, dataId)
193  else:
194  self.log.warn("""ButlerComposite datasets are not tested for mapper functions. Though
195 ButlerComposites duck type as ButlerLocations in some ways, they do not
196 share enough methods to be usefully tested by the same function. Note
197 there are tests of the objects in the package in which they are implemented.""")
198  # This should be the same as above. Testing that both the generic
199  # and specific interface work for mapping the raw.
200  location = self.mapper.map("raw", dataId)
201  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
202  self._test_map_test_map(location, dataId)
203 
205  """
206  Test expansion of incomplete information of the available data in this
207  obs package's testdata repo.
208  """
209  for query, expect in self.mapper_datamapper_data.queryMetadata:
210  # queryMetadata returns tuples of available items of the 2nd
211  # parameter.
212  result = self.mapper.queryMetadata("raw", self.mapper_datamapper_data.query_format, query)
213  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
214 
216  self.assertTrue(self.mapper.canStandardize("raw"))
217  self.assertFalse(self.mapper.canStandardize("camera"))
218  if not self.mapper_datamapper_data.test_config_metadata:
219  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
220  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
221 
222  def _test_validate(self, dataId):
223  self.assertEqual(self.mapper.validate(dataId), dataId)
224 
225  def test_validate(self):
226  self._test_validate_test_validate({'visit': 1, 'filter': 'g'})
227  self._test_validate_test_validate({'visit': 2, 'filter': 'r'})
228  self._test_validate_test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
229  # NOTE: when DM-7909 is completed, add assertRaises test here.
230  # visit must be an integers
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_python_std_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None, test_config_metadata=True)
Definition: mapper_tests.py:58
def _test_map(self, butlerLocation, dataId)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
daf::base::PropertySet * set
Definition: fits.cc:912