LSSTApplications  18.1.0
LSSTDataManagementBasePackage
mapper_tests.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2016 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import abc
23 import os
24 import inspect
25 
26 import lsst.afw.geom
27 import lsst.utils.tests
29 import lsst.afw.image
30 import collections
31 
32 __all__ = ["MapperTests"]
33 
34 
35 class MapperTests(metaclass=abc.ABCMeta):
36  """
37  Generic tests of obs_* package mapper functionality.
38 
39  In the subclasses's setUp():
40  * Call setUp_mapper() to fill in required parameters.
41  """
42 
43  def setUp_mapper(self,
44  output=None,
45  path_to_raw=None,
46  keys=None,
47  query_format=None,
48  queryMetadata=None,
49  metadata_output_path=None,
50  map_python_type=None,
51  map_python_std_type=None,
52  map_cpp_type=None,
53  map_storage_name=None,
54  raw_filename=None,
55  default_level=None,
56  raw_levels=None,
57  test_config_metadata=True,
58  ):
59  """
60  Set up the necessary variables for mapper tests.
61 
62  Parameters
63  ----------
64 
65  output : `str`
66  full path to output repository (can be the same as data_dir input repository)
67  path_to_raw : `str`
68  full path to the raw file referenced by dataIds['raw']
69  keys : `set`
70  dictionary keys that this mapper should contain
71  query_format : `list`
72  format list for the results portion of queryMetadata
73  queryMetadata : `tuple` of (`dict`, `tuple`)
74  dataIds and the results of calling them in queryMetadata
75  metadata_output_path : `str`
76  path to metadata output associated with dataIds['raw']
77  map_python_type : `type`
78  full python type specification returned by the mapper for dataIds['raw']
79  map_python_std_type : `type`
80  full python type specification returned by the mapper for dataIds['raw'] after standardization
81  map_cpp_type : `str`
82  C++ type specification returned by the mapper for dataIds['raw']
83  map_storage_name : `str`
84  butler name for the storage type dataIds['raw']
85  raw_filename : `str`
86  Name of the raw files returned by the mapper for dataIds['raw']
87  default_level : `str`
88  value returned from mapper.getDefaultLevel
89  raw_levels : `tuple` of (`str`, `set` of `str`)
90  (level, expect) level and expected mapper return for mapper.getKeys('raw', level)
91  test_config_metadata : `bool`
92  Test persisted config and metadata? These tests may not be appropriate for test stand data.
93  Defaults to True.
94  """
95  fields = ['output',
96  'path_to_raw',
97  'keys',
98  'query_format',
99  'queryMetadata',
100  'metadata_output_path',
101  'map_python_type',
102  'map_python_std_type',
103  'map_cpp_type',
104  'map_storage_name',
105  'raw_filename',
106  'default_level',
107  'raw_levels',
108  'test_config_metadata',
109  ]
110  MapperData = collections.namedtuple("MapperData", fields)
111  self.mapper_data = MapperData(output=output,
112  path_to_raw=path_to_raw,
113  keys=keys,
114  query_format=query_format,
115  queryMetadata=queryMetadata,
116  metadata_output_path=metadata_output_path,
117  map_python_type=map_python_type,
118  map_python_std_type=map_python_std_type,
119  map_cpp_type=map_cpp_type,
120  map_storage_name=map_storage_name,
121  raw_filename=raw_filename,
122  default_level=default_level,
123  raw_levels=raw_levels,
124  test_config_metadata=test_config_metadata,
125  )
126 
128  if not self.mapper_data.test_config_metadata:
129  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
130  dataId = self.dataIds['raw']
131  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
132  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
133  self.assertEqual(butlerLocation.getCppType(), "Config")
134  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
135  processCcd_path = os.path.join("config", "processCcd.py")
136  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
137  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
138  for k, v in dataId.items():
139  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
140  msg="Failed for key={}".format(k))
141 
143  if not self.mapper_data.test_config_metadata:
144  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
145  dataId = self.dataIds['raw']
146  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
147  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
148  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
149  self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
150  self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
151  for k, v in dataId.items():
152  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
153  msg="Failed for key={}".format(k))
154 
155  def test_keys(self):
156  self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
157 
159  if not self.mapper_data.test_config_metadata:
160  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
161  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
162  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
163 
164  def test_get_keys_raw(self):
165  for level, expect in self.mapper_data.raw_levels:
166  result = self.mapper.getKeys("raw", level)
167  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
168 
170  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
171 
172  def _test_map(self, butlerLocation, dataId):
173  self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
174  self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
175  self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
176  locationList = butlerLocation.getLocations()
177  self.assertEqual(len(locationList), 1)
178  fileName = os.path.basename(locationList[0])
179  self.assertEqual(fileName, self.mapper_data.raw_filename)
180  for k, v in dataId.items():
181  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
182  msg="Failed for key={}".format(k))
183 
184  def test_map(self):
185  dataId = self.dataIds['raw']
186  location = self.mapper.map_raw(dataId)
187  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
188  self._test_map(location, dataId)
189  else:
190  self.log.warn("""ButlerComposite datasets are not tested for mapper functions. Though
191 ButlerComposites duck type as ButlerLocations in some ways, they do not
192 share enough methods to be usefully tested by the same function. Note
193 there are tests of the objects in the package in which they are implemented.""")
194  # This should be the same as above. Testing that both the generic and specific interface work for
195  # mapping the raw.
196  location = self.mapper.map("raw", dataId)
197  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
198  self._test_map(location, dataId)
199 
201  """
202  Test expansion of incomplete information of the available data in this
203  obs package's testdata repo.
204  """
205  for query, expect in self.mapper_data.queryMetadata:
206  # queryMetadata returns tuples of available items of the 2nd parameter.
207  result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
208  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
209 
211  self.assertTrue(self.mapper.canStandardize("raw"))
212  self.assertFalse(self.mapper.canStandardize("camera"))
213  if not self.mapper_data.test_config_metadata:
214  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
215  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
216 
217  def _test_validate(self, dataId):
218  self.assertEqual(self.mapper.validate(dataId), dataId)
219 
220  def test_validate(self):
221  self._test_validate({'visit': 1, 'filter': 'g'})
222  self._test_validate({'visit': 2, 'filter': 'r'})
223  self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
224  # NOTE: when DM-7909 is completed, add assertRaises test here.
225  # visit must be an integers
daf::base::PropertySet * set
Definition: fits.cc:884
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:168
def _test_map(self, butlerLocation, dataId)
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_python_std_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None, test_config_metadata=True)
Definition: mapper_tests.py:58
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects...