LSSTApplications  19.0.0-14-gb0260a2+72efe9b372,20.0.0+7927753e06,20.0.0+8829bf0056,20.0.0+995114c5d2,20.0.0+b6f4b2abd1,20.0.0+bddc4f4cbe,20.0.0-1-g253301a+8829bf0056,20.0.0-1-g2b7511a+0d71a2d77f,20.0.0-1-g5b95a8c+7461dd0434,20.0.0-12-g321c96ea+23efe4bbff,20.0.0-16-gfab17e72e+fdf35455f6,20.0.0-2-g0070d88+ba3ffc8f0b,20.0.0-2-g4dae9ad+ee58a624b3,20.0.0-2-g61b8584+5d3db074ba,20.0.0-2-gb780d76+d529cf1a41,20.0.0-2-ged6426c+226a441f5f,20.0.0-2-gf072044+8829bf0056,20.0.0-2-gf1f7952+ee58a624b3,20.0.0-20-geae50cf+e37fec0aee,20.0.0-25-g3dcad98+544a109665,20.0.0-25-g5eafb0f+ee58a624b3,20.0.0-27-g64178ef+f1f297b00a,20.0.0-3-g4cc78c6+e0676b0dc8,20.0.0-3-g8f21e14+4fd2c12c9a,20.0.0-3-gbd60e8c+187b78b4b8,20.0.0-3-gbecbe05+48431fa087,20.0.0-38-ge4adf513+a12e1f8e37,20.0.0-4-g97dc21a+544a109665,20.0.0-4-gb4befbc+087873070b,20.0.0-4-gf910f65+5d3db074ba,20.0.0-5-gdfe0fee+199202a608,20.0.0-5-gfbfe500+d529cf1a41,20.0.0-6-g64f541c+d529cf1a41,20.0.0-6-g9a5b7a1+a1cd37312e,20.0.0-68-ga3f3dda+5fca18c6a4,20.0.0-9-g4aef684+e18322736b,w.2020.45
LSSTDataManagementBasePackage
mapper_tests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import abc
23 import os
24 import inspect
25 
26 import lsst.afw.geom
27 import lsst.utils.tests
29 import lsst.afw.image
30 import collections
31 
32 __all__ = ["MapperTests"]
33 
34 
35 class MapperTests(metaclass=abc.ABCMeta):
36  """
37  Generic tests of obs_* package mapper functionality.
38 
39  In the subclasses's setUp():
40  * Call setUp_mapper() to fill in required parameters.
41  """
42 
43  def setUp_mapper(self,
44  output=None,
45  path_to_raw=None,
46  keys=None,
47  query_format=None,
48  queryMetadata=None,
49  metadata_output_path=None,
50  map_python_type=None,
51  map_python_std_type=None,
52  map_cpp_type=None,
53  map_storage_name=None,
54  raw_filename=None,
55  default_level=None,
56  raw_levels=None,
57  test_config_metadata=True,
58  ):
59  """
60  Set up the necessary variables for mapper tests.
61 
62  Parameters
63  ----------
64 
65  output : `str`
66  full path to output repository (can be the same as data_dir input
67  repository)
68  path_to_raw : `str`
69  full path to the raw file referenced by dataIds['raw']
70  keys : `set`
71  dictionary keys that this mapper should contain
72  query_format : `list`
73  format list for the results portion of queryMetadata
74  queryMetadata : `tuple` of (`dict`, `tuple`)
75  dataIds and the results of calling them in queryMetadata
76  metadata_output_path : `str`
77  path to metadata output associated with dataIds['raw']
78  map_python_type : `type`
79  full python type specification returned by the mapper for
80  dataIds['raw']
81  map_python_std_type : `type`
82  full python type specification returned by the mapper for
83  dataIds['raw'] after standardization
84  map_cpp_type : `str`
85  C++ type specification returned by the mapper for dataIds['raw']
86  map_storage_name : `str`
87  butler name for the storage type dataIds['raw']
88  raw_filename : `str`
89  Name of the raw files returned by the mapper for dataIds['raw']
90  default_level : `str`
91  value returned from mapper.getDefaultLevel
92  raw_levels : `tuple` of (`str`, `set` of `str`)
93  (level, expect) level and expected mapper return for
94  mapper.getKeys('raw', level)
95  test_config_metadata : `bool`
96  Test persisted config and metadata? These tests may not be
97  appropriate for test stand data. Defaults to True.
98  """
99  fields = ['output',
100  'path_to_raw',
101  'keys',
102  'query_format',
103  'queryMetadata',
104  'metadata_output_path',
105  'map_python_type',
106  'map_python_std_type',
107  'map_cpp_type',
108  'map_storage_name',
109  'raw_filename',
110  'default_level',
111  'raw_levels',
112  'test_config_metadata',
113  ]
114  MapperData = collections.namedtuple("MapperData", fields)
115  self.mapper_data = MapperData(output=output,
116  path_to_raw=path_to_raw,
117  keys=keys,
118  query_format=query_format,
119  queryMetadata=queryMetadata,
120  metadata_output_path=metadata_output_path,
121  map_python_type=map_python_type,
122  map_python_std_type=map_python_std_type,
123  map_cpp_type=map_cpp_type,
124  map_storage_name=map_storage_name,
125  raw_filename=raw_filename,
126  default_level=default_level,
127  raw_levels=raw_levels,
128  test_config_metadata=test_config_metadata,
129  )
130 
132  if not self.mapper_data.test_config_metadata:
133  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
134  dataId = self.dataIds['raw']
135  butlerLocation = self.mapper.map("processCcd_config_filename", dataId)
136  self.assertEqual(butlerLocation.getPythonType(), "lsst.pipe.tasks.processCcd.ProcessCcdConfig")
137  self.assertEqual(butlerLocation.getCppType(), "Config")
138  self.assertEqual(butlerLocation.getStorageName(), "ConfigStorage")
139  processCcd_path = os.path.join("config", "processCcd.py")
140  self.assertEqual(self.mapper.root, butlerLocation.getStorage().root)
141  self.assertEqual(butlerLocation.getLocations(), [processCcd_path])
142  for k, v in dataId.items():
143  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
144  msg="Failed for key={}".format(k))
145 
147  if not self.mapper_data.test_config_metadata:
148  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
149  dataId = self.dataIds['raw']
150  butlerLocation = self.mapper.map_processCcd_metadata(dataId)
151  self.assertEqual(butlerLocation.getPythonType(), "lsst.daf.base.PropertySet")
152  self.assertEqual(butlerLocation.getCppType(), "PropertySet")
153  self.assertEqual(butlerLocation.getStorageName(), "YamlStorage")
154  self.assertEqual(butlerLocation.getLocations(), [self.mapper_data.metadata_output_path])
155  for k, v in dataId.items():
156  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
157  msg="Failed for key={}".format(k))
158 
159  def test_keys(self):
160  self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
161 
163  if not self.mapper_data.test_config_metadata:
164  self.skipTest('Skipping %s as requested' % (inspect.currentframe().f_code.co_name))
165  someKeys = set(['raw', 'processCcd_config', 'processCcd_metadata'])
166  self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
167 
168  def test_get_keys_raw(self):
169  for level, expect in self.mapper_data.raw_levels:
170  result = self.mapper.getKeys("raw", level)
171  self.assertEqual(set(result), expect, msg='Failed for level={}'.format(level))
172 
174  self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
175 
176  def _test_map(self, butlerLocation, dataId):
177  self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
178  self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
179  self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
180  locationList = butlerLocation.getLocations()
181  self.assertEqual(len(locationList), 1)
182  fileName = os.path.basename(locationList[0])
183  self.assertEqual(fileName, self.mapper_data.raw_filename)
184  for k, v in dataId.items():
185  self.assertEqual(butlerLocation.getAdditionalData().getScalar(k), v,
186  msg="Failed for key={}".format(k))
187 
188  def test_map(self):
189  dataId = self.dataIds['raw']
190  location = self.mapper.map_raw(dataId)
191  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
192  self._test_map(location, dataId)
193  else:
194  self.log.warn("""ButlerComposite datasets are not tested for mapper functions. Though
195 ButlerComposites duck type as ButlerLocations in some ways, they do not
196 share enough methods to be usefully tested by the same function. Note
197 there are tests of the objects in the package in which they are implemented.""")
198  # This should be the same as above. Testing that both the generic
199  # and specific interface work for mapping the raw.
200  location = self.mapper.map("raw", dataId)
201  if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
202  self._test_map(location, dataId)
203 
205  """
206  Test expansion of incomplete information of the available data in this
207  obs package's testdata repo.
208  """
209  for query, expect in self.mapper_data.queryMetadata:
210  # queryMetadata returns tuples of available items of the 2nd
211  # parameter.
212  result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
213  self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
214 
216  self.assertTrue(self.mapper.canStandardize("raw"))
217  self.assertFalse(self.mapper.canStandardize("camera"))
218  if not self.mapper_data.test_config_metadata:
219  self.assertFalse(self.mapper.canStandardize("processCcd_config"))
220  self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
221 
222  def _test_validate(self, dataId):
223  self.assertEqual(self.mapper.validate(dataId), dataId)
224 
225  def test_validate(self):
226  self._test_validate({'visit': 1, 'filter': 'g'})
227  self._test_validate({'visit': 2, 'filter': 'r'})
228  self._test_validate({'visit': 3, 'filter': 'g', 'tract': 4})
229  # NOTE: when DM-7909 is completed, add assertRaises test here.
230  # visit must be an integers
lsst::afw::image
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
Definition: imageAlgorithm.dox:1
lsst::log.log.logContinued.warn
def warn(fmt, *args)
Definition: logContinued.py:205
lsst.obs.base.mapper_tests.MapperTests.test_get_default_level
def test_get_default_level(self)
Definition: mapper_tests.py:173
lsst.obs.base.mapper_tests.MapperTests._test_map
def _test_map(self, butlerLocation, dataId)
Definition: mapper_tests.py:176
lsst::daf::persistence.butlerLocation.ButlerComposite
Definition: butlerLocation.py:36
lsst.obs.base.mapper_tests.MapperTests.test_map_metadata_data
def test_map_metadata_data(self)
Definition: mapper_tests.py:146
lsst.obs.base.mapper_tests.MapperTests.test_get_dataset_types
def test_get_dataset_types(self)
Definition: mapper_tests.py:162
lsst.obs.base.mapper_tests.MapperTests.mapper_data
mapper_data
Definition: mapper_tests.py:100
astshim.keyMap.keyMapContinued.keys
def keys(self)
Definition: keyMapContinued.py:6
lsst.pex.config.history.format
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
Definition: history.py:174
lsst.obs.base.mapper_tests.MapperTests
Definition: mapper_tests.py:35
lsst.obs.base.mapper_tests.MapperTests.test_keys
def test_keys(self)
Definition: mapper_tests.py:159
lsst.obs.base.mapper_tests.MapperTests.test_validate
def test_validate(self)
Definition: mapper_tests.py:225
lsst.obs.base.mapper_tests.MapperTests._test_validate
def _test_validate(self, dataId)
Definition: mapper_tests.py:222
lsst::utils.tests
Definition: tests.py:1
lsst.obs.base.mapper_tests.MapperTests.test_map
def test_map(self)
Definition: mapper_tests.py:188
lsst.obs.base.mapper_tests.MapperTests.test_get_keys_raw
def test_get_keys_raw(self)
Definition: mapper_tests.py:168
lsst.obs.base.mapper_tests.MapperTests.test_can_standardize
def test_can_standardize(self)
Definition: mapper_tests.py:215
lsst::daf::persistence
Definition: Utils.h:50
lsst.obs.base.mapper_tests.MapperTests.setUp_mapper
def setUp_mapper(self, output=None, path_to_raw=None, keys=None, query_format=None, queryMetadata=None, metadata_output_path=None, map_python_type=None, map_python_std_type=None, map_cpp_type=None, map_storage_name=None, raw_filename=None, default_level=None, raw_levels=None, test_config_metadata=True)
Definition: mapper_tests.py:43
lsst.obs.base.mapper_tests.MapperTests.test_map_config_data
def test_map_config_data(self)
Definition: mapper_tests.py:131
lsst.pex.config.wrap.validate
validate
Definition: wrap.py:295
set
daf::base::PropertySet * set
Definition: fits.cc:912
lsst.obs.base.mapper_tests.MapperTests.test_query_metadata
def test_query_metadata(self)
Definition: mapper_tests.py:204
lsst::afw::geom
Definition: frameSetUtils.h:40