LSST Applications  21.0.0+75b29a8a7f,21.0.0+e70536a077,21.0.0-1-ga51b5d4+62c747d40b,21.0.0-10-gbfb87ad6+3307648ee3,21.0.0-15-gedb9d5423+47cba9fc36,21.0.0-2-g103fe59+fdf0863a2a,21.0.0-2-g1367e85+d38a93257c,21.0.0-2-g45278ab+e70536a077,21.0.0-2-g5242d73+d38a93257c,21.0.0-2-g7f82c8f+e682ffb718,21.0.0-2-g8dde007+d179fbfa6a,21.0.0-2-g8f08a60+9402881886,21.0.0-2-ga326454+e682ffb718,21.0.0-2-ga63a54e+08647d4b1b,21.0.0-2-gde069b7+26c92b3210,21.0.0-2-gecfae73+0445ed2f95,21.0.0-2-gfc62afb+d38a93257c,21.0.0-27-gbbd0d29+ae871e0f33,21.0.0-28-g5fc5e037+feb0e9397b,21.0.0-3-g21c7a62+f4b9c0ff5c,21.0.0-3-g357aad2+57b0bddf0b,21.0.0-3-g4be5c26+d38a93257c,21.0.0-3-g65f322c+3f454acf5d,21.0.0-3-g7d9da8d+75b29a8a7f,21.0.0-3-gaa929c8+9e4ef6332c,21.0.0-3-ge02ed75+4b120a55c4,21.0.0-4-g3300ddd+e70536a077,21.0.0-4-g591bb35+4b120a55c4,21.0.0-4-gc004bbf+4911b9cd27,21.0.0-4-gccdca77+f94adcd104,21.0.0-4-ge8fba5a+2b3a696ff9,21.0.0-5-gb155db7+2c5429117a,21.0.0-5-gdf36809+637e4641ee,21.0.0-6-g00874e7+c9fd7f7160,21.0.0-6-g4e60332+4b120a55c4,21.0.0-7-gc8ca178+40eb9cf840,21.0.0-8-gfbe0b4b+9e4ef6332c,21.0.0-9-g2fd488a+d83b7cd606,w.2021.05
LSST Data Management Base Package
convertTests.py
Go to the documentation of this file.
1 # This file is part of obs_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Unit test base class for the gen2 to gen3 converter.
23 """
24 
25 import abc
26 import itertools
27 import shutil
28 import tempfile
29 import unittest
30 
31 
32 import lsst.afw.image
33 import lsst.afw.table
35 import lsst.daf.butler
36 import lsst.ip.isr
37 from lsst.obs.base.script import convert
38 import lsst.utils.tests
39 from lsst.utils import doImport
40 
41 
42 class ConvertGen2To3TestCase(metaclass=abc.ABCMeta):
43  """Test the `butler convert` command.
44 
45  Subclass this, and then `lsst.utils.tests.TestCase` and set the below
46  attributes. Uses the `butler convert` command line command to do the
47  conversion.
48  """
49 
50  gen2root = ""
51  """Root path to the gen2 repo to be converted."""
52 
53  gen2calib = None
54  """Path to the gen2 calib repo to be converted."""
55 
56  @property
57  @abc.abstractmethod
59  """Full path to the `Instrument` class of the data to be converted,
60  e.g. ``lsst.obs.decam.DarkEnergyCamera``.
61 
62  Returns
63  -------
64  className : `str`
65  The fully qualified instrument class name.
66  """
67  pass
68 
69  @property
70  def instrumentClass(self):
71  """The instrument class."""
72  return doImport(self.instrumentClassNameinstrumentClassName)
73 
74  @property
75  def instrumentName(self):
76  """Name of the instrument for the gen3 registry, e.g. "DECam".
77 
78  Returns
79  -------
80  name : `str`
81  The name of the instrument.
82  """
83  return self.instrumentClassinstrumentClass.getName()
84 
85  config = None
86  """Full path to a config override for ConvertRepoTask, to be applied after
87  the Instrument overrides when running the conversion function."""
88 
89  biases = []
90  """List dataIds to use to load gen3 biases to test that they exist."""
91 
92  biasName = "bias"
93  """Name of the dataset that the biases are loaded into."""
94 
95  flats = []
96  """List dataIds to use to load gen3 flats to test that they exist."""
97 
98  flatName = "flat"
99  """Name of the dataset that the flats are loaded into."""
100 
101  darks = []
102  """List dataIds to use to load gen3 darks to test that they exist."""
103 
104  darkName = "dark"
105  """Name of the dataset that the darks are loaded into."""
106 
107  kwargs = {}
108  """Other keyword arguments to pass directly to the converter function,
109  as a dict."""
110 
111  refcats = []
112  """Names of the reference catalogs to query for the existence of in the
113  converted gen3 repo."""
114 
115  collections = set()
116  """Additional collections that should appear in the gen3 repo.
117 
118  This will automatically be populated by the base `setUp` to include
119  ``"{instrumentName}/raw"``, ``"refcats"`` (if the ``refcats``
120  class attribute is non-empty), and ``"skymaps"`` (if ``skymapName`` is
121  not `None`).
122  """
123 
124  detectorKey = "ccd"
125  """Key to use in a gen2 dataId to refer to a detector."""
126 
127  exposureKey = "visit"
128  """Key to use in a gen2 dataId to refer to a visit or exposure."""
129 
130  calibFilterType = "physical_filter"
131  """Gen3 dimension that corresponds to Gen2 ``filter``. Should be
132  physical_filter or band."""
133 
134  skymapName = None
135  """Name of the Gen3 skymap."""
136 
137  skymapConfig = None
138  """Path to skymap config file defining the new gen3 skymap."""
139 
140  def setUp(self):
141  self.gen3rootgen3root = tempfile.mkdtemp()
142  self.gen2Butlergen2Butler = lsst.daf.persistence.Butler(root=self.gen2rootgen2root, calibRoot=self.gen2calibgen2calib)
143  self.collectionscollections = set(type(self).collections)
144  self.collectionscollections.add(self.instrumentClassinstrumentClass.makeDefaultRawIngestRunName())
145  if len(self.refcatsrefcats) > 0:
146  self.collectionscollections.add("refcats")
147  if self.skymapNameskymapName is not None:
148  self.collectionscollections.add("skymaps")
149 
150  # We always write a default calibration collection
151  # containing at least the camera
152  self.collectionscollections.add(self.instrumentClassinstrumentClass.makeCollectionName("calib"))
153 
154  def tearDown(self):
155  shutil.rmtree(self.gen3rootgen3root, ignore_errors=True)
156 
157  def _run_convert(self):
158  """Convert a gen2 repo to gen3 for testing.
159  """
160 
161  # Turn on logging
162  log = lsst.log.Log.getLogger("convertRepo")
163  log.setLevel(log.INFO)
164  log.info("Converting %s to %s", self.gen2rootgen2root, self.gen3rootgen3root)
165 
166  convert(repo=self.gen3rootgen3root,
167  gen2root=self.gen2rootgen2root,
168  skymap_name=self.skymapNameskymapName,
169  skymap_config=self.skymapConfigskymapConfig,
170  config_file=self.configconfig,
171  calibs=self.gen2calibgen2calib,
172  reruns=None,
173  transfer="auto")
174 
175  def check_raw(self, gen3Butler, exposure, detector):
176  """Check that a raw was converted correctly.
177 
178  Parameters
179  ----------
180  gen3Butler : `lsst.daf.butler.Butler`
181  The Butler to be tested.
182  exposure : `int`
183  The exposure/vist identifier ``get`` from both butlers.
184  detector : `int`
185  The detector identifier to ``get`` from both butlers.
186  """
187  dataIdGen2 = {self.detectorKeydetectorKey: detector, self.exposureKeyexposureKey: exposure}
188  try:
189  gen2Exposure = self.gen2Butlergen2Butler.get("raw", dataId=dataIdGen2)
191  # ignore datasets that don't actually exist in the gen2 butler.
192  return
193  dataIdGen3 = dict(detector=detector, exposure=exposure, instrument=self.instrumentNameinstrumentName)
194  gen3Exposure = gen3Butler.get("raw", dataId=dataIdGen3)
195  # Check that we got an Exposure, but not what type; there is
196  # inconsistency between different obs packages.
197  self.assertIsInstance(gen3Exposure, lsst.afw.image.Exposure)
198  self.assertEqual(gen3Exposure.getInfo().getDetector().getId(), detector)
199  self.assertMaskedImagesEqual(gen2Exposure.maskedImage, gen3Exposure.maskedImage)
200 
201  def check_calibs(self, calibName, calibIds, gen3Butler):
202  """Test that we can get converted bias/dark/flat from the gen3 repo.
203 
204  Note: because there is no clear way to get calibrations from a gen2
205  repo, we just test that the thing we got is an ExposureF here, and
206  assume that formatter testing is handled properly elsewhere.
207 
208  Parameters
209  ----------
210  calibName : `str`
211  The name of the calibration to attempt to get ("bias", "flat").
212  calibIds : `list` of `dict`
213  The list of calibration dataIds to get.
214  gen3Butler : `lsst.daf.butler.Butler`
215  The Butler to use to get the data.
216  """
217  if not calibIds:
218  return
219  collection = self.instrumentClassinstrumentClass.makeCalibrationCollectionName()
220  with self.subTest(dtype=calibName):
221  datasets = {}
222  for assoc in gen3Butler.registry.queryDatasetAssociations(calibName, collections=collection):
223  # There will in general be multiple refs for each data ID
224  # (for different validity ranges), but we'll just test one
225  # anyway to keep the test fast, so we don't care that this
226  # might overwrite.
227  datasets[assoc.ref.dataId] = assoc.ref
228  for dataId in calibIds:
229  standardizedDataId = lsst.daf.butler.DataCoordinate.standardize(
230  dataId,
231  universe=gen3Butler.registry.dimensions
232  )
233  with self.subTest(dataId=standardizedDataId):
234  gen3Exposure = gen3Butler.getDirect(datasets[standardizedDataId])
235  self.assertIsInstance(gen3Exposure, lsst.afw.image.ExposureF)
236 
237  def check_defects(self, gen3Butler, detectors):
238  """Test that we can get converted defects from the gen3 repo.
239 
240  Parameters
241  ----------
242  gen3Butler : `lsst.daf.butler.Butler`
243  The Butler to be tested.
244  detectors : `list` of `int`
245  The detector identifiers to ``get`` from the gen3 butler.
246  """
247  collection = self.instrumentClassinstrumentClass.makeCalibrationCollectionName()
248  datasets = {}
249  for assoc in gen3Butler.registry.queryDatasetAssociations("defects", collections=collection):
250  # There will in general be multiple refs for each data ID
251  # (for different validity ranges), but we'll just test one
252  # anyway to keep the test fast, so we don't care that this
253  # might overwrite.
254  datasets[assoc.ref.dataId] = assoc.ref
255  for detector in detectors:
256  dataId = lsst.daf.butler.DataCoordinate.standardize(
257  detector=detector, instrument=self.instrumentNameinstrumentName,
258  universe=gen3Butler.registry.dimensions
259  )
260  if dataId in datasets:
261  gen3Defects = gen3Butler.getDirect(datasets[dataId])
262  self.assertIsInstance(gen3Defects, lsst.ip.isr.Defects)
263 
264  def check_refcat(self, gen3Butler):
265  """Test that each expected refcat is in the gen3 repo.
266 
267  Parameters
268  ----------
269  gen3Butler : `lsst.daf.butler.Butler`
270  The Butler to be tested.
271  """
272  if len(self.refcatsrefcats) > 0:
273  for refcat in self.refcatsrefcats:
274  query = gen3Butler.registry.queryDatasets(refcat, collections=["refcats"])
275  self.assertGreater(len(list(query)), 0,
276  msg=f"refcat={refcat} has no entries in collection 'refcats'.")
277 
278  def check_collections(self, gen3Butler):
279  """Test that the correct set of collections is in the gen3 repo.
280 
281  Parameters
282  ----------
283  gen3Butler : `lsst.daf.butler.Butler`
284  The Butler to be tested.
285  """
286  # We use assertGreaterEqual because the conversion code may create
287  # multiple RUNS and combine them into CHAINED collections as an
288  # implementation detail; we only care that the high-level collections
289  # exist one way or another.
290  self.assertGreaterEqual(set(gen3Butler.registry.queryCollections()), set(self.collectionscollections),
291  f"Compare with expected collections ({self.collections})")
292 
293  def test_convert(self):
294  """Test that all data are converted correctly.
295  """
296  self._run_convert_run_convert()
297  gen3Butler = lsst.daf.butler.Butler(self.gen3rootgen3root,
298  collections=self.instrumentClassinstrumentClass.makeDefaultRawIngestRunName())
299  self.check_collectionscheck_collections(gen3Butler)
300 
301  # check every raw detector that the gen2 butler knows about
302  detectors = self.gen2Butlergen2Butler.queryMetadata("raw", self.detectorKeydetectorKey)
303  exposures = self.gen2Butlergen2Butler.queryMetadata("raw", self.exposureKeyexposureKey)
304  for exposure, detector in itertools.product(exposures, detectors):
305  with self.subTest(mode="raw", exposure=exposure, detector=detector):
306  self.check_rawcheck_raw(gen3Butler, exposure, detector)
307 
308  self.check_refcatcheck_refcat(gen3Butler)
309  self.check_defectscheck_defects(gen3Butler, detectors)
310  self.check_calibscheck_calibs(self.biasNamebiasName, self.biasesbiases, gen3Butler)
311  self.check_calibscheck_calibs(self.flatNameflatName, self.flatsflats, gen3Butler)
312  self.check_calibscheck_calibs(self.darkNamedarkName, self.darksdarks, gen3Butler)
313 
314 
315 def setup_module(module):
317 
318 
319 if __name__ == "__main__":
321  unittest.main()
A class to contain the data, WCS, and other information needed to describe an image of the sky.
Definition: Exposure.h:72
static Log getLogger(Log const &logger)
Definition: Log.h:760
def check_raw(self, gen3Butler, exposure, detector)
def check_calibs(self, calibName, calibIds, gen3Butler)
Backwards-compatibility support for depersisting the old Calib (FluxMag0/FluxMag0Err) objects.
std::string const & getName() const noexcept
Return a filter's name.
Definition: Filter.h:78
def convert(repo, gen2root, skymap_name, skymap_config, calibs, reruns, config_file, transfer, processes=1)
Definition: convert.py:35
def init()
Definition: tests.py:59
table::Key< int > type
Definition: Detector.cc:163
daf::base::PropertyList * list
Definition: fits.cc:913
daf::base::PropertySet * set
Definition: fits.cc:912