22 """Base class for writing Gen3 raw data ingest tests.
25 __all__ = (
"IngestTestBase",)
34 from lsst.daf.butler
import Butler
35 from lsst.daf.butler.cli.butler
import cli
as butlerCli
36 from lsst.daf.butler.cli.utils
import LogCliRunner
39 from .utils
import getInstrument
44 """Base class for tests of gen3 ingest. Subclass from this, then
45 `unittest.TestCase` to get a working test suite.
49 """Root path to ingest files into. Typically `obs_package/tests/`; the
50 actual directory will be a tempdir under this one.
54 """list of butler data IDs of files that should have been ingested."""
57 """Full path to a file to ingest in tests."""
59 rawIngestTask =
"lsst.obs.base.RawIngestTask"
60 """The task to use in the Ingest test."""
62 curatedCalibrationDatasetTypes =
None
63 """List or tuple of Datasets types that should be present after calling
64 writeCuratedCalibrations. If `None` writeCuratedCalibrations will
65 not be called and the test will be skipped."""
68 """The task to use to define visits from groups of exposures.
69 This is ignored if ``visits`` is `None`.
73 """A dictionary mapping visit data IDs the lists of exposure data IDs that
74 are associated with them.
75 If this is empty (but not `None`), visit definition will be run but no
76 visits will be expected (e.g. because no exposures are on-sky
83 """The fully qualified instrument class name.
88 The fully qualified instrument class name.
94 """The instrument class."""
99 """The name of the instrument.
104 The name of the instrument.
123 if os.path.exists(cls.
root):
124 shutil.rmtree(cls.
root, ignore_errors=
True)
128 Test that RawIngestTask ingested the expected files.
132 files : `list` [`str`], or None
133 List of files to be ingested, or None to use ``self.file``
134 fullCheck : `bool`, optional
135 If `True`, read the full raw dataset and check component
136 consistency. If `False` check that a component can be read
137 but do not read the entire raw exposure.
141 Reading all the ingested test data can be expensive. The code paths
142 for reading the second raw are the same as reading the first so
143 we do not gain anything by doing full checks of everything.
144 Only read full pixel data for first dataset from file.
145 Don't even do that if we are requested not to by the caller.
146 This only really affects files that contain multiple datasets.
149 datasets = butler.registry.queryDatasets(
"raw", collections=self.
outputRun)
150 self.assertEqual(len(
list(datasets)), len(self.
dataIds))
154 metadata = butler.get(
"raw.metadata", dataId)
158 exposure = butler.get(
"raw", dataId)
159 self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
164 wcs = butler.get(
"raw.wcs", dataId)
165 self.assertEqual(wcs, exposure.getWcs())
167 rawImage = butler.get(
"raw.image", dataId)
168 self.assertEqual(rawImage.getBBox(), exposure.getBBox())
173 """Check the state of the repository after ingest.
175 This is an optional hook provided for subclasses; by default it does
180 files : `list` [`str`], or None
181 List of files to be ingested, or None to use ``self.file``
186 def _createRepo(cls):
187 """Use the Click `testing` module to call the butler command line api
188 to create a repository."""
189 runner = LogCliRunner()
190 result = runner.invoke(butlerCli, [
"create", cls.
root])
192 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
194 def _ingestRaws(self, transfer, file=None):
195 """Use the Click `testing` module to call the butler command line api
201 The external data transfer type.
203 Path to a file to ingest instead of the default associated with
208 runner = LogCliRunner()
209 result = runner.invoke(butlerCli, [
"ingest-raws", self.
root, file,
211 "--transfer", transfer,
213 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
216 def _registerInstrument(cls):
217 """Use the Click `testing` module to call the butler command line api
218 to register the instrument."""
219 runner = LogCliRunner()
222 assert result.exit_code == 0, f
"output: {result.output} exception: {result.exception}"
224 def _writeCuratedCalibrations(self):
225 """Use the Click `testing` module to call the butler command line api
226 to write curated calibrations."""
227 runner = LogCliRunner()
228 result = runner.invoke(butlerCli, [
"write-curated-calibrations", self.
root, self.
instrumentName])
229 self.assertEqual(result.exit_code, 0, f
"output: {result.output} exception: {result.exception}")
250 except PermissionError
as err:
251 raise unittest.SkipTest(
"Skipping hard-link test because input data"
252 " is on a different filesystem.")
from err
255 """Test that files already in the directory can be added to the
260 pathInStore =
"prefix-" + os.path.basename(self.
file)
261 newPath = butler.datastore.root.join(pathInStore)
262 os.symlink(os.path.abspath(self.
file), newPath.ospath)
263 self.
_ingestRaws(transfer=
"auto", file=newPath.ospath)
271 uri = butler.getURI(
"raw", self.
dataIds[0])
272 self.assertEqual(uri.relative_to(butler.datastore.root), pathInStore)
275 """Re-ingesting the same data into the repository should fail.
278 with self.assertRaises(Exception):
282 """Test that we can ingest the curated calibrations, and read them
283 with `loadCamera` both before and after.
286 raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
288 butler = Butler(self.
root, writeable=
False)
293 with self.assertRaises(LookupError):
294 lsst.obs.base.loadCamera(butler, {
"exposure": 0}, collections=collection)
301 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
302 self.assertFalse(isVersioned)
311 butler = Butler(self.
root, writeable=
False)
314 with self.subTest(dtype=datasetTypeName):
316 butler.registry.queryDatasetAssociations(
318 collections=collection,
321 self.assertGreater(len(found), 0, f
"Checking {datasetTypeName}")
324 camera, isVersioned = lsst.obs.base.loadCamera(butler, self.
dataIds[0], collections=collection)
325 self.assertTrue(isVersioned)
330 self.skipTest(
"Expected visits were not defined.")
337 script.defineVisits(self.
root, config_file=
None, collections=self.
outputRun,
342 visits = butler.registry.queryDataIds([
"visit"]).expanded().toSet()
343 self.assertCountEqual(visits, self.
visits.
keys())
345 camera = instr.getCamera()
346 for foundVisit, (expectedVisit, expectedExposures)
in zip(visits, self.
visits.
items()):
348 foundExposures = butler.registry.queryDataIds([
"exposure"], dataId=expectedVisit
350 self.assertCountEqual(foundExposures, expectedExposures)
353 self.assertIsNotNone(foundVisit.region)
354 detectorVisitDataIds = butler.registry.queryDataIds([
"visit",
"detector"], dataId=expectedVisit
356 self.assertEqual(len(detectorVisitDataIds), len(camera))
357 for dataId
in detectorVisitDataIds:
358 self.assertTrue(foundVisit.region.contains(dataId.region))