LSST Applications  21.0.0+75b29a8a7f,21.0.0+e70536a077,21.0.0-1-ga51b5d4+62c747d40b,21.0.0-10-gbfb87ad6+3307648ee3,21.0.0-15-gedb9d5423+47cba9fc36,21.0.0-2-g103fe59+fdf0863a2a,21.0.0-2-g1367e85+d38a93257c,21.0.0-2-g45278ab+e70536a077,21.0.0-2-g5242d73+d38a93257c,21.0.0-2-g7f82c8f+e682ffb718,21.0.0-2-g8dde007+d179fbfa6a,21.0.0-2-g8f08a60+9402881886,21.0.0-2-ga326454+e682ffb718,21.0.0-2-ga63a54e+08647d4b1b,21.0.0-2-gde069b7+26c92b3210,21.0.0-2-gecfae73+0445ed2f95,21.0.0-2-gfc62afb+d38a93257c,21.0.0-27-gbbd0d29+ae871e0f33,21.0.0-28-g5fc5e037+feb0e9397b,21.0.0-3-g21c7a62+f4b9c0ff5c,21.0.0-3-g357aad2+57b0bddf0b,21.0.0-3-g4be5c26+d38a93257c,21.0.0-3-g65f322c+3f454acf5d,21.0.0-3-g7d9da8d+75b29a8a7f,21.0.0-3-gaa929c8+9e4ef6332c,21.0.0-3-ge02ed75+4b120a55c4,21.0.0-4-g3300ddd+e70536a077,21.0.0-4-g591bb35+4b120a55c4,21.0.0-4-gc004bbf+4911b9cd27,21.0.0-4-gccdca77+f94adcd104,21.0.0-4-ge8fba5a+2b3a696ff9,21.0.0-5-gb155db7+2c5429117a,21.0.0-5-gdf36809+637e4641ee,21.0.0-6-g00874e7+c9fd7f7160,21.0.0-6-g4e60332+4b120a55c4,21.0.0-7-gc8ca178+40eb9cf840,21.0.0-8-gfbe0b4b+9e4ef6332c,21.0.0-9-g2fd488a+d83b7cd606,w.2021.05
LSST Data Management Base Package
Public Member Functions | Public Attributes | Static Public Attributes | List of all members
lsst.obs.base.ingest_tests.IngestTestBase Class Reference
Inheritance diagram for lsst.obs.base.ingest_tests.IngestTestBase:

Public Member Functions

def instrumentClassName (self)
 
def instrumentClass (self)
 
def instrumentName (self)
 
def setUpClass (cls)
 
def setUp (self)
 
def tearDownClass (cls)
 
def verifyIngest (self, files=None, cli=False, fullCheck=False)
 
def checkRepo (self, files=None)
 
def testLink (self)
 
def testSymLink (self)
 
def testDirect (self)
 
def testCopy (self)
 
def testHardLink (self)
 
def testInPlace (self)
 
def testFailOnConflict (self)
 
def testWriteCuratedCalibrations (self)
 
def testDefineVisits (self)
 

Public Attributes

 root
 
 outputRun
 

Static Public Attributes

string ingestDir = ""
 
list dataIds = []
 
string file = ""
 
 filterLabel = None
 
string rawIngestTask = "lsst.obs.base.RawIngestTask"
 
 curatedCalibrationDatasetTypes = None
 
 defineVisitsTask = lsst.obs.base.DefineVisitsTask
 
dictionary visits = {}
 

Detailed Description

Base class for tests of gen3 ingest. Subclass from this, then
`unittest.TestCase` to get a working test suite.

Definition at line 43 of file ingest_tests.py.

Member Function Documentation

◆ checkRepo()

def lsst.obs.base.ingest_tests.IngestTestBase.checkRepo (   self,
  files = None 
)
Check the state of the repository after ingest.

This is an optional hook provided for subclasses; by default it does
nothing.

Parameters
----------
files : `list` [`str`], or None
    List of files to be ingested, or None to use ``self.file``

Definition at line 185 of file ingest_tests.py.

185  def checkRepo(self, files=None):
186  """Check the state of the repository after ingest.
187 
188  This is an optional hook provided for subclasses; by default it does
189  nothing.
190 
191  Parameters
192  ----------
193  files : `list` [`str`], or None
194  List of files to be ingested, or None to use ``self.file``
195  """
196  pass
197 

◆ instrumentClass()

def lsst.obs.base.ingest_tests.IngestTestBase.instrumentClass (   self)
The instrument class.

Definition at line 97 of file ingest_tests.py.

97  def instrumentClass(self):
98  """The instrument class."""
99  return doImport(self.instrumentClassName)
100 

◆ instrumentClassName()

def lsst.obs.base.ingest_tests.IngestTestBase.instrumentClassName (   self)
The fully qualified instrument class name.

Returns
-------
`str`
    The fully qualified instrument class name.

Definition at line 86 of file ingest_tests.py.

86  def instrumentClassName(self):
87  """The fully qualified instrument class name.
88 
89  Returns
90  -------
91  `str`
92  The fully qualified instrument class name.
93  """
94  pass
95 

◆ instrumentName()

def lsst.obs.base.ingest_tests.IngestTestBase.instrumentName (   self)
The name of the instrument.

Returns
-------
`str`
    The name of the instrument.

Definition at line 102 of file ingest_tests.py.

102  def instrumentName(self):
103  """The name of the instrument.
104 
105  Returns
106  -------
107  `str`
108  The name of the instrument.
109  """
110  return self.instrumentClass.getName()
111 
std::string const & getName() const noexcept
Return a filter's name.
Definition: Filter.h:78

◆ setUp()

def lsst.obs.base.ingest_tests.IngestTestBase.setUp (   self)

Definition at line 121 of file ingest_tests.py.

121  def setUp(self):
122  # Want a unique run name per test
123  self.outputRun = "raw_ingest_" + self.id()
124 

◆ setUpClass()

def lsst.obs.base.ingest_tests.IngestTestBase.setUpClass (   cls)

Definition at line 113 of file ingest_tests.py.

113  def setUpClass(cls):
114  # Use a temporary working directory
115  cls.root = tempfile.mkdtemp(dir=cls.ingestDir)
116  cls._createRepo()
117 
118  # Register the instrument and its static metadata
119  cls._registerInstrument()
120 

◆ tearDownClass()

def lsst.obs.base.ingest_tests.IngestTestBase.tearDownClass (   cls)

Definition at line 126 of file ingest_tests.py.

126  def tearDownClass(cls):
127  if os.path.exists(cls.root):
128  shutil.rmtree(cls.root, ignore_errors=True)
129 

◆ testCopy()

def lsst.obs.base.ingest_tests.IngestTestBase.testCopy (   self)

Definition at line 262 of file ingest_tests.py.

262  def testCopy(self):
263  self._ingestRaws(transfer="copy")
264  # Only test full read of raws for the copy test. No need to do it
265  # in the other tests since the formatter will be the same in all
266  # cases.
267  self.verifyIngest(fullCheck=True)
268 

◆ testDefineVisits()

def lsst.obs.base.ingest_tests.IngestTestBase.testDefineVisits (   self)

Definition at line 355 of file ingest_tests.py.

355  def testDefineVisits(self):
356  if self.visits is None:
357  self.skipTest("Expected visits were not defined.")
358  self._ingestRaws(transfer="link")
359 
360  # Calling defineVisits tests the implementation of the butler command
361  # line interface "define-visits" subcommand. Functions in the script
362  # folder are generally considered protected and should not be used
363  # as public api.
364  script.defineVisits(self.root, config_file=None, collections=self.outputRun,
365  instrument=self.instrumentName)
366 
367  # Test that we got the visits we expected.
368  butler = Butler(self.root, run=self.outputRun)
369  visits = butler.registry.queryDataIds(["visit"]).expanded().toSet()
370  self.assertCountEqual(visits, self.visits.keys())
371  instr = getInstrument(self.instrumentName, butler.registry)
372  camera = instr.getCamera()
373  for foundVisit, (expectedVisit, expectedExposures) in zip(visits, self.visits.items()):
374  # Test that this visit is associated with the expected exposures.
375  foundExposures = butler.registry.queryDataIds(["exposure"], dataId=expectedVisit
376  ).expanded().toSet()
377  self.assertCountEqual(foundExposures, expectedExposures)
378  # Test that we have a visit region, and that it contains all of the
379  # detector+visit regions.
380  self.assertIsNotNone(foundVisit.region)
381  detectorVisitDataIds = butler.registry.queryDataIds(["visit", "detector"], dataId=expectedVisit
382  ).expanded().toSet()
383  self.assertEqual(len(detectorVisitDataIds), len(camera))
384  for dataId in detectorVisitDataIds:
385  self.assertTrue(foundVisit.region.contains(dataId.region))
std::vector< SchemaItem< Flag > > * items
def getInstrument(instrumentName, registry=None)
Definition: utils.py:131

◆ testDirect()

def lsst.obs.base.ingest_tests.IngestTestBase.testDirect (   self)

Definition at line 252 of file ingest_tests.py.

252  def testDirect(self):
253  self._ingestRaws(transfer="direct")
254 
255  # Check that it really did have a URI outside of datastore
256  srcUri = ButlerURI(self.file)
257  butler = Butler(self.root, run=self.outputRun)
258  datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
259  datastoreUri = butler.getURI(datasets[0])
260  self.assertEqual(datastoreUri, srcUri)
261 
daf::base::PropertyList * list
Definition: fits.cc:913

◆ testFailOnConflict()

def lsst.obs.base.ingest_tests.IngestTestBase.testFailOnConflict (   self)
Re-ingesting the same data into the repository should fail.

Definition at line 301 of file ingest_tests.py.

301  def testFailOnConflict(self):
302  """Re-ingesting the same data into the repository should fail.
303  """
304  self._ingestRaws(transfer="symlink")
305  with self.assertRaises(Exception):
306  self._ingestRaws(transfer="symlink")
307 

◆ testHardLink()

def lsst.obs.base.ingest_tests.IngestTestBase.testHardLink (   self)

Definition at line 269 of file ingest_tests.py.

269  def testHardLink(self):
270  try:
271  self._ingestRaws(transfer="hardlink")
272  # Running ingest through the Click testing infrastructure causes
273  # the original exception indicating that we can't hard-link
274  # on this filesystem to be turned into a nonzero exit code, which
275  # then trips the test assertion.
276  except (AssertionError, PermissionError) as err:
277  raise unittest.SkipTest("Skipping hard-link test because input data"
278  " is on a different filesystem.") from err
279  self.verifyIngest()
280 

◆ testInPlace()

def lsst.obs.base.ingest_tests.IngestTestBase.testInPlace (   self)
Test that files already in the directory can be added to the
registry in-place.

Definition at line 281 of file ingest_tests.py.

281  def testInPlace(self):
282  """Test that files already in the directory can be added to the
283  registry in-place.
284  """
285  # symlink into repo root manually
286  butler = Butler(self.root, run=self.outputRun)
287  pathInStore = "prefix-" + os.path.basename(self.file)
288  newPath = butler.datastore.root.join(pathInStore)
289  os.symlink(os.path.abspath(self.file), newPath.ospath)
290  self._ingestRaws(transfer="auto", file=newPath.ospath)
291  self.verifyIngest()
292 
293  # Recreate a butler post-ingest (the earlier one won't see the
294  # ingested files)
295  butler = Butler(self.root, run=self.outputRun)
296 
297  # Check that the URI associated with this path is the right one
298  uri = butler.getURI("raw", self.dataIds[0])
299  self.assertEqual(uri.relative_to(butler.datastore.root), pathInStore)
300 

◆ testLink()

def lsst.obs.base.ingest_tests.IngestTestBase.testLink (   self)

Definition at line 244 of file ingest_tests.py.

244  def testLink(self):
245  self._ingestRaws(transfer="link")
246  self.verifyIngest()
247 

◆ testSymLink()

def lsst.obs.base.ingest_tests.IngestTestBase.testSymLink (   self)

Definition at line 248 of file ingest_tests.py.

248  def testSymLink(self):
249  self._ingestRaws(transfer="symlink")
250  self.verifyIngest()
251 

◆ testWriteCuratedCalibrations()

def lsst.obs.base.ingest_tests.IngestTestBase.testWriteCuratedCalibrations (   self)
Test that we can ingest the curated calibrations, and read them
with `loadCamera` both before and after.

Definition at line 308 of file ingest_tests.py.

308  def testWriteCuratedCalibrations(self):
309  """Test that we can ingest the curated calibrations, and read them
310  with `loadCamera` both before and after.
311  """
312  if self.curatedCalibrationDatasetTypes is None:
313  raise unittest.SkipTest("Class requests disabling of writeCuratedCalibrations test")
314 
315  butler = Butler(self.root, writeable=False)
316  collection = self.instrumentClass.makeCalibrationCollectionName()
317 
318  # Trying to load a camera with a data ID not known to the registry
319  # is an error, because we can't get any temporal information.
320  with self.assertRaises(LookupError):
321  lsst.obs.base.loadCamera(butler, {"exposure": 0}, collections=collection)
322 
323  # Ingest raws in order to get some exposure records.
324  self._ingestRaws(transfer="auto")
325 
326  # Load camera should returned an unversioned camera because there's
327  # nothing in the repo.
328  camera, isVersioned = lsst.obs.base.loadCamera(butler, self.dataIds[0], collections=collection)
329  self.assertFalse(isVersioned)
330  self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
331 
332  self._writeCuratedCalibrations()
333 
334  # Make a new butler instance to make sure we don't have any stale
335  # caches (e.g. of DatasetTypes). Note that we didn't give
336  # _writeCuratedCalibrations the butler instance we had, because it's
337  # trying to test the CLI interface anyway.
338  butler = Butler(self.root, writeable=False)
339 
340  for datasetTypeName in self.curatedCalibrationDatasetTypes:
341  with self.subTest(dtype=datasetTypeName):
342  found = list(
343  butler.registry.queryDatasetAssociations(
344  datasetTypeName,
345  collections=collection,
346  )
347  )
348  self.assertGreater(len(found), 0, f"Checking {datasetTypeName}")
349 
350  # Load camera should returned the versioned camera from the repo.
351  camera, isVersioned = lsst.obs.base.loadCamera(butler, self.dataIds[0], collections=collection)
352  self.assertTrue(isVersioned)
353  self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
354 
An immutable representation of a camera.
Definition: Camera.h:43

◆ verifyIngest()

def lsst.obs.base.ingest_tests.IngestTestBase.verifyIngest (   self,
  files = None,
  cli = False,
  fullCheck = False 
)
Test that RawIngestTask ingested the expected files.

Parameters
----------
files : `list` [`str`], or None
    List of files to be ingested, or None to use ``self.file``
fullCheck : `bool`, optional
    If `True`, read the full raw dataset and check component
    consistency. If `False` check that a component can be read
    but do not read the entire raw exposure.

Notes
-----
Reading all the ingested test data can be expensive. The code paths
for reading the second raw are the same as reading the first so
we do not gain anything by doing full checks of everything.
Only read full pixel data for first dataset from file.
Don't even do that if we are requested not to by the caller.
This only really affects files that contain multiple datasets.

Definition at line 130 of file ingest_tests.py.

130  def verifyIngest(self, files=None, cli=False, fullCheck=False):
131  """
132  Test that RawIngestTask ingested the expected files.
133 
134  Parameters
135  ----------
136  files : `list` [`str`], or None
137  List of files to be ingested, or None to use ``self.file``
138  fullCheck : `bool`, optional
139  If `True`, read the full raw dataset and check component
140  consistency. If `False` check that a component can be read
141  but do not read the entire raw exposure.
142 
143  Notes
144  -----
145  Reading all the ingested test data can be expensive. The code paths
146  for reading the second raw are the same as reading the first so
147  we do not gain anything by doing full checks of everything.
148  Only read full pixel data for first dataset from file.
149  Don't even do that if we are requested not to by the caller.
150  This only really affects files that contain multiple datasets.
151  """
152  butler = Butler(self.root, run=self.outputRun)
153  datasets = list(butler.registry.queryDatasets("raw", collections=self.outputRun))
154  self.assertEqual(len(datasets), len(self.dataIds))
155 
156  # Get the URI to the first dataset and check it is inside the
157  # datastore
158  datasetUri = butler.getURI(datasets[0])
159  self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
160 
161  for dataId in self.dataIds:
162  # Check that we can read metadata from a raw
163  metadata = butler.get("raw.metadata", dataId)
164  if not fullCheck:
165  continue
166  fullCheck = False
167  exposure = butler.get("raw", dataId)
168  self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())
169 
170  # Since components follow a different code path we check that
171  # WCS match and also we check that at least the shape
172  # of the image is the same (rather than doing per-pixel equality)
173  wcs = butler.get("raw.wcs", dataId)
174  self.assertEqual(wcs, exposure.getWcs())
175 
176  rawImage = butler.get("raw.image", dataId)
177  self.assertEqual(rawImage.getBBox(), exposure.getBBox())
178 
179  # check that the filter label got the correct band
180  filterLabel = butler.get("raw.filterLabel", dataId)
181  self.assertEqual(filterLabel, self.filterLabel)
182 
183  self.checkRepo(files=files)
184 

Member Data Documentation

◆ curatedCalibrationDatasetTypes

lsst.obs.base.ingest_tests.IngestTestBase.curatedCalibrationDatasetTypes = None
static

Definition at line 66 of file ingest_tests.py.

◆ dataIds

list lsst.obs.base.ingest_tests.IngestTestBase.dataIds = []
static

Definition at line 53 of file ingest_tests.py.

◆ defineVisitsTask

lsst.obs.base.ingest_tests.IngestTestBase.defineVisitsTask = lsst.obs.base.DefineVisitsTask
static

Definition at line 71 of file ingest_tests.py.

◆ file

string lsst.obs.base.ingest_tests.IngestTestBase.file = ""
static

Definition at line 56 of file ingest_tests.py.

◆ filterLabel

lsst.obs.base.ingest_tests.IngestTestBase.filterLabel = None
static

Definition at line 59 of file ingest_tests.py.

◆ ingestDir

string lsst.obs.base.ingest_tests.IngestTestBase.ingestDir = ""
static

Definition at line 48 of file ingest_tests.py.

◆ outputRun

lsst.obs.base.ingest_tests.IngestTestBase.outputRun

Definition at line 123 of file ingest_tests.py.

◆ rawIngestTask

string lsst.obs.base.ingest_tests.IngestTestBase.rawIngestTask = "lsst.obs.base.RawIngestTask"
static

Definition at line 63 of file ingest_tests.py.

◆ root

lsst.obs.base.ingest_tests.IngestTestBase.root

Definition at line 115 of file ingest_tests.py.

◆ visits

dictionary lsst.obs.base.ingest_tests.IngestTestBase.visits = {}
static

Definition at line 76 of file ingest_tests.py.


The documentation for this class was generated from the following file: