lsst.obs.base.gen3.ingest_tests.IngestTestBase Class Reference

Inheritance diagram for lsst.obs.base.gen3.ingest_tests.IngestTestBase:

Public Member Functions
def	setUp (self)
def	tearDown (self)
def	runIngest (self, files=None)
def	runIngestTest (self, files=None)
def	testSymLink (self)
def	testCopy (self)
def	testHardLink (self)
def	testInPlace (self)
def	testOnConflictFail (self)
def	testOnConflictIgnore (self)
def	testOnConflictStash (self)
def	testOnErrorBreak (self)
def	testOnErrorContinue (self)
def	testOnErrorRollback (self)

Public Attributes
	root
	butler
	config

Static Public Attributes
string	ingestDir = ""
	instrument = None
dictionary	dataId = {}
string	file = ""

Detailed Description

Base class for tests of gen3 ingest. Subclass from this, then
`unittest.TestCase` to get a working test suite.

Definition at line 36 of file ingest_tests.py.

Member Function Documentation

runIngest()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.runIngest	(		self,
			files = None
	)

Initialize and run RawIngestTask on a list of files.

Parameters
----------
files : `list`, [`str`], or None
    List of files to be ingested, or None to use ``self.file``

Definition at line 72 of file ingest_tests.py.

    def runIngest(self, files=None):
        """
        Initialize and run RawIngestTask on a list of files.

        Parameters
        ----------
        files : `list`, [`str`], or None
            List of files to be ingested, or None to use ``self.file``
        """
        if files is None:
            files = [self.file]
        task = RawIngestTask(config=self.config, butler=self.butler)
        task.log.setLevel(task.log.FATAL)  # silence logs, since we expect a lot of warnings
        task.run(files)

runIngestTest()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.runIngestTest	(		self,
			files = None
	)

Test that RawIngestTask ingested the expected files.

Parameters
----------
files : `list`, [`str`], or None
    List of files to be ingested, or None to use ``self.file``

Definition at line 87 of file ingest_tests.py.

    def runIngestTest(self, files=None):
        """
        Test that RawIngestTask ingested the expected files.

        Parameters
        ----------
        files : `list`, [`str`], or None
            List of files to be ingested, or None to use ``self.file``
        """
        self.runIngest(files)
        exposure = self.butler.get("raw", self.dataId)
        metadata = self.butler.get("raw.metadata", self.dataId)
        image = self.butler.get("raw.image", self.dataId)
        self.assertImagesEqual(exposure.image, image)
        self.assertEqual(metadata.toDict(), exposure.getMetadata().toDict())

setUp()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.setUp

(

self

)

Definition at line 55 of file ingest_tests.py.

    def setUp(self):
        # Use a temporary working directory
        self.root = tempfile.mkdtemp(dir=self.ingestDir)
        Butler.makeRepo(self.root)
        self.butler = Butler(self.root, run="raw")

        # Register the instrument and its static metadata
        self.instrument.register(self.butler.registry)

        # Make a default config for test methods to play with
        self.config = RawIngestTask.ConfigClass()
        self.config.onError = "break"

tearDown()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.tearDown

(

self

)

Definition at line 68 of file ingest_tests.py.

    def tearDown(self):
        if os.path.exists(self.root):
            shutil.rmtree(self.root, ignore_errors=True)

testCopy()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testCopy

(

self

)

Definition at line 107 of file ingest_tests.py.

    def testCopy(self):
        self.config.transfer = "copy"
        self.runIngestTest()

testHardLink()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testHardLink

(

self

)

Definition at line 111 of file ingest_tests.py.

    def testHardLink(self):
        self.config.transfer = "hardlink"
        self.runIngestTest()

testInPlace()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testInPlace

(

self

)

Test that files already in the directory can be added to the
registry in-place.

Definition at line 115 of file ingest_tests.py.

    def testInPlace(self):
        """Test that files already in the directory can be added to the
        registry in-place.
        """
        # hardlink into repo root manually
        newPath = os.path.join(self.butler.datastore.root, os.path.basename(self.file))
        os.link(self.file, newPath)
        self.config.transfer = None
        self.runIngestTest([newPath])

testOnConflictFail()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnConflictFail

(

self

)

Re-ingesting the same data into the repository should fail, if
configured to do so.

Definition at line 125 of file ingest_tests.py.

    def testOnConflictFail(self):
        """Re-ingesting the same data into the repository should fail, if
        configured to do so.
        """
        self.config.transfer = "symlink"
        self.config.conflict = "fail"
        self.runIngest()
        with self.assertRaises(Exception):
            self.runIngest()

testOnConflictIgnore()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnConflictIgnore

(

self

)

Re-ingesting the same data into the repository does not fail, if
configured to ignore conflict errors.

Definition at line 135 of file ingest_tests.py.

    def testOnConflictIgnore(self):
        """Re-ingesting the same data into the repository does not fail, if
        configured to ignore conflict errors.
        """
        self.config.transfer = "symlink"
        self.config.conflict = "ignore"
        self.runIngest()   # this one should succeed
        n1, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
        self.runIngest()   # this one should silently fail
        n2, = self.butler.registry.query("SELECT COUNT(*) FROM Dataset")
        self.assertEqual(n1, n2)

testOnConflictStash()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnConflictStash

(

self

)

Re-ingesting the same data will be put into a different collection,
if configured to do so.

Definition at line 147 of file ingest_tests.py.

    def testOnConflictStash(self):
        """Re-ingesting the same data will be put into a different collection,
        if configured to do so.
        """
        self.config.transfer = "symlink"
        self.config.conflict = "ignore"
        self.config.stash = "stash"
        self.runIngest()   # this one should write to 'raw'
        self.runIngest()   # this one should write to 'stash'
        dt = self.butler.registry.getDatasetType("raw.metadata")
        ref1 = self.butler.registry.find(self.butler.collection, dt, self.dataId)
        ref2 = self.butler.registry.find("stash", dt, self.dataId)
        self.assertNotEqual(ref1.id, ref2.id)
        self.assertEqual(self.butler.get(ref1).toDict(), self.butler.getDirect(ref2).toDict())

testOnErrorBreak()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnErrorBreak

(

self

)

Test that errors do not roll back success, when configured to do so.

Failing to ingest a nonexistent file after ingesting the valid one should
leave the valid one in the registry, despite raising an exception.

Definition at line 162 of file ingest_tests.py.

    def testOnErrorBreak(self):
        """Test that errors do not roll back success, when configured to do so.

        Failing to ingest a nonexistent file after ingesting the valid one should
        leave the valid one in the registry, despite raising an exception.
        """
        self.config.transfer = "symlink"
        self.config.onError = "break"
        with self.assertRaises(Exception):
            self.runIngest(files=[self.file, "nonexistent.fits"])
        dt = self.butler.registry.getDatasetType("raw.metadata")
        self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))

testOnErrorContinue()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnErrorContinue

(

self

)

Failing to ingest nonexistent files before and after ingesting the
valid one should leave the valid one in the registry and not raise
an exception.

Definition at line 175 of file ingest_tests.py.

    def testOnErrorContinue(self):
        """Failing to ingest nonexistent files before and after ingesting the
        valid one should leave the valid one in the registry and not raise
        an exception.
        """
        self.config.transfer = "symlink"
        self.config.onError = "continue"
        self.runIngest(files=["nonexistent.fits", self.file, "still-not-here.fits"])
        dt = self.butler.registry.getDatasetType("raw.metadata")
        self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))

testOnErrorRollback()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testOnErrorRollback

(

self

)

Failing to ingest nonexistent files after ingesting the
valid one should leave the registry unchanged.

Definition at line 186 of file ingest_tests.py.

    def testOnErrorRollback(self):
        """Failing to ingest nonexistent files after ingesting the
        valid one should leave the registry unchanged.
        """
        self.config.transfer = "symlink"
        self.config.onError = "rollback"
        with self.assertRaises(Exception):
            self.runIngest(file=[self.file, "nonexistent.fits"])
        try:
            dt = self.butler.registry.getDatasetType("raw.metadata")
        except KeyError:
            # If we also rollback registering the DatasetType, that's fine,
            # but not required.
            pass
        else:
            self.assertIsNotNone(self.butler.registry.find(self.butler.collection, dt, self.dataId))

testSymLink()

def lsst.obs.base.gen3.ingest_tests.IngestTestBase.testSymLink

(

self

)

Definition at line 103 of file ingest_tests.py.

    def testSymLink(self):
        self.config.transfer = "symlink"
        self.runIngestTest()

Member Data Documentation

butler

lsst.obs.base.gen3.ingest_tests.IngestTestBase.butler

Definition at line 59 of file ingest_tests.py.

config

lsst.obs.base.gen3.ingest_tests.IngestTestBase.config

Definition at line 65 of file ingest_tests.py.

dataId

dictionary lsst.obs.base.gen3.ingest_tests.IngestTestBase.dataId = {}

static

Definition at line 49 of file ingest_tests.py.

file

string lsst.obs.base.gen3.ingest_tests.IngestTestBase.file = ""

static

Definition at line 52 of file ingest_tests.py.

ingestDir

string lsst.obs.base.gen3.ingest_tests.IngestTestBase.ingestDir = ""

static

Definition at line 41 of file ingest_tests.py.

instrument

lsst.obs.base.gen3.ingest_tests.IngestTestBase.instrument = None

static

Definition at line 46 of file ingest_tests.py.

root

lsst.obs.base.gen3.ingest_tests.IngestTestBase.root

Definition at line 57 of file ingest_tests.py.

---

The documentation for this class was generated from the following file:

• /j/snowflake/release/lsstsw/stack/Linux64/obs_base/18.1.0-18-gb5d19ff+1/python/lsst/obs/base/gen3/ingest_tests.py

---

Generated on Mon Sep 23 2019 07:03:02 for LSSTApplications by   1.8.13