23 """Support code for running unit tests"""
40 __all__ = [
"init",
"MemoryTestCase",
"ExecutablesTestCase",
"getTempFilePath",
41 "TestCase",
"assertFloatsAlmostEqual",
"assertFloatsNotEqual",
"assertFloatsEqual",
42 "debugger",
"classParameters",
"methodParameters"]
48 def _get_open_files():
49 """Return a set containing the list of files currently open in this
55 Set containing the list of open files.
61 """Initialize the memory tester and file descriptor leak tester."""
64 open_files = _get_open_files()
68 """Sort supplied test suites such that MemoryTestCases are at the end.
70 `lsst.utils.tests.MemoryTestCase` tests should always run after any other
76 Sequence of test suites.
80 suite : `unittest.TestSuite`
81 A combined `~unittest.TestSuite` with
82 `~lsst.utils.tests.MemoryTestCase` at the end.
85 suite = unittest.TestSuite()
87 for test_suite
in tests:
94 for method
in test_suite:
95 bases = inspect.getmro(method.__class__)
97 if bases
is not None and MemoryTestCase
in bases:
98 memtests.append(test_suite)
100 suite.addTests(test_suite)
102 if isinstance(test_suite, MemoryTestCase):
103 memtests.append(test_suite)
105 suite.addTest(test_suite)
106 suite.addTests(memtests)
117 unittest.defaultTestLoader.suiteClass = suiteClassWrapper
121 """Check for resource leaks."""
125 """Reset the leak counter when the tests have been completed"""
129 """Check if any file descriptors are open since init() called."""
132 now_open = _get_open_files()
135 now_open =
set(f
for f
in now_open
if not f.endswith(
".car")
136 and not f.startswith(
"/proc/")
137 and not f.endswith(
".ttf")
138 and not (f.startswith(
"/var/lib/")
and f.endswith(
"/passwd"))
139 and not f.endswith(
"astropy.log"))
141 diff = now_open.difference(open_files)
144 print(
"File open: %s" % f)
145 self.fail(
"Failed to close %d file%s" % (len(diff),
"s" if len(diff) != 1
else ""))
149 """Test that executables can be run and return good status.
151 The test methods are dynamically created. Callers
152 must subclass this class in their own test file and invoke
153 the create_executable_tests() class method to register the tests.
155 TESTS_DISCOVERED = -1
159 """Abort testing if automated test creation was enabled and
160 no tests were found."""
163 raise RuntimeError(
"No executables discovered.")
166 """This test exists to ensure that there is at least one test to be
167 executed. This allows the test runner to trigger the class set up
168 machinery to test whether there are some executables to test."""
172 """Check an executable runs and returns good status.
174 Prints output to standard out. On bad exit status the test
175 fails. If the executable can not be located the test is skipped.
180 Path to an executable. ``root_dir`` is not used if this is an
182 root_dir : `str`, optional
183 Directory containing executable. Ignored if `None`.
184 args : `list` or `tuple`, optional
185 Arguments to be provided to the executable.
186 msg : `str`, optional
187 Message to use when the test fails. Can be `None` for default
193 The executable did not return 0 exit status.
196 if root_dir
is not None and not os.path.isabs(executable):
197 executable = os.path.join(root_dir, executable)
200 sp_args = [executable]
201 argstr =
"no arguments"
204 argstr =
'arguments "' +
" ".join(args) +
'"'
206 print(
"Running executable '{}' with {}...".
format(executable, argstr))
207 if not os.path.exists(executable):
208 self.skipTest(
"Executable {} is unexpectedly missing".
format(executable))
211 output = subprocess.check_output(sp_args)
212 except subprocess.CalledProcessError
as e:
214 failmsg =
"Bad exit status from '{}': {}".
format(executable, e.returncode)
215 print(output.decode(
'utf-8'))
222 def _build_test_method(cls, executable, root_dir):
223 """Build a test method and attach to class.
225 A test method is created for the supplied excutable located
226 in the supplied root directory. This method is attached to the class
227 so that the test runner will discover the test and run it.
232 The class in which to create the tests.
234 Name of executable. Can be absolute path.
236 Path to executable. Not used if executable path is absolute.
238 if not os.path.isabs(executable):
239 executable = os.path.abspath(os.path.join(root_dir, executable))
242 test_name =
"test_exe_" + executable.replace(
"/",
"_")
245 def test_executable_runs(*args):
247 self.assertExecutable(executable)
250 test_executable_runs.__name__ = test_name
251 setattr(cls, test_name, test_executable_runs)
255 """Discover executables to test and create corresponding test methods.
257 Scans the directory containing the supplied reference file
258 (usually ``__file__`` supplied from the test class) to look for
259 executables. If executables are found a test method is created
260 for each one. That test method will run the executable and
261 check the returned value.
263 Executable scripts with a ``.py`` extension and shared libraries
264 are ignored by the scanner.
266 This class method must be called before test discovery.
271 Path to a file within the directory to be searched.
272 If the files are in the same location as the test file, then
273 ``__file__`` can be used.
274 executables : `list` or `tuple`, optional
275 Sequence of executables that can override the automated
276 detection. If an executable mentioned here is not found, a
277 skipped test will be created for it, rather than a failed
282 >>> cls.create_executable_tests(__file__)
286 ref_dir = os.path.abspath(os.path.dirname(ref_file))
288 if executables
is None:
291 for root, dirs, files
in os.walk(ref_dir):
294 if not f.endswith(
".py")
and not f.endswith(
".so"):
295 full_path = os.path.join(root, f)
296 if os.access(full_path, os.X_OK):
297 executables.append(full_path)
306 for e
in executables:
310 @contextlib.contextmanager
312 """Return a path suitable for a temporary file and try to delete the
315 If the with block completes successfully then the file is deleted,
316 if possible; failure results in a printed warning.
317 If a file is remains when it should not, a RuntimeError exception is
318 raised. This exception is also raised if a file is not present on context
319 manager exit when one is expected to exist.
320 If the block exits with an exception the file if left on disk so it can be
321 examined. The file name has a random component such that nested context
322 managers can be used with the same file suffix.
328 File name extension, e.g. ``.fits``.
329 expectOutput : `bool`, optional
330 If `True`, a file should be created within the context manager.
331 If `False`, a file should not be present when the context manager
337 Path for a temporary file. The path is a combination of the caller's
338 file path and the name of the top-level function
344 # file tests/testFoo.py
346 import lsst.utils.tests
347 class FooTestCase(unittest.TestCase):
348 def testBasics(self):
352 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile:
353 # if tests/.tests exists then
354 # tmpFile = "tests/.tests/testFoo_testBasics.fits"
355 # otherwise tmpFile = "testFoo_testBasics.fits"
357 # at the end of this "with" block the path tmpFile will be
358 # deleted, but only if the file exists and the "with"
359 # block terminated normally (rather than with an exception)
362 stack = inspect.stack()
364 for i
in range(2, len(stack)):
365 frameInfo = inspect.getframeinfo(stack[i][0])
367 callerFilePath = frameInfo.filename
368 callerFuncName = frameInfo.function
369 elif callerFilePath == frameInfo.filename:
371 callerFuncName = frameInfo.function
375 callerDir, callerFileNameWithExt = os.path.split(callerFilePath)
376 callerFileName = os.path.splitext(callerFileNameWithExt)[0]
377 outDir = os.path.join(callerDir,
".tests")
378 if not os.path.isdir(outDir):
380 prefix =
"%s_%s-" % (callerFileName, callerFuncName)
381 outPath = tempfile.mktemp(dir=outDir, suffix=ext, prefix=prefix)
382 if os.path.exists(outPath):
385 warnings.warn(
"Unexpectedly found pre-existing tempfile named %r" % (outPath,),
394 fileExists = os.path.exists(outPath)
397 raise RuntimeError(
"Temp file expected named {} but none found".
format(outPath))
400 raise RuntimeError(
"Unexpectedly discovered temp file named {}".
format(outPath))
407 warnings.warn(
"Warning: could not remove file %r: %s" % (outPath, e), stacklevel=3)
411 """Subclass of unittest.TestCase that adds some custom assertions for
417 """A decorator to add a free function to our custom TestCase class, while also
418 making it available as a free function.
420 setattr(TestCase, func.__name__, func)
425 """Decorator to enter the debugger when there's an uncaught exception
427 To use, just slap a ``@debugger()`` on your function.
429 You may provide specific exception classes to catch as arguments to
430 the decorator function, e.g.,
431 ``@debugger(RuntimeError, NotImplementedError)``.
432 This defaults to just `AssertionError`, for use on `unittest.TestCase`
435 Code provided by "Rosh Oxymoron" on StackOverflow:
436 http://stackoverflow.com/questions/4398967/python-unit-testing-automatically-running-the-debugger-when-a-test-fails
440 Consider using ``pytest --pdb`` instead of this decorator.
443 exceptions = (Exception, )
447 def wrapper(*args, **kwargs):
449 return f(*args, **kwargs)
453 pdb.post_mortem(sys.exc_info()[2])
459 """Plot the comparison of two 2-d NumPy arrays.
463 lhs : `numpy.ndarray`
464 LHS values to compare; a 2-d NumPy array
465 rhs : `numpy.ndarray`
466 RHS values to compare; a 2-d NumPy array
467 bad : `numpy.ndarray`
468 A 2-d boolean NumPy array of values to emphasize in the plots
469 diff : `numpy.ndarray`
470 difference array; a 2-d NumPy array, or None to show lhs-rhs
472 Filename to save the plot to. If None, the plot will be displayed in
477 This method uses `matplotlib` and imports it internally; it should be
478 wrapped in a try/except block within packages that do not depend on
479 `matplotlib` (including `~lsst.utils`).
481 from matplotlib
import pyplot
487 badImage = numpy.zeros(bad.shape + (4,), dtype=numpy.uint8)
488 badImage[:, :, 0] = 255
489 badImage[:, :, 1] = 0
490 badImage[:, :, 2] = 0
491 badImage[:, :, 3] = 255*bad
492 vmin1 = numpy.minimum(numpy.min(lhs), numpy.min(rhs))
493 vmax1 = numpy.maximum(numpy.max(lhs), numpy.max(rhs))
494 vmin2 = numpy.min(diff)
495 vmax2 = numpy.max(diff)
496 for n, (image, title)
in enumerate([(lhs,
"lhs"), (rhs,
"rhs"), (diff,
"diff")]):
497 pyplot.subplot(2, 3, n + 1)
498 im1 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
499 vmin=vmin1, vmax=vmax1)
501 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
504 pyplot.subplot(2, 3, n + 4)
505 im2 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
506 vmin=vmin2, vmax=vmax2)
508 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
511 pyplot.subplots_adjust(left=0.05, bottom=0.05, top=0.92, right=0.75, wspace=0.05, hspace=0.05)
512 cax1 = pyplot.axes([0.8, 0.55, 0.05, 0.4])
513 pyplot.colorbar(im1, cax=cax1)
514 cax2 = pyplot.axes([0.8, 0.05, 0.05, 0.4])
515 pyplot.colorbar(im2, cax=cax2)
517 pyplot.savefig(plotFileName)
524 atol=sys.float_info.epsilon, relTo=None,
525 printFailures=True, plotOnFailure=False,
526 plotFileName=None, invert=False, msg=None,
528 """Highly-configurable floating point comparisons for scalars and arrays.
530 The test assertion will fail if all elements ``lhs`` and ``rhs`` are not
531 equal to within the tolerances specified by ``rtol`` and ``atol``.
532 More precisely, the comparison is:
534 ``abs(lhs - rhs) <= relTo*rtol OR abs(lhs - rhs) <= atol``
536 If ``rtol`` or ``atol`` is `None`, that term in the comparison is not
539 When not specified, ``relTo`` is the elementwise maximum of the absolute
540 values of ``lhs`` and ``rhs``. If set manually, it should usually be set
541 to either ``lhs`` or ``rhs``, or a scalar value typical of what is
546 testCase : `unittest.TestCase`
547 Instance the test is part of.
548 lhs : scalar or array-like
549 LHS value(s) to compare; may be a scalar or array-like of any
551 rhs : scalar or array-like
552 RHS value(s) to compare; may be a scalar or array-like of any
554 rtol : `float`, optional
555 Relative tolerance for comparison; defaults to double-precision
557 atol : `float`, optional
558 Absolute tolerance for comparison; defaults to double-precision
560 relTo : `float`, optional
561 Value to which comparison with rtol is relative.
562 printFailures : `bool`, optional
563 Upon failure, print all inequal elements as part of the message.
564 plotOnFailure : `bool`, optional
565 Upon failure, plot the originals and their residual with matplotlib.
566 Only 2-d arrays are supported.
567 plotFileName : `str`, optional
568 Filename to save the plot to. If `None`, the plot will be displayed in
570 invert : `bool`, optional
571 If `True`, invert the comparison and fail only if any elements *are*
572 equal. Used to implement `~lsst.utils.tests.assertFloatsNotEqual`,
573 which should generally be used instead for clarity.
575 msg : `str`, optional
576 String to append to the error message when assert fails.
577 ignoreNaNs : `bool`, optional
578 If `True` (`False` is default) mask out any NaNs from operand arrays
579 before performing comparisons if they are in the same locations; NaNs
580 in different locations are trigger test assertion failures, even when
581 ``invert=True``. Scalar NaNs are treated like arrays containing only
582 NaNs of the same shape as the other operand, and no comparisons are
583 performed if both sides are scalar NaNs.
588 The values are not almost equal.
591 lhsMask = numpy.isnan(lhs)
592 rhsMask = numpy.isnan(rhs)
593 if not numpy.all(lhsMask == rhsMask):
594 testCase.fail(f
"lhs has {lhsMask.sum()} NaN values and rhs has {rhsMask.sum()} NaN values, "
595 f
"in different locations.")
596 if numpy.all(lhsMask):
597 assert numpy.all(rhsMask),
"Should be guaranteed by previous if."
601 assert not numpy.all(rhsMask),
"Should be guaranteed by prevoius two ifs."
605 if numpy.any(lhsMask):
606 lhs = lhs[numpy.logical_not(lhsMask)]
607 if numpy.any(rhsMask):
608 rhs = rhs[numpy.logical_not(rhsMask)]
609 if not numpy.isfinite(lhs).
all():
610 testCase.fail(
"Non-finite values in lhs")
611 if not numpy.isfinite(rhs).
all():
612 testCase.fail(
"Non-finite values in rhs")
614 absDiff = numpy.abs(lhs - rhs)
617 relTo = numpy.maximum(numpy.abs(lhs), numpy.abs(rhs))
619 relTo = numpy.abs(relTo)
620 bad = absDiff > rtol*relTo
622 bad = numpy.logical_and(bad, absDiff > atol)
625 raise ValueError(
"rtol and atol cannot both be None")
627 failed = numpy.any(bad)
630 bad = numpy.logical_not(bad)
632 failStr =
"are the same"
638 if numpy.isscalar(bad):
640 errMsg = [
"%s %s %s; diff=%s with atol=%s"
641 % (lhs, cmpStr, rhs, absDiff, atol)]
643 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s"
644 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol)]
646 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s, atol=%s"
647 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol, atol)]
649 errMsg = [
"%d/%d elements %s with rtol=%s, atol=%s"
650 % (bad.sum(), bad.size, failStr, rtol, atol)]
652 if len(lhs.shape) != 2
or len(rhs.shape) != 2:
653 raise ValueError(
"plotOnFailure is only valid for 2-d arrays")
655 plotImageDiff(lhs, rhs, bad, diff=diff, plotFileName=plotFileName)
657 errMsg.append(
"Failure plot requested but matplotlib could not be imported.")
662 if numpy.isscalar(relTo):
663 relTo = numpy.ones(bad.shape, dtype=float) * relTo
664 if numpy.isscalar(lhs):
665 lhs = numpy.ones(bad.shape, dtype=float) * lhs
666 if numpy.isscalar(rhs):
667 rhs = numpy.ones(bad.shape, dtype=float) * rhs
669 for a, b, diff
in zip(lhs[bad], rhs[bad], absDiff[bad]):
670 errMsg.append(
"%s %s %s (diff=%s)" % (a, cmpStr, b, diff))
672 for a, b, diff, rel
in zip(lhs[bad], rhs[bad], absDiff[bad], relTo[bad]):
673 errMsg.append(
"%s %s %s (diff=%s/%s=%s)" % (a, cmpStr, b, diff, rel, diff/rel))
677 testCase.assertFalse(failed, msg=
"\n".join(errMsg))
682 """Fail a test if the given floating point values are equal to within the
685 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with
686 ``rtol=atol=0``) for more information.
690 testCase : `unittest.TestCase`
691 Instance the test is part of.
692 lhs : scalar or array-like
693 LHS value(s) to compare; may be a scalar or array-like of any
695 rhs : scalar or array-like
696 RHS value(s) to compare; may be a scalar or array-like of any
702 The values are almost equal.
710 Assert that lhs == rhs (both numeric types, whether scalar or array).
712 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with
713 ``rtol=atol=0``) for more information.
717 testCase : `unittest.TestCase`
718 Instance the test is part of.
719 lhs : scalar or array-like
720 LHS value(s) to compare; may be a scalar or array-like of any
722 rhs : scalar or array-like
723 RHS value(s) to compare; may be a scalar or array-like of any
729 The values are not equal.
734 def _settingsIterator(settings):
735 """Return an iterator for the provided test settings
739 settings : `dict` (`str`: iterable)
740 Lists of test parameters. Each should be an iterable of the same length.
741 If a string is provided as an iterable, it will be converted to a list
747 If the ``settings`` are not of the same length.
751 parameters : `dict` (`str`: anything)
754 for name, values
in settings.items():
755 if isinstance(values, str):
757 settings[name] = [values]
758 num = len(
next(
iter(settings.values())))
759 for name, values
in settings.items():
760 assert len(values) == num, f
"Length mismatch for setting {name}: {len(values)} vs {num}"
761 for ii
in range(num):
762 values = [settings[kk][ii]
for kk
in settings]
763 yield dict(zip(settings, values))
767 """Class decorator for generating unit tests
769 This decorator generates classes with class variables according to the
770 supplied ``settings``.
774 **settings : `dict` (`str`: iterable)
775 The lists of test parameters to set as class variables in turn. Each
776 should be an iterable of the same length.
782 @classParameters(foo=[1, 2], bar=[3, 4])
783 class MyTestCase(unittest.TestCase):
786 will generate two classes, as if you wrote::
788 class MyTestCase_1_3(unittest.TestCase):
793 class MyTestCase_2_4(unittest.TestCase):
798 Note that the values are embedded in the class name.
801 module = sys.modules[cls.__module__].__dict__
802 for params
in _settingsIterator(settings):
803 name = f
"{cls.__name__}_{'_'.join(str(vv) for vv in params.values())}"
804 bindings = dict(cls.__dict__)
805 bindings.update(params)
806 module[name] =
type(name, (cls,), bindings)
811 """Method decorator for unit tests
813 This decorator iterates over the supplied settings, using
814 ``TestCase.subTest`` to communicate the values in the event of a failure.
818 **settings : `dict` (`str`: iterable)
819 The lists of test parameters. Each should be an iterable of the same
826 @methodParameters(foo=[1, 2], bar=[3, 4])
827 def testSomething(self, foo, bar):
832 testSomething(foo=1, bar=3)
833 testSomething(foo=2, bar=4)
836 @functools.wraps(func)
837 def wrapper(self, *args, **kwargs):
838 for params
in _settingsIterator(settings):
839 kwargs.update(params)
840 with self.subTest(**params):
841 func(self, *args, **kwargs)
846 def _cartesianProduct(settings):
847 """Return the cartesian product of the settings
851 settings : `dict` mapping `str` to `iterable`
852 Parameter combinations.
856 product : `dict` mapping `str` to `iterable`
857 Parameter combinations covering the cartesian product (all possible
858 combinations) of the input parameters.
863 cartesianProduct({"foo": [1, 2], "bar": ["black", "white"]})
867 {"foo": [1, 1, 2, 2], "bar": ["black", "white", "black", "white"]}
869 product = {kk: []
for kk
in settings}
870 for values
in itertools.product(*settings.values()):
871 for kk, vv
in zip(settings.keys(), values):
877 """Class decorator for generating unit tests
879 This decorator generates classes with class variables according to the
880 cartesian product of the supplied ``settings``.
884 **settings : `dict` (`str`: iterable)
885 The lists of test parameters to set as class variables in turn. Each
886 should be an iterable.
892 @classParametersProduct(foo=[1, 2], bar=[3, 4])
893 class MyTestCase(unittest.TestCase):
896 will generate four classes, as if you wrote::
898 class MyTestCase_1_3(unittest.TestCase):
903 class MyTestCase_1_4(unittest.TestCase):
908 class MyTestCase_2_3(unittest.TestCase):
913 class MyTestCase_2_4(unittest.TestCase):
918 Note that the values are embedded in the class name.
924 """Method decorator for unit tests
926 This decorator iterates over the cartesian product of the supplied settings,
927 using ``TestCase.subTest`` to communicate the values in the event of a
932 **settings : `dict` (`str`: iterable)
933 The parameter combinations to test. Each should be an iterable.
938 @methodParametersProduct(foo=[1, 2], bar=["black", "white"])
939 def testSomething(self, foo, bar):
944 testSomething(foo=1, bar="black")
945 testSomething(foo=1, bar="white")
946 testSomething(foo=2, bar="black")
947 testSomething(foo=2, bar="white")
952 @contextlib.contextmanager
954 """Context manager that creates and destroys a temporary directory.
956 The difference from `tempfile.TemporaryDirectory` is that this ignores
957 errors when deleting a directory, which may happen with some filesystems.
959 tmpdir = tempfile.mkdtemp()
961 shutil.rmtree(tmpdir, ignore_errors=
True)
def assertExecutable(self, executable, root_dir=None, args=None, msg=None)
def create_executable_tests(cls, ref_file, executables=None)
def _build_test_method(cls, executable, root_dir)
def testFileDescriptorLeaks(self)
daf::base::PropertySet * set
std::shared_ptr< FrameSet > append(FrameSet const &first, FrameSet const &second)
Construct a FrameSet that performs two transformations in series.
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def assertFloatsEqual(testCase, lhs, rhs, **kwargs)
def classParameters(**settings)
def assertFloatsAlmostEqual(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_info.epsilon, relTo=None, printFailures=True, plotOnFailure=False, plotFileName=None, invert=False, msg=None, ignoreNaNs=False)
def methodParameters(**settings)
def methodParametersProduct(**settings)
def debugger(*exceptions)
def assertFloatsNotEqual(testCase, lhs, rhs, **kwds)
def getTempFilePath(ext, expectOutput=True)
def plotImageDiff(lhs, rhs, bad=None, diff=None, plotFileName=None)
def suiteClassWrapper(tests)
def classParametersProduct(**settings)