23 """Support code for running unit tests""" 37 __all__ = [
"init",
"run",
"MemoryTestCase",
"ExecutablesTestCase",
"getTempFilePath",
38 "TestCase",
"assertFloatsAlmostEqual",
"assertFloatsNotEqual",
"assertFloatsEqual"]
61 def _get_open_files():
62 """Return a set containing the list of files currently open in this 68 Set containing the list of open files. 76 """Initialize the memory tester and file descriptor leak tester.""" 80 memId0 = dafBase.Citizen.getNextMemId()
82 open_files = _get_open_files()
85 def run(suite, exit=True):
86 """Run a test suite and report the test return status to caller or shell. 88 .. note:: Deprecated in 13_0 89 Use `unittest.main()` instead, which automatically detects 90 all tests in a test case and does not require a test suite. 94 suite : `unittest.TestSuite` 96 exit : `bool`, optional 97 If `True`, Python process will exit with the test exit status. 102 If ``exit`` is `False`, will return 0 if the tests passed, or 1 if 106 warnings.warn(
"lsst.utils.tests.run() is deprecated; please use unittest.main() instead",
107 DeprecationWarning, stacklevel=2)
109 if unittest.TextTestRunner().
run(suite).wasSuccessful():
121 """Sort supplied test suites such that MemoryTestCases are at the end. 123 `lsst.utils.tests.MemoryTestCase` tests should always run after any other 129 Sequence of test suites. 133 suite : `unittest.TestSuite` 134 A combined `~unittest.TestSuite` with 135 `~lsst.utils.tests.MemoryTestCase` at the end. 138 suite = unittest.TestSuite()
140 for test_suite
in tests:
147 for method
in test_suite:
148 bases = inspect.getmro(method.__class__)
150 if bases
is not None and MemoryTestCase
in bases:
151 memtests.append(test_suite)
153 suite.addTests(test_suite)
155 if isinstance(test_suite, MemoryTestCase):
156 memtests.append(test_suite)
158 suite.addTest(test_suite)
159 suite.addTests(memtests)
170 unittest.defaultTestLoader.suiteClass = suiteClassWrapper
174 """Check for memory leaks since memId0 was allocated""" 181 """Reset the leak counter when the tests have been completed""" 185 """Check for memory leaks in the preceding tests""" 188 global memId0, nleakPrintMax
189 nleak = dafBase.Citizen.census(0, memId0)
191 plural =
"s" if nleak != 1
else "" 192 print(
"\n%d Object%s leaked:" % (nleak, plural))
194 if nleak <= nleakPrintMax:
195 print(dafBase.Citizen.census(memId0))
197 census = dafBase.Citizen.census()
199 for i
in range(nleakPrintMax - 1, -1, -1):
200 print(census[i].repr())
202 self.fail(
"Leaked %d block%s" % (nleak, plural))
205 """Check if any file descriptors are open since init() called.""" 207 self.skipTest(
"Unable to test file descriptor leaks. psutil unavailable.")
210 now_open = _get_open_files()
213 now_open =
set(f
for f
in now_open
if not f.endswith(
".car")
and 214 not f.startswith(
"/proc/")
and 215 not f.endswith(
".ttf")
and 216 not (f.startswith(
"/var/lib/")
and f.endswith(
"/passwd"))
and 217 not f.endswith(
"astropy.log"))
219 diff = now_open.difference(open_files)
222 print(
"File open: %s" % f)
223 self.fail(
"Failed to close %d file%s" % (len(diff),
"s" if len(diff) != 1
else ""))
227 """Test that executables can be run and return good status. 229 The test methods are dynamically created. Callers 230 must subclass this class in their own test file and invoke 231 the create_executable_tests() class method to register the tests. 233 TESTS_DISCOVERED = -1
237 """Abort testing if automated test creation was enabled and 238 no tests were found.""" 241 raise Exception(
"No executables discovered.")
244 """This test exists to ensure that there is at least one test to be 245 executed. This allows the test runner to trigger the class set up 246 machinery to test whether there are some executables to test.""" 250 """Check an executable runs and returns good status. 252 Prints output to standard out. On bad exit status the test 253 fails. If the executable can not be located the test is skipped. 258 Path to an executable. ``root_dir`` is not used if this is an 260 root_dir : `str`, optional 261 Directory containing executable. Ignored if `None`. 262 args : `list` or `tuple`, optional 263 Arguments to be provided to the executable. 264 msg : `str`, optional 265 Message to use when the test fails. Can be `None` for default 271 The executable did not return 0 exit status. 274 if root_dir
is not None and not os.path.isabs(executable):
275 executable = os.path.join(root_dir, executable)
278 sp_args = [executable]
279 argstr =
"no arguments" 282 argstr =
'arguments "' +
" ".join(args) +
'"' 284 print(
"Running executable '{}' with {}...".
format(executable, argstr))
285 if not os.path.exists(executable):
286 self.skipTest(
"Executable {} is unexpectedly missing".
format(executable))
289 output = subprocess.check_output(sp_args)
290 except subprocess.CalledProcessError
as e:
292 failmsg =
"Bad exit status from '{}': {}".
format(executable, e.returncode)
293 print(output.decode(
'utf-8'))
300 def _build_test_method(cls, executable, root_dir):
301 """Build a test method and attach to class. 303 A test method is created for the supplied excutable located 304 in the supplied root directory. This method is attached to the class 305 so that the test runner will discover the test and run it. 310 The class in which to create the tests. 312 Name of executable. Can be absolute path. 314 Path to executable. Not used if executable path is absolute. 316 if not os.path.isabs(executable):
317 executable = os.path.abspath(os.path.join(root_dir, executable))
320 test_name =
"test_exe_" + executable.replace(
"/",
"_")
323 def test_executable_runs(*args):
325 self.assertExecutable(executable)
328 test_executable_runs.__name__ = test_name
329 setattr(cls, test_name, test_executable_runs)
333 """Discover executables to test and create corresponding test methods. 335 Scans the directory containing the supplied reference file 336 (usually ``__file__`` supplied from the test class) to look for 337 executables. If executables are found a test method is created 338 for each one. That test method will run the executable and 339 check the returned value. 341 Executable scripts with a ``.py`` extension and shared libraries 342 are ignored by the scanner. 344 This class method must be called before test discovery. 349 Path to a file within the directory to be searched. 350 If the files are in the same location as the test file, then 351 ``__file__`` can be used. 352 executables : `list` or `tuple`, optional 353 Sequence of executables that can override the automated 354 detection. If an executable mentioned here is not found, a 355 skipped test will be created for it, rather than a failed 360 >>> cls.create_executable_tests(__file__) 364 ref_dir = os.path.abspath(os.path.dirname(ref_file))
366 if executables
is None:
369 for root, dirs, files
in os.walk(ref_dir):
372 if not f.endswith(
".py")
and not f.endswith(
".so"):
373 full_path = os.path.join(root, f)
374 if os.access(full_path, os.X_OK):
375 executables.append(full_path)
384 for e
in executables:
388 @contextlib.contextmanager
390 """Return a path suitable for a temporary file and try to delete the 393 If the with block completes successfully then the file is deleted, 394 if possible; failure results in a printed warning. 395 If a file is remains when it should not, a RuntimeError exception is 396 raised. This exception is also raised if a file is not present on context 397 manager exit when one is expected to exist. 398 If the block exits with an exception the file if left on disk so it can be 399 examined. The file name has a random component such that nested context 400 managers can be used with the same file suffix. 406 File name extension, e.g. ``.fits``. 407 expectOutput : `bool`, optional 408 If `True`, a file should be created within the context manager. 409 If `False`, a file should not be present when the context manager 415 Path for a temporary file. The path is a combination of the caller's 416 file path and the name of the top-level function 422 # file tests/testFoo.py 424 import lsst.utils.tests 425 class FooTestCase(unittest.TestCase): 426 def testBasics(self): 430 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 431 # if tests/.tests exists then 432 # tmpFile = "tests/.tests/testFoo_testBasics.fits" 433 # otherwise tmpFile = "testFoo_testBasics.fits" 435 # at the end of this "with" block the path tmpFile will be 436 # deleted, but only if the file exists and the "with" 437 # block terminated normally (rather than with an exception) 440 stack = inspect.stack()
442 for i
in range(2, len(stack)):
443 frameInfo = inspect.getframeinfo(stack[i][0])
445 callerFilePath = frameInfo.filename
446 callerFuncName = frameInfo.function
447 elif callerFilePath == frameInfo.filename:
449 callerFuncName = frameInfo.function
453 callerDir, callerFileNameWithExt = os.path.split(callerFilePath)
454 callerFileName = os.path.splitext(callerFileNameWithExt)[0]
455 outDir = os.path.join(callerDir,
".tests")
456 if not os.path.isdir(outDir):
458 prefix =
"%s_%s-" % (callerFileName, callerFuncName)
459 outPath = tempfile.mktemp(dir=outDir, suffix=ext, prefix=prefix)
460 if os.path.exists(outPath):
463 warnings.warn(
"Unexpectedly found pre-existing tempfile named %r" % (outPath,),
472 fileExists = os.path.exists(outPath)
475 raise RuntimeError(
"Temp file expected named {} but none found".
format(outPath))
478 raise RuntimeError(
"Unexpectedly discovered temp file named {}".
format(outPath))
485 warnings.warn(
"Warning: could not remove file %r: %s" % (outPath, e), stacklevel=3)
489 """Subclass of unittest.TestCase that adds some custom assertions for 495 """A decorator to add a free function to our custom TestCase class, while also 496 making it available as a free function. 498 setattr(TestCase, func.__name__, func)
504 """.. note:: Deprecated in 12_0""" 505 warnings.warn(
"assertRaisesLsstCpp is deprecated; please just use TestCase.assertRaises",
506 DeprecationWarning, stacklevel=2)
507 return testcase.assertRaises(excClass, callableObj, *args, **kwargs)
511 """Decorator to enter the debugger when there's an uncaught exception 513 To use, just slap a ``@debugger()`` on your function. 515 You may provide specific exception classes to catch as arguments to 516 the decorator function, e.g., 517 ``@debugger(RuntimeError, NotImplementedError)``. 518 This defaults to just `AssertionError`, for use on `unittest.TestCase` 521 Code provided by "Rosh Oxymoron" on StackOverflow: 522 http://stackoverflow.com/questions/4398967/python-unit-testing-automatically-running-the-debugger-when-a-test-fails 526 Consider using ``pytest --pdb`` instead of this decorator. 529 exceptions = (AssertionError, )
533 def wrapper(*args, **kwargs):
535 return f(*args, **kwargs)
539 pdb.post_mortem(sys.exc_info()[2])
545 """Plot the comparison of two 2-d NumPy arrays. 549 lhs : `numpy.ndarray` 550 LHS values to compare; a 2-d NumPy array 551 rhs : `numpy.ndarray` 552 RHS values to compare; a 2-d NumPy array 553 bad : `numpy.ndarray` 554 A 2-d boolean NumPy array of values to emphasize in the plots 555 diff : `numpy.ndarray` 556 difference array; a 2-d NumPy array, or None to show lhs-rhs 558 Filename to save the plot to. If None, the plot will be displayed in 563 This method uses `matplotlib` and imports it internally; it should be 564 wrapped in a try/except block within packages that do not depend on 565 `matplotlib` (including `~lsst.utils`). 567 from matplotlib
import pyplot
573 badImage = numpy.zeros(bad.shape + (4,), dtype=numpy.uint8)
574 badImage[:, :, 0] = 255
575 badImage[:, :, 1] = 0
576 badImage[:, :, 2] = 0
577 badImage[:, :, 3] = 255*bad
578 vmin1 = numpy.minimum(numpy.min(lhs), numpy.min(rhs))
579 vmax1 = numpy.maximum(numpy.max(lhs), numpy.max(rhs))
580 vmin2 = numpy.min(diff)
581 vmax2 = numpy.max(diff)
582 for n, (image, title)
in enumerate([(lhs,
"lhs"), (rhs,
"rhs"), (diff,
"diff")]):
583 pyplot.subplot(2, 3, n + 1)
584 im1 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
585 vmin=vmin1, vmax=vmax1)
587 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
590 pyplot.subplot(2, 3, n + 4)
591 im2 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
592 vmin=vmin2, vmax=vmax2)
594 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
597 pyplot.subplots_adjust(left=0.05, bottom=0.05, top=0.92, right=0.75, wspace=0.05, hspace=0.05)
598 cax1 = pyplot.axes([0.8, 0.55, 0.05, 0.4])
599 pyplot.colorbar(im1, cax=cax1)
600 cax2 = pyplot.axes([0.8, 0.05, 0.05, 0.4])
601 pyplot.colorbar(im2, cax=cax2)
603 pyplot.savefig(plotFileName)
610 atol=sys.float_info.epsilon, relTo=None,
611 printFailures=True, plotOnFailure=False,
612 plotFileName=None, invert=False, msg=None):
613 """Highly-configurable floating point comparisons for scalars and arrays. 615 The test assertion will fail if all elements ``lhs`` and ``rhs`` are not 616 equal to within the tolerances specified by ``rtol`` and ``atol``. 617 More precisely, the comparison is: 619 ``abs(lhs - rhs) <= relTo*rtol OR abs(lhs - rhs) <= atol`` 621 If ``rtol`` or ``atol`` is `None`, that term in the comparison is not 624 When not specified, ``relTo`` is the elementwise maximum of the absolute 625 values of ``lhs`` and ``rhs``. If set manually, it should usually be set 626 to either ``lhs`` or ``rhs``, or a scalar value typical of what is 631 testCase : `unittest.TestCase` 632 Instance the test is part of. 633 lhs : scalar or array-like 634 LHS value(s) to compare; may be a scalar or array-like of any 636 rhs : scalar or array-like 637 RHS value(s) to compare; may be a scalar or array-like of any 639 rtol : `float`, optional 640 Relative tolerance for comparison; defaults to double-precision 642 atol : `float`, optional 643 Absolute tolerance for comparison; defaults to double-precision 645 relTo : `float`, optional 646 Value to which comparison with rtol is relative. 647 printFailures : `bool`, optional 648 Upon failure, print all inequal elements as part of the message. 649 plotOnFailure : `bool`, optional 650 Upon failure, plot the originals and their residual with matplotlib. 651 Only 2-d arrays are supported. 652 plotFileName : `str`, optional 653 Filename to save the plot to. If `None`, the plot will be displayed in 655 invert : `bool`, optional 656 If `True`, invert the comparison and fail only if any elements *are* 657 equal. Used to implement `~lsst.utils.tests.assertFloatsNotEqual`, 658 which should generally be used instead for clarity. 659 msg : `str`, optional 660 String to append to the error message when assert fails. 665 The values are not almost equal. 667 if not numpy.isfinite(lhs).
all():
668 testCase.fail(
"Non-finite values in lhs")
669 if not numpy.isfinite(rhs).
all():
670 testCase.fail(
"Non-finite values in rhs")
672 absDiff = numpy.abs(lhs - rhs)
675 relTo = numpy.maximum(numpy.abs(lhs), numpy.abs(rhs))
677 relTo = numpy.abs(relTo)
678 bad = absDiff > rtol*relTo
680 bad = numpy.logical_and(bad, absDiff > atol)
683 raise ValueError(
"rtol and atol cannot both be None")
685 failed = numpy.any(bad)
688 bad = numpy.logical_not(bad)
690 failStr =
"are the same" 696 if numpy.isscalar(bad):
698 errMsg = [
"%s %s %s; diff=%s with atol=%s" 699 % (lhs, cmpStr, rhs, absDiff, atol)]
701 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s" 702 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol)]
704 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s, atol=%s" 705 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol, atol)]
707 errMsg = [
"%d/%d elements %s with rtol=%s, atol=%s" 708 % (bad.sum(), bad.size, failStr, rtol, atol)]
710 if len(lhs.shape) != 2
or len(rhs.shape) != 2:
711 raise ValueError(
"plotOnFailure is only valid for 2-d arrays")
713 plotImageDiff(lhs, rhs, bad, diff=diff, plotFileName=plotFileName)
715 errMsg.append(
"Failure plot requested but matplotlib could not be imported.")
720 if numpy.isscalar(relTo):
721 relTo = numpy.ones(bad.shape, dtype=float) * relTo
722 if numpy.isscalar(lhs):
723 lhs = numpy.ones(bad.shape, dtype=float) * lhs
724 if numpy.isscalar(rhs):
725 rhs = numpy.ones(bad.shape, dtype=float) * rhs
727 for a, b, diff
in zip(lhs[bad], rhs[bad], absDiff[bad]):
728 errMsg.append(
"%s %s %s (diff=%s)" % (a, cmpStr, b, diff))
730 for a, b, diff, rel
in zip(lhs[bad], rhs[bad], absDiff[bad], relTo[bad]):
731 errMsg.append(
"%s %s %s (diff=%s/%s=%s)" % (a, cmpStr, b, diff, rel, diff/rel))
735 testCase.assertFalse(failed, msg=
"\n".join(errMsg))
740 """Fail a test if the given floating point values are equal to within the 743 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with 744 ``rtol=atol=0``) for more information. 748 testCase : `unittest.TestCase` 749 Instance the test is part of. 750 lhs : scalar or array-like 751 LHS value(s) to compare; may be a scalar or array-like of any 753 rhs : scalar or array-like 754 RHS value(s) to compare; may be a scalar or array-like of any 760 The values are almost equal. 768 Assert that lhs == rhs (both numeric types, whether scalar or array). 770 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with 771 ``rtol=atol=0``) for more information. 775 testCase : `unittest.TestCase` 776 Instance the test is part of. 777 lhs : scalar or array-like 778 LHS value(s) to compare; may be a scalar or array-like of any 780 rhs : scalar or array-like 781 RHS value(s) to compare; may be a scalar or array-like of any 787 The values are not equal. 794 """.. note:: Deprecated in 12_0""" 795 warnings.warn(
"assertClose is deprecated; please use TestCase.assertFloatsAlmostEqual",
796 DeprecationWarning, stacklevel=2)
802 """.. note:: Deprecated in 12_0""" 803 warnings.warn(
"assertNotClose is deprecated; please use TestCase.assertFloatsNotEqual",
804 DeprecationWarning, stacklevel=2)
def suiteClassWrapper(tests)
def assertExecutable(self, executable, root_dir=None, args=None, msg=None)
def assertFloatsEqual(testCase, lhs, rhs, kwargs)
def plotImageDiff(lhs, rhs, bad=None, diff=None, plotFileName=None)
def assertClose(args, kwargs)
daf::base::PropertySet * set
bool all(CoordinateExpr< N > const &expr) noexcept
Return true if all elements are true.
def _build_test_method(cls, executable, root_dir)
def assertFloatsAlmostEqual(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_info.epsilon, relTo=None, printFailures=True, plotOnFailure=False, plotFileName=None, invert=False, msg=None)
def format(config, name=None, writeSourceLine=True, prefix="", verbose=False)
def assertFloatsNotEqual(testCase, lhs, rhs, kwds)
def run(suite, exit=True)
def getTempFilePath(ext, expectOutput=True)
def assertNotClose(args, kwargs)
def create_executable_tests(cls, ref_file, executables=None)
def testFileDescriptorLeaks(self)
def assertRaisesLsstCpp(testcase, excClass, callableObj, args, kwargs)