Coverage for tests/testspec.py: 66%

1"""TestSpec testing framework.""" 

2 

3from collections.abc import Callable 

4from enum import Enum 

5import traceback 

6from typing import Any, NamedTuple, Optional, Sequence 

7import unittest 

8 

9 

10# Sentinel value used to indicate that no expected value is set 

11NO_EXPECTED_VALUE = object() 

12""" 

13A sentinel value used to indicate that no expected value is set. 

14""" 

15 

16 

17class TestSpec(NamedTuple): 

18 """A generic unit test specification class. 

19 

20 It allow tests to be specified declaratively while providing a large amount 

21 of flexibility. 

22 

23 Args: 

24 name (str): 

25 Identifying name for the test. 

26 action (Callable[..., Any]): 

27 A reference to a callable function or method to be invoked for the test. 

28 args (Sequence[Any], default = []): 

29 Sequence of positional arguments to be passed to the `action` function or method. 

30 kwargs (dict[str, Any], default = {}): 

31 Dictionary containing keyword arguments to be passed to the `action` function or method. 

32 expected (Any, default=NO_EXPECTED_VALUE ): 

33 Expected value (if any) that is expected to be returned by the `action` function or method. 

34 If there is no expected value, the special class NoExpectedValue is used to flag it. 

35 This is used so that the specific return value of None can be distinguished from no 

36 particular value or any value at all is expected to be returned from the function or method. 

37 obj: Optional[Any] = None 

38 validate_obj: Optional[Callable[[Any], bool]] = None 

39 validate_result: Optional[Callable[[Any], bool]] = None 

40 exception: Optional[type[Exception]] = None 

41 exception_tag: Optional[str] = None 

42 display_on_fail: Optional[Callable[[], str]] = None 

43 """ 

44 name: str 

45 action: Callable[..., Any] 

46 args: Optional[list[Any]] = None 

47 kwargs: Optional[dict[str, Any]] = None 

48 expected: Any = NO_EXPECTED_VALUE 

49 obj: Optional[Any] = None 

50 validate_obj: Optional[Callable[[Any], bool]] = None 

51 validate_result: Optional[Callable[[Any], bool]] = None 

52 exception: Optional[type[Exception]] = None 

53 exception_tag: Optional[str | Enum] = None 

54 display_on_fail: Optional[Callable[[], str]] = None 

55 

56 

57def run_tests_list(test_case: unittest.TestCase, test_specs: Sequence[TestSpec]) -> None: 

58 """Run a list of tests based on the provided TestSpec entries. 

59 

60 This function iterates over the list of TestSpec entries and runs each test using 

61 the `run_test` function. It allows for a clean and organized way to execute multiple tests. 

62 

63 Args: 

64 test_case (unittest.TestCase): The test case instance that will run the tests. 

65 test_specs (list[TestSpec]): A list of TestSpec entries, each representing a test to be run. 

66 """ 

67 for spec in test_specs: 

68 run_test(test_case, spec) 

69 

70 

71def run_test(test_case: unittest.TestCase, spec: TestSpec) -> None: # pylint: disable=too-many-branches 

72 """Run a single test based on the provided TestSpec entry. 

73 This function executes the action specified in the entry, checks the result against 

74 the expected value, and reports any errors. 

75 

76 Args: 

77 test_case (unittest.TestCase): The test case instance that will run the test. 

78 spec (TestSpec): The test configuration entry containing all necessary information for the test. 

79 """ 

80 with test_case.subTest(msg=spec.name): 

81 test_description: str = f"{spec.name}" 

82 errors: list[str] = [] 

83 try: 

84 # Use empty list/dict if the spec field is None 

85 pos_args = spec.args if spec.args is not None else [] 

86 kw_args = spec.kwargs if spec.kwargs is not None else {} 

87 found: Any = spec.action(*pos_args, **kw_args) 

88 if spec.exception: 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true

89 errors.append("returned result instead of raising exception") 

90 

91 else: 

92 if spec.validate_result and not spec.validate_result(found): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true

93 errors.append(f"failed result validation: found={found}") 

94 if spec.validate_obj and not spec.validate_obj(spec.obj): 94 ↛ 95line 94 didn't jump to line 95 because the condition on line 94 was never true

95 errors.append(f"failed object validation: obj={spec.obj}") 

96 if spec.expected is not NO_EXPECTED_VALUE and spec.expected != found: 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 errors.append(f"expected={spec.expected}, found={found}") 

98 if isinstance(spec.display_on_fail, Callable): 

99 errors.append(spec.display_on_fail()) 

100 except Exception as err: # pylint: disable=broad-exception-caught 

101 if spec.exception is None: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true

102 errors.append(f"Did not expect exception. Caught exception {repr(err)}") 

103 errors.append("stacktrace = ") 

104 errors.append("\n".join(traceback.format_tb(tb=err.__traceback__))) 

105 

106 elif not isinstance(err, spec.exception): 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true

107 errors.append( 

108 f"Unexpected exception type: expected={spec.exception}, " 

109 f"found = {type(err)}" 

110 ) 

111 elif spec.exception_tag: 

112 # Case 1: The expected tag is an Enum member. 

113 # This requires the exception object to have a 'tag_code' attribute. 

114 if isinstance(spec.exception_tag, Enum): 114 ↛ 126line 114 didn't jump to line 126 because the condition on line 114 was always true

115 if not hasattr(err, 'tag_code'): 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true

116 errors.append( 

117 "Exception is missing the 'tag_code' attribute required for Enum tag validation.") 

118 else: 

119 actual_tag = getattr(err, 'tag_code') 

120 if actual_tag != spec.exception_tag: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true

121 errors.append(f"Unexpected exception tag: expected={spec.exception_tag}, " 

122 f"found={actual_tag}") 

123 # Case 2: The expected tag is a string. 

124 # This performs a substring search in the exception's string representation. 

125 else: 

126 if str(spec.exception_tag) not in str(err): 

127 errors.append( 

128 f"Correct exception type, but tag '{spec.exception_tag}' " 

129 f"not found in exception message: {repr(err)}" 

130 ) 

131 if errors: 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true

132 test_case.fail(msg=test_description + ": " + "\n".join(errors))