Maxim nexpaq / nexpaq_dev

Preliminary main mbed library for nexpaq development

features/frameworks/utest/README.md@0:6c56fb4bc5f0, 2016-11-04 (annotated)

Committer:
nexpaq
Date:
Fri Nov 04 20:27:58 2016 +0000
Revision:
0:6c56fb4bc5f0
Moving to library for sharing updates

Who changed what in which revision?

UserRevisionLine numberNew contents of line
nexpaq 0:6c56fb4bc5f0 1 # utest: Asynchronous C++ Test Harness
nexpaq 0:6c56fb4bc5f0 2
nexpaq 0:6c56fb4bc5f0 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
nexpaq 0:6c56fb4bc5f0 4
nexpaq 0:6c56fb4bc5f0 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
nexpaq 0:6c56fb4bc5f0 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
nexpaq 0:6c56fb4bc5f0 7
nexpaq 0:6c56fb4bc5f0 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
nexpaq 0:6c56fb4bc5f0 9
nexpaq 0:6c56fb4bc5f0 10 ## Theory of Operation
nexpaq 0:6c56fb4bc5f0 11
nexpaq 0:6c56fb4bc5f0 12 A test specification contains a setup handler, several test cases and a teardown handler.
nexpaq 0:6c56fb4bc5f0 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
nexpaq 0:6c56fb4bc5f0 14
nexpaq 0:6c56fb4bc5f0 15 The order of handler execution is:
nexpaq 0:6c56fb4bc5f0 16
nexpaq 0:6c56fb4bc5f0 17 1. Test setup handler.
nexpaq 0:6c56fb4bc5f0 18 1. For each test case:
nexpaq 0:6c56fb4bc5f0 19 1. Test case setup handler.
nexpaq 0:6c56fb4bc5f0 20 1. Test case execution handler.
nexpaq 0:6c56fb4bc5f0 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
nexpaq 0:6c56fb4bc5f0 22 1. (repeat test case execution handler if specified.)
nexpaq 0:6c56fb4bc5f0 23 1. Test case teardown handler.
nexpaq 0:6c56fb4bc5f0 24 1. Test teardown handler.
nexpaq 0:6c56fb4bc5f0 25
nexpaq 0:6c56fb4bc5f0 26 ## Example
nexpaq 0:6c56fb4bc5f0 27
nexpaq 0:6c56fb4bc5f0 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
nexpaq 0:6c56fb4bc5f0 29
nexpaq 0:6c56fb4bc5f0 30 ```cpp
nexpaq 0:6c56fb4bc5f0 31 #include "mbed-drivers/test_env.h"
nexpaq 0:6c56fb4bc5f0 32 #include "utest/utest.h"
nexpaq 0:6c56fb4bc5f0 33 #include "unity/unity.h"
nexpaq 0:6c56fb4bc5f0 34
nexpaq 0:6c56fb4bc5f0 35 using namespace utest::v1;
nexpaq 0:6c56fb4bc5f0 36
nexpaq 0:6c56fb4bc5f0 37 void test_simple() {
nexpaq 0:6c56fb4bc5f0 38 TEST_ASSERT_EQUAL(0, 0);
nexpaq 0:6c56fb4bc5f0 39 printf("Simple test called\n");
nexpaq 0:6c56fb4bc5f0 40 }
nexpaq 0:6c56fb4bc5f0 41
nexpaq 0:6c56fb4bc5f0 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
nexpaq 0:6c56fb4bc5f0 43 // Call the default handler for proper reporting
nexpaq 0:6c56fb4bc5f0 44 status_t status = greentea_case_setup_handler(source, index_of_case);
nexpaq 0:6c56fb4bc5f0 45 printf("Setting up for '%s'\n", source->get_description());
nexpaq 0:6c56fb4bc5f0 46 return status;
nexpaq 0:6c56fb4bc5f0 47 }
nexpaq 0:6c56fb4bc5f0 48 control_t test_repeats(const size_t call_count) {
nexpaq 0:6c56fb4bc5f0 49 printf("Called for the %u. time\n", call_count);
nexpaq 0:6c56fb4bc5f0 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
nexpaq 0:6c56fb4bc5f0 51 // Specify how often this test is repeated ie. n total calls
nexpaq 0:6c56fb4bc5f0 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
nexpaq 0:6c56fb4bc5f0 53 }
nexpaq 0:6c56fb4bc5f0 54
nexpaq 0:6c56fb4bc5f0 55 void test_callback_validate() {
nexpaq 0:6c56fb4bc5f0 56 // You may also use assertions here!
nexpaq 0:6c56fb4bc5f0 57 TEST_ASSERT_EQUAL_PTR(0, 0);
nexpaq 0:6c56fb4bc5f0 58 // Validate the callback
nexpaq 0:6c56fb4bc5f0 59 Harness::validate_callback();
nexpaq 0:6c56fb4bc5f0 60 }
nexpaq 0:6c56fb4bc5f0 61 control_t test_asynchronous() {
nexpaq 0:6c56fb4bc5f0 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
nexpaq 0:6c56fb4bc5f0 63 // Set up a callback in the future. This may also be an interrupt!
nexpaq 0:6c56fb4bc5f0 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
nexpaq 0:6c56fb4bc5f0 65 // Set a 200ms timeout starting from now
nexpaq 0:6c56fb4bc5f0 66 return CaseTimeout(200);
nexpaq 0:6c56fb4bc5f0 67 }
nexpaq 0:6c56fb4bc5f0 68
nexpaq 0:6c56fb4bc5f0 69 control_t test_asynchronous_timeout(const size_t call_count) {
nexpaq 0:6c56fb4bc5f0 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
nexpaq 0:6c56fb4bc5f0 71 // Set a 200ms timeout starting from now,
nexpaq 0:6c56fb4bc5f0 72 // but automatically repeat only this handler on timeout.
nexpaq 0:6c56fb4bc5f0 73 if (call_count >= 5) {
nexpaq 0:6c56fb4bc5f0 74 // but after the 5th call, the callback finally gets validated
nexpaq 0:6c56fb4bc5f0 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
nexpaq 0:6c56fb4bc5f0 76 }
nexpaq 0:6c56fb4bc5f0 77 return CaseRepeatHandlerOnTimeout(200);
nexpaq 0:6c56fb4bc5f0 78 }
nexpaq 0:6c56fb4bc5f0 79
nexpaq 0:6c56fb4bc5f0 80 // Custom setup handler required for proper Greentea support
nexpaq 0:6c56fb4bc5f0 81 status_t greentea_setup(const size_t number_of_cases) {
nexpaq 0:6c56fb4bc5f0 82 GREENTEA_SETUP(20, "default_auto");
nexpaq 0:6c56fb4bc5f0 83 // Call the default reporting function
nexpaq 0:6c56fb4bc5f0 84 return greentea_test_setup_handler(number_of_cases);
nexpaq 0:6c56fb4bc5f0 85 }
nexpaq 0:6c56fb4bc5f0 86
nexpaq 0:6c56fb4bc5f0 87 // Specify all your test cases here
nexpaq 0:6c56fb4bc5f0 88 Case cases[] = {
nexpaq 0:6c56fb4bc5f0 89 Case("Simple Test", test_simple),
nexpaq 0:6c56fb4bc5f0 90 Case("Repeating Test", test_repeats_setup, test_repeats),
nexpaq 0:6c56fb4bc5f0 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
nexpaq 0:6c56fb4bc5f0 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
nexpaq 0:6c56fb4bc5f0 93 };
nexpaq 0:6c56fb4bc5f0 94
nexpaq 0:6c56fb4bc5f0 95 // Declare your test specification with a custom setup handler
nexpaq 0:6c56fb4bc5f0 96 Specification specification(greentea_setup, cases);
nexpaq 0:6c56fb4bc5f0 97
nexpaq 0:6c56fb4bc5f0 98 void app_start(int, char**)
nexpaq 0:6c56fb4bc5f0 99 { // Run the test specification
nexpaq 0:6c56fb4bc5f0 100 Harness::run(specification);
nexpaq 0:6c56fb4bc5f0 101 }
nexpaq 0:6c56fb4bc5f0 102 ```
nexpaq 0:6c56fb4bc5f0 103
nexpaq 0:6c56fb4bc5f0 104 Running this test will output the following:
nexpaq 0:6c56fb4bc5f0 105
nexpaq 0:6c56fb4bc5f0 106 ```
nexpaq 0:6c56fb4bc5f0 107 {{timeout;20}}
nexpaq 0:6c56fb4bc5f0 108 {{host_test_name;default_auto}}
nexpaq 0:6c56fb4bc5f0 109 {{description;utest greentea example}}
nexpaq 0:6c56fb4bc5f0 110 {{test_id;MBED_OS}}
nexpaq 0:6c56fb4bc5f0 111 {{start}}
nexpaq 0:6c56fb4bc5f0 112 >>> Running 4 test cases...
nexpaq 0:6c56fb4bc5f0 113
nexpaq 0:6c56fb4bc5f0 114 >>> Running case #1: 'Simple Test'...
nexpaq 0:6c56fb4bc5f0 115 Simple test called
nexpaq 0:6c56fb4bc5f0 116 >>> 'Simple Test': 1 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 117
nexpaq 0:6c56fb4bc5f0 118 >>> Running case #2: 'Repeating Test'...
nexpaq 0:6c56fb4bc5f0 119 Setting up for 'Repeating Test'
nexpaq 0:6c56fb4bc5f0 120 Called for the 1. time
nexpaq 0:6c56fb4bc5f0 121 >>> 'Repeating Test': 1 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 122
nexpaq 0:6c56fb4bc5f0 123 >>> Running case #2: 'Repeating Test'...
nexpaq 0:6c56fb4bc5f0 124 Setting up for 'Repeating Test'
nexpaq 0:6c56fb4bc5f0 125 Called for the 2. time
nexpaq 0:6c56fb4bc5f0 126 >>> 'Repeating Test': 2 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 127
nexpaq 0:6c56fb4bc5f0 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
nexpaq 0:6c56fb4bc5f0 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 130
nexpaq 0:6c56fb4bc5f0 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
nexpaq 0:6c56fb4bc5f0 132 >>> failure with reason 'Ignored: Timed Out'
nexpaq 0:6c56fb4bc5f0 133 >>> failure with reason 'Ignored: Timed Out'
nexpaq 0:6c56fb4bc5f0 134 >>> failure with reason 'Ignored: Timed Out'
nexpaq 0:6c56fb4bc5f0 135 >>> failure with reason 'Ignored: Timed Out'
nexpaq 0:6c56fb4bc5f0 136 >>> failure with reason 'Ignored: Timed Out'
nexpaq 0:6c56fb4bc5f0 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 138
nexpaq 0:6c56fb4bc5f0 139 >>> Test cases: 4 passed, 0 failed
nexpaq 0:6c56fb4bc5f0 140 {{success}}
nexpaq 0:6c56fb4bc5f0 141 {{end}}
nexpaq 0:6c56fb4bc5f0 142 ```
nexpaq 0:6c56fb4bc5f0 143
nexpaq 0:6c56fb4bc5f0 144 ## Detailed Description
nexpaq 0:6c56fb4bc5f0 145
nexpaq 0:6c56fb4bc5f0 146 ### Handlers
nexpaq 0:6c56fb4bc5f0 147
nexpaq 0:6c56fb4bc5f0 148 There are five handler types you can, but do not have to, override to customize operation.
nexpaq 0:6c56fb4bc5f0 149 Please see the `utest/types.h` file for a detailed description.
nexpaq 0:6c56fb4bc5f0 150
nexpaq 0:6c56fb4bc5f0 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
nexpaq 0:6c56fb4bc5f0 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
nexpaq 0:6c56fb4bc5f0 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
nexpaq 0:6c56fb4bc5f0 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
nexpaq 0:6c56fb4bc5f0 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
nexpaq 0:6c56fb4bc5f0 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
nexpaq 0:6c56fb4bc5f0 157
nexpaq 0:6c56fb4bc5f0 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
nexpaq 0:6c56fb4bc5f0 159
nexpaq 0:6c56fb4bc5f0 160 ### Test Case Handlers
nexpaq 0:6c56fb4bc5f0 161
nexpaq 0:6c56fb4bc5f0 162 There are three test case handlers:
nexpaq 0:6c56fb4bc5f0 163
nexpaq 0:6c56fb4bc5f0 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
nexpaq 0:6c56fb4bc5f0 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
nexpaq 0:6c56fb4bc5f0 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
nexpaq 0:6c56fb4bc5f0 167
nexpaq 0:6c56fb4bc5f0 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
nexpaq 0:6c56fb4bc5f0 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
nexpaq 0:6c56fb4bc5f0 170
nexpaq 0:6c56fb4bc5f0 171 1. Description (required).
nexpaq 0:6c56fb4bc5f0 172 1. Setup handler (optional).
nexpaq 0:6c56fb4bc5f0 173 1. Test case handler (required).
nexpaq 0:6c56fb4bc5f0 174 1. Teardown handler (optional).
nexpaq 0:6c56fb4bc5f0 175 1. Failure handler (optional).
nexpaq 0:6c56fb4bc5f0 176
nexpaq 0:6c56fb4bc5f0 177 #### Test Case Attributes
nexpaq 0:6c56fb4bc5f0 178
nexpaq 0:6c56fb4bc5f0 179 You can modify test case behavior by returning `control_t` modifiers:
nexpaq 0:6c56fb4bc5f0 180
nexpaq 0:6c56fb4bc5f0 181 - `CaseNext`: never repeats and immediately moves to next test case
nexpaq 0:6c56fb4bc5f0 182 - `CaseNoRepeat`: never repeats.
nexpaq 0:6c56fb4bc5f0 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
nexpaq 0:6c56fb4bc5f0 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
nexpaq 0:6c56fb4bc5f0 185 - `CaseNoTimeout`: immediately moves to next test case.
nexpaq 0:6c56fb4bc5f0 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
nexpaq 0:6c56fb4bc5f0 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
nexpaq 0:6c56fb4bc5f0 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
nexpaq 0:6c56fb4bc5f0 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
nexpaq 0:6c56fb4bc5f0 190
nexpaq 0:6c56fb4bc5f0 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
nexpaq 0:6c56fb4bc5f0 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
nexpaq 0:6c56fb4bc5f0 193
nexpaq 0:6c56fb4bc5f0 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
nexpaq 0:6c56fb4bc5f0 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
nexpaq 0:6c56fb4bc5f0 196
nexpaq 0:6c56fb4bc5f0 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
nexpaq 0:6c56fb4bc5f0 198 This will schedule the execution of the next test case.
nexpaq 0:6c56fb4bc5f0 199
nexpaq 0:6c56fb4bc5f0 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
nexpaq 0:6c56fb4bc5f0 201
nexpaq 0:6c56fb4bc5f0 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
nexpaq 0:6c56fb4bc5f0 203
nexpaq 0:6c56fb4bc5f0 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
nexpaq 0:6c56fb4bc5f0 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
nexpaq 0:6c56fb4bc5f0 206
nexpaq 0:6c56fb4bc5f0 207 ### Failure Handlers
nexpaq 0:6c56fb4bc5f0 208
nexpaq 0:6c56fb4bc5f0 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
nexpaq 0:6c56fb4bc5f0 210
nexpaq 0:6c56fb4bc5f0 211 The failure reasons are:
nexpaq 0:6c56fb4bc5f0 212
nexpaq 0:6c56fb4bc5f0 213 - `REASON_NONE`: No failure occurred
nexpaq 0:6c56fb4bc5f0 214 - `REASON_UNKNOWN`: An unknown failure occurred
nexpaq 0:6c56fb4bc5f0 215 - `REASON_CASES`: A failure occurred in at least one test case
nexpaq 0:6c56fb4bc5f0 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
nexpaq 0:6c56fb4bc5f0 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
nexpaq 0:6c56fb4bc5f0 218 - `REASON_ASSERTION`: An assertion failed
nexpaq 0:6c56fb4bc5f0 219 - `REASON_TEST_SETUP`: Test setup failed
nexpaq 0:6c56fb4bc5f0 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
nexpaq 0:6c56fb4bc5f0 221 - `REASON_CASE_SETUP`: Case setup failed
nexpaq 0:6c56fb4bc5f0 222 - `REASON_CASE_HANDLER`: Case handler failed
nexpaq 0:6c56fb4bc5f0 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
nexpaq 0:6c56fb4bc5f0 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
nexpaq 0:6c56fb4bc5f0 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
nexpaq 0:6c56fb4bc5f0 226
nexpaq 0:6c56fb4bc5f0 227 The failure locations are:
nexpaq 0:6c56fb4bc5f0 228
nexpaq 0:6c56fb4bc5f0 229 - `LOCATION_NONE`: No location information
nexpaq 0:6c56fb4bc5f0 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
nexpaq 0:6c56fb4bc5f0 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
nexpaq 0:6c56fb4bc5f0 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
nexpaq 0:6c56fb4bc5f0 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
nexpaq 0:6c56fb4bc5f0 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
nexpaq 0:6c56fb4bc5f0 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
nexpaq 0:6c56fb4bc5f0 236
nexpaq 0:6c56fb4bc5f0 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
nexpaq 0:6c56fb4bc5f0 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
nexpaq 0:6c56fb4bc5f0 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
nexpaq 0:6c56fb4bc5f0 240
nexpaq 0:6c56fb4bc5f0 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
nexpaq 0:6c56fb4bc5f0 242
nexpaq 0:6c56fb4bc5f0 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
nexpaq 0:6c56fb4bc5f0 244
nexpaq 0:6c56fb4bc5f0 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
nexpaq 0:6c56fb4bc5f0 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
nexpaq 0:6c56fb4bc5f0 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
nexpaq 0:6c56fb4bc5f0 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
nexpaq 0:6c56fb4bc5f0 249
nexpaq 0:6c56fb4bc5f0 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
nexpaq 0:6c56fb4bc5f0 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
nexpaq 0:6c56fb4bc5f0 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
nexpaq 0:6c56fb4bc5f0 253
nexpaq 0:6c56fb4bc5f0 254 ### Default Handlers
nexpaq 0:6c56fb4bc5f0 255
nexpaq 0:6c56fb4bc5f0 256 Three sets of default handlers with different behaviors are provided for your convenience:
nexpaq 0:6c56fb4bc5f0 257
nexpaq 0:6c56fb4bc5f0 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
nexpaq 0:6c56fb4bc5f0 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
nexpaq 0:6c56fb4bc5f0 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
nexpaq 0:6c56fb4bc5f0 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
nexpaq 0:6c56fb4bc5f0 262
nexpaq 0:6c56fb4bc5f0 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
nexpaq 0:6c56fb4bc5f0 264
nexpaq 0:6c56fb4bc5f0 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
nexpaq 0:6c56fb4bc5f0 266
nexpaq 0:6c56fb4bc5f0 267 ```cpp
nexpaq 0:6c56fb4bc5f0 268 // Declare your test specification with a custom setup handler
nexpaq 0:6c56fb4bc5f0 269 // and set the default handlers to the predefined “greentea continue” behavior
nexpaq 0:6c56fb4bc5f0 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
nexpaq 0:6c56fb4bc5f0 271 ```
nexpaq 0:6c56fb4bc5f0 272
nexpaq 0:6c56fb4bc5f0 273 ### Custom Handlers
nexpaq 0:6c56fb4bc5f0 274
nexpaq 0:6c56fb4bc5f0 275 You may override any of the default handlers with your own custom handler.
nexpaq 0:6c56fb4bc5f0 276
nexpaq 0:6c56fb4bc5f0 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
nexpaq 0:6c56fb4bc5f0 278 To explicitly invoke the default handler, use the `default_handler` hint.
nexpaq 0:6c56fb4bc5f0 279
nexpaq 0:6c56fb4bc5f0 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
nexpaq 0:6c56fb4bc5f0 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
nexpaq 0:6c56fb4bc5f0 282
nexpaq 0:6c56fb4bc5f0 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
nexpaq 0:6c56fb4bc5f0 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
nexpaq 0:6c56fb4bc5f0 285
nexpaq 0:6c56fb4bc5f0 286 For the `Specification` class the order of arguments is:
nexpaq 0:6c56fb4bc5f0 287
nexpaq 0:6c56fb4bc5f0 288 1. Test setup handler (optional).
nexpaq 0:6c56fb4bc5f0 289 1. Array of test cases (required).
nexpaq 0:6c56fb4bc5f0 290 1. Test teardown handler (optional).
nexpaq 0:6c56fb4bc5f0 291 1. Test failure handler (optional).
nexpaq 0:6c56fb4bc5f0 292 1. Default handlers (optional).
nexpaq 0:6c56fb4bc5f0 293
nexpaq 0:6c56fb4bc5f0 294 ### Test Case Attribute Arbitration
nexpaq 0:6c56fb4bc5f0 295
nexpaq 0:6c56fb4bc5f0 296 When adding conflicting modifiers together
nexpaq 0:6c56fb4bc5f0 297
nexpaq 0:6c56fb4bc5f0 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
nexpaq 0:6c56fb4bc5f0 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
nexpaq 0:6c56fb4bc5f0 300 - `CaseNext` always wins arbitration.
nexpaq 0:6c56fb4bc5f0 301
nexpaq 0:6c56fb4bc5f0 302 The following table shows this arbitration logic in detail:
nexpaq 0:6c56fb4bc5f0 303
nexpaq 0:6c56fb4bc5f0 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
nexpaq 0:6c56fb4bc5f0 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
nexpaq 0:6c56fb4bc5f0 306 | `CaseNext` | no repeat &<br> no timeout
nexpaq 0:6c56fb4bc5f0 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
nexpaq 0:6c56fb4bc5f0 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
nexpaq 0:6c56fb4bc5f0 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
nexpaq 0:6c56fb4bc5f0 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
nexpaq 0:6c56fb4bc5f0 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
nexpaq 0:6c56fb4bc5f0 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
nexpaq 0:6c56fb4bc5f0 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
nexpaq 0:6c56fb4bc5f0 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
nexpaq 0:6c56fb4bc5f0 315
nexpaq 0:6c56fb4bc5f0 316 ### Atomicity
nexpaq 0:6c56fb4bc5f0 317
nexpaq 0:6c56fb4bc5f0 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
nexpaq 0:6c56fb4bc5f0 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
nexpaq 0:6c56fb4bc5f0 320
nexpaq 0:6c56fb4bc5f0 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
nexpaq 0:6c56fb4bc5f0 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
nexpaq 0:6c56fb4bc5f0 323
nexpaq 0:6c56fb4bc5f0 324 ### Custom Scheduler
nexpaq 0:6c56fb4bc5f0 325
nexpaq 0:6c56fb4bc5f0 326 By default, a Timeout object is used for scheduling the harness operations.
nexpaq 0:6c56fb4bc5f0 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
nexpaq 0:6c56fb4bc5f0 328
nexpaq 0:6c56fb4bc5f0 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
nexpaq 0:6c56fb4bc5f0 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
nexpaq 0:6c56fb4bc5f0 331
nexpaq 0:6c56fb4bc5f0 332 There are two functions you need to implement:
nexpaq 0:6c56fb4bc5f0 333
nexpaq 0:6c56fb4bc5f0 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
nexpaq 0:6c56fb4bc5f0 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
nexpaq 0:6c56fb4bc5f0 336
nexpaq 0:6c56fb4bc5f0 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
nexpaq 0:6c56fb4bc5f0 338
nexpaq 0:6c56fb4bc5f0 339 ### Example Synchronous Scheduler
nexpaq 0:6c56fb4bc5f0 340
nexpaq 0:6c56fb4bc5f0 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
nexpaq 0:6c56fb4bc5f0 342 ```cpp
nexpaq 0:6c56fb4bc5f0 343 volatile utest_v1_harness_callback_t minimal_callback;
nexpaq 0:6c56fb4bc5f0 344
nexpaq 0:6c56fb4bc5f0 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
nexpaq 0:6c56fb4bc5f0 346 minimal_callback = callback;
nexpaq 0:6c56fb4bc5f0 347 // this scheduler does not support asynchronous callbacks
nexpaq 0:6c56fb4bc5f0 348 return (delay_ms ? NULL : (void*)1);
nexpaq 0:6c56fb4bc5f0 349 }
nexpaq 0:6c56fb4bc5f0 350 static int32_t utest_minimal_cancel(void*) {
nexpaq 0:6c56fb4bc5f0 351 return -1; // canceling not supported either
nexpaq 0:6c56fb4bc5f0 352 }
nexpaq 0:6c56fb4bc5f0 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
nexpaq 0:6c56fb4bc5f0 354
nexpaq 0:6c56fb4bc5f0 355 // [...] Add your test cases and specification here.
nexpaq 0:6c56fb4bc5f0 356
nexpaq 0:6c56fb4bc5f0 357 void main() // or whatever your custom entry point is
nexpaq 0:6c56fb4bc5f0 358 {
nexpaq 0:6c56fb4bc5f0 359 // You MUST set the custom scheduler before running the specification.
nexpaq 0:6c56fb4bc5f0 360 Harness::set_scheduler(utest_minimal_scheduler);
nexpaq 0:6c56fb4bc5f0 361 Harness::run(specification);
nexpaq 0:6c56fb4bc5f0 362
nexpaq 0:6c56fb4bc5f0 363 while(1) {
nexpaq 0:6c56fb4bc5f0 364 if (minimal_callback) {
nexpaq 0:6c56fb4bc5f0 365 // copy the callback and reset the shared memory
nexpaq 0:6c56fb4bc5f0 366 utest_v1_harness_callback_t callback = minimal_callback;
nexpaq 0:6c56fb4bc5f0 367 minimal_callback = NULL;
nexpaq 0:6c56fb4bc5f0 368 callback(); // execute the copied callback
nexpaq 0:6c56fb4bc5f0 369 }
nexpaq 0:6c56fb4bc5f0 370 }
nexpaq 0:6c56fb4bc5f0 371 }
nexpaq 0:6c56fb4bc5f0 372 ```
nexpaq 0:6c56fb4bc5f0 373
nexpaq 0:6c56fb4bc5f0 374 ### Example Asynchronous Scheduler
nexpaq 0:6c56fb4bc5f0 375
nexpaq 0:6c56fb4bc5f0 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
nexpaq 0:6c56fb4bc5f0 377 ```cpp
nexpaq 0:6c56fb4bc5f0 378 volatile utest_v1_harness_callback_t minimal_callback;
nexpaq 0:6c56fb4bc5f0 379 volatile utest_v1_harness_callback_t ticker_callback;
nexpaq 0:6c56fb4bc5f0 380 const ticker_data_t *ticker_data;
nexpaq 0:6c56fb4bc5f0 381 ticker_event_t ticker_event;
nexpaq 0:6c56fb4bc5f0 382
nexpaq 0:6c56fb4bc5f0 383 static void ticker_handler(uint32_t) {
nexpaq 0:6c56fb4bc5f0 384 minimal_callback = ticker_callback; // interrupt context!
nexpaq 0:6c56fb4bc5f0 385 }
nexpaq 0:6c56fb4bc5f0 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
nexpaq 0:6c56fb4bc5f0 387 if (delay_ms) {
nexpaq 0:6c56fb4bc5f0 388 ticker_callback = callback;
nexpaq 0:6c56fb4bc5f0 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
nexpaq 0:6c56fb4bc5f0 390 }
nexpaq 0:6c56fb4bc5f0 391 else minimal_callback = callback;
nexpaq 0:6c56fb4bc5f0 392 return (void*)1;
nexpaq 0:6c56fb4bc5f0 393 }
nexpaq 0:6c56fb4bc5f0 394 static int32_t utest_minimal_cancel(void*) {
nexpaq 0:6c56fb4bc5f0 395 ticker_remove_event(ticker_data, &ticker_event);
nexpaq 0:6c56fb4bc5f0 396 return 0; // canceling is supported
nexpaq 0:6c56fb4bc5f0 397 }
nexpaq 0:6c56fb4bc5f0 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
nexpaq 0:6c56fb4bc5f0 399
nexpaq 0:6c56fb4bc5f0 400 // [...] Add your test cases and specification here.
nexpaq 0:6c56fb4bc5f0 401
nexpaq 0:6c56fb4bc5f0 402 void main() // or whatever your custom entry point is
nexpaq 0:6c56fb4bc5f0 403 {
nexpaq 0:6c56fb4bc5f0 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
nexpaq 0:6c56fb4bc5f0 405 ticker_set_handler(ticker_data, ticker_handler);
nexpaq 0:6c56fb4bc5f0 406 // You MUST set the custom scheduler before running the specification.
nexpaq 0:6c56fb4bc5f0 407 Harness::set_scheduler(utest_minimal_scheduler);
nexpaq 0:6c56fb4bc5f0 408 Harness::run(specification);
nexpaq 0:6c56fb4bc5f0 409
nexpaq 0:6c56fb4bc5f0 410 while(1) {
nexpaq 0:6c56fb4bc5f0 411 if (minimal_callback) {
nexpaq 0:6c56fb4bc5f0 412 // copy the callback and reset the shared memory
nexpaq 0:6c56fb4bc5f0 413 utest_v1_harness_callback_t callback = minimal_callback;
nexpaq 0:6c56fb4bc5f0 414 minimal_callback = NULL;
nexpaq 0:6c56fb4bc5f0 415 callback(); // execute the copied callback
nexpaq 0:6c56fb4bc5f0 416 }
nexpaq 0:6c56fb4bc5f0 417 }
nexpaq 0:6c56fb4bc5f0 418 }
nexpaq 0:6c56fb4bc5f0 419 ```