erkin yucel / mbed-os

mbed os with nrf51 internal bandgap enabled to read battery level

Dependents:   BLE_file_test BLE_Blink ExternalEncoder

features/frameworks/utest/README.md@0:f269e3021894, 2016-10-23 (annotated)

Committer:
elessair
Date:
Sun Oct 23 15:10:02 2016 +0000
Revision:
0:f269e3021894
Initial commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
elessair 0:f269e3021894 1 # utest: Asynchronous C++ Test Harness
elessair 0:f269e3021894 2
elessair 0:f269e3021894 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
elessair 0:f269e3021894 4
elessair 0:f269e3021894 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
elessair 0:f269e3021894 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
elessair 0:f269e3021894 7
elessair 0:f269e3021894 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
elessair 0:f269e3021894 9
elessair 0:f269e3021894 10 ## Theory of Operation
elessair 0:f269e3021894 11
elessair 0:f269e3021894 12 A test specification contains a setup handler, several test cases and a teardown handler.
elessair 0:f269e3021894 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
elessair 0:f269e3021894 14
elessair 0:f269e3021894 15 The order of handler execution is:
elessair 0:f269e3021894 16
elessair 0:f269e3021894 17 1. Test setup handler.
elessair 0:f269e3021894 18 1. For each test case:
elessair 0:f269e3021894 19 1. Test case setup handler.
elessair 0:f269e3021894 20 1. Test case execution handler.
elessair 0:f269e3021894 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
elessair 0:f269e3021894 22 1. (repeat test case execution handler if specified.)
elessair 0:f269e3021894 23 1. Test case teardown handler.
elessair 0:f269e3021894 24 1. Test teardown handler.
elessair 0:f269e3021894 25
elessair 0:f269e3021894 26 ## Example
elessair 0:f269e3021894 27
elessair 0:f269e3021894 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
elessair 0:f269e3021894 29
elessair 0:f269e3021894 30 ```cpp
elessair 0:f269e3021894 31 #include "mbed-drivers/test_env.h"
elessair 0:f269e3021894 32 #include "utest/utest.h"
elessair 0:f269e3021894 33 #include "unity/unity.h"
elessair 0:f269e3021894 34
elessair 0:f269e3021894 35 using namespace utest::v1;
elessair 0:f269e3021894 36
elessair 0:f269e3021894 37 void test_simple() {
elessair 0:f269e3021894 38 TEST_ASSERT_EQUAL(0, 0);
elessair 0:f269e3021894 39 printf("Simple test called\n");
elessair 0:f269e3021894 40 }
elessair 0:f269e3021894 41
elessair 0:f269e3021894 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
elessair 0:f269e3021894 43 // Call the default handler for proper reporting
elessair 0:f269e3021894 44 status_t status = greentea_case_setup_handler(source, index_of_case);
elessair 0:f269e3021894 45 printf("Setting up for '%s'\n", source->get_description());
elessair 0:f269e3021894 46 return status;
elessair 0:f269e3021894 47 }
elessair 0:f269e3021894 48 control_t test_repeats(const size_t call_count) {
elessair 0:f269e3021894 49 printf("Called for the %u. time\n", call_count);
elessair 0:f269e3021894 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
elessair 0:f269e3021894 51 // Specify how often this test is repeated ie. n total calls
elessair 0:f269e3021894 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
elessair 0:f269e3021894 53 }
elessair 0:f269e3021894 54
elessair 0:f269e3021894 55 void test_callback_validate() {
elessair 0:f269e3021894 56 // You may also use assertions here!
elessair 0:f269e3021894 57 TEST_ASSERT_EQUAL_PTR(0, 0);
elessair 0:f269e3021894 58 // Validate the callback
elessair 0:f269e3021894 59 Harness::validate_callback();
elessair 0:f269e3021894 60 }
elessair 0:f269e3021894 61 control_t test_asynchronous() {
elessair 0:f269e3021894 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
elessair 0:f269e3021894 63 // Set up a callback in the future. This may also be an interrupt!
elessair 0:f269e3021894 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
elessair 0:f269e3021894 65 // Set a 200ms timeout starting from now
elessair 0:f269e3021894 66 return CaseTimeout(200);
elessair 0:f269e3021894 67 }
elessair 0:f269e3021894 68
elessair 0:f269e3021894 69 control_t test_asynchronous_timeout(const size_t call_count) {
elessair 0:f269e3021894 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
elessair 0:f269e3021894 71 // Set a 200ms timeout starting from now,
elessair 0:f269e3021894 72 // but automatically repeat only this handler on timeout.
elessair 0:f269e3021894 73 if (call_count >= 5) {
elessair 0:f269e3021894 74 // but after the 5th call, the callback finally gets validated
elessair 0:f269e3021894 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
elessair 0:f269e3021894 76 }
elessair 0:f269e3021894 77 return CaseRepeatHandlerOnTimeout(200);
elessair 0:f269e3021894 78 }
elessair 0:f269e3021894 79
elessair 0:f269e3021894 80 // Custom setup handler required for proper Greentea support
elessair 0:f269e3021894 81 status_t greentea_setup(const size_t number_of_cases) {
elessair 0:f269e3021894 82 GREENTEA_SETUP(20, "default_auto");
elessair 0:f269e3021894 83 // Call the default reporting function
elessair 0:f269e3021894 84 return greentea_test_setup_handler(number_of_cases);
elessair 0:f269e3021894 85 }
elessair 0:f269e3021894 86
elessair 0:f269e3021894 87 // Specify all your test cases here
elessair 0:f269e3021894 88 Case cases[] = {
elessair 0:f269e3021894 89 Case("Simple Test", test_simple),
elessair 0:f269e3021894 90 Case("Repeating Test", test_repeats_setup, test_repeats),
elessair 0:f269e3021894 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
elessair 0:f269e3021894 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
elessair 0:f269e3021894 93 };
elessair 0:f269e3021894 94
elessair 0:f269e3021894 95 // Declare your test specification with a custom setup handler
elessair 0:f269e3021894 96 Specification specification(greentea_setup, cases);
elessair 0:f269e3021894 97
elessair 0:f269e3021894 98 void app_start(int, char**)
elessair 0:f269e3021894 99 { // Run the test specification
elessair 0:f269e3021894 100 Harness::run(specification);
elessair 0:f269e3021894 101 }
elessair 0:f269e3021894 102 ```
elessair 0:f269e3021894 103
elessair 0:f269e3021894 104 Running this test will output the following:
elessair 0:f269e3021894 105
elessair 0:f269e3021894 106 ```
elessair 0:f269e3021894 107 {{timeout;20}}
elessair 0:f269e3021894 108 {{host_test_name;default_auto}}
elessair 0:f269e3021894 109 {{description;utest greentea example}}
elessair 0:f269e3021894 110 {{test_id;MBED_OS}}
elessair 0:f269e3021894 111 {{start}}
elessair 0:f269e3021894 112 >>> Running 4 test cases...
elessair 0:f269e3021894 113
elessair 0:f269e3021894 114 >>> Running case #1: 'Simple Test'...
elessair 0:f269e3021894 115 Simple test called
elessair 0:f269e3021894 116 >>> 'Simple Test': 1 passed, 0 failed
elessair 0:f269e3021894 117
elessair 0:f269e3021894 118 >>> Running case #2: 'Repeating Test'...
elessair 0:f269e3021894 119 Setting up for 'Repeating Test'
elessair 0:f269e3021894 120 Called for the 1. time
elessair 0:f269e3021894 121 >>> 'Repeating Test': 1 passed, 0 failed
elessair 0:f269e3021894 122
elessair 0:f269e3021894 123 >>> Running case #2: 'Repeating Test'...
elessair 0:f269e3021894 124 Setting up for 'Repeating Test'
elessair 0:f269e3021894 125 Called for the 2. time
elessair 0:f269e3021894 126 >>> 'Repeating Test': 2 passed, 0 failed
elessair 0:f269e3021894 127
elessair 0:f269e3021894 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
elessair 0:f269e3021894 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
elessair 0:f269e3021894 130
elessair 0:f269e3021894 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
elessair 0:f269e3021894 132 >>> failure with reason 'Ignored: Timed Out'
elessair 0:f269e3021894 133 >>> failure with reason 'Ignored: Timed Out'
elessair 0:f269e3021894 134 >>> failure with reason 'Ignored: Timed Out'
elessair 0:f269e3021894 135 >>> failure with reason 'Ignored: Timed Out'
elessair 0:f269e3021894 136 >>> failure with reason 'Ignored: Timed Out'
elessair 0:f269e3021894 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
elessair 0:f269e3021894 138
elessair 0:f269e3021894 139 >>> Test cases: 4 passed, 0 failed
elessair 0:f269e3021894 140 {{success}}
elessair 0:f269e3021894 141 {{end}}
elessair 0:f269e3021894 142 ```
elessair 0:f269e3021894 143
elessair 0:f269e3021894 144 ## Detailed Description
elessair 0:f269e3021894 145
elessair 0:f269e3021894 146 ### Handlers
elessair 0:f269e3021894 147
elessair 0:f269e3021894 148 There are five handler types you can, but do not have to, override to customize operation.
elessair 0:f269e3021894 149 Please see the `utest/types.h` file for a detailed description.
elessair 0:f269e3021894 150
elessair 0:f269e3021894 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
elessair 0:f269e3021894 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
elessair 0:f269e3021894 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
elessair 0:f269e3021894 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
elessair 0:f269e3021894 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
elessair 0:f269e3021894 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
elessair 0:f269e3021894 157
elessair 0:f269e3021894 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
elessair 0:f269e3021894 159
elessair 0:f269e3021894 160 ### Test Case Handlers
elessair 0:f269e3021894 161
elessair 0:f269e3021894 162 There are three test case handlers:
elessair 0:f269e3021894 163
elessair 0:f269e3021894 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
elessair 0:f269e3021894 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
elessair 0:f269e3021894 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
elessair 0:f269e3021894 167
elessair 0:f269e3021894 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
elessair 0:f269e3021894 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
elessair 0:f269e3021894 170
elessair 0:f269e3021894 171 1. Description (required).
elessair 0:f269e3021894 172 1. Setup handler (optional).
elessair 0:f269e3021894 173 1. Test case handler (required).
elessair 0:f269e3021894 174 1. Teardown handler (optional).
elessair 0:f269e3021894 175 1. Failure handler (optional).
elessair 0:f269e3021894 176
elessair 0:f269e3021894 177 #### Test Case Attributes
elessair 0:f269e3021894 178
elessair 0:f269e3021894 179 You can modify test case behavior by returning `control_t` modifiers:
elessair 0:f269e3021894 180
elessair 0:f269e3021894 181 - `CaseNext`: never repeats and immediately moves to next test case
elessair 0:f269e3021894 182 - `CaseNoRepeat`: never repeats.
elessair 0:f269e3021894 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
elessair 0:f269e3021894 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
elessair 0:f269e3021894 185 - `CaseNoTimeout`: immediately moves to next test case.
elessair 0:f269e3021894 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
elessair 0:f269e3021894 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
elessair 0:f269e3021894 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
elessair 0:f269e3021894 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
elessair 0:f269e3021894 190
elessair 0:f269e3021894 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
elessair 0:f269e3021894 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
elessair 0:f269e3021894 193
elessair 0:f269e3021894 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
elessair 0:f269e3021894 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
elessair 0:f269e3021894 196
elessair 0:f269e3021894 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
elessair 0:f269e3021894 198 This will schedule the execution of the next test case.
elessair 0:f269e3021894 199
elessair 0:f269e3021894 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
elessair 0:f269e3021894 201
elessair 0:f269e3021894 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
elessair 0:f269e3021894 203
elessair 0:f269e3021894 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
elessair 0:f269e3021894 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
elessair 0:f269e3021894 206
elessair 0:f269e3021894 207 ### Failure Handlers
elessair 0:f269e3021894 208
elessair 0:f269e3021894 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
elessair 0:f269e3021894 210
elessair 0:f269e3021894 211 The failure reasons are:
elessair 0:f269e3021894 212
elessair 0:f269e3021894 213 - `REASON_NONE`: No failure occurred
elessair 0:f269e3021894 214 - `REASON_UNKNOWN`: An unknown failure occurred
elessair 0:f269e3021894 215 - `REASON_CASES`: A failure occurred in at least one test case
elessair 0:f269e3021894 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
elessair 0:f269e3021894 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
elessair 0:f269e3021894 218 - `REASON_ASSERTION`: An assertion failed
elessair 0:f269e3021894 219 - `REASON_TEST_SETUP`: Test setup failed
elessair 0:f269e3021894 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
elessair 0:f269e3021894 221 - `REASON_CASE_SETUP`: Case setup failed
elessair 0:f269e3021894 222 - `REASON_CASE_HANDLER`: Case handler failed
elessair 0:f269e3021894 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
elessair 0:f269e3021894 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
elessair 0:f269e3021894 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
elessair 0:f269e3021894 226
elessair 0:f269e3021894 227 The failure locations are:
elessair 0:f269e3021894 228
elessair 0:f269e3021894 229 - `LOCATION_NONE`: No location information
elessair 0:f269e3021894 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
elessair 0:f269e3021894 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
elessair 0:f269e3021894 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
elessair 0:f269e3021894 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
elessair 0:f269e3021894 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
elessair 0:f269e3021894 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
elessair 0:f269e3021894 236
elessair 0:f269e3021894 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
elessair 0:f269e3021894 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
elessair 0:f269e3021894 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
elessair 0:f269e3021894 240
elessair 0:f269e3021894 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
elessair 0:f269e3021894 242
elessair 0:f269e3021894 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
elessair 0:f269e3021894 244
elessair 0:f269e3021894 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
elessair 0:f269e3021894 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
elessair 0:f269e3021894 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
elessair 0:f269e3021894 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
elessair 0:f269e3021894 249
elessair 0:f269e3021894 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
elessair 0:f269e3021894 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
elessair 0:f269e3021894 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
elessair 0:f269e3021894 253
elessair 0:f269e3021894 254 ### Default Handlers
elessair 0:f269e3021894 255
elessair 0:f269e3021894 256 Three sets of default handlers with different behaviors are provided for your convenience:
elessair 0:f269e3021894 257
elessair 0:f269e3021894 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
elessair 0:f269e3021894 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
elessair 0:f269e3021894 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
elessair 0:f269e3021894 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
elessair 0:f269e3021894 262
elessair 0:f269e3021894 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
elessair 0:f269e3021894 264
elessair 0:f269e3021894 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
elessair 0:f269e3021894 266
elessair 0:f269e3021894 267 ```cpp
elessair 0:f269e3021894 268 // Declare your test specification with a custom setup handler
elessair 0:f269e3021894 269 // and set the default handlers to the predefined “greentea continue” behavior
elessair 0:f269e3021894 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
elessair 0:f269e3021894 271 ```
elessair 0:f269e3021894 272
elessair 0:f269e3021894 273 ### Custom Handlers
elessair 0:f269e3021894 274
elessair 0:f269e3021894 275 You may override any of the default handlers with your own custom handler.
elessair 0:f269e3021894 276
elessair 0:f269e3021894 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
elessair 0:f269e3021894 278 To explicitly invoke the default handler, use the `default_handler` hint.
elessair 0:f269e3021894 279
elessair 0:f269e3021894 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
elessair 0:f269e3021894 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
elessair 0:f269e3021894 282
elessair 0:f269e3021894 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
elessair 0:f269e3021894 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
elessair 0:f269e3021894 285
elessair 0:f269e3021894 286 For the `Specification` class the order of arguments is:
elessair 0:f269e3021894 287
elessair 0:f269e3021894 288 1. Test setup handler (optional).
elessair 0:f269e3021894 289 1. Array of test cases (required).
elessair 0:f269e3021894 290 1. Test teardown handler (optional).
elessair 0:f269e3021894 291 1. Test failure handler (optional).
elessair 0:f269e3021894 292 1. Default handlers (optional).
elessair 0:f269e3021894 293
elessair 0:f269e3021894 294 ### Test Case Attribute Arbitration
elessair 0:f269e3021894 295
elessair 0:f269e3021894 296 When adding conflicting modifiers together
elessair 0:f269e3021894 297
elessair 0:f269e3021894 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
elessair 0:f269e3021894 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
elessair 0:f269e3021894 300 - `CaseNext` always wins arbitration.
elessair 0:f269e3021894 301
elessair 0:f269e3021894 302 The following table shows this arbitration logic in detail:
elessair 0:f269e3021894 303
elessair 0:f269e3021894 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
elessair 0:f269e3021894 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
elessair 0:f269e3021894 306 | `CaseNext` | no repeat &<br> no timeout
elessair 0:f269e3021894 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
elessair 0:f269e3021894 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
elessair 0:f269e3021894 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
elessair 0:f269e3021894 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
elessair 0:f269e3021894 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
elessair 0:f269e3021894 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
elessair 0:f269e3021894 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
elessair 0:f269e3021894 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
elessair 0:f269e3021894 315
elessair 0:f269e3021894 316 ### Atomicity
elessair 0:f269e3021894 317
elessair 0:f269e3021894 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
elessair 0:f269e3021894 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
elessair 0:f269e3021894 320
elessair 0:f269e3021894 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
elessair 0:f269e3021894 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
elessair 0:f269e3021894 323
elessair 0:f269e3021894 324 ### Custom Scheduler
elessair 0:f269e3021894 325
elessair 0:f269e3021894 326 By default, a Timeout object is used for scheduling the harness operations.
elessair 0:f269e3021894 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
elessair 0:f269e3021894 328
elessair 0:f269e3021894 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
elessair 0:f269e3021894 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
elessair 0:f269e3021894 331
elessair 0:f269e3021894 332 There are two functions you need to implement:
elessair 0:f269e3021894 333
elessair 0:f269e3021894 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
elessair 0:f269e3021894 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
elessair 0:f269e3021894 336
elessair 0:f269e3021894 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
elessair 0:f269e3021894 338
elessair 0:f269e3021894 339 ### Example Synchronous Scheduler
elessair 0:f269e3021894 340
elessair 0:f269e3021894 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
elessair 0:f269e3021894 342 ```cpp
elessair 0:f269e3021894 343 volatile utest_v1_harness_callback_t minimal_callback;
elessair 0:f269e3021894 344
elessair 0:f269e3021894 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
elessair 0:f269e3021894 346 minimal_callback = callback;
elessair 0:f269e3021894 347 // this scheduler does not support asynchronous callbacks
elessair 0:f269e3021894 348 return (delay_ms ? NULL : (void*)1);
elessair 0:f269e3021894 349 }
elessair 0:f269e3021894 350 static int32_t utest_minimal_cancel(void*) {
elessair 0:f269e3021894 351 return -1; // canceling not supported either
elessair 0:f269e3021894 352 }
elessair 0:f269e3021894 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
elessair 0:f269e3021894 354
elessair 0:f269e3021894 355 // [...] Add your test cases and specification here.
elessair 0:f269e3021894 356
elessair 0:f269e3021894 357 void main() // or whatever your custom entry point is
elessair 0:f269e3021894 358 {
elessair 0:f269e3021894 359 // You MUST set the custom scheduler before running the specification.
elessair 0:f269e3021894 360 Harness::set_scheduler(utest_minimal_scheduler);
elessair 0:f269e3021894 361 Harness::run(specification);
elessair 0:f269e3021894 362
elessair 0:f269e3021894 363 while(1) {
elessair 0:f269e3021894 364 if (minimal_callback) {
elessair 0:f269e3021894 365 // copy the callback and reset the shared memory
elessair 0:f269e3021894 366 utest_v1_harness_callback_t callback = minimal_callback;
elessair 0:f269e3021894 367 minimal_callback = NULL;
elessair 0:f269e3021894 368 callback(); // execute the copied callback
elessair 0:f269e3021894 369 }
elessair 0:f269e3021894 370 }
elessair 0:f269e3021894 371 }
elessair 0:f269e3021894 372 ```
elessair 0:f269e3021894 373
elessair 0:f269e3021894 374 ### Example Asynchronous Scheduler
elessair 0:f269e3021894 375
elessair 0:f269e3021894 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
elessair 0:f269e3021894 377 ```cpp
elessair 0:f269e3021894 378 volatile utest_v1_harness_callback_t minimal_callback;
elessair 0:f269e3021894 379 volatile utest_v1_harness_callback_t ticker_callback;
elessair 0:f269e3021894 380 const ticker_data_t *ticker_data;
elessair 0:f269e3021894 381 ticker_event_t ticker_event;
elessair 0:f269e3021894 382
elessair 0:f269e3021894 383 static void ticker_handler(uint32_t) {
elessair 0:f269e3021894 384 minimal_callback = ticker_callback; // interrupt context!
elessair 0:f269e3021894 385 }
elessair 0:f269e3021894 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
elessair 0:f269e3021894 387 if (delay_ms) {
elessair 0:f269e3021894 388 ticker_callback = callback;
elessair 0:f269e3021894 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
elessair 0:f269e3021894 390 }
elessair 0:f269e3021894 391 else minimal_callback = callback;
elessair 0:f269e3021894 392 return (void*)1;
elessair 0:f269e3021894 393 }
elessair 0:f269e3021894 394 static int32_t utest_minimal_cancel(void*) {
elessair 0:f269e3021894 395 ticker_remove_event(ticker_data, &ticker_event);
elessair 0:f269e3021894 396 return 0; // canceling is supported
elessair 0:f269e3021894 397 }
elessair 0:f269e3021894 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
elessair 0:f269e3021894 399
elessair 0:f269e3021894 400 // [...] Add your test cases and specification here.
elessair 0:f269e3021894 401
elessair 0:f269e3021894 402 void main() // or whatever your custom entry point is
elessair 0:f269e3021894 403 {
elessair 0:f269e3021894 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
elessair 0:f269e3021894 405 ticker_set_handler(ticker_data, ticker_handler);
elessair 0:f269e3021894 406 // You MUST set the custom scheduler before running the specification.
elessair 0:f269e3021894 407 Harness::set_scheduler(utest_minimal_scheduler);
elessair 0:f269e3021894 408 Harness::run(specification);
elessair 0:f269e3021894 409
elessair 0:f269e3021894 410 while(1) {
elessair 0:f269e3021894 411 if (minimal_callback) {
elessair 0:f269e3021894 412 // copy the callback and reset the shared memory
elessair 0:f269e3021894 413 utest_v1_harness_callback_t callback = minimal_callback;
elessair 0:f269e3021894 414 minimal_callback = NULL;
elessair 0:f269e3021894 415 callback(); // execute the copied callback
elessair 0:f269e3021894 416 }
elessair 0:f269e3021894 417 }
elessair 0:f269e3021894 418 }
elessair 0:f269e3021894 419 ```