Kev Mann / mbed-dev-OS5_10_4

RTC auf true

features/frameworks/utest/README.md@0:38ceb79fef03, 2018-11-28 (annotated)

Committer:
kevman
Date:
Wed Nov 28 15:10:15 2018 +0000
Revision:
0:38ceb79fef03
RTC modified

Who changed what in which revision?

UserRevisionLine numberNew contents of line
kevman 0:38ceb79fef03 1 # utest: Asynchronous C++ Test Harness
kevman 0:38ceb79fef03 2
kevman 0:38ceb79fef03 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
kevman 0:38ceb79fef03 4
kevman 0:38ceb79fef03 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
kevman 0:38ceb79fef03 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
kevman 0:38ceb79fef03 7
kevman 0:38ceb79fef03 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
kevman 0:38ceb79fef03 9
kevman 0:38ceb79fef03 10 ## Theory of Operation
kevman 0:38ceb79fef03 11
kevman 0:38ceb79fef03 12 A test specification contains a setup handler, several test cases and a teardown handler.
kevman 0:38ceb79fef03 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
kevman 0:38ceb79fef03 14
kevman 0:38ceb79fef03 15 The order of handler execution is:
kevman 0:38ceb79fef03 16
kevman 0:38ceb79fef03 17 1. Test setup handler.
kevman 0:38ceb79fef03 18 1. For each test case:
kevman 0:38ceb79fef03 19 1. Test case setup handler.
kevman 0:38ceb79fef03 20 1. Test case execution handler.
kevman 0:38ceb79fef03 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
kevman 0:38ceb79fef03 22 1. (repeat test case execution handler if specified.)
kevman 0:38ceb79fef03 23 1. Test case teardown handler.
kevman 0:38ceb79fef03 24 1. Test teardown handler.
kevman 0:38ceb79fef03 25
kevman 0:38ceb79fef03 26 ## Example
kevman 0:38ceb79fef03 27
kevman 0:38ceb79fef03 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
kevman 0:38ceb79fef03 29
kevman 0:38ceb79fef03 30 ```cpp
kevman 0:38ceb79fef03 31 #include "mbed-drivers/test_env.h"
kevman 0:38ceb79fef03 32 #include "utest/utest.h"
kevman 0:38ceb79fef03 33 #include "unity/unity.h"
kevman 0:38ceb79fef03 34
kevman 0:38ceb79fef03 35 using namespace utest::v1;
kevman 0:38ceb79fef03 36
kevman 0:38ceb79fef03 37 void test_simple() {
kevman 0:38ceb79fef03 38 TEST_ASSERT_EQUAL(0, 0);
kevman 0:38ceb79fef03 39 printf("Simple test called\n");
kevman 0:38ceb79fef03 40 }
kevman 0:38ceb79fef03 41
kevman 0:38ceb79fef03 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
kevman 0:38ceb79fef03 43 // Call the default handler for proper reporting
kevman 0:38ceb79fef03 44 status_t status = greentea_case_setup_handler(source, index_of_case);
kevman 0:38ceb79fef03 45 printf("Setting up for '%s'\n", source->get_description());
kevman 0:38ceb79fef03 46 return status;
kevman 0:38ceb79fef03 47 }
kevman 0:38ceb79fef03 48 control_t test_repeats(const size_t call_count) {
kevman 0:38ceb79fef03 49 printf("Called for the %u. time\n", call_count);
kevman 0:38ceb79fef03 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
kevman 0:38ceb79fef03 51 // Specify how often this test is repeated ie. n total calls
kevman 0:38ceb79fef03 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
kevman 0:38ceb79fef03 53 }
kevman 0:38ceb79fef03 54
kevman 0:38ceb79fef03 55 void test_callback_validate() {
kevman 0:38ceb79fef03 56 // You may also use assertions here!
kevman 0:38ceb79fef03 57 TEST_ASSERT_EQUAL_PTR(0, 0);
kevman 0:38ceb79fef03 58 // Validate the callback
kevman 0:38ceb79fef03 59 Harness::validate_callback();
kevman 0:38ceb79fef03 60 }
kevman 0:38ceb79fef03 61 control_t test_asynchronous() {
kevman 0:38ceb79fef03 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
kevman 0:38ceb79fef03 63 // Set up a callback in the future. This may also be an interrupt!
kevman 0:38ceb79fef03 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
kevman 0:38ceb79fef03 65 // Set a 200ms timeout starting from now
kevman 0:38ceb79fef03 66 return CaseTimeout(200);
kevman 0:38ceb79fef03 67 }
kevman 0:38ceb79fef03 68
kevman 0:38ceb79fef03 69 control_t test_asynchronous_timeout(const size_t call_count) {
kevman 0:38ceb79fef03 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
kevman 0:38ceb79fef03 71 // Set a 200ms timeout starting from now,
kevman 0:38ceb79fef03 72 // but automatically repeat only this handler on timeout.
kevman 0:38ceb79fef03 73 if (call_count >= 5) {
kevman 0:38ceb79fef03 74 // but after the 5th call, the callback finally gets validated
kevman 0:38ceb79fef03 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
kevman 0:38ceb79fef03 76 }
kevman 0:38ceb79fef03 77 return CaseRepeatHandlerOnTimeout(200);
kevman 0:38ceb79fef03 78 }
kevman 0:38ceb79fef03 79
kevman 0:38ceb79fef03 80 // Custom setup handler required for proper Greentea support
kevman 0:38ceb79fef03 81 status_t greentea_setup(const size_t number_of_cases) {
kevman 0:38ceb79fef03 82 GREENTEA_SETUP(20, "default_auto");
kevman 0:38ceb79fef03 83 // Call the default reporting function
kevman 0:38ceb79fef03 84 return greentea_test_setup_handler(number_of_cases);
kevman 0:38ceb79fef03 85 }
kevman 0:38ceb79fef03 86
kevman 0:38ceb79fef03 87 // Specify all your test cases here
kevman 0:38ceb79fef03 88 Case cases[] = {
kevman 0:38ceb79fef03 89 Case("Simple Test", test_simple),
kevman 0:38ceb79fef03 90 Case("Repeating Test", test_repeats_setup, test_repeats),
kevman 0:38ceb79fef03 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
kevman 0:38ceb79fef03 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
kevman 0:38ceb79fef03 93 };
kevman 0:38ceb79fef03 94
kevman 0:38ceb79fef03 95 // Declare your test specification with a custom setup handler
kevman 0:38ceb79fef03 96 Specification specification(greentea_setup, cases);
kevman 0:38ceb79fef03 97
kevman 0:38ceb79fef03 98 void app_start(int, char**)
kevman 0:38ceb79fef03 99 { // Run the test specification
kevman 0:38ceb79fef03 100 Harness::run(specification);
kevman 0:38ceb79fef03 101 }
kevman 0:38ceb79fef03 102 ```
kevman 0:38ceb79fef03 103
kevman 0:38ceb79fef03 104 Running this test will output the following:
kevman 0:38ceb79fef03 105
kevman 0:38ceb79fef03 106 ```
kevman 0:38ceb79fef03 107 {{timeout;20}}
kevman 0:38ceb79fef03 108 {{host_test_name;default_auto}}
kevman 0:38ceb79fef03 109 {{description;utest greentea example}}
kevman 0:38ceb79fef03 110 {{test_id;MBED_OS}}
kevman 0:38ceb79fef03 111 {{start}}
kevman 0:38ceb79fef03 112 >>> Running 4 test cases...
kevman 0:38ceb79fef03 113
kevman 0:38ceb79fef03 114 >>> Running case #1: 'Simple Test'...
kevman 0:38ceb79fef03 115 Simple test called
kevman 0:38ceb79fef03 116 >>> 'Simple Test': 1 passed, 0 failed
kevman 0:38ceb79fef03 117
kevman 0:38ceb79fef03 118 >>> Running case #2: 'Repeating Test'...
kevman 0:38ceb79fef03 119 Setting up for 'Repeating Test'
kevman 0:38ceb79fef03 120 Called for the 1. time
kevman 0:38ceb79fef03 121 >>> 'Repeating Test': 1 passed, 0 failed
kevman 0:38ceb79fef03 122
kevman 0:38ceb79fef03 123 >>> Running case #2: 'Repeating Test'...
kevman 0:38ceb79fef03 124 Setting up for 'Repeating Test'
kevman 0:38ceb79fef03 125 Called for the 2. time
kevman 0:38ceb79fef03 126 >>> 'Repeating Test': 2 passed, 0 failed
kevman 0:38ceb79fef03 127
kevman 0:38ceb79fef03 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
kevman 0:38ceb79fef03 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
kevman 0:38ceb79fef03 130
kevman 0:38ceb79fef03 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
kevman 0:38ceb79fef03 132 >>> failure with reason 'Ignored: Timed Out'
kevman 0:38ceb79fef03 133 >>> failure with reason 'Ignored: Timed Out'
kevman 0:38ceb79fef03 134 >>> failure with reason 'Ignored: Timed Out'
kevman 0:38ceb79fef03 135 >>> failure with reason 'Ignored: Timed Out'
kevman 0:38ceb79fef03 136 >>> failure with reason 'Ignored: Timed Out'
kevman 0:38ceb79fef03 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
kevman 0:38ceb79fef03 138
kevman 0:38ceb79fef03 139 >>> Test cases: 4 passed, 0 failed
kevman 0:38ceb79fef03 140 {{success}}
kevman 0:38ceb79fef03 141 {{end}}
kevman 0:38ceb79fef03 142 ```
kevman 0:38ceb79fef03 143
kevman 0:38ceb79fef03 144 ## Detailed Description
kevman 0:38ceb79fef03 145
kevman 0:38ceb79fef03 146 ### Handlers
kevman 0:38ceb79fef03 147
kevman 0:38ceb79fef03 148 There are five handler types you can, but do not have to, override to customize operation.
kevman 0:38ceb79fef03 149 Please see the `utest/types.h` file for a detailed description.
kevman 0:38ceb79fef03 150
kevman 0:38ceb79fef03 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
kevman 0:38ceb79fef03 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
kevman 0:38ceb79fef03 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
kevman 0:38ceb79fef03 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
kevman 0:38ceb79fef03 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
kevman 0:38ceb79fef03 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
kevman 0:38ceb79fef03 157
kevman 0:38ceb79fef03 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
kevman 0:38ceb79fef03 159
kevman 0:38ceb79fef03 160 ### Test Case Handlers
kevman 0:38ceb79fef03 161
kevman 0:38ceb79fef03 162 There are three test case handlers:
kevman 0:38ceb79fef03 163
kevman 0:38ceb79fef03 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
kevman 0:38ceb79fef03 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
kevman 0:38ceb79fef03 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
kevman 0:38ceb79fef03 167
kevman 0:38ceb79fef03 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
kevman 0:38ceb79fef03 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
kevman 0:38ceb79fef03 170
kevman 0:38ceb79fef03 171 1. Description (required).
kevman 0:38ceb79fef03 172 1. Setup handler (optional).
kevman 0:38ceb79fef03 173 1. Test case handler (required).
kevman 0:38ceb79fef03 174 1. Teardown handler (optional).
kevman 0:38ceb79fef03 175 1. Failure handler (optional).
kevman 0:38ceb79fef03 176
kevman 0:38ceb79fef03 177 #### Test Case Attributes
kevman 0:38ceb79fef03 178
kevman 0:38ceb79fef03 179 You can modify test case behavior by returning `control_t` modifiers:
kevman 0:38ceb79fef03 180
kevman 0:38ceb79fef03 181 - `CaseNext`: never repeats and immediately moves to next test case
kevman 0:38ceb79fef03 182 - `CaseNoRepeat`: never repeats.
kevman 0:38ceb79fef03 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
kevman 0:38ceb79fef03 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
kevman 0:38ceb79fef03 185 - `CaseNoTimeout`: immediately moves to next test case.
kevman 0:38ceb79fef03 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
kevman 0:38ceb79fef03 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
kevman 0:38ceb79fef03 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
kevman 0:38ceb79fef03 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
kevman 0:38ceb79fef03 190
kevman 0:38ceb79fef03 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
kevman 0:38ceb79fef03 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
kevman 0:38ceb79fef03 193
kevman 0:38ceb79fef03 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
kevman 0:38ceb79fef03 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
kevman 0:38ceb79fef03 196
kevman 0:38ceb79fef03 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
kevman 0:38ceb79fef03 198 This will schedule the execution of the next test case.
kevman 0:38ceb79fef03 199
kevman 0:38ceb79fef03 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
kevman 0:38ceb79fef03 201
kevman 0:38ceb79fef03 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
kevman 0:38ceb79fef03 203
kevman 0:38ceb79fef03 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
kevman 0:38ceb79fef03 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
kevman 0:38ceb79fef03 206
kevman 0:38ceb79fef03 207 ### Failure Handlers
kevman 0:38ceb79fef03 208
kevman 0:38ceb79fef03 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
kevman 0:38ceb79fef03 210
kevman 0:38ceb79fef03 211 The failure reasons are:
kevman 0:38ceb79fef03 212
kevman 0:38ceb79fef03 213 - `REASON_NONE`: No failure occurred
kevman 0:38ceb79fef03 214 - `REASON_UNKNOWN`: An unknown failure occurred
kevman 0:38ceb79fef03 215 - `REASON_CASES`: A failure occurred in at least one test case
kevman 0:38ceb79fef03 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
kevman 0:38ceb79fef03 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
kevman 0:38ceb79fef03 218 - `REASON_ASSERTION`: An assertion failed
kevman 0:38ceb79fef03 219 - `REASON_TEST_SETUP`: Test setup failed
kevman 0:38ceb79fef03 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
kevman 0:38ceb79fef03 221 - `REASON_CASE_SETUP`: Case setup failed
kevman 0:38ceb79fef03 222 - `REASON_CASE_HANDLER`: Case handler failed
kevman 0:38ceb79fef03 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
kevman 0:38ceb79fef03 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
kevman 0:38ceb79fef03 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
kevman 0:38ceb79fef03 226
kevman 0:38ceb79fef03 227 The failure locations are:
kevman 0:38ceb79fef03 228
kevman 0:38ceb79fef03 229 - `LOCATION_NONE`: No location information
kevman 0:38ceb79fef03 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
kevman 0:38ceb79fef03 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
kevman 0:38ceb79fef03 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
kevman 0:38ceb79fef03 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
kevman 0:38ceb79fef03 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
kevman 0:38ceb79fef03 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
kevman 0:38ceb79fef03 236
kevman 0:38ceb79fef03 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
kevman 0:38ceb79fef03 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
kevman 0:38ceb79fef03 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
kevman 0:38ceb79fef03 240
kevman 0:38ceb79fef03 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
kevman 0:38ceb79fef03 242
kevman 0:38ceb79fef03 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
kevman 0:38ceb79fef03 244
kevman 0:38ceb79fef03 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
kevman 0:38ceb79fef03 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
kevman 0:38ceb79fef03 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
kevman 0:38ceb79fef03 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
kevman 0:38ceb79fef03 249
kevman 0:38ceb79fef03 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
kevman 0:38ceb79fef03 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
kevman 0:38ceb79fef03 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
kevman 0:38ceb79fef03 253
kevman 0:38ceb79fef03 254 ### Default Handlers
kevman 0:38ceb79fef03 255
kevman 0:38ceb79fef03 256 Three sets of default handlers with different behaviors are provided for your convenience:
kevman 0:38ceb79fef03 257
kevman 0:38ceb79fef03 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
kevman 0:38ceb79fef03 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
kevman 0:38ceb79fef03 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
kevman 0:38ceb79fef03 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
kevman 0:38ceb79fef03 262
kevman 0:38ceb79fef03 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
kevman 0:38ceb79fef03 264
kevman 0:38ceb79fef03 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
kevman 0:38ceb79fef03 266
kevman 0:38ceb79fef03 267 ```cpp
kevman 0:38ceb79fef03 268 // Declare your test specification with a custom setup handler
kevman 0:38ceb79fef03 269 // and set the default handlers to the predefined “greentea continue” behavior
kevman 0:38ceb79fef03 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
kevman 0:38ceb79fef03 271 ```
kevman 0:38ceb79fef03 272
kevman 0:38ceb79fef03 273 ### Custom Handlers
kevman 0:38ceb79fef03 274
kevman 0:38ceb79fef03 275 You may override any of the default handlers with your own custom handler.
kevman 0:38ceb79fef03 276
kevman 0:38ceb79fef03 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
kevman 0:38ceb79fef03 278 To explicitly invoke the default handler, use the `default_handler` hint.
kevman 0:38ceb79fef03 279
kevman 0:38ceb79fef03 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
kevman 0:38ceb79fef03 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
kevman 0:38ceb79fef03 282
kevman 0:38ceb79fef03 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
kevman 0:38ceb79fef03 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
kevman 0:38ceb79fef03 285
kevman 0:38ceb79fef03 286 For the `Specification` class the order of arguments is:
kevman 0:38ceb79fef03 287
kevman 0:38ceb79fef03 288 1. Test setup handler (optional).
kevman 0:38ceb79fef03 289 1. Array of test cases (required).
kevman 0:38ceb79fef03 290 1. Test teardown handler (optional).
kevman 0:38ceb79fef03 291 1. Test failure handler (optional).
kevman 0:38ceb79fef03 292 1. Default handlers (optional).
kevman 0:38ceb79fef03 293
kevman 0:38ceb79fef03 294 ### Test Case Attribute Arbitration
kevman 0:38ceb79fef03 295
kevman 0:38ceb79fef03 296 When adding conflicting modifiers together
kevman 0:38ceb79fef03 297
kevman 0:38ceb79fef03 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
kevman 0:38ceb79fef03 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
kevman 0:38ceb79fef03 300 - `CaseNext` always wins arbitration.
kevman 0:38ceb79fef03 301
kevman 0:38ceb79fef03 302 The following table shows this arbitration logic in detail:
kevman 0:38ceb79fef03 303
kevman 0:38ceb79fef03 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
kevman 0:38ceb79fef03 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
kevman 0:38ceb79fef03 306 | `CaseNext` | no repeat &<br> no timeout
kevman 0:38ceb79fef03 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
kevman 0:38ceb79fef03 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
kevman 0:38ceb79fef03 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
kevman 0:38ceb79fef03 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
kevman 0:38ceb79fef03 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
kevman 0:38ceb79fef03 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
kevman 0:38ceb79fef03 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
kevman 0:38ceb79fef03 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
kevman 0:38ceb79fef03 315
kevman 0:38ceb79fef03 316 ### Atomicity
kevman 0:38ceb79fef03 317
kevman 0:38ceb79fef03 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
kevman 0:38ceb79fef03 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
kevman 0:38ceb79fef03 320
kevman 0:38ceb79fef03 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
kevman 0:38ceb79fef03 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
kevman 0:38ceb79fef03 323
kevman 0:38ceb79fef03 324 ### Custom Scheduler
kevman 0:38ceb79fef03 325
kevman 0:38ceb79fef03 326 By default, a Timeout object is used for scheduling the harness operations.
kevman 0:38ceb79fef03 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
kevman 0:38ceb79fef03 328
kevman 0:38ceb79fef03 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
kevman 0:38ceb79fef03 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
kevman 0:38ceb79fef03 331
kevman 0:38ceb79fef03 332 There are two functions you need to implement:
kevman 0:38ceb79fef03 333
kevman 0:38ceb79fef03 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
kevman 0:38ceb79fef03 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
kevman 0:38ceb79fef03 336
kevman 0:38ceb79fef03 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
kevman 0:38ceb79fef03 338
kevman 0:38ceb79fef03 339 ### Example Synchronous Scheduler
kevman 0:38ceb79fef03 340
kevman 0:38ceb79fef03 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
kevman 0:38ceb79fef03 342 ```cpp
kevman 0:38ceb79fef03 343 volatile utest_v1_harness_callback_t minimal_callback;
kevman 0:38ceb79fef03 344
kevman 0:38ceb79fef03 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
kevman 0:38ceb79fef03 346 minimal_callback = callback;
kevman 0:38ceb79fef03 347 // this scheduler does not support asynchronous callbacks
kevman 0:38ceb79fef03 348 return (delay_ms ? NULL : (void*)1);
kevman 0:38ceb79fef03 349 }
kevman 0:38ceb79fef03 350 static int32_t utest_minimal_cancel(void*) {
kevman 0:38ceb79fef03 351 return -1; // canceling not supported either
kevman 0:38ceb79fef03 352 }
kevman 0:38ceb79fef03 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
kevman 0:38ceb79fef03 354
kevman 0:38ceb79fef03 355 // [...] Add your test cases and specification here.
kevman 0:38ceb79fef03 356
kevman 0:38ceb79fef03 357 void main() // or whatever your custom entry point is
kevman 0:38ceb79fef03 358 {
kevman 0:38ceb79fef03 359 // You MUST set the custom scheduler before running the specification.
kevman 0:38ceb79fef03 360 Harness::set_scheduler(utest_minimal_scheduler);
kevman 0:38ceb79fef03 361 Harness::run(specification);
kevman 0:38ceb79fef03 362
kevman 0:38ceb79fef03 363 while(1) {
kevman 0:38ceb79fef03 364 if (minimal_callback) {
kevman 0:38ceb79fef03 365 // copy the callback and reset the shared memory
kevman 0:38ceb79fef03 366 utest_v1_harness_callback_t callback = minimal_callback;
kevman 0:38ceb79fef03 367 minimal_callback = NULL;
kevman 0:38ceb79fef03 368 callback(); // execute the copied callback
kevman 0:38ceb79fef03 369 }
kevman 0:38ceb79fef03 370 }
kevman 0:38ceb79fef03 371 }
kevman 0:38ceb79fef03 372 ```
kevman 0:38ceb79fef03 373
kevman 0:38ceb79fef03 374 ### Example Asynchronous Scheduler
kevman 0:38ceb79fef03 375
kevman 0:38ceb79fef03 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
kevman 0:38ceb79fef03 377 ```cpp
kevman 0:38ceb79fef03 378 volatile utest_v1_harness_callback_t minimal_callback;
kevman 0:38ceb79fef03 379 volatile utest_v1_harness_callback_t ticker_callback;
kevman 0:38ceb79fef03 380 const ticker_data_t *ticker_data;
kevman 0:38ceb79fef03 381 ticker_event_t ticker_event;
kevman 0:38ceb79fef03 382
kevman 0:38ceb79fef03 383 static void ticker_handler(uint32_t) {
kevman 0:38ceb79fef03 384 minimal_callback = ticker_callback; // interrupt context!
kevman 0:38ceb79fef03 385 }
kevman 0:38ceb79fef03 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
kevman 0:38ceb79fef03 387 if (delay_ms) {
kevman 0:38ceb79fef03 388 ticker_callback = callback;
kevman 0:38ceb79fef03 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
kevman 0:38ceb79fef03 390 }
kevman 0:38ceb79fef03 391 else minimal_callback = callback;
kevman 0:38ceb79fef03 392 return (void*)1;
kevman 0:38ceb79fef03 393 }
kevman 0:38ceb79fef03 394 static int32_t utest_minimal_cancel(void*) {
kevman 0:38ceb79fef03 395 ticker_remove_event(ticker_data, &ticker_event);
kevman 0:38ceb79fef03 396 return 0; // canceling is supported
kevman 0:38ceb79fef03 397 }
kevman 0:38ceb79fef03 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
kevman 0:38ceb79fef03 399
kevman 0:38ceb79fef03 400 // [...] Add your test cases and specification here.
kevman 0:38ceb79fef03 401
kevman 0:38ceb79fef03 402 void main() // or whatever your custom entry point is
kevman 0:38ceb79fef03 403 {
kevman 0:38ceb79fef03 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
kevman 0:38ceb79fef03 405 ticker_set_handler(ticker_data, ticker_handler);
kevman 0:38ceb79fef03 406 // You MUST set the custom scheduler before running the specification.
kevman 0:38ceb79fef03 407 Harness::set_scheduler(utest_minimal_scheduler);
kevman 0:38ceb79fef03 408 Harness::run(specification);
kevman 0:38ceb79fef03 409
kevman 0:38ceb79fef03 410 while(1) {
kevman 0:38ceb79fef03 411 if (minimal_callback) {
kevman 0:38ceb79fef03 412 // copy the callback and reset the shared memory
kevman 0:38ceb79fef03 413 utest_v1_harness_callback_t callback = minimal_callback;
kevman 0:38ceb79fef03 414 minimal_callback = NULL;
kevman 0:38ceb79fef03 415 callback(); // execute the copied callback
kevman 0:38ceb79fef03 416 }
kevman 0:38ceb79fef03 417 }
kevman 0:38ceb79fef03 418 }
kevman 0:38ceb79fef03 419 ```