Daiki Kato / mbed-os-lychee

mbed-os for GR-LYCHEE

Dependents:   mbed-os-example-blinky-gr-lychee GR-Boads_Camera_sample GR-Boards_Audio_Recoder GR-Boads_Camera_DisplayApp ... more

features/frameworks/utest/README.md@0:f782d9c66c49, 2018-02-02 (annotated)

Committer:
dkato
Date:
Fri Feb 02 05:42:23 2018 +0000
Revision:
0:f782d9c66c49
mbed-os for GR-LYCHEE

Who changed what in which revision?

UserRevisionLine numberNew contents of line
dkato 0:f782d9c66c49 1 # utest: Asynchronous C++ Test Harness
dkato 0:f782d9c66c49 2
dkato 0:f782d9c66c49 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
dkato 0:f782d9c66c49 4
dkato 0:f782d9c66c49 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
dkato 0:f782d9c66c49 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
dkato 0:f782d9c66c49 7
dkato 0:f782d9c66c49 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
dkato 0:f782d9c66c49 9
dkato 0:f782d9c66c49 10 ## Theory of Operation
dkato 0:f782d9c66c49 11
dkato 0:f782d9c66c49 12 A test specification contains a setup handler, several test cases and a teardown handler.
dkato 0:f782d9c66c49 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
dkato 0:f782d9c66c49 14
dkato 0:f782d9c66c49 15 The order of handler execution is:
dkato 0:f782d9c66c49 16
dkato 0:f782d9c66c49 17 1. Test setup handler.
dkato 0:f782d9c66c49 18 1. For each test case:
dkato 0:f782d9c66c49 19 1. Test case setup handler.
dkato 0:f782d9c66c49 20 1. Test case execution handler.
dkato 0:f782d9c66c49 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
dkato 0:f782d9c66c49 22 1. (repeat test case execution handler if specified.)
dkato 0:f782d9c66c49 23 1. Test case teardown handler.
dkato 0:f782d9c66c49 24 1. Test teardown handler.
dkato 0:f782d9c66c49 25
dkato 0:f782d9c66c49 26 ## Example
dkato 0:f782d9c66c49 27
dkato 0:f782d9c66c49 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
dkato 0:f782d9c66c49 29
dkato 0:f782d9c66c49 30 ```cpp
dkato 0:f782d9c66c49 31 #include "mbed-drivers/test_env.h"
dkato 0:f782d9c66c49 32 #include "utest/utest.h"
dkato 0:f782d9c66c49 33 #include "unity/unity.h"
dkato 0:f782d9c66c49 34
dkato 0:f782d9c66c49 35 using namespace utest::v1;
dkato 0:f782d9c66c49 36
dkato 0:f782d9c66c49 37 void test_simple() {
dkato 0:f782d9c66c49 38 TEST_ASSERT_EQUAL(0, 0);
dkato 0:f782d9c66c49 39 printf("Simple test called\n");
dkato 0:f782d9c66c49 40 }
dkato 0:f782d9c66c49 41
dkato 0:f782d9c66c49 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
dkato 0:f782d9c66c49 43 // Call the default handler for proper reporting
dkato 0:f782d9c66c49 44 status_t status = greentea_case_setup_handler(source, index_of_case);
dkato 0:f782d9c66c49 45 printf("Setting up for '%s'\n", source->get_description());
dkato 0:f782d9c66c49 46 return status;
dkato 0:f782d9c66c49 47 }
dkato 0:f782d9c66c49 48 control_t test_repeats(const size_t call_count) {
dkato 0:f782d9c66c49 49 printf("Called for the %u. time\n", call_count);
dkato 0:f782d9c66c49 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
dkato 0:f782d9c66c49 51 // Specify how often this test is repeated ie. n total calls
dkato 0:f782d9c66c49 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
dkato 0:f782d9c66c49 53 }
dkato 0:f782d9c66c49 54
dkato 0:f782d9c66c49 55 void test_callback_validate() {
dkato 0:f782d9c66c49 56 // You may also use assertions here!
dkato 0:f782d9c66c49 57 TEST_ASSERT_EQUAL_PTR(0, 0);
dkato 0:f782d9c66c49 58 // Validate the callback
dkato 0:f782d9c66c49 59 Harness::validate_callback();
dkato 0:f782d9c66c49 60 }
dkato 0:f782d9c66c49 61 control_t test_asynchronous() {
dkato 0:f782d9c66c49 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
dkato 0:f782d9c66c49 63 // Set up a callback in the future. This may also be an interrupt!
dkato 0:f782d9c66c49 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
dkato 0:f782d9c66c49 65 // Set a 200ms timeout starting from now
dkato 0:f782d9c66c49 66 return CaseTimeout(200);
dkato 0:f782d9c66c49 67 }
dkato 0:f782d9c66c49 68
dkato 0:f782d9c66c49 69 control_t test_asynchronous_timeout(const size_t call_count) {
dkato 0:f782d9c66c49 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
dkato 0:f782d9c66c49 71 // Set a 200ms timeout starting from now,
dkato 0:f782d9c66c49 72 // but automatically repeat only this handler on timeout.
dkato 0:f782d9c66c49 73 if (call_count >= 5) {
dkato 0:f782d9c66c49 74 // but after the 5th call, the callback finally gets validated
dkato 0:f782d9c66c49 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
dkato 0:f782d9c66c49 76 }
dkato 0:f782d9c66c49 77 return CaseRepeatHandlerOnTimeout(200);
dkato 0:f782d9c66c49 78 }
dkato 0:f782d9c66c49 79
dkato 0:f782d9c66c49 80 // Custom setup handler required for proper Greentea support
dkato 0:f782d9c66c49 81 status_t greentea_setup(const size_t number_of_cases) {
dkato 0:f782d9c66c49 82 GREENTEA_SETUP(20, "default_auto");
dkato 0:f782d9c66c49 83 // Call the default reporting function
dkato 0:f782d9c66c49 84 return greentea_test_setup_handler(number_of_cases);
dkato 0:f782d9c66c49 85 }
dkato 0:f782d9c66c49 86
dkato 0:f782d9c66c49 87 // Specify all your test cases here
dkato 0:f782d9c66c49 88 Case cases[] = {
dkato 0:f782d9c66c49 89 Case("Simple Test", test_simple),
dkato 0:f782d9c66c49 90 Case("Repeating Test", test_repeats_setup, test_repeats),
dkato 0:f782d9c66c49 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
dkato 0:f782d9c66c49 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
dkato 0:f782d9c66c49 93 };
dkato 0:f782d9c66c49 94
dkato 0:f782d9c66c49 95 // Declare your test specification with a custom setup handler
dkato 0:f782d9c66c49 96 Specification specification(greentea_setup, cases);
dkato 0:f782d9c66c49 97
dkato 0:f782d9c66c49 98 void app_start(int, char**)
dkato 0:f782d9c66c49 99 { // Run the test specification
dkato 0:f782d9c66c49 100 Harness::run(specification);
dkato 0:f782d9c66c49 101 }
dkato 0:f782d9c66c49 102 ```
dkato 0:f782d9c66c49 103
dkato 0:f782d9c66c49 104 Running this test will output the following:
dkato 0:f782d9c66c49 105
dkato 0:f782d9c66c49 106 ```
dkato 0:f782d9c66c49 107 {{timeout;20}}
dkato 0:f782d9c66c49 108 {{host_test_name;default_auto}}
dkato 0:f782d9c66c49 109 {{description;utest greentea example}}
dkato 0:f782d9c66c49 110 {{test_id;MBED_OS}}
dkato 0:f782d9c66c49 111 {{start}}
dkato 0:f782d9c66c49 112 >>> Running 4 test cases...
dkato 0:f782d9c66c49 113
dkato 0:f782d9c66c49 114 >>> Running case #1: 'Simple Test'...
dkato 0:f782d9c66c49 115 Simple test called
dkato 0:f782d9c66c49 116 >>> 'Simple Test': 1 passed, 0 failed
dkato 0:f782d9c66c49 117
dkato 0:f782d9c66c49 118 >>> Running case #2: 'Repeating Test'...
dkato 0:f782d9c66c49 119 Setting up for 'Repeating Test'
dkato 0:f782d9c66c49 120 Called for the 1. time
dkato 0:f782d9c66c49 121 >>> 'Repeating Test': 1 passed, 0 failed
dkato 0:f782d9c66c49 122
dkato 0:f782d9c66c49 123 >>> Running case #2: 'Repeating Test'...
dkato 0:f782d9c66c49 124 Setting up for 'Repeating Test'
dkato 0:f782d9c66c49 125 Called for the 2. time
dkato 0:f782d9c66c49 126 >>> 'Repeating Test': 2 passed, 0 failed
dkato 0:f782d9c66c49 127
dkato 0:f782d9c66c49 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
dkato 0:f782d9c66c49 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
dkato 0:f782d9c66c49 130
dkato 0:f782d9c66c49 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
dkato 0:f782d9c66c49 132 >>> failure with reason 'Ignored: Timed Out'
dkato 0:f782d9c66c49 133 >>> failure with reason 'Ignored: Timed Out'
dkato 0:f782d9c66c49 134 >>> failure with reason 'Ignored: Timed Out'
dkato 0:f782d9c66c49 135 >>> failure with reason 'Ignored: Timed Out'
dkato 0:f782d9c66c49 136 >>> failure with reason 'Ignored: Timed Out'
dkato 0:f782d9c66c49 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
dkato 0:f782d9c66c49 138
dkato 0:f782d9c66c49 139 >>> Test cases: 4 passed, 0 failed
dkato 0:f782d9c66c49 140 {{success}}
dkato 0:f782d9c66c49 141 {{end}}
dkato 0:f782d9c66c49 142 ```
dkato 0:f782d9c66c49 143
dkato 0:f782d9c66c49 144 ## Detailed Description
dkato 0:f782d9c66c49 145
dkato 0:f782d9c66c49 146 ### Handlers
dkato 0:f782d9c66c49 147
dkato 0:f782d9c66c49 148 There are five handler types you can, but do not have to, override to customize operation.
dkato 0:f782d9c66c49 149 Please see the `utest/types.h` file for a detailed description.
dkato 0:f782d9c66c49 150
dkato 0:f782d9c66c49 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
dkato 0:f782d9c66c49 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
dkato 0:f782d9c66c49 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
dkato 0:f782d9c66c49 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
dkato 0:f782d9c66c49 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
dkato 0:f782d9c66c49 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
dkato 0:f782d9c66c49 157
dkato 0:f782d9c66c49 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
dkato 0:f782d9c66c49 159
dkato 0:f782d9c66c49 160 ### Test Case Handlers
dkato 0:f782d9c66c49 161
dkato 0:f782d9c66c49 162 There are three test case handlers:
dkato 0:f782d9c66c49 163
dkato 0:f782d9c66c49 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
dkato 0:f782d9c66c49 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
dkato 0:f782d9c66c49 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
dkato 0:f782d9c66c49 167
dkato 0:f782d9c66c49 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
dkato 0:f782d9c66c49 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
dkato 0:f782d9c66c49 170
dkato 0:f782d9c66c49 171 1. Description (required).
dkato 0:f782d9c66c49 172 1. Setup handler (optional).
dkato 0:f782d9c66c49 173 1. Test case handler (required).
dkato 0:f782d9c66c49 174 1. Teardown handler (optional).
dkato 0:f782d9c66c49 175 1. Failure handler (optional).
dkato 0:f782d9c66c49 176
dkato 0:f782d9c66c49 177 #### Test Case Attributes
dkato 0:f782d9c66c49 178
dkato 0:f782d9c66c49 179 You can modify test case behavior by returning `control_t` modifiers:
dkato 0:f782d9c66c49 180
dkato 0:f782d9c66c49 181 - `CaseNext`: never repeats and immediately moves to next test case
dkato 0:f782d9c66c49 182 - `CaseNoRepeat`: never repeats.
dkato 0:f782d9c66c49 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
dkato 0:f782d9c66c49 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
dkato 0:f782d9c66c49 185 - `CaseNoTimeout`: immediately moves to next test case.
dkato 0:f782d9c66c49 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
dkato 0:f782d9c66c49 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
dkato 0:f782d9c66c49 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
dkato 0:f782d9c66c49 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
dkato 0:f782d9c66c49 190
dkato 0:f782d9c66c49 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
dkato 0:f782d9c66c49 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
dkato 0:f782d9c66c49 193
dkato 0:f782d9c66c49 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
dkato 0:f782d9c66c49 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
dkato 0:f782d9c66c49 196
dkato 0:f782d9c66c49 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
dkato 0:f782d9c66c49 198 This will schedule the execution of the next test case.
dkato 0:f782d9c66c49 199
dkato 0:f782d9c66c49 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
dkato 0:f782d9c66c49 201
dkato 0:f782d9c66c49 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
dkato 0:f782d9c66c49 203
dkato 0:f782d9c66c49 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
dkato 0:f782d9c66c49 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
dkato 0:f782d9c66c49 206
dkato 0:f782d9c66c49 207 ### Failure Handlers
dkato 0:f782d9c66c49 208
dkato 0:f782d9c66c49 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
dkato 0:f782d9c66c49 210
dkato 0:f782d9c66c49 211 The failure reasons are:
dkato 0:f782d9c66c49 212
dkato 0:f782d9c66c49 213 - `REASON_NONE`: No failure occurred
dkato 0:f782d9c66c49 214 - `REASON_UNKNOWN`: An unknown failure occurred
dkato 0:f782d9c66c49 215 - `REASON_CASES`: A failure occurred in at least one test case
dkato 0:f782d9c66c49 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
dkato 0:f782d9c66c49 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
dkato 0:f782d9c66c49 218 - `REASON_ASSERTION`: An assertion failed
dkato 0:f782d9c66c49 219 - `REASON_TEST_SETUP`: Test setup failed
dkato 0:f782d9c66c49 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
dkato 0:f782d9c66c49 221 - `REASON_CASE_SETUP`: Case setup failed
dkato 0:f782d9c66c49 222 - `REASON_CASE_HANDLER`: Case handler failed
dkato 0:f782d9c66c49 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
dkato 0:f782d9c66c49 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
dkato 0:f782d9c66c49 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
dkato 0:f782d9c66c49 226
dkato 0:f782d9c66c49 227 The failure locations are:
dkato 0:f782d9c66c49 228
dkato 0:f782d9c66c49 229 - `LOCATION_NONE`: No location information
dkato 0:f782d9c66c49 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
dkato 0:f782d9c66c49 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
dkato 0:f782d9c66c49 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
dkato 0:f782d9c66c49 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
dkato 0:f782d9c66c49 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
dkato 0:f782d9c66c49 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
dkato 0:f782d9c66c49 236
dkato 0:f782d9c66c49 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
dkato 0:f782d9c66c49 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
dkato 0:f782d9c66c49 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
dkato 0:f782d9c66c49 240
dkato 0:f782d9c66c49 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
dkato 0:f782d9c66c49 242
dkato 0:f782d9c66c49 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
dkato 0:f782d9c66c49 244
dkato 0:f782d9c66c49 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
dkato 0:f782d9c66c49 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
dkato 0:f782d9c66c49 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
dkato 0:f782d9c66c49 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
dkato 0:f782d9c66c49 249
dkato 0:f782d9c66c49 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
dkato 0:f782d9c66c49 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
dkato 0:f782d9c66c49 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
dkato 0:f782d9c66c49 253
dkato 0:f782d9c66c49 254 ### Default Handlers
dkato 0:f782d9c66c49 255
dkato 0:f782d9c66c49 256 Three sets of default handlers with different behaviors are provided for your convenience:
dkato 0:f782d9c66c49 257
dkato 0:f782d9c66c49 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
dkato 0:f782d9c66c49 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
dkato 0:f782d9c66c49 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
dkato 0:f782d9c66c49 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
dkato 0:f782d9c66c49 262
dkato 0:f782d9c66c49 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
dkato 0:f782d9c66c49 264
dkato 0:f782d9c66c49 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
dkato 0:f782d9c66c49 266
dkato 0:f782d9c66c49 267 ```cpp
dkato 0:f782d9c66c49 268 // Declare your test specification with a custom setup handler
dkato 0:f782d9c66c49 269 // and set the default handlers to the predefined “greentea continue” behavior
dkato 0:f782d9c66c49 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
dkato 0:f782d9c66c49 271 ```
dkato 0:f782d9c66c49 272
dkato 0:f782d9c66c49 273 ### Custom Handlers
dkato 0:f782d9c66c49 274
dkato 0:f782d9c66c49 275 You may override any of the default handlers with your own custom handler.
dkato 0:f782d9c66c49 276
dkato 0:f782d9c66c49 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
dkato 0:f782d9c66c49 278 To explicitly invoke the default handler, use the `default_handler` hint.
dkato 0:f782d9c66c49 279
dkato 0:f782d9c66c49 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
dkato 0:f782d9c66c49 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
dkato 0:f782d9c66c49 282
dkato 0:f782d9c66c49 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
dkato 0:f782d9c66c49 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
dkato 0:f782d9c66c49 285
dkato 0:f782d9c66c49 286 For the `Specification` class the order of arguments is:
dkato 0:f782d9c66c49 287
dkato 0:f782d9c66c49 288 1. Test setup handler (optional).
dkato 0:f782d9c66c49 289 1. Array of test cases (required).
dkato 0:f782d9c66c49 290 1. Test teardown handler (optional).
dkato 0:f782d9c66c49 291 1. Test failure handler (optional).
dkato 0:f782d9c66c49 292 1. Default handlers (optional).
dkato 0:f782d9c66c49 293
dkato 0:f782d9c66c49 294 ### Test Case Attribute Arbitration
dkato 0:f782d9c66c49 295
dkato 0:f782d9c66c49 296 When adding conflicting modifiers together
dkato 0:f782d9c66c49 297
dkato 0:f782d9c66c49 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
dkato 0:f782d9c66c49 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
dkato 0:f782d9c66c49 300 - `CaseNext` always wins arbitration.
dkato 0:f782d9c66c49 301
dkato 0:f782d9c66c49 302 The following table shows this arbitration logic in detail:
dkato 0:f782d9c66c49 303
dkato 0:f782d9c66c49 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
dkato 0:f782d9c66c49 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
dkato 0:f782d9c66c49 306 | `CaseNext` | no repeat &<br> no timeout
dkato 0:f782d9c66c49 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
dkato 0:f782d9c66c49 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
dkato 0:f782d9c66c49 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
dkato 0:f782d9c66c49 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
dkato 0:f782d9c66c49 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
dkato 0:f782d9c66c49 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
dkato 0:f782d9c66c49 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
dkato 0:f782d9c66c49 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
dkato 0:f782d9c66c49 315
dkato 0:f782d9c66c49 316 ### Atomicity
dkato 0:f782d9c66c49 317
dkato 0:f782d9c66c49 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
dkato 0:f782d9c66c49 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
dkato 0:f782d9c66c49 320
dkato 0:f782d9c66c49 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
dkato 0:f782d9c66c49 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
dkato 0:f782d9c66c49 323
dkato 0:f782d9c66c49 324 ### Custom Scheduler
dkato 0:f782d9c66c49 325
dkato 0:f782d9c66c49 326 By default, a Timeout object is used for scheduling the harness operations.
dkato 0:f782d9c66c49 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
dkato 0:f782d9c66c49 328
dkato 0:f782d9c66c49 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
dkato 0:f782d9c66c49 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
dkato 0:f782d9c66c49 331
dkato 0:f782d9c66c49 332 There are two functions you need to implement:
dkato 0:f782d9c66c49 333
dkato 0:f782d9c66c49 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
dkato 0:f782d9c66c49 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
dkato 0:f782d9c66c49 336
dkato 0:f782d9c66c49 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
dkato 0:f782d9c66c49 338
dkato 0:f782d9c66c49 339 ### Example Synchronous Scheduler
dkato 0:f782d9c66c49 340
dkato 0:f782d9c66c49 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
dkato 0:f782d9c66c49 342 ```cpp
dkato 0:f782d9c66c49 343 volatile utest_v1_harness_callback_t minimal_callback;
dkato 0:f782d9c66c49 344
dkato 0:f782d9c66c49 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
dkato 0:f782d9c66c49 346 minimal_callback = callback;
dkato 0:f782d9c66c49 347 // this scheduler does not support asynchronous callbacks
dkato 0:f782d9c66c49 348 return (delay_ms ? NULL : (void*)1);
dkato 0:f782d9c66c49 349 }
dkato 0:f782d9c66c49 350 static int32_t utest_minimal_cancel(void*) {
dkato 0:f782d9c66c49 351 return -1; // canceling not supported either
dkato 0:f782d9c66c49 352 }
dkato 0:f782d9c66c49 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
dkato 0:f782d9c66c49 354
dkato 0:f782d9c66c49 355 // [...] Add your test cases and specification here.
dkato 0:f782d9c66c49 356
dkato 0:f782d9c66c49 357 void main() // or whatever your custom entry point is
dkato 0:f782d9c66c49 358 {
dkato 0:f782d9c66c49 359 // You MUST set the custom scheduler before running the specification.
dkato 0:f782d9c66c49 360 Harness::set_scheduler(utest_minimal_scheduler);
dkato 0:f782d9c66c49 361 Harness::run(specification);
dkato 0:f782d9c66c49 362
dkato 0:f782d9c66c49 363 while(1) {
dkato 0:f782d9c66c49 364 if (minimal_callback) {
dkato 0:f782d9c66c49 365 // copy the callback and reset the shared memory
dkato 0:f782d9c66c49 366 utest_v1_harness_callback_t callback = minimal_callback;
dkato 0:f782d9c66c49 367 minimal_callback = NULL;
dkato 0:f782d9c66c49 368 callback(); // execute the copied callback
dkato 0:f782d9c66c49 369 }
dkato 0:f782d9c66c49 370 }
dkato 0:f782d9c66c49 371 }
dkato 0:f782d9c66c49 372 ```
dkato 0:f782d9c66c49 373
dkato 0:f782d9c66c49 374 ### Example Asynchronous Scheduler
dkato 0:f782d9c66c49 375
dkato 0:f782d9c66c49 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
dkato 0:f782d9c66c49 377 ```cpp
dkato 0:f782d9c66c49 378 volatile utest_v1_harness_callback_t minimal_callback;
dkato 0:f782d9c66c49 379 volatile utest_v1_harness_callback_t ticker_callback;
dkato 0:f782d9c66c49 380 const ticker_data_t *ticker_data;
dkato 0:f782d9c66c49 381 ticker_event_t ticker_event;
dkato 0:f782d9c66c49 382
dkato 0:f782d9c66c49 383 static void ticker_handler(uint32_t) {
dkato 0:f782d9c66c49 384 minimal_callback = ticker_callback; // interrupt context!
dkato 0:f782d9c66c49 385 }
dkato 0:f782d9c66c49 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
dkato 0:f782d9c66c49 387 if (delay_ms) {
dkato 0:f782d9c66c49 388 ticker_callback = callback;
dkato 0:f782d9c66c49 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
dkato 0:f782d9c66c49 390 }
dkato 0:f782d9c66c49 391 else minimal_callback = callback;
dkato 0:f782d9c66c49 392 return (void*)1;
dkato 0:f782d9c66c49 393 }
dkato 0:f782d9c66c49 394 static int32_t utest_minimal_cancel(void*) {
dkato 0:f782d9c66c49 395 ticker_remove_event(ticker_data, &ticker_event);
dkato 0:f782d9c66c49 396 return 0; // canceling is supported
dkato 0:f782d9c66c49 397 }
dkato 0:f782d9c66c49 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
dkato 0:f782d9c66c49 399
dkato 0:f782d9c66c49 400 // [...] Add your test cases and specification here.
dkato 0:f782d9c66c49 401
dkato 0:f782d9c66c49 402 void main() // or whatever your custom entry point is
dkato 0:f782d9c66c49 403 {
dkato 0:f782d9c66c49 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
dkato 0:f782d9c66c49 405 ticker_set_handler(ticker_data, ticker_handler);
dkato 0:f782d9c66c49 406 // You MUST set the custom scheduler before running the specification.
dkato 0:f782d9c66c49 407 Harness::set_scheduler(utest_minimal_scheduler);
dkato 0:f782d9c66c49 408 Harness::run(specification);
dkato 0:f782d9c66c49 409
dkato 0:f782d9c66c49 410 while(1) {
dkato 0:f782d9c66c49 411 if (minimal_callback) {
dkato 0:f782d9c66c49 412 // copy the callback and reset the shared memory
dkato 0:f782d9c66c49 413 utest_v1_harness_callback_t callback = minimal_callback;
dkato 0:f782d9c66c49 414 minimal_callback = NULL;
dkato 0:f782d9c66c49 415 callback(); // execute the copied callback
dkato 0:f782d9c66c49 416 }
dkato 0:f782d9c66c49 417 }
dkato 0:f782d9c66c49 418 }
dkato 0:f782d9c66c49 419 ```