Magnificent7 / Hackathon

Includes library modifications to allow access to AIN_4 (AIN_0 / 5)

mbd_os/features/frameworks/utest/README.md@0:eafc3fd41f75, 2016-09-20 (annotated)

Committer:
bryantaylor
Date:
Tue Sep 20 21:26:12 2016 +0000
Revision:
0:eafc3fd41f75
hackathon

Who changed what in which revision?

UserRevisionLine numberNew contents of line
bryantaylor 0:eafc3fd41f75 1 # utest: Asynchronous C++ Test Harness
bryantaylor 0:eafc3fd41f75 2
bryantaylor 0:eafc3fd41f75 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
bryantaylor 0:eafc3fd41f75 4
bryantaylor 0:eafc3fd41f75 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
bryantaylor 0:eafc3fd41f75 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
bryantaylor 0:eafc3fd41f75 7
bryantaylor 0:eafc3fd41f75 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
bryantaylor 0:eafc3fd41f75 9
bryantaylor 0:eafc3fd41f75 10 ## Theory of Operation
bryantaylor 0:eafc3fd41f75 11
bryantaylor 0:eafc3fd41f75 12 A test specification contains a setup handler, several test cases and a teardown handler.
bryantaylor 0:eafc3fd41f75 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
bryantaylor 0:eafc3fd41f75 14
bryantaylor 0:eafc3fd41f75 15 The order of handler execution is:
bryantaylor 0:eafc3fd41f75 16
bryantaylor 0:eafc3fd41f75 17 1. Test setup handler.
bryantaylor 0:eafc3fd41f75 18 1. For each test case:
bryantaylor 0:eafc3fd41f75 19 1. Test case setup handler.
bryantaylor 0:eafc3fd41f75 20 1. Test case execution handler.
bryantaylor 0:eafc3fd41f75 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
bryantaylor 0:eafc3fd41f75 22 1. (repeat test case execution handler if specified.)
bryantaylor 0:eafc3fd41f75 23 1. Test case teardown handler.
bryantaylor 0:eafc3fd41f75 24 1. Test teardown handler.
bryantaylor 0:eafc3fd41f75 25
bryantaylor 0:eafc3fd41f75 26 ## Example
bryantaylor 0:eafc3fd41f75 27
bryantaylor 0:eafc3fd41f75 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
bryantaylor 0:eafc3fd41f75 29
bryantaylor 0:eafc3fd41f75 30 ```cpp
bryantaylor 0:eafc3fd41f75 31 #include "mbed-drivers/test_env.h"
bryantaylor 0:eafc3fd41f75 32 #include "utest/utest.h"
bryantaylor 0:eafc3fd41f75 33 #include "unity/unity.h"
bryantaylor 0:eafc3fd41f75 34
bryantaylor 0:eafc3fd41f75 35 using namespace utest::v1;
bryantaylor 0:eafc3fd41f75 36
bryantaylor 0:eafc3fd41f75 37 void test_simple() {
bryantaylor 0:eafc3fd41f75 38 TEST_ASSERT_EQUAL(0, 0);
bryantaylor 0:eafc3fd41f75 39 printf("Simple test called\n");
bryantaylor 0:eafc3fd41f75 40 }
bryantaylor 0:eafc3fd41f75 41
bryantaylor 0:eafc3fd41f75 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
bryantaylor 0:eafc3fd41f75 43 // Call the default handler for proper reporting
bryantaylor 0:eafc3fd41f75 44 status_t status = greentea_case_setup_handler(source, index_of_case);
bryantaylor 0:eafc3fd41f75 45 printf("Setting up for '%s'\n", source->get_description());
bryantaylor 0:eafc3fd41f75 46 return status;
bryantaylor 0:eafc3fd41f75 47 }
bryantaylor 0:eafc3fd41f75 48 control_t test_repeats(const size_t call_count) {
bryantaylor 0:eafc3fd41f75 49 printf("Called for the %u. time\n", call_count);
bryantaylor 0:eafc3fd41f75 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
bryantaylor 0:eafc3fd41f75 51 // Specify how often this test is repeated ie. n total calls
bryantaylor 0:eafc3fd41f75 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
bryantaylor 0:eafc3fd41f75 53 }
bryantaylor 0:eafc3fd41f75 54
bryantaylor 0:eafc3fd41f75 55 void test_callback_validate() {
bryantaylor 0:eafc3fd41f75 56 // You may also use assertions here!
bryantaylor 0:eafc3fd41f75 57 TEST_ASSERT_EQUAL_PTR(0, 0);
bryantaylor 0:eafc3fd41f75 58 // Validate the callback
bryantaylor 0:eafc3fd41f75 59 Harness::validate_callback();
bryantaylor 0:eafc3fd41f75 60 }
bryantaylor 0:eafc3fd41f75 61 control_t test_asynchronous() {
bryantaylor 0:eafc3fd41f75 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
bryantaylor 0:eafc3fd41f75 63 // Set up a callback in the future. This may also be an interrupt!
bryantaylor 0:eafc3fd41f75 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
bryantaylor 0:eafc3fd41f75 65 // Set a 200ms timeout starting from now
bryantaylor 0:eafc3fd41f75 66 return CaseTimeout(200);
bryantaylor 0:eafc3fd41f75 67 }
bryantaylor 0:eafc3fd41f75 68
bryantaylor 0:eafc3fd41f75 69 control_t test_asynchronous_timeout(const size_t call_count) {
bryantaylor 0:eafc3fd41f75 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
bryantaylor 0:eafc3fd41f75 71 // Set a 200ms timeout starting from now,
bryantaylor 0:eafc3fd41f75 72 // but automatically repeat only this handler on timeout.
bryantaylor 0:eafc3fd41f75 73 if (call_count >= 5) {
bryantaylor 0:eafc3fd41f75 74 // but after the 5th call, the callback finally gets validated
bryantaylor 0:eafc3fd41f75 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
bryantaylor 0:eafc3fd41f75 76 }
bryantaylor 0:eafc3fd41f75 77 return CaseRepeatHandlerOnTimeout(200);
bryantaylor 0:eafc3fd41f75 78 }
bryantaylor 0:eafc3fd41f75 79
bryantaylor 0:eafc3fd41f75 80 // Custom setup handler required for proper Greentea support
bryantaylor 0:eafc3fd41f75 81 status_t greentea_setup(const size_t number_of_cases) {
bryantaylor 0:eafc3fd41f75 82 GREENTEA_SETUP(20, "default_auto");
bryantaylor 0:eafc3fd41f75 83 // Call the default reporting function
bryantaylor 0:eafc3fd41f75 84 return greentea_test_setup_handler(number_of_cases);
bryantaylor 0:eafc3fd41f75 85 }
bryantaylor 0:eafc3fd41f75 86
bryantaylor 0:eafc3fd41f75 87 // Specify all your test cases here
bryantaylor 0:eafc3fd41f75 88 Case cases[] = {
bryantaylor 0:eafc3fd41f75 89 Case("Simple Test", test_simple),
bryantaylor 0:eafc3fd41f75 90 Case("Repeating Test", test_repeats_setup, test_repeats),
bryantaylor 0:eafc3fd41f75 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
bryantaylor 0:eafc3fd41f75 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
bryantaylor 0:eafc3fd41f75 93 };
bryantaylor 0:eafc3fd41f75 94
bryantaylor 0:eafc3fd41f75 95 // Declare your test specification with a custom setup handler
bryantaylor 0:eafc3fd41f75 96 Specification specification(greentea_setup, cases);
bryantaylor 0:eafc3fd41f75 97
bryantaylor 0:eafc3fd41f75 98 void app_start(int, char**)
bryantaylor 0:eafc3fd41f75 99 { // Run the test specification
bryantaylor 0:eafc3fd41f75 100 Harness::run(specification);
bryantaylor 0:eafc3fd41f75 101 }
bryantaylor 0:eafc3fd41f75 102 ```
bryantaylor 0:eafc3fd41f75 103
bryantaylor 0:eafc3fd41f75 104 Running this test will output the following:
bryantaylor 0:eafc3fd41f75 105
bryantaylor 0:eafc3fd41f75 106 ```
bryantaylor 0:eafc3fd41f75 107 {{timeout;20}}
bryantaylor 0:eafc3fd41f75 108 {{host_test_name;default_auto}}
bryantaylor 0:eafc3fd41f75 109 {{description;utest greentea example}}
bryantaylor 0:eafc3fd41f75 110 {{test_id;MBED_OS}}
bryantaylor 0:eafc3fd41f75 111 {{start}}
bryantaylor 0:eafc3fd41f75 112 >>> Running 4 test cases...
bryantaylor 0:eafc3fd41f75 113
bryantaylor 0:eafc3fd41f75 114 >>> Running case #1: 'Simple Test'...
bryantaylor 0:eafc3fd41f75 115 Simple test called
bryantaylor 0:eafc3fd41f75 116 >>> 'Simple Test': 1 passed, 0 failed
bryantaylor 0:eafc3fd41f75 117
bryantaylor 0:eafc3fd41f75 118 >>> Running case #2: 'Repeating Test'...
bryantaylor 0:eafc3fd41f75 119 Setting up for 'Repeating Test'
bryantaylor 0:eafc3fd41f75 120 Called for the 1. time
bryantaylor 0:eafc3fd41f75 121 >>> 'Repeating Test': 1 passed, 0 failed
bryantaylor 0:eafc3fd41f75 122
bryantaylor 0:eafc3fd41f75 123 >>> Running case #2: 'Repeating Test'...
bryantaylor 0:eafc3fd41f75 124 Setting up for 'Repeating Test'
bryantaylor 0:eafc3fd41f75 125 Called for the 2. time
bryantaylor 0:eafc3fd41f75 126 >>> 'Repeating Test': 2 passed, 0 failed
bryantaylor 0:eafc3fd41f75 127
bryantaylor 0:eafc3fd41f75 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
bryantaylor 0:eafc3fd41f75 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
bryantaylor 0:eafc3fd41f75 130
bryantaylor 0:eafc3fd41f75 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
bryantaylor 0:eafc3fd41f75 132 >>> failure with reason 'Ignored: Timed Out'
bryantaylor 0:eafc3fd41f75 133 >>> failure with reason 'Ignored: Timed Out'
bryantaylor 0:eafc3fd41f75 134 >>> failure with reason 'Ignored: Timed Out'
bryantaylor 0:eafc3fd41f75 135 >>> failure with reason 'Ignored: Timed Out'
bryantaylor 0:eafc3fd41f75 136 >>> failure with reason 'Ignored: Timed Out'
bryantaylor 0:eafc3fd41f75 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
bryantaylor 0:eafc3fd41f75 138
bryantaylor 0:eafc3fd41f75 139 >>> Test cases: 4 passed, 0 failed
bryantaylor 0:eafc3fd41f75 140 {{success}}
bryantaylor 0:eafc3fd41f75 141 {{end}}
bryantaylor 0:eafc3fd41f75 142 ```
bryantaylor 0:eafc3fd41f75 143
bryantaylor 0:eafc3fd41f75 144 ## Detailed Description
bryantaylor 0:eafc3fd41f75 145
bryantaylor 0:eafc3fd41f75 146 ### Handlers
bryantaylor 0:eafc3fd41f75 147
bryantaylor 0:eafc3fd41f75 148 There are five handler types you can, but do not have to, override to customize operation.
bryantaylor 0:eafc3fd41f75 149 Please see the `utest/types.h` file for a detailed description.
bryantaylor 0:eafc3fd41f75 150
bryantaylor 0:eafc3fd41f75 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
bryantaylor 0:eafc3fd41f75 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
bryantaylor 0:eafc3fd41f75 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
bryantaylor 0:eafc3fd41f75 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
bryantaylor 0:eafc3fd41f75 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
bryantaylor 0:eafc3fd41f75 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
bryantaylor 0:eafc3fd41f75 157
bryantaylor 0:eafc3fd41f75 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
bryantaylor 0:eafc3fd41f75 159
bryantaylor 0:eafc3fd41f75 160 ### Test Case Handlers
bryantaylor 0:eafc3fd41f75 161
bryantaylor 0:eafc3fd41f75 162 There are three test case handlers:
bryantaylor 0:eafc3fd41f75 163
bryantaylor 0:eafc3fd41f75 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
bryantaylor 0:eafc3fd41f75 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
bryantaylor 0:eafc3fd41f75 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
bryantaylor 0:eafc3fd41f75 167
bryantaylor 0:eafc3fd41f75 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
bryantaylor 0:eafc3fd41f75 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
bryantaylor 0:eafc3fd41f75 170
bryantaylor 0:eafc3fd41f75 171 1. Description (required).
bryantaylor 0:eafc3fd41f75 172 1. Setup handler (optional).
bryantaylor 0:eafc3fd41f75 173 1. Test case handler (required).
bryantaylor 0:eafc3fd41f75 174 1. Teardown handler (optional).
bryantaylor 0:eafc3fd41f75 175 1. Failure handler (optional).
bryantaylor 0:eafc3fd41f75 176
bryantaylor 0:eafc3fd41f75 177 #### Test Case Attributes
bryantaylor 0:eafc3fd41f75 178
bryantaylor 0:eafc3fd41f75 179 You can modify test case behavior by returning `control_t` modifiers:
bryantaylor 0:eafc3fd41f75 180
bryantaylor 0:eafc3fd41f75 181 - `CaseNext`: never repeats and immediately moves to next test case
bryantaylor 0:eafc3fd41f75 182 - `CaseNoRepeat`: never repeats.
bryantaylor 0:eafc3fd41f75 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
bryantaylor 0:eafc3fd41f75 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
bryantaylor 0:eafc3fd41f75 185 - `CaseNoTimeout`: immediately moves to next test case.
bryantaylor 0:eafc3fd41f75 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
bryantaylor 0:eafc3fd41f75 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
bryantaylor 0:eafc3fd41f75 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
bryantaylor 0:eafc3fd41f75 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
bryantaylor 0:eafc3fd41f75 190
bryantaylor 0:eafc3fd41f75 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
bryantaylor 0:eafc3fd41f75 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
bryantaylor 0:eafc3fd41f75 193
bryantaylor 0:eafc3fd41f75 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
bryantaylor 0:eafc3fd41f75 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
bryantaylor 0:eafc3fd41f75 196
bryantaylor 0:eafc3fd41f75 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
bryantaylor 0:eafc3fd41f75 198 This will schedule the execution of the next test case.
bryantaylor 0:eafc3fd41f75 199
bryantaylor 0:eafc3fd41f75 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
bryantaylor 0:eafc3fd41f75 201
bryantaylor 0:eafc3fd41f75 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
bryantaylor 0:eafc3fd41f75 203
bryantaylor 0:eafc3fd41f75 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
bryantaylor 0:eafc3fd41f75 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
bryantaylor 0:eafc3fd41f75 206
bryantaylor 0:eafc3fd41f75 207 ### Failure Handlers
bryantaylor 0:eafc3fd41f75 208
bryantaylor 0:eafc3fd41f75 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
bryantaylor 0:eafc3fd41f75 210
bryantaylor 0:eafc3fd41f75 211 The failure reasons are:
bryantaylor 0:eafc3fd41f75 212
bryantaylor 0:eafc3fd41f75 213 - `REASON_NONE`: No failure occurred
bryantaylor 0:eafc3fd41f75 214 - `REASON_UNKNOWN`: An unknown failure occurred
bryantaylor 0:eafc3fd41f75 215 - `REASON_CASES`: A failure occurred in at least one test case
bryantaylor 0:eafc3fd41f75 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
bryantaylor 0:eafc3fd41f75 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
bryantaylor 0:eafc3fd41f75 218 - `REASON_ASSERTION`: An assertion failed
bryantaylor 0:eafc3fd41f75 219 - `REASON_TEST_SETUP`: Test setup failed
bryantaylor 0:eafc3fd41f75 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
bryantaylor 0:eafc3fd41f75 221 - `REASON_CASE_SETUP`: Case setup failed
bryantaylor 0:eafc3fd41f75 222 - `REASON_CASE_HANDLER`: Case handler failed
bryantaylor 0:eafc3fd41f75 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
bryantaylor 0:eafc3fd41f75 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
bryantaylor 0:eafc3fd41f75 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
bryantaylor 0:eafc3fd41f75 226
bryantaylor 0:eafc3fd41f75 227 The failure locations are:
bryantaylor 0:eafc3fd41f75 228
bryantaylor 0:eafc3fd41f75 229 - `LOCATION_NONE`: No location information
bryantaylor 0:eafc3fd41f75 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
bryantaylor 0:eafc3fd41f75 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
bryantaylor 0:eafc3fd41f75 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
bryantaylor 0:eafc3fd41f75 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
bryantaylor 0:eafc3fd41f75 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
bryantaylor 0:eafc3fd41f75 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
bryantaylor 0:eafc3fd41f75 236
bryantaylor 0:eafc3fd41f75 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
bryantaylor 0:eafc3fd41f75 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
bryantaylor 0:eafc3fd41f75 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
bryantaylor 0:eafc3fd41f75 240
bryantaylor 0:eafc3fd41f75 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
bryantaylor 0:eafc3fd41f75 242
bryantaylor 0:eafc3fd41f75 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
bryantaylor 0:eafc3fd41f75 244
bryantaylor 0:eafc3fd41f75 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
bryantaylor 0:eafc3fd41f75 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
bryantaylor 0:eafc3fd41f75 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
bryantaylor 0:eafc3fd41f75 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
bryantaylor 0:eafc3fd41f75 249
bryantaylor 0:eafc3fd41f75 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
bryantaylor 0:eafc3fd41f75 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
bryantaylor 0:eafc3fd41f75 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
bryantaylor 0:eafc3fd41f75 253
bryantaylor 0:eafc3fd41f75 254 ### Default Handlers
bryantaylor 0:eafc3fd41f75 255
bryantaylor 0:eafc3fd41f75 256 Three sets of default handlers with different behaviors are provided for your convenience:
bryantaylor 0:eafc3fd41f75 257
bryantaylor 0:eafc3fd41f75 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
bryantaylor 0:eafc3fd41f75 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
bryantaylor 0:eafc3fd41f75 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
bryantaylor 0:eafc3fd41f75 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
bryantaylor 0:eafc3fd41f75 262
bryantaylor 0:eafc3fd41f75 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
bryantaylor 0:eafc3fd41f75 264
bryantaylor 0:eafc3fd41f75 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
bryantaylor 0:eafc3fd41f75 266
bryantaylor 0:eafc3fd41f75 267 ```cpp
bryantaylor 0:eafc3fd41f75 268 // Declare your test specification with a custom setup handler
bryantaylor 0:eafc3fd41f75 269 // and set the default handlers to the predefined “greentea continue” behavior
bryantaylor 0:eafc3fd41f75 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
bryantaylor 0:eafc3fd41f75 271 ```
bryantaylor 0:eafc3fd41f75 272
bryantaylor 0:eafc3fd41f75 273 ### Custom Handlers
bryantaylor 0:eafc3fd41f75 274
bryantaylor 0:eafc3fd41f75 275 You may override any of the default handlers with your own custom handler.
bryantaylor 0:eafc3fd41f75 276
bryantaylor 0:eafc3fd41f75 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
bryantaylor 0:eafc3fd41f75 278 To explicitly invoke the default handler, use the `default_handler` hint.
bryantaylor 0:eafc3fd41f75 279
bryantaylor 0:eafc3fd41f75 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
bryantaylor 0:eafc3fd41f75 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
bryantaylor 0:eafc3fd41f75 282
bryantaylor 0:eafc3fd41f75 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
bryantaylor 0:eafc3fd41f75 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
bryantaylor 0:eafc3fd41f75 285
bryantaylor 0:eafc3fd41f75 286 For the `Specification` class the order of arguments is:
bryantaylor 0:eafc3fd41f75 287
bryantaylor 0:eafc3fd41f75 288 1. Test setup handler (optional).
bryantaylor 0:eafc3fd41f75 289 1. Array of test cases (required).
bryantaylor 0:eafc3fd41f75 290 1. Test teardown handler (optional).
bryantaylor 0:eafc3fd41f75 291 1. Test failure handler (optional).
bryantaylor 0:eafc3fd41f75 292 1. Default handlers (optional).
bryantaylor 0:eafc3fd41f75 293
bryantaylor 0:eafc3fd41f75 294 ### Test Case Attribute Arbitration
bryantaylor 0:eafc3fd41f75 295
bryantaylor 0:eafc3fd41f75 296 When adding conflicting modifiers together
bryantaylor 0:eafc3fd41f75 297
bryantaylor 0:eafc3fd41f75 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
bryantaylor 0:eafc3fd41f75 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
bryantaylor 0:eafc3fd41f75 300 - `CaseNext` always wins arbitration.
bryantaylor 0:eafc3fd41f75 301
bryantaylor 0:eafc3fd41f75 302 The following table shows this arbitration logic in detail:
bryantaylor 0:eafc3fd41f75 303
bryantaylor 0:eafc3fd41f75 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
bryantaylor 0:eafc3fd41f75 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
bryantaylor 0:eafc3fd41f75 306 | `CaseNext` | no repeat &<br> no timeout
bryantaylor 0:eafc3fd41f75 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
bryantaylor 0:eafc3fd41f75 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
bryantaylor 0:eafc3fd41f75 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
bryantaylor 0:eafc3fd41f75 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
bryantaylor 0:eafc3fd41f75 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
bryantaylor 0:eafc3fd41f75 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
bryantaylor 0:eafc3fd41f75 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
bryantaylor 0:eafc3fd41f75 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
bryantaylor 0:eafc3fd41f75 315
bryantaylor 0:eafc3fd41f75 316 ### Atomicity
bryantaylor 0:eafc3fd41f75 317
bryantaylor 0:eafc3fd41f75 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
bryantaylor 0:eafc3fd41f75 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
bryantaylor 0:eafc3fd41f75 320
bryantaylor 0:eafc3fd41f75 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
bryantaylor 0:eafc3fd41f75 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
bryantaylor 0:eafc3fd41f75 323
bryantaylor 0:eafc3fd41f75 324 ### Custom Scheduler
bryantaylor 0:eafc3fd41f75 325
bryantaylor 0:eafc3fd41f75 326 By default, a Timeout object is used for scheduling the harness operations.
bryantaylor 0:eafc3fd41f75 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
bryantaylor 0:eafc3fd41f75 328
bryantaylor 0:eafc3fd41f75 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
bryantaylor 0:eafc3fd41f75 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
bryantaylor 0:eafc3fd41f75 331
bryantaylor 0:eafc3fd41f75 332 There are two functions you need to implement:
bryantaylor 0:eafc3fd41f75 333
bryantaylor 0:eafc3fd41f75 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
bryantaylor 0:eafc3fd41f75 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
bryantaylor 0:eafc3fd41f75 336
bryantaylor 0:eafc3fd41f75 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
bryantaylor 0:eafc3fd41f75 338
bryantaylor 0:eafc3fd41f75 339 ### Example Synchronous Scheduler
bryantaylor 0:eafc3fd41f75 340
bryantaylor 0:eafc3fd41f75 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
bryantaylor 0:eafc3fd41f75 342 ```cpp
bryantaylor 0:eafc3fd41f75 343 volatile utest_v1_harness_callback_t minimal_callback;
bryantaylor 0:eafc3fd41f75 344
bryantaylor 0:eafc3fd41f75 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
bryantaylor 0:eafc3fd41f75 346 minimal_callback = callback;
bryantaylor 0:eafc3fd41f75 347 // this scheduler does not support asynchronous callbacks
bryantaylor 0:eafc3fd41f75 348 return (delay_ms ? NULL : (void*)1);
bryantaylor 0:eafc3fd41f75 349 }
bryantaylor 0:eafc3fd41f75 350 static int32_t utest_minimal_cancel(void*) {
bryantaylor 0:eafc3fd41f75 351 return -1; // canceling not supported either
bryantaylor 0:eafc3fd41f75 352 }
bryantaylor 0:eafc3fd41f75 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
bryantaylor 0:eafc3fd41f75 354
bryantaylor 0:eafc3fd41f75 355 // [...] Add your test cases and specification here.
bryantaylor 0:eafc3fd41f75 356
bryantaylor 0:eafc3fd41f75 357 void main() // or whatever your custom entry point is
bryantaylor 0:eafc3fd41f75 358 {
bryantaylor 0:eafc3fd41f75 359 // You MUST set the custom scheduler before running the specification.
bryantaylor 0:eafc3fd41f75 360 Harness::set_scheduler(utest_minimal_scheduler);
bryantaylor 0:eafc3fd41f75 361 Harness::run(specification);
bryantaylor 0:eafc3fd41f75 362
bryantaylor 0:eafc3fd41f75 363 while(1) {
bryantaylor 0:eafc3fd41f75 364 if (minimal_callback) {
bryantaylor 0:eafc3fd41f75 365 // copy the callback and reset the shared memory
bryantaylor 0:eafc3fd41f75 366 utest_v1_harness_callback_t callback = minimal_callback;
bryantaylor 0:eafc3fd41f75 367 minimal_callback = NULL;
bryantaylor 0:eafc3fd41f75 368 callback(); // execute the copied callback
bryantaylor 0:eafc3fd41f75 369 }
bryantaylor 0:eafc3fd41f75 370 }
bryantaylor 0:eafc3fd41f75 371 }
bryantaylor 0:eafc3fd41f75 372 ```
bryantaylor 0:eafc3fd41f75 373
bryantaylor 0:eafc3fd41f75 374 ### Example Asynchronous Scheduler
bryantaylor 0:eafc3fd41f75 375
bryantaylor 0:eafc3fd41f75 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
bryantaylor 0:eafc3fd41f75 377 ```cpp
bryantaylor 0:eafc3fd41f75 378 volatile utest_v1_harness_callback_t minimal_callback;
bryantaylor 0:eafc3fd41f75 379 volatile utest_v1_harness_callback_t ticker_callback;
bryantaylor 0:eafc3fd41f75 380 const ticker_data_t *ticker_data;
bryantaylor 0:eafc3fd41f75 381 ticker_event_t ticker_event;
bryantaylor 0:eafc3fd41f75 382
bryantaylor 0:eafc3fd41f75 383 static void ticker_handler(uint32_t) {
bryantaylor 0:eafc3fd41f75 384 minimal_callback = ticker_callback; // interrupt context!
bryantaylor 0:eafc3fd41f75 385 }
bryantaylor 0:eafc3fd41f75 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
bryantaylor 0:eafc3fd41f75 387 if (delay_ms) {
bryantaylor 0:eafc3fd41f75 388 ticker_callback = callback;
bryantaylor 0:eafc3fd41f75 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
bryantaylor 0:eafc3fd41f75 390 }
bryantaylor 0:eafc3fd41f75 391 else minimal_callback = callback;
bryantaylor 0:eafc3fd41f75 392 return (void*)1;
bryantaylor 0:eafc3fd41f75 393 }
bryantaylor 0:eafc3fd41f75 394 static int32_t utest_minimal_cancel(void*) {
bryantaylor 0:eafc3fd41f75 395 ticker_remove_event(ticker_data, &ticker_event);
bryantaylor 0:eafc3fd41f75 396 return 0; // canceling is supported
bryantaylor 0:eafc3fd41f75 397 }
bryantaylor 0:eafc3fd41f75 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
bryantaylor 0:eafc3fd41f75 399
bryantaylor 0:eafc3fd41f75 400 // [...] Add your test cases and specification here.
bryantaylor 0:eafc3fd41f75 401
bryantaylor 0:eafc3fd41f75 402 void main() // or whatever your custom entry point is
bryantaylor 0:eafc3fd41f75 403 {
bryantaylor 0:eafc3fd41f75 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
bryantaylor 0:eafc3fd41f75 405 ticker_set_handler(ticker_data, ticker_handler);
bryantaylor 0:eafc3fd41f75 406 // You MUST set the custom scheduler before running the specification.
bryantaylor 0:eafc3fd41f75 407 Harness::set_scheduler(utest_minimal_scheduler);
bryantaylor 0:eafc3fd41f75 408 Harness::run(specification);
bryantaylor 0:eafc3fd41f75 409
bryantaylor 0:eafc3fd41f75 410 while(1) {
bryantaylor 0:eafc3fd41f75 411 if (minimal_callback) {
bryantaylor 0:eafc3fd41f75 412 // copy the callback and reset the shared memory
bryantaylor 0:eafc3fd41f75 413 utest_v1_harness_callback_t callback = minimal_callback;
bryantaylor 0:eafc3fd41f75 414 minimal_callback = NULL;
bryantaylor 0:eafc3fd41f75 415 callback(); // execute the copied callback
bryantaylor 0:eafc3fd41f75 416 }
bryantaylor 0:eafc3fd41f75 417 }
bryantaylor 0:eafc3fd41f75 418 }
bryantaylor 0:eafc3fd41f75 419 ```