Marco Zecchini / Mbed OS Example_RTOS

Rtos API example

mbed-os/features/frameworks/utest/README.md@0:9fca2b23d0ba, 2019-02-23 (annotated)

Committer:
marcozecchini
Date:
Sat Feb 23 12:13:36 2019 +0000
Revision:
0:9fca2b23d0ba
final commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
marcozecchini 0:9fca2b23d0ba 1 # utest: Asynchronous C++ Test Harness
marcozecchini 0:9fca2b23d0ba 2
marcozecchini 0:9fca2b23d0ba 3 This test harness allows you to execute a specified series of (asynchronous) C++ test cases with sensible default reporting and useful customization options.
marcozecchini 0:9fca2b23d0ba 4
marcozecchini 0:9fca2b23d0ba 5 Please note that this is a purposefully lean test harness, only dealing with test execution and providing default reporting handlers. It specifically does not support auto-discovery of test cases and does not provide you with test macros or other convenience functions.
marcozecchini 0:9fca2b23d0ba 6 Instead, the macros in the [unity module](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity) can be used for this purpose. However, you are not required to use these, and can use your own macros if you wish.
marcozecchini 0:9fca2b23d0ba 7
marcozecchini 0:9fca2b23d0ba 8 Furthermore, test failure recovery through the use of exceptions or `longjmp` is not supported; the test will either continue and ignore failures or die by busy-waiting.
marcozecchini 0:9fca2b23d0ba 9
marcozecchini 0:9fca2b23d0ba 10 ## Theory of Operation
marcozecchini 0:9fca2b23d0ba 11
marcozecchini 0:9fca2b23d0ba 12 A test specification contains a setup handler, several test cases and a teardown handler.
marcozecchini 0:9fca2b23d0ba 13 Each test case contains a textual description, setup, teardown and failure handler as well as the actual test handler.
marcozecchini 0:9fca2b23d0ba 14
marcozecchini 0:9fca2b23d0ba 15 The order of handler execution is:
marcozecchini 0:9fca2b23d0ba 16
marcozecchini 0:9fca2b23d0ba 17 1. Test setup handler.
marcozecchini 0:9fca2b23d0ba 18 1. For each test case:
marcozecchini 0:9fca2b23d0ba 19 1. Test case setup handler.
marcozecchini 0:9fca2b23d0ba 20 1. Test case execution handler.
marcozecchini 0:9fca2b23d0ba 21 1. (wait for timeout or callback validation in case of an asynchronous test case.)
marcozecchini 0:9fca2b23d0ba 22 1. (repeat test case execution handler if specified.)
marcozecchini 0:9fca2b23d0ba 23 1. Test case teardown handler.
marcozecchini 0:9fca2b23d0ba 24 1. Test teardown handler.
marcozecchini 0:9fca2b23d0ba 25
marcozecchini 0:9fca2b23d0ba 26 ## Example
marcozecchini 0:9fca2b23d0ba 27
marcozecchini 0:9fca2b23d0ba 28 The following example showcases a lot of functionality and proper integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea), while making use of the [unity test macros](https://github.com/ARMmbed/mbed-os/tree/master/features/frameworks/unity):
marcozecchini 0:9fca2b23d0ba 29
marcozecchini 0:9fca2b23d0ba 30 ```cpp
marcozecchini 0:9fca2b23d0ba 31 #include "mbed-drivers/test_env.h"
marcozecchini 0:9fca2b23d0ba 32 #include "utest/utest.h"
marcozecchini 0:9fca2b23d0ba 33 #include "unity/unity.h"
marcozecchini 0:9fca2b23d0ba 34
marcozecchini 0:9fca2b23d0ba 35 using namespace utest::v1;
marcozecchini 0:9fca2b23d0ba 36
marcozecchini 0:9fca2b23d0ba 37 void test_simple() {
marcozecchini 0:9fca2b23d0ba 38 TEST_ASSERT_EQUAL(0, 0);
marcozecchini 0:9fca2b23d0ba 39 printf("Simple test called\n");
marcozecchini 0:9fca2b23d0ba 40 }
marcozecchini 0:9fca2b23d0ba 41
marcozecchini 0:9fca2b23d0ba 42 status_t test_repeats_setup(const Case *const source, const size_t index_of_case) {
marcozecchini 0:9fca2b23d0ba 43 // Call the default handler for proper reporting
marcozecchini 0:9fca2b23d0ba 44 status_t status = greentea_case_setup_handler(source, index_of_case);
marcozecchini 0:9fca2b23d0ba 45 printf("Setting up for '%s'\n", source->get_description());
marcozecchini 0:9fca2b23d0ba 46 return status;
marcozecchini 0:9fca2b23d0ba 47 }
marcozecchini 0:9fca2b23d0ba 48 control_t test_repeats(const size_t call_count) {
marcozecchini 0:9fca2b23d0ba 49 printf("Called for the %u. time\n", call_count);
marcozecchini 0:9fca2b23d0ba 50 TEST_ASSERT_NOT_EQUAL(3, call_count);
marcozecchini 0:9fca2b23d0ba 51 // Specify how often this test is repeated ie. n total calls
marcozecchini 0:9fca2b23d0ba 52 return (call_count < 2) ? CaseRepeatAll : CaseNext;
marcozecchini 0:9fca2b23d0ba 53 }
marcozecchini 0:9fca2b23d0ba 54
marcozecchini 0:9fca2b23d0ba 55 void test_callback_validate() {
marcozecchini 0:9fca2b23d0ba 56 // You may also use assertions here!
marcozecchini 0:9fca2b23d0ba 57 TEST_ASSERT_EQUAL_PTR(0, 0);
marcozecchini 0:9fca2b23d0ba 58 // Validate the callback
marcozecchini 0:9fca2b23d0ba 59 Harness::validate_callback();
marcozecchini 0:9fca2b23d0ba 60 }
marcozecchini 0:9fca2b23d0ba 61 control_t test_asynchronous() {
marcozecchini 0:9fca2b23d0ba 62 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
marcozecchini 0:9fca2b23d0ba 63 // Set up a callback in the future. This may also be an interrupt!
marcozecchini 0:9fca2b23d0ba 64 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
marcozecchini 0:9fca2b23d0ba 65 // Set a 200ms timeout starting from now
marcozecchini 0:9fca2b23d0ba 66 return CaseTimeout(200);
marcozecchini 0:9fca2b23d0ba 67 }
marcozecchini 0:9fca2b23d0ba 68
marcozecchini 0:9fca2b23d0ba 69 control_t test_asynchronous_timeout(const size_t call_count) {
marcozecchini 0:9fca2b23d0ba 70 TEST_ASSERT_TRUE_MESSAGE(true, "(true == false) o_O");
marcozecchini 0:9fca2b23d0ba 71 // Set a 200ms timeout starting from now,
marcozecchini 0:9fca2b23d0ba 72 // but automatically repeat only this handler on timeout.
marcozecchini 0:9fca2b23d0ba 73 if (call_count >= 5) {
marcozecchini 0:9fca2b23d0ba 74 // but after the 5th call, the callback finally gets validated
marcozecchini 0:9fca2b23d0ba 75 minar::Scheduler::postCallback(test_callback_validate).delay(minar::milliseconds(100));
marcozecchini 0:9fca2b23d0ba 76 }
marcozecchini 0:9fca2b23d0ba 77 return CaseRepeatHandlerOnTimeout(200);
marcozecchini 0:9fca2b23d0ba 78 }
marcozecchini 0:9fca2b23d0ba 79
marcozecchini 0:9fca2b23d0ba 80 // Custom setup handler required for proper Greentea support
marcozecchini 0:9fca2b23d0ba 81 status_t greentea_setup(const size_t number_of_cases) {
marcozecchini 0:9fca2b23d0ba 82 GREENTEA_SETUP(20, "default_auto");
marcozecchini 0:9fca2b23d0ba 83 // Call the default reporting function
marcozecchini 0:9fca2b23d0ba 84 return greentea_test_setup_handler(number_of_cases);
marcozecchini 0:9fca2b23d0ba 85 }
marcozecchini 0:9fca2b23d0ba 86
marcozecchini 0:9fca2b23d0ba 87 // Specify all your test cases here
marcozecchini 0:9fca2b23d0ba 88 Case cases[] = {
marcozecchini 0:9fca2b23d0ba 89 Case("Simple Test", test_simple),
marcozecchini 0:9fca2b23d0ba 90 Case("Repeating Test", test_repeats_setup, test_repeats),
marcozecchini 0:9fca2b23d0ba 91 Case("Asynchronous Test (200ms timeout)", test_asynchronous),
marcozecchini 0:9fca2b23d0ba 92 Case("Asynchronous Timeout Repeat", test_asynchronous_timeout)
marcozecchini 0:9fca2b23d0ba 93 };
marcozecchini 0:9fca2b23d0ba 94
marcozecchini 0:9fca2b23d0ba 95 // Declare your test specification with a custom setup handler
marcozecchini 0:9fca2b23d0ba 96 Specification specification(greentea_setup, cases);
marcozecchini 0:9fca2b23d0ba 97
marcozecchini 0:9fca2b23d0ba 98 void app_start(int, char**)
marcozecchini 0:9fca2b23d0ba 99 { // Run the test specification
marcozecchini 0:9fca2b23d0ba 100 Harness::run(specification);
marcozecchini 0:9fca2b23d0ba 101 }
marcozecchini 0:9fca2b23d0ba 102 ```
marcozecchini 0:9fca2b23d0ba 103
marcozecchini 0:9fca2b23d0ba 104 Running this test will output the following:
marcozecchini 0:9fca2b23d0ba 105
marcozecchini 0:9fca2b23d0ba 106 ```
marcozecchini 0:9fca2b23d0ba 107 {{timeout;20}}
marcozecchini 0:9fca2b23d0ba 108 {{host_test_name;default_auto}}
marcozecchini 0:9fca2b23d0ba 109 {{description;utest greentea example}}
marcozecchini 0:9fca2b23d0ba 110 {{test_id;MBED_OS}}
marcozecchini 0:9fca2b23d0ba 111 {{start}}
marcozecchini 0:9fca2b23d0ba 112 >>> Running 4 test cases...
marcozecchini 0:9fca2b23d0ba 113
marcozecchini 0:9fca2b23d0ba 114 >>> Running case #1: 'Simple Test'...
marcozecchini 0:9fca2b23d0ba 115 Simple test called
marcozecchini 0:9fca2b23d0ba 116 >>> 'Simple Test': 1 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 117
marcozecchini 0:9fca2b23d0ba 118 >>> Running case #2: 'Repeating Test'...
marcozecchini 0:9fca2b23d0ba 119 Setting up for 'Repeating Test'
marcozecchini 0:9fca2b23d0ba 120 Called for the 1. time
marcozecchini 0:9fca2b23d0ba 121 >>> 'Repeating Test': 1 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 122
marcozecchini 0:9fca2b23d0ba 123 >>> Running case #2: 'Repeating Test'...
marcozecchini 0:9fca2b23d0ba 124 Setting up for 'Repeating Test'
marcozecchini 0:9fca2b23d0ba 125 Called for the 2. time
marcozecchini 0:9fca2b23d0ba 126 >>> 'Repeating Test': 2 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 127
marcozecchini 0:9fca2b23d0ba 128 >>> Running case #3: 'Asynchronous Test (200ms timeout)'...
marcozecchini 0:9fca2b23d0ba 129 >>> 'Asynchronous Test (200ms timeout)': 1 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 130
marcozecchini 0:9fca2b23d0ba 131 >>> Running case #4: 'Asynchronous Timeout Repeat'...
marcozecchini 0:9fca2b23d0ba 132 >>> failure with reason 'Ignored: Timed Out'
marcozecchini 0:9fca2b23d0ba 133 >>> failure with reason 'Ignored: Timed Out'
marcozecchini 0:9fca2b23d0ba 134 >>> failure with reason 'Ignored: Timed Out'
marcozecchini 0:9fca2b23d0ba 135 >>> failure with reason 'Ignored: Timed Out'
marcozecchini 0:9fca2b23d0ba 136 >>> failure with reason 'Ignored: Timed Out'
marcozecchini 0:9fca2b23d0ba 137 >>> 'Asynchronous Timeout Repeat': 1 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 138
marcozecchini 0:9fca2b23d0ba 139 >>> Test cases: 4 passed, 0 failed
marcozecchini 0:9fca2b23d0ba 140 {{success}}
marcozecchini 0:9fca2b23d0ba 141 {{end}}
marcozecchini 0:9fca2b23d0ba 142 ```
marcozecchini 0:9fca2b23d0ba 143
marcozecchini 0:9fca2b23d0ba 144 ## Detailed Description
marcozecchini 0:9fca2b23d0ba 145
marcozecchini 0:9fca2b23d0ba 146 ### Handlers
marcozecchini 0:9fca2b23d0ba 147
marcozecchini 0:9fca2b23d0ba 148 There are five handler types you can, but do not have to, override to customize operation.
marcozecchini 0:9fca2b23d0ba 149 Please see the `utest/types.h` file for a detailed description.
marcozecchini 0:9fca2b23d0ba 150
marcozecchini 0:9fca2b23d0ba 151 1. `status_t test_setup_handler_t(const size_t number_of_cases)`: called before execution of any test case.
marcozecchini 0:9fca2b23d0ba 152 1. `void test_teardown_handler_t(const size_t passed, const size_t failed, const failure_t failure)`: called after execution of all test cases, and if testing is aborted.
marcozecchini 0:9fca2b23d0ba 153 1. `void test_failure_handler_t(const failure_t failure)`: called whenever a failure occurs anywhere in the specification.
marcozecchini 0:9fca2b23d0ba 154 1. `status_t case_setup_handler_t(const Case *const source, const size_t index_of_case)`: called before execution of each test case.
marcozecchini 0:9fca2b23d0ba 155 1. `status_t case_teardown_handler_t(const Case *const source, const size_t passed, const size_t failed, const failure_t reason)`: called after execution of each test case, and if testing is aborted.
marcozecchini 0:9fca2b23d0ba 156 1. `status_t case_failure_handler_t(const Case *const source, const failure_t reason)`: called whenever a failure occurs during the execution of a test case.
marcozecchini 0:9fca2b23d0ba 157
marcozecchini 0:9fca2b23d0ba 158 All handlers are defaulted for integration with the [Greentea testing automation framework](https://github.com/ARMmbed/greentea).
marcozecchini 0:9fca2b23d0ba 159
marcozecchini 0:9fca2b23d0ba 160 ### Test Case Handlers
marcozecchini 0:9fca2b23d0ba 161
marcozecchini 0:9fca2b23d0ba 162 There are three test case handlers:
marcozecchini 0:9fca2b23d0ba 163
marcozecchini 0:9fca2b23d0ba 164 1. `void case_handler_t(void)`: executes once, if the case setup succeeded.
marcozecchini 0:9fca2b23d0ba 165 1. `control_t case_control_handler_t(void)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
marcozecchini 0:9fca2b23d0ba 166 1. `control_t case_call_count_handler_t(const size_t call_count)`: executes (asynchronously) as many times as you specify, if the case setup succeeded.
marcozecchini 0:9fca2b23d0ba 167
marcozecchini 0:9fca2b23d0ba 168 To specify a test case you must wrap it into a `Case` class: `Case("mandatory description", case_handler)`. You may override the setup, teardown and failure handlers in this wrapper class as well.
marcozecchini 0:9fca2b23d0ba 169 The `Case` constructor is overloaded to allow you a comfortable declaration of all your callbacks and the order of arguments is:
marcozecchini 0:9fca2b23d0ba 170
marcozecchini 0:9fca2b23d0ba 171 1. Description (required).
marcozecchini 0:9fca2b23d0ba 172 1. Setup handler (optional).
marcozecchini 0:9fca2b23d0ba 173 1. Test case handler (required).
marcozecchini 0:9fca2b23d0ba 174 1. Teardown handler (optional).
marcozecchini 0:9fca2b23d0ba 175 1. Failure handler (optional).
marcozecchini 0:9fca2b23d0ba 176
marcozecchini 0:9fca2b23d0ba 177 #### Test Case Attributes
marcozecchini 0:9fca2b23d0ba 178
marcozecchini 0:9fca2b23d0ba 179 You can modify test case behavior by returning `control_t` modifiers:
marcozecchini 0:9fca2b23d0ba 180
marcozecchini 0:9fca2b23d0ba 181 - `CaseNext`: never repeats and immediately moves to next test case
marcozecchini 0:9fca2b23d0ba 182 - `CaseNoRepeat`: never repeats.
marcozecchini 0:9fca2b23d0ba 183 - `CaseRepeatAll`: repeats test case **with** setup and teardown handlers.
marcozecchini 0:9fca2b23d0ba 184 - `CaseRepeatHandler`: repeats test case **without** set and teardown handlers.
marcozecchini 0:9fca2b23d0ba 185 - `CaseNoTimeout`: immediately moves to next test case.
marcozecchini 0:9fca2b23d0ba 186 - `CaseAwait`: waits indefinitely for callback validation (*use with caution*).
marcozecchini 0:9fca2b23d0ba 187 - `CaseTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, times out after that (fails with `REASON_TIMEOUT`).
marcozecchini 0:9fca2b23d0ba 188 - `CaseRepeatAllOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **with** setup and teardown handlers on time out.
marcozecchini 0:9fca2b23d0ba 189 - `CaseRepeatHandlerOnTimeout(uint32_t ms)`: waits for callback validation for `ms` milliseconds, repeats test case **without** setup and teardown handlers on time out.
marcozecchini 0:9fca2b23d0ba 190
marcozecchini 0:9fca2b23d0ba 191 Returning `CaseRepeatAll` from your test case handler tells the test harness to repeat the test handler. You can use the `call_count` (starts counting at 1) to decide when to stop.
marcozecchini 0:9fca2b23d0ba 192 By default the setup and teardown handlers are called on every repeated test cases, however, you may only repeat the case handler by returning `CaseRepeatHandler`. To stop the harness from repeating the test case, return `CaseNext`.
marcozecchini 0:9fca2b23d0ba 193
marcozecchini 0:9fca2b23d0ba 194 For asynchronous test cases, you must return a `CaseTimeout(uint32_t ms)`.
marcozecchini 0:9fca2b23d0ba 195 If you want to automatically repeat the test case on a timeout, use `CaseRepeatAllOnTimeout(uint32_t ms)` and `CaseRepeatHandlerOnTimeout(uint32_t ms)`.
marcozecchini 0:9fca2b23d0ba 196
marcozecchini 0:9fca2b23d0ba 197 To validate your callback, you must call `Harness::validate_callback()` in your asynchronous callback before the timeout fires.
marcozecchini 0:9fca2b23d0ba 198 This will schedule the execution of the next test case.
marcozecchini 0:9fca2b23d0ba 199
marcozecchini 0:9fca2b23d0ba 200 For repeating asynchronous cases, you can "add" both attributes together: `CaseTimeout(200) + CaseRepeatAll` will wait for 200ms for the callback validation and then repeat the test case. See the section on arbitration logic for more details.
marcozecchini 0:9fca2b23d0ba 201
marcozecchini 0:9fca2b23d0ba 202 Note that you can also add attributes during callback validation, however, only repeat attributes are considered. This allows you to return `CaseTimeout(500)` to wait up to 500ms for the callback validation and delegate the decision to repeat to the time the callback occurs: `Harness::validate_callback(CaseRepeatHandler)`.
marcozecchini 0:9fca2b23d0ba 203
marcozecchini 0:9fca2b23d0ba 204 Keep in mind that you can only validate a callback once. If you need to wait for several callbacks, you need to write your own helper function that validates the expected callback only when all your custom callbacks arrive.
marcozecchini 0:9fca2b23d0ba 205 This custom functionality is purposefully not part of this test harness, you can achieve it externally with additional code.
marcozecchini 0:9fca2b23d0ba 206
marcozecchini 0:9fca2b23d0ba 207 ### Failure Handlers
marcozecchini 0:9fca2b23d0ba 208
marcozecchini 0:9fca2b23d0ba 209 A failure may occur during any phase of the test. The appropriate failure handler is then called with `failure_t`, which contains the failure reason and location.
marcozecchini 0:9fca2b23d0ba 210
marcozecchini 0:9fca2b23d0ba 211 The failure reasons are:
marcozecchini 0:9fca2b23d0ba 212
marcozecchini 0:9fca2b23d0ba 213 - `REASON_NONE`: No failure occurred
marcozecchini 0:9fca2b23d0ba 214 - `REASON_UNKNOWN`: An unknown failure occurred
marcozecchini 0:9fca2b23d0ba 215 - `REASON_CASES`: A failure occurred in at least one test case
marcozecchini 0:9fca2b23d0ba 216 - `REASON_EMPTY_CASE`: The test case contains only empty handlers
marcozecchini 0:9fca2b23d0ba 217 - `REASON_TIMEOUT`: An expected asynchronous call timed out
marcozecchini 0:9fca2b23d0ba 218 - `REASON_ASSERTION`: An assertion failed
marcozecchini 0:9fca2b23d0ba 219 - `REASON_TEST_SETUP`: Test setup failed
marcozecchini 0:9fca2b23d0ba 220 - `REASON_TEST_TEARDOWN`: Test teardown failed
marcozecchini 0:9fca2b23d0ba 221 - `REASON_CASE_SETUP`: Case setup failed
marcozecchini 0:9fca2b23d0ba 222 - `REASON_CASE_HANDLER`: Case handler failed
marcozecchini 0:9fca2b23d0ba 223 - `REASON_CASE_TEARDOWN`: Case teardown failed
marcozecchini 0:9fca2b23d0ba 224 - `REASON_CASE_INDEX`: Case index returned from test setup or case teardown handler is invalid
marcozecchini 0:9fca2b23d0ba 225 - `REASON_SCHEDULER`: Underlying scheduler is not asynchronous
marcozecchini 0:9fca2b23d0ba 226
marcozecchini 0:9fca2b23d0ba 227 The failure locations are:
marcozecchini 0:9fca2b23d0ba 228
marcozecchini 0:9fca2b23d0ba 229 - `LOCATION_NONE`: No location information
marcozecchini 0:9fca2b23d0ba 230 - `LOCATION_UNKNOWN`: A failure occurred in an unknown location
marcozecchini 0:9fca2b23d0ba 231 - `LOCATION_TEST_SETUP`: A failure occurred in the test setup
marcozecchini 0:9fca2b23d0ba 232 - `LOCATION_TEST_TEARDOWN`: A failure occurred in the test teardown
marcozecchini 0:9fca2b23d0ba 233 - `LOCATION_CASE_SETUP`: A failure occurred in the case setup
marcozecchini 0:9fca2b23d0ba 234 - `LOCATION_CASE_HANDLER`: A failure occurred in the case handler
marcozecchini 0:9fca2b23d0ba 235 - `LOCATION_CASE_TEARDOWN`: A failure occurred in the case teardown
marcozecchini 0:9fca2b23d0ba 236
marcozecchini 0:9fca2b23d0ba 237 If the setup or teardown handlers fail, they may return a `STATUS_ABORT` code, which will call the failure handler with the appropriate failure reason (`REASON_CASE_{SETUP|TEARDOWN}`) and failure location (`LOCATION_CASE_{SETUP|TEARDOWN}`).
marcozecchini 0:9fca2b23d0ba 238 If the setup handler fails, the test case is never executed. Instead, the teardown handler is called in an attempt to salvage the situation.
marcozecchini 0:9fca2b23d0ba 239 Please note that if a teardown handler fails, the system can be considered too unstable to continue testing.
marcozecchini 0:9fca2b23d0ba 240
marcozecchini 0:9fca2b23d0ba 241 You may also raise a failure manually by calling `Harness::raise_failure(failure_reason_t reason)`. In fact, this is how you can integrate assertion failures from custom test macros, as done with the unity macros, which raise a failure with the `REASON_ASSERTION` reason.
marcozecchini 0:9fca2b23d0ba 242
marcozecchini 0:9fca2b23d0ba 243 When waiting for an asynchronous callback, if the timeout fires, `REASON_TIMEOUT` is raised.
marcozecchini 0:9fca2b23d0ba 244
marcozecchini 0:9fca2b23d0ba 245 The failure handler decides whether to continue or abort testing by returning `STATUS_CONTINUE` or `STATUS_ABORT` respectively.
marcozecchini 0:9fca2b23d0ba 246 You can also ignore any raised failure by returning `STATUS_IGNORE` and the harness will then not count this failure.
marcozecchini 0:9fca2b23d0ba 247 In case of an abort, the test harness dies by busy waiting in a forever loop.
marcozecchini 0:9fca2b23d0ba 248 This is needed because we cannot unwind the stack without exception support, and the asynchronous nature of the test harness breaks with using `longjmp`s.
marcozecchini 0:9fca2b23d0ba 249
marcozecchini 0:9fca2b23d0ba 250 Note that when `REASON_IGNORE` is `OR`ed into the failure reason, the failure handler is expected to return `STATUS_IGNORE`.
marcozecchini 0:9fca2b23d0ba 251 This is done automatically for test cases repeating after a timeout, and the default failure handlers also report this failure, but tell the harness to ignore it.
marcozecchini 0:9fca2b23d0ba 252 Furthermore, the unity macros may decide to ignore assertion failures as well, in which case the assertion is ignored intentionally.
marcozecchini 0:9fca2b23d0ba 253
marcozecchini 0:9fca2b23d0ba 254 ### Default Handlers
marcozecchini 0:9fca2b23d0ba 255
marcozecchini 0:9fca2b23d0ba 256 Three sets of default handlers with different behaviors are provided for your convenience:
marcozecchini 0:9fca2b23d0ba 257
marcozecchini 0:9fca2b23d0ba 258 1. `greentea_abort_handlers` (default): Greentea-style reporting, aborts on the first failure, but requires custom test setup handler.
marcozecchini 0:9fca2b23d0ba 259 1. `greentea_continue_handlers`: Greentea-style reporting, always continues testing, but requires custom test setup handler.
marcozecchini 0:9fca2b23d0ba 260 1. `verbose_continue_handlers`: always continues testing and reporting, except when a teardown failed.
marcozecchini 0:9fca2b23d0ba 261 1. `selftest_handlers`: Greentea-style reporting, but aborts on the first assertion failure raised. This allows the use of unity macros for self testing without recursive failure handler calls.
marcozecchini 0:9fca2b23d0ba 262
marcozecchini 0:9fca2b23d0ba 263 These default handlers are called when you have not overridden a custom handler, and they only contain reporting functionality and do not modify global state.
marcozecchini 0:9fca2b23d0ba 264
marcozecchini 0:9fca2b23d0ba 265 You can specify which default handlers you want to use when wrapping your test cases in the `Specification` class:
marcozecchini 0:9fca2b23d0ba 266
marcozecchini 0:9fca2b23d0ba 267 ```cpp
marcozecchini 0:9fca2b23d0ba 268 // Declare your test specification with a custom setup handler
marcozecchini 0:9fca2b23d0ba 269 // and set the default handlers to the predefined “greentea continue” behavior
marcozecchini 0:9fca2b23d0ba 270 Specification specification(greentea_setup, cases, greentea_continue_handlers);
marcozecchini 0:9fca2b23d0ba 271 ```
marcozecchini 0:9fca2b23d0ba 272
marcozecchini 0:9fca2b23d0ba 273 ### Custom Handlers
marcozecchini 0:9fca2b23d0ba 274
marcozecchini 0:9fca2b23d0ba 275 You may override any of the default handlers with your own custom handler.
marcozecchini 0:9fca2b23d0ba 276
marcozecchini 0:9fca2b23d0ba 277 To ignore a handler completely and not call a custom or default handler, you may use the `ignore_handler` hint.
marcozecchini 0:9fca2b23d0ba 278 To explicitly invoke the default handler, use the `default_handler` hint.
marcozecchini 0:9fca2b23d0ba 279
marcozecchini 0:9fca2b23d0ba 280 To use your own custom handler, provide a function with the correct signature for the handler that you want to customize and provide it in your test case wrapper or specification wrapper.
marcozecchini 0:9fca2b23d0ba 281 To turn a `failure_t` into a meaningful string use the `stringify(failure_t)` method.
marcozecchini 0:9fca2b23d0ba 282
marcozecchini 0:9fca2b23d0ba 283 **We strongly recommend that you call the predefined `greentea_*` handlers inside your custom callback, as they report the current condition in a properly formatted fashion.**
marcozecchini 0:9fca2b23d0ba 284 By calling these handlers inside your custom callback your unit test does not need to be modified if the test logging needs to be changed in the future.
marcozecchini 0:9fca2b23d0ba 285
marcozecchini 0:9fca2b23d0ba 286 For the `Specification` class the order of arguments is:
marcozecchini 0:9fca2b23d0ba 287
marcozecchini 0:9fca2b23d0ba 288 1. Test setup handler (optional).
marcozecchini 0:9fca2b23d0ba 289 1. Array of test cases (required).
marcozecchini 0:9fca2b23d0ba 290 1. Test teardown handler (optional).
marcozecchini 0:9fca2b23d0ba 291 1. Test failure handler (optional).
marcozecchini 0:9fca2b23d0ba 292 1. Default handlers (optional).
marcozecchini 0:9fca2b23d0ba 293
marcozecchini 0:9fca2b23d0ba 294 ### Test Case Attribute Arbitration
marcozecchini 0:9fca2b23d0ba 295
marcozecchini 0:9fca2b23d0ba 296 When adding conflicting modifiers together
marcozecchini 0:9fca2b23d0ba 297
marcozecchini 0:9fca2b23d0ba 298 - the more restrictive (=shorter) timeout is chosen, but `CaseNoTimeout` always wins arbitration: `CaseNoTimeout` > `CaseTimeout(100)` > `CaseTimeout(200)` > `CaseAwait`.
marcozecchini 0:9fca2b23d0ba 299 - the more invasive repeat method is chosen, but `CaseNoRepeat` always wins arbitration: `CaseNoRepeat` > `CaseRepeatAll`/`CaseRepeatAllOnTimeout(ms)` > `CaseRepeatHandler`/`CaseRepeatHandlerOnTimeout(ms)`.
marcozecchini 0:9fca2b23d0ba 300 - `CaseNext` always wins arbitration.
marcozecchini 0:9fca2b23d0ba 301
marcozecchini 0:9fca2b23d0ba 302 The following table shows this arbitration logic in detail:
marcozecchini 0:9fca2b23d0ba 303
marcozecchini 0:9fca2b23d0ba 304 | + | `CaseNext` | `CaseNoRepeat` | `CaseRepeatAll` | `CaseRepeatHandler` | `CaseNoTimeout` | `CaseAwait` | `CaseTimeout(aa)` | `CaseRepeatAllOnTimeout(aa)` | `CaseRepeatHandlerOnTimeout(aa)`
marcozecchini 0:9fca2b23d0ba 305 |:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
marcozecchini 0:9fca2b23d0ba 306 | `CaseNext` | no repeat &<br> no timeout
marcozecchini 0:9fca2b23d0ba 307 | `CaseNoRepeat` | no repeat &<br> no timeout | no repeat
marcozecchini 0:9fca2b23d0ba 308 | `CaseRepeatAll` | no repeat &<br> no timeout | no repeat | repeat all
marcozecchini 0:9fca2b23d0ba 309 | `CaseRepeatHandler` | no repeat &<br> no timeout | no repeat | repeat all | repeat handler
marcozecchini 0:9fca2b23d0ba 310 | `CaseNoTimeout` | no repeat &<br> no timeout | no repeat &<br> no timeout | repeat all &<br> no timeout | repeat handler &<br> no timeout | no timeout
marcozecchini 0:9fca2b23d0ba 311 | `CaseAwait` | no repeat &<br> no timeout | no repeat &<br> infinite timeout | repeat all &<br> infinite timeout | repeat handler &<br> infinite timeout | no timeout | infinite timeout
marcozecchini 0:9fca2b23d0ba 312 | `CaseTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all &<br> `bb`ms timeout | repeat handler &<br> `bb`ms timeout | no timeout | `bb`ms timeout | `min(aa,bb)`ms timeout
marcozecchini 0:9fca2b23d0ba 313 | `CaseRepeatAllOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat all & no timeout | repeat all on `bb`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout |
marcozecchini 0:9fca2b23d0ba 314 | `CaseRepeatHandlerOnTimeout(bb)` | no repeat &<br> no timeout | no repeat &<br> `bb`ms timeout | repeat all on validate & repeat all on `bb`ms timeout | repeat handler on validate & repeat handler on `bb`ms timeout | repeat handler & no timeout | repeat handler on `bb`ms timeout | repeat handler on `min(aa,bb)`ms timeout | repeat all on `min(aa,bb)`ms timeout | repeat handler on `min(aa,bb)`ms timeout
marcozecchini 0:9fca2b23d0ba 315
marcozecchini 0:9fca2b23d0ba 316 ### Atomicity
marcozecchini 0:9fca2b23d0ba 317
marcozecchini 0:9fca2b23d0ba 318 All handlers execute with interrupts enabled, **except the case failure handler!**.
marcozecchini 0:9fca2b23d0ba 319 This means you can write test cases that poll for interrupts to be completed inside any handler, except the failure handler.
marcozecchini 0:9fca2b23d0ba 320
marcozecchini 0:9fca2b23d0ba 321 If you setup an interrupt that validates its callback using `Harness::validate_callback()` inside a test case and it fires before the test case completed, the validation will be buffered.
marcozecchini 0:9fca2b23d0ba 322 If the test case then returns a timeout value, but the callback is already validated, the test harness just continues normally.
marcozecchini 0:9fca2b23d0ba 323
marcozecchini 0:9fca2b23d0ba 324 ### Custom Scheduler
marcozecchini 0:9fca2b23d0ba 325
marcozecchini 0:9fca2b23d0ba 326 By default, a Timeout object is used for scheduling the harness operations.
marcozecchini 0:9fca2b23d0ba 327 In case this is not available you can provide your own custom scheduler implementation and make the harness use it with the `Harness::set_scheduler(your_custom_implementation)` function.
marcozecchini 0:9fca2b23d0ba 328
marcozecchini 0:9fca2b23d0ba 329 The scheduler requirements are very simple: Execute a `void(void)` function in you main loop (with a delay of *N* ms). Only one function is scheduled by the harness *at any given time*.
marcozecchini 0:9fca2b23d0ba 330 Note that you do not need to implement the delay functionality, if your tests do not require timeouts. You will still be able to use repeating test cases, but an error is thrown if your tests attempt to use a timeout, when your underlying scheduler does not support it.
marcozecchini 0:9fca2b23d0ba 331
marcozecchini 0:9fca2b23d0ba 332 There are two functions you need to implement:
marcozecchini 0:9fca2b23d0ba 333
marcozecchini 0:9fca2b23d0ba 334 - `void* post_callback(const utest_v1_harness_callback_t callback, const uint32_t delay_ms)`: schedules a `void(void)` callback function in *N* ms.
marcozecchini 0:9fca2b23d0ba 335 - `int32_t cancel_callback_t(void *handle)`: cancels an asynchronous callback.
marcozecchini 0:9fca2b23d0ba 336
marcozecchini 0:9fca2b23d0ba 337 Please see [their doxygen documentation for implementation details](utest/scheduler.h).
marcozecchini 0:9fca2b23d0ba 338
marcozecchini 0:9fca2b23d0ba 339 ### Example Synchronous Scheduler
marcozecchini 0:9fca2b23d0ba 340
marcozecchini 0:9fca2b23d0ba 341 Here is the most [basic scheduler implementation without any asynchronous support](test/minimal_scheduler/main.cpp). Note that this does not require any hardware support at all, but you cannot use timeouts in your test cases!
marcozecchini 0:9fca2b23d0ba 342 ```cpp
marcozecchini 0:9fca2b23d0ba 343 volatile utest_v1_harness_callback_t minimal_callback;
marcozecchini 0:9fca2b23d0ba 344
marcozecchini 0:9fca2b23d0ba 345 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
marcozecchini 0:9fca2b23d0ba 346 minimal_callback = callback;
marcozecchini 0:9fca2b23d0ba 347 // this scheduler does not support asynchronous callbacks
marcozecchini 0:9fca2b23d0ba 348 return (delay_ms ? NULL : (void*)1);
marcozecchini 0:9fca2b23d0ba 349 }
marcozecchini 0:9fca2b23d0ba 350 static int32_t utest_minimal_cancel(void*) {
marcozecchini 0:9fca2b23d0ba 351 return -1; // canceling not supported either
marcozecchini 0:9fca2b23d0ba 352 }
marcozecchini 0:9fca2b23d0ba 353 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
marcozecchini 0:9fca2b23d0ba 354
marcozecchini 0:9fca2b23d0ba 355 // [...] Add your test cases and specification here.
marcozecchini 0:9fca2b23d0ba 356
marcozecchini 0:9fca2b23d0ba 357 void main() // or whatever your custom entry point is
marcozecchini 0:9fca2b23d0ba 358 {
marcozecchini 0:9fca2b23d0ba 359 // You MUST set the custom scheduler before running the specification.
marcozecchini 0:9fca2b23d0ba 360 Harness::set_scheduler(utest_minimal_scheduler);
marcozecchini 0:9fca2b23d0ba 361 Harness::run(specification);
marcozecchini 0:9fca2b23d0ba 362
marcozecchini 0:9fca2b23d0ba 363 while(1) {
marcozecchini 0:9fca2b23d0ba 364 if (minimal_callback) {
marcozecchini 0:9fca2b23d0ba 365 // copy the callback and reset the shared memory
marcozecchini 0:9fca2b23d0ba 366 utest_v1_harness_callback_t callback = minimal_callback;
marcozecchini 0:9fca2b23d0ba 367 minimal_callback = NULL;
marcozecchini 0:9fca2b23d0ba 368 callback(); // execute the copied callback
marcozecchini 0:9fca2b23d0ba 369 }
marcozecchini 0:9fca2b23d0ba 370 }
marcozecchini 0:9fca2b23d0ba 371 }
marcozecchini 0:9fca2b23d0ba 372 ```
marcozecchini 0:9fca2b23d0ba 373
marcozecchini 0:9fca2b23d0ba 374 ### Example Asynchronous Scheduler
marcozecchini 0:9fca2b23d0ba 375
marcozecchini 0:9fca2b23d0ba 376 Here is the a [complete scheduler implementation with any asynchronous support](test/minimal_scheduler_async/main.cpp). Note that this does require at least a hardware timer, in this case we have used `mbed-hal/us_ticker`! Note that you must not execute the callback in the timer interrupt context, but in the main loop context!
marcozecchini 0:9fca2b23d0ba 377 ```cpp
marcozecchini 0:9fca2b23d0ba 378 volatile utest_v1_harness_callback_t minimal_callback;
marcozecchini 0:9fca2b23d0ba 379 volatile utest_v1_harness_callback_t ticker_callback;
marcozecchini 0:9fca2b23d0ba 380 const ticker_data_t *ticker_data;
marcozecchini 0:9fca2b23d0ba 381 ticker_event_t ticker_event;
marcozecchini 0:9fca2b23d0ba 382
marcozecchini 0:9fca2b23d0ba 383 static void ticker_handler(uint32_t) {
marcozecchini 0:9fca2b23d0ba 384 minimal_callback = ticker_callback; // interrupt context!
marcozecchini 0:9fca2b23d0ba 385 }
marcozecchini 0:9fca2b23d0ba 386 static void* utest_minimal_post(const utest_v1_harness_callback_t callback, const uint32_t delay_ms) {
marcozecchini 0:9fca2b23d0ba 387 if (delay_ms) {
marcozecchini 0:9fca2b23d0ba 388 ticker_callback = callback;
marcozecchini 0:9fca2b23d0ba 389 ticker_insert_event(ticker_data, &ticker_event, ticker_read(ticker_data) + delay_ms * 1000, 0);
marcozecchini 0:9fca2b23d0ba 390 }
marcozecchini 0:9fca2b23d0ba 391 else minimal_callback = callback;
marcozecchini 0:9fca2b23d0ba 392 return (void*)1;
marcozecchini 0:9fca2b23d0ba 393 }
marcozecchini 0:9fca2b23d0ba 394 static int32_t utest_minimal_cancel(void*) {
marcozecchini 0:9fca2b23d0ba 395 ticker_remove_event(ticker_data, &ticker_event);
marcozecchini 0:9fca2b23d0ba 396 return 0; // canceling is supported
marcozecchini 0:9fca2b23d0ba 397 }
marcozecchini 0:9fca2b23d0ba 398 static const utest_v1_scheduler_t utest_minimal_scheduler = {utest_minimal_post, utest_minimal_cancel};
marcozecchini 0:9fca2b23d0ba 399
marcozecchini 0:9fca2b23d0ba 400 // [...] Add your test cases and specification here.
marcozecchini 0:9fca2b23d0ba 401
marcozecchini 0:9fca2b23d0ba 402 void main() // or whatever your custom entry point is
marcozecchini 0:9fca2b23d0ba 403 {
marcozecchini 0:9fca2b23d0ba 404 ticker_data = get_us_ticker_data(); // initialize the ticker data.
marcozecchini 0:9fca2b23d0ba 405 ticker_set_handler(ticker_data, ticker_handler);
marcozecchini 0:9fca2b23d0ba 406 // You MUST set the custom scheduler before running the specification.
marcozecchini 0:9fca2b23d0ba 407 Harness::set_scheduler(utest_minimal_scheduler);
marcozecchini 0:9fca2b23d0ba 408 Harness::run(specification);
marcozecchini 0:9fca2b23d0ba 409
marcozecchini 0:9fca2b23d0ba 410 while(1) {
marcozecchini 0:9fca2b23d0ba 411 if (minimal_callback) {
marcozecchini 0:9fca2b23d0ba 412 // copy the callback and reset the shared memory
marcozecchini 0:9fca2b23d0ba 413 utest_v1_harness_callback_t callback = minimal_callback;
marcozecchini 0:9fca2b23d0ba 414 minimal_callback = NULL;
marcozecchini 0:9fca2b23d0ba 415 callback(); // execute the copied callback
marcozecchini 0:9fca2b23d0ba 416 }
marcozecchini 0:9fca2b23d0ba 417 }
marcozecchini 0:9fca2b23d0ba 418 }
marcozecchini 0:9fca2b23d0ba 419 ```