utest_harness.cpp

00001 /****************************************************************************
00002  * Copyright (c) 2015, ARM Limited, All Rights Reserved
00003  * SPDX-License-Identifier: Apache-2.0
00004  *
00005  * Licensed under the Apache License, Version 2.0 (the "License"); you may
00006  * not use this file except in compliance with the License.
00007  * You may obtain a copy of the License at
00008  *
00009  * http://www.apache.org/licenses/LICENSE-2.0
00010  *
00011  * Unless required by applicable law or agreed to in writing, software
00012  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
00013  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014  * See the License for the specific language governing permissions and
00015  * limitations under the License.
00016  ****************************************************************************
00017  */
00018 
00019 #include "utest/utest_harness.h"
00020 #include "utest/utest_stack_trace.h"
00021 #include "utest/utest_serial.h"
00022 
00023 #include <stdlib.h>
00024 
00025 using namespace utest::v1;
00026 
00027 namespace
00028 {
00029     const Case *test_cases = NULL;
00030     size_t test_length = 0;
00031 
00032     size_t test_index_of_case = 0;
00033 
00034     size_t test_passed = 0;
00035     size_t test_failed = 0;
00036 
00037     const Case *case_current = NULL;
00038     size_t case_index = 0;
00039     base_control_t case_control =  { REPEAT_SETUP_TEARDOWN, TIMEOUT_UNDECLR };
00040     size_t case_repeat_count = 1;
00041 
00042     void *case_timeout_handle = NULL;
00043     size_t case_validation_count = 0;
00044     bool case_timeout_occurred = false;
00045 
00046     size_t case_passed = 0;
00047     size_t case_failed = 0;
00048     size_t case_failed_before = 0;
00049 
00050     struct DefaultHandlers : public handlers_t { 
00051         DefaultHandlers() : handlers_t(default_handlers) { }
00052         DefaultHandlers(const handlers_t& other) : handlers_t(other) { }
00053     };
00054 
00055     SingletonPtr<DefaultHandlers> defaults;
00056     SingletonPtr<DefaultHandlers> handlers;
00057 
00058     location_t location = LOCATION_UNKNOWN;
00059 
00060     utest_v1_scheduler_t scheduler = {NULL, NULL, NULL, NULL};
00061 }
00062 
00063 static void die() {
00064     UTEST_LOG_FUNCTION();
00065     while(1) ;
00066 }
00067 
00068 static bool is_scheduler_valid(const utest_v1_scheduler_t scheduler)
00069 {
00070     UTEST_LOG_FUNCTION();
00071     return (scheduler.init && scheduler.post && scheduler.cancel && scheduler.run);
00072 }
00073 
00074 bool Harness::set_scheduler(const utest_v1_scheduler_t scheduler)
00075 {
00076     UTEST_LOG_FUNCTION();
00077     if (is_scheduler_valid(scheduler)) {
00078         ::scheduler = scheduler;
00079         return true;
00080     }
00081     return false;
00082 }
00083 
00084 
00085 void Harness::notify_testcases()
00086 {
00087     for(unsigned i = 0; i < test_length; i++) {
00088         utest::v1::greentea_testcase_notification_handler(test_cases[i].get_description());
00089     }
00090 }
00091 
00092 bool Harness::run(const Specification& specification, size_t)
00093 {
00094     UTEST_LOG_FUNCTION();
00095     return run(specification);
00096 }
00097 
00098 bool Harness::run(const Specification& specification)
00099 {
00100     UTEST_LOG_FUNCTION();
00101     // check if a specification is currently running
00102     if (is_busy ())
00103         return false;
00104 
00105     // if the scheduler is invalid, this is the first time we are calling
00106     if (!is_scheduler_valid(scheduler))
00107         scheduler = utest_v1_get_scheduler();
00108 
00109     // if the scheduler is still invalid, abort
00110     if (!is_scheduler_valid(scheduler))
00111         return false;
00112 
00113     // if the scheduler failed to initialize, abort
00114     if (scheduler.init() != 0)
00115         return false;
00116     test_cases  = specification.cases;
00117     test_length = specification.length;
00118     *defaults.get()    = specification.defaults;
00119     handlers->test_setup    = defaults->get_handler(specification.setup_handler);
00120     handlers->test_teardown = defaults->get_handler(specification.teardown_handler);
00121     handlers->test_failure  = defaults->get_handler(specification.failure_handler);
00122 
00123     test_index_of_case = 0;
00124     test_passed = 0;
00125     test_failed = 0;
00126 
00127     case_passed = 0;
00128     case_failed = 0;
00129     case_failed_before = 0;
00130 
00131     location = LOCATION_TEST_SETUP;
00132     int setup_status = 0;
00133     failure_t failure(REASON_NONE, location);
00134 
00135     if (handlers->test_setup) {
00136         setup_status = handlers->test_setup(test_length);
00137         if (setup_status == STATUS_CONTINUE) setup_status = 0;
00138         else if (setup_status < STATUS_CONTINUE)     failure.reason = REASON_TEST_SETUP;
00139         else if (setup_status > signed(test_length)) failure.reason = REASON_CASE_INDEX;
00140     }
00141 
00142     if (failure.reason != REASON_NONE) {
00143         if (handlers->test_failure) handlers->test_failure(failure);
00144         if (handlers->test_teardown) handlers->test_teardown(0, 0, failure);
00145         test_cases = NULL;
00146         exit(1);
00147         return true;
00148     }
00149 
00150     notify_testcases();
00151 
00152     case_index = setup_status;
00153     case_current = &test_cases[case_index];
00154 
00155     scheduler.post(run_next_case, 0);
00156     if (scheduler.run() != 0) {
00157         failure.reason = REASON_SCHEDULER;
00158         if (handlers->test_failure) handlers->test_failure(failure);
00159         if (handlers->test_teardown) handlers->test_teardown(0, 0, failure);
00160         test_cases = NULL;
00161         exit(1);
00162         return true;
00163     }
00164     return true;
00165 }
00166 
00167 void Harness::raise_failure(const failure_reason_t reason)
00168 {
00169     UTEST_LOG_FUNCTION();
00170     // ignore a failure, if the Harness has not been initialized.
00171     // this allows using unity assertion macros without setting up utest.
00172     if (test_cases == NULL) return;
00173 
00174     utest::v1::status_t fail_status = STATUS_ABORT;
00175     if (handlers->test_failure) handlers->test_failure(failure_t(reason, location));
00176     if (handlers->case_failure) fail_status = handlers->case_failure(case_current, failure_t(reason, location));
00177 
00178     {
00179         UTEST_ENTER_CRITICAL_SECTION;
00180 
00181         if (fail_status != STATUS_IGNORE) case_failed++;
00182 
00183         if ((fail_status == STATUS_ABORT) && case_timeout_handle)
00184         {
00185             scheduler.cancel(case_timeout_handle);
00186             case_timeout_handle = NULL;
00187         }
00188         UTEST_LEAVE_CRITICAL_SECTION;
00189     }
00190 
00191     if (fail_status == STATUS_ABORT || reason & REASON_CASE_SETUP) {
00192         if (handlers->case_teardown && location != LOCATION_CASE_TEARDOWN) {
00193             location_t fail_loc(location);
00194             location = LOCATION_CASE_TEARDOWN;
00195 
00196             utest::v1::status_t teardown_status = handlers->case_teardown(case_current, case_passed, case_failed, failure_t(reason, fail_loc));
00197             if (teardown_status < STATUS_CONTINUE) raise_failure(REASON_CASE_TEARDOWN);
00198             else if (teardown_status > signed(test_length)) raise_failure(REASON_CASE_INDEX);
00199             else if (teardown_status >= 0) case_index = teardown_status - 1;
00200 
00201             // Restore case failure location once we have dealt with case teardown
00202             location = fail_loc;
00203             handlers->case_teardown = NULL;
00204         }
00205     }
00206     if (fail_status == STATUS_ABORT) {
00207         test_failed++;
00208         failure_t fail(reason, location);
00209         location = LOCATION_TEST_TEARDOWN;
00210         if (handlers->test_teardown) handlers->test_teardown(test_passed, test_failed, fail);
00211         exit(test_failed);
00212         die();
00213     }
00214 }
00215 
00216 void Harness::schedule_next_case()
00217 {
00218     UTEST_LOG_FUNCTION();
00219     if (!case_timeout_occurred && case_failed_before == case_failed) {
00220         case_passed++;
00221     }
00222 
00223     if (case_control.repeat & REPEAT_SETUP_TEARDOWN || !(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
00224         location = LOCATION_CASE_TEARDOWN;
00225 
00226         if (handlers->case_teardown) {
00227             utest::v1::status_t status = handlers->case_teardown(case_current, case_passed, case_failed,
00228                                                      case_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
00229             if (status < STATUS_CONTINUE)          raise_failure(REASON_CASE_TEARDOWN);
00230             else if (status > signed(test_length)) raise_failure(REASON_CASE_INDEX);
00231             else if (status >= 0) case_index = status - 1;
00232         }
00233     }
00234 
00235     if (!(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
00236         if (case_failed > 0) test_failed++;
00237         else test_passed++;
00238 
00239         case_control = control_t(REPEAT_SETUP_TEARDOWN);
00240         case_index++;
00241         case_current = &test_cases[case_index];
00242         case_passed = 0;
00243         case_failed = 0;
00244         case_failed_before = 0;
00245         case_repeat_count = 1;
00246         test_index_of_case++;
00247     }
00248     scheduler.post(run_next_case, 0);
00249 }
00250 
00251 void Harness::handle_timeout()
00252 {
00253     UTEST_LOG_FUNCTION();
00254     {
00255         UTEST_ENTER_CRITICAL_SECTION;
00256 
00257         if (case_timeout_handle != NULL) {
00258             case_timeout_handle = NULL;
00259             case_timeout_occurred = true;
00260         }
00261         UTEST_LEAVE_CRITICAL_SECTION;
00262     }
00263     if (case_timeout_occurred) {
00264         raise_failure(failure_reason_t(REASON_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? REASON_IGNORE : 0)));
00265         scheduler.post(schedule_next_case, 0);
00266     }
00267 }
00268 
00269 void Harness::validate_callback(const control_t control)
00270 {
00271     UTEST_LOG_FUNCTION();
00272     UTEST_ENTER_CRITICAL_SECTION;
00273     case_validation_count++;
00274 
00275     if (case_timeout_handle != NULL || case_control.timeout == TIMEOUT_FOREVER)
00276     {
00277         scheduler.cancel(case_timeout_handle);
00278         case_timeout_handle = NULL;
00279         control_t merged_control = case_control + control;
00280         case_control.repeat = repeat_t(merged_control.repeat & ~REPEAT_ON_TIMEOUT);
00281         case_control.timeout = TIMEOUT_NONE;
00282         scheduler.post(schedule_next_case, 0);
00283     }
00284     UTEST_LEAVE_CRITICAL_SECTION;
00285 }
00286 
00287 bool Harness::is_busy ()
00288 {
00289     UTEST_LOG_FUNCTION();
00290     UTEST_ENTER_CRITICAL_SECTION;
00291     bool res = false;
00292     
00293     if (test_cases && case_current) {    
00294         res = (case_current < (test_cases + test_length));
00295     }
00296     
00297     UTEST_LEAVE_CRITICAL_SECTION;
00298     return res;
00299 }
00300 
00301 void Harness::run_next_case()
00302 {
00303     UTEST_LOG_FUNCTION();
00304     if(case_current < (test_cases + test_length))
00305     {
00306         handlers->case_setup    = defaults->get_handler(case_current->setup_handler);
00307         handlers->case_teardown = defaults->get_handler(case_current->teardown_handler);
00308         handlers->case_failure  = defaults->get_handler(case_current->failure_handler);
00309 
00310         if (case_current->is_empty ()) {
00311             location = LOCATION_UNKNOWN;
00312             raise_failure(REASON_EMPTY_CASE);
00313             schedule_next_case();
00314             return;
00315         }
00316 
00317         repeat_t setup_repeat;
00318         {
00319             UTEST_ENTER_CRITICAL_SECTION;
00320             case_validation_count = 0;
00321             case_timeout_occurred = false;
00322             setup_repeat = case_control.repeat;
00323             case_control = control_t();
00324             UTEST_LEAVE_CRITICAL_SECTION;
00325         }
00326 
00327         if (setup_repeat & REPEAT_SETUP_TEARDOWN) {
00328             location = LOCATION_CASE_SETUP;
00329             if (handlers->case_setup && (handlers->case_setup(case_current, test_index_of_case) != STATUS_CONTINUE)) {
00330                 raise_failure(REASON_CASE_SETUP);
00331                 schedule_next_case();
00332                 return;
00333             }
00334         }
00335 
00336         case_failed_before = case_failed;
00337         location = LOCATION_CASE_HANDLER;
00338 
00339         if (case_current->handler) {
00340             case_current->handler();
00341         } else if (case_current->control_handler) {
00342             case_control = case_control + case_current->control_handler();
00343         } else if (case_current->repeat_count_handler) {
00344             case_control = case_control + case_current->repeat_count_handler(case_repeat_count);
00345         }
00346         case_repeat_count++;
00347 
00348         {
00349             UTEST_ENTER_CRITICAL_SECTION;
00350             if (case_validation_count) case_control.repeat = repeat_t(case_control.repeat & ~REPEAT_ON_TIMEOUT);
00351 
00352             // if timeout valid
00353             if (case_control.timeout < TIMEOUT_UNDECLR && case_validation_count == 0) {
00354                 // if await validation _with_ timeout
00355                 if (case_control.timeout < TIMEOUT_FOREVER) {
00356                     case_timeout_handle = scheduler.post(handle_timeout, case_control.timeout);
00357                     if (case_timeout_handle == NULL) {
00358                         raise_failure(REASON_SCHEDULER);
00359                         schedule_next_case();
00360                     }
00361                 }
00362             }
00363             else {
00364                 scheduler.post(schedule_next_case, 0);
00365             }
00366             UTEST_LEAVE_CRITICAL_SECTION;
00367         }
00368     }
00369     else if (handlers->test_teardown) {
00370         location = LOCATION_TEST_TEARDOWN;
00371         handlers->test_teardown(test_passed, test_failed, test_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
00372         test_cases = NULL;
00373         exit(test_failed);
00374     } else {
00375         exit(test_failed);
00376     }
00377 }