utest_harness.cpp

00001 /****************************************************************************
00002  * Copyright (c) 2015, ARM Limited, All Rights Reserved
00003  * SPDX-License-Identifier: Apache-2.0
00004  *
00005  * Licensed under the Apache License, Version 2.0 (the "License"); you may
00006  * not use this file except in compliance with the License.
00007  * You may obtain a copy of the License at
00008  *
00009  * http://www.apache.org/licenses/LICENSE-2.0
00010  *
00011  * Unless required by applicable law or agreed to in writing, software
00012  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
00013  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014  * See the License for the specific language governing permissions and
00015  * limitations under the License.
00016  ****************************************************************************
00017  */
00018 
00019 #include "utest/utest_harness.h"
00020 #include "utest/utest_stack_trace.h"
00021 #include "utest/utest_serial.h"
00022 
00023 #include <stdlib.h>
00024 
00025 using namespace utest::v1;
00026 
00027 namespace
00028 {
00029     const Case *test_cases = NULL;
00030     size_t test_length = 0;
00031 
00032     size_t test_index_of_case = 0;
00033 
00034     size_t test_passed = 0;
00035     size_t test_failed = 0;
00036 
00037     const Case *case_current = NULL;
00038     size_t case_index = 0;
00039     base_control_t case_control =  { REPEAT_SETUP_TEARDOWN, TIMEOUT_UNDECLR };
00040     size_t case_repeat_count = 1;
00041 
00042     void *case_timeout_handle = NULL;
00043     size_t case_validation_count = 0;
00044     bool case_timeout_occurred = false;
00045 
00046     size_t case_passed = 0;
00047     size_t case_failed = 0;
00048     size_t case_failed_before = 0;
00049 
00050     struct DefaultHandlers : public handlers_t { 
00051         DefaultHandlers() : handlers_t(default_handlers) { }
00052         DefaultHandlers(const handlers_t& other) : handlers_t(other) { }
00053     };
00054 
00055     SingletonPtr<DefaultHandlers> defaults;
00056     SingletonPtr<DefaultHandlers> handlers;
00057 
00058     location_t location = LOCATION_UNKNOWN;
00059 
00060     utest_v1_scheduler_t scheduler = {NULL, NULL, NULL, NULL};
00061 }
00062 
00063 static void die() {
00064     UTEST_LOG_FUNCTION();
00065     while(1) ;
00066 }
00067 
00068 static bool is_scheduler_valid(const utest_v1_scheduler_t scheduler)
00069 {
00070     UTEST_LOG_FUNCTION();
00071     return (scheduler.init && scheduler.post && scheduler.cancel && scheduler.run);
00072 }
00073 
00074 bool Harness::set_scheduler(const utest_v1_scheduler_t scheduler)
00075 {
00076     UTEST_LOG_FUNCTION();
00077     if (is_scheduler_valid(scheduler)) {
00078         ::scheduler = scheduler;
00079         return true;
00080     }
00081     return false;
00082 }
00083 
00084 
00085 void Harness::notify_testcases()
00086 {
00087     for(unsigned i = 0; i < test_length; i++) {
00088         utest::v1::greentea_testcase_notification_handler(test_cases[i].get_description());
00089     }
00090 }
00091 
00092 bool Harness::run(const Specification& specification, size_t)
00093 {
00094     UTEST_LOG_FUNCTION();
00095     return run(specification);
00096 }
00097 
00098 bool Harness::run(const Specification& specification)
00099 {
00100     UTEST_LOG_FUNCTION();
00101     // check if a specification is currently running
00102     if (is_busy ())
00103         return false;
00104 
00105     // if the scheduler is invalid, this is the first time we are calling
00106     if (!is_scheduler_valid(scheduler))
00107         scheduler = utest_v1_get_scheduler();
00108 
00109     // if the scheduler is still invalid, abort
00110     if (!is_scheduler_valid(scheduler))
00111         return false;
00112 
00113     // if the scheduler failed to initialize, abort
00114     if (scheduler.init() != 0)
00115         return false;
00116     test_cases  = specification.cases;
00117     test_length = specification.length;
00118     *defaults.get()    = specification.defaults;
00119     handlers->test_setup    = defaults->get_handler(specification.setup_handler);
00120     handlers->test_teardown = defaults->get_handler(specification.teardown_handler);
00121     handlers->test_failure  = defaults->get_handler(specification.failure_handler);
00122 
00123     test_index_of_case = 0;
00124     test_passed = 0;
00125     test_failed = 0;
00126 
00127     case_passed = 0;
00128     case_failed = 0;
00129     case_failed_before = 0;
00130 
00131     location = LOCATION_TEST_SETUP;
00132     int setup_status = 0;
00133     failure_t failure(REASON_NONE, location);
00134 
00135     if (handlers->test_setup) {
00136         setup_status = handlers->test_setup(test_length);
00137         if (setup_status == STATUS_CONTINUE) setup_status = 0;
00138         else if (setup_status < STATUS_CONTINUE)     failure.reason = REASON_TEST_SETUP;
00139         else if (setup_status > signed(test_length)) failure.reason = REASON_CASE_INDEX;
00140     }
00141 
00142     if (failure.reason != REASON_NONE) {
00143         if (handlers->test_failure) handlers->test_failure(failure);
00144         if (handlers->test_teardown) handlers->test_teardown(0, 0, failure);
00145         test_cases = NULL;
00146         exit(1);
00147     }
00148 
00149     notify_testcases();
00150 
00151     case_index = setup_status;
00152     case_current = &test_cases[case_index];
00153 
00154     scheduler.post(run_next_case, 0);
00155     if (scheduler.run() != 0) {
00156         failure.reason = REASON_SCHEDULER;
00157         if (handlers->test_failure) handlers->test_failure(failure);
00158         if (handlers->test_teardown) handlers->test_teardown(0, 0, failure);
00159         test_cases = NULL;
00160         exit(1);
00161     }
00162     return true;
00163 }
00164 
00165 void Harness::raise_failure(const failure_reason_t reason)
00166 {
00167     UTEST_LOG_FUNCTION();
00168     // ignore a failure, if the Harness has not been initialized.
00169     // this allows using unity assertion macros without setting up utest.
00170     if (test_cases == NULL) return;
00171 
00172     utest::v1::status_t fail_status = STATUS_ABORT;
00173     if (handlers->test_failure) handlers->test_failure(failure_t(reason, location));
00174     if (handlers->case_failure) fail_status = handlers->case_failure(case_current, failure_t(reason, location));
00175 
00176     {
00177         UTEST_ENTER_CRITICAL_SECTION;
00178 
00179         if (fail_status != STATUS_IGNORE) case_failed++;
00180 
00181         if ((fail_status == STATUS_ABORT) && case_timeout_handle)
00182         {
00183             scheduler.cancel(case_timeout_handle);
00184             case_timeout_handle = NULL;
00185         }
00186         UTEST_LEAVE_CRITICAL_SECTION;
00187     }
00188 
00189     if (fail_status == STATUS_ABORT || reason & REASON_CASE_SETUP) {
00190         if (handlers->case_teardown && location != LOCATION_CASE_TEARDOWN) {
00191             location_t fail_loc(location);
00192             location = LOCATION_CASE_TEARDOWN;
00193 
00194             utest::v1::status_t teardown_status = handlers->case_teardown(case_current, case_passed, case_failed, failure_t(reason, fail_loc));
00195             if (teardown_status < STATUS_CONTINUE) raise_failure(REASON_CASE_TEARDOWN);
00196             else if (teardown_status > signed(test_length)) raise_failure(REASON_CASE_INDEX);
00197             else if (teardown_status >= 0) case_index = teardown_status - 1;
00198 
00199             // Restore case failure location once we have dealt with case teardown
00200             location = fail_loc;
00201             handlers->case_teardown = NULL;
00202         }
00203     }
00204     if (fail_status == STATUS_ABORT) {
00205         test_failed++;
00206         failure_t fail(reason, location);
00207         location = LOCATION_TEST_TEARDOWN;
00208         if (handlers->test_teardown) handlers->test_teardown(test_passed, test_failed, fail);
00209         exit(test_failed);
00210         die();
00211     }
00212 }
00213 
00214 void Harness::schedule_next_case()
00215 {
00216     UTEST_LOG_FUNCTION();
00217     if (!case_timeout_occurred && case_failed_before == case_failed) {
00218         case_passed++;
00219     }
00220 
00221     if (case_control.repeat & REPEAT_SETUP_TEARDOWN || !(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
00222         location = LOCATION_CASE_TEARDOWN;
00223 
00224         if (handlers->case_teardown) {
00225             utest::v1::status_t status = handlers->case_teardown(case_current, case_passed, case_failed,
00226                                                      case_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
00227             if (status < STATUS_CONTINUE)          raise_failure(REASON_CASE_TEARDOWN);
00228             else if (status > signed(test_length)) raise_failure(REASON_CASE_INDEX);
00229             else if (status >= 0) case_index = status - 1;
00230         }
00231     }
00232 
00233     if (!(case_control.repeat & (REPEAT_ON_TIMEOUT | REPEAT_ON_VALIDATE))) {
00234         if (case_failed > 0) test_failed++;
00235         else test_passed++;
00236 
00237         case_control = control_t(REPEAT_SETUP_TEARDOWN);
00238         case_index++;
00239         case_current = &test_cases[case_index];
00240         case_passed = 0;
00241         case_failed = 0;
00242         case_failed_before = 0;
00243         case_repeat_count = 1;
00244         test_index_of_case++;
00245     }
00246     scheduler.post(run_next_case, 0);
00247 }
00248 
00249 void Harness::handle_timeout()
00250 {
00251     UTEST_LOG_FUNCTION();
00252     {
00253         UTEST_ENTER_CRITICAL_SECTION;
00254 
00255         if (case_timeout_handle != NULL) {
00256             case_timeout_handle = NULL;
00257             case_timeout_occurred = true;
00258         }
00259         UTEST_LEAVE_CRITICAL_SECTION;
00260     }
00261     if (case_timeout_occurred) {
00262         raise_failure(failure_reason_t(REASON_TIMEOUT | ((case_control.repeat & REPEAT_ON_TIMEOUT) ? REASON_IGNORE : 0)));
00263         scheduler.post(schedule_next_case, 0);
00264     }
00265 }
00266 
00267 void Harness::validate_callback(const control_t control)
00268 {
00269     UTEST_LOG_FUNCTION();
00270     UTEST_ENTER_CRITICAL_SECTION;
00271     case_validation_count++;
00272 
00273     if (case_timeout_handle != NULL || case_control.timeout == TIMEOUT_FOREVER)
00274     {
00275         scheduler.cancel(case_timeout_handle);
00276         case_timeout_handle = NULL;
00277         control_t merged_control = case_control + control;
00278         case_control.repeat = repeat_t(merged_control.repeat & ~REPEAT_ON_TIMEOUT);
00279         case_control.timeout = TIMEOUT_NONE;
00280         scheduler.post(schedule_next_case, 0);
00281     }
00282     UTEST_LEAVE_CRITICAL_SECTION;
00283 }
00284 
00285 bool Harness::is_busy ()
00286 {
00287     UTEST_LOG_FUNCTION();
00288     UTEST_ENTER_CRITICAL_SECTION;
00289     bool res = false;
00290     
00291     if (test_cases && case_current) {    
00292         res = (case_current < (test_cases + test_length));
00293     }
00294     
00295     UTEST_LEAVE_CRITICAL_SECTION;
00296     return res;
00297 }
00298 
00299 void Harness::run_next_case()
00300 {
00301     UTEST_LOG_FUNCTION();
00302     if(case_current < (test_cases + test_length))
00303     {
00304         handlers->case_setup    = defaults->get_handler(case_current->setup_handler);
00305         handlers->case_teardown = defaults->get_handler(case_current->teardown_handler);
00306         handlers->case_failure  = defaults->get_handler(case_current->failure_handler);
00307 
00308         if (case_current->is_empty ()) {
00309             location = LOCATION_UNKNOWN;
00310             raise_failure(REASON_EMPTY_CASE);
00311             schedule_next_case();
00312             return;
00313         }
00314 
00315         repeat_t setup_repeat;
00316         {
00317             UTEST_ENTER_CRITICAL_SECTION;
00318             case_validation_count = 0;
00319             case_timeout_occurred = false;
00320             setup_repeat = case_control.repeat;
00321             case_control = control_t();
00322             UTEST_LEAVE_CRITICAL_SECTION;
00323         }
00324 
00325         if (setup_repeat & REPEAT_SETUP_TEARDOWN) {
00326             location = LOCATION_CASE_SETUP;
00327             if (handlers->case_setup && (handlers->case_setup(case_current, test_index_of_case) != STATUS_CONTINUE)) {
00328                 raise_failure(REASON_CASE_SETUP);
00329                 schedule_next_case();
00330                 return;
00331             }
00332         }
00333 
00334         case_failed_before = case_failed;
00335         location = LOCATION_CASE_HANDLER;
00336 
00337         if (case_current->handler) {
00338             case_current->handler();
00339         } else if (case_current->control_handler ) {
00340             case_control = case_control + case_current->control_handler ();
00341         } else if (case_current->repeat_count_handler ) {
00342             case_control = case_control + case_current->repeat_count_handler (case_repeat_count);
00343         }
00344         case_repeat_count++;
00345 
00346         {
00347             UTEST_ENTER_CRITICAL_SECTION;
00348             if (case_validation_count) case_control.repeat = repeat_t(case_control.repeat & ~REPEAT_ON_TIMEOUT);
00349 
00350             // if timeout valid
00351             if (case_control.timeout < TIMEOUT_UNDECLR && case_validation_count == 0) {
00352                 // if await validation _with_ timeout
00353                 if (case_control.timeout < TIMEOUT_FOREVER) {
00354                     case_timeout_handle = scheduler.post(handle_timeout, case_control.timeout);
00355                     if (case_timeout_handle == NULL) {
00356                         raise_failure(REASON_SCHEDULER);
00357                         schedule_next_case();
00358                     }
00359                 }
00360             }
00361             else {
00362                 scheduler.post(schedule_next_case, 0);
00363             }
00364             UTEST_LEAVE_CRITICAL_SECTION;
00365         }
00366     }
00367     else if (handlers->test_teardown) {
00368         location = LOCATION_TEST_TEARDOWN;
00369         handlers->test_teardown(test_passed, test_failed, test_failed ? failure_t(REASON_CASES, LOCATION_UNKNOWN) : failure_t(REASON_NONE));
00370         test_cases = NULL;
00371         exit(test_failed);
00372     } else {
00373         exit(test_failed);
00374     }
00375 }