Nitin Shukla / LED_Demo

nkjnm

Dependencies:   MAX44000 nexpaq_mdk

Fork of LED_Demo by Maxim nexpaq

mbd_os/tools/test_api.py@1:55a6170b404f, 2016-09-17 (annotated)

Committer:
nexpaq
Date:
Sat Sep 17 16:32:05 2016 +0000
Revision:
1:55a6170b404f
checking in for sharing

Who changed what in which revision?

UserRevisionLine numberNew contents of line
nexpaq 1:55a6170b404f 1 """
nexpaq 1:55a6170b404f 2 mbed SDK
nexpaq 1:55a6170b404f 3 Copyright (c) 2011-2014 ARM Limited
nexpaq 1:55a6170b404f 4
nexpaq 1:55a6170b404f 5 Licensed under the Apache License, Version 2.0 (the "License");
nexpaq 1:55a6170b404f 6 you may not use this file except in compliance with the License.
nexpaq 1:55a6170b404f 7 You may obtain a copy of the License at
nexpaq 1:55a6170b404f 8
nexpaq 1:55a6170b404f 9 http://www.apache.org/licenses/LICENSE-2.0
nexpaq 1:55a6170b404f 10
nexpaq 1:55a6170b404f 11 Unless required by applicable law or agreed to in writing, software
nexpaq 1:55a6170b404f 12 distributed under the License is distributed on an "AS IS" BASIS,
nexpaq 1:55a6170b404f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
nexpaq 1:55a6170b404f 14 See the License for the specific language governing permissions and
nexpaq 1:55a6170b404f 15 limitations under the License.
nexpaq 1:55a6170b404f 16
nexpaq 1:55a6170b404f 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
nexpaq 1:55a6170b404f 18 """
nexpaq 1:55a6170b404f 19
nexpaq 1:55a6170b404f 20 import os
nexpaq 1:55a6170b404f 21 import re
nexpaq 1:55a6170b404f 22 import sys
nexpaq 1:55a6170b404f 23 import json
nexpaq 1:55a6170b404f 24 import uuid
nexpaq 1:55a6170b404f 25 import pprint
nexpaq 1:55a6170b404f 26 import random
nexpaq 1:55a6170b404f 27 import argparse
nexpaq 1:55a6170b404f 28 import datetime
nexpaq 1:55a6170b404f 29 import threading
nexpaq 1:55a6170b404f 30 import ctypes
nexpaq 1:55a6170b404f 31 from types import ListType
nexpaq 1:55a6170b404f 32 from colorama import Fore, Back, Style
nexpaq 1:55a6170b404f 33 from prettytable import PrettyTable
nexpaq 1:55a6170b404f 34 from copy import copy
nexpaq 1:55a6170b404f 35
nexpaq 1:55a6170b404f 36 from time import sleep, time
nexpaq 1:55a6170b404f 37 from Queue import Queue, Empty
nexpaq 1:55a6170b404f 38 from os.path import join, exists, basename, relpath
nexpaq 1:55a6170b404f 39 from threading import Thread, Lock
nexpaq 1:55a6170b404f 40 from subprocess import Popen, PIPE
nexpaq 1:55a6170b404f 41
nexpaq 1:55a6170b404f 42 # Imports related to mbed build api
nexpaq 1:55a6170b404f 43 from tools.tests import TESTS
nexpaq 1:55a6170b404f 44 from tools.tests import TEST_MAP
nexpaq 1:55a6170b404f 45 from tools.paths import BUILD_DIR
nexpaq 1:55a6170b404f 46 from tools.paths import HOST_TESTS
nexpaq 1:55a6170b404f 47 from tools.utils import ToolException
nexpaq 1:55a6170b404f 48 from tools.utils import NotSupportedException
nexpaq 1:55a6170b404f 49 from tools.utils import construct_enum
nexpaq 1:55a6170b404f 50 from tools.memap import MemapParser
nexpaq 1:55a6170b404f 51 from tools.targets import TARGET_MAP
nexpaq 1:55a6170b404f 52 from tools.test_db import BaseDBAccess
nexpaq 1:55a6170b404f 53 from tools.build_api import build_project, build_mbed_libs, build_lib
nexpaq 1:55a6170b404f 54 from tools.build_api import get_target_supported_toolchains
nexpaq 1:55a6170b404f 55 from tools.build_api import write_build_report
nexpaq 1:55a6170b404f 56 from tools.build_api import prep_report
nexpaq 1:55a6170b404f 57 from tools.build_api import prep_properties
nexpaq 1:55a6170b404f 58 from tools.build_api import create_result
nexpaq 1:55a6170b404f 59 from tools.build_api import add_result_to_report
nexpaq 1:55a6170b404f 60 from tools.build_api import prepare_toolchain
nexpaq 1:55a6170b404f 61 from tools.build_api import scan_resources
nexpaq 1:55a6170b404f 62 from tools.libraries import LIBRARIES, LIBRARY_MAP
nexpaq 1:55a6170b404f 63 from tools.toolchains import TOOLCHAIN_PATHS
nexpaq 1:55a6170b404f 64 from tools.toolchains import TOOLCHAINS
nexpaq 1:55a6170b404f 65 from tools.test_exporters import ReportExporter, ResultExporterType
nexpaq 1:55a6170b404f 66 from tools.utils import argparse_filestring_type
nexpaq 1:55a6170b404f 67 from tools.utils import argparse_uppercase_type
nexpaq 1:55a6170b404f 68 from tools.utils import argparse_lowercase_type
nexpaq 1:55a6170b404f 69 from tools.utils import argparse_many
nexpaq 1:55a6170b404f 70 from tools.utils import get_path_depth
nexpaq 1:55a6170b404f 71
nexpaq 1:55a6170b404f 72 import tools.host_tests.host_tests_plugins as host_tests_plugins
nexpaq 1:55a6170b404f 73
nexpaq 1:55a6170b404f 74 try:
nexpaq 1:55a6170b404f 75 import mbed_lstools
nexpaq 1:55a6170b404f 76 from tools.compliance.ioper_runner import get_available_oper_test_scopes
nexpaq 1:55a6170b404f 77 except:
nexpaq 1:55a6170b404f 78 pass
nexpaq 1:55a6170b404f 79
nexpaq 1:55a6170b404f 80
nexpaq 1:55a6170b404f 81 class ProcessObserver(Thread):
nexpaq 1:55a6170b404f 82 def __init__(self, proc):
nexpaq 1:55a6170b404f 83 Thread.__init__(self)
nexpaq 1:55a6170b404f 84 self.proc = proc
nexpaq 1:55a6170b404f 85 self.queue = Queue()
nexpaq 1:55a6170b404f 86 self.daemon = True
nexpaq 1:55a6170b404f 87 self.active = True
nexpaq 1:55a6170b404f 88 self.start()
nexpaq 1:55a6170b404f 89
nexpaq 1:55a6170b404f 90 def run(self):
nexpaq 1:55a6170b404f 91 while self.active:
nexpaq 1:55a6170b404f 92 c = self.proc.stdout.read(1)
nexpaq 1:55a6170b404f 93 self.queue.put(c)
nexpaq 1:55a6170b404f 94
nexpaq 1:55a6170b404f 95 def stop(self):
nexpaq 1:55a6170b404f 96 self.active = False
nexpaq 1:55a6170b404f 97 try:
nexpaq 1:55a6170b404f 98 self.proc.terminate()
nexpaq 1:55a6170b404f 99 except Exception, _:
nexpaq 1:55a6170b404f 100 pass
nexpaq 1:55a6170b404f 101
nexpaq 1:55a6170b404f 102
nexpaq 1:55a6170b404f 103 class SingleTestExecutor(threading.Thread):
nexpaq 1:55a6170b404f 104 """ Example: Single test class in separate thread usage
nexpaq 1:55a6170b404f 105 """
nexpaq 1:55a6170b404f 106 def __init__(self, single_test):
nexpaq 1:55a6170b404f 107 self.single_test = single_test
nexpaq 1:55a6170b404f 108 threading.Thread.__init__(self)
nexpaq 1:55a6170b404f 109
nexpaq 1:55a6170b404f 110 def run(self):
nexpaq 1:55a6170b404f 111 start = time()
nexpaq 1:55a6170b404f 112 # Execute tests depending on options and filter applied
nexpaq 1:55a6170b404f 113 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
nexpaq 1:55a6170b404f 114 elapsed_time = time() - start
nexpaq 1:55a6170b404f 115
nexpaq 1:55a6170b404f 116 # Human readable summary
nexpaq 1:55a6170b404f 117 if not self.single_test.opts_suppress_summary:
nexpaq 1:55a6170b404f 118 # prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 119 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
nexpaq 1:55a6170b404f 120 if self.single_test.opts_test_x_toolchain_summary:
nexpaq 1:55a6170b404f 121 # prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 122 # table shows text x toolchain test result matrix
nexpaq 1:55a6170b404f 123 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
nexpaq 1:55a6170b404f 124 print "Completed in %.2f sec"% (elapsed_time)
nexpaq 1:55a6170b404f 125
nexpaq 1:55a6170b404f 126
nexpaq 1:55a6170b404f 127 class SingleTestRunner(object):
nexpaq 1:55a6170b404f 128 """ Object wrapper for single test run which may involve multiple MUTs
nexpaq 1:55a6170b404f 129 """
nexpaq 1:55a6170b404f 130 RE_DETECT_TESTCASE_RESULT = None
nexpaq 1:55a6170b404f 131
nexpaq 1:55a6170b404f 132 # Return codes for test script
nexpaq 1:55a6170b404f 133 TEST_RESULT_OK = "OK"
nexpaq 1:55a6170b404f 134 TEST_RESULT_FAIL = "FAIL"
nexpaq 1:55a6170b404f 135 TEST_RESULT_ERROR = "ERROR"
nexpaq 1:55a6170b404f 136 TEST_RESULT_UNDEF = "UNDEF"
nexpaq 1:55a6170b404f 137 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
nexpaq 1:55a6170b404f 138 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
nexpaq 1:55a6170b404f 139 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
nexpaq 1:55a6170b404f 140 TEST_RESULT_TIMEOUT = "TIMEOUT"
nexpaq 1:55a6170b404f 141 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
nexpaq 1:55a6170b404f 142 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
nexpaq 1:55a6170b404f 143 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
nexpaq 1:55a6170b404f 144 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
nexpaq 1:55a6170b404f 145
nexpaq 1:55a6170b404f 146 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
nexpaq 1:55a6170b404f 147 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
nexpaq 1:55a6170b404f 148 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
nexpaq 1:55a6170b404f 149
nexpaq 1:55a6170b404f 150 muts = {} # MUTs descriptor (from external file)
nexpaq 1:55a6170b404f 151 test_spec = {} # Test specification (from external file)
nexpaq 1:55a6170b404f 152
nexpaq 1:55a6170b404f 153 # mbed test suite -> SingleTestRunner
nexpaq 1:55a6170b404f 154 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
nexpaq 1:55a6170b404f 155 "failure" : TEST_RESULT_FAIL,
nexpaq 1:55a6170b404f 156 "error" : TEST_RESULT_ERROR,
nexpaq 1:55a6170b404f 157 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
nexpaq 1:55a6170b404f 158 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
nexpaq 1:55a6170b404f 159 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
nexpaq 1:55a6170b404f 160 "timeout" : TEST_RESULT_TIMEOUT,
nexpaq 1:55a6170b404f 161 "no_image" : TEST_RESULT_NO_IMAGE,
nexpaq 1:55a6170b404f 162 "end" : TEST_RESULT_UNDEF,
nexpaq 1:55a6170b404f 163 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
nexpaq 1:55a6170b404f 164 "build_failed" : TEST_RESULT_BUILD_FAILED,
nexpaq 1:55a6170b404f 165 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
nexpaq 1:55a6170b404f 166 }
nexpaq 1:55a6170b404f 167
nexpaq 1:55a6170b404f 168 def __init__(self,
nexpaq 1:55a6170b404f 169 _global_loops_count=1,
nexpaq 1:55a6170b404f 170 _test_loops_list=None,
nexpaq 1:55a6170b404f 171 _muts={},
nexpaq 1:55a6170b404f 172 _clean=False,
nexpaq 1:55a6170b404f 173 _opts_db_url=None,
nexpaq 1:55a6170b404f 174 _opts_log_file_name=None,
nexpaq 1:55a6170b404f 175 _opts_report_html_file_name=None,
nexpaq 1:55a6170b404f 176 _opts_report_junit_file_name=None,
nexpaq 1:55a6170b404f 177 _opts_report_build_file_name=None,
nexpaq 1:55a6170b404f 178 _opts_report_text_file_name=None,
nexpaq 1:55a6170b404f 179 _opts_build_report={},
nexpaq 1:55a6170b404f 180 _opts_build_properties={},
nexpaq 1:55a6170b404f 181 _test_spec={},
nexpaq 1:55a6170b404f 182 _opts_goanna_for_mbed_sdk=None,
nexpaq 1:55a6170b404f 183 _opts_goanna_for_tests=None,
nexpaq 1:55a6170b404f 184 _opts_shuffle_test_order=False,
nexpaq 1:55a6170b404f 185 _opts_shuffle_test_seed=None,
nexpaq 1:55a6170b404f 186 _opts_test_by_names=None,
nexpaq 1:55a6170b404f 187 _opts_peripheral_by_names=None,
nexpaq 1:55a6170b404f 188 _opts_test_only_peripheral=False,
nexpaq 1:55a6170b404f 189 _opts_test_only_common=False,
nexpaq 1:55a6170b404f 190 _opts_verbose_skipped_tests=False,
nexpaq 1:55a6170b404f 191 _opts_verbose_test_result_only=False,
nexpaq 1:55a6170b404f 192 _opts_verbose=False,
nexpaq 1:55a6170b404f 193 _opts_firmware_global_name=None,
nexpaq 1:55a6170b404f 194 _opts_only_build_tests=False,
nexpaq 1:55a6170b404f 195 _opts_parallel_test_exec=False,
nexpaq 1:55a6170b404f 196 _opts_suppress_summary=False,
nexpaq 1:55a6170b404f 197 _opts_test_x_toolchain_summary=False,
nexpaq 1:55a6170b404f 198 _opts_copy_method=None,
nexpaq 1:55a6170b404f 199 _opts_mut_reset_type=None,
nexpaq 1:55a6170b404f 200 _opts_jobs=None,
nexpaq 1:55a6170b404f 201 _opts_waterfall_test=None,
nexpaq 1:55a6170b404f 202 _opts_consolidate_waterfall_test=None,
nexpaq 1:55a6170b404f 203 _opts_extend_test_timeout=None,
nexpaq 1:55a6170b404f 204 _opts_auto_detect=None,
nexpaq 1:55a6170b404f 205 _opts_include_non_automated=False):
nexpaq 1:55a6170b404f 206 """ Let's try hard to init this object
nexpaq 1:55a6170b404f 207 """
nexpaq 1:55a6170b404f 208 from colorama import init
nexpaq 1:55a6170b404f 209 init()
nexpaq 1:55a6170b404f 210
nexpaq 1:55a6170b404f 211 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
nexpaq 1:55a6170b404f 212 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
nexpaq 1:55a6170b404f 213 # Settings related to test loops counters
nexpaq 1:55a6170b404f 214 try:
nexpaq 1:55a6170b404f 215 _global_loops_count = int(_global_loops_count)
nexpaq 1:55a6170b404f 216 except:
nexpaq 1:55a6170b404f 217 _global_loops_count = 1
nexpaq 1:55a6170b404f 218 if _global_loops_count < 1:
nexpaq 1:55a6170b404f 219 _global_loops_count = 1
nexpaq 1:55a6170b404f 220 self.GLOBAL_LOOPS_COUNT = _global_loops_count
nexpaq 1:55a6170b404f 221 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
nexpaq 1:55a6170b404f 222 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
nexpaq 1:55a6170b404f 223
nexpaq 1:55a6170b404f 224 self.shuffle_random_seed = 0.0
nexpaq 1:55a6170b404f 225 self.SHUFFLE_SEED_ROUND = 10
nexpaq 1:55a6170b404f 226
nexpaq 1:55a6170b404f 227 # MUT list and test specification storage
nexpaq 1:55a6170b404f 228 self.muts = _muts
nexpaq 1:55a6170b404f 229 self.test_spec = _test_spec
nexpaq 1:55a6170b404f 230
nexpaq 1:55a6170b404f 231 # Settings passed e.g. from command line
nexpaq 1:55a6170b404f 232 self.opts_db_url = _opts_db_url
nexpaq 1:55a6170b404f 233 self.opts_log_file_name = _opts_log_file_name
nexpaq 1:55a6170b404f 234 self.opts_report_html_file_name = _opts_report_html_file_name
nexpaq 1:55a6170b404f 235 self.opts_report_junit_file_name = _opts_report_junit_file_name
nexpaq 1:55a6170b404f 236 self.opts_report_build_file_name = _opts_report_build_file_name
nexpaq 1:55a6170b404f 237 self.opts_report_text_file_name = _opts_report_text_file_name
nexpaq 1:55a6170b404f 238 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
nexpaq 1:55a6170b404f 239 self.opts_goanna_for_tests = _opts_goanna_for_tests
nexpaq 1:55a6170b404f 240 self.opts_shuffle_test_order = _opts_shuffle_test_order
nexpaq 1:55a6170b404f 241 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
nexpaq 1:55a6170b404f 242 self.opts_test_by_names = _opts_test_by_names
nexpaq 1:55a6170b404f 243 self.opts_peripheral_by_names = _opts_peripheral_by_names
nexpaq 1:55a6170b404f 244 self.opts_test_only_peripheral = _opts_test_only_peripheral
nexpaq 1:55a6170b404f 245 self.opts_test_only_common = _opts_test_only_common
nexpaq 1:55a6170b404f 246 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
nexpaq 1:55a6170b404f 247 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
nexpaq 1:55a6170b404f 248 self.opts_verbose = _opts_verbose
nexpaq 1:55a6170b404f 249 self.opts_firmware_global_name = _opts_firmware_global_name
nexpaq 1:55a6170b404f 250 self.opts_only_build_tests = _opts_only_build_tests
nexpaq 1:55a6170b404f 251 self.opts_parallel_test_exec = _opts_parallel_test_exec
nexpaq 1:55a6170b404f 252 self.opts_suppress_summary = _opts_suppress_summary
nexpaq 1:55a6170b404f 253 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
nexpaq 1:55a6170b404f 254 self.opts_copy_method = _opts_copy_method
nexpaq 1:55a6170b404f 255 self.opts_mut_reset_type = _opts_mut_reset_type
nexpaq 1:55a6170b404f 256 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
nexpaq 1:55a6170b404f 257 self.opts_waterfall_test = _opts_waterfall_test
nexpaq 1:55a6170b404f 258 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
nexpaq 1:55a6170b404f 259 self.opts_extend_test_timeout = _opts_extend_test_timeout
nexpaq 1:55a6170b404f 260 self.opts_clean = _clean
nexpaq 1:55a6170b404f 261 self.opts_auto_detect = _opts_auto_detect
nexpaq 1:55a6170b404f 262 self.opts_include_non_automated = _opts_include_non_automated
nexpaq 1:55a6170b404f 263
nexpaq 1:55a6170b404f 264 self.build_report = _opts_build_report
nexpaq 1:55a6170b404f 265 self.build_properties = _opts_build_properties
nexpaq 1:55a6170b404f 266
nexpaq 1:55a6170b404f 267 # File / screen logger initialization
nexpaq 1:55a6170b404f 268 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
nexpaq 1:55a6170b404f 269
nexpaq 1:55a6170b404f 270 # Database related initializations
nexpaq 1:55a6170b404f 271 self.db_logger = factory_db_logger(self.opts_db_url)
nexpaq 1:55a6170b404f 272 self.db_logger_build_id = None # Build ID (database index of build_id table)
nexpaq 1:55a6170b404f 273 # Let's connect to database to set up credentials and confirm database is ready
nexpaq 1:55a6170b404f 274 if self.db_logger:
nexpaq 1:55a6170b404f 275 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
nexpaq 1:55a6170b404f 276 if self.db_logger.is_connected():
nexpaq 1:55a6170b404f 277 # Get hostname and uname so we can use it as build description
nexpaq 1:55a6170b404f 278 # when creating new build_id in external database
nexpaq 1:55a6170b404f 279 (_hostname, _uname) = self.db_logger.get_hostname()
nexpaq 1:55a6170b404f 280 _host_location = os.path.dirname(os.path.abspath(__file__))
nexpaq 1:55a6170b404f 281 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
nexpaq 1:55a6170b404f 282 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
nexpaq 1:55a6170b404f 283 self.db_logger.disconnect()
nexpaq 1:55a6170b404f 284
nexpaq 1:55a6170b404f 285 def dump_options(self):
nexpaq 1:55a6170b404f 286 """ Function returns data structure with common settings passed to SingelTestRunner
nexpaq 1:55a6170b404f 287 It can be used for example to fill _extra fields in database storing test suite single run data
nexpaq 1:55a6170b404f 288 Example:
nexpaq 1:55a6170b404f 289 data = self.dump_options()
nexpaq 1:55a6170b404f 290 or
nexpaq 1:55a6170b404f 291 data_str = json.dumps(self.dump_options())
nexpaq 1:55a6170b404f 292 """
nexpaq 1:55a6170b404f 293 result = {"db_url" : str(self.opts_db_url),
nexpaq 1:55a6170b404f 294 "log_file_name" : str(self.opts_log_file_name),
nexpaq 1:55a6170b404f 295 "shuffle_test_order" : str(self.opts_shuffle_test_order),
nexpaq 1:55a6170b404f 296 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
nexpaq 1:55a6170b404f 297 "test_by_names" : str(self.opts_test_by_names),
nexpaq 1:55a6170b404f 298 "peripheral_by_names" : str(self.opts_peripheral_by_names),
nexpaq 1:55a6170b404f 299 "test_only_peripheral" : str(self.opts_test_only_peripheral),
nexpaq 1:55a6170b404f 300 "test_only_common" : str(self.opts_test_only_common),
nexpaq 1:55a6170b404f 301 "verbose" : str(self.opts_verbose),
nexpaq 1:55a6170b404f 302 "firmware_global_name" : str(self.opts_firmware_global_name),
nexpaq 1:55a6170b404f 303 "only_build_tests" : str(self.opts_only_build_tests),
nexpaq 1:55a6170b404f 304 "copy_method" : str(self.opts_copy_method),
nexpaq 1:55a6170b404f 305 "mut_reset_type" : str(self.opts_mut_reset_type),
nexpaq 1:55a6170b404f 306 "jobs" : str(self.opts_jobs),
nexpaq 1:55a6170b404f 307 "extend_test_timeout" : str(self.opts_extend_test_timeout),
nexpaq 1:55a6170b404f 308 "_dummy" : ''
nexpaq 1:55a6170b404f 309 }
nexpaq 1:55a6170b404f 310 return result
nexpaq 1:55a6170b404f 311
nexpaq 1:55a6170b404f 312 def shuffle_random_func(self):
nexpaq 1:55a6170b404f 313 return self.shuffle_random_seed
nexpaq 1:55a6170b404f 314
nexpaq 1:55a6170b404f 315 def is_shuffle_seed_float(self):
nexpaq 1:55a6170b404f 316 """ return true if function parameter can be converted to float
nexpaq 1:55a6170b404f 317 """
nexpaq 1:55a6170b404f 318 result = True
nexpaq 1:55a6170b404f 319 try:
nexpaq 1:55a6170b404f 320 float(self.shuffle_random_seed)
nexpaq 1:55a6170b404f 321 except ValueError:
nexpaq 1:55a6170b404f 322 result = False
nexpaq 1:55a6170b404f 323 return result
nexpaq 1:55a6170b404f 324
nexpaq 1:55a6170b404f 325 # This will store target / toolchain specific properties
nexpaq 1:55a6170b404f 326 test_suite_properties_ext = {} # target : toolchain
nexpaq 1:55a6170b404f 327 # Here we store test results
nexpaq 1:55a6170b404f 328 test_summary = []
nexpaq 1:55a6170b404f 329 # Here we store test results in extended data structure
nexpaq 1:55a6170b404f 330 test_summary_ext = {}
nexpaq 1:55a6170b404f 331 execute_thread_slice_lock = Lock()
nexpaq 1:55a6170b404f 332
nexpaq 1:55a6170b404f 333 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
nexpaq 1:55a6170b404f 334 for toolchain in toolchains:
nexpaq 1:55a6170b404f 335 tt_id = "%s::%s" % (toolchain, target)
nexpaq 1:55a6170b404f 336
nexpaq 1:55a6170b404f 337 T = TARGET_MAP[target]
nexpaq 1:55a6170b404f 338
nexpaq 1:55a6170b404f 339 # print target, toolchain
nexpaq 1:55a6170b404f 340 # Test suite properties returned to external tools like CI
nexpaq 1:55a6170b404f 341 test_suite_properties = {
nexpaq 1:55a6170b404f 342 'jobs': self.opts_jobs,
nexpaq 1:55a6170b404f 343 'clean': clean,
nexpaq 1:55a6170b404f 344 'target': target,
nexpaq 1:55a6170b404f 345 'vendor': T.extra_labels[0],
nexpaq 1:55a6170b404f 346 'test_ids': ', '.join(test_ids),
nexpaq 1:55a6170b404f 347 'toolchain': toolchain,
nexpaq 1:55a6170b404f 348 'shuffle_random_seed': self.shuffle_random_seed
nexpaq 1:55a6170b404f 349 }
nexpaq 1:55a6170b404f 350
nexpaq 1:55a6170b404f 351
nexpaq 1:55a6170b404f 352 # print '=== %s::%s ===' % (target, toolchain)
nexpaq 1:55a6170b404f 353 # Let's build our test
nexpaq 1:55a6170b404f 354 if target not in TARGET_MAP:
nexpaq 1:55a6170b404f 355 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
nexpaq 1:55a6170b404f 356 continue
nexpaq 1:55a6170b404f 357
nexpaq 1:55a6170b404f 358 build_mbed_libs_options = ["analyze"] if self.opts_goanna_for_mbed_sdk else None
nexpaq 1:55a6170b404f 359 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
nexpaq 1:55a6170b404f 360
nexpaq 1:55a6170b404f 361
nexpaq 1:55a6170b404f 362 try:
nexpaq 1:55a6170b404f 363 build_mbed_libs_result = build_mbed_libs(T,
nexpaq 1:55a6170b404f 364 toolchain,
nexpaq 1:55a6170b404f 365 options=build_mbed_libs_options,
nexpaq 1:55a6170b404f 366 clean=clean_mbed_libs_options,
nexpaq 1:55a6170b404f 367 verbose=self.opts_verbose,
nexpaq 1:55a6170b404f 368 jobs=self.opts_jobs,
nexpaq 1:55a6170b404f 369 report=build_report,
nexpaq 1:55a6170b404f 370 properties=build_properties)
nexpaq 1:55a6170b404f 371
nexpaq 1:55a6170b404f 372 if not build_mbed_libs_result:
nexpaq 1:55a6170b404f 373 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
nexpaq 1:55a6170b404f 374 continue
nexpaq 1:55a6170b404f 375
nexpaq 1:55a6170b404f 376 except ToolException:
nexpaq 1:55a6170b404f 377 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
nexpaq 1:55a6170b404f 378 continue
nexpaq 1:55a6170b404f 379
nexpaq 1:55a6170b404f 380 build_dir = join(BUILD_DIR, "test", target, toolchain)
nexpaq 1:55a6170b404f 381
nexpaq 1:55a6170b404f 382 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
nexpaq 1:55a6170b404f 383 test_suite_properties['build_dir'] = build_dir
nexpaq 1:55a6170b404f 384 test_suite_properties['skipped'] = []
nexpaq 1:55a6170b404f 385
nexpaq 1:55a6170b404f 386 # Enumerate through all tests and shuffle test order if requested
nexpaq 1:55a6170b404f 387 test_map_keys = sorted(TEST_MAP.keys())
nexpaq 1:55a6170b404f 388
nexpaq 1:55a6170b404f 389 if self.opts_shuffle_test_order:
nexpaq 1:55a6170b404f 390 random.shuffle(test_map_keys, self.shuffle_random_func)
nexpaq 1:55a6170b404f 391 # Update database with shuffle seed f applicable
nexpaq 1:55a6170b404f 392 if self.db_logger:
nexpaq 1:55a6170b404f 393 self.db_logger.reconnect();
nexpaq 1:55a6170b404f 394 if self.db_logger.is_connected():
nexpaq 1:55a6170b404f 395 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
nexpaq 1:55a6170b404f 396 self.db_logger.disconnect();
nexpaq 1:55a6170b404f 397
nexpaq 1:55a6170b404f 398 if self.db_logger:
nexpaq 1:55a6170b404f 399 self.db_logger.reconnect();
nexpaq 1:55a6170b404f 400 if self.db_logger.is_connected():
nexpaq 1:55a6170b404f 401 # Update MUTs and Test Specification in database
nexpaq 1:55a6170b404f 402 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
nexpaq 1:55a6170b404f 403 # Update Extra information in database (some options passed to test suite)
nexpaq 1:55a6170b404f 404 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
nexpaq 1:55a6170b404f 405 self.db_logger.disconnect();
nexpaq 1:55a6170b404f 406
nexpaq 1:55a6170b404f 407 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
nexpaq 1:55a6170b404f 408 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
nexpaq 1:55a6170b404f 409
nexpaq 1:55a6170b404f 410 for skipped_test_id in skipped_test_map_keys:
nexpaq 1:55a6170b404f 411 test_suite_properties['skipped'].append(skipped_test_id)
nexpaq 1:55a6170b404f 412
nexpaq 1:55a6170b404f 413
nexpaq 1:55a6170b404f 414 # First pass through all tests and determine which libraries need to be built
nexpaq 1:55a6170b404f 415 libraries = []
nexpaq 1:55a6170b404f 416 for test_id in valid_test_map_keys:
nexpaq 1:55a6170b404f 417 test = TEST_MAP[test_id]
nexpaq 1:55a6170b404f 418
nexpaq 1:55a6170b404f 419 # Detect which lib should be added to test
nexpaq 1:55a6170b404f 420 # Some libs have to compiled like RTOS or ETH
nexpaq 1:55a6170b404f 421 for lib in LIBRARIES:
nexpaq 1:55a6170b404f 422 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
nexpaq 1:55a6170b404f 423 libraries.append(lib['id'])
nexpaq 1:55a6170b404f 424
nexpaq 1:55a6170b404f 425
nexpaq 1:55a6170b404f 426 build_project_options = ["analyze"] if self.opts_goanna_for_tests else None
nexpaq 1:55a6170b404f 427 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
nexpaq 1:55a6170b404f 428
nexpaq 1:55a6170b404f 429 # Build all required libraries
nexpaq 1:55a6170b404f 430 for lib_id in libraries:
nexpaq 1:55a6170b404f 431 try:
nexpaq 1:55a6170b404f 432 build_lib(lib_id,
nexpaq 1:55a6170b404f 433 T,
nexpaq 1:55a6170b404f 434 toolchain,
nexpaq 1:55a6170b404f 435 options=build_project_options,
nexpaq 1:55a6170b404f 436 verbose=self.opts_verbose,
nexpaq 1:55a6170b404f 437 clean=clean_mbed_libs_options,
nexpaq 1:55a6170b404f 438 jobs=self.opts_jobs,
nexpaq 1:55a6170b404f 439 report=build_report,
nexpaq 1:55a6170b404f 440 properties=build_properties)
nexpaq 1:55a6170b404f 441
nexpaq 1:55a6170b404f 442 except ToolException:
nexpaq 1:55a6170b404f 443 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
nexpaq 1:55a6170b404f 444 continue
nexpaq 1:55a6170b404f 445
nexpaq 1:55a6170b404f 446
nexpaq 1:55a6170b404f 447 for test_id in valid_test_map_keys:
nexpaq 1:55a6170b404f 448 test = TEST_MAP[test_id]
nexpaq 1:55a6170b404f 449
nexpaq 1:55a6170b404f 450 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
nexpaq 1:55a6170b404f 451
nexpaq 1:55a6170b404f 452 # TODO: move this 2 below loops to separate function
nexpaq 1:55a6170b404f 453 INC_DIRS = []
nexpaq 1:55a6170b404f 454 for lib_id in libraries:
nexpaq 1:55a6170b404f 455 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
nexpaq 1:55a6170b404f 456 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
nexpaq 1:55a6170b404f 457
nexpaq 1:55a6170b404f 458 MACROS = []
nexpaq 1:55a6170b404f 459 for lib_id in libraries:
nexpaq 1:55a6170b404f 460 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
nexpaq 1:55a6170b404f 461 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
nexpaq 1:55a6170b404f 462 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
nexpaq 1:55a6170b404f 463 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
nexpaq 1:55a6170b404f 464 test_uuid = uuid.uuid4()
nexpaq 1:55a6170b404f 465 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
nexpaq 1:55a6170b404f 466
nexpaq 1:55a6170b404f 467 # Prepare extended test results data structure (it can be used to generate detailed test report)
nexpaq 1:55a6170b404f 468 if target not in self.test_summary_ext:
nexpaq 1:55a6170b404f 469 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
nexpaq 1:55a6170b404f 470 if toolchain not in self.test_summary_ext[target]:
nexpaq 1:55a6170b404f 471 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
nexpaq 1:55a6170b404f 472
nexpaq 1:55a6170b404f 473 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
nexpaq 1:55a6170b404f 474
nexpaq 1:55a6170b404f 475 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
nexpaq 1:55a6170b404f 476 try:
nexpaq 1:55a6170b404f 477 path = build_project(test.source_dir,
nexpaq 1:55a6170b404f 478 join(build_dir, test_id),
nexpaq 1:55a6170b404f 479 T,
nexpaq 1:55a6170b404f 480 toolchain,
nexpaq 1:55a6170b404f 481 test.dependencies,
nexpaq 1:55a6170b404f 482 options=build_project_options,
nexpaq 1:55a6170b404f 483 clean=clean_project_options,
nexpaq 1:55a6170b404f 484 verbose=self.opts_verbose,
nexpaq 1:55a6170b404f 485 name=project_name,
nexpaq 1:55a6170b404f 486 macros=MACROS,
nexpaq 1:55a6170b404f 487 inc_dirs=INC_DIRS,
nexpaq 1:55a6170b404f 488 jobs=self.opts_jobs,
nexpaq 1:55a6170b404f 489 report=build_report,
nexpaq 1:55a6170b404f 490 properties=build_properties,
nexpaq 1:55a6170b404f 491 project_id=test_id,
nexpaq 1:55a6170b404f 492 project_description=test.get_description())
nexpaq 1:55a6170b404f 493
nexpaq 1:55a6170b404f 494 except Exception, e:
nexpaq 1:55a6170b404f 495 project_name_str = project_name if project_name is not None else test_id
nexpaq 1:55a6170b404f 496
nexpaq 1:55a6170b404f 497
nexpaq 1:55a6170b404f 498 test_result = self.TEST_RESULT_FAIL
nexpaq 1:55a6170b404f 499
nexpaq 1:55a6170b404f 500 if isinstance(e, ToolException):
nexpaq 1:55a6170b404f 501 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
nexpaq 1:55a6170b404f 502 test_result = self.TEST_RESULT_BUILD_FAILED
nexpaq 1:55a6170b404f 503 elif isinstance(e, NotSupportedException):
nexpaq 1:55a6170b404f 504 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
nexpaq 1:55a6170b404f 505 test_result = self.TEST_RESULT_NOT_SUPPORTED
nexpaq 1:55a6170b404f 506
nexpaq 1:55a6170b404f 507
nexpaq 1:55a6170b404f 508 # Append test results to global test summary
nexpaq 1:55a6170b404f 509 self.test_summary.append(
nexpaq 1:55a6170b404f 510 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
nexpaq 1:55a6170b404f 511 )
nexpaq 1:55a6170b404f 512
nexpaq 1:55a6170b404f 513 # Add detailed test result to test summary structure
nexpaq 1:55a6170b404f 514 if test_id not in self.test_summary_ext[target][toolchain]:
nexpaq 1:55a6170b404f 515 self.test_summary_ext[target][toolchain][test_id] = []
nexpaq 1:55a6170b404f 516
nexpaq 1:55a6170b404f 517 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
nexpaq 1:55a6170b404f 518 'result' : test_result,
nexpaq 1:55a6170b404f 519 'output' : '',
nexpaq 1:55a6170b404f 520 'target_name' : target,
nexpaq 1:55a6170b404f 521 'target_name_unique': target,
nexpaq 1:55a6170b404f 522 'toolchain_name' : toolchain,
nexpaq 1:55a6170b404f 523 'id' : test_id,
nexpaq 1:55a6170b404f 524 'description' : test.get_description(),
nexpaq 1:55a6170b404f 525 'elapsed_time' : 0,
nexpaq 1:55a6170b404f 526 'duration' : 0,
nexpaq 1:55a6170b404f 527 'copy_method' : None
nexpaq 1:55a6170b404f 528 }})
nexpaq 1:55a6170b404f 529 continue
nexpaq 1:55a6170b404f 530
nexpaq 1:55a6170b404f 531 if self.opts_only_build_tests:
nexpaq 1:55a6170b404f 532 # With this option we are skipping testing phase
nexpaq 1:55a6170b404f 533 continue
nexpaq 1:55a6170b404f 534
nexpaq 1:55a6170b404f 535 # Test duration can be increased by global value
nexpaq 1:55a6170b404f 536 test_duration = test.duration
nexpaq 1:55a6170b404f 537 if self.opts_extend_test_timeout is not None:
nexpaq 1:55a6170b404f 538 test_duration += self.opts_extend_test_timeout
nexpaq 1:55a6170b404f 539
nexpaq 1:55a6170b404f 540 # For an automated test the duration act as a timeout after
nexpaq 1:55a6170b404f 541 # which the test gets interrupted
nexpaq 1:55a6170b404f 542 test_spec = self.shape_test_request(target, path, test_id, test_duration)
nexpaq 1:55a6170b404f 543 test_loops = self.get_test_loop_count(test_id)
nexpaq 1:55a6170b404f 544
nexpaq 1:55a6170b404f 545 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
nexpaq 1:55a6170b404f 546 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
nexpaq 1:55a6170b404f 547 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
nexpaq 1:55a6170b404f 548
nexpaq 1:55a6170b404f 549 # read MUTs, test specification and perform tests
nexpaq 1:55a6170b404f 550 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
nexpaq 1:55a6170b404f 551
nexpaq 1:55a6170b404f 552 if handle_results is None:
nexpaq 1:55a6170b404f 553 continue
nexpaq 1:55a6170b404f 554
nexpaq 1:55a6170b404f 555 for handle_result in handle_results:
nexpaq 1:55a6170b404f 556 if handle_result:
nexpaq 1:55a6170b404f 557 single_test_result, detailed_test_results = handle_result
nexpaq 1:55a6170b404f 558 else:
nexpaq 1:55a6170b404f 559 continue
nexpaq 1:55a6170b404f 560
nexpaq 1:55a6170b404f 561 # Append test results to global test summary
nexpaq 1:55a6170b404f 562 if single_test_result is not None:
nexpaq 1:55a6170b404f 563 self.test_summary.append(single_test_result)
nexpaq 1:55a6170b404f 564
nexpaq 1:55a6170b404f 565 # Add detailed test result to test summary structure
nexpaq 1:55a6170b404f 566 if target not in self.test_summary_ext[target][toolchain]:
nexpaq 1:55a6170b404f 567 if test_id not in self.test_summary_ext[target][toolchain]:
nexpaq 1:55a6170b404f 568 self.test_summary_ext[target][toolchain][test_id] = []
nexpaq 1:55a6170b404f 569
nexpaq 1:55a6170b404f 570 append_test_result = detailed_test_results
nexpaq 1:55a6170b404f 571
nexpaq 1:55a6170b404f 572 # If waterfall and consolidate-waterfall options are enabled,
nexpaq 1:55a6170b404f 573 # only include the last test result in the report.
nexpaq 1:55a6170b404f 574 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
nexpaq 1:55a6170b404f 575 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
nexpaq 1:55a6170b404f 576
nexpaq 1:55a6170b404f 577 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
nexpaq 1:55a6170b404f 578
nexpaq 1:55a6170b404f 579 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
nexpaq 1:55a6170b404f 580 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
nexpaq 1:55a6170b404f 581
nexpaq 1:55a6170b404f 582 q.put(target + '_'.join(toolchains))
nexpaq 1:55a6170b404f 583 return
nexpaq 1:55a6170b404f 584
nexpaq 1:55a6170b404f 585 def execute(self):
nexpaq 1:55a6170b404f 586 clean = self.test_spec.get('clean', False)
nexpaq 1:55a6170b404f 587 test_ids = self.test_spec.get('test_ids', [])
nexpaq 1:55a6170b404f 588 q = Queue()
nexpaq 1:55a6170b404f 589
nexpaq 1:55a6170b404f 590 # Generate seed for shuffle if seed is not provided in
nexpaq 1:55a6170b404f 591 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
nexpaq 1:55a6170b404f 592 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
nexpaq 1:55a6170b404f 593 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
nexpaq 1:55a6170b404f 594
nexpaq 1:55a6170b404f 595
nexpaq 1:55a6170b404f 596 if self.opts_parallel_test_exec:
nexpaq 1:55a6170b404f 597 ###################################################################
nexpaq 1:55a6170b404f 598 # Experimental, parallel test execution per singletest instance.
nexpaq 1:55a6170b404f 599 ###################################################################
nexpaq 1:55a6170b404f 600 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
nexpaq 1:55a6170b404f 601 # Note: We are building here in parallel for each target separately!
nexpaq 1:55a6170b404f 602 # So we are not building the same thing multiple times and compilers
nexpaq 1:55a6170b404f 603 # in separate threads do not collide.
nexpaq 1:55a6170b404f 604 # Inside execute_thread_slice() function function handle() will be called to
nexpaq 1:55a6170b404f 605 # get information about available MUTs (per target).
nexpaq 1:55a6170b404f 606 for target, toolchains in self.test_spec['targets'].iteritems():
nexpaq 1:55a6170b404f 607 self.test_suite_properties_ext[target] = {}
nexpaq 1:55a6170b404f 608 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
nexpaq 1:55a6170b404f 609 t.daemon = True
nexpaq 1:55a6170b404f 610 t.start()
nexpaq 1:55a6170b404f 611 execute_threads.append(t)
nexpaq 1:55a6170b404f 612
nexpaq 1:55a6170b404f 613 for t in execute_threads:
nexpaq 1:55a6170b404f 614 q.get() # t.join() would block some threads because we should not wait in any order for thread end
nexpaq 1:55a6170b404f 615 else:
nexpaq 1:55a6170b404f 616 # Serialized (not parallel) test execution
nexpaq 1:55a6170b404f 617 for target, toolchains in self.test_spec['targets'].iteritems():
nexpaq 1:55a6170b404f 618 if target not in self.test_suite_properties_ext:
nexpaq 1:55a6170b404f 619 self.test_suite_properties_ext[target] = {}
nexpaq 1:55a6170b404f 620
nexpaq 1:55a6170b404f 621 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
nexpaq 1:55a6170b404f 622 q.get()
nexpaq 1:55a6170b404f 623
nexpaq 1:55a6170b404f 624 if self.db_logger:
nexpaq 1:55a6170b404f 625 self.db_logger.reconnect();
nexpaq 1:55a6170b404f 626 if self.db_logger.is_connected():
nexpaq 1:55a6170b404f 627 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
nexpaq 1:55a6170b404f 628 self.db_logger.disconnect();
nexpaq 1:55a6170b404f 629
nexpaq 1:55a6170b404f 630 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
nexpaq 1:55a6170b404f 631
nexpaq 1:55a6170b404f 632 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
nexpaq 1:55a6170b404f 633 valid_test_map_keys = []
nexpaq 1:55a6170b404f 634
nexpaq 1:55a6170b404f 635 for test_id in test_map_keys:
nexpaq 1:55a6170b404f 636 test = TEST_MAP[test_id]
nexpaq 1:55a6170b404f 637 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
nexpaq 1:55a6170b404f 638 continue
nexpaq 1:55a6170b404f 639
nexpaq 1:55a6170b404f 640 if test_ids and test_id not in test_ids:
nexpaq 1:55a6170b404f 641 continue
nexpaq 1:55a6170b404f 642
nexpaq 1:55a6170b404f 643 if self.opts_test_only_peripheral and not test.peripherals:
nexpaq 1:55a6170b404f 644 if self.opts_verbose_skipped_tests:
nexpaq 1:55a6170b404f 645 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
nexpaq 1:55a6170b404f 646 continue
nexpaq 1:55a6170b404f 647
nexpaq 1:55a6170b404f 648 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
nexpaq 1:55a6170b404f 649 # We will skip tests not forced with -p option
nexpaq 1:55a6170b404f 650 if self.opts_verbose_skipped_tests:
nexpaq 1:55a6170b404f 651 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
nexpaq 1:55a6170b404f 652 continue
nexpaq 1:55a6170b404f 653
nexpaq 1:55a6170b404f 654 if self.opts_test_only_common and test.peripherals:
nexpaq 1:55a6170b404f 655 if self.opts_verbose_skipped_tests:
nexpaq 1:55a6170b404f 656 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
nexpaq 1:55a6170b404f 657 continue
nexpaq 1:55a6170b404f 658
nexpaq 1:55a6170b404f 659 if not include_non_automated and not test.automated:
nexpaq 1:55a6170b404f 660 if self.opts_verbose_skipped_tests:
nexpaq 1:55a6170b404f 661 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
nexpaq 1:55a6170b404f 662 continue
nexpaq 1:55a6170b404f 663
nexpaq 1:55a6170b404f 664 if test.is_supported(target, toolchain):
nexpaq 1:55a6170b404f 665 if test.peripherals is None and self.opts_only_build_tests:
nexpaq 1:55a6170b404f 666 # When users are using 'build only flag' and test do not have
nexpaq 1:55a6170b404f 667 # specified peripherals we can allow test building by default
nexpaq 1:55a6170b404f 668 pass
nexpaq 1:55a6170b404f 669 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
nexpaq 1:55a6170b404f 670 # If we force peripheral with option -p we expect test
nexpaq 1:55a6170b404f 671 # to pass even if peripheral is not in MUTs file.
nexpaq 1:55a6170b404f 672 pass
nexpaq 1:55a6170b404f 673 elif not self.is_peripherals_available(target, test.peripherals):
nexpaq 1:55a6170b404f 674 if self.opts_verbose_skipped_tests:
nexpaq 1:55a6170b404f 675 if test.peripherals:
nexpaq 1:55a6170b404f 676 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
nexpaq 1:55a6170b404f 677 else:
nexpaq 1:55a6170b404f 678 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
nexpaq 1:55a6170b404f 679 continue
nexpaq 1:55a6170b404f 680
nexpaq 1:55a6170b404f 681 # The test has made it through all the filters, so add it to the valid tests list
nexpaq 1:55a6170b404f 682 valid_test_map_keys.append(test_id)
nexpaq 1:55a6170b404f 683
nexpaq 1:55a6170b404f 684 return valid_test_map_keys
nexpaq 1:55a6170b404f 685
nexpaq 1:55a6170b404f 686 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
nexpaq 1:55a6170b404f 687 # NOTE: This will not preserve order
nexpaq 1:55a6170b404f 688 return list(set(all_test_map_keys) - set(valid_test_map_keys))
nexpaq 1:55a6170b404f 689
nexpaq 1:55a6170b404f 690 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
nexpaq 1:55a6170b404f 691 """ Prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 692 table shows text x toolchain test result matrix
nexpaq 1:55a6170b404f 693 """
nexpaq 1:55a6170b404f 694 RESULT_INDEX = 0
nexpaq 1:55a6170b404f 695 TARGET_INDEX = 1
nexpaq 1:55a6170b404f 696 TOOLCHAIN_INDEX = 2
nexpaq 1:55a6170b404f 697 TEST_INDEX = 3
nexpaq 1:55a6170b404f 698 DESC_INDEX = 4
nexpaq 1:55a6170b404f 699
nexpaq 1:55a6170b404f 700 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
nexpaq 1:55a6170b404f 701 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
nexpaq 1:55a6170b404f 702 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
nexpaq 1:55a6170b404f 703 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
nexpaq 1:55a6170b404f 704
nexpaq 1:55a6170b404f 705 result = "Test summary:\n"
nexpaq 1:55a6170b404f 706 for target in unique_targets:
nexpaq 1:55a6170b404f 707 result_dict = {} # test : { toolchain : result }
nexpaq 1:55a6170b404f 708 unique_target_toolchains = []
nexpaq 1:55a6170b404f 709 for test in test_summary:
nexpaq 1:55a6170b404f 710 if test[TARGET_INDEX] == target:
nexpaq 1:55a6170b404f 711 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
nexpaq 1:55a6170b404f 712 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
nexpaq 1:55a6170b404f 713 if test[TEST_INDEX] not in result_dict:
nexpaq 1:55a6170b404f 714 result_dict[test[TEST_INDEX]] = {}
nexpaq 1:55a6170b404f 715 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
nexpaq 1:55a6170b404f 716
nexpaq 1:55a6170b404f 717 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
nexpaq 1:55a6170b404f 718 pt = PrettyTable(pt_cols)
nexpaq 1:55a6170b404f 719 for col in pt_cols:
nexpaq 1:55a6170b404f 720 pt.align[col] = "l"
nexpaq 1:55a6170b404f 721 pt.padding_width = 1 # One space between column edges and contents (default)
nexpaq 1:55a6170b404f 722
nexpaq 1:55a6170b404f 723 for test in unique_tests:
nexpaq 1:55a6170b404f 724 if test in result_dict:
nexpaq 1:55a6170b404f 725 test_results = result_dict[test]
nexpaq 1:55a6170b404f 726 if test in unique_test_desc:
nexpaq 1:55a6170b404f 727 row = [target, test, unique_test_desc[test]]
nexpaq 1:55a6170b404f 728 for toolchain in unique_toolchains:
nexpaq 1:55a6170b404f 729 if toolchain in test_results:
nexpaq 1:55a6170b404f 730 row.append(test_results[toolchain])
nexpaq 1:55a6170b404f 731 pt.add_row(row)
nexpaq 1:55a6170b404f 732 result += pt.get_string()
nexpaq 1:55a6170b404f 733 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
nexpaq 1:55a6170b404f 734 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
nexpaq 1:55a6170b404f 735 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
nexpaq 1:55a6170b404f 736 return result
nexpaq 1:55a6170b404f 737
nexpaq 1:55a6170b404f 738 def generate_test_summary(self, test_summary, shuffle_seed=None):
nexpaq 1:55a6170b404f 739 """ Prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 740 table shows target x test results matrix across
nexpaq 1:55a6170b404f 741 """
nexpaq 1:55a6170b404f 742 success_code = 0 # Success code that can be leter returned to
nexpaq 1:55a6170b404f 743 result = "Test summary:\n"
nexpaq 1:55a6170b404f 744 # Pretty table package is used to print results
nexpaq 1:55a6170b404f 745 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
nexpaq 1:55a6170b404f 746 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
nexpaq 1:55a6170b404f 747 pt.align["Result"] = "l" # Left align
nexpaq 1:55a6170b404f 748 pt.align["Target"] = "l" # Left align
nexpaq 1:55a6170b404f 749 pt.align["Toolchain"] = "l" # Left align
nexpaq 1:55a6170b404f 750 pt.align["Test ID"] = "l" # Left align
nexpaq 1:55a6170b404f 751 pt.align["Test Description"] = "l" # Left align
nexpaq 1:55a6170b404f 752 pt.padding_width = 1 # One space between column edges and contents (default)
nexpaq 1:55a6170b404f 753
nexpaq 1:55a6170b404f 754 result_dict = {self.TEST_RESULT_OK : 0,
nexpaq 1:55a6170b404f 755 self.TEST_RESULT_FAIL : 0,
nexpaq 1:55a6170b404f 756 self.TEST_RESULT_ERROR : 0,
nexpaq 1:55a6170b404f 757 self.TEST_RESULT_UNDEF : 0,
nexpaq 1:55a6170b404f 758 self.TEST_RESULT_IOERR_COPY : 0,
nexpaq 1:55a6170b404f 759 self.TEST_RESULT_IOERR_DISK : 0,
nexpaq 1:55a6170b404f 760 self.TEST_RESULT_IOERR_SERIAL : 0,
nexpaq 1:55a6170b404f 761 self.TEST_RESULT_NO_IMAGE : 0,
nexpaq 1:55a6170b404f 762 self.TEST_RESULT_TIMEOUT : 0,
nexpaq 1:55a6170b404f 763 self.TEST_RESULT_MBED_ASSERT : 0,
nexpaq 1:55a6170b404f 764 self.TEST_RESULT_BUILD_FAILED : 0,
nexpaq 1:55a6170b404f 765 self.TEST_RESULT_NOT_SUPPORTED : 0
nexpaq 1:55a6170b404f 766 }
nexpaq 1:55a6170b404f 767
nexpaq 1:55a6170b404f 768 for test in test_summary:
nexpaq 1:55a6170b404f 769 if test[0] in result_dict:
nexpaq 1:55a6170b404f 770 result_dict[test[0]] += 1
nexpaq 1:55a6170b404f 771 pt.add_row(test)
nexpaq 1:55a6170b404f 772 result += pt.get_string()
nexpaq 1:55a6170b404f 773 result += "\n"
nexpaq 1:55a6170b404f 774
nexpaq 1:55a6170b404f 775 # Print result count
nexpaq 1:55a6170b404f 776 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
nexpaq 1:55a6170b404f 777 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
nexpaq 1:55a6170b404f 778 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
nexpaq 1:55a6170b404f 779 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
nexpaq 1:55a6170b404f 780 return result
nexpaq 1:55a6170b404f 781
nexpaq 1:55a6170b404f 782 def test_loop_list_to_dict(self, test_loops_str):
nexpaq 1:55a6170b404f 783 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
nexpaq 1:55a6170b404f 784 """
nexpaq 1:55a6170b404f 785 result = {}
nexpaq 1:55a6170b404f 786 if test_loops_str:
nexpaq 1:55a6170b404f 787 test_loops = test_loops_str
nexpaq 1:55a6170b404f 788 for test_loop in test_loops:
nexpaq 1:55a6170b404f 789 test_loop_count = test_loop.split('=')
nexpaq 1:55a6170b404f 790 if len(test_loop_count) == 2:
nexpaq 1:55a6170b404f 791 _test_id, _test_loops = test_loop_count
nexpaq 1:55a6170b404f 792 try:
nexpaq 1:55a6170b404f 793 _test_loops = int(_test_loops)
nexpaq 1:55a6170b404f 794 except:
nexpaq 1:55a6170b404f 795 continue
nexpaq 1:55a6170b404f 796 result[_test_id] = _test_loops
nexpaq 1:55a6170b404f 797 return result
nexpaq 1:55a6170b404f 798
nexpaq 1:55a6170b404f 799 def get_test_loop_count(self, test_id):
nexpaq 1:55a6170b404f 800 """ This function returns no. of loops per test (deducted by test_id_.
nexpaq 1:55a6170b404f 801 If test is not in list of redefined loop counts it will use default value.
nexpaq 1:55a6170b404f 802 """
nexpaq 1:55a6170b404f 803 result = self.GLOBAL_LOOPS_COUNT
nexpaq 1:55a6170b404f 804 if test_id in self.TEST_LOOPS_DICT:
nexpaq 1:55a6170b404f 805 result = self.TEST_LOOPS_DICT[test_id]
nexpaq 1:55a6170b404f 806 return result
nexpaq 1:55a6170b404f 807
nexpaq 1:55a6170b404f 808 def delete_file(self, file_path):
nexpaq 1:55a6170b404f 809 """ Remove file from the system
nexpaq 1:55a6170b404f 810 """
nexpaq 1:55a6170b404f 811 result = True
nexpaq 1:55a6170b404f 812 resutl_msg = ""
nexpaq 1:55a6170b404f 813 try:
nexpaq 1:55a6170b404f 814 os.remove(file_path)
nexpaq 1:55a6170b404f 815 except Exception, e:
nexpaq 1:55a6170b404f 816 resutl_msg = e
nexpaq 1:55a6170b404f 817 result = False
nexpaq 1:55a6170b404f 818 return result, resutl_msg
nexpaq 1:55a6170b404f 819
nexpaq 1:55a6170b404f 820 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
nexpaq 1:55a6170b404f 821 """ Test is being invoked for given MUT.
nexpaq 1:55a6170b404f 822 """
nexpaq 1:55a6170b404f 823 # Get test information, image and test timeout
nexpaq 1:55a6170b404f 824 test_id = data['test_id']
nexpaq 1:55a6170b404f 825 test = TEST_MAP[test_id]
nexpaq 1:55a6170b404f 826 test_description = TEST_MAP[test_id].get_description()
nexpaq 1:55a6170b404f 827 image = data["image"]
nexpaq 1:55a6170b404f 828 duration = data.get("duration", 10)
nexpaq 1:55a6170b404f 829
nexpaq 1:55a6170b404f 830 if mut is None:
nexpaq 1:55a6170b404f 831 print "Error: No Mbed available: MUT[%s]" % data['mcu']
nexpaq 1:55a6170b404f 832 return None
nexpaq 1:55a6170b404f 833
nexpaq 1:55a6170b404f 834 mcu = mut['mcu']
nexpaq 1:55a6170b404f 835 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
nexpaq 1:55a6170b404f 836
nexpaq 1:55a6170b404f 837 if self.db_logger:
nexpaq 1:55a6170b404f 838 self.db_logger.reconnect()
nexpaq 1:55a6170b404f 839
nexpaq 1:55a6170b404f 840 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
nexpaq 1:55a6170b404f 841
nexpaq 1:55a6170b404f 842 # Tests can be looped so test results must be stored for the same test
nexpaq 1:55a6170b404f 843 test_all_result = []
nexpaq 1:55a6170b404f 844 # Test results for one test ran few times
nexpaq 1:55a6170b404f 845 detailed_test_results = {} # { Loop_number: { results ... } }
nexpaq 1:55a6170b404f 846
nexpaq 1:55a6170b404f 847 for test_index in range(test_loops):
nexpaq 1:55a6170b404f 848
nexpaq 1:55a6170b404f 849 # If mbedls is available and we are auto detecting MUT info,
nexpaq 1:55a6170b404f 850 # update MUT info (mounting may changed)
nexpaq 1:55a6170b404f 851 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
nexpaq 1:55a6170b404f 852 platform_name_filter = [mcu]
nexpaq 1:55a6170b404f 853 muts_list = {}
nexpaq 1:55a6170b404f 854 found = False
nexpaq 1:55a6170b404f 855
nexpaq 1:55a6170b404f 856 for i in range(0, 60):
nexpaq 1:55a6170b404f 857 print('Looking for %s with MBEDLS' % mcu)
nexpaq 1:55a6170b404f 858 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
nexpaq 1:55a6170b404f 859
nexpaq 1:55a6170b404f 860 if 1 not in muts_list:
nexpaq 1:55a6170b404f 861 sleep(3)
nexpaq 1:55a6170b404f 862 else:
nexpaq 1:55a6170b404f 863 found = True
nexpaq 1:55a6170b404f 864 break
nexpaq 1:55a6170b404f 865
nexpaq 1:55a6170b404f 866 if not found:
nexpaq 1:55a6170b404f 867 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
nexpaq 1:55a6170b404f 868 return None
nexpaq 1:55a6170b404f 869 else:
nexpaq 1:55a6170b404f 870 mut = muts_list[1]
nexpaq 1:55a6170b404f 871
nexpaq 1:55a6170b404f 872 disk = mut.get('disk')
nexpaq 1:55a6170b404f 873 port = mut.get('port')
nexpaq 1:55a6170b404f 874
nexpaq 1:55a6170b404f 875 if disk is None or port is None:
nexpaq 1:55a6170b404f 876 return None
nexpaq 1:55a6170b404f 877
nexpaq 1:55a6170b404f 878 target_by_mcu = TARGET_MAP[mut['mcu']]
nexpaq 1:55a6170b404f 879 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
nexpaq 1:55a6170b404f 880 # Some extra stuff can be declared in MUTs structure
nexpaq 1:55a6170b404f 881 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
nexpaq 1:55a6170b404f 882 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
nexpaq 1:55a6170b404f 883
nexpaq 1:55a6170b404f 884 # When the build and test system were separate, this was relative to a
nexpaq 1:55a6170b404f 885 # base network folder base path: join(NETWORK_BASE_PATH, )
nexpaq 1:55a6170b404f 886 image_path = image
nexpaq 1:55a6170b404f 887
nexpaq 1:55a6170b404f 888 # Host test execution
nexpaq 1:55a6170b404f 889 start_host_exec_time = time()
nexpaq 1:55a6170b404f 890
nexpaq 1:55a6170b404f 891 single_test_result = self.TEST_RESULT_UNDEF # single test run result
nexpaq 1:55a6170b404f 892 _copy_method = selected_copy_method
nexpaq 1:55a6170b404f 893
nexpaq 1:55a6170b404f 894 if not exists(image_path):
nexpaq 1:55a6170b404f 895 single_test_result = self.TEST_RESULT_NO_IMAGE
nexpaq 1:55a6170b404f 896 elapsed_time = 0
nexpaq 1:55a6170b404f 897 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
nexpaq 1:55a6170b404f 898 print single_test_output
nexpaq 1:55a6170b404f 899 else:
nexpaq 1:55a6170b404f 900 # Host test execution
nexpaq 1:55a6170b404f 901 start_host_exec_time = time()
nexpaq 1:55a6170b404f 902
nexpaq 1:55a6170b404f 903 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
nexpaq 1:55a6170b404f 904 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
nexpaq 1:55a6170b404f 905 host_test_result = self.run_host_test(test.host_test,
nexpaq 1:55a6170b404f 906 image_path, disk, port, duration,
nexpaq 1:55a6170b404f 907 micro=target_name,
nexpaq 1:55a6170b404f 908 verbose=host_test_verbose,
nexpaq 1:55a6170b404f 909 reset=host_test_reset,
nexpaq 1:55a6170b404f 910 reset_tout=reset_tout,
nexpaq 1:55a6170b404f 911 copy_method=selected_copy_method,
nexpaq 1:55a6170b404f 912 program_cycle_s=target_by_mcu.program_cycle_s)
nexpaq 1:55a6170b404f 913 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
nexpaq 1:55a6170b404f 914
nexpaq 1:55a6170b404f 915 # Store test result
nexpaq 1:55a6170b404f 916 test_all_result.append(single_test_result)
nexpaq 1:55a6170b404f 917 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
nexpaq 1:55a6170b404f 918 elapsed_time = single_testduration # TIme of single test case execution after reset
nexpaq 1:55a6170b404f 919
nexpaq 1:55a6170b404f 920 detailed_test_results[test_index] = {
nexpaq 1:55a6170b404f 921 'result' : single_test_result,
nexpaq 1:55a6170b404f 922 'output' : single_test_output,
nexpaq 1:55a6170b404f 923 'target_name' : target_name,
nexpaq 1:55a6170b404f 924 'target_name_unique' : target_name_unique,
nexpaq 1:55a6170b404f 925 'toolchain_name' : toolchain_name,
nexpaq 1:55a6170b404f 926 'id' : test_id,
nexpaq 1:55a6170b404f 927 'description' : test_description,
nexpaq 1:55a6170b404f 928 'elapsed_time' : round(elapsed_time, 2),
nexpaq 1:55a6170b404f 929 'duration' : single_timeout,
nexpaq 1:55a6170b404f 930 'copy_method' : _copy_method,
nexpaq 1:55a6170b404f 931 }
nexpaq 1:55a6170b404f 932
nexpaq 1:55a6170b404f 933 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
nexpaq 1:55a6170b404f 934 test_id, test_description, elapsed_time, single_timeout)
nexpaq 1:55a6170b404f 935
nexpaq 1:55a6170b404f 936 # Update database entries for ongoing test
nexpaq 1:55a6170b404f 937 if self.db_logger and self.db_logger.is_connected():
nexpaq 1:55a6170b404f 938 test_type = 'SingleTest'
nexpaq 1:55a6170b404f 939 self.db_logger.insert_test_entry(self.db_logger_build_id,
nexpaq 1:55a6170b404f 940 target_name,
nexpaq 1:55a6170b404f 941 toolchain_name,
nexpaq 1:55a6170b404f 942 test_type,
nexpaq 1:55a6170b404f 943 test_id,
nexpaq 1:55a6170b404f 944 single_test_result,
nexpaq 1:55a6170b404f 945 single_test_output,
nexpaq 1:55a6170b404f 946 elapsed_time,
nexpaq 1:55a6170b404f 947 single_timeout,
nexpaq 1:55a6170b404f 948 test_index)
nexpaq 1:55a6170b404f 949
nexpaq 1:55a6170b404f 950 # If we perform waterfall test we test until we get OK and we stop testing
nexpaq 1:55a6170b404f 951 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
nexpaq 1:55a6170b404f 952 break
nexpaq 1:55a6170b404f 953
nexpaq 1:55a6170b404f 954 if self.db_logger:
nexpaq 1:55a6170b404f 955 self.db_logger.disconnect()
nexpaq 1:55a6170b404f 956
nexpaq 1:55a6170b404f 957 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
nexpaq 1:55a6170b404f 958 target_name_unique,
nexpaq 1:55a6170b404f 959 toolchain_name,
nexpaq 1:55a6170b404f 960 test_id,
nexpaq 1:55a6170b404f 961 test_description,
nexpaq 1:55a6170b404f 962 round(elapsed_time, 2),
nexpaq 1:55a6170b404f 963 single_timeout,
nexpaq 1:55a6170b404f 964 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
nexpaq 1:55a6170b404f 965
nexpaq 1:55a6170b404f 966 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
nexpaq 1:55a6170b404f 967 """ Function determines MUT's mbed disk/port and copies binary to
nexpaq 1:55a6170b404f 968 target.
nexpaq 1:55a6170b404f 969 """
nexpaq 1:55a6170b404f 970 handle_results = []
nexpaq 1:55a6170b404f 971 data = json.loads(test_spec)
nexpaq 1:55a6170b404f 972
nexpaq 1:55a6170b404f 973 # Find a suitable MUT:
nexpaq 1:55a6170b404f 974 mut = None
nexpaq 1:55a6170b404f 975 for id, m in self.muts.iteritems():
nexpaq 1:55a6170b404f 976 if m['mcu'] == data['mcu']:
nexpaq 1:55a6170b404f 977 mut = m
nexpaq 1:55a6170b404f 978 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
nexpaq 1:55a6170b404f 979 handle_results.append(handle_result)
nexpaq 1:55a6170b404f 980
nexpaq 1:55a6170b404f 981 return handle_results
nexpaq 1:55a6170b404f 982
nexpaq 1:55a6170b404f 983 def print_test_result(self, test_result, target_name, toolchain_name,
nexpaq 1:55a6170b404f 984 test_id, test_description, elapsed_time, duration):
nexpaq 1:55a6170b404f 985 """ Use specific convention to print test result and related data
nexpaq 1:55a6170b404f 986 """
nexpaq 1:55a6170b404f 987 tokens = []
nexpaq 1:55a6170b404f 988 tokens.append("TargetTest")
nexpaq 1:55a6170b404f 989 tokens.append(target_name)
nexpaq 1:55a6170b404f 990 tokens.append(toolchain_name)
nexpaq 1:55a6170b404f 991 tokens.append(test_id)
nexpaq 1:55a6170b404f 992 tokens.append(test_description)
nexpaq 1:55a6170b404f 993 separator = "::"
nexpaq 1:55a6170b404f 994 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
nexpaq 1:55a6170b404f 995 result = separator.join(tokens) + " [" + test_result +"]" + time_info
nexpaq 1:55a6170b404f 996 return Fore.MAGENTA + result + Fore.RESET
nexpaq 1:55a6170b404f 997
nexpaq 1:55a6170b404f 998 def shape_test_loop_ok_result_count(self, test_all_result):
nexpaq 1:55a6170b404f 999 """ Reformats list of results to simple string
nexpaq 1:55a6170b404f 1000 """
nexpaq 1:55a6170b404f 1001 test_loop_count = len(test_all_result)
nexpaq 1:55a6170b404f 1002 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
nexpaq 1:55a6170b404f 1003 return "%d/%d"% (test_loop_ok_result, test_loop_count)
nexpaq 1:55a6170b404f 1004
nexpaq 1:55a6170b404f 1005 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
nexpaq 1:55a6170b404f 1006 """ Reformats list of results to simple string
nexpaq 1:55a6170b404f 1007 """
nexpaq 1:55a6170b404f 1008 result = self.TEST_RESULT_FAIL
nexpaq 1:55a6170b404f 1009
nexpaq 1:55a6170b404f 1010 if all(test_all_result[0] == res for res in test_all_result):
nexpaq 1:55a6170b404f 1011 result = test_all_result[0]
nexpaq 1:55a6170b404f 1012 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
nexpaq 1:55a6170b404f 1013 result = self.TEST_RESULT_OK
nexpaq 1:55a6170b404f 1014
nexpaq 1:55a6170b404f 1015 return result
nexpaq 1:55a6170b404f 1016
nexpaq 1:55a6170b404f 1017 def run_host_test(self, name, image_path, disk, port, duration,
nexpaq 1:55a6170b404f 1018 micro=None, reset=None, reset_tout=None,
nexpaq 1:55a6170b404f 1019 verbose=False, copy_method=None, program_cycle_s=None):
nexpaq 1:55a6170b404f 1020 """ Function creates new process with host test configured with particular test case.
nexpaq 1:55a6170b404f 1021 Function also is pooling for serial port activity from process to catch all data
nexpaq 1:55a6170b404f 1022 printed by test runner and host test during test execution
nexpaq 1:55a6170b404f 1023 """
nexpaq 1:55a6170b404f 1024
nexpaq 1:55a6170b404f 1025 def get_char_from_queue(obs):
nexpaq 1:55a6170b404f 1026 """ Get character from queue safe way
nexpaq 1:55a6170b404f 1027 """
nexpaq 1:55a6170b404f 1028 try:
nexpaq 1:55a6170b404f 1029 c = obs.queue.get(block=True, timeout=0.5)
nexpaq 1:55a6170b404f 1030 except Empty, _:
nexpaq 1:55a6170b404f 1031 c = None
nexpaq 1:55a6170b404f 1032 return c
nexpaq 1:55a6170b404f 1033
nexpaq 1:55a6170b404f 1034 def filter_queue_char(c):
nexpaq 1:55a6170b404f 1035 """ Filters out non ASCII characters from serial port
nexpaq 1:55a6170b404f 1036 """
nexpaq 1:55a6170b404f 1037 if ord(c) not in range(128):
nexpaq 1:55a6170b404f 1038 c = ' '
nexpaq 1:55a6170b404f 1039 return c
nexpaq 1:55a6170b404f 1040
nexpaq 1:55a6170b404f 1041 def get_test_result(output):
nexpaq 1:55a6170b404f 1042 """ Parse test 'output' data
nexpaq 1:55a6170b404f 1043 """
nexpaq 1:55a6170b404f 1044 result = self.TEST_RESULT_TIMEOUT
nexpaq 1:55a6170b404f 1045 for line in "".join(output).splitlines():
nexpaq 1:55a6170b404f 1046 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
nexpaq 1:55a6170b404f 1047 if search_result and len(search_result.groups()):
nexpaq 1:55a6170b404f 1048 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
nexpaq 1:55a6170b404f 1049 break
nexpaq 1:55a6170b404f 1050 return result
nexpaq 1:55a6170b404f 1051
nexpaq 1:55a6170b404f 1052 def get_auto_property_value(property_name, line):
nexpaq 1:55a6170b404f 1053 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
nexpaq 1:55a6170b404f 1054 Returns string
nexpaq 1:55a6170b404f 1055 """
nexpaq 1:55a6170b404f 1056 result = None
nexpaq 1:55a6170b404f 1057 if re.search("HOST: Property '%s'"% property_name, line) is not None:
nexpaq 1:55a6170b404f 1058 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
nexpaq 1:55a6170b404f 1059 if property is not None and len(property.groups()) == 1:
nexpaq 1:55a6170b404f 1060 result = property.groups()[0]
nexpaq 1:55a6170b404f 1061 return result
nexpaq 1:55a6170b404f 1062
nexpaq 1:55a6170b404f 1063 # print "{%s} port:%s disk:%s" % (name, port, disk),
nexpaq 1:55a6170b404f 1064 cmd = ["python",
nexpaq 1:55a6170b404f 1065 '%s.py'% name,
nexpaq 1:55a6170b404f 1066 '-d', disk,
nexpaq 1:55a6170b404f 1067 '-f', '"%s"'% image_path,
nexpaq 1:55a6170b404f 1068 '-p', port,
nexpaq 1:55a6170b404f 1069 '-t', str(duration),
nexpaq 1:55a6170b404f 1070 '-C', str(program_cycle_s)]
nexpaq 1:55a6170b404f 1071
nexpaq 1:55a6170b404f 1072 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
nexpaq 1:55a6170b404f 1073 cmd += ['--auto']
nexpaq 1:55a6170b404f 1074
nexpaq 1:55a6170b404f 1075 # Add extra parameters to host_test
nexpaq 1:55a6170b404f 1076 if copy_method is not None:
nexpaq 1:55a6170b404f 1077 cmd += ["-c", copy_method]
nexpaq 1:55a6170b404f 1078 if micro is not None:
nexpaq 1:55a6170b404f 1079 cmd += ["-m", micro]
nexpaq 1:55a6170b404f 1080 if reset is not None:
nexpaq 1:55a6170b404f 1081 cmd += ["-r", reset]
nexpaq 1:55a6170b404f 1082 if reset_tout is not None:
nexpaq 1:55a6170b404f 1083 cmd += ["-R", str(reset_tout)]
nexpaq 1:55a6170b404f 1084
nexpaq 1:55a6170b404f 1085 if verbose:
nexpaq 1:55a6170b404f 1086 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
nexpaq 1:55a6170b404f 1087 print "Test::Output::Start"
nexpaq 1:55a6170b404f 1088
nexpaq 1:55a6170b404f 1089 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
nexpaq 1:55a6170b404f 1090 obs = ProcessObserver(proc)
nexpaq 1:55a6170b404f 1091 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
nexpaq 1:55a6170b404f 1092 line = ''
nexpaq 1:55a6170b404f 1093 output = []
nexpaq 1:55a6170b404f 1094 start_time = time()
nexpaq 1:55a6170b404f 1095 while (time() - start_time) < (2 * duration):
nexpaq 1:55a6170b404f 1096 c = get_char_from_queue(obs)
nexpaq 1:55a6170b404f 1097 if c:
nexpaq 1:55a6170b404f 1098 if verbose:
nexpaq 1:55a6170b404f 1099 sys.stdout.write(c)
nexpaq 1:55a6170b404f 1100 c = filter_queue_char(c)
nexpaq 1:55a6170b404f 1101 output.append(c)
nexpaq 1:55a6170b404f 1102 # Give the mbed under test a way to communicate the end of the test
nexpaq 1:55a6170b404f 1103 if c in ['\n', '\r']:
nexpaq 1:55a6170b404f 1104
nexpaq 1:55a6170b404f 1105 # Checking for auto-detection information from the test about MUT reset moment
nexpaq 1:55a6170b404f 1106 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
nexpaq 1:55a6170b404f 1107 # We will update this marker only once to prevent multiple time resets
nexpaq 1:55a6170b404f 1108 update_once_flag['reset_target'] = True
nexpaq 1:55a6170b404f 1109 start_time = time()
nexpaq 1:55a6170b404f 1110
nexpaq 1:55a6170b404f 1111 # Checking for auto-detection information from the test about timeout
nexpaq 1:55a6170b404f 1112 auto_timeout_val = get_auto_property_value('timeout', line)
nexpaq 1:55a6170b404f 1113 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
nexpaq 1:55a6170b404f 1114 # We will update this marker only once to prevent multiple time resets
nexpaq 1:55a6170b404f 1115 update_once_flag['timeout'] = True
nexpaq 1:55a6170b404f 1116 duration = int(auto_timeout_val)
nexpaq 1:55a6170b404f 1117
nexpaq 1:55a6170b404f 1118 # Detect mbed assert:
nexpaq 1:55a6170b404f 1119 if 'mbed assertation failed: ' in line:
nexpaq 1:55a6170b404f 1120 output.append('{{mbed_assert}}')
nexpaq 1:55a6170b404f 1121 break
nexpaq 1:55a6170b404f 1122
nexpaq 1:55a6170b404f 1123 # Check for test end
nexpaq 1:55a6170b404f 1124 if '{end}' in line:
nexpaq 1:55a6170b404f 1125 break
nexpaq 1:55a6170b404f 1126 line = ''
nexpaq 1:55a6170b404f 1127 else:
nexpaq 1:55a6170b404f 1128 line += c
nexpaq 1:55a6170b404f 1129 end_time = time()
nexpaq 1:55a6170b404f 1130 testcase_duration = end_time - start_time # Test case duration from reset to {end}
nexpaq 1:55a6170b404f 1131
nexpaq 1:55a6170b404f 1132 c = get_char_from_queue(obs)
nexpaq 1:55a6170b404f 1133
nexpaq 1:55a6170b404f 1134 if c:
nexpaq 1:55a6170b404f 1135 if verbose:
nexpaq 1:55a6170b404f 1136 sys.stdout.write(c)
nexpaq 1:55a6170b404f 1137 c = filter_queue_char(c)
nexpaq 1:55a6170b404f 1138 output.append(c)
nexpaq 1:55a6170b404f 1139
nexpaq 1:55a6170b404f 1140 if verbose:
nexpaq 1:55a6170b404f 1141 print "Test::Output::Finish"
nexpaq 1:55a6170b404f 1142 # Stop test process
nexpaq 1:55a6170b404f 1143 obs.stop()
nexpaq 1:55a6170b404f 1144
nexpaq 1:55a6170b404f 1145 result = get_test_result(output)
nexpaq 1:55a6170b404f 1146 return (result, "".join(output), testcase_duration, duration)
nexpaq 1:55a6170b404f 1147
nexpaq 1:55a6170b404f 1148 def is_peripherals_available(self, target_mcu_name, peripherals=None):
nexpaq 1:55a6170b404f 1149 """ Checks if specified target should run specific peripheral test case defined in MUTs file
nexpaq 1:55a6170b404f 1150 """
nexpaq 1:55a6170b404f 1151 if peripherals is not None:
nexpaq 1:55a6170b404f 1152 peripherals = set(peripherals)
nexpaq 1:55a6170b404f 1153 for id, mut in self.muts.iteritems():
nexpaq 1:55a6170b404f 1154 # Target MCU name check
nexpaq 1:55a6170b404f 1155 if mut["mcu"] != target_mcu_name:
nexpaq 1:55a6170b404f 1156 continue
nexpaq 1:55a6170b404f 1157 # Peripherals check
nexpaq 1:55a6170b404f 1158 if peripherals is not None:
nexpaq 1:55a6170b404f 1159 if 'peripherals' not in mut:
nexpaq 1:55a6170b404f 1160 continue
nexpaq 1:55a6170b404f 1161 if not peripherals.issubset(set(mut['peripherals'])):
nexpaq 1:55a6170b404f 1162 continue
nexpaq 1:55a6170b404f 1163 return True
nexpaq 1:55a6170b404f 1164 return False
nexpaq 1:55a6170b404f 1165
nexpaq 1:55a6170b404f 1166 def shape_test_request(self, mcu, image_path, test_id, duration=10):
nexpaq 1:55a6170b404f 1167 """ Function prepares JSON structure describing test specification
nexpaq 1:55a6170b404f 1168 """
nexpaq 1:55a6170b404f 1169 test_spec = {
nexpaq 1:55a6170b404f 1170 "mcu": mcu,
nexpaq 1:55a6170b404f 1171 "image": image_path,
nexpaq 1:55a6170b404f 1172 "duration": duration,
nexpaq 1:55a6170b404f 1173 "test_id": test_id,
nexpaq 1:55a6170b404f 1174 }
nexpaq 1:55a6170b404f 1175 return json.dumps(test_spec)
nexpaq 1:55a6170b404f 1176
nexpaq 1:55a6170b404f 1177
nexpaq 1:55a6170b404f 1178 def get_unique_value_from_summary(test_summary, index):
nexpaq 1:55a6170b404f 1179 """ Gets list of unique target names
nexpaq 1:55a6170b404f 1180 """
nexpaq 1:55a6170b404f 1181 result = []
nexpaq 1:55a6170b404f 1182 for test in test_summary:
nexpaq 1:55a6170b404f 1183 target_name = test[index]
nexpaq 1:55a6170b404f 1184 if target_name not in result:
nexpaq 1:55a6170b404f 1185 result.append(target_name)
nexpaq 1:55a6170b404f 1186 return sorted(result)
nexpaq 1:55a6170b404f 1187
nexpaq 1:55a6170b404f 1188
nexpaq 1:55a6170b404f 1189 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
nexpaq 1:55a6170b404f 1190 """ Gets list of unique target names and return dictionary
nexpaq 1:55a6170b404f 1191 """
nexpaq 1:55a6170b404f 1192 result = {}
nexpaq 1:55a6170b404f 1193 for test in test_summary:
nexpaq 1:55a6170b404f 1194 key = test[index_key]
nexpaq 1:55a6170b404f 1195 val = test[index_val]
nexpaq 1:55a6170b404f 1196 if key not in result:
nexpaq 1:55a6170b404f 1197 result[key] = val
nexpaq 1:55a6170b404f 1198 return result
nexpaq 1:55a6170b404f 1199
nexpaq 1:55a6170b404f 1200
nexpaq 1:55a6170b404f 1201 def show_json_file_format_error(json_spec_filename, line, column):
nexpaq 1:55a6170b404f 1202 """ Prints JSON broken content
nexpaq 1:55a6170b404f 1203 """
nexpaq 1:55a6170b404f 1204 with open(json_spec_filename) as data_file:
nexpaq 1:55a6170b404f 1205 line_no = 1
nexpaq 1:55a6170b404f 1206 for json_line in data_file:
nexpaq 1:55a6170b404f 1207 if line_no + 5 >= line: # Print last few lines before error
nexpaq 1:55a6170b404f 1208 print 'Line %d:\t'%line_no + json_line, # Prints line
nexpaq 1:55a6170b404f 1209 if line_no == line:
nexpaq 1:55a6170b404f 1210 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
nexpaq 1:55a6170b404f 1211 break
nexpaq 1:55a6170b404f 1212 line_no += 1
nexpaq 1:55a6170b404f 1213
nexpaq 1:55a6170b404f 1214
nexpaq 1:55a6170b404f 1215 def json_format_error_defect_pos(json_error_msg):
nexpaq 1:55a6170b404f 1216 """ Gets first error line and column in JSON file format.
nexpaq 1:55a6170b404f 1217 Parsed from exception thrown by json.loads() string
nexpaq 1:55a6170b404f 1218 """
nexpaq 1:55a6170b404f 1219 result = None
nexpaq 1:55a6170b404f 1220 line, column = 0, 0
nexpaq 1:55a6170b404f 1221 # Line value search
nexpaq 1:55a6170b404f 1222 line_search = re.search('line [0-9]+', json_error_msg)
nexpaq 1:55a6170b404f 1223 if line_search is not None:
nexpaq 1:55a6170b404f 1224 ls = line_search.group().split(' ')
nexpaq 1:55a6170b404f 1225 if len(ls) == 2:
nexpaq 1:55a6170b404f 1226 line = int(ls[1])
nexpaq 1:55a6170b404f 1227 # Column position search
nexpaq 1:55a6170b404f 1228 column_search = re.search('column [0-9]+', json_error_msg)
nexpaq 1:55a6170b404f 1229 if column_search is not None:
nexpaq 1:55a6170b404f 1230 cs = column_search.group().split(' ')
nexpaq 1:55a6170b404f 1231 if len(cs) == 2:
nexpaq 1:55a6170b404f 1232 column = int(cs[1])
nexpaq 1:55a6170b404f 1233 result = [line, column]
nexpaq 1:55a6170b404f 1234 return result
nexpaq 1:55a6170b404f 1235
nexpaq 1:55a6170b404f 1236
nexpaq 1:55a6170b404f 1237 def get_json_data_from_file(json_spec_filename, verbose=False):
nexpaq 1:55a6170b404f 1238 """ Loads from file JSON formatted string to data structure
nexpaq 1:55a6170b404f 1239 """
nexpaq 1:55a6170b404f 1240 result = None
nexpaq 1:55a6170b404f 1241 try:
nexpaq 1:55a6170b404f 1242 with open(json_spec_filename) as data_file:
nexpaq 1:55a6170b404f 1243 try:
nexpaq 1:55a6170b404f 1244 result = json.load(data_file)
nexpaq 1:55a6170b404f 1245 except ValueError as json_error_msg:
nexpaq 1:55a6170b404f 1246 result = None
nexpaq 1:55a6170b404f 1247 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
nexpaq 1:55a6170b404f 1248 # We can print where error occurred inside JSON file if we can parse exception msg
nexpaq 1:55a6170b404f 1249 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
nexpaq 1:55a6170b404f 1250 if json_format_defect_pos is not None:
nexpaq 1:55a6170b404f 1251 line = json_format_defect_pos[0]
nexpaq 1:55a6170b404f 1252 column = json_format_defect_pos[1]
nexpaq 1:55a6170b404f 1253 print
nexpaq 1:55a6170b404f 1254 show_json_file_format_error(json_spec_filename, line, column)
nexpaq 1:55a6170b404f 1255
nexpaq 1:55a6170b404f 1256 except IOError as fileopen_error_msg:
nexpaq 1:55a6170b404f 1257 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
nexpaq 1:55a6170b404f 1258 print
nexpaq 1:55a6170b404f 1259 if verbose and result:
nexpaq 1:55a6170b404f 1260 pp = pprint.PrettyPrinter(indent=4)
nexpaq 1:55a6170b404f 1261 pp.pprint(result)
nexpaq 1:55a6170b404f 1262 return result
nexpaq 1:55a6170b404f 1263
nexpaq 1:55a6170b404f 1264
nexpaq 1:55a6170b404f 1265 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
nexpaq 1:55a6170b404f 1266 """ Prints MUTs configuration passed to test script for verboseness
nexpaq 1:55a6170b404f 1267 """
nexpaq 1:55a6170b404f 1268 muts_info_cols = []
nexpaq 1:55a6170b404f 1269 # We need to check all unique properties for each defined MUT
nexpaq 1:55a6170b404f 1270 for k in json_data:
nexpaq 1:55a6170b404f 1271 mut_info = json_data[k]
nexpaq 1:55a6170b404f 1272 for mut_property in mut_info:
nexpaq 1:55a6170b404f 1273 if mut_property not in muts_info_cols:
nexpaq 1:55a6170b404f 1274 muts_info_cols.append(mut_property)
nexpaq 1:55a6170b404f 1275
nexpaq 1:55a6170b404f 1276 # Prepare pretty table object to display all MUTs
nexpaq 1:55a6170b404f 1277 pt_cols = ["index"] + muts_info_cols
nexpaq 1:55a6170b404f 1278 pt = PrettyTable(pt_cols)
nexpaq 1:55a6170b404f 1279 for col in pt_cols:
nexpaq 1:55a6170b404f 1280 pt.align[col] = "l"
nexpaq 1:55a6170b404f 1281
nexpaq 1:55a6170b404f 1282 # Add rows to pretty print object
nexpaq 1:55a6170b404f 1283 for k in json_data:
nexpaq 1:55a6170b404f 1284 row = [k]
nexpaq 1:55a6170b404f 1285 mut_info = json_data[k]
nexpaq 1:55a6170b404f 1286
nexpaq 1:55a6170b404f 1287 add_row = True
nexpaq 1:55a6170b404f 1288 if platform_filter and 'mcu' in mut_info:
nexpaq 1:55a6170b404f 1289 add_row = re.search(platform_filter, mut_info['mcu']) is not None
nexpaq 1:55a6170b404f 1290 if add_row:
nexpaq 1:55a6170b404f 1291 for col in muts_info_cols:
nexpaq 1:55a6170b404f 1292 cell_val = mut_info[col] if col in mut_info else None
nexpaq 1:55a6170b404f 1293 if type(cell_val) == ListType:
nexpaq 1:55a6170b404f 1294 cell_val = join_delim.join(cell_val)
nexpaq 1:55a6170b404f 1295 row.append(cell_val)
nexpaq 1:55a6170b404f 1296 pt.add_row(row)
nexpaq 1:55a6170b404f 1297 return pt.get_string()
nexpaq 1:55a6170b404f 1298
nexpaq 1:55a6170b404f 1299
nexpaq 1:55a6170b404f 1300 def print_test_configuration_from_json(json_data, join_delim=", "):
nexpaq 1:55a6170b404f 1301 """ Prints test specification configuration passed to test script for verboseness
nexpaq 1:55a6170b404f 1302 """
nexpaq 1:55a6170b404f 1303 toolchains_info_cols = []
nexpaq 1:55a6170b404f 1304 # We need to check all toolchains for each device
nexpaq 1:55a6170b404f 1305 for k in json_data:
nexpaq 1:55a6170b404f 1306 # k should be 'targets'
nexpaq 1:55a6170b404f 1307 targets = json_data[k]
nexpaq 1:55a6170b404f 1308 for target in targets:
nexpaq 1:55a6170b404f 1309 toolchains = targets[target]
nexpaq 1:55a6170b404f 1310 for toolchain in toolchains:
nexpaq 1:55a6170b404f 1311 if toolchain not in toolchains_info_cols:
nexpaq 1:55a6170b404f 1312 toolchains_info_cols.append(toolchain)
nexpaq 1:55a6170b404f 1313
nexpaq 1:55a6170b404f 1314 # Prepare pretty table object to display test specification
nexpaq 1:55a6170b404f 1315 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
nexpaq 1:55a6170b404f 1316 pt = PrettyTable(pt_cols)
nexpaq 1:55a6170b404f 1317 for col in pt_cols:
nexpaq 1:55a6170b404f 1318 pt.align[col] = "l"
nexpaq 1:55a6170b404f 1319
nexpaq 1:55a6170b404f 1320 # { target : [conflicted toolchains] }
nexpaq 1:55a6170b404f 1321 toolchain_conflicts = {}
nexpaq 1:55a6170b404f 1322 toolchain_path_conflicts = []
nexpaq 1:55a6170b404f 1323 for k in json_data:
nexpaq 1:55a6170b404f 1324 # k should be 'targets'
nexpaq 1:55a6170b404f 1325 targets = json_data[k]
nexpaq 1:55a6170b404f 1326 for target in targets:
nexpaq 1:55a6170b404f 1327 target_supported_toolchains = get_target_supported_toolchains(target)
nexpaq 1:55a6170b404f 1328 if not target_supported_toolchains:
nexpaq 1:55a6170b404f 1329 target_supported_toolchains = []
nexpaq 1:55a6170b404f 1330 target_name = target if target in TARGET_MAP else "%s*"% target
nexpaq 1:55a6170b404f 1331 row = [target_name]
nexpaq 1:55a6170b404f 1332 toolchains = targets[target]
nexpaq 1:55a6170b404f 1333
nexpaq 1:55a6170b404f 1334 for toolchain in sorted(toolchains_info_cols):
nexpaq 1:55a6170b404f 1335 # Check for conflicts: target vs toolchain
nexpaq 1:55a6170b404f 1336 conflict = False
nexpaq 1:55a6170b404f 1337 conflict_path = False
nexpaq 1:55a6170b404f 1338 if toolchain in toolchains:
nexpaq 1:55a6170b404f 1339 if toolchain not in target_supported_toolchains:
nexpaq 1:55a6170b404f 1340 conflict = True
nexpaq 1:55a6170b404f 1341 if target not in toolchain_conflicts:
nexpaq 1:55a6170b404f 1342 toolchain_conflicts[target] = []
nexpaq 1:55a6170b404f 1343 toolchain_conflicts[target].append(toolchain)
nexpaq 1:55a6170b404f 1344 # Add marker inside table about target usage / conflict
nexpaq 1:55a6170b404f 1345 cell_val = 'Yes' if toolchain in toolchains else '-'
nexpaq 1:55a6170b404f 1346 if conflict:
nexpaq 1:55a6170b404f 1347 cell_val += '*'
nexpaq 1:55a6170b404f 1348 # Check for conflicts: toolchain vs toolchain path
nexpaq 1:55a6170b404f 1349 if toolchain in TOOLCHAIN_PATHS:
nexpaq 1:55a6170b404f 1350 toolchain_path = TOOLCHAIN_PATHS[toolchain]
nexpaq 1:55a6170b404f 1351 if not os.path.isdir(toolchain_path):
nexpaq 1:55a6170b404f 1352 conflict_path = True
nexpaq 1:55a6170b404f 1353 if toolchain not in toolchain_path_conflicts:
nexpaq 1:55a6170b404f 1354 toolchain_path_conflicts.append(toolchain)
nexpaq 1:55a6170b404f 1355 if conflict_path:
nexpaq 1:55a6170b404f 1356 cell_val += '#'
nexpaq 1:55a6170b404f 1357 row.append(cell_val)
nexpaq 1:55a6170b404f 1358 pt.add_row(row)
nexpaq 1:55a6170b404f 1359
nexpaq 1:55a6170b404f 1360 # generate result string
nexpaq 1:55a6170b404f 1361 result = pt.get_string() # Test specification table
nexpaq 1:55a6170b404f 1362 if toolchain_conflicts or toolchain_path_conflicts:
nexpaq 1:55a6170b404f 1363 result += "\n"
nexpaq 1:55a6170b404f 1364 result += "Toolchain conflicts:\n"
nexpaq 1:55a6170b404f 1365 for target in toolchain_conflicts:
nexpaq 1:55a6170b404f 1366 if target not in TARGET_MAP:
nexpaq 1:55a6170b404f 1367 result += "\t* Target %s unknown\n"% (target)
nexpaq 1:55a6170b404f 1368 conflict_target_list = join_delim.join(toolchain_conflicts[target])
nexpaq 1:55a6170b404f 1369 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
nexpaq 1:55a6170b404f 1370 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
nexpaq 1:55a6170b404f 1371
nexpaq 1:55a6170b404f 1372 for toolchain in toolchain_path_conflicts:
nexpaq 1:55a6170b404f 1373 # Let's check toolchain configuration
nexpaq 1:55a6170b404f 1374 if toolchain in TOOLCHAIN_PATHS:
nexpaq 1:55a6170b404f 1375 toolchain_path = TOOLCHAIN_PATHS[toolchain]
nexpaq 1:55a6170b404f 1376 if not os.path.isdir(toolchain_path):
nexpaq 1:55a6170b404f 1377 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
nexpaq 1:55a6170b404f 1378 return result
nexpaq 1:55a6170b404f 1379
nexpaq 1:55a6170b404f 1380
nexpaq 1:55a6170b404f 1381 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
nexpaq 1:55a6170b404f 1382 """ Generates table summary with all test cases and additional test cases
nexpaq 1:55a6170b404f 1383 information using pretty print functionality. Allows test suite user to
nexpaq 1:55a6170b404f 1384 see test cases
nexpaq 1:55a6170b404f 1385 """
nexpaq 1:55a6170b404f 1386 # get all unique test ID prefixes
nexpaq 1:55a6170b404f 1387 unique_test_id = []
nexpaq 1:55a6170b404f 1388 for test in TESTS:
nexpaq 1:55a6170b404f 1389 split = test['id'].split('_')[:-1]
nexpaq 1:55a6170b404f 1390 test_id_prefix = '_'.join(split)
nexpaq 1:55a6170b404f 1391 if test_id_prefix not in unique_test_id:
nexpaq 1:55a6170b404f 1392 unique_test_id.append(test_id_prefix)
nexpaq 1:55a6170b404f 1393 unique_test_id.sort()
nexpaq 1:55a6170b404f 1394 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
nexpaq 1:55a6170b404f 1395 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
nexpaq 1:55a6170b404f 1396
nexpaq 1:55a6170b404f 1397 test_properties = ['id',
nexpaq 1:55a6170b404f 1398 'automated',
nexpaq 1:55a6170b404f 1399 'description',
nexpaq 1:55a6170b404f 1400 'peripherals',
nexpaq 1:55a6170b404f 1401 'host_test',
nexpaq 1:55a6170b404f 1402 'duration'] if cols is None else cols
nexpaq 1:55a6170b404f 1403
nexpaq 1:55a6170b404f 1404 # All tests status table print
nexpaq 1:55a6170b404f 1405 pt = PrettyTable(test_properties)
nexpaq 1:55a6170b404f 1406 for col in test_properties:
nexpaq 1:55a6170b404f 1407 pt.align[col] = "l"
nexpaq 1:55a6170b404f 1408 pt.align['duration'] = "r"
nexpaq 1:55a6170b404f 1409
nexpaq 1:55a6170b404f 1410 counter_all = 0
nexpaq 1:55a6170b404f 1411 counter_automated = 0
nexpaq 1:55a6170b404f 1412 pt.padding_width = 1 # One space between column edges and contents (default)
nexpaq 1:55a6170b404f 1413
nexpaq 1:55a6170b404f 1414 for test_id in sorted(TEST_MAP.keys()):
nexpaq 1:55a6170b404f 1415 if platform_filter is not None:
nexpaq 1:55a6170b404f 1416 # FIlter out platforms using regex
nexpaq 1:55a6170b404f 1417 if re.search(platform_filter, test_id) is None:
nexpaq 1:55a6170b404f 1418 continue
nexpaq 1:55a6170b404f 1419 row = []
nexpaq 1:55a6170b404f 1420 test = TEST_MAP[test_id]
nexpaq 1:55a6170b404f 1421 split = test_id.split('_')[:-1]
nexpaq 1:55a6170b404f 1422 test_id_prefix = '_'.join(split)
nexpaq 1:55a6170b404f 1423
nexpaq 1:55a6170b404f 1424 for col in test_properties:
nexpaq 1:55a6170b404f 1425 col_value = test[col]
nexpaq 1:55a6170b404f 1426 if type(test[col]) == ListType:
nexpaq 1:55a6170b404f 1427 col_value = join_delim.join(test[col])
nexpaq 1:55a6170b404f 1428 elif test[col] == None:
nexpaq 1:55a6170b404f 1429 col_value = "-"
nexpaq 1:55a6170b404f 1430
nexpaq 1:55a6170b404f 1431 row.append(col_value)
nexpaq 1:55a6170b404f 1432 if test['automated'] == True:
nexpaq 1:55a6170b404f 1433 counter_dict_test_id_types[test_id_prefix] += 1
nexpaq 1:55a6170b404f 1434 counter_automated += 1
nexpaq 1:55a6170b404f 1435 pt.add_row(row)
nexpaq 1:55a6170b404f 1436 # Update counters
nexpaq 1:55a6170b404f 1437 counter_all += 1
nexpaq 1:55a6170b404f 1438 counter_dict_test_id_types_all[test_id_prefix] += 1
nexpaq 1:55a6170b404f 1439 result = pt.get_string()
nexpaq 1:55a6170b404f 1440 result += "\n\n"
nexpaq 1:55a6170b404f 1441
nexpaq 1:55a6170b404f 1442 if result_summary and not platform_filter:
nexpaq 1:55a6170b404f 1443 # Automation result summary
nexpaq 1:55a6170b404f 1444 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
nexpaq 1:55a6170b404f 1445 pt = PrettyTable(test_id_cols)
nexpaq 1:55a6170b404f 1446 pt.align['automated'] = "r"
nexpaq 1:55a6170b404f 1447 pt.align['all'] = "r"
nexpaq 1:55a6170b404f 1448 pt.align['percent [%]'] = "r"
nexpaq 1:55a6170b404f 1449
nexpaq 1:55a6170b404f 1450 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
nexpaq 1:55a6170b404f 1451 str_progress = progress_bar(percent_progress, 75)
nexpaq 1:55a6170b404f 1452 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
nexpaq 1:55a6170b404f 1453 result += "Automation coverage:\n"
nexpaq 1:55a6170b404f 1454 result += pt.get_string()
nexpaq 1:55a6170b404f 1455 result += "\n\n"
nexpaq 1:55a6170b404f 1456
nexpaq 1:55a6170b404f 1457 # Test automation coverage table print
nexpaq 1:55a6170b404f 1458 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
nexpaq 1:55a6170b404f 1459 pt = PrettyTable(test_id_cols)
nexpaq 1:55a6170b404f 1460 pt.align['id'] = "l"
nexpaq 1:55a6170b404f 1461 pt.align['automated'] = "r"
nexpaq 1:55a6170b404f 1462 pt.align['all'] = "r"
nexpaq 1:55a6170b404f 1463 pt.align['percent [%]'] = "r"
nexpaq 1:55a6170b404f 1464 for unique_id in unique_test_id:
nexpaq 1:55a6170b404f 1465 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
nexpaq 1:55a6170b404f 1466 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
nexpaq 1:55a6170b404f 1467 str_progress = progress_bar(percent_progress, 75)
nexpaq 1:55a6170b404f 1468 row = [unique_id,
nexpaq 1:55a6170b404f 1469 counter_dict_test_id_types[unique_id],
nexpaq 1:55a6170b404f 1470 counter_dict_test_id_types_all[unique_id],
nexpaq 1:55a6170b404f 1471 percent_progress,
nexpaq 1:55a6170b404f 1472 "[" + str_progress + "]"]
nexpaq 1:55a6170b404f 1473 pt.add_row(row)
nexpaq 1:55a6170b404f 1474 result += "Test automation coverage:\n"
nexpaq 1:55a6170b404f 1475 result += pt.get_string()
nexpaq 1:55a6170b404f 1476 result += "\n\n"
nexpaq 1:55a6170b404f 1477 return result
nexpaq 1:55a6170b404f 1478
nexpaq 1:55a6170b404f 1479
nexpaq 1:55a6170b404f 1480 def progress_bar(percent_progress, saturation=0):
nexpaq 1:55a6170b404f 1481 """ This function creates progress bar with optional simple saturation mark
nexpaq 1:55a6170b404f 1482 """
nexpaq 1:55a6170b404f 1483 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
nexpaq 1:55a6170b404f 1484 str_progress = '#' * step + '.' * int(50 - step)
nexpaq 1:55a6170b404f 1485 c = '!' if str_progress[38] == '.' else '|'
nexpaq 1:55a6170b404f 1486 if saturation > 0:
nexpaq 1:55a6170b404f 1487 saturation = saturation / 2
nexpaq 1:55a6170b404f 1488 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
nexpaq 1:55a6170b404f 1489 return str_progress
nexpaq 1:55a6170b404f 1490
nexpaq 1:55a6170b404f 1491
nexpaq 1:55a6170b404f 1492 def singletest_in_cli_mode(single_test):
nexpaq 1:55a6170b404f 1493 """ Runs SingleTestRunner object in CLI (Command line interface) mode
nexpaq 1:55a6170b404f 1494
nexpaq 1:55a6170b404f 1495 @return returns success code (0 == success) for building and running tests
nexpaq 1:55a6170b404f 1496 """
nexpaq 1:55a6170b404f 1497 start = time()
nexpaq 1:55a6170b404f 1498 # Execute tests depending on options and filter applied
nexpaq 1:55a6170b404f 1499 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
nexpaq 1:55a6170b404f 1500 elapsed_time = time() - start
nexpaq 1:55a6170b404f 1501
nexpaq 1:55a6170b404f 1502 # Human readable summary
nexpaq 1:55a6170b404f 1503 if not single_test.opts_suppress_summary:
nexpaq 1:55a6170b404f 1504 # prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 1505 print single_test.generate_test_summary(test_summary, shuffle_seed)
nexpaq 1:55a6170b404f 1506 if single_test.opts_test_x_toolchain_summary:
nexpaq 1:55a6170b404f 1507 # prints well-formed summary with results (SQL table like)
nexpaq 1:55a6170b404f 1508 # table shows text x toolchain test result matrix
nexpaq 1:55a6170b404f 1509 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
nexpaq 1:55a6170b404f 1510
nexpaq 1:55a6170b404f 1511 print "Completed in %.2f sec"% (elapsed_time)
nexpaq 1:55a6170b404f 1512 print
nexpaq 1:55a6170b404f 1513 # Write summary of the builds
nexpaq 1:55a6170b404f 1514
nexpaq 1:55a6170b404f 1515 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
nexpaq 1:55a6170b404f 1516 status = print_report_exporter.report(build_report)
nexpaq 1:55a6170b404f 1517
nexpaq 1:55a6170b404f 1518 # Store extra reports in files
nexpaq 1:55a6170b404f 1519 if single_test.opts_report_html_file_name:
nexpaq 1:55a6170b404f 1520 # Export results in form of HTML report to separate file
nexpaq 1:55a6170b404f 1521 report_exporter = ReportExporter(ResultExporterType.HTML)
nexpaq 1:55a6170b404f 1522 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
nexpaq 1:55a6170b404f 1523 if single_test.opts_report_junit_file_name:
nexpaq 1:55a6170b404f 1524 # Export results in form of JUnit XML report to separate file
nexpaq 1:55a6170b404f 1525 report_exporter = ReportExporter(ResultExporterType.JUNIT)
nexpaq 1:55a6170b404f 1526 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
nexpaq 1:55a6170b404f 1527 if single_test.opts_report_text_file_name:
nexpaq 1:55a6170b404f 1528 # Export results in form of a text file
nexpaq 1:55a6170b404f 1529 report_exporter = ReportExporter(ResultExporterType.TEXT)
nexpaq 1:55a6170b404f 1530 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
nexpaq 1:55a6170b404f 1531 if single_test.opts_report_build_file_name:
nexpaq 1:55a6170b404f 1532 # Export build results as html report to sparate file
nexpaq 1:55a6170b404f 1533 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
nexpaq 1:55a6170b404f 1534 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
nexpaq 1:55a6170b404f 1535
nexpaq 1:55a6170b404f 1536 # Returns True if no build failures of the test projects or their dependencies
nexpaq 1:55a6170b404f 1537 return status
nexpaq 1:55a6170b404f 1538
nexpaq 1:55a6170b404f 1539 class TestLogger():
nexpaq 1:55a6170b404f 1540 """ Super-class for logging and printing ongoing events for test suite pass
nexpaq 1:55a6170b404f 1541 """
nexpaq 1:55a6170b404f 1542 def __init__(self, store_log=True):
nexpaq 1:55a6170b404f 1543 """ We can control if logger actually stores log in memory
nexpaq 1:55a6170b404f 1544 or just handled all log entries immediately
nexpaq 1:55a6170b404f 1545 """
nexpaq 1:55a6170b404f 1546 self.log = []
nexpaq 1:55a6170b404f 1547 self.log_to_file = False
nexpaq 1:55a6170b404f 1548 self.log_file_name = None
nexpaq 1:55a6170b404f 1549 self.store_log = store_log
nexpaq 1:55a6170b404f 1550
nexpaq 1:55a6170b404f 1551 self.LogType = construct_enum(INFO='Info',
nexpaq 1:55a6170b404f 1552 WARN='Warning',
nexpaq 1:55a6170b404f 1553 NOTIF='Notification',
nexpaq 1:55a6170b404f 1554 ERROR='Error',
nexpaq 1:55a6170b404f 1555 EXCEPT='Exception')
nexpaq 1:55a6170b404f 1556
nexpaq 1:55a6170b404f 1557 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
nexpaq 1:55a6170b404f 1558 APPEND=2) # Append to existing log file
nexpaq 1:55a6170b404f 1559
nexpaq 1:55a6170b404f 1560 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
nexpaq 1:55a6170b404f 1561 """ Log one line of text
nexpaq 1:55a6170b404f 1562 """
nexpaq 1:55a6170b404f 1563 log_timestamp = time()
nexpaq 1:55a6170b404f 1564 log_entry = {'log_type' : LogType,
nexpaq 1:55a6170b404f 1565 'log_timestamp' : log_timestamp,
nexpaq 1:55a6170b404f 1566 'log_line' : log_line,
nexpaq 1:55a6170b404f 1567 '_future' : None
nexpaq 1:55a6170b404f 1568 }
nexpaq 1:55a6170b404f 1569 # Store log in memory
nexpaq 1:55a6170b404f 1570 if self.store_log:
nexpaq 1:55a6170b404f 1571 self.log.append(log_entry)
nexpaq 1:55a6170b404f 1572 return log_entry
nexpaq 1:55a6170b404f 1573
nexpaq 1:55a6170b404f 1574
nexpaq 1:55a6170b404f 1575 class CLITestLogger(TestLogger):
nexpaq 1:55a6170b404f 1576 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
nexpaq 1:55a6170b404f 1577 """
nexpaq 1:55a6170b404f 1578 def __init__(self, store_log=True, file_name=None):
nexpaq 1:55a6170b404f 1579 TestLogger.__init__(self)
nexpaq 1:55a6170b404f 1580 self.log_file_name = file_name
nexpaq 1:55a6170b404f 1581 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
nexpaq 1:55a6170b404f 1582 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
nexpaq 1:55a6170b404f 1583
nexpaq 1:55a6170b404f 1584 def log_print(self, log_entry, timestamp=True):
nexpaq 1:55a6170b404f 1585 """ Prints on screen formatted log entry
nexpaq 1:55a6170b404f 1586 """
nexpaq 1:55a6170b404f 1587 ts = log_entry['log_timestamp']
nexpaq 1:55a6170b404f 1588 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
nexpaq 1:55a6170b404f 1589 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
nexpaq 1:55a6170b404f 1590 return timestamp_str + log_line_str
nexpaq 1:55a6170b404f 1591
nexpaq 1:55a6170b404f 1592 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
nexpaq 1:55a6170b404f 1593 """ Logs line, if log file output was specified log line will be appended
nexpaq 1:55a6170b404f 1594 at the end of log file
nexpaq 1:55a6170b404f 1595 """
nexpaq 1:55a6170b404f 1596 log_entry = TestLogger.log_line(self, LogType, log_line)
nexpaq 1:55a6170b404f 1597 log_line_str = self.log_print(log_entry, timestamp)
nexpaq 1:55a6170b404f 1598 if self.log_file_name is not None:
nexpaq 1:55a6170b404f 1599 try:
nexpaq 1:55a6170b404f 1600 with open(self.log_file_name, 'a') as f:
nexpaq 1:55a6170b404f 1601 f.write(log_line_str + line_delim)
nexpaq 1:55a6170b404f 1602 except IOError:
nexpaq 1:55a6170b404f 1603 pass
nexpaq 1:55a6170b404f 1604 return log_line_str
nexpaq 1:55a6170b404f 1605
nexpaq 1:55a6170b404f 1606
nexpaq 1:55a6170b404f 1607 def factory_db_logger(db_url):
nexpaq 1:55a6170b404f 1608 """ Factory database driver depending on database type supplied in database connection string db_url
nexpaq 1:55a6170b404f 1609 """
nexpaq 1:55a6170b404f 1610 if db_url is not None:
nexpaq 1:55a6170b404f 1611 from tools.test_mysql import MySQLDBAccess
nexpaq 1:55a6170b404f 1612 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
nexpaq 1:55a6170b404f 1613 if connection_info is not None:
nexpaq 1:55a6170b404f 1614 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
nexpaq 1:55a6170b404f 1615 if db_type == 'mysql':
nexpaq 1:55a6170b404f 1616 return MySQLDBAccess()
nexpaq 1:55a6170b404f 1617 return None
nexpaq 1:55a6170b404f 1618
nexpaq 1:55a6170b404f 1619
nexpaq 1:55a6170b404f 1620 def detect_database_verbose(db_url):
nexpaq 1:55a6170b404f 1621 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
nexpaq 1:55a6170b404f 1622 """
nexpaq 1:55a6170b404f 1623 result = BaseDBAccess().parse_db_connection_string(db_url)
nexpaq 1:55a6170b404f 1624 if result is not None:
nexpaq 1:55a6170b404f 1625 # Parsing passed
nexpaq 1:55a6170b404f 1626 (db_type, username, password, host, db_name) = result
nexpaq 1:55a6170b404f 1627 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
nexpaq 1:55a6170b404f 1628 # Let's try to connect
nexpaq 1:55a6170b404f 1629 db_ = factory_db_logger(db_url)
nexpaq 1:55a6170b404f 1630 if db_ is not None:
nexpaq 1:55a6170b404f 1631 print "Connecting to database '%s'..."% db_url,
nexpaq 1:55a6170b404f 1632 db_.connect(host, username, password, db_name)
nexpaq 1:55a6170b404f 1633 if db_.is_connected():
nexpaq 1:55a6170b404f 1634 print "ok"
nexpaq 1:55a6170b404f 1635 print "Detecting database..."
nexpaq 1:55a6170b404f 1636 print db_.detect_database(verbose=True)
nexpaq 1:55a6170b404f 1637 print "Disconnecting...",
nexpaq 1:55a6170b404f 1638 db_.disconnect()
nexpaq 1:55a6170b404f 1639 print "done"
nexpaq 1:55a6170b404f 1640 else:
nexpaq 1:55a6170b404f 1641 print "Database type '%s' unknown"% db_type
nexpaq 1:55a6170b404f 1642 else:
nexpaq 1:55a6170b404f 1643 print "Parse error: '%s' - DB Url error"% (db_url)
nexpaq 1:55a6170b404f 1644
nexpaq 1:55a6170b404f 1645
nexpaq 1:55a6170b404f 1646 def get_module_avail(module_name):
nexpaq 1:55a6170b404f 1647 """ This function returns True if module_name is already impored module
nexpaq 1:55a6170b404f 1648 """
nexpaq 1:55a6170b404f 1649 return module_name in sys.modules.keys()
nexpaq 1:55a6170b404f 1650
nexpaq 1:55a6170b404f 1651
nexpaq 1:55a6170b404f 1652 def get_autodetected_MUTS_list(platform_name_filter=None):
nexpaq 1:55a6170b404f 1653 oldError = None
nexpaq 1:55a6170b404f 1654 if os.name == 'nt':
nexpaq 1:55a6170b404f 1655 # Disable Windows error box temporarily
nexpaq 1:55a6170b404f 1656 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
nexpaq 1:55a6170b404f 1657
nexpaq 1:55a6170b404f 1658 mbeds = mbed_lstools.create()
nexpaq 1:55a6170b404f 1659 detect_muts_list = mbeds.list_mbeds()
nexpaq 1:55a6170b404f 1660
nexpaq 1:55a6170b404f 1661 if os.name == 'nt':
nexpaq 1:55a6170b404f 1662 ctypes.windll.kernel32.SetErrorMode(oldError)
nexpaq 1:55a6170b404f 1663
nexpaq 1:55a6170b404f 1664 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
nexpaq 1:55a6170b404f 1665
nexpaq 1:55a6170b404f 1666 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
nexpaq 1:55a6170b404f 1667 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
nexpaq 1:55a6170b404f 1668 If function fails to auto-detect devices it will return empty dictionary.
nexpaq 1:55a6170b404f 1669
nexpaq 1:55a6170b404f 1670 if get_module_avail('mbed_lstools'):
nexpaq 1:55a6170b404f 1671 mbeds = mbed_lstools.create()
nexpaq 1:55a6170b404f 1672 mbeds_list = mbeds.list_mbeds()
nexpaq 1:55a6170b404f 1673
nexpaq 1:55a6170b404f 1674 @param mbeds_list list of mbeds captured from mbed_lstools
nexpaq 1:55a6170b404f 1675 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
nexpaq 1:55a6170b404f 1676 """
nexpaq 1:55a6170b404f 1677 result = {} # Should be in muts_all.json format
nexpaq 1:55a6170b404f 1678 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
nexpaq 1:55a6170b404f 1679 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
nexpaq 1:55a6170b404f 1680 index = 1
nexpaq 1:55a6170b404f 1681 for mut in mbeds_list:
nexpaq 1:55a6170b404f 1682 # Filter the MUTS if a filter is specified
nexpaq 1:55a6170b404f 1683
nexpaq 1:55a6170b404f 1684 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
nexpaq 1:55a6170b404f 1685 continue
nexpaq 1:55a6170b404f 1686
nexpaq 1:55a6170b404f 1687 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
nexpaq 1:55a6170b404f 1688 # if not we are creating our own unique value (last few chars from platform's target_id).
nexpaq 1:55a6170b404f 1689 m = {'mcu': mut['platform_name'],
nexpaq 1:55a6170b404f 1690 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
nexpaq 1:55a6170b404f 1691 'port': mut['serial_port'],
nexpaq 1:55a6170b404f 1692 'disk': mut['mount_point'],
nexpaq 1:55a6170b404f 1693 'peripherals': [] # No peripheral detection
nexpaq 1:55a6170b404f 1694 }
nexpaq 1:55a6170b404f 1695 if index not in result:
nexpaq 1:55a6170b404f 1696 result[index] = {}
nexpaq 1:55a6170b404f 1697 result[index] = m
nexpaq 1:55a6170b404f 1698 index += 1
nexpaq 1:55a6170b404f 1699 return result
nexpaq 1:55a6170b404f 1700
nexpaq 1:55a6170b404f 1701
nexpaq 1:55a6170b404f 1702 def get_autodetected_TEST_SPEC(mbeds_list,
nexpaq 1:55a6170b404f 1703 use_default_toolchain=True,
nexpaq 1:55a6170b404f 1704 use_supported_toolchains=False,
nexpaq 1:55a6170b404f 1705 toolchain_filter=None,
nexpaq 1:55a6170b404f 1706 platform_name_filter=None):
nexpaq 1:55a6170b404f 1707 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
nexpaq 1:55a6170b404f 1708 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
nexpaq 1:55a6170b404f 1709
nexpaq 1:55a6170b404f 1710 use_default_toolchain - if True add default toolchain to test_spec
nexpaq 1:55a6170b404f 1711 use_supported_toolchains - if True add all supported toolchains to test_spec
nexpaq 1:55a6170b404f 1712 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
nexpaq 1:55a6170b404f 1713 """
nexpaq 1:55a6170b404f 1714 result = {'targets': {} }
nexpaq 1:55a6170b404f 1715
nexpaq 1:55a6170b404f 1716 for mut in mbeds_list:
nexpaq 1:55a6170b404f 1717 mcu = mut['mcu']
nexpaq 1:55a6170b404f 1718 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
nexpaq 1:55a6170b404f 1719 if mcu in TARGET_MAP:
nexpaq 1:55a6170b404f 1720 default_toolchain = TARGET_MAP[mcu].default_toolchain
nexpaq 1:55a6170b404f 1721 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
nexpaq 1:55a6170b404f 1722
nexpaq 1:55a6170b404f 1723 # Decide which toolchains should be added to test specification toolchain pool for each target
nexpaq 1:55a6170b404f 1724 toolchains = []
nexpaq 1:55a6170b404f 1725 if use_default_toolchain:
nexpaq 1:55a6170b404f 1726 toolchains.append(default_toolchain)
nexpaq 1:55a6170b404f 1727 if use_supported_toolchains:
nexpaq 1:55a6170b404f 1728 toolchains += supported_toolchains
nexpaq 1:55a6170b404f 1729 if toolchain_filter is not None:
nexpaq 1:55a6170b404f 1730 all_toolchains = supported_toolchains + [default_toolchain]
nexpaq 1:55a6170b404f 1731 for toolchain in toolchain_filter:
nexpaq 1:55a6170b404f 1732 if toolchain in all_toolchains:
nexpaq 1:55a6170b404f 1733 toolchains.append(toolchain)
nexpaq 1:55a6170b404f 1734
nexpaq 1:55a6170b404f 1735 result['targets'][mcu] = list(set(toolchains))
nexpaq 1:55a6170b404f 1736 return result
nexpaq 1:55a6170b404f 1737
nexpaq 1:55a6170b404f 1738
nexpaq 1:55a6170b404f 1739 def get_default_test_options_parser():
nexpaq 1:55a6170b404f 1740 """ Get common test script options used by CLI, web services etc.
nexpaq 1:55a6170b404f 1741 """
nexpaq 1:55a6170b404f 1742 parser = argparse.ArgumentParser()
nexpaq 1:55a6170b404f 1743 parser.add_argument('-i', '--tests',
nexpaq 1:55a6170b404f 1744 dest='test_spec_filename',
nexpaq 1:55a6170b404f 1745 metavar="FILE",
nexpaq 1:55a6170b404f 1746 type=argparse_filestring_type,
nexpaq 1:55a6170b404f 1747 help='Points to file with test specification')
nexpaq 1:55a6170b404f 1748
nexpaq 1:55a6170b404f 1749 parser.add_argument('-M', '--MUTS',
nexpaq 1:55a6170b404f 1750 dest='muts_spec_filename',
nexpaq 1:55a6170b404f 1751 metavar="FILE",
nexpaq 1:55a6170b404f 1752 type=argparse_filestring_type,
nexpaq 1:55a6170b404f 1753 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
nexpaq 1:55a6170b404f 1754
nexpaq 1:55a6170b404f 1755 parser.add_argument("-j", "--jobs",
nexpaq 1:55a6170b404f 1756 dest='jobs',
nexpaq 1:55a6170b404f 1757 metavar="NUMBER",
nexpaq 1:55a6170b404f 1758 type=int,
nexpaq 1:55a6170b404f 1759 help="Define number of compilation jobs. Default value is 1")
nexpaq 1:55a6170b404f 1760
nexpaq 1:55a6170b404f 1761 if get_module_avail('mbed_lstools'):
nexpaq 1:55a6170b404f 1762 # Additional features available when mbed_lstools is installed on host and imported
nexpaq 1:55a6170b404f 1763 # mbed_lstools allow users to detect connected to host mbed-enabled devices
nexpaq 1:55a6170b404f 1764 parser.add_argument('--auto',
nexpaq 1:55a6170b404f 1765 dest='auto_detect',
nexpaq 1:55a6170b404f 1766 action="store_true",
nexpaq 1:55a6170b404f 1767 help='Use mbed-ls module to detect all connected mbed devices')
nexpaq 1:55a6170b404f 1768
nexpaq 1:55a6170b404f 1769 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
nexpaq 1:55a6170b404f 1770 parser.add_argument('--tc',
nexpaq 1:55a6170b404f 1771 dest='toolchains_filter',
nexpaq 1:55a6170b404f 1772 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
nexpaq 1:55a6170b404f 1773 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
nexpaq 1:55a6170b404f 1774
nexpaq 1:55a6170b404f 1775 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
nexpaq 1:55a6170b404f 1776 parser.add_argument('--oper',
nexpaq 1:55a6170b404f 1777 dest='operability_checks',
nexpaq 1:55a6170b404f 1778 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
nexpaq 1:55a6170b404f 1779 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
nexpaq 1:55a6170b404f 1780
nexpaq 1:55a6170b404f 1781 parser.add_argument('--clean',
nexpaq 1:55a6170b404f 1782 dest='clean',
nexpaq 1:55a6170b404f 1783 action="store_true",
nexpaq 1:55a6170b404f 1784 help='Clean the build directory')
nexpaq 1:55a6170b404f 1785
nexpaq 1:55a6170b404f 1786 parser.add_argument('-P', '--only-peripherals',
nexpaq 1:55a6170b404f 1787 dest='test_only_peripheral',
nexpaq 1:55a6170b404f 1788 default=False,
nexpaq 1:55a6170b404f 1789 action="store_true",
nexpaq 1:55a6170b404f 1790 help='Test only peripheral declared for MUT and skip common tests')
nexpaq 1:55a6170b404f 1791
nexpaq 1:55a6170b404f 1792 parser.add_argument('-C', '--only-commons',
nexpaq 1:55a6170b404f 1793 dest='test_only_common',
nexpaq 1:55a6170b404f 1794 default=False,
nexpaq 1:55a6170b404f 1795 action="store_true",
nexpaq 1:55a6170b404f 1796 help='Test only board internals. Skip perpherials tests and perform common tests')
nexpaq 1:55a6170b404f 1797
nexpaq 1:55a6170b404f 1798 parser.add_argument('-n', '--test-by-names',
nexpaq 1:55a6170b404f 1799 dest='test_by_names',
nexpaq 1:55a6170b404f 1800 type=argparse_many(str),
nexpaq 1:55a6170b404f 1801 help='Runs only test enumerated it this switch. Use comma to separate test case names')
nexpaq 1:55a6170b404f 1802
nexpaq 1:55a6170b404f 1803 parser.add_argument('-p', '--peripheral-by-names',
nexpaq 1:55a6170b404f 1804 dest='peripheral_by_names',
nexpaq 1:55a6170b404f 1805 type=argparse_many(str),
nexpaq 1:55a6170b404f 1806 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
nexpaq 1:55a6170b404f 1807
nexpaq 1:55a6170b404f 1808 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
nexpaq 1:55a6170b404f 1809 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
nexpaq 1:55a6170b404f 1810
nexpaq 1:55a6170b404f 1811 parser.add_argument('-c', '--copy-method',
nexpaq 1:55a6170b404f 1812 dest='copy_method',
nexpaq 1:55a6170b404f 1813 type=argparse_uppercase_type(copy_methods, "flash method"),
nexpaq 1:55a6170b404f 1814 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
nexpaq 1:55a6170b404f 1815
nexpaq 1:55a6170b404f 1816 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
nexpaq 1:55a6170b404f 1817 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
nexpaq 1:55a6170b404f 1818
nexpaq 1:55a6170b404f 1819 parser.add_argument('-r', '--reset-type',
nexpaq 1:55a6170b404f 1820 dest='mut_reset_type',
nexpaq 1:55a6170b404f 1821 default=None,
nexpaq 1:55a6170b404f 1822 type=argparse_uppercase_type(reset_methods, "reset method"),
nexpaq 1:55a6170b404f 1823 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
nexpaq 1:55a6170b404f 1824
nexpaq 1:55a6170b404f 1825 parser.add_argument('-g', '--goanna-for-tests',
nexpaq 1:55a6170b404f 1826 dest='goanna_for_tests',
nexpaq 1:55a6170b404f 1827 action="store_true",
nexpaq 1:55a6170b404f 1828 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
nexpaq 1:55a6170b404f 1829
nexpaq 1:55a6170b404f 1830 parser.add_argument('-G', '--goanna-for-sdk',
nexpaq 1:55a6170b404f 1831 dest='goanna_for_mbed_sdk',
nexpaq 1:55a6170b404f 1832 action="store_true",
nexpaq 1:55a6170b404f 1833 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
nexpaq 1:55a6170b404f 1834
nexpaq 1:55a6170b404f 1835 parser.add_argument('-s', '--suppress-summary',
nexpaq 1:55a6170b404f 1836 dest='suppress_summary',
nexpaq 1:55a6170b404f 1837 default=False,
nexpaq 1:55a6170b404f 1838 action="store_true",
nexpaq 1:55a6170b404f 1839 help='Suppresses display of wellformatted table with test results')
nexpaq 1:55a6170b404f 1840
nexpaq 1:55a6170b404f 1841 parser.add_argument('-t', '--test-summary',
nexpaq 1:55a6170b404f 1842 dest='test_x_toolchain_summary',
nexpaq 1:55a6170b404f 1843 default=False,
nexpaq 1:55a6170b404f 1844 action="store_true",
nexpaq 1:55a6170b404f 1845 help='Displays wellformatted table with test x toolchain test result per target')
nexpaq 1:55a6170b404f 1846
nexpaq 1:55a6170b404f 1847 parser.add_argument('-A', '--test-automation-report',
nexpaq 1:55a6170b404f 1848 dest='test_automation_report',
nexpaq 1:55a6170b404f 1849 default=False,
nexpaq 1:55a6170b404f 1850 action="store_true",
nexpaq 1:55a6170b404f 1851 help='Prints information about all tests and exits')
nexpaq 1:55a6170b404f 1852
nexpaq 1:55a6170b404f 1853 parser.add_argument('-R', '--test-case-report',
nexpaq 1:55a6170b404f 1854 dest='test_case_report',
nexpaq 1:55a6170b404f 1855 default=False,
nexpaq 1:55a6170b404f 1856 action="store_true",
nexpaq 1:55a6170b404f 1857 help='Prints information about all test cases and exits')
nexpaq 1:55a6170b404f 1858
nexpaq 1:55a6170b404f 1859 parser.add_argument("-S", "--supported-toolchains",
nexpaq 1:55a6170b404f 1860 action="store_true",
nexpaq 1:55a6170b404f 1861 dest="supported_toolchains",
nexpaq 1:55a6170b404f 1862 default=False,
nexpaq 1:55a6170b404f 1863 help="Displays supported matrix of MCUs and toolchains")
nexpaq 1:55a6170b404f 1864
nexpaq 1:55a6170b404f 1865 parser.add_argument("-O", "--only-build",
nexpaq 1:55a6170b404f 1866 action="store_true",
nexpaq 1:55a6170b404f 1867 dest="only_build_tests",
nexpaq 1:55a6170b404f 1868 default=False,
nexpaq 1:55a6170b404f 1869 help="Only build tests, skips actual test procedures (flashing etc.)")
nexpaq 1:55a6170b404f 1870
nexpaq 1:55a6170b404f 1871 parser.add_argument('--parallel',
nexpaq 1:55a6170b404f 1872 dest='parallel_test_exec',
nexpaq 1:55a6170b404f 1873 default=False,
nexpaq 1:55a6170b404f 1874 action="store_true",
nexpaq 1:55a6170b404f 1875 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
nexpaq 1:55a6170b404f 1876
nexpaq 1:55a6170b404f 1877 parser.add_argument('--config',
nexpaq 1:55a6170b404f 1878 dest='verbose_test_configuration_only',
nexpaq 1:55a6170b404f 1879 default=False,
nexpaq 1:55a6170b404f 1880 action="store_true",
nexpaq 1:55a6170b404f 1881 help='Displays full test specification and MUTs configration and exits')
nexpaq 1:55a6170b404f 1882
nexpaq 1:55a6170b404f 1883 parser.add_argument('--loops',
nexpaq 1:55a6170b404f 1884 dest='test_loops_list',
nexpaq 1:55a6170b404f 1885 type=argparse_many(str),
nexpaq 1:55a6170b404f 1886 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
nexpaq 1:55a6170b404f 1887
nexpaq 1:55a6170b404f 1888 parser.add_argument('--global-loops',
nexpaq 1:55a6170b404f 1889 dest='test_global_loops_value',
nexpaq 1:55a6170b404f 1890 type=int,
nexpaq 1:55a6170b404f 1891 help='Set global number of test loops per test. Default value is set 1')
nexpaq 1:55a6170b404f 1892
nexpaq 1:55a6170b404f 1893 parser.add_argument('--consolidate-waterfall',
nexpaq 1:55a6170b404f 1894 dest='consolidate_waterfall_test',
nexpaq 1:55a6170b404f 1895 default=False,
nexpaq 1:55a6170b404f 1896 action="store_true",
nexpaq 1:55a6170b404f 1897 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
nexpaq 1:55a6170b404f 1898
nexpaq 1:55a6170b404f 1899 parser.add_argument('-W', '--waterfall',
nexpaq 1:55a6170b404f 1900 dest='waterfall_test',
nexpaq 1:55a6170b404f 1901 default=False,
nexpaq 1:55a6170b404f 1902 action="store_true",
nexpaq 1:55a6170b404f 1903 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
nexpaq 1:55a6170b404f 1904
nexpaq 1:55a6170b404f 1905 parser.add_argument('-N', '--firmware-name',
nexpaq 1:55a6170b404f 1906 dest='firmware_global_name',
nexpaq 1:55a6170b404f 1907 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
nexpaq 1:55a6170b404f 1908
nexpaq 1:55a6170b404f 1909 parser.add_argument('-u', '--shuffle',
nexpaq 1:55a6170b404f 1910 dest='shuffle_test_order',
nexpaq 1:55a6170b404f 1911 default=False,
nexpaq 1:55a6170b404f 1912 action="store_true",
nexpaq 1:55a6170b404f 1913 help='Shuffles test execution order')
nexpaq 1:55a6170b404f 1914
nexpaq 1:55a6170b404f 1915 parser.add_argument('--shuffle-seed',
nexpaq 1:55a6170b404f 1916 dest='shuffle_test_seed',
nexpaq 1:55a6170b404f 1917 default=None,
nexpaq 1:55a6170b404f 1918 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
nexpaq 1:55a6170b404f 1919
nexpaq 1:55a6170b404f 1920 parser.add_argument('-f', '--filter',
nexpaq 1:55a6170b404f 1921 dest='general_filter_regex',
nexpaq 1:55a6170b404f 1922 type=argparse_many(str),
nexpaq 1:55a6170b404f 1923 default=None,
nexpaq 1:55a6170b404f 1924 help='For some commands you can use filter to filter out results')
nexpaq 1:55a6170b404f 1925
nexpaq 1:55a6170b404f 1926 parser.add_argument('--inc-timeout',
nexpaq 1:55a6170b404f 1927 dest='extend_test_timeout',
nexpaq 1:55a6170b404f 1928 metavar="NUMBER",
nexpaq 1:55a6170b404f 1929 type=int,
nexpaq 1:55a6170b404f 1930 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
nexpaq 1:55a6170b404f 1931
nexpaq 1:55a6170b404f 1932 parser.add_argument('--db',
nexpaq 1:55a6170b404f 1933 dest='db_url',
nexpaq 1:55a6170b404f 1934 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
nexpaq 1:55a6170b404f 1935
nexpaq 1:55a6170b404f 1936 parser.add_argument('-l', '--log',
nexpaq 1:55a6170b404f 1937 dest='log_file_name',
nexpaq 1:55a6170b404f 1938 help='Log events to external file (note not all console entries may be visible in log file)')
nexpaq 1:55a6170b404f 1939
nexpaq 1:55a6170b404f 1940 parser.add_argument('--report-html',
nexpaq 1:55a6170b404f 1941 dest='report_html_file_name',
nexpaq 1:55a6170b404f 1942 help='You can log test suite results in form of HTML report')
nexpaq 1:55a6170b404f 1943
nexpaq 1:55a6170b404f 1944 parser.add_argument('--report-junit',
nexpaq 1:55a6170b404f 1945 dest='report_junit_file_name',
nexpaq 1:55a6170b404f 1946 help='You can log test suite results in form of JUnit compliant XML report')
nexpaq 1:55a6170b404f 1947
nexpaq 1:55a6170b404f 1948 parser.add_argument("--report-build",
nexpaq 1:55a6170b404f 1949 dest="report_build_file_name",
nexpaq 1:55a6170b404f 1950 help="Output the build results to a junit xml file")
nexpaq 1:55a6170b404f 1951
nexpaq 1:55a6170b404f 1952 parser.add_argument("--report-text",
nexpaq 1:55a6170b404f 1953 dest="report_text_file_name",
nexpaq 1:55a6170b404f 1954 help="Output the build results to a text file")
nexpaq 1:55a6170b404f 1955
nexpaq 1:55a6170b404f 1956 parser.add_argument('--verbose-skipped',
nexpaq 1:55a6170b404f 1957 dest='verbose_skipped_tests',
nexpaq 1:55a6170b404f 1958 default=False,
nexpaq 1:55a6170b404f 1959 action="store_true",
nexpaq 1:55a6170b404f 1960 help='Prints some extra information about skipped tests')
nexpaq 1:55a6170b404f 1961
nexpaq 1:55a6170b404f 1962 parser.add_argument('-V', '--verbose-test-result',
nexpaq 1:55a6170b404f 1963 dest='verbose_test_result_only',
nexpaq 1:55a6170b404f 1964 default=False,
nexpaq 1:55a6170b404f 1965 action="store_true",
nexpaq 1:55a6170b404f 1966 help='Prints test serial output')
nexpaq 1:55a6170b404f 1967
nexpaq 1:55a6170b404f 1968 parser.add_argument('-v', '--verbose',
nexpaq 1:55a6170b404f 1969 dest='verbose',
nexpaq 1:55a6170b404f 1970 default=False,
nexpaq 1:55a6170b404f 1971 action="store_true",
nexpaq 1:55a6170b404f 1972 help='Verbose mode (prints some extra information)')
nexpaq 1:55a6170b404f 1973
nexpaq 1:55a6170b404f 1974 parser.add_argument('--version',
nexpaq 1:55a6170b404f 1975 dest='version',
nexpaq 1:55a6170b404f 1976 default=False,
nexpaq 1:55a6170b404f 1977 action="store_true",
nexpaq 1:55a6170b404f 1978 help='Prints script version and exits')
nexpaq 1:55a6170b404f 1979 return parser
nexpaq 1:55a6170b404f 1980
nexpaq 1:55a6170b404f 1981 def test_path_to_name(path, base):
nexpaq 1:55a6170b404f 1982 """Change all slashes in a path into hyphens
nexpaq 1:55a6170b404f 1983 This creates a unique cross-platform test name based on the path
nexpaq 1:55a6170b404f 1984 This can eventually be overriden by a to-be-determined meta-data mechanism"""
nexpaq 1:55a6170b404f 1985 name_parts = []
nexpaq 1:55a6170b404f 1986 head, tail = os.path.split(relpath(path,base))
nexpaq 1:55a6170b404f 1987 while (tail and tail != "."):
nexpaq 1:55a6170b404f 1988 name_parts.insert(0, tail)
nexpaq 1:55a6170b404f 1989 head, tail = os.path.split(head)
nexpaq 1:55a6170b404f 1990
nexpaq 1:55a6170b404f 1991 return "-".join(name_parts).lower()
nexpaq 1:55a6170b404f 1992
nexpaq 1:55a6170b404f 1993 def find_tests(base_dir, target_name, toolchain_name, options=None, app_config=None):
nexpaq 1:55a6170b404f 1994 """ Finds all tests in a directory recursively
nexpaq 1:55a6170b404f 1995 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
nexpaq 1:55a6170b404f 1996 target_name: name of the target to use for scanning (ex. 'K64F')
nexpaq 1:55a6170b404f 1997 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
nexpaq 1:55a6170b404f 1998 options: Compile options to pass to the toolchain (ex. ['debug-info'])
nexpaq 1:55a6170b404f 1999 app_config - location of a chosen mbed_app.json file
nexpaq 1:55a6170b404f 2000 """
nexpaq 1:55a6170b404f 2001
nexpaq 1:55a6170b404f 2002 tests = {}
nexpaq 1:55a6170b404f 2003
nexpaq 1:55a6170b404f 2004 # Prepare the toolchain
nexpaq 1:55a6170b404f 2005 toolchain = prepare_toolchain([base_dir], target_name, toolchain_name, options=options,
nexpaq 1:55a6170b404f 2006 silent=True, app_config=app_config)
nexpaq 1:55a6170b404f 2007
nexpaq 1:55a6170b404f 2008 # Scan the directory for paths to probe for 'TESTS' folders
nexpaq 1:55a6170b404f 2009 base_resources = scan_resources([base_dir], toolchain)
nexpaq 1:55a6170b404f 2010
nexpaq 1:55a6170b404f 2011 dirs = base_resources.inc_dirs
nexpaq 1:55a6170b404f 2012 for directory in dirs:
nexpaq 1:55a6170b404f 2013 subdirs = os.listdir(directory)
nexpaq 1:55a6170b404f 2014
nexpaq 1:55a6170b404f 2015 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
nexpaq 1:55a6170b404f 2016 if 'TESTS' in subdirs:
nexpaq 1:55a6170b404f 2017 walk_base_dir = join(directory, 'TESTS')
nexpaq 1:55a6170b404f 2018 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
nexpaq 1:55a6170b404f 2019
nexpaq 1:55a6170b404f 2020 # Loop through all subdirectories
nexpaq 1:55a6170b404f 2021 for d in test_resources.inc_dirs:
nexpaq 1:55a6170b404f 2022
nexpaq 1:55a6170b404f 2023 # If the test case folder is not called 'host_tests' and it is
nexpaq 1:55a6170b404f 2024 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
nexpaq 1:55a6170b404f 2025 # then add it to the tests
nexpaq 1:55a6170b404f 2026 path_depth = get_path_depth(relpath(d, walk_base_dir))
nexpaq 1:55a6170b404f 2027 if path_depth == 2:
nexpaq 1:55a6170b404f 2028 test_group_directory_path, test_case_directory = os.path.split(d)
nexpaq 1:55a6170b404f 2029 test_group_directory = os.path.basename(test_group_directory_path)
nexpaq 1:55a6170b404f 2030
nexpaq 1:55a6170b404f 2031 # Check to make sure discoverd folder is not in a host test directory
nexpaq 1:55a6170b404f 2032 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
nexpaq 1:55a6170b404f 2033 test_name = test_path_to_name(d, base_dir)
nexpaq 1:55a6170b404f 2034 tests[test_name] = d
nexpaq 1:55a6170b404f 2035
nexpaq 1:55a6170b404f 2036 return tests
nexpaq 1:55a6170b404f 2037
nexpaq 1:55a6170b404f 2038 def print_tests(tests, format="list", sort=True):
nexpaq 1:55a6170b404f 2039 """Given a dictionary of tests (as returned from "find_tests"), print them
nexpaq 1:55a6170b404f 2040 in the specified format"""
nexpaq 1:55a6170b404f 2041 if format == "list":
nexpaq 1:55a6170b404f 2042 for test_name in sorted(tests.keys()):
nexpaq 1:55a6170b404f 2043 test_path = tests[test_name]
nexpaq 1:55a6170b404f 2044 print "Test Case:"
nexpaq 1:55a6170b404f 2045 print " Name: %s" % test_name
nexpaq 1:55a6170b404f 2046 print " Path: %s" % test_path
nexpaq 1:55a6170b404f 2047 elif format == "json":
nexpaq 1:55a6170b404f 2048 print json.dumps(tests, indent=2)
nexpaq 1:55a6170b404f 2049 else:
nexpaq 1:55a6170b404f 2050 print "Unknown format '%s'" % format
nexpaq 1:55a6170b404f 2051 sys.exit(1)
nexpaq 1:55a6170b404f 2052
nexpaq 1:55a6170b404f 2053 def norm_relative_path(path, start):
nexpaq 1:55a6170b404f 2054 """This function will create a normalized, relative path. It mimics the
nexpaq 1:55a6170b404f 2055 python os.path.relpath function, but also normalizes a Windows-syle path
nexpaq 1:55a6170b404f 2056 that use backslashes to a Unix style path that uses forward slashes."""
nexpaq 1:55a6170b404f 2057 path = os.path.normpath(path)
nexpaq 1:55a6170b404f 2058 path = os.path.relpath(path, start)
nexpaq 1:55a6170b404f 2059 path = path.replace("\\", "/")
nexpaq 1:55a6170b404f 2060 return path
nexpaq 1:55a6170b404f 2061
nexpaq 1:55a6170b404f 2062 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
nexpaq 1:55a6170b404f 2063 options=None, clean=False, notify=None, verbose=False, jobs=1,
nexpaq 1:55a6170b404f 2064 macros=None, silent=False, report=None, properties=None,
nexpaq 1:55a6170b404f 2065 continue_on_build_fail=False, app_config=None):
nexpaq 1:55a6170b404f 2066 """Given the data structure from 'find_tests' and the typical build parameters,
nexpaq 1:55a6170b404f 2067 build all the tests
nexpaq 1:55a6170b404f 2068
nexpaq 1:55a6170b404f 2069 Returns a tuple of the build result (True or False) followed by the test
nexpaq 1:55a6170b404f 2070 build data structure"""
nexpaq 1:55a6170b404f 2071
nexpaq 1:55a6170b404f 2072 execution_directory = "."
nexpaq 1:55a6170b404f 2073 base_path = norm_relative_path(build_path, execution_directory)
nexpaq 1:55a6170b404f 2074
nexpaq 1:55a6170b404f 2075 target_name = target if isinstance(target, str) else target.name
nexpaq 1:55a6170b404f 2076
nexpaq 1:55a6170b404f 2077 test_build = {
nexpaq 1:55a6170b404f 2078 "platform": target_name,
nexpaq 1:55a6170b404f 2079 "toolchain": toolchain_name,
nexpaq 1:55a6170b404f 2080 "base_path": base_path,
nexpaq 1:55a6170b404f 2081 "baud_rate": 9600,
nexpaq 1:55a6170b404f 2082 "binary_type": "bootable",
nexpaq 1:55a6170b404f 2083 "tests": {}
nexpaq 1:55a6170b404f 2084 }
nexpaq 1:55a6170b404f 2085
nexpaq 1:55a6170b404f 2086 result = True
nexpaq 1:55a6170b404f 2087
nexpaq 1:55a6170b404f 2088 map_outputs_total = list()
nexpaq 1:55a6170b404f 2089 for test_name, test_path in tests.iteritems():
nexpaq 1:55a6170b404f 2090 test_build_path = os.path.join(build_path, test_path)
nexpaq 1:55a6170b404f 2091 src_path = base_source_paths + [test_path]
nexpaq 1:55a6170b404f 2092 bin_file = None
nexpaq 1:55a6170b404f 2093 test_case_folder_name = os.path.basename(test_path)
nexpaq 1:55a6170b404f 2094
nexpaq 1:55a6170b404f 2095
nexpaq 1:55a6170b404f 2096 try:
nexpaq 1:55a6170b404f 2097 bin_file = build_project(src_path, test_build_path, target, toolchain_name,
nexpaq 1:55a6170b404f 2098 options=options,
nexpaq 1:55a6170b404f 2099 jobs=jobs,
nexpaq 1:55a6170b404f 2100 clean=clean,
nexpaq 1:55a6170b404f 2101 macros=macros,
nexpaq 1:55a6170b404f 2102 name=test_case_folder_name,
nexpaq 1:55a6170b404f 2103 project_id=test_name,
nexpaq 1:55a6170b404f 2104 report=report,
nexpaq 1:55a6170b404f 2105 properties=properties,
nexpaq 1:55a6170b404f 2106 verbose=verbose,
nexpaq 1:55a6170b404f 2107 app_config=app_config)
nexpaq 1:55a6170b404f 2108
nexpaq 1:55a6170b404f 2109 except Exception, e:
nexpaq 1:55a6170b404f 2110 if not isinstance(e, NotSupportedException):
nexpaq 1:55a6170b404f 2111 result = False
nexpaq 1:55a6170b404f 2112
nexpaq 1:55a6170b404f 2113 if continue_on_build_fail:
nexpaq 1:55a6170b404f 2114 continue
nexpaq 1:55a6170b404f 2115 else:
nexpaq 1:55a6170b404f 2116 break
nexpaq 1:55a6170b404f 2117
nexpaq 1:55a6170b404f 2118 # If a clean build was carried out last time, disable it for the next build.
nexpaq 1:55a6170b404f 2119 # Otherwise the previously built test will be deleted.
nexpaq 1:55a6170b404f 2120 if clean:
nexpaq 1:55a6170b404f 2121 clean = False
nexpaq 1:55a6170b404f 2122
nexpaq 1:55a6170b404f 2123 # Normalize the path
nexpaq 1:55a6170b404f 2124 if bin_file:
nexpaq 1:55a6170b404f 2125 bin_file = norm_relative_path(bin_file, execution_directory)
nexpaq 1:55a6170b404f 2126
nexpaq 1:55a6170b404f 2127 test_build['tests'][test_name] = {
nexpaq 1:55a6170b404f 2128 "binaries": [
nexpaq 1:55a6170b404f 2129 {
nexpaq 1:55a6170b404f 2130 "path": bin_file
nexpaq 1:55a6170b404f 2131 }
nexpaq 1:55a6170b404f 2132 ]
nexpaq 1:55a6170b404f 2133 }
nexpaq 1:55a6170b404f 2134
nexpaq 1:55a6170b404f 2135 print 'Image: %s'% bin_file
nexpaq 1:55a6170b404f 2136
nexpaq 1:55a6170b404f 2137 test_builds = {}
nexpaq 1:55a6170b404f 2138 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
nexpaq 1:55a6170b404f 2139
nexpaq 1:55a6170b404f 2140
nexpaq 1:55a6170b404f 2141 return result, test_builds
nexpaq 1:55a6170b404f 2142
nexpaq 1:55a6170b404f 2143
nexpaq 1:55a6170b404f 2144 def test_spec_from_test_builds(test_builds):
nexpaq 1:55a6170b404f 2145 return {
nexpaq 1:55a6170b404f 2146 "builds": test_builds
nexpaq 1:55a6170b404f 2147 }