Anders Blomdell / mbed-sdk-tools

Clone of official tools

test_api.py@41:2a77626a4c21, 2017-10-25 (annotated)

Committer:
theotherjimmy
Date:
Wed Oct 25 14:46:50 2017 -0500
Revision:
41:2a77626a4c21
Parent:
40:7d3fa6b99b2b
Child:
43:2a7da56ebd24
Update to track Mbed OS 5.6.3

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """
screamer 0:66f3b5499f7f 2 mbed SDK
screamer 0:66f3b5499f7f 3 Copyright (c) 2011-2014 ARM Limited
screamer 0:66f3b5499f7f 4
screamer 0:66f3b5499f7f 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 6 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 7 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 8
screamer 0:66f3b5499f7f 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 14 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 15 limitations under the License.
screamer 0:66f3b5499f7f 16
screamer 0:66f3b5499f7f 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer 0:66f3b5499f7f 18 """
screamer 0:66f3b5499f7f 19
screamer 0:66f3b5499f7f 20 import os
screamer 0:66f3b5499f7f 21 import re
screamer 0:66f3b5499f7f 22 import sys
screamer 0:66f3b5499f7f 23 import json
screamer 0:66f3b5499f7f 24 import uuid
screamer 0:66f3b5499f7f 25 import pprint
screamer 0:66f3b5499f7f 26 import random
screamer 22:9e85236d8716 27 import argparse
screamer 0:66f3b5499f7f 28 import datetime
screamer 0:66f3b5499f7f 29 import threading
screamer 0:66f3b5499f7f 30 import ctypes
screamer 0:66f3b5499f7f 31 from types import ListType
screamer 0:66f3b5499f7f 32 from colorama import Fore, Back, Style
screamer 0:66f3b5499f7f 33 from prettytable import PrettyTable
screamer 24:25bff2709c20 34 from copy import copy
screamer 0:66f3b5499f7f 35
screamer 0:66f3b5499f7f 36 from time import sleep, time
screamer 0:66f3b5499f7f 37 from Queue import Queue, Empty
screamer 24:25bff2709c20 38 from os.path import join, exists, basename, relpath
screamer 0:66f3b5499f7f 39 from threading import Thread, Lock
The Other Jimmy 31:8ea194f6145b 40 from multiprocessing import Pool, cpu_count
screamer 0:66f3b5499f7f 41 from subprocess import Popen, PIPE
screamer 0:66f3b5499f7f 42
screamer 0:66f3b5499f7f 43 # Imports related to mbed build api
screamer 0:66f3b5499f7f 44 from tools.tests import TESTS
screamer 0:66f3b5499f7f 45 from tools.tests import TEST_MAP
screamer 0:66f3b5499f7f 46 from tools.paths import BUILD_DIR
screamer 0:66f3b5499f7f 47 from tools.paths import HOST_TESTS
screamer 0:66f3b5499f7f 48 from tools.utils import ToolException
screamer 0:66f3b5499f7f 49 from tools.utils import NotSupportedException
screamer 0:66f3b5499f7f 50 from tools.utils import construct_enum
screamer 22:9e85236d8716 51 from tools.memap import MemapParser
screamer 0:66f3b5499f7f 52 from tools.targets import TARGET_MAP
theotherjimmy 41:2a77626a4c21 53 import tools.test_configs as TestConfig
screamer 0:66f3b5499f7f 54 from tools.test_db import BaseDBAccess
screamer 0:66f3b5499f7f 55 from tools.build_api import build_project, build_mbed_libs, build_lib
screamer 0:66f3b5499f7f 56 from tools.build_api import get_target_supported_toolchains
screamer 0:66f3b5499f7f 57 from tools.build_api import write_build_report
screamer 0:66f3b5499f7f 58 from tools.build_api import prep_report
screamer 0:66f3b5499f7f 59 from tools.build_api import prep_properties
screamer 0:66f3b5499f7f 60 from tools.build_api import create_result
screamer 0:66f3b5499f7f 61 from tools.build_api import add_result_to_report
screamer 24:25bff2709c20 62 from tools.build_api import prepare_toolchain
screamer 24:25bff2709c20 63 from tools.build_api import scan_resources
The Other Jimmy 31:8ea194f6145b 64 from tools.build_api import get_config
screamer 0:66f3b5499f7f 65 from tools.libraries import LIBRARIES, LIBRARY_MAP
The Other Jimmy 31:8ea194f6145b 66 from tools.options import extract_profile
screamer 20:835f6355470d 67 from tools.toolchains import TOOLCHAIN_PATHS
screamer 22:9e85236d8716 68 from tools.toolchains import TOOLCHAINS
screamer 0:66f3b5499f7f 69 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 22:9e85236d8716 70 from tools.utils import argparse_filestring_type
screamer 22:9e85236d8716 71 from tools.utils import argparse_uppercase_type
screamer 22:9e85236d8716 72 from tools.utils import argparse_lowercase_type
screamer 22:9e85236d8716 73 from tools.utils import argparse_many
screamer 24:25bff2709c20 74 from tools.utils import get_path_depth
screamer 0:66f3b5499f7f 75
screamer 0:66f3b5499f7f 76 import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer 0:66f3b5499f7f 77
screamer 0:66f3b5499f7f 78 try:
screamer 0:66f3b5499f7f 79 import mbed_lstools
screamer 0:66f3b5499f7f 80 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:66f3b5499f7f 81 except:
screamer 0:66f3b5499f7f 82 pass
screamer 0:66f3b5499f7f 83
screamer 0:66f3b5499f7f 84
screamer 0:66f3b5499f7f 85 class ProcessObserver(Thread):
screamer 0:66f3b5499f7f 86 def __init__(self, proc):
screamer 0:66f3b5499f7f 87 Thread.__init__(self)
screamer 0:66f3b5499f7f 88 self.proc = proc
screamer 0:66f3b5499f7f 89 self.queue = Queue()
screamer 0:66f3b5499f7f 90 self.daemon = True
screamer 0:66f3b5499f7f 91 self.active = True
screamer 0:66f3b5499f7f 92 self.start()
screamer 0:66f3b5499f7f 93
screamer 0:66f3b5499f7f 94 def run(self):
screamer 0:66f3b5499f7f 95 while self.active:
screamer 0:66f3b5499f7f 96 c = self.proc.stdout.read(1)
screamer 0:66f3b5499f7f 97 self.queue.put(c)
screamer 0:66f3b5499f7f 98
screamer 0:66f3b5499f7f 99 def stop(self):
screamer 0:66f3b5499f7f 100 self.active = False
screamer 0:66f3b5499f7f 101 try:
screamer 0:66f3b5499f7f 102 self.proc.terminate()
screamer 0:66f3b5499f7f 103 except Exception, _:
screamer 0:66f3b5499f7f 104 pass
screamer 0:66f3b5499f7f 105
screamer 0:66f3b5499f7f 106
screamer 0:66f3b5499f7f 107 class SingleTestExecutor(threading.Thread):
screamer 0:66f3b5499f7f 108 """ Example: Single test class in separate thread usage
screamer 0:66f3b5499f7f 109 """
screamer 0:66f3b5499f7f 110 def __init__(self, single_test):
screamer 0:66f3b5499f7f 111 self.single_test = single_test
screamer 0:66f3b5499f7f 112 threading.Thread.__init__(self)
screamer 0:66f3b5499f7f 113
screamer 0:66f3b5499f7f 114 def run(self):
screamer 0:66f3b5499f7f 115 start = time()
screamer 0:66f3b5499f7f 116 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 117 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer 0:66f3b5499f7f 118 elapsed_time = time() - start
screamer 0:66f3b5499f7f 119
screamer 0:66f3b5499f7f 120 # Human readable summary
screamer 0:66f3b5499f7f 121 if not self.single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 122 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 123 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 124 if self.single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 125 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 126 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 127 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 128 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 129
screamer 0:66f3b5499f7f 130
screamer 0:66f3b5499f7f 131 class SingleTestRunner(object):
screamer 0:66f3b5499f7f 132 """ Object wrapper for single test run which may involve multiple MUTs
screamer 0:66f3b5499f7f 133 """
screamer 0:66f3b5499f7f 134 RE_DETECT_TESTCASE_RESULT = None
screamer 0:66f3b5499f7f 135
screamer 0:66f3b5499f7f 136 # Return codes for test script
screamer 0:66f3b5499f7f 137 TEST_RESULT_OK = "OK"
screamer 0:66f3b5499f7f 138 TEST_RESULT_FAIL = "FAIL"
screamer 0:66f3b5499f7f 139 TEST_RESULT_ERROR = "ERROR"
screamer 0:66f3b5499f7f 140 TEST_RESULT_UNDEF = "UNDEF"
screamer 0:66f3b5499f7f 141 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer 0:66f3b5499f7f 142 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer 0:66f3b5499f7f 143 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer 0:66f3b5499f7f 144 TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer 0:66f3b5499f7f 145 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer 0:66f3b5499f7f 146 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer 0:66f3b5499f7f 147 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer 0:66f3b5499f7f 148 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer 0:66f3b5499f7f 149
screamer 0:66f3b5499f7f 150 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer 0:66f3b5499f7f 151 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer 0:66f3b5499f7f 152 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer 0:66f3b5499f7f 153
screamer 0:66f3b5499f7f 154 muts = {} # MUTs descriptor (from external file)
screamer 0:66f3b5499f7f 155 test_spec = {} # Test specification (from external file)
screamer 0:66f3b5499f7f 156
screamer 0:66f3b5499f7f 157 # mbed test suite -> SingleTestRunner
screamer 0:66f3b5499f7f 158 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer 0:66f3b5499f7f 159 "failure" : TEST_RESULT_FAIL,
screamer 0:66f3b5499f7f 160 "error" : TEST_RESULT_ERROR,
screamer 0:66f3b5499f7f 161 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer 0:66f3b5499f7f 162 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer 0:66f3b5499f7f 163 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer 0:66f3b5499f7f 164 "timeout" : TEST_RESULT_TIMEOUT,
screamer 0:66f3b5499f7f 165 "no_image" : TEST_RESULT_NO_IMAGE,
screamer 0:66f3b5499f7f 166 "end" : TEST_RESULT_UNDEF,
screamer 0:66f3b5499f7f 167 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer 0:66f3b5499f7f 168 "build_failed" : TEST_RESULT_BUILD_FAILED,
screamer 0:66f3b5499f7f 169 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 170 }
screamer 0:66f3b5499f7f 171
screamer 0:66f3b5499f7f 172 def __init__(self,
screamer 0:66f3b5499f7f 173 _global_loops_count=1,
screamer 0:66f3b5499f7f 174 _test_loops_list=None,
screamer 0:66f3b5499f7f 175 _muts={},
screamer 0:66f3b5499f7f 176 _clean=False,
The Other Jimmy 31:8ea194f6145b 177 _parser=None,
The Other Jimmy 31:8ea194f6145b 178 _opts=None,
screamer 0:66f3b5499f7f 179 _opts_db_url=None,
screamer 0:66f3b5499f7f 180 _opts_log_file_name=None,
screamer 0:66f3b5499f7f 181 _opts_report_html_file_name=None,
screamer 0:66f3b5499f7f 182 _opts_report_junit_file_name=None,
screamer 0:66f3b5499f7f 183 _opts_report_build_file_name=None,
screamer 22:9e85236d8716 184 _opts_report_text_file_name=None,
screamer 0:66f3b5499f7f 185 _opts_build_report={},
screamer 0:66f3b5499f7f 186 _opts_build_properties={},
screamer 0:66f3b5499f7f 187 _test_spec={},
screamer 0:66f3b5499f7f 188 _opts_goanna_for_mbed_sdk=None,
screamer 0:66f3b5499f7f 189 _opts_goanna_for_tests=None,
screamer 0:66f3b5499f7f 190 _opts_shuffle_test_order=False,
screamer 0:66f3b5499f7f 191 _opts_shuffle_test_seed=None,
screamer 0:66f3b5499f7f 192 _opts_test_by_names=None,
screamer 0:66f3b5499f7f 193 _opts_peripheral_by_names=None,
screamer 0:66f3b5499f7f 194 _opts_test_only_peripheral=False,
screamer 0:66f3b5499f7f 195 _opts_test_only_common=False,
screamer 0:66f3b5499f7f 196 _opts_verbose_skipped_tests=False,
screamer 0:66f3b5499f7f 197 _opts_verbose_test_result_only=False,
screamer 0:66f3b5499f7f 198 _opts_verbose=False,
screamer 0:66f3b5499f7f 199 _opts_firmware_global_name=None,
screamer 0:66f3b5499f7f 200 _opts_only_build_tests=False,
screamer 0:66f3b5499f7f 201 _opts_parallel_test_exec=False,
screamer 0:66f3b5499f7f 202 _opts_suppress_summary=False,
screamer 0:66f3b5499f7f 203 _opts_test_x_toolchain_summary=False,
screamer 0:66f3b5499f7f 204 _opts_copy_method=None,
screamer 0:66f3b5499f7f 205 _opts_mut_reset_type=None,
screamer 0:66f3b5499f7f 206 _opts_jobs=None,
screamer 0:66f3b5499f7f 207 _opts_waterfall_test=None,
screamer 0:66f3b5499f7f 208 _opts_consolidate_waterfall_test=None,
screamer 0:66f3b5499f7f 209 _opts_extend_test_timeout=None,
screamer 0:66f3b5499f7f 210 _opts_auto_detect=None,
screamer 0:66f3b5499f7f 211 _opts_include_non_automated=False):
screamer 0:66f3b5499f7f 212 """ Let's try hard to init this object
screamer 0:66f3b5499f7f 213 """
screamer 0:66f3b5499f7f 214 from colorama import init
screamer 0:66f3b5499f7f 215 init()
screamer 0:66f3b5499f7f 216
screamer 0:66f3b5499f7f 217 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer 0:66f3b5499f7f 218 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer 0:66f3b5499f7f 219 # Settings related to test loops counters
screamer 0:66f3b5499f7f 220 try:
screamer 0:66f3b5499f7f 221 _global_loops_count = int(_global_loops_count)
screamer 0:66f3b5499f7f 222 except:
screamer 0:66f3b5499f7f 223 _global_loops_count = 1
screamer 0:66f3b5499f7f 224 if _global_loops_count < 1:
screamer 0:66f3b5499f7f 225 _global_loops_count = 1
screamer 0:66f3b5499f7f 226 self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer 0:66f3b5499f7f 227 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer 0:66f3b5499f7f 228 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer 0:66f3b5499f7f 229
screamer 0:66f3b5499f7f 230 self.shuffle_random_seed = 0.0
screamer 0:66f3b5499f7f 231 self.SHUFFLE_SEED_ROUND = 10
screamer 0:66f3b5499f7f 232
screamer 0:66f3b5499f7f 233 # MUT list and test specification storage
screamer 0:66f3b5499f7f 234 self.muts = _muts
screamer 0:66f3b5499f7f 235 self.test_spec = _test_spec
screamer 0:66f3b5499f7f 236
screamer 0:66f3b5499f7f 237 # Settings passed e.g. from command line
screamer 0:66f3b5499f7f 238 self.opts_db_url = _opts_db_url
screamer 0:66f3b5499f7f 239 self.opts_log_file_name = _opts_log_file_name
screamer 0:66f3b5499f7f 240 self.opts_report_html_file_name = _opts_report_html_file_name
screamer 0:66f3b5499f7f 241 self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer 0:66f3b5499f7f 242 self.opts_report_build_file_name = _opts_report_build_file_name
screamer 22:9e85236d8716 243 self.opts_report_text_file_name = _opts_report_text_file_name
screamer 0:66f3b5499f7f 244 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer 0:66f3b5499f7f 245 self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer 0:66f3b5499f7f 246 self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer 0:66f3b5499f7f 247 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer 0:66f3b5499f7f 248 self.opts_test_by_names = _opts_test_by_names
screamer 0:66f3b5499f7f 249 self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer 0:66f3b5499f7f 250 self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer 0:66f3b5499f7f 251 self.opts_test_only_common = _opts_test_only_common
screamer 0:66f3b5499f7f 252 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer 0:66f3b5499f7f 253 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer 0:66f3b5499f7f 254 self.opts_verbose = _opts_verbose
screamer 0:66f3b5499f7f 255 self.opts_firmware_global_name = _opts_firmware_global_name
screamer 0:66f3b5499f7f 256 self.opts_only_build_tests = _opts_only_build_tests
screamer 0:66f3b5499f7f 257 self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer 0:66f3b5499f7f 258 self.opts_suppress_summary = _opts_suppress_summary
screamer 0:66f3b5499f7f 259 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer 0:66f3b5499f7f 260 self.opts_copy_method = _opts_copy_method
screamer 0:66f3b5499f7f 261 self.opts_mut_reset_type = _opts_mut_reset_type
screamer 0:66f3b5499f7f 262 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer 0:66f3b5499f7f 263 self.opts_waterfall_test = _opts_waterfall_test
screamer 0:66f3b5499f7f 264 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer 0:66f3b5499f7f 265 self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer 0:66f3b5499f7f 266 self.opts_clean = _clean
The Other Jimmy 31:8ea194f6145b 267 self.opts_parser = _parser
The Other Jimmy 31:8ea194f6145b 268 self.opts = _opts
screamer 0:66f3b5499f7f 269 self.opts_auto_detect = _opts_auto_detect
screamer 0:66f3b5499f7f 270 self.opts_include_non_automated = _opts_include_non_automated
screamer 0:66f3b5499f7f 271
screamer 0:66f3b5499f7f 272 self.build_report = _opts_build_report
screamer 0:66f3b5499f7f 273 self.build_properties = _opts_build_properties
screamer 0:66f3b5499f7f 274
screamer 0:66f3b5499f7f 275 # File / screen logger initialization
screamer 0:66f3b5499f7f 276 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer 0:66f3b5499f7f 277
screamer 0:66f3b5499f7f 278 # Database related initializations
screamer 0:66f3b5499f7f 279 self.db_logger = factory_db_logger(self.opts_db_url)
screamer 0:66f3b5499f7f 280 self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer 0:66f3b5499f7f 281 # Let's connect to database to set up credentials and confirm database is ready
screamer 0:66f3b5499f7f 282 if self.db_logger:
screamer 0:66f3b5499f7f 283 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer 0:66f3b5499f7f 284 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 285 # Get hostname and uname so we can use it as build description
screamer 0:66f3b5499f7f 286 # when creating new build_id in external database
screamer 0:66f3b5499f7f 287 (_hostname, _uname) = self.db_logger.get_hostname()
screamer 0:66f3b5499f7f 288 _host_location = os.path.dirname(os.path.abspath(__file__))
screamer 0:66f3b5499f7f 289 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer 0:66f3b5499f7f 290 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer 0:66f3b5499f7f 291 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 292
screamer 0:66f3b5499f7f 293 def dump_options(self):
screamer 0:66f3b5499f7f 294 """ Function returns data structure with common settings passed to SingelTestRunner
screamer 0:66f3b5499f7f 295 It can be used for example to fill _extra fields in database storing test suite single run data
screamer 0:66f3b5499f7f 296 Example:
screamer 0:66f3b5499f7f 297 data = self.dump_options()
screamer 0:66f3b5499f7f 298 or
screamer 0:66f3b5499f7f 299 data_str = json.dumps(self.dump_options())
screamer 0:66f3b5499f7f 300 """
screamer 0:66f3b5499f7f 301 result = {"db_url" : str(self.opts_db_url),
screamer 0:66f3b5499f7f 302 "log_file_name" : str(self.opts_log_file_name),
screamer 0:66f3b5499f7f 303 "shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer 0:66f3b5499f7f 304 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer 0:66f3b5499f7f 305 "test_by_names" : str(self.opts_test_by_names),
screamer 0:66f3b5499f7f 306 "peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer 0:66f3b5499f7f 307 "test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer 0:66f3b5499f7f 308 "test_only_common" : str(self.opts_test_only_common),
screamer 0:66f3b5499f7f 309 "verbose" : str(self.opts_verbose),
screamer 0:66f3b5499f7f 310 "firmware_global_name" : str(self.opts_firmware_global_name),
screamer 0:66f3b5499f7f 311 "only_build_tests" : str(self.opts_only_build_tests),
screamer 0:66f3b5499f7f 312 "copy_method" : str(self.opts_copy_method),
screamer 0:66f3b5499f7f 313 "mut_reset_type" : str(self.opts_mut_reset_type),
screamer 0:66f3b5499f7f 314 "jobs" : str(self.opts_jobs),
screamer 0:66f3b5499f7f 315 "extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer 0:66f3b5499f7f 316 "_dummy" : ''
screamer 0:66f3b5499f7f 317 }
screamer 0:66f3b5499f7f 318 return result
screamer 0:66f3b5499f7f 319
screamer 0:66f3b5499f7f 320 def shuffle_random_func(self):
screamer 0:66f3b5499f7f 321 return self.shuffle_random_seed
screamer 0:66f3b5499f7f 322
screamer 0:66f3b5499f7f 323 def is_shuffle_seed_float(self):
screamer 0:66f3b5499f7f 324 """ return true if function parameter can be converted to float
screamer 0:66f3b5499f7f 325 """
screamer 0:66f3b5499f7f 326 result = True
screamer 0:66f3b5499f7f 327 try:
screamer 0:66f3b5499f7f 328 float(self.shuffle_random_seed)
screamer 0:66f3b5499f7f 329 except ValueError:
screamer 0:66f3b5499f7f 330 result = False
screamer 0:66f3b5499f7f 331 return result
screamer 0:66f3b5499f7f 332
screamer 0:66f3b5499f7f 333 # This will store target / toolchain specific properties
screamer 0:66f3b5499f7f 334 test_suite_properties_ext = {} # target : toolchain
screamer 0:66f3b5499f7f 335 # Here we store test results
screamer 0:66f3b5499f7f 336 test_summary = []
screamer 0:66f3b5499f7f 337 # Here we store test results in extended data structure
screamer 0:66f3b5499f7f 338 test_summary_ext = {}
screamer 0:66f3b5499f7f 339 execute_thread_slice_lock = Lock()
screamer 0:66f3b5499f7f 340
screamer 0:66f3b5499f7f 341 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer 0:66f3b5499f7f 342 for toolchain in toolchains:
screamer 0:66f3b5499f7f 343 tt_id = "%s::%s" % (toolchain, target)
screamer 0:66f3b5499f7f 344
screamer 0:66f3b5499f7f 345 T = TARGET_MAP[target]
screamer 0:66f3b5499f7f 346
screamer 0:66f3b5499f7f 347 # print target, toolchain
screamer 0:66f3b5499f7f 348 # Test suite properties returned to external tools like CI
screamer 0:66f3b5499f7f 349 test_suite_properties = {
screamer 0:66f3b5499f7f 350 'jobs': self.opts_jobs,
screamer 0:66f3b5499f7f 351 'clean': clean,
screamer 0:66f3b5499f7f 352 'target': target,
screamer 0:66f3b5499f7f 353 'vendor': T.extra_labels[0],
screamer 0:66f3b5499f7f 354 'test_ids': ', '.join(test_ids),
screamer 0:66f3b5499f7f 355 'toolchain': toolchain,
screamer 0:66f3b5499f7f 356 'shuffle_random_seed': self.shuffle_random_seed
screamer 0:66f3b5499f7f 357 }
screamer 0:66f3b5499f7f 358
screamer 0:66f3b5499f7f 359
screamer 0:66f3b5499f7f 360 # print '=== %s::%s ===' % (target, toolchain)
screamer 0:66f3b5499f7f 361 # Let's build our test
screamer 0:66f3b5499f7f 362 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 363 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
screamer 0:66f3b5499f7f 364 continue
screamer 0:66f3b5499f7f 365
screamer 0:66f3b5499f7f 366 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 367
The Other Jimmy 31:8ea194f6145b 368 profile = extract_profile(self.opts_parser, self.opts, toolchain)
theotherjimmy 40:7d3fa6b99b2b 369 stats_depth = self.opts.stats_depth or 2
The Other Jimmy 31:8ea194f6145b 370
screamer 0:66f3b5499f7f 371
screamer 0:66f3b5499f7f 372 try:
screamer 0:66f3b5499f7f 373 build_mbed_libs_result = build_mbed_libs(T,
screamer 0:66f3b5499f7f 374 toolchain,
screamer 0:66f3b5499f7f 375 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 376 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 377 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 378 report=build_report,
The Other Jimmy 31:8ea194f6145b 379 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 380 build_profile=profile)
screamer 0:66f3b5499f7f 381
screamer 0:66f3b5499f7f 382 if not build_mbed_libs_result:
screamer 0:66f3b5499f7f 383 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
screamer 0:66f3b5499f7f 384 continue
screamer 0:66f3b5499f7f 385
screamer 0:66f3b5499f7f 386 except ToolException:
screamer 0:66f3b5499f7f 387 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
screamer 0:66f3b5499f7f 388 continue
screamer 0:66f3b5499f7f 389
screamer 0:66f3b5499f7f 390 build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer 0:66f3b5499f7f 391
screamer 0:66f3b5499f7f 392 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer 0:66f3b5499f7f 393 test_suite_properties['build_dir'] = build_dir
screamer 0:66f3b5499f7f 394 test_suite_properties['skipped'] = []
screamer 0:66f3b5499f7f 395
screamer 0:66f3b5499f7f 396 # Enumerate through all tests and shuffle test order if requested
screamer 0:66f3b5499f7f 397 test_map_keys = sorted(TEST_MAP.keys())
screamer 0:66f3b5499f7f 398
screamer 0:66f3b5499f7f 399 if self.opts_shuffle_test_order:
screamer 0:66f3b5499f7f 400 random.shuffle(test_map_keys, self.shuffle_random_func)
screamer 0:66f3b5499f7f 401 # Update database with shuffle seed f applicable
screamer 0:66f3b5499f7f 402 if self.db_logger:
screamer 0:66f3b5499f7f 403 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 404 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 405 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
screamer 0:66f3b5499f7f 406 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 407
screamer 0:66f3b5499f7f 408 if self.db_logger:
screamer 0:66f3b5499f7f 409 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 410 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 411 # Update MUTs and Test Specification in database
screamer 0:66f3b5499f7f 412 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
screamer 0:66f3b5499f7f 413 # Update Extra information in database (some options passed to test suite)
screamer 0:66f3b5499f7f 414 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
screamer 0:66f3b5499f7f 415 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 416
screamer 0:66f3b5499f7f 417 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer 0:66f3b5499f7f 418 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer 0:66f3b5499f7f 419
screamer 0:66f3b5499f7f 420 for skipped_test_id in skipped_test_map_keys:
screamer 0:66f3b5499f7f 421 test_suite_properties['skipped'].append(skipped_test_id)
screamer 0:66f3b5499f7f 422
screamer 0:66f3b5499f7f 423
screamer 0:66f3b5499f7f 424 # First pass through all tests and determine which libraries need to be built
screamer 0:66f3b5499f7f 425 libraries = []
screamer 0:66f3b5499f7f 426 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 427 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 428
screamer 0:66f3b5499f7f 429 # Detect which lib should be added to test
screamer 0:66f3b5499f7f 430 # Some libs have to compiled like RTOS or ETH
screamer 0:66f3b5499f7f 431 for lib in LIBRARIES:
screamer 0:66f3b5499f7f 432 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer 0:66f3b5499f7f 433 libraries.append(lib['id'])
screamer 0:66f3b5499f7f 434
screamer 0:66f3b5499f7f 435
screamer 0:66f3b5499f7f 436 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 437
screamer 0:66f3b5499f7f 438 # Build all required libraries
screamer 0:66f3b5499f7f 439 for lib_id in libraries:
screamer 0:66f3b5499f7f 440 try:
screamer 0:66f3b5499f7f 441 build_lib(lib_id,
screamer 0:66f3b5499f7f 442 T,
screamer 0:66f3b5499f7f 443 toolchain,
screamer 0:66f3b5499f7f 444 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 445 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 446 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 447 report=build_report,
The Other Jimmy 31:8ea194f6145b 448 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 449 build_profile=profile)
screamer 0:66f3b5499f7f 450
screamer 0:66f3b5499f7f 451 except ToolException:
screamer 0:66f3b5499f7f 452 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
screamer 0:66f3b5499f7f 453 continue
screamer 0:66f3b5499f7f 454
screamer 0:66f3b5499f7f 455
screamer 0:66f3b5499f7f 456 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 457 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 458
screamer 0:66f3b5499f7f 459 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer 0:66f3b5499f7f 460
screamer 0:66f3b5499f7f 461 # TODO: move this 2 below loops to separate function
screamer 0:66f3b5499f7f 462 INC_DIRS = []
screamer 0:66f3b5499f7f 463 for lib_id in libraries:
screamer 0:66f3b5499f7f 464 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer 0:66f3b5499f7f 465 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer 0:66f3b5499f7f 466
screamer 0:66f3b5499f7f 467 MACROS = []
screamer 0:66f3b5499f7f 468 for lib_id in libraries:
screamer 0:66f3b5499f7f 469 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer 0:66f3b5499f7f 470 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer 0:66f3b5499f7f 471 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer 0:66f3b5499f7f 472 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer 0:66f3b5499f7f 473 test_uuid = uuid.uuid4()
screamer 0:66f3b5499f7f 474 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer 0:66f3b5499f7f 475
screamer 0:66f3b5499f7f 476 # Prepare extended test results data structure (it can be used to generate detailed test report)
screamer 0:66f3b5499f7f 477 if target not in self.test_summary_ext:
screamer 0:66f3b5499f7f 478 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer 0:66f3b5499f7f 479 if toolchain not in self.test_summary_ext[target]:
screamer 0:66f3b5499f7f 480 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer 0:66f3b5499f7f 481
screamer 0:66f3b5499f7f 482 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer 0:66f3b5499f7f 483
screamer 0:66f3b5499f7f 484 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer 0:66f3b5499f7f 485 try:
theotherjimmy 40:7d3fa6b99b2b 486 path = build_project(test.source_dir, join(build_dir, test_id), T,
theotherjimmy 40:7d3fa6b99b2b 487 toolchain, test.dependencies, clean=clean_project_options,
theotherjimmy 40:7d3fa6b99b2b 488 verbose=self.opts_verbose, name=project_name, macros=MACROS,
theotherjimmy 40:7d3fa6b99b2b 489 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
theotherjimmy 40:7d3fa6b99b2b 490 properties=build_properties, project_id=test_id,
theotherjimmy 40:7d3fa6b99b2b 491 project_description=test.get_description(),
theotherjimmy 40:7d3fa6b99b2b 492 build_profile=profile, stats_depth=stats_depth)
screamer 0:66f3b5499f7f 493
screamer 0:66f3b5499f7f 494 except Exception, e:
screamer 0:66f3b5499f7f 495 project_name_str = project_name if project_name is not None else test_id
screamer 0:66f3b5499f7f 496
screamer 0:66f3b5499f7f 497
screamer 0:66f3b5499f7f 498 test_result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 499
screamer 0:66f3b5499f7f 500 if isinstance(e, ToolException):
screamer 0:66f3b5499f7f 501 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
screamer 0:66f3b5499f7f 502 test_result = self.TEST_RESULT_BUILD_FAILED
screamer 0:66f3b5499f7f 503 elif isinstance(e, NotSupportedException):
screamer 0:66f3b5499f7f 504 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
screamer 0:66f3b5499f7f 505 test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 506
screamer 0:66f3b5499f7f 507
screamer 0:66f3b5499f7f 508 # Append test results to global test summary
screamer 0:66f3b5499f7f 509 self.test_summary.append(
screamer 0:66f3b5499f7f 510 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
screamer 0:66f3b5499f7f 511 )
screamer 0:66f3b5499f7f 512
screamer 0:66f3b5499f7f 513 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 514 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 515 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 516
screamer 0:66f3b5499f7f 517 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer 0:66f3b5499f7f 518 'result' : test_result,
screamer 0:66f3b5499f7f 519 'output' : '',
screamer 0:66f3b5499f7f 520 'target_name' : target,
screamer 0:66f3b5499f7f 521 'target_name_unique': target,
screamer 0:66f3b5499f7f 522 'toolchain_name' : toolchain,
screamer 0:66f3b5499f7f 523 'id' : test_id,
screamer 0:66f3b5499f7f 524 'description' : test.get_description(),
screamer 0:66f3b5499f7f 525 'elapsed_time' : 0,
screamer 0:66f3b5499f7f 526 'duration' : 0,
screamer 0:66f3b5499f7f 527 'copy_method' : None
screamer 0:66f3b5499f7f 528 }})
screamer 0:66f3b5499f7f 529 continue
screamer 0:66f3b5499f7f 530
screamer 0:66f3b5499f7f 531 if self.opts_only_build_tests:
screamer 0:66f3b5499f7f 532 # With this option we are skipping testing phase
screamer 0:66f3b5499f7f 533 continue
screamer 0:66f3b5499f7f 534
screamer 0:66f3b5499f7f 535 # Test duration can be increased by global value
screamer 0:66f3b5499f7f 536 test_duration = test.duration
screamer 0:66f3b5499f7f 537 if self.opts_extend_test_timeout is not None:
screamer 0:66f3b5499f7f 538 test_duration += self.opts_extend_test_timeout
screamer 0:66f3b5499f7f 539
screamer 0:66f3b5499f7f 540 # For an automated test the duration act as a timeout after
screamer 0:66f3b5499f7f 541 # which the test gets interrupted
screamer 0:66f3b5499f7f 542 test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer 0:66f3b5499f7f 543 test_loops = self.get_test_loop_count(test_id)
screamer 0:66f3b5499f7f 544
screamer 0:66f3b5499f7f 545 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer 0:66f3b5499f7f 546 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer 0:66f3b5499f7f 547 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer 0:66f3b5499f7f 548
screamer 0:66f3b5499f7f 549 # read MUTs, test specification and perform tests
screamer 0:66f3b5499f7f 550 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer 0:66f3b5499f7f 551
screamer 0:66f3b5499f7f 552 if handle_results is None:
screamer 0:66f3b5499f7f 553 continue
screamer 0:66f3b5499f7f 554
screamer 0:66f3b5499f7f 555 for handle_result in handle_results:
screamer 0:66f3b5499f7f 556 if handle_result:
screamer 0:66f3b5499f7f 557 single_test_result, detailed_test_results = handle_result
screamer 0:66f3b5499f7f 558 else:
screamer 0:66f3b5499f7f 559 continue
screamer 0:66f3b5499f7f 560
screamer 0:66f3b5499f7f 561 # Append test results to global test summary
screamer 0:66f3b5499f7f 562 if single_test_result is not None:
screamer 0:66f3b5499f7f 563 self.test_summary.append(single_test_result)
screamer 0:66f3b5499f7f 564
screamer 0:66f3b5499f7f 565 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 566 if target not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 567 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 568 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 569
screamer 0:66f3b5499f7f 570 append_test_result = detailed_test_results
screamer 0:66f3b5499f7f 571
screamer 0:66f3b5499f7f 572 # If waterfall and consolidate-waterfall options are enabled,
screamer 0:66f3b5499f7f 573 # only include the last test result in the report.
screamer 0:66f3b5499f7f 574 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer 0:66f3b5499f7f 575 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer 0:66f3b5499f7f 576
screamer 0:66f3b5499f7f 577 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer 0:66f3b5499f7f 578
screamer 0:66f3b5499f7f 579 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer 0:66f3b5499f7f 580 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer 0:66f3b5499f7f 581
screamer 0:66f3b5499f7f 582 q.put(target + '_'.join(toolchains))
screamer 0:66f3b5499f7f 583 return
screamer 0:66f3b5499f7f 584
screamer 0:66f3b5499f7f 585 def execute(self):
screamer 0:66f3b5499f7f 586 clean = self.test_spec.get('clean', False)
screamer 0:66f3b5499f7f 587 test_ids = self.test_spec.get('test_ids', [])
screamer 0:66f3b5499f7f 588 q = Queue()
screamer 0:66f3b5499f7f 589
screamer 0:66f3b5499f7f 590 # Generate seed for shuffle if seed is not provided in
screamer 0:66f3b5499f7f 591 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 592 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer 0:66f3b5499f7f 593 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 594
screamer 0:66f3b5499f7f 595
screamer 0:66f3b5499f7f 596 if self.opts_parallel_test_exec:
screamer 0:66f3b5499f7f 597 ###################################################################
screamer 0:66f3b5499f7f 598 # Experimental, parallel test execution per singletest instance.
screamer 0:66f3b5499f7f 599 ###################################################################
screamer 0:66f3b5499f7f 600 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer 0:66f3b5499f7f 601 # Note: We are building here in parallel for each target separately!
screamer 0:66f3b5499f7f 602 # So we are not building the same thing multiple times and compilers
screamer 0:66f3b5499f7f 603 # in separate threads do not collide.
screamer 0:66f3b5499f7f 604 # Inside execute_thread_slice() function function handle() will be called to
screamer 0:66f3b5499f7f 605 # get information about available MUTs (per target).
screamer 0:66f3b5499f7f 606 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 607 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 608 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer 0:66f3b5499f7f 609 t.daemon = True
screamer 0:66f3b5499f7f 610 t.start()
screamer 0:66f3b5499f7f 611 execute_threads.append(t)
screamer 0:66f3b5499f7f 612
screamer 0:66f3b5499f7f 613 for t in execute_threads:
screamer 0:66f3b5499f7f 614 q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer 0:66f3b5499f7f 615 else:
screamer 0:66f3b5499f7f 616 # Serialized (not parallel) test execution
screamer 0:66f3b5499f7f 617 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 618 if target not in self.test_suite_properties_ext:
screamer 0:66f3b5499f7f 619 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 620
screamer 0:66f3b5499f7f 621 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer 0:66f3b5499f7f 622 q.get()
screamer 0:66f3b5499f7f 623
screamer 0:66f3b5499f7f 624 if self.db_logger:
screamer 0:66f3b5499f7f 625 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 626 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 627 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer 0:66f3b5499f7f 628 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 629
screamer 0:66f3b5499f7f 630 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer 0:66f3b5499f7f 631
screamer 0:66f3b5499f7f 632 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer 0:66f3b5499f7f 633 valid_test_map_keys = []
screamer 0:66f3b5499f7f 634
screamer 0:66f3b5499f7f 635 for test_id in test_map_keys:
screamer 0:66f3b5499f7f 636 test = TEST_MAP[test_id]
screamer 22:9e85236d8716 637 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
screamer 0:66f3b5499f7f 638 continue
screamer 0:66f3b5499f7f 639
screamer 0:66f3b5499f7f 640 if test_ids and test_id not in test_ids:
screamer 0:66f3b5499f7f 641 continue
screamer 0:66f3b5499f7f 642
screamer 0:66f3b5499f7f 643 if self.opts_test_only_peripheral and not test.peripherals:
screamer 0:66f3b5499f7f 644 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 645 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 646 continue
screamer 0:66f3b5499f7f 647
screamer 22:9e85236d8716 648 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
screamer 0:66f3b5499f7f 649 # We will skip tests not forced with -p option
screamer 0:66f3b5499f7f 650 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 651 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 652 continue
screamer 0:66f3b5499f7f 653
screamer 0:66f3b5499f7f 654 if self.opts_test_only_common and test.peripherals:
screamer 0:66f3b5499f7f 655 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 656 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 657 continue
screamer 0:66f3b5499f7f 658
screamer 0:66f3b5499f7f 659 if not include_non_automated and not test.automated:
screamer 0:66f3b5499f7f 660 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 661 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 662 continue
screamer 0:66f3b5499f7f 663
screamer 0:66f3b5499f7f 664 if test.is_supported(target, toolchain):
screamer 0:66f3b5499f7f 665 if test.peripherals is None and self.opts_only_build_tests:
screamer 0:66f3b5499f7f 666 # When users are using 'build only flag' and test do not have
screamer 0:66f3b5499f7f 667 # specified peripherals we can allow test building by default
screamer 0:66f3b5499f7f 668 pass
screamer 22:9e85236d8716 669 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
screamer 0:66f3b5499f7f 670 # If we force peripheral with option -p we expect test
screamer 0:66f3b5499f7f 671 # to pass even if peripheral is not in MUTs file.
screamer 0:66f3b5499f7f 672 pass
screamer 0:66f3b5499f7f 673 elif not self.is_peripherals_available(target, test.peripherals):
screamer 0:66f3b5499f7f 674 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 675 if test.peripherals:
screamer 0:66f3b5499f7f 676 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
screamer 0:66f3b5499f7f 677 else:
screamer 0:66f3b5499f7f 678 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
screamer 0:66f3b5499f7f 679 continue
screamer 0:66f3b5499f7f 680
screamer 0:66f3b5499f7f 681 # The test has made it through all the filters, so add it to the valid tests list
screamer 0:66f3b5499f7f 682 valid_test_map_keys.append(test_id)
screamer 0:66f3b5499f7f 683
screamer 0:66f3b5499f7f 684 return valid_test_map_keys
screamer 0:66f3b5499f7f 685
screamer 0:66f3b5499f7f 686 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer 0:66f3b5499f7f 687 # NOTE: This will not preserve order
screamer 0:66f3b5499f7f 688 return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer 0:66f3b5499f7f 689
screamer 0:66f3b5499f7f 690 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 691 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 692 table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 693 """
screamer 0:66f3b5499f7f 694 RESULT_INDEX = 0
screamer 0:66f3b5499f7f 695 TARGET_INDEX = 1
screamer 0:66f3b5499f7f 696 TOOLCHAIN_INDEX = 2
screamer 0:66f3b5499f7f 697 TEST_INDEX = 3
screamer 0:66f3b5499f7f 698 DESC_INDEX = 4
screamer 0:66f3b5499f7f 699
screamer 0:66f3b5499f7f 700 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer 0:66f3b5499f7f 701 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer 0:66f3b5499f7f 702 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer 0:66f3b5499f7f 703 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer 0:66f3b5499f7f 704
screamer 0:66f3b5499f7f 705 result = "Test summary:\n"
screamer 0:66f3b5499f7f 706 for target in unique_targets:
screamer 0:66f3b5499f7f 707 result_dict = {} # test : { toolchain : result }
screamer 0:66f3b5499f7f 708 unique_target_toolchains = []
screamer 0:66f3b5499f7f 709 for test in test_summary:
screamer 0:66f3b5499f7f 710 if test[TARGET_INDEX] == target:
screamer 0:66f3b5499f7f 711 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer 0:66f3b5499f7f 712 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer 0:66f3b5499f7f 713 if test[TEST_INDEX] not in result_dict:
screamer 0:66f3b5499f7f 714 result_dict[test[TEST_INDEX]] = {}
screamer 0:66f3b5499f7f 715 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer 0:66f3b5499f7f 716
screamer 0:66f3b5499f7f 717 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
screamer 0:66f3b5499f7f 718 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 719 for col in pt_cols:
screamer 0:66f3b5499f7f 720 pt.align[col] = "l"
screamer 0:66f3b5499f7f 721 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 722
screamer 0:66f3b5499f7f 723 for test in unique_tests:
screamer 0:66f3b5499f7f 724 if test in result_dict:
screamer 0:66f3b5499f7f 725 test_results = result_dict[test]
screamer 0:66f3b5499f7f 726 if test in unique_test_desc:
screamer 0:66f3b5499f7f 727 row = [target, test, unique_test_desc[test]]
screamer 0:66f3b5499f7f 728 for toolchain in unique_toolchains:
screamer 0:66f3b5499f7f 729 if toolchain in test_results:
screamer 0:66f3b5499f7f 730 row.append(test_results[toolchain])
screamer 0:66f3b5499f7f 731 pt.add_row(row)
screamer 0:66f3b5499f7f 732 result += pt.get_string()
screamer 0:66f3b5499f7f 733 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 734 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 735 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 736 return result
screamer 0:66f3b5499f7f 737
screamer 0:66f3b5499f7f 738 def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 739 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 740 table shows target x test results matrix across
screamer 0:66f3b5499f7f 741 """
screamer 0:66f3b5499f7f 742 success_code = 0 # Success code that can be leter returned to
screamer 0:66f3b5499f7f 743 result = "Test summary:\n"
screamer 0:66f3b5499f7f 744 # Pretty table package is used to print results
screamer 0:66f3b5499f7f 745 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
screamer 0:66f3b5499f7f 746 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
screamer 0:66f3b5499f7f 747 pt.align["Result"] = "l" # Left align
screamer 0:66f3b5499f7f 748 pt.align["Target"] = "l" # Left align
screamer 0:66f3b5499f7f 749 pt.align["Toolchain"] = "l" # Left align
screamer 0:66f3b5499f7f 750 pt.align["Test ID"] = "l" # Left align
screamer 0:66f3b5499f7f 751 pt.align["Test Description"] = "l" # Left align
screamer 0:66f3b5499f7f 752 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 753
screamer 0:66f3b5499f7f 754 result_dict = {self.TEST_RESULT_OK : 0,
screamer 0:66f3b5499f7f 755 self.TEST_RESULT_FAIL : 0,
screamer 0:66f3b5499f7f 756 self.TEST_RESULT_ERROR : 0,
screamer 0:66f3b5499f7f 757 self.TEST_RESULT_UNDEF : 0,
screamer 0:66f3b5499f7f 758 self.TEST_RESULT_IOERR_COPY : 0,
screamer 0:66f3b5499f7f 759 self.TEST_RESULT_IOERR_DISK : 0,
screamer 0:66f3b5499f7f 760 self.TEST_RESULT_IOERR_SERIAL : 0,
screamer 0:66f3b5499f7f 761 self.TEST_RESULT_NO_IMAGE : 0,
screamer 0:66f3b5499f7f 762 self.TEST_RESULT_TIMEOUT : 0,
screamer 0:66f3b5499f7f 763 self.TEST_RESULT_MBED_ASSERT : 0,
screamer 0:66f3b5499f7f 764 self.TEST_RESULT_BUILD_FAILED : 0,
screamer 0:66f3b5499f7f 765 self.TEST_RESULT_NOT_SUPPORTED : 0
screamer 0:66f3b5499f7f 766 }
screamer 0:66f3b5499f7f 767
screamer 0:66f3b5499f7f 768 for test in test_summary:
screamer 0:66f3b5499f7f 769 if test[0] in result_dict:
screamer 0:66f3b5499f7f 770 result_dict[test[0]] += 1
screamer 0:66f3b5499f7f 771 pt.add_row(test)
screamer 0:66f3b5499f7f 772 result += pt.get_string()
screamer 0:66f3b5499f7f 773 result += "\n"
screamer 0:66f3b5499f7f 774
screamer 0:66f3b5499f7f 775 # Print result count
screamer 0:66f3b5499f7f 776 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
screamer 0:66f3b5499f7f 777 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 778 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 779 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 780 return result
screamer 0:66f3b5499f7f 781
screamer 0:66f3b5499f7f 782 def test_loop_list_to_dict(self, test_loops_str):
screamer 0:66f3b5499f7f 783 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer 0:66f3b5499f7f 784 """
screamer 0:66f3b5499f7f 785 result = {}
screamer 0:66f3b5499f7f 786 if test_loops_str:
screamer 22:9e85236d8716 787 test_loops = test_loops_str
screamer 0:66f3b5499f7f 788 for test_loop in test_loops:
screamer 0:66f3b5499f7f 789 test_loop_count = test_loop.split('=')
screamer 0:66f3b5499f7f 790 if len(test_loop_count) == 2:
screamer 0:66f3b5499f7f 791 _test_id, _test_loops = test_loop_count
screamer 0:66f3b5499f7f 792 try:
screamer 0:66f3b5499f7f 793 _test_loops = int(_test_loops)
screamer 0:66f3b5499f7f 794 except:
screamer 0:66f3b5499f7f 795 continue
screamer 0:66f3b5499f7f 796 result[_test_id] = _test_loops
screamer 0:66f3b5499f7f 797 return result
screamer 0:66f3b5499f7f 798
screamer 0:66f3b5499f7f 799 def get_test_loop_count(self, test_id):
screamer 0:66f3b5499f7f 800 """ This function returns no. of loops per test (deducted by test_id_.
screamer 0:66f3b5499f7f 801 If test is not in list of redefined loop counts it will use default value.
screamer 0:66f3b5499f7f 802 """
screamer 0:66f3b5499f7f 803 result = self.GLOBAL_LOOPS_COUNT
screamer 0:66f3b5499f7f 804 if test_id in self.TEST_LOOPS_DICT:
screamer 0:66f3b5499f7f 805 result = self.TEST_LOOPS_DICT[test_id]
screamer 0:66f3b5499f7f 806 return result
screamer 0:66f3b5499f7f 807
screamer 0:66f3b5499f7f 808 def delete_file(self, file_path):
screamer 0:66f3b5499f7f 809 """ Remove file from the system
screamer 0:66f3b5499f7f 810 """
screamer 0:66f3b5499f7f 811 result = True
screamer 0:66f3b5499f7f 812 resutl_msg = ""
screamer 0:66f3b5499f7f 813 try:
screamer 0:66f3b5499f7f 814 os.remove(file_path)
screamer 0:66f3b5499f7f 815 except Exception, e:
screamer 0:66f3b5499f7f 816 resutl_msg = e
screamer 0:66f3b5499f7f 817 result = False
screamer 0:66f3b5499f7f 818 return result, resutl_msg
screamer 0:66f3b5499f7f 819
screamer 0:66f3b5499f7f 820 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 821 """ Test is being invoked for given MUT.
screamer 0:66f3b5499f7f 822 """
screamer 0:66f3b5499f7f 823 # Get test information, image and test timeout
screamer 0:66f3b5499f7f 824 test_id = data['test_id']
screamer 0:66f3b5499f7f 825 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 826 test_description = TEST_MAP[test_id].get_description()
screamer 0:66f3b5499f7f 827 image = data["image"]
screamer 0:66f3b5499f7f 828 duration = data.get("duration", 10)
screamer 0:66f3b5499f7f 829
screamer 0:66f3b5499f7f 830 if mut is None:
screamer 0:66f3b5499f7f 831 print "Error: No Mbed available: MUT[%s]" % data['mcu']
screamer 0:66f3b5499f7f 832 return None
screamer 0:66f3b5499f7f 833
screamer 0:66f3b5499f7f 834 mcu = mut['mcu']
screamer 0:66f3b5499f7f 835 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer 0:66f3b5499f7f 836
screamer 0:66f3b5499f7f 837 if self.db_logger:
screamer 0:66f3b5499f7f 838 self.db_logger.reconnect()
screamer 0:66f3b5499f7f 839
screamer 0:66f3b5499f7f 840 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer 0:66f3b5499f7f 841
screamer 0:66f3b5499f7f 842 # Tests can be looped so test results must be stored for the same test
screamer 0:66f3b5499f7f 843 test_all_result = []
screamer 0:66f3b5499f7f 844 # Test results for one test ran few times
screamer 0:66f3b5499f7f 845 detailed_test_results = {} # { Loop_number: { results ... } }
screamer 0:66f3b5499f7f 846
screamer 0:66f3b5499f7f 847 for test_index in range(test_loops):
screamer 0:66f3b5499f7f 848
screamer 0:66f3b5499f7f 849 # If mbedls is available and we are auto detecting MUT info,
screamer 0:66f3b5499f7f 850 # update MUT info (mounting may changed)
screamer 0:66f3b5499f7f 851 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 852 platform_name_filter = [mcu]
screamer 0:66f3b5499f7f 853 muts_list = {}
screamer 0:66f3b5499f7f 854 found = False
screamer 0:66f3b5499f7f 855
screamer 0:66f3b5499f7f 856 for i in range(0, 60):
screamer 0:66f3b5499f7f 857 print('Looking for %s with MBEDLS' % mcu)
screamer 0:66f3b5499f7f 858 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 859
screamer 0:66f3b5499f7f 860 if 1 not in muts_list:
screamer 0:66f3b5499f7f 861 sleep(3)
screamer 0:66f3b5499f7f 862 else:
screamer 0:66f3b5499f7f 863 found = True
screamer 0:66f3b5499f7f 864 break
screamer 0:66f3b5499f7f 865
screamer 0:66f3b5499f7f 866 if not found:
screamer 0:66f3b5499f7f 867 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
screamer 0:66f3b5499f7f 868 return None
screamer 0:66f3b5499f7f 869 else:
screamer 0:66f3b5499f7f 870 mut = muts_list[1]
screamer 0:66f3b5499f7f 871
screamer 0:66f3b5499f7f 872 disk = mut.get('disk')
screamer 0:66f3b5499f7f 873 port = mut.get('port')
screamer 0:66f3b5499f7f 874
screamer 0:66f3b5499f7f 875 if disk is None or port is None:
screamer 0:66f3b5499f7f 876 return None
screamer 0:66f3b5499f7f 877
screamer 0:66f3b5499f7f 878 target_by_mcu = TARGET_MAP[mut['mcu']]
screamer 0:66f3b5499f7f 879 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer 0:66f3b5499f7f 880 # Some extra stuff can be declared in MUTs structure
screamer 0:66f3b5499f7f 881 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer 0:66f3b5499f7f 882 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer 0:66f3b5499f7f 883
screamer 0:66f3b5499f7f 884 # When the build and test system were separate, this was relative to a
screamer 0:66f3b5499f7f 885 # base network folder base path: join(NETWORK_BASE_PATH, )
screamer 0:66f3b5499f7f 886 image_path = image
screamer 0:66f3b5499f7f 887
screamer 0:66f3b5499f7f 888 # Host test execution
screamer 0:66f3b5499f7f 889 start_host_exec_time = time()
screamer 0:66f3b5499f7f 890
screamer 0:66f3b5499f7f 891 single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer 0:66f3b5499f7f 892 _copy_method = selected_copy_method
screamer 0:66f3b5499f7f 893
screamer 0:66f3b5499f7f 894 if not exists(image_path):
screamer 0:66f3b5499f7f 895 single_test_result = self.TEST_RESULT_NO_IMAGE
screamer 0:66f3b5499f7f 896 elapsed_time = 0
screamer 0:66f3b5499f7f 897 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
screamer 0:66f3b5499f7f 898 print single_test_output
screamer 0:66f3b5499f7f 899 else:
screamer 0:66f3b5499f7f 900 # Host test execution
screamer 0:66f3b5499f7f 901 start_host_exec_time = time()
screamer 0:66f3b5499f7f 902
screamer 0:66f3b5499f7f 903 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer 0:66f3b5499f7f 904 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer 0:66f3b5499f7f 905 host_test_result = self.run_host_test(test.host_test,
screamer 0:66f3b5499f7f 906 image_path, disk, port, duration,
screamer 0:66f3b5499f7f 907 micro=target_name,
screamer 0:66f3b5499f7f 908 verbose=host_test_verbose,
screamer 0:66f3b5499f7f 909 reset=host_test_reset,
screamer 0:66f3b5499f7f 910 reset_tout=reset_tout,
screamer 0:66f3b5499f7f 911 copy_method=selected_copy_method,
screamer 13:ab47a20b66f0 912 program_cycle_s=target_by_mcu.program_cycle_s)
screamer 0:66f3b5499f7f 913 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer 0:66f3b5499f7f 914
screamer 0:66f3b5499f7f 915 # Store test result
screamer 0:66f3b5499f7f 916 test_all_result.append(single_test_result)
screamer 0:66f3b5499f7f 917 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer 0:66f3b5499f7f 918 elapsed_time = single_testduration # TIme of single test case execution after reset
screamer 0:66f3b5499f7f 919
screamer 0:66f3b5499f7f 920 detailed_test_results[test_index] = {
screamer 0:66f3b5499f7f 921 'result' : single_test_result,
screamer 0:66f3b5499f7f 922 'output' : single_test_output,
screamer 0:66f3b5499f7f 923 'target_name' : target_name,
screamer 0:66f3b5499f7f 924 'target_name_unique' : target_name_unique,
screamer 0:66f3b5499f7f 925 'toolchain_name' : toolchain_name,
screamer 0:66f3b5499f7f 926 'id' : test_id,
screamer 0:66f3b5499f7f 927 'description' : test_description,
screamer 0:66f3b5499f7f 928 'elapsed_time' : round(elapsed_time, 2),
screamer 0:66f3b5499f7f 929 'duration' : single_timeout,
screamer 0:66f3b5499f7f 930 'copy_method' : _copy_method,
screamer 0:66f3b5499f7f 931 }
screamer 0:66f3b5499f7f 932
screamer 0:66f3b5499f7f 933 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
screamer 0:66f3b5499f7f 934 test_id, test_description, elapsed_time, single_timeout)
screamer 0:66f3b5499f7f 935
screamer 0:66f3b5499f7f 936 # Update database entries for ongoing test
screamer 0:66f3b5499f7f 937 if self.db_logger and self.db_logger.is_connected():
screamer 0:66f3b5499f7f 938 test_type = 'SingleTest'
screamer 0:66f3b5499f7f 939 self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer 0:66f3b5499f7f 940 target_name,
screamer 0:66f3b5499f7f 941 toolchain_name,
screamer 0:66f3b5499f7f 942 test_type,
screamer 0:66f3b5499f7f 943 test_id,
screamer 0:66f3b5499f7f 944 single_test_result,
screamer 0:66f3b5499f7f 945 single_test_output,
screamer 0:66f3b5499f7f 946 elapsed_time,
screamer 0:66f3b5499f7f 947 single_timeout,
screamer 0:66f3b5499f7f 948 test_index)
screamer 0:66f3b5499f7f 949
screamer 0:66f3b5499f7f 950 # If we perform waterfall test we test until we get OK and we stop testing
screamer 0:66f3b5499f7f 951 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer 0:66f3b5499f7f 952 break
screamer 0:66f3b5499f7f 953
screamer 0:66f3b5499f7f 954 if self.db_logger:
screamer 0:66f3b5499f7f 955 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 956
screamer 0:66f3b5499f7f 957 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer 0:66f3b5499f7f 958 target_name_unique,
screamer 0:66f3b5499f7f 959 toolchain_name,
screamer 0:66f3b5499f7f 960 test_id,
screamer 0:66f3b5499f7f 961 test_description,
screamer 0:66f3b5499f7f 962 round(elapsed_time, 2),
screamer 0:66f3b5499f7f 963 single_timeout,
screamer 0:66f3b5499f7f 964 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer 0:66f3b5499f7f 965
screamer 0:66f3b5499f7f 966 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 967 """ Function determines MUT's mbed disk/port and copies binary to
screamer 0:66f3b5499f7f 968 target.
screamer 0:66f3b5499f7f 969 """
screamer 0:66f3b5499f7f 970 handle_results = []
screamer 0:66f3b5499f7f 971 data = json.loads(test_spec)
screamer 0:66f3b5499f7f 972
screamer 0:66f3b5499f7f 973 # Find a suitable MUT:
screamer 0:66f3b5499f7f 974 mut = None
screamer 0:66f3b5499f7f 975 for id, m in self.muts.iteritems():
screamer 0:66f3b5499f7f 976 if m['mcu'] == data['mcu']:
screamer 0:66f3b5499f7f 977 mut = m
screamer 0:66f3b5499f7f 978 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer 0:66f3b5499f7f 979 handle_results.append(handle_result)
screamer 0:66f3b5499f7f 980
screamer 0:66f3b5499f7f 981 return handle_results
screamer 0:66f3b5499f7f 982
screamer 0:66f3b5499f7f 983 def print_test_result(self, test_result, target_name, toolchain_name,
screamer 0:66f3b5499f7f 984 test_id, test_description, elapsed_time, duration):
screamer 0:66f3b5499f7f 985 """ Use specific convention to print test result and related data
screamer 0:66f3b5499f7f 986 """
screamer 0:66f3b5499f7f 987 tokens = []
screamer 0:66f3b5499f7f 988 tokens.append("TargetTest")
screamer 0:66f3b5499f7f 989 tokens.append(target_name)
screamer 0:66f3b5499f7f 990 tokens.append(toolchain_name)
screamer 0:66f3b5499f7f 991 tokens.append(test_id)
screamer 0:66f3b5499f7f 992 tokens.append(test_description)
screamer 0:66f3b5499f7f 993 separator = "::"
screamer 0:66f3b5499f7f 994 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer 0:66f3b5499f7f 995 result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer 0:66f3b5499f7f 996 return Fore.MAGENTA + result + Fore.RESET
screamer 0:66f3b5499f7f 997
screamer 0:66f3b5499f7f 998 def shape_test_loop_ok_result_count(self, test_all_result):
screamer 0:66f3b5499f7f 999 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1000 """
screamer 0:66f3b5499f7f 1001 test_loop_count = len(test_all_result)
screamer 0:66f3b5499f7f 1002 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer 0:66f3b5499f7f 1003 return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer 0:66f3b5499f7f 1004
screamer 0:66f3b5499f7f 1005 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer 0:66f3b5499f7f 1006 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1007 """
screamer 0:66f3b5499f7f 1008 result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 1009
screamer 0:66f3b5499f7f 1010 if all(test_all_result[0] == res for res in test_all_result):
screamer 0:66f3b5499f7f 1011 result = test_all_result[0]
screamer 0:66f3b5499f7f 1012 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer 0:66f3b5499f7f 1013 result = self.TEST_RESULT_OK
screamer 0:66f3b5499f7f 1014
screamer 0:66f3b5499f7f 1015 return result
screamer 0:66f3b5499f7f 1016
screamer 0:66f3b5499f7f 1017 def run_host_test(self, name, image_path, disk, port, duration,
screamer 0:66f3b5499f7f 1018 micro=None, reset=None, reset_tout=None,
screamer 0:66f3b5499f7f 1019 verbose=False, copy_method=None, program_cycle_s=None):
screamer 0:66f3b5499f7f 1020 """ Function creates new process with host test configured with particular test case.
screamer 0:66f3b5499f7f 1021 Function also is pooling for serial port activity from process to catch all data
screamer 0:66f3b5499f7f 1022 printed by test runner and host test during test execution
screamer 0:66f3b5499f7f 1023 """
screamer 0:66f3b5499f7f 1024
screamer 0:66f3b5499f7f 1025 def get_char_from_queue(obs):
screamer 0:66f3b5499f7f 1026 """ Get character from queue safe way
screamer 0:66f3b5499f7f 1027 """
screamer 0:66f3b5499f7f 1028 try:
screamer 0:66f3b5499f7f 1029 c = obs.queue.get(block=True, timeout=0.5)
screamer 0:66f3b5499f7f 1030 except Empty, _:
screamer 0:66f3b5499f7f 1031 c = None
screamer 0:66f3b5499f7f 1032 return c
screamer 0:66f3b5499f7f 1033
screamer 0:66f3b5499f7f 1034 def filter_queue_char(c):
screamer 0:66f3b5499f7f 1035 """ Filters out non ASCII characters from serial port
screamer 0:66f3b5499f7f 1036 """
screamer 0:66f3b5499f7f 1037 if ord(c) not in range(128):
screamer 0:66f3b5499f7f 1038 c = ' '
screamer 0:66f3b5499f7f 1039 return c
screamer 0:66f3b5499f7f 1040
screamer 0:66f3b5499f7f 1041 def get_test_result(output):
screamer 0:66f3b5499f7f 1042 """ Parse test 'output' data
screamer 0:66f3b5499f7f 1043 """
screamer 0:66f3b5499f7f 1044 result = self.TEST_RESULT_TIMEOUT
screamer 0:66f3b5499f7f 1045 for line in "".join(output).splitlines():
screamer 0:66f3b5499f7f 1046 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer 0:66f3b5499f7f 1047 if search_result and len(search_result.groups()):
screamer 0:66f3b5499f7f 1048 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer 0:66f3b5499f7f 1049 break
screamer 0:66f3b5499f7f 1050 return result
screamer 0:66f3b5499f7f 1051
screamer 0:66f3b5499f7f 1052 def get_auto_property_value(property_name, line):
screamer 0:66f3b5499f7f 1053 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer 0:66f3b5499f7f 1054 Returns string
screamer 0:66f3b5499f7f 1055 """
screamer 0:66f3b5499f7f 1056 result = None
screamer 0:66f3b5499f7f 1057 if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer 0:66f3b5499f7f 1058 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer 0:66f3b5499f7f 1059 if property is not None and len(property.groups()) == 1:
screamer 0:66f3b5499f7f 1060 result = property.groups()[0]
screamer 0:66f3b5499f7f 1061 return result
screamer 0:66f3b5499f7f 1062
screamer 0:66f3b5499f7f 1063 # print "{%s} port:%s disk:%s" % (name, port, disk),
screamer 0:66f3b5499f7f 1064 cmd = ["python",
screamer 0:66f3b5499f7f 1065 '%s.py'% name,
screamer 0:66f3b5499f7f 1066 '-d', disk,
screamer 0:66f3b5499f7f 1067 '-f', '"%s"'% image_path,
screamer 0:66f3b5499f7f 1068 '-p', port,
screamer 0:66f3b5499f7f 1069 '-t', str(duration),
screamer 0:66f3b5499f7f 1070 '-C', str(program_cycle_s)]
screamer 0:66f3b5499f7f 1071
screamer 0:66f3b5499f7f 1072 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 1073 cmd += ['--auto']
screamer 0:66f3b5499f7f 1074
screamer 0:66f3b5499f7f 1075 # Add extra parameters to host_test
screamer 0:66f3b5499f7f 1076 if copy_method is not None:
screamer 0:66f3b5499f7f 1077 cmd += ["-c", copy_method]
screamer 0:66f3b5499f7f 1078 if micro is not None:
screamer 0:66f3b5499f7f 1079 cmd += ["-m", micro]
screamer 0:66f3b5499f7f 1080 if reset is not None:
screamer 0:66f3b5499f7f 1081 cmd += ["-r", reset]
screamer 0:66f3b5499f7f 1082 if reset_tout is not None:
screamer 0:66f3b5499f7f 1083 cmd += ["-R", str(reset_tout)]
screamer 0:66f3b5499f7f 1084
screamer 0:66f3b5499f7f 1085 if verbose:
screamer 0:66f3b5499f7f 1086 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
screamer 0:66f3b5499f7f 1087 print "Test::Output::Start"
screamer 0:66f3b5499f7f 1088
screamer 0:66f3b5499f7f 1089 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer 0:66f3b5499f7f 1090 obs = ProcessObserver(proc)
screamer 0:66f3b5499f7f 1091 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer 0:66f3b5499f7f 1092 line = ''
screamer 0:66f3b5499f7f 1093 output = []
screamer 0:66f3b5499f7f 1094 start_time = time()
screamer 0:66f3b5499f7f 1095 while (time() - start_time) < (2 * duration):
screamer 0:66f3b5499f7f 1096 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1097 if c:
screamer 0:66f3b5499f7f 1098 if verbose:
screamer 0:66f3b5499f7f 1099 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1100 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1101 output.append(c)
screamer 0:66f3b5499f7f 1102 # Give the mbed under test a way to communicate the end of the test
screamer 0:66f3b5499f7f 1103 if c in ['\n', '\r']:
screamer 0:66f3b5499f7f 1104
screamer 0:66f3b5499f7f 1105 # Checking for auto-detection information from the test about MUT reset moment
screamer 0:66f3b5499f7f 1106 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer 0:66f3b5499f7f 1107 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1108 update_once_flag['reset_target'] = True
screamer 0:66f3b5499f7f 1109 start_time = time()
screamer 0:66f3b5499f7f 1110
screamer 0:66f3b5499f7f 1111 # Checking for auto-detection information from the test about timeout
screamer 0:66f3b5499f7f 1112 auto_timeout_val = get_auto_property_value('timeout', line)
screamer 0:66f3b5499f7f 1113 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer 0:66f3b5499f7f 1114 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1115 update_once_flag['timeout'] = True
screamer 0:66f3b5499f7f 1116 duration = int(auto_timeout_val)
screamer 0:66f3b5499f7f 1117
screamer 0:66f3b5499f7f 1118 # Detect mbed assert:
screamer 0:66f3b5499f7f 1119 if 'mbed assertation failed: ' in line:
screamer 0:66f3b5499f7f 1120 output.append('{{mbed_assert}}')
screamer 0:66f3b5499f7f 1121 break
screamer 0:66f3b5499f7f 1122
screamer 0:66f3b5499f7f 1123 # Check for test end
screamer 0:66f3b5499f7f 1124 if '{end}' in line:
screamer 0:66f3b5499f7f 1125 break
screamer 0:66f3b5499f7f 1126 line = ''
screamer 0:66f3b5499f7f 1127 else:
screamer 0:66f3b5499f7f 1128 line += c
screamer 0:66f3b5499f7f 1129 end_time = time()
screamer 0:66f3b5499f7f 1130 testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer 0:66f3b5499f7f 1131
screamer 0:66f3b5499f7f 1132 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1133
screamer 0:66f3b5499f7f 1134 if c:
screamer 0:66f3b5499f7f 1135 if verbose:
screamer 0:66f3b5499f7f 1136 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1137 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1138 output.append(c)
screamer 0:66f3b5499f7f 1139
screamer 0:66f3b5499f7f 1140 if verbose:
screamer 0:66f3b5499f7f 1141 print "Test::Output::Finish"
screamer 0:66f3b5499f7f 1142 # Stop test process
screamer 0:66f3b5499f7f 1143 obs.stop()
screamer 0:66f3b5499f7f 1144
screamer 0:66f3b5499f7f 1145 result = get_test_result(output)
screamer 0:66f3b5499f7f 1146 return (result, "".join(output), testcase_duration, duration)
screamer 0:66f3b5499f7f 1147
screamer 0:66f3b5499f7f 1148 def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer 0:66f3b5499f7f 1149 """ Checks if specified target should run specific peripheral test case defined in MUTs file
screamer 0:66f3b5499f7f 1150 """
screamer 0:66f3b5499f7f 1151 if peripherals is not None:
screamer 0:66f3b5499f7f 1152 peripherals = set(peripherals)
screamer 0:66f3b5499f7f 1153 for id, mut in self.muts.iteritems():
screamer 0:66f3b5499f7f 1154 # Target MCU name check
screamer 0:66f3b5499f7f 1155 if mut["mcu"] != target_mcu_name:
screamer 0:66f3b5499f7f 1156 continue
screamer 0:66f3b5499f7f 1157 # Peripherals check
screamer 0:66f3b5499f7f 1158 if peripherals is not None:
screamer 0:66f3b5499f7f 1159 if 'peripherals' not in mut:
screamer 0:66f3b5499f7f 1160 continue
screamer 0:66f3b5499f7f 1161 if not peripherals.issubset(set(mut['peripherals'])):
screamer 0:66f3b5499f7f 1162 continue
screamer 0:66f3b5499f7f 1163 return True
screamer 0:66f3b5499f7f 1164 return False
screamer 0:66f3b5499f7f 1165
screamer 0:66f3b5499f7f 1166 def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer 0:66f3b5499f7f 1167 """ Function prepares JSON structure describing test specification
screamer 0:66f3b5499f7f 1168 """
screamer 0:66f3b5499f7f 1169 test_spec = {
screamer 0:66f3b5499f7f 1170 "mcu": mcu,
screamer 0:66f3b5499f7f 1171 "image": image_path,
screamer 0:66f3b5499f7f 1172 "duration": duration,
screamer 0:66f3b5499f7f 1173 "test_id": test_id,
screamer 0:66f3b5499f7f 1174 }
screamer 0:66f3b5499f7f 1175 return json.dumps(test_spec)
screamer 0:66f3b5499f7f 1176
screamer 0:66f3b5499f7f 1177
screamer 0:66f3b5499f7f 1178 def get_unique_value_from_summary(test_summary, index):
screamer 0:66f3b5499f7f 1179 """ Gets list of unique target names
screamer 0:66f3b5499f7f 1180 """
screamer 0:66f3b5499f7f 1181 result = []
screamer 0:66f3b5499f7f 1182 for test in test_summary:
screamer 0:66f3b5499f7f 1183 target_name = test[index]
screamer 0:66f3b5499f7f 1184 if target_name not in result:
screamer 0:66f3b5499f7f 1185 result.append(target_name)
screamer 0:66f3b5499f7f 1186 return sorted(result)
screamer 0:66f3b5499f7f 1187
screamer 0:66f3b5499f7f 1188
screamer 0:66f3b5499f7f 1189 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer 0:66f3b5499f7f 1190 """ Gets list of unique target names and return dictionary
screamer 0:66f3b5499f7f 1191 """
screamer 0:66f3b5499f7f 1192 result = {}
screamer 0:66f3b5499f7f 1193 for test in test_summary:
screamer 0:66f3b5499f7f 1194 key = test[index_key]
screamer 0:66f3b5499f7f 1195 val = test[index_val]
screamer 0:66f3b5499f7f 1196 if key not in result:
screamer 0:66f3b5499f7f 1197 result[key] = val
screamer 0:66f3b5499f7f 1198 return result
screamer 0:66f3b5499f7f 1199
screamer 0:66f3b5499f7f 1200
screamer 0:66f3b5499f7f 1201 def show_json_file_format_error(json_spec_filename, line, column):
screamer 0:66f3b5499f7f 1202 """ Prints JSON broken content
screamer 0:66f3b5499f7f 1203 """
screamer 0:66f3b5499f7f 1204 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1205 line_no = 1
screamer 0:66f3b5499f7f 1206 for json_line in data_file:
screamer 0:66f3b5499f7f 1207 if line_no + 5 >= line: # Print last few lines before error
screamer 0:66f3b5499f7f 1208 print 'Line %d:\t'%line_no + json_line, # Prints line
screamer 0:66f3b5499f7f 1209 if line_no == line:
screamer 0:66f3b5499f7f 1210 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
screamer 0:66f3b5499f7f 1211 break
screamer 0:66f3b5499f7f 1212 line_no += 1
screamer 0:66f3b5499f7f 1213
screamer 0:66f3b5499f7f 1214
screamer 0:66f3b5499f7f 1215 def json_format_error_defect_pos(json_error_msg):
screamer 0:66f3b5499f7f 1216 """ Gets first error line and column in JSON file format.
screamer 0:66f3b5499f7f 1217 Parsed from exception thrown by json.loads() string
screamer 0:66f3b5499f7f 1218 """
screamer 0:66f3b5499f7f 1219 result = None
screamer 0:66f3b5499f7f 1220 line, column = 0, 0
screamer 0:66f3b5499f7f 1221 # Line value search
screamer 0:66f3b5499f7f 1222 line_search = re.search('line [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1223 if line_search is not None:
screamer 0:66f3b5499f7f 1224 ls = line_search.group().split(' ')
screamer 0:66f3b5499f7f 1225 if len(ls) == 2:
screamer 0:66f3b5499f7f 1226 line = int(ls[1])
screamer 0:66f3b5499f7f 1227 # Column position search
screamer 0:66f3b5499f7f 1228 column_search = re.search('column [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1229 if column_search is not None:
screamer 0:66f3b5499f7f 1230 cs = column_search.group().split(' ')
screamer 0:66f3b5499f7f 1231 if len(cs) == 2:
screamer 0:66f3b5499f7f 1232 column = int(cs[1])
screamer 0:66f3b5499f7f 1233 result = [line, column]
screamer 0:66f3b5499f7f 1234 return result
screamer 0:66f3b5499f7f 1235
screamer 0:66f3b5499f7f 1236
screamer 0:66f3b5499f7f 1237 def get_json_data_from_file(json_spec_filename, verbose=False):
screamer 0:66f3b5499f7f 1238 """ Loads from file JSON formatted string to data structure
screamer 0:66f3b5499f7f 1239 """
screamer 0:66f3b5499f7f 1240 result = None
screamer 0:66f3b5499f7f 1241 try:
screamer 0:66f3b5499f7f 1242 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1243 try:
screamer 0:66f3b5499f7f 1244 result = json.load(data_file)
screamer 0:66f3b5499f7f 1245 except ValueError as json_error_msg:
screamer 0:66f3b5499f7f 1246 result = None
screamer 0:66f3b5499f7f 1247 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
screamer 0:66f3b5499f7f 1248 # We can print where error occurred inside JSON file if we can parse exception msg
screamer 0:66f3b5499f7f 1249 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer 0:66f3b5499f7f 1250 if json_format_defect_pos is not None:
screamer 0:66f3b5499f7f 1251 line = json_format_defect_pos[0]
screamer 0:66f3b5499f7f 1252 column = json_format_defect_pos[1]
screamer 0:66f3b5499f7f 1253 print
screamer 0:66f3b5499f7f 1254 show_json_file_format_error(json_spec_filename, line, column)
screamer 0:66f3b5499f7f 1255
screamer 0:66f3b5499f7f 1256 except IOError as fileopen_error_msg:
screamer 0:66f3b5499f7f 1257 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
screamer 0:66f3b5499f7f 1258 print
screamer 0:66f3b5499f7f 1259 if verbose and result:
screamer 0:66f3b5499f7f 1260 pp = pprint.PrettyPrinter(indent=4)
screamer 0:66f3b5499f7f 1261 pp.pprint(result)
screamer 0:66f3b5499f7f 1262 return result
screamer 0:66f3b5499f7f 1263
screamer 0:66f3b5499f7f 1264
screamer 0:66f3b5499f7f 1265 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer 0:66f3b5499f7f 1266 """ Prints MUTs configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1267 """
screamer 0:66f3b5499f7f 1268 muts_info_cols = []
screamer 0:66f3b5499f7f 1269 # We need to check all unique properties for each defined MUT
screamer 0:66f3b5499f7f 1270 for k in json_data:
screamer 0:66f3b5499f7f 1271 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1272 for mut_property in mut_info:
screamer 0:66f3b5499f7f 1273 if mut_property not in muts_info_cols:
screamer 0:66f3b5499f7f 1274 muts_info_cols.append(mut_property)
screamer 0:66f3b5499f7f 1275
screamer 0:66f3b5499f7f 1276 # Prepare pretty table object to display all MUTs
screamer 0:66f3b5499f7f 1277 pt_cols = ["index"] + muts_info_cols
screamer 0:66f3b5499f7f 1278 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1279 for col in pt_cols:
screamer 0:66f3b5499f7f 1280 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1281
screamer 0:66f3b5499f7f 1282 # Add rows to pretty print object
screamer 0:66f3b5499f7f 1283 for k in json_data:
screamer 0:66f3b5499f7f 1284 row = [k]
screamer 0:66f3b5499f7f 1285 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1286
screamer 0:66f3b5499f7f 1287 add_row = True
screamer 0:66f3b5499f7f 1288 if platform_filter and 'mcu' in mut_info:
screamer 0:66f3b5499f7f 1289 add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer 0:66f3b5499f7f 1290 if add_row:
screamer 0:66f3b5499f7f 1291 for col in muts_info_cols:
screamer 0:66f3b5499f7f 1292 cell_val = mut_info[col] if col in mut_info else None
screamer 0:66f3b5499f7f 1293 if type(cell_val) == ListType:
screamer 0:66f3b5499f7f 1294 cell_val = join_delim.join(cell_val)
screamer 0:66f3b5499f7f 1295 row.append(cell_val)
screamer 0:66f3b5499f7f 1296 pt.add_row(row)
screamer 0:66f3b5499f7f 1297 return pt.get_string()
screamer 0:66f3b5499f7f 1298
screamer 0:66f3b5499f7f 1299
screamer 0:66f3b5499f7f 1300 def print_test_configuration_from_json(json_data, join_delim=", "):
screamer 0:66f3b5499f7f 1301 """ Prints test specification configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1302 """
screamer 0:66f3b5499f7f 1303 toolchains_info_cols = []
screamer 0:66f3b5499f7f 1304 # We need to check all toolchains for each device
screamer 0:66f3b5499f7f 1305 for k in json_data:
screamer 0:66f3b5499f7f 1306 # k should be 'targets'
screamer 0:66f3b5499f7f 1307 targets = json_data[k]
screamer 0:66f3b5499f7f 1308 for target in targets:
screamer 0:66f3b5499f7f 1309 toolchains = targets[target]
screamer 0:66f3b5499f7f 1310 for toolchain in toolchains:
screamer 0:66f3b5499f7f 1311 if toolchain not in toolchains_info_cols:
screamer 0:66f3b5499f7f 1312 toolchains_info_cols.append(toolchain)
screamer 0:66f3b5499f7f 1313
screamer 0:66f3b5499f7f 1314 # Prepare pretty table object to display test specification
screamer 0:66f3b5499f7f 1315 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
screamer 0:66f3b5499f7f 1316 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1317 for col in pt_cols:
screamer 0:66f3b5499f7f 1318 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1319
screamer 0:66f3b5499f7f 1320 # { target : [conflicted toolchains] }
screamer 0:66f3b5499f7f 1321 toolchain_conflicts = {}
screamer 0:66f3b5499f7f 1322 toolchain_path_conflicts = []
screamer 0:66f3b5499f7f 1323 for k in json_data:
screamer 0:66f3b5499f7f 1324 # k should be 'targets'
screamer 0:66f3b5499f7f 1325 targets = json_data[k]
screamer 0:66f3b5499f7f 1326 for target in targets:
screamer 0:66f3b5499f7f 1327 target_supported_toolchains = get_target_supported_toolchains(target)
screamer 0:66f3b5499f7f 1328 if not target_supported_toolchains:
screamer 0:66f3b5499f7f 1329 target_supported_toolchains = []
screamer 0:66f3b5499f7f 1330 target_name = target if target in TARGET_MAP else "%s*"% target
screamer 0:66f3b5499f7f 1331 row = [target_name]
screamer 0:66f3b5499f7f 1332 toolchains = targets[target]
screamer 0:66f3b5499f7f 1333
screamer 0:66f3b5499f7f 1334 for toolchain in sorted(toolchains_info_cols):
screamer 0:66f3b5499f7f 1335 # Check for conflicts: target vs toolchain
screamer 0:66f3b5499f7f 1336 conflict = False
screamer 0:66f3b5499f7f 1337 conflict_path = False
screamer 0:66f3b5499f7f 1338 if toolchain in toolchains:
screamer 0:66f3b5499f7f 1339 if toolchain not in target_supported_toolchains:
screamer 0:66f3b5499f7f 1340 conflict = True
screamer 0:66f3b5499f7f 1341 if target not in toolchain_conflicts:
screamer 0:66f3b5499f7f 1342 toolchain_conflicts[target] = []
screamer 0:66f3b5499f7f 1343 toolchain_conflicts[target].append(toolchain)
screamer 0:66f3b5499f7f 1344 # Add marker inside table about target usage / conflict
screamer 0:66f3b5499f7f 1345 cell_val = 'Yes' if toolchain in toolchains else '-'
screamer 0:66f3b5499f7f 1346 if conflict:
screamer 0:66f3b5499f7f 1347 cell_val += '*'
screamer 0:66f3b5499f7f 1348 # Check for conflicts: toolchain vs toolchain path
screamer 20:835f6355470d 1349 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1350 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1351 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1352 conflict_path = True
screamer 0:66f3b5499f7f 1353 if toolchain not in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1354 toolchain_path_conflicts.append(toolchain)
screamer 0:66f3b5499f7f 1355 if conflict_path:
screamer 0:66f3b5499f7f 1356 cell_val += '#'
screamer 0:66f3b5499f7f 1357 row.append(cell_val)
screamer 0:66f3b5499f7f 1358 pt.add_row(row)
screamer 0:66f3b5499f7f 1359
screamer 0:66f3b5499f7f 1360 # generate result string
screamer 0:66f3b5499f7f 1361 result = pt.get_string() # Test specification table
screamer 0:66f3b5499f7f 1362 if toolchain_conflicts or toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1363 result += "\n"
screamer 0:66f3b5499f7f 1364 result += "Toolchain conflicts:\n"
screamer 0:66f3b5499f7f 1365 for target in toolchain_conflicts:
screamer 0:66f3b5499f7f 1366 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 1367 result += "\t* Target %s unknown\n"% (target)
screamer 0:66f3b5499f7f 1368 conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer 0:66f3b5499f7f 1369 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer 0:66f3b5499f7f 1370 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer 0:66f3b5499f7f 1371
screamer 0:66f3b5499f7f 1372 for toolchain in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1373 # Let's check toolchain configuration
screamer 20:835f6355470d 1374 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1375 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1376 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1377 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer 0:66f3b5499f7f 1378 return result
screamer 0:66f3b5499f7f 1379
screamer 0:66f3b5499f7f 1380
screamer 0:66f3b5499f7f 1381 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer 0:66f3b5499f7f 1382 """ Generates table summary with all test cases and additional test cases
screamer 0:66f3b5499f7f 1383 information using pretty print functionality. Allows test suite user to
screamer 0:66f3b5499f7f 1384 see test cases
screamer 0:66f3b5499f7f 1385 """
screamer 0:66f3b5499f7f 1386 # get all unique test ID prefixes
screamer 0:66f3b5499f7f 1387 unique_test_id = []
screamer 0:66f3b5499f7f 1388 for test in TESTS:
screamer 0:66f3b5499f7f 1389 split = test['id'].split('_')[:-1]
screamer 0:66f3b5499f7f 1390 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1391 if test_id_prefix not in unique_test_id:
screamer 0:66f3b5499f7f 1392 unique_test_id.append(test_id_prefix)
screamer 0:66f3b5499f7f 1393 unique_test_id.sort()
screamer 0:66f3b5499f7f 1394 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1395 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1396
screamer 0:66f3b5499f7f 1397 test_properties = ['id',
screamer 0:66f3b5499f7f 1398 'automated',
screamer 0:66f3b5499f7f 1399 'description',
screamer 0:66f3b5499f7f 1400 'peripherals',
screamer 0:66f3b5499f7f 1401 'host_test',
screamer 0:66f3b5499f7f 1402 'duration'] if cols is None else cols
screamer 0:66f3b5499f7f 1403
screamer 0:66f3b5499f7f 1404 # All tests status table print
screamer 0:66f3b5499f7f 1405 pt = PrettyTable(test_properties)
screamer 0:66f3b5499f7f 1406 for col in test_properties:
screamer 0:66f3b5499f7f 1407 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1408 pt.align['duration'] = "r"
screamer 0:66f3b5499f7f 1409
screamer 0:66f3b5499f7f 1410 counter_all = 0
screamer 0:66f3b5499f7f 1411 counter_automated = 0
screamer 0:66f3b5499f7f 1412 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 1413
screamer 0:66f3b5499f7f 1414 for test_id in sorted(TEST_MAP.keys()):
screamer 0:66f3b5499f7f 1415 if platform_filter is not None:
screamer 0:66f3b5499f7f 1416 # FIlter out platforms using regex
screamer 0:66f3b5499f7f 1417 if re.search(platform_filter, test_id) is None:
screamer 0:66f3b5499f7f 1418 continue
screamer 0:66f3b5499f7f 1419 row = []
screamer 0:66f3b5499f7f 1420 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 1421 split = test_id.split('_')[:-1]
screamer 0:66f3b5499f7f 1422 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1423
screamer 0:66f3b5499f7f 1424 for col in test_properties:
screamer 0:66f3b5499f7f 1425 col_value = test[col]
screamer 0:66f3b5499f7f 1426 if type(test[col]) == ListType:
screamer 0:66f3b5499f7f 1427 col_value = join_delim.join(test[col])
screamer 0:66f3b5499f7f 1428 elif test[col] == None:
screamer 0:66f3b5499f7f 1429 col_value = "-"
screamer 0:66f3b5499f7f 1430
screamer 0:66f3b5499f7f 1431 row.append(col_value)
screamer 0:66f3b5499f7f 1432 if test['automated'] == True:
screamer 0:66f3b5499f7f 1433 counter_dict_test_id_types[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1434 counter_automated += 1
screamer 0:66f3b5499f7f 1435 pt.add_row(row)
screamer 0:66f3b5499f7f 1436 # Update counters
screamer 0:66f3b5499f7f 1437 counter_all += 1
screamer 0:66f3b5499f7f 1438 counter_dict_test_id_types_all[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1439 result = pt.get_string()
screamer 0:66f3b5499f7f 1440 result += "\n\n"
screamer 0:66f3b5499f7f 1441
screamer 0:66f3b5499f7f 1442 if result_summary and not platform_filter:
screamer 0:66f3b5499f7f 1443 # Automation result summary
screamer 0:66f3b5499f7f 1444 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1445 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1446 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1447 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1448 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1449
screamer 0:66f3b5499f7f 1450 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer 0:66f3b5499f7f 1451 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1452 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer 0:66f3b5499f7f 1453 result += "Automation coverage:\n"
screamer 0:66f3b5499f7f 1454 result += pt.get_string()
screamer 0:66f3b5499f7f 1455 result += "\n\n"
screamer 0:66f3b5499f7f 1456
screamer 0:66f3b5499f7f 1457 # Test automation coverage table print
screamer 0:66f3b5499f7f 1458 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1459 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1460 pt.align['id'] = "l"
screamer 0:66f3b5499f7f 1461 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1462 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1463 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1464 for unique_id in unique_test_id:
screamer 0:66f3b5499f7f 1465 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer 0:66f3b5499f7f 1466 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer 0:66f3b5499f7f 1467 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1468 row = [unique_id,
screamer 0:66f3b5499f7f 1469 counter_dict_test_id_types[unique_id],
screamer 0:66f3b5499f7f 1470 counter_dict_test_id_types_all[unique_id],
screamer 0:66f3b5499f7f 1471 percent_progress,
screamer 0:66f3b5499f7f 1472 "[" + str_progress + "]"]
screamer 0:66f3b5499f7f 1473 pt.add_row(row)
screamer 0:66f3b5499f7f 1474 result += "Test automation coverage:\n"
screamer 0:66f3b5499f7f 1475 result += pt.get_string()
screamer 0:66f3b5499f7f 1476 result += "\n\n"
screamer 0:66f3b5499f7f 1477 return result
screamer 0:66f3b5499f7f 1478
screamer 0:66f3b5499f7f 1479
screamer 0:66f3b5499f7f 1480 def progress_bar(percent_progress, saturation=0):
screamer 0:66f3b5499f7f 1481 """ This function creates progress bar with optional simple saturation mark
screamer 0:66f3b5499f7f 1482 """
screamer 0:66f3b5499f7f 1483 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer 0:66f3b5499f7f 1484 str_progress = '#' * step + '.' * int(50 - step)
screamer 0:66f3b5499f7f 1485 c = '!' if str_progress[38] == '.' else '|'
screamer 0:66f3b5499f7f 1486 if saturation > 0:
screamer 0:66f3b5499f7f 1487 saturation = saturation / 2
screamer 0:66f3b5499f7f 1488 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer 0:66f3b5499f7f 1489 return str_progress
screamer 0:66f3b5499f7f 1490
screamer 0:66f3b5499f7f 1491
screamer 0:66f3b5499f7f 1492 def singletest_in_cli_mode(single_test):
screamer 0:66f3b5499f7f 1493 """ Runs SingleTestRunner object in CLI (Command line interface) mode
screamer 0:66f3b5499f7f 1494
screamer 0:66f3b5499f7f 1495 @return returns success code (0 == success) for building and running tests
screamer 0:66f3b5499f7f 1496 """
screamer 0:66f3b5499f7f 1497 start = time()
screamer 0:66f3b5499f7f 1498 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 1499 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer 0:66f3b5499f7f 1500 elapsed_time = time() - start
screamer 0:66f3b5499f7f 1501
screamer 0:66f3b5499f7f 1502 # Human readable summary
screamer 0:66f3b5499f7f 1503 if not single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 1504 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1505 print single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1506 if single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 1507 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1508 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 1509 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1510
screamer 0:66f3b5499f7f 1511 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 1512 print
screamer 0:66f3b5499f7f 1513 # Write summary of the builds
screamer 0:66f3b5499f7f 1514
screamer 0:66f3b5499f7f 1515 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 0:66f3b5499f7f 1516 status = print_report_exporter.report(build_report)
screamer 0:66f3b5499f7f 1517
screamer 0:66f3b5499f7f 1518 # Store extra reports in files
screamer 0:66f3b5499f7f 1519 if single_test.opts_report_html_file_name:
screamer 0:66f3b5499f7f 1520 # Export results in form of HTML report to separate file
screamer 0:66f3b5499f7f 1521 report_exporter = ReportExporter(ResultExporterType.HTML)
screamer 0:66f3b5499f7f 1522 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1523 if single_test.opts_report_junit_file_name:
screamer 0:66f3b5499f7f 1524 # Export results in form of JUnit XML report to separate file
screamer 0:66f3b5499f7f 1525 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 0:66f3b5499f7f 1526 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer 22:9e85236d8716 1527 if single_test.opts_report_text_file_name:
screamer 22:9e85236d8716 1528 # Export results in form of a text file
screamer 22:9e85236d8716 1529 report_exporter = ReportExporter(ResultExporterType.TEXT)
screamer 22:9e85236d8716 1530 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1531 if single_test.opts_report_build_file_name:
screamer 0:66f3b5499f7f 1532 # Export build results as html report to sparate file
screamer 0:66f3b5499f7f 1533 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 0:66f3b5499f7f 1534 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer 0:66f3b5499f7f 1535
screamer 0:66f3b5499f7f 1536 # Returns True if no build failures of the test projects or their dependencies
screamer 0:66f3b5499f7f 1537 return status
screamer 0:66f3b5499f7f 1538
screamer 0:66f3b5499f7f 1539 class TestLogger():
screamer 0:66f3b5499f7f 1540 """ Super-class for logging and printing ongoing events for test suite pass
screamer 0:66f3b5499f7f 1541 """
screamer 0:66f3b5499f7f 1542 def __init__(self, store_log=True):
screamer 0:66f3b5499f7f 1543 """ We can control if logger actually stores log in memory
screamer 0:66f3b5499f7f 1544 or just handled all log entries immediately
screamer 0:66f3b5499f7f 1545 """
screamer 0:66f3b5499f7f 1546 self.log = []
screamer 0:66f3b5499f7f 1547 self.log_to_file = False
screamer 0:66f3b5499f7f 1548 self.log_file_name = None
screamer 0:66f3b5499f7f 1549 self.store_log = store_log
screamer 0:66f3b5499f7f 1550
screamer 0:66f3b5499f7f 1551 self.LogType = construct_enum(INFO='Info',
screamer 0:66f3b5499f7f 1552 WARN='Warning',
screamer 0:66f3b5499f7f 1553 NOTIF='Notification',
screamer 0:66f3b5499f7f 1554 ERROR='Error',
screamer 0:66f3b5499f7f 1555 EXCEPT='Exception')
screamer 0:66f3b5499f7f 1556
screamer 0:66f3b5499f7f 1557 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer 0:66f3b5499f7f 1558 APPEND=2) # Append to existing log file
screamer 0:66f3b5499f7f 1559
screamer 0:66f3b5499f7f 1560 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1561 """ Log one line of text
screamer 0:66f3b5499f7f 1562 """
screamer 0:66f3b5499f7f 1563 log_timestamp = time()
screamer 0:66f3b5499f7f 1564 log_entry = {'log_type' : LogType,
screamer 0:66f3b5499f7f 1565 'log_timestamp' : log_timestamp,
screamer 0:66f3b5499f7f 1566 'log_line' : log_line,
screamer 0:66f3b5499f7f 1567 '_future' : None
screamer 0:66f3b5499f7f 1568 }
screamer 0:66f3b5499f7f 1569 # Store log in memory
screamer 0:66f3b5499f7f 1570 if self.store_log:
screamer 0:66f3b5499f7f 1571 self.log.append(log_entry)
screamer 0:66f3b5499f7f 1572 return log_entry
screamer 0:66f3b5499f7f 1573
screamer 0:66f3b5499f7f 1574
screamer 0:66f3b5499f7f 1575 class CLITestLogger(TestLogger):
screamer 0:66f3b5499f7f 1576 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer 0:66f3b5499f7f 1577 """
screamer 0:66f3b5499f7f 1578 def __init__(self, store_log=True, file_name=None):
screamer 0:66f3b5499f7f 1579 TestLogger.__init__(self)
screamer 0:66f3b5499f7f 1580 self.log_file_name = file_name
screamer 0:66f3b5499f7f 1581 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer 0:66f3b5499f7f 1582 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer 0:66f3b5499f7f 1583
screamer 0:66f3b5499f7f 1584 def log_print(self, log_entry, timestamp=True):
screamer 0:66f3b5499f7f 1585 """ Prints on screen formatted log entry
screamer 0:66f3b5499f7f 1586 """
screamer 0:66f3b5499f7f 1587 ts = log_entry['log_timestamp']
screamer 0:66f3b5499f7f 1588 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer 0:66f3b5499f7f 1589 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer 0:66f3b5499f7f 1590 return timestamp_str + log_line_str
screamer 0:66f3b5499f7f 1591
screamer 0:66f3b5499f7f 1592 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1593 """ Logs line, if log file output was specified log line will be appended
screamer 0:66f3b5499f7f 1594 at the end of log file
screamer 0:66f3b5499f7f 1595 """
screamer 0:66f3b5499f7f 1596 log_entry = TestLogger.log_line(self, LogType, log_line)
screamer 0:66f3b5499f7f 1597 log_line_str = self.log_print(log_entry, timestamp)
screamer 0:66f3b5499f7f 1598 if self.log_file_name is not None:
screamer 0:66f3b5499f7f 1599 try:
screamer 0:66f3b5499f7f 1600 with open(self.log_file_name, 'a') as f:
screamer 0:66f3b5499f7f 1601 f.write(log_line_str + line_delim)
screamer 0:66f3b5499f7f 1602 except IOError:
screamer 0:66f3b5499f7f 1603 pass
screamer 0:66f3b5499f7f 1604 return log_line_str
screamer 0:66f3b5499f7f 1605
screamer 0:66f3b5499f7f 1606
screamer 0:66f3b5499f7f 1607 def factory_db_logger(db_url):
screamer 0:66f3b5499f7f 1608 """ Factory database driver depending on database type supplied in database connection string db_url
screamer 0:66f3b5499f7f 1609 """
screamer 0:66f3b5499f7f 1610 if db_url is not None:
screamer 0:66f3b5499f7f 1611 from tools.test_mysql import MySQLDBAccess
screamer 0:66f3b5499f7f 1612 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1613 if connection_info is not None:
screamer 0:66f3b5499f7f 1614 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1615 if db_type == 'mysql':
screamer 0:66f3b5499f7f 1616 return MySQLDBAccess()
screamer 0:66f3b5499f7f 1617 return None
screamer 0:66f3b5499f7f 1618
screamer 0:66f3b5499f7f 1619
screamer 0:66f3b5499f7f 1620 def detect_database_verbose(db_url):
screamer 0:66f3b5499f7f 1621 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer 0:66f3b5499f7f 1622 """
screamer 0:66f3b5499f7f 1623 result = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1624 if result is not None:
screamer 0:66f3b5499f7f 1625 # Parsing passed
screamer 0:66f3b5499f7f 1626 (db_type, username, password, host, db_name) = result
screamer 0:66f3b5499f7f 1627 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer 0:66f3b5499f7f 1628 # Let's try to connect
screamer 0:66f3b5499f7f 1629 db_ = factory_db_logger(db_url)
screamer 0:66f3b5499f7f 1630 if db_ is not None:
screamer 0:66f3b5499f7f 1631 print "Connecting to database '%s'..."% db_url,
screamer 0:66f3b5499f7f 1632 db_.connect(host, username, password, db_name)
screamer 0:66f3b5499f7f 1633 if db_.is_connected():
screamer 0:66f3b5499f7f 1634 print "ok"
screamer 0:66f3b5499f7f 1635 print "Detecting database..."
screamer 0:66f3b5499f7f 1636 print db_.detect_database(verbose=True)
screamer 0:66f3b5499f7f 1637 print "Disconnecting...",
screamer 0:66f3b5499f7f 1638 db_.disconnect()
screamer 0:66f3b5499f7f 1639 print "done"
screamer 0:66f3b5499f7f 1640 else:
screamer 0:66f3b5499f7f 1641 print "Database type '%s' unknown"% db_type
screamer 0:66f3b5499f7f 1642 else:
screamer 0:66f3b5499f7f 1643 print "Parse error: '%s' - DB Url error"% (db_url)
screamer 0:66f3b5499f7f 1644
screamer 0:66f3b5499f7f 1645
screamer 0:66f3b5499f7f 1646 def get_module_avail(module_name):
theotherjimmy 41:2a77626a4c21 1647 """ This function returns True if module_name is already imported module
screamer 0:66f3b5499f7f 1648 """
screamer 0:66f3b5499f7f 1649 return module_name in sys.modules.keys()
screamer 0:66f3b5499f7f 1650
screamer 0:66f3b5499f7f 1651 def get_autodetected_MUTS_list(platform_name_filter=None):
screamer 0:66f3b5499f7f 1652 oldError = None
screamer 0:66f3b5499f7f 1653 if os.name == 'nt':
screamer 0:66f3b5499f7f 1654 # Disable Windows error box temporarily
screamer 0:66f3b5499f7f 1655 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer 0:66f3b5499f7f 1656
screamer 0:66f3b5499f7f 1657 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1658 detect_muts_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1659
screamer 0:66f3b5499f7f 1660 if os.name == 'nt':
screamer 0:66f3b5499f7f 1661 ctypes.windll.kernel32.SetErrorMode(oldError)
screamer 0:66f3b5499f7f 1662
screamer 0:66f3b5499f7f 1663 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 1664
screamer 0:66f3b5499f7f 1665 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer 0:66f3b5499f7f 1666 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer 0:66f3b5499f7f 1667 If function fails to auto-detect devices it will return empty dictionary.
screamer 0:66f3b5499f7f 1668
screamer 0:66f3b5499f7f 1669 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1670 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1671 mbeds_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1672
screamer 0:66f3b5499f7f 1673 @param mbeds_list list of mbeds captured from mbed_lstools
screamer 0:66f3b5499f7f 1674 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer 0:66f3b5499f7f 1675 """
screamer 0:66f3b5499f7f 1676 result = {} # Should be in muts_all.json format
screamer 0:66f3b5499f7f 1677 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer 0:66f3b5499f7f 1678 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer 0:66f3b5499f7f 1679 index = 1
screamer 0:66f3b5499f7f 1680 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1681 # Filter the MUTS if a filter is specified
screamer 0:66f3b5499f7f 1682
screamer 0:66f3b5499f7f 1683 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer 0:66f3b5499f7f 1684 continue
screamer 0:66f3b5499f7f 1685
screamer 0:66f3b5499f7f 1686 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer 0:66f3b5499f7f 1687 # if not we are creating our own unique value (last few chars from platform's target_id).
screamer 0:66f3b5499f7f 1688 m = {'mcu': mut['platform_name'],
screamer 0:66f3b5499f7f 1689 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer 0:66f3b5499f7f 1690 'port': mut['serial_port'],
screamer 0:66f3b5499f7f 1691 'disk': mut['mount_point'],
screamer 0:66f3b5499f7f 1692 'peripherals': [] # No peripheral detection
screamer 0:66f3b5499f7f 1693 }
screamer 0:66f3b5499f7f 1694 if index not in result:
screamer 0:66f3b5499f7f 1695 result[index] = {}
screamer 0:66f3b5499f7f 1696 result[index] = m
screamer 0:66f3b5499f7f 1697 index += 1
screamer 0:66f3b5499f7f 1698 return result
screamer 0:66f3b5499f7f 1699
screamer 0:66f3b5499f7f 1700
screamer 0:66f3b5499f7f 1701 def get_autodetected_TEST_SPEC(mbeds_list,
screamer 0:66f3b5499f7f 1702 use_default_toolchain=True,
screamer 0:66f3b5499f7f 1703 use_supported_toolchains=False,
screamer 0:66f3b5499f7f 1704 toolchain_filter=None,
screamer 0:66f3b5499f7f 1705 platform_name_filter=None):
screamer 0:66f3b5499f7f 1706 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer 0:66f3b5499f7f 1707 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer 0:66f3b5499f7f 1708
screamer 0:66f3b5499f7f 1709 use_default_toolchain - if True add default toolchain to test_spec
screamer 0:66f3b5499f7f 1710 use_supported_toolchains - if True add all supported toolchains to test_spec
screamer 0:66f3b5499f7f 1711 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer 0:66f3b5499f7f 1712 """
screamer 0:66f3b5499f7f 1713 result = {'targets': {} }
screamer 0:66f3b5499f7f 1714
screamer 0:66f3b5499f7f 1715 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1716 mcu = mut['mcu']
screamer 0:66f3b5499f7f 1717 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer 0:66f3b5499f7f 1718 if mcu in TARGET_MAP:
screamer 0:66f3b5499f7f 1719 default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer 0:66f3b5499f7f 1720 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer 0:66f3b5499f7f 1721
screamer 0:66f3b5499f7f 1722 # Decide which toolchains should be added to test specification toolchain pool for each target
screamer 0:66f3b5499f7f 1723 toolchains = []
screamer 0:66f3b5499f7f 1724 if use_default_toolchain:
screamer 0:66f3b5499f7f 1725 toolchains.append(default_toolchain)
screamer 0:66f3b5499f7f 1726 if use_supported_toolchains:
screamer 0:66f3b5499f7f 1727 toolchains += supported_toolchains
screamer 0:66f3b5499f7f 1728 if toolchain_filter is not None:
screamer 0:66f3b5499f7f 1729 all_toolchains = supported_toolchains + [default_toolchain]
screamer 22:9e85236d8716 1730 for toolchain in toolchain_filter:
screamer 0:66f3b5499f7f 1731 if toolchain in all_toolchains:
screamer 0:66f3b5499f7f 1732 toolchains.append(toolchain)
screamer 0:66f3b5499f7f 1733
screamer 0:66f3b5499f7f 1734 result['targets'][mcu] = list(set(toolchains))
screamer 0:66f3b5499f7f 1735 return result
screamer 0:66f3b5499f7f 1736
screamer 0:66f3b5499f7f 1737
screamer 0:66f3b5499f7f 1738 def get_default_test_options_parser():
screamer 0:66f3b5499f7f 1739 """ Get common test script options used by CLI, web services etc.
screamer 0:66f3b5499f7f 1740 """
screamer 22:9e85236d8716 1741 parser = argparse.ArgumentParser()
screamer 22:9e85236d8716 1742 parser.add_argument('-i', '--tests',
screamer 22:9e85236d8716 1743 dest='test_spec_filename',
screamer 22:9e85236d8716 1744 metavar="FILE",
screamer 22:9e85236d8716 1745 type=argparse_filestring_type,
screamer 22:9e85236d8716 1746 help='Points to file with test specification')
screamer 0:66f3b5499f7f 1747
screamer 22:9e85236d8716 1748 parser.add_argument('-M', '--MUTS',
screamer 22:9e85236d8716 1749 dest='muts_spec_filename',
screamer 22:9e85236d8716 1750 metavar="FILE",
screamer 22:9e85236d8716 1751 type=argparse_filestring_type,
screamer 22:9e85236d8716 1752 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer 0:66f3b5499f7f 1753
screamer 22:9e85236d8716 1754 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 1755 dest='jobs',
screamer 22:9e85236d8716 1756 metavar="NUMBER",
screamer 22:9e85236d8716 1757 type=int,
screamer 22:9e85236d8716 1758 help="Define number of compilation jobs. Default value is 1")
screamer 0:66f3b5499f7f 1759
screamer 0:66f3b5499f7f 1760 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1761 # Additional features available when mbed_lstools is installed on host and imported
screamer 0:66f3b5499f7f 1762 # mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer 22:9e85236d8716 1763 parser.add_argument('--auto',
screamer 22:9e85236d8716 1764 dest='auto_detect',
screamer 22:9e85236d8716 1765 action="store_true",
screamer 22:9e85236d8716 1766 help='Use mbed-ls module to detect all connected mbed devices')
screamer 0:66f3b5499f7f 1767
screamer 22:9e85236d8716 1768 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
screamer 22:9e85236d8716 1769 parser.add_argument('--tc',
screamer 22:9e85236d8716 1770 dest='toolchains_filter',
screamer 22:9e85236d8716 1771 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
screamer 22:9e85236d8716 1772 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer 0:66f3b5499f7f 1773
screamer 0:66f3b5499f7f 1774 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer 22:9e85236d8716 1775 parser.add_argument('--oper',
screamer 22:9e85236d8716 1776 dest='operability_checks',
screamer 22:9e85236d8716 1777 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
screamer 22:9e85236d8716 1778 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer 0:66f3b5499f7f 1779
screamer 22:9e85236d8716 1780 parser.add_argument('--clean',
screamer 22:9e85236d8716 1781 dest='clean',
screamer 22:9e85236d8716 1782 action="store_true",
screamer 22:9e85236d8716 1783 help='Clean the build directory')
screamer 0:66f3b5499f7f 1784
screamer 22:9e85236d8716 1785 parser.add_argument('-P', '--only-peripherals',
screamer 22:9e85236d8716 1786 dest='test_only_peripheral',
screamer 22:9e85236d8716 1787 default=False,
screamer 22:9e85236d8716 1788 action="store_true",
screamer 22:9e85236d8716 1789 help='Test only peripheral declared for MUT and skip common tests')
screamer 0:66f3b5499f7f 1790
The Other Jimmy 31:8ea194f6145b 1791 parser.add_argument("--profile", dest="profile", action="append",
The Other Jimmy 31:8ea194f6145b 1792 type=argparse_filestring_type,
The Other Jimmy 31:8ea194f6145b 1793 default=[])
The Other Jimmy 31:8ea194f6145b 1794
screamer 22:9e85236d8716 1795 parser.add_argument('-C', '--only-commons',
screamer 22:9e85236d8716 1796 dest='test_only_common',
screamer 22:9e85236d8716 1797 default=False,
screamer 22:9e85236d8716 1798 action="store_true",
screamer 22:9e85236d8716 1799 help='Test only board internals. Skip perpherials tests and perform common tests')
screamer 0:66f3b5499f7f 1800
screamer 22:9e85236d8716 1801 parser.add_argument('-n', '--test-by-names',
screamer 22:9e85236d8716 1802 dest='test_by_names',
screamer 22:9e85236d8716 1803 type=argparse_many(str),
screamer 22:9e85236d8716 1804 help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer 0:66f3b5499f7f 1805
screamer 22:9e85236d8716 1806 parser.add_argument('-p', '--peripheral-by-names',
screamer 0:66f3b5499f7f 1807 dest='peripheral_by_names',
screamer 22:9e85236d8716 1808 type=argparse_many(str),
screamer 0:66f3b5499f7f 1809 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer 0:66f3b5499f7f 1810
screamer 0:66f3b5499f7f 1811 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer 0:66f3b5499f7f 1812 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer 0:66f3b5499f7f 1813
screamer 22:9e85236d8716 1814 parser.add_argument('-c', '--copy-method',
screamer 22:9e85236d8716 1815 dest='copy_method',
screamer 22:9e85236d8716 1816 type=argparse_uppercase_type(copy_methods, "flash method"),
screamer 22:9e85236d8716 1817 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer 0:66f3b5499f7f 1818
screamer 0:66f3b5499f7f 1819 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer 0:66f3b5499f7f 1820 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer 0:66f3b5499f7f 1821
screamer 22:9e85236d8716 1822 parser.add_argument('-r', '--reset-type',
screamer 22:9e85236d8716 1823 dest='mut_reset_type',
screamer 22:9e85236d8716 1824 default=None,
screamer 22:9e85236d8716 1825 type=argparse_uppercase_type(reset_methods, "reset method"),
screamer 22:9e85236d8716 1826 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer 0:66f3b5499f7f 1827
screamer 22:9e85236d8716 1828 parser.add_argument('-g', '--goanna-for-tests',
screamer 22:9e85236d8716 1829 dest='goanna_for_tests',
screamer 22:9e85236d8716 1830 action="store_true",
screamer 22:9e85236d8716 1831 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1832
screamer 22:9e85236d8716 1833 parser.add_argument('-G', '--goanna-for-sdk',
screamer 22:9e85236d8716 1834 dest='goanna_for_mbed_sdk',
screamer 22:9e85236d8716 1835 action="store_true",
screamer 22:9e85236d8716 1836 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1837
screamer 22:9e85236d8716 1838 parser.add_argument('-s', '--suppress-summary',
screamer 22:9e85236d8716 1839 dest='suppress_summary',
screamer 22:9e85236d8716 1840 default=False,
screamer 22:9e85236d8716 1841 action="store_true",
screamer 22:9e85236d8716 1842 help='Suppresses display of wellformatted table with test results')
screamer 22:9e85236d8716 1843
screamer 22:9e85236d8716 1844 parser.add_argument('-t', '--test-summary',
screamer 22:9e85236d8716 1845 dest='test_x_toolchain_summary',
screamer 22:9e85236d8716 1846 default=False,
screamer 22:9e85236d8716 1847 action="store_true",
screamer 22:9e85236d8716 1848 help='Displays wellformatted table with test x toolchain test result per target')
screamer 0:66f3b5499f7f 1849
screamer 22:9e85236d8716 1850 parser.add_argument('-A', '--test-automation-report',
screamer 22:9e85236d8716 1851 dest='test_automation_report',
screamer 22:9e85236d8716 1852 default=False,
screamer 22:9e85236d8716 1853 action="store_true",
screamer 22:9e85236d8716 1854 help='Prints information about all tests and exits')
screamer 0:66f3b5499f7f 1855
screamer 22:9e85236d8716 1856 parser.add_argument('-R', '--test-case-report',
screamer 22:9e85236d8716 1857 dest='test_case_report',
screamer 22:9e85236d8716 1858 default=False,
screamer 22:9e85236d8716 1859 action="store_true",
screamer 22:9e85236d8716 1860 help='Prints information about all test cases and exits')
screamer 0:66f3b5499f7f 1861
screamer 22:9e85236d8716 1862 parser.add_argument("-S", "--supported-toolchains",
screamer 22:9e85236d8716 1863 action="store_true",
screamer 22:9e85236d8716 1864 dest="supported_toolchains",
screamer 22:9e85236d8716 1865 default=False,
screamer 22:9e85236d8716 1866 help="Displays supported matrix of MCUs and toolchains")
screamer 0:66f3b5499f7f 1867
screamer 22:9e85236d8716 1868 parser.add_argument("-O", "--only-build",
screamer 22:9e85236d8716 1869 action="store_true",
screamer 22:9e85236d8716 1870 dest="only_build_tests",
screamer 22:9e85236d8716 1871 default=False,
screamer 22:9e85236d8716 1872 help="Only build tests, skips actual test procedures (flashing etc.)")
screamer 0:66f3b5499f7f 1873
screamer 22:9e85236d8716 1874 parser.add_argument('--parallel',
screamer 22:9e85236d8716 1875 dest='parallel_test_exec',
screamer 22:9e85236d8716 1876 default=False,
screamer 22:9e85236d8716 1877 action="store_true",
screamer 22:9e85236d8716 1878 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer 0:66f3b5499f7f 1879
screamer 22:9e85236d8716 1880 parser.add_argument('--config',
screamer 22:9e85236d8716 1881 dest='verbose_test_configuration_only',
screamer 22:9e85236d8716 1882 default=False,
screamer 22:9e85236d8716 1883 action="store_true",
screamer 22:9e85236d8716 1884 help='Displays full test specification and MUTs configration and exits')
screamer 22:9e85236d8716 1885
screamer 22:9e85236d8716 1886 parser.add_argument('--loops',
screamer 22:9e85236d8716 1887 dest='test_loops_list',
screamer 22:9e85236d8716 1888 type=argparse_many(str),
screamer 22:9e85236d8716 1889 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer 0:66f3b5499f7f 1890
screamer 22:9e85236d8716 1891 parser.add_argument('--global-loops',
screamer 22:9e85236d8716 1892 dest='test_global_loops_value',
screamer 22:9e85236d8716 1893 type=int,
screamer 22:9e85236d8716 1894 help='Set global number of test loops per test. Default value is set 1')
screamer 0:66f3b5499f7f 1895
screamer 22:9e85236d8716 1896 parser.add_argument('--consolidate-waterfall',
screamer 22:9e85236d8716 1897 dest='consolidate_waterfall_test',
screamer 22:9e85236d8716 1898 default=False,
screamer 22:9e85236d8716 1899 action="store_true",
screamer 22:9e85236d8716 1900 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
screamer 0:66f3b5499f7f 1901
screamer 22:9e85236d8716 1902 parser.add_argument('-W', '--waterfall',
screamer 22:9e85236d8716 1903 dest='waterfall_test',
screamer 22:9e85236d8716 1904 default=False,
screamer 22:9e85236d8716 1905 action="store_true",
screamer 22:9e85236d8716 1906 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
screamer 0:66f3b5499f7f 1907
screamer 22:9e85236d8716 1908 parser.add_argument('-N', '--firmware-name',
screamer 22:9e85236d8716 1909 dest='firmware_global_name',
screamer 22:9e85236d8716 1910 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer 0:66f3b5499f7f 1911
screamer 22:9e85236d8716 1912 parser.add_argument('-u', '--shuffle',
screamer 22:9e85236d8716 1913 dest='shuffle_test_order',
screamer 22:9e85236d8716 1914 default=False,
screamer 22:9e85236d8716 1915 action="store_true",
screamer 22:9e85236d8716 1916 help='Shuffles test execution order')
screamer 0:66f3b5499f7f 1917
screamer 22:9e85236d8716 1918 parser.add_argument('--shuffle-seed',
screamer 22:9e85236d8716 1919 dest='shuffle_test_seed',
screamer 22:9e85236d8716 1920 default=None,
screamer 22:9e85236d8716 1921 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer 0:66f3b5499f7f 1922
screamer 22:9e85236d8716 1923 parser.add_argument('-f', '--filter',
screamer 22:9e85236d8716 1924 dest='general_filter_regex',
screamer 22:9e85236d8716 1925 type=argparse_many(str),
screamer 22:9e85236d8716 1926 default=None,
screamer 22:9e85236d8716 1927 help='For some commands you can use filter to filter out results')
screamer 0:66f3b5499f7f 1928
screamer 22:9e85236d8716 1929 parser.add_argument('--inc-timeout',
screamer 22:9e85236d8716 1930 dest='extend_test_timeout',
screamer 22:9e85236d8716 1931 metavar="NUMBER",
screamer 22:9e85236d8716 1932 type=int,
screamer 22:9e85236d8716 1933 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer 22:9e85236d8716 1934
screamer 22:9e85236d8716 1935 parser.add_argument('--db',
screamer 22:9e85236d8716 1936 dest='db_url',
screamer 22:9e85236d8716 1937 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer 0:66f3b5499f7f 1938
screamer 22:9e85236d8716 1939 parser.add_argument('-l', '--log',
screamer 22:9e85236d8716 1940 dest='log_file_name',
screamer 22:9e85236d8716 1941 help='Log events to external file (note not all console entries may be visible in log file)')
screamer 0:66f3b5499f7f 1942
screamer 22:9e85236d8716 1943 parser.add_argument('--report-html',
screamer 22:9e85236d8716 1944 dest='report_html_file_name',
screamer 22:9e85236d8716 1945 help='You can log test suite results in form of HTML report')
screamer 0:66f3b5499f7f 1946
screamer 22:9e85236d8716 1947 parser.add_argument('--report-junit',
screamer 22:9e85236d8716 1948 dest='report_junit_file_name',
screamer 22:9e85236d8716 1949 help='You can log test suite results in form of JUnit compliant XML report')
screamer 0:66f3b5499f7f 1950
screamer 22:9e85236d8716 1951 parser.add_argument("--report-build",
screamer 22:9e85236d8716 1952 dest="report_build_file_name",
screamer 22:9e85236d8716 1953 help="Output the build results to a junit xml file")
screamer 0:66f3b5499f7f 1954
screamer 22:9e85236d8716 1955 parser.add_argument("--report-text",
screamer 22:9e85236d8716 1956 dest="report_text_file_name",
screamer 22:9e85236d8716 1957 help="Output the build results to a text file")
screamer 0:66f3b5499f7f 1958
screamer 22:9e85236d8716 1959 parser.add_argument('--verbose-skipped',
screamer 22:9e85236d8716 1960 dest='verbose_skipped_tests',
screamer 22:9e85236d8716 1961 default=False,
screamer 22:9e85236d8716 1962 action="store_true",
screamer 22:9e85236d8716 1963 help='Prints some extra information about skipped tests')
screamer 0:66f3b5499f7f 1964
screamer 22:9e85236d8716 1965 parser.add_argument('-V', '--verbose-test-result',
screamer 22:9e85236d8716 1966 dest='verbose_test_result_only',
screamer 22:9e85236d8716 1967 default=False,
screamer 22:9e85236d8716 1968 action="store_true",
screamer 22:9e85236d8716 1969 help='Prints test serial output')
screamer 0:66f3b5499f7f 1970
screamer 22:9e85236d8716 1971 parser.add_argument('-v', '--verbose',
screamer 22:9e85236d8716 1972 dest='verbose',
screamer 22:9e85236d8716 1973 default=False,
screamer 22:9e85236d8716 1974 action="store_true",
screamer 22:9e85236d8716 1975 help='Verbose mode (prints some extra information)')
screamer 0:66f3b5499f7f 1976
screamer 22:9e85236d8716 1977 parser.add_argument('--version',
screamer 22:9e85236d8716 1978 dest='version',
screamer 22:9e85236d8716 1979 default=False,
screamer 22:9e85236d8716 1980 action="store_true",
screamer 22:9e85236d8716 1981 help='Prints script version and exits')
theotherjimmy 40:7d3fa6b99b2b 1982
theotherjimmy 40:7d3fa6b99b2b 1983 parser.add_argument('--stats-depth',
theotherjimmy 40:7d3fa6b99b2b 1984 dest='stats_depth',
theotherjimmy 40:7d3fa6b99b2b 1985 default=2,
theotherjimmy 40:7d3fa6b99b2b 1986 type=int,
theotherjimmy 40:7d3fa6b99b2b 1987 help="Depth level for static memory report")
screamer 0:66f3b5499f7f 1988 return parser
screamer 0:66f3b5499f7f 1989
The Other Jimmy 31:8ea194f6145b 1990 def test_path_to_name(path, base):
screamer 0:66f3b5499f7f 1991 """Change all slashes in a path into hyphens
screamer 0:66f3b5499f7f 1992 This creates a unique cross-platform test name based on the path
screamer 0:66f3b5499f7f 1993 This can eventually be overriden by a to-be-determined meta-data mechanism"""
screamer 0:66f3b5499f7f 1994 name_parts = []
The Other Jimmy 31:8ea194f6145b 1995 head, tail = os.path.split(relpath(path,base))
screamer 0:66f3b5499f7f 1996 while (tail and tail != "."):
screamer 0:66f3b5499f7f 1997 name_parts.insert(0, tail)
screamer 0:66f3b5499f7f 1998 head, tail = os.path.split(head)
screamer 22:9e85236d8716 1999
screamer 13:ab47a20b66f0 2000 return "-".join(name_parts).lower()
screamer 0:66f3b5499f7f 2001
theotherjimmy 41:2a77626a4c21 2002 def get_test_config(config_name, target_name):
theotherjimmy 41:2a77626a4c21 2003 """Finds the path to a test configuration file
theotherjimmy 41:2a77626a4c21 2004 config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
theotherjimmy 41:2a77626a4c21 2005 target_name: name of target to determing if mbed OS interface given is valid
theotherjimmy 41:2a77626a4c21 2006 returns path to config, boolean of whether it is a module or mbed OS interface
theotherjimmy 41:2a77626a4c21 2007 """
theotherjimmy 41:2a77626a4c21 2008 # If they passed in a full path
theotherjimmy 41:2a77626a4c21 2009 if exists(config_name):
theotherjimmy 41:2a77626a4c21 2010 # This is a module config
theotherjimmy 41:2a77626a4c21 2011 return config_name
theotherjimmy 41:2a77626a4c21 2012 # Otherwise find the path to configuration file based on mbed OS interface
theotherjimmy 41:2a77626a4c21 2013 return TestConfig.get_config_path(config_name, target_name)
theotherjimmy 41:2a77626a4c21 2014
The Other Jimmy 31:8ea194f6145b 2015 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
screamer 24:25bff2709c20 2016 """ Finds all tests in a directory recursively
screamer 24:25bff2709c20 2017 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
screamer 24:25bff2709c20 2018 target_name: name of the target to use for scanning (ex. 'K64F')
screamer 24:25bff2709c20 2019 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
screamer 24:25bff2709c20 2020 options: Compile options to pass to the toolchain (ex. ['debug-info'])
The Other Jimmy 31:8ea194f6145b 2021 app_config - location of a chosen mbed_app.json file
screamer 24:25bff2709c20 2022 """
screamer 24:25bff2709c20 2023
screamer 24:25bff2709c20 2024 tests = {}
screamer 24:25bff2709c20 2025
screamer 24:25bff2709c20 2026 # Prepare the toolchain
The Other Jimmy 36:96847d42f010 2027 toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2028 silent=True, app_config=app_config)
screamer 24:25bff2709c20 2029
screamer 24:25bff2709c20 2030 # Scan the directory for paths to probe for 'TESTS' folders
The Other Jimmy 31:8ea194f6145b 2031 base_resources = scan_resources([base_dir], toolchain)
screamer 24:25bff2709c20 2032
screamer 24:25bff2709c20 2033 dirs = base_resources.inc_dirs
screamer 24:25bff2709c20 2034 for directory in dirs:
screamer 24:25bff2709c20 2035 subdirs = os.listdir(directory)
screamer 22:9e85236d8716 2036
screamer 24:25bff2709c20 2037 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
screamer 24:25bff2709c20 2038 if 'TESTS' in subdirs:
screamer 24:25bff2709c20 2039 walk_base_dir = join(directory, 'TESTS')
screamer 24:25bff2709c20 2040 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
screamer 24:25bff2709c20 2041
screamer 24:25bff2709c20 2042 # Loop through all subdirectories
screamer 24:25bff2709c20 2043 for d in test_resources.inc_dirs:
screamer 22:9e85236d8716 2044
screamer 24:25bff2709c20 2045 # If the test case folder is not called 'host_tests' and it is
screamer 24:25bff2709c20 2046 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
screamer 24:25bff2709c20 2047 # then add it to the tests
screamer 24:25bff2709c20 2048 path_depth = get_path_depth(relpath(d, walk_base_dir))
screamer 24:25bff2709c20 2049 if path_depth == 2:
screamer 24:25bff2709c20 2050 test_group_directory_path, test_case_directory = os.path.split(d)
screamer 24:25bff2709c20 2051 test_group_directory = os.path.basename(test_group_directory_path)
theotherjimmy 40:7d3fa6b99b2b 2052
screamer 24:25bff2709c20 2053 # Check to make sure discoverd folder is not in a host test directory
screamer 24:25bff2709c20 2054 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
The Other Jimmy 31:8ea194f6145b 2055 test_name = test_path_to_name(d, base_dir)
screamer 24:25bff2709c20 2056 tests[test_name] = d
screamer 22:9e85236d8716 2057
screamer 13:ab47a20b66f0 2058 return tests
screamer 22:9e85236d8716 2059
screamer 13:ab47a20b66f0 2060 def print_tests(tests, format="list", sort=True):
screamer 0:66f3b5499f7f 2061 """Given a dictionary of tests (as returned from "find_tests"), print them
screamer 0:66f3b5499f7f 2062 in the specified format"""
screamer 0:66f3b5499f7f 2063 if format == "list":
screamer 13:ab47a20b66f0 2064 for test_name in sorted(tests.keys()):
screamer 13:ab47a20b66f0 2065 test_path = tests[test_name]
screamer 0:66f3b5499f7f 2066 print "Test Case:"
screamer 0:66f3b5499f7f 2067 print " Name: %s" % test_name
screamer 0:66f3b5499f7f 2068 print " Path: %s" % test_path
screamer 0:66f3b5499f7f 2069 elif format == "json":
screamer 0:66f3b5499f7f 2070 print json.dumps(tests, indent=2)
screamer 0:66f3b5499f7f 2071 else:
screamer 0:66f3b5499f7f 2072 print "Unknown format '%s'" % format
screamer 0:66f3b5499f7f 2073 sys.exit(1)
screamer 0:66f3b5499f7f 2074
screamer 13:ab47a20b66f0 2075 def norm_relative_path(path, start):
screamer 13:ab47a20b66f0 2076 """This function will create a normalized, relative path. It mimics the
screamer 13:ab47a20b66f0 2077 python os.path.relpath function, but also normalizes a Windows-syle path
screamer 13:ab47a20b66f0 2078 that use backslashes to a Unix style path that uses forward slashes."""
screamer 13:ab47a20b66f0 2079 path = os.path.normpath(path)
screamer 13:ab47a20b66f0 2080 path = os.path.relpath(path, start)
screamer 13:ab47a20b66f0 2081 path = path.replace("\\", "/")
screamer 13:ab47a20b66f0 2082 return path
screamer 13:ab47a20b66f0 2083
The Other Jimmy 31:8ea194f6145b 2084
The Other Jimmy 31:8ea194f6145b 2085 def build_test_worker(*args, **kwargs):
The Other Jimmy 31:8ea194f6145b 2086 """This is a worker function for the parallel building of tests. The `args`
The Other Jimmy 31:8ea194f6145b 2087 and `kwargs` are passed directly to `build_project`. It returns a dictionary
The Other Jimmy 31:8ea194f6145b 2088 with the following structure:
The Other Jimmy 31:8ea194f6145b 2089
The Other Jimmy 31:8ea194f6145b 2090 {
The Other Jimmy 31:8ea194f6145b 2091 'result': `True` if no exceptions were thrown, `False` otherwise
The Other Jimmy 31:8ea194f6145b 2092 'reason': Instance of exception that was thrown on failure
The Other Jimmy 31:8ea194f6145b 2093 'bin_file': Path to the created binary if `build_project` was
The Other Jimmy 31:8ea194f6145b 2094 successful. Not present otherwise
The Other Jimmy 31:8ea194f6145b 2095 'kwargs': The keyword arguments that were passed to `build_project`.
The Other Jimmy 31:8ea194f6145b 2096 This includes arguments that were modified (ex. report)
The Other Jimmy 31:8ea194f6145b 2097 }
The Other Jimmy 31:8ea194f6145b 2098 """
The Other Jimmy 31:8ea194f6145b 2099 bin_file = None
The Other Jimmy 31:8ea194f6145b 2100 ret = {
The Other Jimmy 31:8ea194f6145b 2101 'result': False,
The Other Jimmy 31:8ea194f6145b 2102 'args': args,
The Other Jimmy 31:8ea194f6145b 2103 'kwargs': kwargs
The Other Jimmy 31:8ea194f6145b 2104 }
The Other Jimmy 31:8ea194f6145b 2105
The Other Jimmy 31:8ea194f6145b 2106 # Use parent TOOLCHAIN_PATHS variable
The Other Jimmy 31:8ea194f6145b 2107 for key, value in kwargs['toolchain_paths'].iteritems():
The Other Jimmy 31:8ea194f6145b 2108 TOOLCHAIN_PATHS[key] = value
The Other Jimmy 31:8ea194f6145b 2109
The Other Jimmy 31:8ea194f6145b 2110 del kwargs['toolchain_paths']
The Other Jimmy 31:8ea194f6145b 2111
The Other Jimmy 31:8ea194f6145b 2112 try:
The Other Jimmy 31:8ea194f6145b 2113 bin_file = build_project(*args, **kwargs)
The Other Jimmy 31:8ea194f6145b 2114 ret['result'] = True
The Other Jimmy 31:8ea194f6145b 2115 ret['bin_file'] = bin_file
The Other Jimmy 31:8ea194f6145b 2116 ret['kwargs'] = kwargs
The Other Jimmy 31:8ea194f6145b 2117
The Other Jimmy 31:8ea194f6145b 2118 except NotSupportedException, e:
The Other Jimmy 31:8ea194f6145b 2119 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2120 except ToolException, e:
The Other Jimmy 31:8ea194f6145b 2121 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2122 except KeyboardInterrupt, e:
The Other Jimmy 31:8ea194f6145b 2123 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2124 except:
The Other Jimmy 31:8ea194f6145b 2125 # Print unhandled exceptions here
The Other Jimmy 31:8ea194f6145b 2126 import traceback
The Other Jimmy 31:8ea194f6145b 2127 traceback.print_exc(file=sys.stdout)
The Other Jimmy 31:8ea194f6145b 2128
The Other Jimmy 31:8ea194f6145b 2129 return ret
The Other Jimmy 31:8ea194f6145b 2130
The Other Jimmy 31:8ea194f6145b 2131
screamer 0:66f3b5499f7f 2132 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2133 clean=False, notify=None, verbose=False, jobs=1, macros=None,
The Other Jimmy 31:8ea194f6145b 2134 silent=False, report=None, properties=None,
The Other Jimmy 31:8ea194f6145b 2135 continue_on_build_fail=False, app_config=None,
theotherjimmy 40:7d3fa6b99b2b 2136 build_profile=None, stats_depth=None):
screamer 0:66f3b5499f7f 2137 """Given the data structure from 'find_tests' and the typical build parameters,
screamer 13:ab47a20b66f0 2138 build all the tests
screamer 22:9e85236d8716 2139
screamer 13:ab47a20b66f0 2140 Returns a tuple of the build result (True or False) followed by the test
screamer 13:ab47a20b66f0 2141 build data structure"""
screamer 22:9e85236d8716 2142
screamer 13:ab47a20b66f0 2143 execution_directory = "."
screamer 13:ab47a20b66f0 2144 base_path = norm_relative_path(build_path, execution_directory)
screamer 13:ab47a20b66f0 2145
screamer 13:ab47a20b66f0 2146 target_name = target if isinstance(target, str) else target.name
The Other Jimmy 35:da9c89f8be7d 2147 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2148
The Other Jimmy 31:8ea194f6145b 2149 baud_rate = 9600
The Other Jimmy 31:8ea194f6145b 2150 if 'platform.stdio-baud-rate' in cfg:
The Other Jimmy 31:8ea194f6145b 2151 baud_rate = cfg['platform.stdio-baud-rate'].value
The Other Jimmy 31:8ea194f6145b 2152
screamer 0:66f3b5499f7f 2153 test_build = {
screamer 13:ab47a20b66f0 2154 "platform": target_name,
screamer 0:66f3b5499f7f 2155 "toolchain": toolchain_name,
screamer 13:ab47a20b66f0 2156 "base_path": base_path,
The Other Jimmy 31:8ea194f6145b 2157 "baud_rate": baud_rate,
screamer 0:66f3b5499f7f 2158 "binary_type": "bootable",
screamer 0:66f3b5499f7f 2159 "tests": {}
screamer 0:66f3b5499f7f 2160 }
screamer 22:9e85236d8716 2161
screamer 13:ab47a20b66f0 2162 result = True
screamer 22:9e85236d8716 2163
The Other Jimmy 31:8ea194f6145b 2164 jobs_count = int(jobs if jobs else cpu_count())
The Other Jimmy 31:8ea194f6145b 2165 p = Pool(processes=jobs_count)
The Other Jimmy 31:8ea194f6145b 2166 results = []
screamer 0:66f3b5499f7f 2167 for test_name, test_path in tests.iteritems():
screamer 0:66f3b5499f7f 2168 test_build_path = os.path.join(build_path, test_path)
screamer 0:66f3b5499f7f 2169 src_path = base_source_paths + [test_path]
screamer 13:ab47a20b66f0 2170 bin_file = None
screamer 24:25bff2709c20 2171 test_case_folder_name = os.path.basename(test_path)
theotherjimmy 40:7d3fa6b99b2b 2172
The Other Jimmy 31:8ea194f6145b 2173 args = (src_path, test_build_path, target, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2174 kwargs = {
The Other Jimmy 31:8ea194f6145b 2175 'jobs': 1,
The Other Jimmy 31:8ea194f6145b 2176 'clean': clean,
The Other Jimmy 31:8ea194f6145b 2177 'macros': macros,
The Other Jimmy 31:8ea194f6145b 2178 'name': test_case_folder_name,
The Other Jimmy 31:8ea194f6145b 2179 'project_id': test_name,
The Other Jimmy 31:8ea194f6145b 2180 'report': report,
The Other Jimmy 31:8ea194f6145b 2181 'properties': properties,
The Other Jimmy 31:8ea194f6145b 2182 'verbose': verbose,
The Other Jimmy 31:8ea194f6145b 2183 'app_config': app_config,
The Other Jimmy 31:8ea194f6145b 2184 'build_profile': build_profile,
The Other Jimmy 31:8ea194f6145b 2185 'silent': True,
theotherjimmy 40:7d3fa6b99b2b 2186 'toolchain_paths': TOOLCHAIN_PATHS,
theotherjimmy 40:7d3fa6b99b2b 2187 'stats_depth': stats_depth
The Other Jimmy 31:8ea194f6145b 2188 }
theotherjimmy 40:7d3fa6b99b2b 2189
The Other Jimmy 31:8ea194f6145b 2190 results.append(p.apply_async(build_test_worker, args, kwargs))
The Other Jimmy 31:8ea194f6145b 2191
The Other Jimmy 31:8ea194f6145b 2192 p.close()
The Other Jimmy 31:8ea194f6145b 2193 result = True
The Other Jimmy 31:8ea194f6145b 2194 itr = 0
The Other Jimmy 31:8ea194f6145b 2195 while len(results):
The Other Jimmy 31:8ea194f6145b 2196 itr += 1
The Other Jimmy 31:8ea194f6145b 2197 if itr > 360000:
The Other Jimmy 31:8ea194f6145b 2198 p.terminate()
The Other Jimmy 31:8ea194f6145b 2199 p.join()
The Other Jimmy 31:8ea194f6145b 2200 raise ToolException("Compile did not finish in 10 minutes")
The Other Jimmy 31:8ea194f6145b 2201 else:
The Other Jimmy 31:8ea194f6145b 2202 sleep(0.01)
The Other Jimmy 31:8ea194f6145b 2203 pending = 0
The Other Jimmy 31:8ea194f6145b 2204 for r in results:
The Other Jimmy 31:8ea194f6145b 2205 if r.ready() is True:
The Other Jimmy 31:8ea194f6145b 2206 try:
The Other Jimmy 31:8ea194f6145b 2207 worker_result = r.get()
The Other Jimmy 31:8ea194f6145b 2208 results.remove(r)
screamer 13:ab47a20b66f0 2209
The Other Jimmy 31:8ea194f6145b 2210 # Take report from the kwargs and merge it into existing report
theotherjimmy 40:7d3fa6b99b2b 2211 if report:
theotherjimmy 40:7d3fa6b99b2b 2212 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
theotherjimmy 40:7d3fa6b99b2b 2213 for test_key in report_entry.keys():
theotherjimmy 40:7d3fa6b99b2b 2214 report[target_name][toolchain_name][test_key] = report_entry[test_key]
theotherjimmy 40:7d3fa6b99b2b 2215
The Other Jimmy 31:8ea194f6145b 2216 # Set the overall result to a failure if a build failure occurred
The Other Jimmy 35:da9c89f8be7d 2217 if ('reason' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2218 not worker_result['reason'] and
The Other Jimmy 35:da9c89f8be7d 2219 not isinstance(worker_result['reason'], NotSupportedException)):
The Other Jimmy 31:8ea194f6145b 2220 result = False
The Other Jimmy 31:8ea194f6145b 2221 break
The Other Jimmy 31:8ea194f6145b 2222
The Other Jimmy 31:8ea194f6145b 2223 # Adding binary path to test build result
The Other Jimmy 35:da9c89f8be7d 2224 if ('result' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2225 worker_result['result'] and
The Other Jimmy 35:da9c89f8be7d 2226 'bin_file' in worker_result):
The Other Jimmy 31:8ea194f6145b 2227 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
screamer 22:9e85236d8716 2228
The Other Jimmy 31:8ea194f6145b 2229 test_build['tests'][worker_result['kwargs']['project_id']] = {
The Other Jimmy 31:8ea194f6145b 2230 "binaries": [
The Other Jimmy 31:8ea194f6145b 2231 {
The Other Jimmy 31:8ea194f6145b 2232 "path": bin_file
The Other Jimmy 31:8ea194f6145b 2233 }
The Other Jimmy 31:8ea194f6145b 2234 ]
The Other Jimmy 31:8ea194f6145b 2235 }
The Other Jimmy 31:8ea194f6145b 2236
The Other Jimmy 31:8ea194f6145b 2237 test_key = worker_result['kwargs']['project_id'].upper()
theotherjimmy 40:7d3fa6b99b2b 2238 if report:
theotherjimmy 40:7d3fa6b99b2b 2239 print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
The Other Jimmy 31:8ea194f6145b 2240 print 'Image: %s\n' % bin_file
The Other Jimmy 31:8ea194f6145b 2241
The Other Jimmy 31:8ea194f6145b 2242 except:
The Other Jimmy 31:8ea194f6145b 2243 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2244 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2245 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2246 p.terminate()
The Other Jimmy 31:8ea194f6145b 2247 p.join()
The Other Jimmy 31:8ea194f6145b 2248 raise
screamer 13:ab47a20b66f0 2249 else:
The Other Jimmy 31:8ea194f6145b 2250 pending += 1
The Other Jimmy 31:8ea194f6145b 2251 if pending >= jobs_count:
The Other Jimmy 31:8ea194f6145b 2252 break
screamer 22:9e85236d8716 2253
The Other Jimmy 31:8ea194f6145b 2254 # Break as soon as possible if there is a failure and we are not
The Other Jimmy 31:8ea194f6145b 2255 # continuing on build failures
The Other Jimmy 31:8ea194f6145b 2256 if not result and not continue_on_build_fail:
The Other Jimmy 31:8ea194f6145b 2257 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2258 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2259 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2260 p.terminate()
The Other Jimmy 31:8ea194f6145b 2261 break
screamer 22:9e85236d8716 2262
The Other Jimmy 31:8ea194f6145b 2263 p.join()
screamer 22:9e85236d8716 2264
screamer 0:66f3b5499f7f 2265 test_builds = {}
screamer 13:ab47a20b66f0 2266 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
screamer 22:9e85236d8716 2267
screamer 13:ab47a20b66f0 2268 return result, test_builds
screamer 22:9e85236d8716 2269
screamer 0:66f3b5499f7f 2270
screamer 13:ab47a20b66f0 2271 def test_spec_from_test_builds(test_builds):
screamer 0:66f3b5499f7f 2272 return {
screamer 0:66f3b5499f7f 2273 "builds": test_builds
screamer 0:66f3b5499f7f 2274 }