Anders Blomdell / mbed-sdk-tools

Clone of official tools

test_api.py@40:7d3fa6b99b2b, 2017-10-10 (annotated)

Committer:
theotherjimmy
Date:
Tue Oct 10 16:56:30 2017 -0500
Revision:
40:7d3fa6b99b2b
Parent:
36:96847d42f010
Child:
41:2a77626a4c21
Update to tools release 5.6.1

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """
screamer 0:66f3b5499f7f 2 mbed SDK
screamer 0:66f3b5499f7f 3 Copyright (c) 2011-2014 ARM Limited
screamer 0:66f3b5499f7f 4
screamer 0:66f3b5499f7f 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 6 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 7 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 8
screamer 0:66f3b5499f7f 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 14 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 15 limitations under the License.
screamer 0:66f3b5499f7f 16
screamer 0:66f3b5499f7f 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer 0:66f3b5499f7f 18 """
screamer 0:66f3b5499f7f 19
screamer 0:66f3b5499f7f 20 import os
screamer 0:66f3b5499f7f 21 import re
screamer 0:66f3b5499f7f 22 import sys
screamer 0:66f3b5499f7f 23 import json
screamer 0:66f3b5499f7f 24 import uuid
screamer 0:66f3b5499f7f 25 import pprint
screamer 0:66f3b5499f7f 26 import random
screamer 22:9e85236d8716 27 import argparse
screamer 0:66f3b5499f7f 28 import datetime
screamer 0:66f3b5499f7f 29 import threading
screamer 0:66f3b5499f7f 30 import ctypes
screamer 0:66f3b5499f7f 31 from types import ListType
screamer 0:66f3b5499f7f 32 from colorama import Fore, Back, Style
screamer 0:66f3b5499f7f 33 from prettytable import PrettyTable
screamer 24:25bff2709c20 34 from copy import copy
screamer 0:66f3b5499f7f 35
screamer 0:66f3b5499f7f 36 from time import sleep, time
screamer 0:66f3b5499f7f 37 from Queue import Queue, Empty
screamer 24:25bff2709c20 38 from os.path import join, exists, basename, relpath
screamer 0:66f3b5499f7f 39 from threading import Thread, Lock
The Other Jimmy 31:8ea194f6145b 40 from multiprocessing import Pool, cpu_count
screamer 0:66f3b5499f7f 41 from subprocess import Popen, PIPE
screamer 0:66f3b5499f7f 42
screamer 0:66f3b5499f7f 43 # Imports related to mbed build api
screamer 0:66f3b5499f7f 44 from tools.tests import TESTS
screamer 0:66f3b5499f7f 45 from tools.tests import TEST_MAP
screamer 0:66f3b5499f7f 46 from tools.paths import BUILD_DIR
screamer 0:66f3b5499f7f 47 from tools.paths import HOST_TESTS
screamer 0:66f3b5499f7f 48 from tools.utils import ToolException
screamer 0:66f3b5499f7f 49 from tools.utils import NotSupportedException
screamer 0:66f3b5499f7f 50 from tools.utils import construct_enum
screamer 22:9e85236d8716 51 from tools.memap import MemapParser
screamer 0:66f3b5499f7f 52 from tools.targets import TARGET_MAP
screamer 0:66f3b5499f7f 53 from tools.test_db import BaseDBAccess
screamer 0:66f3b5499f7f 54 from tools.build_api import build_project, build_mbed_libs, build_lib
screamer 0:66f3b5499f7f 55 from tools.build_api import get_target_supported_toolchains
screamer 0:66f3b5499f7f 56 from tools.build_api import write_build_report
screamer 0:66f3b5499f7f 57 from tools.build_api import prep_report
screamer 0:66f3b5499f7f 58 from tools.build_api import prep_properties
screamer 0:66f3b5499f7f 59 from tools.build_api import create_result
screamer 0:66f3b5499f7f 60 from tools.build_api import add_result_to_report
screamer 24:25bff2709c20 61 from tools.build_api import prepare_toolchain
screamer 24:25bff2709c20 62 from tools.build_api import scan_resources
The Other Jimmy 31:8ea194f6145b 63 from tools.build_api import get_config
screamer 0:66f3b5499f7f 64 from tools.libraries import LIBRARIES, LIBRARY_MAP
The Other Jimmy 31:8ea194f6145b 65 from tools.options import extract_profile
screamer 20:835f6355470d 66 from tools.toolchains import TOOLCHAIN_PATHS
screamer 22:9e85236d8716 67 from tools.toolchains import TOOLCHAINS
screamer 0:66f3b5499f7f 68 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 22:9e85236d8716 69 from tools.utils import argparse_filestring_type
screamer 22:9e85236d8716 70 from tools.utils import argparse_uppercase_type
screamer 22:9e85236d8716 71 from tools.utils import argparse_lowercase_type
screamer 22:9e85236d8716 72 from tools.utils import argparse_many
screamer 24:25bff2709c20 73 from tools.utils import get_path_depth
screamer 0:66f3b5499f7f 74
screamer 0:66f3b5499f7f 75 import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer 0:66f3b5499f7f 76
screamer 0:66f3b5499f7f 77 try:
screamer 0:66f3b5499f7f 78 import mbed_lstools
screamer 0:66f3b5499f7f 79 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:66f3b5499f7f 80 except:
screamer 0:66f3b5499f7f 81 pass
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83
screamer 0:66f3b5499f7f 84 class ProcessObserver(Thread):
screamer 0:66f3b5499f7f 85 def __init__(self, proc):
screamer 0:66f3b5499f7f 86 Thread.__init__(self)
screamer 0:66f3b5499f7f 87 self.proc = proc
screamer 0:66f3b5499f7f 88 self.queue = Queue()
screamer 0:66f3b5499f7f 89 self.daemon = True
screamer 0:66f3b5499f7f 90 self.active = True
screamer 0:66f3b5499f7f 91 self.start()
screamer 0:66f3b5499f7f 92
screamer 0:66f3b5499f7f 93 def run(self):
screamer 0:66f3b5499f7f 94 while self.active:
screamer 0:66f3b5499f7f 95 c = self.proc.stdout.read(1)
screamer 0:66f3b5499f7f 96 self.queue.put(c)
screamer 0:66f3b5499f7f 97
screamer 0:66f3b5499f7f 98 def stop(self):
screamer 0:66f3b5499f7f 99 self.active = False
screamer 0:66f3b5499f7f 100 try:
screamer 0:66f3b5499f7f 101 self.proc.terminate()
screamer 0:66f3b5499f7f 102 except Exception, _:
screamer 0:66f3b5499f7f 103 pass
screamer 0:66f3b5499f7f 104
screamer 0:66f3b5499f7f 105
screamer 0:66f3b5499f7f 106 class SingleTestExecutor(threading.Thread):
screamer 0:66f3b5499f7f 107 """ Example: Single test class in separate thread usage
screamer 0:66f3b5499f7f 108 """
screamer 0:66f3b5499f7f 109 def __init__(self, single_test):
screamer 0:66f3b5499f7f 110 self.single_test = single_test
screamer 0:66f3b5499f7f 111 threading.Thread.__init__(self)
screamer 0:66f3b5499f7f 112
screamer 0:66f3b5499f7f 113 def run(self):
screamer 0:66f3b5499f7f 114 start = time()
screamer 0:66f3b5499f7f 115 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 116 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer 0:66f3b5499f7f 117 elapsed_time = time() - start
screamer 0:66f3b5499f7f 118
screamer 0:66f3b5499f7f 119 # Human readable summary
screamer 0:66f3b5499f7f 120 if not self.single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 121 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 122 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 123 if self.single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 124 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 125 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 126 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 127 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 128
screamer 0:66f3b5499f7f 129
screamer 0:66f3b5499f7f 130 class SingleTestRunner(object):
screamer 0:66f3b5499f7f 131 """ Object wrapper for single test run which may involve multiple MUTs
screamer 0:66f3b5499f7f 132 """
screamer 0:66f3b5499f7f 133 RE_DETECT_TESTCASE_RESULT = None
screamer 0:66f3b5499f7f 134
screamer 0:66f3b5499f7f 135 # Return codes for test script
screamer 0:66f3b5499f7f 136 TEST_RESULT_OK = "OK"
screamer 0:66f3b5499f7f 137 TEST_RESULT_FAIL = "FAIL"
screamer 0:66f3b5499f7f 138 TEST_RESULT_ERROR = "ERROR"
screamer 0:66f3b5499f7f 139 TEST_RESULT_UNDEF = "UNDEF"
screamer 0:66f3b5499f7f 140 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer 0:66f3b5499f7f 141 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer 0:66f3b5499f7f 142 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer 0:66f3b5499f7f 143 TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer 0:66f3b5499f7f 144 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer 0:66f3b5499f7f 145 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer 0:66f3b5499f7f 146 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer 0:66f3b5499f7f 147 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer 0:66f3b5499f7f 148
screamer 0:66f3b5499f7f 149 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer 0:66f3b5499f7f 150 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer 0:66f3b5499f7f 151 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer 0:66f3b5499f7f 152
screamer 0:66f3b5499f7f 153 muts = {} # MUTs descriptor (from external file)
screamer 0:66f3b5499f7f 154 test_spec = {} # Test specification (from external file)
screamer 0:66f3b5499f7f 155
screamer 0:66f3b5499f7f 156 # mbed test suite -> SingleTestRunner
screamer 0:66f3b5499f7f 157 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer 0:66f3b5499f7f 158 "failure" : TEST_RESULT_FAIL,
screamer 0:66f3b5499f7f 159 "error" : TEST_RESULT_ERROR,
screamer 0:66f3b5499f7f 160 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer 0:66f3b5499f7f 161 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer 0:66f3b5499f7f 162 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer 0:66f3b5499f7f 163 "timeout" : TEST_RESULT_TIMEOUT,
screamer 0:66f3b5499f7f 164 "no_image" : TEST_RESULT_NO_IMAGE,
screamer 0:66f3b5499f7f 165 "end" : TEST_RESULT_UNDEF,
screamer 0:66f3b5499f7f 166 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer 0:66f3b5499f7f 167 "build_failed" : TEST_RESULT_BUILD_FAILED,
screamer 0:66f3b5499f7f 168 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 169 }
screamer 0:66f3b5499f7f 170
screamer 0:66f3b5499f7f 171 def __init__(self,
screamer 0:66f3b5499f7f 172 _global_loops_count=1,
screamer 0:66f3b5499f7f 173 _test_loops_list=None,
screamer 0:66f3b5499f7f 174 _muts={},
screamer 0:66f3b5499f7f 175 _clean=False,
The Other Jimmy 31:8ea194f6145b 176 _parser=None,
The Other Jimmy 31:8ea194f6145b 177 _opts=None,
screamer 0:66f3b5499f7f 178 _opts_db_url=None,
screamer 0:66f3b5499f7f 179 _opts_log_file_name=None,
screamer 0:66f3b5499f7f 180 _opts_report_html_file_name=None,
screamer 0:66f3b5499f7f 181 _opts_report_junit_file_name=None,
screamer 0:66f3b5499f7f 182 _opts_report_build_file_name=None,
screamer 22:9e85236d8716 183 _opts_report_text_file_name=None,
screamer 0:66f3b5499f7f 184 _opts_build_report={},
screamer 0:66f3b5499f7f 185 _opts_build_properties={},
screamer 0:66f3b5499f7f 186 _test_spec={},
screamer 0:66f3b5499f7f 187 _opts_goanna_for_mbed_sdk=None,
screamer 0:66f3b5499f7f 188 _opts_goanna_for_tests=None,
screamer 0:66f3b5499f7f 189 _opts_shuffle_test_order=False,
screamer 0:66f3b5499f7f 190 _opts_shuffle_test_seed=None,
screamer 0:66f3b5499f7f 191 _opts_test_by_names=None,
screamer 0:66f3b5499f7f 192 _opts_peripheral_by_names=None,
screamer 0:66f3b5499f7f 193 _opts_test_only_peripheral=False,
screamer 0:66f3b5499f7f 194 _opts_test_only_common=False,
screamer 0:66f3b5499f7f 195 _opts_verbose_skipped_tests=False,
screamer 0:66f3b5499f7f 196 _opts_verbose_test_result_only=False,
screamer 0:66f3b5499f7f 197 _opts_verbose=False,
screamer 0:66f3b5499f7f 198 _opts_firmware_global_name=None,
screamer 0:66f3b5499f7f 199 _opts_only_build_tests=False,
screamer 0:66f3b5499f7f 200 _opts_parallel_test_exec=False,
screamer 0:66f3b5499f7f 201 _opts_suppress_summary=False,
screamer 0:66f3b5499f7f 202 _opts_test_x_toolchain_summary=False,
screamer 0:66f3b5499f7f 203 _opts_copy_method=None,
screamer 0:66f3b5499f7f 204 _opts_mut_reset_type=None,
screamer 0:66f3b5499f7f 205 _opts_jobs=None,
screamer 0:66f3b5499f7f 206 _opts_waterfall_test=None,
screamer 0:66f3b5499f7f 207 _opts_consolidate_waterfall_test=None,
screamer 0:66f3b5499f7f 208 _opts_extend_test_timeout=None,
screamer 0:66f3b5499f7f 209 _opts_auto_detect=None,
screamer 0:66f3b5499f7f 210 _opts_include_non_automated=False):
screamer 0:66f3b5499f7f 211 """ Let's try hard to init this object
screamer 0:66f3b5499f7f 212 """
screamer 0:66f3b5499f7f 213 from colorama import init
screamer 0:66f3b5499f7f 214 init()
screamer 0:66f3b5499f7f 215
screamer 0:66f3b5499f7f 216 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer 0:66f3b5499f7f 217 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer 0:66f3b5499f7f 218 # Settings related to test loops counters
screamer 0:66f3b5499f7f 219 try:
screamer 0:66f3b5499f7f 220 _global_loops_count = int(_global_loops_count)
screamer 0:66f3b5499f7f 221 except:
screamer 0:66f3b5499f7f 222 _global_loops_count = 1
screamer 0:66f3b5499f7f 223 if _global_loops_count < 1:
screamer 0:66f3b5499f7f 224 _global_loops_count = 1
screamer 0:66f3b5499f7f 225 self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer 0:66f3b5499f7f 226 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer 0:66f3b5499f7f 227 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer 0:66f3b5499f7f 228
screamer 0:66f3b5499f7f 229 self.shuffle_random_seed = 0.0
screamer 0:66f3b5499f7f 230 self.SHUFFLE_SEED_ROUND = 10
screamer 0:66f3b5499f7f 231
screamer 0:66f3b5499f7f 232 # MUT list and test specification storage
screamer 0:66f3b5499f7f 233 self.muts = _muts
screamer 0:66f3b5499f7f 234 self.test_spec = _test_spec
screamer 0:66f3b5499f7f 235
screamer 0:66f3b5499f7f 236 # Settings passed e.g. from command line
screamer 0:66f3b5499f7f 237 self.opts_db_url = _opts_db_url
screamer 0:66f3b5499f7f 238 self.opts_log_file_name = _opts_log_file_name
screamer 0:66f3b5499f7f 239 self.opts_report_html_file_name = _opts_report_html_file_name
screamer 0:66f3b5499f7f 240 self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer 0:66f3b5499f7f 241 self.opts_report_build_file_name = _opts_report_build_file_name
screamer 22:9e85236d8716 242 self.opts_report_text_file_name = _opts_report_text_file_name
screamer 0:66f3b5499f7f 243 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer 0:66f3b5499f7f 244 self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer 0:66f3b5499f7f 245 self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer 0:66f3b5499f7f 246 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer 0:66f3b5499f7f 247 self.opts_test_by_names = _opts_test_by_names
screamer 0:66f3b5499f7f 248 self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer 0:66f3b5499f7f 249 self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer 0:66f3b5499f7f 250 self.opts_test_only_common = _opts_test_only_common
screamer 0:66f3b5499f7f 251 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer 0:66f3b5499f7f 252 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer 0:66f3b5499f7f 253 self.opts_verbose = _opts_verbose
screamer 0:66f3b5499f7f 254 self.opts_firmware_global_name = _opts_firmware_global_name
screamer 0:66f3b5499f7f 255 self.opts_only_build_tests = _opts_only_build_tests
screamer 0:66f3b5499f7f 256 self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer 0:66f3b5499f7f 257 self.opts_suppress_summary = _opts_suppress_summary
screamer 0:66f3b5499f7f 258 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer 0:66f3b5499f7f 259 self.opts_copy_method = _opts_copy_method
screamer 0:66f3b5499f7f 260 self.opts_mut_reset_type = _opts_mut_reset_type
screamer 0:66f3b5499f7f 261 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer 0:66f3b5499f7f 262 self.opts_waterfall_test = _opts_waterfall_test
screamer 0:66f3b5499f7f 263 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer 0:66f3b5499f7f 264 self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer 0:66f3b5499f7f 265 self.opts_clean = _clean
The Other Jimmy 31:8ea194f6145b 266 self.opts_parser = _parser
The Other Jimmy 31:8ea194f6145b 267 self.opts = _opts
screamer 0:66f3b5499f7f 268 self.opts_auto_detect = _opts_auto_detect
screamer 0:66f3b5499f7f 269 self.opts_include_non_automated = _opts_include_non_automated
screamer 0:66f3b5499f7f 270
screamer 0:66f3b5499f7f 271 self.build_report = _opts_build_report
screamer 0:66f3b5499f7f 272 self.build_properties = _opts_build_properties
screamer 0:66f3b5499f7f 273
screamer 0:66f3b5499f7f 274 # File / screen logger initialization
screamer 0:66f3b5499f7f 275 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer 0:66f3b5499f7f 276
screamer 0:66f3b5499f7f 277 # Database related initializations
screamer 0:66f3b5499f7f 278 self.db_logger = factory_db_logger(self.opts_db_url)
screamer 0:66f3b5499f7f 279 self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer 0:66f3b5499f7f 280 # Let's connect to database to set up credentials and confirm database is ready
screamer 0:66f3b5499f7f 281 if self.db_logger:
screamer 0:66f3b5499f7f 282 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer 0:66f3b5499f7f 283 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 284 # Get hostname and uname so we can use it as build description
screamer 0:66f3b5499f7f 285 # when creating new build_id in external database
screamer 0:66f3b5499f7f 286 (_hostname, _uname) = self.db_logger.get_hostname()
screamer 0:66f3b5499f7f 287 _host_location = os.path.dirname(os.path.abspath(__file__))
screamer 0:66f3b5499f7f 288 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer 0:66f3b5499f7f 289 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer 0:66f3b5499f7f 290 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 291
screamer 0:66f3b5499f7f 292 def dump_options(self):
screamer 0:66f3b5499f7f 293 """ Function returns data structure with common settings passed to SingelTestRunner
screamer 0:66f3b5499f7f 294 It can be used for example to fill _extra fields in database storing test suite single run data
screamer 0:66f3b5499f7f 295 Example:
screamer 0:66f3b5499f7f 296 data = self.dump_options()
screamer 0:66f3b5499f7f 297 or
screamer 0:66f3b5499f7f 298 data_str = json.dumps(self.dump_options())
screamer 0:66f3b5499f7f 299 """
screamer 0:66f3b5499f7f 300 result = {"db_url" : str(self.opts_db_url),
screamer 0:66f3b5499f7f 301 "log_file_name" : str(self.opts_log_file_name),
screamer 0:66f3b5499f7f 302 "shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer 0:66f3b5499f7f 303 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer 0:66f3b5499f7f 304 "test_by_names" : str(self.opts_test_by_names),
screamer 0:66f3b5499f7f 305 "peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer 0:66f3b5499f7f 306 "test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer 0:66f3b5499f7f 307 "test_only_common" : str(self.opts_test_only_common),
screamer 0:66f3b5499f7f 308 "verbose" : str(self.opts_verbose),
screamer 0:66f3b5499f7f 309 "firmware_global_name" : str(self.opts_firmware_global_name),
screamer 0:66f3b5499f7f 310 "only_build_tests" : str(self.opts_only_build_tests),
screamer 0:66f3b5499f7f 311 "copy_method" : str(self.opts_copy_method),
screamer 0:66f3b5499f7f 312 "mut_reset_type" : str(self.opts_mut_reset_type),
screamer 0:66f3b5499f7f 313 "jobs" : str(self.opts_jobs),
screamer 0:66f3b5499f7f 314 "extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer 0:66f3b5499f7f 315 "_dummy" : ''
screamer 0:66f3b5499f7f 316 }
screamer 0:66f3b5499f7f 317 return result
screamer 0:66f3b5499f7f 318
screamer 0:66f3b5499f7f 319 def shuffle_random_func(self):
screamer 0:66f3b5499f7f 320 return self.shuffle_random_seed
screamer 0:66f3b5499f7f 321
screamer 0:66f3b5499f7f 322 def is_shuffle_seed_float(self):
screamer 0:66f3b5499f7f 323 """ return true if function parameter can be converted to float
screamer 0:66f3b5499f7f 324 """
screamer 0:66f3b5499f7f 325 result = True
screamer 0:66f3b5499f7f 326 try:
screamer 0:66f3b5499f7f 327 float(self.shuffle_random_seed)
screamer 0:66f3b5499f7f 328 except ValueError:
screamer 0:66f3b5499f7f 329 result = False
screamer 0:66f3b5499f7f 330 return result
screamer 0:66f3b5499f7f 331
screamer 0:66f3b5499f7f 332 # This will store target / toolchain specific properties
screamer 0:66f3b5499f7f 333 test_suite_properties_ext = {} # target : toolchain
screamer 0:66f3b5499f7f 334 # Here we store test results
screamer 0:66f3b5499f7f 335 test_summary = []
screamer 0:66f3b5499f7f 336 # Here we store test results in extended data structure
screamer 0:66f3b5499f7f 337 test_summary_ext = {}
screamer 0:66f3b5499f7f 338 execute_thread_slice_lock = Lock()
screamer 0:66f3b5499f7f 339
screamer 0:66f3b5499f7f 340 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer 0:66f3b5499f7f 341 for toolchain in toolchains:
screamer 0:66f3b5499f7f 342 tt_id = "%s::%s" % (toolchain, target)
screamer 0:66f3b5499f7f 343
screamer 0:66f3b5499f7f 344 T = TARGET_MAP[target]
screamer 0:66f3b5499f7f 345
screamer 0:66f3b5499f7f 346 # print target, toolchain
screamer 0:66f3b5499f7f 347 # Test suite properties returned to external tools like CI
screamer 0:66f3b5499f7f 348 test_suite_properties = {
screamer 0:66f3b5499f7f 349 'jobs': self.opts_jobs,
screamer 0:66f3b5499f7f 350 'clean': clean,
screamer 0:66f3b5499f7f 351 'target': target,
screamer 0:66f3b5499f7f 352 'vendor': T.extra_labels[0],
screamer 0:66f3b5499f7f 353 'test_ids': ', '.join(test_ids),
screamer 0:66f3b5499f7f 354 'toolchain': toolchain,
screamer 0:66f3b5499f7f 355 'shuffle_random_seed': self.shuffle_random_seed
screamer 0:66f3b5499f7f 356 }
screamer 0:66f3b5499f7f 357
screamer 0:66f3b5499f7f 358
screamer 0:66f3b5499f7f 359 # print '=== %s::%s ===' % (target, toolchain)
screamer 0:66f3b5499f7f 360 # Let's build our test
screamer 0:66f3b5499f7f 361 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 362 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
screamer 0:66f3b5499f7f 363 continue
screamer 0:66f3b5499f7f 364
screamer 0:66f3b5499f7f 365 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 366
The Other Jimmy 31:8ea194f6145b 367 profile = extract_profile(self.opts_parser, self.opts, toolchain)
theotherjimmy 40:7d3fa6b99b2b 368 stats_depth = self.opts.stats_depth or 2
The Other Jimmy 31:8ea194f6145b 369
screamer 0:66f3b5499f7f 370
screamer 0:66f3b5499f7f 371 try:
screamer 0:66f3b5499f7f 372 build_mbed_libs_result = build_mbed_libs(T,
screamer 0:66f3b5499f7f 373 toolchain,
screamer 0:66f3b5499f7f 374 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 375 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 376 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 377 report=build_report,
The Other Jimmy 31:8ea194f6145b 378 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 379 build_profile=profile)
screamer 0:66f3b5499f7f 380
screamer 0:66f3b5499f7f 381 if not build_mbed_libs_result:
screamer 0:66f3b5499f7f 382 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
screamer 0:66f3b5499f7f 383 continue
screamer 0:66f3b5499f7f 384
screamer 0:66f3b5499f7f 385 except ToolException:
screamer 0:66f3b5499f7f 386 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
screamer 0:66f3b5499f7f 387 continue
screamer 0:66f3b5499f7f 388
screamer 0:66f3b5499f7f 389 build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer 0:66f3b5499f7f 390
screamer 0:66f3b5499f7f 391 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer 0:66f3b5499f7f 392 test_suite_properties['build_dir'] = build_dir
screamer 0:66f3b5499f7f 393 test_suite_properties['skipped'] = []
screamer 0:66f3b5499f7f 394
screamer 0:66f3b5499f7f 395 # Enumerate through all tests and shuffle test order if requested
screamer 0:66f3b5499f7f 396 test_map_keys = sorted(TEST_MAP.keys())
screamer 0:66f3b5499f7f 397
screamer 0:66f3b5499f7f 398 if self.opts_shuffle_test_order:
screamer 0:66f3b5499f7f 399 random.shuffle(test_map_keys, self.shuffle_random_func)
screamer 0:66f3b5499f7f 400 # Update database with shuffle seed f applicable
screamer 0:66f3b5499f7f 401 if self.db_logger:
screamer 0:66f3b5499f7f 402 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 403 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 404 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
screamer 0:66f3b5499f7f 405 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 406
screamer 0:66f3b5499f7f 407 if self.db_logger:
screamer 0:66f3b5499f7f 408 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 409 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 410 # Update MUTs and Test Specification in database
screamer 0:66f3b5499f7f 411 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
screamer 0:66f3b5499f7f 412 # Update Extra information in database (some options passed to test suite)
screamer 0:66f3b5499f7f 413 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
screamer 0:66f3b5499f7f 414 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 415
screamer 0:66f3b5499f7f 416 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer 0:66f3b5499f7f 417 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer 0:66f3b5499f7f 418
screamer 0:66f3b5499f7f 419 for skipped_test_id in skipped_test_map_keys:
screamer 0:66f3b5499f7f 420 test_suite_properties['skipped'].append(skipped_test_id)
screamer 0:66f3b5499f7f 421
screamer 0:66f3b5499f7f 422
screamer 0:66f3b5499f7f 423 # First pass through all tests and determine which libraries need to be built
screamer 0:66f3b5499f7f 424 libraries = []
screamer 0:66f3b5499f7f 425 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 426 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 427
screamer 0:66f3b5499f7f 428 # Detect which lib should be added to test
screamer 0:66f3b5499f7f 429 # Some libs have to compiled like RTOS or ETH
screamer 0:66f3b5499f7f 430 for lib in LIBRARIES:
screamer 0:66f3b5499f7f 431 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer 0:66f3b5499f7f 432 libraries.append(lib['id'])
screamer 0:66f3b5499f7f 433
screamer 0:66f3b5499f7f 434
screamer 0:66f3b5499f7f 435 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 436
screamer 0:66f3b5499f7f 437 # Build all required libraries
screamer 0:66f3b5499f7f 438 for lib_id in libraries:
screamer 0:66f3b5499f7f 439 try:
screamer 0:66f3b5499f7f 440 build_lib(lib_id,
screamer 0:66f3b5499f7f 441 T,
screamer 0:66f3b5499f7f 442 toolchain,
screamer 0:66f3b5499f7f 443 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 444 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 445 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 446 report=build_report,
The Other Jimmy 31:8ea194f6145b 447 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 448 build_profile=profile)
screamer 0:66f3b5499f7f 449
screamer 0:66f3b5499f7f 450 except ToolException:
screamer 0:66f3b5499f7f 451 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
screamer 0:66f3b5499f7f 452 continue
screamer 0:66f3b5499f7f 453
screamer 0:66f3b5499f7f 454
screamer 0:66f3b5499f7f 455 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 456 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 457
screamer 0:66f3b5499f7f 458 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer 0:66f3b5499f7f 459
screamer 0:66f3b5499f7f 460 # TODO: move this 2 below loops to separate function
screamer 0:66f3b5499f7f 461 INC_DIRS = []
screamer 0:66f3b5499f7f 462 for lib_id in libraries:
screamer 0:66f3b5499f7f 463 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer 0:66f3b5499f7f 464 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer 0:66f3b5499f7f 465
screamer 0:66f3b5499f7f 466 MACROS = []
screamer 0:66f3b5499f7f 467 for lib_id in libraries:
screamer 0:66f3b5499f7f 468 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer 0:66f3b5499f7f 469 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer 0:66f3b5499f7f 470 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer 0:66f3b5499f7f 471 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer 0:66f3b5499f7f 472 test_uuid = uuid.uuid4()
screamer 0:66f3b5499f7f 473 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer 0:66f3b5499f7f 474
screamer 0:66f3b5499f7f 475 # Prepare extended test results data structure (it can be used to generate detailed test report)
screamer 0:66f3b5499f7f 476 if target not in self.test_summary_ext:
screamer 0:66f3b5499f7f 477 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer 0:66f3b5499f7f 478 if toolchain not in self.test_summary_ext[target]:
screamer 0:66f3b5499f7f 479 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer 0:66f3b5499f7f 480
screamer 0:66f3b5499f7f 481 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer 0:66f3b5499f7f 482
screamer 0:66f3b5499f7f 483 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer 0:66f3b5499f7f 484 try:
theotherjimmy 40:7d3fa6b99b2b 485 path = build_project(test.source_dir, join(build_dir, test_id), T,
theotherjimmy 40:7d3fa6b99b2b 486 toolchain, test.dependencies, clean=clean_project_options,
theotherjimmy 40:7d3fa6b99b2b 487 verbose=self.opts_verbose, name=project_name, macros=MACROS,
theotherjimmy 40:7d3fa6b99b2b 488 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
theotherjimmy 40:7d3fa6b99b2b 489 properties=build_properties, project_id=test_id,
theotherjimmy 40:7d3fa6b99b2b 490 project_description=test.get_description(),
theotherjimmy 40:7d3fa6b99b2b 491 build_profile=profile, stats_depth=stats_depth)
screamer 0:66f3b5499f7f 492
screamer 0:66f3b5499f7f 493 except Exception, e:
screamer 0:66f3b5499f7f 494 project_name_str = project_name if project_name is not None else test_id
screamer 0:66f3b5499f7f 495
screamer 0:66f3b5499f7f 496
screamer 0:66f3b5499f7f 497 test_result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 498
screamer 0:66f3b5499f7f 499 if isinstance(e, ToolException):
screamer 0:66f3b5499f7f 500 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
screamer 0:66f3b5499f7f 501 test_result = self.TEST_RESULT_BUILD_FAILED
screamer 0:66f3b5499f7f 502 elif isinstance(e, NotSupportedException):
screamer 0:66f3b5499f7f 503 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
screamer 0:66f3b5499f7f 504 test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 505
screamer 0:66f3b5499f7f 506
screamer 0:66f3b5499f7f 507 # Append test results to global test summary
screamer 0:66f3b5499f7f 508 self.test_summary.append(
screamer 0:66f3b5499f7f 509 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
screamer 0:66f3b5499f7f 510 )
screamer 0:66f3b5499f7f 511
screamer 0:66f3b5499f7f 512 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 513 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 514 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 515
screamer 0:66f3b5499f7f 516 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer 0:66f3b5499f7f 517 'result' : test_result,
screamer 0:66f3b5499f7f 518 'output' : '',
screamer 0:66f3b5499f7f 519 'target_name' : target,
screamer 0:66f3b5499f7f 520 'target_name_unique': target,
screamer 0:66f3b5499f7f 521 'toolchain_name' : toolchain,
screamer 0:66f3b5499f7f 522 'id' : test_id,
screamer 0:66f3b5499f7f 523 'description' : test.get_description(),
screamer 0:66f3b5499f7f 524 'elapsed_time' : 0,
screamer 0:66f3b5499f7f 525 'duration' : 0,
screamer 0:66f3b5499f7f 526 'copy_method' : None
screamer 0:66f3b5499f7f 527 }})
screamer 0:66f3b5499f7f 528 continue
screamer 0:66f3b5499f7f 529
screamer 0:66f3b5499f7f 530 if self.opts_only_build_tests:
screamer 0:66f3b5499f7f 531 # With this option we are skipping testing phase
screamer 0:66f3b5499f7f 532 continue
screamer 0:66f3b5499f7f 533
screamer 0:66f3b5499f7f 534 # Test duration can be increased by global value
screamer 0:66f3b5499f7f 535 test_duration = test.duration
screamer 0:66f3b5499f7f 536 if self.opts_extend_test_timeout is not None:
screamer 0:66f3b5499f7f 537 test_duration += self.opts_extend_test_timeout
screamer 0:66f3b5499f7f 538
screamer 0:66f3b5499f7f 539 # For an automated test the duration act as a timeout after
screamer 0:66f3b5499f7f 540 # which the test gets interrupted
screamer 0:66f3b5499f7f 541 test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer 0:66f3b5499f7f 542 test_loops = self.get_test_loop_count(test_id)
screamer 0:66f3b5499f7f 543
screamer 0:66f3b5499f7f 544 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer 0:66f3b5499f7f 545 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer 0:66f3b5499f7f 546 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer 0:66f3b5499f7f 547
screamer 0:66f3b5499f7f 548 # read MUTs, test specification and perform tests
screamer 0:66f3b5499f7f 549 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer 0:66f3b5499f7f 550
screamer 0:66f3b5499f7f 551 if handle_results is None:
screamer 0:66f3b5499f7f 552 continue
screamer 0:66f3b5499f7f 553
screamer 0:66f3b5499f7f 554 for handle_result in handle_results:
screamer 0:66f3b5499f7f 555 if handle_result:
screamer 0:66f3b5499f7f 556 single_test_result, detailed_test_results = handle_result
screamer 0:66f3b5499f7f 557 else:
screamer 0:66f3b5499f7f 558 continue
screamer 0:66f3b5499f7f 559
screamer 0:66f3b5499f7f 560 # Append test results to global test summary
screamer 0:66f3b5499f7f 561 if single_test_result is not None:
screamer 0:66f3b5499f7f 562 self.test_summary.append(single_test_result)
screamer 0:66f3b5499f7f 563
screamer 0:66f3b5499f7f 564 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 565 if target not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 566 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 567 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 568
screamer 0:66f3b5499f7f 569 append_test_result = detailed_test_results
screamer 0:66f3b5499f7f 570
screamer 0:66f3b5499f7f 571 # If waterfall and consolidate-waterfall options are enabled,
screamer 0:66f3b5499f7f 572 # only include the last test result in the report.
screamer 0:66f3b5499f7f 573 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer 0:66f3b5499f7f 574 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer 0:66f3b5499f7f 575
screamer 0:66f3b5499f7f 576 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer 0:66f3b5499f7f 577
screamer 0:66f3b5499f7f 578 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer 0:66f3b5499f7f 579 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer 0:66f3b5499f7f 580
screamer 0:66f3b5499f7f 581 q.put(target + '_'.join(toolchains))
screamer 0:66f3b5499f7f 582 return
screamer 0:66f3b5499f7f 583
screamer 0:66f3b5499f7f 584 def execute(self):
screamer 0:66f3b5499f7f 585 clean = self.test_spec.get('clean', False)
screamer 0:66f3b5499f7f 586 test_ids = self.test_spec.get('test_ids', [])
screamer 0:66f3b5499f7f 587 q = Queue()
screamer 0:66f3b5499f7f 588
screamer 0:66f3b5499f7f 589 # Generate seed for shuffle if seed is not provided in
screamer 0:66f3b5499f7f 590 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 591 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer 0:66f3b5499f7f 592 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 593
screamer 0:66f3b5499f7f 594
screamer 0:66f3b5499f7f 595 if self.opts_parallel_test_exec:
screamer 0:66f3b5499f7f 596 ###################################################################
screamer 0:66f3b5499f7f 597 # Experimental, parallel test execution per singletest instance.
screamer 0:66f3b5499f7f 598 ###################################################################
screamer 0:66f3b5499f7f 599 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer 0:66f3b5499f7f 600 # Note: We are building here in parallel for each target separately!
screamer 0:66f3b5499f7f 601 # So we are not building the same thing multiple times and compilers
screamer 0:66f3b5499f7f 602 # in separate threads do not collide.
screamer 0:66f3b5499f7f 603 # Inside execute_thread_slice() function function handle() will be called to
screamer 0:66f3b5499f7f 604 # get information about available MUTs (per target).
screamer 0:66f3b5499f7f 605 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 606 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 607 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer 0:66f3b5499f7f 608 t.daemon = True
screamer 0:66f3b5499f7f 609 t.start()
screamer 0:66f3b5499f7f 610 execute_threads.append(t)
screamer 0:66f3b5499f7f 611
screamer 0:66f3b5499f7f 612 for t in execute_threads:
screamer 0:66f3b5499f7f 613 q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer 0:66f3b5499f7f 614 else:
screamer 0:66f3b5499f7f 615 # Serialized (not parallel) test execution
screamer 0:66f3b5499f7f 616 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 617 if target not in self.test_suite_properties_ext:
screamer 0:66f3b5499f7f 618 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 619
screamer 0:66f3b5499f7f 620 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer 0:66f3b5499f7f 621 q.get()
screamer 0:66f3b5499f7f 622
screamer 0:66f3b5499f7f 623 if self.db_logger:
screamer 0:66f3b5499f7f 624 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 625 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 626 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer 0:66f3b5499f7f 627 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 628
screamer 0:66f3b5499f7f 629 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer 0:66f3b5499f7f 630
screamer 0:66f3b5499f7f 631 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer 0:66f3b5499f7f 632 valid_test_map_keys = []
screamer 0:66f3b5499f7f 633
screamer 0:66f3b5499f7f 634 for test_id in test_map_keys:
screamer 0:66f3b5499f7f 635 test = TEST_MAP[test_id]
screamer 22:9e85236d8716 636 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
screamer 0:66f3b5499f7f 637 continue
screamer 0:66f3b5499f7f 638
screamer 0:66f3b5499f7f 639 if test_ids and test_id not in test_ids:
screamer 0:66f3b5499f7f 640 continue
screamer 0:66f3b5499f7f 641
screamer 0:66f3b5499f7f 642 if self.opts_test_only_peripheral and not test.peripherals:
screamer 0:66f3b5499f7f 643 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 644 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 645 continue
screamer 0:66f3b5499f7f 646
screamer 22:9e85236d8716 647 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
screamer 0:66f3b5499f7f 648 # We will skip tests not forced with -p option
screamer 0:66f3b5499f7f 649 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 650 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 651 continue
screamer 0:66f3b5499f7f 652
screamer 0:66f3b5499f7f 653 if self.opts_test_only_common and test.peripherals:
screamer 0:66f3b5499f7f 654 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 655 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 656 continue
screamer 0:66f3b5499f7f 657
screamer 0:66f3b5499f7f 658 if not include_non_automated and not test.automated:
screamer 0:66f3b5499f7f 659 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 660 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 661 continue
screamer 0:66f3b5499f7f 662
screamer 0:66f3b5499f7f 663 if test.is_supported(target, toolchain):
screamer 0:66f3b5499f7f 664 if test.peripherals is None and self.opts_only_build_tests:
screamer 0:66f3b5499f7f 665 # When users are using 'build only flag' and test do not have
screamer 0:66f3b5499f7f 666 # specified peripherals we can allow test building by default
screamer 0:66f3b5499f7f 667 pass
screamer 22:9e85236d8716 668 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
screamer 0:66f3b5499f7f 669 # If we force peripheral with option -p we expect test
screamer 0:66f3b5499f7f 670 # to pass even if peripheral is not in MUTs file.
screamer 0:66f3b5499f7f 671 pass
screamer 0:66f3b5499f7f 672 elif not self.is_peripherals_available(target, test.peripherals):
screamer 0:66f3b5499f7f 673 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 674 if test.peripherals:
screamer 0:66f3b5499f7f 675 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
screamer 0:66f3b5499f7f 676 else:
screamer 0:66f3b5499f7f 677 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
screamer 0:66f3b5499f7f 678 continue
screamer 0:66f3b5499f7f 679
screamer 0:66f3b5499f7f 680 # The test has made it through all the filters, so add it to the valid tests list
screamer 0:66f3b5499f7f 681 valid_test_map_keys.append(test_id)
screamer 0:66f3b5499f7f 682
screamer 0:66f3b5499f7f 683 return valid_test_map_keys
screamer 0:66f3b5499f7f 684
screamer 0:66f3b5499f7f 685 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer 0:66f3b5499f7f 686 # NOTE: This will not preserve order
screamer 0:66f3b5499f7f 687 return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer 0:66f3b5499f7f 688
screamer 0:66f3b5499f7f 689 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 690 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 691 table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 692 """
screamer 0:66f3b5499f7f 693 RESULT_INDEX = 0
screamer 0:66f3b5499f7f 694 TARGET_INDEX = 1
screamer 0:66f3b5499f7f 695 TOOLCHAIN_INDEX = 2
screamer 0:66f3b5499f7f 696 TEST_INDEX = 3
screamer 0:66f3b5499f7f 697 DESC_INDEX = 4
screamer 0:66f3b5499f7f 698
screamer 0:66f3b5499f7f 699 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer 0:66f3b5499f7f 700 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer 0:66f3b5499f7f 701 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer 0:66f3b5499f7f 702 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer 0:66f3b5499f7f 703
screamer 0:66f3b5499f7f 704 result = "Test summary:\n"
screamer 0:66f3b5499f7f 705 for target in unique_targets:
screamer 0:66f3b5499f7f 706 result_dict = {} # test : { toolchain : result }
screamer 0:66f3b5499f7f 707 unique_target_toolchains = []
screamer 0:66f3b5499f7f 708 for test in test_summary:
screamer 0:66f3b5499f7f 709 if test[TARGET_INDEX] == target:
screamer 0:66f3b5499f7f 710 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer 0:66f3b5499f7f 711 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer 0:66f3b5499f7f 712 if test[TEST_INDEX] not in result_dict:
screamer 0:66f3b5499f7f 713 result_dict[test[TEST_INDEX]] = {}
screamer 0:66f3b5499f7f 714 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer 0:66f3b5499f7f 715
screamer 0:66f3b5499f7f 716 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
screamer 0:66f3b5499f7f 717 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 718 for col in pt_cols:
screamer 0:66f3b5499f7f 719 pt.align[col] = "l"
screamer 0:66f3b5499f7f 720 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 721
screamer 0:66f3b5499f7f 722 for test in unique_tests:
screamer 0:66f3b5499f7f 723 if test in result_dict:
screamer 0:66f3b5499f7f 724 test_results = result_dict[test]
screamer 0:66f3b5499f7f 725 if test in unique_test_desc:
screamer 0:66f3b5499f7f 726 row = [target, test, unique_test_desc[test]]
screamer 0:66f3b5499f7f 727 for toolchain in unique_toolchains:
screamer 0:66f3b5499f7f 728 if toolchain in test_results:
screamer 0:66f3b5499f7f 729 row.append(test_results[toolchain])
screamer 0:66f3b5499f7f 730 pt.add_row(row)
screamer 0:66f3b5499f7f 731 result += pt.get_string()
screamer 0:66f3b5499f7f 732 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 733 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 734 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 735 return result
screamer 0:66f3b5499f7f 736
screamer 0:66f3b5499f7f 737 def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 738 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 739 table shows target x test results matrix across
screamer 0:66f3b5499f7f 740 """
screamer 0:66f3b5499f7f 741 success_code = 0 # Success code that can be leter returned to
screamer 0:66f3b5499f7f 742 result = "Test summary:\n"
screamer 0:66f3b5499f7f 743 # Pretty table package is used to print results
screamer 0:66f3b5499f7f 744 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
screamer 0:66f3b5499f7f 745 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
screamer 0:66f3b5499f7f 746 pt.align["Result"] = "l" # Left align
screamer 0:66f3b5499f7f 747 pt.align["Target"] = "l" # Left align
screamer 0:66f3b5499f7f 748 pt.align["Toolchain"] = "l" # Left align
screamer 0:66f3b5499f7f 749 pt.align["Test ID"] = "l" # Left align
screamer 0:66f3b5499f7f 750 pt.align["Test Description"] = "l" # Left align
screamer 0:66f3b5499f7f 751 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 752
screamer 0:66f3b5499f7f 753 result_dict = {self.TEST_RESULT_OK : 0,
screamer 0:66f3b5499f7f 754 self.TEST_RESULT_FAIL : 0,
screamer 0:66f3b5499f7f 755 self.TEST_RESULT_ERROR : 0,
screamer 0:66f3b5499f7f 756 self.TEST_RESULT_UNDEF : 0,
screamer 0:66f3b5499f7f 757 self.TEST_RESULT_IOERR_COPY : 0,
screamer 0:66f3b5499f7f 758 self.TEST_RESULT_IOERR_DISK : 0,
screamer 0:66f3b5499f7f 759 self.TEST_RESULT_IOERR_SERIAL : 0,
screamer 0:66f3b5499f7f 760 self.TEST_RESULT_NO_IMAGE : 0,
screamer 0:66f3b5499f7f 761 self.TEST_RESULT_TIMEOUT : 0,
screamer 0:66f3b5499f7f 762 self.TEST_RESULT_MBED_ASSERT : 0,
screamer 0:66f3b5499f7f 763 self.TEST_RESULT_BUILD_FAILED : 0,
screamer 0:66f3b5499f7f 764 self.TEST_RESULT_NOT_SUPPORTED : 0
screamer 0:66f3b5499f7f 765 }
screamer 0:66f3b5499f7f 766
screamer 0:66f3b5499f7f 767 for test in test_summary:
screamer 0:66f3b5499f7f 768 if test[0] in result_dict:
screamer 0:66f3b5499f7f 769 result_dict[test[0]] += 1
screamer 0:66f3b5499f7f 770 pt.add_row(test)
screamer 0:66f3b5499f7f 771 result += pt.get_string()
screamer 0:66f3b5499f7f 772 result += "\n"
screamer 0:66f3b5499f7f 773
screamer 0:66f3b5499f7f 774 # Print result count
screamer 0:66f3b5499f7f 775 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
screamer 0:66f3b5499f7f 776 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 777 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 778 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 779 return result
screamer 0:66f3b5499f7f 780
screamer 0:66f3b5499f7f 781 def test_loop_list_to_dict(self, test_loops_str):
screamer 0:66f3b5499f7f 782 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer 0:66f3b5499f7f 783 """
screamer 0:66f3b5499f7f 784 result = {}
screamer 0:66f3b5499f7f 785 if test_loops_str:
screamer 22:9e85236d8716 786 test_loops = test_loops_str
screamer 0:66f3b5499f7f 787 for test_loop in test_loops:
screamer 0:66f3b5499f7f 788 test_loop_count = test_loop.split('=')
screamer 0:66f3b5499f7f 789 if len(test_loop_count) == 2:
screamer 0:66f3b5499f7f 790 _test_id, _test_loops = test_loop_count
screamer 0:66f3b5499f7f 791 try:
screamer 0:66f3b5499f7f 792 _test_loops = int(_test_loops)
screamer 0:66f3b5499f7f 793 except:
screamer 0:66f3b5499f7f 794 continue
screamer 0:66f3b5499f7f 795 result[_test_id] = _test_loops
screamer 0:66f3b5499f7f 796 return result
screamer 0:66f3b5499f7f 797
screamer 0:66f3b5499f7f 798 def get_test_loop_count(self, test_id):
screamer 0:66f3b5499f7f 799 """ This function returns no. of loops per test (deducted by test_id_.
screamer 0:66f3b5499f7f 800 If test is not in list of redefined loop counts it will use default value.
screamer 0:66f3b5499f7f 801 """
screamer 0:66f3b5499f7f 802 result = self.GLOBAL_LOOPS_COUNT
screamer 0:66f3b5499f7f 803 if test_id in self.TEST_LOOPS_DICT:
screamer 0:66f3b5499f7f 804 result = self.TEST_LOOPS_DICT[test_id]
screamer 0:66f3b5499f7f 805 return result
screamer 0:66f3b5499f7f 806
screamer 0:66f3b5499f7f 807 def delete_file(self, file_path):
screamer 0:66f3b5499f7f 808 """ Remove file from the system
screamer 0:66f3b5499f7f 809 """
screamer 0:66f3b5499f7f 810 result = True
screamer 0:66f3b5499f7f 811 resutl_msg = ""
screamer 0:66f3b5499f7f 812 try:
screamer 0:66f3b5499f7f 813 os.remove(file_path)
screamer 0:66f3b5499f7f 814 except Exception, e:
screamer 0:66f3b5499f7f 815 resutl_msg = e
screamer 0:66f3b5499f7f 816 result = False
screamer 0:66f3b5499f7f 817 return result, resutl_msg
screamer 0:66f3b5499f7f 818
screamer 0:66f3b5499f7f 819 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 820 """ Test is being invoked for given MUT.
screamer 0:66f3b5499f7f 821 """
screamer 0:66f3b5499f7f 822 # Get test information, image and test timeout
screamer 0:66f3b5499f7f 823 test_id = data['test_id']
screamer 0:66f3b5499f7f 824 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 825 test_description = TEST_MAP[test_id].get_description()
screamer 0:66f3b5499f7f 826 image = data["image"]
screamer 0:66f3b5499f7f 827 duration = data.get("duration", 10)
screamer 0:66f3b5499f7f 828
screamer 0:66f3b5499f7f 829 if mut is None:
screamer 0:66f3b5499f7f 830 print "Error: No Mbed available: MUT[%s]" % data['mcu']
screamer 0:66f3b5499f7f 831 return None
screamer 0:66f3b5499f7f 832
screamer 0:66f3b5499f7f 833 mcu = mut['mcu']
screamer 0:66f3b5499f7f 834 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer 0:66f3b5499f7f 835
screamer 0:66f3b5499f7f 836 if self.db_logger:
screamer 0:66f3b5499f7f 837 self.db_logger.reconnect()
screamer 0:66f3b5499f7f 838
screamer 0:66f3b5499f7f 839 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer 0:66f3b5499f7f 840
screamer 0:66f3b5499f7f 841 # Tests can be looped so test results must be stored for the same test
screamer 0:66f3b5499f7f 842 test_all_result = []
screamer 0:66f3b5499f7f 843 # Test results for one test ran few times
screamer 0:66f3b5499f7f 844 detailed_test_results = {} # { Loop_number: { results ... } }
screamer 0:66f3b5499f7f 845
screamer 0:66f3b5499f7f 846 for test_index in range(test_loops):
screamer 0:66f3b5499f7f 847
screamer 0:66f3b5499f7f 848 # If mbedls is available and we are auto detecting MUT info,
screamer 0:66f3b5499f7f 849 # update MUT info (mounting may changed)
screamer 0:66f3b5499f7f 850 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 851 platform_name_filter = [mcu]
screamer 0:66f3b5499f7f 852 muts_list = {}
screamer 0:66f3b5499f7f 853 found = False
screamer 0:66f3b5499f7f 854
screamer 0:66f3b5499f7f 855 for i in range(0, 60):
screamer 0:66f3b5499f7f 856 print('Looking for %s with MBEDLS' % mcu)
screamer 0:66f3b5499f7f 857 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 858
screamer 0:66f3b5499f7f 859 if 1 not in muts_list:
screamer 0:66f3b5499f7f 860 sleep(3)
screamer 0:66f3b5499f7f 861 else:
screamer 0:66f3b5499f7f 862 found = True
screamer 0:66f3b5499f7f 863 break
screamer 0:66f3b5499f7f 864
screamer 0:66f3b5499f7f 865 if not found:
screamer 0:66f3b5499f7f 866 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
screamer 0:66f3b5499f7f 867 return None
screamer 0:66f3b5499f7f 868 else:
screamer 0:66f3b5499f7f 869 mut = muts_list[1]
screamer 0:66f3b5499f7f 870
screamer 0:66f3b5499f7f 871 disk = mut.get('disk')
screamer 0:66f3b5499f7f 872 port = mut.get('port')
screamer 0:66f3b5499f7f 873
screamer 0:66f3b5499f7f 874 if disk is None or port is None:
screamer 0:66f3b5499f7f 875 return None
screamer 0:66f3b5499f7f 876
screamer 0:66f3b5499f7f 877 target_by_mcu = TARGET_MAP[mut['mcu']]
screamer 0:66f3b5499f7f 878 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer 0:66f3b5499f7f 879 # Some extra stuff can be declared in MUTs structure
screamer 0:66f3b5499f7f 880 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer 0:66f3b5499f7f 881 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer 0:66f3b5499f7f 882
screamer 0:66f3b5499f7f 883 # When the build and test system were separate, this was relative to a
screamer 0:66f3b5499f7f 884 # base network folder base path: join(NETWORK_BASE_PATH, )
screamer 0:66f3b5499f7f 885 image_path = image
screamer 0:66f3b5499f7f 886
screamer 0:66f3b5499f7f 887 # Host test execution
screamer 0:66f3b5499f7f 888 start_host_exec_time = time()
screamer 0:66f3b5499f7f 889
screamer 0:66f3b5499f7f 890 single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer 0:66f3b5499f7f 891 _copy_method = selected_copy_method
screamer 0:66f3b5499f7f 892
screamer 0:66f3b5499f7f 893 if not exists(image_path):
screamer 0:66f3b5499f7f 894 single_test_result = self.TEST_RESULT_NO_IMAGE
screamer 0:66f3b5499f7f 895 elapsed_time = 0
screamer 0:66f3b5499f7f 896 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
screamer 0:66f3b5499f7f 897 print single_test_output
screamer 0:66f3b5499f7f 898 else:
screamer 0:66f3b5499f7f 899 # Host test execution
screamer 0:66f3b5499f7f 900 start_host_exec_time = time()
screamer 0:66f3b5499f7f 901
screamer 0:66f3b5499f7f 902 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer 0:66f3b5499f7f 903 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer 0:66f3b5499f7f 904 host_test_result = self.run_host_test(test.host_test,
screamer 0:66f3b5499f7f 905 image_path, disk, port, duration,
screamer 0:66f3b5499f7f 906 micro=target_name,
screamer 0:66f3b5499f7f 907 verbose=host_test_verbose,
screamer 0:66f3b5499f7f 908 reset=host_test_reset,
screamer 0:66f3b5499f7f 909 reset_tout=reset_tout,
screamer 0:66f3b5499f7f 910 copy_method=selected_copy_method,
screamer 13:ab47a20b66f0 911 program_cycle_s=target_by_mcu.program_cycle_s)
screamer 0:66f3b5499f7f 912 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer 0:66f3b5499f7f 913
screamer 0:66f3b5499f7f 914 # Store test result
screamer 0:66f3b5499f7f 915 test_all_result.append(single_test_result)
screamer 0:66f3b5499f7f 916 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer 0:66f3b5499f7f 917 elapsed_time = single_testduration # TIme of single test case execution after reset
screamer 0:66f3b5499f7f 918
screamer 0:66f3b5499f7f 919 detailed_test_results[test_index] = {
screamer 0:66f3b5499f7f 920 'result' : single_test_result,
screamer 0:66f3b5499f7f 921 'output' : single_test_output,
screamer 0:66f3b5499f7f 922 'target_name' : target_name,
screamer 0:66f3b5499f7f 923 'target_name_unique' : target_name_unique,
screamer 0:66f3b5499f7f 924 'toolchain_name' : toolchain_name,
screamer 0:66f3b5499f7f 925 'id' : test_id,
screamer 0:66f3b5499f7f 926 'description' : test_description,
screamer 0:66f3b5499f7f 927 'elapsed_time' : round(elapsed_time, 2),
screamer 0:66f3b5499f7f 928 'duration' : single_timeout,
screamer 0:66f3b5499f7f 929 'copy_method' : _copy_method,
screamer 0:66f3b5499f7f 930 }
screamer 0:66f3b5499f7f 931
screamer 0:66f3b5499f7f 932 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
screamer 0:66f3b5499f7f 933 test_id, test_description, elapsed_time, single_timeout)
screamer 0:66f3b5499f7f 934
screamer 0:66f3b5499f7f 935 # Update database entries for ongoing test
screamer 0:66f3b5499f7f 936 if self.db_logger and self.db_logger.is_connected():
screamer 0:66f3b5499f7f 937 test_type = 'SingleTest'
screamer 0:66f3b5499f7f 938 self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer 0:66f3b5499f7f 939 target_name,
screamer 0:66f3b5499f7f 940 toolchain_name,
screamer 0:66f3b5499f7f 941 test_type,
screamer 0:66f3b5499f7f 942 test_id,
screamer 0:66f3b5499f7f 943 single_test_result,
screamer 0:66f3b5499f7f 944 single_test_output,
screamer 0:66f3b5499f7f 945 elapsed_time,
screamer 0:66f3b5499f7f 946 single_timeout,
screamer 0:66f3b5499f7f 947 test_index)
screamer 0:66f3b5499f7f 948
screamer 0:66f3b5499f7f 949 # If we perform waterfall test we test until we get OK and we stop testing
screamer 0:66f3b5499f7f 950 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer 0:66f3b5499f7f 951 break
screamer 0:66f3b5499f7f 952
screamer 0:66f3b5499f7f 953 if self.db_logger:
screamer 0:66f3b5499f7f 954 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 955
screamer 0:66f3b5499f7f 956 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer 0:66f3b5499f7f 957 target_name_unique,
screamer 0:66f3b5499f7f 958 toolchain_name,
screamer 0:66f3b5499f7f 959 test_id,
screamer 0:66f3b5499f7f 960 test_description,
screamer 0:66f3b5499f7f 961 round(elapsed_time, 2),
screamer 0:66f3b5499f7f 962 single_timeout,
screamer 0:66f3b5499f7f 963 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer 0:66f3b5499f7f 964
screamer 0:66f3b5499f7f 965 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 966 """ Function determines MUT's mbed disk/port and copies binary to
screamer 0:66f3b5499f7f 967 target.
screamer 0:66f3b5499f7f 968 """
screamer 0:66f3b5499f7f 969 handle_results = []
screamer 0:66f3b5499f7f 970 data = json.loads(test_spec)
screamer 0:66f3b5499f7f 971
screamer 0:66f3b5499f7f 972 # Find a suitable MUT:
screamer 0:66f3b5499f7f 973 mut = None
screamer 0:66f3b5499f7f 974 for id, m in self.muts.iteritems():
screamer 0:66f3b5499f7f 975 if m['mcu'] == data['mcu']:
screamer 0:66f3b5499f7f 976 mut = m
screamer 0:66f3b5499f7f 977 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer 0:66f3b5499f7f 978 handle_results.append(handle_result)
screamer 0:66f3b5499f7f 979
screamer 0:66f3b5499f7f 980 return handle_results
screamer 0:66f3b5499f7f 981
screamer 0:66f3b5499f7f 982 def print_test_result(self, test_result, target_name, toolchain_name,
screamer 0:66f3b5499f7f 983 test_id, test_description, elapsed_time, duration):
screamer 0:66f3b5499f7f 984 """ Use specific convention to print test result and related data
screamer 0:66f3b5499f7f 985 """
screamer 0:66f3b5499f7f 986 tokens = []
screamer 0:66f3b5499f7f 987 tokens.append("TargetTest")
screamer 0:66f3b5499f7f 988 tokens.append(target_name)
screamer 0:66f3b5499f7f 989 tokens.append(toolchain_name)
screamer 0:66f3b5499f7f 990 tokens.append(test_id)
screamer 0:66f3b5499f7f 991 tokens.append(test_description)
screamer 0:66f3b5499f7f 992 separator = "::"
screamer 0:66f3b5499f7f 993 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer 0:66f3b5499f7f 994 result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer 0:66f3b5499f7f 995 return Fore.MAGENTA + result + Fore.RESET
screamer 0:66f3b5499f7f 996
screamer 0:66f3b5499f7f 997 def shape_test_loop_ok_result_count(self, test_all_result):
screamer 0:66f3b5499f7f 998 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 999 """
screamer 0:66f3b5499f7f 1000 test_loop_count = len(test_all_result)
screamer 0:66f3b5499f7f 1001 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer 0:66f3b5499f7f 1002 return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer 0:66f3b5499f7f 1003
screamer 0:66f3b5499f7f 1004 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer 0:66f3b5499f7f 1005 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1006 """
screamer 0:66f3b5499f7f 1007 result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 1008
screamer 0:66f3b5499f7f 1009 if all(test_all_result[0] == res for res in test_all_result):
screamer 0:66f3b5499f7f 1010 result = test_all_result[0]
screamer 0:66f3b5499f7f 1011 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer 0:66f3b5499f7f 1012 result = self.TEST_RESULT_OK
screamer 0:66f3b5499f7f 1013
screamer 0:66f3b5499f7f 1014 return result
screamer 0:66f3b5499f7f 1015
screamer 0:66f3b5499f7f 1016 def run_host_test(self, name, image_path, disk, port, duration,
screamer 0:66f3b5499f7f 1017 micro=None, reset=None, reset_tout=None,
screamer 0:66f3b5499f7f 1018 verbose=False, copy_method=None, program_cycle_s=None):
screamer 0:66f3b5499f7f 1019 """ Function creates new process with host test configured with particular test case.
screamer 0:66f3b5499f7f 1020 Function also is pooling for serial port activity from process to catch all data
screamer 0:66f3b5499f7f 1021 printed by test runner and host test during test execution
screamer 0:66f3b5499f7f 1022 """
screamer 0:66f3b5499f7f 1023
screamer 0:66f3b5499f7f 1024 def get_char_from_queue(obs):
screamer 0:66f3b5499f7f 1025 """ Get character from queue safe way
screamer 0:66f3b5499f7f 1026 """
screamer 0:66f3b5499f7f 1027 try:
screamer 0:66f3b5499f7f 1028 c = obs.queue.get(block=True, timeout=0.5)
screamer 0:66f3b5499f7f 1029 except Empty, _:
screamer 0:66f3b5499f7f 1030 c = None
screamer 0:66f3b5499f7f 1031 return c
screamer 0:66f3b5499f7f 1032
screamer 0:66f3b5499f7f 1033 def filter_queue_char(c):
screamer 0:66f3b5499f7f 1034 """ Filters out non ASCII characters from serial port
screamer 0:66f3b5499f7f 1035 """
screamer 0:66f3b5499f7f 1036 if ord(c) not in range(128):
screamer 0:66f3b5499f7f 1037 c = ' '
screamer 0:66f3b5499f7f 1038 return c
screamer 0:66f3b5499f7f 1039
screamer 0:66f3b5499f7f 1040 def get_test_result(output):
screamer 0:66f3b5499f7f 1041 """ Parse test 'output' data
screamer 0:66f3b5499f7f 1042 """
screamer 0:66f3b5499f7f 1043 result = self.TEST_RESULT_TIMEOUT
screamer 0:66f3b5499f7f 1044 for line in "".join(output).splitlines():
screamer 0:66f3b5499f7f 1045 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer 0:66f3b5499f7f 1046 if search_result and len(search_result.groups()):
screamer 0:66f3b5499f7f 1047 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer 0:66f3b5499f7f 1048 break
screamer 0:66f3b5499f7f 1049 return result
screamer 0:66f3b5499f7f 1050
screamer 0:66f3b5499f7f 1051 def get_auto_property_value(property_name, line):
screamer 0:66f3b5499f7f 1052 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer 0:66f3b5499f7f 1053 Returns string
screamer 0:66f3b5499f7f 1054 """
screamer 0:66f3b5499f7f 1055 result = None
screamer 0:66f3b5499f7f 1056 if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer 0:66f3b5499f7f 1057 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer 0:66f3b5499f7f 1058 if property is not None and len(property.groups()) == 1:
screamer 0:66f3b5499f7f 1059 result = property.groups()[0]
screamer 0:66f3b5499f7f 1060 return result
screamer 0:66f3b5499f7f 1061
screamer 0:66f3b5499f7f 1062 # print "{%s} port:%s disk:%s" % (name, port, disk),
screamer 0:66f3b5499f7f 1063 cmd = ["python",
screamer 0:66f3b5499f7f 1064 '%s.py'% name,
screamer 0:66f3b5499f7f 1065 '-d', disk,
screamer 0:66f3b5499f7f 1066 '-f', '"%s"'% image_path,
screamer 0:66f3b5499f7f 1067 '-p', port,
screamer 0:66f3b5499f7f 1068 '-t', str(duration),
screamer 0:66f3b5499f7f 1069 '-C', str(program_cycle_s)]
screamer 0:66f3b5499f7f 1070
screamer 0:66f3b5499f7f 1071 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 1072 cmd += ['--auto']
screamer 0:66f3b5499f7f 1073
screamer 0:66f3b5499f7f 1074 # Add extra parameters to host_test
screamer 0:66f3b5499f7f 1075 if copy_method is not None:
screamer 0:66f3b5499f7f 1076 cmd += ["-c", copy_method]
screamer 0:66f3b5499f7f 1077 if micro is not None:
screamer 0:66f3b5499f7f 1078 cmd += ["-m", micro]
screamer 0:66f3b5499f7f 1079 if reset is not None:
screamer 0:66f3b5499f7f 1080 cmd += ["-r", reset]
screamer 0:66f3b5499f7f 1081 if reset_tout is not None:
screamer 0:66f3b5499f7f 1082 cmd += ["-R", str(reset_tout)]
screamer 0:66f3b5499f7f 1083
screamer 0:66f3b5499f7f 1084 if verbose:
screamer 0:66f3b5499f7f 1085 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
screamer 0:66f3b5499f7f 1086 print "Test::Output::Start"
screamer 0:66f3b5499f7f 1087
screamer 0:66f3b5499f7f 1088 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer 0:66f3b5499f7f 1089 obs = ProcessObserver(proc)
screamer 0:66f3b5499f7f 1090 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer 0:66f3b5499f7f 1091 line = ''
screamer 0:66f3b5499f7f 1092 output = []
screamer 0:66f3b5499f7f 1093 start_time = time()
screamer 0:66f3b5499f7f 1094 while (time() - start_time) < (2 * duration):
screamer 0:66f3b5499f7f 1095 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1096 if c:
screamer 0:66f3b5499f7f 1097 if verbose:
screamer 0:66f3b5499f7f 1098 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1099 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1100 output.append(c)
screamer 0:66f3b5499f7f 1101 # Give the mbed under test a way to communicate the end of the test
screamer 0:66f3b5499f7f 1102 if c in ['\n', '\r']:
screamer 0:66f3b5499f7f 1103
screamer 0:66f3b5499f7f 1104 # Checking for auto-detection information from the test about MUT reset moment
screamer 0:66f3b5499f7f 1105 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer 0:66f3b5499f7f 1106 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1107 update_once_flag['reset_target'] = True
screamer 0:66f3b5499f7f 1108 start_time = time()
screamer 0:66f3b5499f7f 1109
screamer 0:66f3b5499f7f 1110 # Checking for auto-detection information from the test about timeout
screamer 0:66f3b5499f7f 1111 auto_timeout_val = get_auto_property_value('timeout', line)
screamer 0:66f3b5499f7f 1112 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer 0:66f3b5499f7f 1113 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1114 update_once_flag['timeout'] = True
screamer 0:66f3b5499f7f 1115 duration = int(auto_timeout_val)
screamer 0:66f3b5499f7f 1116
screamer 0:66f3b5499f7f 1117 # Detect mbed assert:
screamer 0:66f3b5499f7f 1118 if 'mbed assertation failed: ' in line:
screamer 0:66f3b5499f7f 1119 output.append('{{mbed_assert}}')
screamer 0:66f3b5499f7f 1120 break
screamer 0:66f3b5499f7f 1121
screamer 0:66f3b5499f7f 1122 # Check for test end
screamer 0:66f3b5499f7f 1123 if '{end}' in line:
screamer 0:66f3b5499f7f 1124 break
screamer 0:66f3b5499f7f 1125 line = ''
screamer 0:66f3b5499f7f 1126 else:
screamer 0:66f3b5499f7f 1127 line += c
screamer 0:66f3b5499f7f 1128 end_time = time()
screamer 0:66f3b5499f7f 1129 testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer 0:66f3b5499f7f 1130
screamer 0:66f3b5499f7f 1131 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1132
screamer 0:66f3b5499f7f 1133 if c:
screamer 0:66f3b5499f7f 1134 if verbose:
screamer 0:66f3b5499f7f 1135 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1136 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1137 output.append(c)
screamer 0:66f3b5499f7f 1138
screamer 0:66f3b5499f7f 1139 if verbose:
screamer 0:66f3b5499f7f 1140 print "Test::Output::Finish"
screamer 0:66f3b5499f7f 1141 # Stop test process
screamer 0:66f3b5499f7f 1142 obs.stop()
screamer 0:66f3b5499f7f 1143
screamer 0:66f3b5499f7f 1144 result = get_test_result(output)
screamer 0:66f3b5499f7f 1145 return (result, "".join(output), testcase_duration, duration)
screamer 0:66f3b5499f7f 1146
screamer 0:66f3b5499f7f 1147 def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer 0:66f3b5499f7f 1148 """ Checks if specified target should run specific peripheral test case defined in MUTs file
screamer 0:66f3b5499f7f 1149 """
screamer 0:66f3b5499f7f 1150 if peripherals is not None:
screamer 0:66f3b5499f7f 1151 peripherals = set(peripherals)
screamer 0:66f3b5499f7f 1152 for id, mut in self.muts.iteritems():
screamer 0:66f3b5499f7f 1153 # Target MCU name check
screamer 0:66f3b5499f7f 1154 if mut["mcu"] != target_mcu_name:
screamer 0:66f3b5499f7f 1155 continue
screamer 0:66f3b5499f7f 1156 # Peripherals check
screamer 0:66f3b5499f7f 1157 if peripherals is not None:
screamer 0:66f3b5499f7f 1158 if 'peripherals' not in mut:
screamer 0:66f3b5499f7f 1159 continue
screamer 0:66f3b5499f7f 1160 if not peripherals.issubset(set(mut['peripherals'])):
screamer 0:66f3b5499f7f 1161 continue
screamer 0:66f3b5499f7f 1162 return True
screamer 0:66f3b5499f7f 1163 return False
screamer 0:66f3b5499f7f 1164
screamer 0:66f3b5499f7f 1165 def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer 0:66f3b5499f7f 1166 """ Function prepares JSON structure describing test specification
screamer 0:66f3b5499f7f 1167 """
screamer 0:66f3b5499f7f 1168 test_spec = {
screamer 0:66f3b5499f7f 1169 "mcu": mcu,
screamer 0:66f3b5499f7f 1170 "image": image_path,
screamer 0:66f3b5499f7f 1171 "duration": duration,
screamer 0:66f3b5499f7f 1172 "test_id": test_id,
screamer 0:66f3b5499f7f 1173 }
screamer 0:66f3b5499f7f 1174 return json.dumps(test_spec)
screamer 0:66f3b5499f7f 1175
screamer 0:66f3b5499f7f 1176
screamer 0:66f3b5499f7f 1177 def get_unique_value_from_summary(test_summary, index):
screamer 0:66f3b5499f7f 1178 """ Gets list of unique target names
screamer 0:66f3b5499f7f 1179 """
screamer 0:66f3b5499f7f 1180 result = []
screamer 0:66f3b5499f7f 1181 for test in test_summary:
screamer 0:66f3b5499f7f 1182 target_name = test[index]
screamer 0:66f3b5499f7f 1183 if target_name not in result:
screamer 0:66f3b5499f7f 1184 result.append(target_name)
screamer 0:66f3b5499f7f 1185 return sorted(result)
screamer 0:66f3b5499f7f 1186
screamer 0:66f3b5499f7f 1187
screamer 0:66f3b5499f7f 1188 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer 0:66f3b5499f7f 1189 """ Gets list of unique target names and return dictionary
screamer 0:66f3b5499f7f 1190 """
screamer 0:66f3b5499f7f 1191 result = {}
screamer 0:66f3b5499f7f 1192 for test in test_summary:
screamer 0:66f3b5499f7f 1193 key = test[index_key]
screamer 0:66f3b5499f7f 1194 val = test[index_val]
screamer 0:66f3b5499f7f 1195 if key not in result:
screamer 0:66f3b5499f7f 1196 result[key] = val
screamer 0:66f3b5499f7f 1197 return result
screamer 0:66f3b5499f7f 1198
screamer 0:66f3b5499f7f 1199
screamer 0:66f3b5499f7f 1200 def show_json_file_format_error(json_spec_filename, line, column):
screamer 0:66f3b5499f7f 1201 """ Prints JSON broken content
screamer 0:66f3b5499f7f 1202 """
screamer 0:66f3b5499f7f 1203 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1204 line_no = 1
screamer 0:66f3b5499f7f 1205 for json_line in data_file:
screamer 0:66f3b5499f7f 1206 if line_no + 5 >= line: # Print last few lines before error
screamer 0:66f3b5499f7f 1207 print 'Line %d:\t'%line_no + json_line, # Prints line
screamer 0:66f3b5499f7f 1208 if line_no == line:
screamer 0:66f3b5499f7f 1209 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
screamer 0:66f3b5499f7f 1210 break
screamer 0:66f3b5499f7f 1211 line_no += 1
screamer 0:66f3b5499f7f 1212
screamer 0:66f3b5499f7f 1213
screamer 0:66f3b5499f7f 1214 def json_format_error_defect_pos(json_error_msg):
screamer 0:66f3b5499f7f 1215 """ Gets first error line and column in JSON file format.
screamer 0:66f3b5499f7f 1216 Parsed from exception thrown by json.loads() string
screamer 0:66f3b5499f7f 1217 """
screamer 0:66f3b5499f7f 1218 result = None
screamer 0:66f3b5499f7f 1219 line, column = 0, 0
screamer 0:66f3b5499f7f 1220 # Line value search
screamer 0:66f3b5499f7f 1221 line_search = re.search('line [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1222 if line_search is not None:
screamer 0:66f3b5499f7f 1223 ls = line_search.group().split(' ')
screamer 0:66f3b5499f7f 1224 if len(ls) == 2:
screamer 0:66f3b5499f7f 1225 line = int(ls[1])
screamer 0:66f3b5499f7f 1226 # Column position search
screamer 0:66f3b5499f7f 1227 column_search = re.search('column [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1228 if column_search is not None:
screamer 0:66f3b5499f7f 1229 cs = column_search.group().split(' ')
screamer 0:66f3b5499f7f 1230 if len(cs) == 2:
screamer 0:66f3b5499f7f 1231 column = int(cs[1])
screamer 0:66f3b5499f7f 1232 result = [line, column]
screamer 0:66f3b5499f7f 1233 return result
screamer 0:66f3b5499f7f 1234
screamer 0:66f3b5499f7f 1235
screamer 0:66f3b5499f7f 1236 def get_json_data_from_file(json_spec_filename, verbose=False):
screamer 0:66f3b5499f7f 1237 """ Loads from file JSON formatted string to data structure
screamer 0:66f3b5499f7f 1238 """
screamer 0:66f3b5499f7f 1239 result = None
screamer 0:66f3b5499f7f 1240 try:
screamer 0:66f3b5499f7f 1241 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1242 try:
screamer 0:66f3b5499f7f 1243 result = json.load(data_file)
screamer 0:66f3b5499f7f 1244 except ValueError as json_error_msg:
screamer 0:66f3b5499f7f 1245 result = None
screamer 0:66f3b5499f7f 1246 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
screamer 0:66f3b5499f7f 1247 # We can print where error occurred inside JSON file if we can parse exception msg
screamer 0:66f3b5499f7f 1248 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer 0:66f3b5499f7f 1249 if json_format_defect_pos is not None:
screamer 0:66f3b5499f7f 1250 line = json_format_defect_pos[0]
screamer 0:66f3b5499f7f 1251 column = json_format_defect_pos[1]
screamer 0:66f3b5499f7f 1252 print
screamer 0:66f3b5499f7f 1253 show_json_file_format_error(json_spec_filename, line, column)
screamer 0:66f3b5499f7f 1254
screamer 0:66f3b5499f7f 1255 except IOError as fileopen_error_msg:
screamer 0:66f3b5499f7f 1256 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
screamer 0:66f3b5499f7f 1257 print
screamer 0:66f3b5499f7f 1258 if verbose and result:
screamer 0:66f3b5499f7f 1259 pp = pprint.PrettyPrinter(indent=4)
screamer 0:66f3b5499f7f 1260 pp.pprint(result)
screamer 0:66f3b5499f7f 1261 return result
screamer 0:66f3b5499f7f 1262
screamer 0:66f3b5499f7f 1263
screamer 0:66f3b5499f7f 1264 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer 0:66f3b5499f7f 1265 """ Prints MUTs configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1266 """
screamer 0:66f3b5499f7f 1267 muts_info_cols = []
screamer 0:66f3b5499f7f 1268 # We need to check all unique properties for each defined MUT
screamer 0:66f3b5499f7f 1269 for k in json_data:
screamer 0:66f3b5499f7f 1270 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1271 for mut_property in mut_info:
screamer 0:66f3b5499f7f 1272 if mut_property not in muts_info_cols:
screamer 0:66f3b5499f7f 1273 muts_info_cols.append(mut_property)
screamer 0:66f3b5499f7f 1274
screamer 0:66f3b5499f7f 1275 # Prepare pretty table object to display all MUTs
screamer 0:66f3b5499f7f 1276 pt_cols = ["index"] + muts_info_cols
screamer 0:66f3b5499f7f 1277 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1278 for col in pt_cols:
screamer 0:66f3b5499f7f 1279 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1280
screamer 0:66f3b5499f7f 1281 # Add rows to pretty print object
screamer 0:66f3b5499f7f 1282 for k in json_data:
screamer 0:66f3b5499f7f 1283 row = [k]
screamer 0:66f3b5499f7f 1284 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1285
screamer 0:66f3b5499f7f 1286 add_row = True
screamer 0:66f3b5499f7f 1287 if platform_filter and 'mcu' in mut_info:
screamer 0:66f3b5499f7f 1288 add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer 0:66f3b5499f7f 1289 if add_row:
screamer 0:66f3b5499f7f 1290 for col in muts_info_cols:
screamer 0:66f3b5499f7f 1291 cell_val = mut_info[col] if col in mut_info else None
screamer 0:66f3b5499f7f 1292 if type(cell_val) == ListType:
screamer 0:66f3b5499f7f 1293 cell_val = join_delim.join(cell_val)
screamer 0:66f3b5499f7f 1294 row.append(cell_val)
screamer 0:66f3b5499f7f 1295 pt.add_row(row)
screamer 0:66f3b5499f7f 1296 return pt.get_string()
screamer 0:66f3b5499f7f 1297
screamer 0:66f3b5499f7f 1298
screamer 0:66f3b5499f7f 1299 def print_test_configuration_from_json(json_data, join_delim=", "):
screamer 0:66f3b5499f7f 1300 """ Prints test specification configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1301 """
screamer 0:66f3b5499f7f 1302 toolchains_info_cols = []
screamer 0:66f3b5499f7f 1303 # We need to check all toolchains for each device
screamer 0:66f3b5499f7f 1304 for k in json_data:
screamer 0:66f3b5499f7f 1305 # k should be 'targets'
screamer 0:66f3b5499f7f 1306 targets = json_data[k]
screamer 0:66f3b5499f7f 1307 for target in targets:
screamer 0:66f3b5499f7f 1308 toolchains = targets[target]
screamer 0:66f3b5499f7f 1309 for toolchain in toolchains:
screamer 0:66f3b5499f7f 1310 if toolchain not in toolchains_info_cols:
screamer 0:66f3b5499f7f 1311 toolchains_info_cols.append(toolchain)
screamer 0:66f3b5499f7f 1312
screamer 0:66f3b5499f7f 1313 # Prepare pretty table object to display test specification
screamer 0:66f3b5499f7f 1314 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
screamer 0:66f3b5499f7f 1315 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1316 for col in pt_cols:
screamer 0:66f3b5499f7f 1317 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1318
screamer 0:66f3b5499f7f 1319 # { target : [conflicted toolchains] }
screamer 0:66f3b5499f7f 1320 toolchain_conflicts = {}
screamer 0:66f3b5499f7f 1321 toolchain_path_conflicts = []
screamer 0:66f3b5499f7f 1322 for k in json_data:
screamer 0:66f3b5499f7f 1323 # k should be 'targets'
screamer 0:66f3b5499f7f 1324 targets = json_data[k]
screamer 0:66f3b5499f7f 1325 for target in targets:
screamer 0:66f3b5499f7f 1326 target_supported_toolchains = get_target_supported_toolchains(target)
screamer 0:66f3b5499f7f 1327 if not target_supported_toolchains:
screamer 0:66f3b5499f7f 1328 target_supported_toolchains = []
screamer 0:66f3b5499f7f 1329 target_name = target if target in TARGET_MAP else "%s*"% target
screamer 0:66f3b5499f7f 1330 row = [target_name]
screamer 0:66f3b5499f7f 1331 toolchains = targets[target]
screamer 0:66f3b5499f7f 1332
screamer 0:66f3b5499f7f 1333 for toolchain in sorted(toolchains_info_cols):
screamer 0:66f3b5499f7f 1334 # Check for conflicts: target vs toolchain
screamer 0:66f3b5499f7f 1335 conflict = False
screamer 0:66f3b5499f7f 1336 conflict_path = False
screamer 0:66f3b5499f7f 1337 if toolchain in toolchains:
screamer 0:66f3b5499f7f 1338 if toolchain not in target_supported_toolchains:
screamer 0:66f3b5499f7f 1339 conflict = True
screamer 0:66f3b5499f7f 1340 if target not in toolchain_conflicts:
screamer 0:66f3b5499f7f 1341 toolchain_conflicts[target] = []
screamer 0:66f3b5499f7f 1342 toolchain_conflicts[target].append(toolchain)
screamer 0:66f3b5499f7f 1343 # Add marker inside table about target usage / conflict
screamer 0:66f3b5499f7f 1344 cell_val = 'Yes' if toolchain in toolchains else '-'
screamer 0:66f3b5499f7f 1345 if conflict:
screamer 0:66f3b5499f7f 1346 cell_val += '*'
screamer 0:66f3b5499f7f 1347 # Check for conflicts: toolchain vs toolchain path
screamer 20:835f6355470d 1348 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1349 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1350 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1351 conflict_path = True
screamer 0:66f3b5499f7f 1352 if toolchain not in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1353 toolchain_path_conflicts.append(toolchain)
screamer 0:66f3b5499f7f 1354 if conflict_path:
screamer 0:66f3b5499f7f 1355 cell_val += '#'
screamer 0:66f3b5499f7f 1356 row.append(cell_val)
screamer 0:66f3b5499f7f 1357 pt.add_row(row)
screamer 0:66f3b5499f7f 1358
screamer 0:66f3b5499f7f 1359 # generate result string
screamer 0:66f3b5499f7f 1360 result = pt.get_string() # Test specification table
screamer 0:66f3b5499f7f 1361 if toolchain_conflicts or toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1362 result += "\n"
screamer 0:66f3b5499f7f 1363 result += "Toolchain conflicts:\n"
screamer 0:66f3b5499f7f 1364 for target in toolchain_conflicts:
screamer 0:66f3b5499f7f 1365 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 1366 result += "\t* Target %s unknown\n"% (target)
screamer 0:66f3b5499f7f 1367 conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer 0:66f3b5499f7f 1368 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer 0:66f3b5499f7f 1369 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer 0:66f3b5499f7f 1370
screamer 0:66f3b5499f7f 1371 for toolchain in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1372 # Let's check toolchain configuration
screamer 20:835f6355470d 1373 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1374 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1375 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1376 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer 0:66f3b5499f7f 1377 return result
screamer 0:66f3b5499f7f 1378
screamer 0:66f3b5499f7f 1379
screamer 0:66f3b5499f7f 1380 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer 0:66f3b5499f7f 1381 """ Generates table summary with all test cases and additional test cases
screamer 0:66f3b5499f7f 1382 information using pretty print functionality. Allows test suite user to
screamer 0:66f3b5499f7f 1383 see test cases
screamer 0:66f3b5499f7f 1384 """
screamer 0:66f3b5499f7f 1385 # get all unique test ID prefixes
screamer 0:66f3b5499f7f 1386 unique_test_id = []
screamer 0:66f3b5499f7f 1387 for test in TESTS:
screamer 0:66f3b5499f7f 1388 split = test['id'].split('_')[:-1]
screamer 0:66f3b5499f7f 1389 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1390 if test_id_prefix not in unique_test_id:
screamer 0:66f3b5499f7f 1391 unique_test_id.append(test_id_prefix)
screamer 0:66f3b5499f7f 1392 unique_test_id.sort()
screamer 0:66f3b5499f7f 1393 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1394 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1395
screamer 0:66f3b5499f7f 1396 test_properties = ['id',
screamer 0:66f3b5499f7f 1397 'automated',
screamer 0:66f3b5499f7f 1398 'description',
screamer 0:66f3b5499f7f 1399 'peripherals',
screamer 0:66f3b5499f7f 1400 'host_test',
screamer 0:66f3b5499f7f 1401 'duration'] if cols is None else cols
screamer 0:66f3b5499f7f 1402
screamer 0:66f3b5499f7f 1403 # All tests status table print
screamer 0:66f3b5499f7f 1404 pt = PrettyTable(test_properties)
screamer 0:66f3b5499f7f 1405 for col in test_properties:
screamer 0:66f3b5499f7f 1406 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1407 pt.align['duration'] = "r"
screamer 0:66f3b5499f7f 1408
screamer 0:66f3b5499f7f 1409 counter_all = 0
screamer 0:66f3b5499f7f 1410 counter_automated = 0
screamer 0:66f3b5499f7f 1411 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 1412
screamer 0:66f3b5499f7f 1413 for test_id in sorted(TEST_MAP.keys()):
screamer 0:66f3b5499f7f 1414 if platform_filter is not None:
screamer 0:66f3b5499f7f 1415 # FIlter out platforms using regex
screamer 0:66f3b5499f7f 1416 if re.search(platform_filter, test_id) is None:
screamer 0:66f3b5499f7f 1417 continue
screamer 0:66f3b5499f7f 1418 row = []
screamer 0:66f3b5499f7f 1419 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 1420 split = test_id.split('_')[:-1]
screamer 0:66f3b5499f7f 1421 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1422
screamer 0:66f3b5499f7f 1423 for col in test_properties:
screamer 0:66f3b5499f7f 1424 col_value = test[col]
screamer 0:66f3b5499f7f 1425 if type(test[col]) == ListType:
screamer 0:66f3b5499f7f 1426 col_value = join_delim.join(test[col])
screamer 0:66f3b5499f7f 1427 elif test[col] == None:
screamer 0:66f3b5499f7f 1428 col_value = "-"
screamer 0:66f3b5499f7f 1429
screamer 0:66f3b5499f7f 1430 row.append(col_value)
screamer 0:66f3b5499f7f 1431 if test['automated'] == True:
screamer 0:66f3b5499f7f 1432 counter_dict_test_id_types[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1433 counter_automated += 1
screamer 0:66f3b5499f7f 1434 pt.add_row(row)
screamer 0:66f3b5499f7f 1435 # Update counters
screamer 0:66f3b5499f7f 1436 counter_all += 1
screamer 0:66f3b5499f7f 1437 counter_dict_test_id_types_all[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1438 result = pt.get_string()
screamer 0:66f3b5499f7f 1439 result += "\n\n"
screamer 0:66f3b5499f7f 1440
screamer 0:66f3b5499f7f 1441 if result_summary and not platform_filter:
screamer 0:66f3b5499f7f 1442 # Automation result summary
screamer 0:66f3b5499f7f 1443 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1444 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1445 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1446 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1447 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1448
screamer 0:66f3b5499f7f 1449 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer 0:66f3b5499f7f 1450 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1451 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer 0:66f3b5499f7f 1452 result += "Automation coverage:\n"
screamer 0:66f3b5499f7f 1453 result += pt.get_string()
screamer 0:66f3b5499f7f 1454 result += "\n\n"
screamer 0:66f3b5499f7f 1455
screamer 0:66f3b5499f7f 1456 # Test automation coverage table print
screamer 0:66f3b5499f7f 1457 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1458 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1459 pt.align['id'] = "l"
screamer 0:66f3b5499f7f 1460 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1461 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1462 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1463 for unique_id in unique_test_id:
screamer 0:66f3b5499f7f 1464 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer 0:66f3b5499f7f 1465 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer 0:66f3b5499f7f 1466 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1467 row = [unique_id,
screamer 0:66f3b5499f7f 1468 counter_dict_test_id_types[unique_id],
screamer 0:66f3b5499f7f 1469 counter_dict_test_id_types_all[unique_id],
screamer 0:66f3b5499f7f 1470 percent_progress,
screamer 0:66f3b5499f7f 1471 "[" + str_progress + "]"]
screamer 0:66f3b5499f7f 1472 pt.add_row(row)
screamer 0:66f3b5499f7f 1473 result += "Test automation coverage:\n"
screamer 0:66f3b5499f7f 1474 result += pt.get_string()
screamer 0:66f3b5499f7f 1475 result += "\n\n"
screamer 0:66f3b5499f7f 1476 return result
screamer 0:66f3b5499f7f 1477
screamer 0:66f3b5499f7f 1478
screamer 0:66f3b5499f7f 1479 def progress_bar(percent_progress, saturation=0):
screamer 0:66f3b5499f7f 1480 """ This function creates progress bar with optional simple saturation mark
screamer 0:66f3b5499f7f 1481 """
screamer 0:66f3b5499f7f 1482 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer 0:66f3b5499f7f 1483 str_progress = '#' * step + '.' * int(50 - step)
screamer 0:66f3b5499f7f 1484 c = '!' if str_progress[38] == '.' else '|'
screamer 0:66f3b5499f7f 1485 if saturation > 0:
screamer 0:66f3b5499f7f 1486 saturation = saturation / 2
screamer 0:66f3b5499f7f 1487 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer 0:66f3b5499f7f 1488 return str_progress
screamer 0:66f3b5499f7f 1489
screamer 0:66f3b5499f7f 1490
screamer 0:66f3b5499f7f 1491 def singletest_in_cli_mode(single_test):
screamer 0:66f3b5499f7f 1492 """ Runs SingleTestRunner object in CLI (Command line interface) mode
screamer 0:66f3b5499f7f 1493
screamer 0:66f3b5499f7f 1494 @return returns success code (0 == success) for building and running tests
screamer 0:66f3b5499f7f 1495 """
screamer 0:66f3b5499f7f 1496 start = time()
screamer 0:66f3b5499f7f 1497 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 1498 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer 0:66f3b5499f7f 1499 elapsed_time = time() - start
screamer 0:66f3b5499f7f 1500
screamer 0:66f3b5499f7f 1501 # Human readable summary
screamer 0:66f3b5499f7f 1502 if not single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 1503 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1504 print single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1505 if single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 1506 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1507 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 1508 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1509
screamer 0:66f3b5499f7f 1510 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 1511 print
screamer 0:66f3b5499f7f 1512 # Write summary of the builds
screamer 0:66f3b5499f7f 1513
screamer 0:66f3b5499f7f 1514 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 0:66f3b5499f7f 1515 status = print_report_exporter.report(build_report)
screamer 0:66f3b5499f7f 1516
screamer 0:66f3b5499f7f 1517 # Store extra reports in files
screamer 0:66f3b5499f7f 1518 if single_test.opts_report_html_file_name:
screamer 0:66f3b5499f7f 1519 # Export results in form of HTML report to separate file
screamer 0:66f3b5499f7f 1520 report_exporter = ReportExporter(ResultExporterType.HTML)
screamer 0:66f3b5499f7f 1521 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1522 if single_test.opts_report_junit_file_name:
screamer 0:66f3b5499f7f 1523 # Export results in form of JUnit XML report to separate file
screamer 0:66f3b5499f7f 1524 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 0:66f3b5499f7f 1525 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer 22:9e85236d8716 1526 if single_test.opts_report_text_file_name:
screamer 22:9e85236d8716 1527 # Export results in form of a text file
screamer 22:9e85236d8716 1528 report_exporter = ReportExporter(ResultExporterType.TEXT)
screamer 22:9e85236d8716 1529 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1530 if single_test.opts_report_build_file_name:
screamer 0:66f3b5499f7f 1531 # Export build results as html report to sparate file
screamer 0:66f3b5499f7f 1532 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 0:66f3b5499f7f 1533 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer 0:66f3b5499f7f 1534
screamer 0:66f3b5499f7f 1535 # Returns True if no build failures of the test projects or their dependencies
screamer 0:66f3b5499f7f 1536 return status
screamer 0:66f3b5499f7f 1537
screamer 0:66f3b5499f7f 1538 class TestLogger():
screamer 0:66f3b5499f7f 1539 """ Super-class for logging and printing ongoing events for test suite pass
screamer 0:66f3b5499f7f 1540 """
screamer 0:66f3b5499f7f 1541 def __init__(self, store_log=True):
screamer 0:66f3b5499f7f 1542 """ We can control if logger actually stores log in memory
screamer 0:66f3b5499f7f 1543 or just handled all log entries immediately
screamer 0:66f3b5499f7f 1544 """
screamer 0:66f3b5499f7f 1545 self.log = []
screamer 0:66f3b5499f7f 1546 self.log_to_file = False
screamer 0:66f3b5499f7f 1547 self.log_file_name = None
screamer 0:66f3b5499f7f 1548 self.store_log = store_log
screamer 0:66f3b5499f7f 1549
screamer 0:66f3b5499f7f 1550 self.LogType = construct_enum(INFO='Info',
screamer 0:66f3b5499f7f 1551 WARN='Warning',
screamer 0:66f3b5499f7f 1552 NOTIF='Notification',
screamer 0:66f3b5499f7f 1553 ERROR='Error',
screamer 0:66f3b5499f7f 1554 EXCEPT='Exception')
screamer 0:66f3b5499f7f 1555
screamer 0:66f3b5499f7f 1556 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer 0:66f3b5499f7f 1557 APPEND=2) # Append to existing log file
screamer 0:66f3b5499f7f 1558
screamer 0:66f3b5499f7f 1559 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1560 """ Log one line of text
screamer 0:66f3b5499f7f 1561 """
screamer 0:66f3b5499f7f 1562 log_timestamp = time()
screamer 0:66f3b5499f7f 1563 log_entry = {'log_type' : LogType,
screamer 0:66f3b5499f7f 1564 'log_timestamp' : log_timestamp,
screamer 0:66f3b5499f7f 1565 'log_line' : log_line,
screamer 0:66f3b5499f7f 1566 '_future' : None
screamer 0:66f3b5499f7f 1567 }
screamer 0:66f3b5499f7f 1568 # Store log in memory
screamer 0:66f3b5499f7f 1569 if self.store_log:
screamer 0:66f3b5499f7f 1570 self.log.append(log_entry)
screamer 0:66f3b5499f7f 1571 return log_entry
screamer 0:66f3b5499f7f 1572
screamer 0:66f3b5499f7f 1573
screamer 0:66f3b5499f7f 1574 class CLITestLogger(TestLogger):
screamer 0:66f3b5499f7f 1575 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer 0:66f3b5499f7f 1576 """
screamer 0:66f3b5499f7f 1577 def __init__(self, store_log=True, file_name=None):
screamer 0:66f3b5499f7f 1578 TestLogger.__init__(self)
screamer 0:66f3b5499f7f 1579 self.log_file_name = file_name
screamer 0:66f3b5499f7f 1580 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer 0:66f3b5499f7f 1581 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer 0:66f3b5499f7f 1582
screamer 0:66f3b5499f7f 1583 def log_print(self, log_entry, timestamp=True):
screamer 0:66f3b5499f7f 1584 """ Prints on screen formatted log entry
screamer 0:66f3b5499f7f 1585 """
screamer 0:66f3b5499f7f 1586 ts = log_entry['log_timestamp']
screamer 0:66f3b5499f7f 1587 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer 0:66f3b5499f7f 1588 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer 0:66f3b5499f7f 1589 return timestamp_str + log_line_str
screamer 0:66f3b5499f7f 1590
screamer 0:66f3b5499f7f 1591 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1592 """ Logs line, if log file output was specified log line will be appended
screamer 0:66f3b5499f7f 1593 at the end of log file
screamer 0:66f3b5499f7f 1594 """
screamer 0:66f3b5499f7f 1595 log_entry = TestLogger.log_line(self, LogType, log_line)
screamer 0:66f3b5499f7f 1596 log_line_str = self.log_print(log_entry, timestamp)
screamer 0:66f3b5499f7f 1597 if self.log_file_name is not None:
screamer 0:66f3b5499f7f 1598 try:
screamer 0:66f3b5499f7f 1599 with open(self.log_file_name, 'a') as f:
screamer 0:66f3b5499f7f 1600 f.write(log_line_str + line_delim)
screamer 0:66f3b5499f7f 1601 except IOError:
screamer 0:66f3b5499f7f 1602 pass
screamer 0:66f3b5499f7f 1603 return log_line_str
screamer 0:66f3b5499f7f 1604
screamer 0:66f3b5499f7f 1605
screamer 0:66f3b5499f7f 1606 def factory_db_logger(db_url):
screamer 0:66f3b5499f7f 1607 """ Factory database driver depending on database type supplied in database connection string db_url
screamer 0:66f3b5499f7f 1608 """
screamer 0:66f3b5499f7f 1609 if db_url is not None:
screamer 0:66f3b5499f7f 1610 from tools.test_mysql import MySQLDBAccess
screamer 0:66f3b5499f7f 1611 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1612 if connection_info is not None:
screamer 0:66f3b5499f7f 1613 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1614 if db_type == 'mysql':
screamer 0:66f3b5499f7f 1615 return MySQLDBAccess()
screamer 0:66f3b5499f7f 1616 return None
screamer 0:66f3b5499f7f 1617
screamer 0:66f3b5499f7f 1618
screamer 0:66f3b5499f7f 1619 def detect_database_verbose(db_url):
screamer 0:66f3b5499f7f 1620 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer 0:66f3b5499f7f 1621 """
screamer 0:66f3b5499f7f 1622 result = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1623 if result is not None:
screamer 0:66f3b5499f7f 1624 # Parsing passed
screamer 0:66f3b5499f7f 1625 (db_type, username, password, host, db_name) = result
screamer 0:66f3b5499f7f 1626 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer 0:66f3b5499f7f 1627 # Let's try to connect
screamer 0:66f3b5499f7f 1628 db_ = factory_db_logger(db_url)
screamer 0:66f3b5499f7f 1629 if db_ is not None:
screamer 0:66f3b5499f7f 1630 print "Connecting to database '%s'..."% db_url,
screamer 0:66f3b5499f7f 1631 db_.connect(host, username, password, db_name)
screamer 0:66f3b5499f7f 1632 if db_.is_connected():
screamer 0:66f3b5499f7f 1633 print "ok"
screamer 0:66f3b5499f7f 1634 print "Detecting database..."
screamer 0:66f3b5499f7f 1635 print db_.detect_database(verbose=True)
screamer 0:66f3b5499f7f 1636 print "Disconnecting...",
screamer 0:66f3b5499f7f 1637 db_.disconnect()
screamer 0:66f3b5499f7f 1638 print "done"
screamer 0:66f3b5499f7f 1639 else:
screamer 0:66f3b5499f7f 1640 print "Database type '%s' unknown"% db_type
screamer 0:66f3b5499f7f 1641 else:
screamer 0:66f3b5499f7f 1642 print "Parse error: '%s' - DB Url error"% (db_url)
screamer 0:66f3b5499f7f 1643
screamer 0:66f3b5499f7f 1644
screamer 0:66f3b5499f7f 1645 def get_module_avail(module_name):
screamer 0:66f3b5499f7f 1646 """ This function returns True if module_name is already impored module
screamer 0:66f3b5499f7f 1647 """
screamer 0:66f3b5499f7f 1648 return module_name in sys.modules.keys()
screamer 0:66f3b5499f7f 1649
screamer 0:66f3b5499f7f 1650
screamer 0:66f3b5499f7f 1651 def get_autodetected_MUTS_list(platform_name_filter=None):
screamer 0:66f3b5499f7f 1652 oldError = None
screamer 0:66f3b5499f7f 1653 if os.name == 'nt':
screamer 0:66f3b5499f7f 1654 # Disable Windows error box temporarily
screamer 0:66f3b5499f7f 1655 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer 0:66f3b5499f7f 1656
screamer 0:66f3b5499f7f 1657 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1658 detect_muts_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1659
screamer 0:66f3b5499f7f 1660 if os.name == 'nt':
screamer 0:66f3b5499f7f 1661 ctypes.windll.kernel32.SetErrorMode(oldError)
screamer 0:66f3b5499f7f 1662
screamer 0:66f3b5499f7f 1663 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 1664
screamer 0:66f3b5499f7f 1665 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer 0:66f3b5499f7f 1666 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer 0:66f3b5499f7f 1667 If function fails to auto-detect devices it will return empty dictionary.
screamer 0:66f3b5499f7f 1668
screamer 0:66f3b5499f7f 1669 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1670 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1671 mbeds_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1672
screamer 0:66f3b5499f7f 1673 @param mbeds_list list of mbeds captured from mbed_lstools
screamer 0:66f3b5499f7f 1674 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer 0:66f3b5499f7f 1675 """
screamer 0:66f3b5499f7f 1676 result = {} # Should be in muts_all.json format
screamer 0:66f3b5499f7f 1677 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer 0:66f3b5499f7f 1678 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer 0:66f3b5499f7f 1679 index = 1
screamer 0:66f3b5499f7f 1680 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1681 # Filter the MUTS if a filter is specified
screamer 0:66f3b5499f7f 1682
screamer 0:66f3b5499f7f 1683 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer 0:66f3b5499f7f 1684 continue
screamer 0:66f3b5499f7f 1685
screamer 0:66f3b5499f7f 1686 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer 0:66f3b5499f7f 1687 # if not we are creating our own unique value (last few chars from platform's target_id).
screamer 0:66f3b5499f7f 1688 m = {'mcu': mut['platform_name'],
screamer 0:66f3b5499f7f 1689 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer 0:66f3b5499f7f 1690 'port': mut['serial_port'],
screamer 0:66f3b5499f7f 1691 'disk': mut['mount_point'],
screamer 0:66f3b5499f7f 1692 'peripherals': [] # No peripheral detection
screamer 0:66f3b5499f7f 1693 }
screamer 0:66f3b5499f7f 1694 if index not in result:
screamer 0:66f3b5499f7f 1695 result[index] = {}
screamer 0:66f3b5499f7f 1696 result[index] = m
screamer 0:66f3b5499f7f 1697 index += 1
screamer 0:66f3b5499f7f 1698 return result
screamer 0:66f3b5499f7f 1699
screamer 0:66f3b5499f7f 1700
screamer 0:66f3b5499f7f 1701 def get_autodetected_TEST_SPEC(mbeds_list,
screamer 0:66f3b5499f7f 1702 use_default_toolchain=True,
screamer 0:66f3b5499f7f 1703 use_supported_toolchains=False,
screamer 0:66f3b5499f7f 1704 toolchain_filter=None,
screamer 0:66f3b5499f7f 1705 platform_name_filter=None):
screamer 0:66f3b5499f7f 1706 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer 0:66f3b5499f7f 1707 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer 0:66f3b5499f7f 1708
screamer 0:66f3b5499f7f 1709 use_default_toolchain - if True add default toolchain to test_spec
screamer 0:66f3b5499f7f 1710 use_supported_toolchains - if True add all supported toolchains to test_spec
screamer 0:66f3b5499f7f 1711 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer 0:66f3b5499f7f 1712 """
screamer 0:66f3b5499f7f 1713 result = {'targets': {} }
screamer 0:66f3b5499f7f 1714
screamer 0:66f3b5499f7f 1715 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1716 mcu = mut['mcu']
screamer 0:66f3b5499f7f 1717 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer 0:66f3b5499f7f 1718 if mcu in TARGET_MAP:
screamer 0:66f3b5499f7f 1719 default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer 0:66f3b5499f7f 1720 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer 0:66f3b5499f7f 1721
screamer 0:66f3b5499f7f 1722 # Decide which toolchains should be added to test specification toolchain pool for each target
screamer 0:66f3b5499f7f 1723 toolchains = []
screamer 0:66f3b5499f7f 1724 if use_default_toolchain:
screamer 0:66f3b5499f7f 1725 toolchains.append(default_toolchain)
screamer 0:66f3b5499f7f 1726 if use_supported_toolchains:
screamer 0:66f3b5499f7f 1727 toolchains += supported_toolchains
screamer 0:66f3b5499f7f 1728 if toolchain_filter is not None:
screamer 0:66f3b5499f7f 1729 all_toolchains = supported_toolchains + [default_toolchain]
screamer 22:9e85236d8716 1730 for toolchain in toolchain_filter:
screamer 0:66f3b5499f7f 1731 if toolchain in all_toolchains:
screamer 0:66f3b5499f7f 1732 toolchains.append(toolchain)
screamer 0:66f3b5499f7f 1733
screamer 0:66f3b5499f7f 1734 result['targets'][mcu] = list(set(toolchains))
screamer 0:66f3b5499f7f 1735 return result
screamer 0:66f3b5499f7f 1736
screamer 0:66f3b5499f7f 1737
screamer 0:66f3b5499f7f 1738 def get_default_test_options_parser():
screamer 0:66f3b5499f7f 1739 """ Get common test script options used by CLI, web services etc.
screamer 0:66f3b5499f7f 1740 """
screamer 22:9e85236d8716 1741 parser = argparse.ArgumentParser()
screamer 22:9e85236d8716 1742 parser.add_argument('-i', '--tests',
screamer 22:9e85236d8716 1743 dest='test_spec_filename',
screamer 22:9e85236d8716 1744 metavar="FILE",
screamer 22:9e85236d8716 1745 type=argparse_filestring_type,
screamer 22:9e85236d8716 1746 help='Points to file with test specification')
screamer 0:66f3b5499f7f 1747
screamer 22:9e85236d8716 1748 parser.add_argument('-M', '--MUTS',
screamer 22:9e85236d8716 1749 dest='muts_spec_filename',
screamer 22:9e85236d8716 1750 metavar="FILE",
screamer 22:9e85236d8716 1751 type=argparse_filestring_type,
screamer 22:9e85236d8716 1752 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer 0:66f3b5499f7f 1753
screamer 22:9e85236d8716 1754 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 1755 dest='jobs',
screamer 22:9e85236d8716 1756 metavar="NUMBER",
screamer 22:9e85236d8716 1757 type=int,
screamer 22:9e85236d8716 1758 help="Define number of compilation jobs. Default value is 1")
screamer 0:66f3b5499f7f 1759
screamer 0:66f3b5499f7f 1760 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1761 # Additional features available when mbed_lstools is installed on host and imported
screamer 0:66f3b5499f7f 1762 # mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer 22:9e85236d8716 1763 parser.add_argument('--auto',
screamer 22:9e85236d8716 1764 dest='auto_detect',
screamer 22:9e85236d8716 1765 action="store_true",
screamer 22:9e85236d8716 1766 help='Use mbed-ls module to detect all connected mbed devices')
screamer 0:66f3b5499f7f 1767
screamer 22:9e85236d8716 1768 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
screamer 22:9e85236d8716 1769 parser.add_argument('--tc',
screamer 22:9e85236d8716 1770 dest='toolchains_filter',
screamer 22:9e85236d8716 1771 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
screamer 22:9e85236d8716 1772 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer 0:66f3b5499f7f 1773
screamer 0:66f3b5499f7f 1774 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer 22:9e85236d8716 1775 parser.add_argument('--oper',
screamer 22:9e85236d8716 1776 dest='operability_checks',
screamer 22:9e85236d8716 1777 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
screamer 22:9e85236d8716 1778 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer 0:66f3b5499f7f 1779
screamer 22:9e85236d8716 1780 parser.add_argument('--clean',
screamer 22:9e85236d8716 1781 dest='clean',
screamer 22:9e85236d8716 1782 action="store_true",
screamer 22:9e85236d8716 1783 help='Clean the build directory')
screamer 0:66f3b5499f7f 1784
screamer 22:9e85236d8716 1785 parser.add_argument('-P', '--only-peripherals',
screamer 22:9e85236d8716 1786 dest='test_only_peripheral',
screamer 22:9e85236d8716 1787 default=False,
screamer 22:9e85236d8716 1788 action="store_true",
screamer 22:9e85236d8716 1789 help='Test only peripheral declared for MUT and skip common tests')
screamer 0:66f3b5499f7f 1790
The Other Jimmy 31:8ea194f6145b 1791 parser.add_argument("--profile", dest="profile", action="append",
The Other Jimmy 31:8ea194f6145b 1792 type=argparse_filestring_type,
The Other Jimmy 31:8ea194f6145b 1793 default=[])
The Other Jimmy 31:8ea194f6145b 1794
screamer 22:9e85236d8716 1795 parser.add_argument('-C', '--only-commons',
screamer 22:9e85236d8716 1796 dest='test_only_common',
screamer 22:9e85236d8716 1797 default=False,
screamer 22:9e85236d8716 1798 action="store_true",
screamer 22:9e85236d8716 1799 help='Test only board internals. Skip perpherials tests and perform common tests')
screamer 0:66f3b5499f7f 1800
screamer 22:9e85236d8716 1801 parser.add_argument('-n', '--test-by-names',
screamer 22:9e85236d8716 1802 dest='test_by_names',
screamer 22:9e85236d8716 1803 type=argparse_many(str),
screamer 22:9e85236d8716 1804 help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer 0:66f3b5499f7f 1805
screamer 22:9e85236d8716 1806 parser.add_argument('-p', '--peripheral-by-names',
screamer 0:66f3b5499f7f 1807 dest='peripheral_by_names',
screamer 22:9e85236d8716 1808 type=argparse_many(str),
screamer 0:66f3b5499f7f 1809 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer 0:66f3b5499f7f 1810
screamer 0:66f3b5499f7f 1811 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer 0:66f3b5499f7f 1812 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer 0:66f3b5499f7f 1813
screamer 22:9e85236d8716 1814 parser.add_argument('-c', '--copy-method',
screamer 22:9e85236d8716 1815 dest='copy_method',
screamer 22:9e85236d8716 1816 type=argparse_uppercase_type(copy_methods, "flash method"),
screamer 22:9e85236d8716 1817 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer 0:66f3b5499f7f 1818
screamer 0:66f3b5499f7f 1819 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer 0:66f3b5499f7f 1820 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer 0:66f3b5499f7f 1821
screamer 22:9e85236d8716 1822 parser.add_argument('-r', '--reset-type',
screamer 22:9e85236d8716 1823 dest='mut_reset_type',
screamer 22:9e85236d8716 1824 default=None,
screamer 22:9e85236d8716 1825 type=argparse_uppercase_type(reset_methods, "reset method"),
screamer 22:9e85236d8716 1826 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer 0:66f3b5499f7f 1827
screamer 22:9e85236d8716 1828 parser.add_argument('-g', '--goanna-for-tests',
screamer 22:9e85236d8716 1829 dest='goanna_for_tests',
screamer 22:9e85236d8716 1830 action="store_true",
screamer 22:9e85236d8716 1831 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1832
screamer 22:9e85236d8716 1833 parser.add_argument('-G', '--goanna-for-sdk',
screamer 22:9e85236d8716 1834 dest='goanna_for_mbed_sdk',
screamer 22:9e85236d8716 1835 action="store_true",
screamer 22:9e85236d8716 1836 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1837
screamer 22:9e85236d8716 1838 parser.add_argument('-s', '--suppress-summary',
screamer 22:9e85236d8716 1839 dest='suppress_summary',
screamer 22:9e85236d8716 1840 default=False,
screamer 22:9e85236d8716 1841 action="store_true",
screamer 22:9e85236d8716 1842 help='Suppresses display of wellformatted table with test results')
screamer 22:9e85236d8716 1843
screamer 22:9e85236d8716 1844 parser.add_argument('-t', '--test-summary',
screamer 22:9e85236d8716 1845 dest='test_x_toolchain_summary',
screamer 22:9e85236d8716 1846 default=False,
screamer 22:9e85236d8716 1847 action="store_true",
screamer 22:9e85236d8716 1848 help='Displays wellformatted table with test x toolchain test result per target')
screamer 0:66f3b5499f7f 1849
screamer 22:9e85236d8716 1850 parser.add_argument('-A', '--test-automation-report',
screamer 22:9e85236d8716 1851 dest='test_automation_report',
screamer 22:9e85236d8716 1852 default=False,
screamer 22:9e85236d8716 1853 action="store_true",
screamer 22:9e85236d8716 1854 help='Prints information about all tests and exits')
screamer 0:66f3b5499f7f 1855
screamer 22:9e85236d8716 1856 parser.add_argument('-R', '--test-case-report',
screamer 22:9e85236d8716 1857 dest='test_case_report',
screamer 22:9e85236d8716 1858 default=False,
screamer 22:9e85236d8716 1859 action="store_true",
screamer 22:9e85236d8716 1860 help='Prints information about all test cases and exits')
screamer 0:66f3b5499f7f 1861
screamer 22:9e85236d8716 1862 parser.add_argument("-S", "--supported-toolchains",
screamer 22:9e85236d8716 1863 action="store_true",
screamer 22:9e85236d8716 1864 dest="supported_toolchains",
screamer 22:9e85236d8716 1865 default=False,
screamer 22:9e85236d8716 1866 help="Displays supported matrix of MCUs and toolchains")
screamer 0:66f3b5499f7f 1867
screamer 22:9e85236d8716 1868 parser.add_argument("-O", "--only-build",
screamer 22:9e85236d8716 1869 action="store_true",
screamer 22:9e85236d8716 1870 dest="only_build_tests",
screamer 22:9e85236d8716 1871 default=False,
screamer 22:9e85236d8716 1872 help="Only build tests, skips actual test procedures (flashing etc.)")
screamer 0:66f3b5499f7f 1873
screamer 22:9e85236d8716 1874 parser.add_argument('--parallel',
screamer 22:9e85236d8716 1875 dest='parallel_test_exec',
screamer 22:9e85236d8716 1876 default=False,
screamer 22:9e85236d8716 1877 action="store_true",
screamer 22:9e85236d8716 1878 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer 0:66f3b5499f7f 1879
screamer 22:9e85236d8716 1880 parser.add_argument('--config',
screamer 22:9e85236d8716 1881 dest='verbose_test_configuration_only',
screamer 22:9e85236d8716 1882 default=False,
screamer 22:9e85236d8716 1883 action="store_true",
screamer 22:9e85236d8716 1884 help='Displays full test specification and MUTs configration and exits')
screamer 22:9e85236d8716 1885
screamer 22:9e85236d8716 1886 parser.add_argument('--loops',
screamer 22:9e85236d8716 1887 dest='test_loops_list',
screamer 22:9e85236d8716 1888 type=argparse_many(str),
screamer 22:9e85236d8716 1889 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer 0:66f3b5499f7f 1890
screamer 22:9e85236d8716 1891 parser.add_argument('--global-loops',
screamer 22:9e85236d8716 1892 dest='test_global_loops_value',
screamer 22:9e85236d8716 1893 type=int,
screamer 22:9e85236d8716 1894 help='Set global number of test loops per test. Default value is set 1')
screamer 0:66f3b5499f7f 1895
screamer 22:9e85236d8716 1896 parser.add_argument('--consolidate-waterfall',
screamer 22:9e85236d8716 1897 dest='consolidate_waterfall_test',
screamer 22:9e85236d8716 1898 default=False,
screamer 22:9e85236d8716 1899 action="store_true",
screamer 22:9e85236d8716 1900 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
screamer 0:66f3b5499f7f 1901
screamer 22:9e85236d8716 1902 parser.add_argument('-W', '--waterfall',
screamer 22:9e85236d8716 1903 dest='waterfall_test',
screamer 22:9e85236d8716 1904 default=False,
screamer 22:9e85236d8716 1905 action="store_true",
screamer 22:9e85236d8716 1906 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
screamer 0:66f3b5499f7f 1907
screamer 22:9e85236d8716 1908 parser.add_argument('-N', '--firmware-name',
screamer 22:9e85236d8716 1909 dest='firmware_global_name',
screamer 22:9e85236d8716 1910 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer 0:66f3b5499f7f 1911
screamer 22:9e85236d8716 1912 parser.add_argument('-u', '--shuffle',
screamer 22:9e85236d8716 1913 dest='shuffle_test_order',
screamer 22:9e85236d8716 1914 default=False,
screamer 22:9e85236d8716 1915 action="store_true",
screamer 22:9e85236d8716 1916 help='Shuffles test execution order')
screamer 0:66f3b5499f7f 1917
screamer 22:9e85236d8716 1918 parser.add_argument('--shuffle-seed',
screamer 22:9e85236d8716 1919 dest='shuffle_test_seed',
screamer 22:9e85236d8716 1920 default=None,
screamer 22:9e85236d8716 1921 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer 0:66f3b5499f7f 1922
screamer 22:9e85236d8716 1923 parser.add_argument('-f', '--filter',
screamer 22:9e85236d8716 1924 dest='general_filter_regex',
screamer 22:9e85236d8716 1925 type=argparse_many(str),
screamer 22:9e85236d8716 1926 default=None,
screamer 22:9e85236d8716 1927 help='For some commands you can use filter to filter out results')
screamer 0:66f3b5499f7f 1928
screamer 22:9e85236d8716 1929 parser.add_argument('--inc-timeout',
screamer 22:9e85236d8716 1930 dest='extend_test_timeout',
screamer 22:9e85236d8716 1931 metavar="NUMBER",
screamer 22:9e85236d8716 1932 type=int,
screamer 22:9e85236d8716 1933 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer 22:9e85236d8716 1934
screamer 22:9e85236d8716 1935 parser.add_argument('--db',
screamer 22:9e85236d8716 1936 dest='db_url',
screamer 22:9e85236d8716 1937 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer 0:66f3b5499f7f 1938
screamer 22:9e85236d8716 1939 parser.add_argument('-l', '--log',
screamer 22:9e85236d8716 1940 dest='log_file_name',
screamer 22:9e85236d8716 1941 help='Log events to external file (note not all console entries may be visible in log file)')
screamer 0:66f3b5499f7f 1942
screamer 22:9e85236d8716 1943 parser.add_argument('--report-html',
screamer 22:9e85236d8716 1944 dest='report_html_file_name',
screamer 22:9e85236d8716 1945 help='You can log test suite results in form of HTML report')
screamer 0:66f3b5499f7f 1946
screamer 22:9e85236d8716 1947 parser.add_argument('--report-junit',
screamer 22:9e85236d8716 1948 dest='report_junit_file_name',
screamer 22:9e85236d8716 1949 help='You can log test suite results in form of JUnit compliant XML report')
screamer 0:66f3b5499f7f 1950
screamer 22:9e85236d8716 1951 parser.add_argument("--report-build",
screamer 22:9e85236d8716 1952 dest="report_build_file_name",
screamer 22:9e85236d8716 1953 help="Output the build results to a junit xml file")
screamer 0:66f3b5499f7f 1954
screamer 22:9e85236d8716 1955 parser.add_argument("--report-text",
screamer 22:9e85236d8716 1956 dest="report_text_file_name",
screamer 22:9e85236d8716 1957 help="Output the build results to a text file")
screamer 0:66f3b5499f7f 1958
screamer 22:9e85236d8716 1959 parser.add_argument('--verbose-skipped',
screamer 22:9e85236d8716 1960 dest='verbose_skipped_tests',
screamer 22:9e85236d8716 1961 default=False,
screamer 22:9e85236d8716 1962 action="store_true",
screamer 22:9e85236d8716 1963 help='Prints some extra information about skipped tests')
screamer 0:66f3b5499f7f 1964
screamer 22:9e85236d8716 1965 parser.add_argument('-V', '--verbose-test-result',
screamer 22:9e85236d8716 1966 dest='verbose_test_result_only',
screamer 22:9e85236d8716 1967 default=False,
screamer 22:9e85236d8716 1968 action="store_true",
screamer 22:9e85236d8716 1969 help='Prints test serial output')
screamer 0:66f3b5499f7f 1970
screamer 22:9e85236d8716 1971 parser.add_argument('-v', '--verbose',
screamer 22:9e85236d8716 1972 dest='verbose',
screamer 22:9e85236d8716 1973 default=False,
screamer 22:9e85236d8716 1974 action="store_true",
screamer 22:9e85236d8716 1975 help='Verbose mode (prints some extra information)')
screamer 0:66f3b5499f7f 1976
screamer 22:9e85236d8716 1977 parser.add_argument('--version',
screamer 22:9e85236d8716 1978 dest='version',
screamer 22:9e85236d8716 1979 default=False,
screamer 22:9e85236d8716 1980 action="store_true",
screamer 22:9e85236d8716 1981 help='Prints script version and exits')
theotherjimmy 40:7d3fa6b99b2b 1982
theotherjimmy 40:7d3fa6b99b2b 1983 parser.add_argument('--stats-depth',
theotherjimmy 40:7d3fa6b99b2b 1984 dest='stats_depth',
theotherjimmy 40:7d3fa6b99b2b 1985 default=2,
theotherjimmy 40:7d3fa6b99b2b 1986 type=int,
theotherjimmy 40:7d3fa6b99b2b 1987 help="Depth level for static memory report")
screamer 0:66f3b5499f7f 1988 return parser
screamer 0:66f3b5499f7f 1989
The Other Jimmy 31:8ea194f6145b 1990 def test_path_to_name(path, base):
screamer 0:66f3b5499f7f 1991 """Change all slashes in a path into hyphens
screamer 0:66f3b5499f7f 1992 This creates a unique cross-platform test name based on the path
screamer 0:66f3b5499f7f 1993 This can eventually be overriden by a to-be-determined meta-data mechanism"""
screamer 0:66f3b5499f7f 1994 name_parts = []
The Other Jimmy 31:8ea194f6145b 1995 head, tail = os.path.split(relpath(path,base))
screamer 0:66f3b5499f7f 1996 while (tail and tail != "."):
screamer 0:66f3b5499f7f 1997 name_parts.insert(0, tail)
screamer 0:66f3b5499f7f 1998 head, tail = os.path.split(head)
screamer 22:9e85236d8716 1999
screamer 13:ab47a20b66f0 2000 return "-".join(name_parts).lower()
screamer 0:66f3b5499f7f 2001
The Other Jimmy 31:8ea194f6145b 2002 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
screamer 24:25bff2709c20 2003 """ Finds all tests in a directory recursively
screamer 24:25bff2709c20 2004 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
screamer 24:25bff2709c20 2005 target_name: name of the target to use for scanning (ex. 'K64F')
screamer 24:25bff2709c20 2006 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
screamer 24:25bff2709c20 2007 options: Compile options to pass to the toolchain (ex. ['debug-info'])
The Other Jimmy 31:8ea194f6145b 2008 app_config - location of a chosen mbed_app.json file
screamer 24:25bff2709c20 2009 """
screamer 24:25bff2709c20 2010
screamer 24:25bff2709c20 2011 tests = {}
screamer 24:25bff2709c20 2012
screamer 24:25bff2709c20 2013 # Prepare the toolchain
The Other Jimmy 36:96847d42f010 2014 toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2015 silent=True, app_config=app_config)
screamer 24:25bff2709c20 2016
screamer 24:25bff2709c20 2017 # Scan the directory for paths to probe for 'TESTS' folders
The Other Jimmy 31:8ea194f6145b 2018 base_resources = scan_resources([base_dir], toolchain)
screamer 24:25bff2709c20 2019
screamer 24:25bff2709c20 2020 dirs = base_resources.inc_dirs
screamer 24:25bff2709c20 2021 for directory in dirs:
screamer 24:25bff2709c20 2022 subdirs = os.listdir(directory)
screamer 22:9e85236d8716 2023
screamer 24:25bff2709c20 2024 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
screamer 24:25bff2709c20 2025 if 'TESTS' in subdirs:
screamer 24:25bff2709c20 2026 walk_base_dir = join(directory, 'TESTS')
screamer 24:25bff2709c20 2027 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
screamer 24:25bff2709c20 2028
screamer 24:25bff2709c20 2029 # Loop through all subdirectories
screamer 24:25bff2709c20 2030 for d in test_resources.inc_dirs:
screamer 22:9e85236d8716 2031
screamer 24:25bff2709c20 2032 # If the test case folder is not called 'host_tests' and it is
screamer 24:25bff2709c20 2033 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
screamer 24:25bff2709c20 2034 # then add it to the tests
screamer 24:25bff2709c20 2035 path_depth = get_path_depth(relpath(d, walk_base_dir))
screamer 24:25bff2709c20 2036 if path_depth == 2:
screamer 24:25bff2709c20 2037 test_group_directory_path, test_case_directory = os.path.split(d)
screamer 24:25bff2709c20 2038 test_group_directory = os.path.basename(test_group_directory_path)
theotherjimmy 40:7d3fa6b99b2b 2039
screamer 24:25bff2709c20 2040 # Check to make sure discoverd folder is not in a host test directory
screamer 24:25bff2709c20 2041 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
The Other Jimmy 31:8ea194f6145b 2042 test_name = test_path_to_name(d, base_dir)
screamer 24:25bff2709c20 2043 tests[test_name] = d
screamer 22:9e85236d8716 2044
screamer 13:ab47a20b66f0 2045 return tests
screamer 22:9e85236d8716 2046
screamer 13:ab47a20b66f0 2047 def print_tests(tests, format="list", sort=True):
screamer 0:66f3b5499f7f 2048 """Given a dictionary of tests (as returned from "find_tests"), print them
screamer 0:66f3b5499f7f 2049 in the specified format"""
screamer 0:66f3b5499f7f 2050 if format == "list":
screamer 13:ab47a20b66f0 2051 for test_name in sorted(tests.keys()):
screamer 13:ab47a20b66f0 2052 test_path = tests[test_name]
screamer 0:66f3b5499f7f 2053 print "Test Case:"
screamer 0:66f3b5499f7f 2054 print " Name: %s" % test_name
screamer 0:66f3b5499f7f 2055 print " Path: %s" % test_path
screamer 0:66f3b5499f7f 2056 elif format == "json":
screamer 0:66f3b5499f7f 2057 print json.dumps(tests, indent=2)
screamer 0:66f3b5499f7f 2058 else:
screamer 0:66f3b5499f7f 2059 print "Unknown format '%s'" % format
screamer 0:66f3b5499f7f 2060 sys.exit(1)
screamer 0:66f3b5499f7f 2061
screamer 13:ab47a20b66f0 2062 def norm_relative_path(path, start):
screamer 13:ab47a20b66f0 2063 """This function will create a normalized, relative path. It mimics the
screamer 13:ab47a20b66f0 2064 python os.path.relpath function, but also normalizes a Windows-syle path
screamer 13:ab47a20b66f0 2065 that use backslashes to a Unix style path that uses forward slashes."""
screamer 13:ab47a20b66f0 2066 path = os.path.normpath(path)
screamer 13:ab47a20b66f0 2067 path = os.path.relpath(path, start)
screamer 13:ab47a20b66f0 2068 path = path.replace("\\", "/")
screamer 13:ab47a20b66f0 2069 return path
screamer 13:ab47a20b66f0 2070
The Other Jimmy 31:8ea194f6145b 2071
The Other Jimmy 31:8ea194f6145b 2072 def build_test_worker(*args, **kwargs):
The Other Jimmy 31:8ea194f6145b 2073 """This is a worker function for the parallel building of tests. The `args`
The Other Jimmy 31:8ea194f6145b 2074 and `kwargs` are passed directly to `build_project`. It returns a dictionary
The Other Jimmy 31:8ea194f6145b 2075 with the following structure:
The Other Jimmy 31:8ea194f6145b 2076
The Other Jimmy 31:8ea194f6145b 2077 {
The Other Jimmy 31:8ea194f6145b 2078 'result': `True` if no exceptions were thrown, `False` otherwise
The Other Jimmy 31:8ea194f6145b 2079 'reason': Instance of exception that was thrown on failure
The Other Jimmy 31:8ea194f6145b 2080 'bin_file': Path to the created binary if `build_project` was
The Other Jimmy 31:8ea194f6145b 2081 successful. Not present otherwise
The Other Jimmy 31:8ea194f6145b 2082 'kwargs': The keyword arguments that were passed to `build_project`.
The Other Jimmy 31:8ea194f6145b 2083 This includes arguments that were modified (ex. report)
The Other Jimmy 31:8ea194f6145b 2084 }
The Other Jimmy 31:8ea194f6145b 2085 """
The Other Jimmy 31:8ea194f6145b 2086 bin_file = None
The Other Jimmy 31:8ea194f6145b 2087 ret = {
The Other Jimmy 31:8ea194f6145b 2088 'result': False,
The Other Jimmy 31:8ea194f6145b 2089 'args': args,
The Other Jimmy 31:8ea194f6145b 2090 'kwargs': kwargs
The Other Jimmy 31:8ea194f6145b 2091 }
The Other Jimmy 31:8ea194f6145b 2092
The Other Jimmy 31:8ea194f6145b 2093 # Use parent TOOLCHAIN_PATHS variable
The Other Jimmy 31:8ea194f6145b 2094 for key, value in kwargs['toolchain_paths'].iteritems():
The Other Jimmy 31:8ea194f6145b 2095 TOOLCHAIN_PATHS[key] = value
The Other Jimmy 31:8ea194f6145b 2096
The Other Jimmy 31:8ea194f6145b 2097 del kwargs['toolchain_paths']
The Other Jimmy 31:8ea194f6145b 2098
The Other Jimmy 31:8ea194f6145b 2099 try:
The Other Jimmy 31:8ea194f6145b 2100 bin_file = build_project(*args, **kwargs)
The Other Jimmy 31:8ea194f6145b 2101 ret['result'] = True
The Other Jimmy 31:8ea194f6145b 2102 ret['bin_file'] = bin_file
The Other Jimmy 31:8ea194f6145b 2103 ret['kwargs'] = kwargs
The Other Jimmy 31:8ea194f6145b 2104
The Other Jimmy 31:8ea194f6145b 2105 except NotSupportedException, e:
The Other Jimmy 31:8ea194f6145b 2106 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2107 except ToolException, e:
The Other Jimmy 31:8ea194f6145b 2108 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2109 except KeyboardInterrupt, e:
The Other Jimmy 31:8ea194f6145b 2110 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2111 except:
The Other Jimmy 31:8ea194f6145b 2112 # Print unhandled exceptions here
The Other Jimmy 31:8ea194f6145b 2113 import traceback
The Other Jimmy 31:8ea194f6145b 2114 traceback.print_exc(file=sys.stdout)
The Other Jimmy 31:8ea194f6145b 2115
The Other Jimmy 31:8ea194f6145b 2116 return ret
The Other Jimmy 31:8ea194f6145b 2117
The Other Jimmy 31:8ea194f6145b 2118
screamer 0:66f3b5499f7f 2119 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2120 clean=False, notify=None, verbose=False, jobs=1, macros=None,
The Other Jimmy 31:8ea194f6145b 2121 silent=False, report=None, properties=None,
The Other Jimmy 31:8ea194f6145b 2122 continue_on_build_fail=False, app_config=None,
theotherjimmy 40:7d3fa6b99b2b 2123 build_profile=None, stats_depth=None):
screamer 0:66f3b5499f7f 2124 """Given the data structure from 'find_tests' and the typical build parameters,
screamer 13:ab47a20b66f0 2125 build all the tests
screamer 22:9e85236d8716 2126
screamer 13:ab47a20b66f0 2127 Returns a tuple of the build result (True or False) followed by the test
screamer 13:ab47a20b66f0 2128 build data structure"""
screamer 22:9e85236d8716 2129
screamer 13:ab47a20b66f0 2130 execution_directory = "."
screamer 13:ab47a20b66f0 2131 base_path = norm_relative_path(build_path, execution_directory)
screamer 13:ab47a20b66f0 2132
screamer 13:ab47a20b66f0 2133 target_name = target if isinstance(target, str) else target.name
The Other Jimmy 35:da9c89f8be7d 2134 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2135
The Other Jimmy 31:8ea194f6145b 2136 baud_rate = 9600
The Other Jimmy 31:8ea194f6145b 2137 if 'platform.stdio-baud-rate' in cfg:
The Other Jimmy 31:8ea194f6145b 2138 baud_rate = cfg['platform.stdio-baud-rate'].value
The Other Jimmy 31:8ea194f6145b 2139
screamer 0:66f3b5499f7f 2140 test_build = {
screamer 13:ab47a20b66f0 2141 "platform": target_name,
screamer 0:66f3b5499f7f 2142 "toolchain": toolchain_name,
screamer 13:ab47a20b66f0 2143 "base_path": base_path,
The Other Jimmy 31:8ea194f6145b 2144 "baud_rate": baud_rate,
screamer 0:66f3b5499f7f 2145 "binary_type": "bootable",
screamer 0:66f3b5499f7f 2146 "tests": {}
screamer 0:66f3b5499f7f 2147 }
screamer 22:9e85236d8716 2148
screamer 13:ab47a20b66f0 2149 result = True
screamer 22:9e85236d8716 2150
The Other Jimmy 31:8ea194f6145b 2151 jobs_count = int(jobs if jobs else cpu_count())
The Other Jimmy 31:8ea194f6145b 2152 p = Pool(processes=jobs_count)
The Other Jimmy 31:8ea194f6145b 2153 results = []
screamer 0:66f3b5499f7f 2154 for test_name, test_path in tests.iteritems():
screamer 0:66f3b5499f7f 2155 test_build_path = os.path.join(build_path, test_path)
screamer 0:66f3b5499f7f 2156 src_path = base_source_paths + [test_path]
screamer 13:ab47a20b66f0 2157 bin_file = None
screamer 24:25bff2709c20 2158 test_case_folder_name = os.path.basename(test_path)
theotherjimmy 40:7d3fa6b99b2b 2159
The Other Jimmy 31:8ea194f6145b 2160 args = (src_path, test_build_path, target, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2161 kwargs = {
The Other Jimmy 31:8ea194f6145b 2162 'jobs': 1,
The Other Jimmy 31:8ea194f6145b 2163 'clean': clean,
The Other Jimmy 31:8ea194f6145b 2164 'macros': macros,
The Other Jimmy 31:8ea194f6145b 2165 'name': test_case_folder_name,
The Other Jimmy 31:8ea194f6145b 2166 'project_id': test_name,
The Other Jimmy 31:8ea194f6145b 2167 'report': report,
The Other Jimmy 31:8ea194f6145b 2168 'properties': properties,
The Other Jimmy 31:8ea194f6145b 2169 'verbose': verbose,
The Other Jimmy 31:8ea194f6145b 2170 'app_config': app_config,
The Other Jimmy 31:8ea194f6145b 2171 'build_profile': build_profile,
The Other Jimmy 31:8ea194f6145b 2172 'silent': True,
theotherjimmy 40:7d3fa6b99b2b 2173 'toolchain_paths': TOOLCHAIN_PATHS,
theotherjimmy 40:7d3fa6b99b2b 2174 'stats_depth': stats_depth
The Other Jimmy 31:8ea194f6145b 2175 }
theotherjimmy 40:7d3fa6b99b2b 2176
The Other Jimmy 31:8ea194f6145b 2177 results.append(p.apply_async(build_test_worker, args, kwargs))
The Other Jimmy 31:8ea194f6145b 2178
The Other Jimmy 31:8ea194f6145b 2179 p.close()
The Other Jimmy 31:8ea194f6145b 2180 result = True
The Other Jimmy 31:8ea194f6145b 2181 itr = 0
The Other Jimmy 31:8ea194f6145b 2182 while len(results):
The Other Jimmy 31:8ea194f6145b 2183 itr += 1
The Other Jimmy 31:8ea194f6145b 2184 if itr > 360000:
The Other Jimmy 31:8ea194f6145b 2185 p.terminate()
The Other Jimmy 31:8ea194f6145b 2186 p.join()
The Other Jimmy 31:8ea194f6145b 2187 raise ToolException("Compile did not finish in 10 minutes")
The Other Jimmy 31:8ea194f6145b 2188 else:
The Other Jimmy 31:8ea194f6145b 2189 sleep(0.01)
The Other Jimmy 31:8ea194f6145b 2190 pending = 0
The Other Jimmy 31:8ea194f6145b 2191 for r in results:
The Other Jimmy 31:8ea194f6145b 2192 if r.ready() is True:
The Other Jimmy 31:8ea194f6145b 2193 try:
The Other Jimmy 31:8ea194f6145b 2194 worker_result = r.get()
The Other Jimmy 31:8ea194f6145b 2195 results.remove(r)
screamer 13:ab47a20b66f0 2196
The Other Jimmy 31:8ea194f6145b 2197 # Take report from the kwargs and merge it into existing report
theotherjimmy 40:7d3fa6b99b2b 2198 if report:
theotherjimmy 40:7d3fa6b99b2b 2199 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
theotherjimmy 40:7d3fa6b99b2b 2200 for test_key in report_entry.keys():
theotherjimmy 40:7d3fa6b99b2b 2201 report[target_name][toolchain_name][test_key] = report_entry[test_key]
theotherjimmy 40:7d3fa6b99b2b 2202
The Other Jimmy 31:8ea194f6145b 2203 # Set the overall result to a failure if a build failure occurred
The Other Jimmy 35:da9c89f8be7d 2204 if ('reason' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2205 not worker_result['reason'] and
The Other Jimmy 35:da9c89f8be7d 2206 not isinstance(worker_result['reason'], NotSupportedException)):
The Other Jimmy 31:8ea194f6145b 2207 result = False
The Other Jimmy 31:8ea194f6145b 2208 break
The Other Jimmy 31:8ea194f6145b 2209
The Other Jimmy 31:8ea194f6145b 2210 # Adding binary path to test build result
The Other Jimmy 35:da9c89f8be7d 2211 if ('result' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2212 worker_result['result'] and
The Other Jimmy 35:da9c89f8be7d 2213 'bin_file' in worker_result):
The Other Jimmy 31:8ea194f6145b 2214 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
screamer 22:9e85236d8716 2215
The Other Jimmy 31:8ea194f6145b 2216 test_build['tests'][worker_result['kwargs']['project_id']] = {
The Other Jimmy 31:8ea194f6145b 2217 "binaries": [
The Other Jimmy 31:8ea194f6145b 2218 {
The Other Jimmy 31:8ea194f6145b 2219 "path": bin_file
The Other Jimmy 31:8ea194f6145b 2220 }
The Other Jimmy 31:8ea194f6145b 2221 ]
The Other Jimmy 31:8ea194f6145b 2222 }
The Other Jimmy 31:8ea194f6145b 2223
The Other Jimmy 31:8ea194f6145b 2224 test_key = worker_result['kwargs']['project_id'].upper()
theotherjimmy 40:7d3fa6b99b2b 2225 if report:
theotherjimmy 40:7d3fa6b99b2b 2226 print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
The Other Jimmy 31:8ea194f6145b 2227 print 'Image: %s\n' % bin_file
The Other Jimmy 31:8ea194f6145b 2228
The Other Jimmy 31:8ea194f6145b 2229 except:
The Other Jimmy 31:8ea194f6145b 2230 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2231 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2232 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2233 p.terminate()
The Other Jimmy 31:8ea194f6145b 2234 p.join()
The Other Jimmy 31:8ea194f6145b 2235 raise
screamer 13:ab47a20b66f0 2236 else:
The Other Jimmy 31:8ea194f6145b 2237 pending += 1
The Other Jimmy 31:8ea194f6145b 2238 if pending >= jobs_count:
The Other Jimmy 31:8ea194f6145b 2239 break
screamer 22:9e85236d8716 2240
The Other Jimmy 31:8ea194f6145b 2241 # Break as soon as possible if there is a failure and we are not
The Other Jimmy 31:8ea194f6145b 2242 # continuing on build failures
The Other Jimmy 31:8ea194f6145b 2243 if not result and not continue_on_build_fail:
The Other Jimmy 31:8ea194f6145b 2244 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2245 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2246 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2247 p.terminate()
The Other Jimmy 31:8ea194f6145b 2248 break
screamer 22:9e85236d8716 2249
The Other Jimmy 31:8ea194f6145b 2250 p.join()
screamer 22:9e85236d8716 2251
screamer 0:66f3b5499f7f 2252 test_builds = {}
screamer 13:ab47a20b66f0 2253 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
screamer 22:9e85236d8716 2254
screamer 13:ab47a20b66f0 2255 return result, test_builds
screamer 22:9e85236d8716 2256
screamer 0:66f3b5499f7f 2257
screamer 13:ab47a20b66f0 2258 def test_spec_from_test_builds(test_builds):
screamer 0:66f3b5499f7f 2259 return {
screamer 0:66f3b5499f7f 2260 "builds": test_builds
screamer 0:66f3b5499f7f 2261 }