Anders Blomdell / mbed-sdk-tools

Clone of official tools

test_api.py@35:da9c89f8be7d, 2017-02-15 (annotated)

Committer:
The Other Jimmy
Date:
Wed Feb 15 13:53:18 2017 -0600
Revision:
35:da9c89f8be7d
Parent:
31:8ea194f6145b
Child:
36:96847d42f010
Update tools to mbed-os 5.3.5

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """
screamer 0:66f3b5499f7f 2 mbed SDK
screamer 0:66f3b5499f7f 3 Copyright (c) 2011-2014 ARM Limited
screamer 0:66f3b5499f7f 4
screamer 0:66f3b5499f7f 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 6 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 7 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 8
screamer 0:66f3b5499f7f 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 14 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 15 limitations under the License.
screamer 0:66f3b5499f7f 16
screamer 0:66f3b5499f7f 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer 0:66f3b5499f7f 18 """
screamer 0:66f3b5499f7f 19
screamer 0:66f3b5499f7f 20 import os
screamer 0:66f3b5499f7f 21 import re
screamer 0:66f3b5499f7f 22 import sys
screamer 0:66f3b5499f7f 23 import json
screamer 0:66f3b5499f7f 24 import uuid
screamer 0:66f3b5499f7f 25 import pprint
screamer 0:66f3b5499f7f 26 import random
screamer 22:9e85236d8716 27 import argparse
screamer 0:66f3b5499f7f 28 import datetime
screamer 0:66f3b5499f7f 29 import threading
screamer 0:66f3b5499f7f 30 import ctypes
screamer 0:66f3b5499f7f 31 from types import ListType
screamer 0:66f3b5499f7f 32 from colorama import Fore, Back, Style
screamer 0:66f3b5499f7f 33 from prettytable import PrettyTable
screamer 24:25bff2709c20 34 from copy import copy
screamer 0:66f3b5499f7f 35
screamer 0:66f3b5499f7f 36 from time import sleep, time
screamer 0:66f3b5499f7f 37 from Queue import Queue, Empty
screamer 24:25bff2709c20 38 from os.path import join, exists, basename, relpath
screamer 0:66f3b5499f7f 39 from threading import Thread, Lock
The Other Jimmy 31:8ea194f6145b 40 from multiprocessing import Pool, cpu_count
screamer 0:66f3b5499f7f 41 from subprocess import Popen, PIPE
screamer 0:66f3b5499f7f 42
screamer 0:66f3b5499f7f 43 # Imports related to mbed build api
screamer 0:66f3b5499f7f 44 from tools.tests import TESTS
screamer 0:66f3b5499f7f 45 from tools.tests import TEST_MAP
screamer 0:66f3b5499f7f 46 from tools.paths import BUILD_DIR
screamer 0:66f3b5499f7f 47 from tools.paths import HOST_TESTS
screamer 0:66f3b5499f7f 48 from tools.utils import ToolException
screamer 0:66f3b5499f7f 49 from tools.utils import NotSupportedException
screamer 0:66f3b5499f7f 50 from tools.utils import construct_enum
screamer 22:9e85236d8716 51 from tools.memap import MemapParser
screamer 0:66f3b5499f7f 52 from tools.targets import TARGET_MAP
screamer 0:66f3b5499f7f 53 from tools.test_db import BaseDBAccess
screamer 0:66f3b5499f7f 54 from tools.build_api import build_project, build_mbed_libs, build_lib
screamer 0:66f3b5499f7f 55 from tools.build_api import get_target_supported_toolchains
screamer 0:66f3b5499f7f 56 from tools.build_api import write_build_report
screamer 0:66f3b5499f7f 57 from tools.build_api import prep_report
screamer 0:66f3b5499f7f 58 from tools.build_api import prep_properties
screamer 0:66f3b5499f7f 59 from tools.build_api import create_result
screamer 0:66f3b5499f7f 60 from tools.build_api import add_result_to_report
screamer 24:25bff2709c20 61 from tools.build_api import prepare_toolchain
screamer 24:25bff2709c20 62 from tools.build_api import scan_resources
The Other Jimmy 31:8ea194f6145b 63 from tools.build_api import get_config
screamer 0:66f3b5499f7f 64 from tools.libraries import LIBRARIES, LIBRARY_MAP
The Other Jimmy 31:8ea194f6145b 65 from tools.options import extract_profile
screamer 20:835f6355470d 66 from tools.toolchains import TOOLCHAIN_PATHS
screamer 22:9e85236d8716 67 from tools.toolchains import TOOLCHAINS
screamer 0:66f3b5499f7f 68 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 22:9e85236d8716 69 from tools.utils import argparse_filestring_type
screamer 22:9e85236d8716 70 from tools.utils import argparse_uppercase_type
screamer 22:9e85236d8716 71 from tools.utils import argparse_lowercase_type
screamer 22:9e85236d8716 72 from tools.utils import argparse_many
screamer 24:25bff2709c20 73 from tools.utils import get_path_depth
screamer 0:66f3b5499f7f 74
screamer 0:66f3b5499f7f 75 import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer 0:66f3b5499f7f 76
screamer 0:66f3b5499f7f 77 try:
screamer 0:66f3b5499f7f 78 import mbed_lstools
screamer 0:66f3b5499f7f 79 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:66f3b5499f7f 80 except:
screamer 0:66f3b5499f7f 81 pass
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83
screamer 0:66f3b5499f7f 84 class ProcessObserver(Thread):
screamer 0:66f3b5499f7f 85 def __init__(self, proc):
screamer 0:66f3b5499f7f 86 Thread.__init__(self)
screamer 0:66f3b5499f7f 87 self.proc = proc
screamer 0:66f3b5499f7f 88 self.queue = Queue()
screamer 0:66f3b5499f7f 89 self.daemon = True
screamer 0:66f3b5499f7f 90 self.active = True
screamer 0:66f3b5499f7f 91 self.start()
screamer 0:66f3b5499f7f 92
screamer 0:66f3b5499f7f 93 def run(self):
screamer 0:66f3b5499f7f 94 while self.active:
screamer 0:66f3b5499f7f 95 c = self.proc.stdout.read(1)
screamer 0:66f3b5499f7f 96 self.queue.put(c)
screamer 0:66f3b5499f7f 97
screamer 0:66f3b5499f7f 98 def stop(self):
screamer 0:66f3b5499f7f 99 self.active = False
screamer 0:66f3b5499f7f 100 try:
screamer 0:66f3b5499f7f 101 self.proc.terminate()
screamer 0:66f3b5499f7f 102 except Exception, _:
screamer 0:66f3b5499f7f 103 pass
screamer 0:66f3b5499f7f 104
screamer 0:66f3b5499f7f 105
screamer 0:66f3b5499f7f 106 class SingleTestExecutor(threading.Thread):
screamer 0:66f3b5499f7f 107 """ Example: Single test class in separate thread usage
screamer 0:66f3b5499f7f 108 """
screamer 0:66f3b5499f7f 109 def __init__(self, single_test):
screamer 0:66f3b5499f7f 110 self.single_test = single_test
screamer 0:66f3b5499f7f 111 threading.Thread.__init__(self)
screamer 0:66f3b5499f7f 112
screamer 0:66f3b5499f7f 113 def run(self):
screamer 0:66f3b5499f7f 114 start = time()
screamer 0:66f3b5499f7f 115 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 116 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer 0:66f3b5499f7f 117 elapsed_time = time() - start
screamer 0:66f3b5499f7f 118
screamer 0:66f3b5499f7f 119 # Human readable summary
screamer 0:66f3b5499f7f 120 if not self.single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 121 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 122 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 123 if self.single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 124 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 125 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 126 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 127 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 128
screamer 0:66f3b5499f7f 129
screamer 0:66f3b5499f7f 130 class SingleTestRunner(object):
screamer 0:66f3b5499f7f 131 """ Object wrapper for single test run which may involve multiple MUTs
screamer 0:66f3b5499f7f 132 """
screamer 0:66f3b5499f7f 133 RE_DETECT_TESTCASE_RESULT = None
screamer 0:66f3b5499f7f 134
screamer 0:66f3b5499f7f 135 # Return codes for test script
screamer 0:66f3b5499f7f 136 TEST_RESULT_OK = "OK"
screamer 0:66f3b5499f7f 137 TEST_RESULT_FAIL = "FAIL"
screamer 0:66f3b5499f7f 138 TEST_RESULT_ERROR = "ERROR"
screamer 0:66f3b5499f7f 139 TEST_RESULT_UNDEF = "UNDEF"
screamer 0:66f3b5499f7f 140 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer 0:66f3b5499f7f 141 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer 0:66f3b5499f7f 142 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer 0:66f3b5499f7f 143 TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer 0:66f3b5499f7f 144 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer 0:66f3b5499f7f 145 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer 0:66f3b5499f7f 146 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer 0:66f3b5499f7f 147 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer 0:66f3b5499f7f 148
screamer 0:66f3b5499f7f 149 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer 0:66f3b5499f7f 150 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer 0:66f3b5499f7f 151 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer 0:66f3b5499f7f 152
screamer 0:66f3b5499f7f 153 muts = {} # MUTs descriptor (from external file)
screamer 0:66f3b5499f7f 154 test_spec = {} # Test specification (from external file)
screamer 0:66f3b5499f7f 155
screamer 0:66f3b5499f7f 156 # mbed test suite -> SingleTestRunner
screamer 0:66f3b5499f7f 157 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer 0:66f3b5499f7f 158 "failure" : TEST_RESULT_FAIL,
screamer 0:66f3b5499f7f 159 "error" : TEST_RESULT_ERROR,
screamer 0:66f3b5499f7f 160 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer 0:66f3b5499f7f 161 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer 0:66f3b5499f7f 162 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer 0:66f3b5499f7f 163 "timeout" : TEST_RESULT_TIMEOUT,
screamer 0:66f3b5499f7f 164 "no_image" : TEST_RESULT_NO_IMAGE,
screamer 0:66f3b5499f7f 165 "end" : TEST_RESULT_UNDEF,
screamer 0:66f3b5499f7f 166 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer 0:66f3b5499f7f 167 "build_failed" : TEST_RESULT_BUILD_FAILED,
screamer 0:66f3b5499f7f 168 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 169 }
screamer 0:66f3b5499f7f 170
screamer 0:66f3b5499f7f 171 def __init__(self,
screamer 0:66f3b5499f7f 172 _global_loops_count=1,
screamer 0:66f3b5499f7f 173 _test_loops_list=None,
screamer 0:66f3b5499f7f 174 _muts={},
screamer 0:66f3b5499f7f 175 _clean=False,
The Other Jimmy 31:8ea194f6145b 176 _parser=None,
The Other Jimmy 31:8ea194f6145b 177 _opts=None,
screamer 0:66f3b5499f7f 178 _opts_db_url=None,
screamer 0:66f3b5499f7f 179 _opts_log_file_name=None,
screamer 0:66f3b5499f7f 180 _opts_report_html_file_name=None,
screamer 0:66f3b5499f7f 181 _opts_report_junit_file_name=None,
screamer 0:66f3b5499f7f 182 _opts_report_build_file_name=None,
screamer 22:9e85236d8716 183 _opts_report_text_file_name=None,
screamer 0:66f3b5499f7f 184 _opts_build_report={},
screamer 0:66f3b5499f7f 185 _opts_build_properties={},
screamer 0:66f3b5499f7f 186 _test_spec={},
screamer 0:66f3b5499f7f 187 _opts_goanna_for_mbed_sdk=None,
screamer 0:66f3b5499f7f 188 _opts_goanna_for_tests=None,
screamer 0:66f3b5499f7f 189 _opts_shuffle_test_order=False,
screamer 0:66f3b5499f7f 190 _opts_shuffle_test_seed=None,
screamer 0:66f3b5499f7f 191 _opts_test_by_names=None,
screamer 0:66f3b5499f7f 192 _opts_peripheral_by_names=None,
screamer 0:66f3b5499f7f 193 _opts_test_only_peripheral=False,
screamer 0:66f3b5499f7f 194 _opts_test_only_common=False,
screamer 0:66f3b5499f7f 195 _opts_verbose_skipped_tests=False,
screamer 0:66f3b5499f7f 196 _opts_verbose_test_result_only=False,
screamer 0:66f3b5499f7f 197 _opts_verbose=False,
screamer 0:66f3b5499f7f 198 _opts_firmware_global_name=None,
screamer 0:66f3b5499f7f 199 _opts_only_build_tests=False,
screamer 0:66f3b5499f7f 200 _opts_parallel_test_exec=False,
screamer 0:66f3b5499f7f 201 _opts_suppress_summary=False,
screamer 0:66f3b5499f7f 202 _opts_test_x_toolchain_summary=False,
screamer 0:66f3b5499f7f 203 _opts_copy_method=None,
screamer 0:66f3b5499f7f 204 _opts_mut_reset_type=None,
screamer 0:66f3b5499f7f 205 _opts_jobs=None,
screamer 0:66f3b5499f7f 206 _opts_waterfall_test=None,
screamer 0:66f3b5499f7f 207 _opts_consolidate_waterfall_test=None,
screamer 0:66f3b5499f7f 208 _opts_extend_test_timeout=None,
screamer 0:66f3b5499f7f 209 _opts_auto_detect=None,
screamer 0:66f3b5499f7f 210 _opts_include_non_automated=False):
screamer 0:66f3b5499f7f 211 """ Let's try hard to init this object
screamer 0:66f3b5499f7f 212 """
screamer 0:66f3b5499f7f 213 from colorama import init
screamer 0:66f3b5499f7f 214 init()
screamer 0:66f3b5499f7f 215
screamer 0:66f3b5499f7f 216 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer 0:66f3b5499f7f 217 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer 0:66f3b5499f7f 218 # Settings related to test loops counters
screamer 0:66f3b5499f7f 219 try:
screamer 0:66f3b5499f7f 220 _global_loops_count = int(_global_loops_count)
screamer 0:66f3b5499f7f 221 except:
screamer 0:66f3b5499f7f 222 _global_loops_count = 1
screamer 0:66f3b5499f7f 223 if _global_loops_count < 1:
screamer 0:66f3b5499f7f 224 _global_loops_count = 1
screamer 0:66f3b5499f7f 225 self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer 0:66f3b5499f7f 226 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer 0:66f3b5499f7f 227 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer 0:66f3b5499f7f 228
screamer 0:66f3b5499f7f 229 self.shuffle_random_seed = 0.0
screamer 0:66f3b5499f7f 230 self.SHUFFLE_SEED_ROUND = 10
screamer 0:66f3b5499f7f 231
screamer 0:66f3b5499f7f 232 # MUT list and test specification storage
screamer 0:66f3b5499f7f 233 self.muts = _muts
screamer 0:66f3b5499f7f 234 self.test_spec = _test_spec
screamer 0:66f3b5499f7f 235
screamer 0:66f3b5499f7f 236 # Settings passed e.g. from command line
screamer 0:66f3b5499f7f 237 self.opts_db_url = _opts_db_url
screamer 0:66f3b5499f7f 238 self.opts_log_file_name = _opts_log_file_name
screamer 0:66f3b5499f7f 239 self.opts_report_html_file_name = _opts_report_html_file_name
screamer 0:66f3b5499f7f 240 self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer 0:66f3b5499f7f 241 self.opts_report_build_file_name = _opts_report_build_file_name
screamer 22:9e85236d8716 242 self.opts_report_text_file_name = _opts_report_text_file_name
screamer 0:66f3b5499f7f 243 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer 0:66f3b5499f7f 244 self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer 0:66f3b5499f7f 245 self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer 0:66f3b5499f7f 246 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer 0:66f3b5499f7f 247 self.opts_test_by_names = _opts_test_by_names
screamer 0:66f3b5499f7f 248 self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer 0:66f3b5499f7f 249 self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer 0:66f3b5499f7f 250 self.opts_test_only_common = _opts_test_only_common
screamer 0:66f3b5499f7f 251 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer 0:66f3b5499f7f 252 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer 0:66f3b5499f7f 253 self.opts_verbose = _opts_verbose
screamer 0:66f3b5499f7f 254 self.opts_firmware_global_name = _opts_firmware_global_name
screamer 0:66f3b5499f7f 255 self.opts_only_build_tests = _opts_only_build_tests
screamer 0:66f3b5499f7f 256 self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer 0:66f3b5499f7f 257 self.opts_suppress_summary = _opts_suppress_summary
screamer 0:66f3b5499f7f 258 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer 0:66f3b5499f7f 259 self.opts_copy_method = _opts_copy_method
screamer 0:66f3b5499f7f 260 self.opts_mut_reset_type = _opts_mut_reset_type
screamer 0:66f3b5499f7f 261 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer 0:66f3b5499f7f 262 self.opts_waterfall_test = _opts_waterfall_test
screamer 0:66f3b5499f7f 263 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer 0:66f3b5499f7f 264 self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer 0:66f3b5499f7f 265 self.opts_clean = _clean
The Other Jimmy 31:8ea194f6145b 266 self.opts_parser = _parser
The Other Jimmy 31:8ea194f6145b 267 self.opts = _opts
screamer 0:66f3b5499f7f 268 self.opts_auto_detect = _opts_auto_detect
screamer 0:66f3b5499f7f 269 self.opts_include_non_automated = _opts_include_non_automated
screamer 0:66f3b5499f7f 270
screamer 0:66f3b5499f7f 271 self.build_report = _opts_build_report
screamer 0:66f3b5499f7f 272 self.build_properties = _opts_build_properties
screamer 0:66f3b5499f7f 273
screamer 0:66f3b5499f7f 274 # File / screen logger initialization
screamer 0:66f3b5499f7f 275 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer 0:66f3b5499f7f 276
screamer 0:66f3b5499f7f 277 # Database related initializations
screamer 0:66f3b5499f7f 278 self.db_logger = factory_db_logger(self.opts_db_url)
screamer 0:66f3b5499f7f 279 self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer 0:66f3b5499f7f 280 # Let's connect to database to set up credentials and confirm database is ready
screamer 0:66f3b5499f7f 281 if self.db_logger:
screamer 0:66f3b5499f7f 282 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer 0:66f3b5499f7f 283 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 284 # Get hostname and uname so we can use it as build description
screamer 0:66f3b5499f7f 285 # when creating new build_id in external database
screamer 0:66f3b5499f7f 286 (_hostname, _uname) = self.db_logger.get_hostname()
screamer 0:66f3b5499f7f 287 _host_location = os.path.dirname(os.path.abspath(__file__))
screamer 0:66f3b5499f7f 288 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer 0:66f3b5499f7f 289 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer 0:66f3b5499f7f 290 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 291
screamer 0:66f3b5499f7f 292 def dump_options(self):
screamer 0:66f3b5499f7f 293 """ Function returns data structure with common settings passed to SingelTestRunner
screamer 0:66f3b5499f7f 294 It can be used for example to fill _extra fields in database storing test suite single run data
screamer 0:66f3b5499f7f 295 Example:
screamer 0:66f3b5499f7f 296 data = self.dump_options()
screamer 0:66f3b5499f7f 297 or
screamer 0:66f3b5499f7f 298 data_str = json.dumps(self.dump_options())
screamer 0:66f3b5499f7f 299 """
screamer 0:66f3b5499f7f 300 result = {"db_url" : str(self.opts_db_url),
screamer 0:66f3b5499f7f 301 "log_file_name" : str(self.opts_log_file_name),
screamer 0:66f3b5499f7f 302 "shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer 0:66f3b5499f7f 303 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer 0:66f3b5499f7f 304 "test_by_names" : str(self.opts_test_by_names),
screamer 0:66f3b5499f7f 305 "peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer 0:66f3b5499f7f 306 "test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer 0:66f3b5499f7f 307 "test_only_common" : str(self.opts_test_only_common),
screamer 0:66f3b5499f7f 308 "verbose" : str(self.opts_verbose),
screamer 0:66f3b5499f7f 309 "firmware_global_name" : str(self.opts_firmware_global_name),
screamer 0:66f3b5499f7f 310 "only_build_tests" : str(self.opts_only_build_tests),
screamer 0:66f3b5499f7f 311 "copy_method" : str(self.opts_copy_method),
screamer 0:66f3b5499f7f 312 "mut_reset_type" : str(self.opts_mut_reset_type),
screamer 0:66f3b5499f7f 313 "jobs" : str(self.opts_jobs),
screamer 0:66f3b5499f7f 314 "extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer 0:66f3b5499f7f 315 "_dummy" : ''
screamer 0:66f3b5499f7f 316 }
screamer 0:66f3b5499f7f 317 return result
screamer 0:66f3b5499f7f 318
screamer 0:66f3b5499f7f 319 def shuffle_random_func(self):
screamer 0:66f3b5499f7f 320 return self.shuffle_random_seed
screamer 0:66f3b5499f7f 321
screamer 0:66f3b5499f7f 322 def is_shuffle_seed_float(self):
screamer 0:66f3b5499f7f 323 """ return true if function parameter can be converted to float
screamer 0:66f3b5499f7f 324 """
screamer 0:66f3b5499f7f 325 result = True
screamer 0:66f3b5499f7f 326 try:
screamer 0:66f3b5499f7f 327 float(self.shuffle_random_seed)
screamer 0:66f3b5499f7f 328 except ValueError:
screamer 0:66f3b5499f7f 329 result = False
screamer 0:66f3b5499f7f 330 return result
screamer 0:66f3b5499f7f 331
screamer 0:66f3b5499f7f 332 # This will store target / toolchain specific properties
screamer 0:66f3b5499f7f 333 test_suite_properties_ext = {} # target : toolchain
screamer 0:66f3b5499f7f 334 # Here we store test results
screamer 0:66f3b5499f7f 335 test_summary = []
screamer 0:66f3b5499f7f 336 # Here we store test results in extended data structure
screamer 0:66f3b5499f7f 337 test_summary_ext = {}
screamer 0:66f3b5499f7f 338 execute_thread_slice_lock = Lock()
screamer 0:66f3b5499f7f 339
screamer 0:66f3b5499f7f 340 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer 0:66f3b5499f7f 341 for toolchain in toolchains:
screamer 0:66f3b5499f7f 342 tt_id = "%s::%s" % (toolchain, target)
screamer 0:66f3b5499f7f 343
screamer 0:66f3b5499f7f 344 T = TARGET_MAP[target]
screamer 0:66f3b5499f7f 345
screamer 0:66f3b5499f7f 346 # print target, toolchain
screamer 0:66f3b5499f7f 347 # Test suite properties returned to external tools like CI
screamer 0:66f3b5499f7f 348 test_suite_properties = {
screamer 0:66f3b5499f7f 349 'jobs': self.opts_jobs,
screamer 0:66f3b5499f7f 350 'clean': clean,
screamer 0:66f3b5499f7f 351 'target': target,
screamer 0:66f3b5499f7f 352 'vendor': T.extra_labels[0],
screamer 0:66f3b5499f7f 353 'test_ids': ', '.join(test_ids),
screamer 0:66f3b5499f7f 354 'toolchain': toolchain,
screamer 0:66f3b5499f7f 355 'shuffle_random_seed': self.shuffle_random_seed
screamer 0:66f3b5499f7f 356 }
screamer 0:66f3b5499f7f 357
screamer 0:66f3b5499f7f 358
screamer 0:66f3b5499f7f 359 # print '=== %s::%s ===' % (target, toolchain)
screamer 0:66f3b5499f7f 360 # Let's build our test
screamer 0:66f3b5499f7f 361 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 362 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
screamer 0:66f3b5499f7f 363 continue
screamer 0:66f3b5499f7f 364
screamer 0:66f3b5499f7f 365 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 366
The Other Jimmy 31:8ea194f6145b 367 profile = extract_profile(self.opts_parser, self.opts, toolchain)
The Other Jimmy 31:8ea194f6145b 368
screamer 0:66f3b5499f7f 369
screamer 0:66f3b5499f7f 370 try:
screamer 0:66f3b5499f7f 371 build_mbed_libs_result = build_mbed_libs(T,
screamer 0:66f3b5499f7f 372 toolchain,
screamer 0:66f3b5499f7f 373 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 374 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 375 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 376 report=build_report,
The Other Jimmy 31:8ea194f6145b 377 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 378 build_profile=profile)
screamer 0:66f3b5499f7f 379
screamer 0:66f3b5499f7f 380 if not build_mbed_libs_result:
screamer 0:66f3b5499f7f 381 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
screamer 0:66f3b5499f7f 382 continue
screamer 0:66f3b5499f7f 383
screamer 0:66f3b5499f7f 384 except ToolException:
screamer 0:66f3b5499f7f 385 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
screamer 0:66f3b5499f7f 386 continue
screamer 0:66f3b5499f7f 387
screamer 0:66f3b5499f7f 388 build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer 0:66f3b5499f7f 389
screamer 0:66f3b5499f7f 390 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer 0:66f3b5499f7f 391 test_suite_properties['build_dir'] = build_dir
screamer 0:66f3b5499f7f 392 test_suite_properties['skipped'] = []
screamer 0:66f3b5499f7f 393
screamer 0:66f3b5499f7f 394 # Enumerate through all tests and shuffle test order if requested
screamer 0:66f3b5499f7f 395 test_map_keys = sorted(TEST_MAP.keys())
screamer 0:66f3b5499f7f 396
screamer 0:66f3b5499f7f 397 if self.opts_shuffle_test_order:
screamer 0:66f3b5499f7f 398 random.shuffle(test_map_keys, self.shuffle_random_func)
screamer 0:66f3b5499f7f 399 # Update database with shuffle seed f applicable
screamer 0:66f3b5499f7f 400 if self.db_logger:
screamer 0:66f3b5499f7f 401 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 402 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 403 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
screamer 0:66f3b5499f7f 404 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 405
screamer 0:66f3b5499f7f 406 if self.db_logger:
screamer 0:66f3b5499f7f 407 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 408 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 409 # Update MUTs and Test Specification in database
screamer 0:66f3b5499f7f 410 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
screamer 0:66f3b5499f7f 411 # Update Extra information in database (some options passed to test suite)
screamer 0:66f3b5499f7f 412 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
screamer 0:66f3b5499f7f 413 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 414
screamer 0:66f3b5499f7f 415 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer 0:66f3b5499f7f 416 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer 0:66f3b5499f7f 417
screamer 0:66f3b5499f7f 418 for skipped_test_id in skipped_test_map_keys:
screamer 0:66f3b5499f7f 419 test_suite_properties['skipped'].append(skipped_test_id)
screamer 0:66f3b5499f7f 420
screamer 0:66f3b5499f7f 421
screamer 0:66f3b5499f7f 422 # First pass through all tests and determine which libraries need to be built
screamer 0:66f3b5499f7f 423 libraries = []
screamer 0:66f3b5499f7f 424 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 425 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 426
screamer 0:66f3b5499f7f 427 # Detect which lib should be added to test
screamer 0:66f3b5499f7f 428 # Some libs have to compiled like RTOS or ETH
screamer 0:66f3b5499f7f 429 for lib in LIBRARIES:
screamer 0:66f3b5499f7f 430 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer 0:66f3b5499f7f 431 libraries.append(lib['id'])
screamer 0:66f3b5499f7f 432
screamer 0:66f3b5499f7f 433
screamer 0:66f3b5499f7f 434 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 435
screamer 0:66f3b5499f7f 436 # Build all required libraries
screamer 0:66f3b5499f7f 437 for lib_id in libraries:
screamer 0:66f3b5499f7f 438 try:
screamer 0:66f3b5499f7f 439 build_lib(lib_id,
screamer 0:66f3b5499f7f 440 T,
screamer 0:66f3b5499f7f 441 toolchain,
screamer 0:66f3b5499f7f 442 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 443 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 444 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 445 report=build_report,
The Other Jimmy 31:8ea194f6145b 446 properties=build_properties,
The Other Jimmy 31:8ea194f6145b 447 build_profile=profile)
screamer 0:66f3b5499f7f 448
screamer 0:66f3b5499f7f 449 except ToolException:
screamer 0:66f3b5499f7f 450 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
screamer 0:66f3b5499f7f 451 continue
screamer 0:66f3b5499f7f 452
screamer 0:66f3b5499f7f 453
screamer 0:66f3b5499f7f 454 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 455 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 456
screamer 0:66f3b5499f7f 457 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer 0:66f3b5499f7f 458
screamer 0:66f3b5499f7f 459 # TODO: move this 2 below loops to separate function
screamer 0:66f3b5499f7f 460 INC_DIRS = []
screamer 0:66f3b5499f7f 461 for lib_id in libraries:
screamer 0:66f3b5499f7f 462 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer 0:66f3b5499f7f 463 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer 0:66f3b5499f7f 464
screamer 0:66f3b5499f7f 465 MACROS = []
screamer 0:66f3b5499f7f 466 for lib_id in libraries:
screamer 0:66f3b5499f7f 467 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer 0:66f3b5499f7f 468 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer 0:66f3b5499f7f 469 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer 0:66f3b5499f7f 470 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer 0:66f3b5499f7f 471 test_uuid = uuid.uuid4()
screamer 0:66f3b5499f7f 472 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer 0:66f3b5499f7f 473
screamer 0:66f3b5499f7f 474 # Prepare extended test results data structure (it can be used to generate detailed test report)
screamer 0:66f3b5499f7f 475 if target not in self.test_summary_ext:
screamer 0:66f3b5499f7f 476 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer 0:66f3b5499f7f 477 if toolchain not in self.test_summary_ext[target]:
screamer 0:66f3b5499f7f 478 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer 0:66f3b5499f7f 479
screamer 0:66f3b5499f7f 480 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer 0:66f3b5499f7f 481
screamer 0:66f3b5499f7f 482 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer 0:66f3b5499f7f 483 try:
screamer 0:66f3b5499f7f 484 path = build_project(test.source_dir,
screamer 0:66f3b5499f7f 485 join(build_dir, test_id),
screamer 0:66f3b5499f7f 486 T,
screamer 0:66f3b5499f7f 487 toolchain,
screamer 0:66f3b5499f7f 488 test.dependencies,
screamer 0:66f3b5499f7f 489 clean=clean_project_options,
screamer 0:66f3b5499f7f 490 verbose=self.opts_verbose,
screamer 0:66f3b5499f7f 491 name=project_name,
screamer 0:66f3b5499f7f 492 macros=MACROS,
screamer 0:66f3b5499f7f 493 inc_dirs=INC_DIRS,
screamer 0:66f3b5499f7f 494 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 495 report=build_report,
screamer 0:66f3b5499f7f 496 properties=build_properties,
screamer 0:66f3b5499f7f 497 project_id=test_id,
The Other Jimmy 31:8ea194f6145b 498 project_description=test.get_description(),
The Other Jimmy 31:8ea194f6145b 499 build_profile=profile)
screamer 0:66f3b5499f7f 500
screamer 0:66f3b5499f7f 501 except Exception, e:
screamer 0:66f3b5499f7f 502 project_name_str = project_name if project_name is not None else test_id
screamer 0:66f3b5499f7f 503
screamer 0:66f3b5499f7f 504
screamer 0:66f3b5499f7f 505 test_result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 506
screamer 0:66f3b5499f7f 507 if isinstance(e, ToolException):
screamer 0:66f3b5499f7f 508 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
screamer 0:66f3b5499f7f 509 test_result = self.TEST_RESULT_BUILD_FAILED
screamer 0:66f3b5499f7f 510 elif isinstance(e, NotSupportedException):
screamer 0:66f3b5499f7f 511 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
screamer 0:66f3b5499f7f 512 test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 513
screamer 0:66f3b5499f7f 514
screamer 0:66f3b5499f7f 515 # Append test results to global test summary
screamer 0:66f3b5499f7f 516 self.test_summary.append(
screamer 0:66f3b5499f7f 517 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
screamer 0:66f3b5499f7f 518 )
screamer 0:66f3b5499f7f 519
screamer 0:66f3b5499f7f 520 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 521 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 522 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 523
screamer 0:66f3b5499f7f 524 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer 0:66f3b5499f7f 525 'result' : test_result,
screamer 0:66f3b5499f7f 526 'output' : '',
screamer 0:66f3b5499f7f 527 'target_name' : target,
screamer 0:66f3b5499f7f 528 'target_name_unique': target,
screamer 0:66f3b5499f7f 529 'toolchain_name' : toolchain,
screamer 0:66f3b5499f7f 530 'id' : test_id,
screamer 0:66f3b5499f7f 531 'description' : test.get_description(),
screamer 0:66f3b5499f7f 532 'elapsed_time' : 0,
screamer 0:66f3b5499f7f 533 'duration' : 0,
screamer 0:66f3b5499f7f 534 'copy_method' : None
screamer 0:66f3b5499f7f 535 }})
screamer 0:66f3b5499f7f 536 continue
screamer 0:66f3b5499f7f 537
screamer 0:66f3b5499f7f 538 if self.opts_only_build_tests:
screamer 0:66f3b5499f7f 539 # With this option we are skipping testing phase
screamer 0:66f3b5499f7f 540 continue
screamer 0:66f3b5499f7f 541
screamer 0:66f3b5499f7f 542 # Test duration can be increased by global value
screamer 0:66f3b5499f7f 543 test_duration = test.duration
screamer 0:66f3b5499f7f 544 if self.opts_extend_test_timeout is not None:
screamer 0:66f3b5499f7f 545 test_duration += self.opts_extend_test_timeout
screamer 0:66f3b5499f7f 546
screamer 0:66f3b5499f7f 547 # For an automated test the duration act as a timeout after
screamer 0:66f3b5499f7f 548 # which the test gets interrupted
screamer 0:66f3b5499f7f 549 test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer 0:66f3b5499f7f 550 test_loops = self.get_test_loop_count(test_id)
screamer 0:66f3b5499f7f 551
screamer 0:66f3b5499f7f 552 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer 0:66f3b5499f7f 553 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer 0:66f3b5499f7f 554 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer 0:66f3b5499f7f 555
screamer 0:66f3b5499f7f 556 # read MUTs, test specification and perform tests
screamer 0:66f3b5499f7f 557 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer 0:66f3b5499f7f 558
screamer 0:66f3b5499f7f 559 if handle_results is None:
screamer 0:66f3b5499f7f 560 continue
screamer 0:66f3b5499f7f 561
screamer 0:66f3b5499f7f 562 for handle_result in handle_results:
screamer 0:66f3b5499f7f 563 if handle_result:
screamer 0:66f3b5499f7f 564 single_test_result, detailed_test_results = handle_result
screamer 0:66f3b5499f7f 565 else:
screamer 0:66f3b5499f7f 566 continue
screamer 0:66f3b5499f7f 567
screamer 0:66f3b5499f7f 568 # Append test results to global test summary
screamer 0:66f3b5499f7f 569 if single_test_result is not None:
screamer 0:66f3b5499f7f 570 self.test_summary.append(single_test_result)
screamer 0:66f3b5499f7f 571
screamer 0:66f3b5499f7f 572 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 573 if target not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 574 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 575 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 576
screamer 0:66f3b5499f7f 577 append_test_result = detailed_test_results
screamer 0:66f3b5499f7f 578
screamer 0:66f3b5499f7f 579 # If waterfall and consolidate-waterfall options are enabled,
screamer 0:66f3b5499f7f 580 # only include the last test result in the report.
screamer 0:66f3b5499f7f 581 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer 0:66f3b5499f7f 582 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer 0:66f3b5499f7f 583
screamer 0:66f3b5499f7f 584 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer 0:66f3b5499f7f 585
screamer 0:66f3b5499f7f 586 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer 0:66f3b5499f7f 587 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer 0:66f3b5499f7f 588
screamer 0:66f3b5499f7f 589 q.put(target + '_'.join(toolchains))
screamer 0:66f3b5499f7f 590 return
screamer 0:66f3b5499f7f 591
screamer 0:66f3b5499f7f 592 def execute(self):
screamer 0:66f3b5499f7f 593 clean = self.test_spec.get('clean', False)
screamer 0:66f3b5499f7f 594 test_ids = self.test_spec.get('test_ids', [])
screamer 0:66f3b5499f7f 595 q = Queue()
screamer 0:66f3b5499f7f 596
screamer 0:66f3b5499f7f 597 # Generate seed for shuffle if seed is not provided in
screamer 0:66f3b5499f7f 598 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 599 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer 0:66f3b5499f7f 600 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 601
screamer 0:66f3b5499f7f 602
screamer 0:66f3b5499f7f 603 if self.opts_parallel_test_exec:
screamer 0:66f3b5499f7f 604 ###################################################################
screamer 0:66f3b5499f7f 605 # Experimental, parallel test execution per singletest instance.
screamer 0:66f3b5499f7f 606 ###################################################################
screamer 0:66f3b5499f7f 607 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer 0:66f3b5499f7f 608 # Note: We are building here in parallel for each target separately!
screamer 0:66f3b5499f7f 609 # So we are not building the same thing multiple times and compilers
screamer 0:66f3b5499f7f 610 # in separate threads do not collide.
screamer 0:66f3b5499f7f 611 # Inside execute_thread_slice() function function handle() will be called to
screamer 0:66f3b5499f7f 612 # get information about available MUTs (per target).
screamer 0:66f3b5499f7f 613 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 614 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 615 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer 0:66f3b5499f7f 616 t.daemon = True
screamer 0:66f3b5499f7f 617 t.start()
screamer 0:66f3b5499f7f 618 execute_threads.append(t)
screamer 0:66f3b5499f7f 619
screamer 0:66f3b5499f7f 620 for t in execute_threads:
screamer 0:66f3b5499f7f 621 q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer 0:66f3b5499f7f 622 else:
screamer 0:66f3b5499f7f 623 # Serialized (not parallel) test execution
screamer 0:66f3b5499f7f 624 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:66f3b5499f7f 625 if target not in self.test_suite_properties_ext:
screamer 0:66f3b5499f7f 626 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 627
screamer 0:66f3b5499f7f 628 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer 0:66f3b5499f7f 629 q.get()
screamer 0:66f3b5499f7f 630
screamer 0:66f3b5499f7f 631 if self.db_logger:
screamer 0:66f3b5499f7f 632 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 633 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 634 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer 0:66f3b5499f7f 635 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 636
screamer 0:66f3b5499f7f 637 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer 0:66f3b5499f7f 638
screamer 0:66f3b5499f7f 639 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer 0:66f3b5499f7f 640 valid_test_map_keys = []
screamer 0:66f3b5499f7f 641
screamer 0:66f3b5499f7f 642 for test_id in test_map_keys:
screamer 0:66f3b5499f7f 643 test = TEST_MAP[test_id]
screamer 22:9e85236d8716 644 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
screamer 0:66f3b5499f7f 645 continue
screamer 0:66f3b5499f7f 646
screamer 0:66f3b5499f7f 647 if test_ids and test_id not in test_ids:
screamer 0:66f3b5499f7f 648 continue
screamer 0:66f3b5499f7f 649
screamer 0:66f3b5499f7f 650 if self.opts_test_only_peripheral and not test.peripherals:
screamer 0:66f3b5499f7f 651 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 652 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 653 continue
screamer 0:66f3b5499f7f 654
screamer 22:9e85236d8716 655 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
screamer 0:66f3b5499f7f 656 # We will skip tests not forced with -p option
screamer 0:66f3b5499f7f 657 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 658 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 659 continue
screamer 0:66f3b5499f7f 660
screamer 0:66f3b5499f7f 661 if self.opts_test_only_common and test.peripherals:
screamer 0:66f3b5499f7f 662 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 663 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 664 continue
screamer 0:66f3b5499f7f 665
screamer 0:66f3b5499f7f 666 if not include_non_automated and not test.automated:
screamer 0:66f3b5499f7f 667 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 668 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
screamer 0:66f3b5499f7f 669 continue
screamer 0:66f3b5499f7f 670
screamer 0:66f3b5499f7f 671 if test.is_supported(target, toolchain):
screamer 0:66f3b5499f7f 672 if test.peripherals is None and self.opts_only_build_tests:
screamer 0:66f3b5499f7f 673 # When users are using 'build only flag' and test do not have
screamer 0:66f3b5499f7f 674 # specified peripherals we can allow test building by default
screamer 0:66f3b5499f7f 675 pass
screamer 22:9e85236d8716 676 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
screamer 0:66f3b5499f7f 677 # If we force peripheral with option -p we expect test
screamer 0:66f3b5499f7f 678 # to pass even if peripheral is not in MUTs file.
screamer 0:66f3b5499f7f 679 pass
screamer 0:66f3b5499f7f 680 elif not self.is_peripherals_available(target, test.peripherals):
screamer 0:66f3b5499f7f 681 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 682 if test.peripherals:
screamer 0:66f3b5499f7f 683 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
screamer 0:66f3b5499f7f 684 else:
screamer 0:66f3b5499f7f 685 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
screamer 0:66f3b5499f7f 686 continue
screamer 0:66f3b5499f7f 687
screamer 0:66f3b5499f7f 688 # The test has made it through all the filters, so add it to the valid tests list
screamer 0:66f3b5499f7f 689 valid_test_map_keys.append(test_id)
screamer 0:66f3b5499f7f 690
screamer 0:66f3b5499f7f 691 return valid_test_map_keys
screamer 0:66f3b5499f7f 692
screamer 0:66f3b5499f7f 693 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer 0:66f3b5499f7f 694 # NOTE: This will not preserve order
screamer 0:66f3b5499f7f 695 return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer 0:66f3b5499f7f 696
screamer 0:66f3b5499f7f 697 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 698 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 699 table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 700 """
screamer 0:66f3b5499f7f 701 RESULT_INDEX = 0
screamer 0:66f3b5499f7f 702 TARGET_INDEX = 1
screamer 0:66f3b5499f7f 703 TOOLCHAIN_INDEX = 2
screamer 0:66f3b5499f7f 704 TEST_INDEX = 3
screamer 0:66f3b5499f7f 705 DESC_INDEX = 4
screamer 0:66f3b5499f7f 706
screamer 0:66f3b5499f7f 707 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer 0:66f3b5499f7f 708 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer 0:66f3b5499f7f 709 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer 0:66f3b5499f7f 710 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer 0:66f3b5499f7f 711
screamer 0:66f3b5499f7f 712 result = "Test summary:\n"
screamer 0:66f3b5499f7f 713 for target in unique_targets:
screamer 0:66f3b5499f7f 714 result_dict = {} # test : { toolchain : result }
screamer 0:66f3b5499f7f 715 unique_target_toolchains = []
screamer 0:66f3b5499f7f 716 for test in test_summary:
screamer 0:66f3b5499f7f 717 if test[TARGET_INDEX] == target:
screamer 0:66f3b5499f7f 718 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer 0:66f3b5499f7f 719 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer 0:66f3b5499f7f 720 if test[TEST_INDEX] not in result_dict:
screamer 0:66f3b5499f7f 721 result_dict[test[TEST_INDEX]] = {}
screamer 0:66f3b5499f7f 722 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer 0:66f3b5499f7f 723
screamer 0:66f3b5499f7f 724 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
screamer 0:66f3b5499f7f 725 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 726 for col in pt_cols:
screamer 0:66f3b5499f7f 727 pt.align[col] = "l"
screamer 0:66f3b5499f7f 728 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 729
screamer 0:66f3b5499f7f 730 for test in unique_tests:
screamer 0:66f3b5499f7f 731 if test in result_dict:
screamer 0:66f3b5499f7f 732 test_results = result_dict[test]
screamer 0:66f3b5499f7f 733 if test in unique_test_desc:
screamer 0:66f3b5499f7f 734 row = [target, test, unique_test_desc[test]]
screamer 0:66f3b5499f7f 735 for toolchain in unique_toolchains:
screamer 0:66f3b5499f7f 736 if toolchain in test_results:
screamer 0:66f3b5499f7f 737 row.append(test_results[toolchain])
screamer 0:66f3b5499f7f 738 pt.add_row(row)
screamer 0:66f3b5499f7f 739 result += pt.get_string()
screamer 0:66f3b5499f7f 740 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 741 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 742 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 743 return result
screamer 0:66f3b5499f7f 744
screamer 0:66f3b5499f7f 745 def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 746 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 747 table shows target x test results matrix across
screamer 0:66f3b5499f7f 748 """
screamer 0:66f3b5499f7f 749 success_code = 0 # Success code that can be leter returned to
screamer 0:66f3b5499f7f 750 result = "Test summary:\n"
screamer 0:66f3b5499f7f 751 # Pretty table package is used to print results
screamer 0:66f3b5499f7f 752 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
screamer 0:66f3b5499f7f 753 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
screamer 0:66f3b5499f7f 754 pt.align["Result"] = "l" # Left align
screamer 0:66f3b5499f7f 755 pt.align["Target"] = "l" # Left align
screamer 0:66f3b5499f7f 756 pt.align["Toolchain"] = "l" # Left align
screamer 0:66f3b5499f7f 757 pt.align["Test ID"] = "l" # Left align
screamer 0:66f3b5499f7f 758 pt.align["Test Description"] = "l" # Left align
screamer 0:66f3b5499f7f 759 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 760
screamer 0:66f3b5499f7f 761 result_dict = {self.TEST_RESULT_OK : 0,
screamer 0:66f3b5499f7f 762 self.TEST_RESULT_FAIL : 0,
screamer 0:66f3b5499f7f 763 self.TEST_RESULT_ERROR : 0,
screamer 0:66f3b5499f7f 764 self.TEST_RESULT_UNDEF : 0,
screamer 0:66f3b5499f7f 765 self.TEST_RESULT_IOERR_COPY : 0,
screamer 0:66f3b5499f7f 766 self.TEST_RESULT_IOERR_DISK : 0,
screamer 0:66f3b5499f7f 767 self.TEST_RESULT_IOERR_SERIAL : 0,
screamer 0:66f3b5499f7f 768 self.TEST_RESULT_NO_IMAGE : 0,
screamer 0:66f3b5499f7f 769 self.TEST_RESULT_TIMEOUT : 0,
screamer 0:66f3b5499f7f 770 self.TEST_RESULT_MBED_ASSERT : 0,
screamer 0:66f3b5499f7f 771 self.TEST_RESULT_BUILD_FAILED : 0,
screamer 0:66f3b5499f7f 772 self.TEST_RESULT_NOT_SUPPORTED : 0
screamer 0:66f3b5499f7f 773 }
screamer 0:66f3b5499f7f 774
screamer 0:66f3b5499f7f 775 for test in test_summary:
screamer 0:66f3b5499f7f 776 if test[0] in result_dict:
screamer 0:66f3b5499f7f 777 result_dict[test[0]] += 1
screamer 0:66f3b5499f7f 778 pt.add_row(test)
screamer 0:66f3b5499f7f 779 result += pt.get_string()
screamer 0:66f3b5499f7f 780 result += "\n"
screamer 0:66f3b5499f7f 781
screamer 0:66f3b5499f7f 782 # Print result count
screamer 0:66f3b5499f7f 783 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
screamer 0:66f3b5499f7f 784 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 785 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 786 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 787 return result
screamer 0:66f3b5499f7f 788
screamer 0:66f3b5499f7f 789 def test_loop_list_to_dict(self, test_loops_str):
screamer 0:66f3b5499f7f 790 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer 0:66f3b5499f7f 791 """
screamer 0:66f3b5499f7f 792 result = {}
screamer 0:66f3b5499f7f 793 if test_loops_str:
screamer 22:9e85236d8716 794 test_loops = test_loops_str
screamer 0:66f3b5499f7f 795 for test_loop in test_loops:
screamer 0:66f3b5499f7f 796 test_loop_count = test_loop.split('=')
screamer 0:66f3b5499f7f 797 if len(test_loop_count) == 2:
screamer 0:66f3b5499f7f 798 _test_id, _test_loops = test_loop_count
screamer 0:66f3b5499f7f 799 try:
screamer 0:66f3b5499f7f 800 _test_loops = int(_test_loops)
screamer 0:66f3b5499f7f 801 except:
screamer 0:66f3b5499f7f 802 continue
screamer 0:66f3b5499f7f 803 result[_test_id] = _test_loops
screamer 0:66f3b5499f7f 804 return result
screamer 0:66f3b5499f7f 805
screamer 0:66f3b5499f7f 806 def get_test_loop_count(self, test_id):
screamer 0:66f3b5499f7f 807 """ This function returns no. of loops per test (deducted by test_id_.
screamer 0:66f3b5499f7f 808 If test is not in list of redefined loop counts it will use default value.
screamer 0:66f3b5499f7f 809 """
screamer 0:66f3b5499f7f 810 result = self.GLOBAL_LOOPS_COUNT
screamer 0:66f3b5499f7f 811 if test_id in self.TEST_LOOPS_DICT:
screamer 0:66f3b5499f7f 812 result = self.TEST_LOOPS_DICT[test_id]
screamer 0:66f3b5499f7f 813 return result
screamer 0:66f3b5499f7f 814
screamer 0:66f3b5499f7f 815 def delete_file(self, file_path):
screamer 0:66f3b5499f7f 816 """ Remove file from the system
screamer 0:66f3b5499f7f 817 """
screamer 0:66f3b5499f7f 818 result = True
screamer 0:66f3b5499f7f 819 resutl_msg = ""
screamer 0:66f3b5499f7f 820 try:
screamer 0:66f3b5499f7f 821 os.remove(file_path)
screamer 0:66f3b5499f7f 822 except Exception, e:
screamer 0:66f3b5499f7f 823 resutl_msg = e
screamer 0:66f3b5499f7f 824 result = False
screamer 0:66f3b5499f7f 825 return result, resutl_msg
screamer 0:66f3b5499f7f 826
screamer 0:66f3b5499f7f 827 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 828 """ Test is being invoked for given MUT.
screamer 0:66f3b5499f7f 829 """
screamer 0:66f3b5499f7f 830 # Get test information, image and test timeout
screamer 0:66f3b5499f7f 831 test_id = data['test_id']
screamer 0:66f3b5499f7f 832 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 833 test_description = TEST_MAP[test_id].get_description()
screamer 0:66f3b5499f7f 834 image = data["image"]
screamer 0:66f3b5499f7f 835 duration = data.get("duration", 10)
screamer 0:66f3b5499f7f 836
screamer 0:66f3b5499f7f 837 if mut is None:
screamer 0:66f3b5499f7f 838 print "Error: No Mbed available: MUT[%s]" % data['mcu']
screamer 0:66f3b5499f7f 839 return None
screamer 0:66f3b5499f7f 840
screamer 0:66f3b5499f7f 841 mcu = mut['mcu']
screamer 0:66f3b5499f7f 842 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer 0:66f3b5499f7f 843
screamer 0:66f3b5499f7f 844 if self.db_logger:
screamer 0:66f3b5499f7f 845 self.db_logger.reconnect()
screamer 0:66f3b5499f7f 846
screamer 0:66f3b5499f7f 847 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer 0:66f3b5499f7f 848
screamer 0:66f3b5499f7f 849 # Tests can be looped so test results must be stored for the same test
screamer 0:66f3b5499f7f 850 test_all_result = []
screamer 0:66f3b5499f7f 851 # Test results for one test ran few times
screamer 0:66f3b5499f7f 852 detailed_test_results = {} # { Loop_number: { results ... } }
screamer 0:66f3b5499f7f 853
screamer 0:66f3b5499f7f 854 for test_index in range(test_loops):
screamer 0:66f3b5499f7f 855
screamer 0:66f3b5499f7f 856 # If mbedls is available and we are auto detecting MUT info,
screamer 0:66f3b5499f7f 857 # update MUT info (mounting may changed)
screamer 0:66f3b5499f7f 858 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 859 platform_name_filter = [mcu]
screamer 0:66f3b5499f7f 860 muts_list = {}
screamer 0:66f3b5499f7f 861 found = False
screamer 0:66f3b5499f7f 862
screamer 0:66f3b5499f7f 863 for i in range(0, 60):
screamer 0:66f3b5499f7f 864 print('Looking for %s with MBEDLS' % mcu)
screamer 0:66f3b5499f7f 865 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 866
screamer 0:66f3b5499f7f 867 if 1 not in muts_list:
screamer 0:66f3b5499f7f 868 sleep(3)
screamer 0:66f3b5499f7f 869 else:
screamer 0:66f3b5499f7f 870 found = True
screamer 0:66f3b5499f7f 871 break
screamer 0:66f3b5499f7f 872
screamer 0:66f3b5499f7f 873 if not found:
screamer 0:66f3b5499f7f 874 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
screamer 0:66f3b5499f7f 875 return None
screamer 0:66f3b5499f7f 876 else:
screamer 0:66f3b5499f7f 877 mut = muts_list[1]
screamer 0:66f3b5499f7f 878
screamer 0:66f3b5499f7f 879 disk = mut.get('disk')
screamer 0:66f3b5499f7f 880 port = mut.get('port')
screamer 0:66f3b5499f7f 881
screamer 0:66f3b5499f7f 882 if disk is None or port is None:
screamer 0:66f3b5499f7f 883 return None
screamer 0:66f3b5499f7f 884
screamer 0:66f3b5499f7f 885 target_by_mcu = TARGET_MAP[mut['mcu']]
screamer 0:66f3b5499f7f 886 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer 0:66f3b5499f7f 887 # Some extra stuff can be declared in MUTs structure
screamer 0:66f3b5499f7f 888 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer 0:66f3b5499f7f 889 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer 0:66f3b5499f7f 890
screamer 0:66f3b5499f7f 891 # When the build and test system were separate, this was relative to a
screamer 0:66f3b5499f7f 892 # base network folder base path: join(NETWORK_BASE_PATH, )
screamer 0:66f3b5499f7f 893 image_path = image
screamer 0:66f3b5499f7f 894
screamer 0:66f3b5499f7f 895 # Host test execution
screamer 0:66f3b5499f7f 896 start_host_exec_time = time()
screamer 0:66f3b5499f7f 897
screamer 0:66f3b5499f7f 898 single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer 0:66f3b5499f7f 899 _copy_method = selected_copy_method
screamer 0:66f3b5499f7f 900
screamer 0:66f3b5499f7f 901 if not exists(image_path):
screamer 0:66f3b5499f7f 902 single_test_result = self.TEST_RESULT_NO_IMAGE
screamer 0:66f3b5499f7f 903 elapsed_time = 0
screamer 0:66f3b5499f7f 904 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
screamer 0:66f3b5499f7f 905 print single_test_output
screamer 0:66f3b5499f7f 906 else:
screamer 0:66f3b5499f7f 907 # Host test execution
screamer 0:66f3b5499f7f 908 start_host_exec_time = time()
screamer 0:66f3b5499f7f 909
screamer 0:66f3b5499f7f 910 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer 0:66f3b5499f7f 911 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer 0:66f3b5499f7f 912 host_test_result = self.run_host_test(test.host_test,
screamer 0:66f3b5499f7f 913 image_path, disk, port, duration,
screamer 0:66f3b5499f7f 914 micro=target_name,
screamer 0:66f3b5499f7f 915 verbose=host_test_verbose,
screamer 0:66f3b5499f7f 916 reset=host_test_reset,
screamer 0:66f3b5499f7f 917 reset_tout=reset_tout,
screamer 0:66f3b5499f7f 918 copy_method=selected_copy_method,
screamer 13:ab47a20b66f0 919 program_cycle_s=target_by_mcu.program_cycle_s)
screamer 0:66f3b5499f7f 920 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer 0:66f3b5499f7f 921
screamer 0:66f3b5499f7f 922 # Store test result
screamer 0:66f3b5499f7f 923 test_all_result.append(single_test_result)
screamer 0:66f3b5499f7f 924 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer 0:66f3b5499f7f 925 elapsed_time = single_testduration # TIme of single test case execution after reset
screamer 0:66f3b5499f7f 926
screamer 0:66f3b5499f7f 927 detailed_test_results[test_index] = {
screamer 0:66f3b5499f7f 928 'result' : single_test_result,
screamer 0:66f3b5499f7f 929 'output' : single_test_output,
screamer 0:66f3b5499f7f 930 'target_name' : target_name,
screamer 0:66f3b5499f7f 931 'target_name_unique' : target_name_unique,
screamer 0:66f3b5499f7f 932 'toolchain_name' : toolchain_name,
screamer 0:66f3b5499f7f 933 'id' : test_id,
screamer 0:66f3b5499f7f 934 'description' : test_description,
screamer 0:66f3b5499f7f 935 'elapsed_time' : round(elapsed_time, 2),
screamer 0:66f3b5499f7f 936 'duration' : single_timeout,
screamer 0:66f3b5499f7f 937 'copy_method' : _copy_method,
screamer 0:66f3b5499f7f 938 }
screamer 0:66f3b5499f7f 939
screamer 0:66f3b5499f7f 940 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
screamer 0:66f3b5499f7f 941 test_id, test_description, elapsed_time, single_timeout)
screamer 0:66f3b5499f7f 942
screamer 0:66f3b5499f7f 943 # Update database entries for ongoing test
screamer 0:66f3b5499f7f 944 if self.db_logger and self.db_logger.is_connected():
screamer 0:66f3b5499f7f 945 test_type = 'SingleTest'
screamer 0:66f3b5499f7f 946 self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer 0:66f3b5499f7f 947 target_name,
screamer 0:66f3b5499f7f 948 toolchain_name,
screamer 0:66f3b5499f7f 949 test_type,
screamer 0:66f3b5499f7f 950 test_id,
screamer 0:66f3b5499f7f 951 single_test_result,
screamer 0:66f3b5499f7f 952 single_test_output,
screamer 0:66f3b5499f7f 953 elapsed_time,
screamer 0:66f3b5499f7f 954 single_timeout,
screamer 0:66f3b5499f7f 955 test_index)
screamer 0:66f3b5499f7f 956
screamer 0:66f3b5499f7f 957 # If we perform waterfall test we test until we get OK and we stop testing
screamer 0:66f3b5499f7f 958 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer 0:66f3b5499f7f 959 break
screamer 0:66f3b5499f7f 960
screamer 0:66f3b5499f7f 961 if self.db_logger:
screamer 0:66f3b5499f7f 962 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 963
screamer 0:66f3b5499f7f 964 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer 0:66f3b5499f7f 965 target_name_unique,
screamer 0:66f3b5499f7f 966 toolchain_name,
screamer 0:66f3b5499f7f 967 test_id,
screamer 0:66f3b5499f7f 968 test_description,
screamer 0:66f3b5499f7f 969 round(elapsed_time, 2),
screamer 0:66f3b5499f7f 970 single_timeout,
screamer 0:66f3b5499f7f 971 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer 0:66f3b5499f7f 972
screamer 0:66f3b5499f7f 973 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 974 """ Function determines MUT's mbed disk/port and copies binary to
screamer 0:66f3b5499f7f 975 target.
screamer 0:66f3b5499f7f 976 """
screamer 0:66f3b5499f7f 977 handle_results = []
screamer 0:66f3b5499f7f 978 data = json.loads(test_spec)
screamer 0:66f3b5499f7f 979
screamer 0:66f3b5499f7f 980 # Find a suitable MUT:
screamer 0:66f3b5499f7f 981 mut = None
screamer 0:66f3b5499f7f 982 for id, m in self.muts.iteritems():
screamer 0:66f3b5499f7f 983 if m['mcu'] == data['mcu']:
screamer 0:66f3b5499f7f 984 mut = m
screamer 0:66f3b5499f7f 985 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer 0:66f3b5499f7f 986 handle_results.append(handle_result)
screamer 0:66f3b5499f7f 987
screamer 0:66f3b5499f7f 988 return handle_results
screamer 0:66f3b5499f7f 989
screamer 0:66f3b5499f7f 990 def print_test_result(self, test_result, target_name, toolchain_name,
screamer 0:66f3b5499f7f 991 test_id, test_description, elapsed_time, duration):
screamer 0:66f3b5499f7f 992 """ Use specific convention to print test result and related data
screamer 0:66f3b5499f7f 993 """
screamer 0:66f3b5499f7f 994 tokens = []
screamer 0:66f3b5499f7f 995 tokens.append("TargetTest")
screamer 0:66f3b5499f7f 996 tokens.append(target_name)
screamer 0:66f3b5499f7f 997 tokens.append(toolchain_name)
screamer 0:66f3b5499f7f 998 tokens.append(test_id)
screamer 0:66f3b5499f7f 999 tokens.append(test_description)
screamer 0:66f3b5499f7f 1000 separator = "::"
screamer 0:66f3b5499f7f 1001 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer 0:66f3b5499f7f 1002 result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer 0:66f3b5499f7f 1003 return Fore.MAGENTA + result + Fore.RESET
screamer 0:66f3b5499f7f 1004
screamer 0:66f3b5499f7f 1005 def shape_test_loop_ok_result_count(self, test_all_result):
screamer 0:66f3b5499f7f 1006 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1007 """
screamer 0:66f3b5499f7f 1008 test_loop_count = len(test_all_result)
screamer 0:66f3b5499f7f 1009 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer 0:66f3b5499f7f 1010 return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer 0:66f3b5499f7f 1011
screamer 0:66f3b5499f7f 1012 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer 0:66f3b5499f7f 1013 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1014 """
screamer 0:66f3b5499f7f 1015 result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 1016
screamer 0:66f3b5499f7f 1017 if all(test_all_result[0] == res for res in test_all_result):
screamer 0:66f3b5499f7f 1018 result = test_all_result[0]
screamer 0:66f3b5499f7f 1019 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer 0:66f3b5499f7f 1020 result = self.TEST_RESULT_OK
screamer 0:66f3b5499f7f 1021
screamer 0:66f3b5499f7f 1022 return result
screamer 0:66f3b5499f7f 1023
screamer 0:66f3b5499f7f 1024 def run_host_test(self, name, image_path, disk, port, duration,
screamer 0:66f3b5499f7f 1025 micro=None, reset=None, reset_tout=None,
screamer 0:66f3b5499f7f 1026 verbose=False, copy_method=None, program_cycle_s=None):
screamer 0:66f3b5499f7f 1027 """ Function creates new process with host test configured with particular test case.
screamer 0:66f3b5499f7f 1028 Function also is pooling for serial port activity from process to catch all data
screamer 0:66f3b5499f7f 1029 printed by test runner and host test during test execution
screamer 0:66f3b5499f7f 1030 """
screamer 0:66f3b5499f7f 1031
screamer 0:66f3b5499f7f 1032 def get_char_from_queue(obs):
screamer 0:66f3b5499f7f 1033 """ Get character from queue safe way
screamer 0:66f3b5499f7f 1034 """
screamer 0:66f3b5499f7f 1035 try:
screamer 0:66f3b5499f7f 1036 c = obs.queue.get(block=True, timeout=0.5)
screamer 0:66f3b5499f7f 1037 except Empty, _:
screamer 0:66f3b5499f7f 1038 c = None
screamer 0:66f3b5499f7f 1039 return c
screamer 0:66f3b5499f7f 1040
screamer 0:66f3b5499f7f 1041 def filter_queue_char(c):
screamer 0:66f3b5499f7f 1042 """ Filters out non ASCII characters from serial port
screamer 0:66f3b5499f7f 1043 """
screamer 0:66f3b5499f7f 1044 if ord(c) not in range(128):
screamer 0:66f3b5499f7f 1045 c = ' '
screamer 0:66f3b5499f7f 1046 return c
screamer 0:66f3b5499f7f 1047
screamer 0:66f3b5499f7f 1048 def get_test_result(output):
screamer 0:66f3b5499f7f 1049 """ Parse test 'output' data
screamer 0:66f3b5499f7f 1050 """
screamer 0:66f3b5499f7f 1051 result = self.TEST_RESULT_TIMEOUT
screamer 0:66f3b5499f7f 1052 for line in "".join(output).splitlines():
screamer 0:66f3b5499f7f 1053 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer 0:66f3b5499f7f 1054 if search_result and len(search_result.groups()):
screamer 0:66f3b5499f7f 1055 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer 0:66f3b5499f7f 1056 break
screamer 0:66f3b5499f7f 1057 return result
screamer 0:66f3b5499f7f 1058
screamer 0:66f3b5499f7f 1059 def get_auto_property_value(property_name, line):
screamer 0:66f3b5499f7f 1060 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer 0:66f3b5499f7f 1061 Returns string
screamer 0:66f3b5499f7f 1062 """
screamer 0:66f3b5499f7f 1063 result = None
screamer 0:66f3b5499f7f 1064 if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer 0:66f3b5499f7f 1065 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer 0:66f3b5499f7f 1066 if property is not None and len(property.groups()) == 1:
screamer 0:66f3b5499f7f 1067 result = property.groups()[0]
screamer 0:66f3b5499f7f 1068 return result
screamer 0:66f3b5499f7f 1069
screamer 0:66f3b5499f7f 1070 # print "{%s} port:%s disk:%s" % (name, port, disk),
screamer 0:66f3b5499f7f 1071 cmd = ["python",
screamer 0:66f3b5499f7f 1072 '%s.py'% name,
screamer 0:66f3b5499f7f 1073 '-d', disk,
screamer 0:66f3b5499f7f 1074 '-f', '"%s"'% image_path,
screamer 0:66f3b5499f7f 1075 '-p', port,
screamer 0:66f3b5499f7f 1076 '-t', str(duration),
screamer 0:66f3b5499f7f 1077 '-C', str(program_cycle_s)]
screamer 0:66f3b5499f7f 1078
screamer 0:66f3b5499f7f 1079 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 1080 cmd += ['--auto']
screamer 0:66f3b5499f7f 1081
screamer 0:66f3b5499f7f 1082 # Add extra parameters to host_test
screamer 0:66f3b5499f7f 1083 if copy_method is not None:
screamer 0:66f3b5499f7f 1084 cmd += ["-c", copy_method]
screamer 0:66f3b5499f7f 1085 if micro is not None:
screamer 0:66f3b5499f7f 1086 cmd += ["-m", micro]
screamer 0:66f3b5499f7f 1087 if reset is not None:
screamer 0:66f3b5499f7f 1088 cmd += ["-r", reset]
screamer 0:66f3b5499f7f 1089 if reset_tout is not None:
screamer 0:66f3b5499f7f 1090 cmd += ["-R", str(reset_tout)]
screamer 0:66f3b5499f7f 1091
screamer 0:66f3b5499f7f 1092 if verbose:
screamer 0:66f3b5499f7f 1093 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
screamer 0:66f3b5499f7f 1094 print "Test::Output::Start"
screamer 0:66f3b5499f7f 1095
screamer 0:66f3b5499f7f 1096 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer 0:66f3b5499f7f 1097 obs = ProcessObserver(proc)
screamer 0:66f3b5499f7f 1098 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer 0:66f3b5499f7f 1099 line = ''
screamer 0:66f3b5499f7f 1100 output = []
screamer 0:66f3b5499f7f 1101 start_time = time()
screamer 0:66f3b5499f7f 1102 while (time() - start_time) < (2 * duration):
screamer 0:66f3b5499f7f 1103 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1104 if c:
screamer 0:66f3b5499f7f 1105 if verbose:
screamer 0:66f3b5499f7f 1106 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1107 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1108 output.append(c)
screamer 0:66f3b5499f7f 1109 # Give the mbed under test a way to communicate the end of the test
screamer 0:66f3b5499f7f 1110 if c in ['\n', '\r']:
screamer 0:66f3b5499f7f 1111
screamer 0:66f3b5499f7f 1112 # Checking for auto-detection information from the test about MUT reset moment
screamer 0:66f3b5499f7f 1113 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer 0:66f3b5499f7f 1114 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1115 update_once_flag['reset_target'] = True
screamer 0:66f3b5499f7f 1116 start_time = time()
screamer 0:66f3b5499f7f 1117
screamer 0:66f3b5499f7f 1118 # Checking for auto-detection information from the test about timeout
screamer 0:66f3b5499f7f 1119 auto_timeout_val = get_auto_property_value('timeout', line)
screamer 0:66f3b5499f7f 1120 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer 0:66f3b5499f7f 1121 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1122 update_once_flag['timeout'] = True
screamer 0:66f3b5499f7f 1123 duration = int(auto_timeout_val)
screamer 0:66f3b5499f7f 1124
screamer 0:66f3b5499f7f 1125 # Detect mbed assert:
screamer 0:66f3b5499f7f 1126 if 'mbed assertation failed: ' in line:
screamer 0:66f3b5499f7f 1127 output.append('{{mbed_assert}}')
screamer 0:66f3b5499f7f 1128 break
screamer 0:66f3b5499f7f 1129
screamer 0:66f3b5499f7f 1130 # Check for test end
screamer 0:66f3b5499f7f 1131 if '{end}' in line:
screamer 0:66f3b5499f7f 1132 break
screamer 0:66f3b5499f7f 1133 line = ''
screamer 0:66f3b5499f7f 1134 else:
screamer 0:66f3b5499f7f 1135 line += c
screamer 0:66f3b5499f7f 1136 end_time = time()
screamer 0:66f3b5499f7f 1137 testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer 0:66f3b5499f7f 1138
screamer 0:66f3b5499f7f 1139 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1140
screamer 0:66f3b5499f7f 1141 if c:
screamer 0:66f3b5499f7f 1142 if verbose:
screamer 0:66f3b5499f7f 1143 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1144 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1145 output.append(c)
screamer 0:66f3b5499f7f 1146
screamer 0:66f3b5499f7f 1147 if verbose:
screamer 0:66f3b5499f7f 1148 print "Test::Output::Finish"
screamer 0:66f3b5499f7f 1149 # Stop test process
screamer 0:66f3b5499f7f 1150 obs.stop()
screamer 0:66f3b5499f7f 1151
screamer 0:66f3b5499f7f 1152 result = get_test_result(output)
screamer 0:66f3b5499f7f 1153 return (result, "".join(output), testcase_duration, duration)
screamer 0:66f3b5499f7f 1154
screamer 0:66f3b5499f7f 1155 def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer 0:66f3b5499f7f 1156 """ Checks if specified target should run specific peripheral test case defined in MUTs file
screamer 0:66f3b5499f7f 1157 """
screamer 0:66f3b5499f7f 1158 if peripherals is not None:
screamer 0:66f3b5499f7f 1159 peripherals = set(peripherals)
screamer 0:66f3b5499f7f 1160 for id, mut in self.muts.iteritems():
screamer 0:66f3b5499f7f 1161 # Target MCU name check
screamer 0:66f3b5499f7f 1162 if mut["mcu"] != target_mcu_name:
screamer 0:66f3b5499f7f 1163 continue
screamer 0:66f3b5499f7f 1164 # Peripherals check
screamer 0:66f3b5499f7f 1165 if peripherals is not None:
screamer 0:66f3b5499f7f 1166 if 'peripherals' not in mut:
screamer 0:66f3b5499f7f 1167 continue
screamer 0:66f3b5499f7f 1168 if not peripherals.issubset(set(mut['peripherals'])):
screamer 0:66f3b5499f7f 1169 continue
screamer 0:66f3b5499f7f 1170 return True
screamer 0:66f3b5499f7f 1171 return False
screamer 0:66f3b5499f7f 1172
screamer 0:66f3b5499f7f 1173 def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer 0:66f3b5499f7f 1174 """ Function prepares JSON structure describing test specification
screamer 0:66f3b5499f7f 1175 """
screamer 0:66f3b5499f7f 1176 test_spec = {
screamer 0:66f3b5499f7f 1177 "mcu": mcu,
screamer 0:66f3b5499f7f 1178 "image": image_path,
screamer 0:66f3b5499f7f 1179 "duration": duration,
screamer 0:66f3b5499f7f 1180 "test_id": test_id,
screamer 0:66f3b5499f7f 1181 }
screamer 0:66f3b5499f7f 1182 return json.dumps(test_spec)
screamer 0:66f3b5499f7f 1183
screamer 0:66f3b5499f7f 1184
screamer 0:66f3b5499f7f 1185 def get_unique_value_from_summary(test_summary, index):
screamer 0:66f3b5499f7f 1186 """ Gets list of unique target names
screamer 0:66f3b5499f7f 1187 """
screamer 0:66f3b5499f7f 1188 result = []
screamer 0:66f3b5499f7f 1189 for test in test_summary:
screamer 0:66f3b5499f7f 1190 target_name = test[index]
screamer 0:66f3b5499f7f 1191 if target_name not in result:
screamer 0:66f3b5499f7f 1192 result.append(target_name)
screamer 0:66f3b5499f7f 1193 return sorted(result)
screamer 0:66f3b5499f7f 1194
screamer 0:66f3b5499f7f 1195
screamer 0:66f3b5499f7f 1196 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer 0:66f3b5499f7f 1197 """ Gets list of unique target names and return dictionary
screamer 0:66f3b5499f7f 1198 """
screamer 0:66f3b5499f7f 1199 result = {}
screamer 0:66f3b5499f7f 1200 for test in test_summary:
screamer 0:66f3b5499f7f 1201 key = test[index_key]
screamer 0:66f3b5499f7f 1202 val = test[index_val]
screamer 0:66f3b5499f7f 1203 if key not in result:
screamer 0:66f3b5499f7f 1204 result[key] = val
screamer 0:66f3b5499f7f 1205 return result
screamer 0:66f3b5499f7f 1206
screamer 0:66f3b5499f7f 1207
screamer 0:66f3b5499f7f 1208 def show_json_file_format_error(json_spec_filename, line, column):
screamer 0:66f3b5499f7f 1209 """ Prints JSON broken content
screamer 0:66f3b5499f7f 1210 """
screamer 0:66f3b5499f7f 1211 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1212 line_no = 1
screamer 0:66f3b5499f7f 1213 for json_line in data_file:
screamer 0:66f3b5499f7f 1214 if line_no + 5 >= line: # Print last few lines before error
screamer 0:66f3b5499f7f 1215 print 'Line %d:\t'%line_no + json_line, # Prints line
screamer 0:66f3b5499f7f 1216 if line_no == line:
screamer 0:66f3b5499f7f 1217 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
screamer 0:66f3b5499f7f 1218 break
screamer 0:66f3b5499f7f 1219 line_no += 1
screamer 0:66f3b5499f7f 1220
screamer 0:66f3b5499f7f 1221
screamer 0:66f3b5499f7f 1222 def json_format_error_defect_pos(json_error_msg):
screamer 0:66f3b5499f7f 1223 """ Gets first error line and column in JSON file format.
screamer 0:66f3b5499f7f 1224 Parsed from exception thrown by json.loads() string
screamer 0:66f3b5499f7f 1225 """
screamer 0:66f3b5499f7f 1226 result = None
screamer 0:66f3b5499f7f 1227 line, column = 0, 0
screamer 0:66f3b5499f7f 1228 # Line value search
screamer 0:66f3b5499f7f 1229 line_search = re.search('line [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1230 if line_search is not None:
screamer 0:66f3b5499f7f 1231 ls = line_search.group().split(' ')
screamer 0:66f3b5499f7f 1232 if len(ls) == 2:
screamer 0:66f3b5499f7f 1233 line = int(ls[1])
screamer 0:66f3b5499f7f 1234 # Column position search
screamer 0:66f3b5499f7f 1235 column_search = re.search('column [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1236 if column_search is not None:
screamer 0:66f3b5499f7f 1237 cs = column_search.group().split(' ')
screamer 0:66f3b5499f7f 1238 if len(cs) == 2:
screamer 0:66f3b5499f7f 1239 column = int(cs[1])
screamer 0:66f3b5499f7f 1240 result = [line, column]
screamer 0:66f3b5499f7f 1241 return result
screamer 0:66f3b5499f7f 1242
screamer 0:66f3b5499f7f 1243
screamer 0:66f3b5499f7f 1244 def get_json_data_from_file(json_spec_filename, verbose=False):
screamer 0:66f3b5499f7f 1245 """ Loads from file JSON formatted string to data structure
screamer 0:66f3b5499f7f 1246 """
screamer 0:66f3b5499f7f 1247 result = None
screamer 0:66f3b5499f7f 1248 try:
screamer 0:66f3b5499f7f 1249 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1250 try:
screamer 0:66f3b5499f7f 1251 result = json.load(data_file)
screamer 0:66f3b5499f7f 1252 except ValueError as json_error_msg:
screamer 0:66f3b5499f7f 1253 result = None
screamer 0:66f3b5499f7f 1254 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
screamer 0:66f3b5499f7f 1255 # We can print where error occurred inside JSON file if we can parse exception msg
screamer 0:66f3b5499f7f 1256 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer 0:66f3b5499f7f 1257 if json_format_defect_pos is not None:
screamer 0:66f3b5499f7f 1258 line = json_format_defect_pos[0]
screamer 0:66f3b5499f7f 1259 column = json_format_defect_pos[1]
screamer 0:66f3b5499f7f 1260 print
screamer 0:66f3b5499f7f 1261 show_json_file_format_error(json_spec_filename, line, column)
screamer 0:66f3b5499f7f 1262
screamer 0:66f3b5499f7f 1263 except IOError as fileopen_error_msg:
screamer 0:66f3b5499f7f 1264 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
screamer 0:66f3b5499f7f 1265 print
screamer 0:66f3b5499f7f 1266 if verbose and result:
screamer 0:66f3b5499f7f 1267 pp = pprint.PrettyPrinter(indent=4)
screamer 0:66f3b5499f7f 1268 pp.pprint(result)
screamer 0:66f3b5499f7f 1269 return result
screamer 0:66f3b5499f7f 1270
screamer 0:66f3b5499f7f 1271
screamer 0:66f3b5499f7f 1272 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer 0:66f3b5499f7f 1273 """ Prints MUTs configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1274 """
screamer 0:66f3b5499f7f 1275 muts_info_cols = []
screamer 0:66f3b5499f7f 1276 # We need to check all unique properties for each defined MUT
screamer 0:66f3b5499f7f 1277 for k in json_data:
screamer 0:66f3b5499f7f 1278 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1279 for mut_property in mut_info:
screamer 0:66f3b5499f7f 1280 if mut_property not in muts_info_cols:
screamer 0:66f3b5499f7f 1281 muts_info_cols.append(mut_property)
screamer 0:66f3b5499f7f 1282
screamer 0:66f3b5499f7f 1283 # Prepare pretty table object to display all MUTs
screamer 0:66f3b5499f7f 1284 pt_cols = ["index"] + muts_info_cols
screamer 0:66f3b5499f7f 1285 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1286 for col in pt_cols:
screamer 0:66f3b5499f7f 1287 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1288
screamer 0:66f3b5499f7f 1289 # Add rows to pretty print object
screamer 0:66f3b5499f7f 1290 for k in json_data:
screamer 0:66f3b5499f7f 1291 row = [k]
screamer 0:66f3b5499f7f 1292 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1293
screamer 0:66f3b5499f7f 1294 add_row = True
screamer 0:66f3b5499f7f 1295 if platform_filter and 'mcu' in mut_info:
screamer 0:66f3b5499f7f 1296 add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer 0:66f3b5499f7f 1297 if add_row:
screamer 0:66f3b5499f7f 1298 for col in muts_info_cols:
screamer 0:66f3b5499f7f 1299 cell_val = mut_info[col] if col in mut_info else None
screamer 0:66f3b5499f7f 1300 if type(cell_val) == ListType:
screamer 0:66f3b5499f7f 1301 cell_val = join_delim.join(cell_val)
screamer 0:66f3b5499f7f 1302 row.append(cell_val)
screamer 0:66f3b5499f7f 1303 pt.add_row(row)
screamer 0:66f3b5499f7f 1304 return pt.get_string()
screamer 0:66f3b5499f7f 1305
screamer 0:66f3b5499f7f 1306
screamer 0:66f3b5499f7f 1307 def print_test_configuration_from_json(json_data, join_delim=", "):
screamer 0:66f3b5499f7f 1308 """ Prints test specification configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1309 """
screamer 0:66f3b5499f7f 1310 toolchains_info_cols = []
screamer 0:66f3b5499f7f 1311 # We need to check all toolchains for each device
screamer 0:66f3b5499f7f 1312 for k in json_data:
screamer 0:66f3b5499f7f 1313 # k should be 'targets'
screamer 0:66f3b5499f7f 1314 targets = json_data[k]
screamer 0:66f3b5499f7f 1315 for target in targets:
screamer 0:66f3b5499f7f 1316 toolchains = targets[target]
screamer 0:66f3b5499f7f 1317 for toolchain in toolchains:
screamer 0:66f3b5499f7f 1318 if toolchain not in toolchains_info_cols:
screamer 0:66f3b5499f7f 1319 toolchains_info_cols.append(toolchain)
screamer 0:66f3b5499f7f 1320
screamer 0:66f3b5499f7f 1321 # Prepare pretty table object to display test specification
screamer 0:66f3b5499f7f 1322 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
screamer 0:66f3b5499f7f 1323 pt = PrettyTable(pt_cols)
screamer 0:66f3b5499f7f 1324 for col in pt_cols:
screamer 0:66f3b5499f7f 1325 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1326
screamer 0:66f3b5499f7f 1327 # { target : [conflicted toolchains] }
screamer 0:66f3b5499f7f 1328 toolchain_conflicts = {}
screamer 0:66f3b5499f7f 1329 toolchain_path_conflicts = []
screamer 0:66f3b5499f7f 1330 for k in json_data:
screamer 0:66f3b5499f7f 1331 # k should be 'targets'
screamer 0:66f3b5499f7f 1332 targets = json_data[k]
screamer 0:66f3b5499f7f 1333 for target in targets:
screamer 0:66f3b5499f7f 1334 target_supported_toolchains = get_target_supported_toolchains(target)
screamer 0:66f3b5499f7f 1335 if not target_supported_toolchains:
screamer 0:66f3b5499f7f 1336 target_supported_toolchains = []
screamer 0:66f3b5499f7f 1337 target_name = target if target in TARGET_MAP else "%s*"% target
screamer 0:66f3b5499f7f 1338 row = [target_name]
screamer 0:66f3b5499f7f 1339 toolchains = targets[target]
screamer 0:66f3b5499f7f 1340
screamer 0:66f3b5499f7f 1341 for toolchain in sorted(toolchains_info_cols):
screamer 0:66f3b5499f7f 1342 # Check for conflicts: target vs toolchain
screamer 0:66f3b5499f7f 1343 conflict = False
screamer 0:66f3b5499f7f 1344 conflict_path = False
screamer 0:66f3b5499f7f 1345 if toolchain in toolchains:
screamer 0:66f3b5499f7f 1346 if toolchain not in target_supported_toolchains:
screamer 0:66f3b5499f7f 1347 conflict = True
screamer 0:66f3b5499f7f 1348 if target not in toolchain_conflicts:
screamer 0:66f3b5499f7f 1349 toolchain_conflicts[target] = []
screamer 0:66f3b5499f7f 1350 toolchain_conflicts[target].append(toolchain)
screamer 0:66f3b5499f7f 1351 # Add marker inside table about target usage / conflict
screamer 0:66f3b5499f7f 1352 cell_val = 'Yes' if toolchain in toolchains else '-'
screamer 0:66f3b5499f7f 1353 if conflict:
screamer 0:66f3b5499f7f 1354 cell_val += '*'
screamer 0:66f3b5499f7f 1355 # Check for conflicts: toolchain vs toolchain path
screamer 20:835f6355470d 1356 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1357 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1358 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1359 conflict_path = True
screamer 0:66f3b5499f7f 1360 if toolchain not in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1361 toolchain_path_conflicts.append(toolchain)
screamer 0:66f3b5499f7f 1362 if conflict_path:
screamer 0:66f3b5499f7f 1363 cell_val += '#'
screamer 0:66f3b5499f7f 1364 row.append(cell_val)
screamer 0:66f3b5499f7f 1365 pt.add_row(row)
screamer 0:66f3b5499f7f 1366
screamer 0:66f3b5499f7f 1367 # generate result string
screamer 0:66f3b5499f7f 1368 result = pt.get_string() # Test specification table
screamer 0:66f3b5499f7f 1369 if toolchain_conflicts or toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1370 result += "\n"
screamer 0:66f3b5499f7f 1371 result += "Toolchain conflicts:\n"
screamer 0:66f3b5499f7f 1372 for target in toolchain_conflicts:
screamer 0:66f3b5499f7f 1373 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 1374 result += "\t* Target %s unknown\n"% (target)
screamer 0:66f3b5499f7f 1375 conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer 0:66f3b5499f7f 1376 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer 0:66f3b5499f7f 1377 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer 0:66f3b5499f7f 1378
screamer 0:66f3b5499f7f 1379 for toolchain in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1380 # Let's check toolchain configuration
screamer 20:835f6355470d 1381 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1382 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1383 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1384 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer 0:66f3b5499f7f 1385 return result
screamer 0:66f3b5499f7f 1386
screamer 0:66f3b5499f7f 1387
screamer 0:66f3b5499f7f 1388 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer 0:66f3b5499f7f 1389 """ Generates table summary with all test cases and additional test cases
screamer 0:66f3b5499f7f 1390 information using pretty print functionality. Allows test suite user to
screamer 0:66f3b5499f7f 1391 see test cases
screamer 0:66f3b5499f7f 1392 """
screamer 0:66f3b5499f7f 1393 # get all unique test ID prefixes
screamer 0:66f3b5499f7f 1394 unique_test_id = []
screamer 0:66f3b5499f7f 1395 for test in TESTS:
screamer 0:66f3b5499f7f 1396 split = test['id'].split('_')[:-1]
screamer 0:66f3b5499f7f 1397 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1398 if test_id_prefix not in unique_test_id:
screamer 0:66f3b5499f7f 1399 unique_test_id.append(test_id_prefix)
screamer 0:66f3b5499f7f 1400 unique_test_id.sort()
screamer 0:66f3b5499f7f 1401 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1402 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1403
screamer 0:66f3b5499f7f 1404 test_properties = ['id',
screamer 0:66f3b5499f7f 1405 'automated',
screamer 0:66f3b5499f7f 1406 'description',
screamer 0:66f3b5499f7f 1407 'peripherals',
screamer 0:66f3b5499f7f 1408 'host_test',
screamer 0:66f3b5499f7f 1409 'duration'] if cols is None else cols
screamer 0:66f3b5499f7f 1410
screamer 0:66f3b5499f7f 1411 # All tests status table print
screamer 0:66f3b5499f7f 1412 pt = PrettyTable(test_properties)
screamer 0:66f3b5499f7f 1413 for col in test_properties:
screamer 0:66f3b5499f7f 1414 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1415 pt.align['duration'] = "r"
screamer 0:66f3b5499f7f 1416
screamer 0:66f3b5499f7f 1417 counter_all = 0
screamer 0:66f3b5499f7f 1418 counter_automated = 0
screamer 0:66f3b5499f7f 1419 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 1420
screamer 0:66f3b5499f7f 1421 for test_id in sorted(TEST_MAP.keys()):
screamer 0:66f3b5499f7f 1422 if platform_filter is not None:
screamer 0:66f3b5499f7f 1423 # FIlter out platforms using regex
screamer 0:66f3b5499f7f 1424 if re.search(platform_filter, test_id) is None:
screamer 0:66f3b5499f7f 1425 continue
screamer 0:66f3b5499f7f 1426 row = []
screamer 0:66f3b5499f7f 1427 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 1428 split = test_id.split('_')[:-1]
screamer 0:66f3b5499f7f 1429 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1430
screamer 0:66f3b5499f7f 1431 for col in test_properties:
screamer 0:66f3b5499f7f 1432 col_value = test[col]
screamer 0:66f3b5499f7f 1433 if type(test[col]) == ListType:
screamer 0:66f3b5499f7f 1434 col_value = join_delim.join(test[col])
screamer 0:66f3b5499f7f 1435 elif test[col] == None:
screamer 0:66f3b5499f7f 1436 col_value = "-"
screamer 0:66f3b5499f7f 1437
screamer 0:66f3b5499f7f 1438 row.append(col_value)
screamer 0:66f3b5499f7f 1439 if test['automated'] == True:
screamer 0:66f3b5499f7f 1440 counter_dict_test_id_types[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1441 counter_automated += 1
screamer 0:66f3b5499f7f 1442 pt.add_row(row)
screamer 0:66f3b5499f7f 1443 # Update counters
screamer 0:66f3b5499f7f 1444 counter_all += 1
screamer 0:66f3b5499f7f 1445 counter_dict_test_id_types_all[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1446 result = pt.get_string()
screamer 0:66f3b5499f7f 1447 result += "\n\n"
screamer 0:66f3b5499f7f 1448
screamer 0:66f3b5499f7f 1449 if result_summary and not platform_filter:
screamer 0:66f3b5499f7f 1450 # Automation result summary
screamer 0:66f3b5499f7f 1451 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1452 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1453 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1454 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1455 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1456
screamer 0:66f3b5499f7f 1457 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer 0:66f3b5499f7f 1458 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1459 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer 0:66f3b5499f7f 1460 result += "Automation coverage:\n"
screamer 0:66f3b5499f7f 1461 result += pt.get_string()
screamer 0:66f3b5499f7f 1462 result += "\n\n"
screamer 0:66f3b5499f7f 1463
screamer 0:66f3b5499f7f 1464 # Test automation coverage table print
screamer 0:66f3b5499f7f 1465 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
screamer 0:66f3b5499f7f 1466 pt = PrettyTable(test_id_cols)
screamer 0:66f3b5499f7f 1467 pt.align['id'] = "l"
screamer 0:66f3b5499f7f 1468 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1469 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1470 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1471 for unique_id in unique_test_id:
screamer 0:66f3b5499f7f 1472 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer 0:66f3b5499f7f 1473 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer 0:66f3b5499f7f 1474 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1475 row = [unique_id,
screamer 0:66f3b5499f7f 1476 counter_dict_test_id_types[unique_id],
screamer 0:66f3b5499f7f 1477 counter_dict_test_id_types_all[unique_id],
screamer 0:66f3b5499f7f 1478 percent_progress,
screamer 0:66f3b5499f7f 1479 "[" + str_progress + "]"]
screamer 0:66f3b5499f7f 1480 pt.add_row(row)
screamer 0:66f3b5499f7f 1481 result += "Test automation coverage:\n"
screamer 0:66f3b5499f7f 1482 result += pt.get_string()
screamer 0:66f3b5499f7f 1483 result += "\n\n"
screamer 0:66f3b5499f7f 1484 return result
screamer 0:66f3b5499f7f 1485
screamer 0:66f3b5499f7f 1486
screamer 0:66f3b5499f7f 1487 def progress_bar(percent_progress, saturation=0):
screamer 0:66f3b5499f7f 1488 """ This function creates progress bar with optional simple saturation mark
screamer 0:66f3b5499f7f 1489 """
screamer 0:66f3b5499f7f 1490 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer 0:66f3b5499f7f 1491 str_progress = '#' * step + '.' * int(50 - step)
screamer 0:66f3b5499f7f 1492 c = '!' if str_progress[38] == '.' else '|'
screamer 0:66f3b5499f7f 1493 if saturation > 0:
screamer 0:66f3b5499f7f 1494 saturation = saturation / 2
screamer 0:66f3b5499f7f 1495 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer 0:66f3b5499f7f 1496 return str_progress
screamer 0:66f3b5499f7f 1497
screamer 0:66f3b5499f7f 1498
screamer 0:66f3b5499f7f 1499 def singletest_in_cli_mode(single_test):
screamer 0:66f3b5499f7f 1500 """ Runs SingleTestRunner object in CLI (Command line interface) mode
screamer 0:66f3b5499f7f 1501
screamer 0:66f3b5499f7f 1502 @return returns success code (0 == success) for building and running tests
screamer 0:66f3b5499f7f 1503 """
screamer 0:66f3b5499f7f 1504 start = time()
screamer 0:66f3b5499f7f 1505 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 1506 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer 0:66f3b5499f7f 1507 elapsed_time = time() - start
screamer 0:66f3b5499f7f 1508
screamer 0:66f3b5499f7f 1509 # Human readable summary
screamer 0:66f3b5499f7f 1510 if not single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 1511 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1512 print single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1513 if single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 1514 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1515 # table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 1516 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:66f3b5499f7f 1517
screamer 0:66f3b5499f7f 1518 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:66f3b5499f7f 1519 print
screamer 0:66f3b5499f7f 1520 # Write summary of the builds
screamer 0:66f3b5499f7f 1521
screamer 0:66f3b5499f7f 1522 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 0:66f3b5499f7f 1523 status = print_report_exporter.report(build_report)
screamer 0:66f3b5499f7f 1524
screamer 0:66f3b5499f7f 1525 # Store extra reports in files
screamer 0:66f3b5499f7f 1526 if single_test.opts_report_html_file_name:
screamer 0:66f3b5499f7f 1527 # Export results in form of HTML report to separate file
screamer 0:66f3b5499f7f 1528 report_exporter = ReportExporter(ResultExporterType.HTML)
screamer 0:66f3b5499f7f 1529 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1530 if single_test.opts_report_junit_file_name:
screamer 0:66f3b5499f7f 1531 # Export results in form of JUnit XML report to separate file
screamer 0:66f3b5499f7f 1532 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 0:66f3b5499f7f 1533 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer 22:9e85236d8716 1534 if single_test.opts_report_text_file_name:
screamer 22:9e85236d8716 1535 # Export results in form of a text file
screamer 22:9e85236d8716 1536 report_exporter = ReportExporter(ResultExporterType.TEXT)
screamer 22:9e85236d8716 1537 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1538 if single_test.opts_report_build_file_name:
screamer 0:66f3b5499f7f 1539 # Export build results as html report to sparate file
screamer 0:66f3b5499f7f 1540 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 0:66f3b5499f7f 1541 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer 0:66f3b5499f7f 1542
screamer 0:66f3b5499f7f 1543 # Returns True if no build failures of the test projects or their dependencies
screamer 0:66f3b5499f7f 1544 return status
screamer 0:66f3b5499f7f 1545
screamer 0:66f3b5499f7f 1546 class TestLogger():
screamer 0:66f3b5499f7f 1547 """ Super-class for logging and printing ongoing events for test suite pass
screamer 0:66f3b5499f7f 1548 """
screamer 0:66f3b5499f7f 1549 def __init__(self, store_log=True):
screamer 0:66f3b5499f7f 1550 """ We can control if logger actually stores log in memory
screamer 0:66f3b5499f7f 1551 or just handled all log entries immediately
screamer 0:66f3b5499f7f 1552 """
screamer 0:66f3b5499f7f 1553 self.log = []
screamer 0:66f3b5499f7f 1554 self.log_to_file = False
screamer 0:66f3b5499f7f 1555 self.log_file_name = None
screamer 0:66f3b5499f7f 1556 self.store_log = store_log
screamer 0:66f3b5499f7f 1557
screamer 0:66f3b5499f7f 1558 self.LogType = construct_enum(INFO='Info',
screamer 0:66f3b5499f7f 1559 WARN='Warning',
screamer 0:66f3b5499f7f 1560 NOTIF='Notification',
screamer 0:66f3b5499f7f 1561 ERROR='Error',
screamer 0:66f3b5499f7f 1562 EXCEPT='Exception')
screamer 0:66f3b5499f7f 1563
screamer 0:66f3b5499f7f 1564 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer 0:66f3b5499f7f 1565 APPEND=2) # Append to existing log file
screamer 0:66f3b5499f7f 1566
screamer 0:66f3b5499f7f 1567 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1568 """ Log one line of text
screamer 0:66f3b5499f7f 1569 """
screamer 0:66f3b5499f7f 1570 log_timestamp = time()
screamer 0:66f3b5499f7f 1571 log_entry = {'log_type' : LogType,
screamer 0:66f3b5499f7f 1572 'log_timestamp' : log_timestamp,
screamer 0:66f3b5499f7f 1573 'log_line' : log_line,
screamer 0:66f3b5499f7f 1574 '_future' : None
screamer 0:66f3b5499f7f 1575 }
screamer 0:66f3b5499f7f 1576 # Store log in memory
screamer 0:66f3b5499f7f 1577 if self.store_log:
screamer 0:66f3b5499f7f 1578 self.log.append(log_entry)
screamer 0:66f3b5499f7f 1579 return log_entry
screamer 0:66f3b5499f7f 1580
screamer 0:66f3b5499f7f 1581
screamer 0:66f3b5499f7f 1582 class CLITestLogger(TestLogger):
screamer 0:66f3b5499f7f 1583 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer 0:66f3b5499f7f 1584 """
screamer 0:66f3b5499f7f 1585 def __init__(self, store_log=True, file_name=None):
screamer 0:66f3b5499f7f 1586 TestLogger.__init__(self)
screamer 0:66f3b5499f7f 1587 self.log_file_name = file_name
screamer 0:66f3b5499f7f 1588 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer 0:66f3b5499f7f 1589 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer 0:66f3b5499f7f 1590
screamer 0:66f3b5499f7f 1591 def log_print(self, log_entry, timestamp=True):
screamer 0:66f3b5499f7f 1592 """ Prints on screen formatted log entry
screamer 0:66f3b5499f7f 1593 """
screamer 0:66f3b5499f7f 1594 ts = log_entry['log_timestamp']
screamer 0:66f3b5499f7f 1595 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer 0:66f3b5499f7f 1596 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer 0:66f3b5499f7f 1597 return timestamp_str + log_line_str
screamer 0:66f3b5499f7f 1598
screamer 0:66f3b5499f7f 1599 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1600 """ Logs line, if log file output was specified log line will be appended
screamer 0:66f3b5499f7f 1601 at the end of log file
screamer 0:66f3b5499f7f 1602 """
screamer 0:66f3b5499f7f 1603 log_entry = TestLogger.log_line(self, LogType, log_line)
screamer 0:66f3b5499f7f 1604 log_line_str = self.log_print(log_entry, timestamp)
screamer 0:66f3b5499f7f 1605 if self.log_file_name is not None:
screamer 0:66f3b5499f7f 1606 try:
screamer 0:66f3b5499f7f 1607 with open(self.log_file_name, 'a') as f:
screamer 0:66f3b5499f7f 1608 f.write(log_line_str + line_delim)
screamer 0:66f3b5499f7f 1609 except IOError:
screamer 0:66f3b5499f7f 1610 pass
screamer 0:66f3b5499f7f 1611 return log_line_str
screamer 0:66f3b5499f7f 1612
screamer 0:66f3b5499f7f 1613
screamer 0:66f3b5499f7f 1614 def factory_db_logger(db_url):
screamer 0:66f3b5499f7f 1615 """ Factory database driver depending on database type supplied in database connection string db_url
screamer 0:66f3b5499f7f 1616 """
screamer 0:66f3b5499f7f 1617 if db_url is not None:
screamer 0:66f3b5499f7f 1618 from tools.test_mysql import MySQLDBAccess
screamer 0:66f3b5499f7f 1619 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1620 if connection_info is not None:
screamer 0:66f3b5499f7f 1621 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1622 if db_type == 'mysql':
screamer 0:66f3b5499f7f 1623 return MySQLDBAccess()
screamer 0:66f3b5499f7f 1624 return None
screamer 0:66f3b5499f7f 1625
screamer 0:66f3b5499f7f 1626
screamer 0:66f3b5499f7f 1627 def detect_database_verbose(db_url):
screamer 0:66f3b5499f7f 1628 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer 0:66f3b5499f7f 1629 """
screamer 0:66f3b5499f7f 1630 result = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1631 if result is not None:
screamer 0:66f3b5499f7f 1632 # Parsing passed
screamer 0:66f3b5499f7f 1633 (db_type, username, password, host, db_name) = result
screamer 0:66f3b5499f7f 1634 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer 0:66f3b5499f7f 1635 # Let's try to connect
screamer 0:66f3b5499f7f 1636 db_ = factory_db_logger(db_url)
screamer 0:66f3b5499f7f 1637 if db_ is not None:
screamer 0:66f3b5499f7f 1638 print "Connecting to database '%s'..."% db_url,
screamer 0:66f3b5499f7f 1639 db_.connect(host, username, password, db_name)
screamer 0:66f3b5499f7f 1640 if db_.is_connected():
screamer 0:66f3b5499f7f 1641 print "ok"
screamer 0:66f3b5499f7f 1642 print "Detecting database..."
screamer 0:66f3b5499f7f 1643 print db_.detect_database(verbose=True)
screamer 0:66f3b5499f7f 1644 print "Disconnecting...",
screamer 0:66f3b5499f7f 1645 db_.disconnect()
screamer 0:66f3b5499f7f 1646 print "done"
screamer 0:66f3b5499f7f 1647 else:
screamer 0:66f3b5499f7f 1648 print "Database type '%s' unknown"% db_type
screamer 0:66f3b5499f7f 1649 else:
screamer 0:66f3b5499f7f 1650 print "Parse error: '%s' - DB Url error"% (db_url)
screamer 0:66f3b5499f7f 1651
screamer 0:66f3b5499f7f 1652
screamer 0:66f3b5499f7f 1653 def get_module_avail(module_name):
screamer 0:66f3b5499f7f 1654 """ This function returns True if module_name is already impored module
screamer 0:66f3b5499f7f 1655 """
screamer 0:66f3b5499f7f 1656 return module_name in sys.modules.keys()
screamer 0:66f3b5499f7f 1657
screamer 0:66f3b5499f7f 1658
screamer 0:66f3b5499f7f 1659 def get_autodetected_MUTS_list(platform_name_filter=None):
screamer 0:66f3b5499f7f 1660 oldError = None
screamer 0:66f3b5499f7f 1661 if os.name == 'nt':
screamer 0:66f3b5499f7f 1662 # Disable Windows error box temporarily
screamer 0:66f3b5499f7f 1663 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer 0:66f3b5499f7f 1664
screamer 0:66f3b5499f7f 1665 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1666 detect_muts_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1667
screamer 0:66f3b5499f7f 1668 if os.name == 'nt':
screamer 0:66f3b5499f7f 1669 ctypes.windll.kernel32.SetErrorMode(oldError)
screamer 0:66f3b5499f7f 1670
screamer 0:66f3b5499f7f 1671 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 1672
screamer 0:66f3b5499f7f 1673 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer 0:66f3b5499f7f 1674 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer 0:66f3b5499f7f 1675 If function fails to auto-detect devices it will return empty dictionary.
screamer 0:66f3b5499f7f 1676
screamer 0:66f3b5499f7f 1677 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1678 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1679 mbeds_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1680
screamer 0:66f3b5499f7f 1681 @param mbeds_list list of mbeds captured from mbed_lstools
screamer 0:66f3b5499f7f 1682 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer 0:66f3b5499f7f 1683 """
screamer 0:66f3b5499f7f 1684 result = {} # Should be in muts_all.json format
screamer 0:66f3b5499f7f 1685 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer 0:66f3b5499f7f 1686 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer 0:66f3b5499f7f 1687 index = 1
screamer 0:66f3b5499f7f 1688 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1689 # Filter the MUTS if a filter is specified
screamer 0:66f3b5499f7f 1690
screamer 0:66f3b5499f7f 1691 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer 0:66f3b5499f7f 1692 continue
screamer 0:66f3b5499f7f 1693
screamer 0:66f3b5499f7f 1694 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer 0:66f3b5499f7f 1695 # if not we are creating our own unique value (last few chars from platform's target_id).
screamer 0:66f3b5499f7f 1696 m = {'mcu': mut['platform_name'],
screamer 0:66f3b5499f7f 1697 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer 0:66f3b5499f7f 1698 'port': mut['serial_port'],
screamer 0:66f3b5499f7f 1699 'disk': mut['mount_point'],
screamer 0:66f3b5499f7f 1700 'peripherals': [] # No peripheral detection
screamer 0:66f3b5499f7f 1701 }
screamer 0:66f3b5499f7f 1702 if index not in result:
screamer 0:66f3b5499f7f 1703 result[index] = {}
screamer 0:66f3b5499f7f 1704 result[index] = m
screamer 0:66f3b5499f7f 1705 index += 1
screamer 0:66f3b5499f7f 1706 return result
screamer 0:66f3b5499f7f 1707
screamer 0:66f3b5499f7f 1708
screamer 0:66f3b5499f7f 1709 def get_autodetected_TEST_SPEC(mbeds_list,
screamer 0:66f3b5499f7f 1710 use_default_toolchain=True,
screamer 0:66f3b5499f7f 1711 use_supported_toolchains=False,
screamer 0:66f3b5499f7f 1712 toolchain_filter=None,
screamer 0:66f3b5499f7f 1713 platform_name_filter=None):
screamer 0:66f3b5499f7f 1714 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer 0:66f3b5499f7f 1715 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer 0:66f3b5499f7f 1716
screamer 0:66f3b5499f7f 1717 use_default_toolchain - if True add default toolchain to test_spec
screamer 0:66f3b5499f7f 1718 use_supported_toolchains - if True add all supported toolchains to test_spec
screamer 0:66f3b5499f7f 1719 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer 0:66f3b5499f7f 1720 """
screamer 0:66f3b5499f7f 1721 result = {'targets': {} }
screamer 0:66f3b5499f7f 1722
screamer 0:66f3b5499f7f 1723 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1724 mcu = mut['mcu']
screamer 0:66f3b5499f7f 1725 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer 0:66f3b5499f7f 1726 if mcu in TARGET_MAP:
screamer 0:66f3b5499f7f 1727 default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer 0:66f3b5499f7f 1728 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer 0:66f3b5499f7f 1729
screamer 0:66f3b5499f7f 1730 # Decide which toolchains should be added to test specification toolchain pool for each target
screamer 0:66f3b5499f7f 1731 toolchains = []
screamer 0:66f3b5499f7f 1732 if use_default_toolchain:
screamer 0:66f3b5499f7f 1733 toolchains.append(default_toolchain)
screamer 0:66f3b5499f7f 1734 if use_supported_toolchains:
screamer 0:66f3b5499f7f 1735 toolchains += supported_toolchains
screamer 0:66f3b5499f7f 1736 if toolchain_filter is not None:
screamer 0:66f3b5499f7f 1737 all_toolchains = supported_toolchains + [default_toolchain]
screamer 22:9e85236d8716 1738 for toolchain in toolchain_filter:
screamer 0:66f3b5499f7f 1739 if toolchain in all_toolchains:
screamer 0:66f3b5499f7f 1740 toolchains.append(toolchain)
screamer 0:66f3b5499f7f 1741
screamer 0:66f3b5499f7f 1742 result['targets'][mcu] = list(set(toolchains))
screamer 0:66f3b5499f7f 1743 return result
screamer 0:66f3b5499f7f 1744
screamer 0:66f3b5499f7f 1745
screamer 0:66f3b5499f7f 1746 def get_default_test_options_parser():
screamer 0:66f3b5499f7f 1747 """ Get common test script options used by CLI, web services etc.
screamer 0:66f3b5499f7f 1748 """
screamer 22:9e85236d8716 1749 parser = argparse.ArgumentParser()
screamer 22:9e85236d8716 1750 parser.add_argument('-i', '--tests',
screamer 22:9e85236d8716 1751 dest='test_spec_filename',
screamer 22:9e85236d8716 1752 metavar="FILE",
screamer 22:9e85236d8716 1753 type=argparse_filestring_type,
screamer 22:9e85236d8716 1754 help='Points to file with test specification')
screamer 0:66f3b5499f7f 1755
screamer 22:9e85236d8716 1756 parser.add_argument('-M', '--MUTS',
screamer 22:9e85236d8716 1757 dest='muts_spec_filename',
screamer 22:9e85236d8716 1758 metavar="FILE",
screamer 22:9e85236d8716 1759 type=argparse_filestring_type,
screamer 22:9e85236d8716 1760 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer 0:66f3b5499f7f 1761
screamer 22:9e85236d8716 1762 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 1763 dest='jobs',
screamer 22:9e85236d8716 1764 metavar="NUMBER",
screamer 22:9e85236d8716 1765 type=int,
screamer 22:9e85236d8716 1766 help="Define number of compilation jobs. Default value is 1")
screamer 0:66f3b5499f7f 1767
screamer 0:66f3b5499f7f 1768 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1769 # Additional features available when mbed_lstools is installed on host and imported
screamer 0:66f3b5499f7f 1770 # mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer 22:9e85236d8716 1771 parser.add_argument('--auto',
screamer 22:9e85236d8716 1772 dest='auto_detect',
screamer 22:9e85236d8716 1773 action="store_true",
screamer 22:9e85236d8716 1774 help='Use mbed-ls module to detect all connected mbed devices')
screamer 0:66f3b5499f7f 1775
screamer 22:9e85236d8716 1776 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
screamer 22:9e85236d8716 1777 parser.add_argument('--tc',
screamer 22:9e85236d8716 1778 dest='toolchains_filter',
screamer 22:9e85236d8716 1779 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
screamer 22:9e85236d8716 1780 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer 0:66f3b5499f7f 1781
screamer 0:66f3b5499f7f 1782 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer 22:9e85236d8716 1783 parser.add_argument('--oper',
screamer 22:9e85236d8716 1784 dest='operability_checks',
screamer 22:9e85236d8716 1785 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
screamer 22:9e85236d8716 1786 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer 0:66f3b5499f7f 1787
screamer 22:9e85236d8716 1788 parser.add_argument('--clean',
screamer 22:9e85236d8716 1789 dest='clean',
screamer 22:9e85236d8716 1790 action="store_true",
screamer 22:9e85236d8716 1791 help='Clean the build directory')
screamer 0:66f3b5499f7f 1792
screamer 22:9e85236d8716 1793 parser.add_argument('-P', '--only-peripherals',
screamer 22:9e85236d8716 1794 dest='test_only_peripheral',
screamer 22:9e85236d8716 1795 default=False,
screamer 22:9e85236d8716 1796 action="store_true",
screamer 22:9e85236d8716 1797 help='Test only peripheral declared for MUT and skip common tests')
screamer 0:66f3b5499f7f 1798
The Other Jimmy 31:8ea194f6145b 1799 parser.add_argument("--profile", dest="profile", action="append",
The Other Jimmy 31:8ea194f6145b 1800 type=argparse_filestring_type,
The Other Jimmy 31:8ea194f6145b 1801 default=[])
The Other Jimmy 31:8ea194f6145b 1802
screamer 22:9e85236d8716 1803 parser.add_argument('-C', '--only-commons',
screamer 22:9e85236d8716 1804 dest='test_only_common',
screamer 22:9e85236d8716 1805 default=False,
screamer 22:9e85236d8716 1806 action="store_true",
screamer 22:9e85236d8716 1807 help='Test only board internals. Skip perpherials tests and perform common tests')
screamer 0:66f3b5499f7f 1808
screamer 22:9e85236d8716 1809 parser.add_argument('-n', '--test-by-names',
screamer 22:9e85236d8716 1810 dest='test_by_names',
screamer 22:9e85236d8716 1811 type=argparse_many(str),
screamer 22:9e85236d8716 1812 help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer 0:66f3b5499f7f 1813
screamer 22:9e85236d8716 1814 parser.add_argument('-p', '--peripheral-by-names',
screamer 0:66f3b5499f7f 1815 dest='peripheral_by_names',
screamer 22:9e85236d8716 1816 type=argparse_many(str),
screamer 0:66f3b5499f7f 1817 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer 0:66f3b5499f7f 1818
screamer 0:66f3b5499f7f 1819 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer 0:66f3b5499f7f 1820 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer 0:66f3b5499f7f 1821
screamer 22:9e85236d8716 1822 parser.add_argument('-c', '--copy-method',
screamer 22:9e85236d8716 1823 dest='copy_method',
screamer 22:9e85236d8716 1824 type=argparse_uppercase_type(copy_methods, "flash method"),
screamer 22:9e85236d8716 1825 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer 0:66f3b5499f7f 1826
screamer 0:66f3b5499f7f 1827 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer 0:66f3b5499f7f 1828 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer 0:66f3b5499f7f 1829
screamer 22:9e85236d8716 1830 parser.add_argument('-r', '--reset-type',
screamer 22:9e85236d8716 1831 dest='mut_reset_type',
screamer 22:9e85236d8716 1832 default=None,
screamer 22:9e85236d8716 1833 type=argparse_uppercase_type(reset_methods, "reset method"),
screamer 22:9e85236d8716 1834 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer 0:66f3b5499f7f 1835
screamer 22:9e85236d8716 1836 parser.add_argument('-g', '--goanna-for-tests',
screamer 22:9e85236d8716 1837 dest='goanna_for_tests',
screamer 22:9e85236d8716 1838 action="store_true",
screamer 22:9e85236d8716 1839 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1840
screamer 22:9e85236d8716 1841 parser.add_argument('-G', '--goanna-for-sdk',
screamer 22:9e85236d8716 1842 dest='goanna_for_mbed_sdk',
screamer 22:9e85236d8716 1843 action="store_true",
screamer 22:9e85236d8716 1844 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1845
screamer 22:9e85236d8716 1846 parser.add_argument('-s', '--suppress-summary',
screamer 22:9e85236d8716 1847 dest='suppress_summary',
screamer 22:9e85236d8716 1848 default=False,
screamer 22:9e85236d8716 1849 action="store_true",
screamer 22:9e85236d8716 1850 help='Suppresses display of wellformatted table with test results')
screamer 22:9e85236d8716 1851
screamer 22:9e85236d8716 1852 parser.add_argument('-t', '--test-summary',
screamer 22:9e85236d8716 1853 dest='test_x_toolchain_summary',
screamer 22:9e85236d8716 1854 default=False,
screamer 22:9e85236d8716 1855 action="store_true",
screamer 22:9e85236d8716 1856 help='Displays wellformatted table with test x toolchain test result per target')
screamer 0:66f3b5499f7f 1857
screamer 22:9e85236d8716 1858 parser.add_argument('-A', '--test-automation-report',
screamer 22:9e85236d8716 1859 dest='test_automation_report',
screamer 22:9e85236d8716 1860 default=False,
screamer 22:9e85236d8716 1861 action="store_true",
screamer 22:9e85236d8716 1862 help='Prints information about all tests and exits')
screamer 0:66f3b5499f7f 1863
screamer 22:9e85236d8716 1864 parser.add_argument('-R', '--test-case-report',
screamer 22:9e85236d8716 1865 dest='test_case_report',
screamer 22:9e85236d8716 1866 default=False,
screamer 22:9e85236d8716 1867 action="store_true",
screamer 22:9e85236d8716 1868 help='Prints information about all test cases and exits')
screamer 0:66f3b5499f7f 1869
screamer 22:9e85236d8716 1870 parser.add_argument("-S", "--supported-toolchains",
screamer 22:9e85236d8716 1871 action="store_true",
screamer 22:9e85236d8716 1872 dest="supported_toolchains",
screamer 22:9e85236d8716 1873 default=False,
screamer 22:9e85236d8716 1874 help="Displays supported matrix of MCUs and toolchains")
screamer 0:66f3b5499f7f 1875
screamer 22:9e85236d8716 1876 parser.add_argument("-O", "--only-build",
screamer 22:9e85236d8716 1877 action="store_true",
screamer 22:9e85236d8716 1878 dest="only_build_tests",
screamer 22:9e85236d8716 1879 default=False,
screamer 22:9e85236d8716 1880 help="Only build tests, skips actual test procedures (flashing etc.)")
screamer 0:66f3b5499f7f 1881
screamer 22:9e85236d8716 1882 parser.add_argument('--parallel',
screamer 22:9e85236d8716 1883 dest='parallel_test_exec',
screamer 22:9e85236d8716 1884 default=False,
screamer 22:9e85236d8716 1885 action="store_true",
screamer 22:9e85236d8716 1886 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer 0:66f3b5499f7f 1887
screamer 22:9e85236d8716 1888 parser.add_argument('--config',
screamer 22:9e85236d8716 1889 dest='verbose_test_configuration_only',
screamer 22:9e85236d8716 1890 default=False,
screamer 22:9e85236d8716 1891 action="store_true",
screamer 22:9e85236d8716 1892 help='Displays full test specification and MUTs configration and exits')
screamer 22:9e85236d8716 1893
screamer 22:9e85236d8716 1894 parser.add_argument('--loops',
screamer 22:9e85236d8716 1895 dest='test_loops_list',
screamer 22:9e85236d8716 1896 type=argparse_many(str),
screamer 22:9e85236d8716 1897 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer 0:66f3b5499f7f 1898
screamer 22:9e85236d8716 1899 parser.add_argument('--global-loops',
screamer 22:9e85236d8716 1900 dest='test_global_loops_value',
screamer 22:9e85236d8716 1901 type=int,
screamer 22:9e85236d8716 1902 help='Set global number of test loops per test. Default value is set 1')
screamer 0:66f3b5499f7f 1903
screamer 22:9e85236d8716 1904 parser.add_argument('--consolidate-waterfall',
screamer 22:9e85236d8716 1905 dest='consolidate_waterfall_test',
screamer 22:9e85236d8716 1906 default=False,
screamer 22:9e85236d8716 1907 action="store_true",
screamer 22:9e85236d8716 1908 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
screamer 0:66f3b5499f7f 1909
screamer 22:9e85236d8716 1910 parser.add_argument('-W', '--waterfall',
screamer 22:9e85236d8716 1911 dest='waterfall_test',
screamer 22:9e85236d8716 1912 default=False,
screamer 22:9e85236d8716 1913 action="store_true",
screamer 22:9e85236d8716 1914 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
screamer 0:66f3b5499f7f 1915
screamer 22:9e85236d8716 1916 parser.add_argument('-N', '--firmware-name',
screamer 22:9e85236d8716 1917 dest='firmware_global_name',
screamer 22:9e85236d8716 1918 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer 0:66f3b5499f7f 1919
screamer 22:9e85236d8716 1920 parser.add_argument('-u', '--shuffle',
screamer 22:9e85236d8716 1921 dest='shuffle_test_order',
screamer 22:9e85236d8716 1922 default=False,
screamer 22:9e85236d8716 1923 action="store_true",
screamer 22:9e85236d8716 1924 help='Shuffles test execution order')
screamer 0:66f3b5499f7f 1925
screamer 22:9e85236d8716 1926 parser.add_argument('--shuffle-seed',
screamer 22:9e85236d8716 1927 dest='shuffle_test_seed',
screamer 22:9e85236d8716 1928 default=None,
screamer 22:9e85236d8716 1929 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer 0:66f3b5499f7f 1930
screamer 22:9e85236d8716 1931 parser.add_argument('-f', '--filter',
screamer 22:9e85236d8716 1932 dest='general_filter_regex',
screamer 22:9e85236d8716 1933 type=argparse_many(str),
screamer 22:9e85236d8716 1934 default=None,
screamer 22:9e85236d8716 1935 help='For some commands you can use filter to filter out results')
screamer 0:66f3b5499f7f 1936
screamer 22:9e85236d8716 1937 parser.add_argument('--inc-timeout',
screamer 22:9e85236d8716 1938 dest='extend_test_timeout',
screamer 22:9e85236d8716 1939 metavar="NUMBER",
screamer 22:9e85236d8716 1940 type=int,
screamer 22:9e85236d8716 1941 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer 22:9e85236d8716 1942
screamer 22:9e85236d8716 1943 parser.add_argument('--db',
screamer 22:9e85236d8716 1944 dest='db_url',
screamer 22:9e85236d8716 1945 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer 0:66f3b5499f7f 1946
screamer 22:9e85236d8716 1947 parser.add_argument('-l', '--log',
screamer 22:9e85236d8716 1948 dest='log_file_name',
screamer 22:9e85236d8716 1949 help='Log events to external file (note not all console entries may be visible in log file)')
screamer 0:66f3b5499f7f 1950
screamer 22:9e85236d8716 1951 parser.add_argument('--report-html',
screamer 22:9e85236d8716 1952 dest='report_html_file_name',
screamer 22:9e85236d8716 1953 help='You can log test suite results in form of HTML report')
screamer 0:66f3b5499f7f 1954
screamer 22:9e85236d8716 1955 parser.add_argument('--report-junit',
screamer 22:9e85236d8716 1956 dest='report_junit_file_name',
screamer 22:9e85236d8716 1957 help='You can log test suite results in form of JUnit compliant XML report')
screamer 0:66f3b5499f7f 1958
screamer 22:9e85236d8716 1959 parser.add_argument("--report-build",
screamer 22:9e85236d8716 1960 dest="report_build_file_name",
screamer 22:9e85236d8716 1961 help="Output the build results to a junit xml file")
screamer 0:66f3b5499f7f 1962
screamer 22:9e85236d8716 1963 parser.add_argument("--report-text",
screamer 22:9e85236d8716 1964 dest="report_text_file_name",
screamer 22:9e85236d8716 1965 help="Output the build results to a text file")
screamer 0:66f3b5499f7f 1966
screamer 22:9e85236d8716 1967 parser.add_argument('--verbose-skipped',
screamer 22:9e85236d8716 1968 dest='verbose_skipped_tests',
screamer 22:9e85236d8716 1969 default=False,
screamer 22:9e85236d8716 1970 action="store_true",
screamer 22:9e85236d8716 1971 help='Prints some extra information about skipped tests')
screamer 0:66f3b5499f7f 1972
screamer 22:9e85236d8716 1973 parser.add_argument('-V', '--verbose-test-result',
screamer 22:9e85236d8716 1974 dest='verbose_test_result_only',
screamer 22:9e85236d8716 1975 default=False,
screamer 22:9e85236d8716 1976 action="store_true",
screamer 22:9e85236d8716 1977 help='Prints test serial output')
screamer 0:66f3b5499f7f 1978
screamer 22:9e85236d8716 1979 parser.add_argument('-v', '--verbose',
screamer 22:9e85236d8716 1980 dest='verbose',
screamer 22:9e85236d8716 1981 default=False,
screamer 22:9e85236d8716 1982 action="store_true",
screamer 22:9e85236d8716 1983 help='Verbose mode (prints some extra information)')
screamer 0:66f3b5499f7f 1984
screamer 22:9e85236d8716 1985 parser.add_argument('--version',
screamer 22:9e85236d8716 1986 dest='version',
screamer 22:9e85236d8716 1987 default=False,
screamer 22:9e85236d8716 1988 action="store_true",
screamer 22:9e85236d8716 1989 help='Prints script version and exits')
screamer 0:66f3b5499f7f 1990 return parser
screamer 0:66f3b5499f7f 1991
The Other Jimmy 31:8ea194f6145b 1992 def test_path_to_name(path, base):
screamer 0:66f3b5499f7f 1993 """Change all slashes in a path into hyphens
screamer 0:66f3b5499f7f 1994 This creates a unique cross-platform test name based on the path
screamer 0:66f3b5499f7f 1995 This can eventually be overriden by a to-be-determined meta-data mechanism"""
screamer 0:66f3b5499f7f 1996 name_parts = []
The Other Jimmy 31:8ea194f6145b 1997 head, tail = os.path.split(relpath(path,base))
screamer 0:66f3b5499f7f 1998 while (tail and tail != "."):
screamer 0:66f3b5499f7f 1999 name_parts.insert(0, tail)
screamer 0:66f3b5499f7f 2000 head, tail = os.path.split(head)
screamer 22:9e85236d8716 2001
screamer 13:ab47a20b66f0 2002 return "-".join(name_parts).lower()
screamer 0:66f3b5499f7f 2003
The Other Jimmy 31:8ea194f6145b 2004 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
screamer 24:25bff2709c20 2005 """ Finds all tests in a directory recursively
screamer 24:25bff2709c20 2006 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
screamer 24:25bff2709c20 2007 target_name: name of the target to use for scanning (ex. 'K64F')
screamer 24:25bff2709c20 2008 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
screamer 24:25bff2709c20 2009 options: Compile options to pass to the toolchain (ex. ['debug-info'])
The Other Jimmy 31:8ea194f6145b 2010 app_config - location of a chosen mbed_app.json file
screamer 24:25bff2709c20 2011 """
screamer 24:25bff2709c20 2012
screamer 24:25bff2709c20 2013 tests = {}
screamer 24:25bff2709c20 2014
screamer 24:25bff2709c20 2015 # Prepare the toolchain
The Other Jimmy 31:8ea194f6145b 2016 toolchain = prepare_toolchain([base_dir], target_name, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2017 silent=True, app_config=app_config)
screamer 24:25bff2709c20 2018
screamer 24:25bff2709c20 2019 # Scan the directory for paths to probe for 'TESTS' folders
The Other Jimmy 31:8ea194f6145b 2020 base_resources = scan_resources([base_dir], toolchain)
screamer 24:25bff2709c20 2021
screamer 24:25bff2709c20 2022 dirs = base_resources.inc_dirs
screamer 24:25bff2709c20 2023 for directory in dirs:
screamer 24:25bff2709c20 2024 subdirs = os.listdir(directory)
screamer 22:9e85236d8716 2025
screamer 24:25bff2709c20 2026 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
screamer 24:25bff2709c20 2027 if 'TESTS' in subdirs:
screamer 24:25bff2709c20 2028 walk_base_dir = join(directory, 'TESTS')
screamer 24:25bff2709c20 2029 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
screamer 24:25bff2709c20 2030
screamer 24:25bff2709c20 2031 # Loop through all subdirectories
screamer 24:25bff2709c20 2032 for d in test_resources.inc_dirs:
screamer 22:9e85236d8716 2033
screamer 24:25bff2709c20 2034 # If the test case folder is not called 'host_tests' and it is
screamer 24:25bff2709c20 2035 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
screamer 24:25bff2709c20 2036 # then add it to the tests
screamer 24:25bff2709c20 2037 path_depth = get_path_depth(relpath(d, walk_base_dir))
screamer 24:25bff2709c20 2038 if path_depth == 2:
screamer 24:25bff2709c20 2039 test_group_directory_path, test_case_directory = os.path.split(d)
screamer 24:25bff2709c20 2040 test_group_directory = os.path.basename(test_group_directory_path)
screamer 24:25bff2709c20 2041
screamer 24:25bff2709c20 2042 # Check to make sure discoverd folder is not in a host test directory
screamer 24:25bff2709c20 2043 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
The Other Jimmy 31:8ea194f6145b 2044 test_name = test_path_to_name(d, base_dir)
screamer 24:25bff2709c20 2045 tests[test_name] = d
screamer 22:9e85236d8716 2046
screamer 13:ab47a20b66f0 2047 return tests
screamer 22:9e85236d8716 2048
screamer 13:ab47a20b66f0 2049 def print_tests(tests, format="list", sort=True):
screamer 0:66f3b5499f7f 2050 """Given a dictionary of tests (as returned from "find_tests"), print them
screamer 0:66f3b5499f7f 2051 in the specified format"""
screamer 0:66f3b5499f7f 2052 if format == "list":
screamer 13:ab47a20b66f0 2053 for test_name in sorted(tests.keys()):
screamer 13:ab47a20b66f0 2054 test_path = tests[test_name]
screamer 0:66f3b5499f7f 2055 print "Test Case:"
screamer 0:66f3b5499f7f 2056 print " Name: %s" % test_name
screamer 0:66f3b5499f7f 2057 print " Path: %s" % test_path
screamer 0:66f3b5499f7f 2058 elif format == "json":
screamer 0:66f3b5499f7f 2059 print json.dumps(tests, indent=2)
screamer 0:66f3b5499f7f 2060 else:
screamer 0:66f3b5499f7f 2061 print "Unknown format '%s'" % format
screamer 0:66f3b5499f7f 2062 sys.exit(1)
screamer 0:66f3b5499f7f 2063
screamer 13:ab47a20b66f0 2064 def norm_relative_path(path, start):
screamer 13:ab47a20b66f0 2065 """This function will create a normalized, relative path. It mimics the
screamer 13:ab47a20b66f0 2066 python os.path.relpath function, but also normalizes a Windows-syle path
screamer 13:ab47a20b66f0 2067 that use backslashes to a Unix style path that uses forward slashes."""
screamer 13:ab47a20b66f0 2068 path = os.path.normpath(path)
screamer 13:ab47a20b66f0 2069 path = os.path.relpath(path, start)
screamer 13:ab47a20b66f0 2070 path = path.replace("\\", "/")
screamer 13:ab47a20b66f0 2071 return path
screamer 13:ab47a20b66f0 2072
The Other Jimmy 31:8ea194f6145b 2073
The Other Jimmy 31:8ea194f6145b 2074 def build_test_worker(*args, **kwargs):
The Other Jimmy 31:8ea194f6145b 2075 """This is a worker function for the parallel building of tests. The `args`
The Other Jimmy 31:8ea194f6145b 2076 and `kwargs` are passed directly to `build_project`. It returns a dictionary
The Other Jimmy 31:8ea194f6145b 2077 with the following structure:
The Other Jimmy 31:8ea194f6145b 2078
The Other Jimmy 31:8ea194f6145b 2079 {
The Other Jimmy 31:8ea194f6145b 2080 'result': `True` if no exceptions were thrown, `False` otherwise
The Other Jimmy 31:8ea194f6145b 2081 'reason': Instance of exception that was thrown on failure
The Other Jimmy 31:8ea194f6145b 2082 'bin_file': Path to the created binary if `build_project` was
The Other Jimmy 31:8ea194f6145b 2083 successful. Not present otherwise
The Other Jimmy 31:8ea194f6145b 2084 'kwargs': The keyword arguments that were passed to `build_project`.
The Other Jimmy 31:8ea194f6145b 2085 This includes arguments that were modified (ex. report)
The Other Jimmy 31:8ea194f6145b 2086 }
The Other Jimmy 31:8ea194f6145b 2087 """
The Other Jimmy 31:8ea194f6145b 2088 bin_file = None
The Other Jimmy 31:8ea194f6145b 2089 ret = {
The Other Jimmy 31:8ea194f6145b 2090 'result': False,
The Other Jimmy 31:8ea194f6145b 2091 'args': args,
The Other Jimmy 31:8ea194f6145b 2092 'kwargs': kwargs
The Other Jimmy 31:8ea194f6145b 2093 }
The Other Jimmy 31:8ea194f6145b 2094
The Other Jimmy 31:8ea194f6145b 2095 # Use parent TOOLCHAIN_PATHS variable
The Other Jimmy 31:8ea194f6145b 2096 for key, value in kwargs['toolchain_paths'].iteritems():
The Other Jimmy 31:8ea194f6145b 2097 TOOLCHAIN_PATHS[key] = value
The Other Jimmy 31:8ea194f6145b 2098
The Other Jimmy 31:8ea194f6145b 2099 del kwargs['toolchain_paths']
The Other Jimmy 31:8ea194f6145b 2100
The Other Jimmy 31:8ea194f6145b 2101 try:
The Other Jimmy 31:8ea194f6145b 2102 bin_file = build_project(*args, **kwargs)
The Other Jimmy 31:8ea194f6145b 2103 ret['result'] = True
The Other Jimmy 31:8ea194f6145b 2104 ret['bin_file'] = bin_file
The Other Jimmy 31:8ea194f6145b 2105 ret['kwargs'] = kwargs
The Other Jimmy 31:8ea194f6145b 2106
The Other Jimmy 31:8ea194f6145b 2107 except NotSupportedException, e:
The Other Jimmy 31:8ea194f6145b 2108 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2109 except ToolException, e:
The Other Jimmy 31:8ea194f6145b 2110 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2111 except KeyboardInterrupt, e:
The Other Jimmy 31:8ea194f6145b 2112 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2113 except:
The Other Jimmy 31:8ea194f6145b 2114 # Print unhandled exceptions here
The Other Jimmy 31:8ea194f6145b 2115 import traceback
The Other Jimmy 31:8ea194f6145b 2116 traceback.print_exc(file=sys.stdout)
The Other Jimmy 31:8ea194f6145b 2117
The Other Jimmy 31:8ea194f6145b 2118 return ret
The Other Jimmy 31:8ea194f6145b 2119
The Other Jimmy 31:8ea194f6145b 2120
screamer 0:66f3b5499f7f 2121 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
The Other Jimmy 31:8ea194f6145b 2122 clean=False, notify=None, verbose=False, jobs=1, macros=None,
The Other Jimmy 31:8ea194f6145b 2123 silent=False, report=None, properties=None,
The Other Jimmy 31:8ea194f6145b 2124 continue_on_build_fail=False, app_config=None,
The Other Jimmy 31:8ea194f6145b 2125 build_profile=None):
screamer 0:66f3b5499f7f 2126 """Given the data structure from 'find_tests' and the typical build parameters,
screamer 13:ab47a20b66f0 2127 build all the tests
screamer 22:9e85236d8716 2128
screamer 13:ab47a20b66f0 2129 Returns a tuple of the build result (True or False) followed by the test
screamer 13:ab47a20b66f0 2130 build data structure"""
screamer 22:9e85236d8716 2131
screamer 13:ab47a20b66f0 2132 execution_directory = "."
screamer 13:ab47a20b66f0 2133 base_path = norm_relative_path(build_path, execution_directory)
screamer 13:ab47a20b66f0 2134
screamer 13:ab47a20b66f0 2135 target_name = target if isinstance(target, str) else target.name
The Other Jimmy 35:da9c89f8be7d 2136 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2137
The Other Jimmy 31:8ea194f6145b 2138 baud_rate = 9600
The Other Jimmy 31:8ea194f6145b 2139 if 'platform.stdio-baud-rate' in cfg:
The Other Jimmy 31:8ea194f6145b 2140 baud_rate = cfg['platform.stdio-baud-rate'].value
The Other Jimmy 31:8ea194f6145b 2141
screamer 0:66f3b5499f7f 2142 test_build = {
screamer 13:ab47a20b66f0 2143 "platform": target_name,
screamer 0:66f3b5499f7f 2144 "toolchain": toolchain_name,
screamer 13:ab47a20b66f0 2145 "base_path": base_path,
The Other Jimmy 31:8ea194f6145b 2146 "baud_rate": baud_rate,
screamer 0:66f3b5499f7f 2147 "binary_type": "bootable",
screamer 0:66f3b5499f7f 2148 "tests": {}
screamer 0:66f3b5499f7f 2149 }
screamer 22:9e85236d8716 2150
screamer 13:ab47a20b66f0 2151 result = True
screamer 22:9e85236d8716 2152
The Other Jimmy 31:8ea194f6145b 2153 jobs_count = int(jobs if jobs else cpu_count())
The Other Jimmy 31:8ea194f6145b 2154 p = Pool(processes=jobs_count)
The Other Jimmy 31:8ea194f6145b 2155 results = []
screamer 0:66f3b5499f7f 2156 for test_name, test_path in tests.iteritems():
screamer 0:66f3b5499f7f 2157 test_build_path = os.path.join(build_path, test_path)
screamer 0:66f3b5499f7f 2158 src_path = base_source_paths + [test_path]
screamer 13:ab47a20b66f0 2159 bin_file = None
screamer 24:25bff2709c20 2160 test_case_folder_name = os.path.basename(test_path)
screamer 24:25bff2709c20 2161
The Other Jimmy 31:8ea194f6145b 2162 args = (src_path, test_build_path, target, toolchain_name)
The Other Jimmy 31:8ea194f6145b 2163 kwargs = {
The Other Jimmy 31:8ea194f6145b 2164 'jobs': 1,
The Other Jimmy 31:8ea194f6145b 2165 'clean': clean,
The Other Jimmy 31:8ea194f6145b 2166 'macros': macros,
The Other Jimmy 31:8ea194f6145b 2167 'name': test_case_folder_name,
The Other Jimmy 31:8ea194f6145b 2168 'project_id': test_name,
The Other Jimmy 31:8ea194f6145b 2169 'report': report,
The Other Jimmy 31:8ea194f6145b 2170 'properties': properties,
The Other Jimmy 31:8ea194f6145b 2171 'verbose': verbose,
The Other Jimmy 31:8ea194f6145b 2172 'app_config': app_config,
The Other Jimmy 31:8ea194f6145b 2173 'build_profile': build_profile,
The Other Jimmy 31:8ea194f6145b 2174 'silent': True,
The Other Jimmy 31:8ea194f6145b 2175 'toolchain_paths': TOOLCHAIN_PATHS
The Other Jimmy 31:8ea194f6145b 2176 }
screamer 24:25bff2709c20 2177
The Other Jimmy 31:8ea194f6145b 2178 results.append(p.apply_async(build_test_worker, args, kwargs))
The Other Jimmy 31:8ea194f6145b 2179
The Other Jimmy 31:8ea194f6145b 2180 p.close()
The Other Jimmy 31:8ea194f6145b 2181 result = True
The Other Jimmy 31:8ea194f6145b 2182 itr = 0
The Other Jimmy 31:8ea194f6145b 2183 while len(results):
The Other Jimmy 31:8ea194f6145b 2184 itr += 1
The Other Jimmy 31:8ea194f6145b 2185 if itr > 360000:
The Other Jimmy 31:8ea194f6145b 2186 p.terminate()
The Other Jimmy 31:8ea194f6145b 2187 p.join()
The Other Jimmy 31:8ea194f6145b 2188 raise ToolException("Compile did not finish in 10 minutes")
The Other Jimmy 31:8ea194f6145b 2189 else:
The Other Jimmy 31:8ea194f6145b 2190 sleep(0.01)
The Other Jimmy 31:8ea194f6145b 2191 pending = 0
The Other Jimmy 31:8ea194f6145b 2192 for r in results:
The Other Jimmy 31:8ea194f6145b 2193 if r.ready() is True:
The Other Jimmy 31:8ea194f6145b 2194 try:
The Other Jimmy 31:8ea194f6145b 2195 worker_result = r.get()
The Other Jimmy 31:8ea194f6145b 2196 results.remove(r)
screamer 13:ab47a20b66f0 2197
The Other Jimmy 31:8ea194f6145b 2198 # Take report from the kwargs and merge it into existing report
The Other Jimmy 31:8ea194f6145b 2199 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
The Other Jimmy 31:8ea194f6145b 2200 for test_key in report_entry.keys():
The Other Jimmy 31:8ea194f6145b 2201 report[target_name][toolchain_name][test_key] = report_entry[test_key]
The Other Jimmy 31:8ea194f6145b 2202
The Other Jimmy 31:8ea194f6145b 2203 # Set the overall result to a failure if a build failure occurred
The Other Jimmy 35:da9c89f8be7d 2204 if ('reason' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2205 not worker_result['reason'] and
The Other Jimmy 35:da9c89f8be7d 2206 not isinstance(worker_result['reason'], NotSupportedException)):
The Other Jimmy 31:8ea194f6145b 2207 result = False
The Other Jimmy 31:8ea194f6145b 2208 break
The Other Jimmy 31:8ea194f6145b 2209
The Other Jimmy 31:8ea194f6145b 2210 # Adding binary path to test build result
The Other Jimmy 35:da9c89f8be7d 2211 if ('result' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2212 worker_result['result'] and
The Other Jimmy 35:da9c89f8be7d 2213 'bin_file' in worker_result):
The Other Jimmy 31:8ea194f6145b 2214 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
screamer 22:9e85236d8716 2215
The Other Jimmy 31:8ea194f6145b 2216 test_build['tests'][worker_result['kwargs']['project_id']] = {
The Other Jimmy 31:8ea194f6145b 2217 "binaries": [
The Other Jimmy 31:8ea194f6145b 2218 {
The Other Jimmy 31:8ea194f6145b 2219 "path": bin_file
The Other Jimmy 31:8ea194f6145b 2220 }
The Other Jimmy 31:8ea194f6145b 2221 ]
The Other Jimmy 31:8ea194f6145b 2222 }
The Other Jimmy 31:8ea194f6145b 2223
The Other Jimmy 31:8ea194f6145b 2224 test_key = worker_result['kwargs']['project_id'].upper()
The Other Jimmy 31:8ea194f6145b 2225 print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
The Other Jimmy 31:8ea194f6145b 2226 print 'Image: %s\n' % bin_file
The Other Jimmy 31:8ea194f6145b 2227
The Other Jimmy 31:8ea194f6145b 2228 except:
The Other Jimmy 31:8ea194f6145b 2229 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2230 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2231 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2232 p.terminate()
The Other Jimmy 31:8ea194f6145b 2233 p.join()
The Other Jimmy 31:8ea194f6145b 2234 raise
screamer 13:ab47a20b66f0 2235 else:
The Other Jimmy 31:8ea194f6145b 2236 pending += 1
The Other Jimmy 31:8ea194f6145b 2237 if pending >= jobs_count:
The Other Jimmy 31:8ea194f6145b 2238 break
screamer 22:9e85236d8716 2239
The Other Jimmy 31:8ea194f6145b 2240 # Break as soon as possible if there is a failure and we are not
The Other Jimmy 31:8ea194f6145b 2241 # continuing on build failures
The Other Jimmy 31:8ea194f6145b 2242 if not result and not continue_on_build_fail:
The Other Jimmy 31:8ea194f6145b 2243 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2244 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2245 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2246 p.terminate()
The Other Jimmy 31:8ea194f6145b 2247 break
screamer 22:9e85236d8716 2248
The Other Jimmy 31:8ea194f6145b 2249 p.join()
screamer 22:9e85236d8716 2250
screamer 0:66f3b5499f7f 2251 test_builds = {}
screamer 13:ab47a20b66f0 2252 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
screamer 22:9e85236d8716 2253
screamer 13:ab47a20b66f0 2254 return result, test_builds
screamer 22:9e85236d8716 2255
screamer 0:66f3b5499f7f 2256
screamer 13:ab47a20b66f0 2257 def test_spec_from_test_builds(test_builds):
screamer 0:66f3b5499f7f 2258 return {
screamer 0:66f3b5499f7f 2259 "builds": test_builds
screamer 0:66f3b5499f7f 2260 }