Anders Blomdell / mbed-sdk-tools

Clone of official tools

test_api.py@43:2a7da56ebd24, 2018-09-25 (annotated)

Committer:
theotherjimmy
Date:
Tue Sep 25 13:43:09 2018 -0500
Revision:
43:2a7da56ebd24
Parent:
41:2a77626a4c21 
Release 5.10.0

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """
screamer 0:66f3b5499f7f 2 mbed SDK
screamer 0:66f3b5499f7f 3 Copyright (c) 2011-2014 ARM Limited
screamer 0:66f3b5499f7f 4
screamer 0:66f3b5499f7f 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 6 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 7 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 8
screamer 0:66f3b5499f7f 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 14 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 15 limitations under the License.
screamer 0:66f3b5499f7f 16
screamer 0:66f3b5499f7f 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer 0:66f3b5499f7f 18 """
theotherjimmy 43:2a7da56ebd24 19 from __future__ import print_function
theotherjimmy 43:2a7da56ebd24 20 import six
screamer 0:66f3b5499f7f 21
screamer 0:66f3b5499f7f 22 import os
screamer 0:66f3b5499f7f 23 import re
screamer 0:66f3b5499f7f 24 import sys
screamer 0:66f3b5499f7f 25 import json
screamer 0:66f3b5499f7f 26 import uuid
screamer 0:66f3b5499f7f 27 import pprint
screamer 0:66f3b5499f7f 28 import random
screamer 22:9e85236d8716 29 import argparse
screamer 0:66f3b5499f7f 30 import datetime
screamer 0:66f3b5499f7f 31 import threading
screamer 0:66f3b5499f7f 32 import ctypes
theotherjimmy 43:2a7da56ebd24 33 import functools
screamer 0:66f3b5499f7f 34 from colorama import Fore, Back, Style
theotherjimmy 43:2a7da56ebd24 35 from prettytable import PrettyTable, HEADER
theotherjimmy 43:2a7da56ebd24 36 from copy import copy, deepcopy
screamer 0:66f3b5499f7f 37
screamer 0:66f3b5499f7f 38 from time import sleep, time
theotherjimmy 43:2a7da56ebd24 39 try:
theotherjimmy 43:2a7da56ebd24 40 from Queue import Queue, Empty
theotherjimmy 43:2a7da56ebd24 41 except ImportError:
theotherjimmy 43:2a7da56ebd24 42 from queue import Queue, Empty
theotherjimmy 43:2a7da56ebd24 43 from os.path import join, exists, basename, relpath, isdir, isfile
screamer 0:66f3b5499f7f 44 from threading import Thread, Lock
The Other Jimmy 31:8ea194f6145b 45 from multiprocessing import Pool, cpu_count
screamer 0:66f3b5499f7f 46 from subprocess import Popen, PIPE
screamer 0:66f3b5499f7f 47
screamer 0:66f3b5499f7f 48 # Imports related to mbed build api
screamer 0:66f3b5499f7f 49 from tools.tests import TESTS
screamer 0:66f3b5499f7f 50 from tools.tests import TEST_MAP
screamer 0:66f3b5499f7f 51 from tools.paths import BUILD_DIR
screamer 0:66f3b5499f7f 52 from tools.paths import HOST_TESTS
screamer 0:66f3b5499f7f 53 from tools.utils import ToolException
screamer 0:66f3b5499f7f 54 from tools.utils import NotSupportedException
screamer 0:66f3b5499f7f 55 from tools.utils import construct_enum
screamer 22:9e85236d8716 56 from tools.memap import MemapParser
theotherjimmy 43:2a7da56ebd24 57 from tools.targets import TARGET_MAP, Target
theotherjimmy 43:2a7da56ebd24 58 from tools.config import Config
theotherjimmy 41:2a77626a4c21 59 import tools.test_configs as TestConfig
screamer 0:66f3b5499f7f 60 from tools.test_db import BaseDBAccess
screamer 0:66f3b5499f7f 61 from tools.build_api import build_project, build_mbed_libs, build_lib
screamer 0:66f3b5499f7f 62 from tools.build_api import get_target_supported_toolchains
screamer 0:66f3b5499f7f 63 from tools.build_api import write_build_report
screamer 0:66f3b5499f7f 64 from tools.build_api import prep_report
screamer 0:66f3b5499f7f 65 from tools.build_api import prep_properties
screamer 0:66f3b5499f7f 66 from tools.build_api import create_result
screamer 0:66f3b5499f7f 67 from tools.build_api import add_result_to_report
screamer 24:25bff2709c20 68 from tools.build_api import prepare_toolchain
The Other Jimmy 31:8ea194f6145b 69 from tools.build_api import get_config
theotherjimmy 43:2a7da56ebd24 70 from tools.resources import Resources, MbedIgnoreSet, IGNORE_FILENAME
screamer 0:66f3b5499f7f 71 from tools.libraries import LIBRARIES, LIBRARY_MAP
The Other Jimmy 31:8ea194f6145b 72 from tools.options import extract_profile
screamer 20:835f6355470d 73 from tools.toolchains import TOOLCHAIN_PATHS
screamer 22:9e85236d8716 74 from tools.toolchains import TOOLCHAINS
screamer 0:66f3b5499f7f 75 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 22:9e85236d8716 76 from tools.utils import argparse_filestring_type
screamer 22:9e85236d8716 77 from tools.utils import argparse_uppercase_type
screamer 22:9e85236d8716 78 from tools.utils import argparse_lowercase_type
screamer 22:9e85236d8716 79 from tools.utils import argparse_many
theotherjimmy 43:2a7da56ebd24 80 from tools.notifier.mock import MockNotifier
theotherjimmy 43:2a7da56ebd24 81 from tools.notifier.term import TerminalNotifier
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83 import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer 0:66f3b5499f7f 84
screamer 0:66f3b5499f7f 85 try:
screamer 0:66f3b5499f7f 86 import mbed_lstools
screamer 0:66f3b5499f7f 87 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:66f3b5499f7f 88 except:
screamer 0:66f3b5499f7f 89 pass
screamer 0:66f3b5499f7f 90
screamer 0:66f3b5499f7f 91
screamer 0:66f3b5499f7f 92 class ProcessObserver(Thread):
screamer 0:66f3b5499f7f 93 def __init__(self, proc):
screamer 0:66f3b5499f7f 94 Thread.__init__(self)
screamer 0:66f3b5499f7f 95 self.proc = proc
screamer 0:66f3b5499f7f 96 self.queue = Queue()
screamer 0:66f3b5499f7f 97 self.daemon = True
screamer 0:66f3b5499f7f 98 self.active = True
screamer 0:66f3b5499f7f 99 self.start()
screamer 0:66f3b5499f7f 100
screamer 0:66f3b5499f7f 101 def run(self):
screamer 0:66f3b5499f7f 102 while self.active:
screamer 0:66f3b5499f7f 103 c = self.proc.stdout.read(1)
screamer 0:66f3b5499f7f 104 self.queue.put(c)
screamer 0:66f3b5499f7f 105
screamer 0:66f3b5499f7f 106 def stop(self):
screamer 0:66f3b5499f7f 107 self.active = False
screamer 0:66f3b5499f7f 108 try:
screamer 0:66f3b5499f7f 109 self.proc.terminate()
theotherjimmy 43:2a7da56ebd24 110 except Exception:
screamer 0:66f3b5499f7f 111 pass
screamer 0:66f3b5499f7f 112
screamer 0:66f3b5499f7f 113
screamer 0:66f3b5499f7f 114 class SingleTestExecutor(threading.Thread):
screamer 0:66f3b5499f7f 115 """ Example: Single test class in separate thread usage
screamer 0:66f3b5499f7f 116 """
screamer 0:66f3b5499f7f 117 def __init__(self, single_test):
screamer 0:66f3b5499f7f 118 self.single_test = single_test
screamer 0:66f3b5499f7f 119 threading.Thread.__init__(self)
screamer 0:66f3b5499f7f 120
screamer 0:66f3b5499f7f 121 def run(self):
screamer 0:66f3b5499f7f 122 start = time()
screamer 0:66f3b5499f7f 123 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 124 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer 0:66f3b5499f7f 125 elapsed_time = time() - start
screamer 0:66f3b5499f7f 126
screamer 0:66f3b5499f7f 127 # Human readable summary
screamer 0:66f3b5499f7f 128 if not self.single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 129 # prints well-formed summary with results (SQL table like)
theotherjimmy 43:2a7da56ebd24 130 print(self.single_test.generate_test_summary(test_summary,
theotherjimmy 43:2a7da56ebd24 131 shuffle_seed))
screamer 0:66f3b5499f7f 132 if self.single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 133 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 134 # table shows text x toolchain test result matrix
theotherjimmy 43:2a7da56ebd24 135 print(self.single_test.generate_test_summary_by_target(
theotherjimmy 43:2a7da56ebd24 136 test_summary, shuffle_seed))
theotherjimmy 43:2a7da56ebd24 137 print("Completed in %.2f sec"% (elapsed_time))
screamer 0:66f3b5499f7f 138
screamer 0:66f3b5499f7f 139
screamer 0:66f3b5499f7f 140 class SingleTestRunner(object):
screamer 0:66f3b5499f7f 141 """ Object wrapper for single test run which may involve multiple MUTs
screamer 0:66f3b5499f7f 142 """
screamer 0:66f3b5499f7f 143 RE_DETECT_TESTCASE_RESULT = None
screamer 0:66f3b5499f7f 144
screamer 0:66f3b5499f7f 145 # Return codes for test script
screamer 0:66f3b5499f7f 146 TEST_RESULT_OK = "OK"
screamer 0:66f3b5499f7f 147 TEST_RESULT_FAIL = "FAIL"
screamer 0:66f3b5499f7f 148 TEST_RESULT_ERROR = "ERROR"
screamer 0:66f3b5499f7f 149 TEST_RESULT_UNDEF = "UNDEF"
screamer 0:66f3b5499f7f 150 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer 0:66f3b5499f7f 151 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer 0:66f3b5499f7f 152 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer 0:66f3b5499f7f 153 TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer 0:66f3b5499f7f 154 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer 0:66f3b5499f7f 155 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer 0:66f3b5499f7f 156 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer 0:66f3b5499f7f 157 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer 0:66f3b5499f7f 158
screamer 0:66f3b5499f7f 159 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer 0:66f3b5499f7f 160 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer 0:66f3b5499f7f 161 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer 0:66f3b5499f7f 162
screamer 0:66f3b5499f7f 163 muts = {} # MUTs descriptor (from external file)
screamer 0:66f3b5499f7f 164 test_spec = {} # Test specification (from external file)
screamer 0:66f3b5499f7f 165
screamer 0:66f3b5499f7f 166 # mbed test suite -> SingleTestRunner
screamer 0:66f3b5499f7f 167 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer 0:66f3b5499f7f 168 "failure" : TEST_RESULT_FAIL,
screamer 0:66f3b5499f7f 169 "error" : TEST_RESULT_ERROR,
screamer 0:66f3b5499f7f 170 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer 0:66f3b5499f7f 171 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer 0:66f3b5499f7f 172 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer 0:66f3b5499f7f 173 "timeout" : TEST_RESULT_TIMEOUT,
screamer 0:66f3b5499f7f 174 "no_image" : TEST_RESULT_NO_IMAGE,
screamer 0:66f3b5499f7f 175 "end" : TEST_RESULT_UNDEF,
screamer 0:66f3b5499f7f 176 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer 0:66f3b5499f7f 177 "build_failed" : TEST_RESULT_BUILD_FAILED,
screamer 0:66f3b5499f7f 178 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 179 }
screamer 0:66f3b5499f7f 180
screamer 0:66f3b5499f7f 181 def __init__(self,
screamer 0:66f3b5499f7f 182 _global_loops_count=1,
screamer 0:66f3b5499f7f 183 _test_loops_list=None,
screamer 0:66f3b5499f7f 184 _muts={},
screamer 0:66f3b5499f7f 185 _clean=False,
The Other Jimmy 31:8ea194f6145b 186 _parser=None,
The Other Jimmy 31:8ea194f6145b 187 _opts=None,
screamer 0:66f3b5499f7f 188 _opts_db_url=None,
screamer 0:66f3b5499f7f 189 _opts_log_file_name=None,
screamer 0:66f3b5499f7f 190 _opts_report_html_file_name=None,
screamer 0:66f3b5499f7f 191 _opts_report_junit_file_name=None,
screamer 0:66f3b5499f7f 192 _opts_report_build_file_name=None,
screamer 22:9e85236d8716 193 _opts_report_text_file_name=None,
screamer 0:66f3b5499f7f 194 _opts_build_report={},
screamer 0:66f3b5499f7f 195 _opts_build_properties={},
screamer 0:66f3b5499f7f 196 _test_spec={},
screamer 0:66f3b5499f7f 197 _opts_goanna_for_mbed_sdk=None,
screamer 0:66f3b5499f7f 198 _opts_goanna_for_tests=None,
screamer 0:66f3b5499f7f 199 _opts_shuffle_test_order=False,
screamer 0:66f3b5499f7f 200 _opts_shuffle_test_seed=None,
screamer 0:66f3b5499f7f 201 _opts_test_by_names=None,
screamer 0:66f3b5499f7f 202 _opts_peripheral_by_names=None,
screamer 0:66f3b5499f7f 203 _opts_test_only_peripheral=False,
screamer 0:66f3b5499f7f 204 _opts_test_only_common=False,
screamer 0:66f3b5499f7f 205 _opts_verbose_skipped_tests=False,
screamer 0:66f3b5499f7f 206 _opts_verbose_test_result_only=False,
screamer 0:66f3b5499f7f 207 _opts_verbose=False,
screamer 0:66f3b5499f7f 208 _opts_firmware_global_name=None,
screamer 0:66f3b5499f7f 209 _opts_only_build_tests=False,
screamer 0:66f3b5499f7f 210 _opts_parallel_test_exec=False,
screamer 0:66f3b5499f7f 211 _opts_suppress_summary=False,
screamer 0:66f3b5499f7f 212 _opts_test_x_toolchain_summary=False,
screamer 0:66f3b5499f7f 213 _opts_copy_method=None,
screamer 0:66f3b5499f7f 214 _opts_mut_reset_type=None,
screamer 0:66f3b5499f7f 215 _opts_jobs=None,
screamer 0:66f3b5499f7f 216 _opts_waterfall_test=None,
screamer 0:66f3b5499f7f 217 _opts_consolidate_waterfall_test=None,
screamer 0:66f3b5499f7f 218 _opts_extend_test_timeout=None,
screamer 0:66f3b5499f7f 219 _opts_auto_detect=None,
screamer 0:66f3b5499f7f 220 _opts_include_non_automated=False):
screamer 0:66f3b5499f7f 221 """ Let's try hard to init this object
screamer 0:66f3b5499f7f 222 """
screamer 0:66f3b5499f7f 223 from colorama import init
screamer 0:66f3b5499f7f 224 init()
screamer 0:66f3b5499f7f 225
screamer 0:66f3b5499f7f 226 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer 0:66f3b5499f7f 227 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer 0:66f3b5499f7f 228 # Settings related to test loops counters
screamer 0:66f3b5499f7f 229 try:
screamer 0:66f3b5499f7f 230 _global_loops_count = int(_global_loops_count)
screamer 0:66f3b5499f7f 231 except:
screamer 0:66f3b5499f7f 232 _global_loops_count = 1
screamer 0:66f3b5499f7f 233 if _global_loops_count < 1:
screamer 0:66f3b5499f7f 234 _global_loops_count = 1
screamer 0:66f3b5499f7f 235 self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer 0:66f3b5499f7f 236 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer 0:66f3b5499f7f 237 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer 0:66f3b5499f7f 238
screamer 0:66f3b5499f7f 239 self.shuffle_random_seed = 0.0
screamer 0:66f3b5499f7f 240 self.SHUFFLE_SEED_ROUND = 10
screamer 0:66f3b5499f7f 241
screamer 0:66f3b5499f7f 242 # MUT list and test specification storage
screamer 0:66f3b5499f7f 243 self.muts = _muts
screamer 0:66f3b5499f7f 244 self.test_spec = _test_spec
screamer 0:66f3b5499f7f 245
screamer 0:66f3b5499f7f 246 # Settings passed e.g. from command line
screamer 0:66f3b5499f7f 247 self.opts_db_url = _opts_db_url
screamer 0:66f3b5499f7f 248 self.opts_log_file_name = _opts_log_file_name
screamer 0:66f3b5499f7f 249 self.opts_report_html_file_name = _opts_report_html_file_name
screamer 0:66f3b5499f7f 250 self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer 0:66f3b5499f7f 251 self.opts_report_build_file_name = _opts_report_build_file_name
screamer 22:9e85236d8716 252 self.opts_report_text_file_name = _opts_report_text_file_name
screamer 0:66f3b5499f7f 253 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer 0:66f3b5499f7f 254 self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer 0:66f3b5499f7f 255 self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer 0:66f3b5499f7f 256 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer 0:66f3b5499f7f 257 self.opts_test_by_names = _opts_test_by_names
screamer 0:66f3b5499f7f 258 self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer 0:66f3b5499f7f 259 self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer 0:66f3b5499f7f 260 self.opts_test_only_common = _opts_test_only_common
screamer 0:66f3b5499f7f 261 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer 0:66f3b5499f7f 262 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer 0:66f3b5499f7f 263 self.opts_verbose = _opts_verbose
screamer 0:66f3b5499f7f 264 self.opts_firmware_global_name = _opts_firmware_global_name
screamer 0:66f3b5499f7f 265 self.opts_only_build_tests = _opts_only_build_tests
screamer 0:66f3b5499f7f 266 self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer 0:66f3b5499f7f 267 self.opts_suppress_summary = _opts_suppress_summary
screamer 0:66f3b5499f7f 268 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer 0:66f3b5499f7f 269 self.opts_copy_method = _opts_copy_method
screamer 0:66f3b5499f7f 270 self.opts_mut_reset_type = _opts_mut_reset_type
screamer 0:66f3b5499f7f 271 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer 0:66f3b5499f7f 272 self.opts_waterfall_test = _opts_waterfall_test
screamer 0:66f3b5499f7f 273 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer 0:66f3b5499f7f 274 self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer 0:66f3b5499f7f 275 self.opts_clean = _clean
The Other Jimmy 31:8ea194f6145b 276 self.opts_parser = _parser
The Other Jimmy 31:8ea194f6145b 277 self.opts = _opts
screamer 0:66f3b5499f7f 278 self.opts_auto_detect = _opts_auto_detect
screamer 0:66f3b5499f7f 279 self.opts_include_non_automated = _opts_include_non_automated
screamer 0:66f3b5499f7f 280
screamer 0:66f3b5499f7f 281 self.build_report = _opts_build_report
screamer 0:66f3b5499f7f 282 self.build_properties = _opts_build_properties
screamer 0:66f3b5499f7f 283
screamer 0:66f3b5499f7f 284 # File / screen logger initialization
screamer 0:66f3b5499f7f 285 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer 0:66f3b5499f7f 286
screamer 0:66f3b5499f7f 287 # Database related initializations
screamer 0:66f3b5499f7f 288 self.db_logger = factory_db_logger(self.opts_db_url)
screamer 0:66f3b5499f7f 289 self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer 0:66f3b5499f7f 290 # Let's connect to database to set up credentials and confirm database is ready
screamer 0:66f3b5499f7f 291 if self.db_logger:
screamer 0:66f3b5499f7f 292 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer 0:66f3b5499f7f 293 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 294 # Get hostname and uname so we can use it as build description
screamer 0:66f3b5499f7f 295 # when creating new build_id in external database
screamer 0:66f3b5499f7f 296 (_hostname, _uname) = self.db_logger.get_hostname()
screamer 0:66f3b5499f7f 297 _host_location = os.path.dirname(os.path.abspath(__file__))
screamer 0:66f3b5499f7f 298 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer 0:66f3b5499f7f 299 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer 0:66f3b5499f7f 300 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 301
screamer 0:66f3b5499f7f 302 def dump_options(self):
screamer 0:66f3b5499f7f 303 """ Function returns data structure with common settings passed to SingelTestRunner
screamer 0:66f3b5499f7f 304 It can be used for example to fill _extra fields in database storing test suite single run data
screamer 0:66f3b5499f7f 305 Example:
screamer 0:66f3b5499f7f 306 data = self.dump_options()
screamer 0:66f3b5499f7f 307 or
screamer 0:66f3b5499f7f 308 data_str = json.dumps(self.dump_options())
screamer 0:66f3b5499f7f 309 """
screamer 0:66f3b5499f7f 310 result = {"db_url" : str(self.opts_db_url),
screamer 0:66f3b5499f7f 311 "log_file_name" : str(self.opts_log_file_name),
screamer 0:66f3b5499f7f 312 "shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer 0:66f3b5499f7f 313 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer 0:66f3b5499f7f 314 "test_by_names" : str(self.opts_test_by_names),
screamer 0:66f3b5499f7f 315 "peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer 0:66f3b5499f7f 316 "test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer 0:66f3b5499f7f 317 "test_only_common" : str(self.opts_test_only_common),
screamer 0:66f3b5499f7f 318 "verbose" : str(self.opts_verbose),
screamer 0:66f3b5499f7f 319 "firmware_global_name" : str(self.opts_firmware_global_name),
screamer 0:66f3b5499f7f 320 "only_build_tests" : str(self.opts_only_build_tests),
screamer 0:66f3b5499f7f 321 "copy_method" : str(self.opts_copy_method),
screamer 0:66f3b5499f7f 322 "mut_reset_type" : str(self.opts_mut_reset_type),
screamer 0:66f3b5499f7f 323 "jobs" : str(self.opts_jobs),
screamer 0:66f3b5499f7f 324 "extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer 0:66f3b5499f7f 325 "_dummy" : ''
screamer 0:66f3b5499f7f 326 }
screamer 0:66f3b5499f7f 327 return result
screamer 0:66f3b5499f7f 328
screamer 0:66f3b5499f7f 329 def shuffle_random_func(self):
screamer 0:66f3b5499f7f 330 return self.shuffle_random_seed
screamer 0:66f3b5499f7f 331
screamer 0:66f3b5499f7f 332 def is_shuffle_seed_float(self):
screamer 0:66f3b5499f7f 333 """ return true if function parameter can be converted to float
screamer 0:66f3b5499f7f 334 """
screamer 0:66f3b5499f7f 335 result = True
screamer 0:66f3b5499f7f 336 try:
screamer 0:66f3b5499f7f 337 float(self.shuffle_random_seed)
screamer 0:66f3b5499f7f 338 except ValueError:
screamer 0:66f3b5499f7f 339 result = False
screamer 0:66f3b5499f7f 340 return result
screamer 0:66f3b5499f7f 341
screamer 0:66f3b5499f7f 342 # This will store target / toolchain specific properties
screamer 0:66f3b5499f7f 343 test_suite_properties_ext = {} # target : toolchain
screamer 0:66f3b5499f7f 344 # Here we store test results
screamer 0:66f3b5499f7f 345 test_summary = []
screamer 0:66f3b5499f7f 346 # Here we store test results in extended data structure
screamer 0:66f3b5499f7f 347 test_summary_ext = {}
screamer 0:66f3b5499f7f 348 execute_thread_slice_lock = Lock()
screamer 0:66f3b5499f7f 349
screamer 0:66f3b5499f7f 350 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer 0:66f3b5499f7f 351 for toolchain in toolchains:
screamer 0:66f3b5499f7f 352 tt_id = "%s::%s" % (toolchain, target)
screamer 0:66f3b5499f7f 353
screamer 0:66f3b5499f7f 354 T = TARGET_MAP[target]
screamer 0:66f3b5499f7f 355
screamer 0:66f3b5499f7f 356 # print target, toolchain
screamer 0:66f3b5499f7f 357 # Test suite properties returned to external tools like CI
screamer 0:66f3b5499f7f 358 test_suite_properties = {
screamer 0:66f3b5499f7f 359 'jobs': self.opts_jobs,
screamer 0:66f3b5499f7f 360 'clean': clean,
screamer 0:66f3b5499f7f 361 'target': target,
screamer 0:66f3b5499f7f 362 'vendor': T.extra_labels[0],
screamer 0:66f3b5499f7f 363 'test_ids': ', '.join(test_ids),
screamer 0:66f3b5499f7f 364 'toolchain': toolchain,
screamer 0:66f3b5499f7f 365 'shuffle_random_seed': self.shuffle_random_seed
screamer 0:66f3b5499f7f 366 }
screamer 0:66f3b5499f7f 367
screamer 0:66f3b5499f7f 368
screamer 0:66f3b5499f7f 369 # print '=== %s::%s ===' % (target, toolchain)
screamer 0:66f3b5499f7f 370 # Let's build our test
screamer 0:66f3b5499f7f 371 if target not in TARGET_MAP:
theotherjimmy 43:2a7da56ebd24 372 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 373 self.logger.LogType.NOTIF,
theotherjimmy 43:2a7da56ebd24 374 'Skipped tests for %s target. Target platform not found' %
theotherjimmy 43:2a7da56ebd24 375 (target)))
screamer 0:66f3b5499f7f 376 continue
screamer 0:66f3b5499f7f 377
theotherjimmy 43:2a7da56ebd24 378 clean_mbed_libs_options = (self.opts_goanna_for_mbed_sdk or
theotherjimmy 43:2a7da56ebd24 379 self.opts_clean or clean)
screamer 0:66f3b5499f7f 380
The Other Jimmy 31:8ea194f6145b 381 profile = extract_profile(self.opts_parser, self.opts, toolchain)
theotherjimmy 40:7d3fa6b99b2b 382 stats_depth = self.opts.stats_depth or 2
The Other Jimmy 31:8ea194f6145b 383
screamer 0:66f3b5499f7f 384 try:
theotherjimmy 43:2a7da56ebd24 385 build_mbed_libs_result = build_mbed_libs(
theotherjimmy 43:2a7da56ebd24 386 T, toolchain,
theotherjimmy 43:2a7da56ebd24 387 clean=clean_mbed_libs_options,
theotherjimmy 43:2a7da56ebd24 388 jobs=self.opts_jobs,
theotherjimmy 43:2a7da56ebd24 389 report=build_report,
theotherjimmy 43:2a7da56ebd24 390 properties=build_properties,
theotherjimmy 43:2a7da56ebd24 391 build_profile=profile,
theotherjimmy 43:2a7da56ebd24 392 notify=TerminalNotifier())
screamer 0:66f3b5499f7f 393
screamer 0:66f3b5499f7f 394 if not build_mbed_libs_result:
theotherjimmy 43:2a7da56ebd24 395 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 396 self.logger.LogType.NOTIF,
theotherjimmy 43:2a7da56ebd24 397 'Skipped tests for %s target. Toolchain %s is not '
theotherjimmy 43:2a7da56ebd24 398 'supported for this target'% (T.name, toolchain)))
screamer 0:66f3b5499f7f 399 continue
screamer 0:66f3b5499f7f 400
screamer 0:66f3b5499f7f 401 except ToolException:
theotherjimmy 43:2a7da56ebd24 402 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 403 self.logger.LogType.ERROR,
theotherjimmy 43:2a7da56ebd24 404 'There were errors while building MBED libs for %s using %s'
theotherjimmy 43:2a7da56ebd24 405 % (target, toolchain)))
screamer 0:66f3b5499f7f 406 continue
screamer 0:66f3b5499f7f 407
screamer 0:66f3b5499f7f 408 build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer 0:66f3b5499f7f 409
screamer 0:66f3b5499f7f 410 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer 0:66f3b5499f7f 411 test_suite_properties['build_dir'] = build_dir
screamer 0:66f3b5499f7f 412 test_suite_properties['skipped'] = []
screamer 0:66f3b5499f7f 413
screamer 0:66f3b5499f7f 414 # Enumerate through all tests and shuffle test order if requested
screamer 0:66f3b5499f7f 415 test_map_keys = sorted(TEST_MAP.keys())
screamer 0:66f3b5499f7f 416
screamer 0:66f3b5499f7f 417 if self.opts_shuffle_test_order:
screamer 0:66f3b5499f7f 418 random.shuffle(test_map_keys, self.shuffle_random_func)
screamer 0:66f3b5499f7f 419 # Update database with shuffle seed f applicable
screamer 0:66f3b5499f7f 420 if self.db_logger:
screamer 0:66f3b5499f7f 421 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 422 if self.db_logger.is_connected():
theotherjimmy 43:2a7da56ebd24 423 self.db_logger.update_build_id_info(
theotherjimmy 43:2a7da56ebd24 424 self.db_logger_build_id,
theotherjimmy 43:2a7da56ebd24 425 _shuffle_seed=self.shuffle_random_func())
screamer 0:66f3b5499f7f 426 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 427
screamer 0:66f3b5499f7f 428 if self.db_logger:
screamer 0:66f3b5499f7f 429 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 430 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 431 # Update MUTs and Test Specification in database
theotherjimmy 43:2a7da56ebd24 432 self.db_logger.update_build_id_info(
theotherjimmy 43:2a7da56ebd24 433 self.db_logger_build_id,
theotherjimmy 43:2a7da56ebd24 434 _muts=self.muts, _test_spec=self.test_spec)
screamer 0:66f3b5499f7f 435 # Update Extra information in database (some options passed to test suite)
theotherjimmy 43:2a7da56ebd24 436 self.db_logger.update_build_id_info(
theotherjimmy 43:2a7da56ebd24 437 self.db_logger_build_id,
theotherjimmy 43:2a7da56ebd24 438 _extra=json.dumps(self.dump_options()))
screamer 0:66f3b5499f7f 439 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 440
screamer 0:66f3b5499f7f 441 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer 0:66f3b5499f7f 442 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer 0:66f3b5499f7f 443
screamer 0:66f3b5499f7f 444 for skipped_test_id in skipped_test_map_keys:
screamer 0:66f3b5499f7f 445 test_suite_properties['skipped'].append(skipped_test_id)
screamer 0:66f3b5499f7f 446
screamer 0:66f3b5499f7f 447
screamer 0:66f3b5499f7f 448 # First pass through all tests and determine which libraries need to be built
screamer 0:66f3b5499f7f 449 libraries = []
screamer 0:66f3b5499f7f 450 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 451 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 452
screamer 0:66f3b5499f7f 453 # Detect which lib should be added to test
screamer 0:66f3b5499f7f 454 # Some libs have to compiled like RTOS or ETH
screamer 0:66f3b5499f7f 455 for lib in LIBRARIES:
screamer 0:66f3b5499f7f 456 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer 0:66f3b5499f7f 457 libraries.append(lib['id'])
screamer 0:66f3b5499f7f 458
screamer 0:66f3b5499f7f 459
screamer 0:66f3b5499f7f 460 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer 0:66f3b5499f7f 461
screamer 0:66f3b5499f7f 462 # Build all required libraries
screamer 0:66f3b5499f7f 463 for lib_id in libraries:
screamer 0:66f3b5499f7f 464 try:
screamer 0:66f3b5499f7f 465 build_lib(lib_id,
screamer 0:66f3b5499f7f 466 T,
screamer 0:66f3b5499f7f 467 toolchain,
screamer 0:66f3b5499f7f 468 clean=clean_mbed_libs_options,
screamer 0:66f3b5499f7f 469 jobs=self.opts_jobs,
screamer 0:66f3b5499f7f 470 report=build_report,
The Other Jimmy 31:8ea194f6145b 471 properties=build_properties,
theotherjimmy 43:2a7da56ebd24 472 build_profile=profile,
theotherjimmy 43:2a7da56ebd24 473 notify=TerminalNotifier())
screamer 0:66f3b5499f7f 474
screamer 0:66f3b5499f7f 475 except ToolException:
theotherjimmy 43:2a7da56ebd24 476 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 477 self.logger.LogType.ERROR,
theotherjimmy 43:2a7da56ebd24 478 'There were errors while building library %s' % lib_id))
screamer 0:66f3b5499f7f 479 continue
screamer 0:66f3b5499f7f 480
screamer 0:66f3b5499f7f 481
screamer 0:66f3b5499f7f 482 for test_id in valid_test_map_keys:
screamer 0:66f3b5499f7f 483 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 484
screamer 0:66f3b5499f7f 485 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer 0:66f3b5499f7f 486
screamer 0:66f3b5499f7f 487 # TODO: move this 2 below loops to separate function
screamer 0:66f3b5499f7f 488 INC_DIRS = []
screamer 0:66f3b5499f7f 489 for lib_id in libraries:
screamer 0:66f3b5499f7f 490 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer 0:66f3b5499f7f 491 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer 0:66f3b5499f7f 492
screamer 0:66f3b5499f7f 493 MACROS = []
screamer 0:66f3b5499f7f 494 for lib_id in libraries:
screamer 0:66f3b5499f7f 495 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer 0:66f3b5499f7f 496 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer 0:66f3b5499f7f 497 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer 0:66f3b5499f7f 498 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer 0:66f3b5499f7f 499 test_uuid = uuid.uuid4()
screamer 0:66f3b5499f7f 500 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer 0:66f3b5499f7f 501
screamer 0:66f3b5499f7f 502 # Prepare extended test results data structure (it can be used to generate detailed test report)
screamer 0:66f3b5499f7f 503 if target not in self.test_summary_ext:
screamer 0:66f3b5499f7f 504 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer 0:66f3b5499f7f 505 if toolchain not in self.test_summary_ext[target]:
screamer 0:66f3b5499f7f 506 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer 0:66f3b5499f7f 507
screamer 0:66f3b5499f7f 508 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer 0:66f3b5499f7f 509
screamer 0:66f3b5499f7f 510 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer 0:66f3b5499f7f 511 try:
theotherjimmy 43:2a7da56ebd24 512 path = build_project(
theotherjimmy 43:2a7da56ebd24 513 test.source_dir, join(build_dir, test_id), T,
theotherjimmy 40:7d3fa6b99b2b 514 toolchain, test.dependencies, clean=clean_project_options,
theotherjimmy 43:2a7da56ebd24 515 name=project_name, macros=MACROS,
theotherjimmy 40:7d3fa6b99b2b 516 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
theotherjimmy 40:7d3fa6b99b2b 517 properties=build_properties, project_id=test_id,
theotherjimmy 40:7d3fa6b99b2b 518 project_description=test.get_description(),
theotherjimmy 43:2a7da56ebd24 519 build_profile=profile, stats_depth=stats_depth,
theotherjimmy 43:2a7da56ebd24 520 notify=TerminalNotifier(),
theotherjimmy 43:2a7da56ebd24 521 )
screamer 0:66f3b5499f7f 522
theotherjimmy 43:2a7da56ebd24 523 except Exception as e:
screamer 0:66f3b5499f7f 524 project_name_str = project_name if project_name is not None else test_id
screamer 0:66f3b5499f7f 525
screamer 0:66f3b5499f7f 526
screamer 0:66f3b5499f7f 527 test_result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 528
screamer 0:66f3b5499f7f 529 if isinstance(e, ToolException):
theotherjimmy 43:2a7da56ebd24 530 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 531 self.logger.LogType.ERROR,
theotherjimmy 43:2a7da56ebd24 532 'There were errors while building project %s' %
theotherjimmy 43:2a7da56ebd24 533 project_name_str))
screamer 0:66f3b5499f7f 534 test_result = self.TEST_RESULT_BUILD_FAILED
screamer 0:66f3b5499f7f 535 elif isinstance(e, NotSupportedException):
theotherjimmy 43:2a7da56ebd24 536 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 537 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 538 'Project %s is not supported' % project_name_str))
screamer 0:66f3b5499f7f 539 test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer 0:66f3b5499f7f 540
screamer 0:66f3b5499f7f 541
screamer 0:66f3b5499f7f 542 # Append test results to global test summary
screamer 0:66f3b5499f7f 543 self.test_summary.append(
theotherjimmy 43:2a7da56ebd24 544 (test_result, target, toolchain, test_id,
theotherjimmy 43:2a7da56ebd24 545 test.get_description(), 0, 0, '-')
screamer 0:66f3b5499f7f 546 )
screamer 0:66f3b5499f7f 547
screamer 0:66f3b5499f7f 548 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 549 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 550 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 551
screamer 0:66f3b5499f7f 552 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer 0:66f3b5499f7f 553 'result' : test_result,
screamer 0:66f3b5499f7f 554 'output' : '',
screamer 0:66f3b5499f7f 555 'target_name' : target,
screamer 0:66f3b5499f7f 556 'target_name_unique': target,
screamer 0:66f3b5499f7f 557 'toolchain_name' : toolchain,
screamer 0:66f3b5499f7f 558 'id' : test_id,
screamer 0:66f3b5499f7f 559 'description' : test.get_description(),
screamer 0:66f3b5499f7f 560 'elapsed_time' : 0,
screamer 0:66f3b5499f7f 561 'duration' : 0,
screamer 0:66f3b5499f7f 562 'copy_method' : None
screamer 0:66f3b5499f7f 563 }})
screamer 0:66f3b5499f7f 564 continue
screamer 0:66f3b5499f7f 565
screamer 0:66f3b5499f7f 566 if self.opts_only_build_tests:
screamer 0:66f3b5499f7f 567 # With this option we are skipping testing phase
screamer 0:66f3b5499f7f 568 continue
screamer 0:66f3b5499f7f 569
screamer 0:66f3b5499f7f 570 # Test duration can be increased by global value
screamer 0:66f3b5499f7f 571 test_duration = test.duration
screamer 0:66f3b5499f7f 572 if self.opts_extend_test_timeout is not None:
screamer 0:66f3b5499f7f 573 test_duration += self.opts_extend_test_timeout
screamer 0:66f3b5499f7f 574
screamer 0:66f3b5499f7f 575 # For an automated test the duration act as a timeout after
screamer 0:66f3b5499f7f 576 # which the test gets interrupted
screamer 0:66f3b5499f7f 577 test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer 0:66f3b5499f7f 578 test_loops = self.get_test_loop_count(test_id)
screamer 0:66f3b5499f7f 579
screamer 0:66f3b5499f7f 580 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer 0:66f3b5499f7f 581 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer 0:66f3b5499f7f 582 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer 0:66f3b5499f7f 583
screamer 0:66f3b5499f7f 584 # read MUTs, test specification and perform tests
screamer 0:66f3b5499f7f 585 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer 0:66f3b5499f7f 586
screamer 0:66f3b5499f7f 587 if handle_results is None:
screamer 0:66f3b5499f7f 588 continue
screamer 0:66f3b5499f7f 589
screamer 0:66f3b5499f7f 590 for handle_result in handle_results:
screamer 0:66f3b5499f7f 591 if handle_result:
screamer 0:66f3b5499f7f 592 single_test_result, detailed_test_results = handle_result
screamer 0:66f3b5499f7f 593 else:
screamer 0:66f3b5499f7f 594 continue
screamer 0:66f3b5499f7f 595
screamer 0:66f3b5499f7f 596 # Append test results to global test summary
screamer 0:66f3b5499f7f 597 if single_test_result is not None:
screamer 0:66f3b5499f7f 598 self.test_summary.append(single_test_result)
screamer 0:66f3b5499f7f 599
screamer 0:66f3b5499f7f 600 # Add detailed test result to test summary structure
screamer 0:66f3b5499f7f 601 if target not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 602 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:66f3b5499f7f 603 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:66f3b5499f7f 604
screamer 0:66f3b5499f7f 605 append_test_result = detailed_test_results
screamer 0:66f3b5499f7f 606
screamer 0:66f3b5499f7f 607 # If waterfall and consolidate-waterfall options are enabled,
screamer 0:66f3b5499f7f 608 # only include the last test result in the report.
screamer 0:66f3b5499f7f 609 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer 0:66f3b5499f7f 610 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer 0:66f3b5499f7f 611
screamer 0:66f3b5499f7f 612 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer 0:66f3b5499f7f 613
screamer 0:66f3b5499f7f 614 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer 0:66f3b5499f7f 615 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer 0:66f3b5499f7f 616
screamer 0:66f3b5499f7f 617 q.put(target + '_'.join(toolchains))
screamer 0:66f3b5499f7f 618 return
screamer 0:66f3b5499f7f 619
screamer 0:66f3b5499f7f 620 def execute(self):
screamer 0:66f3b5499f7f 621 clean = self.test_spec.get('clean', False)
screamer 0:66f3b5499f7f 622 test_ids = self.test_spec.get('test_ids', [])
screamer 0:66f3b5499f7f 623 q = Queue()
screamer 0:66f3b5499f7f 624
screamer 0:66f3b5499f7f 625 # Generate seed for shuffle if seed is not provided in
screamer 0:66f3b5499f7f 626 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 627 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer 0:66f3b5499f7f 628 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer 0:66f3b5499f7f 629
screamer 0:66f3b5499f7f 630
screamer 0:66f3b5499f7f 631 if self.opts_parallel_test_exec:
screamer 0:66f3b5499f7f 632 ###################################################################
screamer 0:66f3b5499f7f 633 # Experimental, parallel test execution per singletest instance.
screamer 0:66f3b5499f7f 634 ###################################################################
screamer 0:66f3b5499f7f 635 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer 0:66f3b5499f7f 636 # Note: We are building here in parallel for each target separately!
screamer 0:66f3b5499f7f 637 # So we are not building the same thing multiple times and compilers
screamer 0:66f3b5499f7f 638 # in separate threads do not collide.
screamer 0:66f3b5499f7f 639 # Inside execute_thread_slice() function function handle() will be called to
screamer 0:66f3b5499f7f 640 # get information about available MUTs (per target).
theotherjimmy 43:2a7da56ebd24 641 for target, toolchains in self.test_spec['targets'].items():
screamer 0:66f3b5499f7f 642 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 643 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer 0:66f3b5499f7f 644 t.daemon = True
screamer 0:66f3b5499f7f 645 t.start()
screamer 0:66f3b5499f7f 646 execute_threads.append(t)
screamer 0:66f3b5499f7f 647
screamer 0:66f3b5499f7f 648 for t in execute_threads:
screamer 0:66f3b5499f7f 649 q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer 0:66f3b5499f7f 650 else:
screamer 0:66f3b5499f7f 651 # Serialized (not parallel) test execution
theotherjimmy 43:2a7da56ebd24 652 for target, toolchains in self.test_spec['targets'].items():
screamer 0:66f3b5499f7f 653 if target not in self.test_suite_properties_ext:
screamer 0:66f3b5499f7f 654 self.test_suite_properties_ext[target] = {}
screamer 0:66f3b5499f7f 655
screamer 0:66f3b5499f7f 656 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer 0:66f3b5499f7f 657 q.get()
screamer 0:66f3b5499f7f 658
screamer 0:66f3b5499f7f 659 if self.db_logger:
screamer 0:66f3b5499f7f 660 self.db_logger.reconnect();
screamer 0:66f3b5499f7f 661 if self.db_logger.is_connected():
screamer 0:66f3b5499f7f 662 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer 0:66f3b5499f7f 663 self.db_logger.disconnect();
screamer 0:66f3b5499f7f 664
screamer 0:66f3b5499f7f 665 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer 0:66f3b5499f7f 666
screamer 0:66f3b5499f7f 667 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer 0:66f3b5499f7f 668 valid_test_map_keys = []
screamer 0:66f3b5499f7f 669
screamer 0:66f3b5499f7f 670 for test_id in test_map_keys:
screamer 0:66f3b5499f7f 671 test = TEST_MAP[test_id]
screamer 22:9e85236d8716 672 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
screamer 0:66f3b5499f7f 673 continue
screamer 0:66f3b5499f7f 674
screamer 0:66f3b5499f7f 675 if test_ids and test_id not in test_ids:
screamer 0:66f3b5499f7f 676 continue
screamer 0:66f3b5499f7f 677
screamer 0:66f3b5499f7f 678 if self.opts_test_only_peripheral and not test.peripherals:
screamer 0:66f3b5499f7f 679 if self.opts_verbose_skipped_tests:
theotherjimmy 43:2a7da56ebd24 680 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 681 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 682 'Common test skipped for target %s' % target))
screamer 0:66f3b5499f7f 683 continue
screamer 0:66f3b5499f7f 684
theotherjimmy 43:2a7da56ebd24 685 if (self.opts_peripheral_by_names and test.peripherals and
theotherjimmy 43:2a7da56ebd24 686 not any((i in self.opts_peripheral_by_names)
theotherjimmy 43:2a7da56ebd24 687 for i in test.peripherals)):
screamer 0:66f3b5499f7f 688 # We will skip tests not forced with -p option
screamer 0:66f3b5499f7f 689 if self.opts_verbose_skipped_tests:
theotherjimmy 43:2a7da56ebd24 690 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 691 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 692 'Common test skipped for target %s' % target))
screamer 0:66f3b5499f7f 693 continue
screamer 0:66f3b5499f7f 694
screamer 0:66f3b5499f7f 695 if self.opts_test_only_common and test.peripherals:
screamer 0:66f3b5499f7f 696 if self.opts_verbose_skipped_tests:
theotherjimmy 43:2a7da56ebd24 697 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 698 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 699 'Peripheral test skipped for target %s' % target))
screamer 0:66f3b5499f7f 700 continue
screamer 0:66f3b5499f7f 701
screamer 0:66f3b5499f7f 702 if not include_non_automated and not test.automated:
screamer 0:66f3b5499f7f 703 if self.opts_verbose_skipped_tests:
theotherjimmy 43:2a7da56ebd24 704 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 705 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 706 'Non automated test skipped for target %s' % target))
screamer 0:66f3b5499f7f 707 continue
screamer 0:66f3b5499f7f 708
screamer 0:66f3b5499f7f 709 if test.is_supported(target, toolchain):
screamer 0:66f3b5499f7f 710 if test.peripherals is None and self.opts_only_build_tests:
screamer 0:66f3b5499f7f 711 # When users are using 'build only flag' and test do not have
screamer 0:66f3b5499f7f 712 # specified peripherals we can allow test building by default
screamer 0:66f3b5499f7f 713 pass
screamer 22:9e85236d8716 714 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
screamer 0:66f3b5499f7f 715 # If we force peripheral with option -p we expect test
screamer 0:66f3b5499f7f 716 # to pass even if peripheral is not in MUTs file.
screamer 0:66f3b5499f7f 717 pass
screamer 0:66f3b5499f7f 718 elif not self.is_peripherals_available(target, test.peripherals):
screamer 0:66f3b5499f7f 719 if self.opts_verbose_skipped_tests:
screamer 0:66f3b5499f7f 720 if test.peripherals:
theotherjimmy 43:2a7da56ebd24 721 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 722 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 723 'Peripheral %s test skipped for target %s' %
theotherjimmy 43:2a7da56ebd24 724 (",".join(test.peripherals), target)))
screamer 0:66f3b5499f7f 725 else:
theotherjimmy 43:2a7da56ebd24 726 print(self.logger.log_line(
theotherjimmy 43:2a7da56ebd24 727 self.logger.LogType.INFO,
theotherjimmy 43:2a7da56ebd24 728 'Test %s skipped for target %s' %
theotherjimmy 43:2a7da56ebd24 729 (test_id, target)))
screamer 0:66f3b5499f7f 730 continue
screamer 0:66f3b5499f7f 731
screamer 0:66f3b5499f7f 732 # The test has made it through all the filters, so add it to the valid tests list
screamer 0:66f3b5499f7f 733 valid_test_map_keys.append(test_id)
screamer 0:66f3b5499f7f 734
screamer 0:66f3b5499f7f 735 return valid_test_map_keys
screamer 0:66f3b5499f7f 736
screamer 0:66f3b5499f7f 737 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer 0:66f3b5499f7f 738 # NOTE: This will not preserve order
screamer 0:66f3b5499f7f 739 return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer 0:66f3b5499f7f 740
screamer 0:66f3b5499f7f 741 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 742 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 743 table shows text x toolchain test result matrix
screamer 0:66f3b5499f7f 744 """
screamer 0:66f3b5499f7f 745 RESULT_INDEX = 0
screamer 0:66f3b5499f7f 746 TARGET_INDEX = 1
screamer 0:66f3b5499f7f 747 TOOLCHAIN_INDEX = 2
screamer 0:66f3b5499f7f 748 TEST_INDEX = 3
screamer 0:66f3b5499f7f 749 DESC_INDEX = 4
screamer 0:66f3b5499f7f 750
screamer 0:66f3b5499f7f 751 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer 0:66f3b5499f7f 752 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer 0:66f3b5499f7f 753 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer 0:66f3b5499f7f 754 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer 0:66f3b5499f7f 755
screamer 0:66f3b5499f7f 756 result = "Test summary:\n"
screamer 0:66f3b5499f7f 757 for target in unique_targets:
screamer 0:66f3b5499f7f 758 result_dict = {} # test : { toolchain : result }
screamer 0:66f3b5499f7f 759 unique_target_toolchains = []
screamer 0:66f3b5499f7f 760 for test in test_summary:
screamer 0:66f3b5499f7f 761 if test[TARGET_INDEX] == target:
screamer 0:66f3b5499f7f 762 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer 0:66f3b5499f7f 763 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer 0:66f3b5499f7f 764 if test[TEST_INDEX] not in result_dict:
screamer 0:66f3b5499f7f 765 result_dict[test[TEST_INDEX]] = {}
screamer 0:66f3b5499f7f 766 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer 0:66f3b5499f7f 767
screamer 0:66f3b5499f7f 768 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
theotherjimmy 43:2a7da56ebd24 769 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 770 for col in pt_cols:
screamer 0:66f3b5499f7f 771 pt.align[col] = "l"
screamer 0:66f3b5499f7f 772 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 773
screamer 0:66f3b5499f7f 774 for test in unique_tests:
screamer 0:66f3b5499f7f 775 if test in result_dict:
screamer 0:66f3b5499f7f 776 test_results = result_dict[test]
screamer 0:66f3b5499f7f 777 if test in unique_test_desc:
screamer 0:66f3b5499f7f 778 row = [target, test, unique_test_desc[test]]
screamer 0:66f3b5499f7f 779 for toolchain in unique_toolchains:
screamer 0:66f3b5499f7f 780 if toolchain in test_results:
screamer 0:66f3b5499f7f 781 row.append(test_results[toolchain])
screamer 0:66f3b5499f7f 782 pt.add_row(row)
screamer 0:66f3b5499f7f 783 result += pt.get_string()
screamer 0:66f3b5499f7f 784 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 785 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 786 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 787 return result
screamer 0:66f3b5499f7f 788
screamer 0:66f3b5499f7f 789 def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer 0:66f3b5499f7f 790 """ Prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 791 table shows target x test results matrix across
screamer 0:66f3b5499f7f 792 """
screamer 0:66f3b5499f7f 793 success_code = 0 # Success code that can be leter returned to
screamer 0:66f3b5499f7f 794 result = "Test summary:\n"
screamer 0:66f3b5499f7f 795 # Pretty table package is used to print results
screamer 0:66f3b5499f7f 796 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
theotherjimmy 43:2a7da56ebd24 797 "Elapsed Time (sec)", "Timeout (sec)", "Loops"], junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 798 pt.align["Result"] = "l" # Left align
screamer 0:66f3b5499f7f 799 pt.align["Target"] = "l" # Left align
screamer 0:66f3b5499f7f 800 pt.align["Toolchain"] = "l" # Left align
screamer 0:66f3b5499f7f 801 pt.align["Test ID"] = "l" # Left align
screamer 0:66f3b5499f7f 802 pt.align["Test Description"] = "l" # Left align
screamer 0:66f3b5499f7f 803 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 804
screamer 0:66f3b5499f7f 805 result_dict = {self.TEST_RESULT_OK : 0,
screamer 0:66f3b5499f7f 806 self.TEST_RESULT_FAIL : 0,
screamer 0:66f3b5499f7f 807 self.TEST_RESULT_ERROR : 0,
screamer 0:66f3b5499f7f 808 self.TEST_RESULT_UNDEF : 0,
screamer 0:66f3b5499f7f 809 self.TEST_RESULT_IOERR_COPY : 0,
screamer 0:66f3b5499f7f 810 self.TEST_RESULT_IOERR_DISK : 0,
screamer 0:66f3b5499f7f 811 self.TEST_RESULT_IOERR_SERIAL : 0,
screamer 0:66f3b5499f7f 812 self.TEST_RESULT_NO_IMAGE : 0,
screamer 0:66f3b5499f7f 813 self.TEST_RESULT_TIMEOUT : 0,
screamer 0:66f3b5499f7f 814 self.TEST_RESULT_MBED_ASSERT : 0,
screamer 0:66f3b5499f7f 815 self.TEST_RESULT_BUILD_FAILED : 0,
screamer 0:66f3b5499f7f 816 self.TEST_RESULT_NOT_SUPPORTED : 0
screamer 0:66f3b5499f7f 817 }
screamer 0:66f3b5499f7f 818
screamer 0:66f3b5499f7f 819 for test in test_summary:
screamer 0:66f3b5499f7f 820 if test[0] in result_dict:
screamer 0:66f3b5499f7f 821 result_dict[test[0]] += 1
screamer 0:66f3b5499f7f 822 pt.add_row(test)
screamer 0:66f3b5499f7f 823 result += pt.get_string()
screamer 0:66f3b5499f7f 824 result += "\n"
screamer 0:66f3b5499f7f 825
screamer 0:66f3b5499f7f 826 # Print result count
theotherjimmy 43:2a7da56ebd24 827 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()])
screamer 0:66f3b5499f7f 828 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer 0:66f3b5499f7f 829 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:66f3b5499f7f 830 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:66f3b5499f7f 831 return result
screamer 0:66f3b5499f7f 832
screamer 0:66f3b5499f7f 833 def test_loop_list_to_dict(self, test_loops_str):
screamer 0:66f3b5499f7f 834 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer 0:66f3b5499f7f 835 """
screamer 0:66f3b5499f7f 836 result = {}
screamer 0:66f3b5499f7f 837 if test_loops_str:
screamer 22:9e85236d8716 838 test_loops = test_loops_str
screamer 0:66f3b5499f7f 839 for test_loop in test_loops:
screamer 0:66f3b5499f7f 840 test_loop_count = test_loop.split('=')
screamer 0:66f3b5499f7f 841 if len(test_loop_count) == 2:
screamer 0:66f3b5499f7f 842 _test_id, _test_loops = test_loop_count
screamer 0:66f3b5499f7f 843 try:
screamer 0:66f3b5499f7f 844 _test_loops = int(_test_loops)
screamer 0:66f3b5499f7f 845 except:
screamer 0:66f3b5499f7f 846 continue
screamer 0:66f3b5499f7f 847 result[_test_id] = _test_loops
screamer 0:66f3b5499f7f 848 return result
screamer 0:66f3b5499f7f 849
screamer 0:66f3b5499f7f 850 def get_test_loop_count(self, test_id):
screamer 0:66f3b5499f7f 851 """ This function returns no. of loops per test (deducted by test_id_.
screamer 0:66f3b5499f7f 852 If test is not in list of redefined loop counts it will use default value.
screamer 0:66f3b5499f7f 853 """
screamer 0:66f3b5499f7f 854 result = self.GLOBAL_LOOPS_COUNT
screamer 0:66f3b5499f7f 855 if test_id in self.TEST_LOOPS_DICT:
screamer 0:66f3b5499f7f 856 result = self.TEST_LOOPS_DICT[test_id]
screamer 0:66f3b5499f7f 857 return result
screamer 0:66f3b5499f7f 858
screamer 0:66f3b5499f7f 859 def delete_file(self, file_path):
screamer 0:66f3b5499f7f 860 """ Remove file from the system
screamer 0:66f3b5499f7f 861 """
screamer 0:66f3b5499f7f 862 result = True
screamer 0:66f3b5499f7f 863 resutl_msg = ""
screamer 0:66f3b5499f7f 864 try:
screamer 0:66f3b5499f7f 865 os.remove(file_path)
theotherjimmy 43:2a7da56ebd24 866 except Exception as e:
screamer 0:66f3b5499f7f 867 resutl_msg = e
screamer 0:66f3b5499f7f 868 result = False
screamer 0:66f3b5499f7f 869 return result, resutl_msg
screamer 0:66f3b5499f7f 870
screamer 0:66f3b5499f7f 871 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 872 """ Test is being invoked for given MUT.
screamer 0:66f3b5499f7f 873 """
screamer 0:66f3b5499f7f 874 # Get test information, image and test timeout
screamer 0:66f3b5499f7f 875 test_id = data['test_id']
screamer 0:66f3b5499f7f 876 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 877 test_description = TEST_MAP[test_id].get_description()
screamer 0:66f3b5499f7f 878 image = data["image"]
screamer 0:66f3b5499f7f 879 duration = data.get("duration", 10)
screamer 0:66f3b5499f7f 880
screamer 0:66f3b5499f7f 881 if mut is None:
theotherjimmy 43:2a7da56ebd24 882 print("Error: No Mbed available: MUT[%s]" % data['mcu'])
screamer 0:66f3b5499f7f 883 return None
screamer 0:66f3b5499f7f 884
screamer 0:66f3b5499f7f 885 mcu = mut['mcu']
screamer 0:66f3b5499f7f 886 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer 0:66f3b5499f7f 887
screamer 0:66f3b5499f7f 888 if self.db_logger:
screamer 0:66f3b5499f7f 889 self.db_logger.reconnect()
screamer 0:66f3b5499f7f 890
screamer 0:66f3b5499f7f 891 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer 0:66f3b5499f7f 892
screamer 0:66f3b5499f7f 893 # Tests can be looped so test results must be stored for the same test
screamer 0:66f3b5499f7f 894 test_all_result = []
screamer 0:66f3b5499f7f 895 # Test results for one test ran few times
screamer 0:66f3b5499f7f 896 detailed_test_results = {} # { Loop_number: { results ... } }
screamer 0:66f3b5499f7f 897
screamer 0:66f3b5499f7f 898 for test_index in range(test_loops):
screamer 0:66f3b5499f7f 899
screamer 0:66f3b5499f7f 900 # If mbedls is available and we are auto detecting MUT info,
screamer 0:66f3b5499f7f 901 # update MUT info (mounting may changed)
screamer 0:66f3b5499f7f 902 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 903 platform_name_filter = [mcu]
screamer 0:66f3b5499f7f 904 muts_list = {}
screamer 0:66f3b5499f7f 905 found = False
screamer 0:66f3b5499f7f 906
screamer 0:66f3b5499f7f 907 for i in range(0, 60):
screamer 0:66f3b5499f7f 908 print('Looking for %s with MBEDLS' % mcu)
screamer 0:66f3b5499f7f 909 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 910
screamer 0:66f3b5499f7f 911 if 1 not in muts_list:
screamer 0:66f3b5499f7f 912 sleep(3)
screamer 0:66f3b5499f7f 913 else:
screamer 0:66f3b5499f7f 914 found = True
screamer 0:66f3b5499f7f 915 break
screamer 0:66f3b5499f7f 916
screamer 0:66f3b5499f7f 917 if not found:
theotherjimmy 43:2a7da56ebd24 918 print("Error: mbed not found with MBEDLS: %s" % data['mcu'])
screamer 0:66f3b5499f7f 919 return None
screamer 0:66f3b5499f7f 920 else:
screamer 0:66f3b5499f7f 921 mut = muts_list[1]
screamer 0:66f3b5499f7f 922
screamer 0:66f3b5499f7f 923 disk = mut.get('disk')
screamer 0:66f3b5499f7f 924 port = mut.get('port')
screamer 0:66f3b5499f7f 925
screamer 0:66f3b5499f7f 926 if disk is None or port is None:
screamer 0:66f3b5499f7f 927 return None
screamer 0:66f3b5499f7f 928
screamer 0:66f3b5499f7f 929 target_by_mcu = TARGET_MAP[mut['mcu']]
screamer 0:66f3b5499f7f 930 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer 0:66f3b5499f7f 931 # Some extra stuff can be declared in MUTs structure
screamer 0:66f3b5499f7f 932 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer 0:66f3b5499f7f 933 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer 0:66f3b5499f7f 934
screamer 0:66f3b5499f7f 935 # When the build and test system were separate, this was relative to a
screamer 0:66f3b5499f7f 936 # base network folder base path: join(NETWORK_BASE_PATH, )
screamer 0:66f3b5499f7f 937 image_path = image
screamer 0:66f3b5499f7f 938
screamer 0:66f3b5499f7f 939 # Host test execution
screamer 0:66f3b5499f7f 940 start_host_exec_time = time()
screamer 0:66f3b5499f7f 941
screamer 0:66f3b5499f7f 942 single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer 0:66f3b5499f7f 943 _copy_method = selected_copy_method
screamer 0:66f3b5499f7f 944
screamer 0:66f3b5499f7f 945 if not exists(image_path):
screamer 0:66f3b5499f7f 946 single_test_result = self.TEST_RESULT_NO_IMAGE
screamer 0:66f3b5499f7f 947 elapsed_time = 0
screamer 0:66f3b5499f7f 948 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
theotherjimmy 43:2a7da56ebd24 949 print(single_test_output)
screamer 0:66f3b5499f7f 950 else:
screamer 0:66f3b5499f7f 951 # Host test execution
screamer 0:66f3b5499f7f 952 start_host_exec_time = time()
screamer 0:66f3b5499f7f 953
screamer 0:66f3b5499f7f 954 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer 0:66f3b5499f7f 955 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer 0:66f3b5499f7f 956 host_test_result = self.run_host_test(test.host_test,
screamer 0:66f3b5499f7f 957 image_path, disk, port, duration,
screamer 0:66f3b5499f7f 958 micro=target_name,
screamer 0:66f3b5499f7f 959 verbose=host_test_verbose,
screamer 0:66f3b5499f7f 960 reset=host_test_reset,
screamer 0:66f3b5499f7f 961 reset_tout=reset_tout,
screamer 0:66f3b5499f7f 962 copy_method=selected_copy_method,
screamer 13:ab47a20b66f0 963 program_cycle_s=target_by_mcu.program_cycle_s)
screamer 0:66f3b5499f7f 964 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer 0:66f3b5499f7f 965
screamer 0:66f3b5499f7f 966 # Store test result
screamer 0:66f3b5499f7f 967 test_all_result.append(single_test_result)
screamer 0:66f3b5499f7f 968 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer 0:66f3b5499f7f 969 elapsed_time = single_testduration # TIme of single test case execution after reset
screamer 0:66f3b5499f7f 970
screamer 0:66f3b5499f7f 971 detailed_test_results[test_index] = {
screamer 0:66f3b5499f7f 972 'result' : single_test_result,
screamer 0:66f3b5499f7f 973 'output' : single_test_output,
screamer 0:66f3b5499f7f 974 'target_name' : target_name,
screamer 0:66f3b5499f7f 975 'target_name_unique' : target_name_unique,
screamer 0:66f3b5499f7f 976 'toolchain_name' : toolchain_name,
screamer 0:66f3b5499f7f 977 'id' : test_id,
screamer 0:66f3b5499f7f 978 'description' : test_description,
screamer 0:66f3b5499f7f 979 'elapsed_time' : round(elapsed_time, 2),
screamer 0:66f3b5499f7f 980 'duration' : single_timeout,
screamer 0:66f3b5499f7f 981 'copy_method' : _copy_method,
screamer 0:66f3b5499f7f 982 }
screamer 0:66f3b5499f7f 983
theotherjimmy 43:2a7da56ebd24 984 print(self.print_test_result(
theotherjimmy 43:2a7da56ebd24 985 single_test_result, target_name_unique, toolchain_name, test_id,
theotherjimmy 43:2a7da56ebd24 986 test_description, elapsed_time, single_timeout))
screamer 0:66f3b5499f7f 987
screamer 0:66f3b5499f7f 988 # Update database entries for ongoing test
screamer 0:66f3b5499f7f 989 if self.db_logger and self.db_logger.is_connected():
screamer 0:66f3b5499f7f 990 test_type = 'SingleTest'
screamer 0:66f3b5499f7f 991 self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer 0:66f3b5499f7f 992 target_name,
screamer 0:66f3b5499f7f 993 toolchain_name,
screamer 0:66f3b5499f7f 994 test_type,
screamer 0:66f3b5499f7f 995 test_id,
screamer 0:66f3b5499f7f 996 single_test_result,
screamer 0:66f3b5499f7f 997 single_test_output,
screamer 0:66f3b5499f7f 998 elapsed_time,
screamer 0:66f3b5499f7f 999 single_timeout,
screamer 0:66f3b5499f7f 1000 test_index)
screamer 0:66f3b5499f7f 1001
screamer 0:66f3b5499f7f 1002 # If we perform waterfall test we test until we get OK and we stop testing
screamer 0:66f3b5499f7f 1003 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer 0:66f3b5499f7f 1004 break
screamer 0:66f3b5499f7f 1005
screamer 0:66f3b5499f7f 1006 if self.db_logger:
screamer 0:66f3b5499f7f 1007 self.db_logger.disconnect()
screamer 0:66f3b5499f7f 1008
screamer 0:66f3b5499f7f 1009 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer 0:66f3b5499f7f 1010 target_name_unique,
screamer 0:66f3b5499f7f 1011 toolchain_name,
screamer 0:66f3b5499f7f 1012 test_id,
screamer 0:66f3b5499f7f 1013 test_description,
screamer 0:66f3b5499f7f 1014 round(elapsed_time, 2),
screamer 0:66f3b5499f7f 1015 single_timeout,
screamer 0:66f3b5499f7f 1016 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer 0:66f3b5499f7f 1017
screamer 0:66f3b5499f7f 1018 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer 0:66f3b5499f7f 1019 """ Function determines MUT's mbed disk/port and copies binary to
screamer 0:66f3b5499f7f 1020 target.
screamer 0:66f3b5499f7f 1021 """
screamer 0:66f3b5499f7f 1022 handle_results = []
screamer 0:66f3b5499f7f 1023 data = json.loads(test_spec)
screamer 0:66f3b5499f7f 1024
screamer 0:66f3b5499f7f 1025 # Find a suitable MUT:
screamer 0:66f3b5499f7f 1026 mut = None
theotherjimmy 43:2a7da56ebd24 1027 for id, m in self.muts.items():
screamer 0:66f3b5499f7f 1028 if m['mcu'] == data['mcu']:
screamer 0:66f3b5499f7f 1029 mut = m
screamer 0:66f3b5499f7f 1030 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer 0:66f3b5499f7f 1031 handle_results.append(handle_result)
screamer 0:66f3b5499f7f 1032
screamer 0:66f3b5499f7f 1033 return handle_results
screamer 0:66f3b5499f7f 1034
screamer 0:66f3b5499f7f 1035 def print_test_result(self, test_result, target_name, toolchain_name,
screamer 0:66f3b5499f7f 1036 test_id, test_description, elapsed_time, duration):
screamer 0:66f3b5499f7f 1037 """ Use specific convention to print test result and related data
screamer 0:66f3b5499f7f 1038 """
screamer 0:66f3b5499f7f 1039 tokens = []
screamer 0:66f3b5499f7f 1040 tokens.append("TargetTest")
screamer 0:66f3b5499f7f 1041 tokens.append(target_name)
screamer 0:66f3b5499f7f 1042 tokens.append(toolchain_name)
screamer 0:66f3b5499f7f 1043 tokens.append(test_id)
screamer 0:66f3b5499f7f 1044 tokens.append(test_description)
screamer 0:66f3b5499f7f 1045 separator = "::"
screamer 0:66f3b5499f7f 1046 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer 0:66f3b5499f7f 1047 result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer 0:66f3b5499f7f 1048 return Fore.MAGENTA + result + Fore.RESET
screamer 0:66f3b5499f7f 1049
screamer 0:66f3b5499f7f 1050 def shape_test_loop_ok_result_count(self, test_all_result):
screamer 0:66f3b5499f7f 1051 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1052 """
screamer 0:66f3b5499f7f 1053 test_loop_count = len(test_all_result)
screamer 0:66f3b5499f7f 1054 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer 0:66f3b5499f7f 1055 return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer 0:66f3b5499f7f 1056
screamer 0:66f3b5499f7f 1057 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer 0:66f3b5499f7f 1058 """ Reformats list of results to simple string
screamer 0:66f3b5499f7f 1059 """
screamer 0:66f3b5499f7f 1060 result = self.TEST_RESULT_FAIL
screamer 0:66f3b5499f7f 1061
screamer 0:66f3b5499f7f 1062 if all(test_all_result[0] == res for res in test_all_result):
screamer 0:66f3b5499f7f 1063 result = test_all_result[0]
screamer 0:66f3b5499f7f 1064 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer 0:66f3b5499f7f 1065 result = self.TEST_RESULT_OK
screamer 0:66f3b5499f7f 1066
screamer 0:66f3b5499f7f 1067 return result
screamer 0:66f3b5499f7f 1068
screamer 0:66f3b5499f7f 1069 def run_host_test(self, name, image_path, disk, port, duration,
screamer 0:66f3b5499f7f 1070 micro=None, reset=None, reset_tout=None,
screamer 0:66f3b5499f7f 1071 verbose=False, copy_method=None, program_cycle_s=None):
screamer 0:66f3b5499f7f 1072 """ Function creates new process with host test configured with particular test case.
screamer 0:66f3b5499f7f 1073 Function also is pooling for serial port activity from process to catch all data
screamer 0:66f3b5499f7f 1074 printed by test runner and host test during test execution
screamer 0:66f3b5499f7f 1075 """
screamer 0:66f3b5499f7f 1076
screamer 0:66f3b5499f7f 1077 def get_char_from_queue(obs):
screamer 0:66f3b5499f7f 1078 """ Get character from queue safe way
screamer 0:66f3b5499f7f 1079 """
screamer 0:66f3b5499f7f 1080 try:
screamer 0:66f3b5499f7f 1081 c = obs.queue.get(block=True, timeout=0.5)
theotherjimmy 43:2a7da56ebd24 1082 except Empty:
screamer 0:66f3b5499f7f 1083 c = None
screamer 0:66f3b5499f7f 1084 return c
screamer 0:66f3b5499f7f 1085
screamer 0:66f3b5499f7f 1086 def filter_queue_char(c):
screamer 0:66f3b5499f7f 1087 """ Filters out non ASCII characters from serial port
screamer 0:66f3b5499f7f 1088 """
screamer 0:66f3b5499f7f 1089 if ord(c) not in range(128):
screamer 0:66f3b5499f7f 1090 c = ' '
screamer 0:66f3b5499f7f 1091 return c
screamer 0:66f3b5499f7f 1092
screamer 0:66f3b5499f7f 1093 def get_test_result(output):
screamer 0:66f3b5499f7f 1094 """ Parse test 'output' data
screamer 0:66f3b5499f7f 1095 """
screamer 0:66f3b5499f7f 1096 result = self.TEST_RESULT_TIMEOUT
screamer 0:66f3b5499f7f 1097 for line in "".join(output).splitlines():
screamer 0:66f3b5499f7f 1098 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer 0:66f3b5499f7f 1099 if search_result and len(search_result.groups()):
screamer 0:66f3b5499f7f 1100 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer 0:66f3b5499f7f 1101 break
screamer 0:66f3b5499f7f 1102 return result
screamer 0:66f3b5499f7f 1103
screamer 0:66f3b5499f7f 1104 def get_auto_property_value(property_name, line):
screamer 0:66f3b5499f7f 1105 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer 0:66f3b5499f7f 1106 Returns string
screamer 0:66f3b5499f7f 1107 """
screamer 0:66f3b5499f7f 1108 result = None
screamer 0:66f3b5499f7f 1109 if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer 0:66f3b5499f7f 1110 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer 0:66f3b5499f7f 1111 if property is not None and len(property.groups()) == 1:
screamer 0:66f3b5499f7f 1112 result = property.groups()[0]
screamer 0:66f3b5499f7f 1113 return result
screamer 0:66f3b5499f7f 1114
screamer 0:66f3b5499f7f 1115 cmd = ["python",
screamer 0:66f3b5499f7f 1116 '%s.py'% name,
screamer 0:66f3b5499f7f 1117 '-d', disk,
screamer 0:66f3b5499f7f 1118 '-f', '"%s"'% image_path,
screamer 0:66f3b5499f7f 1119 '-p', port,
screamer 0:66f3b5499f7f 1120 '-t', str(duration),
screamer 0:66f3b5499f7f 1121 '-C', str(program_cycle_s)]
screamer 0:66f3b5499f7f 1122
screamer 0:66f3b5499f7f 1123 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:66f3b5499f7f 1124 cmd += ['--auto']
screamer 0:66f3b5499f7f 1125
screamer 0:66f3b5499f7f 1126 # Add extra parameters to host_test
screamer 0:66f3b5499f7f 1127 if copy_method is not None:
screamer 0:66f3b5499f7f 1128 cmd += ["-c", copy_method]
screamer 0:66f3b5499f7f 1129 if micro is not None:
screamer 0:66f3b5499f7f 1130 cmd += ["-m", micro]
screamer 0:66f3b5499f7f 1131 if reset is not None:
screamer 0:66f3b5499f7f 1132 cmd += ["-r", reset]
screamer 0:66f3b5499f7f 1133 if reset_tout is not None:
screamer 0:66f3b5499f7f 1134 cmd += ["-R", str(reset_tout)]
screamer 0:66f3b5499f7f 1135
screamer 0:66f3b5499f7f 1136 if verbose:
theotherjimmy 43:2a7da56ebd24 1137 print(Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET)
theotherjimmy 43:2a7da56ebd24 1138 print("Test::Output::Start")
screamer 0:66f3b5499f7f 1139
screamer 0:66f3b5499f7f 1140 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer 0:66f3b5499f7f 1141 obs = ProcessObserver(proc)
screamer 0:66f3b5499f7f 1142 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer 0:66f3b5499f7f 1143 line = ''
screamer 0:66f3b5499f7f 1144 output = []
screamer 0:66f3b5499f7f 1145 start_time = time()
screamer 0:66f3b5499f7f 1146 while (time() - start_time) < (2 * duration):
screamer 0:66f3b5499f7f 1147 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1148 if c:
screamer 0:66f3b5499f7f 1149 if verbose:
screamer 0:66f3b5499f7f 1150 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1151 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1152 output.append(c)
screamer 0:66f3b5499f7f 1153 # Give the mbed under test a way to communicate the end of the test
screamer 0:66f3b5499f7f 1154 if c in ['\n', '\r']:
screamer 0:66f3b5499f7f 1155
screamer 0:66f3b5499f7f 1156 # Checking for auto-detection information from the test about MUT reset moment
screamer 0:66f3b5499f7f 1157 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer 0:66f3b5499f7f 1158 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1159 update_once_flag['reset_target'] = True
screamer 0:66f3b5499f7f 1160 start_time = time()
screamer 0:66f3b5499f7f 1161
screamer 0:66f3b5499f7f 1162 # Checking for auto-detection information from the test about timeout
screamer 0:66f3b5499f7f 1163 auto_timeout_val = get_auto_property_value('timeout', line)
screamer 0:66f3b5499f7f 1164 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer 0:66f3b5499f7f 1165 # We will update this marker only once to prevent multiple time resets
screamer 0:66f3b5499f7f 1166 update_once_flag['timeout'] = True
screamer 0:66f3b5499f7f 1167 duration = int(auto_timeout_val)
screamer 0:66f3b5499f7f 1168
screamer 0:66f3b5499f7f 1169 # Detect mbed assert:
screamer 0:66f3b5499f7f 1170 if 'mbed assertation failed: ' in line:
screamer 0:66f3b5499f7f 1171 output.append('{{mbed_assert}}')
screamer 0:66f3b5499f7f 1172 break
screamer 0:66f3b5499f7f 1173
screamer 0:66f3b5499f7f 1174 # Check for test end
screamer 0:66f3b5499f7f 1175 if '{end}' in line:
screamer 0:66f3b5499f7f 1176 break
screamer 0:66f3b5499f7f 1177 line = ''
screamer 0:66f3b5499f7f 1178 else:
screamer 0:66f3b5499f7f 1179 line += c
screamer 0:66f3b5499f7f 1180 end_time = time()
screamer 0:66f3b5499f7f 1181 testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer 0:66f3b5499f7f 1182
screamer 0:66f3b5499f7f 1183 c = get_char_from_queue(obs)
screamer 0:66f3b5499f7f 1184
screamer 0:66f3b5499f7f 1185 if c:
screamer 0:66f3b5499f7f 1186 if verbose:
screamer 0:66f3b5499f7f 1187 sys.stdout.write(c)
screamer 0:66f3b5499f7f 1188 c = filter_queue_char(c)
screamer 0:66f3b5499f7f 1189 output.append(c)
screamer 0:66f3b5499f7f 1190
screamer 0:66f3b5499f7f 1191 if verbose:
theotherjimmy 43:2a7da56ebd24 1192 print("Test::Output::Finish")
screamer 0:66f3b5499f7f 1193 # Stop test process
screamer 0:66f3b5499f7f 1194 obs.stop()
screamer 0:66f3b5499f7f 1195
screamer 0:66f3b5499f7f 1196 result = get_test_result(output)
screamer 0:66f3b5499f7f 1197 return (result, "".join(output), testcase_duration, duration)
screamer 0:66f3b5499f7f 1198
screamer 0:66f3b5499f7f 1199 def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer 0:66f3b5499f7f 1200 """ Checks if specified target should run specific peripheral test case defined in MUTs file
screamer 0:66f3b5499f7f 1201 """
screamer 0:66f3b5499f7f 1202 if peripherals is not None:
screamer 0:66f3b5499f7f 1203 peripherals = set(peripherals)
theotherjimmy 43:2a7da56ebd24 1204 for id, mut in self.muts.items():
screamer 0:66f3b5499f7f 1205 # Target MCU name check
screamer 0:66f3b5499f7f 1206 if mut["mcu"] != target_mcu_name:
screamer 0:66f3b5499f7f 1207 continue
screamer 0:66f3b5499f7f 1208 # Peripherals check
screamer 0:66f3b5499f7f 1209 if peripherals is not None:
screamer 0:66f3b5499f7f 1210 if 'peripherals' not in mut:
screamer 0:66f3b5499f7f 1211 continue
screamer 0:66f3b5499f7f 1212 if not peripherals.issubset(set(mut['peripherals'])):
screamer 0:66f3b5499f7f 1213 continue
screamer 0:66f3b5499f7f 1214 return True
screamer 0:66f3b5499f7f 1215 return False
screamer 0:66f3b5499f7f 1216
screamer 0:66f3b5499f7f 1217 def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer 0:66f3b5499f7f 1218 """ Function prepares JSON structure describing test specification
screamer 0:66f3b5499f7f 1219 """
screamer 0:66f3b5499f7f 1220 test_spec = {
screamer 0:66f3b5499f7f 1221 "mcu": mcu,
screamer 0:66f3b5499f7f 1222 "image": image_path,
screamer 0:66f3b5499f7f 1223 "duration": duration,
screamer 0:66f3b5499f7f 1224 "test_id": test_id,
screamer 0:66f3b5499f7f 1225 }
screamer 0:66f3b5499f7f 1226 return json.dumps(test_spec)
screamer 0:66f3b5499f7f 1227
screamer 0:66f3b5499f7f 1228
screamer 0:66f3b5499f7f 1229 def get_unique_value_from_summary(test_summary, index):
screamer 0:66f3b5499f7f 1230 """ Gets list of unique target names
screamer 0:66f3b5499f7f 1231 """
screamer 0:66f3b5499f7f 1232 result = []
screamer 0:66f3b5499f7f 1233 for test in test_summary:
screamer 0:66f3b5499f7f 1234 target_name = test[index]
screamer 0:66f3b5499f7f 1235 if target_name not in result:
screamer 0:66f3b5499f7f 1236 result.append(target_name)
screamer 0:66f3b5499f7f 1237 return sorted(result)
screamer 0:66f3b5499f7f 1238
screamer 0:66f3b5499f7f 1239
screamer 0:66f3b5499f7f 1240 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer 0:66f3b5499f7f 1241 """ Gets list of unique target names and return dictionary
screamer 0:66f3b5499f7f 1242 """
screamer 0:66f3b5499f7f 1243 result = {}
screamer 0:66f3b5499f7f 1244 for test in test_summary:
screamer 0:66f3b5499f7f 1245 key = test[index_key]
screamer 0:66f3b5499f7f 1246 val = test[index_val]
screamer 0:66f3b5499f7f 1247 if key not in result:
screamer 0:66f3b5499f7f 1248 result[key] = val
screamer 0:66f3b5499f7f 1249 return result
screamer 0:66f3b5499f7f 1250
screamer 0:66f3b5499f7f 1251
screamer 0:66f3b5499f7f 1252 def show_json_file_format_error(json_spec_filename, line, column):
screamer 0:66f3b5499f7f 1253 """ Prints JSON broken content
screamer 0:66f3b5499f7f 1254 """
screamer 0:66f3b5499f7f 1255 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1256 line_no = 1
screamer 0:66f3b5499f7f 1257 for json_line in data_file:
screamer 0:66f3b5499f7f 1258 if line_no + 5 >= line: # Print last few lines before error
theotherjimmy 43:2a7da56ebd24 1259 print('Line %d:\t'%line_no + json_line)
screamer 0:66f3b5499f7f 1260 if line_no == line:
theotherjimmy 43:2a7da56ebd24 1261 print('%s\t%s^' (' ' * len('Line %d:' % line_no),
theotherjimmy 43:2a7da56ebd24 1262 '-' * (column - 1)))
screamer 0:66f3b5499f7f 1263 break
screamer 0:66f3b5499f7f 1264 line_no += 1
screamer 0:66f3b5499f7f 1265
screamer 0:66f3b5499f7f 1266
screamer 0:66f3b5499f7f 1267 def json_format_error_defect_pos(json_error_msg):
screamer 0:66f3b5499f7f 1268 """ Gets first error line and column in JSON file format.
screamer 0:66f3b5499f7f 1269 Parsed from exception thrown by json.loads() string
screamer 0:66f3b5499f7f 1270 """
screamer 0:66f3b5499f7f 1271 result = None
screamer 0:66f3b5499f7f 1272 line, column = 0, 0
screamer 0:66f3b5499f7f 1273 # Line value search
screamer 0:66f3b5499f7f 1274 line_search = re.search('line [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1275 if line_search is not None:
screamer 0:66f3b5499f7f 1276 ls = line_search.group().split(' ')
screamer 0:66f3b5499f7f 1277 if len(ls) == 2:
screamer 0:66f3b5499f7f 1278 line = int(ls[1])
screamer 0:66f3b5499f7f 1279 # Column position search
screamer 0:66f3b5499f7f 1280 column_search = re.search('column [0-9]+', json_error_msg)
screamer 0:66f3b5499f7f 1281 if column_search is not None:
screamer 0:66f3b5499f7f 1282 cs = column_search.group().split(' ')
screamer 0:66f3b5499f7f 1283 if len(cs) == 2:
screamer 0:66f3b5499f7f 1284 column = int(cs[1])
screamer 0:66f3b5499f7f 1285 result = [line, column]
screamer 0:66f3b5499f7f 1286 return result
screamer 0:66f3b5499f7f 1287
screamer 0:66f3b5499f7f 1288
screamer 0:66f3b5499f7f 1289 def get_json_data_from_file(json_spec_filename, verbose=False):
screamer 0:66f3b5499f7f 1290 """ Loads from file JSON formatted string to data structure
screamer 0:66f3b5499f7f 1291 """
screamer 0:66f3b5499f7f 1292 result = None
screamer 0:66f3b5499f7f 1293 try:
screamer 0:66f3b5499f7f 1294 with open(json_spec_filename) as data_file:
screamer 0:66f3b5499f7f 1295 try:
screamer 0:66f3b5499f7f 1296 result = json.load(data_file)
screamer 0:66f3b5499f7f 1297 except ValueError as json_error_msg:
screamer 0:66f3b5499f7f 1298 result = None
theotherjimmy 43:2a7da56ebd24 1299 print('JSON file %s parsing failed. Reason: %s' %
theotherjimmy 43:2a7da56ebd24 1300 (json_spec_filename, json_error_msg))
screamer 0:66f3b5499f7f 1301 # We can print where error occurred inside JSON file if we can parse exception msg
screamer 0:66f3b5499f7f 1302 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer 0:66f3b5499f7f 1303 if json_format_defect_pos is not None:
screamer 0:66f3b5499f7f 1304 line = json_format_defect_pos[0]
screamer 0:66f3b5499f7f 1305 column = json_format_defect_pos[1]
theotherjimmy 43:2a7da56ebd24 1306 print()
screamer 0:66f3b5499f7f 1307 show_json_file_format_error(json_spec_filename, line, column)
screamer 0:66f3b5499f7f 1308
screamer 0:66f3b5499f7f 1309 except IOError as fileopen_error_msg:
theotherjimmy 43:2a7da56ebd24 1310 print('JSON file %s not opened. Reason: %s\n'%
theotherjimmy 43:2a7da56ebd24 1311 (json_spec_filename, fileopen_error_msg))
screamer 0:66f3b5499f7f 1312 if verbose and result:
screamer 0:66f3b5499f7f 1313 pp = pprint.PrettyPrinter(indent=4)
screamer 0:66f3b5499f7f 1314 pp.pprint(result)
screamer 0:66f3b5499f7f 1315 return result
screamer 0:66f3b5499f7f 1316
screamer 0:66f3b5499f7f 1317
screamer 0:66f3b5499f7f 1318 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer 0:66f3b5499f7f 1319 """ Prints MUTs configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1320 """
screamer 0:66f3b5499f7f 1321 muts_info_cols = []
screamer 0:66f3b5499f7f 1322 # We need to check all unique properties for each defined MUT
screamer 0:66f3b5499f7f 1323 for k in json_data:
screamer 0:66f3b5499f7f 1324 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1325 for mut_property in mut_info:
screamer 0:66f3b5499f7f 1326 if mut_property not in muts_info_cols:
screamer 0:66f3b5499f7f 1327 muts_info_cols.append(mut_property)
screamer 0:66f3b5499f7f 1328
screamer 0:66f3b5499f7f 1329 # Prepare pretty table object to display all MUTs
screamer 0:66f3b5499f7f 1330 pt_cols = ["index"] + muts_info_cols
theotherjimmy 43:2a7da56ebd24 1331 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 1332 for col in pt_cols:
screamer 0:66f3b5499f7f 1333 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1334
screamer 0:66f3b5499f7f 1335 # Add rows to pretty print object
screamer 0:66f3b5499f7f 1336 for k in json_data:
screamer 0:66f3b5499f7f 1337 row = [k]
screamer 0:66f3b5499f7f 1338 mut_info = json_data[k]
screamer 0:66f3b5499f7f 1339
screamer 0:66f3b5499f7f 1340 add_row = True
screamer 0:66f3b5499f7f 1341 if platform_filter and 'mcu' in mut_info:
screamer 0:66f3b5499f7f 1342 add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer 0:66f3b5499f7f 1343 if add_row:
screamer 0:66f3b5499f7f 1344 for col in muts_info_cols:
screamer 0:66f3b5499f7f 1345 cell_val = mut_info[col] if col in mut_info else None
theotherjimmy 43:2a7da56ebd24 1346 if isinstance(cell_val, list):
screamer 0:66f3b5499f7f 1347 cell_val = join_delim.join(cell_val)
screamer 0:66f3b5499f7f 1348 row.append(cell_val)
screamer 0:66f3b5499f7f 1349 pt.add_row(row)
screamer 0:66f3b5499f7f 1350 return pt.get_string()
screamer 0:66f3b5499f7f 1351
screamer 0:66f3b5499f7f 1352
screamer 0:66f3b5499f7f 1353 def print_test_configuration_from_json(json_data, join_delim=", "):
screamer 0:66f3b5499f7f 1354 """ Prints test specification configuration passed to test script for verboseness
screamer 0:66f3b5499f7f 1355 """
screamer 0:66f3b5499f7f 1356 toolchains_info_cols = []
screamer 0:66f3b5499f7f 1357 # We need to check all toolchains for each device
screamer 0:66f3b5499f7f 1358 for k in json_data:
screamer 0:66f3b5499f7f 1359 # k should be 'targets'
screamer 0:66f3b5499f7f 1360 targets = json_data[k]
screamer 0:66f3b5499f7f 1361 for target in targets:
screamer 0:66f3b5499f7f 1362 toolchains = targets[target]
screamer 0:66f3b5499f7f 1363 for toolchain in toolchains:
screamer 0:66f3b5499f7f 1364 if toolchain not in toolchains_info_cols:
screamer 0:66f3b5499f7f 1365 toolchains_info_cols.append(toolchain)
screamer 0:66f3b5499f7f 1366
screamer 0:66f3b5499f7f 1367 # Prepare pretty table object to display test specification
screamer 0:66f3b5499f7f 1368 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
theotherjimmy 43:2a7da56ebd24 1369 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 1370 for col in pt_cols:
screamer 0:66f3b5499f7f 1371 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1372
screamer 0:66f3b5499f7f 1373 # { target : [conflicted toolchains] }
screamer 0:66f3b5499f7f 1374 toolchain_conflicts = {}
screamer 0:66f3b5499f7f 1375 toolchain_path_conflicts = []
screamer 0:66f3b5499f7f 1376 for k in json_data:
screamer 0:66f3b5499f7f 1377 # k should be 'targets'
screamer 0:66f3b5499f7f 1378 targets = json_data[k]
screamer 0:66f3b5499f7f 1379 for target in targets:
screamer 0:66f3b5499f7f 1380 target_supported_toolchains = get_target_supported_toolchains(target)
screamer 0:66f3b5499f7f 1381 if not target_supported_toolchains:
screamer 0:66f3b5499f7f 1382 target_supported_toolchains = []
screamer 0:66f3b5499f7f 1383 target_name = target if target in TARGET_MAP else "%s*"% target
screamer 0:66f3b5499f7f 1384 row = [target_name]
screamer 0:66f3b5499f7f 1385 toolchains = targets[target]
screamer 0:66f3b5499f7f 1386
screamer 0:66f3b5499f7f 1387 for toolchain in sorted(toolchains_info_cols):
screamer 0:66f3b5499f7f 1388 # Check for conflicts: target vs toolchain
screamer 0:66f3b5499f7f 1389 conflict = False
screamer 0:66f3b5499f7f 1390 conflict_path = False
screamer 0:66f3b5499f7f 1391 if toolchain in toolchains:
screamer 0:66f3b5499f7f 1392 if toolchain not in target_supported_toolchains:
screamer 0:66f3b5499f7f 1393 conflict = True
screamer 0:66f3b5499f7f 1394 if target not in toolchain_conflicts:
screamer 0:66f3b5499f7f 1395 toolchain_conflicts[target] = []
screamer 0:66f3b5499f7f 1396 toolchain_conflicts[target].append(toolchain)
screamer 0:66f3b5499f7f 1397 # Add marker inside table about target usage / conflict
screamer 0:66f3b5499f7f 1398 cell_val = 'Yes' if toolchain in toolchains else '-'
screamer 0:66f3b5499f7f 1399 if conflict:
screamer 0:66f3b5499f7f 1400 cell_val += '*'
screamer 0:66f3b5499f7f 1401 # Check for conflicts: toolchain vs toolchain path
screamer 20:835f6355470d 1402 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1403 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1404 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1405 conflict_path = True
screamer 0:66f3b5499f7f 1406 if toolchain not in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1407 toolchain_path_conflicts.append(toolchain)
screamer 0:66f3b5499f7f 1408 if conflict_path:
screamer 0:66f3b5499f7f 1409 cell_val += '#'
screamer 0:66f3b5499f7f 1410 row.append(cell_val)
screamer 0:66f3b5499f7f 1411 pt.add_row(row)
screamer 0:66f3b5499f7f 1412
screamer 0:66f3b5499f7f 1413 # generate result string
screamer 0:66f3b5499f7f 1414 result = pt.get_string() # Test specification table
screamer 0:66f3b5499f7f 1415 if toolchain_conflicts or toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1416 result += "\n"
screamer 0:66f3b5499f7f 1417 result += "Toolchain conflicts:\n"
screamer 0:66f3b5499f7f 1418 for target in toolchain_conflicts:
screamer 0:66f3b5499f7f 1419 if target not in TARGET_MAP:
screamer 0:66f3b5499f7f 1420 result += "\t* Target %s unknown\n"% (target)
screamer 0:66f3b5499f7f 1421 conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer 0:66f3b5499f7f 1422 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer 0:66f3b5499f7f 1423 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer 0:66f3b5499f7f 1424
screamer 0:66f3b5499f7f 1425 for toolchain in toolchain_path_conflicts:
screamer 0:66f3b5499f7f 1426 # Let's check toolchain configuration
screamer 20:835f6355470d 1427 if toolchain in TOOLCHAIN_PATHS:
screamer 20:835f6355470d 1428 toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer 0:66f3b5499f7f 1429 if not os.path.isdir(toolchain_path):
screamer 0:66f3b5499f7f 1430 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer 0:66f3b5499f7f 1431 return result
screamer 0:66f3b5499f7f 1432
screamer 0:66f3b5499f7f 1433
screamer 0:66f3b5499f7f 1434 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer 0:66f3b5499f7f 1435 """ Generates table summary with all test cases and additional test cases
screamer 0:66f3b5499f7f 1436 information using pretty print functionality. Allows test suite user to
screamer 0:66f3b5499f7f 1437 see test cases
screamer 0:66f3b5499f7f 1438 """
screamer 0:66f3b5499f7f 1439 # get all unique test ID prefixes
screamer 0:66f3b5499f7f 1440 unique_test_id = []
screamer 0:66f3b5499f7f 1441 for test in TESTS:
screamer 0:66f3b5499f7f 1442 split = test['id'].split('_')[:-1]
screamer 0:66f3b5499f7f 1443 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1444 if test_id_prefix not in unique_test_id:
screamer 0:66f3b5499f7f 1445 unique_test_id.append(test_id_prefix)
screamer 0:66f3b5499f7f 1446 unique_test_id.sort()
screamer 0:66f3b5499f7f 1447 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1448 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer 0:66f3b5499f7f 1449
screamer 0:66f3b5499f7f 1450 test_properties = ['id',
screamer 0:66f3b5499f7f 1451 'automated',
screamer 0:66f3b5499f7f 1452 'description',
screamer 0:66f3b5499f7f 1453 'peripherals',
screamer 0:66f3b5499f7f 1454 'host_test',
screamer 0:66f3b5499f7f 1455 'duration'] if cols is None else cols
screamer 0:66f3b5499f7f 1456
screamer 0:66f3b5499f7f 1457 # All tests status table print
theotherjimmy 43:2a7da56ebd24 1458 pt = PrettyTable(test_properties, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 1459 for col in test_properties:
screamer 0:66f3b5499f7f 1460 pt.align[col] = "l"
screamer 0:66f3b5499f7f 1461 pt.align['duration'] = "r"
screamer 0:66f3b5499f7f 1462
screamer 0:66f3b5499f7f 1463 counter_all = 0
screamer 0:66f3b5499f7f 1464 counter_automated = 0
screamer 0:66f3b5499f7f 1465 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:66f3b5499f7f 1466
screamer 0:66f3b5499f7f 1467 for test_id in sorted(TEST_MAP.keys()):
screamer 0:66f3b5499f7f 1468 if platform_filter is not None:
screamer 0:66f3b5499f7f 1469 # FIlter out platforms using regex
screamer 0:66f3b5499f7f 1470 if re.search(platform_filter, test_id) is None:
screamer 0:66f3b5499f7f 1471 continue
screamer 0:66f3b5499f7f 1472 row = []
screamer 0:66f3b5499f7f 1473 test = TEST_MAP[test_id]
screamer 0:66f3b5499f7f 1474 split = test_id.split('_')[:-1]
screamer 0:66f3b5499f7f 1475 test_id_prefix = '_'.join(split)
screamer 0:66f3b5499f7f 1476
screamer 0:66f3b5499f7f 1477 for col in test_properties:
screamer 0:66f3b5499f7f 1478 col_value = test[col]
theotherjimmy 43:2a7da56ebd24 1479 if isinstance(test[col], list):
screamer 0:66f3b5499f7f 1480 col_value = join_delim.join(test[col])
screamer 0:66f3b5499f7f 1481 elif test[col] == None:
screamer 0:66f3b5499f7f 1482 col_value = "-"
screamer 0:66f3b5499f7f 1483
screamer 0:66f3b5499f7f 1484 row.append(col_value)
screamer 0:66f3b5499f7f 1485 if test['automated'] == True:
screamer 0:66f3b5499f7f 1486 counter_dict_test_id_types[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1487 counter_automated += 1
screamer 0:66f3b5499f7f 1488 pt.add_row(row)
screamer 0:66f3b5499f7f 1489 # Update counters
screamer 0:66f3b5499f7f 1490 counter_all += 1
screamer 0:66f3b5499f7f 1491 counter_dict_test_id_types_all[test_id_prefix] += 1
screamer 0:66f3b5499f7f 1492 result = pt.get_string()
screamer 0:66f3b5499f7f 1493 result += "\n\n"
screamer 0:66f3b5499f7f 1494
screamer 0:66f3b5499f7f 1495 if result_summary and not platform_filter:
screamer 0:66f3b5499f7f 1496 # Automation result summary
screamer 0:66f3b5499f7f 1497 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
theotherjimmy 43:2a7da56ebd24 1498 pt = PrettyTable(test_id_cols, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 1499 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1500 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1501 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1502
screamer 0:66f3b5499f7f 1503 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer 0:66f3b5499f7f 1504 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1505 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer 0:66f3b5499f7f 1506 result += "Automation coverage:\n"
screamer 0:66f3b5499f7f 1507 result += pt.get_string()
screamer 0:66f3b5499f7f 1508 result += "\n\n"
screamer 0:66f3b5499f7f 1509
screamer 0:66f3b5499f7f 1510 # Test automation coverage table print
screamer 0:66f3b5499f7f 1511 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
theotherjimmy 43:2a7da56ebd24 1512 pt = PrettyTable(test_id_cols, junction_char="|", hrules=HEADER)
screamer 0:66f3b5499f7f 1513 pt.align['id'] = "l"
screamer 0:66f3b5499f7f 1514 pt.align['automated'] = "r"
screamer 0:66f3b5499f7f 1515 pt.align['all'] = "r"
screamer 0:66f3b5499f7f 1516 pt.align['percent [%]'] = "r"
screamer 0:66f3b5499f7f 1517 for unique_id in unique_test_id:
screamer 0:66f3b5499f7f 1518 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer 0:66f3b5499f7f 1519 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer 0:66f3b5499f7f 1520 str_progress = progress_bar(percent_progress, 75)
screamer 0:66f3b5499f7f 1521 row = [unique_id,
screamer 0:66f3b5499f7f 1522 counter_dict_test_id_types[unique_id],
screamer 0:66f3b5499f7f 1523 counter_dict_test_id_types_all[unique_id],
screamer 0:66f3b5499f7f 1524 percent_progress,
screamer 0:66f3b5499f7f 1525 "[" + str_progress + "]"]
screamer 0:66f3b5499f7f 1526 pt.add_row(row)
screamer 0:66f3b5499f7f 1527 result += "Test automation coverage:\n"
screamer 0:66f3b5499f7f 1528 result += pt.get_string()
screamer 0:66f3b5499f7f 1529 result += "\n\n"
screamer 0:66f3b5499f7f 1530 return result
screamer 0:66f3b5499f7f 1531
screamer 0:66f3b5499f7f 1532
screamer 0:66f3b5499f7f 1533 def progress_bar(percent_progress, saturation=0):
screamer 0:66f3b5499f7f 1534 """ This function creates progress bar with optional simple saturation mark
screamer 0:66f3b5499f7f 1535 """
screamer 0:66f3b5499f7f 1536 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer 0:66f3b5499f7f 1537 str_progress = '#' * step + '.' * int(50 - step)
screamer 0:66f3b5499f7f 1538 c = '!' if str_progress[38] == '.' else '|'
screamer 0:66f3b5499f7f 1539 if saturation > 0:
screamer 0:66f3b5499f7f 1540 saturation = saturation / 2
screamer 0:66f3b5499f7f 1541 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer 0:66f3b5499f7f 1542 return str_progress
screamer 0:66f3b5499f7f 1543
screamer 0:66f3b5499f7f 1544
screamer 0:66f3b5499f7f 1545 def singletest_in_cli_mode(single_test):
screamer 0:66f3b5499f7f 1546 """ Runs SingleTestRunner object in CLI (Command line interface) mode
screamer 0:66f3b5499f7f 1547
screamer 0:66f3b5499f7f 1548 @return returns success code (0 == success) for building and running tests
screamer 0:66f3b5499f7f 1549 """
screamer 0:66f3b5499f7f 1550 start = time()
screamer 0:66f3b5499f7f 1551 # Execute tests depending on options and filter applied
screamer 0:66f3b5499f7f 1552 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer 0:66f3b5499f7f 1553 elapsed_time = time() - start
screamer 0:66f3b5499f7f 1554
screamer 0:66f3b5499f7f 1555 # Human readable summary
screamer 0:66f3b5499f7f 1556 if not single_test.opts_suppress_summary:
screamer 0:66f3b5499f7f 1557 # prints well-formed summary with results (SQL table like)
theotherjimmy 43:2a7da56ebd24 1558 print(single_test.generate_test_summary(test_summary, shuffle_seed))
screamer 0:66f3b5499f7f 1559 if single_test.opts_test_x_toolchain_summary:
screamer 0:66f3b5499f7f 1560 # prints well-formed summary with results (SQL table like)
screamer 0:66f3b5499f7f 1561 # table shows text x toolchain test result matrix
theotherjimmy 43:2a7da56ebd24 1562 print(single_test.generate_test_summary_by_target(test_summary,
theotherjimmy 43:2a7da56ebd24 1563 shuffle_seed))
screamer 0:66f3b5499f7f 1564
theotherjimmy 43:2a7da56ebd24 1565 print("Completed in %.2f sec" % elapsed_time)
screamer 0:66f3b5499f7f 1566 print
screamer 0:66f3b5499f7f 1567 # Write summary of the builds
screamer 0:66f3b5499f7f 1568
screamer 0:66f3b5499f7f 1569 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 0:66f3b5499f7f 1570 status = print_report_exporter.report(build_report)
screamer 0:66f3b5499f7f 1571
screamer 0:66f3b5499f7f 1572 # Store extra reports in files
screamer 0:66f3b5499f7f 1573 if single_test.opts_report_html_file_name:
screamer 0:66f3b5499f7f 1574 # Export results in form of HTML report to separate file
screamer 0:66f3b5499f7f 1575 report_exporter = ReportExporter(ResultExporterType.HTML)
screamer 0:66f3b5499f7f 1576 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1577 if single_test.opts_report_junit_file_name:
screamer 0:66f3b5499f7f 1578 # Export results in form of JUnit XML report to separate file
screamer 0:66f3b5499f7f 1579 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 0:66f3b5499f7f 1580 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer 22:9e85236d8716 1581 if single_test.opts_report_text_file_name:
screamer 22:9e85236d8716 1582 # Export results in form of a text file
screamer 22:9e85236d8716 1583 report_exporter = ReportExporter(ResultExporterType.TEXT)
screamer 22:9e85236d8716 1584 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:66f3b5499f7f 1585 if single_test.opts_report_build_file_name:
screamer 0:66f3b5499f7f 1586 # Export build results as html report to sparate file
screamer 0:66f3b5499f7f 1587 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 0:66f3b5499f7f 1588 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer 0:66f3b5499f7f 1589
screamer 0:66f3b5499f7f 1590 # Returns True if no build failures of the test projects or their dependencies
screamer 0:66f3b5499f7f 1591 return status
screamer 0:66f3b5499f7f 1592
screamer 0:66f3b5499f7f 1593 class TestLogger():
screamer 0:66f3b5499f7f 1594 """ Super-class for logging and printing ongoing events for test suite pass
screamer 0:66f3b5499f7f 1595 """
screamer 0:66f3b5499f7f 1596 def __init__(self, store_log=True):
screamer 0:66f3b5499f7f 1597 """ We can control if logger actually stores log in memory
screamer 0:66f3b5499f7f 1598 or just handled all log entries immediately
screamer 0:66f3b5499f7f 1599 """
screamer 0:66f3b5499f7f 1600 self.log = []
screamer 0:66f3b5499f7f 1601 self.log_to_file = False
screamer 0:66f3b5499f7f 1602 self.log_file_name = None
screamer 0:66f3b5499f7f 1603 self.store_log = store_log
screamer 0:66f3b5499f7f 1604
screamer 0:66f3b5499f7f 1605 self.LogType = construct_enum(INFO='Info',
screamer 0:66f3b5499f7f 1606 WARN='Warning',
screamer 0:66f3b5499f7f 1607 NOTIF='Notification',
screamer 0:66f3b5499f7f 1608 ERROR='Error',
screamer 0:66f3b5499f7f 1609 EXCEPT='Exception')
screamer 0:66f3b5499f7f 1610
screamer 0:66f3b5499f7f 1611 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer 0:66f3b5499f7f 1612 APPEND=2) # Append to existing log file
screamer 0:66f3b5499f7f 1613
screamer 0:66f3b5499f7f 1614 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1615 """ Log one line of text
screamer 0:66f3b5499f7f 1616 """
screamer 0:66f3b5499f7f 1617 log_timestamp = time()
screamer 0:66f3b5499f7f 1618 log_entry = {'log_type' : LogType,
screamer 0:66f3b5499f7f 1619 'log_timestamp' : log_timestamp,
screamer 0:66f3b5499f7f 1620 'log_line' : log_line,
screamer 0:66f3b5499f7f 1621 '_future' : None
screamer 0:66f3b5499f7f 1622 }
screamer 0:66f3b5499f7f 1623 # Store log in memory
screamer 0:66f3b5499f7f 1624 if self.store_log:
screamer 0:66f3b5499f7f 1625 self.log.append(log_entry)
screamer 0:66f3b5499f7f 1626 return log_entry
screamer 0:66f3b5499f7f 1627
screamer 0:66f3b5499f7f 1628
screamer 0:66f3b5499f7f 1629 class CLITestLogger(TestLogger):
screamer 0:66f3b5499f7f 1630 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer 0:66f3b5499f7f 1631 """
screamer 0:66f3b5499f7f 1632 def __init__(self, store_log=True, file_name=None):
screamer 0:66f3b5499f7f 1633 TestLogger.__init__(self)
screamer 0:66f3b5499f7f 1634 self.log_file_name = file_name
screamer 0:66f3b5499f7f 1635 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer 0:66f3b5499f7f 1636 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer 0:66f3b5499f7f 1637
screamer 0:66f3b5499f7f 1638 def log_print(self, log_entry, timestamp=True):
screamer 0:66f3b5499f7f 1639 """ Prints on screen formatted log entry
screamer 0:66f3b5499f7f 1640 """
screamer 0:66f3b5499f7f 1641 ts = log_entry['log_timestamp']
screamer 0:66f3b5499f7f 1642 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer 0:66f3b5499f7f 1643 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer 0:66f3b5499f7f 1644 return timestamp_str + log_line_str
screamer 0:66f3b5499f7f 1645
screamer 0:66f3b5499f7f 1646 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:66f3b5499f7f 1647 """ Logs line, if log file output was specified log line will be appended
screamer 0:66f3b5499f7f 1648 at the end of log file
screamer 0:66f3b5499f7f 1649 """
screamer 0:66f3b5499f7f 1650 log_entry = TestLogger.log_line(self, LogType, log_line)
screamer 0:66f3b5499f7f 1651 log_line_str = self.log_print(log_entry, timestamp)
screamer 0:66f3b5499f7f 1652 if self.log_file_name is not None:
screamer 0:66f3b5499f7f 1653 try:
screamer 0:66f3b5499f7f 1654 with open(self.log_file_name, 'a') as f:
screamer 0:66f3b5499f7f 1655 f.write(log_line_str + line_delim)
screamer 0:66f3b5499f7f 1656 except IOError:
screamer 0:66f3b5499f7f 1657 pass
screamer 0:66f3b5499f7f 1658 return log_line_str
screamer 0:66f3b5499f7f 1659
screamer 0:66f3b5499f7f 1660
screamer 0:66f3b5499f7f 1661 def factory_db_logger(db_url):
screamer 0:66f3b5499f7f 1662 """ Factory database driver depending on database type supplied in database connection string db_url
screamer 0:66f3b5499f7f 1663 """
screamer 0:66f3b5499f7f 1664 if db_url is not None:
screamer 0:66f3b5499f7f 1665 from tools.test_mysql import MySQLDBAccess
screamer 0:66f3b5499f7f 1666 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1667 if connection_info is not None:
screamer 0:66f3b5499f7f 1668 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1669 if db_type == 'mysql':
screamer 0:66f3b5499f7f 1670 return MySQLDBAccess()
screamer 0:66f3b5499f7f 1671 return None
screamer 0:66f3b5499f7f 1672
screamer 0:66f3b5499f7f 1673
screamer 0:66f3b5499f7f 1674 def detect_database_verbose(db_url):
screamer 0:66f3b5499f7f 1675 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer 0:66f3b5499f7f 1676 """
screamer 0:66f3b5499f7f 1677 result = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:66f3b5499f7f 1678 if result is not None:
screamer 0:66f3b5499f7f 1679 # Parsing passed
screamer 0:66f3b5499f7f 1680 (db_type, username, password, host, db_name) = result
screamer 0:66f3b5499f7f 1681 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer 0:66f3b5499f7f 1682 # Let's try to connect
screamer 0:66f3b5499f7f 1683 db_ = factory_db_logger(db_url)
screamer 0:66f3b5499f7f 1684 if db_ is not None:
theotherjimmy 43:2a7da56ebd24 1685 print("Connecting to database '%s'..." % db_url)
screamer 0:66f3b5499f7f 1686 db_.connect(host, username, password, db_name)
screamer 0:66f3b5499f7f 1687 if db_.is_connected():
theotherjimmy 43:2a7da56ebd24 1688 print("ok")
theotherjimmy 43:2a7da56ebd24 1689 print("Detecting database...")
theotherjimmy 43:2a7da56ebd24 1690 print(db_.detect_database(verbose=True))
theotherjimmy 43:2a7da56ebd24 1691 print("Disconnecting...")
screamer 0:66f3b5499f7f 1692 db_.disconnect()
theotherjimmy 43:2a7da56ebd24 1693 print("done")
screamer 0:66f3b5499f7f 1694 else:
theotherjimmy 43:2a7da56ebd24 1695 print("Database type '%s' unknown" % db_type)
screamer 0:66f3b5499f7f 1696 else:
theotherjimmy 43:2a7da56ebd24 1697 print("Parse error: '%s' - DB Url error" % db_url)
screamer 0:66f3b5499f7f 1698
screamer 0:66f3b5499f7f 1699
screamer 0:66f3b5499f7f 1700 def get_module_avail(module_name):
theotherjimmy 41:2a77626a4c21 1701 """ This function returns True if module_name is already imported module
screamer 0:66f3b5499f7f 1702 """
screamer 0:66f3b5499f7f 1703 return module_name in sys.modules.keys()
screamer 0:66f3b5499f7f 1704
screamer 0:66f3b5499f7f 1705 def get_autodetected_MUTS_list(platform_name_filter=None):
screamer 0:66f3b5499f7f 1706 oldError = None
screamer 0:66f3b5499f7f 1707 if os.name == 'nt':
screamer 0:66f3b5499f7f 1708 # Disable Windows error box temporarily
screamer 0:66f3b5499f7f 1709 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer 0:66f3b5499f7f 1710
screamer 0:66f3b5499f7f 1711 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1712 detect_muts_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1713
screamer 0:66f3b5499f7f 1714 if os.name == 'nt':
screamer 0:66f3b5499f7f 1715 ctypes.windll.kernel32.SetErrorMode(oldError)
screamer 0:66f3b5499f7f 1716
screamer 0:66f3b5499f7f 1717 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 1718
screamer 0:66f3b5499f7f 1719 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer 0:66f3b5499f7f 1720 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer 0:66f3b5499f7f 1721 If function fails to auto-detect devices it will return empty dictionary.
screamer 0:66f3b5499f7f 1722
screamer 0:66f3b5499f7f 1723 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1724 mbeds = mbed_lstools.create()
screamer 0:66f3b5499f7f 1725 mbeds_list = mbeds.list_mbeds()
screamer 0:66f3b5499f7f 1726
screamer 0:66f3b5499f7f 1727 @param mbeds_list list of mbeds captured from mbed_lstools
screamer 0:66f3b5499f7f 1728 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer 0:66f3b5499f7f 1729 """
screamer 0:66f3b5499f7f 1730 result = {} # Should be in muts_all.json format
screamer 0:66f3b5499f7f 1731 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer 0:66f3b5499f7f 1732 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer 0:66f3b5499f7f 1733 index = 1
screamer 0:66f3b5499f7f 1734 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1735 # Filter the MUTS if a filter is specified
screamer 0:66f3b5499f7f 1736
screamer 0:66f3b5499f7f 1737 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer 0:66f3b5499f7f 1738 continue
screamer 0:66f3b5499f7f 1739
screamer 0:66f3b5499f7f 1740 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer 0:66f3b5499f7f 1741 # if not we are creating our own unique value (last few chars from platform's target_id).
screamer 0:66f3b5499f7f 1742 m = {'mcu': mut['platform_name'],
screamer 0:66f3b5499f7f 1743 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer 0:66f3b5499f7f 1744 'port': mut['serial_port'],
screamer 0:66f3b5499f7f 1745 'disk': mut['mount_point'],
screamer 0:66f3b5499f7f 1746 'peripherals': [] # No peripheral detection
screamer 0:66f3b5499f7f 1747 }
screamer 0:66f3b5499f7f 1748 if index not in result:
screamer 0:66f3b5499f7f 1749 result[index] = {}
screamer 0:66f3b5499f7f 1750 result[index] = m
screamer 0:66f3b5499f7f 1751 index += 1
screamer 0:66f3b5499f7f 1752 return result
screamer 0:66f3b5499f7f 1753
screamer 0:66f3b5499f7f 1754
screamer 0:66f3b5499f7f 1755 def get_autodetected_TEST_SPEC(mbeds_list,
screamer 0:66f3b5499f7f 1756 use_default_toolchain=True,
screamer 0:66f3b5499f7f 1757 use_supported_toolchains=False,
screamer 0:66f3b5499f7f 1758 toolchain_filter=None,
screamer 0:66f3b5499f7f 1759 platform_name_filter=None):
screamer 0:66f3b5499f7f 1760 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer 0:66f3b5499f7f 1761 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer 0:66f3b5499f7f 1762
screamer 0:66f3b5499f7f 1763 use_default_toolchain - if True add default toolchain to test_spec
screamer 0:66f3b5499f7f 1764 use_supported_toolchains - if True add all supported toolchains to test_spec
screamer 0:66f3b5499f7f 1765 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer 0:66f3b5499f7f 1766 """
screamer 0:66f3b5499f7f 1767 result = {'targets': {} }
screamer 0:66f3b5499f7f 1768
screamer 0:66f3b5499f7f 1769 for mut in mbeds_list:
screamer 0:66f3b5499f7f 1770 mcu = mut['mcu']
screamer 0:66f3b5499f7f 1771 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer 0:66f3b5499f7f 1772 if mcu in TARGET_MAP:
screamer 0:66f3b5499f7f 1773 default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer 0:66f3b5499f7f 1774 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer 0:66f3b5499f7f 1775
screamer 0:66f3b5499f7f 1776 # Decide which toolchains should be added to test specification toolchain pool for each target
screamer 0:66f3b5499f7f 1777 toolchains = []
screamer 0:66f3b5499f7f 1778 if use_default_toolchain:
screamer 0:66f3b5499f7f 1779 toolchains.append(default_toolchain)
screamer 0:66f3b5499f7f 1780 if use_supported_toolchains:
screamer 0:66f3b5499f7f 1781 toolchains += supported_toolchains
screamer 0:66f3b5499f7f 1782 if toolchain_filter is not None:
screamer 0:66f3b5499f7f 1783 all_toolchains = supported_toolchains + [default_toolchain]
screamer 22:9e85236d8716 1784 for toolchain in toolchain_filter:
screamer 0:66f3b5499f7f 1785 if toolchain in all_toolchains:
screamer 0:66f3b5499f7f 1786 toolchains.append(toolchain)
screamer 0:66f3b5499f7f 1787
screamer 0:66f3b5499f7f 1788 result['targets'][mcu] = list(set(toolchains))
screamer 0:66f3b5499f7f 1789 return result
screamer 0:66f3b5499f7f 1790
screamer 0:66f3b5499f7f 1791
screamer 0:66f3b5499f7f 1792 def get_default_test_options_parser():
screamer 0:66f3b5499f7f 1793 """ Get common test script options used by CLI, web services etc.
screamer 0:66f3b5499f7f 1794 """
screamer 22:9e85236d8716 1795 parser = argparse.ArgumentParser()
screamer 22:9e85236d8716 1796 parser.add_argument('-i', '--tests',
screamer 22:9e85236d8716 1797 dest='test_spec_filename',
screamer 22:9e85236d8716 1798 metavar="FILE",
screamer 22:9e85236d8716 1799 type=argparse_filestring_type,
screamer 22:9e85236d8716 1800 help='Points to file with test specification')
screamer 0:66f3b5499f7f 1801
screamer 22:9e85236d8716 1802 parser.add_argument('-M', '--MUTS',
screamer 22:9e85236d8716 1803 dest='muts_spec_filename',
screamer 22:9e85236d8716 1804 metavar="FILE",
screamer 22:9e85236d8716 1805 type=argparse_filestring_type,
screamer 22:9e85236d8716 1806 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer 0:66f3b5499f7f 1807
screamer 22:9e85236d8716 1808 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 1809 dest='jobs',
screamer 22:9e85236d8716 1810 metavar="NUMBER",
screamer 22:9e85236d8716 1811 type=int,
screamer 22:9e85236d8716 1812 help="Define number of compilation jobs. Default value is 1")
screamer 0:66f3b5499f7f 1813
screamer 0:66f3b5499f7f 1814 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 1815 # Additional features available when mbed_lstools is installed on host and imported
screamer 0:66f3b5499f7f 1816 # mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer 22:9e85236d8716 1817 parser.add_argument('--auto',
screamer 22:9e85236d8716 1818 dest='auto_detect',
screamer 22:9e85236d8716 1819 action="store_true",
screamer 22:9e85236d8716 1820 help='Use mbed-ls module to detect all connected mbed devices')
screamer 0:66f3b5499f7f 1821
screamer 22:9e85236d8716 1822 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
screamer 22:9e85236d8716 1823 parser.add_argument('--tc',
screamer 22:9e85236d8716 1824 dest='toolchains_filter',
screamer 22:9e85236d8716 1825 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
screamer 22:9e85236d8716 1826 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer 0:66f3b5499f7f 1827
screamer 0:66f3b5499f7f 1828 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer 22:9e85236d8716 1829 parser.add_argument('--oper',
screamer 22:9e85236d8716 1830 dest='operability_checks',
screamer 22:9e85236d8716 1831 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
screamer 22:9e85236d8716 1832 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer 0:66f3b5499f7f 1833
screamer 22:9e85236d8716 1834 parser.add_argument('--clean',
screamer 22:9e85236d8716 1835 dest='clean',
screamer 22:9e85236d8716 1836 action="store_true",
screamer 22:9e85236d8716 1837 help='Clean the build directory')
screamer 0:66f3b5499f7f 1838
screamer 22:9e85236d8716 1839 parser.add_argument('-P', '--only-peripherals',
screamer 22:9e85236d8716 1840 dest='test_only_peripheral',
screamer 22:9e85236d8716 1841 default=False,
screamer 22:9e85236d8716 1842 action="store_true",
screamer 22:9e85236d8716 1843 help='Test only peripheral declared for MUT and skip common tests')
screamer 0:66f3b5499f7f 1844
The Other Jimmy 31:8ea194f6145b 1845 parser.add_argument("--profile", dest="profile", action="append",
The Other Jimmy 31:8ea194f6145b 1846 type=argparse_filestring_type,
The Other Jimmy 31:8ea194f6145b 1847 default=[])
The Other Jimmy 31:8ea194f6145b 1848
screamer 22:9e85236d8716 1849 parser.add_argument('-C', '--only-commons',
screamer 22:9e85236d8716 1850 dest='test_only_common',
screamer 22:9e85236d8716 1851 default=False,
screamer 22:9e85236d8716 1852 action="store_true",
screamer 22:9e85236d8716 1853 help='Test only board internals. Skip perpherials tests and perform common tests')
screamer 0:66f3b5499f7f 1854
screamer 22:9e85236d8716 1855 parser.add_argument('-n', '--test-by-names',
screamer 22:9e85236d8716 1856 dest='test_by_names',
screamer 22:9e85236d8716 1857 type=argparse_many(str),
screamer 22:9e85236d8716 1858 help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer 0:66f3b5499f7f 1859
screamer 22:9e85236d8716 1860 parser.add_argument('-p', '--peripheral-by-names',
screamer 0:66f3b5499f7f 1861 dest='peripheral_by_names',
screamer 22:9e85236d8716 1862 type=argparse_many(str),
screamer 0:66f3b5499f7f 1863 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer 0:66f3b5499f7f 1864
screamer 0:66f3b5499f7f 1865 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer 0:66f3b5499f7f 1866 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer 0:66f3b5499f7f 1867
screamer 22:9e85236d8716 1868 parser.add_argument('-c', '--copy-method',
screamer 22:9e85236d8716 1869 dest='copy_method',
screamer 22:9e85236d8716 1870 type=argparse_uppercase_type(copy_methods, "flash method"),
screamer 22:9e85236d8716 1871 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer 0:66f3b5499f7f 1872
screamer 0:66f3b5499f7f 1873 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer 0:66f3b5499f7f 1874 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer 0:66f3b5499f7f 1875
screamer 22:9e85236d8716 1876 parser.add_argument('-r', '--reset-type',
screamer 22:9e85236d8716 1877 dest='mut_reset_type',
screamer 22:9e85236d8716 1878 default=None,
screamer 22:9e85236d8716 1879 type=argparse_uppercase_type(reset_methods, "reset method"),
screamer 22:9e85236d8716 1880 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer 0:66f3b5499f7f 1881
screamer 22:9e85236d8716 1882 parser.add_argument('-g', '--goanna-for-tests',
screamer 22:9e85236d8716 1883 dest='goanna_for_tests',
screamer 22:9e85236d8716 1884 action="store_true",
screamer 22:9e85236d8716 1885 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1886
screamer 22:9e85236d8716 1887 parser.add_argument('-G', '--goanna-for-sdk',
screamer 22:9e85236d8716 1888 dest='goanna_for_mbed_sdk',
screamer 22:9e85236d8716 1889 action="store_true",
screamer 22:9e85236d8716 1890 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer 0:66f3b5499f7f 1891
screamer 22:9e85236d8716 1892 parser.add_argument('-s', '--suppress-summary',
screamer 22:9e85236d8716 1893 dest='suppress_summary',
screamer 22:9e85236d8716 1894 default=False,
screamer 22:9e85236d8716 1895 action="store_true",
screamer 22:9e85236d8716 1896 help='Suppresses display of wellformatted table with test results')
screamer 22:9e85236d8716 1897
screamer 22:9e85236d8716 1898 parser.add_argument('-t', '--test-summary',
screamer 22:9e85236d8716 1899 dest='test_x_toolchain_summary',
screamer 22:9e85236d8716 1900 default=False,
screamer 22:9e85236d8716 1901 action="store_true",
screamer 22:9e85236d8716 1902 help='Displays wellformatted table with test x toolchain test result per target')
screamer 0:66f3b5499f7f 1903
screamer 22:9e85236d8716 1904 parser.add_argument('-A', '--test-automation-report',
screamer 22:9e85236d8716 1905 dest='test_automation_report',
screamer 22:9e85236d8716 1906 default=False,
screamer 22:9e85236d8716 1907 action="store_true",
screamer 22:9e85236d8716 1908 help='Prints information about all tests and exits')
screamer 0:66f3b5499f7f 1909
screamer 22:9e85236d8716 1910 parser.add_argument('-R', '--test-case-report',
screamer 22:9e85236d8716 1911 dest='test_case_report',
screamer 22:9e85236d8716 1912 default=False,
screamer 22:9e85236d8716 1913 action="store_true",
screamer 22:9e85236d8716 1914 help='Prints information about all test cases and exits')
screamer 0:66f3b5499f7f 1915
screamer 22:9e85236d8716 1916 parser.add_argument("-S", "--supported-toolchains",
screamer 22:9e85236d8716 1917 action="store_true",
screamer 22:9e85236d8716 1918 dest="supported_toolchains",
screamer 22:9e85236d8716 1919 default=False,
screamer 22:9e85236d8716 1920 help="Displays supported matrix of MCUs and toolchains")
screamer 0:66f3b5499f7f 1921
screamer 22:9e85236d8716 1922 parser.add_argument("-O", "--only-build",
screamer 22:9e85236d8716 1923 action="store_true",
screamer 22:9e85236d8716 1924 dest="only_build_tests",
screamer 22:9e85236d8716 1925 default=False,
screamer 22:9e85236d8716 1926 help="Only build tests, skips actual test procedures (flashing etc.)")
screamer 0:66f3b5499f7f 1927
screamer 22:9e85236d8716 1928 parser.add_argument('--parallel',
screamer 22:9e85236d8716 1929 dest='parallel_test_exec',
screamer 22:9e85236d8716 1930 default=False,
screamer 22:9e85236d8716 1931 action="store_true",
screamer 22:9e85236d8716 1932 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer 0:66f3b5499f7f 1933
screamer 22:9e85236d8716 1934 parser.add_argument('--config',
screamer 22:9e85236d8716 1935 dest='verbose_test_configuration_only',
screamer 22:9e85236d8716 1936 default=False,
screamer 22:9e85236d8716 1937 action="store_true",
screamer 22:9e85236d8716 1938 help='Displays full test specification and MUTs configration and exits')
screamer 22:9e85236d8716 1939
screamer 22:9e85236d8716 1940 parser.add_argument('--loops',
screamer 22:9e85236d8716 1941 dest='test_loops_list',
screamer 22:9e85236d8716 1942 type=argparse_many(str),
screamer 22:9e85236d8716 1943 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer 0:66f3b5499f7f 1944
screamer 22:9e85236d8716 1945 parser.add_argument('--global-loops',
screamer 22:9e85236d8716 1946 dest='test_global_loops_value',
screamer 22:9e85236d8716 1947 type=int,
screamer 22:9e85236d8716 1948 help='Set global number of test loops per test. Default value is set 1')
screamer 0:66f3b5499f7f 1949
screamer 22:9e85236d8716 1950 parser.add_argument('--consolidate-waterfall',
screamer 22:9e85236d8716 1951 dest='consolidate_waterfall_test',
screamer 22:9e85236d8716 1952 default=False,
screamer 22:9e85236d8716 1953 action="store_true",
screamer 22:9e85236d8716 1954 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
screamer 0:66f3b5499f7f 1955
screamer 22:9e85236d8716 1956 parser.add_argument('-W', '--waterfall',
screamer 22:9e85236d8716 1957 dest='waterfall_test',
screamer 22:9e85236d8716 1958 default=False,
screamer 22:9e85236d8716 1959 action="store_true",
screamer 22:9e85236d8716 1960 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
screamer 0:66f3b5499f7f 1961
screamer 22:9e85236d8716 1962 parser.add_argument('-N', '--firmware-name',
screamer 22:9e85236d8716 1963 dest='firmware_global_name',
screamer 22:9e85236d8716 1964 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer 0:66f3b5499f7f 1965
screamer 22:9e85236d8716 1966 parser.add_argument('-u', '--shuffle',
screamer 22:9e85236d8716 1967 dest='shuffle_test_order',
screamer 22:9e85236d8716 1968 default=False,
screamer 22:9e85236d8716 1969 action="store_true",
screamer 22:9e85236d8716 1970 help='Shuffles test execution order')
screamer 0:66f3b5499f7f 1971
screamer 22:9e85236d8716 1972 parser.add_argument('--shuffle-seed',
screamer 22:9e85236d8716 1973 dest='shuffle_test_seed',
screamer 22:9e85236d8716 1974 default=None,
screamer 22:9e85236d8716 1975 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer 0:66f3b5499f7f 1976
screamer 22:9e85236d8716 1977 parser.add_argument('-f', '--filter',
screamer 22:9e85236d8716 1978 dest='general_filter_regex',
screamer 22:9e85236d8716 1979 type=argparse_many(str),
screamer 22:9e85236d8716 1980 default=None,
screamer 22:9e85236d8716 1981 help='For some commands you can use filter to filter out results')
screamer 0:66f3b5499f7f 1982
screamer 22:9e85236d8716 1983 parser.add_argument('--inc-timeout',
screamer 22:9e85236d8716 1984 dest='extend_test_timeout',
screamer 22:9e85236d8716 1985 metavar="NUMBER",
screamer 22:9e85236d8716 1986 type=int,
screamer 22:9e85236d8716 1987 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer 22:9e85236d8716 1988
screamer 22:9e85236d8716 1989 parser.add_argument('--db',
screamer 22:9e85236d8716 1990 dest='db_url',
screamer 22:9e85236d8716 1991 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer 0:66f3b5499f7f 1992
screamer 22:9e85236d8716 1993 parser.add_argument('-l', '--log',
screamer 22:9e85236d8716 1994 dest='log_file_name',
screamer 22:9e85236d8716 1995 help='Log events to external file (note not all console entries may be visible in log file)')
screamer 0:66f3b5499f7f 1996
screamer 22:9e85236d8716 1997 parser.add_argument('--report-html',
screamer 22:9e85236d8716 1998 dest='report_html_file_name',
screamer 22:9e85236d8716 1999 help='You can log test suite results in form of HTML report')
screamer 0:66f3b5499f7f 2000
screamer 22:9e85236d8716 2001 parser.add_argument('--report-junit',
screamer 22:9e85236d8716 2002 dest='report_junit_file_name',
screamer 22:9e85236d8716 2003 help='You can log test suite results in form of JUnit compliant XML report')
screamer 0:66f3b5499f7f 2004
screamer 22:9e85236d8716 2005 parser.add_argument("--report-build",
screamer 22:9e85236d8716 2006 dest="report_build_file_name",
screamer 22:9e85236d8716 2007 help="Output the build results to a junit xml file")
screamer 0:66f3b5499f7f 2008
screamer 22:9e85236d8716 2009 parser.add_argument("--report-text",
screamer 22:9e85236d8716 2010 dest="report_text_file_name",
screamer 22:9e85236d8716 2011 help="Output the build results to a text file")
screamer 0:66f3b5499f7f 2012
screamer 22:9e85236d8716 2013 parser.add_argument('--verbose-skipped',
screamer 22:9e85236d8716 2014 dest='verbose_skipped_tests',
screamer 22:9e85236d8716 2015 default=False,
screamer 22:9e85236d8716 2016 action="store_true",
screamer 22:9e85236d8716 2017 help='Prints some extra information about skipped tests')
screamer 0:66f3b5499f7f 2018
screamer 22:9e85236d8716 2019 parser.add_argument('-V', '--verbose-test-result',
screamer 22:9e85236d8716 2020 dest='verbose_test_result_only',
screamer 22:9e85236d8716 2021 default=False,
screamer 22:9e85236d8716 2022 action="store_true",
screamer 22:9e85236d8716 2023 help='Prints test serial output')
screamer 0:66f3b5499f7f 2024
screamer 22:9e85236d8716 2025 parser.add_argument('-v', '--verbose',
screamer 22:9e85236d8716 2026 dest='verbose',
screamer 22:9e85236d8716 2027 default=False,
screamer 22:9e85236d8716 2028 action="store_true",
screamer 22:9e85236d8716 2029 help='Verbose mode (prints some extra information)')
screamer 0:66f3b5499f7f 2030
screamer 22:9e85236d8716 2031 parser.add_argument('--version',
screamer 22:9e85236d8716 2032 dest='version',
screamer 22:9e85236d8716 2033 default=False,
screamer 22:9e85236d8716 2034 action="store_true",
screamer 22:9e85236d8716 2035 help='Prints script version and exits')
theotherjimmy 40:7d3fa6b99b2b 2036
theotherjimmy 40:7d3fa6b99b2b 2037 parser.add_argument('--stats-depth',
theotherjimmy 40:7d3fa6b99b2b 2038 dest='stats_depth',
theotherjimmy 40:7d3fa6b99b2b 2039 default=2,
theotherjimmy 40:7d3fa6b99b2b 2040 type=int,
theotherjimmy 40:7d3fa6b99b2b 2041 help="Depth level for static memory report")
screamer 0:66f3b5499f7f 2042 return parser
screamer 0:66f3b5499f7f 2043
The Other Jimmy 31:8ea194f6145b 2044 def test_path_to_name(path, base):
screamer 0:66f3b5499f7f 2045 """Change all slashes in a path into hyphens
screamer 0:66f3b5499f7f 2046 This creates a unique cross-platform test name based on the path
screamer 0:66f3b5499f7f 2047 This can eventually be overriden by a to-be-determined meta-data mechanism"""
screamer 0:66f3b5499f7f 2048 name_parts = []
The Other Jimmy 31:8ea194f6145b 2049 head, tail = os.path.split(relpath(path,base))
screamer 0:66f3b5499f7f 2050 while (tail and tail != "."):
screamer 0:66f3b5499f7f 2051 name_parts.insert(0, tail)
screamer 0:66f3b5499f7f 2052 head, tail = os.path.split(head)
screamer 22:9e85236d8716 2053
screamer 13:ab47a20b66f0 2054 return "-".join(name_parts).lower()
screamer 0:66f3b5499f7f 2055
theotherjimmy 41:2a77626a4c21 2056 def get_test_config(config_name, target_name):
theotherjimmy 41:2a77626a4c21 2057 """Finds the path to a test configuration file
theotherjimmy 41:2a77626a4c21 2058 config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
theotherjimmy 41:2a77626a4c21 2059 target_name: name of target to determing if mbed OS interface given is valid
theotherjimmy 43:2a7da56ebd24 2060 returns path to config, will return None if no valid config is found
theotherjimmy 41:2a77626a4c21 2061 """
theotherjimmy 41:2a77626a4c21 2062 # If they passed in a full path
theotherjimmy 41:2a77626a4c21 2063 if exists(config_name):
theotherjimmy 41:2a77626a4c21 2064 # This is a module config
theotherjimmy 41:2a77626a4c21 2065 return config_name
theotherjimmy 41:2a77626a4c21 2066 # Otherwise find the path to configuration file based on mbed OS interface
theotherjimmy 41:2a77626a4c21 2067 return TestConfig.get_config_path(config_name, target_name)
theotherjimmy 41:2a77626a4c21 2068
theotherjimmy 43:2a7da56ebd24 2069
theotherjimmy 43:2a7da56ebd24 2070 def find_tests(base_dir, target_name, toolchain_name, icetea, greentea, app_config=None):
screamer 24:25bff2709c20 2071 """ Finds all tests in a directory recursively
theotherjimmy 43:2a7da56ebd24 2072 :param base_dir: path to the directory to scan for tests (ex. 'path/to/project')
theotherjimmy 43:2a7da56ebd24 2073 :param target_name: name of the target to use for scanning (ex. 'K64F')
theotherjimmy 43:2a7da56ebd24 2074 :param toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
theotherjimmy 43:2a7da56ebd24 2075 :param icetea: icetea enabled
theotherjimmy 43:2a7da56ebd24 2076 :param greentea: greentea enabled
theotherjimmy 43:2a7da56ebd24 2077 :param app_config - location of a chosen mbed_app.json file
theotherjimmy 43:2a7da56ebd24 2078
theotherjimmy 43:2a7da56ebd24 2079 returns a dictionary where keys are the test name, and the values are
theotherjimmy 43:2a7da56ebd24 2080 lists of paths needed to biuld the test.
screamer 24:25bff2709c20 2081 """
screamer 24:25bff2709c20 2082
theotherjimmy 43:2a7da56ebd24 2083 # Temporary structure: tests referenced by (name, base, group, case) tuple
screamer 24:25bff2709c20 2084 tests = {}
theotherjimmy 43:2a7da56ebd24 2085 # List of common folders: (predicate function, path) tuple
theotherjimmy 43:2a7da56ebd24 2086 commons = []
screamer 24:25bff2709c20 2087
theotherjimmy 43:2a7da56ebd24 2088 config = Config(target_name, base_dir, app_config)
screamer 24:25bff2709c20 2089
screamer 24:25bff2709c20 2090 # Scan the directory for paths to probe for 'TESTS' folders
theotherjimmy 43:2a7da56ebd24 2091 base_resources = Resources(MockNotifier(), collect_ignores=True)
theotherjimmy 43:2a7da56ebd24 2092 base_resources.scan_with_config(base_dir, config)
screamer 24:25bff2709c20 2093
theotherjimmy 43:2a7da56ebd24 2094 if greentea:
theotherjimmy 43:2a7da56ebd24 2095 dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TESTS']
theotherjimmy 43:2a7da56ebd24 2096 ignoreset = MbedIgnoreSet()
screamer 22:9e85236d8716 2097
theotherjimmy 43:2a7da56ebd24 2098 for directory in dirs:
theotherjimmy 43:2a7da56ebd24 2099 ignorefile = join(directory, IGNORE_FILENAME)
theotherjimmy 43:2a7da56ebd24 2100 if isfile(ignorefile):
theotherjimmy 43:2a7da56ebd24 2101 ignoreset.add_mbedignore(directory, ignorefile)
theotherjimmy 43:2a7da56ebd24 2102 for test_group_directory in os.listdir(directory):
theotherjimmy 43:2a7da56ebd24 2103 grp_dir = join(directory, test_group_directory)
theotherjimmy 43:2a7da56ebd24 2104 if not isdir(grp_dir) or ignoreset.is_ignored(grp_dir):
theotherjimmy 43:2a7da56ebd24 2105 continue
theotherjimmy 43:2a7da56ebd24 2106 grpignorefile = join(grp_dir, IGNORE_FILENAME)
theotherjimmy 43:2a7da56ebd24 2107 if isfile(grpignorefile):
theotherjimmy 43:2a7da56ebd24 2108 ignoreset.add_mbedignore(grp_dir, grpignorefile)
theotherjimmy 43:2a7da56ebd24 2109 for test_case_directory in os.listdir(grp_dir):
theotherjimmy 43:2a7da56ebd24 2110 d = join(directory, test_group_directory, test_case_directory)
theotherjimmy 43:2a7da56ebd24 2111 if not isdir(d) or ignoreset.is_ignored(d):
theotherjimmy 43:2a7da56ebd24 2112 continue
theotherjimmy 43:2a7da56ebd24 2113 special_dirs = ['host_tests', 'COMMON']
theotherjimmy 43:2a7da56ebd24 2114 if test_group_directory not in special_dirs and test_case_directory not in special_dirs:
theotherjimmy 43:2a7da56ebd24 2115 test_name = test_path_to_name(d, base_dir)
theotherjimmy 43:2a7da56ebd24 2116 tests[(test_name, directory, test_group_directory, test_case_directory)] = [d]
theotherjimmy 43:2a7da56ebd24 2117 if test_case_directory == 'COMMON':
theotherjimmy 43:2a7da56ebd24 2118 def predicate(base_pred, group_pred, name_base_group_case):
theotherjimmy 43:2a7da56ebd24 2119 (name, base, group, case) = name_base_group_case
theotherjimmy 43:2a7da56ebd24 2120 return base == base_pred and group == group_pred
screamer 22:9e85236d8716 2121
theotherjimmy 43:2a7da56ebd24 2122 commons.append((functools.partial(predicate, directory, test_group_directory), d))
theotherjimmy 43:2a7da56ebd24 2123 if test_group_directory == 'COMMON':
theotherjimmy 43:2a7da56ebd24 2124 def predicate(base_pred, name_base_group_case):
theotherjimmy 43:2a7da56ebd24 2125 (name, base, group, case) = name_base_group_case
theotherjimmy 43:2a7da56ebd24 2126 return base == base_pred
theotherjimmy 43:2a7da56ebd24 2127
theotherjimmy 43:2a7da56ebd24 2128 commons.append((functools.partial(predicate, directory), grp_dir))
theotherjimmy 40:7d3fa6b99b2b 2129
theotherjimmy 43:2a7da56ebd24 2130 if icetea:
theotherjimmy 43:2a7da56ebd24 2131 dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TEST_APPS']
theotherjimmy 43:2a7da56ebd24 2132 for directory in dirs:
theotherjimmy 43:2a7da56ebd24 2133 if not isdir(directory):
theotherjimmy 43:2a7da56ebd24 2134 continue
theotherjimmy 43:2a7da56ebd24 2135 for subdir in os.listdir(directory):
theotherjimmy 43:2a7da56ebd24 2136 d = join(directory, subdir)
theotherjimmy 43:2a7da56ebd24 2137 if not isdir(d):
theotherjimmy 43:2a7da56ebd24 2138 continue
theotherjimmy 43:2a7da56ebd24 2139 if 'device' == subdir:
theotherjimmy 43:2a7da56ebd24 2140 for test_dir in os.listdir(d):
theotherjimmy 43:2a7da56ebd24 2141 test_dir_path = join(d, test_dir)
theotherjimmy 43:2a7da56ebd24 2142 test_name = test_path_to_name(test_dir_path, base_dir)
theotherjimmy 43:2a7da56ebd24 2143 tests[(test_name, directory, subdir, test_dir)] = [test_dir_path]
screamer 22:9e85236d8716 2144
theotherjimmy 43:2a7da56ebd24 2145 # Apply common directories
theotherjimmy 43:2a7da56ebd24 2146 for pred, path in commons:
theotherjimmy 43:2a7da56ebd24 2147 for test_identity, test_paths in six.iteritems(tests):
theotherjimmy 43:2a7da56ebd24 2148 if pred(test_identity):
theotherjimmy 43:2a7da56ebd24 2149 test_paths.append(path)
theotherjimmy 43:2a7da56ebd24 2150
theotherjimmy 43:2a7da56ebd24 2151 # Drop identity besides name
theotherjimmy 43:2a7da56ebd24 2152 return {name: paths for (name, _, _, _), paths in six.iteritems(tests)}
theotherjimmy 43:2a7da56ebd24 2153
screamer 22:9e85236d8716 2154
screamer 13:ab47a20b66f0 2155 def print_tests(tests, format="list", sort=True):
screamer 0:66f3b5499f7f 2156 """Given a dictionary of tests (as returned from "find_tests"), print them
screamer 0:66f3b5499f7f 2157 in the specified format"""
screamer 0:66f3b5499f7f 2158 if format == "list":
screamer 13:ab47a20b66f0 2159 for test_name in sorted(tests.keys()):
theotherjimmy 43:2a7da56ebd24 2160 test_path = tests[test_name][0]
theotherjimmy 43:2a7da56ebd24 2161 print("Test Case:")
theotherjimmy 43:2a7da56ebd24 2162 print(" Name: %s" % test_name)
theotherjimmy 43:2a7da56ebd24 2163 print(" Path: %s" % test_path)
screamer 0:66f3b5499f7f 2164 elif format == "json":
theotherjimmy 43:2a7da56ebd24 2165 print(json.dumps({test_name: test_path[0] for test_name, test_paths
theotherjimmy 43:2a7da56ebd24 2166 in tests}, indent=2))
screamer 0:66f3b5499f7f 2167 else:
theotherjimmy 43:2a7da56ebd24 2168 print("Unknown format '%s'" % format)
screamer 0:66f3b5499f7f 2169 sys.exit(1)
screamer 0:66f3b5499f7f 2170
screamer 13:ab47a20b66f0 2171 def norm_relative_path(path, start):
screamer 13:ab47a20b66f0 2172 """This function will create a normalized, relative path. It mimics the
screamer 13:ab47a20b66f0 2173 python os.path.relpath function, but also normalizes a Windows-syle path
screamer 13:ab47a20b66f0 2174 that use backslashes to a Unix style path that uses forward slashes."""
screamer 13:ab47a20b66f0 2175 path = os.path.normpath(path)
screamer 13:ab47a20b66f0 2176 path = os.path.relpath(path, start)
screamer 13:ab47a20b66f0 2177 path = path.replace("\\", "/")
screamer 13:ab47a20b66f0 2178 return path
screamer 13:ab47a20b66f0 2179
The Other Jimmy 31:8ea194f6145b 2180
The Other Jimmy 31:8ea194f6145b 2181 def build_test_worker(*args, **kwargs):
The Other Jimmy 31:8ea194f6145b 2182 """This is a worker function for the parallel building of tests. The `args`
The Other Jimmy 31:8ea194f6145b 2183 and `kwargs` are passed directly to `build_project`. It returns a dictionary
The Other Jimmy 31:8ea194f6145b 2184 with the following structure:
The Other Jimmy 31:8ea194f6145b 2185
The Other Jimmy 31:8ea194f6145b 2186 {
The Other Jimmy 31:8ea194f6145b 2187 'result': `True` if no exceptions were thrown, `False` otherwise
The Other Jimmy 31:8ea194f6145b 2188 'reason': Instance of exception that was thrown on failure
The Other Jimmy 31:8ea194f6145b 2189 'bin_file': Path to the created binary if `build_project` was
The Other Jimmy 31:8ea194f6145b 2190 successful. Not present otherwise
The Other Jimmy 31:8ea194f6145b 2191 'kwargs': The keyword arguments that were passed to `build_project`.
The Other Jimmy 31:8ea194f6145b 2192 This includes arguments that were modified (ex. report)
The Other Jimmy 31:8ea194f6145b 2193 }
The Other Jimmy 31:8ea194f6145b 2194 """
The Other Jimmy 31:8ea194f6145b 2195 bin_file = None
The Other Jimmy 31:8ea194f6145b 2196 ret = {
The Other Jimmy 31:8ea194f6145b 2197 'result': False,
The Other Jimmy 31:8ea194f6145b 2198 'args': args,
The Other Jimmy 31:8ea194f6145b 2199 'kwargs': kwargs
The Other Jimmy 31:8ea194f6145b 2200 }
The Other Jimmy 31:8ea194f6145b 2201
The Other Jimmy 31:8ea194f6145b 2202 # Use parent TOOLCHAIN_PATHS variable
theotherjimmy 43:2a7da56ebd24 2203 for key, value in kwargs['toolchain_paths'].items():
The Other Jimmy 31:8ea194f6145b 2204 TOOLCHAIN_PATHS[key] = value
The Other Jimmy 31:8ea194f6145b 2205
The Other Jimmy 31:8ea194f6145b 2206 del kwargs['toolchain_paths']
The Other Jimmy 31:8ea194f6145b 2207
The Other Jimmy 31:8ea194f6145b 2208 try:
theotherjimmy 43:2a7da56ebd24 2209 bin_file, _ = build_project(*args, **kwargs)
The Other Jimmy 31:8ea194f6145b 2210 ret['result'] = True
The Other Jimmy 31:8ea194f6145b 2211 ret['bin_file'] = bin_file
The Other Jimmy 31:8ea194f6145b 2212 ret['kwargs'] = kwargs
The Other Jimmy 31:8ea194f6145b 2213
theotherjimmy 43:2a7da56ebd24 2214 except NotSupportedException as e:
The Other Jimmy 31:8ea194f6145b 2215 ret['reason'] = e
theotherjimmy 43:2a7da56ebd24 2216 except ToolException as e:
The Other Jimmy 31:8ea194f6145b 2217 ret['reason'] = e
theotherjimmy 43:2a7da56ebd24 2218 except KeyboardInterrupt as e:
The Other Jimmy 31:8ea194f6145b 2219 ret['reason'] = e
The Other Jimmy 31:8ea194f6145b 2220 except:
The Other Jimmy 31:8ea194f6145b 2221 # Print unhandled exceptions here
The Other Jimmy 31:8ea194f6145b 2222 import traceback
The Other Jimmy 31:8ea194f6145b 2223 traceback.print_exc(file=sys.stdout)
The Other Jimmy 31:8ea194f6145b 2224
The Other Jimmy 31:8ea194f6145b 2225 return ret
The Other Jimmy 31:8ea194f6145b 2226
The Other Jimmy 31:8ea194f6145b 2227
screamer 0:66f3b5499f7f 2228 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
theotherjimmy 43:2a7da56ebd24 2229 clean=False, notify=None, jobs=1, macros=None,
The Other Jimmy 31:8ea194f6145b 2230 silent=False, report=None, properties=None,
The Other Jimmy 31:8ea194f6145b 2231 continue_on_build_fail=False, app_config=None,
theotherjimmy 43:2a7da56ebd24 2232 build_profile=None, stats_depth=None, ignore=None):
screamer 0:66f3b5499f7f 2233 """Given the data structure from 'find_tests' and the typical build parameters,
screamer 13:ab47a20b66f0 2234 build all the tests
screamer 22:9e85236d8716 2235
screamer 13:ab47a20b66f0 2236 Returns a tuple of the build result (True or False) followed by the test
screamer 13:ab47a20b66f0 2237 build data structure"""
screamer 22:9e85236d8716 2238
screamer 13:ab47a20b66f0 2239 execution_directory = "."
screamer 13:ab47a20b66f0 2240 base_path = norm_relative_path(build_path, execution_directory)
screamer 13:ab47a20b66f0 2241
theotherjimmy 43:2a7da56ebd24 2242 if isinstance(target, Target):
theotherjimmy 43:2a7da56ebd24 2243 target_name = target.name
theotherjimmy 43:2a7da56ebd24 2244 else:
theotherjimmy 43:2a7da56ebd24 2245 target_name = target
theotherjimmy 43:2a7da56ebd24 2246 target = TARGET_MAP[target_name]
theotherjimmy 43:2a7da56ebd24 2247 cfg, _, _ = get_config(base_source_paths, target, app_config=app_config)
The Other Jimmy 31:8ea194f6145b 2248
The Other Jimmy 31:8ea194f6145b 2249 baud_rate = 9600
The Other Jimmy 31:8ea194f6145b 2250 if 'platform.stdio-baud-rate' in cfg:
The Other Jimmy 31:8ea194f6145b 2251 baud_rate = cfg['platform.stdio-baud-rate'].value
The Other Jimmy 31:8ea194f6145b 2252
screamer 0:66f3b5499f7f 2253 test_build = {
screamer 13:ab47a20b66f0 2254 "platform": target_name,
screamer 0:66f3b5499f7f 2255 "toolchain": toolchain_name,
screamer 13:ab47a20b66f0 2256 "base_path": base_path,
The Other Jimmy 31:8ea194f6145b 2257 "baud_rate": baud_rate,
screamer 0:66f3b5499f7f 2258 "binary_type": "bootable",
theotherjimmy 43:2a7da56ebd24 2259 "tests": {},
theotherjimmy 43:2a7da56ebd24 2260 "test_apps": {}
screamer 0:66f3b5499f7f 2261 }
screamer 22:9e85236d8716 2262
screamer 13:ab47a20b66f0 2263 result = True
screamer 22:9e85236d8716 2264
The Other Jimmy 31:8ea194f6145b 2265 jobs_count = int(jobs if jobs else cpu_count())
The Other Jimmy 31:8ea194f6145b 2266 p = Pool(processes=jobs_count)
The Other Jimmy 31:8ea194f6145b 2267 results = []
theotherjimmy 43:2a7da56ebd24 2268 for test_name, test_paths in tests.items():
theotherjimmy 43:2a7da56ebd24 2269 if not isinstance(test_paths, list):
theotherjimmy 43:2a7da56ebd24 2270 test_paths = [test_paths]
theotherjimmy 43:2a7da56ebd24 2271
theotherjimmy 43:2a7da56ebd24 2272 test_build_path = os.path.join(build_path, test_paths[0])
theotherjimmy 43:2a7da56ebd24 2273 src_paths = base_source_paths + test_paths
screamer 13:ab47a20b66f0 2274 bin_file = None
theotherjimmy 43:2a7da56ebd24 2275 test_case_folder_name = os.path.basename(test_paths[0])
theotherjimmy 40:7d3fa6b99b2b 2276
theotherjimmy 43:2a7da56ebd24 2277 args = (src_paths, test_build_path, deepcopy(target), toolchain_name)
The Other Jimmy 31:8ea194f6145b 2278 kwargs = {
The Other Jimmy 31:8ea194f6145b 2279 'jobs': 1,
The Other Jimmy 31:8ea194f6145b 2280 'clean': clean,
The Other Jimmy 31:8ea194f6145b 2281 'macros': macros,
The Other Jimmy 31:8ea194f6145b 2282 'name': test_case_folder_name,
The Other Jimmy 31:8ea194f6145b 2283 'project_id': test_name,
The Other Jimmy 31:8ea194f6145b 2284 'report': report,
The Other Jimmy 31:8ea194f6145b 2285 'properties': properties,
The Other Jimmy 31:8ea194f6145b 2286 'app_config': app_config,
The Other Jimmy 31:8ea194f6145b 2287 'build_profile': build_profile,
theotherjimmy 40:7d3fa6b99b2b 2288 'toolchain_paths': TOOLCHAIN_PATHS,
theotherjimmy 43:2a7da56ebd24 2289 'stats_depth': stats_depth,
theotherjimmy 43:2a7da56ebd24 2290 'notify': MockNotifier()
The Other Jimmy 31:8ea194f6145b 2291 }
theotherjimmy 40:7d3fa6b99b2b 2292
The Other Jimmy 31:8ea194f6145b 2293 results.append(p.apply_async(build_test_worker, args, kwargs))
The Other Jimmy 31:8ea194f6145b 2294
The Other Jimmy 31:8ea194f6145b 2295 p.close()
The Other Jimmy 31:8ea194f6145b 2296 result = True
The Other Jimmy 31:8ea194f6145b 2297 itr = 0
The Other Jimmy 31:8ea194f6145b 2298 while len(results):
The Other Jimmy 31:8ea194f6145b 2299 itr += 1
The Other Jimmy 31:8ea194f6145b 2300 if itr > 360000:
The Other Jimmy 31:8ea194f6145b 2301 p.terminate()
The Other Jimmy 31:8ea194f6145b 2302 p.join()
The Other Jimmy 31:8ea194f6145b 2303 raise ToolException("Compile did not finish in 10 minutes")
The Other Jimmy 31:8ea194f6145b 2304 else:
The Other Jimmy 31:8ea194f6145b 2305 sleep(0.01)
The Other Jimmy 31:8ea194f6145b 2306 pending = 0
The Other Jimmy 31:8ea194f6145b 2307 for r in results:
The Other Jimmy 31:8ea194f6145b 2308 if r.ready() is True:
The Other Jimmy 31:8ea194f6145b 2309 try:
The Other Jimmy 31:8ea194f6145b 2310 worker_result = r.get()
The Other Jimmy 31:8ea194f6145b 2311 results.remove(r)
screamer 13:ab47a20b66f0 2312
theotherjimmy 43:2a7da56ebd24 2313 # Push all deferred notifications out to the actual notifier
theotherjimmy 43:2a7da56ebd24 2314 new_notify = deepcopy(notify)
theotherjimmy 43:2a7da56ebd24 2315 for message in worker_result['kwargs']['notify'].messages:
theotherjimmy 43:2a7da56ebd24 2316 new_notify.notify(message)
theotherjimmy 43:2a7da56ebd24 2317
The Other Jimmy 31:8ea194f6145b 2318 # Take report from the kwargs and merge it into existing report
theotherjimmy 40:7d3fa6b99b2b 2319 if report:
theotherjimmy 40:7d3fa6b99b2b 2320 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
theotherjimmy 43:2a7da56ebd24 2321 report_entry[worker_result['kwargs']['project_id'].upper()][0][0]['output'] = new_notify.get_output()
theotherjimmy 40:7d3fa6b99b2b 2322 for test_key in report_entry.keys():
theotherjimmy 40:7d3fa6b99b2b 2323 report[target_name][toolchain_name][test_key] = report_entry[test_key]
theotherjimmy 40:7d3fa6b99b2b 2324
The Other Jimmy 31:8ea194f6145b 2325 # Set the overall result to a failure if a build failure occurred
The Other Jimmy 35:da9c89f8be7d 2326 if ('reason' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2327 not worker_result['reason'] and
The Other Jimmy 35:da9c89f8be7d 2328 not isinstance(worker_result['reason'], NotSupportedException)):
The Other Jimmy 31:8ea194f6145b 2329 result = False
The Other Jimmy 31:8ea194f6145b 2330 break
The Other Jimmy 31:8ea194f6145b 2331
theotherjimmy 43:2a7da56ebd24 2332
The Other Jimmy 31:8ea194f6145b 2333 # Adding binary path to test build result
The Other Jimmy 35:da9c89f8be7d 2334 if ('result' in worker_result and
The Other Jimmy 35:da9c89f8be7d 2335 worker_result['result'] and
The Other Jimmy 35:da9c89f8be7d 2336 'bin_file' in worker_result):
The Other Jimmy 31:8ea194f6145b 2337 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
screamer 22:9e85236d8716 2338
theotherjimmy 43:2a7da56ebd24 2339 test_key = 'test_apps' if 'test_apps-' in worker_result['kwargs']['project_id'] else 'tests'
theotherjimmy 43:2a7da56ebd24 2340 test_build[test_key][worker_result['kwargs']['project_id']] = {
The Other Jimmy 31:8ea194f6145b 2341 "binaries": [
The Other Jimmy 31:8ea194f6145b 2342 {
The Other Jimmy 31:8ea194f6145b 2343 "path": bin_file
The Other Jimmy 31:8ea194f6145b 2344 }
The Other Jimmy 31:8ea194f6145b 2345 ]
The Other Jimmy 31:8ea194f6145b 2346 }
The Other Jimmy 31:8ea194f6145b 2347
The Other Jimmy 31:8ea194f6145b 2348 test_key = worker_result['kwargs']['project_id'].upper()
theotherjimmy 43:2a7da56ebd24 2349 print('Image: %s\n' % bin_file)
The Other Jimmy 31:8ea194f6145b 2350
The Other Jimmy 31:8ea194f6145b 2351 except:
The Other Jimmy 31:8ea194f6145b 2352 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2353 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2354 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2355 p.terminate()
The Other Jimmy 31:8ea194f6145b 2356 p.join()
The Other Jimmy 31:8ea194f6145b 2357 raise
screamer 13:ab47a20b66f0 2358 else:
The Other Jimmy 31:8ea194f6145b 2359 pending += 1
The Other Jimmy 31:8ea194f6145b 2360 if pending >= jobs_count:
The Other Jimmy 31:8ea194f6145b 2361 break
screamer 22:9e85236d8716 2362
The Other Jimmy 31:8ea194f6145b 2363 # Break as soon as possible if there is a failure and we are not
The Other Jimmy 31:8ea194f6145b 2364 # continuing on build failures
The Other Jimmy 31:8ea194f6145b 2365 if not result and not continue_on_build_fail:
The Other Jimmy 31:8ea194f6145b 2366 if p._taskqueue.queue:
The Other Jimmy 31:8ea194f6145b 2367 p._taskqueue.queue.clear()
The Other Jimmy 31:8ea194f6145b 2368 sleep(0.5)
The Other Jimmy 31:8ea194f6145b 2369 p.terminate()
The Other Jimmy 31:8ea194f6145b 2370 break
screamer 22:9e85236d8716 2371
The Other Jimmy 31:8ea194f6145b 2372 p.join()
screamer 22:9e85236d8716 2373
screamer 0:66f3b5499f7f 2374 test_builds = {}
screamer 13:ab47a20b66f0 2375 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
screamer 22:9e85236d8716 2376
screamer 13:ab47a20b66f0 2377 return result, test_builds
screamer 22:9e85236d8716 2378
screamer 0:66f3b5499f7f 2379
screamer 13:ab47a20b66f0 2380 def test_spec_from_test_builds(test_builds):
screamer 0:66f3b5499f7f 2381 return {
screamer 0:66f3b5499f7f 2382 "builds": test_builds
theotherjimmy 43:2a7da56ebd24 2383 }