Marco Zecchini / Mbed OS Example_RTOS

Rtos API example

mbed-os/tools/test_api.py@0:9fca2b23d0ba, 2019-02-23 (annotated)

Committer:
marcozecchini
Date:
Sat Feb 23 12:13:36 2019 +0000
Revision:
0:9fca2b23d0ba
final commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
marcozecchini 0:9fca2b23d0ba 1 """
marcozecchini 0:9fca2b23d0ba 2 mbed SDK
marcozecchini 0:9fca2b23d0ba 3 Copyright (c) 2011-2014 ARM Limited
marcozecchini 0:9fca2b23d0ba 4
marcozecchini 0:9fca2b23d0ba 5 Licensed under the Apache License, Version 2.0 (the "License");
marcozecchini 0:9fca2b23d0ba 6 you may not use this file except in compliance with the License.
marcozecchini 0:9fca2b23d0ba 7 You may obtain a copy of the License at
marcozecchini 0:9fca2b23d0ba 8
marcozecchini 0:9fca2b23d0ba 9 http://www.apache.org/licenses/LICENSE-2.0
marcozecchini 0:9fca2b23d0ba 10
marcozecchini 0:9fca2b23d0ba 11 Unless required by applicable law or agreed to in writing, software
marcozecchini 0:9fca2b23d0ba 12 distributed under the License is distributed on an "AS IS" BASIS,
marcozecchini 0:9fca2b23d0ba 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
marcozecchini 0:9fca2b23d0ba 14 See the License for the specific language governing permissions and
marcozecchini 0:9fca2b23d0ba 15 limitations under the License.
marcozecchini 0:9fca2b23d0ba 16
marcozecchini 0:9fca2b23d0ba 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
marcozecchini 0:9fca2b23d0ba 18 """
marcozecchini 0:9fca2b23d0ba 19
marcozecchini 0:9fca2b23d0ba 20 import os
marcozecchini 0:9fca2b23d0ba 21 import re
marcozecchini 0:9fca2b23d0ba 22 import sys
marcozecchini 0:9fca2b23d0ba 23 import json
marcozecchini 0:9fca2b23d0ba 24 import uuid
marcozecchini 0:9fca2b23d0ba 25 import pprint
marcozecchini 0:9fca2b23d0ba 26 import random
marcozecchini 0:9fca2b23d0ba 27 import argparse
marcozecchini 0:9fca2b23d0ba 28 import datetime
marcozecchini 0:9fca2b23d0ba 29 import threading
marcozecchini 0:9fca2b23d0ba 30 import ctypes
marcozecchini 0:9fca2b23d0ba 31 from types import ListType
marcozecchini 0:9fca2b23d0ba 32 from colorama import Fore, Back, Style
marcozecchini 0:9fca2b23d0ba 33 from prettytable import PrettyTable
marcozecchini 0:9fca2b23d0ba 34 from copy import copy
marcozecchini 0:9fca2b23d0ba 35
marcozecchini 0:9fca2b23d0ba 36 from time import sleep, time
marcozecchini 0:9fca2b23d0ba 37 from Queue import Queue, Empty
marcozecchini 0:9fca2b23d0ba 38 from os.path import join, exists, basename, relpath
marcozecchini 0:9fca2b23d0ba 39 from threading import Thread, Lock
marcozecchini 0:9fca2b23d0ba 40 from multiprocessing import Pool, cpu_count
marcozecchini 0:9fca2b23d0ba 41 from subprocess import Popen, PIPE
marcozecchini 0:9fca2b23d0ba 42
marcozecchini 0:9fca2b23d0ba 43 # Imports related to mbed build api
marcozecchini 0:9fca2b23d0ba 44 from tools.tests import TESTS
marcozecchini 0:9fca2b23d0ba 45 from tools.tests import TEST_MAP
marcozecchini 0:9fca2b23d0ba 46 from tools.paths import BUILD_DIR
marcozecchini 0:9fca2b23d0ba 47 from tools.paths import HOST_TESTS
marcozecchini 0:9fca2b23d0ba 48 from tools.utils import ToolException
marcozecchini 0:9fca2b23d0ba 49 from tools.utils import NotSupportedException
marcozecchini 0:9fca2b23d0ba 50 from tools.utils import construct_enum
marcozecchini 0:9fca2b23d0ba 51 from tools.memap import MemapParser
marcozecchini 0:9fca2b23d0ba 52 from tools.targets import TARGET_MAP
marcozecchini 0:9fca2b23d0ba 53 import tools.test_configs as TestConfig
marcozecchini 0:9fca2b23d0ba 54 from tools.test_db import BaseDBAccess
marcozecchini 0:9fca2b23d0ba 55 from tools.build_api import build_project, build_mbed_libs, build_lib
marcozecchini 0:9fca2b23d0ba 56 from tools.build_api import get_target_supported_toolchains
marcozecchini 0:9fca2b23d0ba 57 from tools.build_api import write_build_report
marcozecchini 0:9fca2b23d0ba 58 from tools.build_api import prep_report
marcozecchini 0:9fca2b23d0ba 59 from tools.build_api import prep_properties
marcozecchini 0:9fca2b23d0ba 60 from tools.build_api import create_result
marcozecchini 0:9fca2b23d0ba 61 from tools.build_api import add_result_to_report
marcozecchini 0:9fca2b23d0ba 62 from tools.build_api import prepare_toolchain
marcozecchini 0:9fca2b23d0ba 63 from tools.build_api import scan_resources
marcozecchini 0:9fca2b23d0ba 64 from tools.build_api import get_config
marcozecchini 0:9fca2b23d0ba 65 from tools.libraries import LIBRARIES, LIBRARY_MAP
marcozecchini 0:9fca2b23d0ba 66 from tools.options import extract_profile
marcozecchini 0:9fca2b23d0ba 67 from tools.toolchains import TOOLCHAIN_PATHS
marcozecchini 0:9fca2b23d0ba 68 from tools.toolchains import TOOLCHAINS
marcozecchini 0:9fca2b23d0ba 69 from tools.test_exporters import ReportExporter, ResultExporterType
marcozecchini 0:9fca2b23d0ba 70 from tools.utils import argparse_filestring_type
marcozecchini 0:9fca2b23d0ba 71 from tools.utils import argparse_uppercase_type
marcozecchini 0:9fca2b23d0ba 72 from tools.utils import argparse_lowercase_type
marcozecchini 0:9fca2b23d0ba 73 from tools.utils import argparse_many
marcozecchini 0:9fca2b23d0ba 74 from tools.utils import get_path_depth
marcozecchini 0:9fca2b23d0ba 75
marcozecchini 0:9fca2b23d0ba 76 import tools.host_tests.host_tests_plugins as host_tests_plugins
marcozecchini 0:9fca2b23d0ba 77
marcozecchini 0:9fca2b23d0ba 78 try:
marcozecchini 0:9fca2b23d0ba 79 import mbed_lstools
marcozecchini 0:9fca2b23d0ba 80 from tools.compliance.ioper_runner import get_available_oper_test_scopes
marcozecchini 0:9fca2b23d0ba 81 except:
marcozecchini 0:9fca2b23d0ba 82 pass
marcozecchini 0:9fca2b23d0ba 83
marcozecchini 0:9fca2b23d0ba 84
marcozecchini 0:9fca2b23d0ba 85 class ProcessObserver(Thread):
marcozecchini 0:9fca2b23d0ba 86 def __init__(self, proc):
marcozecchini 0:9fca2b23d0ba 87 Thread.__init__(self)
marcozecchini 0:9fca2b23d0ba 88 self.proc = proc
marcozecchini 0:9fca2b23d0ba 89 self.queue = Queue()
marcozecchini 0:9fca2b23d0ba 90 self.daemon = True
marcozecchini 0:9fca2b23d0ba 91 self.active = True
marcozecchini 0:9fca2b23d0ba 92 self.start()
marcozecchini 0:9fca2b23d0ba 93
marcozecchini 0:9fca2b23d0ba 94 def run(self):
marcozecchini 0:9fca2b23d0ba 95 while self.active:
marcozecchini 0:9fca2b23d0ba 96 c = self.proc.stdout.read(1)
marcozecchini 0:9fca2b23d0ba 97 self.queue.put(c)
marcozecchini 0:9fca2b23d0ba 98
marcozecchini 0:9fca2b23d0ba 99 def stop(self):
marcozecchini 0:9fca2b23d0ba 100 self.active = False
marcozecchini 0:9fca2b23d0ba 101 try:
marcozecchini 0:9fca2b23d0ba 102 self.proc.terminate()
marcozecchini 0:9fca2b23d0ba 103 except Exception, _:
marcozecchini 0:9fca2b23d0ba 104 pass
marcozecchini 0:9fca2b23d0ba 105
marcozecchini 0:9fca2b23d0ba 106
marcozecchini 0:9fca2b23d0ba 107 class SingleTestExecutor(threading.Thread):
marcozecchini 0:9fca2b23d0ba 108 """ Example: Single test class in separate thread usage
marcozecchini 0:9fca2b23d0ba 109 """
marcozecchini 0:9fca2b23d0ba 110 def __init__(self, single_test):
marcozecchini 0:9fca2b23d0ba 111 self.single_test = single_test
marcozecchini 0:9fca2b23d0ba 112 threading.Thread.__init__(self)
marcozecchini 0:9fca2b23d0ba 113
marcozecchini 0:9fca2b23d0ba 114 def run(self):
marcozecchini 0:9fca2b23d0ba 115 start = time()
marcozecchini 0:9fca2b23d0ba 116 # Execute tests depending on options and filter applied
marcozecchini 0:9fca2b23d0ba 117 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
marcozecchini 0:9fca2b23d0ba 118 elapsed_time = time() - start
marcozecchini 0:9fca2b23d0ba 119
marcozecchini 0:9fca2b23d0ba 120 # Human readable summary
marcozecchini 0:9fca2b23d0ba 121 if not self.single_test.opts_suppress_summary:
marcozecchini 0:9fca2b23d0ba 122 # prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 123 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
marcozecchini 0:9fca2b23d0ba 124 if self.single_test.opts_test_x_toolchain_summary:
marcozecchini 0:9fca2b23d0ba 125 # prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 126 # table shows text x toolchain test result matrix
marcozecchini 0:9fca2b23d0ba 127 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
marcozecchini 0:9fca2b23d0ba 128 print "Completed in %.2f sec"% (elapsed_time)
marcozecchini 0:9fca2b23d0ba 129
marcozecchini 0:9fca2b23d0ba 130
marcozecchini 0:9fca2b23d0ba 131 class SingleTestRunner(object):
marcozecchini 0:9fca2b23d0ba 132 """ Object wrapper for single test run which may involve multiple MUTs
marcozecchini 0:9fca2b23d0ba 133 """
marcozecchini 0:9fca2b23d0ba 134 RE_DETECT_TESTCASE_RESULT = None
marcozecchini 0:9fca2b23d0ba 135
marcozecchini 0:9fca2b23d0ba 136 # Return codes for test script
marcozecchini 0:9fca2b23d0ba 137 TEST_RESULT_OK = "OK"
marcozecchini 0:9fca2b23d0ba 138 TEST_RESULT_FAIL = "FAIL"
marcozecchini 0:9fca2b23d0ba 139 TEST_RESULT_ERROR = "ERROR"
marcozecchini 0:9fca2b23d0ba 140 TEST_RESULT_UNDEF = "UNDEF"
marcozecchini 0:9fca2b23d0ba 141 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
marcozecchini 0:9fca2b23d0ba 142 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
marcozecchini 0:9fca2b23d0ba 143 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
marcozecchini 0:9fca2b23d0ba 144 TEST_RESULT_TIMEOUT = "TIMEOUT"
marcozecchini 0:9fca2b23d0ba 145 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
marcozecchini 0:9fca2b23d0ba 146 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
marcozecchini 0:9fca2b23d0ba 147 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
marcozecchini 0:9fca2b23d0ba 148 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
marcozecchini 0:9fca2b23d0ba 149
marcozecchini 0:9fca2b23d0ba 150 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
marcozecchini 0:9fca2b23d0ba 151 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
marcozecchini 0:9fca2b23d0ba 152 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
marcozecchini 0:9fca2b23d0ba 153
marcozecchini 0:9fca2b23d0ba 154 muts = {} # MUTs descriptor (from external file)
marcozecchini 0:9fca2b23d0ba 155 test_spec = {} # Test specification (from external file)
marcozecchini 0:9fca2b23d0ba 156
marcozecchini 0:9fca2b23d0ba 157 # mbed test suite -> SingleTestRunner
marcozecchini 0:9fca2b23d0ba 158 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
marcozecchini 0:9fca2b23d0ba 159 "failure" : TEST_RESULT_FAIL,
marcozecchini 0:9fca2b23d0ba 160 "error" : TEST_RESULT_ERROR,
marcozecchini 0:9fca2b23d0ba 161 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
marcozecchini 0:9fca2b23d0ba 162 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
marcozecchini 0:9fca2b23d0ba 163 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
marcozecchini 0:9fca2b23d0ba 164 "timeout" : TEST_RESULT_TIMEOUT,
marcozecchini 0:9fca2b23d0ba 165 "no_image" : TEST_RESULT_NO_IMAGE,
marcozecchini 0:9fca2b23d0ba 166 "end" : TEST_RESULT_UNDEF,
marcozecchini 0:9fca2b23d0ba 167 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
marcozecchini 0:9fca2b23d0ba 168 "build_failed" : TEST_RESULT_BUILD_FAILED,
marcozecchini 0:9fca2b23d0ba 169 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
marcozecchini 0:9fca2b23d0ba 170 }
marcozecchini 0:9fca2b23d0ba 171
marcozecchini 0:9fca2b23d0ba 172 def __init__(self,
marcozecchini 0:9fca2b23d0ba 173 _global_loops_count=1,
marcozecchini 0:9fca2b23d0ba 174 _test_loops_list=None,
marcozecchini 0:9fca2b23d0ba 175 _muts={},
marcozecchini 0:9fca2b23d0ba 176 _clean=False,
marcozecchini 0:9fca2b23d0ba 177 _parser=None,
marcozecchini 0:9fca2b23d0ba 178 _opts=None,
marcozecchini 0:9fca2b23d0ba 179 _opts_db_url=None,
marcozecchini 0:9fca2b23d0ba 180 _opts_log_file_name=None,
marcozecchini 0:9fca2b23d0ba 181 _opts_report_html_file_name=None,
marcozecchini 0:9fca2b23d0ba 182 _opts_report_junit_file_name=None,
marcozecchini 0:9fca2b23d0ba 183 _opts_report_build_file_name=None,
marcozecchini 0:9fca2b23d0ba 184 _opts_report_text_file_name=None,
marcozecchini 0:9fca2b23d0ba 185 _opts_build_report={},
marcozecchini 0:9fca2b23d0ba 186 _opts_build_properties={},
marcozecchini 0:9fca2b23d0ba 187 _test_spec={},
marcozecchini 0:9fca2b23d0ba 188 _opts_goanna_for_mbed_sdk=None,
marcozecchini 0:9fca2b23d0ba 189 _opts_goanna_for_tests=None,
marcozecchini 0:9fca2b23d0ba 190 _opts_shuffle_test_order=False,
marcozecchini 0:9fca2b23d0ba 191 _opts_shuffle_test_seed=None,
marcozecchini 0:9fca2b23d0ba 192 _opts_test_by_names=None,
marcozecchini 0:9fca2b23d0ba 193 _opts_peripheral_by_names=None,
marcozecchini 0:9fca2b23d0ba 194 _opts_test_only_peripheral=False,
marcozecchini 0:9fca2b23d0ba 195 _opts_test_only_common=False,
marcozecchini 0:9fca2b23d0ba 196 _opts_verbose_skipped_tests=False,
marcozecchini 0:9fca2b23d0ba 197 _opts_verbose_test_result_only=False,
marcozecchini 0:9fca2b23d0ba 198 _opts_verbose=False,
marcozecchini 0:9fca2b23d0ba 199 _opts_firmware_global_name=None,
marcozecchini 0:9fca2b23d0ba 200 _opts_only_build_tests=False,
marcozecchini 0:9fca2b23d0ba 201 _opts_parallel_test_exec=False,
marcozecchini 0:9fca2b23d0ba 202 _opts_suppress_summary=False,
marcozecchini 0:9fca2b23d0ba 203 _opts_test_x_toolchain_summary=False,
marcozecchini 0:9fca2b23d0ba 204 _opts_copy_method=None,
marcozecchini 0:9fca2b23d0ba 205 _opts_mut_reset_type=None,
marcozecchini 0:9fca2b23d0ba 206 _opts_jobs=None,
marcozecchini 0:9fca2b23d0ba 207 _opts_waterfall_test=None,
marcozecchini 0:9fca2b23d0ba 208 _opts_consolidate_waterfall_test=None,
marcozecchini 0:9fca2b23d0ba 209 _opts_extend_test_timeout=None,
marcozecchini 0:9fca2b23d0ba 210 _opts_auto_detect=None,
marcozecchini 0:9fca2b23d0ba 211 _opts_include_non_automated=False):
marcozecchini 0:9fca2b23d0ba 212 """ Let's try hard to init this object
marcozecchini 0:9fca2b23d0ba 213 """
marcozecchini 0:9fca2b23d0ba 214 from colorama import init
marcozecchini 0:9fca2b23d0ba 215 init()
marcozecchini 0:9fca2b23d0ba 216
marcozecchini 0:9fca2b23d0ba 217 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
marcozecchini 0:9fca2b23d0ba 218 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
marcozecchini 0:9fca2b23d0ba 219 # Settings related to test loops counters
marcozecchini 0:9fca2b23d0ba 220 try:
marcozecchini 0:9fca2b23d0ba 221 _global_loops_count = int(_global_loops_count)
marcozecchini 0:9fca2b23d0ba 222 except:
marcozecchini 0:9fca2b23d0ba 223 _global_loops_count = 1
marcozecchini 0:9fca2b23d0ba 224 if _global_loops_count < 1:
marcozecchini 0:9fca2b23d0ba 225 _global_loops_count = 1
marcozecchini 0:9fca2b23d0ba 226 self.GLOBAL_LOOPS_COUNT = _global_loops_count
marcozecchini 0:9fca2b23d0ba 227 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
marcozecchini 0:9fca2b23d0ba 228 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
marcozecchini 0:9fca2b23d0ba 229
marcozecchini 0:9fca2b23d0ba 230 self.shuffle_random_seed = 0.0
marcozecchini 0:9fca2b23d0ba 231 self.SHUFFLE_SEED_ROUND = 10
marcozecchini 0:9fca2b23d0ba 232
marcozecchini 0:9fca2b23d0ba 233 # MUT list and test specification storage
marcozecchini 0:9fca2b23d0ba 234 self.muts = _muts
marcozecchini 0:9fca2b23d0ba 235 self.test_spec = _test_spec
marcozecchini 0:9fca2b23d0ba 236
marcozecchini 0:9fca2b23d0ba 237 # Settings passed e.g. from command line
marcozecchini 0:9fca2b23d0ba 238 self.opts_db_url = _opts_db_url
marcozecchini 0:9fca2b23d0ba 239 self.opts_log_file_name = _opts_log_file_name
marcozecchini 0:9fca2b23d0ba 240 self.opts_report_html_file_name = _opts_report_html_file_name
marcozecchini 0:9fca2b23d0ba 241 self.opts_report_junit_file_name = _opts_report_junit_file_name
marcozecchini 0:9fca2b23d0ba 242 self.opts_report_build_file_name = _opts_report_build_file_name
marcozecchini 0:9fca2b23d0ba 243 self.opts_report_text_file_name = _opts_report_text_file_name
marcozecchini 0:9fca2b23d0ba 244 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
marcozecchini 0:9fca2b23d0ba 245 self.opts_goanna_for_tests = _opts_goanna_for_tests
marcozecchini 0:9fca2b23d0ba 246 self.opts_shuffle_test_order = _opts_shuffle_test_order
marcozecchini 0:9fca2b23d0ba 247 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
marcozecchini 0:9fca2b23d0ba 248 self.opts_test_by_names = _opts_test_by_names
marcozecchini 0:9fca2b23d0ba 249 self.opts_peripheral_by_names = _opts_peripheral_by_names
marcozecchini 0:9fca2b23d0ba 250 self.opts_test_only_peripheral = _opts_test_only_peripheral
marcozecchini 0:9fca2b23d0ba 251 self.opts_test_only_common = _opts_test_only_common
marcozecchini 0:9fca2b23d0ba 252 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
marcozecchini 0:9fca2b23d0ba 253 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
marcozecchini 0:9fca2b23d0ba 254 self.opts_verbose = _opts_verbose
marcozecchini 0:9fca2b23d0ba 255 self.opts_firmware_global_name = _opts_firmware_global_name
marcozecchini 0:9fca2b23d0ba 256 self.opts_only_build_tests = _opts_only_build_tests
marcozecchini 0:9fca2b23d0ba 257 self.opts_parallel_test_exec = _opts_parallel_test_exec
marcozecchini 0:9fca2b23d0ba 258 self.opts_suppress_summary = _opts_suppress_summary
marcozecchini 0:9fca2b23d0ba 259 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
marcozecchini 0:9fca2b23d0ba 260 self.opts_copy_method = _opts_copy_method
marcozecchini 0:9fca2b23d0ba 261 self.opts_mut_reset_type = _opts_mut_reset_type
marcozecchini 0:9fca2b23d0ba 262 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
marcozecchini 0:9fca2b23d0ba 263 self.opts_waterfall_test = _opts_waterfall_test
marcozecchini 0:9fca2b23d0ba 264 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
marcozecchini 0:9fca2b23d0ba 265 self.opts_extend_test_timeout = _opts_extend_test_timeout
marcozecchini 0:9fca2b23d0ba 266 self.opts_clean = _clean
marcozecchini 0:9fca2b23d0ba 267 self.opts_parser = _parser
marcozecchini 0:9fca2b23d0ba 268 self.opts = _opts
marcozecchini 0:9fca2b23d0ba 269 self.opts_auto_detect = _opts_auto_detect
marcozecchini 0:9fca2b23d0ba 270 self.opts_include_non_automated = _opts_include_non_automated
marcozecchini 0:9fca2b23d0ba 271
marcozecchini 0:9fca2b23d0ba 272 self.build_report = _opts_build_report
marcozecchini 0:9fca2b23d0ba 273 self.build_properties = _opts_build_properties
marcozecchini 0:9fca2b23d0ba 274
marcozecchini 0:9fca2b23d0ba 275 # File / screen logger initialization
marcozecchini 0:9fca2b23d0ba 276 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
marcozecchini 0:9fca2b23d0ba 277
marcozecchini 0:9fca2b23d0ba 278 # Database related initializations
marcozecchini 0:9fca2b23d0ba 279 self.db_logger = factory_db_logger(self.opts_db_url)
marcozecchini 0:9fca2b23d0ba 280 self.db_logger_build_id = None # Build ID (database index of build_id table)
marcozecchini 0:9fca2b23d0ba 281 # Let's connect to database to set up credentials and confirm database is ready
marcozecchini 0:9fca2b23d0ba 282 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 283 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
marcozecchini 0:9fca2b23d0ba 284 if self.db_logger.is_connected():
marcozecchini 0:9fca2b23d0ba 285 # Get hostname and uname so we can use it as build description
marcozecchini 0:9fca2b23d0ba 286 # when creating new build_id in external database
marcozecchini 0:9fca2b23d0ba 287 (_hostname, _uname) = self.db_logger.get_hostname()
marcozecchini 0:9fca2b23d0ba 288 _host_location = os.path.dirname(os.path.abspath(__file__))
marcozecchini 0:9fca2b23d0ba 289 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
marcozecchini 0:9fca2b23d0ba 290 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
marcozecchini 0:9fca2b23d0ba 291 self.db_logger.disconnect()
marcozecchini 0:9fca2b23d0ba 292
marcozecchini 0:9fca2b23d0ba 293 def dump_options(self):
marcozecchini 0:9fca2b23d0ba 294 """ Function returns data structure with common settings passed to SingelTestRunner
marcozecchini 0:9fca2b23d0ba 295 It can be used for example to fill _extra fields in database storing test suite single run data
marcozecchini 0:9fca2b23d0ba 296 Example:
marcozecchini 0:9fca2b23d0ba 297 data = self.dump_options()
marcozecchini 0:9fca2b23d0ba 298 or
marcozecchini 0:9fca2b23d0ba 299 data_str = json.dumps(self.dump_options())
marcozecchini 0:9fca2b23d0ba 300 """
marcozecchini 0:9fca2b23d0ba 301 result = {"db_url" : str(self.opts_db_url),
marcozecchini 0:9fca2b23d0ba 302 "log_file_name" : str(self.opts_log_file_name),
marcozecchini 0:9fca2b23d0ba 303 "shuffle_test_order" : str(self.opts_shuffle_test_order),
marcozecchini 0:9fca2b23d0ba 304 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
marcozecchini 0:9fca2b23d0ba 305 "test_by_names" : str(self.opts_test_by_names),
marcozecchini 0:9fca2b23d0ba 306 "peripheral_by_names" : str(self.opts_peripheral_by_names),
marcozecchini 0:9fca2b23d0ba 307 "test_only_peripheral" : str(self.opts_test_only_peripheral),
marcozecchini 0:9fca2b23d0ba 308 "test_only_common" : str(self.opts_test_only_common),
marcozecchini 0:9fca2b23d0ba 309 "verbose" : str(self.opts_verbose),
marcozecchini 0:9fca2b23d0ba 310 "firmware_global_name" : str(self.opts_firmware_global_name),
marcozecchini 0:9fca2b23d0ba 311 "only_build_tests" : str(self.opts_only_build_tests),
marcozecchini 0:9fca2b23d0ba 312 "copy_method" : str(self.opts_copy_method),
marcozecchini 0:9fca2b23d0ba 313 "mut_reset_type" : str(self.opts_mut_reset_type),
marcozecchini 0:9fca2b23d0ba 314 "jobs" : str(self.opts_jobs),
marcozecchini 0:9fca2b23d0ba 315 "extend_test_timeout" : str(self.opts_extend_test_timeout),
marcozecchini 0:9fca2b23d0ba 316 "_dummy" : ''
marcozecchini 0:9fca2b23d0ba 317 }
marcozecchini 0:9fca2b23d0ba 318 return result
marcozecchini 0:9fca2b23d0ba 319
marcozecchini 0:9fca2b23d0ba 320 def shuffle_random_func(self):
marcozecchini 0:9fca2b23d0ba 321 return self.shuffle_random_seed
marcozecchini 0:9fca2b23d0ba 322
marcozecchini 0:9fca2b23d0ba 323 def is_shuffle_seed_float(self):
marcozecchini 0:9fca2b23d0ba 324 """ return true if function parameter can be converted to float
marcozecchini 0:9fca2b23d0ba 325 """
marcozecchini 0:9fca2b23d0ba 326 result = True
marcozecchini 0:9fca2b23d0ba 327 try:
marcozecchini 0:9fca2b23d0ba 328 float(self.shuffle_random_seed)
marcozecchini 0:9fca2b23d0ba 329 except ValueError:
marcozecchini 0:9fca2b23d0ba 330 result = False
marcozecchini 0:9fca2b23d0ba 331 return result
marcozecchini 0:9fca2b23d0ba 332
marcozecchini 0:9fca2b23d0ba 333 # This will store target / toolchain specific properties
marcozecchini 0:9fca2b23d0ba 334 test_suite_properties_ext = {} # target : toolchain
marcozecchini 0:9fca2b23d0ba 335 # Here we store test results
marcozecchini 0:9fca2b23d0ba 336 test_summary = []
marcozecchini 0:9fca2b23d0ba 337 # Here we store test results in extended data structure
marcozecchini 0:9fca2b23d0ba 338 test_summary_ext = {}
marcozecchini 0:9fca2b23d0ba 339 execute_thread_slice_lock = Lock()
marcozecchini 0:9fca2b23d0ba 340
marcozecchini 0:9fca2b23d0ba 341 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
marcozecchini 0:9fca2b23d0ba 342 for toolchain in toolchains:
marcozecchini 0:9fca2b23d0ba 343 tt_id = "%s::%s" % (toolchain, target)
marcozecchini 0:9fca2b23d0ba 344
marcozecchini 0:9fca2b23d0ba 345 T = TARGET_MAP[target]
marcozecchini 0:9fca2b23d0ba 346
marcozecchini 0:9fca2b23d0ba 347 # print target, toolchain
marcozecchini 0:9fca2b23d0ba 348 # Test suite properties returned to external tools like CI
marcozecchini 0:9fca2b23d0ba 349 test_suite_properties = {
marcozecchini 0:9fca2b23d0ba 350 'jobs': self.opts_jobs,
marcozecchini 0:9fca2b23d0ba 351 'clean': clean,
marcozecchini 0:9fca2b23d0ba 352 'target': target,
marcozecchini 0:9fca2b23d0ba 353 'vendor': T.extra_labels[0],
marcozecchini 0:9fca2b23d0ba 354 'test_ids': ', '.join(test_ids),
marcozecchini 0:9fca2b23d0ba 355 'toolchain': toolchain,
marcozecchini 0:9fca2b23d0ba 356 'shuffle_random_seed': self.shuffle_random_seed
marcozecchini 0:9fca2b23d0ba 357 }
marcozecchini 0:9fca2b23d0ba 358
marcozecchini 0:9fca2b23d0ba 359
marcozecchini 0:9fca2b23d0ba 360 # print '=== %s::%s ===' % (target, toolchain)
marcozecchini 0:9fca2b23d0ba 361 # Let's build our test
marcozecchini 0:9fca2b23d0ba 362 if target not in TARGET_MAP:
marcozecchini 0:9fca2b23d0ba 363 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
marcozecchini 0:9fca2b23d0ba 364 continue
marcozecchini 0:9fca2b23d0ba 365
marcozecchini 0:9fca2b23d0ba 366 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
marcozecchini 0:9fca2b23d0ba 367
marcozecchini 0:9fca2b23d0ba 368 profile = extract_profile(self.opts_parser, self.opts, toolchain)
marcozecchini 0:9fca2b23d0ba 369 stats_depth = self.opts.stats_depth or 2
marcozecchini 0:9fca2b23d0ba 370
marcozecchini 0:9fca2b23d0ba 371
marcozecchini 0:9fca2b23d0ba 372 try:
marcozecchini 0:9fca2b23d0ba 373 build_mbed_libs_result = build_mbed_libs(T,
marcozecchini 0:9fca2b23d0ba 374 toolchain,
marcozecchini 0:9fca2b23d0ba 375 clean=clean_mbed_libs_options,
marcozecchini 0:9fca2b23d0ba 376 verbose=self.opts_verbose,
marcozecchini 0:9fca2b23d0ba 377 jobs=self.opts_jobs,
marcozecchini 0:9fca2b23d0ba 378 report=build_report,
marcozecchini 0:9fca2b23d0ba 379 properties=build_properties,
marcozecchini 0:9fca2b23d0ba 380 build_profile=profile)
marcozecchini 0:9fca2b23d0ba 381
marcozecchini 0:9fca2b23d0ba 382 if not build_mbed_libs_result:
marcozecchini 0:9fca2b23d0ba 383 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
marcozecchini 0:9fca2b23d0ba 384 continue
marcozecchini 0:9fca2b23d0ba 385
marcozecchini 0:9fca2b23d0ba 386 except ToolException:
marcozecchini 0:9fca2b23d0ba 387 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
marcozecchini 0:9fca2b23d0ba 388 continue
marcozecchini 0:9fca2b23d0ba 389
marcozecchini 0:9fca2b23d0ba 390 build_dir = join(BUILD_DIR, "test", target, toolchain)
marcozecchini 0:9fca2b23d0ba 391
marcozecchini 0:9fca2b23d0ba 392 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
marcozecchini 0:9fca2b23d0ba 393 test_suite_properties['build_dir'] = build_dir
marcozecchini 0:9fca2b23d0ba 394 test_suite_properties['skipped'] = []
marcozecchini 0:9fca2b23d0ba 395
marcozecchini 0:9fca2b23d0ba 396 # Enumerate through all tests and shuffle test order if requested
marcozecchini 0:9fca2b23d0ba 397 test_map_keys = sorted(TEST_MAP.keys())
marcozecchini 0:9fca2b23d0ba 398
marcozecchini 0:9fca2b23d0ba 399 if self.opts_shuffle_test_order:
marcozecchini 0:9fca2b23d0ba 400 random.shuffle(test_map_keys, self.shuffle_random_func)
marcozecchini 0:9fca2b23d0ba 401 # Update database with shuffle seed f applicable
marcozecchini 0:9fca2b23d0ba 402 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 403 self.db_logger.reconnect();
marcozecchini 0:9fca2b23d0ba 404 if self.db_logger.is_connected():
marcozecchini 0:9fca2b23d0ba 405 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
marcozecchini 0:9fca2b23d0ba 406 self.db_logger.disconnect();
marcozecchini 0:9fca2b23d0ba 407
marcozecchini 0:9fca2b23d0ba 408 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 409 self.db_logger.reconnect();
marcozecchini 0:9fca2b23d0ba 410 if self.db_logger.is_connected():
marcozecchini 0:9fca2b23d0ba 411 # Update MUTs and Test Specification in database
marcozecchini 0:9fca2b23d0ba 412 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
marcozecchini 0:9fca2b23d0ba 413 # Update Extra information in database (some options passed to test suite)
marcozecchini 0:9fca2b23d0ba 414 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
marcozecchini 0:9fca2b23d0ba 415 self.db_logger.disconnect();
marcozecchini 0:9fca2b23d0ba 416
marcozecchini 0:9fca2b23d0ba 417 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
marcozecchini 0:9fca2b23d0ba 418 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
marcozecchini 0:9fca2b23d0ba 419
marcozecchini 0:9fca2b23d0ba 420 for skipped_test_id in skipped_test_map_keys:
marcozecchini 0:9fca2b23d0ba 421 test_suite_properties['skipped'].append(skipped_test_id)
marcozecchini 0:9fca2b23d0ba 422
marcozecchini 0:9fca2b23d0ba 423
marcozecchini 0:9fca2b23d0ba 424 # First pass through all tests and determine which libraries need to be built
marcozecchini 0:9fca2b23d0ba 425 libraries = []
marcozecchini 0:9fca2b23d0ba 426 for test_id in valid_test_map_keys:
marcozecchini 0:9fca2b23d0ba 427 test = TEST_MAP[test_id]
marcozecchini 0:9fca2b23d0ba 428
marcozecchini 0:9fca2b23d0ba 429 # Detect which lib should be added to test
marcozecchini 0:9fca2b23d0ba 430 # Some libs have to compiled like RTOS or ETH
marcozecchini 0:9fca2b23d0ba 431 for lib in LIBRARIES:
marcozecchini 0:9fca2b23d0ba 432 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
marcozecchini 0:9fca2b23d0ba 433 libraries.append(lib['id'])
marcozecchini 0:9fca2b23d0ba 434
marcozecchini 0:9fca2b23d0ba 435
marcozecchini 0:9fca2b23d0ba 436 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
marcozecchini 0:9fca2b23d0ba 437
marcozecchini 0:9fca2b23d0ba 438 # Build all required libraries
marcozecchini 0:9fca2b23d0ba 439 for lib_id in libraries:
marcozecchini 0:9fca2b23d0ba 440 try:
marcozecchini 0:9fca2b23d0ba 441 build_lib(lib_id,
marcozecchini 0:9fca2b23d0ba 442 T,
marcozecchini 0:9fca2b23d0ba 443 toolchain,
marcozecchini 0:9fca2b23d0ba 444 verbose=self.opts_verbose,
marcozecchini 0:9fca2b23d0ba 445 clean=clean_mbed_libs_options,
marcozecchini 0:9fca2b23d0ba 446 jobs=self.opts_jobs,
marcozecchini 0:9fca2b23d0ba 447 report=build_report,
marcozecchini 0:9fca2b23d0ba 448 properties=build_properties,
marcozecchini 0:9fca2b23d0ba 449 build_profile=profile)
marcozecchini 0:9fca2b23d0ba 450
marcozecchini 0:9fca2b23d0ba 451 except ToolException:
marcozecchini 0:9fca2b23d0ba 452 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
marcozecchini 0:9fca2b23d0ba 453 continue
marcozecchini 0:9fca2b23d0ba 454
marcozecchini 0:9fca2b23d0ba 455
marcozecchini 0:9fca2b23d0ba 456 for test_id in valid_test_map_keys:
marcozecchini 0:9fca2b23d0ba 457 test = TEST_MAP[test_id]
marcozecchini 0:9fca2b23d0ba 458
marcozecchini 0:9fca2b23d0ba 459 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
marcozecchini 0:9fca2b23d0ba 460
marcozecchini 0:9fca2b23d0ba 461 # TODO: move this 2 below loops to separate function
marcozecchini 0:9fca2b23d0ba 462 INC_DIRS = []
marcozecchini 0:9fca2b23d0ba 463 for lib_id in libraries:
marcozecchini 0:9fca2b23d0ba 464 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
marcozecchini 0:9fca2b23d0ba 465 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
marcozecchini 0:9fca2b23d0ba 466
marcozecchini 0:9fca2b23d0ba 467 MACROS = []
marcozecchini 0:9fca2b23d0ba 468 for lib_id in libraries:
marcozecchini 0:9fca2b23d0ba 469 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
marcozecchini 0:9fca2b23d0ba 470 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
marcozecchini 0:9fca2b23d0ba 471 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
marcozecchini 0:9fca2b23d0ba 472 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
marcozecchini 0:9fca2b23d0ba 473 test_uuid = uuid.uuid4()
marcozecchini 0:9fca2b23d0ba 474 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
marcozecchini 0:9fca2b23d0ba 475
marcozecchini 0:9fca2b23d0ba 476 # Prepare extended test results data structure (it can be used to generate detailed test report)
marcozecchini 0:9fca2b23d0ba 477 if target not in self.test_summary_ext:
marcozecchini 0:9fca2b23d0ba 478 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
marcozecchini 0:9fca2b23d0ba 479 if toolchain not in self.test_summary_ext[target]:
marcozecchini 0:9fca2b23d0ba 480 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
marcozecchini 0:9fca2b23d0ba 481
marcozecchini 0:9fca2b23d0ba 482 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
marcozecchini 0:9fca2b23d0ba 483
marcozecchini 0:9fca2b23d0ba 484 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
marcozecchini 0:9fca2b23d0ba 485 try:
marcozecchini 0:9fca2b23d0ba 486 path = build_project(test.source_dir, join(build_dir, test_id), T,
marcozecchini 0:9fca2b23d0ba 487 toolchain, test.dependencies, clean=clean_project_options,
marcozecchini 0:9fca2b23d0ba 488 verbose=self.opts_verbose, name=project_name, macros=MACROS,
marcozecchini 0:9fca2b23d0ba 489 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
marcozecchini 0:9fca2b23d0ba 490 properties=build_properties, project_id=test_id,
marcozecchini 0:9fca2b23d0ba 491 project_description=test.get_description(),
marcozecchini 0:9fca2b23d0ba 492 build_profile=profile, stats_depth=stats_depth)
marcozecchini 0:9fca2b23d0ba 493
marcozecchini 0:9fca2b23d0ba 494 except Exception, e:
marcozecchini 0:9fca2b23d0ba 495 project_name_str = project_name if project_name is not None else test_id
marcozecchini 0:9fca2b23d0ba 496
marcozecchini 0:9fca2b23d0ba 497
marcozecchini 0:9fca2b23d0ba 498 test_result = self.TEST_RESULT_FAIL
marcozecchini 0:9fca2b23d0ba 499
marcozecchini 0:9fca2b23d0ba 500 if isinstance(e, ToolException):
marcozecchini 0:9fca2b23d0ba 501 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
marcozecchini 0:9fca2b23d0ba 502 test_result = self.TEST_RESULT_BUILD_FAILED
marcozecchini 0:9fca2b23d0ba 503 elif isinstance(e, NotSupportedException):
marcozecchini 0:9fca2b23d0ba 504 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
marcozecchini 0:9fca2b23d0ba 505 test_result = self.TEST_RESULT_NOT_SUPPORTED
marcozecchini 0:9fca2b23d0ba 506
marcozecchini 0:9fca2b23d0ba 507
marcozecchini 0:9fca2b23d0ba 508 # Append test results to global test summary
marcozecchini 0:9fca2b23d0ba 509 self.test_summary.append(
marcozecchini 0:9fca2b23d0ba 510 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
marcozecchini 0:9fca2b23d0ba 511 )
marcozecchini 0:9fca2b23d0ba 512
marcozecchini 0:9fca2b23d0ba 513 # Add detailed test result to test summary structure
marcozecchini 0:9fca2b23d0ba 514 if test_id not in self.test_summary_ext[target][toolchain]:
marcozecchini 0:9fca2b23d0ba 515 self.test_summary_ext[target][toolchain][test_id] = []
marcozecchini 0:9fca2b23d0ba 516
marcozecchini 0:9fca2b23d0ba 517 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
marcozecchini 0:9fca2b23d0ba 518 'result' : test_result,
marcozecchini 0:9fca2b23d0ba 519 'output' : '',
marcozecchini 0:9fca2b23d0ba 520 'target_name' : target,
marcozecchini 0:9fca2b23d0ba 521 'target_name_unique': target,
marcozecchini 0:9fca2b23d0ba 522 'toolchain_name' : toolchain,
marcozecchini 0:9fca2b23d0ba 523 'id' : test_id,
marcozecchini 0:9fca2b23d0ba 524 'description' : test.get_description(),
marcozecchini 0:9fca2b23d0ba 525 'elapsed_time' : 0,
marcozecchini 0:9fca2b23d0ba 526 'duration' : 0,
marcozecchini 0:9fca2b23d0ba 527 'copy_method' : None
marcozecchini 0:9fca2b23d0ba 528 }})
marcozecchini 0:9fca2b23d0ba 529 continue
marcozecchini 0:9fca2b23d0ba 530
marcozecchini 0:9fca2b23d0ba 531 if self.opts_only_build_tests:
marcozecchini 0:9fca2b23d0ba 532 # With this option we are skipping testing phase
marcozecchini 0:9fca2b23d0ba 533 continue
marcozecchini 0:9fca2b23d0ba 534
marcozecchini 0:9fca2b23d0ba 535 # Test duration can be increased by global value
marcozecchini 0:9fca2b23d0ba 536 test_duration = test.duration
marcozecchini 0:9fca2b23d0ba 537 if self.opts_extend_test_timeout is not None:
marcozecchini 0:9fca2b23d0ba 538 test_duration += self.opts_extend_test_timeout
marcozecchini 0:9fca2b23d0ba 539
marcozecchini 0:9fca2b23d0ba 540 # For an automated test the duration act as a timeout after
marcozecchini 0:9fca2b23d0ba 541 # which the test gets interrupted
marcozecchini 0:9fca2b23d0ba 542 test_spec = self.shape_test_request(target, path, test_id, test_duration)
marcozecchini 0:9fca2b23d0ba 543 test_loops = self.get_test_loop_count(test_id)
marcozecchini 0:9fca2b23d0ba 544
marcozecchini 0:9fca2b23d0ba 545 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
marcozecchini 0:9fca2b23d0ba 546 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
marcozecchini 0:9fca2b23d0ba 547 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
marcozecchini 0:9fca2b23d0ba 548
marcozecchini 0:9fca2b23d0ba 549 # read MUTs, test specification and perform tests
marcozecchini 0:9fca2b23d0ba 550 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
marcozecchini 0:9fca2b23d0ba 551
marcozecchini 0:9fca2b23d0ba 552 if handle_results is None:
marcozecchini 0:9fca2b23d0ba 553 continue
marcozecchini 0:9fca2b23d0ba 554
marcozecchini 0:9fca2b23d0ba 555 for handle_result in handle_results:
marcozecchini 0:9fca2b23d0ba 556 if handle_result:
marcozecchini 0:9fca2b23d0ba 557 single_test_result, detailed_test_results = handle_result
marcozecchini 0:9fca2b23d0ba 558 else:
marcozecchini 0:9fca2b23d0ba 559 continue
marcozecchini 0:9fca2b23d0ba 560
marcozecchini 0:9fca2b23d0ba 561 # Append test results to global test summary
marcozecchini 0:9fca2b23d0ba 562 if single_test_result is not None:
marcozecchini 0:9fca2b23d0ba 563 self.test_summary.append(single_test_result)
marcozecchini 0:9fca2b23d0ba 564
marcozecchini 0:9fca2b23d0ba 565 # Add detailed test result to test summary structure
marcozecchini 0:9fca2b23d0ba 566 if target not in self.test_summary_ext[target][toolchain]:
marcozecchini 0:9fca2b23d0ba 567 if test_id not in self.test_summary_ext[target][toolchain]:
marcozecchini 0:9fca2b23d0ba 568 self.test_summary_ext[target][toolchain][test_id] = []
marcozecchini 0:9fca2b23d0ba 569
marcozecchini 0:9fca2b23d0ba 570 append_test_result = detailed_test_results
marcozecchini 0:9fca2b23d0ba 571
marcozecchini 0:9fca2b23d0ba 572 # If waterfall and consolidate-waterfall options are enabled,
marcozecchini 0:9fca2b23d0ba 573 # only include the last test result in the report.
marcozecchini 0:9fca2b23d0ba 574 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
marcozecchini 0:9fca2b23d0ba 575 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
marcozecchini 0:9fca2b23d0ba 576
marcozecchini 0:9fca2b23d0ba 577 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
marcozecchini 0:9fca2b23d0ba 578
marcozecchini 0:9fca2b23d0ba 579 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
marcozecchini 0:9fca2b23d0ba 580 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
marcozecchini 0:9fca2b23d0ba 581
marcozecchini 0:9fca2b23d0ba 582 q.put(target + '_'.join(toolchains))
marcozecchini 0:9fca2b23d0ba 583 return
marcozecchini 0:9fca2b23d0ba 584
marcozecchini 0:9fca2b23d0ba 585 def execute(self):
marcozecchini 0:9fca2b23d0ba 586 clean = self.test_spec.get('clean', False)
marcozecchini 0:9fca2b23d0ba 587 test_ids = self.test_spec.get('test_ids', [])
marcozecchini 0:9fca2b23d0ba 588 q = Queue()
marcozecchini 0:9fca2b23d0ba 589
marcozecchini 0:9fca2b23d0ba 590 # Generate seed for shuffle if seed is not provided in
marcozecchini 0:9fca2b23d0ba 591 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
marcozecchini 0:9fca2b23d0ba 592 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
marcozecchini 0:9fca2b23d0ba 593 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
marcozecchini 0:9fca2b23d0ba 594
marcozecchini 0:9fca2b23d0ba 595
marcozecchini 0:9fca2b23d0ba 596 if self.opts_parallel_test_exec:
marcozecchini 0:9fca2b23d0ba 597 ###################################################################
marcozecchini 0:9fca2b23d0ba 598 # Experimental, parallel test execution per singletest instance.
marcozecchini 0:9fca2b23d0ba 599 ###################################################################
marcozecchini 0:9fca2b23d0ba 600 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
marcozecchini 0:9fca2b23d0ba 601 # Note: We are building here in parallel for each target separately!
marcozecchini 0:9fca2b23d0ba 602 # So we are not building the same thing multiple times and compilers
marcozecchini 0:9fca2b23d0ba 603 # in separate threads do not collide.
marcozecchini 0:9fca2b23d0ba 604 # Inside execute_thread_slice() function function handle() will be called to
marcozecchini 0:9fca2b23d0ba 605 # get information about available MUTs (per target).
marcozecchini 0:9fca2b23d0ba 606 for target, toolchains in self.test_spec['targets'].iteritems():
marcozecchini 0:9fca2b23d0ba 607 self.test_suite_properties_ext[target] = {}
marcozecchini 0:9fca2b23d0ba 608 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
marcozecchini 0:9fca2b23d0ba 609 t.daemon = True
marcozecchini 0:9fca2b23d0ba 610 t.start()
marcozecchini 0:9fca2b23d0ba 611 execute_threads.append(t)
marcozecchini 0:9fca2b23d0ba 612
marcozecchini 0:9fca2b23d0ba 613 for t in execute_threads:
marcozecchini 0:9fca2b23d0ba 614 q.get() # t.join() would block some threads because we should not wait in any order for thread end
marcozecchini 0:9fca2b23d0ba 615 else:
marcozecchini 0:9fca2b23d0ba 616 # Serialized (not parallel) test execution
marcozecchini 0:9fca2b23d0ba 617 for target, toolchains in self.test_spec['targets'].iteritems():
marcozecchini 0:9fca2b23d0ba 618 if target not in self.test_suite_properties_ext:
marcozecchini 0:9fca2b23d0ba 619 self.test_suite_properties_ext[target] = {}
marcozecchini 0:9fca2b23d0ba 620
marcozecchini 0:9fca2b23d0ba 621 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
marcozecchini 0:9fca2b23d0ba 622 q.get()
marcozecchini 0:9fca2b23d0ba 623
marcozecchini 0:9fca2b23d0ba 624 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 625 self.db_logger.reconnect();
marcozecchini 0:9fca2b23d0ba 626 if self.db_logger.is_connected():
marcozecchini 0:9fca2b23d0ba 627 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
marcozecchini 0:9fca2b23d0ba 628 self.db_logger.disconnect();
marcozecchini 0:9fca2b23d0ba 629
marcozecchini 0:9fca2b23d0ba 630 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
marcozecchini 0:9fca2b23d0ba 631
marcozecchini 0:9fca2b23d0ba 632 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
marcozecchini 0:9fca2b23d0ba 633 valid_test_map_keys = []
marcozecchini 0:9fca2b23d0ba 634
marcozecchini 0:9fca2b23d0ba 635 for test_id in test_map_keys:
marcozecchini 0:9fca2b23d0ba 636 test = TEST_MAP[test_id]
marcozecchini 0:9fca2b23d0ba 637 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
marcozecchini 0:9fca2b23d0ba 638 continue
marcozecchini 0:9fca2b23d0ba 639
marcozecchini 0:9fca2b23d0ba 640 if test_ids and test_id not in test_ids:
marcozecchini 0:9fca2b23d0ba 641 continue
marcozecchini 0:9fca2b23d0ba 642
marcozecchini 0:9fca2b23d0ba 643 if self.opts_test_only_peripheral and not test.peripherals:
marcozecchini 0:9fca2b23d0ba 644 if self.opts_verbose_skipped_tests:
marcozecchini 0:9fca2b23d0ba 645 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
marcozecchini 0:9fca2b23d0ba 646 continue
marcozecchini 0:9fca2b23d0ba 647
marcozecchini 0:9fca2b23d0ba 648 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
marcozecchini 0:9fca2b23d0ba 649 # We will skip tests not forced with -p option
marcozecchini 0:9fca2b23d0ba 650 if self.opts_verbose_skipped_tests:
marcozecchini 0:9fca2b23d0ba 651 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
marcozecchini 0:9fca2b23d0ba 652 continue
marcozecchini 0:9fca2b23d0ba 653
marcozecchini 0:9fca2b23d0ba 654 if self.opts_test_only_common and test.peripherals:
marcozecchini 0:9fca2b23d0ba 655 if self.opts_verbose_skipped_tests:
marcozecchini 0:9fca2b23d0ba 656 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
marcozecchini 0:9fca2b23d0ba 657 continue
marcozecchini 0:9fca2b23d0ba 658
marcozecchini 0:9fca2b23d0ba 659 if not include_non_automated and not test.automated:
marcozecchini 0:9fca2b23d0ba 660 if self.opts_verbose_skipped_tests:
marcozecchini 0:9fca2b23d0ba 661 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
marcozecchini 0:9fca2b23d0ba 662 continue
marcozecchini 0:9fca2b23d0ba 663
marcozecchini 0:9fca2b23d0ba 664 if test.is_supported(target, toolchain):
marcozecchini 0:9fca2b23d0ba 665 if test.peripherals is None and self.opts_only_build_tests:
marcozecchini 0:9fca2b23d0ba 666 # When users are using 'build only flag' and test do not have
marcozecchini 0:9fca2b23d0ba 667 # specified peripherals we can allow test building by default
marcozecchini 0:9fca2b23d0ba 668 pass
marcozecchini 0:9fca2b23d0ba 669 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
marcozecchini 0:9fca2b23d0ba 670 # If we force peripheral with option -p we expect test
marcozecchini 0:9fca2b23d0ba 671 # to pass even if peripheral is not in MUTs file.
marcozecchini 0:9fca2b23d0ba 672 pass
marcozecchini 0:9fca2b23d0ba 673 elif not self.is_peripherals_available(target, test.peripherals):
marcozecchini 0:9fca2b23d0ba 674 if self.opts_verbose_skipped_tests:
marcozecchini 0:9fca2b23d0ba 675 if test.peripherals:
marcozecchini 0:9fca2b23d0ba 676 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
marcozecchini 0:9fca2b23d0ba 677 else:
marcozecchini 0:9fca2b23d0ba 678 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
marcozecchini 0:9fca2b23d0ba 679 continue
marcozecchini 0:9fca2b23d0ba 680
marcozecchini 0:9fca2b23d0ba 681 # The test has made it through all the filters, so add it to the valid tests list
marcozecchini 0:9fca2b23d0ba 682 valid_test_map_keys.append(test_id)
marcozecchini 0:9fca2b23d0ba 683
marcozecchini 0:9fca2b23d0ba 684 return valid_test_map_keys
marcozecchini 0:9fca2b23d0ba 685
marcozecchini 0:9fca2b23d0ba 686 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
marcozecchini 0:9fca2b23d0ba 687 # NOTE: This will not preserve order
marcozecchini 0:9fca2b23d0ba 688 return list(set(all_test_map_keys) - set(valid_test_map_keys))
marcozecchini 0:9fca2b23d0ba 689
marcozecchini 0:9fca2b23d0ba 690 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
marcozecchini 0:9fca2b23d0ba 691 """ Prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 692 table shows text x toolchain test result matrix
marcozecchini 0:9fca2b23d0ba 693 """
marcozecchini 0:9fca2b23d0ba 694 RESULT_INDEX = 0
marcozecchini 0:9fca2b23d0ba 695 TARGET_INDEX = 1
marcozecchini 0:9fca2b23d0ba 696 TOOLCHAIN_INDEX = 2
marcozecchini 0:9fca2b23d0ba 697 TEST_INDEX = 3
marcozecchini 0:9fca2b23d0ba 698 DESC_INDEX = 4
marcozecchini 0:9fca2b23d0ba 699
marcozecchini 0:9fca2b23d0ba 700 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
marcozecchini 0:9fca2b23d0ba 701 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
marcozecchini 0:9fca2b23d0ba 702 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
marcozecchini 0:9fca2b23d0ba 703 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
marcozecchini 0:9fca2b23d0ba 704
marcozecchini 0:9fca2b23d0ba 705 result = "Test summary:\n"
marcozecchini 0:9fca2b23d0ba 706 for target in unique_targets:
marcozecchini 0:9fca2b23d0ba 707 result_dict = {} # test : { toolchain : result }
marcozecchini 0:9fca2b23d0ba 708 unique_target_toolchains = []
marcozecchini 0:9fca2b23d0ba 709 for test in test_summary:
marcozecchini 0:9fca2b23d0ba 710 if test[TARGET_INDEX] == target:
marcozecchini 0:9fca2b23d0ba 711 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
marcozecchini 0:9fca2b23d0ba 712 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
marcozecchini 0:9fca2b23d0ba 713 if test[TEST_INDEX] not in result_dict:
marcozecchini 0:9fca2b23d0ba 714 result_dict[test[TEST_INDEX]] = {}
marcozecchini 0:9fca2b23d0ba 715 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
marcozecchini 0:9fca2b23d0ba 716
marcozecchini 0:9fca2b23d0ba 717 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
marcozecchini 0:9fca2b23d0ba 718 pt = PrettyTable(pt_cols)
marcozecchini 0:9fca2b23d0ba 719 for col in pt_cols:
marcozecchini 0:9fca2b23d0ba 720 pt.align[col] = "l"
marcozecchini 0:9fca2b23d0ba 721 pt.padding_width = 1 # One space between column edges and contents (default)
marcozecchini 0:9fca2b23d0ba 722
marcozecchini 0:9fca2b23d0ba 723 for test in unique_tests:
marcozecchini 0:9fca2b23d0ba 724 if test in result_dict:
marcozecchini 0:9fca2b23d0ba 725 test_results = result_dict[test]
marcozecchini 0:9fca2b23d0ba 726 if test in unique_test_desc:
marcozecchini 0:9fca2b23d0ba 727 row = [target, test, unique_test_desc[test]]
marcozecchini 0:9fca2b23d0ba 728 for toolchain in unique_toolchains:
marcozecchini 0:9fca2b23d0ba 729 if toolchain in test_results:
marcozecchini 0:9fca2b23d0ba 730 row.append(test_results[toolchain])
marcozecchini 0:9fca2b23d0ba 731 pt.add_row(row)
marcozecchini 0:9fca2b23d0ba 732 result += pt.get_string()
marcozecchini 0:9fca2b23d0ba 733 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
marcozecchini 0:9fca2b23d0ba 734 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
marcozecchini 0:9fca2b23d0ba 735 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
marcozecchini 0:9fca2b23d0ba 736 return result
marcozecchini 0:9fca2b23d0ba 737
marcozecchini 0:9fca2b23d0ba 738 def generate_test_summary(self, test_summary, shuffle_seed=None):
marcozecchini 0:9fca2b23d0ba 739 """ Prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 740 table shows target x test results matrix across
marcozecchini 0:9fca2b23d0ba 741 """
marcozecchini 0:9fca2b23d0ba 742 success_code = 0 # Success code that can be leter returned to
marcozecchini 0:9fca2b23d0ba 743 result = "Test summary:\n"
marcozecchini 0:9fca2b23d0ba 744 # Pretty table package is used to print results
marcozecchini 0:9fca2b23d0ba 745 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
marcozecchini 0:9fca2b23d0ba 746 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
marcozecchini 0:9fca2b23d0ba 747 pt.align["Result"] = "l" # Left align
marcozecchini 0:9fca2b23d0ba 748 pt.align["Target"] = "l" # Left align
marcozecchini 0:9fca2b23d0ba 749 pt.align["Toolchain"] = "l" # Left align
marcozecchini 0:9fca2b23d0ba 750 pt.align["Test ID"] = "l" # Left align
marcozecchini 0:9fca2b23d0ba 751 pt.align["Test Description"] = "l" # Left align
marcozecchini 0:9fca2b23d0ba 752 pt.padding_width = 1 # One space between column edges and contents (default)
marcozecchini 0:9fca2b23d0ba 753
marcozecchini 0:9fca2b23d0ba 754 result_dict = {self.TEST_RESULT_OK : 0,
marcozecchini 0:9fca2b23d0ba 755 self.TEST_RESULT_FAIL : 0,
marcozecchini 0:9fca2b23d0ba 756 self.TEST_RESULT_ERROR : 0,
marcozecchini 0:9fca2b23d0ba 757 self.TEST_RESULT_UNDEF : 0,
marcozecchini 0:9fca2b23d0ba 758 self.TEST_RESULT_IOERR_COPY : 0,
marcozecchini 0:9fca2b23d0ba 759 self.TEST_RESULT_IOERR_DISK : 0,
marcozecchini 0:9fca2b23d0ba 760 self.TEST_RESULT_IOERR_SERIAL : 0,
marcozecchini 0:9fca2b23d0ba 761 self.TEST_RESULT_NO_IMAGE : 0,
marcozecchini 0:9fca2b23d0ba 762 self.TEST_RESULT_TIMEOUT : 0,
marcozecchini 0:9fca2b23d0ba 763 self.TEST_RESULT_MBED_ASSERT : 0,
marcozecchini 0:9fca2b23d0ba 764 self.TEST_RESULT_BUILD_FAILED : 0,
marcozecchini 0:9fca2b23d0ba 765 self.TEST_RESULT_NOT_SUPPORTED : 0
marcozecchini 0:9fca2b23d0ba 766 }
marcozecchini 0:9fca2b23d0ba 767
marcozecchini 0:9fca2b23d0ba 768 for test in test_summary:
marcozecchini 0:9fca2b23d0ba 769 if test[0] in result_dict:
marcozecchini 0:9fca2b23d0ba 770 result_dict[test[0]] += 1
marcozecchini 0:9fca2b23d0ba 771 pt.add_row(test)
marcozecchini 0:9fca2b23d0ba 772 result += pt.get_string()
marcozecchini 0:9fca2b23d0ba 773 result += "\n"
marcozecchini 0:9fca2b23d0ba 774
marcozecchini 0:9fca2b23d0ba 775 # Print result count
marcozecchini 0:9fca2b23d0ba 776 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
marcozecchini 0:9fca2b23d0ba 777 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
marcozecchini 0:9fca2b23d0ba 778 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
marcozecchini 0:9fca2b23d0ba 779 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
marcozecchini 0:9fca2b23d0ba 780 return result
marcozecchini 0:9fca2b23d0ba 781
marcozecchini 0:9fca2b23d0ba 782 def test_loop_list_to_dict(self, test_loops_str):
marcozecchini 0:9fca2b23d0ba 783 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
marcozecchini 0:9fca2b23d0ba 784 """
marcozecchini 0:9fca2b23d0ba 785 result = {}
marcozecchini 0:9fca2b23d0ba 786 if test_loops_str:
marcozecchini 0:9fca2b23d0ba 787 test_loops = test_loops_str
marcozecchini 0:9fca2b23d0ba 788 for test_loop in test_loops:
marcozecchini 0:9fca2b23d0ba 789 test_loop_count = test_loop.split('=')
marcozecchini 0:9fca2b23d0ba 790 if len(test_loop_count) == 2:
marcozecchini 0:9fca2b23d0ba 791 _test_id, _test_loops = test_loop_count
marcozecchini 0:9fca2b23d0ba 792 try:
marcozecchini 0:9fca2b23d0ba 793 _test_loops = int(_test_loops)
marcozecchini 0:9fca2b23d0ba 794 except:
marcozecchini 0:9fca2b23d0ba 795 continue
marcozecchini 0:9fca2b23d0ba 796 result[_test_id] = _test_loops
marcozecchini 0:9fca2b23d0ba 797 return result
marcozecchini 0:9fca2b23d0ba 798
marcozecchini 0:9fca2b23d0ba 799 def get_test_loop_count(self, test_id):
marcozecchini 0:9fca2b23d0ba 800 """ This function returns no. of loops per test (deducted by test_id_.
marcozecchini 0:9fca2b23d0ba 801 If test is not in list of redefined loop counts it will use default value.
marcozecchini 0:9fca2b23d0ba 802 """
marcozecchini 0:9fca2b23d0ba 803 result = self.GLOBAL_LOOPS_COUNT
marcozecchini 0:9fca2b23d0ba 804 if test_id in self.TEST_LOOPS_DICT:
marcozecchini 0:9fca2b23d0ba 805 result = self.TEST_LOOPS_DICT[test_id]
marcozecchini 0:9fca2b23d0ba 806 return result
marcozecchini 0:9fca2b23d0ba 807
marcozecchini 0:9fca2b23d0ba 808 def delete_file(self, file_path):
marcozecchini 0:9fca2b23d0ba 809 """ Remove file from the system
marcozecchini 0:9fca2b23d0ba 810 """
marcozecchini 0:9fca2b23d0ba 811 result = True
marcozecchini 0:9fca2b23d0ba 812 resutl_msg = ""
marcozecchini 0:9fca2b23d0ba 813 try:
marcozecchini 0:9fca2b23d0ba 814 os.remove(file_path)
marcozecchini 0:9fca2b23d0ba 815 except Exception, e:
marcozecchini 0:9fca2b23d0ba 816 resutl_msg = e
marcozecchini 0:9fca2b23d0ba 817 result = False
marcozecchini 0:9fca2b23d0ba 818 return result, resutl_msg
marcozecchini 0:9fca2b23d0ba 819
marcozecchini 0:9fca2b23d0ba 820 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
marcozecchini 0:9fca2b23d0ba 821 """ Test is being invoked for given MUT.
marcozecchini 0:9fca2b23d0ba 822 """
marcozecchini 0:9fca2b23d0ba 823 # Get test information, image and test timeout
marcozecchini 0:9fca2b23d0ba 824 test_id = data['test_id']
marcozecchini 0:9fca2b23d0ba 825 test = TEST_MAP[test_id]
marcozecchini 0:9fca2b23d0ba 826 test_description = TEST_MAP[test_id].get_description()
marcozecchini 0:9fca2b23d0ba 827 image = data["image"]
marcozecchini 0:9fca2b23d0ba 828 duration = data.get("duration", 10)
marcozecchini 0:9fca2b23d0ba 829
marcozecchini 0:9fca2b23d0ba 830 if mut is None:
marcozecchini 0:9fca2b23d0ba 831 print "Error: No Mbed available: MUT[%s]" % data['mcu']
marcozecchini 0:9fca2b23d0ba 832 return None
marcozecchini 0:9fca2b23d0ba 833
marcozecchini 0:9fca2b23d0ba 834 mcu = mut['mcu']
marcozecchini 0:9fca2b23d0ba 835 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
marcozecchini 0:9fca2b23d0ba 836
marcozecchini 0:9fca2b23d0ba 837 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 838 self.db_logger.reconnect()
marcozecchini 0:9fca2b23d0ba 839
marcozecchini 0:9fca2b23d0ba 840 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
marcozecchini 0:9fca2b23d0ba 841
marcozecchini 0:9fca2b23d0ba 842 # Tests can be looped so test results must be stored for the same test
marcozecchini 0:9fca2b23d0ba 843 test_all_result = []
marcozecchini 0:9fca2b23d0ba 844 # Test results for one test ran few times
marcozecchini 0:9fca2b23d0ba 845 detailed_test_results = {} # { Loop_number: { results ... } }
marcozecchini 0:9fca2b23d0ba 846
marcozecchini 0:9fca2b23d0ba 847 for test_index in range(test_loops):
marcozecchini 0:9fca2b23d0ba 848
marcozecchini 0:9fca2b23d0ba 849 # If mbedls is available and we are auto detecting MUT info,
marcozecchini 0:9fca2b23d0ba 850 # update MUT info (mounting may changed)
marcozecchini 0:9fca2b23d0ba 851 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
marcozecchini 0:9fca2b23d0ba 852 platform_name_filter = [mcu]
marcozecchini 0:9fca2b23d0ba 853 muts_list = {}
marcozecchini 0:9fca2b23d0ba 854 found = False
marcozecchini 0:9fca2b23d0ba 855
marcozecchini 0:9fca2b23d0ba 856 for i in range(0, 60):
marcozecchini 0:9fca2b23d0ba 857 print('Looking for %s with MBEDLS' % mcu)
marcozecchini 0:9fca2b23d0ba 858 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
marcozecchini 0:9fca2b23d0ba 859
marcozecchini 0:9fca2b23d0ba 860 if 1 not in muts_list:
marcozecchini 0:9fca2b23d0ba 861 sleep(3)
marcozecchini 0:9fca2b23d0ba 862 else:
marcozecchini 0:9fca2b23d0ba 863 found = True
marcozecchini 0:9fca2b23d0ba 864 break
marcozecchini 0:9fca2b23d0ba 865
marcozecchini 0:9fca2b23d0ba 866 if not found:
marcozecchini 0:9fca2b23d0ba 867 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
marcozecchini 0:9fca2b23d0ba 868 return None
marcozecchini 0:9fca2b23d0ba 869 else:
marcozecchini 0:9fca2b23d0ba 870 mut = muts_list[1]
marcozecchini 0:9fca2b23d0ba 871
marcozecchini 0:9fca2b23d0ba 872 disk = mut.get('disk')
marcozecchini 0:9fca2b23d0ba 873 port = mut.get('port')
marcozecchini 0:9fca2b23d0ba 874
marcozecchini 0:9fca2b23d0ba 875 if disk is None or port is None:
marcozecchini 0:9fca2b23d0ba 876 return None
marcozecchini 0:9fca2b23d0ba 877
marcozecchini 0:9fca2b23d0ba 878 target_by_mcu = TARGET_MAP[mut['mcu']]
marcozecchini 0:9fca2b23d0ba 879 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
marcozecchini 0:9fca2b23d0ba 880 # Some extra stuff can be declared in MUTs structure
marcozecchini 0:9fca2b23d0ba 881 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
marcozecchini 0:9fca2b23d0ba 882 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
marcozecchini 0:9fca2b23d0ba 883
marcozecchini 0:9fca2b23d0ba 884 # When the build and test system were separate, this was relative to a
marcozecchini 0:9fca2b23d0ba 885 # base network folder base path: join(NETWORK_BASE_PATH, )
marcozecchini 0:9fca2b23d0ba 886 image_path = image
marcozecchini 0:9fca2b23d0ba 887
marcozecchini 0:9fca2b23d0ba 888 # Host test execution
marcozecchini 0:9fca2b23d0ba 889 start_host_exec_time = time()
marcozecchini 0:9fca2b23d0ba 890
marcozecchini 0:9fca2b23d0ba 891 single_test_result = self.TEST_RESULT_UNDEF # single test run result
marcozecchini 0:9fca2b23d0ba 892 _copy_method = selected_copy_method
marcozecchini 0:9fca2b23d0ba 893
marcozecchini 0:9fca2b23d0ba 894 if not exists(image_path):
marcozecchini 0:9fca2b23d0ba 895 single_test_result = self.TEST_RESULT_NO_IMAGE
marcozecchini 0:9fca2b23d0ba 896 elapsed_time = 0
marcozecchini 0:9fca2b23d0ba 897 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
marcozecchini 0:9fca2b23d0ba 898 print single_test_output
marcozecchini 0:9fca2b23d0ba 899 else:
marcozecchini 0:9fca2b23d0ba 900 # Host test execution
marcozecchini 0:9fca2b23d0ba 901 start_host_exec_time = time()
marcozecchini 0:9fca2b23d0ba 902
marcozecchini 0:9fca2b23d0ba 903 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
marcozecchini 0:9fca2b23d0ba 904 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
marcozecchini 0:9fca2b23d0ba 905 host_test_result = self.run_host_test(test.host_test,
marcozecchini 0:9fca2b23d0ba 906 image_path, disk, port, duration,
marcozecchini 0:9fca2b23d0ba 907 micro=target_name,
marcozecchini 0:9fca2b23d0ba 908 verbose=host_test_verbose,
marcozecchini 0:9fca2b23d0ba 909 reset=host_test_reset,
marcozecchini 0:9fca2b23d0ba 910 reset_tout=reset_tout,
marcozecchini 0:9fca2b23d0ba 911 copy_method=selected_copy_method,
marcozecchini 0:9fca2b23d0ba 912 program_cycle_s=target_by_mcu.program_cycle_s)
marcozecchini 0:9fca2b23d0ba 913 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
marcozecchini 0:9fca2b23d0ba 914
marcozecchini 0:9fca2b23d0ba 915 # Store test result
marcozecchini 0:9fca2b23d0ba 916 test_all_result.append(single_test_result)
marcozecchini 0:9fca2b23d0ba 917 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
marcozecchini 0:9fca2b23d0ba 918 elapsed_time = single_testduration # TIme of single test case execution after reset
marcozecchini 0:9fca2b23d0ba 919
marcozecchini 0:9fca2b23d0ba 920 detailed_test_results[test_index] = {
marcozecchini 0:9fca2b23d0ba 921 'result' : single_test_result,
marcozecchini 0:9fca2b23d0ba 922 'output' : single_test_output,
marcozecchini 0:9fca2b23d0ba 923 'target_name' : target_name,
marcozecchini 0:9fca2b23d0ba 924 'target_name_unique' : target_name_unique,
marcozecchini 0:9fca2b23d0ba 925 'toolchain_name' : toolchain_name,
marcozecchini 0:9fca2b23d0ba 926 'id' : test_id,
marcozecchini 0:9fca2b23d0ba 927 'description' : test_description,
marcozecchini 0:9fca2b23d0ba 928 'elapsed_time' : round(elapsed_time, 2),
marcozecchini 0:9fca2b23d0ba 929 'duration' : single_timeout,
marcozecchini 0:9fca2b23d0ba 930 'copy_method' : _copy_method,
marcozecchini 0:9fca2b23d0ba 931 }
marcozecchini 0:9fca2b23d0ba 932
marcozecchini 0:9fca2b23d0ba 933 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
marcozecchini 0:9fca2b23d0ba 934 test_id, test_description, elapsed_time, single_timeout)
marcozecchini 0:9fca2b23d0ba 935
marcozecchini 0:9fca2b23d0ba 936 # Update database entries for ongoing test
marcozecchini 0:9fca2b23d0ba 937 if self.db_logger and self.db_logger.is_connected():
marcozecchini 0:9fca2b23d0ba 938 test_type = 'SingleTest'
marcozecchini 0:9fca2b23d0ba 939 self.db_logger.insert_test_entry(self.db_logger_build_id,
marcozecchini 0:9fca2b23d0ba 940 target_name,
marcozecchini 0:9fca2b23d0ba 941 toolchain_name,
marcozecchini 0:9fca2b23d0ba 942 test_type,
marcozecchini 0:9fca2b23d0ba 943 test_id,
marcozecchini 0:9fca2b23d0ba 944 single_test_result,
marcozecchini 0:9fca2b23d0ba 945 single_test_output,
marcozecchini 0:9fca2b23d0ba 946 elapsed_time,
marcozecchini 0:9fca2b23d0ba 947 single_timeout,
marcozecchini 0:9fca2b23d0ba 948 test_index)
marcozecchini 0:9fca2b23d0ba 949
marcozecchini 0:9fca2b23d0ba 950 # If we perform waterfall test we test until we get OK and we stop testing
marcozecchini 0:9fca2b23d0ba 951 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
marcozecchini 0:9fca2b23d0ba 952 break
marcozecchini 0:9fca2b23d0ba 953
marcozecchini 0:9fca2b23d0ba 954 if self.db_logger:
marcozecchini 0:9fca2b23d0ba 955 self.db_logger.disconnect()
marcozecchini 0:9fca2b23d0ba 956
marcozecchini 0:9fca2b23d0ba 957 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
marcozecchini 0:9fca2b23d0ba 958 target_name_unique,
marcozecchini 0:9fca2b23d0ba 959 toolchain_name,
marcozecchini 0:9fca2b23d0ba 960 test_id,
marcozecchini 0:9fca2b23d0ba 961 test_description,
marcozecchini 0:9fca2b23d0ba 962 round(elapsed_time, 2),
marcozecchini 0:9fca2b23d0ba 963 single_timeout,
marcozecchini 0:9fca2b23d0ba 964 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
marcozecchini 0:9fca2b23d0ba 965
marcozecchini 0:9fca2b23d0ba 966 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
marcozecchini 0:9fca2b23d0ba 967 """ Function determines MUT's mbed disk/port and copies binary to
marcozecchini 0:9fca2b23d0ba 968 target.
marcozecchini 0:9fca2b23d0ba 969 """
marcozecchini 0:9fca2b23d0ba 970 handle_results = []
marcozecchini 0:9fca2b23d0ba 971 data = json.loads(test_spec)
marcozecchini 0:9fca2b23d0ba 972
marcozecchini 0:9fca2b23d0ba 973 # Find a suitable MUT:
marcozecchini 0:9fca2b23d0ba 974 mut = None
marcozecchini 0:9fca2b23d0ba 975 for id, m in self.muts.iteritems():
marcozecchini 0:9fca2b23d0ba 976 if m['mcu'] == data['mcu']:
marcozecchini 0:9fca2b23d0ba 977 mut = m
marcozecchini 0:9fca2b23d0ba 978 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
marcozecchini 0:9fca2b23d0ba 979 handle_results.append(handle_result)
marcozecchini 0:9fca2b23d0ba 980
marcozecchini 0:9fca2b23d0ba 981 return handle_results
marcozecchini 0:9fca2b23d0ba 982
marcozecchini 0:9fca2b23d0ba 983 def print_test_result(self, test_result, target_name, toolchain_name,
marcozecchini 0:9fca2b23d0ba 984 test_id, test_description, elapsed_time, duration):
marcozecchini 0:9fca2b23d0ba 985 """ Use specific convention to print test result and related data
marcozecchini 0:9fca2b23d0ba 986 """
marcozecchini 0:9fca2b23d0ba 987 tokens = []
marcozecchini 0:9fca2b23d0ba 988 tokens.append("TargetTest")
marcozecchini 0:9fca2b23d0ba 989 tokens.append(target_name)
marcozecchini 0:9fca2b23d0ba 990 tokens.append(toolchain_name)
marcozecchini 0:9fca2b23d0ba 991 tokens.append(test_id)
marcozecchini 0:9fca2b23d0ba 992 tokens.append(test_description)
marcozecchini 0:9fca2b23d0ba 993 separator = "::"
marcozecchini 0:9fca2b23d0ba 994 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
marcozecchini 0:9fca2b23d0ba 995 result = separator.join(tokens) + " [" + test_result +"]" + time_info
marcozecchini 0:9fca2b23d0ba 996 return Fore.MAGENTA + result + Fore.RESET
marcozecchini 0:9fca2b23d0ba 997
marcozecchini 0:9fca2b23d0ba 998 def shape_test_loop_ok_result_count(self, test_all_result):
marcozecchini 0:9fca2b23d0ba 999 """ Reformats list of results to simple string
marcozecchini 0:9fca2b23d0ba 1000 """
marcozecchini 0:9fca2b23d0ba 1001 test_loop_count = len(test_all_result)
marcozecchini 0:9fca2b23d0ba 1002 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
marcozecchini 0:9fca2b23d0ba 1003 return "%d/%d"% (test_loop_ok_result, test_loop_count)
marcozecchini 0:9fca2b23d0ba 1004
marcozecchini 0:9fca2b23d0ba 1005 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
marcozecchini 0:9fca2b23d0ba 1006 """ Reformats list of results to simple string
marcozecchini 0:9fca2b23d0ba 1007 """
marcozecchini 0:9fca2b23d0ba 1008 result = self.TEST_RESULT_FAIL
marcozecchini 0:9fca2b23d0ba 1009
marcozecchini 0:9fca2b23d0ba 1010 if all(test_all_result[0] == res for res in test_all_result):
marcozecchini 0:9fca2b23d0ba 1011 result = test_all_result[0]
marcozecchini 0:9fca2b23d0ba 1012 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
marcozecchini 0:9fca2b23d0ba 1013 result = self.TEST_RESULT_OK
marcozecchini 0:9fca2b23d0ba 1014
marcozecchini 0:9fca2b23d0ba 1015 return result
marcozecchini 0:9fca2b23d0ba 1016
marcozecchini 0:9fca2b23d0ba 1017 def run_host_test(self, name, image_path, disk, port, duration,
marcozecchini 0:9fca2b23d0ba 1018 micro=None, reset=None, reset_tout=None,
marcozecchini 0:9fca2b23d0ba 1019 verbose=False, copy_method=None, program_cycle_s=None):
marcozecchini 0:9fca2b23d0ba 1020 """ Function creates new process with host test configured with particular test case.
marcozecchini 0:9fca2b23d0ba 1021 Function also is pooling for serial port activity from process to catch all data
marcozecchini 0:9fca2b23d0ba 1022 printed by test runner and host test during test execution
marcozecchini 0:9fca2b23d0ba 1023 """
marcozecchini 0:9fca2b23d0ba 1024
marcozecchini 0:9fca2b23d0ba 1025 def get_char_from_queue(obs):
marcozecchini 0:9fca2b23d0ba 1026 """ Get character from queue safe way
marcozecchini 0:9fca2b23d0ba 1027 """
marcozecchini 0:9fca2b23d0ba 1028 try:
marcozecchini 0:9fca2b23d0ba 1029 c = obs.queue.get(block=True, timeout=0.5)
marcozecchini 0:9fca2b23d0ba 1030 except Empty, _:
marcozecchini 0:9fca2b23d0ba 1031 c = None
marcozecchini 0:9fca2b23d0ba 1032 return c
marcozecchini 0:9fca2b23d0ba 1033
marcozecchini 0:9fca2b23d0ba 1034 def filter_queue_char(c):
marcozecchini 0:9fca2b23d0ba 1035 """ Filters out non ASCII characters from serial port
marcozecchini 0:9fca2b23d0ba 1036 """
marcozecchini 0:9fca2b23d0ba 1037 if ord(c) not in range(128):
marcozecchini 0:9fca2b23d0ba 1038 c = ' '
marcozecchini 0:9fca2b23d0ba 1039 return c
marcozecchini 0:9fca2b23d0ba 1040
marcozecchini 0:9fca2b23d0ba 1041 def get_test_result(output):
marcozecchini 0:9fca2b23d0ba 1042 """ Parse test 'output' data
marcozecchini 0:9fca2b23d0ba 1043 """
marcozecchini 0:9fca2b23d0ba 1044 result = self.TEST_RESULT_TIMEOUT
marcozecchini 0:9fca2b23d0ba 1045 for line in "".join(output).splitlines():
marcozecchini 0:9fca2b23d0ba 1046 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
marcozecchini 0:9fca2b23d0ba 1047 if search_result and len(search_result.groups()):
marcozecchini 0:9fca2b23d0ba 1048 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
marcozecchini 0:9fca2b23d0ba 1049 break
marcozecchini 0:9fca2b23d0ba 1050 return result
marcozecchini 0:9fca2b23d0ba 1051
marcozecchini 0:9fca2b23d0ba 1052 def get_auto_property_value(property_name, line):
marcozecchini 0:9fca2b23d0ba 1053 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
marcozecchini 0:9fca2b23d0ba 1054 Returns string
marcozecchini 0:9fca2b23d0ba 1055 """
marcozecchini 0:9fca2b23d0ba 1056 result = None
marcozecchini 0:9fca2b23d0ba 1057 if re.search("HOST: Property '%s'"% property_name, line) is not None:
marcozecchini 0:9fca2b23d0ba 1058 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
marcozecchini 0:9fca2b23d0ba 1059 if property is not None and len(property.groups()) == 1:
marcozecchini 0:9fca2b23d0ba 1060 result = property.groups()[0]
marcozecchini 0:9fca2b23d0ba 1061 return result
marcozecchini 0:9fca2b23d0ba 1062
marcozecchini 0:9fca2b23d0ba 1063 # print "{%s} port:%s disk:%s" % (name, port, disk),
marcozecchini 0:9fca2b23d0ba 1064 cmd = ["python",
marcozecchini 0:9fca2b23d0ba 1065 '%s.py'% name,
marcozecchini 0:9fca2b23d0ba 1066 '-d', disk,
marcozecchini 0:9fca2b23d0ba 1067 '-f', '"%s"'% image_path,
marcozecchini 0:9fca2b23d0ba 1068 '-p', port,
marcozecchini 0:9fca2b23d0ba 1069 '-t', str(duration),
marcozecchini 0:9fca2b23d0ba 1070 '-C', str(program_cycle_s)]
marcozecchini 0:9fca2b23d0ba 1071
marcozecchini 0:9fca2b23d0ba 1072 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
marcozecchini 0:9fca2b23d0ba 1073 cmd += ['--auto']
marcozecchini 0:9fca2b23d0ba 1074
marcozecchini 0:9fca2b23d0ba 1075 # Add extra parameters to host_test
marcozecchini 0:9fca2b23d0ba 1076 if copy_method is not None:
marcozecchini 0:9fca2b23d0ba 1077 cmd += ["-c", copy_method]
marcozecchini 0:9fca2b23d0ba 1078 if micro is not None:
marcozecchini 0:9fca2b23d0ba 1079 cmd += ["-m", micro]
marcozecchini 0:9fca2b23d0ba 1080 if reset is not None:
marcozecchini 0:9fca2b23d0ba 1081 cmd += ["-r", reset]
marcozecchini 0:9fca2b23d0ba 1082 if reset_tout is not None:
marcozecchini 0:9fca2b23d0ba 1083 cmd += ["-R", str(reset_tout)]
marcozecchini 0:9fca2b23d0ba 1084
marcozecchini 0:9fca2b23d0ba 1085 if verbose:
marcozecchini 0:9fca2b23d0ba 1086 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
marcozecchini 0:9fca2b23d0ba 1087 print "Test::Output::Start"
marcozecchini 0:9fca2b23d0ba 1088
marcozecchini 0:9fca2b23d0ba 1089 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
marcozecchini 0:9fca2b23d0ba 1090 obs = ProcessObserver(proc)
marcozecchini 0:9fca2b23d0ba 1091 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
marcozecchini 0:9fca2b23d0ba 1092 line = ''
marcozecchini 0:9fca2b23d0ba 1093 output = []
marcozecchini 0:9fca2b23d0ba 1094 start_time = time()
marcozecchini 0:9fca2b23d0ba 1095 while (time() - start_time) < (2 * duration):
marcozecchini 0:9fca2b23d0ba 1096 c = get_char_from_queue(obs)
marcozecchini 0:9fca2b23d0ba 1097 if c:
marcozecchini 0:9fca2b23d0ba 1098 if verbose:
marcozecchini 0:9fca2b23d0ba 1099 sys.stdout.write(c)
marcozecchini 0:9fca2b23d0ba 1100 c = filter_queue_char(c)
marcozecchini 0:9fca2b23d0ba 1101 output.append(c)
marcozecchini 0:9fca2b23d0ba 1102 # Give the mbed under test a way to communicate the end of the test
marcozecchini 0:9fca2b23d0ba 1103 if c in ['\n', '\r']:
marcozecchini 0:9fca2b23d0ba 1104
marcozecchini 0:9fca2b23d0ba 1105 # Checking for auto-detection information from the test about MUT reset moment
marcozecchini 0:9fca2b23d0ba 1106 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
marcozecchini 0:9fca2b23d0ba 1107 # We will update this marker only once to prevent multiple time resets
marcozecchini 0:9fca2b23d0ba 1108 update_once_flag['reset_target'] = True
marcozecchini 0:9fca2b23d0ba 1109 start_time = time()
marcozecchini 0:9fca2b23d0ba 1110
marcozecchini 0:9fca2b23d0ba 1111 # Checking for auto-detection information from the test about timeout
marcozecchini 0:9fca2b23d0ba 1112 auto_timeout_val = get_auto_property_value('timeout', line)
marcozecchini 0:9fca2b23d0ba 1113 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
marcozecchini 0:9fca2b23d0ba 1114 # We will update this marker only once to prevent multiple time resets
marcozecchini 0:9fca2b23d0ba 1115 update_once_flag['timeout'] = True
marcozecchini 0:9fca2b23d0ba 1116 duration = int(auto_timeout_val)
marcozecchini 0:9fca2b23d0ba 1117
marcozecchini 0:9fca2b23d0ba 1118 # Detect mbed assert:
marcozecchini 0:9fca2b23d0ba 1119 if 'mbed assertation failed: ' in line:
marcozecchini 0:9fca2b23d0ba 1120 output.append('{{mbed_assert}}')
marcozecchini 0:9fca2b23d0ba 1121 break
marcozecchini 0:9fca2b23d0ba 1122
marcozecchini 0:9fca2b23d0ba 1123 # Check for test end
marcozecchini 0:9fca2b23d0ba 1124 if '{end}' in line:
marcozecchini 0:9fca2b23d0ba 1125 break
marcozecchini 0:9fca2b23d0ba 1126 line = ''
marcozecchini 0:9fca2b23d0ba 1127 else:
marcozecchini 0:9fca2b23d0ba 1128 line += c
marcozecchini 0:9fca2b23d0ba 1129 end_time = time()
marcozecchini 0:9fca2b23d0ba 1130 testcase_duration = end_time - start_time # Test case duration from reset to {end}
marcozecchini 0:9fca2b23d0ba 1131
marcozecchini 0:9fca2b23d0ba 1132 c = get_char_from_queue(obs)
marcozecchini 0:9fca2b23d0ba 1133
marcozecchini 0:9fca2b23d0ba 1134 if c:
marcozecchini 0:9fca2b23d0ba 1135 if verbose:
marcozecchini 0:9fca2b23d0ba 1136 sys.stdout.write(c)
marcozecchini 0:9fca2b23d0ba 1137 c = filter_queue_char(c)
marcozecchini 0:9fca2b23d0ba 1138 output.append(c)
marcozecchini 0:9fca2b23d0ba 1139
marcozecchini 0:9fca2b23d0ba 1140 if verbose:
marcozecchini 0:9fca2b23d0ba 1141 print "Test::Output::Finish"
marcozecchini 0:9fca2b23d0ba 1142 # Stop test process
marcozecchini 0:9fca2b23d0ba 1143 obs.stop()
marcozecchini 0:9fca2b23d0ba 1144
marcozecchini 0:9fca2b23d0ba 1145 result = get_test_result(output)
marcozecchini 0:9fca2b23d0ba 1146 return (result, "".join(output), testcase_duration, duration)
marcozecchini 0:9fca2b23d0ba 1147
marcozecchini 0:9fca2b23d0ba 1148 def is_peripherals_available(self, target_mcu_name, peripherals=None):
marcozecchini 0:9fca2b23d0ba 1149 """ Checks if specified target should run specific peripheral test case defined in MUTs file
marcozecchini 0:9fca2b23d0ba 1150 """
marcozecchini 0:9fca2b23d0ba 1151 if peripherals is not None:
marcozecchini 0:9fca2b23d0ba 1152 peripherals = set(peripherals)
marcozecchini 0:9fca2b23d0ba 1153 for id, mut in self.muts.iteritems():
marcozecchini 0:9fca2b23d0ba 1154 # Target MCU name check
marcozecchini 0:9fca2b23d0ba 1155 if mut["mcu"] != target_mcu_name:
marcozecchini 0:9fca2b23d0ba 1156 continue
marcozecchini 0:9fca2b23d0ba 1157 # Peripherals check
marcozecchini 0:9fca2b23d0ba 1158 if peripherals is not None:
marcozecchini 0:9fca2b23d0ba 1159 if 'peripherals' not in mut:
marcozecchini 0:9fca2b23d0ba 1160 continue
marcozecchini 0:9fca2b23d0ba 1161 if not peripherals.issubset(set(mut['peripherals'])):
marcozecchini 0:9fca2b23d0ba 1162 continue
marcozecchini 0:9fca2b23d0ba 1163 return True
marcozecchini 0:9fca2b23d0ba 1164 return False
marcozecchini 0:9fca2b23d0ba 1165
marcozecchini 0:9fca2b23d0ba 1166 def shape_test_request(self, mcu, image_path, test_id, duration=10):
marcozecchini 0:9fca2b23d0ba 1167 """ Function prepares JSON structure describing test specification
marcozecchini 0:9fca2b23d0ba 1168 """
marcozecchini 0:9fca2b23d0ba 1169 test_spec = {
marcozecchini 0:9fca2b23d0ba 1170 "mcu": mcu,
marcozecchini 0:9fca2b23d0ba 1171 "image": image_path,
marcozecchini 0:9fca2b23d0ba 1172 "duration": duration,
marcozecchini 0:9fca2b23d0ba 1173 "test_id": test_id,
marcozecchini 0:9fca2b23d0ba 1174 }
marcozecchini 0:9fca2b23d0ba 1175 return json.dumps(test_spec)
marcozecchini 0:9fca2b23d0ba 1176
marcozecchini 0:9fca2b23d0ba 1177
marcozecchini 0:9fca2b23d0ba 1178 def get_unique_value_from_summary(test_summary, index):
marcozecchini 0:9fca2b23d0ba 1179 """ Gets list of unique target names
marcozecchini 0:9fca2b23d0ba 1180 """
marcozecchini 0:9fca2b23d0ba 1181 result = []
marcozecchini 0:9fca2b23d0ba 1182 for test in test_summary:
marcozecchini 0:9fca2b23d0ba 1183 target_name = test[index]
marcozecchini 0:9fca2b23d0ba 1184 if target_name not in result:
marcozecchini 0:9fca2b23d0ba 1185 result.append(target_name)
marcozecchini 0:9fca2b23d0ba 1186 return sorted(result)
marcozecchini 0:9fca2b23d0ba 1187
marcozecchini 0:9fca2b23d0ba 1188
marcozecchini 0:9fca2b23d0ba 1189 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
marcozecchini 0:9fca2b23d0ba 1190 """ Gets list of unique target names and return dictionary
marcozecchini 0:9fca2b23d0ba 1191 """
marcozecchini 0:9fca2b23d0ba 1192 result = {}
marcozecchini 0:9fca2b23d0ba 1193 for test in test_summary:
marcozecchini 0:9fca2b23d0ba 1194 key = test[index_key]
marcozecchini 0:9fca2b23d0ba 1195 val = test[index_val]
marcozecchini 0:9fca2b23d0ba 1196 if key not in result:
marcozecchini 0:9fca2b23d0ba 1197 result[key] = val
marcozecchini 0:9fca2b23d0ba 1198 return result
marcozecchini 0:9fca2b23d0ba 1199
marcozecchini 0:9fca2b23d0ba 1200
marcozecchini 0:9fca2b23d0ba 1201 def show_json_file_format_error(json_spec_filename, line, column):
marcozecchini 0:9fca2b23d0ba 1202 """ Prints JSON broken content
marcozecchini 0:9fca2b23d0ba 1203 """
marcozecchini 0:9fca2b23d0ba 1204 with open(json_spec_filename) as data_file:
marcozecchini 0:9fca2b23d0ba 1205 line_no = 1
marcozecchini 0:9fca2b23d0ba 1206 for json_line in data_file:
marcozecchini 0:9fca2b23d0ba 1207 if line_no + 5 >= line: # Print last few lines before error
marcozecchini 0:9fca2b23d0ba 1208 print 'Line %d:\t'%line_no + json_line, # Prints line
marcozecchini 0:9fca2b23d0ba 1209 if line_no == line:
marcozecchini 0:9fca2b23d0ba 1210 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
marcozecchini 0:9fca2b23d0ba 1211 break
marcozecchini 0:9fca2b23d0ba 1212 line_no += 1
marcozecchini 0:9fca2b23d0ba 1213
marcozecchini 0:9fca2b23d0ba 1214
marcozecchini 0:9fca2b23d0ba 1215 def json_format_error_defect_pos(json_error_msg):
marcozecchini 0:9fca2b23d0ba 1216 """ Gets first error line and column in JSON file format.
marcozecchini 0:9fca2b23d0ba 1217 Parsed from exception thrown by json.loads() string
marcozecchini 0:9fca2b23d0ba 1218 """
marcozecchini 0:9fca2b23d0ba 1219 result = None
marcozecchini 0:9fca2b23d0ba 1220 line, column = 0, 0
marcozecchini 0:9fca2b23d0ba 1221 # Line value search
marcozecchini 0:9fca2b23d0ba 1222 line_search = re.search('line [0-9]+', json_error_msg)
marcozecchini 0:9fca2b23d0ba 1223 if line_search is not None:
marcozecchini 0:9fca2b23d0ba 1224 ls = line_search.group().split(' ')
marcozecchini 0:9fca2b23d0ba 1225 if len(ls) == 2:
marcozecchini 0:9fca2b23d0ba 1226 line = int(ls[1])
marcozecchini 0:9fca2b23d0ba 1227 # Column position search
marcozecchini 0:9fca2b23d0ba 1228 column_search = re.search('column [0-9]+', json_error_msg)
marcozecchini 0:9fca2b23d0ba 1229 if column_search is not None:
marcozecchini 0:9fca2b23d0ba 1230 cs = column_search.group().split(' ')
marcozecchini 0:9fca2b23d0ba 1231 if len(cs) == 2:
marcozecchini 0:9fca2b23d0ba 1232 column = int(cs[1])
marcozecchini 0:9fca2b23d0ba 1233 result = [line, column]
marcozecchini 0:9fca2b23d0ba 1234 return result
marcozecchini 0:9fca2b23d0ba 1235
marcozecchini 0:9fca2b23d0ba 1236
marcozecchini 0:9fca2b23d0ba 1237 def get_json_data_from_file(json_spec_filename, verbose=False):
marcozecchini 0:9fca2b23d0ba 1238 """ Loads from file JSON formatted string to data structure
marcozecchini 0:9fca2b23d0ba 1239 """
marcozecchini 0:9fca2b23d0ba 1240 result = None
marcozecchini 0:9fca2b23d0ba 1241 try:
marcozecchini 0:9fca2b23d0ba 1242 with open(json_spec_filename) as data_file:
marcozecchini 0:9fca2b23d0ba 1243 try:
marcozecchini 0:9fca2b23d0ba 1244 result = json.load(data_file)
marcozecchini 0:9fca2b23d0ba 1245 except ValueError as json_error_msg:
marcozecchini 0:9fca2b23d0ba 1246 result = None
marcozecchini 0:9fca2b23d0ba 1247 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
marcozecchini 0:9fca2b23d0ba 1248 # We can print where error occurred inside JSON file if we can parse exception msg
marcozecchini 0:9fca2b23d0ba 1249 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
marcozecchini 0:9fca2b23d0ba 1250 if json_format_defect_pos is not None:
marcozecchini 0:9fca2b23d0ba 1251 line = json_format_defect_pos[0]
marcozecchini 0:9fca2b23d0ba 1252 column = json_format_defect_pos[1]
marcozecchini 0:9fca2b23d0ba 1253 print
marcozecchini 0:9fca2b23d0ba 1254 show_json_file_format_error(json_spec_filename, line, column)
marcozecchini 0:9fca2b23d0ba 1255
marcozecchini 0:9fca2b23d0ba 1256 except IOError as fileopen_error_msg:
marcozecchini 0:9fca2b23d0ba 1257 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
marcozecchini 0:9fca2b23d0ba 1258 print
marcozecchini 0:9fca2b23d0ba 1259 if verbose and result:
marcozecchini 0:9fca2b23d0ba 1260 pp = pprint.PrettyPrinter(indent=4)
marcozecchini 0:9fca2b23d0ba 1261 pp.pprint(result)
marcozecchini 0:9fca2b23d0ba 1262 return result
marcozecchini 0:9fca2b23d0ba 1263
marcozecchini 0:9fca2b23d0ba 1264
marcozecchini 0:9fca2b23d0ba 1265 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
marcozecchini 0:9fca2b23d0ba 1266 """ Prints MUTs configuration passed to test script for verboseness
marcozecchini 0:9fca2b23d0ba 1267 """
marcozecchini 0:9fca2b23d0ba 1268 muts_info_cols = []
marcozecchini 0:9fca2b23d0ba 1269 # We need to check all unique properties for each defined MUT
marcozecchini 0:9fca2b23d0ba 1270 for k in json_data:
marcozecchini 0:9fca2b23d0ba 1271 mut_info = json_data[k]
marcozecchini 0:9fca2b23d0ba 1272 for mut_property in mut_info:
marcozecchini 0:9fca2b23d0ba 1273 if mut_property not in muts_info_cols:
marcozecchini 0:9fca2b23d0ba 1274 muts_info_cols.append(mut_property)
marcozecchini 0:9fca2b23d0ba 1275
marcozecchini 0:9fca2b23d0ba 1276 # Prepare pretty table object to display all MUTs
marcozecchini 0:9fca2b23d0ba 1277 pt_cols = ["index"] + muts_info_cols
marcozecchini 0:9fca2b23d0ba 1278 pt = PrettyTable(pt_cols)
marcozecchini 0:9fca2b23d0ba 1279 for col in pt_cols:
marcozecchini 0:9fca2b23d0ba 1280 pt.align[col] = "l"
marcozecchini 0:9fca2b23d0ba 1281
marcozecchini 0:9fca2b23d0ba 1282 # Add rows to pretty print object
marcozecchini 0:9fca2b23d0ba 1283 for k in json_data:
marcozecchini 0:9fca2b23d0ba 1284 row = [k]
marcozecchini 0:9fca2b23d0ba 1285 mut_info = json_data[k]
marcozecchini 0:9fca2b23d0ba 1286
marcozecchini 0:9fca2b23d0ba 1287 add_row = True
marcozecchini 0:9fca2b23d0ba 1288 if platform_filter and 'mcu' in mut_info:
marcozecchini 0:9fca2b23d0ba 1289 add_row = re.search(platform_filter, mut_info['mcu']) is not None
marcozecchini 0:9fca2b23d0ba 1290 if add_row:
marcozecchini 0:9fca2b23d0ba 1291 for col in muts_info_cols:
marcozecchini 0:9fca2b23d0ba 1292 cell_val = mut_info[col] if col in mut_info else None
marcozecchini 0:9fca2b23d0ba 1293 if type(cell_val) == ListType:
marcozecchini 0:9fca2b23d0ba 1294 cell_val = join_delim.join(cell_val)
marcozecchini 0:9fca2b23d0ba 1295 row.append(cell_val)
marcozecchini 0:9fca2b23d0ba 1296 pt.add_row(row)
marcozecchini 0:9fca2b23d0ba 1297 return pt.get_string()
marcozecchini 0:9fca2b23d0ba 1298
marcozecchini 0:9fca2b23d0ba 1299
marcozecchini 0:9fca2b23d0ba 1300 def print_test_configuration_from_json(json_data, join_delim=", "):
marcozecchini 0:9fca2b23d0ba 1301 """ Prints test specification configuration passed to test script for verboseness
marcozecchini 0:9fca2b23d0ba 1302 """
marcozecchini 0:9fca2b23d0ba 1303 toolchains_info_cols = []
marcozecchini 0:9fca2b23d0ba 1304 # We need to check all toolchains for each device
marcozecchini 0:9fca2b23d0ba 1305 for k in json_data:
marcozecchini 0:9fca2b23d0ba 1306 # k should be 'targets'
marcozecchini 0:9fca2b23d0ba 1307 targets = json_data[k]
marcozecchini 0:9fca2b23d0ba 1308 for target in targets:
marcozecchini 0:9fca2b23d0ba 1309 toolchains = targets[target]
marcozecchini 0:9fca2b23d0ba 1310 for toolchain in toolchains:
marcozecchini 0:9fca2b23d0ba 1311 if toolchain not in toolchains_info_cols:
marcozecchini 0:9fca2b23d0ba 1312 toolchains_info_cols.append(toolchain)
marcozecchini 0:9fca2b23d0ba 1313
marcozecchini 0:9fca2b23d0ba 1314 # Prepare pretty table object to display test specification
marcozecchini 0:9fca2b23d0ba 1315 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
marcozecchini 0:9fca2b23d0ba 1316 pt = PrettyTable(pt_cols)
marcozecchini 0:9fca2b23d0ba 1317 for col in pt_cols:
marcozecchini 0:9fca2b23d0ba 1318 pt.align[col] = "l"
marcozecchini 0:9fca2b23d0ba 1319
marcozecchini 0:9fca2b23d0ba 1320 # { target : [conflicted toolchains] }
marcozecchini 0:9fca2b23d0ba 1321 toolchain_conflicts = {}
marcozecchini 0:9fca2b23d0ba 1322 toolchain_path_conflicts = []
marcozecchini 0:9fca2b23d0ba 1323 for k in json_data:
marcozecchini 0:9fca2b23d0ba 1324 # k should be 'targets'
marcozecchini 0:9fca2b23d0ba 1325 targets = json_data[k]
marcozecchini 0:9fca2b23d0ba 1326 for target in targets:
marcozecchini 0:9fca2b23d0ba 1327 target_supported_toolchains = get_target_supported_toolchains(target)
marcozecchini 0:9fca2b23d0ba 1328 if not target_supported_toolchains:
marcozecchini 0:9fca2b23d0ba 1329 target_supported_toolchains = []
marcozecchini 0:9fca2b23d0ba 1330 target_name = target if target in TARGET_MAP else "%s*"% target
marcozecchini 0:9fca2b23d0ba 1331 row = [target_name]
marcozecchini 0:9fca2b23d0ba 1332 toolchains = targets[target]
marcozecchini 0:9fca2b23d0ba 1333
marcozecchini 0:9fca2b23d0ba 1334 for toolchain in sorted(toolchains_info_cols):
marcozecchini 0:9fca2b23d0ba 1335 # Check for conflicts: target vs toolchain
marcozecchini 0:9fca2b23d0ba 1336 conflict = False
marcozecchini 0:9fca2b23d0ba 1337 conflict_path = False
marcozecchini 0:9fca2b23d0ba 1338 if toolchain in toolchains:
marcozecchini 0:9fca2b23d0ba 1339 if toolchain not in target_supported_toolchains:
marcozecchini 0:9fca2b23d0ba 1340 conflict = True
marcozecchini 0:9fca2b23d0ba 1341 if target not in toolchain_conflicts:
marcozecchini 0:9fca2b23d0ba 1342 toolchain_conflicts[target] = []
marcozecchini 0:9fca2b23d0ba 1343 toolchain_conflicts[target].append(toolchain)
marcozecchini 0:9fca2b23d0ba 1344 # Add marker inside table about target usage / conflict
marcozecchini 0:9fca2b23d0ba 1345 cell_val = 'Yes' if toolchain in toolchains else '-'
marcozecchini 0:9fca2b23d0ba 1346 if conflict:
marcozecchini 0:9fca2b23d0ba 1347 cell_val += '*'
marcozecchini 0:9fca2b23d0ba 1348 # Check for conflicts: toolchain vs toolchain path
marcozecchini 0:9fca2b23d0ba 1349 if toolchain in TOOLCHAIN_PATHS:
marcozecchini 0:9fca2b23d0ba 1350 toolchain_path = TOOLCHAIN_PATHS[toolchain]
marcozecchini 0:9fca2b23d0ba 1351 if not os.path.isdir(toolchain_path):
marcozecchini 0:9fca2b23d0ba 1352 conflict_path = True
marcozecchini 0:9fca2b23d0ba 1353 if toolchain not in toolchain_path_conflicts:
marcozecchini 0:9fca2b23d0ba 1354 toolchain_path_conflicts.append(toolchain)
marcozecchini 0:9fca2b23d0ba 1355 if conflict_path:
marcozecchini 0:9fca2b23d0ba 1356 cell_val += '#'
marcozecchini 0:9fca2b23d0ba 1357 row.append(cell_val)
marcozecchini 0:9fca2b23d0ba 1358 pt.add_row(row)
marcozecchini 0:9fca2b23d0ba 1359
marcozecchini 0:9fca2b23d0ba 1360 # generate result string
marcozecchini 0:9fca2b23d0ba 1361 result = pt.get_string() # Test specification table
marcozecchini 0:9fca2b23d0ba 1362 if toolchain_conflicts or toolchain_path_conflicts:
marcozecchini 0:9fca2b23d0ba 1363 result += "\n"
marcozecchini 0:9fca2b23d0ba 1364 result += "Toolchain conflicts:\n"
marcozecchini 0:9fca2b23d0ba 1365 for target in toolchain_conflicts:
marcozecchini 0:9fca2b23d0ba 1366 if target not in TARGET_MAP:
marcozecchini 0:9fca2b23d0ba 1367 result += "\t* Target %s unknown\n"% (target)
marcozecchini 0:9fca2b23d0ba 1368 conflict_target_list = join_delim.join(toolchain_conflicts[target])
marcozecchini 0:9fca2b23d0ba 1369 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
marcozecchini 0:9fca2b23d0ba 1370 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
marcozecchini 0:9fca2b23d0ba 1371
marcozecchini 0:9fca2b23d0ba 1372 for toolchain in toolchain_path_conflicts:
marcozecchini 0:9fca2b23d0ba 1373 # Let's check toolchain configuration
marcozecchini 0:9fca2b23d0ba 1374 if toolchain in TOOLCHAIN_PATHS:
marcozecchini 0:9fca2b23d0ba 1375 toolchain_path = TOOLCHAIN_PATHS[toolchain]
marcozecchini 0:9fca2b23d0ba 1376 if not os.path.isdir(toolchain_path):
marcozecchini 0:9fca2b23d0ba 1377 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
marcozecchini 0:9fca2b23d0ba 1378 return result
marcozecchini 0:9fca2b23d0ba 1379
marcozecchini 0:9fca2b23d0ba 1380
marcozecchini 0:9fca2b23d0ba 1381 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
marcozecchini 0:9fca2b23d0ba 1382 """ Generates table summary with all test cases and additional test cases
marcozecchini 0:9fca2b23d0ba 1383 information using pretty print functionality. Allows test suite user to
marcozecchini 0:9fca2b23d0ba 1384 see test cases
marcozecchini 0:9fca2b23d0ba 1385 """
marcozecchini 0:9fca2b23d0ba 1386 # get all unique test ID prefixes
marcozecchini 0:9fca2b23d0ba 1387 unique_test_id = []
marcozecchini 0:9fca2b23d0ba 1388 for test in TESTS:
marcozecchini 0:9fca2b23d0ba 1389 split = test['id'].split('_')[:-1]
marcozecchini 0:9fca2b23d0ba 1390 test_id_prefix = '_'.join(split)
marcozecchini 0:9fca2b23d0ba 1391 if test_id_prefix not in unique_test_id:
marcozecchini 0:9fca2b23d0ba 1392 unique_test_id.append(test_id_prefix)
marcozecchini 0:9fca2b23d0ba 1393 unique_test_id.sort()
marcozecchini 0:9fca2b23d0ba 1394 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
marcozecchini 0:9fca2b23d0ba 1395 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
marcozecchini 0:9fca2b23d0ba 1396
marcozecchini 0:9fca2b23d0ba 1397 test_properties = ['id',
marcozecchini 0:9fca2b23d0ba 1398 'automated',
marcozecchini 0:9fca2b23d0ba 1399 'description',
marcozecchini 0:9fca2b23d0ba 1400 'peripherals',
marcozecchini 0:9fca2b23d0ba 1401 'host_test',
marcozecchini 0:9fca2b23d0ba 1402 'duration'] if cols is None else cols
marcozecchini 0:9fca2b23d0ba 1403
marcozecchini 0:9fca2b23d0ba 1404 # All tests status table print
marcozecchini 0:9fca2b23d0ba 1405 pt = PrettyTable(test_properties)
marcozecchini 0:9fca2b23d0ba 1406 for col in test_properties:
marcozecchini 0:9fca2b23d0ba 1407 pt.align[col] = "l"
marcozecchini 0:9fca2b23d0ba 1408 pt.align['duration'] = "r"
marcozecchini 0:9fca2b23d0ba 1409
marcozecchini 0:9fca2b23d0ba 1410 counter_all = 0
marcozecchini 0:9fca2b23d0ba 1411 counter_automated = 0
marcozecchini 0:9fca2b23d0ba 1412 pt.padding_width = 1 # One space between column edges and contents (default)
marcozecchini 0:9fca2b23d0ba 1413
marcozecchini 0:9fca2b23d0ba 1414 for test_id in sorted(TEST_MAP.keys()):
marcozecchini 0:9fca2b23d0ba 1415 if platform_filter is not None:
marcozecchini 0:9fca2b23d0ba 1416 # FIlter out platforms using regex
marcozecchini 0:9fca2b23d0ba 1417 if re.search(platform_filter, test_id) is None:
marcozecchini 0:9fca2b23d0ba 1418 continue
marcozecchini 0:9fca2b23d0ba 1419 row = []
marcozecchini 0:9fca2b23d0ba 1420 test = TEST_MAP[test_id]
marcozecchini 0:9fca2b23d0ba 1421 split = test_id.split('_')[:-1]
marcozecchini 0:9fca2b23d0ba 1422 test_id_prefix = '_'.join(split)
marcozecchini 0:9fca2b23d0ba 1423
marcozecchini 0:9fca2b23d0ba 1424 for col in test_properties:
marcozecchini 0:9fca2b23d0ba 1425 col_value = test[col]
marcozecchini 0:9fca2b23d0ba 1426 if type(test[col]) == ListType:
marcozecchini 0:9fca2b23d0ba 1427 col_value = join_delim.join(test[col])
marcozecchini 0:9fca2b23d0ba 1428 elif test[col] == None:
marcozecchini 0:9fca2b23d0ba 1429 col_value = "-"
marcozecchini 0:9fca2b23d0ba 1430
marcozecchini 0:9fca2b23d0ba 1431 row.append(col_value)
marcozecchini 0:9fca2b23d0ba 1432 if test['automated'] == True:
marcozecchini 0:9fca2b23d0ba 1433 counter_dict_test_id_types[test_id_prefix] += 1
marcozecchini 0:9fca2b23d0ba 1434 counter_automated += 1
marcozecchini 0:9fca2b23d0ba 1435 pt.add_row(row)
marcozecchini 0:9fca2b23d0ba 1436 # Update counters
marcozecchini 0:9fca2b23d0ba 1437 counter_all += 1
marcozecchini 0:9fca2b23d0ba 1438 counter_dict_test_id_types_all[test_id_prefix] += 1
marcozecchini 0:9fca2b23d0ba 1439 result = pt.get_string()
marcozecchini 0:9fca2b23d0ba 1440 result += "\n\n"
marcozecchini 0:9fca2b23d0ba 1441
marcozecchini 0:9fca2b23d0ba 1442 if result_summary and not platform_filter:
marcozecchini 0:9fca2b23d0ba 1443 # Automation result summary
marcozecchini 0:9fca2b23d0ba 1444 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
marcozecchini 0:9fca2b23d0ba 1445 pt = PrettyTable(test_id_cols)
marcozecchini 0:9fca2b23d0ba 1446 pt.align['automated'] = "r"
marcozecchini 0:9fca2b23d0ba 1447 pt.align['all'] = "r"
marcozecchini 0:9fca2b23d0ba 1448 pt.align['percent [%]'] = "r"
marcozecchini 0:9fca2b23d0ba 1449
marcozecchini 0:9fca2b23d0ba 1450 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
marcozecchini 0:9fca2b23d0ba 1451 str_progress = progress_bar(percent_progress, 75)
marcozecchini 0:9fca2b23d0ba 1452 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
marcozecchini 0:9fca2b23d0ba 1453 result += "Automation coverage:\n"
marcozecchini 0:9fca2b23d0ba 1454 result += pt.get_string()
marcozecchini 0:9fca2b23d0ba 1455 result += "\n\n"
marcozecchini 0:9fca2b23d0ba 1456
marcozecchini 0:9fca2b23d0ba 1457 # Test automation coverage table print
marcozecchini 0:9fca2b23d0ba 1458 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
marcozecchini 0:9fca2b23d0ba 1459 pt = PrettyTable(test_id_cols)
marcozecchini 0:9fca2b23d0ba 1460 pt.align['id'] = "l"
marcozecchini 0:9fca2b23d0ba 1461 pt.align['automated'] = "r"
marcozecchini 0:9fca2b23d0ba 1462 pt.align['all'] = "r"
marcozecchini 0:9fca2b23d0ba 1463 pt.align['percent [%]'] = "r"
marcozecchini 0:9fca2b23d0ba 1464 for unique_id in unique_test_id:
marcozecchini 0:9fca2b23d0ba 1465 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
marcozecchini 0:9fca2b23d0ba 1466 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
marcozecchini 0:9fca2b23d0ba 1467 str_progress = progress_bar(percent_progress, 75)
marcozecchini 0:9fca2b23d0ba 1468 row = [unique_id,
marcozecchini 0:9fca2b23d0ba 1469 counter_dict_test_id_types[unique_id],
marcozecchini 0:9fca2b23d0ba 1470 counter_dict_test_id_types_all[unique_id],
marcozecchini 0:9fca2b23d0ba 1471 percent_progress,
marcozecchini 0:9fca2b23d0ba 1472 "[" + str_progress + "]"]
marcozecchini 0:9fca2b23d0ba 1473 pt.add_row(row)
marcozecchini 0:9fca2b23d0ba 1474 result += "Test automation coverage:\n"
marcozecchini 0:9fca2b23d0ba 1475 result += pt.get_string()
marcozecchini 0:9fca2b23d0ba 1476 result += "\n\n"
marcozecchini 0:9fca2b23d0ba 1477 return result
marcozecchini 0:9fca2b23d0ba 1478
marcozecchini 0:9fca2b23d0ba 1479
marcozecchini 0:9fca2b23d0ba 1480 def progress_bar(percent_progress, saturation=0):
marcozecchini 0:9fca2b23d0ba 1481 """ This function creates progress bar with optional simple saturation mark
marcozecchini 0:9fca2b23d0ba 1482 """
marcozecchini 0:9fca2b23d0ba 1483 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
marcozecchini 0:9fca2b23d0ba 1484 str_progress = '#' * step + '.' * int(50 - step)
marcozecchini 0:9fca2b23d0ba 1485 c = '!' if str_progress[38] == '.' else '|'
marcozecchini 0:9fca2b23d0ba 1486 if saturation > 0:
marcozecchini 0:9fca2b23d0ba 1487 saturation = saturation / 2
marcozecchini 0:9fca2b23d0ba 1488 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
marcozecchini 0:9fca2b23d0ba 1489 return str_progress
marcozecchini 0:9fca2b23d0ba 1490
marcozecchini 0:9fca2b23d0ba 1491
marcozecchini 0:9fca2b23d0ba 1492 def singletest_in_cli_mode(single_test):
marcozecchini 0:9fca2b23d0ba 1493 """ Runs SingleTestRunner object in CLI (Command line interface) mode
marcozecchini 0:9fca2b23d0ba 1494
marcozecchini 0:9fca2b23d0ba 1495 @return returns success code (0 == success) for building and running tests
marcozecchini 0:9fca2b23d0ba 1496 """
marcozecchini 0:9fca2b23d0ba 1497 start = time()
marcozecchini 0:9fca2b23d0ba 1498 # Execute tests depending on options and filter applied
marcozecchini 0:9fca2b23d0ba 1499 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
marcozecchini 0:9fca2b23d0ba 1500 elapsed_time = time() - start
marcozecchini 0:9fca2b23d0ba 1501
marcozecchini 0:9fca2b23d0ba 1502 # Human readable summary
marcozecchini 0:9fca2b23d0ba 1503 if not single_test.opts_suppress_summary:
marcozecchini 0:9fca2b23d0ba 1504 # prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 1505 print single_test.generate_test_summary(test_summary, shuffle_seed)
marcozecchini 0:9fca2b23d0ba 1506 if single_test.opts_test_x_toolchain_summary:
marcozecchini 0:9fca2b23d0ba 1507 # prints well-formed summary with results (SQL table like)
marcozecchini 0:9fca2b23d0ba 1508 # table shows text x toolchain test result matrix
marcozecchini 0:9fca2b23d0ba 1509 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
marcozecchini 0:9fca2b23d0ba 1510
marcozecchini 0:9fca2b23d0ba 1511 print "Completed in %.2f sec"% (elapsed_time)
marcozecchini 0:9fca2b23d0ba 1512 print
marcozecchini 0:9fca2b23d0ba 1513 # Write summary of the builds
marcozecchini 0:9fca2b23d0ba 1514
marcozecchini 0:9fca2b23d0ba 1515 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
marcozecchini 0:9fca2b23d0ba 1516 status = print_report_exporter.report(build_report)
marcozecchini 0:9fca2b23d0ba 1517
marcozecchini 0:9fca2b23d0ba 1518 # Store extra reports in files
marcozecchini 0:9fca2b23d0ba 1519 if single_test.opts_report_html_file_name:
marcozecchini 0:9fca2b23d0ba 1520 # Export results in form of HTML report to separate file
marcozecchini 0:9fca2b23d0ba 1521 report_exporter = ReportExporter(ResultExporterType.HTML)
marcozecchini 0:9fca2b23d0ba 1522 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
marcozecchini 0:9fca2b23d0ba 1523 if single_test.opts_report_junit_file_name:
marcozecchini 0:9fca2b23d0ba 1524 # Export results in form of JUnit XML report to separate file
marcozecchini 0:9fca2b23d0ba 1525 report_exporter = ReportExporter(ResultExporterType.JUNIT)
marcozecchini 0:9fca2b23d0ba 1526 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
marcozecchini 0:9fca2b23d0ba 1527 if single_test.opts_report_text_file_name:
marcozecchini 0:9fca2b23d0ba 1528 # Export results in form of a text file
marcozecchini 0:9fca2b23d0ba 1529 report_exporter = ReportExporter(ResultExporterType.TEXT)
marcozecchini 0:9fca2b23d0ba 1530 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
marcozecchini 0:9fca2b23d0ba 1531 if single_test.opts_report_build_file_name:
marcozecchini 0:9fca2b23d0ba 1532 # Export build results as html report to sparate file
marcozecchini 0:9fca2b23d0ba 1533 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
marcozecchini 0:9fca2b23d0ba 1534 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
marcozecchini 0:9fca2b23d0ba 1535
marcozecchini 0:9fca2b23d0ba 1536 # Returns True if no build failures of the test projects or their dependencies
marcozecchini 0:9fca2b23d0ba 1537 return status
marcozecchini 0:9fca2b23d0ba 1538
marcozecchini 0:9fca2b23d0ba 1539 class TestLogger():
marcozecchini 0:9fca2b23d0ba 1540 """ Super-class for logging and printing ongoing events for test suite pass
marcozecchini 0:9fca2b23d0ba 1541 """
marcozecchini 0:9fca2b23d0ba 1542 def __init__(self, store_log=True):
marcozecchini 0:9fca2b23d0ba 1543 """ We can control if logger actually stores log in memory
marcozecchini 0:9fca2b23d0ba 1544 or just handled all log entries immediately
marcozecchini 0:9fca2b23d0ba 1545 """
marcozecchini 0:9fca2b23d0ba 1546 self.log = []
marcozecchini 0:9fca2b23d0ba 1547 self.log_to_file = False
marcozecchini 0:9fca2b23d0ba 1548 self.log_file_name = None
marcozecchini 0:9fca2b23d0ba 1549 self.store_log = store_log
marcozecchini 0:9fca2b23d0ba 1550
marcozecchini 0:9fca2b23d0ba 1551 self.LogType = construct_enum(INFO='Info',
marcozecchini 0:9fca2b23d0ba 1552 WARN='Warning',
marcozecchini 0:9fca2b23d0ba 1553 NOTIF='Notification',
marcozecchini 0:9fca2b23d0ba 1554 ERROR='Error',
marcozecchini 0:9fca2b23d0ba 1555 EXCEPT='Exception')
marcozecchini 0:9fca2b23d0ba 1556
marcozecchini 0:9fca2b23d0ba 1557 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
marcozecchini 0:9fca2b23d0ba 1558 APPEND=2) # Append to existing log file
marcozecchini 0:9fca2b23d0ba 1559
marcozecchini 0:9fca2b23d0ba 1560 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
marcozecchini 0:9fca2b23d0ba 1561 """ Log one line of text
marcozecchini 0:9fca2b23d0ba 1562 """
marcozecchini 0:9fca2b23d0ba 1563 log_timestamp = time()
marcozecchini 0:9fca2b23d0ba 1564 log_entry = {'log_type' : LogType,
marcozecchini 0:9fca2b23d0ba 1565 'log_timestamp' : log_timestamp,
marcozecchini 0:9fca2b23d0ba 1566 'log_line' : log_line,
marcozecchini 0:9fca2b23d0ba 1567 '_future' : None
marcozecchini 0:9fca2b23d0ba 1568 }
marcozecchini 0:9fca2b23d0ba 1569 # Store log in memory
marcozecchini 0:9fca2b23d0ba 1570 if self.store_log:
marcozecchini 0:9fca2b23d0ba 1571 self.log.append(log_entry)
marcozecchini 0:9fca2b23d0ba 1572 return log_entry
marcozecchini 0:9fca2b23d0ba 1573
marcozecchini 0:9fca2b23d0ba 1574
marcozecchini 0:9fca2b23d0ba 1575 class CLITestLogger(TestLogger):
marcozecchini 0:9fca2b23d0ba 1576 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
marcozecchini 0:9fca2b23d0ba 1577 """
marcozecchini 0:9fca2b23d0ba 1578 def __init__(self, store_log=True, file_name=None):
marcozecchini 0:9fca2b23d0ba 1579 TestLogger.__init__(self)
marcozecchini 0:9fca2b23d0ba 1580 self.log_file_name = file_name
marcozecchini 0:9fca2b23d0ba 1581 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
marcozecchini 0:9fca2b23d0ba 1582 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
marcozecchini 0:9fca2b23d0ba 1583
marcozecchini 0:9fca2b23d0ba 1584 def log_print(self, log_entry, timestamp=True):
marcozecchini 0:9fca2b23d0ba 1585 """ Prints on screen formatted log entry
marcozecchini 0:9fca2b23d0ba 1586 """
marcozecchini 0:9fca2b23d0ba 1587 ts = log_entry['log_timestamp']
marcozecchini 0:9fca2b23d0ba 1588 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
marcozecchini 0:9fca2b23d0ba 1589 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
marcozecchini 0:9fca2b23d0ba 1590 return timestamp_str + log_line_str
marcozecchini 0:9fca2b23d0ba 1591
marcozecchini 0:9fca2b23d0ba 1592 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
marcozecchini 0:9fca2b23d0ba 1593 """ Logs line, if log file output was specified log line will be appended
marcozecchini 0:9fca2b23d0ba 1594 at the end of log file
marcozecchini 0:9fca2b23d0ba 1595 """
marcozecchini 0:9fca2b23d0ba 1596 log_entry = TestLogger.log_line(self, LogType, log_line)
marcozecchini 0:9fca2b23d0ba 1597 log_line_str = self.log_print(log_entry, timestamp)
marcozecchini 0:9fca2b23d0ba 1598 if self.log_file_name is not None:
marcozecchini 0:9fca2b23d0ba 1599 try:
marcozecchini 0:9fca2b23d0ba 1600 with open(self.log_file_name, 'a') as f:
marcozecchini 0:9fca2b23d0ba 1601 f.write(log_line_str + line_delim)
marcozecchini 0:9fca2b23d0ba 1602 except IOError:
marcozecchini 0:9fca2b23d0ba 1603 pass
marcozecchini 0:9fca2b23d0ba 1604 return log_line_str
marcozecchini 0:9fca2b23d0ba 1605
marcozecchini 0:9fca2b23d0ba 1606
marcozecchini 0:9fca2b23d0ba 1607 def factory_db_logger(db_url):
marcozecchini 0:9fca2b23d0ba 1608 """ Factory database driver depending on database type supplied in database connection string db_url
marcozecchini 0:9fca2b23d0ba 1609 """
marcozecchini 0:9fca2b23d0ba 1610 if db_url is not None:
marcozecchini 0:9fca2b23d0ba 1611 from tools.test_mysql import MySQLDBAccess
marcozecchini 0:9fca2b23d0ba 1612 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
marcozecchini 0:9fca2b23d0ba 1613 if connection_info is not None:
marcozecchini 0:9fca2b23d0ba 1614 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
marcozecchini 0:9fca2b23d0ba 1615 if db_type == 'mysql':
marcozecchini 0:9fca2b23d0ba 1616 return MySQLDBAccess()
marcozecchini 0:9fca2b23d0ba 1617 return None
marcozecchini 0:9fca2b23d0ba 1618
marcozecchini 0:9fca2b23d0ba 1619
marcozecchini 0:9fca2b23d0ba 1620 def detect_database_verbose(db_url):
marcozecchini 0:9fca2b23d0ba 1621 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
marcozecchini 0:9fca2b23d0ba 1622 """
marcozecchini 0:9fca2b23d0ba 1623 result = BaseDBAccess().parse_db_connection_string(db_url)
marcozecchini 0:9fca2b23d0ba 1624 if result is not None:
marcozecchini 0:9fca2b23d0ba 1625 # Parsing passed
marcozecchini 0:9fca2b23d0ba 1626 (db_type, username, password, host, db_name) = result
marcozecchini 0:9fca2b23d0ba 1627 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
marcozecchini 0:9fca2b23d0ba 1628 # Let's try to connect
marcozecchini 0:9fca2b23d0ba 1629 db_ = factory_db_logger(db_url)
marcozecchini 0:9fca2b23d0ba 1630 if db_ is not None:
marcozecchini 0:9fca2b23d0ba 1631 print "Connecting to database '%s'..."% db_url,
marcozecchini 0:9fca2b23d0ba 1632 db_.connect(host, username, password, db_name)
marcozecchini 0:9fca2b23d0ba 1633 if db_.is_connected():
marcozecchini 0:9fca2b23d0ba 1634 print "ok"
marcozecchini 0:9fca2b23d0ba 1635 print "Detecting database..."
marcozecchini 0:9fca2b23d0ba 1636 print db_.detect_database(verbose=True)
marcozecchini 0:9fca2b23d0ba 1637 print "Disconnecting...",
marcozecchini 0:9fca2b23d0ba 1638 db_.disconnect()
marcozecchini 0:9fca2b23d0ba 1639 print "done"
marcozecchini 0:9fca2b23d0ba 1640 else:
marcozecchini 0:9fca2b23d0ba 1641 print "Database type '%s' unknown"% db_type
marcozecchini 0:9fca2b23d0ba 1642 else:
marcozecchini 0:9fca2b23d0ba 1643 print "Parse error: '%s' - DB Url error"% (db_url)
marcozecchini 0:9fca2b23d0ba 1644
marcozecchini 0:9fca2b23d0ba 1645
marcozecchini 0:9fca2b23d0ba 1646 def get_module_avail(module_name):
marcozecchini 0:9fca2b23d0ba 1647 """ This function returns True if module_name is already imported module
marcozecchini 0:9fca2b23d0ba 1648 """
marcozecchini 0:9fca2b23d0ba 1649 return module_name in sys.modules.keys()
marcozecchini 0:9fca2b23d0ba 1650
marcozecchini 0:9fca2b23d0ba 1651 def get_autodetected_MUTS_list(platform_name_filter=None):
marcozecchini 0:9fca2b23d0ba 1652 oldError = None
marcozecchini 0:9fca2b23d0ba 1653 if os.name == 'nt':
marcozecchini 0:9fca2b23d0ba 1654 # Disable Windows error box temporarily
marcozecchini 0:9fca2b23d0ba 1655 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
marcozecchini 0:9fca2b23d0ba 1656
marcozecchini 0:9fca2b23d0ba 1657 mbeds = mbed_lstools.create()
marcozecchini 0:9fca2b23d0ba 1658 detect_muts_list = mbeds.list_mbeds()
marcozecchini 0:9fca2b23d0ba 1659
marcozecchini 0:9fca2b23d0ba 1660 if os.name == 'nt':
marcozecchini 0:9fca2b23d0ba 1661 ctypes.windll.kernel32.SetErrorMode(oldError)
marcozecchini 0:9fca2b23d0ba 1662
marcozecchini 0:9fca2b23d0ba 1663 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
marcozecchini 0:9fca2b23d0ba 1664
marcozecchini 0:9fca2b23d0ba 1665 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
marcozecchini 0:9fca2b23d0ba 1666 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
marcozecchini 0:9fca2b23d0ba 1667 If function fails to auto-detect devices it will return empty dictionary.
marcozecchini 0:9fca2b23d0ba 1668
marcozecchini 0:9fca2b23d0ba 1669 if get_module_avail('mbed_lstools'):
marcozecchini 0:9fca2b23d0ba 1670 mbeds = mbed_lstools.create()
marcozecchini 0:9fca2b23d0ba 1671 mbeds_list = mbeds.list_mbeds()
marcozecchini 0:9fca2b23d0ba 1672
marcozecchini 0:9fca2b23d0ba 1673 @param mbeds_list list of mbeds captured from mbed_lstools
marcozecchini 0:9fca2b23d0ba 1674 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
marcozecchini 0:9fca2b23d0ba 1675 """
marcozecchini 0:9fca2b23d0ba 1676 result = {} # Should be in muts_all.json format
marcozecchini 0:9fca2b23d0ba 1677 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
marcozecchini 0:9fca2b23d0ba 1678 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
marcozecchini 0:9fca2b23d0ba 1679 index = 1
marcozecchini 0:9fca2b23d0ba 1680 for mut in mbeds_list:
marcozecchini 0:9fca2b23d0ba 1681 # Filter the MUTS if a filter is specified
marcozecchini 0:9fca2b23d0ba 1682
marcozecchini 0:9fca2b23d0ba 1683 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
marcozecchini 0:9fca2b23d0ba 1684 continue
marcozecchini 0:9fca2b23d0ba 1685
marcozecchini 0:9fca2b23d0ba 1686 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
marcozecchini 0:9fca2b23d0ba 1687 # if not we are creating our own unique value (last few chars from platform's target_id).
marcozecchini 0:9fca2b23d0ba 1688 m = {'mcu': mut['platform_name'],
marcozecchini 0:9fca2b23d0ba 1689 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
marcozecchini 0:9fca2b23d0ba 1690 'port': mut['serial_port'],
marcozecchini 0:9fca2b23d0ba 1691 'disk': mut['mount_point'],
marcozecchini 0:9fca2b23d0ba 1692 'peripherals': [] # No peripheral detection
marcozecchini 0:9fca2b23d0ba 1693 }
marcozecchini 0:9fca2b23d0ba 1694 if index not in result:
marcozecchini 0:9fca2b23d0ba 1695 result[index] = {}
marcozecchini 0:9fca2b23d0ba 1696 result[index] = m
marcozecchini 0:9fca2b23d0ba 1697 index += 1
marcozecchini 0:9fca2b23d0ba 1698 return result
marcozecchini 0:9fca2b23d0ba 1699
marcozecchini 0:9fca2b23d0ba 1700
marcozecchini 0:9fca2b23d0ba 1701 def get_autodetected_TEST_SPEC(mbeds_list,
marcozecchini 0:9fca2b23d0ba 1702 use_default_toolchain=True,
marcozecchini 0:9fca2b23d0ba 1703 use_supported_toolchains=False,
marcozecchini 0:9fca2b23d0ba 1704 toolchain_filter=None,
marcozecchini 0:9fca2b23d0ba 1705 platform_name_filter=None):
marcozecchini 0:9fca2b23d0ba 1706 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
marcozecchini 0:9fca2b23d0ba 1707 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
marcozecchini 0:9fca2b23d0ba 1708
marcozecchini 0:9fca2b23d0ba 1709 use_default_toolchain - if True add default toolchain to test_spec
marcozecchini 0:9fca2b23d0ba 1710 use_supported_toolchains - if True add all supported toolchains to test_spec
marcozecchini 0:9fca2b23d0ba 1711 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
marcozecchini 0:9fca2b23d0ba 1712 """
marcozecchini 0:9fca2b23d0ba 1713 result = {'targets': {} }
marcozecchini 0:9fca2b23d0ba 1714
marcozecchini 0:9fca2b23d0ba 1715 for mut in mbeds_list:
marcozecchini 0:9fca2b23d0ba 1716 mcu = mut['mcu']
marcozecchini 0:9fca2b23d0ba 1717 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
marcozecchini 0:9fca2b23d0ba 1718 if mcu in TARGET_MAP:
marcozecchini 0:9fca2b23d0ba 1719 default_toolchain = TARGET_MAP[mcu].default_toolchain
marcozecchini 0:9fca2b23d0ba 1720 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
marcozecchini 0:9fca2b23d0ba 1721
marcozecchini 0:9fca2b23d0ba 1722 # Decide which toolchains should be added to test specification toolchain pool for each target
marcozecchini 0:9fca2b23d0ba 1723 toolchains = []
marcozecchini 0:9fca2b23d0ba 1724 if use_default_toolchain:
marcozecchini 0:9fca2b23d0ba 1725 toolchains.append(default_toolchain)
marcozecchini 0:9fca2b23d0ba 1726 if use_supported_toolchains:
marcozecchini 0:9fca2b23d0ba 1727 toolchains += supported_toolchains
marcozecchini 0:9fca2b23d0ba 1728 if toolchain_filter is not None:
marcozecchini 0:9fca2b23d0ba 1729 all_toolchains = supported_toolchains + [default_toolchain]
marcozecchini 0:9fca2b23d0ba 1730 for toolchain in toolchain_filter:
marcozecchini 0:9fca2b23d0ba 1731 if toolchain in all_toolchains:
marcozecchini 0:9fca2b23d0ba 1732 toolchains.append(toolchain)
marcozecchini 0:9fca2b23d0ba 1733
marcozecchini 0:9fca2b23d0ba 1734 result['targets'][mcu] = list(set(toolchains))
marcozecchini 0:9fca2b23d0ba 1735 return result
marcozecchini 0:9fca2b23d0ba 1736
marcozecchini 0:9fca2b23d0ba 1737
marcozecchini 0:9fca2b23d0ba 1738 def get_default_test_options_parser():
marcozecchini 0:9fca2b23d0ba 1739 """ Get common test script options used by CLI, web services etc.
marcozecchini 0:9fca2b23d0ba 1740 """
marcozecchini 0:9fca2b23d0ba 1741 parser = argparse.ArgumentParser()
marcozecchini 0:9fca2b23d0ba 1742 parser.add_argument('-i', '--tests',
marcozecchini 0:9fca2b23d0ba 1743 dest='test_spec_filename',
marcozecchini 0:9fca2b23d0ba 1744 metavar="FILE",
marcozecchini 0:9fca2b23d0ba 1745 type=argparse_filestring_type,
marcozecchini 0:9fca2b23d0ba 1746 help='Points to file with test specification')
marcozecchini 0:9fca2b23d0ba 1747
marcozecchini 0:9fca2b23d0ba 1748 parser.add_argument('-M', '--MUTS',
marcozecchini 0:9fca2b23d0ba 1749 dest='muts_spec_filename',
marcozecchini 0:9fca2b23d0ba 1750 metavar="FILE",
marcozecchini 0:9fca2b23d0ba 1751 type=argparse_filestring_type,
marcozecchini 0:9fca2b23d0ba 1752 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
marcozecchini 0:9fca2b23d0ba 1753
marcozecchini 0:9fca2b23d0ba 1754 parser.add_argument("-j", "--jobs",
marcozecchini 0:9fca2b23d0ba 1755 dest='jobs',
marcozecchini 0:9fca2b23d0ba 1756 metavar="NUMBER",
marcozecchini 0:9fca2b23d0ba 1757 type=int,
marcozecchini 0:9fca2b23d0ba 1758 help="Define number of compilation jobs. Default value is 1")
marcozecchini 0:9fca2b23d0ba 1759
marcozecchini 0:9fca2b23d0ba 1760 if get_module_avail('mbed_lstools'):
marcozecchini 0:9fca2b23d0ba 1761 # Additional features available when mbed_lstools is installed on host and imported
marcozecchini 0:9fca2b23d0ba 1762 # mbed_lstools allow users to detect connected to host mbed-enabled devices
marcozecchini 0:9fca2b23d0ba 1763 parser.add_argument('--auto',
marcozecchini 0:9fca2b23d0ba 1764 dest='auto_detect',
marcozecchini 0:9fca2b23d0ba 1765 action="store_true",
marcozecchini 0:9fca2b23d0ba 1766 help='Use mbed-ls module to detect all connected mbed devices')
marcozecchini 0:9fca2b23d0ba 1767
marcozecchini 0:9fca2b23d0ba 1768 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
marcozecchini 0:9fca2b23d0ba 1769 parser.add_argument('--tc',
marcozecchini 0:9fca2b23d0ba 1770 dest='toolchains_filter',
marcozecchini 0:9fca2b23d0ba 1771 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
marcozecchini 0:9fca2b23d0ba 1772 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
marcozecchini 0:9fca2b23d0ba 1773
marcozecchini 0:9fca2b23d0ba 1774 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
marcozecchini 0:9fca2b23d0ba 1775 parser.add_argument('--oper',
marcozecchini 0:9fca2b23d0ba 1776 dest='operability_checks',
marcozecchini 0:9fca2b23d0ba 1777 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
marcozecchini 0:9fca2b23d0ba 1778 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
marcozecchini 0:9fca2b23d0ba 1779
marcozecchini 0:9fca2b23d0ba 1780 parser.add_argument('--clean',
marcozecchini 0:9fca2b23d0ba 1781 dest='clean',
marcozecchini 0:9fca2b23d0ba 1782 action="store_true",
marcozecchini 0:9fca2b23d0ba 1783 help='Clean the build directory')
marcozecchini 0:9fca2b23d0ba 1784
marcozecchini 0:9fca2b23d0ba 1785 parser.add_argument('-P', '--only-peripherals',
marcozecchini 0:9fca2b23d0ba 1786 dest='test_only_peripheral',
marcozecchini 0:9fca2b23d0ba 1787 default=False,
marcozecchini 0:9fca2b23d0ba 1788 action="store_true",
marcozecchini 0:9fca2b23d0ba 1789 help='Test only peripheral declared for MUT and skip common tests')
marcozecchini 0:9fca2b23d0ba 1790
marcozecchini 0:9fca2b23d0ba 1791 parser.add_argument("--profile", dest="profile", action="append",
marcozecchini 0:9fca2b23d0ba 1792 type=argparse_filestring_type,
marcozecchini 0:9fca2b23d0ba 1793 default=[])
marcozecchini 0:9fca2b23d0ba 1794
marcozecchini 0:9fca2b23d0ba 1795 parser.add_argument('-C', '--only-commons',
marcozecchini 0:9fca2b23d0ba 1796 dest='test_only_common',
marcozecchini 0:9fca2b23d0ba 1797 default=False,
marcozecchini 0:9fca2b23d0ba 1798 action="store_true",
marcozecchini 0:9fca2b23d0ba 1799 help='Test only board internals. Skip perpherials tests and perform common tests')
marcozecchini 0:9fca2b23d0ba 1800
marcozecchini 0:9fca2b23d0ba 1801 parser.add_argument('-n', '--test-by-names',
marcozecchini 0:9fca2b23d0ba 1802 dest='test_by_names',
marcozecchini 0:9fca2b23d0ba 1803 type=argparse_many(str),
marcozecchini 0:9fca2b23d0ba 1804 help='Runs only test enumerated it this switch. Use comma to separate test case names')
marcozecchini 0:9fca2b23d0ba 1805
marcozecchini 0:9fca2b23d0ba 1806 parser.add_argument('-p', '--peripheral-by-names',
marcozecchini 0:9fca2b23d0ba 1807 dest='peripheral_by_names',
marcozecchini 0:9fca2b23d0ba 1808 type=argparse_many(str),
marcozecchini 0:9fca2b23d0ba 1809 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
marcozecchini 0:9fca2b23d0ba 1810
marcozecchini 0:9fca2b23d0ba 1811 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
marcozecchini 0:9fca2b23d0ba 1812 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
marcozecchini 0:9fca2b23d0ba 1813
marcozecchini 0:9fca2b23d0ba 1814 parser.add_argument('-c', '--copy-method',
marcozecchini 0:9fca2b23d0ba 1815 dest='copy_method',
marcozecchini 0:9fca2b23d0ba 1816 type=argparse_uppercase_type(copy_methods, "flash method"),
marcozecchini 0:9fca2b23d0ba 1817 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
marcozecchini 0:9fca2b23d0ba 1818
marcozecchini 0:9fca2b23d0ba 1819 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
marcozecchini 0:9fca2b23d0ba 1820 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
marcozecchini 0:9fca2b23d0ba 1821
marcozecchini 0:9fca2b23d0ba 1822 parser.add_argument('-r', '--reset-type',
marcozecchini 0:9fca2b23d0ba 1823 dest='mut_reset_type',
marcozecchini 0:9fca2b23d0ba 1824 default=None,
marcozecchini 0:9fca2b23d0ba 1825 type=argparse_uppercase_type(reset_methods, "reset method"),
marcozecchini 0:9fca2b23d0ba 1826 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
marcozecchini 0:9fca2b23d0ba 1827
marcozecchini 0:9fca2b23d0ba 1828 parser.add_argument('-g', '--goanna-for-tests',
marcozecchini 0:9fca2b23d0ba 1829 dest='goanna_for_tests',
marcozecchini 0:9fca2b23d0ba 1830 action="store_true",
marcozecchini 0:9fca2b23d0ba 1831 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
marcozecchini 0:9fca2b23d0ba 1832
marcozecchini 0:9fca2b23d0ba 1833 parser.add_argument('-G', '--goanna-for-sdk',
marcozecchini 0:9fca2b23d0ba 1834 dest='goanna_for_mbed_sdk',
marcozecchini 0:9fca2b23d0ba 1835 action="store_true",
marcozecchini 0:9fca2b23d0ba 1836 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
marcozecchini 0:9fca2b23d0ba 1837
marcozecchini 0:9fca2b23d0ba 1838 parser.add_argument('-s', '--suppress-summary',
marcozecchini 0:9fca2b23d0ba 1839 dest='suppress_summary',
marcozecchini 0:9fca2b23d0ba 1840 default=False,
marcozecchini 0:9fca2b23d0ba 1841 action="store_true",
marcozecchini 0:9fca2b23d0ba 1842 help='Suppresses display of wellformatted table with test results')
marcozecchini 0:9fca2b23d0ba 1843
marcozecchini 0:9fca2b23d0ba 1844 parser.add_argument('-t', '--test-summary',
marcozecchini 0:9fca2b23d0ba 1845 dest='test_x_toolchain_summary',
marcozecchini 0:9fca2b23d0ba 1846 default=False,
marcozecchini 0:9fca2b23d0ba 1847 action="store_true",
marcozecchini 0:9fca2b23d0ba 1848 help='Displays wellformatted table with test x toolchain test result per target')
marcozecchini 0:9fca2b23d0ba 1849
marcozecchini 0:9fca2b23d0ba 1850 parser.add_argument('-A', '--test-automation-report',
marcozecchini 0:9fca2b23d0ba 1851 dest='test_automation_report',
marcozecchini 0:9fca2b23d0ba 1852 default=False,
marcozecchini 0:9fca2b23d0ba 1853 action="store_true",
marcozecchini 0:9fca2b23d0ba 1854 help='Prints information about all tests and exits')
marcozecchini 0:9fca2b23d0ba 1855
marcozecchini 0:9fca2b23d0ba 1856 parser.add_argument('-R', '--test-case-report',
marcozecchini 0:9fca2b23d0ba 1857 dest='test_case_report',
marcozecchini 0:9fca2b23d0ba 1858 default=False,
marcozecchini 0:9fca2b23d0ba 1859 action="store_true",
marcozecchini 0:9fca2b23d0ba 1860 help='Prints information about all test cases and exits')
marcozecchini 0:9fca2b23d0ba 1861
marcozecchini 0:9fca2b23d0ba 1862 parser.add_argument("-S", "--supported-toolchains",
marcozecchini 0:9fca2b23d0ba 1863 action="store_true",
marcozecchini 0:9fca2b23d0ba 1864 dest="supported_toolchains",
marcozecchini 0:9fca2b23d0ba 1865 default=False,
marcozecchini 0:9fca2b23d0ba 1866 help="Displays supported matrix of MCUs and toolchains")
marcozecchini 0:9fca2b23d0ba 1867
marcozecchini 0:9fca2b23d0ba 1868 parser.add_argument("-O", "--only-build",
marcozecchini 0:9fca2b23d0ba 1869 action="store_true",
marcozecchini 0:9fca2b23d0ba 1870 dest="only_build_tests",
marcozecchini 0:9fca2b23d0ba 1871 default=False,
marcozecchini 0:9fca2b23d0ba 1872 help="Only build tests, skips actual test procedures (flashing etc.)")
marcozecchini 0:9fca2b23d0ba 1873
marcozecchini 0:9fca2b23d0ba 1874 parser.add_argument('--parallel',
marcozecchini 0:9fca2b23d0ba 1875 dest='parallel_test_exec',
marcozecchini 0:9fca2b23d0ba 1876 default=False,
marcozecchini 0:9fca2b23d0ba 1877 action="store_true",
marcozecchini 0:9fca2b23d0ba 1878 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
marcozecchini 0:9fca2b23d0ba 1879
marcozecchini 0:9fca2b23d0ba 1880 parser.add_argument('--config',
marcozecchini 0:9fca2b23d0ba 1881 dest='verbose_test_configuration_only',
marcozecchini 0:9fca2b23d0ba 1882 default=False,
marcozecchini 0:9fca2b23d0ba 1883 action="store_true",
marcozecchini 0:9fca2b23d0ba 1884 help='Displays full test specification and MUTs configration and exits')
marcozecchini 0:9fca2b23d0ba 1885
marcozecchini 0:9fca2b23d0ba 1886 parser.add_argument('--loops',
marcozecchini 0:9fca2b23d0ba 1887 dest='test_loops_list',
marcozecchini 0:9fca2b23d0ba 1888 type=argparse_many(str),
marcozecchini 0:9fca2b23d0ba 1889 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
marcozecchini 0:9fca2b23d0ba 1890
marcozecchini 0:9fca2b23d0ba 1891 parser.add_argument('--global-loops',
marcozecchini 0:9fca2b23d0ba 1892 dest='test_global_loops_value',
marcozecchini 0:9fca2b23d0ba 1893 type=int,
marcozecchini 0:9fca2b23d0ba 1894 help='Set global number of test loops per test. Default value is set 1')
marcozecchini 0:9fca2b23d0ba 1895
marcozecchini 0:9fca2b23d0ba 1896 parser.add_argument('--consolidate-waterfall',
marcozecchini 0:9fca2b23d0ba 1897 dest='consolidate_waterfall_test',
marcozecchini 0:9fca2b23d0ba 1898 default=False,
marcozecchini 0:9fca2b23d0ba 1899 action="store_true",
marcozecchini 0:9fca2b23d0ba 1900 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
marcozecchini 0:9fca2b23d0ba 1901
marcozecchini 0:9fca2b23d0ba 1902 parser.add_argument('-W', '--waterfall',
marcozecchini 0:9fca2b23d0ba 1903 dest='waterfall_test',
marcozecchini 0:9fca2b23d0ba 1904 default=False,
marcozecchini 0:9fca2b23d0ba 1905 action="store_true",
marcozecchini 0:9fca2b23d0ba 1906 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
marcozecchini 0:9fca2b23d0ba 1907
marcozecchini 0:9fca2b23d0ba 1908 parser.add_argument('-N', '--firmware-name',
marcozecchini 0:9fca2b23d0ba 1909 dest='firmware_global_name',
marcozecchini 0:9fca2b23d0ba 1910 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
marcozecchini 0:9fca2b23d0ba 1911
marcozecchini 0:9fca2b23d0ba 1912 parser.add_argument('-u', '--shuffle',
marcozecchini 0:9fca2b23d0ba 1913 dest='shuffle_test_order',
marcozecchini 0:9fca2b23d0ba 1914 default=False,
marcozecchini 0:9fca2b23d0ba 1915 action="store_true",
marcozecchini 0:9fca2b23d0ba 1916 help='Shuffles test execution order')
marcozecchini 0:9fca2b23d0ba 1917
marcozecchini 0:9fca2b23d0ba 1918 parser.add_argument('--shuffle-seed',
marcozecchini 0:9fca2b23d0ba 1919 dest='shuffle_test_seed',
marcozecchini 0:9fca2b23d0ba 1920 default=None,
marcozecchini 0:9fca2b23d0ba 1921 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
marcozecchini 0:9fca2b23d0ba 1922
marcozecchini 0:9fca2b23d0ba 1923 parser.add_argument('-f', '--filter',
marcozecchini 0:9fca2b23d0ba 1924 dest='general_filter_regex',
marcozecchini 0:9fca2b23d0ba 1925 type=argparse_many(str),
marcozecchini 0:9fca2b23d0ba 1926 default=None,
marcozecchini 0:9fca2b23d0ba 1927 help='For some commands you can use filter to filter out results')
marcozecchini 0:9fca2b23d0ba 1928
marcozecchini 0:9fca2b23d0ba 1929 parser.add_argument('--inc-timeout',
marcozecchini 0:9fca2b23d0ba 1930 dest='extend_test_timeout',
marcozecchini 0:9fca2b23d0ba 1931 metavar="NUMBER",
marcozecchini 0:9fca2b23d0ba 1932 type=int,
marcozecchini 0:9fca2b23d0ba 1933 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
marcozecchini 0:9fca2b23d0ba 1934
marcozecchini 0:9fca2b23d0ba 1935 parser.add_argument('--db',
marcozecchini 0:9fca2b23d0ba 1936 dest='db_url',
marcozecchini 0:9fca2b23d0ba 1937 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
marcozecchini 0:9fca2b23d0ba 1938
marcozecchini 0:9fca2b23d0ba 1939 parser.add_argument('-l', '--log',
marcozecchini 0:9fca2b23d0ba 1940 dest='log_file_name',
marcozecchini 0:9fca2b23d0ba 1941 help='Log events to external file (note not all console entries may be visible in log file)')
marcozecchini 0:9fca2b23d0ba 1942
marcozecchini 0:9fca2b23d0ba 1943 parser.add_argument('--report-html',
marcozecchini 0:9fca2b23d0ba 1944 dest='report_html_file_name',
marcozecchini 0:9fca2b23d0ba 1945 help='You can log test suite results in form of HTML report')
marcozecchini 0:9fca2b23d0ba 1946
marcozecchini 0:9fca2b23d0ba 1947 parser.add_argument('--report-junit',
marcozecchini 0:9fca2b23d0ba 1948 dest='report_junit_file_name',
marcozecchini 0:9fca2b23d0ba 1949 help='You can log test suite results in form of JUnit compliant XML report')
marcozecchini 0:9fca2b23d0ba 1950
marcozecchini 0:9fca2b23d0ba 1951 parser.add_argument("--report-build",
marcozecchini 0:9fca2b23d0ba 1952 dest="report_build_file_name",
marcozecchini 0:9fca2b23d0ba 1953 help="Output the build results to a junit xml file")
marcozecchini 0:9fca2b23d0ba 1954
marcozecchini 0:9fca2b23d0ba 1955 parser.add_argument("--report-text",
marcozecchini 0:9fca2b23d0ba 1956 dest="report_text_file_name",
marcozecchini 0:9fca2b23d0ba 1957 help="Output the build results to a text file")
marcozecchini 0:9fca2b23d0ba 1958
marcozecchini 0:9fca2b23d0ba 1959 parser.add_argument('--verbose-skipped',
marcozecchini 0:9fca2b23d0ba 1960 dest='verbose_skipped_tests',
marcozecchini 0:9fca2b23d0ba 1961 default=False,
marcozecchini 0:9fca2b23d0ba 1962 action="store_true",
marcozecchini 0:9fca2b23d0ba 1963 help='Prints some extra information about skipped tests')
marcozecchini 0:9fca2b23d0ba 1964
marcozecchini 0:9fca2b23d0ba 1965 parser.add_argument('-V', '--verbose-test-result',
marcozecchini 0:9fca2b23d0ba 1966 dest='verbose_test_result_only',
marcozecchini 0:9fca2b23d0ba 1967 default=False,
marcozecchini 0:9fca2b23d0ba 1968 action="store_true",
marcozecchini 0:9fca2b23d0ba 1969 help='Prints test serial output')
marcozecchini 0:9fca2b23d0ba 1970
marcozecchini 0:9fca2b23d0ba 1971 parser.add_argument('-v', '--verbose',
marcozecchini 0:9fca2b23d0ba 1972 dest='verbose',
marcozecchini 0:9fca2b23d0ba 1973 default=False,
marcozecchini 0:9fca2b23d0ba 1974 action="store_true",
marcozecchini 0:9fca2b23d0ba 1975 help='Verbose mode (prints some extra information)')
marcozecchini 0:9fca2b23d0ba 1976
marcozecchini 0:9fca2b23d0ba 1977 parser.add_argument('--version',
marcozecchini 0:9fca2b23d0ba 1978 dest='version',
marcozecchini 0:9fca2b23d0ba 1979 default=False,
marcozecchini 0:9fca2b23d0ba 1980 action="store_true",
marcozecchini 0:9fca2b23d0ba 1981 help='Prints script version and exits')
marcozecchini 0:9fca2b23d0ba 1982
marcozecchini 0:9fca2b23d0ba 1983 parser.add_argument('--stats-depth',
marcozecchini 0:9fca2b23d0ba 1984 dest='stats_depth',
marcozecchini 0:9fca2b23d0ba 1985 default=2,
marcozecchini 0:9fca2b23d0ba 1986 type=int,
marcozecchini 0:9fca2b23d0ba 1987 help="Depth level for static memory report")
marcozecchini 0:9fca2b23d0ba 1988 return parser
marcozecchini 0:9fca2b23d0ba 1989
marcozecchini 0:9fca2b23d0ba 1990 def test_path_to_name(path, base):
marcozecchini 0:9fca2b23d0ba 1991 """Change all slashes in a path into hyphens
marcozecchini 0:9fca2b23d0ba 1992 This creates a unique cross-platform test name based on the path
marcozecchini 0:9fca2b23d0ba 1993 This can eventually be overriden by a to-be-determined meta-data mechanism"""
marcozecchini 0:9fca2b23d0ba 1994 name_parts = []
marcozecchini 0:9fca2b23d0ba 1995 head, tail = os.path.split(relpath(path,base))
marcozecchini 0:9fca2b23d0ba 1996 while (tail and tail != "."):
marcozecchini 0:9fca2b23d0ba 1997 name_parts.insert(0, tail)
marcozecchini 0:9fca2b23d0ba 1998 head, tail = os.path.split(head)
marcozecchini 0:9fca2b23d0ba 1999
marcozecchini 0:9fca2b23d0ba 2000 return "-".join(name_parts).lower()
marcozecchini 0:9fca2b23d0ba 2001
marcozecchini 0:9fca2b23d0ba 2002 def get_test_config(config_name, target_name):
marcozecchini 0:9fca2b23d0ba 2003 """Finds the path to a test configuration file
marcozecchini 0:9fca2b23d0ba 2004 config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
marcozecchini 0:9fca2b23d0ba 2005 target_name: name of target to determing if mbed OS interface given is valid
marcozecchini 0:9fca2b23d0ba 2006 returns path to config, will return None if no valid config is found
marcozecchini 0:9fca2b23d0ba 2007 """
marcozecchini 0:9fca2b23d0ba 2008 # If they passed in a full path
marcozecchini 0:9fca2b23d0ba 2009 if exists(config_name):
marcozecchini 0:9fca2b23d0ba 2010 # This is a module config
marcozecchini 0:9fca2b23d0ba 2011 return config_name
marcozecchini 0:9fca2b23d0ba 2012 # Otherwise find the path to configuration file based on mbed OS interface
marcozecchini 0:9fca2b23d0ba 2013 return TestConfig.get_config_path(config_name, target_name)
marcozecchini 0:9fca2b23d0ba 2014
marcozecchini 0:9fca2b23d0ba 2015 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
marcozecchini 0:9fca2b23d0ba 2016 """ Finds all tests in a directory recursively
marcozecchini 0:9fca2b23d0ba 2017 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
marcozecchini 0:9fca2b23d0ba 2018 target_name: name of the target to use for scanning (ex. 'K64F')
marcozecchini 0:9fca2b23d0ba 2019 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
marcozecchini 0:9fca2b23d0ba 2020 options: Compile options to pass to the toolchain (ex. ['debug-info'])
marcozecchini 0:9fca2b23d0ba 2021 app_config - location of a chosen mbed_app.json file
marcozecchini 0:9fca2b23d0ba 2022 """
marcozecchini 0:9fca2b23d0ba 2023
marcozecchini 0:9fca2b23d0ba 2024 tests = {}
marcozecchini 0:9fca2b23d0ba 2025
marcozecchini 0:9fca2b23d0ba 2026 # Prepare the toolchain
marcozecchini 0:9fca2b23d0ba 2027 toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
marcozecchini 0:9fca2b23d0ba 2028 silent=True, app_config=app_config)
marcozecchini 0:9fca2b23d0ba 2029
marcozecchini 0:9fca2b23d0ba 2030 # Scan the directory for paths to probe for 'TESTS' folders
marcozecchini 0:9fca2b23d0ba 2031 base_resources = scan_resources([base_dir], toolchain)
marcozecchini 0:9fca2b23d0ba 2032
marcozecchini 0:9fca2b23d0ba 2033 dirs = base_resources.inc_dirs
marcozecchini 0:9fca2b23d0ba 2034 for directory in dirs:
marcozecchini 0:9fca2b23d0ba 2035 subdirs = os.listdir(directory)
marcozecchini 0:9fca2b23d0ba 2036
marcozecchini 0:9fca2b23d0ba 2037 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
marcozecchini 0:9fca2b23d0ba 2038 if 'TESTS' in subdirs:
marcozecchini 0:9fca2b23d0ba 2039 walk_base_dir = join(directory, 'TESTS')
marcozecchini 0:9fca2b23d0ba 2040 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
marcozecchini 0:9fca2b23d0ba 2041
marcozecchini 0:9fca2b23d0ba 2042 # Loop through all subdirectories
marcozecchini 0:9fca2b23d0ba 2043 for d in test_resources.inc_dirs:
marcozecchini 0:9fca2b23d0ba 2044
marcozecchini 0:9fca2b23d0ba 2045 # If the test case folder is not called 'host_tests' and it is
marcozecchini 0:9fca2b23d0ba 2046 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
marcozecchini 0:9fca2b23d0ba 2047 # then add it to the tests
marcozecchini 0:9fca2b23d0ba 2048 path_depth = get_path_depth(relpath(d, walk_base_dir))
marcozecchini 0:9fca2b23d0ba 2049 if path_depth == 2:
marcozecchini 0:9fca2b23d0ba 2050 test_group_directory_path, test_case_directory = os.path.split(d)
marcozecchini 0:9fca2b23d0ba 2051 test_group_directory = os.path.basename(test_group_directory_path)
marcozecchini 0:9fca2b23d0ba 2052
marcozecchini 0:9fca2b23d0ba 2053 # Check to make sure discoverd folder is not in a host test directory
marcozecchini 0:9fca2b23d0ba 2054 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
marcozecchini 0:9fca2b23d0ba 2055 test_name = test_path_to_name(d, base_dir)
marcozecchini 0:9fca2b23d0ba 2056 tests[test_name] = d
marcozecchini 0:9fca2b23d0ba 2057
marcozecchini 0:9fca2b23d0ba 2058 return tests
marcozecchini 0:9fca2b23d0ba 2059
marcozecchini 0:9fca2b23d0ba 2060 def print_tests(tests, format="list", sort=True):
marcozecchini 0:9fca2b23d0ba 2061 """Given a dictionary of tests (as returned from "find_tests"), print them
marcozecchini 0:9fca2b23d0ba 2062 in the specified format"""
marcozecchini 0:9fca2b23d0ba 2063 if format == "list":
marcozecchini 0:9fca2b23d0ba 2064 for test_name in sorted(tests.keys()):
marcozecchini 0:9fca2b23d0ba 2065 test_path = tests[test_name]
marcozecchini 0:9fca2b23d0ba 2066 print "Test Case:"
marcozecchini 0:9fca2b23d0ba 2067 print " Name: %s" % test_name
marcozecchini 0:9fca2b23d0ba 2068 print " Path: %s" % test_path
marcozecchini 0:9fca2b23d0ba 2069 elif format == "json":
marcozecchini 0:9fca2b23d0ba 2070 print json.dumps(tests, indent=2)
marcozecchini 0:9fca2b23d0ba 2071 else:
marcozecchini 0:9fca2b23d0ba 2072 print "Unknown format '%s'" % format
marcozecchini 0:9fca2b23d0ba 2073 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 2074
marcozecchini 0:9fca2b23d0ba 2075 def norm_relative_path(path, start):
marcozecchini 0:9fca2b23d0ba 2076 """This function will create a normalized, relative path. It mimics the
marcozecchini 0:9fca2b23d0ba 2077 python os.path.relpath function, but also normalizes a Windows-syle path
marcozecchini 0:9fca2b23d0ba 2078 that use backslashes to a Unix style path that uses forward slashes."""
marcozecchini 0:9fca2b23d0ba 2079 path = os.path.normpath(path)
marcozecchini 0:9fca2b23d0ba 2080 path = os.path.relpath(path, start)
marcozecchini 0:9fca2b23d0ba 2081 path = path.replace("\\", "/")
marcozecchini 0:9fca2b23d0ba 2082 return path
marcozecchini 0:9fca2b23d0ba 2083
marcozecchini 0:9fca2b23d0ba 2084
marcozecchini 0:9fca2b23d0ba 2085 def build_test_worker(*args, **kwargs):
marcozecchini 0:9fca2b23d0ba 2086 """This is a worker function for the parallel building of tests. The `args`
marcozecchini 0:9fca2b23d0ba 2087 and `kwargs` are passed directly to `build_project`. It returns a dictionary
marcozecchini 0:9fca2b23d0ba 2088 with the following structure:
marcozecchini 0:9fca2b23d0ba 2089
marcozecchini 0:9fca2b23d0ba 2090 {
marcozecchini 0:9fca2b23d0ba 2091 'result': `True` if no exceptions were thrown, `False` otherwise
marcozecchini 0:9fca2b23d0ba 2092 'reason': Instance of exception that was thrown on failure
marcozecchini 0:9fca2b23d0ba 2093 'bin_file': Path to the created binary if `build_project` was
marcozecchini 0:9fca2b23d0ba 2094 successful. Not present otherwise
marcozecchini 0:9fca2b23d0ba 2095 'kwargs': The keyword arguments that were passed to `build_project`.
marcozecchini 0:9fca2b23d0ba 2096 This includes arguments that were modified (ex. report)
marcozecchini 0:9fca2b23d0ba 2097 }
marcozecchini 0:9fca2b23d0ba 2098 """
marcozecchini 0:9fca2b23d0ba 2099 bin_file = None
marcozecchini 0:9fca2b23d0ba 2100 ret = {
marcozecchini 0:9fca2b23d0ba 2101 'result': False,
marcozecchini 0:9fca2b23d0ba 2102 'args': args,
marcozecchini 0:9fca2b23d0ba 2103 'kwargs': kwargs
marcozecchini 0:9fca2b23d0ba 2104 }
marcozecchini 0:9fca2b23d0ba 2105
marcozecchini 0:9fca2b23d0ba 2106 # Use parent TOOLCHAIN_PATHS variable
marcozecchini 0:9fca2b23d0ba 2107 for key, value in kwargs['toolchain_paths'].iteritems():
marcozecchini 0:9fca2b23d0ba 2108 TOOLCHAIN_PATHS[key] = value
marcozecchini 0:9fca2b23d0ba 2109
marcozecchini 0:9fca2b23d0ba 2110 del kwargs['toolchain_paths']
marcozecchini 0:9fca2b23d0ba 2111
marcozecchini 0:9fca2b23d0ba 2112 try:
marcozecchini 0:9fca2b23d0ba 2113 bin_file = build_project(*args, **kwargs)
marcozecchini 0:9fca2b23d0ba 2114 ret['result'] = True
marcozecchini 0:9fca2b23d0ba 2115 ret['bin_file'] = bin_file
marcozecchini 0:9fca2b23d0ba 2116 ret['kwargs'] = kwargs
marcozecchini 0:9fca2b23d0ba 2117
marcozecchini 0:9fca2b23d0ba 2118 except NotSupportedException, e:
marcozecchini 0:9fca2b23d0ba 2119 ret['reason'] = e
marcozecchini 0:9fca2b23d0ba 2120 except ToolException, e:
marcozecchini 0:9fca2b23d0ba 2121 ret['reason'] = e
marcozecchini 0:9fca2b23d0ba 2122 except KeyboardInterrupt, e:
marcozecchini 0:9fca2b23d0ba 2123 ret['reason'] = e
marcozecchini 0:9fca2b23d0ba 2124 except:
marcozecchini 0:9fca2b23d0ba 2125 # Print unhandled exceptions here
marcozecchini 0:9fca2b23d0ba 2126 import traceback
marcozecchini 0:9fca2b23d0ba 2127 traceback.print_exc(file=sys.stdout)
marcozecchini 0:9fca2b23d0ba 2128
marcozecchini 0:9fca2b23d0ba 2129 return ret
marcozecchini 0:9fca2b23d0ba 2130
marcozecchini 0:9fca2b23d0ba 2131
marcozecchini 0:9fca2b23d0ba 2132 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
marcozecchini 0:9fca2b23d0ba 2133 clean=False, notify=None, verbose=False, jobs=1, macros=None,
marcozecchini 0:9fca2b23d0ba 2134 silent=False, report=None, properties=None,
marcozecchini 0:9fca2b23d0ba 2135 continue_on_build_fail=False, app_config=None,
marcozecchini 0:9fca2b23d0ba 2136 build_profile=None, stats_depth=None):
marcozecchini 0:9fca2b23d0ba 2137 """Given the data structure from 'find_tests' and the typical build parameters,
marcozecchini 0:9fca2b23d0ba 2138 build all the tests
marcozecchini 0:9fca2b23d0ba 2139
marcozecchini 0:9fca2b23d0ba 2140 Returns a tuple of the build result (True or False) followed by the test
marcozecchini 0:9fca2b23d0ba 2141 build data structure"""
marcozecchini 0:9fca2b23d0ba 2142
marcozecchini 0:9fca2b23d0ba 2143 execution_directory = "."
marcozecchini 0:9fca2b23d0ba 2144 base_path = norm_relative_path(build_path, execution_directory)
marcozecchini 0:9fca2b23d0ba 2145
marcozecchini 0:9fca2b23d0ba 2146 target_name = target if isinstance(target, str) else target.name
marcozecchini 0:9fca2b23d0ba 2147 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
marcozecchini 0:9fca2b23d0ba 2148
marcozecchini 0:9fca2b23d0ba 2149 baud_rate = 9600
marcozecchini 0:9fca2b23d0ba 2150 if 'platform.stdio-baud-rate' in cfg:
marcozecchini 0:9fca2b23d0ba 2151 baud_rate = cfg['platform.stdio-baud-rate'].value
marcozecchini 0:9fca2b23d0ba 2152
marcozecchini 0:9fca2b23d0ba 2153 test_build = {
marcozecchini 0:9fca2b23d0ba 2154 "platform": target_name,
marcozecchini 0:9fca2b23d0ba 2155 "toolchain": toolchain_name,
marcozecchini 0:9fca2b23d0ba 2156 "base_path": base_path,
marcozecchini 0:9fca2b23d0ba 2157 "baud_rate": baud_rate,
marcozecchini 0:9fca2b23d0ba 2158 "binary_type": "bootable",
marcozecchini 0:9fca2b23d0ba 2159 "tests": {}
marcozecchini 0:9fca2b23d0ba 2160 }
marcozecchini 0:9fca2b23d0ba 2161
marcozecchini 0:9fca2b23d0ba 2162 result = True
marcozecchini 0:9fca2b23d0ba 2163
marcozecchini 0:9fca2b23d0ba 2164 jobs_count = int(jobs if jobs else cpu_count())
marcozecchini 0:9fca2b23d0ba 2165 p = Pool(processes=jobs_count)
marcozecchini 0:9fca2b23d0ba 2166 results = []
marcozecchini 0:9fca2b23d0ba 2167 for test_name, test_path in tests.iteritems():
marcozecchini 0:9fca2b23d0ba 2168 test_build_path = os.path.join(build_path, test_path)
marcozecchini 0:9fca2b23d0ba 2169 src_path = base_source_paths + [test_path]
marcozecchini 0:9fca2b23d0ba 2170 bin_file = None
marcozecchini 0:9fca2b23d0ba 2171 test_case_folder_name = os.path.basename(test_path)
marcozecchini 0:9fca2b23d0ba 2172
marcozecchini 0:9fca2b23d0ba 2173 args = (src_path, test_build_path, target, toolchain_name)
marcozecchini 0:9fca2b23d0ba 2174 kwargs = {
marcozecchini 0:9fca2b23d0ba 2175 'jobs': 1,
marcozecchini 0:9fca2b23d0ba 2176 'clean': clean,
marcozecchini 0:9fca2b23d0ba 2177 'macros': macros,
marcozecchini 0:9fca2b23d0ba 2178 'name': test_case_folder_name,
marcozecchini 0:9fca2b23d0ba 2179 'project_id': test_name,
marcozecchini 0:9fca2b23d0ba 2180 'report': report,
marcozecchini 0:9fca2b23d0ba 2181 'properties': properties,
marcozecchini 0:9fca2b23d0ba 2182 'verbose': verbose,
marcozecchini 0:9fca2b23d0ba 2183 'app_config': app_config,
marcozecchini 0:9fca2b23d0ba 2184 'build_profile': build_profile,
marcozecchini 0:9fca2b23d0ba 2185 'silent': True,
marcozecchini 0:9fca2b23d0ba 2186 'toolchain_paths': TOOLCHAIN_PATHS,
marcozecchini 0:9fca2b23d0ba 2187 'stats_depth': stats_depth
marcozecchini 0:9fca2b23d0ba 2188 }
marcozecchini 0:9fca2b23d0ba 2189
marcozecchini 0:9fca2b23d0ba 2190 results.append(p.apply_async(build_test_worker, args, kwargs))
marcozecchini 0:9fca2b23d0ba 2191
marcozecchini 0:9fca2b23d0ba 2192 p.close()
marcozecchini 0:9fca2b23d0ba 2193 result = True
marcozecchini 0:9fca2b23d0ba 2194 itr = 0
marcozecchini 0:9fca2b23d0ba 2195 while len(results):
marcozecchini 0:9fca2b23d0ba 2196 itr += 1
marcozecchini 0:9fca2b23d0ba 2197 if itr > 360000:
marcozecchini 0:9fca2b23d0ba 2198 p.terminate()
marcozecchini 0:9fca2b23d0ba 2199 p.join()
marcozecchini 0:9fca2b23d0ba 2200 raise ToolException("Compile did not finish in 10 minutes")
marcozecchini 0:9fca2b23d0ba 2201 else:
marcozecchini 0:9fca2b23d0ba 2202 sleep(0.01)
marcozecchini 0:9fca2b23d0ba 2203 pending = 0
marcozecchini 0:9fca2b23d0ba 2204 for r in results:
marcozecchini 0:9fca2b23d0ba 2205 if r.ready() is True:
marcozecchini 0:9fca2b23d0ba 2206 try:
marcozecchini 0:9fca2b23d0ba 2207 worker_result = r.get()
marcozecchini 0:9fca2b23d0ba 2208 results.remove(r)
marcozecchini 0:9fca2b23d0ba 2209
marcozecchini 0:9fca2b23d0ba 2210 # Take report from the kwargs and merge it into existing report
marcozecchini 0:9fca2b23d0ba 2211 if report:
marcozecchini 0:9fca2b23d0ba 2212 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
marcozecchini 0:9fca2b23d0ba 2213 for test_key in report_entry.keys():
marcozecchini 0:9fca2b23d0ba 2214 report[target_name][toolchain_name][test_key] = report_entry[test_key]
marcozecchini 0:9fca2b23d0ba 2215
marcozecchini 0:9fca2b23d0ba 2216 # Set the overall result to a failure if a build failure occurred
marcozecchini 0:9fca2b23d0ba 2217 if ('reason' in worker_result and
marcozecchini 0:9fca2b23d0ba 2218 not worker_result['reason'] and
marcozecchini 0:9fca2b23d0ba 2219 not isinstance(worker_result['reason'], NotSupportedException)):
marcozecchini 0:9fca2b23d0ba 2220 result = False
marcozecchini 0:9fca2b23d0ba 2221 break
marcozecchini 0:9fca2b23d0ba 2222
marcozecchini 0:9fca2b23d0ba 2223 # Adding binary path to test build result
marcozecchini 0:9fca2b23d0ba 2224 if ('result' in worker_result and
marcozecchini 0:9fca2b23d0ba 2225 worker_result['result'] and
marcozecchini 0:9fca2b23d0ba 2226 'bin_file' in worker_result):
marcozecchini 0:9fca2b23d0ba 2227 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
marcozecchini 0:9fca2b23d0ba 2228
marcozecchini 0:9fca2b23d0ba 2229 test_build['tests'][worker_result['kwargs']['project_id']] = {
marcozecchini 0:9fca2b23d0ba 2230 "binaries": [
marcozecchini 0:9fca2b23d0ba 2231 {
marcozecchini 0:9fca2b23d0ba 2232 "path": bin_file
marcozecchini 0:9fca2b23d0ba 2233 }
marcozecchini 0:9fca2b23d0ba 2234 ]
marcozecchini 0:9fca2b23d0ba 2235 }
marcozecchini 0:9fca2b23d0ba 2236
marcozecchini 0:9fca2b23d0ba 2237 test_key = worker_result['kwargs']['project_id'].upper()
marcozecchini 0:9fca2b23d0ba 2238 if report:
marcozecchini 0:9fca2b23d0ba 2239 print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
marcozecchini 0:9fca2b23d0ba 2240 print 'Image: %s\n' % bin_file
marcozecchini 0:9fca2b23d0ba 2241
marcozecchini 0:9fca2b23d0ba 2242 except:
marcozecchini 0:9fca2b23d0ba 2243 if p._taskqueue.queue:
marcozecchini 0:9fca2b23d0ba 2244 p._taskqueue.queue.clear()
marcozecchini 0:9fca2b23d0ba 2245 sleep(0.5)
marcozecchini 0:9fca2b23d0ba 2246 p.terminate()
marcozecchini 0:9fca2b23d0ba 2247 p.join()
marcozecchini 0:9fca2b23d0ba 2248 raise
marcozecchini 0:9fca2b23d0ba 2249 else:
marcozecchini 0:9fca2b23d0ba 2250 pending += 1
marcozecchini 0:9fca2b23d0ba 2251 if pending >= jobs_count:
marcozecchini 0:9fca2b23d0ba 2252 break
marcozecchini 0:9fca2b23d0ba 2253
marcozecchini 0:9fca2b23d0ba 2254 # Break as soon as possible if there is a failure and we are not
marcozecchini 0:9fca2b23d0ba 2255 # continuing on build failures
marcozecchini 0:9fca2b23d0ba 2256 if not result and not continue_on_build_fail:
marcozecchini 0:9fca2b23d0ba 2257 if p._taskqueue.queue:
marcozecchini 0:9fca2b23d0ba 2258 p._taskqueue.queue.clear()
marcozecchini 0:9fca2b23d0ba 2259 sleep(0.5)
marcozecchini 0:9fca2b23d0ba 2260 p.terminate()
marcozecchini 0:9fca2b23d0ba 2261 break
marcozecchini 0:9fca2b23d0ba 2262
marcozecchini 0:9fca2b23d0ba 2263 p.join()
marcozecchini 0:9fca2b23d0ba 2264
marcozecchini 0:9fca2b23d0ba 2265 test_builds = {}
marcozecchini 0:9fca2b23d0ba 2266 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
marcozecchini 0:9fca2b23d0ba 2267
marcozecchini 0:9fca2b23d0ba 2268 return result, test_builds
marcozecchini 0:9fca2b23d0ba 2269
marcozecchini 0:9fca2b23d0ba 2270
marcozecchini 0:9fca2b23d0ba 2271 def test_spec_from_test_builds(test_builds):
marcozecchini 0:9fca2b23d0ba 2272 return {
marcozecchini 0:9fca2b23d0ba 2273 "builds": test_builds
marcozecchini 0:9fca2b23d0ba 2274 }