Alessandro Angelino / mbed-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-tools by Morpheus

test_api.py@2:5f044fef8f98, 2016-03-30 (annotated)

Committer:
screamer
Date:
Wed Mar 30 19:28:05 2016 +0100
Revision:
2:5f044fef8f98
Parent:
0:4a2e5f0422d6 
Adjust all paths in python to reflect the mbed OS structure

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:4a2e5f0422d6 1 """
screamer 0:4a2e5f0422d6 2 mbed SDK
screamer 0:4a2e5f0422d6 3 Copyright (c) 2011-2014 ARM Limited
screamer 0:4a2e5f0422d6 4
screamer 0:4a2e5f0422d6 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:4a2e5f0422d6 6 you may not use this file except in compliance with the License.
screamer 0:4a2e5f0422d6 7 You may obtain a copy of the License at
screamer 0:4a2e5f0422d6 8
screamer 0:4a2e5f0422d6 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:4a2e5f0422d6 10
screamer 0:4a2e5f0422d6 11 Unless required by applicable law or agreed to in writing, software
screamer 0:4a2e5f0422d6 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:4a2e5f0422d6 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:4a2e5f0422d6 14 See the License for the specific language governing permissions and
screamer 0:4a2e5f0422d6 15 limitations under the License.
screamer 0:4a2e5f0422d6 16
screamer 0:4a2e5f0422d6 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer 0:4a2e5f0422d6 18 """
screamer 0:4a2e5f0422d6 19
screamer 0:4a2e5f0422d6 20 import os
screamer 0:4a2e5f0422d6 21 import re
screamer 0:4a2e5f0422d6 22 import sys
screamer 0:4a2e5f0422d6 23 import json
screamer 0:4a2e5f0422d6 24 import uuid
screamer 0:4a2e5f0422d6 25 import pprint
screamer 0:4a2e5f0422d6 26 import random
screamer 0:4a2e5f0422d6 27 import optparse
screamer 0:4a2e5f0422d6 28 import datetime
screamer 0:4a2e5f0422d6 29 import threading
screamer 0:4a2e5f0422d6 30 import ctypes
screamer 0:4a2e5f0422d6 31 from types import ListType
screamer 0:4a2e5f0422d6 32 from colorama import Fore, Back, Style
screamer 0:4a2e5f0422d6 33 from prettytable import PrettyTable
screamer 0:4a2e5f0422d6 34
screamer 0:4a2e5f0422d6 35 from time import sleep, time
screamer 0:4a2e5f0422d6 36 from Queue import Queue, Empty
screamer 0:4a2e5f0422d6 37 from os.path import join, exists, basename
screamer 0:4a2e5f0422d6 38 from threading import Thread, Lock
screamer 0:4a2e5f0422d6 39 from subprocess import Popen, PIPE
screamer 0:4a2e5f0422d6 40
screamer 0:4a2e5f0422d6 41 # Imports related to mbed build api
screamer 2:5f044fef8f98 42 from tools.tests import TESTS
screamer 2:5f044fef8f98 43 from tools.tests import TEST_MAP
screamer 2:5f044fef8f98 44 from tools.paths import BUILD_DIR
screamer 2:5f044fef8f98 45 from tools.paths import HOST_TESTS
screamer 2:5f044fef8f98 46 from tools.utils import ToolException
screamer 2:5f044fef8f98 47 from tools.utils import NotSupportedException
screamer 2:5f044fef8f98 48 from tools.utils import construct_enum
screamer 2:5f044fef8f98 49 from tools.targets import TARGET_MAP
screamer 2:5f044fef8f98 50 from tools.test_db import BaseDBAccess
screamer 2:5f044fef8f98 51 from tools.build_api import build_project, build_mbed_libs, build_lib
screamer 2:5f044fef8f98 52 from tools.build_api import get_target_supported_toolchains
screamer 2:5f044fef8f98 53 from tools.build_api import write_build_report
screamer 2:5f044fef8f98 54 from tools.build_api import prep_report
screamer 2:5f044fef8f98 55 from tools.build_api import prep_properties
screamer 2:5f044fef8f98 56 from tools.build_api import create_result
screamer 2:5f044fef8f98 57 from tools.build_api import add_result_to_report
screamer 2:5f044fef8f98 58 from tools.libraries import LIBRARIES, LIBRARY_MAP
screamer 2:5f044fef8f98 59 from tools.toolchains import TOOLCHAIN_BIN_PATH
screamer 2:5f044fef8f98 60 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 0:4a2e5f0422d6 61
screamer 2:5f044fef8f98 62 import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer 0:4a2e5f0422d6 63
screamer 0:4a2e5f0422d6 64 try:
screamer 0:4a2e5f0422d6 65 import mbed_lstools
screamer 2:5f044fef8f98 66 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:4a2e5f0422d6 67 except:
screamer 0:4a2e5f0422d6 68 pass
screamer 0:4a2e5f0422d6 69
screamer 0:4a2e5f0422d6 70
screamer 0:4a2e5f0422d6 71 class ProcessObserver(Thread):
screamer 0:4a2e5f0422d6 72 def __init__(self, proc):
screamer 0:4a2e5f0422d6 73 Thread.__init__(self)
screamer 0:4a2e5f0422d6 74 self.proc = proc
screamer 0:4a2e5f0422d6 75 self.queue = Queue()
screamer 0:4a2e5f0422d6 76 self.daemon = True
screamer 0:4a2e5f0422d6 77 self.active = True
screamer 0:4a2e5f0422d6 78 self.start()
screamer 0:4a2e5f0422d6 79
screamer 0:4a2e5f0422d6 80 def run(self):
screamer 0:4a2e5f0422d6 81 while self.active:
screamer 0:4a2e5f0422d6 82 c = self.proc.stdout.read(1)
screamer 0:4a2e5f0422d6 83 self.queue.put(c)
screamer 0:4a2e5f0422d6 84
screamer 0:4a2e5f0422d6 85 def stop(self):
screamer 0:4a2e5f0422d6 86 self.active = False
screamer 0:4a2e5f0422d6 87 try:
screamer 0:4a2e5f0422d6 88 self.proc.terminate()
screamer 0:4a2e5f0422d6 89 except Exception, _:
screamer 0:4a2e5f0422d6 90 pass
screamer 0:4a2e5f0422d6 91
screamer 0:4a2e5f0422d6 92
screamer 0:4a2e5f0422d6 93 class SingleTestExecutor(threading.Thread):
screamer 0:4a2e5f0422d6 94 """ Example: Single test class in separate thread usage
screamer 0:4a2e5f0422d6 95 """
screamer 0:4a2e5f0422d6 96 def __init__(self, single_test):
screamer 0:4a2e5f0422d6 97 self.single_test = single_test
screamer 0:4a2e5f0422d6 98 threading.Thread.__init__(self)
screamer 0:4a2e5f0422d6 99
screamer 0:4a2e5f0422d6 100 def run(self):
screamer 0:4a2e5f0422d6 101 start = time()
screamer 0:4a2e5f0422d6 102 # Execute tests depending on options and filter applied
screamer 0:4a2e5f0422d6 103 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer 0:4a2e5f0422d6 104 elapsed_time = time() - start
screamer 0:4a2e5f0422d6 105
screamer 0:4a2e5f0422d6 106 # Human readable summary
screamer 0:4a2e5f0422d6 107 if not self.single_test.opts_suppress_summary:
screamer 0:4a2e5f0422d6 108 # prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 109 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:4a2e5f0422d6 110 if self.single_test.opts_test_x_toolchain_summary:
screamer 0:4a2e5f0422d6 111 # prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 112 # table shows text x toolchain test result matrix
screamer 0:4a2e5f0422d6 113 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:4a2e5f0422d6 114 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:4a2e5f0422d6 115
screamer 0:4a2e5f0422d6 116
screamer 0:4a2e5f0422d6 117 class SingleTestRunner(object):
screamer 0:4a2e5f0422d6 118 """ Object wrapper for single test run which may involve multiple MUTs
screamer 0:4a2e5f0422d6 119 """
screamer 0:4a2e5f0422d6 120 RE_DETECT_TESTCASE_RESULT = None
screamer 0:4a2e5f0422d6 121
screamer 0:4a2e5f0422d6 122 # Return codes for test script
screamer 0:4a2e5f0422d6 123 TEST_RESULT_OK = "OK"
screamer 0:4a2e5f0422d6 124 TEST_RESULT_FAIL = "FAIL"
screamer 0:4a2e5f0422d6 125 TEST_RESULT_ERROR = "ERROR"
screamer 0:4a2e5f0422d6 126 TEST_RESULT_UNDEF = "UNDEF"
screamer 0:4a2e5f0422d6 127 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer 0:4a2e5f0422d6 128 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer 0:4a2e5f0422d6 129 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer 0:4a2e5f0422d6 130 TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer 0:4a2e5f0422d6 131 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer 0:4a2e5f0422d6 132 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer 0:4a2e5f0422d6 133 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer 0:4a2e5f0422d6 134 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer 0:4a2e5f0422d6 135
screamer 0:4a2e5f0422d6 136 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer 0:4a2e5f0422d6 137 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer 0:4a2e5f0422d6 138 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer 0:4a2e5f0422d6 139
screamer 0:4a2e5f0422d6 140 muts = {} # MUTs descriptor (from external file)
screamer 0:4a2e5f0422d6 141 test_spec = {} # Test specification (from external file)
screamer 0:4a2e5f0422d6 142
screamer 0:4a2e5f0422d6 143 # mbed test suite -> SingleTestRunner
screamer 0:4a2e5f0422d6 144 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer 0:4a2e5f0422d6 145 "failure" : TEST_RESULT_FAIL,
screamer 0:4a2e5f0422d6 146 "error" : TEST_RESULT_ERROR,
screamer 0:4a2e5f0422d6 147 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer 0:4a2e5f0422d6 148 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer 0:4a2e5f0422d6 149 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer 0:4a2e5f0422d6 150 "timeout" : TEST_RESULT_TIMEOUT,
screamer 0:4a2e5f0422d6 151 "no_image" : TEST_RESULT_NO_IMAGE,
screamer 0:4a2e5f0422d6 152 "end" : TEST_RESULT_UNDEF,
screamer 0:4a2e5f0422d6 153 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer 0:4a2e5f0422d6 154 "build_failed" : TEST_RESULT_BUILD_FAILED,
screamer 0:4a2e5f0422d6 155 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer 0:4a2e5f0422d6 156 }
screamer 0:4a2e5f0422d6 157
screamer 0:4a2e5f0422d6 158 def __init__(self,
screamer 0:4a2e5f0422d6 159 _global_loops_count=1,
screamer 0:4a2e5f0422d6 160 _test_loops_list=None,
screamer 0:4a2e5f0422d6 161 _muts={},
screamer 0:4a2e5f0422d6 162 _clean=False,
screamer 0:4a2e5f0422d6 163 _opts_db_url=None,
screamer 0:4a2e5f0422d6 164 _opts_log_file_name=None,
screamer 0:4a2e5f0422d6 165 _opts_report_html_file_name=None,
screamer 0:4a2e5f0422d6 166 _opts_report_junit_file_name=None,
screamer 0:4a2e5f0422d6 167 _opts_report_build_file_name=None,
screamer 0:4a2e5f0422d6 168 _opts_build_report={},
screamer 0:4a2e5f0422d6 169 _opts_build_properties={},
screamer 0:4a2e5f0422d6 170 _test_spec={},
screamer 0:4a2e5f0422d6 171 _opts_goanna_for_mbed_sdk=None,
screamer 0:4a2e5f0422d6 172 _opts_goanna_for_tests=None,
screamer 0:4a2e5f0422d6 173 _opts_shuffle_test_order=False,
screamer 0:4a2e5f0422d6 174 _opts_shuffle_test_seed=None,
screamer 0:4a2e5f0422d6 175 _opts_test_by_names=None,
screamer 0:4a2e5f0422d6 176 _opts_peripheral_by_names=None,
screamer 0:4a2e5f0422d6 177 _opts_test_only_peripheral=False,
screamer 0:4a2e5f0422d6 178 _opts_test_only_common=False,
screamer 0:4a2e5f0422d6 179 _opts_verbose_skipped_tests=False,
screamer 0:4a2e5f0422d6 180 _opts_verbose_test_result_only=False,
screamer 0:4a2e5f0422d6 181 _opts_verbose=False,
screamer 0:4a2e5f0422d6 182 _opts_firmware_global_name=None,
screamer 0:4a2e5f0422d6 183 _opts_only_build_tests=False,
screamer 0:4a2e5f0422d6 184 _opts_parallel_test_exec=False,
screamer 0:4a2e5f0422d6 185 _opts_suppress_summary=False,
screamer 0:4a2e5f0422d6 186 _opts_test_x_toolchain_summary=False,
screamer 0:4a2e5f0422d6 187 _opts_copy_method=None,
screamer 0:4a2e5f0422d6 188 _opts_mut_reset_type=None,
screamer 0:4a2e5f0422d6 189 _opts_jobs=None,
screamer 0:4a2e5f0422d6 190 _opts_waterfall_test=None,
screamer 0:4a2e5f0422d6 191 _opts_consolidate_waterfall_test=None,
screamer 0:4a2e5f0422d6 192 _opts_extend_test_timeout=None,
screamer 0:4a2e5f0422d6 193 _opts_auto_detect=None,
screamer 0:4a2e5f0422d6 194 _opts_include_non_automated=False):
screamer 0:4a2e5f0422d6 195 """ Let's try hard to init this object
screamer 0:4a2e5f0422d6 196 """
screamer 0:4a2e5f0422d6 197 from colorama import init
screamer 0:4a2e5f0422d6 198 init()
screamer 0:4a2e5f0422d6 199
screamer 0:4a2e5f0422d6 200 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer 0:4a2e5f0422d6 201 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer 0:4a2e5f0422d6 202 # Settings related to test loops counters
screamer 0:4a2e5f0422d6 203 try:
screamer 0:4a2e5f0422d6 204 _global_loops_count = int(_global_loops_count)
screamer 0:4a2e5f0422d6 205 except:
screamer 0:4a2e5f0422d6 206 _global_loops_count = 1
screamer 0:4a2e5f0422d6 207 if _global_loops_count < 1:
screamer 0:4a2e5f0422d6 208 _global_loops_count = 1
screamer 0:4a2e5f0422d6 209 self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer 0:4a2e5f0422d6 210 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer 0:4a2e5f0422d6 211 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer 0:4a2e5f0422d6 212
screamer 0:4a2e5f0422d6 213 self.shuffle_random_seed = 0.0
screamer 0:4a2e5f0422d6 214 self.SHUFFLE_SEED_ROUND = 10
screamer 0:4a2e5f0422d6 215
screamer 0:4a2e5f0422d6 216 # MUT list and test specification storage
screamer 0:4a2e5f0422d6 217 self.muts = _muts
screamer 0:4a2e5f0422d6 218 self.test_spec = _test_spec
screamer 0:4a2e5f0422d6 219
screamer 0:4a2e5f0422d6 220 # Settings passed e.g. from command line
screamer 0:4a2e5f0422d6 221 self.opts_db_url = _opts_db_url
screamer 0:4a2e5f0422d6 222 self.opts_log_file_name = _opts_log_file_name
screamer 0:4a2e5f0422d6 223 self.opts_report_html_file_name = _opts_report_html_file_name
screamer 0:4a2e5f0422d6 224 self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer 0:4a2e5f0422d6 225 self.opts_report_build_file_name = _opts_report_build_file_name
screamer 0:4a2e5f0422d6 226 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer 0:4a2e5f0422d6 227 self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer 0:4a2e5f0422d6 228 self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer 0:4a2e5f0422d6 229 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer 0:4a2e5f0422d6 230 self.opts_test_by_names = _opts_test_by_names
screamer 0:4a2e5f0422d6 231 self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer 0:4a2e5f0422d6 232 self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer 0:4a2e5f0422d6 233 self.opts_test_only_common = _opts_test_only_common
screamer 0:4a2e5f0422d6 234 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer 0:4a2e5f0422d6 235 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer 0:4a2e5f0422d6 236 self.opts_verbose = _opts_verbose
screamer 0:4a2e5f0422d6 237 self.opts_firmware_global_name = _opts_firmware_global_name
screamer 0:4a2e5f0422d6 238 self.opts_only_build_tests = _opts_only_build_tests
screamer 0:4a2e5f0422d6 239 self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer 0:4a2e5f0422d6 240 self.opts_suppress_summary = _opts_suppress_summary
screamer 0:4a2e5f0422d6 241 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer 0:4a2e5f0422d6 242 self.opts_copy_method = _opts_copy_method
screamer 0:4a2e5f0422d6 243 self.opts_mut_reset_type = _opts_mut_reset_type
screamer 0:4a2e5f0422d6 244 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer 0:4a2e5f0422d6 245 self.opts_waterfall_test = _opts_waterfall_test
screamer 0:4a2e5f0422d6 246 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer 0:4a2e5f0422d6 247 self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer 0:4a2e5f0422d6 248 self.opts_clean = _clean
screamer 0:4a2e5f0422d6 249 self.opts_auto_detect = _opts_auto_detect
screamer 0:4a2e5f0422d6 250 self.opts_include_non_automated = _opts_include_non_automated
screamer 0:4a2e5f0422d6 251
screamer 0:4a2e5f0422d6 252 self.build_report = _opts_build_report
screamer 0:4a2e5f0422d6 253 self.build_properties = _opts_build_properties
screamer 0:4a2e5f0422d6 254
screamer 0:4a2e5f0422d6 255 # File / screen logger initialization
screamer 0:4a2e5f0422d6 256 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer 0:4a2e5f0422d6 257
screamer 0:4a2e5f0422d6 258 # Database related initializations
screamer 0:4a2e5f0422d6 259 self.db_logger = factory_db_logger(self.opts_db_url)
screamer 0:4a2e5f0422d6 260 self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer 0:4a2e5f0422d6 261 # Let's connect to database to set up credentials and confirm database is ready
screamer 0:4a2e5f0422d6 262 if self.db_logger:
screamer 0:4a2e5f0422d6 263 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer 0:4a2e5f0422d6 264 if self.db_logger.is_connected():
screamer 0:4a2e5f0422d6 265 # Get hostname and uname so we can use it as build description
screamer 0:4a2e5f0422d6 266 # when creating new build_id in external database
screamer 0:4a2e5f0422d6 267 (_hostname, _uname) = self.db_logger.get_hostname()
screamer 0:4a2e5f0422d6 268 _host_location = os.path.dirname(os.path.abspath(__file__))
screamer 0:4a2e5f0422d6 269 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer 0:4a2e5f0422d6 270 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer 0:4a2e5f0422d6 271 self.db_logger.disconnect()
screamer 0:4a2e5f0422d6 272
screamer 0:4a2e5f0422d6 273 def dump_options(self):
screamer 0:4a2e5f0422d6 274 """ Function returns data structure with common settings passed to SingelTestRunner
screamer 0:4a2e5f0422d6 275 It can be used for example to fill _extra fields in database storing test suite single run data
screamer 0:4a2e5f0422d6 276 Example:
screamer 0:4a2e5f0422d6 277 data = self.dump_options()
screamer 0:4a2e5f0422d6 278 or
screamer 0:4a2e5f0422d6 279 data_str = json.dumps(self.dump_options())
screamer 0:4a2e5f0422d6 280 """
screamer 0:4a2e5f0422d6 281 result = {"db_url" : str(self.opts_db_url),
screamer 0:4a2e5f0422d6 282 "log_file_name" : str(self.opts_log_file_name),
screamer 0:4a2e5f0422d6 283 "shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer 0:4a2e5f0422d6 284 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer 0:4a2e5f0422d6 285 "test_by_names" : str(self.opts_test_by_names),
screamer 0:4a2e5f0422d6 286 "peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer 0:4a2e5f0422d6 287 "test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer 0:4a2e5f0422d6 288 "test_only_common" : str(self.opts_test_only_common),
screamer 0:4a2e5f0422d6 289 "verbose" : str(self.opts_verbose),
screamer 0:4a2e5f0422d6 290 "firmware_global_name" : str(self.opts_firmware_global_name),
screamer 0:4a2e5f0422d6 291 "only_build_tests" : str(self.opts_only_build_tests),
screamer 0:4a2e5f0422d6 292 "copy_method" : str(self.opts_copy_method),
screamer 0:4a2e5f0422d6 293 "mut_reset_type" : str(self.opts_mut_reset_type),
screamer 0:4a2e5f0422d6 294 "jobs" : str(self.opts_jobs),
screamer 0:4a2e5f0422d6 295 "extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer 0:4a2e5f0422d6 296 "_dummy" : ''
screamer 0:4a2e5f0422d6 297 }
screamer 0:4a2e5f0422d6 298 return result
screamer 0:4a2e5f0422d6 299
screamer 0:4a2e5f0422d6 300 def shuffle_random_func(self):
screamer 0:4a2e5f0422d6 301 return self.shuffle_random_seed
screamer 0:4a2e5f0422d6 302
screamer 0:4a2e5f0422d6 303 def is_shuffle_seed_float(self):
screamer 0:4a2e5f0422d6 304 """ return true if function parameter can be converted to float
screamer 0:4a2e5f0422d6 305 """
screamer 0:4a2e5f0422d6 306 result = True
screamer 0:4a2e5f0422d6 307 try:
screamer 0:4a2e5f0422d6 308 float(self.shuffle_random_seed)
screamer 0:4a2e5f0422d6 309 except ValueError:
screamer 0:4a2e5f0422d6 310 result = False
screamer 0:4a2e5f0422d6 311 return result
screamer 0:4a2e5f0422d6 312
screamer 0:4a2e5f0422d6 313 # This will store target / toolchain specific properties
screamer 0:4a2e5f0422d6 314 test_suite_properties_ext = {} # target : toolchain
screamer 0:4a2e5f0422d6 315 # Here we store test results
screamer 0:4a2e5f0422d6 316 test_summary = []
screamer 0:4a2e5f0422d6 317 # Here we store test results in extended data structure
screamer 0:4a2e5f0422d6 318 test_summary_ext = {}
screamer 0:4a2e5f0422d6 319 execute_thread_slice_lock = Lock()
screamer 0:4a2e5f0422d6 320
screamer 0:4a2e5f0422d6 321 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer 0:4a2e5f0422d6 322 for toolchain in toolchains:
screamer 0:4a2e5f0422d6 323 tt_id = "%s::%s" % (toolchain, target)
screamer 0:4a2e5f0422d6 324
screamer 0:4a2e5f0422d6 325 T = TARGET_MAP[target]
screamer 0:4a2e5f0422d6 326
screamer 0:4a2e5f0422d6 327 # print target, toolchain
screamer 0:4a2e5f0422d6 328 # Test suite properties returned to external tools like CI
screamer 0:4a2e5f0422d6 329 test_suite_properties = {
screamer 0:4a2e5f0422d6 330 'jobs': self.opts_jobs,
screamer 0:4a2e5f0422d6 331 'clean': clean,
screamer 0:4a2e5f0422d6 332 'target': target,
screamer 0:4a2e5f0422d6 333 'vendor': T.extra_labels[0],
screamer 0:4a2e5f0422d6 334 'test_ids': ', '.join(test_ids),
screamer 0:4a2e5f0422d6 335 'toolchain': toolchain,
screamer 0:4a2e5f0422d6 336 'shuffle_random_seed': self.shuffle_random_seed
screamer 0:4a2e5f0422d6 337 }
screamer 0:4a2e5f0422d6 338
screamer 0:4a2e5f0422d6 339
screamer 0:4a2e5f0422d6 340 # print '=== %s::%s ===' % (target, toolchain)
screamer 0:4a2e5f0422d6 341 # Let's build our test
screamer 0:4a2e5f0422d6 342 if target not in TARGET_MAP:
screamer 0:4a2e5f0422d6 343 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
screamer 0:4a2e5f0422d6 344 continue
screamer 0:4a2e5f0422d6 345
screamer 0:4a2e5f0422d6 346 build_mbed_libs_options = ["analyze"] if self.opts_goanna_for_mbed_sdk else None
screamer 0:4a2e5f0422d6 347 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
screamer 0:4a2e5f0422d6 348
screamer 0:4a2e5f0422d6 349
screamer 0:4a2e5f0422d6 350 try:
screamer 0:4a2e5f0422d6 351 build_mbed_libs_result = build_mbed_libs(T,
screamer 0:4a2e5f0422d6 352 toolchain,
screamer 0:4a2e5f0422d6 353 options=build_mbed_libs_options,
screamer 0:4a2e5f0422d6 354 clean=clean_mbed_libs_options,
screamer 0:4a2e5f0422d6 355 verbose=self.opts_verbose,
screamer 0:4a2e5f0422d6 356 jobs=self.opts_jobs,
screamer 0:4a2e5f0422d6 357 report=build_report,
screamer 0:4a2e5f0422d6 358 properties=build_properties)
screamer 0:4a2e5f0422d6 359
screamer 0:4a2e5f0422d6 360 if not build_mbed_libs_result:
screamer 0:4a2e5f0422d6 361 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
screamer 0:4a2e5f0422d6 362 continue
screamer 0:4a2e5f0422d6 363
screamer 0:4a2e5f0422d6 364 except ToolException:
screamer 0:4a2e5f0422d6 365 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
screamer 0:4a2e5f0422d6 366 continue
screamer 0:4a2e5f0422d6 367
screamer 0:4a2e5f0422d6 368 build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer 0:4a2e5f0422d6 369
screamer 0:4a2e5f0422d6 370 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer 0:4a2e5f0422d6 371 test_suite_properties['build_dir'] = build_dir
screamer 0:4a2e5f0422d6 372 test_suite_properties['skipped'] = []
screamer 0:4a2e5f0422d6 373
screamer 0:4a2e5f0422d6 374 # Enumerate through all tests and shuffle test order if requested
screamer 0:4a2e5f0422d6 375 test_map_keys = sorted(TEST_MAP.keys())
screamer 0:4a2e5f0422d6 376
screamer 0:4a2e5f0422d6 377 if self.opts_shuffle_test_order:
screamer 0:4a2e5f0422d6 378 random.shuffle(test_map_keys, self.shuffle_random_func)
screamer 0:4a2e5f0422d6 379 # Update database with shuffle seed f applicable
screamer 0:4a2e5f0422d6 380 if self.db_logger:
screamer 0:4a2e5f0422d6 381 self.db_logger.reconnect();
screamer 0:4a2e5f0422d6 382 if self.db_logger.is_connected():
screamer 0:4a2e5f0422d6 383 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
screamer 0:4a2e5f0422d6 384 self.db_logger.disconnect();
screamer 0:4a2e5f0422d6 385
screamer 0:4a2e5f0422d6 386 if self.db_logger:
screamer 0:4a2e5f0422d6 387 self.db_logger.reconnect();
screamer 0:4a2e5f0422d6 388 if self.db_logger.is_connected():
screamer 0:4a2e5f0422d6 389 # Update MUTs and Test Specification in database
screamer 0:4a2e5f0422d6 390 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
screamer 0:4a2e5f0422d6 391 # Update Extra information in database (some options passed to test suite)
screamer 0:4a2e5f0422d6 392 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
screamer 0:4a2e5f0422d6 393 self.db_logger.disconnect();
screamer 0:4a2e5f0422d6 394
screamer 0:4a2e5f0422d6 395 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer 0:4a2e5f0422d6 396 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer 0:4a2e5f0422d6 397
screamer 0:4a2e5f0422d6 398 for skipped_test_id in skipped_test_map_keys:
screamer 0:4a2e5f0422d6 399 test_suite_properties['skipped'].append(skipped_test_id)
screamer 0:4a2e5f0422d6 400
screamer 0:4a2e5f0422d6 401
screamer 0:4a2e5f0422d6 402 # First pass through all tests and determine which libraries need to be built
screamer 0:4a2e5f0422d6 403 libraries = []
screamer 0:4a2e5f0422d6 404 for test_id in valid_test_map_keys:
screamer 0:4a2e5f0422d6 405 test = TEST_MAP[test_id]
screamer 0:4a2e5f0422d6 406
screamer 0:4a2e5f0422d6 407 # Detect which lib should be added to test
screamer 0:4a2e5f0422d6 408 # Some libs have to compiled like RTOS or ETH
screamer 0:4a2e5f0422d6 409 for lib in LIBRARIES:
screamer 0:4a2e5f0422d6 410 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer 0:4a2e5f0422d6 411 libraries.append(lib['id'])
screamer 0:4a2e5f0422d6 412
screamer 0:4a2e5f0422d6 413
screamer 0:4a2e5f0422d6 414 build_project_options = ["analyze"] if self.opts_goanna_for_tests else None
screamer 0:4a2e5f0422d6 415 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer 0:4a2e5f0422d6 416
screamer 0:4a2e5f0422d6 417 # Build all required libraries
screamer 0:4a2e5f0422d6 418 for lib_id in libraries:
screamer 0:4a2e5f0422d6 419 try:
screamer 0:4a2e5f0422d6 420 build_lib(lib_id,
screamer 0:4a2e5f0422d6 421 T,
screamer 0:4a2e5f0422d6 422 toolchain,
screamer 0:4a2e5f0422d6 423 options=build_project_options,
screamer 0:4a2e5f0422d6 424 verbose=self.opts_verbose,
screamer 0:4a2e5f0422d6 425 clean=clean_mbed_libs_options,
screamer 0:4a2e5f0422d6 426 jobs=self.opts_jobs,
screamer 0:4a2e5f0422d6 427 report=build_report,
screamer 0:4a2e5f0422d6 428 properties=build_properties)
screamer 0:4a2e5f0422d6 429
screamer 0:4a2e5f0422d6 430 except ToolException:
screamer 0:4a2e5f0422d6 431 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
screamer 0:4a2e5f0422d6 432 continue
screamer 0:4a2e5f0422d6 433
screamer 0:4a2e5f0422d6 434
screamer 0:4a2e5f0422d6 435 for test_id in valid_test_map_keys:
screamer 0:4a2e5f0422d6 436 test = TEST_MAP[test_id]
screamer 0:4a2e5f0422d6 437
screamer 0:4a2e5f0422d6 438 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer 0:4a2e5f0422d6 439
screamer 0:4a2e5f0422d6 440 # TODO: move this 2 below loops to separate function
screamer 0:4a2e5f0422d6 441 INC_DIRS = []
screamer 0:4a2e5f0422d6 442 for lib_id in libraries:
screamer 0:4a2e5f0422d6 443 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer 0:4a2e5f0422d6 444 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer 0:4a2e5f0422d6 445
screamer 0:4a2e5f0422d6 446 MACROS = []
screamer 0:4a2e5f0422d6 447 for lib_id in libraries:
screamer 0:4a2e5f0422d6 448 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer 0:4a2e5f0422d6 449 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer 0:4a2e5f0422d6 450 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer 0:4a2e5f0422d6 451 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer 0:4a2e5f0422d6 452 test_uuid = uuid.uuid4()
screamer 0:4a2e5f0422d6 453 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer 0:4a2e5f0422d6 454
screamer 0:4a2e5f0422d6 455 # Prepare extended test results data structure (it can be used to generate detailed test report)
screamer 0:4a2e5f0422d6 456 if target not in self.test_summary_ext:
screamer 0:4a2e5f0422d6 457 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer 0:4a2e5f0422d6 458 if toolchain not in self.test_summary_ext[target]:
screamer 0:4a2e5f0422d6 459 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer 0:4a2e5f0422d6 460
screamer 0:4a2e5f0422d6 461 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer 0:4a2e5f0422d6 462
screamer 0:4a2e5f0422d6 463 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer 0:4a2e5f0422d6 464 try:
screamer 0:4a2e5f0422d6 465 path = build_project(test.source_dir,
screamer 0:4a2e5f0422d6 466 join(build_dir, test_id),
screamer 0:4a2e5f0422d6 467 T,
screamer 0:4a2e5f0422d6 468 toolchain,
screamer 0:4a2e5f0422d6 469 test.dependencies,
screamer 0:4a2e5f0422d6 470 options=build_project_options,
screamer 0:4a2e5f0422d6 471 clean=clean_project_options,
screamer 0:4a2e5f0422d6 472 verbose=self.opts_verbose,
screamer 0:4a2e5f0422d6 473 name=project_name,
screamer 0:4a2e5f0422d6 474 macros=MACROS,
screamer 0:4a2e5f0422d6 475 inc_dirs=INC_DIRS,
screamer 0:4a2e5f0422d6 476 jobs=self.opts_jobs,
screamer 0:4a2e5f0422d6 477 report=build_report,
screamer 0:4a2e5f0422d6 478 properties=build_properties,
screamer 0:4a2e5f0422d6 479 project_id=test_id,
screamer 0:4a2e5f0422d6 480 project_description=test.get_description())
screamer 0:4a2e5f0422d6 481
screamer 0:4a2e5f0422d6 482 except Exception, e:
screamer 0:4a2e5f0422d6 483 project_name_str = project_name if project_name is not None else test_id
screamer 0:4a2e5f0422d6 484
screamer 0:4a2e5f0422d6 485
screamer 0:4a2e5f0422d6 486 test_result = self.TEST_RESULT_FAIL
screamer 0:4a2e5f0422d6 487
screamer 0:4a2e5f0422d6 488 if isinstance(e, ToolException):
screamer 0:4a2e5f0422d6 489 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
screamer 0:4a2e5f0422d6 490 test_result = self.TEST_RESULT_BUILD_FAILED
screamer 0:4a2e5f0422d6 491 elif isinstance(e, NotSupportedException):
screamer 0:4a2e5f0422d6 492 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
screamer 0:4a2e5f0422d6 493 test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer 0:4a2e5f0422d6 494
screamer 0:4a2e5f0422d6 495
screamer 0:4a2e5f0422d6 496 # Append test results to global test summary
screamer 0:4a2e5f0422d6 497 self.test_summary.append(
screamer 0:4a2e5f0422d6 498 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
screamer 0:4a2e5f0422d6 499 )
screamer 0:4a2e5f0422d6 500
screamer 0:4a2e5f0422d6 501 # Add detailed test result to test summary structure
screamer 0:4a2e5f0422d6 502 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:4a2e5f0422d6 503 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:4a2e5f0422d6 504
screamer 0:4a2e5f0422d6 505 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer 0:4a2e5f0422d6 506 'result' : test_result,
screamer 0:4a2e5f0422d6 507 'output' : '',
screamer 0:4a2e5f0422d6 508 'target_name' : target,
screamer 0:4a2e5f0422d6 509 'target_name_unique': target,
screamer 0:4a2e5f0422d6 510 'toolchain_name' : toolchain,
screamer 0:4a2e5f0422d6 511 'id' : test_id,
screamer 0:4a2e5f0422d6 512 'description' : test.get_description(),
screamer 0:4a2e5f0422d6 513 'elapsed_time' : 0,
screamer 0:4a2e5f0422d6 514 'duration' : 0,
screamer 0:4a2e5f0422d6 515 'copy_method' : None
screamer 0:4a2e5f0422d6 516 }})
screamer 0:4a2e5f0422d6 517 continue
screamer 0:4a2e5f0422d6 518
screamer 0:4a2e5f0422d6 519 if self.opts_only_build_tests:
screamer 0:4a2e5f0422d6 520 # With this option we are skipping testing phase
screamer 0:4a2e5f0422d6 521 continue
screamer 0:4a2e5f0422d6 522
screamer 0:4a2e5f0422d6 523 # Test duration can be increased by global value
screamer 0:4a2e5f0422d6 524 test_duration = test.duration
screamer 0:4a2e5f0422d6 525 if self.opts_extend_test_timeout is not None:
screamer 0:4a2e5f0422d6 526 test_duration += self.opts_extend_test_timeout
screamer 0:4a2e5f0422d6 527
screamer 0:4a2e5f0422d6 528 # For an automated test the duration act as a timeout after
screamer 0:4a2e5f0422d6 529 # which the test gets interrupted
screamer 0:4a2e5f0422d6 530 test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer 0:4a2e5f0422d6 531 test_loops = self.get_test_loop_count(test_id)
screamer 0:4a2e5f0422d6 532
screamer 0:4a2e5f0422d6 533 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer 0:4a2e5f0422d6 534 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer 0:4a2e5f0422d6 535 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer 0:4a2e5f0422d6 536
screamer 0:4a2e5f0422d6 537 # read MUTs, test specification and perform tests
screamer 0:4a2e5f0422d6 538 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer 0:4a2e5f0422d6 539
screamer 0:4a2e5f0422d6 540 if handle_results is None:
screamer 0:4a2e5f0422d6 541 continue
screamer 0:4a2e5f0422d6 542
screamer 0:4a2e5f0422d6 543 for handle_result in handle_results:
screamer 0:4a2e5f0422d6 544 if handle_result:
screamer 0:4a2e5f0422d6 545 single_test_result, detailed_test_results = handle_result
screamer 0:4a2e5f0422d6 546 else:
screamer 0:4a2e5f0422d6 547 continue
screamer 0:4a2e5f0422d6 548
screamer 0:4a2e5f0422d6 549 # Append test results to global test summary
screamer 0:4a2e5f0422d6 550 if single_test_result is not None:
screamer 0:4a2e5f0422d6 551 self.test_summary.append(single_test_result)
screamer 0:4a2e5f0422d6 552
screamer 0:4a2e5f0422d6 553 # Add detailed test result to test summary structure
screamer 0:4a2e5f0422d6 554 if target not in self.test_summary_ext[target][toolchain]:
screamer 0:4a2e5f0422d6 555 if test_id not in self.test_summary_ext[target][toolchain]:
screamer 0:4a2e5f0422d6 556 self.test_summary_ext[target][toolchain][test_id] = []
screamer 0:4a2e5f0422d6 557
screamer 0:4a2e5f0422d6 558 append_test_result = detailed_test_results
screamer 0:4a2e5f0422d6 559
screamer 0:4a2e5f0422d6 560 # If waterfall and consolidate-waterfall options are enabled,
screamer 0:4a2e5f0422d6 561 # only include the last test result in the report.
screamer 0:4a2e5f0422d6 562 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer 0:4a2e5f0422d6 563 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer 0:4a2e5f0422d6 564
screamer 0:4a2e5f0422d6 565 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer 0:4a2e5f0422d6 566
screamer 0:4a2e5f0422d6 567 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer 0:4a2e5f0422d6 568 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer 0:4a2e5f0422d6 569
screamer 0:4a2e5f0422d6 570 q.put(target + '_'.join(toolchains))
screamer 0:4a2e5f0422d6 571 return
screamer 0:4a2e5f0422d6 572
screamer 0:4a2e5f0422d6 573 def execute(self):
screamer 0:4a2e5f0422d6 574 clean = self.test_spec.get('clean', False)
screamer 0:4a2e5f0422d6 575 test_ids = self.test_spec.get('test_ids', [])
screamer 0:4a2e5f0422d6 576 q = Queue()
screamer 0:4a2e5f0422d6 577
screamer 0:4a2e5f0422d6 578 # Generate seed for shuffle if seed is not provided in
screamer 0:4a2e5f0422d6 579 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer 0:4a2e5f0422d6 580 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer 0:4a2e5f0422d6 581 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer 0:4a2e5f0422d6 582
screamer 0:4a2e5f0422d6 583
screamer 0:4a2e5f0422d6 584 if self.opts_parallel_test_exec:
screamer 0:4a2e5f0422d6 585 ###################################################################
screamer 0:4a2e5f0422d6 586 # Experimental, parallel test execution per singletest instance.
screamer 0:4a2e5f0422d6 587 ###################################################################
screamer 0:4a2e5f0422d6 588 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer 0:4a2e5f0422d6 589 # Note: We are building here in parallel for each target separately!
screamer 0:4a2e5f0422d6 590 # So we are not building the same thing multiple times and compilers
screamer 0:4a2e5f0422d6 591 # in separate threads do not collide.
screamer 0:4a2e5f0422d6 592 # Inside execute_thread_slice() function function handle() will be called to
screamer 0:4a2e5f0422d6 593 # get information about available MUTs (per target).
screamer 0:4a2e5f0422d6 594 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:4a2e5f0422d6 595 self.test_suite_properties_ext[target] = {}
screamer 0:4a2e5f0422d6 596 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer 0:4a2e5f0422d6 597 t.daemon = True
screamer 0:4a2e5f0422d6 598 t.start()
screamer 0:4a2e5f0422d6 599 execute_threads.append(t)
screamer 0:4a2e5f0422d6 600
screamer 0:4a2e5f0422d6 601 for t in execute_threads:
screamer 0:4a2e5f0422d6 602 q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer 0:4a2e5f0422d6 603 else:
screamer 0:4a2e5f0422d6 604 # Serialized (not parallel) test execution
screamer 0:4a2e5f0422d6 605 for target, toolchains in self.test_spec['targets'].iteritems():
screamer 0:4a2e5f0422d6 606 if target not in self.test_suite_properties_ext:
screamer 0:4a2e5f0422d6 607 self.test_suite_properties_ext[target] = {}
screamer 0:4a2e5f0422d6 608
screamer 0:4a2e5f0422d6 609 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer 0:4a2e5f0422d6 610 q.get()
screamer 0:4a2e5f0422d6 611
screamer 0:4a2e5f0422d6 612 if self.db_logger:
screamer 0:4a2e5f0422d6 613 self.db_logger.reconnect();
screamer 0:4a2e5f0422d6 614 if self.db_logger.is_connected():
screamer 0:4a2e5f0422d6 615 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer 0:4a2e5f0422d6 616 self.db_logger.disconnect();
screamer 0:4a2e5f0422d6 617
screamer 0:4a2e5f0422d6 618 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer 0:4a2e5f0422d6 619
screamer 0:4a2e5f0422d6 620 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer 0:4a2e5f0422d6 621 valid_test_map_keys = []
screamer 0:4a2e5f0422d6 622
screamer 0:4a2e5f0422d6 623 for test_id in test_map_keys:
screamer 0:4a2e5f0422d6 624 test = TEST_MAP[test_id]
screamer 0:4a2e5f0422d6 625 if self.opts_test_by_names and test_id not in self.opts_test_by_names.split(','):
screamer 0:4a2e5f0422d6 626 continue
screamer 0:4a2e5f0422d6 627
screamer 0:4a2e5f0422d6 628 if test_ids and test_id not in test_ids:
screamer 0:4a2e5f0422d6 629 continue
screamer 0:4a2e5f0422d6 630
screamer 0:4a2e5f0422d6 631 if self.opts_test_only_peripheral and not test.peripherals:
screamer 0:4a2e5f0422d6 632 if self.opts_verbose_skipped_tests:
screamer 0:4a2e5f0422d6 633 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:4a2e5f0422d6 634 continue
screamer 0:4a2e5f0422d6 635
screamer 0:4a2e5f0422d6 636 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names.split(',')]):
screamer 0:4a2e5f0422d6 637 # We will skip tests not forced with -p option
screamer 0:4a2e5f0422d6 638 if self.opts_verbose_skipped_tests:
screamer 0:4a2e5f0422d6 639 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
screamer 0:4a2e5f0422d6 640 continue
screamer 0:4a2e5f0422d6 641
screamer 0:4a2e5f0422d6 642 if self.opts_test_only_common and test.peripherals:
screamer 0:4a2e5f0422d6 643 if self.opts_verbose_skipped_tests:
screamer 0:4a2e5f0422d6 644 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
screamer 0:4a2e5f0422d6 645 continue
screamer 0:4a2e5f0422d6 646
screamer 0:4a2e5f0422d6 647 if not include_non_automated and not test.automated:
screamer 0:4a2e5f0422d6 648 if self.opts_verbose_skipped_tests:
screamer 0:4a2e5f0422d6 649 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
screamer 0:4a2e5f0422d6 650 continue
screamer 0:4a2e5f0422d6 651
screamer 0:4a2e5f0422d6 652 if test.is_supported(target, toolchain):
screamer 0:4a2e5f0422d6 653 if test.peripherals is None and self.opts_only_build_tests:
screamer 0:4a2e5f0422d6 654 # When users are using 'build only flag' and test do not have
screamer 0:4a2e5f0422d6 655 # specified peripherals we can allow test building by default
screamer 0:4a2e5f0422d6 656 pass
screamer 0:4a2e5f0422d6 657 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names.split(','):
screamer 0:4a2e5f0422d6 658 # If we force peripheral with option -p we expect test
screamer 0:4a2e5f0422d6 659 # to pass even if peripheral is not in MUTs file.
screamer 0:4a2e5f0422d6 660 pass
screamer 0:4a2e5f0422d6 661 elif not self.is_peripherals_available(target, test.peripherals):
screamer 0:4a2e5f0422d6 662 if self.opts_verbose_skipped_tests:
screamer 0:4a2e5f0422d6 663 if test.peripherals:
screamer 0:4a2e5f0422d6 664 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
screamer 0:4a2e5f0422d6 665 else:
screamer 0:4a2e5f0422d6 666 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
screamer 0:4a2e5f0422d6 667 continue
screamer 0:4a2e5f0422d6 668
screamer 0:4a2e5f0422d6 669 # The test has made it through all the filters, so add it to the valid tests list
screamer 0:4a2e5f0422d6 670 valid_test_map_keys.append(test_id)
screamer 0:4a2e5f0422d6 671
screamer 0:4a2e5f0422d6 672 return valid_test_map_keys
screamer 0:4a2e5f0422d6 673
screamer 0:4a2e5f0422d6 674 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer 0:4a2e5f0422d6 675 # NOTE: This will not preserve order
screamer 0:4a2e5f0422d6 676 return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer 0:4a2e5f0422d6 677
screamer 0:4a2e5f0422d6 678 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer 0:4a2e5f0422d6 679 """ Prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 680 table shows text x toolchain test result matrix
screamer 0:4a2e5f0422d6 681 """
screamer 0:4a2e5f0422d6 682 RESULT_INDEX = 0
screamer 0:4a2e5f0422d6 683 TARGET_INDEX = 1
screamer 0:4a2e5f0422d6 684 TOOLCHAIN_INDEX = 2
screamer 0:4a2e5f0422d6 685 TEST_INDEX = 3
screamer 0:4a2e5f0422d6 686 DESC_INDEX = 4
screamer 0:4a2e5f0422d6 687
screamer 0:4a2e5f0422d6 688 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer 0:4a2e5f0422d6 689 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer 0:4a2e5f0422d6 690 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer 0:4a2e5f0422d6 691 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer 0:4a2e5f0422d6 692
screamer 0:4a2e5f0422d6 693 result = "Test summary:\n"
screamer 0:4a2e5f0422d6 694 for target in unique_targets:
screamer 0:4a2e5f0422d6 695 result_dict = {} # test : { toolchain : result }
screamer 0:4a2e5f0422d6 696 unique_target_toolchains = []
screamer 0:4a2e5f0422d6 697 for test in test_summary:
screamer 0:4a2e5f0422d6 698 if test[TARGET_INDEX] == target:
screamer 0:4a2e5f0422d6 699 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer 0:4a2e5f0422d6 700 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer 0:4a2e5f0422d6 701 if test[TEST_INDEX] not in result_dict:
screamer 0:4a2e5f0422d6 702 result_dict[test[TEST_INDEX]] = {}
screamer 0:4a2e5f0422d6 703 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer 0:4a2e5f0422d6 704
screamer 0:4a2e5f0422d6 705 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
screamer 0:4a2e5f0422d6 706 pt = PrettyTable(pt_cols)
screamer 0:4a2e5f0422d6 707 for col in pt_cols:
screamer 0:4a2e5f0422d6 708 pt.align[col] = "l"
screamer 0:4a2e5f0422d6 709 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:4a2e5f0422d6 710
screamer 0:4a2e5f0422d6 711 for test in unique_tests:
screamer 0:4a2e5f0422d6 712 if test in result_dict:
screamer 0:4a2e5f0422d6 713 test_results = result_dict[test]
screamer 0:4a2e5f0422d6 714 if test in unique_test_desc:
screamer 0:4a2e5f0422d6 715 row = [target, test, unique_test_desc[test]]
screamer 0:4a2e5f0422d6 716 for toolchain in unique_toolchains:
screamer 0:4a2e5f0422d6 717 if toolchain in test_results:
screamer 0:4a2e5f0422d6 718 row.append(test_results[toolchain])
screamer 0:4a2e5f0422d6 719 pt.add_row(row)
screamer 0:4a2e5f0422d6 720 result += pt.get_string()
screamer 0:4a2e5f0422d6 721 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer 0:4a2e5f0422d6 722 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:4a2e5f0422d6 723 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:4a2e5f0422d6 724 return result
screamer 0:4a2e5f0422d6 725
screamer 0:4a2e5f0422d6 726 def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer 0:4a2e5f0422d6 727 """ Prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 728 table shows target x test results matrix across
screamer 0:4a2e5f0422d6 729 """
screamer 0:4a2e5f0422d6 730 success_code = 0 # Success code that can be leter returned to
screamer 0:4a2e5f0422d6 731 result = "Test summary:\n"
screamer 0:4a2e5f0422d6 732 # Pretty table package is used to print results
screamer 0:4a2e5f0422d6 733 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
screamer 0:4a2e5f0422d6 734 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
screamer 0:4a2e5f0422d6 735 pt.align["Result"] = "l" # Left align
screamer 0:4a2e5f0422d6 736 pt.align["Target"] = "l" # Left align
screamer 0:4a2e5f0422d6 737 pt.align["Toolchain"] = "l" # Left align
screamer 0:4a2e5f0422d6 738 pt.align["Test ID"] = "l" # Left align
screamer 0:4a2e5f0422d6 739 pt.align["Test Description"] = "l" # Left align
screamer 0:4a2e5f0422d6 740 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:4a2e5f0422d6 741
screamer 0:4a2e5f0422d6 742 result_dict = {self.TEST_RESULT_OK : 0,
screamer 0:4a2e5f0422d6 743 self.TEST_RESULT_FAIL : 0,
screamer 0:4a2e5f0422d6 744 self.TEST_RESULT_ERROR : 0,
screamer 0:4a2e5f0422d6 745 self.TEST_RESULT_UNDEF : 0,
screamer 0:4a2e5f0422d6 746 self.TEST_RESULT_IOERR_COPY : 0,
screamer 0:4a2e5f0422d6 747 self.TEST_RESULT_IOERR_DISK : 0,
screamer 0:4a2e5f0422d6 748 self.TEST_RESULT_IOERR_SERIAL : 0,
screamer 0:4a2e5f0422d6 749 self.TEST_RESULT_NO_IMAGE : 0,
screamer 0:4a2e5f0422d6 750 self.TEST_RESULT_TIMEOUT : 0,
screamer 0:4a2e5f0422d6 751 self.TEST_RESULT_MBED_ASSERT : 0,
screamer 0:4a2e5f0422d6 752 self.TEST_RESULT_BUILD_FAILED : 0,
screamer 0:4a2e5f0422d6 753 self.TEST_RESULT_NOT_SUPPORTED : 0
screamer 0:4a2e5f0422d6 754 }
screamer 0:4a2e5f0422d6 755
screamer 0:4a2e5f0422d6 756 for test in test_summary:
screamer 0:4a2e5f0422d6 757 if test[0] in result_dict:
screamer 0:4a2e5f0422d6 758 result_dict[test[0]] += 1
screamer 0:4a2e5f0422d6 759 pt.add_row(test)
screamer 0:4a2e5f0422d6 760 result += pt.get_string()
screamer 0:4a2e5f0422d6 761 result += "\n"
screamer 0:4a2e5f0422d6 762
screamer 0:4a2e5f0422d6 763 # Print result count
screamer 0:4a2e5f0422d6 764 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
screamer 0:4a2e5f0422d6 765 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer 0:4a2e5f0422d6 766 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer 0:4a2e5f0422d6 767 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer 0:4a2e5f0422d6 768 return result
screamer 0:4a2e5f0422d6 769
screamer 0:4a2e5f0422d6 770 def test_loop_list_to_dict(self, test_loops_str):
screamer 0:4a2e5f0422d6 771 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer 0:4a2e5f0422d6 772 """
screamer 0:4a2e5f0422d6 773 result = {}
screamer 0:4a2e5f0422d6 774 if test_loops_str:
screamer 0:4a2e5f0422d6 775 test_loops = test_loops_str.split(',')
screamer 0:4a2e5f0422d6 776 for test_loop in test_loops:
screamer 0:4a2e5f0422d6 777 test_loop_count = test_loop.split('=')
screamer 0:4a2e5f0422d6 778 if len(test_loop_count) == 2:
screamer 0:4a2e5f0422d6 779 _test_id, _test_loops = test_loop_count
screamer 0:4a2e5f0422d6 780 try:
screamer 0:4a2e5f0422d6 781 _test_loops = int(_test_loops)
screamer 0:4a2e5f0422d6 782 except:
screamer 0:4a2e5f0422d6 783 continue
screamer 0:4a2e5f0422d6 784 result[_test_id] = _test_loops
screamer 0:4a2e5f0422d6 785 return result
screamer 0:4a2e5f0422d6 786
screamer 0:4a2e5f0422d6 787 def get_test_loop_count(self, test_id):
screamer 0:4a2e5f0422d6 788 """ This function returns no. of loops per test (deducted by test_id_.
screamer 0:4a2e5f0422d6 789 If test is not in list of redefined loop counts it will use default value.
screamer 0:4a2e5f0422d6 790 """
screamer 0:4a2e5f0422d6 791 result = self.GLOBAL_LOOPS_COUNT
screamer 0:4a2e5f0422d6 792 if test_id in self.TEST_LOOPS_DICT:
screamer 0:4a2e5f0422d6 793 result = self.TEST_LOOPS_DICT[test_id]
screamer 0:4a2e5f0422d6 794 return result
screamer 0:4a2e5f0422d6 795
screamer 0:4a2e5f0422d6 796 def delete_file(self, file_path):
screamer 0:4a2e5f0422d6 797 """ Remove file from the system
screamer 0:4a2e5f0422d6 798 """
screamer 0:4a2e5f0422d6 799 result = True
screamer 0:4a2e5f0422d6 800 resutl_msg = ""
screamer 0:4a2e5f0422d6 801 try:
screamer 0:4a2e5f0422d6 802 os.remove(file_path)
screamer 0:4a2e5f0422d6 803 except Exception, e:
screamer 0:4a2e5f0422d6 804 resutl_msg = e
screamer 0:4a2e5f0422d6 805 result = False
screamer 0:4a2e5f0422d6 806 return result, resutl_msg
screamer 0:4a2e5f0422d6 807
screamer 0:4a2e5f0422d6 808 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer 0:4a2e5f0422d6 809 """ Test is being invoked for given MUT.
screamer 0:4a2e5f0422d6 810 """
screamer 0:4a2e5f0422d6 811 # Get test information, image and test timeout
screamer 0:4a2e5f0422d6 812 test_id = data['test_id']
screamer 0:4a2e5f0422d6 813 test = TEST_MAP[test_id]
screamer 0:4a2e5f0422d6 814 test_description = TEST_MAP[test_id].get_description()
screamer 0:4a2e5f0422d6 815 image = data["image"]
screamer 0:4a2e5f0422d6 816 duration = data.get("duration", 10)
screamer 0:4a2e5f0422d6 817
screamer 0:4a2e5f0422d6 818 if mut is None:
screamer 0:4a2e5f0422d6 819 print "Error: No Mbed available: MUT[%s]" % data['mcu']
screamer 0:4a2e5f0422d6 820 return None
screamer 0:4a2e5f0422d6 821
screamer 0:4a2e5f0422d6 822 mcu = mut['mcu']
screamer 0:4a2e5f0422d6 823 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer 0:4a2e5f0422d6 824
screamer 0:4a2e5f0422d6 825 if self.db_logger:
screamer 0:4a2e5f0422d6 826 self.db_logger.reconnect()
screamer 0:4a2e5f0422d6 827
screamer 0:4a2e5f0422d6 828 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer 0:4a2e5f0422d6 829
screamer 0:4a2e5f0422d6 830 # Tests can be looped so test results must be stored for the same test
screamer 0:4a2e5f0422d6 831 test_all_result = []
screamer 0:4a2e5f0422d6 832 # Test results for one test ran few times
screamer 0:4a2e5f0422d6 833 detailed_test_results = {} # { Loop_number: { results ... } }
screamer 0:4a2e5f0422d6 834
screamer 0:4a2e5f0422d6 835 for test_index in range(test_loops):
screamer 0:4a2e5f0422d6 836
screamer 0:4a2e5f0422d6 837 # If mbedls is available and we are auto detecting MUT info,
screamer 0:4a2e5f0422d6 838 # update MUT info (mounting may changed)
screamer 0:4a2e5f0422d6 839 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:4a2e5f0422d6 840 platform_name_filter = [mcu]
screamer 0:4a2e5f0422d6 841 muts_list = {}
screamer 0:4a2e5f0422d6 842 found = False
screamer 0:4a2e5f0422d6 843
screamer 0:4a2e5f0422d6 844 for i in range(0, 60):
screamer 0:4a2e5f0422d6 845 print('Looking for %s with MBEDLS' % mcu)
screamer 0:4a2e5f0422d6 846 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer 0:4a2e5f0422d6 847
screamer 0:4a2e5f0422d6 848 if 1 not in muts_list:
screamer 0:4a2e5f0422d6 849 sleep(3)
screamer 0:4a2e5f0422d6 850 else:
screamer 0:4a2e5f0422d6 851 found = True
screamer 0:4a2e5f0422d6 852 break
screamer 0:4a2e5f0422d6 853
screamer 0:4a2e5f0422d6 854 if not found:
screamer 0:4a2e5f0422d6 855 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
screamer 0:4a2e5f0422d6 856 return None
screamer 0:4a2e5f0422d6 857 else:
screamer 0:4a2e5f0422d6 858 mut = muts_list[1]
screamer 0:4a2e5f0422d6 859
screamer 0:4a2e5f0422d6 860 disk = mut.get('disk')
screamer 0:4a2e5f0422d6 861 port = mut.get('port')
screamer 0:4a2e5f0422d6 862
screamer 0:4a2e5f0422d6 863 if disk is None or port is None:
screamer 0:4a2e5f0422d6 864 return None
screamer 0:4a2e5f0422d6 865
screamer 0:4a2e5f0422d6 866 target_by_mcu = TARGET_MAP[mut['mcu']]
screamer 0:4a2e5f0422d6 867 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer 0:4a2e5f0422d6 868 # Some extra stuff can be declared in MUTs structure
screamer 0:4a2e5f0422d6 869 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer 0:4a2e5f0422d6 870 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer 0:4a2e5f0422d6 871
screamer 0:4a2e5f0422d6 872 # When the build and test system were separate, this was relative to a
screamer 0:4a2e5f0422d6 873 # base network folder base path: join(NETWORK_BASE_PATH, )
screamer 0:4a2e5f0422d6 874 image_path = image
screamer 0:4a2e5f0422d6 875
screamer 0:4a2e5f0422d6 876 # Host test execution
screamer 0:4a2e5f0422d6 877 start_host_exec_time = time()
screamer 0:4a2e5f0422d6 878
screamer 0:4a2e5f0422d6 879 single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer 0:4a2e5f0422d6 880 _copy_method = selected_copy_method
screamer 0:4a2e5f0422d6 881
screamer 0:4a2e5f0422d6 882 if not exists(image_path):
screamer 0:4a2e5f0422d6 883 single_test_result = self.TEST_RESULT_NO_IMAGE
screamer 0:4a2e5f0422d6 884 elapsed_time = 0
screamer 0:4a2e5f0422d6 885 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
screamer 0:4a2e5f0422d6 886 print single_test_output
screamer 0:4a2e5f0422d6 887 else:
screamer 0:4a2e5f0422d6 888 # Host test execution
screamer 0:4a2e5f0422d6 889 start_host_exec_time = time()
screamer 0:4a2e5f0422d6 890
screamer 0:4a2e5f0422d6 891 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer 0:4a2e5f0422d6 892 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer 0:4a2e5f0422d6 893 host_test_result = self.run_host_test(test.host_test,
screamer 0:4a2e5f0422d6 894 image_path, disk, port, duration,
screamer 0:4a2e5f0422d6 895 micro=target_name,
screamer 0:4a2e5f0422d6 896 verbose=host_test_verbose,
screamer 0:4a2e5f0422d6 897 reset=host_test_reset,
screamer 0:4a2e5f0422d6 898 reset_tout=reset_tout,
screamer 0:4a2e5f0422d6 899 copy_method=selected_copy_method,
screamer 0:4a2e5f0422d6 900 program_cycle_s=target_by_mcu.program_cycle_s())
screamer 0:4a2e5f0422d6 901 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer 0:4a2e5f0422d6 902
screamer 0:4a2e5f0422d6 903 # Store test result
screamer 0:4a2e5f0422d6 904 test_all_result.append(single_test_result)
screamer 0:4a2e5f0422d6 905 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer 0:4a2e5f0422d6 906 elapsed_time = single_testduration # TIme of single test case execution after reset
screamer 0:4a2e5f0422d6 907
screamer 0:4a2e5f0422d6 908 detailed_test_results[test_index] = {
screamer 0:4a2e5f0422d6 909 'result' : single_test_result,
screamer 0:4a2e5f0422d6 910 'output' : single_test_output,
screamer 0:4a2e5f0422d6 911 'target_name' : target_name,
screamer 0:4a2e5f0422d6 912 'target_name_unique' : target_name_unique,
screamer 0:4a2e5f0422d6 913 'toolchain_name' : toolchain_name,
screamer 0:4a2e5f0422d6 914 'id' : test_id,
screamer 0:4a2e5f0422d6 915 'description' : test_description,
screamer 0:4a2e5f0422d6 916 'elapsed_time' : round(elapsed_time, 2),
screamer 0:4a2e5f0422d6 917 'duration' : single_timeout,
screamer 0:4a2e5f0422d6 918 'copy_method' : _copy_method,
screamer 0:4a2e5f0422d6 919 }
screamer 0:4a2e5f0422d6 920
screamer 0:4a2e5f0422d6 921 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
screamer 0:4a2e5f0422d6 922 test_id, test_description, elapsed_time, single_timeout)
screamer 0:4a2e5f0422d6 923
screamer 0:4a2e5f0422d6 924 # Update database entries for ongoing test
screamer 0:4a2e5f0422d6 925 if self.db_logger and self.db_logger.is_connected():
screamer 0:4a2e5f0422d6 926 test_type = 'SingleTest'
screamer 0:4a2e5f0422d6 927 self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer 0:4a2e5f0422d6 928 target_name,
screamer 0:4a2e5f0422d6 929 toolchain_name,
screamer 0:4a2e5f0422d6 930 test_type,
screamer 0:4a2e5f0422d6 931 test_id,
screamer 0:4a2e5f0422d6 932 single_test_result,
screamer 0:4a2e5f0422d6 933 single_test_output,
screamer 0:4a2e5f0422d6 934 elapsed_time,
screamer 0:4a2e5f0422d6 935 single_timeout,
screamer 0:4a2e5f0422d6 936 test_index)
screamer 0:4a2e5f0422d6 937
screamer 0:4a2e5f0422d6 938 # If we perform waterfall test we test until we get OK and we stop testing
screamer 0:4a2e5f0422d6 939 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer 0:4a2e5f0422d6 940 break
screamer 0:4a2e5f0422d6 941
screamer 0:4a2e5f0422d6 942 if self.db_logger:
screamer 0:4a2e5f0422d6 943 self.db_logger.disconnect()
screamer 0:4a2e5f0422d6 944
screamer 0:4a2e5f0422d6 945 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer 0:4a2e5f0422d6 946 target_name_unique,
screamer 0:4a2e5f0422d6 947 toolchain_name,
screamer 0:4a2e5f0422d6 948 test_id,
screamer 0:4a2e5f0422d6 949 test_description,
screamer 0:4a2e5f0422d6 950 round(elapsed_time, 2),
screamer 0:4a2e5f0422d6 951 single_timeout,
screamer 0:4a2e5f0422d6 952 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer 0:4a2e5f0422d6 953
screamer 0:4a2e5f0422d6 954 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer 0:4a2e5f0422d6 955 """ Function determines MUT's mbed disk/port and copies binary to
screamer 0:4a2e5f0422d6 956 target.
screamer 0:4a2e5f0422d6 957 """
screamer 0:4a2e5f0422d6 958 handle_results = []
screamer 0:4a2e5f0422d6 959 data = json.loads(test_spec)
screamer 0:4a2e5f0422d6 960
screamer 0:4a2e5f0422d6 961 # Find a suitable MUT:
screamer 0:4a2e5f0422d6 962 mut = None
screamer 0:4a2e5f0422d6 963 for id, m in self.muts.iteritems():
screamer 0:4a2e5f0422d6 964 if m['mcu'] == data['mcu']:
screamer 0:4a2e5f0422d6 965 mut = m
screamer 0:4a2e5f0422d6 966 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer 0:4a2e5f0422d6 967 handle_results.append(handle_result)
screamer 0:4a2e5f0422d6 968
screamer 0:4a2e5f0422d6 969 return handle_results
screamer 0:4a2e5f0422d6 970
screamer 0:4a2e5f0422d6 971 def print_test_result(self, test_result, target_name, toolchain_name,
screamer 0:4a2e5f0422d6 972 test_id, test_description, elapsed_time, duration):
screamer 0:4a2e5f0422d6 973 """ Use specific convention to print test result and related data
screamer 0:4a2e5f0422d6 974 """
screamer 0:4a2e5f0422d6 975 tokens = []
screamer 0:4a2e5f0422d6 976 tokens.append("TargetTest")
screamer 0:4a2e5f0422d6 977 tokens.append(target_name)
screamer 0:4a2e5f0422d6 978 tokens.append(toolchain_name)
screamer 0:4a2e5f0422d6 979 tokens.append(test_id)
screamer 0:4a2e5f0422d6 980 tokens.append(test_description)
screamer 0:4a2e5f0422d6 981 separator = "::"
screamer 0:4a2e5f0422d6 982 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer 0:4a2e5f0422d6 983 result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer 0:4a2e5f0422d6 984 return Fore.MAGENTA + result + Fore.RESET
screamer 0:4a2e5f0422d6 985
screamer 0:4a2e5f0422d6 986 def shape_test_loop_ok_result_count(self, test_all_result):
screamer 0:4a2e5f0422d6 987 """ Reformats list of results to simple string
screamer 0:4a2e5f0422d6 988 """
screamer 0:4a2e5f0422d6 989 test_loop_count = len(test_all_result)
screamer 0:4a2e5f0422d6 990 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer 0:4a2e5f0422d6 991 return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer 0:4a2e5f0422d6 992
screamer 0:4a2e5f0422d6 993 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer 0:4a2e5f0422d6 994 """ Reformats list of results to simple string
screamer 0:4a2e5f0422d6 995 """
screamer 0:4a2e5f0422d6 996 result = self.TEST_RESULT_FAIL
screamer 0:4a2e5f0422d6 997
screamer 0:4a2e5f0422d6 998 if all(test_all_result[0] == res for res in test_all_result):
screamer 0:4a2e5f0422d6 999 result = test_all_result[0]
screamer 0:4a2e5f0422d6 1000 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer 0:4a2e5f0422d6 1001 result = self.TEST_RESULT_OK
screamer 0:4a2e5f0422d6 1002
screamer 0:4a2e5f0422d6 1003 return result
screamer 0:4a2e5f0422d6 1004
screamer 0:4a2e5f0422d6 1005 def run_host_test(self, name, image_path, disk, port, duration,
screamer 0:4a2e5f0422d6 1006 micro=None, reset=None, reset_tout=None,
screamer 0:4a2e5f0422d6 1007 verbose=False, copy_method=None, program_cycle_s=None):
screamer 0:4a2e5f0422d6 1008 """ Function creates new process with host test configured with particular test case.
screamer 0:4a2e5f0422d6 1009 Function also is pooling for serial port activity from process to catch all data
screamer 0:4a2e5f0422d6 1010 printed by test runner and host test during test execution
screamer 0:4a2e5f0422d6 1011 """
screamer 0:4a2e5f0422d6 1012
screamer 0:4a2e5f0422d6 1013 def get_char_from_queue(obs):
screamer 0:4a2e5f0422d6 1014 """ Get character from queue safe way
screamer 0:4a2e5f0422d6 1015 """
screamer 0:4a2e5f0422d6 1016 try:
screamer 0:4a2e5f0422d6 1017 c = obs.queue.get(block=True, timeout=0.5)
screamer 0:4a2e5f0422d6 1018 except Empty, _:
screamer 0:4a2e5f0422d6 1019 c = None
screamer 0:4a2e5f0422d6 1020 return c
screamer 0:4a2e5f0422d6 1021
screamer 0:4a2e5f0422d6 1022 def filter_queue_char(c):
screamer 0:4a2e5f0422d6 1023 """ Filters out non ASCII characters from serial port
screamer 0:4a2e5f0422d6 1024 """
screamer 0:4a2e5f0422d6 1025 if ord(c) not in range(128):
screamer 0:4a2e5f0422d6 1026 c = ' '
screamer 0:4a2e5f0422d6 1027 return c
screamer 0:4a2e5f0422d6 1028
screamer 0:4a2e5f0422d6 1029 def get_test_result(output):
screamer 0:4a2e5f0422d6 1030 """ Parse test 'output' data
screamer 0:4a2e5f0422d6 1031 """
screamer 0:4a2e5f0422d6 1032 result = self.TEST_RESULT_TIMEOUT
screamer 0:4a2e5f0422d6 1033 for line in "".join(output).splitlines():
screamer 0:4a2e5f0422d6 1034 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer 0:4a2e5f0422d6 1035 if search_result and len(search_result.groups()):
screamer 0:4a2e5f0422d6 1036 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer 0:4a2e5f0422d6 1037 break
screamer 0:4a2e5f0422d6 1038 return result
screamer 0:4a2e5f0422d6 1039
screamer 0:4a2e5f0422d6 1040 def get_auto_property_value(property_name, line):
screamer 0:4a2e5f0422d6 1041 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer 0:4a2e5f0422d6 1042 Returns string
screamer 0:4a2e5f0422d6 1043 """
screamer 0:4a2e5f0422d6 1044 result = None
screamer 0:4a2e5f0422d6 1045 if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer 0:4a2e5f0422d6 1046 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer 0:4a2e5f0422d6 1047 if property is not None and len(property.groups()) == 1:
screamer 0:4a2e5f0422d6 1048 result = property.groups()[0]
screamer 0:4a2e5f0422d6 1049 return result
screamer 0:4a2e5f0422d6 1050
screamer 0:4a2e5f0422d6 1051 # print "{%s} port:%s disk:%s" % (name, port, disk),
screamer 0:4a2e5f0422d6 1052 cmd = ["python",
screamer 0:4a2e5f0422d6 1053 '%s.py'% name,
screamer 0:4a2e5f0422d6 1054 '-d', disk,
screamer 0:4a2e5f0422d6 1055 '-f', '"%s"'% image_path,
screamer 0:4a2e5f0422d6 1056 '-p', port,
screamer 0:4a2e5f0422d6 1057 '-t', str(duration),
screamer 0:4a2e5f0422d6 1058 '-C', str(program_cycle_s)]
screamer 0:4a2e5f0422d6 1059
screamer 0:4a2e5f0422d6 1060 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer 0:4a2e5f0422d6 1061 cmd += ['--auto']
screamer 0:4a2e5f0422d6 1062
screamer 0:4a2e5f0422d6 1063 # Add extra parameters to host_test
screamer 0:4a2e5f0422d6 1064 if copy_method is not None:
screamer 0:4a2e5f0422d6 1065 cmd += ["-c", copy_method]
screamer 0:4a2e5f0422d6 1066 if micro is not None:
screamer 0:4a2e5f0422d6 1067 cmd += ["-m", micro]
screamer 0:4a2e5f0422d6 1068 if reset is not None:
screamer 0:4a2e5f0422d6 1069 cmd += ["-r", reset]
screamer 0:4a2e5f0422d6 1070 if reset_tout is not None:
screamer 0:4a2e5f0422d6 1071 cmd += ["-R", str(reset_tout)]
screamer 0:4a2e5f0422d6 1072
screamer 0:4a2e5f0422d6 1073 if verbose:
screamer 0:4a2e5f0422d6 1074 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
screamer 0:4a2e5f0422d6 1075 print "Test::Output::Start"
screamer 0:4a2e5f0422d6 1076
screamer 0:4a2e5f0422d6 1077 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer 0:4a2e5f0422d6 1078 obs = ProcessObserver(proc)
screamer 0:4a2e5f0422d6 1079 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer 0:4a2e5f0422d6 1080 line = ''
screamer 0:4a2e5f0422d6 1081 output = []
screamer 0:4a2e5f0422d6 1082 start_time = time()
screamer 0:4a2e5f0422d6 1083 while (time() - start_time) < (2 * duration):
screamer 0:4a2e5f0422d6 1084 c = get_char_from_queue(obs)
screamer 0:4a2e5f0422d6 1085 if c:
screamer 0:4a2e5f0422d6 1086 if verbose:
screamer 0:4a2e5f0422d6 1087 sys.stdout.write(c)
screamer 0:4a2e5f0422d6 1088 c = filter_queue_char(c)
screamer 0:4a2e5f0422d6 1089 output.append(c)
screamer 0:4a2e5f0422d6 1090 # Give the mbed under test a way to communicate the end of the test
screamer 0:4a2e5f0422d6 1091 if c in ['\n', '\r']:
screamer 0:4a2e5f0422d6 1092
screamer 0:4a2e5f0422d6 1093 # Checking for auto-detection information from the test about MUT reset moment
screamer 0:4a2e5f0422d6 1094 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer 0:4a2e5f0422d6 1095 # We will update this marker only once to prevent multiple time resets
screamer 0:4a2e5f0422d6 1096 update_once_flag['reset_target'] = True
screamer 0:4a2e5f0422d6 1097 start_time = time()
screamer 0:4a2e5f0422d6 1098
screamer 0:4a2e5f0422d6 1099 # Checking for auto-detection information from the test about timeout
screamer 0:4a2e5f0422d6 1100 auto_timeout_val = get_auto_property_value('timeout', line)
screamer 0:4a2e5f0422d6 1101 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer 0:4a2e5f0422d6 1102 # We will update this marker only once to prevent multiple time resets
screamer 0:4a2e5f0422d6 1103 update_once_flag['timeout'] = True
screamer 0:4a2e5f0422d6 1104 duration = int(auto_timeout_val)
screamer 0:4a2e5f0422d6 1105
screamer 0:4a2e5f0422d6 1106 # Detect mbed assert:
screamer 0:4a2e5f0422d6 1107 if 'mbed assertation failed: ' in line:
screamer 0:4a2e5f0422d6 1108 output.append('{{mbed_assert}}')
screamer 0:4a2e5f0422d6 1109 break
screamer 0:4a2e5f0422d6 1110
screamer 0:4a2e5f0422d6 1111 # Check for test end
screamer 0:4a2e5f0422d6 1112 if '{end}' in line:
screamer 0:4a2e5f0422d6 1113 break
screamer 0:4a2e5f0422d6 1114 line = ''
screamer 0:4a2e5f0422d6 1115 else:
screamer 0:4a2e5f0422d6 1116 line += c
screamer 0:4a2e5f0422d6 1117 end_time = time()
screamer 0:4a2e5f0422d6 1118 testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer 0:4a2e5f0422d6 1119
screamer 0:4a2e5f0422d6 1120 c = get_char_from_queue(obs)
screamer 0:4a2e5f0422d6 1121
screamer 0:4a2e5f0422d6 1122 if c:
screamer 0:4a2e5f0422d6 1123 if verbose:
screamer 0:4a2e5f0422d6 1124 sys.stdout.write(c)
screamer 0:4a2e5f0422d6 1125 c = filter_queue_char(c)
screamer 0:4a2e5f0422d6 1126 output.append(c)
screamer 0:4a2e5f0422d6 1127
screamer 0:4a2e5f0422d6 1128 if verbose:
screamer 0:4a2e5f0422d6 1129 print "Test::Output::Finish"
screamer 0:4a2e5f0422d6 1130 # Stop test process
screamer 0:4a2e5f0422d6 1131 obs.stop()
screamer 0:4a2e5f0422d6 1132
screamer 0:4a2e5f0422d6 1133 result = get_test_result(output)
screamer 0:4a2e5f0422d6 1134 return (result, "".join(output), testcase_duration, duration)
screamer 0:4a2e5f0422d6 1135
screamer 0:4a2e5f0422d6 1136 def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer 0:4a2e5f0422d6 1137 """ Checks if specified target should run specific peripheral test case defined in MUTs file
screamer 0:4a2e5f0422d6 1138 """
screamer 0:4a2e5f0422d6 1139 if peripherals is not None:
screamer 0:4a2e5f0422d6 1140 peripherals = set(peripherals)
screamer 0:4a2e5f0422d6 1141 for id, mut in self.muts.iteritems():
screamer 0:4a2e5f0422d6 1142 # Target MCU name check
screamer 0:4a2e5f0422d6 1143 if mut["mcu"] != target_mcu_name:
screamer 0:4a2e5f0422d6 1144 continue
screamer 0:4a2e5f0422d6 1145 # Peripherals check
screamer 0:4a2e5f0422d6 1146 if peripherals is not None:
screamer 0:4a2e5f0422d6 1147 if 'peripherals' not in mut:
screamer 0:4a2e5f0422d6 1148 continue
screamer 0:4a2e5f0422d6 1149 if not peripherals.issubset(set(mut['peripherals'])):
screamer 0:4a2e5f0422d6 1150 continue
screamer 0:4a2e5f0422d6 1151 return True
screamer 0:4a2e5f0422d6 1152 return False
screamer 0:4a2e5f0422d6 1153
screamer 0:4a2e5f0422d6 1154 def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer 0:4a2e5f0422d6 1155 """ Function prepares JSON structure describing test specification
screamer 0:4a2e5f0422d6 1156 """
screamer 0:4a2e5f0422d6 1157 test_spec = {
screamer 0:4a2e5f0422d6 1158 "mcu": mcu,
screamer 0:4a2e5f0422d6 1159 "image": image_path,
screamer 0:4a2e5f0422d6 1160 "duration": duration,
screamer 0:4a2e5f0422d6 1161 "test_id": test_id,
screamer 0:4a2e5f0422d6 1162 }
screamer 0:4a2e5f0422d6 1163 return json.dumps(test_spec)
screamer 0:4a2e5f0422d6 1164
screamer 0:4a2e5f0422d6 1165
screamer 0:4a2e5f0422d6 1166 def get_unique_value_from_summary(test_summary, index):
screamer 0:4a2e5f0422d6 1167 """ Gets list of unique target names
screamer 0:4a2e5f0422d6 1168 """
screamer 0:4a2e5f0422d6 1169 result = []
screamer 0:4a2e5f0422d6 1170 for test in test_summary:
screamer 0:4a2e5f0422d6 1171 target_name = test[index]
screamer 0:4a2e5f0422d6 1172 if target_name not in result:
screamer 0:4a2e5f0422d6 1173 result.append(target_name)
screamer 0:4a2e5f0422d6 1174 return sorted(result)
screamer 0:4a2e5f0422d6 1175
screamer 0:4a2e5f0422d6 1176
screamer 0:4a2e5f0422d6 1177 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer 0:4a2e5f0422d6 1178 """ Gets list of unique target names and return dictionary
screamer 0:4a2e5f0422d6 1179 """
screamer 0:4a2e5f0422d6 1180 result = {}
screamer 0:4a2e5f0422d6 1181 for test in test_summary:
screamer 0:4a2e5f0422d6 1182 key = test[index_key]
screamer 0:4a2e5f0422d6 1183 val = test[index_val]
screamer 0:4a2e5f0422d6 1184 if key not in result:
screamer 0:4a2e5f0422d6 1185 result[key] = val
screamer 0:4a2e5f0422d6 1186 return result
screamer 0:4a2e5f0422d6 1187
screamer 0:4a2e5f0422d6 1188
screamer 0:4a2e5f0422d6 1189 def show_json_file_format_error(json_spec_filename, line, column):
screamer 0:4a2e5f0422d6 1190 """ Prints JSON broken content
screamer 0:4a2e5f0422d6 1191 """
screamer 0:4a2e5f0422d6 1192 with open(json_spec_filename) as data_file:
screamer 0:4a2e5f0422d6 1193 line_no = 1
screamer 0:4a2e5f0422d6 1194 for json_line in data_file:
screamer 0:4a2e5f0422d6 1195 if line_no + 5 >= line: # Print last few lines before error
screamer 0:4a2e5f0422d6 1196 print 'Line %d:\t'%line_no + json_line, # Prints line
screamer 0:4a2e5f0422d6 1197 if line_no == line:
screamer 0:4a2e5f0422d6 1198 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
screamer 0:4a2e5f0422d6 1199 break
screamer 0:4a2e5f0422d6 1200 line_no += 1
screamer 0:4a2e5f0422d6 1201
screamer 0:4a2e5f0422d6 1202
screamer 0:4a2e5f0422d6 1203 def json_format_error_defect_pos(json_error_msg):
screamer 0:4a2e5f0422d6 1204 """ Gets first error line and column in JSON file format.
screamer 0:4a2e5f0422d6 1205 Parsed from exception thrown by json.loads() string
screamer 0:4a2e5f0422d6 1206 """
screamer 0:4a2e5f0422d6 1207 result = None
screamer 0:4a2e5f0422d6 1208 line, column = 0, 0
screamer 0:4a2e5f0422d6 1209 # Line value search
screamer 0:4a2e5f0422d6 1210 line_search = re.search('line [0-9]+', json_error_msg)
screamer 0:4a2e5f0422d6 1211 if line_search is not None:
screamer 0:4a2e5f0422d6 1212 ls = line_search.group().split(' ')
screamer 0:4a2e5f0422d6 1213 if len(ls) == 2:
screamer 0:4a2e5f0422d6 1214 line = int(ls[1])
screamer 0:4a2e5f0422d6 1215 # Column position search
screamer 0:4a2e5f0422d6 1216 column_search = re.search('column [0-9]+', json_error_msg)
screamer 0:4a2e5f0422d6 1217 if column_search is not None:
screamer 0:4a2e5f0422d6 1218 cs = column_search.group().split(' ')
screamer 0:4a2e5f0422d6 1219 if len(cs) == 2:
screamer 0:4a2e5f0422d6 1220 column = int(cs[1])
screamer 0:4a2e5f0422d6 1221 result = [line, column]
screamer 0:4a2e5f0422d6 1222 return result
screamer 0:4a2e5f0422d6 1223
screamer 0:4a2e5f0422d6 1224
screamer 0:4a2e5f0422d6 1225 def get_json_data_from_file(json_spec_filename, verbose=False):
screamer 0:4a2e5f0422d6 1226 """ Loads from file JSON formatted string to data structure
screamer 0:4a2e5f0422d6 1227 """
screamer 0:4a2e5f0422d6 1228 result = None
screamer 0:4a2e5f0422d6 1229 try:
screamer 0:4a2e5f0422d6 1230 with open(json_spec_filename) as data_file:
screamer 0:4a2e5f0422d6 1231 try:
screamer 0:4a2e5f0422d6 1232 result = json.load(data_file)
screamer 0:4a2e5f0422d6 1233 except ValueError as json_error_msg:
screamer 0:4a2e5f0422d6 1234 result = None
screamer 0:4a2e5f0422d6 1235 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
screamer 0:4a2e5f0422d6 1236 # We can print where error occurred inside JSON file if we can parse exception msg
screamer 0:4a2e5f0422d6 1237 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer 0:4a2e5f0422d6 1238 if json_format_defect_pos is not None:
screamer 0:4a2e5f0422d6 1239 line = json_format_defect_pos[0]
screamer 0:4a2e5f0422d6 1240 column = json_format_defect_pos[1]
screamer 0:4a2e5f0422d6 1241 print
screamer 0:4a2e5f0422d6 1242 show_json_file_format_error(json_spec_filename, line, column)
screamer 0:4a2e5f0422d6 1243
screamer 0:4a2e5f0422d6 1244 except IOError as fileopen_error_msg:
screamer 0:4a2e5f0422d6 1245 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
screamer 0:4a2e5f0422d6 1246 print
screamer 0:4a2e5f0422d6 1247 if verbose and result:
screamer 0:4a2e5f0422d6 1248 pp = pprint.PrettyPrinter(indent=4)
screamer 0:4a2e5f0422d6 1249 pp.pprint(result)
screamer 0:4a2e5f0422d6 1250 return result
screamer 0:4a2e5f0422d6 1251
screamer 0:4a2e5f0422d6 1252
screamer 0:4a2e5f0422d6 1253 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer 0:4a2e5f0422d6 1254 """ Prints MUTs configuration passed to test script for verboseness
screamer 0:4a2e5f0422d6 1255 """
screamer 0:4a2e5f0422d6 1256 muts_info_cols = []
screamer 0:4a2e5f0422d6 1257 # We need to check all unique properties for each defined MUT
screamer 0:4a2e5f0422d6 1258 for k in json_data:
screamer 0:4a2e5f0422d6 1259 mut_info = json_data[k]
screamer 0:4a2e5f0422d6 1260 for mut_property in mut_info:
screamer 0:4a2e5f0422d6 1261 if mut_property not in muts_info_cols:
screamer 0:4a2e5f0422d6 1262 muts_info_cols.append(mut_property)
screamer 0:4a2e5f0422d6 1263
screamer 0:4a2e5f0422d6 1264 # Prepare pretty table object to display all MUTs
screamer 0:4a2e5f0422d6 1265 pt_cols = ["index"] + muts_info_cols
screamer 0:4a2e5f0422d6 1266 pt = PrettyTable(pt_cols)
screamer 0:4a2e5f0422d6 1267 for col in pt_cols:
screamer 0:4a2e5f0422d6 1268 pt.align[col] = "l"
screamer 0:4a2e5f0422d6 1269
screamer 0:4a2e5f0422d6 1270 # Add rows to pretty print object
screamer 0:4a2e5f0422d6 1271 for k in json_data:
screamer 0:4a2e5f0422d6 1272 row = [k]
screamer 0:4a2e5f0422d6 1273 mut_info = json_data[k]
screamer 0:4a2e5f0422d6 1274
screamer 0:4a2e5f0422d6 1275 add_row = True
screamer 0:4a2e5f0422d6 1276 if platform_filter and 'mcu' in mut_info:
screamer 0:4a2e5f0422d6 1277 add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer 0:4a2e5f0422d6 1278 if add_row:
screamer 0:4a2e5f0422d6 1279 for col in muts_info_cols:
screamer 0:4a2e5f0422d6 1280 cell_val = mut_info[col] if col in mut_info else None
screamer 0:4a2e5f0422d6 1281 if type(cell_val) == ListType:
screamer 0:4a2e5f0422d6 1282 cell_val = join_delim.join(cell_val)
screamer 0:4a2e5f0422d6 1283 row.append(cell_val)
screamer 0:4a2e5f0422d6 1284 pt.add_row(row)
screamer 0:4a2e5f0422d6 1285 return pt.get_string()
screamer 0:4a2e5f0422d6 1286
screamer 0:4a2e5f0422d6 1287
screamer 0:4a2e5f0422d6 1288 def print_test_configuration_from_json(json_data, join_delim=", "):
screamer 0:4a2e5f0422d6 1289 """ Prints test specification configuration passed to test script for verboseness
screamer 0:4a2e5f0422d6 1290 """
screamer 0:4a2e5f0422d6 1291 toolchains_info_cols = []
screamer 0:4a2e5f0422d6 1292 # We need to check all toolchains for each device
screamer 0:4a2e5f0422d6 1293 for k in json_data:
screamer 0:4a2e5f0422d6 1294 # k should be 'targets'
screamer 0:4a2e5f0422d6 1295 targets = json_data[k]
screamer 0:4a2e5f0422d6 1296 for target in targets:
screamer 0:4a2e5f0422d6 1297 toolchains = targets[target]
screamer 0:4a2e5f0422d6 1298 for toolchain in toolchains:
screamer 0:4a2e5f0422d6 1299 if toolchain not in toolchains_info_cols:
screamer 0:4a2e5f0422d6 1300 toolchains_info_cols.append(toolchain)
screamer 0:4a2e5f0422d6 1301
screamer 0:4a2e5f0422d6 1302 # Prepare pretty table object to display test specification
screamer 0:4a2e5f0422d6 1303 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
screamer 0:4a2e5f0422d6 1304 pt = PrettyTable(pt_cols)
screamer 0:4a2e5f0422d6 1305 for col in pt_cols:
screamer 0:4a2e5f0422d6 1306 pt.align[col] = "l"
screamer 0:4a2e5f0422d6 1307
screamer 0:4a2e5f0422d6 1308 # { target : [conflicted toolchains] }
screamer 0:4a2e5f0422d6 1309 toolchain_conflicts = {}
screamer 0:4a2e5f0422d6 1310 toolchain_path_conflicts = []
screamer 0:4a2e5f0422d6 1311 for k in json_data:
screamer 0:4a2e5f0422d6 1312 # k should be 'targets'
screamer 0:4a2e5f0422d6 1313 targets = json_data[k]
screamer 0:4a2e5f0422d6 1314 for target in targets:
screamer 0:4a2e5f0422d6 1315 target_supported_toolchains = get_target_supported_toolchains(target)
screamer 0:4a2e5f0422d6 1316 if not target_supported_toolchains:
screamer 0:4a2e5f0422d6 1317 target_supported_toolchains = []
screamer 0:4a2e5f0422d6 1318 target_name = target if target in TARGET_MAP else "%s*"% target
screamer 0:4a2e5f0422d6 1319 row = [target_name]
screamer 0:4a2e5f0422d6 1320 toolchains = targets[target]
screamer 0:4a2e5f0422d6 1321
screamer 0:4a2e5f0422d6 1322 for toolchain in sorted(toolchains_info_cols):
screamer 0:4a2e5f0422d6 1323 # Check for conflicts: target vs toolchain
screamer 0:4a2e5f0422d6 1324 conflict = False
screamer 0:4a2e5f0422d6 1325 conflict_path = False
screamer 0:4a2e5f0422d6 1326 if toolchain in toolchains:
screamer 0:4a2e5f0422d6 1327 if toolchain not in target_supported_toolchains:
screamer 0:4a2e5f0422d6 1328 conflict = True
screamer 0:4a2e5f0422d6 1329 if target not in toolchain_conflicts:
screamer 0:4a2e5f0422d6 1330 toolchain_conflicts[target] = []
screamer 0:4a2e5f0422d6 1331 toolchain_conflicts[target].append(toolchain)
screamer 0:4a2e5f0422d6 1332 # Add marker inside table about target usage / conflict
screamer 0:4a2e5f0422d6 1333 cell_val = 'Yes' if toolchain in toolchains else '-'
screamer 0:4a2e5f0422d6 1334 if conflict:
screamer 0:4a2e5f0422d6 1335 cell_val += '*'
screamer 0:4a2e5f0422d6 1336 # Check for conflicts: toolchain vs toolchain path
screamer 0:4a2e5f0422d6 1337 if toolchain in TOOLCHAIN_BIN_PATH:
screamer 0:4a2e5f0422d6 1338 toolchain_path = TOOLCHAIN_BIN_PATH[toolchain]
screamer 0:4a2e5f0422d6 1339 if not os.path.isdir(toolchain_path):
screamer 0:4a2e5f0422d6 1340 conflict_path = True
screamer 0:4a2e5f0422d6 1341 if toolchain not in toolchain_path_conflicts:
screamer 0:4a2e5f0422d6 1342 toolchain_path_conflicts.append(toolchain)
screamer 0:4a2e5f0422d6 1343 if conflict_path:
screamer 0:4a2e5f0422d6 1344 cell_val += '#'
screamer 0:4a2e5f0422d6 1345 row.append(cell_val)
screamer 0:4a2e5f0422d6 1346 pt.add_row(row)
screamer 0:4a2e5f0422d6 1347
screamer 0:4a2e5f0422d6 1348 # generate result string
screamer 0:4a2e5f0422d6 1349 result = pt.get_string() # Test specification table
screamer 0:4a2e5f0422d6 1350 if toolchain_conflicts or toolchain_path_conflicts:
screamer 0:4a2e5f0422d6 1351 result += "\n"
screamer 0:4a2e5f0422d6 1352 result += "Toolchain conflicts:\n"
screamer 0:4a2e5f0422d6 1353 for target in toolchain_conflicts:
screamer 0:4a2e5f0422d6 1354 if target not in TARGET_MAP:
screamer 0:4a2e5f0422d6 1355 result += "\t* Target %s unknown\n"% (target)
screamer 0:4a2e5f0422d6 1356 conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer 0:4a2e5f0422d6 1357 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer 0:4a2e5f0422d6 1358 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer 0:4a2e5f0422d6 1359
screamer 0:4a2e5f0422d6 1360 for toolchain in toolchain_path_conflicts:
screamer 0:4a2e5f0422d6 1361 # Let's check toolchain configuration
screamer 0:4a2e5f0422d6 1362 if toolchain in TOOLCHAIN_BIN_PATH:
screamer 0:4a2e5f0422d6 1363 toolchain_path = TOOLCHAIN_BIN_PATH[toolchain]
screamer 0:4a2e5f0422d6 1364 if not os.path.isdir(toolchain_path):
screamer 0:4a2e5f0422d6 1365 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer 0:4a2e5f0422d6 1366 return result
screamer 0:4a2e5f0422d6 1367
screamer 0:4a2e5f0422d6 1368
screamer 0:4a2e5f0422d6 1369 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer 0:4a2e5f0422d6 1370 """ Generates table summary with all test cases and additional test cases
screamer 0:4a2e5f0422d6 1371 information using pretty print functionality. Allows test suite user to
screamer 0:4a2e5f0422d6 1372 see test cases
screamer 0:4a2e5f0422d6 1373 """
screamer 0:4a2e5f0422d6 1374 # get all unique test ID prefixes
screamer 0:4a2e5f0422d6 1375 unique_test_id = []
screamer 0:4a2e5f0422d6 1376 for test in TESTS:
screamer 0:4a2e5f0422d6 1377 split = test['id'].split('_')[:-1]
screamer 0:4a2e5f0422d6 1378 test_id_prefix = '_'.join(split)
screamer 0:4a2e5f0422d6 1379 if test_id_prefix not in unique_test_id:
screamer 0:4a2e5f0422d6 1380 unique_test_id.append(test_id_prefix)
screamer 0:4a2e5f0422d6 1381 unique_test_id.sort()
screamer 0:4a2e5f0422d6 1382 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer 0:4a2e5f0422d6 1383 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer 0:4a2e5f0422d6 1384
screamer 0:4a2e5f0422d6 1385 test_properties = ['id',
screamer 0:4a2e5f0422d6 1386 'automated',
screamer 0:4a2e5f0422d6 1387 'description',
screamer 0:4a2e5f0422d6 1388 'peripherals',
screamer 0:4a2e5f0422d6 1389 'host_test',
screamer 0:4a2e5f0422d6 1390 'duration'] if cols is None else cols
screamer 0:4a2e5f0422d6 1391
screamer 0:4a2e5f0422d6 1392 # All tests status table print
screamer 0:4a2e5f0422d6 1393 pt = PrettyTable(test_properties)
screamer 0:4a2e5f0422d6 1394 for col in test_properties:
screamer 0:4a2e5f0422d6 1395 pt.align[col] = "l"
screamer 0:4a2e5f0422d6 1396 pt.align['duration'] = "r"
screamer 0:4a2e5f0422d6 1397
screamer 0:4a2e5f0422d6 1398 counter_all = 0
screamer 0:4a2e5f0422d6 1399 counter_automated = 0
screamer 0:4a2e5f0422d6 1400 pt.padding_width = 1 # One space between column edges and contents (default)
screamer 0:4a2e5f0422d6 1401
screamer 0:4a2e5f0422d6 1402 for test_id in sorted(TEST_MAP.keys()):
screamer 0:4a2e5f0422d6 1403 if platform_filter is not None:
screamer 0:4a2e5f0422d6 1404 # FIlter out platforms using regex
screamer 0:4a2e5f0422d6 1405 if re.search(platform_filter, test_id) is None:
screamer 0:4a2e5f0422d6 1406 continue
screamer 0:4a2e5f0422d6 1407 row = []
screamer 0:4a2e5f0422d6 1408 test = TEST_MAP[test_id]
screamer 0:4a2e5f0422d6 1409 split = test_id.split('_')[:-1]
screamer 0:4a2e5f0422d6 1410 test_id_prefix = '_'.join(split)
screamer 0:4a2e5f0422d6 1411
screamer 0:4a2e5f0422d6 1412 for col in test_properties:
screamer 0:4a2e5f0422d6 1413 col_value = test[col]
screamer 0:4a2e5f0422d6 1414 if type(test[col]) == ListType:
screamer 0:4a2e5f0422d6 1415 col_value = join_delim.join(test[col])
screamer 0:4a2e5f0422d6 1416 elif test[col] == None:
screamer 0:4a2e5f0422d6 1417 col_value = "-"
screamer 0:4a2e5f0422d6 1418
screamer 0:4a2e5f0422d6 1419 row.append(col_value)
screamer 0:4a2e5f0422d6 1420 if test['automated'] == True:
screamer 0:4a2e5f0422d6 1421 counter_dict_test_id_types[test_id_prefix] += 1
screamer 0:4a2e5f0422d6 1422 counter_automated += 1
screamer 0:4a2e5f0422d6 1423 pt.add_row(row)
screamer 0:4a2e5f0422d6 1424 # Update counters
screamer 0:4a2e5f0422d6 1425 counter_all += 1
screamer 0:4a2e5f0422d6 1426 counter_dict_test_id_types_all[test_id_prefix] += 1
screamer 0:4a2e5f0422d6 1427 result = pt.get_string()
screamer 0:4a2e5f0422d6 1428 result += "\n\n"
screamer 0:4a2e5f0422d6 1429
screamer 0:4a2e5f0422d6 1430 if result_summary and not platform_filter:
screamer 0:4a2e5f0422d6 1431 # Automation result summary
screamer 0:4a2e5f0422d6 1432 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
screamer 0:4a2e5f0422d6 1433 pt = PrettyTable(test_id_cols)
screamer 0:4a2e5f0422d6 1434 pt.align['automated'] = "r"
screamer 0:4a2e5f0422d6 1435 pt.align['all'] = "r"
screamer 0:4a2e5f0422d6 1436 pt.align['percent [%]'] = "r"
screamer 0:4a2e5f0422d6 1437
screamer 0:4a2e5f0422d6 1438 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer 0:4a2e5f0422d6 1439 str_progress = progress_bar(percent_progress, 75)
screamer 0:4a2e5f0422d6 1440 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer 0:4a2e5f0422d6 1441 result += "Automation coverage:\n"
screamer 0:4a2e5f0422d6 1442 result += pt.get_string()
screamer 0:4a2e5f0422d6 1443 result += "\n\n"
screamer 0:4a2e5f0422d6 1444
screamer 0:4a2e5f0422d6 1445 # Test automation coverage table print
screamer 0:4a2e5f0422d6 1446 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
screamer 0:4a2e5f0422d6 1447 pt = PrettyTable(test_id_cols)
screamer 0:4a2e5f0422d6 1448 pt.align['id'] = "l"
screamer 0:4a2e5f0422d6 1449 pt.align['automated'] = "r"
screamer 0:4a2e5f0422d6 1450 pt.align['all'] = "r"
screamer 0:4a2e5f0422d6 1451 pt.align['percent [%]'] = "r"
screamer 0:4a2e5f0422d6 1452 for unique_id in unique_test_id:
screamer 0:4a2e5f0422d6 1453 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer 0:4a2e5f0422d6 1454 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer 0:4a2e5f0422d6 1455 str_progress = progress_bar(percent_progress, 75)
screamer 0:4a2e5f0422d6 1456 row = [unique_id,
screamer 0:4a2e5f0422d6 1457 counter_dict_test_id_types[unique_id],
screamer 0:4a2e5f0422d6 1458 counter_dict_test_id_types_all[unique_id],
screamer 0:4a2e5f0422d6 1459 percent_progress,
screamer 0:4a2e5f0422d6 1460 "[" + str_progress + "]"]
screamer 0:4a2e5f0422d6 1461 pt.add_row(row)
screamer 0:4a2e5f0422d6 1462 result += "Test automation coverage:\n"
screamer 0:4a2e5f0422d6 1463 result += pt.get_string()
screamer 0:4a2e5f0422d6 1464 result += "\n\n"
screamer 0:4a2e5f0422d6 1465 return result
screamer 0:4a2e5f0422d6 1466
screamer 0:4a2e5f0422d6 1467
screamer 0:4a2e5f0422d6 1468 def progress_bar(percent_progress, saturation=0):
screamer 0:4a2e5f0422d6 1469 """ This function creates progress bar with optional simple saturation mark
screamer 0:4a2e5f0422d6 1470 """
screamer 0:4a2e5f0422d6 1471 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer 0:4a2e5f0422d6 1472 str_progress = '#' * step + '.' * int(50 - step)
screamer 0:4a2e5f0422d6 1473 c = '!' if str_progress[38] == '.' else '|'
screamer 0:4a2e5f0422d6 1474 if saturation > 0:
screamer 0:4a2e5f0422d6 1475 saturation = saturation / 2
screamer 0:4a2e5f0422d6 1476 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer 0:4a2e5f0422d6 1477 return str_progress
screamer 0:4a2e5f0422d6 1478
screamer 0:4a2e5f0422d6 1479
screamer 0:4a2e5f0422d6 1480 def singletest_in_cli_mode(single_test):
screamer 0:4a2e5f0422d6 1481 """ Runs SingleTestRunner object in CLI (Command line interface) mode
screamer 0:4a2e5f0422d6 1482
screamer 0:4a2e5f0422d6 1483 @return returns success code (0 == success) for building and running tests
screamer 0:4a2e5f0422d6 1484 """
screamer 0:4a2e5f0422d6 1485 start = time()
screamer 0:4a2e5f0422d6 1486 # Execute tests depending on options and filter applied
screamer 0:4a2e5f0422d6 1487 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer 0:4a2e5f0422d6 1488 elapsed_time = time() - start
screamer 0:4a2e5f0422d6 1489
screamer 0:4a2e5f0422d6 1490 # Human readable summary
screamer 0:4a2e5f0422d6 1491 if not single_test.opts_suppress_summary:
screamer 0:4a2e5f0422d6 1492 # prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 1493 print single_test.generate_test_summary(test_summary, shuffle_seed)
screamer 0:4a2e5f0422d6 1494 if single_test.opts_test_x_toolchain_summary:
screamer 0:4a2e5f0422d6 1495 # prints well-formed summary with results (SQL table like)
screamer 0:4a2e5f0422d6 1496 # table shows text x toolchain test result matrix
screamer 0:4a2e5f0422d6 1497 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
screamer 0:4a2e5f0422d6 1498
screamer 0:4a2e5f0422d6 1499 print "Completed in %.2f sec"% (elapsed_time)
screamer 0:4a2e5f0422d6 1500 print
screamer 0:4a2e5f0422d6 1501 # Write summary of the builds
screamer 0:4a2e5f0422d6 1502
screamer 0:4a2e5f0422d6 1503 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 0:4a2e5f0422d6 1504 status = print_report_exporter.report(build_report)
screamer 0:4a2e5f0422d6 1505
screamer 0:4a2e5f0422d6 1506 # Store extra reports in files
screamer 0:4a2e5f0422d6 1507 if single_test.opts_report_html_file_name:
screamer 0:4a2e5f0422d6 1508 # Export results in form of HTML report to separate file
screamer 0:4a2e5f0422d6 1509 report_exporter = ReportExporter(ResultExporterType.HTML)
screamer 0:4a2e5f0422d6 1510 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:4a2e5f0422d6 1511 if single_test.opts_report_junit_file_name:
screamer 0:4a2e5f0422d6 1512 # Export results in form of JUnit XML report to separate file
screamer 0:4a2e5f0422d6 1513 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 0:4a2e5f0422d6 1514 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer 0:4a2e5f0422d6 1515 if single_test.opts_report_build_file_name:
screamer 0:4a2e5f0422d6 1516 # Export build results as html report to sparate file
screamer 0:4a2e5f0422d6 1517 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 0:4a2e5f0422d6 1518 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer 0:4a2e5f0422d6 1519
screamer 0:4a2e5f0422d6 1520 # Returns True if no build failures of the test projects or their dependencies
screamer 0:4a2e5f0422d6 1521 return status
screamer 0:4a2e5f0422d6 1522
screamer 0:4a2e5f0422d6 1523 class TestLogger():
screamer 0:4a2e5f0422d6 1524 """ Super-class for logging and printing ongoing events for test suite pass
screamer 0:4a2e5f0422d6 1525 """
screamer 0:4a2e5f0422d6 1526 def __init__(self, store_log=True):
screamer 0:4a2e5f0422d6 1527 """ We can control if logger actually stores log in memory
screamer 0:4a2e5f0422d6 1528 or just handled all log entries immediately
screamer 0:4a2e5f0422d6 1529 """
screamer 0:4a2e5f0422d6 1530 self.log = []
screamer 0:4a2e5f0422d6 1531 self.log_to_file = False
screamer 0:4a2e5f0422d6 1532 self.log_file_name = None
screamer 0:4a2e5f0422d6 1533 self.store_log = store_log
screamer 0:4a2e5f0422d6 1534
screamer 0:4a2e5f0422d6 1535 self.LogType = construct_enum(INFO='Info',
screamer 0:4a2e5f0422d6 1536 WARN='Warning',
screamer 0:4a2e5f0422d6 1537 NOTIF='Notification',
screamer 0:4a2e5f0422d6 1538 ERROR='Error',
screamer 0:4a2e5f0422d6 1539 EXCEPT='Exception')
screamer 0:4a2e5f0422d6 1540
screamer 0:4a2e5f0422d6 1541 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer 0:4a2e5f0422d6 1542 APPEND=2) # Append to existing log file
screamer 0:4a2e5f0422d6 1543
screamer 0:4a2e5f0422d6 1544 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:4a2e5f0422d6 1545 """ Log one line of text
screamer 0:4a2e5f0422d6 1546 """
screamer 0:4a2e5f0422d6 1547 log_timestamp = time()
screamer 0:4a2e5f0422d6 1548 log_entry = {'log_type' : LogType,
screamer 0:4a2e5f0422d6 1549 'log_timestamp' : log_timestamp,
screamer 0:4a2e5f0422d6 1550 'log_line' : log_line,
screamer 0:4a2e5f0422d6 1551 '_future' : None
screamer 0:4a2e5f0422d6 1552 }
screamer 0:4a2e5f0422d6 1553 # Store log in memory
screamer 0:4a2e5f0422d6 1554 if self.store_log:
screamer 0:4a2e5f0422d6 1555 self.log.append(log_entry)
screamer 0:4a2e5f0422d6 1556 return log_entry
screamer 0:4a2e5f0422d6 1557
screamer 0:4a2e5f0422d6 1558
screamer 0:4a2e5f0422d6 1559 class CLITestLogger(TestLogger):
screamer 0:4a2e5f0422d6 1560 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer 0:4a2e5f0422d6 1561 """
screamer 0:4a2e5f0422d6 1562 def __init__(self, store_log=True, file_name=None):
screamer 0:4a2e5f0422d6 1563 TestLogger.__init__(self)
screamer 0:4a2e5f0422d6 1564 self.log_file_name = file_name
screamer 0:4a2e5f0422d6 1565 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer 0:4a2e5f0422d6 1566 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer 0:4a2e5f0422d6 1567
screamer 0:4a2e5f0422d6 1568 def log_print(self, log_entry, timestamp=True):
screamer 0:4a2e5f0422d6 1569 """ Prints on screen formatted log entry
screamer 0:4a2e5f0422d6 1570 """
screamer 0:4a2e5f0422d6 1571 ts = log_entry['log_timestamp']
screamer 0:4a2e5f0422d6 1572 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer 0:4a2e5f0422d6 1573 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer 0:4a2e5f0422d6 1574 return timestamp_str + log_line_str
screamer 0:4a2e5f0422d6 1575
screamer 0:4a2e5f0422d6 1576 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer 0:4a2e5f0422d6 1577 """ Logs line, if log file output was specified log line will be appended
screamer 0:4a2e5f0422d6 1578 at the end of log file
screamer 0:4a2e5f0422d6 1579 """
screamer 0:4a2e5f0422d6 1580 log_entry = TestLogger.log_line(self, LogType, log_line)
screamer 0:4a2e5f0422d6 1581 log_line_str = self.log_print(log_entry, timestamp)
screamer 0:4a2e5f0422d6 1582 if self.log_file_name is not None:
screamer 0:4a2e5f0422d6 1583 try:
screamer 0:4a2e5f0422d6 1584 with open(self.log_file_name, 'a') as f:
screamer 0:4a2e5f0422d6 1585 f.write(log_line_str + line_delim)
screamer 0:4a2e5f0422d6 1586 except IOError:
screamer 0:4a2e5f0422d6 1587 pass
screamer 0:4a2e5f0422d6 1588 return log_line_str
screamer 0:4a2e5f0422d6 1589
screamer 0:4a2e5f0422d6 1590
screamer 0:4a2e5f0422d6 1591 def factory_db_logger(db_url):
screamer 0:4a2e5f0422d6 1592 """ Factory database driver depending on database type supplied in database connection string db_url
screamer 0:4a2e5f0422d6 1593 """
screamer 0:4a2e5f0422d6 1594 if db_url is not None:
screamer 2:5f044fef8f98 1595 from tools.test_mysql import MySQLDBAccess
screamer 0:4a2e5f0422d6 1596 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:4a2e5f0422d6 1597 if connection_info is not None:
screamer 0:4a2e5f0422d6 1598 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:4a2e5f0422d6 1599 if db_type == 'mysql':
screamer 0:4a2e5f0422d6 1600 return MySQLDBAccess()
screamer 0:4a2e5f0422d6 1601 return None
screamer 0:4a2e5f0422d6 1602
screamer 0:4a2e5f0422d6 1603
screamer 0:4a2e5f0422d6 1604 def detect_database_verbose(db_url):
screamer 0:4a2e5f0422d6 1605 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer 0:4a2e5f0422d6 1606 """
screamer 0:4a2e5f0422d6 1607 result = BaseDBAccess().parse_db_connection_string(db_url)
screamer 0:4a2e5f0422d6 1608 if result is not None:
screamer 0:4a2e5f0422d6 1609 # Parsing passed
screamer 0:4a2e5f0422d6 1610 (db_type, username, password, host, db_name) = result
screamer 0:4a2e5f0422d6 1611 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer 0:4a2e5f0422d6 1612 # Let's try to connect
screamer 0:4a2e5f0422d6 1613 db_ = factory_db_logger(db_url)
screamer 0:4a2e5f0422d6 1614 if db_ is not None:
screamer 0:4a2e5f0422d6 1615 print "Connecting to database '%s'..."% db_url,
screamer 0:4a2e5f0422d6 1616 db_.connect(host, username, password, db_name)
screamer 0:4a2e5f0422d6 1617 if db_.is_connected():
screamer 0:4a2e5f0422d6 1618 print "ok"
screamer 0:4a2e5f0422d6 1619 print "Detecting database..."
screamer 0:4a2e5f0422d6 1620 print db_.detect_database(verbose=True)
screamer 0:4a2e5f0422d6 1621 print "Disconnecting...",
screamer 0:4a2e5f0422d6 1622 db_.disconnect()
screamer 0:4a2e5f0422d6 1623 print "done"
screamer 0:4a2e5f0422d6 1624 else:
screamer 0:4a2e5f0422d6 1625 print "Database type '%s' unknown"% db_type
screamer 0:4a2e5f0422d6 1626 else:
screamer 0:4a2e5f0422d6 1627 print "Parse error: '%s' - DB Url error"% (db_url)
screamer 0:4a2e5f0422d6 1628
screamer 0:4a2e5f0422d6 1629
screamer 0:4a2e5f0422d6 1630 def get_module_avail(module_name):
screamer 0:4a2e5f0422d6 1631 """ This function returns True if module_name is already impored module
screamer 0:4a2e5f0422d6 1632 """
screamer 0:4a2e5f0422d6 1633 return module_name in sys.modules.keys()
screamer 0:4a2e5f0422d6 1634
screamer 0:4a2e5f0422d6 1635
screamer 0:4a2e5f0422d6 1636 def get_autodetected_MUTS_list(platform_name_filter=None):
screamer 0:4a2e5f0422d6 1637 oldError = None
screamer 0:4a2e5f0422d6 1638 if os.name == 'nt':
screamer 0:4a2e5f0422d6 1639 # Disable Windows error box temporarily
screamer 0:4a2e5f0422d6 1640 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer 0:4a2e5f0422d6 1641
screamer 0:4a2e5f0422d6 1642 mbeds = mbed_lstools.create()
screamer 0:4a2e5f0422d6 1643 detect_muts_list = mbeds.list_mbeds()
screamer 0:4a2e5f0422d6 1644
screamer 0:4a2e5f0422d6 1645 if os.name == 'nt':
screamer 0:4a2e5f0422d6 1646 ctypes.windll.kernel32.SetErrorMode(oldError)
screamer 0:4a2e5f0422d6 1647
screamer 0:4a2e5f0422d6 1648 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer 0:4a2e5f0422d6 1649
screamer 0:4a2e5f0422d6 1650 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer 0:4a2e5f0422d6 1651 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer 0:4a2e5f0422d6 1652 If function fails to auto-detect devices it will return empty dictionary.
screamer 0:4a2e5f0422d6 1653
screamer 0:4a2e5f0422d6 1654 if get_module_avail('mbed_lstools'):
screamer 0:4a2e5f0422d6 1655 mbeds = mbed_lstools.create()
screamer 0:4a2e5f0422d6 1656 mbeds_list = mbeds.list_mbeds()
screamer 0:4a2e5f0422d6 1657
screamer 0:4a2e5f0422d6 1658 @param mbeds_list list of mbeds captured from mbed_lstools
screamer 0:4a2e5f0422d6 1659 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer 0:4a2e5f0422d6 1660 """
screamer 0:4a2e5f0422d6 1661 result = {} # Should be in muts_all.json format
screamer 0:4a2e5f0422d6 1662 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer 0:4a2e5f0422d6 1663 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer 0:4a2e5f0422d6 1664 index = 1
screamer 0:4a2e5f0422d6 1665 for mut in mbeds_list:
screamer 0:4a2e5f0422d6 1666 # Filter the MUTS if a filter is specified
screamer 0:4a2e5f0422d6 1667
screamer 0:4a2e5f0422d6 1668 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer 0:4a2e5f0422d6 1669 continue
screamer 0:4a2e5f0422d6 1670
screamer 0:4a2e5f0422d6 1671 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer 0:4a2e5f0422d6 1672 # if not we are creating our own unique value (last few chars from platform's target_id).
screamer 0:4a2e5f0422d6 1673 m = {'mcu': mut['platform_name'],
screamer 0:4a2e5f0422d6 1674 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer 0:4a2e5f0422d6 1675 'port': mut['serial_port'],
screamer 0:4a2e5f0422d6 1676 'disk': mut['mount_point'],
screamer 0:4a2e5f0422d6 1677 'peripherals': [] # No peripheral detection
screamer 0:4a2e5f0422d6 1678 }
screamer 0:4a2e5f0422d6 1679 if index not in result:
screamer 0:4a2e5f0422d6 1680 result[index] = {}
screamer 0:4a2e5f0422d6 1681 result[index] = m
screamer 0:4a2e5f0422d6 1682 index += 1
screamer 0:4a2e5f0422d6 1683 return result
screamer 0:4a2e5f0422d6 1684
screamer 0:4a2e5f0422d6 1685
screamer 0:4a2e5f0422d6 1686 def get_autodetected_TEST_SPEC(mbeds_list,
screamer 0:4a2e5f0422d6 1687 use_default_toolchain=True,
screamer 0:4a2e5f0422d6 1688 use_supported_toolchains=False,
screamer 0:4a2e5f0422d6 1689 toolchain_filter=None,
screamer 0:4a2e5f0422d6 1690 platform_name_filter=None):
screamer 0:4a2e5f0422d6 1691 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer 0:4a2e5f0422d6 1692 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer 0:4a2e5f0422d6 1693
screamer 0:4a2e5f0422d6 1694 use_default_toolchain - if True add default toolchain to test_spec
screamer 0:4a2e5f0422d6 1695 use_supported_toolchains - if True add all supported toolchains to test_spec
screamer 0:4a2e5f0422d6 1696 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer 0:4a2e5f0422d6 1697 """
screamer 0:4a2e5f0422d6 1698 result = {'targets': {} }
screamer 0:4a2e5f0422d6 1699
screamer 0:4a2e5f0422d6 1700 for mut in mbeds_list:
screamer 0:4a2e5f0422d6 1701 mcu = mut['mcu']
screamer 0:4a2e5f0422d6 1702 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer 0:4a2e5f0422d6 1703 if mcu in TARGET_MAP:
screamer 0:4a2e5f0422d6 1704 default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer 0:4a2e5f0422d6 1705 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer 0:4a2e5f0422d6 1706
screamer 0:4a2e5f0422d6 1707 # Decide which toolchains should be added to test specification toolchain pool for each target
screamer 0:4a2e5f0422d6 1708 toolchains = []
screamer 0:4a2e5f0422d6 1709 if use_default_toolchain:
screamer 0:4a2e5f0422d6 1710 toolchains.append(default_toolchain)
screamer 0:4a2e5f0422d6 1711 if use_supported_toolchains:
screamer 0:4a2e5f0422d6 1712 toolchains += supported_toolchains
screamer 0:4a2e5f0422d6 1713 if toolchain_filter is not None:
screamer 0:4a2e5f0422d6 1714 all_toolchains = supported_toolchains + [default_toolchain]
screamer 0:4a2e5f0422d6 1715 for toolchain in toolchain_filter.split(','):
screamer 0:4a2e5f0422d6 1716 if toolchain in all_toolchains:
screamer 0:4a2e5f0422d6 1717 toolchains.append(toolchain)
screamer 0:4a2e5f0422d6 1718
screamer 0:4a2e5f0422d6 1719 result['targets'][mcu] = list(set(toolchains))
screamer 0:4a2e5f0422d6 1720 return result
screamer 0:4a2e5f0422d6 1721
screamer 0:4a2e5f0422d6 1722
screamer 0:4a2e5f0422d6 1723 def get_default_test_options_parser():
screamer 0:4a2e5f0422d6 1724 """ Get common test script options used by CLI, web services etc.
screamer 0:4a2e5f0422d6 1725 """
screamer 0:4a2e5f0422d6 1726 parser = optparse.OptionParser()
screamer 0:4a2e5f0422d6 1727 parser.add_option('-i', '--tests',
screamer 0:4a2e5f0422d6 1728 dest='test_spec_filename',
screamer 0:4a2e5f0422d6 1729 metavar="FILE",
screamer 0:4a2e5f0422d6 1730 help='Points to file with test specification')
screamer 0:4a2e5f0422d6 1731
screamer 0:4a2e5f0422d6 1732 parser.add_option('-M', '--MUTS',
screamer 0:4a2e5f0422d6 1733 dest='muts_spec_filename',
screamer 0:4a2e5f0422d6 1734 metavar="FILE",
screamer 0:4a2e5f0422d6 1735 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer 0:4a2e5f0422d6 1736
screamer 0:4a2e5f0422d6 1737 parser.add_option("-j", "--jobs",
screamer 0:4a2e5f0422d6 1738 dest='jobs',
screamer 0:4a2e5f0422d6 1739 metavar="NUMBER",
screamer 0:4a2e5f0422d6 1740 type="int",
screamer 0:4a2e5f0422d6 1741 help="Define number of compilation jobs. Default value is 1")
screamer 0:4a2e5f0422d6 1742
screamer 0:4a2e5f0422d6 1743 if get_module_avail('mbed_lstools'):
screamer 0:4a2e5f0422d6 1744 # Additional features available when mbed_lstools is installed on host and imported
screamer 0:4a2e5f0422d6 1745 # mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer 0:4a2e5f0422d6 1746 parser.add_option('', '--auto',
screamer 0:4a2e5f0422d6 1747 dest='auto_detect',
screamer 0:4a2e5f0422d6 1748 metavar=False,
screamer 0:4a2e5f0422d6 1749 action="store_true",
screamer 0:4a2e5f0422d6 1750 help='Use mbed-ls module to detect all connected mbed devices')
screamer 0:4a2e5f0422d6 1751
screamer 0:4a2e5f0422d6 1752 parser.add_option('', '--tc',
screamer 0:4a2e5f0422d6 1753 dest='toolchains_filter',
screamer 0:4a2e5f0422d6 1754 help="Toolchain filter for --auto option. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer 0:4a2e5f0422d6 1755
screamer 0:4a2e5f0422d6 1756 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer 0:4a2e5f0422d6 1757 parser.add_option('', '--oper',
screamer 0:4a2e5f0422d6 1758 dest='operability_checks',
screamer 0:4a2e5f0422d6 1759 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer 0:4a2e5f0422d6 1760
screamer 0:4a2e5f0422d6 1761 parser.add_option('', '--clean',
screamer 0:4a2e5f0422d6 1762 dest='clean',
screamer 0:4a2e5f0422d6 1763 metavar=False,
screamer 0:4a2e5f0422d6 1764 action="store_true",
screamer 0:4a2e5f0422d6 1765 help='Clean the build directory')
screamer 0:4a2e5f0422d6 1766
screamer 0:4a2e5f0422d6 1767 parser.add_option('-P', '--only-peripherals',
screamer 0:4a2e5f0422d6 1768 dest='test_only_peripheral',
screamer 0:4a2e5f0422d6 1769 default=False,
screamer 0:4a2e5f0422d6 1770 action="store_true",
screamer 0:4a2e5f0422d6 1771 help='Test only peripheral declared for MUT and skip common tests')
screamer 0:4a2e5f0422d6 1772
screamer 0:4a2e5f0422d6 1773 parser.add_option('-C', '--only-commons',
screamer 0:4a2e5f0422d6 1774 dest='test_only_common',
screamer 0:4a2e5f0422d6 1775 default=False,
screamer 0:4a2e5f0422d6 1776 action="store_true",
screamer 0:4a2e5f0422d6 1777 help='Test only board internals. Skip perpherials tests and perform common tests')
screamer 0:4a2e5f0422d6 1778
screamer 0:4a2e5f0422d6 1779 parser.add_option('-n', '--test-by-names',
screamer 0:4a2e5f0422d6 1780 dest='test_by_names',
screamer 0:4a2e5f0422d6 1781 help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer 0:4a2e5f0422d6 1782
screamer 0:4a2e5f0422d6 1783 parser.add_option('-p', '--peripheral-by-names',
screamer 0:4a2e5f0422d6 1784 dest='peripheral_by_names',
screamer 0:4a2e5f0422d6 1785 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer 0:4a2e5f0422d6 1786
screamer 0:4a2e5f0422d6 1787 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer 0:4a2e5f0422d6 1788 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer 0:4a2e5f0422d6 1789
screamer 0:4a2e5f0422d6 1790 parser.add_option('-c', '--copy-method',
screamer 0:4a2e5f0422d6 1791 dest='copy_method',
screamer 0:4a2e5f0422d6 1792 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer 0:4a2e5f0422d6 1793
screamer 0:4a2e5f0422d6 1794 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer 0:4a2e5f0422d6 1795 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer 0:4a2e5f0422d6 1796
screamer 0:4a2e5f0422d6 1797 parser.add_option('-r', '--reset-type',
screamer 0:4a2e5f0422d6 1798 dest='mut_reset_type',
screamer 0:4a2e5f0422d6 1799 default=None,
screamer 0:4a2e5f0422d6 1800 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer 0:4a2e5f0422d6 1801
screamer 0:4a2e5f0422d6 1802 parser.add_option('-g', '--goanna-for-tests',
screamer 0:4a2e5f0422d6 1803 dest='goanna_for_tests',
screamer 0:4a2e5f0422d6 1804 metavar=False,
screamer 0:4a2e5f0422d6 1805 action="store_true",
screamer 0:4a2e5f0422d6 1806 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer 0:4a2e5f0422d6 1807
screamer 0:4a2e5f0422d6 1808 parser.add_option('-G', '--goanna-for-sdk',
screamer 0:4a2e5f0422d6 1809 dest='goanna_for_mbed_sdk',
screamer 0:4a2e5f0422d6 1810 metavar=False,
screamer 0:4a2e5f0422d6 1811 action="store_true",
screamer 0:4a2e5f0422d6 1812 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer 0:4a2e5f0422d6 1813
screamer 0:4a2e5f0422d6 1814 parser.add_option('-s', '--suppress-summary',
screamer 0:4a2e5f0422d6 1815 dest='suppress_summary',
screamer 0:4a2e5f0422d6 1816 default=False,
screamer 0:4a2e5f0422d6 1817 action="store_true",
screamer 0:4a2e5f0422d6 1818 help='Suppresses display of wellformatted table with test results')
screamer 0:4a2e5f0422d6 1819
screamer 0:4a2e5f0422d6 1820 parser.add_option('-t', '--test-summary',
screamer 0:4a2e5f0422d6 1821 dest='test_x_toolchain_summary',
screamer 0:4a2e5f0422d6 1822 default=False,
screamer 0:4a2e5f0422d6 1823 action="store_true",
screamer 0:4a2e5f0422d6 1824 help='Displays wellformatted table with test x toolchain test result per target')
screamer 0:4a2e5f0422d6 1825
screamer 0:4a2e5f0422d6 1826 parser.add_option('-A', '--test-automation-report',
screamer 0:4a2e5f0422d6 1827 dest='test_automation_report',
screamer 0:4a2e5f0422d6 1828 default=False,
screamer 0:4a2e5f0422d6 1829 action="store_true",
screamer 0:4a2e5f0422d6 1830 help='Prints information about all tests and exits')
screamer 0:4a2e5f0422d6 1831
screamer 0:4a2e5f0422d6 1832 parser.add_option('-R', '--test-case-report',
screamer 0:4a2e5f0422d6 1833 dest='test_case_report',
screamer 0:4a2e5f0422d6 1834 default=False,
screamer 0:4a2e5f0422d6 1835 action="store_true",
screamer 0:4a2e5f0422d6 1836 help='Prints information about all test cases and exits')
screamer 0:4a2e5f0422d6 1837
screamer 0:4a2e5f0422d6 1838 parser.add_option("-S", "--supported-toolchains",
screamer 0:4a2e5f0422d6 1839 action="store_true",
screamer 0:4a2e5f0422d6 1840 dest="supported_toolchains",
screamer 0:4a2e5f0422d6 1841 default=False,
screamer 0:4a2e5f0422d6 1842 help="Displays supported matrix of MCUs and toolchains")
screamer 0:4a2e5f0422d6 1843
screamer 0:4a2e5f0422d6 1844 parser.add_option("-O", "--only-build",
screamer 0:4a2e5f0422d6 1845 action="store_true",
screamer 0:4a2e5f0422d6 1846 dest="only_build_tests",
screamer 0:4a2e5f0422d6 1847 default=False,
screamer 0:4a2e5f0422d6 1848 help="Only build tests, skips actual test procedures (flashing etc.)")
screamer 0:4a2e5f0422d6 1849
screamer 0:4a2e5f0422d6 1850 parser.add_option('', '--parallel',
screamer 0:4a2e5f0422d6 1851 dest='parallel_test_exec',
screamer 0:4a2e5f0422d6 1852 default=False,
screamer 0:4a2e5f0422d6 1853 action="store_true",
screamer 0:4a2e5f0422d6 1854 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer 0:4a2e5f0422d6 1855
screamer 0:4a2e5f0422d6 1856 parser.add_option('', '--config',
screamer 0:4a2e5f0422d6 1857 dest='verbose_test_configuration_only',
screamer 0:4a2e5f0422d6 1858 default=False,
screamer 0:4a2e5f0422d6 1859 action="store_true",
screamer 0:4a2e5f0422d6 1860 help='Displays full test specification and MUTs configration and exits')
screamer 0:4a2e5f0422d6 1861
screamer 0:4a2e5f0422d6 1862 parser.add_option('', '--loops',
screamer 0:4a2e5f0422d6 1863 dest='test_loops_list',
screamer 0:4a2e5f0422d6 1864 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer 0:4a2e5f0422d6 1865
screamer 0:4a2e5f0422d6 1866 parser.add_option('', '--global-loops',
screamer 0:4a2e5f0422d6 1867 dest='test_global_loops_value',
screamer 0:4a2e5f0422d6 1868 help='Set global number of test loops per test. Default value is set 1')
screamer 0:4a2e5f0422d6 1869
screamer 0:4a2e5f0422d6 1870 parser.add_option('', '--consolidate-waterfall',
screamer 0:4a2e5f0422d6 1871 dest='consolidate_waterfall_test',
screamer 0:4a2e5f0422d6 1872 default=False,
screamer 0:4a2e5f0422d6 1873 action="store_true",
screamer 0:4a2e5f0422d6 1874 help='Used with --waterfall option. Adds only one test to report reflecting outcome of waterfall test.')
screamer 0:4a2e5f0422d6 1875
screamer 0:4a2e5f0422d6 1876 parser.add_option('-W', '--waterfall',
screamer 0:4a2e5f0422d6 1877 dest='waterfall_test',
screamer 0:4a2e5f0422d6 1878 default=False,
screamer 0:4a2e5f0422d6 1879 action="store_true",
screamer 0:4a2e5f0422d6 1880 help='Used with --loops or --global-loops options. Tests until OK result occurs and assumes test passed')
screamer 0:4a2e5f0422d6 1881
screamer 0:4a2e5f0422d6 1882 parser.add_option('-N', '--firmware-name',
screamer 0:4a2e5f0422d6 1883 dest='firmware_global_name',
screamer 0:4a2e5f0422d6 1884 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer 0:4a2e5f0422d6 1885
screamer 0:4a2e5f0422d6 1886 parser.add_option('-u', '--shuffle',
screamer 0:4a2e5f0422d6 1887 dest='shuffle_test_order',
screamer 0:4a2e5f0422d6 1888 default=False,
screamer 0:4a2e5f0422d6 1889 action="store_true",
screamer 0:4a2e5f0422d6 1890 help='Shuffles test execution order')
screamer 0:4a2e5f0422d6 1891
screamer 0:4a2e5f0422d6 1892 parser.add_option('', '--shuffle-seed',
screamer 0:4a2e5f0422d6 1893 dest='shuffle_test_seed',
screamer 0:4a2e5f0422d6 1894 default=None,
screamer 0:4a2e5f0422d6 1895 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer 0:4a2e5f0422d6 1896
screamer 0:4a2e5f0422d6 1897 parser.add_option('-f', '--filter',
screamer 0:4a2e5f0422d6 1898 dest='general_filter_regex',
screamer 0:4a2e5f0422d6 1899 default=None,
screamer 0:4a2e5f0422d6 1900 help='For some commands you can use filter to filter out results')
screamer 0:4a2e5f0422d6 1901
screamer 0:4a2e5f0422d6 1902 parser.add_option('', '--inc-timeout',
screamer 0:4a2e5f0422d6 1903 dest='extend_test_timeout',
screamer 0:4a2e5f0422d6 1904 metavar="NUMBER",
screamer 0:4a2e5f0422d6 1905 type="int",
screamer 0:4a2e5f0422d6 1906 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer 0:4a2e5f0422d6 1907
screamer 0:4a2e5f0422d6 1908 parser.add_option('', '--db',
screamer 0:4a2e5f0422d6 1909 dest='db_url',
screamer 0:4a2e5f0422d6 1910 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer 0:4a2e5f0422d6 1911
screamer 0:4a2e5f0422d6 1912 parser.add_option('-l', '--log',
screamer 0:4a2e5f0422d6 1913 dest='log_file_name',
screamer 0:4a2e5f0422d6 1914 help='Log events to external file (note not all console entries may be visible in log file)')
screamer 0:4a2e5f0422d6 1915
screamer 0:4a2e5f0422d6 1916 parser.add_option('', '--report-html',
screamer 0:4a2e5f0422d6 1917 dest='report_html_file_name',
screamer 0:4a2e5f0422d6 1918 help='You can log test suite results in form of HTML report')
screamer 0:4a2e5f0422d6 1919
screamer 0:4a2e5f0422d6 1920 parser.add_option('', '--report-junit',
screamer 0:4a2e5f0422d6 1921 dest='report_junit_file_name',
screamer 0:4a2e5f0422d6 1922 help='You can log test suite results in form of JUnit compliant XML report')
screamer 0:4a2e5f0422d6 1923
screamer 0:4a2e5f0422d6 1924 parser.add_option("", "--report-build",
screamer 0:4a2e5f0422d6 1925 dest="report_build_file_name",
screamer 0:4a2e5f0422d6 1926 help="Output the build results to a junit xml file")
screamer 0:4a2e5f0422d6 1927
screamer 0:4a2e5f0422d6 1928 parser.add_option('', '--verbose-skipped',
screamer 0:4a2e5f0422d6 1929 dest='verbose_skipped_tests',
screamer 0:4a2e5f0422d6 1930 default=False,
screamer 0:4a2e5f0422d6 1931 action="store_true",
screamer 0:4a2e5f0422d6 1932 help='Prints some extra information about skipped tests')
screamer 0:4a2e5f0422d6 1933
screamer 0:4a2e5f0422d6 1934 parser.add_option('-V', '--verbose-test-result',
screamer 0:4a2e5f0422d6 1935 dest='verbose_test_result_only',
screamer 0:4a2e5f0422d6 1936 default=False,
screamer 0:4a2e5f0422d6 1937 action="store_true",
screamer 0:4a2e5f0422d6 1938 help='Prints test serial output')
screamer 0:4a2e5f0422d6 1939
screamer 0:4a2e5f0422d6 1940 parser.add_option('-v', '--verbose',
screamer 0:4a2e5f0422d6 1941 dest='verbose',
screamer 0:4a2e5f0422d6 1942 default=False,
screamer 0:4a2e5f0422d6 1943 action="store_true",
screamer 0:4a2e5f0422d6 1944 help='Verbose mode (prints some extra information)')
screamer 0:4a2e5f0422d6 1945
screamer 0:4a2e5f0422d6 1946 parser.add_option('', '--version',
screamer 0:4a2e5f0422d6 1947 dest='version',
screamer 0:4a2e5f0422d6 1948 default=False,
screamer 0:4a2e5f0422d6 1949 action="store_true",
screamer 0:4a2e5f0422d6 1950 help='Prints script version and exits')
screamer 0:4a2e5f0422d6 1951 return parser