Magnificent7 / Hackathon

Includes library modifications to allow access to AIN_4 (AIN_0 / 5)

mbd_os/tools/test_api.py@0:eafc3fd41f75, 2016-09-20 (annotated)

Committer:
bryantaylor
Date:
Tue Sep 20 21:26:12 2016 +0000
Revision:
0:eafc3fd41f75
hackathon

Who changed what in which revision?

UserRevisionLine numberNew contents of line
bryantaylor 0:eafc3fd41f75 1 """
bryantaylor 0:eafc3fd41f75 2 mbed SDK
bryantaylor 0:eafc3fd41f75 3 Copyright (c) 2011-2014 ARM Limited
bryantaylor 0:eafc3fd41f75 4
bryantaylor 0:eafc3fd41f75 5 Licensed under the Apache License, Version 2.0 (the "License");
bryantaylor 0:eafc3fd41f75 6 you may not use this file except in compliance with the License.
bryantaylor 0:eafc3fd41f75 7 You may obtain a copy of the License at
bryantaylor 0:eafc3fd41f75 8
bryantaylor 0:eafc3fd41f75 9 http://www.apache.org/licenses/LICENSE-2.0
bryantaylor 0:eafc3fd41f75 10
bryantaylor 0:eafc3fd41f75 11 Unless required by applicable law or agreed to in writing, software
bryantaylor 0:eafc3fd41f75 12 distributed under the License is distributed on an "AS IS" BASIS,
bryantaylor 0:eafc3fd41f75 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
bryantaylor 0:eafc3fd41f75 14 See the License for the specific language governing permissions and
bryantaylor 0:eafc3fd41f75 15 limitations under the License.
bryantaylor 0:eafc3fd41f75 16
bryantaylor 0:eafc3fd41f75 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
bryantaylor 0:eafc3fd41f75 18 """
bryantaylor 0:eafc3fd41f75 19
bryantaylor 0:eafc3fd41f75 20 import os
bryantaylor 0:eafc3fd41f75 21 import re
bryantaylor 0:eafc3fd41f75 22 import sys
bryantaylor 0:eafc3fd41f75 23 import json
bryantaylor 0:eafc3fd41f75 24 import uuid
bryantaylor 0:eafc3fd41f75 25 import pprint
bryantaylor 0:eafc3fd41f75 26 import random
bryantaylor 0:eafc3fd41f75 27 import argparse
bryantaylor 0:eafc3fd41f75 28 import datetime
bryantaylor 0:eafc3fd41f75 29 import threading
bryantaylor 0:eafc3fd41f75 30 import ctypes
bryantaylor 0:eafc3fd41f75 31 from types import ListType
bryantaylor 0:eafc3fd41f75 32 from colorama import Fore, Back, Style
bryantaylor 0:eafc3fd41f75 33 from prettytable import PrettyTable
bryantaylor 0:eafc3fd41f75 34 from copy import copy
bryantaylor 0:eafc3fd41f75 35
bryantaylor 0:eafc3fd41f75 36 from time import sleep, time
bryantaylor 0:eafc3fd41f75 37 from Queue import Queue, Empty
bryantaylor 0:eafc3fd41f75 38 from os.path import join, exists, basename, relpath
bryantaylor 0:eafc3fd41f75 39 from threading import Thread, Lock
bryantaylor 0:eafc3fd41f75 40 from subprocess import Popen, PIPE
bryantaylor 0:eafc3fd41f75 41
bryantaylor 0:eafc3fd41f75 42 # Imports related to mbed build api
bryantaylor 0:eafc3fd41f75 43 from tools.tests import TESTS
bryantaylor 0:eafc3fd41f75 44 from tools.tests import TEST_MAP
bryantaylor 0:eafc3fd41f75 45 from tools.paths import BUILD_DIR
bryantaylor 0:eafc3fd41f75 46 from tools.paths import HOST_TESTS
bryantaylor 0:eafc3fd41f75 47 from tools.utils import ToolException
bryantaylor 0:eafc3fd41f75 48 from tools.utils import NotSupportedException
bryantaylor 0:eafc3fd41f75 49 from tools.utils import construct_enum
bryantaylor 0:eafc3fd41f75 50 from tools.memap import MemapParser
bryantaylor 0:eafc3fd41f75 51 from tools.targets import TARGET_MAP
bryantaylor 0:eafc3fd41f75 52 from tools.test_db import BaseDBAccess
bryantaylor 0:eafc3fd41f75 53 from tools.build_api import build_project, build_mbed_libs, build_lib
bryantaylor 0:eafc3fd41f75 54 from tools.build_api import get_target_supported_toolchains
bryantaylor 0:eafc3fd41f75 55 from tools.build_api import write_build_report
bryantaylor 0:eafc3fd41f75 56 from tools.build_api import prep_report
bryantaylor 0:eafc3fd41f75 57 from tools.build_api import prep_properties
bryantaylor 0:eafc3fd41f75 58 from tools.build_api import create_result
bryantaylor 0:eafc3fd41f75 59 from tools.build_api import add_result_to_report
bryantaylor 0:eafc3fd41f75 60 from tools.build_api import prepare_toolchain
bryantaylor 0:eafc3fd41f75 61 from tools.build_api import scan_resources
bryantaylor 0:eafc3fd41f75 62 from tools.libraries import LIBRARIES, LIBRARY_MAP
bryantaylor 0:eafc3fd41f75 63 from tools.toolchains import TOOLCHAIN_PATHS
bryantaylor 0:eafc3fd41f75 64 from tools.toolchains import TOOLCHAINS
bryantaylor 0:eafc3fd41f75 65 from tools.test_exporters import ReportExporter, ResultExporterType
bryantaylor 0:eafc3fd41f75 66 from tools.utils import argparse_filestring_type
bryantaylor 0:eafc3fd41f75 67 from tools.utils import argparse_uppercase_type
bryantaylor 0:eafc3fd41f75 68 from tools.utils import argparse_lowercase_type
bryantaylor 0:eafc3fd41f75 69 from tools.utils import argparse_many
bryantaylor 0:eafc3fd41f75 70 from tools.utils import get_path_depth
bryantaylor 0:eafc3fd41f75 71
bryantaylor 0:eafc3fd41f75 72 import tools.host_tests.host_tests_plugins as host_tests_plugins
bryantaylor 0:eafc3fd41f75 73
bryantaylor 0:eafc3fd41f75 74 try:
bryantaylor 0:eafc3fd41f75 75 import mbed_lstools
bryantaylor 0:eafc3fd41f75 76 from tools.compliance.ioper_runner import get_available_oper_test_scopes
bryantaylor 0:eafc3fd41f75 77 except:
bryantaylor 0:eafc3fd41f75 78 pass
bryantaylor 0:eafc3fd41f75 79
bryantaylor 0:eafc3fd41f75 80
bryantaylor 0:eafc3fd41f75 81 class ProcessObserver(Thread):
bryantaylor 0:eafc3fd41f75 82 def __init__(self, proc):
bryantaylor 0:eafc3fd41f75 83 Thread.__init__(self)
bryantaylor 0:eafc3fd41f75 84 self.proc = proc
bryantaylor 0:eafc3fd41f75 85 self.queue = Queue()
bryantaylor 0:eafc3fd41f75 86 self.daemon = True
bryantaylor 0:eafc3fd41f75 87 self.active = True
bryantaylor 0:eafc3fd41f75 88 self.start()
bryantaylor 0:eafc3fd41f75 89
bryantaylor 0:eafc3fd41f75 90 def run(self):
bryantaylor 0:eafc3fd41f75 91 while self.active:
bryantaylor 0:eafc3fd41f75 92 c = self.proc.stdout.read(1)
bryantaylor 0:eafc3fd41f75 93 self.queue.put(c)
bryantaylor 0:eafc3fd41f75 94
bryantaylor 0:eafc3fd41f75 95 def stop(self):
bryantaylor 0:eafc3fd41f75 96 self.active = False
bryantaylor 0:eafc3fd41f75 97 try:
bryantaylor 0:eafc3fd41f75 98 self.proc.terminate()
bryantaylor 0:eafc3fd41f75 99 except Exception, _:
bryantaylor 0:eafc3fd41f75 100 pass
bryantaylor 0:eafc3fd41f75 101
bryantaylor 0:eafc3fd41f75 102
bryantaylor 0:eafc3fd41f75 103 class SingleTestExecutor(threading.Thread):
bryantaylor 0:eafc3fd41f75 104 """ Example: Single test class in separate thread usage
bryantaylor 0:eafc3fd41f75 105 """
bryantaylor 0:eafc3fd41f75 106 def __init__(self, single_test):
bryantaylor 0:eafc3fd41f75 107 self.single_test = single_test
bryantaylor 0:eafc3fd41f75 108 threading.Thread.__init__(self)
bryantaylor 0:eafc3fd41f75 109
bryantaylor 0:eafc3fd41f75 110 def run(self):
bryantaylor 0:eafc3fd41f75 111 start = time()
bryantaylor 0:eafc3fd41f75 112 # Execute tests depending on options and filter applied
bryantaylor 0:eafc3fd41f75 113 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
bryantaylor 0:eafc3fd41f75 114 elapsed_time = time() - start
bryantaylor 0:eafc3fd41f75 115
bryantaylor 0:eafc3fd41f75 116 # Human readable summary
bryantaylor 0:eafc3fd41f75 117 if not self.single_test.opts_suppress_summary:
bryantaylor 0:eafc3fd41f75 118 # prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 119 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
bryantaylor 0:eafc3fd41f75 120 if self.single_test.opts_test_x_toolchain_summary:
bryantaylor 0:eafc3fd41f75 121 # prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 122 # table shows text x toolchain test result matrix
bryantaylor 0:eafc3fd41f75 123 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
bryantaylor 0:eafc3fd41f75 124 print "Completed in %.2f sec"% (elapsed_time)
bryantaylor 0:eafc3fd41f75 125
bryantaylor 0:eafc3fd41f75 126
bryantaylor 0:eafc3fd41f75 127 class SingleTestRunner(object):
bryantaylor 0:eafc3fd41f75 128 """ Object wrapper for single test run which may involve multiple MUTs
bryantaylor 0:eafc3fd41f75 129 """
bryantaylor 0:eafc3fd41f75 130 RE_DETECT_TESTCASE_RESULT = None
bryantaylor 0:eafc3fd41f75 131
bryantaylor 0:eafc3fd41f75 132 # Return codes for test script
bryantaylor 0:eafc3fd41f75 133 TEST_RESULT_OK = "OK"
bryantaylor 0:eafc3fd41f75 134 TEST_RESULT_FAIL = "FAIL"
bryantaylor 0:eafc3fd41f75 135 TEST_RESULT_ERROR = "ERROR"
bryantaylor 0:eafc3fd41f75 136 TEST_RESULT_UNDEF = "UNDEF"
bryantaylor 0:eafc3fd41f75 137 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
bryantaylor 0:eafc3fd41f75 138 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
bryantaylor 0:eafc3fd41f75 139 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
bryantaylor 0:eafc3fd41f75 140 TEST_RESULT_TIMEOUT = "TIMEOUT"
bryantaylor 0:eafc3fd41f75 141 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
bryantaylor 0:eafc3fd41f75 142 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
bryantaylor 0:eafc3fd41f75 143 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
bryantaylor 0:eafc3fd41f75 144 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
bryantaylor 0:eafc3fd41f75 145
bryantaylor 0:eafc3fd41f75 146 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
bryantaylor 0:eafc3fd41f75 147 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
bryantaylor 0:eafc3fd41f75 148 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
bryantaylor 0:eafc3fd41f75 149
bryantaylor 0:eafc3fd41f75 150 muts = {} # MUTs descriptor (from external file)
bryantaylor 0:eafc3fd41f75 151 test_spec = {} # Test specification (from external file)
bryantaylor 0:eafc3fd41f75 152
bryantaylor 0:eafc3fd41f75 153 # mbed test suite -> SingleTestRunner
bryantaylor 0:eafc3fd41f75 154 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
bryantaylor 0:eafc3fd41f75 155 "failure" : TEST_RESULT_FAIL,
bryantaylor 0:eafc3fd41f75 156 "error" : TEST_RESULT_ERROR,
bryantaylor 0:eafc3fd41f75 157 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
bryantaylor 0:eafc3fd41f75 158 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
bryantaylor 0:eafc3fd41f75 159 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
bryantaylor 0:eafc3fd41f75 160 "timeout" : TEST_RESULT_TIMEOUT,
bryantaylor 0:eafc3fd41f75 161 "no_image" : TEST_RESULT_NO_IMAGE,
bryantaylor 0:eafc3fd41f75 162 "end" : TEST_RESULT_UNDEF,
bryantaylor 0:eafc3fd41f75 163 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
bryantaylor 0:eafc3fd41f75 164 "build_failed" : TEST_RESULT_BUILD_FAILED,
bryantaylor 0:eafc3fd41f75 165 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
bryantaylor 0:eafc3fd41f75 166 }
bryantaylor 0:eafc3fd41f75 167
bryantaylor 0:eafc3fd41f75 168 def __init__(self,
bryantaylor 0:eafc3fd41f75 169 _global_loops_count=1,
bryantaylor 0:eafc3fd41f75 170 _test_loops_list=None,
bryantaylor 0:eafc3fd41f75 171 _muts={},
bryantaylor 0:eafc3fd41f75 172 _clean=False,
bryantaylor 0:eafc3fd41f75 173 _opts_db_url=None,
bryantaylor 0:eafc3fd41f75 174 _opts_log_file_name=None,
bryantaylor 0:eafc3fd41f75 175 _opts_report_html_file_name=None,
bryantaylor 0:eafc3fd41f75 176 _opts_report_junit_file_name=None,
bryantaylor 0:eafc3fd41f75 177 _opts_report_build_file_name=None,
bryantaylor 0:eafc3fd41f75 178 _opts_report_text_file_name=None,
bryantaylor 0:eafc3fd41f75 179 _opts_build_report={},
bryantaylor 0:eafc3fd41f75 180 _opts_build_properties={},
bryantaylor 0:eafc3fd41f75 181 _test_spec={},
bryantaylor 0:eafc3fd41f75 182 _opts_goanna_for_mbed_sdk=None,
bryantaylor 0:eafc3fd41f75 183 _opts_goanna_for_tests=None,
bryantaylor 0:eafc3fd41f75 184 _opts_shuffle_test_order=False,
bryantaylor 0:eafc3fd41f75 185 _opts_shuffle_test_seed=None,
bryantaylor 0:eafc3fd41f75 186 _opts_test_by_names=None,
bryantaylor 0:eafc3fd41f75 187 _opts_peripheral_by_names=None,
bryantaylor 0:eafc3fd41f75 188 _opts_test_only_peripheral=False,
bryantaylor 0:eafc3fd41f75 189 _opts_test_only_common=False,
bryantaylor 0:eafc3fd41f75 190 _opts_verbose_skipped_tests=False,
bryantaylor 0:eafc3fd41f75 191 _opts_verbose_test_result_only=False,
bryantaylor 0:eafc3fd41f75 192 _opts_verbose=False,
bryantaylor 0:eafc3fd41f75 193 _opts_firmware_global_name=None,
bryantaylor 0:eafc3fd41f75 194 _opts_only_build_tests=False,
bryantaylor 0:eafc3fd41f75 195 _opts_parallel_test_exec=False,
bryantaylor 0:eafc3fd41f75 196 _opts_suppress_summary=False,
bryantaylor 0:eafc3fd41f75 197 _opts_test_x_toolchain_summary=False,
bryantaylor 0:eafc3fd41f75 198 _opts_copy_method=None,
bryantaylor 0:eafc3fd41f75 199 _opts_mut_reset_type=None,
bryantaylor 0:eafc3fd41f75 200 _opts_jobs=None,
bryantaylor 0:eafc3fd41f75 201 _opts_waterfall_test=None,
bryantaylor 0:eafc3fd41f75 202 _opts_consolidate_waterfall_test=None,
bryantaylor 0:eafc3fd41f75 203 _opts_extend_test_timeout=None,
bryantaylor 0:eafc3fd41f75 204 _opts_auto_detect=None,
bryantaylor 0:eafc3fd41f75 205 _opts_include_non_automated=False):
bryantaylor 0:eafc3fd41f75 206 """ Let's try hard to init this object
bryantaylor 0:eafc3fd41f75 207 """
bryantaylor 0:eafc3fd41f75 208 from colorama import init
bryantaylor 0:eafc3fd41f75 209 init()
bryantaylor 0:eafc3fd41f75 210
bryantaylor 0:eafc3fd41f75 211 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
bryantaylor 0:eafc3fd41f75 212 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
bryantaylor 0:eafc3fd41f75 213 # Settings related to test loops counters
bryantaylor 0:eafc3fd41f75 214 try:
bryantaylor 0:eafc3fd41f75 215 _global_loops_count = int(_global_loops_count)
bryantaylor 0:eafc3fd41f75 216 except:
bryantaylor 0:eafc3fd41f75 217 _global_loops_count = 1
bryantaylor 0:eafc3fd41f75 218 if _global_loops_count < 1:
bryantaylor 0:eafc3fd41f75 219 _global_loops_count = 1
bryantaylor 0:eafc3fd41f75 220 self.GLOBAL_LOOPS_COUNT = _global_loops_count
bryantaylor 0:eafc3fd41f75 221 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
bryantaylor 0:eafc3fd41f75 222 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
bryantaylor 0:eafc3fd41f75 223
bryantaylor 0:eafc3fd41f75 224 self.shuffle_random_seed = 0.0
bryantaylor 0:eafc3fd41f75 225 self.SHUFFLE_SEED_ROUND = 10
bryantaylor 0:eafc3fd41f75 226
bryantaylor 0:eafc3fd41f75 227 # MUT list and test specification storage
bryantaylor 0:eafc3fd41f75 228 self.muts = _muts
bryantaylor 0:eafc3fd41f75 229 self.test_spec = _test_spec
bryantaylor 0:eafc3fd41f75 230
bryantaylor 0:eafc3fd41f75 231 # Settings passed e.g. from command line
bryantaylor 0:eafc3fd41f75 232 self.opts_db_url = _opts_db_url
bryantaylor 0:eafc3fd41f75 233 self.opts_log_file_name = _opts_log_file_name
bryantaylor 0:eafc3fd41f75 234 self.opts_report_html_file_name = _opts_report_html_file_name
bryantaylor 0:eafc3fd41f75 235 self.opts_report_junit_file_name = _opts_report_junit_file_name
bryantaylor 0:eafc3fd41f75 236 self.opts_report_build_file_name = _opts_report_build_file_name
bryantaylor 0:eafc3fd41f75 237 self.opts_report_text_file_name = _opts_report_text_file_name
bryantaylor 0:eafc3fd41f75 238 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
bryantaylor 0:eafc3fd41f75 239 self.opts_goanna_for_tests = _opts_goanna_for_tests
bryantaylor 0:eafc3fd41f75 240 self.opts_shuffle_test_order = _opts_shuffle_test_order
bryantaylor 0:eafc3fd41f75 241 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
bryantaylor 0:eafc3fd41f75 242 self.opts_test_by_names = _opts_test_by_names
bryantaylor 0:eafc3fd41f75 243 self.opts_peripheral_by_names = _opts_peripheral_by_names
bryantaylor 0:eafc3fd41f75 244 self.opts_test_only_peripheral = _opts_test_only_peripheral
bryantaylor 0:eafc3fd41f75 245 self.opts_test_only_common = _opts_test_only_common
bryantaylor 0:eafc3fd41f75 246 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
bryantaylor 0:eafc3fd41f75 247 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
bryantaylor 0:eafc3fd41f75 248 self.opts_verbose = _opts_verbose
bryantaylor 0:eafc3fd41f75 249 self.opts_firmware_global_name = _opts_firmware_global_name
bryantaylor 0:eafc3fd41f75 250 self.opts_only_build_tests = _opts_only_build_tests
bryantaylor 0:eafc3fd41f75 251 self.opts_parallel_test_exec = _opts_parallel_test_exec
bryantaylor 0:eafc3fd41f75 252 self.opts_suppress_summary = _opts_suppress_summary
bryantaylor 0:eafc3fd41f75 253 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
bryantaylor 0:eafc3fd41f75 254 self.opts_copy_method = _opts_copy_method
bryantaylor 0:eafc3fd41f75 255 self.opts_mut_reset_type = _opts_mut_reset_type
bryantaylor 0:eafc3fd41f75 256 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
bryantaylor 0:eafc3fd41f75 257 self.opts_waterfall_test = _opts_waterfall_test
bryantaylor 0:eafc3fd41f75 258 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
bryantaylor 0:eafc3fd41f75 259 self.opts_extend_test_timeout = _opts_extend_test_timeout
bryantaylor 0:eafc3fd41f75 260 self.opts_clean = _clean
bryantaylor 0:eafc3fd41f75 261 self.opts_auto_detect = _opts_auto_detect
bryantaylor 0:eafc3fd41f75 262 self.opts_include_non_automated = _opts_include_non_automated
bryantaylor 0:eafc3fd41f75 263
bryantaylor 0:eafc3fd41f75 264 self.build_report = _opts_build_report
bryantaylor 0:eafc3fd41f75 265 self.build_properties = _opts_build_properties
bryantaylor 0:eafc3fd41f75 266
bryantaylor 0:eafc3fd41f75 267 # File / screen logger initialization
bryantaylor 0:eafc3fd41f75 268 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
bryantaylor 0:eafc3fd41f75 269
bryantaylor 0:eafc3fd41f75 270 # Database related initializations
bryantaylor 0:eafc3fd41f75 271 self.db_logger = factory_db_logger(self.opts_db_url)
bryantaylor 0:eafc3fd41f75 272 self.db_logger_build_id = None # Build ID (database index of build_id table)
bryantaylor 0:eafc3fd41f75 273 # Let's connect to database to set up credentials and confirm database is ready
bryantaylor 0:eafc3fd41f75 274 if self.db_logger:
bryantaylor 0:eafc3fd41f75 275 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
bryantaylor 0:eafc3fd41f75 276 if self.db_logger.is_connected():
bryantaylor 0:eafc3fd41f75 277 # Get hostname and uname so we can use it as build description
bryantaylor 0:eafc3fd41f75 278 # when creating new build_id in external database
bryantaylor 0:eafc3fd41f75 279 (_hostname, _uname) = self.db_logger.get_hostname()
bryantaylor 0:eafc3fd41f75 280 _host_location = os.path.dirname(os.path.abspath(__file__))
bryantaylor 0:eafc3fd41f75 281 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
bryantaylor 0:eafc3fd41f75 282 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
bryantaylor 0:eafc3fd41f75 283 self.db_logger.disconnect()
bryantaylor 0:eafc3fd41f75 284
bryantaylor 0:eafc3fd41f75 285 def dump_options(self):
bryantaylor 0:eafc3fd41f75 286 """ Function returns data structure with common settings passed to SingelTestRunner
bryantaylor 0:eafc3fd41f75 287 It can be used for example to fill _extra fields in database storing test suite single run data
bryantaylor 0:eafc3fd41f75 288 Example:
bryantaylor 0:eafc3fd41f75 289 data = self.dump_options()
bryantaylor 0:eafc3fd41f75 290 or
bryantaylor 0:eafc3fd41f75 291 data_str = json.dumps(self.dump_options())
bryantaylor 0:eafc3fd41f75 292 """
bryantaylor 0:eafc3fd41f75 293 result = {"db_url" : str(self.opts_db_url),
bryantaylor 0:eafc3fd41f75 294 "log_file_name" : str(self.opts_log_file_name),
bryantaylor 0:eafc3fd41f75 295 "shuffle_test_order" : str(self.opts_shuffle_test_order),
bryantaylor 0:eafc3fd41f75 296 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
bryantaylor 0:eafc3fd41f75 297 "test_by_names" : str(self.opts_test_by_names),
bryantaylor 0:eafc3fd41f75 298 "peripheral_by_names" : str(self.opts_peripheral_by_names),
bryantaylor 0:eafc3fd41f75 299 "test_only_peripheral" : str(self.opts_test_only_peripheral),
bryantaylor 0:eafc3fd41f75 300 "test_only_common" : str(self.opts_test_only_common),
bryantaylor 0:eafc3fd41f75 301 "verbose" : str(self.opts_verbose),
bryantaylor 0:eafc3fd41f75 302 "firmware_global_name" : str(self.opts_firmware_global_name),
bryantaylor 0:eafc3fd41f75 303 "only_build_tests" : str(self.opts_only_build_tests),
bryantaylor 0:eafc3fd41f75 304 "copy_method" : str(self.opts_copy_method),
bryantaylor 0:eafc3fd41f75 305 "mut_reset_type" : str(self.opts_mut_reset_type),
bryantaylor 0:eafc3fd41f75 306 "jobs" : str(self.opts_jobs),
bryantaylor 0:eafc3fd41f75 307 "extend_test_timeout" : str(self.opts_extend_test_timeout),
bryantaylor 0:eafc3fd41f75 308 "_dummy" : ''
bryantaylor 0:eafc3fd41f75 309 }
bryantaylor 0:eafc3fd41f75 310 return result
bryantaylor 0:eafc3fd41f75 311
bryantaylor 0:eafc3fd41f75 312 def shuffle_random_func(self):
bryantaylor 0:eafc3fd41f75 313 return self.shuffle_random_seed
bryantaylor 0:eafc3fd41f75 314
bryantaylor 0:eafc3fd41f75 315 def is_shuffle_seed_float(self):
bryantaylor 0:eafc3fd41f75 316 """ return true if function parameter can be converted to float
bryantaylor 0:eafc3fd41f75 317 """
bryantaylor 0:eafc3fd41f75 318 result = True
bryantaylor 0:eafc3fd41f75 319 try:
bryantaylor 0:eafc3fd41f75 320 float(self.shuffle_random_seed)
bryantaylor 0:eafc3fd41f75 321 except ValueError:
bryantaylor 0:eafc3fd41f75 322 result = False
bryantaylor 0:eafc3fd41f75 323 return result
bryantaylor 0:eafc3fd41f75 324
bryantaylor 0:eafc3fd41f75 325 # This will store target / toolchain specific properties
bryantaylor 0:eafc3fd41f75 326 test_suite_properties_ext = {} # target : toolchain
bryantaylor 0:eafc3fd41f75 327 # Here we store test results
bryantaylor 0:eafc3fd41f75 328 test_summary = []
bryantaylor 0:eafc3fd41f75 329 # Here we store test results in extended data structure
bryantaylor 0:eafc3fd41f75 330 test_summary_ext = {}
bryantaylor 0:eafc3fd41f75 331 execute_thread_slice_lock = Lock()
bryantaylor 0:eafc3fd41f75 332
bryantaylor 0:eafc3fd41f75 333 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
bryantaylor 0:eafc3fd41f75 334 for toolchain in toolchains:
bryantaylor 0:eafc3fd41f75 335 tt_id = "%s::%s" % (toolchain, target)
bryantaylor 0:eafc3fd41f75 336
bryantaylor 0:eafc3fd41f75 337 T = TARGET_MAP[target]
bryantaylor 0:eafc3fd41f75 338
bryantaylor 0:eafc3fd41f75 339 # print target, toolchain
bryantaylor 0:eafc3fd41f75 340 # Test suite properties returned to external tools like CI
bryantaylor 0:eafc3fd41f75 341 test_suite_properties = {
bryantaylor 0:eafc3fd41f75 342 'jobs': self.opts_jobs,
bryantaylor 0:eafc3fd41f75 343 'clean': clean,
bryantaylor 0:eafc3fd41f75 344 'target': target,
bryantaylor 0:eafc3fd41f75 345 'vendor': T.extra_labels[0],
bryantaylor 0:eafc3fd41f75 346 'test_ids': ', '.join(test_ids),
bryantaylor 0:eafc3fd41f75 347 'toolchain': toolchain,
bryantaylor 0:eafc3fd41f75 348 'shuffle_random_seed': self.shuffle_random_seed
bryantaylor 0:eafc3fd41f75 349 }
bryantaylor 0:eafc3fd41f75 350
bryantaylor 0:eafc3fd41f75 351
bryantaylor 0:eafc3fd41f75 352 # print '=== %s::%s ===' % (target, toolchain)
bryantaylor 0:eafc3fd41f75 353 # Let's build our test
bryantaylor 0:eafc3fd41f75 354 if target not in TARGET_MAP:
bryantaylor 0:eafc3fd41f75 355 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
bryantaylor 0:eafc3fd41f75 356 continue
bryantaylor 0:eafc3fd41f75 357
bryantaylor 0:eafc3fd41f75 358 build_mbed_libs_options = ["analyze"] if self.opts_goanna_for_mbed_sdk else None
bryantaylor 0:eafc3fd41f75 359 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
bryantaylor 0:eafc3fd41f75 360
bryantaylor 0:eafc3fd41f75 361
bryantaylor 0:eafc3fd41f75 362 try:
bryantaylor 0:eafc3fd41f75 363 build_mbed_libs_result = build_mbed_libs(T,
bryantaylor 0:eafc3fd41f75 364 toolchain,
bryantaylor 0:eafc3fd41f75 365 options=build_mbed_libs_options,
bryantaylor 0:eafc3fd41f75 366 clean=clean_mbed_libs_options,
bryantaylor 0:eafc3fd41f75 367 verbose=self.opts_verbose,
bryantaylor 0:eafc3fd41f75 368 jobs=self.opts_jobs,
bryantaylor 0:eafc3fd41f75 369 report=build_report,
bryantaylor 0:eafc3fd41f75 370 properties=build_properties)
bryantaylor 0:eafc3fd41f75 371
bryantaylor 0:eafc3fd41f75 372 if not build_mbed_libs_result:
bryantaylor 0:eafc3fd41f75 373 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
bryantaylor 0:eafc3fd41f75 374 continue
bryantaylor 0:eafc3fd41f75 375
bryantaylor 0:eafc3fd41f75 376 except ToolException:
bryantaylor 0:eafc3fd41f75 377 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
bryantaylor 0:eafc3fd41f75 378 continue
bryantaylor 0:eafc3fd41f75 379
bryantaylor 0:eafc3fd41f75 380 build_dir = join(BUILD_DIR, "test", target, toolchain)
bryantaylor 0:eafc3fd41f75 381
bryantaylor 0:eafc3fd41f75 382 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
bryantaylor 0:eafc3fd41f75 383 test_suite_properties['build_dir'] = build_dir
bryantaylor 0:eafc3fd41f75 384 test_suite_properties['skipped'] = []
bryantaylor 0:eafc3fd41f75 385
bryantaylor 0:eafc3fd41f75 386 # Enumerate through all tests and shuffle test order if requested
bryantaylor 0:eafc3fd41f75 387 test_map_keys = sorted(TEST_MAP.keys())
bryantaylor 0:eafc3fd41f75 388
bryantaylor 0:eafc3fd41f75 389 if self.opts_shuffle_test_order:
bryantaylor 0:eafc3fd41f75 390 random.shuffle(test_map_keys, self.shuffle_random_func)
bryantaylor 0:eafc3fd41f75 391 # Update database with shuffle seed f applicable
bryantaylor 0:eafc3fd41f75 392 if self.db_logger:
bryantaylor 0:eafc3fd41f75 393 self.db_logger.reconnect();
bryantaylor 0:eafc3fd41f75 394 if self.db_logger.is_connected():
bryantaylor 0:eafc3fd41f75 395 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
bryantaylor 0:eafc3fd41f75 396 self.db_logger.disconnect();
bryantaylor 0:eafc3fd41f75 397
bryantaylor 0:eafc3fd41f75 398 if self.db_logger:
bryantaylor 0:eafc3fd41f75 399 self.db_logger.reconnect();
bryantaylor 0:eafc3fd41f75 400 if self.db_logger.is_connected():
bryantaylor 0:eafc3fd41f75 401 # Update MUTs and Test Specification in database
bryantaylor 0:eafc3fd41f75 402 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
bryantaylor 0:eafc3fd41f75 403 # Update Extra information in database (some options passed to test suite)
bryantaylor 0:eafc3fd41f75 404 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
bryantaylor 0:eafc3fd41f75 405 self.db_logger.disconnect();
bryantaylor 0:eafc3fd41f75 406
bryantaylor 0:eafc3fd41f75 407 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
bryantaylor 0:eafc3fd41f75 408 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
bryantaylor 0:eafc3fd41f75 409
bryantaylor 0:eafc3fd41f75 410 for skipped_test_id in skipped_test_map_keys:
bryantaylor 0:eafc3fd41f75 411 test_suite_properties['skipped'].append(skipped_test_id)
bryantaylor 0:eafc3fd41f75 412
bryantaylor 0:eafc3fd41f75 413
bryantaylor 0:eafc3fd41f75 414 # First pass through all tests and determine which libraries need to be built
bryantaylor 0:eafc3fd41f75 415 libraries = []
bryantaylor 0:eafc3fd41f75 416 for test_id in valid_test_map_keys:
bryantaylor 0:eafc3fd41f75 417 test = TEST_MAP[test_id]
bryantaylor 0:eafc3fd41f75 418
bryantaylor 0:eafc3fd41f75 419 # Detect which lib should be added to test
bryantaylor 0:eafc3fd41f75 420 # Some libs have to compiled like RTOS or ETH
bryantaylor 0:eafc3fd41f75 421 for lib in LIBRARIES:
bryantaylor 0:eafc3fd41f75 422 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
bryantaylor 0:eafc3fd41f75 423 libraries.append(lib['id'])
bryantaylor 0:eafc3fd41f75 424
bryantaylor 0:eafc3fd41f75 425
bryantaylor 0:eafc3fd41f75 426 build_project_options = ["analyze"] if self.opts_goanna_for_tests else None
bryantaylor 0:eafc3fd41f75 427 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
bryantaylor 0:eafc3fd41f75 428
bryantaylor 0:eafc3fd41f75 429 # Build all required libraries
bryantaylor 0:eafc3fd41f75 430 for lib_id in libraries:
bryantaylor 0:eafc3fd41f75 431 try:
bryantaylor 0:eafc3fd41f75 432 build_lib(lib_id,
bryantaylor 0:eafc3fd41f75 433 T,
bryantaylor 0:eafc3fd41f75 434 toolchain,
bryantaylor 0:eafc3fd41f75 435 options=build_project_options,
bryantaylor 0:eafc3fd41f75 436 verbose=self.opts_verbose,
bryantaylor 0:eafc3fd41f75 437 clean=clean_mbed_libs_options,
bryantaylor 0:eafc3fd41f75 438 jobs=self.opts_jobs,
bryantaylor 0:eafc3fd41f75 439 report=build_report,
bryantaylor 0:eafc3fd41f75 440 properties=build_properties)
bryantaylor 0:eafc3fd41f75 441
bryantaylor 0:eafc3fd41f75 442 except ToolException:
bryantaylor 0:eafc3fd41f75 443 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
bryantaylor 0:eafc3fd41f75 444 continue
bryantaylor 0:eafc3fd41f75 445
bryantaylor 0:eafc3fd41f75 446
bryantaylor 0:eafc3fd41f75 447 for test_id in valid_test_map_keys:
bryantaylor 0:eafc3fd41f75 448 test = TEST_MAP[test_id]
bryantaylor 0:eafc3fd41f75 449
bryantaylor 0:eafc3fd41f75 450 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
bryantaylor 0:eafc3fd41f75 451
bryantaylor 0:eafc3fd41f75 452 # TODO: move this 2 below loops to separate function
bryantaylor 0:eafc3fd41f75 453 INC_DIRS = []
bryantaylor 0:eafc3fd41f75 454 for lib_id in libraries:
bryantaylor 0:eafc3fd41f75 455 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
bryantaylor 0:eafc3fd41f75 456 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
bryantaylor 0:eafc3fd41f75 457
bryantaylor 0:eafc3fd41f75 458 MACROS = []
bryantaylor 0:eafc3fd41f75 459 for lib_id in libraries:
bryantaylor 0:eafc3fd41f75 460 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
bryantaylor 0:eafc3fd41f75 461 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
bryantaylor 0:eafc3fd41f75 462 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
bryantaylor 0:eafc3fd41f75 463 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
bryantaylor 0:eafc3fd41f75 464 test_uuid = uuid.uuid4()
bryantaylor 0:eafc3fd41f75 465 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
bryantaylor 0:eafc3fd41f75 466
bryantaylor 0:eafc3fd41f75 467 # Prepare extended test results data structure (it can be used to generate detailed test report)
bryantaylor 0:eafc3fd41f75 468 if target not in self.test_summary_ext:
bryantaylor 0:eafc3fd41f75 469 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
bryantaylor 0:eafc3fd41f75 470 if toolchain not in self.test_summary_ext[target]:
bryantaylor 0:eafc3fd41f75 471 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
bryantaylor 0:eafc3fd41f75 472
bryantaylor 0:eafc3fd41f75 473 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
bryantaylor 0:eafc3fd41f75 474
bryantaylor 0:eafc3fd41f75 475 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
bryantaylor 0:eafc3fd41f75 476 try:
bryantaylor 0:eafc3fd41f75 477 path = build_project(test.source_dir,
bryantaylor 0:eafc3fd41f75 478 join(build_dir, test_id),
bryantaylor 0:eafc3fd41f75 479 T,
bryantaylor 0:eafc3fd41f75 480 toolchain,
bryantaylor 0:eafc3fd41f75 481 test.dependencies,
bryantaylor 0:eafc3fd41f75 482 options=build_project_options,
bryantaylor 0:eafc3fd41f75 483 clean=clean_project_options,
bryantaylor 0:eafc3fd41f75 484 verbose=self.opts_verbose,
bryantaylor 0:eafc3fd41f75 485 name=project_name,
bryantaylor 0:eafc3fd41f75 486 macros=MACROS,
bryantaylor 0:eafc3fd41f75 487 inc_dirs=INC_DIRS,
bryantaylor 0:eafc3fd41f75 488 jobs=self.opts_jobs,
bryantaylor 0:eafc3fd41f75 489 report=build_report,
bryantaylor 0:eafc3fd41f75 490 properties=build_properties,
bryantaylor 0:eafc3fd41f75 491 project_id=test_id,
bryantaylor 0:eafc3fd41f75 492 project_description=test.get_description())
bryantaylor 0:eafc3fd41f75 493
bryantaylor 0:eafc3fd41f75 494 except Exception, e:
bryantaylor 0:eafc3fd41f75 495 project_name_str = project_name if project_name is not None else test_id
bryantaylor 0:eafc3fd41f75 496
bryantaylor 0:eafc3fd41f75 497
bryantaylor 0:eafc3fd41f75 498 test_result = self.TEST_RESULT_FAIL
bryantaylor 0:eafc3fd41f75 499
bryantaylor 0:eafc3fd41f75 500 if isinstance(e, ToolException):
bryantaylor 0:eafc3fd41f75 501 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
bryantaylor 0:eafc3fd41f75 502 test_result = self.TEST_RESULT_BUILD_FAILED
bryantaylor 0:eafc3fd41f75 503 elif isinstance(e, NotSupportedException):
bryantaylor 0:eafc3fd41f75 504 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
bryantaylor 0:eafc3fd41f75 505 test_result = self.TEST_RESULT_NOT_SUPPORTED
bryantaylor 0:eafc3fd41f75 506
bryantaylor 0:eafc3fd41f75 507
bryantaylor 0:eafc3fd41f75 508 # Append test results to global test summary
bryantaylor 0:eafc3fd41f75 509 self.test_summary.append(
bryantaylor 0:eafc3fd41f75 510 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
bryantaylor 0:eafc3fd41f75 511 )
bryantaylor 0:eafc3fd41f75 512
bryantaylor 0:eafc3fd41f75 513 # Add detailed test result to test summary structure
bryantaylor 0:eafc3fd41f75 514 if test_id not in self.test_summary_ext[target][toolchain]:
bryantaylor 0:eafc3fd41f75 515 self.test_summary_ext[target][toolchain][test_id] = []
bryantaylor 0:eafc3fd41f75 516
bryantaylor 0:eafc3fd41f75 517 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
bryantaylor 0:eafc3fd41f75 518 'result' : test_result,
bryantaylor 0:eafc3fd41f75 519 'output' : '',
bryantaylor 0:eafc3fd41f75 520 'target_name' : target,
bryantaylor 0:eafc3fd41f75 521 'target_name_unique': target,
bryantaylor 0:eafc3fd41f75 522 'toolchain_name' : toolchain,
bryantaylor 0:eafc3fd41f75 523 'id' : test_id,
bryantaylor 0:eafc3fd41f75 524 'description' : test.get_description(),
bryantaylor 0:eafc3fd41f75 525 'elapsed_time' : 0,
bryantaylor 0:eafc3fd41f75 526 'duration' : 0,
bryantaylor 0:eafc3fd41f75 527 'copy_method' : None
bryantaylor 0:eafc3fd41f75 528 }})
bryantaylor 0:eafc3fd41f75 529 continue
bryantaylor 0:eafc3fd41f75 530
bryantaylor 0:eafc3fd41f75 531 if self.opts_only_build_tests:
bryantaylor 0:eafc3fd41f75 532 # With this option we are skipping testing phase
bryantaylor 0:eafc3fd41f75 533 continue
bryantaylor 0:eafc3fd41f75 534
bryantaylor 0:eafc3fd41f75 535 # Test duration can be increased by global value
bryantaylor 0:eafc3fd41f75 536 test_duration = test.duration
bryantaylor 0:eafc3fd41f75 537 if self.opts_extend_test_timeout is not None:
bryantaylor 0:eafc3fd41f75 538 test_duration += self.opts_extend_test_timeout
bryantaylor 0:eafc3fd41f75 539
bryantaylor 0:eafc3fd41f75 540 # For an automated test the duration act as a timeout after
bryantaylor 0:eafc3fd41f75 541 # which the test gets interrupted
bryantaylor 0:eafc3fd41f75 542 test_spec = self.shape_test_request(target, path, test_id, test_duration)
bryantaylor 0:eafc3fd41f75 543 test_loops = self.get_test_loop_count(test_id)
bryantaylor 0:eafc3fd41f75 544
bryantaylor 0:eafc3fd41f75 545 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
bryantaylor 0:eafc3fd41f75 546 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
bryantaylor 0:eafc3fd41f75 547 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
bryantaylor 0:eafc3fd41f75 548
bryantaylor 0:eafc3fd41f75 549 # read MUTs, test specification and perform tests
bryantaylor 0:eafc3fd41f75 550 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
bryantaylor 0:eafc3fd41f75 551
bryantaylor 0:eafc3fd41f75 552 if handle_results is None:
bryantaylor 0:eafc3fd41f75 553 continue
bryantaylor 0:eafc3fd41f75 554
bryantaylor 0:eafc3fd41f75 555 for handle_result in handle_results:
bryantaylor 0:eafc3fd41f75 556 if handle_result:
bryantaylor 0:eafc3fd41f75 557 single_test_result, detailed_test_results = handle_result
bryantaylor 0:eafc3fd41f75 558 else:
bryantaylor 0:eafc3fd41f75 559 continue
bryantaylor 0:eafc3fd41f75 560
bryantaylor 0:eafc3fd41f75 561 # Append test results to global test summary
bryantaylor 0:eafc3fd41f75 562 if single_test_result is not None:
bryantaylor 0:eafc3fd41f75 563 self.test_summary.append(single_test_result)
bryantaylor 0:eafc3fd41f75 564
bryantaylor 0:eafc3fd41f75 565 # Add detailed test result to test summary structure
bryantaylor 0:eafc3fd41f75 566 if target not in self.test_summary_ext[target][toolchain]:
bryantaylor 0:eafc3fd41f75 567 if test_id not in self.test_summary_ext[target][toolchain]:
bryantaylor 0:eafc3fd41f75 568 self.test_summary_ext[target][toolchain][test_id] = []
bryantaylor 0:eafc3fd41f75 569
bryantaylor 0:eafc3fd41f75 570 append_test_result = detailed_test_results
bryantaylor 0:eafc3fd41f75 571
bryantaylor 0:eafc3fd41f75 572 # If waterfall and consolidate-waterfall options are enabled,
bryantaylor 0:eafc3fd41f75 573 # only include the last test result in the report.
bryantaylor 0:eafc3fd41f75 574 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
bryantaylor 0:eafc3fd41f75 575 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
bryantaylor 0:eafc3fd41f75 576
bryantaylor 0:eafc3fd41f75 577 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
bryantaylor 0:eafc3fd41f75 578
bryantaylor 0:eafc3fd41f75 579 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
bryantaylor 0:eafc3fd41f75 580 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
bryantaylor 0:eafc3fd41f75 581
bryantaylor 0:eafc3fd41f75 582 q.put(target + '_'.join(toolchains))
bryantaylor 0:eafc3fd41f75 583 return
bryantaylor 0:eafc3fd41f75 584
bryantaylor 0:eafc3fd41f75 585 def execute(self):
bryantaylor 0:eafc3fd41f75 586 clean = self.test_spec.get('clean', False)
bryantaylor 0:eafc3fd41f75 587 test_ids = self.test_spec.get('test_ids', [])
bryantaylor 0:eafc3fd41f75 588 q = Queue()
bryantaylor 0:eafc3fd41f75 589
bryantaylor 0:eafc3fd41f75 590 # Generate seed for shuffle if seed is not provided in
bryantaylor 0:eafc3fd41f75 591 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
bryantaylor 0:eafc3fd41f75 592 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
bryantaylor 0:eafc3fd41f75 593 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
bryantaylor 0:eafc3fd41f75 594
bryantaylor 0:eafc3fd41f75 595
bryantaylor 0:eafc3fd41f75 596 if self.opts_parallel_test_exec:
bryantaylor 0:eafc3fd41f75 597 ###################################################################
bryantaylor 0:eafc3fd41f75 598 # Experimental, parallel test execution per singletest instance.
bryantaylor 0:eafc3fd41f75 599 ###################################################################
bryantaylor 0:eafc3fd41f75 600 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
bryantaylor 0:eafc3fd41f75 601 # Note: We are building here in parallel for each target separately!
bryantaylor 0:eafc3fd41f75 602 # So we are not building the same thing multiple times and compilers
bryantaylor 0:eafc3fd41f75 603 # in separate threads do not collide.
bryantaylor 0:eafc3fd41f75 604 # Inside execute_thread_slice() function function handle() will be called to
bryantaylor 0:eafc3fd41f75 605 # get information about available MUTs (per target).
bryantaylor 0:eafc3fd41f75 606 for target, toolchains in self.test_spec['targets'].iteritems():
bryantaylor 0:eafc3fd41f75 607 self.test_suite_properties_ext[target] = {}
bryantaylor 0:eafc3fd41f75 608 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
bryantaylor 0:eafc3fd41f75 609 t.daemon = True
bryantaylor 0:eafc3fd41f75 610 t.start()
bryantaylor 0:eafc3fd41f75 611 execute_threads.append(t)
bryantaylor 0:eafc3fd41f75 612
bryantaylor 0:eafc3fd41f75 613 for t in execute_threads:
bryantaylor 0:eafc3fd41f75 614 q.get() # t.join() would block some threads because we should not wait in any order for thread end
bryantaylor 0:eafc3fd41f75 615 else:
bryantaylor 0:eafc3fd41f75 616 # Serialized (not parallel) test execution
bryantaylor 0:eafc3fd41f75 617 for target, toolchains in self.test_spec['targets'].iteritems():
bryantaylor 0:eafc3fd41f75 618 if target not in self.test_suite_properties_ext:
bryantaylor 0:eafc3fd41f75 619 self.test_suite_properties_ext[target] = {}
bryantaylor 0:eafc3fd41f75 620
bryantaylor 0:eafc3fd41f75 621 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
bryantaylor 0:eafc3fd41f75 622 q.get()
bryantaylor 0:eafc3fd41f75 623
bryantaylor 0:eafc3fd41f75 624 if self.db_logger:
bryantaylor 0:eafc3fd41f75 625 self.db_logger.reconnect();
bryantaylor 0:eafc3fd41f75 626 if self.db_logger.is_connected():
bryantaylor 0:eafc3fd41f75 627 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
bryantaylor 0:eafc3fd41f75 628 self.db_logger.disconnect();
bryantaylor 0:eafc3fd41f75 629
bryantaylor 0:eafc3fd41f75 630 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
bryantaylor 0:eafc3fd41f75 631
bryantaylor 0:eafc3fd41f75 632 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
bryantaylor 0:eafc3fd41f75 633 valid_test_map_keys = []
bryantaylor 0:eafc3fd41f75 634
bryantaylor 0:eafc3fd41f75 635 for test_id in test_map_keys:
bryantaylor 0:eafc3fd41f75 636 test = TEST_MAP[test_id]
bryantaylor 0:eafc3fd41f75 637 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
bryantaylor 0:eafc3fd41f75 638 continue
bryantaylor 0:eafc3fd41f75 639
bryantaylor 0:eafc3fd41f75 640 if test_ids and test_id not in test_ids:
bryantaylor 0:eafc3fd41f75 641 continue
bryantaylor 0:eafc3fd41f75 642
bryantaylor 0:eafc3fd41f75 643 if self.opts_test_only_peripheral and not test.peripherals:
bryantaylor 0:eafc3fd41f75 644 if self.opts_verbose_skipped_tests:
bryantaylor 0:eafc3fd41f75 645 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
bryantaylor 0:eafc3fd41f75 646 continue
bryantaylor 0:eafc3fd41f75 647
bryantaylor 0:eafc3fd41f75 648 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
bryantaylor 0:eafc3fd41f75 649 # We will skip tests not forced with -p option
bryantaylor 0:eafc3fd41f75 650 if self.opts_verbose_skipped_tests:
bryantaylor 0:eafc3fd41f75 651 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
bryantaylor 0:eafc3fd41f75 652 continue
bryantaylor 0:eafc3fd41f75 653
bryantaylor 0:eafc3fd41f75 654 if self.opts_test_only_common and test.peripherals:
bryantaylor 0:eafc3fd41f75 655 if self.opts_verbose_skipped_tests:
bryantaylor 0:eafc3fd41f75 656 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
bryantaylor 0:eafc3fd41f75 657 continue
bryantaylor 0:eafc3fd41f75 658
bryantaylor 0:eafc3fd41f75 659 if not include_non_automated and not test.automated:
bryantaylor 0:eafc3fd41f75 660 if self.opts_verbose_skipped_tests:
bryantaylor 0:eafc3fd41f75 661 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
bryantaylor 0:eafc3fd41f75 662 continue
bryantaylor 0:eafc3fd41f75 663
bryantaylor 0:eafc3fd41f75 664 if test.is_supported(target, toolchain):
bryantaylor 0:eafc3fd41f75 665 if test.peripherals is None and self.opts_only_build_tests:
bryantaylor 0:eafc3fd41f75 666 # When users are using 'build only flag' and test do not have
bryantaylor 0:eafc3fd41f75 667 # specified peripherals we can allow test building by default
bryantaylor 0:eafc3fd41f75 668 pass
bryantaylor 0:eafc3fd41f75 669 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
bryantaylor 0:eafc3fd41f75 670 # If we force peripheral with option -p we expect test
bryantaylor 0:eafc3fd41f75 671 # to pass even if peripheral is not in MUTs file.
bryantaylor 0:eafc3fd41f75 672 pass
bryantaylor 0:eafc3fd41f75 673 elif not self.is_peripherals_available(target, test.peripherals):
bryantaylor 0:eafc3fd41f75 674 if self.opts_verbose_skipped_tests:
bryantaylor 0:eafc3fd41f75 675 if test.peripherals:
bryantaylor 0:eafc3fd41f75 676 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
bryantaylor 0:eafc3fd41f75 677 else:
bryantaylor 0:eafc3fd41f75 678 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
bryantaylor 0:eafc3fd41f75 679 continue
bryantaylor 0:eafc3fd41f75 680
bryantaylor 0:eafc3fd41f75 681 # The test has made it through all the filters, so add it to the valid tests list
bryantaylor 0:eafc3fd41f75 682 valid_test_map_keys.append(test_id)
bryantaylor 0:eafc3fd41f75 683
bryantaylor 0:eafc3fd41f75 684 return valid_test_map_keys
bryantaylor 0:eafc3fd41f75 685
bryantaylor 0:eafc3fd41f75 686 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
bryantaylor 0:eafc3fd41f75 687 # NOTE: This will not preserve order
bryantaylor 0:eafc3fd41f75 688 return list(set(all_test_map_keys) - set(valid_test_map_keys))
bryantaylor 0:eafc3fd41f75 689
bryantaylor 0:eafc3fd41f75 690 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
bryantaylor 0:eafc3fd41f75 691 """ Prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 692 table shows text x toolchain test result matrix
bryantaylor 0:eafc3fd41f75 693 """
bryantaylor 0:eafc3fd41f75 694 RESULT_INDEX = 0
bryantaylor 0:eafc3fd41f75 695 TARGET_INDEX = 1
bryantaylor 0:eafc3fd41f75 696 TOOLCHAIN_INDEX = 2
bryantaylor 0:eafc3fd41f75 697 TEST_INDEX = 3
bryantaylor 0:eafc3fd41f75 698 DESC_INDEX = 4
bryantaylor 0:eafc3fd41f75 699
bryantaylor 0:eafc3fd41f75 700 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
bryantaylor 0:eafc3fd41f75 701 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
bryantaylor 0:eafc3fd41f75 702 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
bryantaylor 0:eafc3fd41f75 703 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
bryantaylor 0:eafc3fd41f75 704
bryantaylor 0:eafc3fd41f75 705 result = "Test summary:\n"
bryantaylor 0:eafc3fd41f75 706 for target in unique_targets:
bryantaylor 0:eafc3fd41f75 707 result_dict = {} # test : { toolchain : result }
bryantaylor 0:eafc3fd41f75 708 unique_target_toolchains = []
bryantaylor 0:eafc3fd41f75 709 for test in test_summary:
bryantaylor 0:eafc3fd41f75 710 if test[TARGET_INDEX] == target:
bryantaylor 0:eafc3fd41f75 711 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
bryantaylor 0:eafc3fd41f75 712 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
bryantaylor 0:eafc3fd41f75 713 if test[TEST_INDEX] not in result_dict:
bryantaylor 0:eafc3fd41f75 714 result_dict[test[TEST_INDEX]] = {}
bryantaylor 0:eafc3fd41f75 715 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
bryantaylor 0:eafc3fd41f75 716
bryantaylor 0:eafc3fd41f75 717 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
bryantaylor 0:eafc3fd41f75 718 pt = PrettyTable(pt_cols)
bryantaylor 0:eafc3fd41f75 719 for col in pt_cols:
bryantaylor 0:eafc3fd41f75 720 pt.align[col] = "l"
bryantaylor 0:eafc3fd41f75 721 pt.padding_width = 1 # One space between column edges and contents (default)
bryantaylor 0:eafc3fd41f75 722
bryantaylor 0:eafc3fd41f75 723 for test in unique_tests:
bryantaylor 0:eafc3fd41f75 724 if test in result_dict:
bryantaylor 0:eafc3fd41f75 725 test_results = result_dict[test]
bryantaylor 0:eafc3fd41f75 726 if test in unique_test_desc:
bryantaylor 0:eafc3fd41f75 727 row = [target, test, unique_test_desc[test]]
bryantaylor 0:eafc3fd41f75 728 for toolchain in unique_toolchains:
bryantaylor 0:eafc3fd41f75 729 if toolchain in test_results:
bryantaylor 0:eafc3fd41f75 730 row.append(test_results[toolchain])
bryantaylor 0:eafc3fd41f75 731 pt.add_row(row)
bryantaylor 0:eafc3fd41f75 732 result += pt.get_string()
bryantaylor 0:eafc3fd41f75 733 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
bryantaylor 0:eafc3fd41f75 734 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
bryantaylor 0:eafc3fd41f75 735 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
bryantaylor 0:eafc3fd41f75 736 return result
bryantaylor 0:eafc3fd41f75 737
bryantaylor 0:eafc3fd41f75 738 def generate_test_summary(self, test_summary, shuffle_seed=None):
bryantaylor 0:eafc3fd41f75 739 """ Prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 740 table shows target x test results matrix across
bryantaylor 0:eafc3fd41f75 741 """
bryantaylor 0:eafc3fd41f75 742 success_code = 0 # Success code that can be leter returned to
bryantaylor 0:eafc3fd41f75 743 result = "Test summary:\n"
bryantaylor 0:eafc3fd41f75 744 # Pretty table package is used to print results
bryantaylor 0:eafc3fd41f75 745 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
bryantaylor 0:eafc3fd41f75 746 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
bryantaylor 0:eafc3fd41f75 747 pt.align["Result"] = "l" # Left align
bryantaylor 0:eafc3fd41f75 748 pt.align["Target"] = "l" # Left align
bryantaylor 0:eafc3fd41f75 749 pt.align["Toolchain"] = "l" # Left align
bryantaylor 0:eafc3fd41f75 750 pt.align["Test ID"] = "l" # Left align
bryantaylor 0:eafc3fd41f75 751 pt.align["Test Description"] = "l" # Left align
bryantaylor 0:eafc3fd41f75 752 pt.padding_width = 1 # One space between column edges and contents (default)
bryantaylor 0:eafc3fd41f75 753
bryantaylor 0:eafc3fd41f75 754 result_dict = {self.TEST_RESULT_OK : 0,
bryantaylor 0:eafc3fd41f75 755 self.TEST_RESULT_FAIL : 0,
bryantaylor 0:eafc3fd41f75 756 self.TEST_RESULT_ERROR : 0,
bryantaylor 0:eafc3fd41f75 757 self.TEST_RESULT_UNDEF : 0,
bryantaylor 0:eafc3fd41f75 758 self.TEST_RESULT_IOERR_COPY : 0,
bryantaylor 0:eafc3fd41f75 759 self.TEST_RESULT_IOERR_DISK : 0,
bryantaylor 0:eafc3fd41f75 760 self.TEST_RESULT_IOERR_SERIAL : 0,
bryantaylor 0:eafc3fd41f75 761 self.TEST_RESULT_NO_IMAGE : 0,
bryantaylor 0:eafc3fd41f75 762 self.TEST_RESULT_TIMEOUT : 0,
bryantaylor 0:eafc3fd41f75 763 self.TEST_RESULT_MBED_ASSERT : 0,
bryantaylor 0:eafc3fd41f75 764 self.TEST_RESULT_BUILD_FAILED : 0,
bryantaylor 0:eafc3fd41f75 765 self.TEST_RESULT_NOT_SUPPORTED : 0
bryantaylor 0:eafc3fd41f75 766 }
bryantaylor 0:eafc3fd41f75 767
bryantaylor 0:eafc3fd41f75 768 for test in test_summary:
bryantaylor 0:eafc3fd41f75 769 if test[0] in result_dict:
bryantaylor 0:eafc3fd41f75 770 result_dict[test[0]] += 1
bryantaylor 0:eafc3fd41f75 771 pt.add_row(test)
bryantaylor 0:eafc3fd41f75 772 result += pt.get_string()
bryantaylor 0:eafc3fd41f75 773 result += "\n"
bryantaylor 0:eafc3fd41f75 774
bryantaylor 0:eafc3fd41f75 775 # Print result count
bryantaylor 0:eafc3fd41f75 776 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
bryantaylor 0:eafc3fd41f75 777 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
bryantaylor 0:eafc3fd41f75 778 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
bryantaylor 0:eafc3fd41f75 779 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
bryantaylor 0:eafc3fd41f75 780 return result
bryantaylor 0:eafc3fd41f75 781
bryantaylor 0:eafc3fd41f75 782 def test_loop_list_to_dict(self, test_loops_str):
bryantaylor 0:eafc3fd41f75 783 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
bryantaylor 0:eafc3fd41f75 784 """
bryantaylor 0:eafc3fd41f75 785 result = {}
bryantaylor 0:eafc3fd41f75 786 if test_loops_str:
bryantaylor 0:eafc3fd41f75 787 test_loops = test_loops_str
bryantaylor 0:eafc3fd41f75 788 for test_loop in test_loops:
bryantaylor 0:eafc3fd41f75 789 test_loop_count = test_loop.split('=')
bryantaylor 0:eafc3fd41f75 790 if len(test_loop_count) == 2:
bryantaylor 0:eafc3fd41f75 791 _test_id, _test_loops = test_loop_count
bryantaylor 0:eafc3fd41f75 792 try:
bryantaylor 0:eafc3fd41f75 793 _test_loops = int(_test_loops)
bryantaylor 0:eafc3fd41f75 794 except:
bryantaylor 0:eafc3fd41f75 795 continue
bryantaylor 0:eafc3fd41f75 796 result[_test_id] = _test_loops
bryantaylor 0:eafc3fd41f75 797 return result
bryantaylor 0:eafc3fd41f75 798
bryantaylor 0:eafc3fd41f75 799 def get_test_loop_count(self, test_id):
bryantaylor 0:eafc3fd41f75 800 """ This function returns no. of loops per test (deducted by test_id_.
bryantaylor 0:eafc3fd41f75 801 If test is not in list of redefined loop counts it will use default value.
bryantaylor 0:eafc3fd41f75 802 """
bryantaylor 0:eafc3fd41f75 803 result = self.GLOBAL_LOOPS_COUNT
bryantaylor 0:eafc3fd41f75 804 if test_id in self.TEST_LOOPS_DICT:
bryantaylor 0:eafc3fd41f75 805 result = self.TEST_LOOPS_DICT[test_id]
bryantaylor 0:eafc3fd41f75 806 return result
bryantaylor 0:eafc3fd41f75 807
bryantaylor 0:eafc3fd41f75 808 def delete_file(self, file_path):
bryantaylor 0:eafc3fd41f75 809 """ Remove file from the system
bryantaylor 0:eafc3fd41f75 810 """
bryantaylor 0:eafc3fd41f75 811 result = True
bryantaylor 0:eafc3fd41f75 812 resutl_msg = ""
bryantaylor 0:eafc3fd41f75 813 try:
bryantaylor 0:eafc3fd41f75 814 os.remove(file_path)
bryantaylor 0:eafc3fd41f75 815 except Exception, e:
bryantaylor 0:eafc3fd41f75 816 resutl_msg = e
bryantaylor 0:eafc3fd41f75 817 result = False
bryantaylor 0:eafc3fd41f75 818 return result, resutl_msg
bryantaylor 0:eafc3fd41f75 819
bryantaylor 0:eafc3fd41f75 820 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
bryantaylor 0:eafc3fd41f75 821 """ Test is being invoked for given MUT.
bryantaylor 0:eafc3fd41f75 822 """
bryantaylor 0:eafc3fd41f75 823 # Get test information, image and test timeout
bryantaylor 0:eafc3fd41f75 824 test_id = data['test_id']
bryantaylor 0:eafc3fd41f75 825 test = TEST_MAP[test_id]
bryantaylor 0:eafc3fd41f75 826 test_description = TEST_MAP[test_id].get_description()
bryantaylor 0:eafc3fd41f75 827 image = data["image"]
bryantaylor 0:eafc3fd41f75 828 duration = data.get("duration", 10)
bryantaylor 0:eafc3fd41f75 829
bryantaylor 0:eafc3fd41f75 830 if mut is None:
bryantaylor 0:eafc3fd41f75 831 print "Error: No Mbed available: MUT[%s]" % data['mcu']
bryantaylor 0:eafc3fd41f75 832 return None
bryantaylor 0:eafc3fd41f75 833
bryantaylor 0:eafc3fd41f75 834 mcu = mut['mcu']
bryantaylor 0:eafc3fd41f75 835 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
bryantaylor 0:eafc3fd41f75 836
bryantaylor 0:eafc3fd41f75 837 if self.db_logger:
bryantaylor 0:eafc3fd41f75 838 self.db_logger.reconnect()
bryantaylor 0:eafc3fd41f75 839
bryantaylor 0:eafc3fd41f75 840 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
bryantaylor 0:eafc3fd41f75 841
bryantaylor 0:eafc3fd41f75 842 # Tests can be looped so test results must be stored for the same test
bryantaylor 0:eafc3fd41f75 843 test_all_result = []
bryantaylor 0:eafc3fd41f75 844 # Test results for one test ran few times
bryantaylor 0:eafc3fd41f75 845 detailed_test_results = {} # { Loop_number: { results ... } }
bryantaylor 0:eafc3fd41f75 846
bryantaylor 0:eafc3fd41f75 847 for test_index in range(test_loops):
bryantaylor 0:eafc3fd41f75 848
bryantaylor 0:eafc3fd41f75 849 # If mbedls is available and we are auto detecting MUT info,
bryantaylor 0:eafc3fd41f75 850 # update MUT info (mounting may changed)
bryantaylor 0:eafc3fd41f75 851 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
bryantaylor 0:eafc3fd41f75 852 platform_name_filter = [mcu]
bryantaylor 0:eafc3fd41f75 853 muts_list = {}
bryantaylor 0:eafc3fd41f75 854 found = False
bryantaylor 0:eafc3fd41f75 855
bryantaylor 0:eafc3fd41f75 856 for i in range(0, 60):
bryantaylor 0:eafc3fd41f75 857 print('Looking for %s with MBEDLS' % mcu)
bryantaylor 0:eafc3fd41f75 858 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
bryantaylor 0:eafc3fd41f75 859
bryantaylor 0:eafc3fd41f75 860 if 1 not in muts_list:
bryantaylor 0:eafc3fd41f75 861 sleep(3)
bryantaylor 0:eafc3fd41f75 862 else:
bryantaylor 0:eafc3fd41f75 863 found = True
bryantaylor 0:eafc3fd41f75 864 break
bryantaylor 0:eafc3fd41f75 865
bryantaylor 0:eafc3fd41f75 866 if not found:
bryantaylor 0:eafc3fd41f75 867 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
bryantaylor 0:eafc3fd41f75 868 return None
bryantaylor 0:eafc3fd41f75 869 else:
bryantaylor 0:eafc3fd41f75 870 mut = muts_list[1]
bryantaylor 0:eafc3fd41f75 871
bryantaylor 0:eafc3fd41f75 872 disk = mut.get('disk')
bryantaylor 0:eafc3fd41f75 873 port = mut.get('port')
bryantaylor 0:eafc3fd41f75 874
bryantaylor 0:eafc3fd41f75 875 if disk is None or port is None:
bryantaylor 0:eafc3fd41f75 876 return None
bryantaylor 0:eafc3fd41f75 877
bryantaylor 0:eafc3fd41f75 878 target_by_mcu = TARGET_MAP[mut['mcu']]
bryantaylor 0:eafc3fd41f75 879 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
bryantaylor 0:eafc3fd41f75 880 # Some extra stuff can be declared in MUTs structure
bryantaylor 0:eafc3fd41f75 881 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
bryantaylor 0:eafc3fd41f75 882 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
bryantaylor 0:eafc3fd41f75 883
bryantaylor 0:eafc3fd41f75 884 # When the build and test system were separate, this was relative to a
bryantaylor 0:eafc3fd41f75 885 # base network folder base path: join(NETWORK_BASE_PATH, )
bryantaylor 0:eafc3fd41f75 886 image_path = image
bryantaylor 0:eafc3fd41f75 887
bryantaylor 0:eafc3fd41f75 888 # Host test execution
bryantaylor 0:eafc3fd41f75 889 start_host_exec_time = time()
bryantaylor 0:eafc3fd41f75 890
bryantaylor 0:eafc3fd41f75 891 single_test_result = self.TEST_RESULT_UNDEF # single test run result
bryantaylor 0:eafc3fd41f75 892 _copy_method = selected_copy_method
bryantaylor 0:eafc3fd41f75 893
bryantaylor 0:eafc3fd41f75 894 if not exists(image_path):
bryantaylor 0:eafc3fd41f75 895 single_test_result = self.TEST_RESULT_NO_IMAGE
bryantaylor 0:eafc3fd41f75 896 elapsed_time = 0
bryantaylor 0:eafc3fd41f75 897 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
bryantaylor 0:eafc3fd41f75 898 print single_test_output
bryantaylor 0:eafc3fd41f75 899 else:
bryantaylor 0:eafc3fd41f75 900 # Host test execution
bryantaylor 0:eafc3fd41f75 901 start_host_exec_time = time()
bryantaylor 0:eafc3fd41f75 902
bryantaylor 0:eafc3fd41f75 903 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
bryantaylor 0:eafc3fd41f75 904 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
bryantaylor 0:eafc3fd41f75 905 host_test_result = self.run_host_test(test.host_test,
bryantaylor 0:eafc3fd41f75 906 image_path, disk, port, duration,
bryantaylor 0:eafc3fd41f75 907 micro=target_name,
bryantaylor 0:eafc3fd41f75 908 verbose=host_test_verbose,
bryantaylor 0:eafc3fd41f75 909 reset=host_test_reset,
bryantaylor 0:eafc3fd41f75 910 reset_tout=reset_tout,
bryantaylor 0:eafc3fd41f75 911 copy_method=selected_copy_method,
bryantaylor 0:eafc3fd41f75 912 program_cycle_s=target_by_mcu.program_cycle_s)
bryantaylor 0:eafc3fd41f75 913 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
bryantaylor 0:eafc3fd41f75 914
bryantaylor 0:eafc3fd41f75 915 # Store test result
bryantaylor 0:eafc3fd41f75 916 test_all_result.append(single_test_result)
bryantaylor 0:eafc3fd41f75 917 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
bryantaylor 0:eafc3fd41f75 918 elapsed_time = single_testduration # TIme of single test case execution after reset
bryantaylor 0:eafc3fd41f75 919
bryantaylor 0:eafc3fd41f75 920 detailed_test_results[test_index] = {
bryantaylor 0:eafc3fd41f75 921 'result' : single_test_result,
bryantaylor 0:eafc3fd41f75 922 'output' : single_test_output,
bryantaylor 0:eafc3fd41f75 923 'target_name' : target_name,
bryantaylor 0:eafc3fd41f75 924 'target_name_unique' : target_name_unique,
bryantaylor 0:eafc3fd41f75 925 'toolchain_name' : toolchain_name,
bryantaylor 0:eafc3fd41f75 926 'id' : test_id,
bryantaylor 0:eafc3fd41f75 927 'description' : test_description,
bryantaylor 0:eafc3fd41f75 928 'elapsed_time' : round(elapsed_time, 2),
bryantaylor 0:eafc3fd41f75 929 'duration' : single_timeout,
bryantaylor 0:eafc3fd41f75 930 'copy_method' : _copy_method,
bryantaylor 0:eafc3fd41f75 931 }
bryantaylor 0:eafc3fd41f75 932
bryantaylor 0:eafc3fd41f75 933 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
bryantaylor 0:eafc3fd41f75 934 test_id, test_description, elapsed_time, single_timeout)
bryantaylor 0:eafc3fd41f75 935
bryantaylor 0:eafc3fd41f75 936 # Update database entries for ongoing test
bryantaylor 0:eafc3fd41f75 937 if self.db_logger and self.db_logger.is_connected():
bryantaylor 0:eafc3fd41f75 938 test_type = 'SingleTest'
bryantaylor 0:eafc3fd41f75 939 self.db_logger.insert_test_entry(self.db_logger_build_id,
bryantaylor 0:eafc3fd41f75 940 target_name,
bryantaylor 0:eafc3fd41f75 941 toolchain_name,
bryantaylor 0:eafc3fd41f75 942 test_type,
bryantaylor 0:eafc3fd41f75 943 test_id,
bryantaylor 0:eafc3fd41f75 944 single_test_result,
bryantaylor 0:eafc3fd41f75 945 single_test_output,
bryantaylor 0:eafc3fd41f75 946 elapsed_time,
bryantaylor 0:eafc3fd41f75 947 single_timeout,
bryantaylor 0:eafc3fd41f75 948 test_index)
bryantaylor 0:eafc3fd41f75 949
bryantaylor 0:eafc3fd41f75 950 # If we perform waterfall test we test until we get OK and we stop testing
bryantaylor 0:eafc3fd41f75 951 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
bryantaylor 0:eafc3fd41f75 952 break
bryantaylor 0:eafc3fd41f75 953
bryantaylor 0:eafc3fd41f75 954 if self.db_logger:
bryantaylor 0:eafc3fd41f75 955 self.db_logger.disconnect()
bryantaylor 0:eafc3fd41f75 956
bryantaylor 0:eafc3fd41f75 957 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
bryantaylor 0:eafc3fd41f75 958 target_name_unique,
bryantaylor 0:eafc3fd41f75 959 toolchain_name,
bryantaylor 0:eafc3fd41f75 960 test_id,
bryantaylor 0:eafc3fd41f75 961 test_description,
bryantaylor 0:eafc3fd41f75 962 round(elapsed_time, 2),
bryantaylor 0:eafc3fd41f75 963 single_timeout,
bryantaylor 0:eafc3fd41f75 964 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
bryantaylor 0:eafc3fd41f75 965
bryantaylor 0:eafc3fd41f75 966 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
bryantaylor 0:eafc3fd41f75 967 """ Function determines MUT's mbed disk/port and copies binary to
bryantaylor 0:eafc3fd41f75 968 target.
bryantaylor 0:eafc3fd41f75 969 """
bryantaylor 0:eafc3fd41f75 970 handle_results = []
bryantaylor 0:eafc3fd41f75 971 data = json.loads(test_spec)
bryantaylor 0:eafc3fd41f75 972
bryantaylor 0:eafc3fd41f75 973 # Find a suitable MUT:
bryantaylor 0:eafc3fd41f75 974 mut = None
bryantaylor 0:eafc3fd41f75 975 for id, m in self.muts.iteritems():
bryantaylor 0:eafc3fd41f75 976 if m['mcu'] == data['mcu']:
bryantaylor 0:eafc3fd41f75 977 mut = m
bryantaylor 0:eafc3fd41f75 978 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
bryantaylor 0:eafc3fd41f75 979 handle_results.append(handle_result)
bryantaylor 0:eafc3fd41f75 980
bryantaylor 0:eafc3fd41f75 981 return handle_results
bryantaylor 0:eafc3fd41f75 982
bryantaylor 0:eafc3fd41f75 983 def print_test_result(self, test_result, target_name, toolchain_name,
bryantaylor 0:eafc3fd41f75 984 test_id, test_description, elapsed_time, duration):
bryantaylor 0:eafc3fd41f75 985 """ Use specific convention to print test result and related data
bryantaylor 0:eafc3fd41f75 986 """
bryantaylor 0:eafc3fd41f75 987 tokens = []
bryantaylor 0:eafc3fd41f75 988 tokens.append("TargetTest")
bryantaylor 0:eafc3fd41f75 989 tokens.append(target_name)
bryantaylor 0:eafc3fd41f75 990 tokens.append(toolchain_name)
bryantaylor 0:eafc3fd41f75 991 tokens.append(test_id)
bryantaylor 0:eafc3fd41f75 992 tokens.append(test_description)
bryantaylor 0:eafc3fd41f75 993 separator = "::"
bryantaylor 0:eafc3fd41f75 994 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
bryantaylor 0:eafc3fd41f75 995 result = separator.join(tokens) + " [" + test_result +"]" + time_info
bryantaylor 0:eafc3fd41f75 996 return Fore.MAGENTA + result + Fore.RESET
bryantaylor 0:eafc3fd41f75 997
bryantaylor 0:eafc3fd41f75 998 def shape_test_loop_ok_result_count(self, test_all_result):
bryantaylor 0:eafc3fd41f75 999 """ Reformats list of results to simple string
bryantaylor 0:eafc3fd41f75 1000 """
bryantaylor 0:eafc3fd41f75 1001 test_loop_count = len(test_all_result)
bryantaylor 0:eafc3fd41f75 1002 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
bryantaylor 0:eafc3fd41f75 1003 return "%d/%d"% (test_loop_ok_result, test_loop_count)
bryantaylor 0:eafc3fd41f75 1004
bryantaylor 0:eafc3fd41f75 1005 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
bryantaylor 0:eafc3fd41f75 1006 """ Reformats list of results to simple string
bryantaylor 0:eafc3fd41f75 1007 """
bryantaylor 0:eafc3fd41f75 1008 result = self.TEST_RESULT_FAIL
bryantaylor 0:eafc3fd41f75 1009
bryantaylor 0:eafc3fd41f75 1010 if all(test_all_result[0] == res for res in test_all_result):
bryantaylor 0:eafc3fd41f75 1011 result = test_all_result[0]
bryantaylor 0:eafc3fd41f75 1012 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
bryantaylor 0:eafc3fd41f75 1013 result = self.TEST_RESULT_OK
bryantaylor 0:eafc3fd41f75 1014
bryantaylor 0:eafc3fd41f75 1015 return result
bryantaylor 0:eafc3fd41f75 1016
bryantaylor 0:eafc3fd41f75 1017 def run_host_test(self, name, image_path, disk, port, duration,
bryantaylor 0:eafc3fd41f75 1018 micro=None, reset=None, reset_tout=None,
bryantaylor 0:eafc3fd41f75 1019 verbose=False, copy_method=None, program_cycle_s=None):
bryantaylor 0:eafc3fd41f75 1020 """ Function creates new process with host test configured with particular test case.
bryantaylor 0:eafc3fd41f75 1021 Function also is pooling for serial port activity from process to catch all data
bryantaylor 0:eafc3fd41f75 1022 printed by test runner and host test during test execution
bryantaylor 0:eafc3fd41f75 1023 """
bryantaylor 0:eafc3fd41f75 1024
bryantaylor 0:eafc3fd41f75 1025 def get_char_from_queue(obs):
bryantaylor 0:eafc3fd41f75 1026 """ Get character from queue safe way
bryantaylor 0:eafc3fd41f75 1027 """
bryantaylor 0:eafc3fd41f75 1028 try:
bryantaylor 0:eafc3fd41f75 1029 c = obs.queue.get(block=True, timeout=0.5)
bryantaylor 0:eafc3fd41f75 1030 except Empty, _:
bryantaylor 0:eafc3fd41f75 1031 c = None
bryantaylor 0:eafc3fd41f75 1032 return c
bryantaylor 0:eafc3fd41f75 1033
bryantaylor 0:eafc3fd41f75 1034 def filter_queue_char(c):
bryantaylor 0:eafc3fd41f75 1035 """ Filters out non ASCII characters from serial port
bryantaylor 0:eafc3fd41f75 1036 """
bryantaylor 0:eafc3fd41f75 1037 if ord(c) not in range(128):
bryantaylor 0:eafc3fd41f75 1038 c = ' '
bryantaylor 0:eafc3fd41f75 1039 return c
bryantaylor 0:eafc3fd41f75 1040
bryantaylor 0:eafc3fd41f75 1041 def get_test_result(output):
bryantaylor 0:eafc3fd41f75 1042 """ Parse test 'output' data
bryantaylor 0:eafc3fd41f75 1043 """
bryantaylor 0:eafc3fd41f75 1044 result = self.TEST_RESULT_TIMEOUT
bryantaylor 0:eafc3fd41f75 1045 for line in "".join(output).splitlines():
bryantaylor 0:eafc3fd41f75 1046 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
bryantaylor 0:eafc3fd41f75 1047 if search_result and len(search_result.groups()):
bryantaylor 0:eafc3fd41f75 1048 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
bryantaylor 0:eafc3fd41f75 1049 break
bryantaylor 0:eafc3fd41f75 1050 return result
bryantaylor 0:eafc3fd41f75 1051
bryantaylor 0:eafc3fd41f75 1052 def get_auto_property_value(property_name, line):
bryantaylor 0:eafc3fd41f75 1053 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
bryantaylor 0:eafc3fd41f75 1054 Returns string
bryantaylor 0:eafc3fd41f75 1055 """
bryantaylor 0:eafc3fd41f75 1056 result = None
bryantaylor 0:eafc3fd41f75 1057 if re.search("HOST: Property '%s'"% property_name, line) is not None:
bryantaylor 0:eafc3fd41f75 1058 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
bryantaylor 0:eafc3fd41f75 1059 if property is not None and len(property.groups()) == 1:
bryantaylor 0:eafc3fd41f75 1060 result = property.groups()[0]
bryantaylor 0:eafc3fd41f75 1061 return result
bryantaylor 0:eafc3fd41f75 1062
bryantaylor 0:eafc3fd41f75 1063 # print "{%s} port:%s disk:%s" % (name, port, disk),
bryantaylor 0:eafc3fd41f75 1064 cmd = ["python",
bryantaylor 0:eafc3fd41f75 1065 '%s.py'% name,
bryantaylor 0:eafc3fd41f75 1066 '-d', disk,
bryantaylor 0:eafc3fd41f75 1067 '-f', '"%s"'% image_path,
bryantaylor 0:eafc3fd41f75 1068 '-p', port,
bryantaylor 0:eafc3fd41f75 1069 '-t', str(duration),
bryantaylor 0:eafc3fd41f75 1070 '-C', str(program_cycle_s)]
bryantaylor 0:eafc3fd41f75 1071
bryantaylor 0:eafc3fd41f75 1072 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
bryantaylor 0:eafc3fd41f75 1073 cmd += ['--auto']
bryantaylor 0:eafc3fd41f75 1074
bryantaylor 0:eafc3fd41f75 1075 # Add extra parameters to host_test
bryantaylor 0:eafc3fd41f75 1076 if copy_method is not None:
bryantaylor 0:eafc3fd41f75 1077 cmd += ["-c", copy_method]
bryantaylor 0:eafc3fd41f75 1078 if micro is not None:
bryantaylor 0:eafc3fd41f75 1079 cmd += ["-m", micro]
bryantaylor 0:eafc3fd41f75 1080 if reset is not None:
bryantaylor 0:eafc3fd41f75 1081 cmd += ["-r", reset]
bryantaylor 0:eafc3fd41f75 1082 if reset_tout is not None:
bryantaylor 0:eafc3fd41f75 1083 cmd += ["-R", str(reset_tout)]
bryantaylor 0:eafc3fd41f75 1084
bryantaylor 0:eafc3fd41f75 1085 if verbose:
bryantaylor 0:eafc3fd41f75 1086 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
bryantaylor 0:eafc3fd41f75 1087 print "Test::Output::Start"
bryantaylor 0:eafc3fd41f75 1088
bryantaylor 0:eafc3fd41f75 1089 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
bryantaylor 0:eafc3fd41f75 1090 obs = ProcessObserver(proc)
bryantaylor 0:eafc3fd41f75 1091 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
bryantaylor 0:eafc3fd41f75 1092 line = ''
bryantaylor 0:eafc3fd41f75 1093 output = []
bryantaylor 0:eafc3fd41f75 1094 start_time = time()
bryantaylor 0:eafc3fd41f75 1095 while (time() - start_time) < (2 * duration):
bryantaylor 0:eafc3fd41f75 1096 c = get_char_from_queue(obs)
bryantaylor 0:eafc3fd41f75 1097 if c:
bryantaylor 0:eafc3fd41f75 1098 if verbose:
bryantaylor 0:eafc3fd41f75 1099 sys.stdout.write(c)
bryantaylor 0:eafc3fd41f75 1100 c = filter_queue_char(c)
bryantaylor 0:eafc3fd41f75 1101 output.append(c)
bryantaylor 0:eafc3fd41f75 1102 # Give the mbed under test a way to communicate the end of the test
bryantaylor 0:eafc3fd41f75 1103 if c in ['\n', '\r']:
bryantaylor 0:eafc3fd41f75 1104
bryantaylor 0:eafc3fd41f75 1105 # Checking for auto-detection information from the test about MUT reset moment
bryantaylor 0:eafc3fd41f75 1106 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
bryantaylor 0:eafc3fd41f75 1107 # We will update this marker only once to prevent multiple time resets
bryantaylor 0:eafc3fd41f75 1108 update_once_flag['reset_target'] = True
bryantaylor 0:eafc3fd41f75 1109 start_time = time()
bryantaylor 0:eafc3fd41f75 1110
bryantaylor 0:eafc3fd41f75 1111 # Checking for auto-detection information from the test about timeout
bryantaylor 0:eafc3fd41f75 1112 auto_timeout_val = get_auto_property_value('timeout', line)
bryantaylor 0:eafc3fd41f75 1113 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
bryantaylor 0:eafc3fd41f75 1114 # We will update this marker only once to prevent multiple time resets
bryantaylor 0:eafc3fd41f75 1115 update_once_flag['timeout'] = True
bryantaylor 0:eafc3fd41f75 1116 duration = int(auto_timeout_val)
bryantaylor 0:eafc3fd41f75 1117
bryantaylor 0:eafc3fd41f75 1118 # Detect mbed assert:
bryantaylor 0:eafc3fd41f75 1119 if 'mbed assertation failed: ' in line:
bryantaylor 0:eafc3fd41f75 1120 output.append('{{mbed_assert}}')
bryantaylor 0:eafc3fd41f75 1121 break
bryantaylor 0:eafc3fd41f75 1122
bryantaylor 0:eafc3fd41f75 1123 # Check for test end
bryantaylor 0:eafc3fd41f75 1124 if '{end}' in line:
bryantaylor 0:eafc3fd41f75 1125 break
bryantaylor 0:eafc3fd41f75 1126 line = ''
bryantaylor 0:eafc3fd41f75 1127 else:
bryantaylor 0:eafc3fd41f75 1128 line += c
bryantaylor 0:eafc3fd41f75 1129 end_time = time()
bryantaylor 0:eafc3fd41f75 1130 testcase_duration = end_time - start_time # Test case duration from reset to {end}
bryantaylor 0:eafc3fd41f75 1131
bryantaylor 0:eafc3fd41f75 1132 c = get_char_from_queue(obs)
bryantaylor 0:eafc3fd41f75 1133
bryantaylor 0:eafc3fd41f75 1134 if c:
bryantaylor 0:eafc3fd41f75 1135 if verbose:
bryantaylor 0:eafc3fd41f75 1136 sys.stdout.write(c)
bryantaylor 0:eafc3fd41f75 1137 c = filter_queue_char(c)
bryantaylor 0:eafc3fd41f75 1138 output.append(c)
bryantaylor 0:eafc3fd41f75 1139
bryantaylor 0:eafc3fd41f75 1140 if verbose:
bryantaylor 0:eafc3fd41f75 1141 print "Test::Output::Finish"
bryantaylor 0:eafc3fd41f75 1142 # Stop test process
bryantaylor 0:eafc3fd41f75 1143 obs.stop()
bryantaylor 0:eafc3fd41f75 1144
bryantaylor 0:eafc3fd41f75 1145 result = get_test_result(output)
bryantaylor 0:eafc3fd41f75 1146 return (result, "".join(output), testcase_duration, duration)
bryantaylor 0:eafc3fd41f75 1147
bryantaylor 0:eafc3fd41f75 1148 def is_peripherals_available(self, target_mcu_name, peripherals=None):
bryantaylor 0:eafc3fd41f75 1149 """ Checks if specified target should run specific peripheral test case defined in MUTs file
bryantaylor 0:eafc3fd41f75 1150 """
bryantaylor 0:eafc3fd41f75 1151 if peripherals is not None:
bryantaylor 0:eafc3fd41f75 1152 peripherals = set(peripherals)
bryantaylor 0:eafc3fd41f75 1153 for id, mut in self.muts.iteritems():
bryantaylor 0:eafc3fd41f75 1154 # Target MCU name check
bryantaylor 0:eafc3fd41f75 1155 if mut["mcu"] != target_mcu_name:
bryantaylor 0:eafc3fd41f75 1156 continue
bryantaylor 0:eafc3fd41f75 1157 # Peripherals check
bryantaylor 0:eafc3fd41f75 1158 if peripherals is not None:
bryantaylor 0:eafc3fd41f75 1159 if 'peripherals' not in mut:
bryantaylor 0:eafc3fd41f75 1160 continue
bryantaylor 0:eafc3fd41f75 1161 if not peripherals.issubset(set(mut['peripherals'])):
bryantaylor 0:eafc3fd41f75 1162 continue
bryantaylor 0:eafc3fd41f75 1163 return True
bryantaylor 0:eafc3fd41f75 1164 return False
bryantaylor 0:eafc3fd41f75 1165
bryantaylor 0:eafc3fd41f75 1166 def shape_test_request(self, mcu, image_path, test_id, duration=10):
bryantaylor 0:eafc3fd41f75 1167 """ Function prepares JSON structure describing test specification
bryantaylor 0:eafc3fd41f75 1168 """
bryantaylor 0:eafc3fd41f75 1169 test_spec = {
bryantaylor 0:eafc3fd41f75 1170 "mcu": mcu,
bryantaylor 0:eafc3fd41f75 1171 "image": image_path,
bryantaylor 0:eafc3fd41f75 1172 "duration": duration,
bryantaylor 0:eafc3fd41f75 1173 "test_id": test_id,
bryantaylor 0:eafc3fd41f75 1174 }
bryantaylor 0:eafc3fd41f75 1175 return json.dumps(test_spec)
bryantaylor 0:eafc3fd41f75 1176
bryantaylor 0:eafc3fd41f75 1177
bryantaylor 0:eafc3fd41f75 1178 def get_unique_value_from_summary(test_summary, index):
bryantaylor 0:eafc3fd41f75 1179 """ Gets list of unique target names
bryantaylor 0:eafc3fd41f75 1180 """
bryantaylor 0:eafc3fd41f75 1181 result = []
bryantaylor 0:eafc3fd41f75 1182 for test in test_summary:
bryantaylor 0:eafc3fd41f75 1183 target_name = test[index]
bryantaylor 0:eafc3fd41f75 1184 if target_name not in result:
bryantaylor 0:eafc3fd41f75 1185 result.append(target_name)
bryantaylor 0:eafc3fd41f75 1186 return sorted(result)
bryantaylor 0:eafc3fd41f75 1187
bryantaylor 0:eafc3fd41f75 1188
bryantaylor 0:eafc3fd41f75 1189 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
bryantaylor 0:eafc3fd41f75 1190 """ Gets list of unique target names and return dictionary
bryantaylor 0:eafc3fd41f75 1191 """
bryantaylor 0:eafc3fd41f75 1192 result = {}
bryantaylor 0:eafc3fd41f75 1193 for test in test_summary:
bryantaylor 0:eafc3fd41f75 1194 key = test[index_key]
bryantaylor 0:eafc3fd41f75 1195 val = test[index_val]
bryantaylor 0:eafc3fd41f75 1196 if key not in result:
bryantaylor 0:eafc3fd41f75 1197 result[key] = val
bryantaylor 0:eafc3fd41f75 1198 return result
bryantaylor 0:eafc3fd41f75 1199
bryantaylor 0:eafc3fd41f75 1200
bryantaylor 0:eafc3fd41f75 1201 def show_json_file_format_error(json_spec_filename, line, column):
bryantaylor 0:eafc3fd41f75 1202 """ Prints JSON broken content
bryantaylor 0:eafc3fd41f75 1203 """
bryantaylor 0:eafc3fd41f75 1204 with open(json_spec_filename) as data_file:
bryantaylor 0:eafc3fd41f75 1205 line_no = 1
bryantaylor 0:eafc3fd41f75 1206 for json_line in data_file:
bryantaylor 0:eafc3fd41f75 1207 if line_no + 5 >= line: # Print last few lines before error
bryantaylor 0:eafc3fd41f75 1208 print 'Line %d:\t'%line_no + json_line, # Prints line
bryantaylor 0:eafc3fd41f75 1209 if line_no == line:
bryantaylor 0:eafc3fd41f75 1210 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
bryantaylor 0:eafc3fd41f75 1211 break
bryantaylor 0:eafc3fd41f75 1212 line_no += 1
bryantaylor 0:eafc3fd41f75 1213
bryantaylor 0:eafc3fd41f75 1214
bryantaylor 0:eafc3fd41f75 1215 def json_format_error_defect_pos(json_error_msg):
bryantaylor 0:eafc3fd41f75 1216 """ Gets first error line and column in JSON file format.
bryantaylor 0:eafc3fd41f75 1217 Parsed from exception thrown by json.loads() string
bryantaylor 0:eafc3fd41f75 1218 """
bryantaylor 0:eafc3fd41f75 1219 result = None
bryantaylor 0:eafc3fd41f75 1220 line, column = 0, 0
bryantaylor 0:eafc3fd41f75 1221 # Line value search
bryantaylor 0:eafc3fd41f75 1222 line_search = re.search('line [0-9]+', json_error_msg)
bryantaylor 0:eafc3fd41f75 1223 if line_search is not None:
bryantaylor 0:eafc3fd41f75 1224 ls = line_search.group().split(' ')
bryantaylor 0:eafc3fd41f75 1225 if len(ls) == 2:
bryantaylor 0:eafc3fd41f75 1226 line = int(ls[1])
bryantaylor 0:eafc3fd41f75 1227 # Column position search
bryantaylor 0:eafc3fd41f75 1228 column_search = re.search('column [0-9]+', json_error_msg)
bryantaylor 0:eafc3fd41f75 1229 if column_search is not None:
bryantaylor 0:eafc3fd41f75 1230 cs = column_search.group().split(' ')
bryantaylor 0:eafc3fd41f75 1231 if len(cs) == 2:
bryantaylor 0:eafc3fd41f75 1232 column = int(cs[1])
bryantaylor 0:eafc3fd41f75 1233 result = [line, column]
bryantaylor 0:eafc3fd41f75 1234 return result
bryantaylor 0:eafc3fd41f75 1235
bryantaylor 0:eafc3fd41f75 1236
bryantaylor 0:eafc3fd41f75 1237 def get_json_data_from_file(json_spec_filename, verbose=False):
bryantaylor 0:eafc3fd41f75 1238 """ Loads from file JSON formatted string to data structure
bryantaylor 0:eafc3fd41f75 1239 """
bryantaylor 0:eafc3fd41f75 1240 result = None
bryantaylor 0:eafc3fd41f75 1241 try:
bryantaylor 0:eafc3fd41f75 1242 with open(json_spec_filename) as data_file:
bryantaylor 0:eafc3fd41f75 1243 try:
bryantaylor 0:eafc3fd41f75 1244 result = json.load(data_file)
bryantaylor 0:eafc3fd41f75 1245 except ValueError as json_error_msg:
bryantaylor 0:eafc3fd41f75 1246 result = None
bryantaylor 0:eafc3fd41f75 1247 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
bryantaylor 0:eafc3fd41f75 1248 # We can print where error occurred inside JSON file if we can parse exception msg
bryantaylor 0:eafc3fd41f75 1249 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
bryantaylor 0:eafc3fd41f75 1250 if json_format_defect_pos is not None:
bryantaylor 0:eafc3fd41f75 1251 line = json_format_defect_pos[0]
bryantaylor 0:eafc3fd41f75 1252 column = json_format_defect_pos[1]
bryantaylor 0:eafc3fd41f75 1253 print
bryantaylor 0:eafc3fd41f75 1254 show_json_file_format_error(json_spec_filename, line, column)
bryantaylor 0:eafc3fd41f75 1255
bryantaylor 0:eafc3fd41f75 1256 except IOError as fileopen_error_msg:
bryantaylor 0:eafc3fd41f75 1257 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
bryantaylor 0:eafc3fd41f75 1258 print
bryantaylor 0:eafc3fd41f75 1259 if verbose and result:
bryantaylor 0:eafc3fd41f75 1260 pp = pprint.PrettyPrinter(indent=4)
bryantaylor 0:eafc3fd41f75 1261 pp.pprint(result)
bryantaylor 0:eafc3fd41f75 1262 return result
bryantaylor 0:eafc3fd41f75 1263
bryantaylor 0:eafc3fd41f75 1264
bryantaylor 0:eafc3fd41f75 1265 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
bryantaylor 0:eafc3fd41f75 1266 """ Prints MUTs configuration passed to test script for verboseness
bryantaylor 0:eafc3fd41f75 1267 """
bryantaylor 0:eafc3fd41f75 1268 muts_info_cols = []
bryantaylor 0:eafc3fd41f75 1269 # We need to check all unique properties for each defined MUT
bryantaylor 0:eafc3fd41f75 1270 for k in json_data:
bryantaylor 0:eafc3fd41f75 1271 mut_info = json_data[k]
bryantaylor 0:eafc3fd41f75 1272 for mut_property in mut_info:
bryantaylor 0:eafc3fd41f75 1273 if mut_property not in muts_info_cols:
bryantaylor 0:eafc3fd41f75 1274 muts_info_cols.append(mut_property)
bryantaylor 0:eafc3fd41f75 1275
bryantaylor 0:eafc3fd41f75 1276 # Prepare pretty table object to display all MUTs
bryantaylor 0:eafc3fd41f75 1277 pt_cols = ["index"] + muts_info_cols
bryantaylor 0:eafc3fd41f75 1278 pt = PrettyTable(pt_cols)
bryantaylor 0:eafc3fd41f75 1279 for col in pt_cols:
bryantaylor 0:eafc3fd41f75 1280 pt.align[col] = "l"
bryantaylor 0:eafc3fd41f75 1281
bryantaylor 0:eafc3fd41f75 1282 # Add rows to pretty print object
bryantaylor 0:eafc3fd41f75 1283 for k in json_data:
bryantaylor 0:eafc3fd41f75 1284 row = [k]
bryantaylor 0:eafc3fd41f75 1285 mut_info = json_data[k]
bryantaylor 0:eafc3fd41f75 1286
bryantaylor 0:eafc3fd41f75 1287 add_row = True
bryantaylor 0:eafc3fd41f75 1288 if platform_filter and 'mcu' in mut_info:
bryantaylor 0:eafc3fd41f75 1289 add_row = re.search(platform_filter, mut_info['mcu']) is not None
bryantaylor 0:eafc3fd41f75 1290 if add_row:
bryantaylor 0:eafc3fd41f75 1291 for col in muts_info_cols:
bryantaylor 0:eafc3fd41f75 1292 cell_val = mut_info[col] if col in mut_info else None
bryantaylor 0:eafc3fd41f75 1293 if type(cell_val) == ListType:
bryantaylor 0:eafc3fd41f75 1294 cell_val = join_delim.join(cell_val)
bryantaylor 0:eafc3fd41f75 1295 row.append(cell_val)
bryantaylor 0:eafc3fd41f75 1296 pt.add_row(row)
bryantaylor 0:eafc3fd41f75 1297 return pt.get_string()
bryantaylor 0:eafc3fd41f75 1298
bryantaylor 0:eafc3fd41f75 1299
bryantaylor 0:eafc3fd41f75 1300 def print_test_configuration_from_json(json_data, join_delim=", "):
bryantaylor 0:eafc3fd41f75 1301 """ Prints test specification configuration passed to test script for verboseness
bryantaylor 0:eafc3fd41f75 1302 """
bryantaylor 0:eafc3fd41f75 1303 toolchains_info_cols = []
bryantaylor 0:eafc3fd41f75 1304 # We need to check all toolchains for each device
bryantaylor 0:eafc3fd41f75 1305 for k in json_data:
bryantaylor 0:eafc3fd41f75 1306 # k should be 'targets'
bryantaylor 0:eafc3fd41f75 1307 targets = json_data[k]
bryantaylor 0:eafc3fd41f75 1308 for target in targets:
bryantaylor 0:eafc3fd41f75 1309 toolchains = targets[target]
bryantaylor 0:eafc3fd41f75 1310 for toolchain in toolchains:
bryantaylor 0:eafc3fd41f75 1311 if toolchain not in toolchains_info_cols:
bryantaylor 0:eafc3fd41f75 1312 toolchains_info_cols.append(toolchain)
bryantaylor 0:eafc3fd41f75 1313
bryantaylor 0:eafc3fd41f75 1314 # Prepare pretty table object to display test specification
bryantaylor 0:eafc3fd41f75 1315 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
bryantaylor 0:eafc3fd41f75 1316 pt = PrettyTable(pt_cols)
bryantaylor 0:eafc3fd41f75 1317 for col in pt_cols:
bryantaylor 0:eafc3fd41f75 1318 pt.align[col] = "l"
bryantaylor 0:eafc3fd41f75 1319
bryantaylor 0:eafc3fd41f75 1320 # { target : [conflicted toolchains] }
bryantaylor 0:eafc3fd41f75 1321 toolchain_conflicts = {}
bryantaylor 0:eafc3fd41f75 1322 toolchain_path_conflicts = []
bryantaylor 0:eafc3fd41f75 1323 for k in json_data:
bryantaylor 0:eafc3fd41f75 1324 # k should be 'targets'
bryantaylor 0:eafc3fd41f75 1325 targets = json_data[k]
bryantaylor 0:eafc3fd41f75 1326 for target in targets:
bryantaylor 0:eafc3fd41f75 1327 target_supported_toolchains = get_target_supported_toolchains(target)
bryantaylor 0:eafc3fd41f75 1328 if not target_supported_toolchains:
bryantaylor 0:eafc3fd41f75 1329 target_supported_toolchains = []
bryantaylor 0:eafc3fd41f75 1330 target_name = target if target in TARGET_MAP else "%s*"% target
bryantaylor 0:eafc3fd41f75 1331 row = [target_name]
bryantaylor 0:eafc3fd41f75 1332 toolchains = targets[target]
bryantaylor 0:eafc3fd41f75 1333
bryantaylor 0:eafc3fd41f75 1334 for toolchain in sorted(toolchains_info_cols):
bryantaylor 0:eafc3fd41f75 1335 # Check for conflicts: target vs toolchain
bryantaylor 0:eafc3fd41f75 1336 conflict = False
bryantaylor 0:eafc3fd41f75 1337 conflict_path = False
bryantaylor 0:eafc3fd41f75 1338 if toolchain in toolchains:
bryantaylor 0:eafc3fd41f75 1339 if toolchain not in target_supported_toolchains:
bryantaylor 0:eafc3fd41f75 1340 conflict = True
bryantaylor 0:eafc3fd41f75 1341 if target not in toolchain_conflicts:
bryantaylor 0:eafc3fd41f75 1342 toolchain_conflicts[target] = []
bryantaylor 0:eafc3fd41f75 1343 toolchain_conflicts[target].append(toolchain)
bryantaylor 0:eafc3fd41f75 1344 # Add marker inside table about target usage / conflict
bryantaylor 0:eafc3fd41f75 1345 cell_val = 'Yes' if toolchain in toolchains else '-'
bryantaylor 0:eafc3fd41f75 1346 if conflict:
bryantaylor 0:eafc3fd41f75 1347 cell_val += '*'
bryantaylor 0:eafc3fd41f75 1348 # Check for conflicts: toolchain vs toolchain path
bryantaylor 0:eafc3fd41f75 1349 if toolchain in TOOLCHAIN_PATHS:
bryantaylor 0:eafc3fd41f75 1350 toolchain_path = TOOLCHAIN_PATHS[toolchain]
bryantaylor 0:eafc3fd41f75 1351 if not os.path.isdir(toolchain_path):
bryantaylor 0:eafc3fd41f75 1352 conflict_path = True
bryantaylor 0:eafc3fd41f75 1353 if toolchain not in toolchain_path_conflicts:
bryantaylor 0:eafc3fd41f75 1354 toolchain_path_conflicts.append(toolchain)
bryantaylor 0:eafc3fd41f75 1355 if conflict_path:
bryantaylor 0:eafc3fd41f75 1356 cell_val += '#'
bryantaylor 0:eafc3fd41f75 1357 row.append(cell_val)
bryantaylor 0:eafc3fd41f75 1358 pt.add_row(row)
bryantaylor 0:eafc3fd41f75 1359
bryantaylor 0:eafc3fd41f75 1360 # generate result string
bryantaylor 0:eafc3fd41f75 1361 result = pt.get_string() # Test specification table
bryantaylor 0:eafc3fd41f75 1362 if toolchain_conflicts or toolchain_path_conflicts:
bryantaylor 0:eafc3fd41f75 1363 result += "\n"
bryantaylor 0:eafc3fd41f75 1364 result += "Toolchain conflicts:\n"
bryantaylor 0:eafc3fd41f75 1365 for target in toolchain_conflicts:
bryantaylor 0:eafc3fd41f75 1366 if target not in TARGET_MAP:
bryantaylor 0:eafc3fd41f75 1367 result += "\t* Target %s unknown\n"% (target)
bryantaylor 0:eafc3fd41f75 1368 conflict_target_list = join_delim.join(toolchain_conflicts[target])
bryantaylor 0:eafc3fd41f75 1369 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
bryantaylor 0:eafc3fd41f75 1370 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
bryantaylor 0:eafc3fd41f75 1371
bryantaylor 0:eafc3fd41f75 1372 for toolchain in toolchain_path_conflicts:
bryantaylor 0:eafc3fd41f75 1373 # Let's check toolchain configuration
bryantaylor 0:eafc3fd41f75 1374 if toolchain in TOOLCHAIN_PATHS:
bryantaylor 0:eafc3fd41f75 1375 toolchain_path = TOOLCHAIN_PATHS[toolchain]
bryantaylor 0:eafc3fd41f75 1376 if not os.path.isdir(toolchain_path):
bryantaylor 0:eafc3fd41f75 1377 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
bryantaylor 0:eafc3fd41f75 1378 return result
bryantaylor 0:eafc3fd41f75 1379
bryantaylor 0:eafc3fd41f75 1380
bryantaylor 0:eafc3fd41f75 1381 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
bryantaylor 0:eafc3fd41f75 1382 """ Generates table summary with all test cases and additional test cases
bryantaylor 0:eafc3fd41f75 1383 information using pretty print functionality. Allows test suite user to
bryantaylor 0:eafc3fd41f75 1384 see test cases
bryantaylor 0:eafc3fd41f75 1385 """
bryantaylor 0:eafc3fd41f75 1386 # get all unique test ID prefixes
bryantaylor 0:eafc3fd41f75 1387 unique_test_id = []
bryantaylor 0:eafc3fd41f75 1388 for test in TESTS:
bryantaylor 0:eafc3fd41f75 1389 split = test['id'].split('_')[:-1]
bryantaylor 0:eafc3fd41f75 1390 test_id_prefix = '_'.join(split)
bryantaylor 0:eafc3fd41f75 1391 if test_id_prefix not in unique_test_id:
bryantaylor 0:eafc3fd41f75 1392 unique_test_id.append(test_id_prefix)
bryantaylor 0:eafc3fd41f75 1393 unique_test_id.sort()
bryantaylor 0:eafc3fd41f75 1394 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
bryantaylor 0:eafc3fd41f75 1395 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
bryantaylor 0:eafc3fd41f75 1396
bryantaylor 0:eafc3fd41f75 1397 test_properties = ['id',
bryantaylor 0:eafc3fd41f75 1398 'automated',
bryantaylor 0:eafc3fd41f75 1399 'description',
bryantaylor 0:eafc3fd41f75 1400 'peripherals',
bryantaylor 0:eafc3fd41f75 1401 'host_test',
bryantaylor 0:eafc3fd41f75 1402 'duration'] if cols is None else cols
bryantaylor 0:eafc3fd41f75 1403
bryantaylor 0:eafc3fd41f75 1404 # All tests status table print
bryantaylor 0:eafc3fd41f75 1405 pt = PrettyTable(test_properties)
bryantaylor 0:eafc3fd41f75 1406 for col in test_properties:
bryantaylor 0:eafc3fd41f75 1407 pt.align[col] = "l"
bryantaylor 0:eafc3fd41f75 1408 pt.align['duration'] = "r"
bryantaylor 0:eafc3fd41f75 1409
bryantaylor 0:eafc3fd41f75 1410 counter_all = 0
bryantaylor 0:eafc3fd41f75 1411 counter_automated = 0
bryantaylor 0:eafc3fd41f75 1412 pt.padding_width = 1 # One space between column edges and contents (default)
bryantaylor 0:eafc3fd41f75 1413
bryantaylor 0:eafc3fd41f75 1414 for test_id in sorted(TEST_MAP.keys()):
bryantaylor 0:eafc3fd41f75 1415 if platform_filter is not None:
bryantaylor 0:eafc3fd41f75 1416 # FIlter out platforms using regex
bryantaylor 0:eafc3fd41f75 1417 if re.search(platform_filter, test_id) is None:
bryantaylor 0:eafc3fd41f75 1418 continue
bryantaylor 0:eafc3fd41f75 1419 row = []
bryantaylor 0:eafc3fd41f75 1420 test = TEST_MAP[test_id]
bryantaylor 0:eafc3fd41f75 1421 split = test_id.split('_')[:-1]
bryantaylor 0:eafc3fd41f75 1422 test_id_prefix = '_'.join(split)
bryantaylor 0:eafc3fd41f75 1423
bryantaylor 0:eafc3fd41f75 1424 for col in test_properties:
bryantaylor 0:eafc3fd41f75 1425 col_value = test[col]
bryantaylor 0:eafc3fd41f75 1426 if type(test[col]) == ListType:
bryantaylor 0:eafc3fd41f75 1427 col_value = join_delim.join(test[col])
bryantaylor 0:eafc3fd41f75 1428 elif test[col] == None:
bryantaylor 0:eafc3fd41f75 1429 col_value = "-"
bryantaylor 0:eafc3fd41f75 1430
bryantaylor 0:eafc3fd41f75 1431 row.append(col_value)
bryantaylor 0:eafc3fd41f75 1432 if test['automated'] == True:
bryantaylor 0:eafc3fd41f75 1433 counter_dict_test_id_types[test_id_prefix] += 1
bryantaylor 0:eafc3fd41f75 1434 counter_automated += 1
bryantaylor 0:eafc3fd41f75 1435 pt.add_row(row)
bryantaylor 0:eafc3fd41f75 1436 # Update counters
bryantaylor 0:eafc3fd41f75 1437 counter_all += 1
bryantaylor 0:eafc3fd41f75 1438 counter_dict_test_id_types_all[test_id_prefix] += 1
bryantaylor 0:eafc3fd41f75 1439 result = pt.get_string()
bryantaylor 0:eafc3fd41f75 1440 result += "\n\n"
bryantaylor 0:eafc3fd41f75 1441
bryantaylor 0:eafc3fd41f75 1442 if result_summary and not platform_filter:
bryantaylor 0:eafc3fd41f75 1443 # Automation result summary
bryantaylor 0:eafc3fd41f75 1444 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
bryantaylor 0:eafc3fd41f75 1445 pt = PrettyTable(test_id_cols)
bryantaylor 0:eafc3fd41f75 1446 pt.align['automated'] = "r"
bryantaylor 0:eafc3fd41f75 1447 pt.align['all'] = "r"
bryantaylor 0:eafc3fd41f75 1448 pt.align['percent [%]'] = "r"
bryantaylor 0:eafc3fd41f75 1449
bryantaylor 0:eafc3fd41f75 1450 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
bryantaylor 0:eafc3fd41f75 1451 str_progress = progress_bar(percent_progress, 75)
bryantaylor 0:eafc3fd41f75 1452 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
bryantaylor 0:eafc3fd41f75 1453 result += "Automation coverage:\n"
bryantaylor 0:eafc3fd41f75 1454 result += pt.get_string()
bryantaylor 0:eafc3fd41f75 1455 result += "\n\n"
bryantaylor 0:eafc3fd41f75 1456
bryantaylor 0:eafc3fd41f75 1457 # Test automation coverage table print
bryantaylor 0:eafc3fd41f75 1458 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
bryantaylor 0:eafc3fd41f75 1459 pt = PrettyTable(test_id_cols)
bryantaylor 0:eafc3fd41f75 1460 pt.align['id'] = "l"
bryantaylor 0:eafc3fd41f75 1461 pt.align['automated'] = "r"
bryantaylor 0:eafc3fd41f75 1462 pt.align['all'] = "r"
bryantaylor 0:eafc3fd41f75 1463 pt.align['percent [%]'] = "r"
bryantaylor 0:eafc3fd41f75 1464 for unique_id in unique_test_id:
bryantaylor 0:eafc3fd41f75 1465 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
bryantaylor 0:eafc3fd41f75 1466 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
bryantaylor 0:eafc3fd41f75 1467 str_progress = progress_bar(percent_progress, 75)
bryantaylor 0:eafc3fd41f75 1468 row = [unique_id,
bryantaylor 0:eafc3fd41f75 1469 counter_dict_test_id_types[unique_id],
bryantaylor 0:eafc3fd41f75 1470 counter_dict_test_id_types_all[unique_id],
bryantaylor 0:eafc3fd41f75 1471 percent_progress,
bryantaylor 0:eafc3fd41f75 1472 "[" + str_progress + "]"]
bryantaylor 0:eafc3fd41f75 1473 pt.add_row(row)
bryantaylor 0:eafc3fd41f75 1474 result += "Test automation coverage:\n"
bryantaylor 0:eafc3fd41f75 1475 result += pt.get_string()
bryantaylor 0:eafc3fd41f75 1476 result += "\n\n"
bryantaylor 0:eafc3fd41f75 1477 return result
bryantaylor 0:eafc3fd41f75 1478
bryantaylor 0:eafc3fd41f75 1479
bryantaylor 0:eafc3fd41f75 1480 def progress_bar(percent_progress, saturation=0):
bryantaylor 0:eafc3fd41f75 1481 """ This function creates progress bar with optional simple saturation mark
bryantaylor 0:eafc3fd41f75 1482 """
bryantaylor 0:eafc3fd41f75 1483 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
bryantaylor 0:eafc3fd41f75 1484 str_progress = '#' * step + '.' * int(50 - step)
bryantaylor 0:eafc3fd41f75 1485 c = '!' if str_progress[38] == '.' else '|'
bryantaylor 0:eafc3fd41f75 1486 if saturation > 0:
bryantaylor 0:eafc3fd41f75 1487 saturation = saturation / 2
bryantaylor 0:eafc3fd41f75 1488 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
bryantaylor 0:eafc3fd41f75 1489 return str_progress
bryantaylor 0:eafc3fd41f75 1490
bryantaylor 0:eafc3fd41f75 1491
bryantaylor 0:eafc3fd41f75 1492 def singletest_in_cli_mode(single_test):
bryantaylor 0:eafc3fd41f75 1493 """ Runs SingleTestRunner object in CLI (Command line interface) mode
bryantaylor 0:eafc3fd41f75 1494
bryantaylor 0:eafc3fd41f75 1495 @return returns success code (0 == success) for building and running tests
bryantaylor 0:eafc3fd41f75 1496 """
bryantaylor 0:eafc3fd41f75 1497 start = time()
bryantaylor 0:eafc3fd41f75 1498 # Execute tests depending on options and filter applied
bryantaylor 0:eafc3fd41f75 1499 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
bryantaylor 0:eafc3fd41f75 1500 elapsed_time = time() - start
bryantaylor 0:eafc3fd41f75 1501
bryantaylor 0:eafc3fd41f75 1502 # Human readable summary
bryantaylor 0:eafc3fd41f75 1503 if not single_test.opts_suppress_summary:
bryantaylor 0:eafc3fd41f75 1504 # prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 1505 print single_test.generate_test_summary(test_summary, shuffle_seed)
bryantaylor 0:eafc3fd41f75 1506 if single_test.opts_test_x_toolchain_summary:
bryantaylor 0:eafc3fd41f75 1507 # prints well-formed summary with results (SQL table like)
bryantaylor 0:eafc3fd41f75 1508 # table shows text x toolchain test result matrix
bryantaylor 0:eafc3fd41f75 1509 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
bryantaylor 0:eafc3fd41f75 1510
bryantaylor 0:eafc3fd41f75 1511 print "Completed in %.2f sec"% (elapsed_time)
bryantaylor 0:eafc3fd41f75 1512 print
bryantaylor 0:eafc3fd41f75 1513 # Write summary of the builds
bryantaylor 0:eafc3fd41f75 1514
bryantaylor 0:eafc3fd41f75 1515 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
bryantaylor 0:eafc3fd41f75 1516 status = print_report_exporter.report(build_report)
bryantaylor 0:eafc3fd41f75 1517
bryantaylor 0:eafc3fd41f75 1518 # Store extra reports in files
bryantaylor 0:eafc3fd41f75 1519 if single_test.opts_report_html_file_name:
bryantaylor 0:eafc3fd41f75 1520 # Export results in form of HTML report to separate file
bryantaylor 0:eafc3fd41f75 1521 report_exporter = ReportExporter(ResultExporterType.HTML)
bryantaylor 0:eafc3fd41f75 1522 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
bryantaylor 0:eafc3fd41f75 1523 if single_test.opts_report_junit_file_name:
bryantaylor 0:eafc3fd41f75 1524 # Export results in form of JUnit XML report to separate file
bryantaylor 0:eafc3fd41f75 1525 report_exporter = ReportExporter(ResultExporterType.JUNIT)
bryantaylor 0:eafc3fd41f75 1526 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
bryantaylor 0:eafc3fd41f75 1527 if single_test.opts_report_text_file_name:
bryantaylor 0:eafc3fd41f75 1528 # Export results in form of a text file
bryantaylor 0:eafc3fd41f75 1529 report_exporter = ReportExporter(ResultExporterType.TEXT)
bryantaylor 0:eafc3fd41f75 1530 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
bryantaylor 0:eafc3fd41f75 1531 if single_test.opts_report_build_file_name:
bryantaylor 0:eafc3fd41f75 1532 # Export build results as html report to sparate file
bryantaylor 0:eafc3fd41f75 1533 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
bryantaylor 0:eafc3fd41f75 1534 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
bryantaylor 0:eafc3fd41f75 1535
bryantaylor 0:eafc3fd41f75 1536 # Returns True if no build failures of the test projects or their dependencies
bryantaylor 0:eafc3fd41f75 1537 return status
bryantaylor 0:eafc3fd41f75 1538
bryantaylor 0:eafc3fd41f75 1539 class TestLogger():
bryantaylor 0:eafc3fd41f75 1540 """ Super-class for logging and printing ongoing events for test suite pass
bryantaylor 0:eafc3fd41f75 1541 """
bryantaylor 0:eafc3fd41f75 1542 def __init__(self, store_log=True):
bryantaylor 0:eafc3fd41f75 1543 """ We can control if logger actually stores log in memory
bryantaylor 0:eafc3fd41f75 1544 or just handled all log entries immediately
bryantaylor 0:eafc3fd41f75 1545 """
bryantaylor 0:eafc3fd41f75 1546 self.log = []
bryantaylor 0:eafc3fd41f75 1547 self.log_to_file = False
bryantaylor 0:eafc3fd41f75 1548 self.log_file_name = None
bryantaylor 0:eafc3fd41f75 1549 self.store_log = store_log
bryantaylor 0:eafc3fd41f75 1550
bryantaylor 0:eafc3fd41f75 1551 self.LogType = construct_enum(INFO='Info',
bryantaylor 0:eafc3fd41f75 1552 WARN='Warning',
bryantaylor 0:eafc3fd41f75 1553 NOTIF='Notification',
bryantaylor 0:eafc3fd41f75 1554 ERROR='Error',
bryantaylor 0:eafc3fd41f75 1555 EXCEPT='Exception')
bryantaylor 0:eafc3fd41f75 1556
bryantaylor 0:eafc3fd41f75 1557 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
bryantaylor 0:eafc3fd41f75 1558 APPEND=2) # Append to existing log file
bryantaylor 0:eafc3fd41f75 1559
bryantaylor 0:eafc3fd41f75 1560 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
bryantaylor 0:eafc3fd41f75 1561 """ Log one line of text
bryantaylor 0:eafc3fd41f75 1562 """
bryantaylor 0:eafc3fd41f75 1563 log_timestamp = time()
bryantaylor 0:eafc3fd41f75 1564 log_entry = {'log_type' : LogType,
bryantaylor 0:eafc3fd41f75 1565 'log_timestamp' : log_timestamp,
bryantaylor 0:eafc3fd41f75 1566 'log_line' : log_line,
bryantaylor 0:eafc3fd41f75 1567 '_future' : None
bryantaylor 0:eafc3fd41f75 1568 }
bryantaylor 0:eafc3fd41f75 1569 # Store log in memory
bryantaylor 0:eafc3fd41f75 1570 if self.store_log:
bryantaylor 0:eafc3fd41f75 1571 self.log.append(log_entry)
bryantaylor 0:eafc3fd41f75 1572 return log_entry
bryantaylor 0:eafc3fd41f75 1573
bryantaylor 0:eafc3fd41f75 1574
bryantaylor 0:eafc3fd41f75 1575 class CLITestLogger(TestLogger):
bryantaylor 0:eafc3fd41f75 1576 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
bryantaylor 0:eafc3fd41f75 1577 """
bryantaylor 0:eafc3fd41f75 1578 def __init__(self, store_log=True, file_name=None):
bryantaylor 0:eafc3fd41f75 1579 TestLogger.__init__(self)
bryantaylor 0:eafc3fd41f75 1580 self.log_file_name = file_name
bryantaylor 0:eafc3fd41f75 1581 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
bryantaylor 0:eafc3fd41f75 1582 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
bryantaylor 0:eafc3fd41f75 1583
bryantaylor 0:eafc3fd41f75 1584 def log_print(self, log_entry, timestamp=True):
bryantaylor 0:eafc3fd41f75 1585 """ Prints on screen formatted log entry
bryantaylor 0:eafc3fd41f75 1586 """
bryantaylor 0:eafc3fd41f75 1587 ts = log_entry['log_timestamp']
bryantaylor 0:eafc3fd41f75 1588 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
bryantaylor 0:eafc3fd41f75 1589 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
bryantaylor 0:eafc3fd41f75 1590 return timestamp_str + log_line_str
bryantaylor 0:eafc3fd41f75 1591
bryantaylor 0:eafc3fd41f75 1592 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
bryantaylor 0:eafc3fd41f75 1593 """ Logs line, if log file output was specified log line will be appended
bryantaylor 0:eafc3fd41f75 1594 at the end of log file
bryantaylor 0:eafc3fd41f75 1595 """
bryantaylor 0:eafc3fd41f75 1596 log_entry = TestLogger.log_line(self, LogType, log_line)
bryantaylor 0:eafc3fd41f75 1597 log_line_str = self.log_print(log_entry, timestamp)
bryantaylor 0:eafc3fd41f75 1598 if self.log_file_name is not None:
bryantaylor 0:eafc3fd41f75 1599 try:
bryantaylor 0:eafc3fd41f75 1600 with open(self.log_file_name, 'a') as f:
bryantaylor 0:eafc3fd41f75 1601 f.write(log_line_str + line_delim)
bryantaylor 0:eafc3fd41f75 1602 except IOError:
bryantaylor 0:eafc3fd41f75 1603 pass
bryantaylor 0:eafc3fd41f75 1604 return log_line_str
bryantaylor 0:eafc3fd41f75 1605
bryantaylor 0:eafc3fd41f75 1606
bryantaylor 0:eafc3fd41f75 1607 def factory_db_logger(db_url):
bryantaylor 0:eafc3fd41f75 1608 """ Factory database driver depending on database type supplied in database connection string db_url
bryantaylor 0:eafc3fd41f75 1609 """
bryantaylor 0:eafc3fd41f75 1610 if db_url is not None:
bryantaylor 0:eafc3fd41f75 1611 from tools.test_mysql import MySQLDBAccess
bryantaylor 0:eafc3fd41f75 1612 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
bryantaylor 0:eafc3fd41f75 1613 if connection_info is not None:
bryantaylor 0:eafc3fd41f75 1614 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
bryantaylor 0:eafc3fd41f75 1615 if db_type == 'mysql':
bryantaylor 0:eafc3fd41f75 1616 return MySQLDBAccess()
bryantaylor 0:eafc3fd41f75 1617 return None
bryantaylor 0:eafc3fd41f75 1618
bryantaylor 0:eafc3fd41f75 1619
bryantaylor 0:eafc3fd41f75 1620 def detect_database_verbose(db_url):
bryantaylor 0:eafc3fd41f75 1621 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
bryantaylor 0:eafc3fd41f75 1622 """
bryantaylor 0:eafc3fd41f75 1623 result = BaseDBAccess().parse_db_connection_string(db_url)
bryantaylor 0:eafc3fd41f75 1624 if result is not None:
bryantaylor 0:eafc3fd41f75 1625 # Parsing passed
bryantaylor 0:eafc3fd41f75 1626 (db_type, username, password, host, db_name) = result
bryantaylor 0:eafc3fd41f75 1627 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
bryantaylor 0:eafc3fd41f75 1628 # Let's try to connect
bryantaylor 0:eafc3fd41f75 1629 db_ = factory_db_logger(db_url)
bryantaylor 0:eafc3fd41f75 1630 if db_ is not None:
bryantaylor 0:eafc3fd41f75 1631 print "Connecting to database '%s'..."% db_url,
bryantaylor 0:eafc3fd41f75 1632 db_.connect(host, username, password, db_name)
bryantaylor 0:eafc3fd41f75 1633 if db_.is_connected():
bryantaylor 0:eafc3fd41f75 1634 print "ok"
bryantaylor 0:eafc3fd41f75 1635 print "Detecting database..."
bryantaylor 0:eafc3fd41f75 1636 print db_.detect_database(verbose=True)
bryantaylor 0:eafc3fd41f75 1637 print "Disconnecting...",
bryantaylor 0:eafc3fd41f75 1638 db_.disconnect()
bryantaylor 0:eafc3fd41f75 1639 print "done"
bryantaylor 0:eafc3fd41f75 1640 else:
bryantaylor 0:eafc3fd41f75 1641 print "Database type '%s' unknown"% db_type
bryantaylor 0:eafc3fd41f75 1642 else:
bryantaylor 0:eafc3fd41f75 1643 print "Parse error: '%s' - DB Url error"% (db_url)
bryantaylor 0:eafc3fd41f75 1644
bryantaylor 0:eafc3fd41f75 1645
bryantaylor 0:eafc3fd41f75 1646 def get_module_avail(module_name):
bryantaylor 0:eafc3fd41f75 1647 """ This function returns True if module_name is already impored module
bryantaylor 0:eafc3fd41f75 1648 """
bryantaylor 0:eafc3fd41f75 1649 return module_name in sys.modules.keys()
bryantaylor 0:eafc3fd41f75 1650
bryantaylor 0:eafc3fd41f75 1651
bryantaylor 0:eafc3fd41f75 1652 def get_autodetected_MUTS_list(platform_name_filter=None):
bryantaylor 0:eafc3fd41f75 1653 oldError = None
bryantaylor 0:eafc3fd41f75 1654 if os.name == 'nt':
bryantaylor 0:eafc3fd41f75 1655 # Disable Windows error box temporarily
bryantaylor 0:eafc3fd41f75 1656 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
bryantaylor 0:eafc3fd41f75 1657
bryantaylor 0:eafc3fd41f75 1658 mbeds = mbed_lstools.create()
bryantaylor 0:eafc3fd41f75 1659 detect_muts_list = mbeds.list_mbeds()
bryantaylor 0:eafc3fd41f75 1660
bryantaylor 0:eafc3fd41f75 1661 if os.name == 'nt':
bryantaylor 0:eafc3fd41f75 1662 ctypes.windll.kernel32.SetErrorMode(oldError)
bryantaylor 0:eafc3fd41f75 1663
bryantaylor 0:eafc3fd41f75 1664 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
bryantaylor 0:eafc3fd41f75 1665
bryantaylor 0:eafc3fd41f75 1666 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
bryantaylor 0:eafc3fd41f75 1667 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
bryantaylor 0:eafc3fd41f75 1668 If function fails to auto-detect devices it will return empty dictionary.
bryantaylor 0:eafc3fd41f75 1669
bryantaylor 0:eafc3fd41f75 1670 if get_module_avail('mbed_lstools'):
bryantaylor 0:eafc3fd41f75 1671 mbeds = mbed_lstools.create()
bryantaylor 0:eafc3fd41f75 1672 mbeds_list = mbeds.list_mbeds()
bryantaylor 0:eafc3fd41f75 1673
bryantaylor 0:eafc3fd41f75 1674 @param mbeds_list list of mbeds captured from mbed_lstools
bryantaylor 0:eafc3fd41f75 1675 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
bryantaylor 0:eafc3fd41f75 1676 """
bryantaylor 0:eafc3fd41f75 1677 result = {} # Should be in muts_all.json format
bryantaylor 0:eafc3fd41f75 1678 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
bryantaylor 0:eafc3fd41f75 1679 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
bryantaylor 0:eafc3fd41f75 1680 index = 1
bryantaylor 0:eafc3fd41f75 1681 for mut in mbeds_list:
bryantaylor 0:eafc3fd41f75 1682 # Filter the MUTS if a filter is specified
bryantaylor 0:eafc3fd41f75 1683
bryantaylor 0:eafc3fd41f75 1684 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
bryantaylor 0:eafc3fd41f75 1685 continue
bryantaylor 0:eafc3fd41f75 1686
bryantaylor 0:eafc3fd41f75 1687 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
bryantaylor 0:eafc3fd41f75 1688 # if not we are creating our own unique value (last few chars from platform's target_id).
bryantaylor 0:eafc3fd41f75 1689 m = {'mcu': mut['platform_name'],
bryantaylor 0:eafc3fd41f75 1690 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
bryantaylor 0:eafc3fd41f75 1691 'port': mut['serial_port'],
bryantaylor 0:eafc3fd41f75 1692 'disk': mut['mount_point'],
bryantaylor 0:eafc3fd41f75 1693 'peripherals': [] # No peripheral detection
bryantaylor 0:eafc3fd41f75 1694 }
bryantaylor 0:eafc3fd41f75 1695 if index not in result:
bryantaylor 0:eafc3fd41f75 1696 result[index] = {}
bryantaylor 0:eafc3fd41f75 1697 result[index] = m
bryantaylor 0:eafc3fd41f75 1698 index += 1
bryantaylor 0:eafc3fd41f75 1699 return result
bryantaylor 0:eafc3fd41f75 1700
bryantaylor 0:eafc3fd41f75 1701
bryantaylor 0:eafc3fd41f75 1702 def get_autodetected_TEST_SPEC(mbeds_list,
bryantaylor 0:eafc3fd41f75 1703 use_default_toolchain=True,
bryantaylor 0:eafc3fd41f75 1704 use_supported_toolchains=False,
bryantaylor 0:eafc3fd41f75 1705 toolchain_filter=None,
bryantaylor 0:eafc3fd41f75 1706 platform_name_filter=None):
bryantaylor 0:eafc3fd41f75 1707 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
bryantaylor 0:eafc3fd41f75 1708 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
bryantaylor 0:eafc3fd41f75 1709
bryantaylor 0:eafc3fd41f75 1710 use_default_toolchain - if True add default toolchain to test_spec
bryantaylor 0:eafc3fd41f75 1711 use_supported_toolchains - if True add all supported toolchains to test_spec
bryantaylor 0:eafc3fd41f75 1712 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
bryantaylor 0:eafc3fd41f75 1713 """
bryantaylor 0:eafc3fd41f75 1714 result = {'targets': {} }
bryantaylor 0:eafc3fd41f75 1715
bryantaylor 0:eafc3fd41f75 1716 for mut in mbeds_list:
bryantaylor 0:eafc3fd41f75 1717 mcu = mut['mcu']
bryantaylor 0:eafc3fd41f75 1718 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
bryantaylor 0:eafc3fd41f75 1719 if mcu in TARGET_MAP:
bryantaylor 0:eafc3fd41f75 1720 default_toolchain = TARGET_MAP[mcu].default_toolchain
bryantaylor 0:eafc3fd41f75 1721 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
bryantaylor 0:eafc3fd41f75 1722
bryantaylor 0:eafc3fd41f75 1723 # Decide which toolchains should be added to test specification toolchain pool for each target
bryantaylor 0:eafc3fd41f75 1724 toolchains = []
bryantaylor 0:eafc3fd41f75 1725 if use_default_toolchain:
bryantaylor 0:eafc3fd41f75 1726 toolchains.append(default_toolchain)
bryantaylor 0:eafc3fd41f75 1727 if use_supported_toolchains:
bryantaylor 0:eafc3fd41f75 1728 toolchains += supported_toolchains
bryantaylor 0:eafc3fd41f75 1729 if toolchain_filter is not None:
bryantaylor 0:eafc3fd41f75 1730 all_toolchains = supported_toolchains + [default_toolchain]
bryantaylor 0:eafc3fd41f75 1731 for toolchain in toolchain_filter:
bryantaylor 0:eafc3fd41f75 1732 if toolchain in all_toolchains:
bryantaylor 0:eafc3fd41f75 1733 toolchains.append(toolchain)
bryantaylor 0:eafc3fd41f75 1734
bryantaylor 0:eafc3fd41f75 1735 result['targets'][mcu] = list(set(toolchains))
bryantaylor 0:eafc3fd41f75 1736 return result
bryantaylor 0:eafc3fd41f75 1737
bryantaylor 0:eafc3fd41f75 1738
bryantaylor 0:eafc3fd41f75 1739 def get_default_test_options_parser():
bryantaylor 0:eafc3fd41f75 1740 """ Get common test script options used by CLI, web services etc.
bryantaylor 0:eafc3fd41f75 1741 """
bryantaylor 0:eafc3fd41f75 1742 parser = argparse.ArgumentParser()
bryantaylor 0:eafc3fd41f75 1743 parser.add_argument('-i', '--tests',
bryantaylor 0:eafc3fd41f75 1744 dest='test_spec_filename',
bryantaylor 0:eafc3fd41f75 1745 metavar="FILE",
bryantaylor 0:eafc3fd41f75 1746 type=argparse_filestring_type,
bryantaylor 0:eafc3fd41f75 1747 help='Points to file with test specification')
bryantaylor 0:eafc3fd41f75 1748
bryantaylor 0:eafc3fd41f75 1749 parser.add_argument('-M', '--MUTS',
bryantaylor 0:eafc3fd41f75 1750 dest='muts_spec_filename',
bryantaylor 0:eafc3fd41f75 1751 metavar="FILE",
bryantaylor 0:eafc3fd41f75 1752 type=argparse_filestring_type,
bryantaylor 0:eafc3fd41f75 1753 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
bryantaylor 0:eafc3fd41f75 1754
bryantaylor 0:eafc3fd41f75 1755 parser.add_argument("-j", "--jobs",
bryantaylor 0:eafc3fd41f75 1756 dest='jobs',
bryantaylor 0:eafc3fd41f75 1757 metavar="NUMBER",
bryantaylor 0:eafc3fd41f75 1758 type=int,
bryantaylor 0:eafc3fd41f75 1759 help="Define number of compilation jobs. Default value is 1")
bryantaylor 0:eafc3fd41f75 1760
bryantaylor 0:eafc3fd41f75 1761 if get_module_avail('mbed_lstools'):
bryantaylor 0:eafc3fd41f75 1762 # Additional features available when mbed_lstools is installed on host and imported
bryantaylor 0:eafc3fd41f75 1763 # mbed_lstools allow users to detect connected to host mbed-enabled devices
bryantaylor 0:eafc3fd41f75 1764 parser.add_argument('--auto',
bryantaylor 0:eafc3fd41f75 1765 dest='auto_detect',
bryantaylor 0:eafc3fd41f75 1766 action="store_true",
bryantaylor 0:eafc3fd41f75 1767 help='Use mbed-ls module to detect all connected mbed devices')
bryantaylor 0:eafc3fd41f75 1768
bryantaylor 0:eafc3fd41f75 1769 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
bryantaylor 0:eafc3fd41f75 1770 parser.add_argument('--tc',
bryantaylor 0:eafc3fd41f75 1771 dest='toolchains_filter',
bryantaylor 0:eafc3fd41f75 1772 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
bryantaylor 0:eafc3fd41f75 1773 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
bryantaylor 0:eafc3fd41f75 1774
bryantaylor 0:eafc3fd41f75 1775 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
bryantaylor 0:eafc3fd41f75 1776 parser.add_argument('--oper',
bryantaylor 0:eafc3fd41f75 1777 dest='operability_checks',
bryantaylor 0:eafc3fd41f75 1778 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
bryantaylor 0:eafc3fd41f75 1779 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
bryantaylor 0:eafc3fd41f75 1780
bryantaylor 0:eafc3fd41f75 1781 parser.add_argument('--clean',
bryantaylor 0:eafc3fd41f75 1782 dest='clean',
bryantaylor 0:eafc3fd41f75 1783 action="store_true",
bryantaylor 0:eafc3fd41f75 1784 help='Clean the build directory')
bryantaylor 0:eafc3fd41f75 1785
bryantaylor 0:eafc3fd41f75 1786 parser.add_argument('-P', '--only-peripherals',
bryantaylor 0:eafc3fd41f75 1787 dest='test_only_peripheral',
bryantaylor 0:eafc3fd41f75 1788 default=False,
bryantaylor 0:eafc3fd41f75 1789 action="store_true",
bryantaylor 0:eafc3fd41f75 1790 help='Test only peripheral declared for MUT and skip common tests')
bryantaylor 0:eafc3fd41f75 1791
bryantaylor 0:eafc3fd41f75 1792 parser.add_argument('-C', '--only-commons',
bryantaylor 0:eafc3fd41f75 1793 dest='test_only_common',
bryantaylor 0:eafc3fd41f75 1794 default=False,
bryantaylor 0:eafc3fd41f75 1795 action="store_true",
bryantaylor 0:eafc3fd41f75 1796 help='Test only board internals. Skip perpherials tests and perform common tests')
bryantaylor 0:eafc3fd41f75 1797
bryantaylor 0:eafc3fd41f75 1798 parser.add_argument('-n', '--test-by-names',
bryantaylor 0:eafc3fd41f75 1799 dest='test_by_names',
bryantaylor 0:eafc3fd41f75 1800 type=argparse_many(str),
bryantaylor 0:eafc3fd41f75 1801 help='Runs only test enumerated it this switch. Use comma to separate test case names')
bryantaylor 0:eafc3fd41f75 1802
bryantaylor 0:eafc3fd41f75 1803 parser.add_argument('-p', '--peripheral-by-names',
bryantaylor 0:eafc3fd41f75 1804 dest='peripheral_by_names',
bryantaylor 0:eafc3fd41f75 1805 type=argparse_many(str),
bryantaylor 0:eafc3fd41f75 1806 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
bryantaylor 0:eafc3fd41f75 1807
bryantaylor 0:eafc3fd41f75 1808 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
bryantaylor 0:eafc3fd41f75 1809 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
bryantaylor 0:eafc3fd41f75 1810
bryantaylor 0:eafc3fd41f75 1811 parser.add_argument('-c', '--copy-method',
bryantaylor 0:eafc3fd41f75 1812 dest='copy_method',
bryantaylor 0:eafc3fd41f75 1813 type=argparse_uppercase_type(copy_methods, "flash method"),
bryantaylor 0:eafc3fd41f75 1814 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
bryantaylor 0:eafc3fd41f75 1815
bryantaylor 0:eafc3fd41f75 1816 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
bryantaylor 0:eafc3fd41f75 1817 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
bryantaylor 0:eafc3fd41f75 1818
bryantaylor 0:eafc3fd41f75 1819 parser.add_argument('-r', '--reset-type',
bryantaylor 0:eafc3fd41f75 1820 dest='mut_reset_type',
bryantaylor 0:eafc3fd41f75 1821 default=None,
bryantaylor 0:eafc3fd41f75 1822 type=argparse_uppercase_type(reset_methods, "reset method"),
bryantaylor 0:eafc3fd41f75 1823 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
bryantaylor 0:eafc3fd41f75 1824
bryantaylor 0:eafc3fd41f75 1825 parser.add_argument('-g', '--goanna-for-tests',
bryantaylor 0:eafc3fd41f75 1826 dest='goanna_for_tests',
bryantaylor 0:eafc3fd41f75 1827 action="store_true",
bryantaylor 0:eafc3fd41f75 1828 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
bryantaylor 0:eafc3fd41f75 1829
bryantaylor 0:eafc3fd41f75 1830 parser.add_argument('-G', '--goanna-for-sdk',
bryantaylor 0:eafc3fd41f75 1831 dest='goanna_for_mbed_sdk',
bryantaylor 0:eafc3fd41f75 1832 action="store_true",
bryantaylor 0:eafc3fd41f75 1833 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
bryantaylor 0:eafc3fd41f75 1834
bryantaylor 0:eafc3fd41f75 1835 parser.add_argument('-s', '--suppress-summary',
bryantaylor 0:eafc3fd41f75 1836 dest='suppress_summary',
bryantaylor 0:eafc3fd41f75 1837 default=False,
bryantaylor 0:eafc3fd41f75 1838 action="store_true",
bryantaylor 0:eafc3fd41f75 1839 help='Suppresses display of wellformatted table with test results')
bryantaylor 0:eafc3fd41f75 1840
bryantaylor 0:eafc3fd41f75 1841 parser.add_argument('-t', '--test-summary',
bryantaylor 0:eafc3fd41f75 1842 dest='test_x_toolchain_summary',
bryantaylor 0:eafc3fd41f75 1843 default=False,
bryantaylor 0:eafc3fd41f75 1844 action="store_true",
bryantaylor 0:eafc3fd41f75 1845 help='Displays wellformatted table with test x toolchain test result per target')
bryantaylor 0:eafc3fd41f75 1846
bryantaylor 0:eafc3fd41f75 1847 parser.add_argument('-A', '--test-automation-report',
bryantaylor 0:eafc3fd41f75 1848 dest='test_automation_report',
bryantaylor 0:eafc3fd41f75 1849 default=False,
bryantaylor 0:eafc3fd41f75 1850 action="store_true",
bryantaylor 0:eafc3fd41f75 1851 help='Prints information about all tests and exits')
bryantaylor 0:eafc3fd41f75 1852
bryantaylor 0:eafc3fd41f75 1853 parser.add_argument('-R', '--test-case-report',
bryantaylor 0:eafc3fd41f75 1854 dest='test_case_report',
bryantaylor 0:eafc3fd41f75 1855 default=False,
bryantaylor 0:eafc3fd41f75 1856 action="store_true",
bryantaylor 0:eafc3fd41f75 1857 help='Prints information about all test cases and exits')
bryantaylor 0:eafc3fd41f75 1858
bryantaylor 0:eafc3fd41f75 1859 parser.add_argument("-S", "--supported-toolchains",
bryantaylor 0:eafc3fd41f75 1860 action="store_true",
bryantaylor 0:eafc3fd41f75 1861 dest="supported_toolchains",
bryantaylor 0:eafc3fd41f75 1862 default=False,
bryantaylor 0:eafc3fd41f75 1863 help="Displays supported matrix of MCUs and toolchains")
bryantaylor 0:eafc3fd41f75 1864
bryantaylor 0:eafc3fd41f75 1865 parser.add_argument("-O", "--only-build",
bryantaylor 0:eafc3fd41f75 1866 action="store_true",
bryantaylor 0:eafc3fd41f75 1867 dest="only_build_tests",
bryantaylor 0:eafc3fd41f75 1868 default=False,
bryantaylor 0:eafc3fd41f75 1869 help="Only build tests, skips actual test procedures (flashing etc.)")
bryantaylor 0:eafc3fd41f75 1870
bryantaylor 0:eafc3fd41f75 1871 parser.add_argument('--parallel',
bryantaylor 0:eafc3fd41f75 1872 dest='parallel_test_exec',
bryantaylor 0:eafc3fd41f75 1873 default=False,
bryantaylor 0:eafc3fd41f75 1874 action="store_true",
bryantaylor 0:eafc3fd41f75 1875 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
bryantaylor 0:eafc3fd41f75 1876
bryantaylor 0:eafc3fd41f75 1877 parser.add_argument('--config',
bryantaylor 0:eafc3fd41f75 1878 dest='verbose_test_configuration_only',
bryantaylor 0:eafc3fd41f75 1879 default=False,
bryantaylor 0:eafc3fd41f75 1880 action="store_true",
bryantaylor 0:eafc3fd41f75 1881 help='Displays full test specification and MUTs configration and exits')
bryantaylor 0:eafc3fd41f75 1882
bryantaylor 0:eafc3fd41f75 1883 parser.add_argument('--loops',
bryantaylor 0:eafc3fd41f75 1884 dest='test_loops_list',
bryantaylor 0:eafc3fd41f75 1885 type=argparse_many(str),
bryantaylor 0:eafc3fd41f75 1886 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
bryantaylor 0:eafc3fd41f75 1887
bryantaylor 0:eafc3fd41f75 1888 parser.add_argument('--global-loops',
bryantaylor 0:eafc3fd41f75 1889 dest='test_global_loops_value',
bryantaylor 0:eafc3fd41f75 1890 type=int,
bryantaylor 0:eafc3fd41f75 1891 help='Set global number of test loops per test. Default value is set 1')
bryantaylor 0:eafc3fd41f75 1892
bryantaylor 0:eafc3fd41f75 1893 parser.add_argument('--consolidate-waterfall',
bryantaylor 0:eafc3fd41f75 1894 dest='consolidate_waterfall_test',
bryantaylor 0:eafc3fd41f75 1895 default=False,
bryantaylor 0:eafc3fd41f75 1896 action="store_true",
bryantaylor 0:eafc3fd41f75 1897 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
bryantaylor 0:eafc3fd41f75 1898
bryantaylor 0:eafc3fd41f75 1899 parser.add_argument('-W', '--waterfall',
bryantaylor 0:eafc3fd41f75 1900 dest='waterfall_test',
bryantaylor 0:eafc3fd41f75 1901 default=False,
bryantaylor 0:eafc3fd41f75 1902 action="store_true",
bryantaylor 0:eafc3fd41f75 1903 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
bryantaylor 0:eafc3fd41f75 1904
bryantaylor 0:eafc3fd41f75 1905 parser.add_argument('-N', '--firmware-name',
bryantaylor 0:eafc3fd41f75 1906 dest='firmware_global_name',
bryantaylor 0:eafc3fd41f75 1907 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
bryantaylor 0:eafc3fd41f75 1908
bryantaylor 0:eafc3fd41f75 1909 parser.add_argument('-u', '--shuffle',
bryantaylor 0:eafc3fd41f75 1910 dest='shuffle_test_order',
bryantaylor 0:eafc3fd41f75 1911 default=False,
bryantaylor 0:eafc3fd41f75 1912 action="store_true",
bryantaylor 0:eafc3fd41f75 1913 help='Shuffles test execution order')
bryantaylor 0:eafc3fd41f75 1914
bryantaylor 0:eafc3fd41f75 1915 parser.add_argument('--shuffle-seed',
bryantaylor 0:eafc3fd41f75 1916 dest='shuffle_test_seed',
bryantaylor 0:eafc3fd41f75 1917 default=None,
bryantaylor 0:eafc3fd41f75 1918 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
bryantaylor 0:eafc3fd41f75 1919
bryantaylor 0:eafc3fd41f75 1920 parser.add_argument('-f', '--filter',
bryantaylor 0:eafc3fd41f75 1921 dest='general_filter_regex',
bryantaylor 0:eafc3fd41f75 1922 type=argparse_many(str),
bryantaylor 0:eafc3fd41f75 1923 default=None,
bryantaylor 0:eafc3fd41f75 1924 help='For some commands you can use filter to filter out results')
bryantaylor 0:eafc3fd41f75 1925
bryantaylor 0:eafc3fd41f75 1926 parser.add_argument('--inc-timeout',
bryantaylor 0:eafc3fd41f75 1927 dest='extend_test_timeout',
bryantaylor 0:eafc3fd41f75 1928 metavar="NUMBER",
bryantaylor 0:eafc3fd41f75 1929 type=int,
bryantaylor 0:eafc3fd41f75 1930 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
bryantaylor 0:eafc3fd41f75 1931
bryantaylor 0:eafc3fd41f75 1932 parser.add_argument('--db',
bryantaylor 0:eafc3fd41f75 1933 dest='db_url',
bryantaylor 0:eafc3fd41f75 1934 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
bryantaylor 0:eafc3fd41f75 1935
bryantaylor 0:eafc3fd41f75 1936 parser.add_argument('-l', '--log',
bryantaylor 0:eafc3fd41f75 1937 dest='log_file_name',
bryantaylor 0:eafc3fd41f75 1938 help='Log events to external file (note not all console entries may be visible in log file)')
bryantaylor 0:eafc3fd41f75 1939
bryantaylor 0:eafc3fd41f75 1940 parser.add_argument('--report-html',
bryantaylor 0:eafc3fd41f75 1941 dest='report_html_file_name',
bryantaylor 0:eafc3fd41f75 1942 help='You can log test suite results in form of HTML report')
bryantaylor 0:eafc3fd41f75 1943
bryantaylor 0:eafc3fd41f75 1944 parser.add_argument('--report-junit',
bryantaylor 0:eafc3fd41f75 1945 dest='report_junit_file_name',
bryantaylor 0:eafc3fd41f75 1946 help='You can log test suite results in form of JUnit compliant XML report')
bryantaylor 0:eafc3fd41f75 1947
bryantaylor 0:eafc3fd41f75 1948 parser.add_argument("--report-build",
bryantaylor 0:eafc3fd41f75 1949 dest="report_build_file_name",
bryantaylor 0:eafc3fd41f75 1950 help="Output the build results to a junit xml file")
bryantaylor 0:eafc3fd41f75 1951
bryantaylor 0:eafc3fd41f75 1952 parser.add_argument("--report-text",
bryantaylor 0:eafc3fd41f75 1953 dest="report_text_file_name",
bryantaylor 0:eafc3fd41f75 1954 help="Output the build results to a text file")
bryantaylor 0:eafc3fd41f75 1955
bryantaylor 0:eafc3fd41f75 1956 parser.add_argument('--verbose-skipped',
bryantaylor 0:eafc3fd41f75 1957 dest='verbose_skipped_tests',
bryantaylor 0:eafc3fd41f75 1958 default=False,
bryantaylor 0:eafc3fd41f75 1959 action="store_true",
bryantaylor 0:eafc3fd41f75 1960 help='Prints some extra information about skipped tests')
bryantaylor 0:eafc3fd41f75 1961
bryantaylor 0:eafc3fd41f75 1962 parser.add_argument('-V', '--verbose-test-result',
bryantaylor 0:eafc3fd41f75 1963 dest='verbose_test_result_only',
bryantaylor 0:eafc3fd41f75 1964 default=False,
bryantaylor 0:eafc3fd41f75 1965 action="store_true",
bryantaylor 0:eafc3fd41f75 1966 help='Prints test serial output')
bryantaylor 0:eafc3fd41f75 1967
bryantaylor 0:eafc3fd41f75 1968 parser.add_argument('-v', '--verbose',
bryantaylor 0:eafc3fd41f75 1969 dest='verbose',
bryantaylor 0:eafc3fd41f75 1970 default=False,
bryantaylor 0:eafc3fd41f75 1971 action="store_true",
bryantaylor 0:eafc3fd41f75 1972 help='Verbose mode (prints some extra information)')
bryantaylor 0:eafc3fd41f75 1973
bryantaylor 0:eafc3fd41f75 1974 parser.add_argument('--version',
bryantaylor 0:eafc3fd41f75 1975 dest='version',
bryantaylor 0:eafc3fd41f75 1976 default=False,
bryantaylor 0:eafc3fd41f75 1977 action="store_true",
bryantaylor 0:eafc3fd41f75 1978 help='Prints script version and exits')
bryantaylor 0:eafc3fd41f75 1979 return parser
bryantaylor 0:eafc3fd41f75 1980
bryantaylor 0:eafc3fd41f75 1981 def test_path_to_name(path, base):
bryantaylor 0:eafc3fd41f75 1982 """Change all slashes in a path into hyphens
bryantaylor 0:eafc3fd41f75 1983 This creates a unique cross-platform test name based on the path
bryantaylor 0:eafc3fd41f75 1984 This can eventually be overriden by a to-be-determined meta-data mechanism"""
bryantaylor 0:eafc3fd41f75 1985 name_parts = []
bryantaylor 0:eafc3fd41f75 1986 head, tail = os.path.split(relpath(path,base))
bryantaylor 0:eafc3fd41f75 1987 while (tail and tail != "."):
bryantaylor 0:eafc3fd41f75 1988 name_parts.insert(0, tail)
bryantaylor 0:eafc3fd41f75 1989 head, tail = os.path.split(head)
bryantaylor 0:eafc3fd41f75 1990
bryantaylor 0:eafc3fd41f75 1991 return "-".join(name_parts).lower()
bryantaylor 0:eafc3fd41f75 1992
bryantaylor 0:eafc3fd41f75 1993 def find_tests(base_dir, target_name, toolchain_name, options=None, app_config=None):
bryantaylor 0:eafc3fd41f75 1994 """ Finds all tests in a directory recursively
bryantaylor 0:eafc3fd41f75 1995 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
bryantaylor 0:eafc3fd41f75 1996 target_name: name of the target to use for scanning (ex. 'K64F')
bryantaylor 0:eafc3fd41f75 1997 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
bryantaylor 0:eafc3fd41f75 1998 options: Compile options to pass to the toolchain (ex. ['debug-info'])
bryantaylor 0:eafc3fd41f75 1999 app_config - location of a chosen mbed_app.json file
bryantaylor 0:eafc3fd41f75 2000 """
bryantaylor 0:eafc3fd41f75 2001
bryantaylor 0:eafc3fd41f75 2002 tests = {}
bryantaylor 0:eafc3fd41f75 2003
bryantaylor 0:eafc3fd41f75 2004 # Prepare the toolchain
bryantaylor 0:eafc3fd41f75 2005 toolchain = prepare_toolchain([base_dir], target_name, toolchain_name, options=options,
bryantaylor 0:eafc3fd41f75 2006 silent=True, app_config=app_config)
bryantaylor 0:eafc3fd41f75 2007
bryantaylor 0:eafc3fd41f75 2008 # Scan the directory for paths to probe for 'TESTS' folders
bryantaylor 0:eafc3fd41f75 2009 base_resources = scan_resources([base_dir], toolchain)
bryantaylor 0:eafc3fd41f75 2010
bryantaylor 0:eafc3fd41f75 2011 dirs = base_resources.inc_dirs
bryantaylor 0:eafc3fd41f75 2012 for directory in dirs:
bryantaylor 0:eafc3fd41f75 2013 subdirs = os.listdir(directory)
bryantaylor 0:eafc3fd41f75 2014
bryantaylor 0:eafc3fd41f75 2015 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
bryantaylor 0:eafc3fd41f75 2016 if 'TESTS' in subdirs:
bryantaylor 0:eafc3fd41f75 2017 walk_base_dir = join(directory, 'TESTS')
bryantaylor 0:eafc3fd41f75 2018 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
bryantaylor 0:eafc3fd41f75 2019
bryantaylor 0:eafc3fd41f75 2020 # Loop through all subdirectories
bryantaylor 0:eafc3fd41f75 2021 for d in test_resources.inc_dirs:
bryantaylor 0:eafc3fd41f75 2022
bryantaylor 0:eafc3fd41f75 2023 # If the test case folder is not called 'host_tests' and it is
bryantaylor 0:eafc3fd41f75 2024 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
bryantaylor 0:eafc3fd41f75 2025 # then add it to the tests
bryantaylor 0:eafc3fd41f75 2026 path_depth = get_path_depth(relpath(d, walk_base_dir))
bryantaylor 0:eafc3fd41f75 2027 if path_depth == 2:
bryantaylor 0:eafc3fd41f75 2028 test_group_directory_path, test_case_directory = os.path.split(d)
bryantaylor 0:eafc3fd41f75 2029 test_group_directory = os.path.basename(test_group_directory_path)
bryantaylor 0:eafc3fd41f75 2030
bryantaylor 0:eafc3fd41f75 2031 # Check to make sure discoverd folder is not in a host test directory
bryantaylor 0:eafc3fd41f75 2032 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
bryantaylor 0:eafc3fd41f75 2033 test_name = test_path_to_name(d, base_dir)
bryantaylor 0:eafc3fd41f75 2034 tests[test_name] = d
bryantaylor 0:eafc3fd41f75 2035
bryantaylor 0:eafc3fd41f75 2036 return tests
bryantaylor 0:eafc3fd41f75 2037
bryantaylor 0:eafc3fd41f75 2038 def print_tests(tests, format="list", sort=True):
bryantaylor 0:eafc3fd41f75 2039 """Given a dictionary of tests (as returned from "find_tests"), print them
bryantaylor 0:eafc3fd41f75 2040 in the specified format"""
bryantaylor 0:eafc3fd41f75 2041 if format == "list":
bryantaylor 0:eafc3fd41f75 2042 for test_name in sorted(tests.keys()):
bryantaylor 0:eafc3fd41f75 2043 test_path = tests[test_name]
bryantaylor 0:eafc3fd41f75 2044 print "Test Case:"
bryantaylor 0:eafc3fd41f75 2045 print " Name: %s" % test_name
bryantaylor 0:eafc3fd41f75 2046 print " Path: %s" % test_path
bryantaylor 0:eafc3fd41f75 2047 elif format == "json":
bryantaylor 0:eafc3fd41f75 2048 print json.dumps(tests, indent=2)
bryantaylor 0:eafc3fd41f75 2049 else:
bryantaylor 0:eafc3fd41f75 2050 print "Unknown format '%s'" % format
bryantaylor 0:eafc3fd41f75 2051 sys.exit(1)
bryantaylor 0:eafc3fd41f75 2052
bryantaylor 0:eafc3fd41f75 2053 def norm_relative_path(path, start):
bryantaylor 0:eafc3fd41f75 2054 """This function will create a normalized, relative path. It mimics the
bryantaylor 0:eafc3fd41f75 2055 python os.path.relpath function, but also normalizes a Windows-syle path
bryantaylor 0:eafc3fd41f75 2056 that use backslashes to a Unix style path that uses forward slashes."""
bryantaylor 0:eafc3fd41f75 2057 path = os.path.normpath(path)
bryantaylor 0:eafc3fd41f75 2058 path = os.path.relpath(path, start)
bryantaylor 0:eafc3fd41f75 2059 path = path.replace("\\", "/")
bryantaylor 0:eafc3fd41f75 2060 return path
bryantaylor 0:eafc3fd41f75 2061
bryantaylor 0:eafc3fd41f75 2062 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
bryantaylor 0:eafc3fd41f75 2063 options=None, clean=False, notify=None, verbose=False, jobs=1,
bryantaylor 0:eafc3fd41f75 2064 macros=None, silent=False, report=None, properties=None,
bryantaylor 0:eafc3fd41f75 2065 continue_on_build_fail=False, app_config=None):
bryantaylor 0:eafc3fd41f75 2066 """Given the data structure from 'find_tests' and the typical build parameters,
bryantaylor 0:eafc3fd41f75 2067 build all the tests
bryantaylor 0:eafc3fd41f75 2068
bryantaylor 0:eafc3fd41f75 2069 Returns a tuple of the build result (True or False) followed by the test
bryantaylor 0:eafc3fd41f75 2070 build data structure"""
bryantaylor 0:eafc3fd41f75 2071
bryantaylor 0:eafc3fd41f75 2072 execution_directory = "."
bryantaylor 0:eafc3fd41f75 2073 base_path = norm_relative_path(build_path, execution_directory)
bryantaylor 0:eafc3fd41f75 2074
bryantaylor 0:eafc3fd41f75 2075 target_name = target if isinstance(target, str) else target.name
bryantaylor 0:eafc3fd41f75 2076
bryantaylor 0:eafc3fd41f75 2077 test_build = {
bryantaylor 0:eafc3fd41f75 2078 "platform": target_name,
bryantaylor 0:eafc3fd41f75 2079 "toolchain": toolchain_name,
bryantaylor 0:eafc3fd41f75 2080 "base_path": base_path,
bryantaylor 0:eafc3fd41f75 2081 "baud_rate": 9600,
bryantaylor 0:eafc3fd41f75 2082 "binary_type": "bootable",
bryantaylor 0:eafc3fd41f75 2083 "tests": {}
bryantaylor 0:eafc3fd41f75 2084 }
bryantaylor 0:eafc3fd41f75 2085
bryantaylor 0:eafc3fd41f75 2086 result = True
bryantaylor 0:eafc3fd41f75 2087
bryantaylor 0:eafc3fd41f75 2088 map_outputs_total = list()
bryantaylor 0:eafc3fd41f75 2089 for test_name, test_path in tests.iteritems():
bryantaylor 0:eafc3fd41f75 2090 test_build_path = os.path.join(build_path, test_path)
bryantaylor 0:eafc3fd41f75 2091 src_path = base_source_paths + [test_path]
bryantaylor 0:eafc3fd41f75 2092 bin_file = None
bryantaylor 0:eafc3fd41f75 2093 test_case_folder_name = os.path.basename(test_path)
bryantaylor 0:eafc3fd41f75 2094
bryantaylor 0:eafc3fd41f75 2095
bryantaylor 0:eafc3fd41f75 2096 try:
bryantaylor 0:eafc3fd41f75 2097 bin_file = build_project(src_path, test_build_path, target, toolchain_name,
bryantaylor 0:eafc3fd41f75 2098 options=options,
bryantaylor 0:eafc3fd41f75 2099 jobs=jobs,
bryantaylor 0:eafc3fd41f75 2100 clean=clean,
bryantaylor 0:eafc3fd41f75 2101 macros=macros,
bryantaylor 0:eafc3fd41f75 2102 name=test_case_folder_name,
bryantaylor 0:eafc3fd41f75 2103 project_id=test_name,
bryantaylor 0:eafc3fd41f75 2104 report=report,
bryantaylor 0:eafc3fd41f75 2105 properties=properties,
bryantaylor 0:eafc3fd41f75 2106 verbose=verbose,
bryantaylor 0:eafc3fd41f75 2107 app_config=app_config)
bryantaylor 0:eafc3fd41f75 2108
bryantaylor 0:eafc3fd41f75 2109 except Exception, e:
bryantaylor 0:eafc3fd41f75 2110 if not isinstance(e, NotSupportedException):
bryantaylor 0:eafc3fd41f75 2111 result = False
bryantaylor 0:eafc3fd41f75 2112
bryantaylor 0:eafc3fd41f75 2113 if continue_on_build_fail:
bryantaylor 0:eafc3fd41f75 2114 continue
bryantaylor 0:eafc3fd41f75 2115 else:
bryantaylor 0:eafc3fd41f75 2116 break
bryantaylor 0:eafc3fd41f75 2117
bryantaylor 0:eafc3fd41f75 2118 # If a clean build was carried out last time, disable it for the next build.
bryantaylor 0:eafc3fd41f75 2119 # Otherwise the previously built test will be deleted.
bryantaylor 0:eafc3fd41f75 2120 if clean:
bryantaylor 0:eafc3fd41f75 2121 clean = False
bryantaylor 0:eafc3fd41f75 2122
bryantaylor 0:eafc3fd41f75 2123 # Normalize the path
bryantaylor 0:eafc3fd41f75 2124 if bin_file:
bryantaylor 0:eafc3fd41f75 2125 bin_file = norm_relative_path(bin_file, execution_directory)
bryantaylor 0:eafc3fd41f75 2126
bryantaylor 0:eafc3fd41f75 2127 test_build['tests'][test_name] = {
bryantaylor 0:eafc3fd41f75 2128 "binaries": [
bryantaylor 0:eafc3fd41f75 2129 {
bryantaylor 0:eafc3fd41f75 2130 "path": bin_file
bryantaylor 0:eafc3fd41f75 2131 }
bryantaylor 0:eafc3fd41f75 2132 ]
bryantaylor 0:eafc3fd41f75 2133 }
bryantaylor 0:eafc3fd41f75 2134
bryantaylor 0:eafc3fd41f75 2135 print 'Image: %s'% bin_file
bryantaylor 0:eafc3fd41f75 2136
bryantaylor 0:eafc3fd41f75 2137 test_builds = {}
bryantaylor 0:eafc3fd41f75 2138 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
bryantaylor 0:eafc3fd41f75 2139
bryantaylor 0:eafc3fd41f75 2140
bryantaylor 0:eafc3fd41f75 2141 return result, test_builds
bryantaylor 0:eafc3fd41f75 2142
bryantaylor 0:eafc3fd41f75 2143
bryantaylor 0:eafc3fd41f75 2144 def test_spec_from_test_builds(test_builds):
bryantaylor 0:eafc3fd41f75 2145 return {
bryantaylor 0:eafc3fd41f75 2146 "builds": test_builds
bryantaylor 0:eafc3fd41f75 2147 }