takashi kadono / Mbed OS Nucleo446_SSD1331

Color Oled(SSD1331) connect to STMicroelectronics Nucleo-F466

Dependencies:   ssd1331

mbed-os/tools/test_api.py@0:8fdf9a60065b, 2018-10-10 (annotated)

Committer:
kadonotakashi
Date:
Wed Oct 10 00:33:53 2018 +0000
Revision:
0:8fdf9a60065b
how to make mbed librry

Who changed what in which revision?

UserRevisionLine numberNew contents of line
kadonotakashi 0:8fdf9a60065b 1 """
kadonotakashi 0:8fdf9a60065b 2 mbed SDK
kadonotakashi 0:8fdf9a60065b 3 Copyright (c) 2011-2014 ARM Limited
kadonotakashi 0:8fdf9a60065b 4
kadonotakashi 0:8fdf9a60065b 5 Licensed under the Apache License, Version 2.0 (the "License");
kadonotakashi 0:8fdf9a60065b 6 you may not use this file except in compliance with the License.
kadonotakashi 0:8fdf9a60065b 7 You may obtain a copy of the License at
kadonotakashi 0:8fdf9a60065b 8
kadonotakashi 0:8fdf9a60065b 9 http://www.apache.org/licenses/LICENSE-2.0
kadonotakashi 0:8fdf9a60065b 10
kadonotakashi 0:8fdf9a60065b 11 Unless required by applicable law or agreed to in writing, software
kadonotakashi 0:8fdf9a60065b 12 distributed under the License is distributed on an "AS IS" BASIS,
kadonotakashi 0:8fdf9a60065b 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
kadonotakashi 0:8fdf9a60065b 14 See the License for the specific language governing permissions and
kadonotakashi 0:8fdf9a60065b 15 limitations under the License.
kadonotakashi 0:8fdf9a60065b 16
kadonotakashi 0:8fdf9a60065b 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
kadonotakashi 0:8fdf9a60065b 18 """
kadonotakashi 0:8fdf9a60065b 19 from __future__ import print_function
kadonotakashi 0:8fdf9a60065b 20 import six
kadonotakashi 0:8fdf9a60065b 21
kadonotakashi 0:8fdf9a60065b 22 import os
kadonotakashi 0:8fdf9a60065b 23 import re
kadonotakashi 0:8fdf9a60065b 24 import sys
kadonotakashi 0:8fdf9a60065b 25 import json
kadonotakashi 0:8fdf9a60065b 26 import uuid
kadonotakashi 0:8fdf9a60065b 27 import pprint
kadonotakashi 0:8fdf9a60065b 28 import random
kadonotakashi 0:8fdf9a60065b 29 import argparse
kadonotakashi 0:8fdf9a60065b 30 import datetime
kadonotakashi 0:8fdf9a60065b 31 import threading
kadonotakashi 0:8fdf9a60065b 32 import ctypes
kadonotakashi 0:8fdf9a60065b 33 import functools
kadonotakashi 0:8fdf9a60065b 34 from colorama import Fore, Back, Style
kadonotakashi 0:8fdf9a60065b 35 from prettytable import PrettyTable, HEADER
kadonotakashi 0:8fdf9a60065b 36 from copy import copy, deepcopy
kadonotakashi 0:8fdf9a60065b 37
kadonotakashi 0:8fdf9a60065b 38 from time import sleep, time
kadonotakashi 0:8fdf9a60065b 39 try:
kadonotakashi 0:8fdf9a60065b 40 from Queue import Queue, Empty
kadonotakashi 0:8fdf9a60065b 41 except ImportError:
kadonotakashi 0:8fdf9a60065b 42 from queue import Queue, Empty
kadonotakashi 0:8fdf9a60065b 43 from os.path import join, exists, basename, relpath, isdir, isfile
kadonotakashi 0:8fdf9a60065b 44 from threading import Thread, Lock
kadonotakashi 0:8fdf9a60065b 45 from multiprocessing import Pool, cpu_count
kadonotakashi 0:8fdf9a60065b 46 from subprocess import Popen, PIPE
kadonotakashi 0:8fdf9a60065b 47
kadonotakashi 0:8fdf9a60065b 48 # Imports related to mbed build api
kadonotakashi 0:8fdf9a60065b 49 from tools.tests import TESTS
kadonotakashi 0:8fdf9a60065b 50 from tools.tests import TEST_MAP
kadonotakashi 0:8fdf9a60065b 51 from tools.paths import BUILD_DIR
kadonotakashi 0:8fdf9a60065b 52 from tools.paths import HOST_TESTS
kadonotakashi 0:8fdf9a60065b 53 from tools.utils import ToolException
kadonotakashi 0:8fdf9a60065b 54 from tools.utils import NotSupportedException
kadonotakashi 0:8fdf9a60065b 55 from tools.utils import construct_enum
kadonotakashi 0:8fdf9a60065b 56 from tools.memap import MemapParser
kadonotakashi 0:8fdf9a60065b 57 from tools.targets import TARGET_MAP, Target
kadonotakashi 0:8fdf9a60065b 58 from tools.config import Config
kadonotakashi 0:8fdf9a60065b 59 import tools.test_configs as TestConfig
kadonotakashi 0:8fdf9a60065b 60 from tools.test_db import BaseDBAccess
kadonotakashi 0:8fdf9a60065b 61 from tools.build_api import build_project, build_mbed_libs, build_lib
kadonotakashi 0:8fdf9a60065b 62 from tools.build_api import get_target_supported_toolchains
kadonotakashi 0:8fdf9a60065b 63 from tools.build_api import write_build_report
kadonotakashi 0:8fdf9a60065b 64 from tools.build_api import prep_report
kadonotakashi 0:8fdf9a60065b 65 from tools.build_api import prep_properties
kadonotakashi 0:8fdf9a60065b 66 from tools.build_api import create_result
kadonotakashi 0:8fdf9a60065b 67 from tools.build_api import add_result_to_report
kadonotakashi 0:8fdf9a60065b 68 from tools.build_api import prepare_toolchain
kadonotakashi 0:8fdf9a60065b 69 from tools.build_api import get_config
kadonotakashi 0:8fdf9a60065b 70 from tools.resources import Resources, MbedIgnoreSet, IGNORE_FILENAME
kadonotakashi 0:8fdf9a60065b 71 from tools.libraries import LIBRARIES, LIBRARY_MAP
kadonotakashi 0:8fdf9a60065b 72 from tools.options import extract_profile
kadonotakashi 0:8fdf9a60065b 73 from tools.toolchains import TOOLCHAIN_PATHS
kadonotakashi 0:8fdf9a60065b 74 from tools.toolchains import TOOLCHAINS
kadonotakashi 0:8fdf9a60065b 75 from tools.test_exporters import ReportExporter, ResultExporterType
kadonotakashi 0:8fdf9a60065b 76 from tools.utils import argparse_filestring_type
kadonotakashi 0:8fdf9a60065b 77 from tools.utils import argparse_uppercase_type
kadonotakashi 0:8fdf9a60065b 78 from tools.utils import argparse_lowercase_type
kadonotakashi 0:8fdf9a60065b 79 from tools.utils import argparse_many
kadonotakashi 0:8fdf9a60065b 80 from tools.notifier.mock import MockNotifier
kadonotakashi 0:8fdf9a60065b 81 from tools.notifier.term import TerminalNotifier
kadonotakashi 0:8fdf9a60065b 82
kadonotakashi 0:8fdf9a60065b 83 import tools.host_tests.host_tests_plugins as host_tests_plugins
kadonotakashi 0:8fdf9a60065b 84
kadonotakashi 0:8fdf9a60065b 85 try:
kadonotakashi 0:8fdf9a60065b 86 import mbed_lstools
kadonotakashi 0:8fdf9a60065b 87 from tools.compliance.ioper_runner import get_available_oper_test_scopes
kadonotakashi 0:8fdf9a60065b 88 except:
kadonotakashi 0:8fdf9a60065b 89 pass
kadonotakashi 0:8fdf9a60065b 90
kadonotakashi 0:8fdf9a60065b 91
kadonotakashi 0:8fdf9a60065b 92 class ProcessObserver(Thread):
kadonotakashi 0:8fdf9a60065b 93 def __init__(self, proc):
kadonotakashi 0:8fdf9a60065b 94 Thread.__init__(self)
kadonotakashi 0:8fdf9a60065b 95 self.proc = proc
kadonotakashi 0:8fdf9a60065b 96 self.queue = Queue()
kadonotakashi 0:8fdf9a60065b 97 self.daemon = True
kadonotakashi 0:8fdf9a60065b 98 self.active = True
kadonotakashi 0:8fdf9a60065b 99 self.start()
kadonotakashi 0:8fdf9a60065b 100
kadonotakashi 0:8fdf9a60065b 101 def run(self):
kadonotakashi 0:8fdf9a60065b 102 while self.active:
kadonotakashi 0:8fdf9a60065b 103 c = self.proc.stdout.read(1)
kadonotakashi 0:8fdf9a60065b 104 self.queue.put(c)
kadonotakashi 0:8fdf9a60065b 105
kadonotakashi 0:8fdf9a60065b 106 def stop(self):
kadonotakashi 0:8fdf9a60065b 107 self.active = False
kadonotakashi 0:8fdf9a60065b 108 try:
kadonotakashi 0:8fdf9a60065b 109 self.proc.terminate()
kadonotakashi 0:8fdf9a60065b 110 except Exception:
kadonotakashi 0:8fdf9a60065b 111 pass
kadonotakashi 0:8fdf9a60065b 112
kadonotakashi 0:8fdf9a60065b 113
kadonotakashi 0:8fdf9a60065b 114 class SingleTestExecutor(threading.Thread):
kadonotakashi 0:8fdf9a60065b 115 """ Example: Single test class in separate thread usage
kadonotakashi 0:8fdf9a60065b 116 """
kadonotakashi 0:8fdf9a60065b 117 def __init__(self, single_test):
kadonotakashi 0:8fdf9a60065b 118 self.single_test = single_test
kadonotakashi 0:8fdf9a60065b 119 threading.Thread.__init__(self)
kadonotakashi 0:8fdf9a60065b 120
kadonotakashi 0:8fdf9a60065b 121 def run(self):
kadonotakashi 0:8fdf9a60065b 122 start = time()
kadonotakashi 0:8fdf9a60065b 123 # Execute tests depending on options and filter applied
kadonotakashi 0:8fdf9a60065b 124 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
kadonotakashi 0:8fdf9a60065b 125 elapsed_time = time() - start
kadonotakashi 0:8fdf9a60065b 126
kadonotakashi 0:8fdf9a60065b 127 # Human readable summary
kadonotakashi 0:8fdf9a60065b 128 if not self.single_test.opts_suppress_summary:
kadonotakashi 0:8fdf9a60065b 129 # prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 130 print(self.single_test.generate_test_summary(test_summary,
kadonotakashi 0:8fdf9a60065b 131 shuffle_seed))
kadonotakashi 0:8fdf9a60065b 132 if self.single_test.opts_test_x_toolchain_summary:
kadonotakashi 0:8fdf9a60065b 133 # prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 134 # table shows text x toolchain test result matrix
kadonotakashi 0:8fdf9a60065b 135 print(self.single_test.generate_test_summary_by_target(
kadonotakashi 0:8fdf9a60065b 136 test_summary, shuffle_seed))
kadonotakashi 0:8fdf9a60065b 137 print("Completed in %.2f sec"% (elapsed_time))
kadonotakashi 0:8fdf9a60065b 138
kadonotakashi 0:8fdf9a60065b 139
kadonotakashi 0:8fdf9a60065b 140 class SingleTestRunner(object):
kadonotakashi 0:8fdf9a60065b 141 """ Object wrapper for single test run which may involve multiple MUTs
kadonotakashi 0:8fdf9a60065b 142 """
kadonotakashi 0:8fdf9a60065b 143 RE_DETECT_TESTCASE_RESULT = None
kadonotakashi 0:8fdf9a60065b 144
kadonotakashi 0:8fdf9a60065b 145 # Return codes for test script
kadonotakashi 0:8fdf9a60065b 146 TEST_RESULT_OK = "OK"
kadonotakashi 0:8fdf9a60065b 147 TEST_RESULT_FAIL = "FAIL"
kadonotakashi 0:8fdf9a60065b 148 TEST_RESULT_ERROR = "ERROR"
kadonotakashi 0:8fdf9a60065b 149 TEST_RESULT_UNDEF = "UNDEF"
kadonotakashi 0:8fdf9a60065b 150 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
kadonotakashi 0:8fdf9a60065b 151 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
kadonotakashi 0:8fdf9a60065b 152 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
kadonotakashi 0:8fdf9a60065b 153 TEST_RESULT_TIMEOUT = "TIMEOUT"
kadonotakashi 0:8fdf9a60065b 154 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
kadonotakashi 0:8fdf9a60065b 155 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
kadonotakashi 0:8fdf9a60065b 156 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
kadonotakashi 0:8fdf9a60065b 157 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
kadonotakashi 0:8fdf9a60065b 158
kadonotakashi 0:8fdf9a60065b 159 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
kadonotakashi 0:8fdf9a60065b 160 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
kadonotakashi 0:8fdf9a60065b 161 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
kadonotakashi 0:8fdf9a60065b 162
kadonotakashi 0:8fdf9a60065b 163 muts = {} # MUTs descriptor (from external file)
kadonotakashi 0:8fdf9a60065b 164 test_spec = {} # Test specification (from external file)
kadonotakashi 0:8fdf9a60065b 165
kadonotakashi 0:8fdf9a60065b 166 # mbed test suite -> SingleTestRunner
kadonotakashi 0:8fdf9a60065b 167 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
kadonotakashi 0:8fdf9a60065b 168 "failure" : TEST_RESULT_FAIL,
kadonotakashi 0:8fdf9a60065b 169 "error" : TEST_RESULT_ERROR,
kadonotakashi 0:8fdf9a60065b 170 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
kadonotakashi 0:8fdf9a60065b 171 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
kadonotakashi 0:8fdf9a60065b 172 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
kadonotakashi 0:8fdf9a60065b 173 "timeout" : TEST_RESULT_TIMEOUT,
kadonotakashi 0:8fdf9a60065b 174 "no_image" : TEST_RESULT_NO_IMAGE,
kadonotakashi 0:8fdf9a60065b 175 "end" : TEST_RESULT_UNDEF,
kadonotakashi 0:8fdf9a60065b 176 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
kadonotakashi 0:8fdf9a60065b 177 "build_failed" : TEST_RESULT_BUILD_FAILED,
kadonotakashi 0:8fdf9a60065b 178 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
kadonotakashi 0:8fdf9a60065b 179 }
kadonotakashi 0:8fdf9a60065b 180
kadonotakashi 0:8fdf9a60065b 181 def __init__(self,
kadonotakashi 0:8fdf9a60065b 182 _global_loops_count=1,
kadonotakashi 0:8fdf9a60065b 183 _test_loops_list=None,
kadonotakashi 0:8fdf9a60065b 184 _muts={},
kadonotakashi 0:8fdf9a60065b 185 _clean=False,
kadonotakashi 0:8fdf9a60065b 186 _parser=None,
kadonotakashi 0:8fdf9a60065b 187 _opts=None,
kadonotakashi 0:8fdf9a60065b 188 _opts_db_url=None,
kadonotakashi 0:8fdf9a60065b 189 _opts_log_file_name=None,
kadonotakashi 0:8fdf9a60065b 190 _opts_report_html_file_name=None,
kadonotakashi 0:8fdf9a60065b 191 _opts_report_junit_file_name=None,
kadonotakashi 0:8fdf9a60065b 192 _opts_report_build_file_name=None,
kadonotakashi 0:8fdf9a60065b 193 _opts_report_text_file_name=None,
kadonotakashi 0:8fdf9a60065b 194 _opts_build_report={},
kadonotakashi 0:8fdf9a60065b 195 _opts_build_properties={},
kadonotakashi 0:8fdf9a60065b 196 _test_spec={},
kadonotakashi 0:8fdf9a60065b 197 _opts_goanna_for_mbed_sdk=None,
kadonotakashi 0:8fdf9a60065b 198 _opts_goanna_for_tests=None,
kadonotakashi 0:8fdf9a60065b 199 _opts_shuffle_test_order=False,
kadonotakashi 0:8fdf9a60065b 200 _opts_shuffle_test_seed=None,
kadonotakashi 0:8fdf9a60065b 201 _opts_test_by_names=None,
kadonotakashi 0:8fdf9a60065b 202 _opts_peripheral_by_names=None,
kadonotakashi 0:8fdf9a60065b 203 _opts_test_only_peripheral=False,
kadonotakashi 0:8fdf9a60065b 204 _opts_test_only_common=False,
kadonotakashi 0:8fdf9a60065b 205 _opts_verbose_skipped_tests=False,
kadonotakashi 0:8fdf9a60065b 206 _opts_verbose_test_result_only=False,
kadonotakashi 0:8fdf9a60065b 207 _opts_verbose=False,
kadonotakashi 0:8fdf9a60065b 208 _opts_firmware_global_name=None,
kadonotakashi 0:8fdf9a60065b 209 _opts_only_build_tests=False,
kadonotakashi 0:8fdf9a60065b 210 _opts_parallel_test_exec=False,
kadonotakashi 0:8fdf9a60065b 211 _opts_suppress_summary=False,
kadonotakashi 0:8fdf9a60065b 212 _opts_test_x_toolchain_summary=False,
kadonotakashi 0:8fdf9a60065b 213 _opts_copy_method=None,
kadonotakashi 0:8fdf9a60065b 214 _opts_mut_reset_type=None,
kadonotakashi 0:8fdf9a60065b 215 _opts_jobs=None,
kadonotakashi 0:8fdf9a60065b 216 _opts_waterfall_test=None,
kadonotakashi 0:8fdf9a60065b 217 _opts_consolidate_waterfall_test=None,
kadonotakashi 0:8fdf9a60065b 218 _opts_extend_test_timeout=None,
kadonotakashi 0:8fdf9a60065b 219 _opts_auto_detect=None,
kadonotakashi 0:8fdf9a60065b 220 _opts_include_non_automated=False):
kadonotakashi 0:8fdf9a60065b 221 """ Let's try hard to init this object
kadonotakashi 0:8fdf9a60065b 222 """
kadonotakashi 0:8fdf9a60065b 223 from colorama import init
kadonotakashi 0:8fdf9a60065b 224 init()
kadonotakashi 0:8fdf9a60065b 225
kadonotakashi 0:8fdf9a60065b 226 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
kadonotakashi 0:8fdf9a60065b 227 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
kadonotakashi 0:8fdf9a60065b 228 # Settings related to test loops counters
kadonotakashi 0:8fdf9a60065b 229 try:
kadonotakashi 0:8fdf9a60065b 230 _global_loops_count = int(_global_loops_count)
kadonotakashi 0:8fdf9a60065b 231 except:
kadonotakashi 0:8fdf9a60065b 232 _global_loops_count = 1
kadonotakashi 0:8fdf9a60065b 233 if _global_loops_count < 1:
kadonotakashi 0:8fdf9a60065b 234 _global_loops_count = 1
kadonotakashi 0:8fdf9a60065b 235 self.GLOBAL_LOOPS_COUNT = _global_loops_count
kadonotakashi 0:8fdf9a60065b 236 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
kadonotakashi 0:8fdf9a60065b 237 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
kadonotakashi 0:8fdf9a60065b 238
kadonotakashi 0:8fdf9a60065b 239 self.shuffle_random_seed = 0.0
kadonotakashi 0:8fdf9a60065b 240 self.SHUFFLE_SEED_ROUND = 10
kadonotakashi 0:8fdf9a60065b 241
kadonotakashi 0:8fdf9a60065b 242 # MUT list and test specification storage
kadonotakashi 0:8fdf9a60065b 243 self.muts = _muts
kadonotakashi 0:8fdf9a60065b 244 self.test_spec = _test_spec
kadonotakashi 0:8fdf9a60065b 245
kadonotakashi 0:8fdf9a60065b 246 # Settings passed e.g. from command line
kadonotakashi 0:8fdf9a60065b 247 self.opts_db_url = _opts_db_url
kadonotakashi 0:8fdf9a60065b 248 self.opts_log_file_name = _opts_log_file_name
kadonotakashi 0:8fdf9a60065b 249 self.opts_report_html_file_name = _opts_report_html_file_name
kadonotakashi 0:8fdf9a60065b 250 self.opts_report_junit_file_name = _opts_report_junit_file_name
kadonotakashi 0:8fdf9a60065b 251 self.opts_report_build_file_name = _opts_report_build_file_name
kadonotakashi 0:8fdf9a60065b 252 self.opts_report_text_file_name = _opts_report_text_file_name
kadonotakashi 0:8fdf9a60065b 253 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
kadonotakashi 0:8fdf9a60065b 254 self.opts_goanna_for_tests = _opts_goanna_for_tests
kadonotakashi 0:8fdf9a60065b 255 self.opts_shuffle_test_order = _opts_shuffle_test_order
kadonotakashi 0:8fdf9a60065b 256 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
kadonotakashi 0:8fdf9a60065b 257 self.opts_test_by_names = _opts_test_by_names
kadonotakashi 0:8fdf9a60065b 258 self.opts_peripheral_by_names = _opts_peripheral_by_names
kadonotakashi 0:8fdf9a60065b 259 self.opts_test_only_peripheral = _opts_test_only_peripheral
kadonotakashi 0:8fdf9a60065b 260 self.opts_test_only_common = _opts_test_only_common
kadonotakashi 0:8fdf9a60065b 261 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
kadonotakashi 0:8fdf9a60065b 262 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
kadonotakashi 0:8fdf9a60065b 263 self.opts_verbose = _opts_verbose
kadonotakashi 0:8fdf9a60065b 264 self.opts_firmware_global_name = _opts_firmware_global_name
kadonotakashi 0:8fdf9a60065b 265 self.opts_only_build_tests = _opts_only_build_tests
kadonotakashi 0:8fdf9a60065b 266 self.opts_parallel_test_exec = _opts_parallel_test_exec
kadonotakashi 0:8fdf9a60065b 267 self.opts_suppress_summary = _opts_suppress_summary
kadonotakashi 0:8fdf9a60065b 268 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
kadonotakashi 0:8fdf9a60065b 269 self.opts_copy_method = _opts_copy_method
kadonotakashi 0:8fdf9a60065b 270 self.opts_mut_reset_type = _opts_mut_reset_type
kadonotakashi 0:8fdf9a60065b 271 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
kadonotakashi 0:8fdf9a60065b 272 self.opts_waterfall_test = _opts_waterfall_test
kadonotakashi 0:8fdf9a60065b 273 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
kadonotakashi 0:8fdf9a60065b 274 self.opts_extend_test_timeout = _opts_extend_test_timeout
kadonotakashi 0:8fdf9a60065b 275 self.opts_clean = _clean
kadonotakashi 0:8fdf9a60065b 276 self.opts_parser = _parser
kadonotakashi 0:8fdf9a60065b 277 self.opts = _opts
kadonotakashi 0:8fdf9a60065b 278 self.opts_auto_detect = _opts_auto_detect
kadonotakashi 0:8fdf9a60065b 279 self.opts_include_non_automated = _opts_include_non_automated
kadonotakashi 0:8fdf9a60065b 280
kadonotakashi 0:8fdf9a60065b 281 self.build_report = _opts_build_report
kadonotakashi 0:8fdf9a60065b 282 self.build_properties = _opts_build_properties
kadonotakashi 0:8fdf9a60065b 283
kadonotakashi 0:8fdf9a60065b 284 # File / screen logger initialization
kadonotakashi 0:8fdf9a60065b 285 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
kadonotakashi 0:8fdf9a60065b 286
kadonotakashi 0:8fdf9a60065b 287 # Database related initializations
kadonotakashi 0:8fdf9a60065b 288 self.db_logger = factory_db_logger(self.opts_db_url)
kadonotakashi 0:8fdf9a60065b 289 self.db_logger_build_id = None # Build ID (database index of build_id table)
kadonotakashi 0:8fdf9a60065b 290 # Let's connect to database to set up credentials and confirm database is ready
kadonotakashi 0:8fdf9a60065b 291 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 292 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
kadonotakashi 0:8fdf9a60065b 293 if self.db_logger.is_connected():
kadonotakashi 0:8fdf9a60065b 294 # Get hostname and uname so we can use it as build description
kadonotakashi 0:8fdf9a60065b 295 # when creating new build_id in external database
kadonotakashi 0:8fdf9a60065b 296 (_hostname, _uname) = self.db_logger.get_hostname()
kadonotakashi 0:8fdf9a60065b 297 _host_location = os.path.dirname(os.path.abspath(__file__))
kadonotakashi 0:8fdf9a60065b 298 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
kadonotakashi 0:8fdf9a60065b 299 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
kadonotakashi 0:8fdf9a60065b 300 self.db_logger.disconnect()
kadonotakashi 0:8fdf9a60065b 301
kadonotakashi 0:8fdf9a60065b 302 def dump_options(self):
kadonotakashi 0:8fdf9a60065b 303 """ Function returns data structure with common settings passed to SingelTestRunner
kadonotakashi 0:8fdf9a60065b 304 It can be used for example to fill _extra fields in database storing test suite single run data
kadonotakashi 0:8fdf9a60065b 305 Example:
kadonotakashi 0:8fdf9a60065b 306 data = self.dump_options()
kadonotakashi 0:8fdf9a60065b 307 or
kadonotakashi 0:8fdf9a60065b 308 data_str = json.dumps(self.dump_options())
kadonotakashi 0:8fdf9a60065b 309 """
kadonotakashi 0:8fdf9a60065b 310 result = {"db_url" : str(self.opts_db_url),
kadonotakashi 0:8fdf9a60065b 311 "log_file_name" : str(self.opts_log_file_name),
kadonotakashi 0:8fdf9a60065b 312 "shuffle_test_order" : str(self.opts_shuffle_test_order),
kadonotakashi 0:8fdf9a60065b 313 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
kadonotakashi 0:8fdf9a60065b 314 "test_by_names" : str(self.opts_test_by_names),
kadonotakashi 0:8fdf9a60065b 315 "peripheral_by_names" : str(self.opts_peripheral_by_names),
kadonotakashi 0:8fdf9a60065b 316 "test_only_peripheral" : str(self.opts_test_only_peripheral),
kadonotakashi 0:8fdf9a60065b 317 "test_only_common" : str(self.opts_test_only_common),
kadonotakashi 0:8fdf9a60065b 318 "verbose" : str(self.opts_verbose),
kadonotakashi 0:8fdf9a60065b 319 "firmware_global_name" : str(self.opts_firmware_global_name),
kadonotakashi 0:8fdf9a60065b 320 "only_build_tests" : str(self.opts_only_build_tests),
kadonotakashi 0:8fdf9a60065b 321 "copy_method" : str(self.opts_copy_method),
kadonotakashi 0:8fdf9a60065b 322 "mut_reset_type" : str(self.opts_mut_reset_type),
kadonotakashi 0:8fdf9a60065b 323 "jobs" : str(self.opts_jobs),
kadonotakashi 0:8fdf9a60065b 324 "extend_test_timeout" : str(self.opts_extend_test_timeout),
kadonotakashi 0:8fdf9a60065b 325 "_dummy" : ''
kadonotakashi 0:8fdf9a60065b 326 }
kadonotakashi 0:8fdf9a60065b 327 return result
kadonotakashi 0:8fdf9a60065b 328
kadonotakashi 0:8fdf9a60065b 329 def shuffle_random_func(self):
kadonotakashi 0:8fdf9a60065b 330 return self.shuffle_random_seed
kadonotakashi 0:8fdf9a60065b 331
kadonotakashi 0:8fdf9a60065b 332 def is_shuffle_seed_float(self):
kadonotakashi 0:8fdf9a60065b 333 """ return true if function parameter can be converted to float
kadonotakashi 0:8fdf9a60065b 334 """
kadonotakashi 0:8fdf9a60065b 335 result = True
kadonotakashi 0:8fdf9a60065b 336 try:
kadonotakashi 0:8fdf9a60065b 337 float(self.shuffle_random_seed)
kadonotakashi 0:8fdf9a60065b 338 except ValueError:
kadonotakashi 0:8fdf9a60065b 339 result = False
kadonotakashi 0:8fdf9a60065b 340 return result
kadonotakashi 0:8fdf9a60065b 341
kadonotakashi 0:8fdf9a60065b 342 # This will store target / toolchain specific properties
kadonotakashi 0:8fdf9a60065b 343 test_suite_properties_ext = {} # target : toolchain
kadonotakashi 0:8fdf9a60065b 344 # Here we store test results
kadonotakashi 0:8fdf9a60065b 345 test_summary = []
kadonotakashi 0:8fdf9a60065b 346 # Here we store test results in extended data structure
kadonotakashi 0:8fdf9a60065b 347 test_summary_ext = {}
kadonotakashi 0:8fdf9a60065b 348 execute_thread_slice_lock = Lock()
kadonotakashi 0:8fdf9a60065b 349
kadonotakashi 0:8fdf9a60065b 350 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
kadonotakashi 0:8fdf9a60065b 351 for toolchain in toolchains:
kadonotakashi 0:8fdf9a60065b 352 tt_id = "%s::%s" % (toolchain, target)
kadonotakashi 0:8fdf9a60065b 353
kadonotakashi 0:8fdf9a60065b 354 T = TARGET_MAP[target]
kadonotakashi 0:8fdf9a60065b 355
kadonotakashi 0:8fdf9a60065b 356 # print target, toolchain
kadonotakashi 0:8fdf9a60065b 357 # Test suite properties returned to external tools like CI
kadonotakashi 0:8fdf9a60065b 358 test_suite_properties = {
kadonotakashi 0:8fdf9a60065b 359 'jobs': self.opts_jobs,
kadonotakashi 0:8fdf9a60065b 360 'clean': clean,
kadonotakashi 0:8fdf9a60065b 361 'target': target,
kadonotakashi 0:8fdf9a60065b 362 'vendor': T.extra_labels[0],
kadonotakashi 0:8fdf9a60065b 363 'test_ids': ', '.join(test_ids),
kadonotakashi 0:8fdf9a60065b 364 'toolchain': toolchain,
kadonotakashi 0:8fdf9a60065b 365 'shuffle_random_seed': self.shuffle_random_seed
kadonotakashi 0:8fdf9a60065b 366 }
kadonotakashi 0:8fdf9a60065b 367
kadonotakashi 0:8fdf9a60065b 368
kadonotakashi 0:8fdf9a60065b 369 # print '=== %s::%s ===' % (target, toolchain)
kadonotakashi 0:8fdf9a60065b 370 # Let's build our test
kadonotakashi 0:8fdf9a60065b 371 if target not in TARGET_MAP:
kadonotakashi 0:8fdf9a60065b 372 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 373 self.logger.LogType.NOTIF,
kadonotakashi 0:8fdf9a60065b 374 'Skipped tests for %s target. Target platform not found' %
kadonotakashi 0:8fdf9a60065b 375 (target)))
kadonotakashi 0:8fdf9a60065b 376 continue
kadonotakashi 0:8fdf9a60065b 377
kadonotakashi 0:8fdf9a60065b 378 clean_mbed_libs_options = (self.opts_goanna_for_mbed_sdk or
kadonotakashi 0:8fdf9a60065b 379 self.opts_clean or clean)
kadonotakashi 0:8fdf9a60065b 380
kadonotakashi 0:8fdf9a60065b 381 profile = extract_profile(self.opts_parser, self.opts, toolchain)
kadonotakashi 0:8fdf9a60065b 382 stats_depth = self.opts.stats_depth or 2
kadonotakashi 0:8fdf9a60065b 383
kadonotakashi 0:8fdf9a60065b 384 try:
kadonotakashi 0:8fdf9a60065b 385 build_mbed_libs_result = build_mbed_libs(
kadonotakashi 0:8fdf9a60065b 386 T, toolchain,
kadonotakashi 0:8fdf9a60065b 387 clean=clean_mbed_libs_options,
kadonotakashi 0:8fdf9a60065b 388 jobs=self.opts_jobs,
kadonotakashi 0:8fdf9a60065b 389 report=build_report,
kadonotakashi 0:8fdf9a60065b 390 properties=build_properties,
kadonotakashi 0:8fdf9a60065b 391 build_profile=profile,
kadonotakashi 0:8fdf9a60065b 392 notify=TerminalNotifier())
kadonotakashi 0:8fdf9a60065b 393
kadonotakashi 0:8fdf9a60065b 394 if not build_mbed_libs_result:
kadonotakashi 0:8fdf9a60065b 395 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 396 self.logger.LogType.NOTIF,
kadonotakashi 0:8fdf9a60065b 397 'Skipped tests for %s target. Toolchain %s is not '
kadonotakashi 0:8fdf9a60065b 398 'supported for this target'% (T.name, toolchain)))
kadonotakashi 0:8fdf9a60065b 399 continue
kadonotakashi 0:8fdf9a60065b 400
kadonotakashi 0:8fdf9a60065b 401 except ToolException:
kadonotakashi 0:8fdf9a60065b 402 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 403 self.logger.LogType.ERROR,
kadonotakashi 0:8fdf9a60065b 404 'There were errors while building MBED libs for %s using %s'
kadonotakashi 0:8fdf9a60065b 405 % (target, toolchain)))
kadonotakashi 0:8fdf9a60065b 406 continue
kadonotakashi 0:8fdf9a60065b 407
kadonotakashi 0:8fdf9a60065b 408 build_dir = join(BUILD_DIR, "test", target, toolchain)
kadonotakashi 0:8fdf9a60065b 409
kadonotakashi 0:8fdf9a60065b 410 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
kadonotakashi 0:8fdf9a60065b 411 test_suite_properties['build_dir'] = build_dir
kadonotakashi 0:8fdf9a60065b 412 test_suite_properties['skipped'] = []
kadonotakashi 0:8fdf9a60065b 413
kadonotakashi 0:8fdf9a60065b 414 # Enumerate through all tests and shuffle test order if requested
kadonotakashi 0:8fdf9a60065b 415 test_map_keys = sorted(TEST_MAP.keys())
kadonotakashi 0:8fdf9a60065b 416
kadonotakashi 0:8fdf9a60065b 417 if self.opts_shuffle_test_order:
kadonotakashi 0:8fdf9a60065b 418 random.shuffle(test_map_keys, self.shuffle_random_func)
kadonotakashi 0:8fdf9a60065b 419 # Update database with shuffle seed f applicable
kadonotakashi 0:8fdf9a60065b 420 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 421 self.db_logger.reconnect();
kadonotakashi 0:8fdf9a60065b 422 if self.db_logger.is_connected():
kadonotakashi 0:8fdf9a60065b 423 self.db_logger.update_build_id_info(
kadonotakashi 0:8fdf9a60065b 424 self.db_logger_build_id,
kadonotakashi 0:8fdf9a60065b 425 _shuffle_seed=self.shuffle_random_func())
kadonotakashi 0:8fdf9a60065b 426 self.db_logger.disconnect();
kadonotakashi 0:8fdf9a60065b 427
kadonotakashi 0:8fdf9a60065b 428 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 429 self.db_logger.reconnect();
kadonotakashi 0:8fdf9a60065b 430 if self.db_logger.is_connected():
kadonotakashi 0:8fdf9a60065b 431 # Update MUTs and Test Specification in database
kadonotakashi 0:8fdf9a60065b 432 self.db_logger.update_build_id_info(
kadonotakashi 0:8fdf9a60065b 433 self.db_logger_build_id,
kadonotakashi 0:8fdf9a60065b 434 _muts=self.muts, _test_spec=self.test_spec)
kadonotakashi 0:8fdf9a60065b 435 # Update Extra information in database (some options passed to test suite)
kadonotakashi 0:8fdf9a60065b 436 self.db_logger.update_build_id_info(
kadonotakashi 0:8fdf9a60065b 437 self.db_logger_build_id,
kadonotakashi 0:8fdf9a60065b 438 _extra=json.dumps(self.dump_options()))
kadonotakashi 0:8fdf9a60065b 439 self.db_logger.disconnect();
kadonotakashi 0:8fdf9a60065b 440
kadonotakashi 0:8fdf9a60065b 441 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
kadonotakashi 0:8fdf9a60065b 442 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
kadonotakashi 0:8fdf9a60065b 443
kadonotakashi 0:8fdf9a60065b 444 for skipped_test_id in skipped_test_map_keys:
kadonotakashi 0:8fdf9a60065b 445 test_suite_properties['skipped'].append(skipped_test_id)
kadonotakashi 0:8fdf9a60065b 446
kadonotakashi 0:8fdf9a60065b 447
kadonotakashi 0:8fdf9a60065b 448 # First pass through all tests and determine which libraries need to be built
kadonotakashi 0:8fdf9a60065b 449 libraries = []
kadonotakashi 0:8fdf9a60065b 450 for test_id in valid_test_map_keys:
kadonotakashi 0:8fdf9a60065b 451 test = TEST_MAP[test_id]
kadonotakashi 0:8fdf9a60065b 452
kadonotakashi 0:8fdf9a60065b 453 # Detect which lib should be added to test
kadonotakashi 0:8fdf9a60065b 454 # Some libs have to compiled like RTOS or ETH
kadonotakashi 0:8fdf9a60065b 455 for lib in LIBRARIES:
kadonotakashi 0:8fdf9a60065b 456 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
kadonotakashi 0:8fdf9a60065b 457 libraries.append(lib['id'])
kadonotakashi 0:8fdf9a60065b 458
kadonotakashi 0:8fdf9a60065b 459
kadonotakashi 0:8fdf9a60065b 460 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
kadonotakashi 0:8fdf9a60065b 461
kadonotakashi 0:8fdf9a60065b 462 # Build all required libraries
kadonotakashi 0:8fdf9a60065b 463 for lib_id in libraries:
kadonotakashi 0:8fdf9a60065b 464 try:
kadonotakashi 0:8fdf9a60065b 465 build_lib(lib_id,
kadonotakashi 0:8fdf9a60065b 466 T,
kadonotakashi 0:8fdf9a60065b 467 toolchain,
kadonotakashi 0:8fdf9a60065b 468 clean=clean_mbed_libs_options,
kadonotakashi 0:8fdf9a60065b 469 jobs=self.opts_jobs,
kadonotakashi 0:8fdf9a60065b 470 report=build_report,
kadonotakashi 0:8fdf9a60065b 471 properties=build_properties,
kadonotakashi 0:8fdf9a60065b 472 build_profile=profile,
kadonotakashi 0:8fdf9a60065b 473 notify=TerminalNotifier())
kadonotakashi 0:8fdf9a60065b 474
kadonotakashi 0:8fdf9a60065b 475 except ToolException:
kadonotakashi 0:8fdf9a60065b 476 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 477 self.logger.LogType.ERROR,
kadonotakashi 0:8fdf9a60065b 478 'There were errors while building library %s' % lib_id))
kadonotakashi 0:8fdf9a60065b 479 continue
kadonotakashi 0:8fdf9a60065b 480
kadonotakashi 0:8fdf9a60065b 481
kadonotakashi 0:8fdf9a60065b 482 for test_id in valid_test_map_keys:
kadonotakashi 0:8fdf9a60065b 483 test = TEST_MAP[test_id]
kadonotakashi 0:8fdf9a60065b 484
kadonotakashi 0:8fdf9a60065b 485 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
kadonotakashi 0:8fdf9a60065b 486
kadonotakashi 0:8fdf9a60065b 487 # TODO: move this 2 below loops to separate function
kadonotakashi 0:8fdf9a60065b 488 INC_DIRS = []
kadonotakashi 0:8fdf9a60065b 489 for lib_id in libraries:
kadonotakashi 0:8fdf9a60065b 490 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
kadonotakashi 0:8fdf9a60065b 491 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
kadonotakashi 0:8fdf9a60065b 492
kadonotakashi 0:8fdf9a60065b 493 MACROS = []
kadonotakashi 0:8fdf9a60065b 494 for lib_id in libraries:
kadonotakashi 0:8fdf9a60065b 495 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
kadonotakashi 0:8fdf9a60065b 496 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
kadonotakashi 0:8fdf9a60065b 497 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
kadonotakashi 0:8fdf9a60065b 498 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
kadonotakashi 0:8fdf9a60065b 499 test_uuid = uuid.uuid4()
kadonotakashi 0:8fdf9a60065b 500 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
kadonotakashi 0:8fdf9a60065b 501
kadonotakashi 0:8fdf9a60065b 502 # Prepare extended test results data structure (it can be used to generate detailed test report)
kadonotakashi 0:8fdf9a60065b 503 if target not in self.test_summary_ext:
kadonotakashi 0:8fdf9a60065b 504 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
kadonotakashi 0:8fdf9a60065b 505 if toolchain not in self.test_summary_ext[target]:
kadonotakashi 0:8fdf9a60065b 506 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
kadonotakashi 0:8fdf9a60065b 507
kadonotakashi 0:8fdf9a60065b 508 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
kadonotakashi 0:8fdf9a60065b 509
kadonotakashi 0:8fdf9a60065b 510 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
kadonotakashi 0:8fdf9a60065b 511 try:
kadonotakashi 0:8fdf9a60065b 512 path = build_project(
kadonotakashi 0:8fdf9a60065b 513 test.source_dir, join(build_dir, test_id), T,
kadonotakashi 0:8fdf9a60065b 514 toolchain, test.dependencies, clean=clean_project_options,
kadonotakashi 0:8fdf9a60065b 515 name=project_name, macros=MACROS,
kadonotakashi 0:8fdf9a60065b 516 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
kadonotakashi 0:8fdf9a60065b 517 properties=build_properties, project_id=test_id,
kadonotakashi 0:8fdf9a60065b 518 project_description=test.get_description(),
kadonotakashi 0:8fdf9a60065b 519 build_profile=profile, stats_depth=stats_depth,
kadonotakashi 0:8fdf9a60065b 520 notify=TerminalNotifier(),
kadonotakashi 0:8fdf9a60065b 521 )
kadonotakashi 0:8fdf9a60065b 522
kadonotakashi 0:8fdf9a60065b 523 except Exception as e:
kadonotakashi 0:8fdf9a60065b 524 project_name_str = project_name if project_name is not None else test_id
kadonotakashi 0:8fdf9a60065b 525
kadonotakashi 0:8fdf9a60065b 526
kadonotakashi 0:8fdf9a60065b 527 test_result = self.TEST_RESULT_FAIL
kadonotakashi 0:8fdf9a60065b 528
kadonotakashi 0:8fdf9a60065b 529 if isinstance(e, ToolException):
kadonotakashi 0:8fdf9a60065b 530 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 531 self.logger.LogType.ERROR,
kadonotakashi 0:8fdf9a60065b 532 'There were errors while building project %s' %
kadonotakashi 0:8fdf9a60065b 533 project_name_str))
kadonotakashi 0:8fdf9a60065b 534 test_result = self.TEST_RESULT_BUILD_FAILED
kadonotakashi 0:8fdf9a60065b 535 elif isinstance(e, NotSupportedException):
kadonotakashi 0:8fdf9a60065b 536 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 537 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 538 'Project %s is not supported' % project_name_str))
kadonotakashi 0:8fdf9a60065b 539 test_result = self.TEST_RESULT_NOT_SUPPORTED
kadonotakashi 0:8fdf9a60065b 540
kadonotakashi 0:8fdf9a60065b 541
kadonotakashi 0:8fdf9a60065b 542 # Append test results to global test summary
kadonotakashi 0:8fdf9a60065b 543 self.test_summary.append(
kadonotakashi 0:8fdf9a60065b 544 (test_result, target, toolchain, test_id,
kadonotakashi 0:8fdf9a60065b 545 test.get_description(), 0, 0, '-')
kadonotakashi 0:8fdf9a60065b 546 )
kadonotakashi 0:8fdf9a60065b 547
kadonotakashi 0:8fdf9a60065b 548 # Add detailed test result to test summary structure
kadonotakashi 0:8fdf9a60065b 549 if test_id not in self.test_summary_ext[target][toolchain]:
kadonotakashi 0:8fdf9a60065b 550 self.test_summary_ext[target][toolchain][test_id] = []
kadonotakashi 0:8fdf9a60065b 551
kadonotakashi 0:8fdf9a60065b 552 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
kadonotakashi 0:8fdf9a60065b 553 'result' : test_result,
kadonotakashi 0:8fdf9a60065b 554 'output' : '',
kadonotakashi 0:8fdf9a60065b 555 'target_name' : target,
kadonotakashi 0:8fdf9a60065b 556 'target_name_unique': target,
kadonotakashi 0:8fdf9a60065b 557 'toolchain_name' : toolchain,
kadonotakashi 0:8fdf9a60065b 558 'id' : test_id,
kadonotakashi 0:8fdf9a60065b 559 'description' : test.get_description(),
kadonotakashi 0:8fdf9a60065b 560 'elapsed_time' : 0,
kadonotakashi 0:8fdf9a60065b 561 'duration' : 0,
kadonotakashi 0:8fdf9a60065b 562 'copy_method' : None
kadonotakashi 0:8fdf9a60065b 563 }})
kadonotakashi 0:8fdf9a60065b 564 continue
kadonotakashi 0:8fdf9a60065b 565
kadonotakashi 0:8fdf9a60065b 566 if self.opts_only_build_tests:
kadonotakashi 0:8fdf9a60065b 567 # With this option we are skipping testing phase
kadonotakashi 0:8fdf9a60065b 568 continue
kadonotakashi 0:8fdf9a60065b 569
kadonotakashi 0:8fdf9a60065b 570 # Test duration can be increased by global value
kadonotakashi 0:8fdf9a60065b 571 test_duration = test.duration
kadonotakashi 0:8fdf9a60065b 572 if self.opts_extend_test_timeout is not None:
kadonotakashi 0:8fdf9a60065b 573 test_duration += self.opts_extend_test_timeout
kadonotakashi 0:8fdf9a60065b 574
kadonotakashi 0:8fdf9a60065b 575 # For an automated test the duration act as a timeout after
kadonotakashi 0:8fdf9a60065b 576 # which the test gets interrupted
kadonotakashi 0:8fdf9a60065b 577 test_spec = self.shape_test_request(target, path, test_id, test_duration)
kadonotakashi 0:8fdf9a60065b 578 test_loops = self.get_test_loop_count(test_id)
kadonotakashi 0:8fdf9a60065b 579
kadonotakashi 0:8fdf9a60065b 580 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
kadonotakashi 0:8fdf9a60065b 581 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
kadonotakashi 0:8fdf9a60065b 582 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
kadonotakashi 0:8fdf9a60065b 583
kadonotakashi 0:8fdf9a60065b 584 # read MUTs, test specification and perform tests
kadonotakashi 0:8fdf9a60065b 585 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
kadonotakashi 0:8fdf9a60065b 586
kadonotakashi 0:8fdf9a60065b 587 if handle_results is None:
kadonotakashi 0:8fdf9a60065b 588 continue
kadonotakashi 0:8fdf9a60065b 589
kadonotakashi 0:8fdf9a60065b 590 for handle_result in handle_results:
kadonotakashi 0:8fdf9a60065b 591 if handle_result:
kadonotakashi 0:8fdf9a60065b 592 single_test_result, detailed_test_results = handle_result
kadonotakashi 0:8fdf9a60065b 593 else:
kadonotakashi 0:8fdf9a60065b 594 continue
kadonotakashi 0:8fdf9a60065b 595
kadonotakashi 0:8fdf9a60065b 596 # Append test results to global test summary
kadonotakashi 0:8fdf9a60065b 597 if single_test_result is not None:
kadonotakashi 0:8fdf9a60065b 598 self.test_summary.append(single_test_result)
kadonotakashi 0:8fdf9a60065b 599
kadonotakashi 0:8fdf9a60065b 600 # Add detailed test result to test summary structure
kadonotakashi 0:8fdf9a60065b 601 if target not in self.test_summary_ext[target][toolchain]:
kadonotakashi 0:8fdf9a60065b 602 if test_id not in self.test_summary_ext[target][toolchain]:
kadonotakashi 0:8fdf9a60065b 603 self.test_summary_ext[target][toolchain][test_id] = []
kadonotakashi 0:8fdf9a60065b 604
kadonotakashi 0:8fdf9a60065b 605 append_test_result = detailed_test_results
kadonotakashi 0:8fdf9a60065b 606
kadonotakashi 0:8fdf9a60065b 607 # If waterfall and consolidate-waterfall options are enabled,
kadonotakashi 0:8fdf9a60065b 608 # only include the last test result in the report.
kadonotakashi 0:8fdf9a60065b 609 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
kadonotakashi 0:8fdf9a60065b 610 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
kadonotakashi 0:8fdf9a60065b 611
kadonotakashi 0:8fdf9a60065b 612 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
kadonotakashi 0:8fdf9a60065b 613
kadonotakashi 0:8fdf9a60065b 614 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
kadonotakashi 0:8fdf9a60065b 615 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
kadonotakashi 0:8fdf9a60065b 616
kadonotakashi 0:8fdf9a60065b 617 q.put(target + '_'.join(toolchains))
kadonotakashi 0:8fdf9a60065b 618 return
kadonotakashi 0:8fdf9a60065b 619
kadonotakashi 0:8fdf9a60065b 620 def execute(self):
kadonotakashi 0:8fdf9a60065b 621 clean = self.test_spec.get('clean', False)
kadonotakashi 0:8fdf9a60065b 622 test_ids = self.test_spec.get('test_ids', [])
kadonotakashi 0:8fdf9a60065b 623 q = Queue()
kadonotakashi 0:8fdf9a60065b 624
kadonotakashi 0:8fdf9a60065b 625 # Generate seed for shuffle if seed is not provided in
kadonotakashi 0:8fdf9a60065b 626 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
kadonotakashi 0:8fdf9a60065b 627 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
kadonotakashi 0:8fdf9a60065b 628 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
kadonotakashi 0:8fdf9a60065b 629
kadonotakashi 0:8fdf9a60065b 630
kadonotakashi 0:8fdf9a60065b 631 if self.opts_parallel_test_exec:
kadonotakashi 0:8fdf9a60065b 632 ###################################################################
kadonotakashi 0:8fdf9a60065b 633 # Experimental, parallel test execution per singletest instance.
kadonotakashi 0:8fdf9a60065b 634 ###################################################################
kadonotakashi 0:8fdf9a60065b 635 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
kadonotakashi 0:8fdf9a60065b 636 # Note: We are building here in parallel for each target separately!
kadonotakashi 0:8fdf9a60065b 637 # So we are not building the same thing multiple times and compilers
kadonotakashi 0:8fdf9a60065b 638 # in separate threads do not collide.
kadonotakashi 0:8fdf9a60065b 639 # Inside execute_thread_slice() function function handle() will be called to
kadonotakashi 0:8fdf9a60065b 640 # get information about available MUTs (per target).
kadonotakashi 0:8fdf9a60065b 641 for target, toolchains in self.test_spec['targets'].items():
kadonotakashi 0:8fdf9a60065b 642 self.test_suite_properties_ext[target] = {}
kadonotakashi 0:8fdf9a60065b 643 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
kadonotakashi 0:8fdf9a60065b 644 t.daemon = True
kadonotakashi 0:8fdf9a60065b 645 t.start()
kadonotakashi 0:8fdf9a60065b 646 execute_threads.append(t)
kadonotakashi 0:8fdf9a60065b 647
kadonotakashi 0:8fdf9a60065b 648 for t in execute_threads:
kadonotakashi 0:8fdf9a60065b 649 q.get() # t.join() would block some threads because we should not wait in any order for thread end
kadonotakashi 0:8fdf9a60065b 650 else:
kadonotakashi 0:8fdf9a60065b 651 # Serialized (not parallel) test execution
kadonotakashi 0:8fdf9a60065b 652 for target, toolchains in self.test_spec['targets'].items():
kadonotakashi 0:8fdf9a60065b 653 if target not in self.test_suite_properties_ext:
kadonotakashi 0:8fdf9a60065b 654 self.test_suite_properties_ext[target] = {}
kadonotakashi 0:8fdf9a60065b 655
kadonotakashi 0:8fdf9a60065b 656 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
kadonotakashi 0:8fdf9a60065b 657 q.get()
kadonotakashi 0:8fdf9a60065b 658
kadonotakashi 0:8fdf9a60065b 659 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 660 self.db_logger.reconnect();
kadonotakashi 0:8fdf9a60065b 661 if self.db_logger.is_connected():
kadonotakashi 0:8fdf9a60065b 662 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
kadonotakashi 0:8fdf9a60065b 663 self.db_logger.disconnect();
kadonotakashi 0:8fdf9a60065b 664
kadonotakashi 0:8fdf9a60065b 665 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
kadonotakashi 0:8fdf9a60065b 666
kadonotakashi 0:8fdf9a60065b 667 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
kadonotakashi 0:8fdf9a60065b 668 valid_test_map_keys = []
kadonotakashi 0:8fdf9a60065b 669
kadonotakashi 0:8fdf9a60065b 670 for test_id in test_map_keys:
kadonotakashi 0:8fdf9a60065b 671 test = TEST_MAP[test_id]
kadonotakashi 0:8fdf9a60065b 672 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
kadonotakashi 0:8fdf9a60065b 673 continue
kadonotakashi 0:8fdf9a60065b 674
kadonotakashi 0:8fdf9a60065b 675 if test_ids and test_id not in test_ids:
kadonotakashi 0:8fdf9a60065b 676 continue
kadonotakashi 0:8fdf9a60065b 677
kadonotakashi 0:8fdf9a60065b 678 if self.opts_test_only_peripheral and not test.peripherals:
kadonotakashi 0:8fdf9a60065b 679 if self.opts_verbose_skipped_tests:
kadonotakashi 0:8fdf9a60065b 680 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 681 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 682 'Common test skipped for target %s' % target))
kadonotakashi 0:8fdf9a60065b 683 continue
kadonotakashi 0:8fdf9a60065b 684
kadonotakashi 0:8fdf9a60065b 685 if (self.opts_peripheral_by_names and test.peripherals and
kadonotakashi 0:8fdf9a60065b 686 not any((i in self.opts_peripheral_by_names)
kadonotakashi 0:8fdf9a60065b 687 for i in test.peripherals)):
kadonotakashi 0:8fdf9a60065b 688 # We will skip tests not forced with -p option
kadonotakashi 0:8fdf9a60065b 689 if self.opts_verbose_skipped_tests:
kadonotakashi 0:8fdf9a60065b 690 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 691 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 692 'Common test skipped for target %s' % target))
kadonotakashi 0:8fdf9a60065b 693 continue
kadonotakashi 0:8fdf9a60065b 694
kadonotakashi 0:8fdf9a60065b 695 if self.opts_test_only_common and test.peripherals:
kadonotakashi 0:8fdf9a60065b 696 if self.opts_verbose_skipped_tests:
kadonotakashi 0:8fdf9a60065b 697 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 698 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 699 'Peripheral test skipped for target %s' % target))
kadonotakashi 0:8fdf9a60065b 700 continue
kadonotakashi 0:8fdf9a60065b 701
kadonotakashi 0:8fdf9a60065b 702 if not include_non_automated and not test.automated:
kadonotakashi 0:8fdf9a60065b 703 if self.opts_verbose_skipped_tests:
kadonotakashi 0:8fdf9a60065b 704 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 705 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 706 'Non automated test skipped for target %s' % target))
kadonotakashi 0:8fdf9a60065b 707 continue
kadonotakashi 0:8fdf9a60065b 708
kadonotakashi 0:8fdf9a60065b 709 if test.is_supported(target, toolchain):
kadonotakashi 0:8fdf9a60065b 710 if test.peripherals is None and self.opts_only_build_tests:
kadonotakashi 0:8fdf9a60065b 711 # When users are using 'build only flag' and test do not have
kadonotakashi 0:8fdf9a60065b 712 # specified peripherals we can allow test building by default
kadonotakashi 0:8fdf9a60065b 713 pass
kadonotakashi 0:8fdf9a60065b 714 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
kadonotakashi 0:8fdf9a60065b 715 # If we force peripheral with option -p we expect test
kadonotakashi 0:8fdf9a60065b 716 # to pass even if peripheral is not in MUTs file.
kadonotakashi 0:8fdf9a60065b 717 pass
kadonotakashi 0:8fdf9a60065b 718 elif not self.is_peripherals_available(target, test.peripherals):
kadonotakashi 0:8fdf9a60065b 719 if self.opts_verbose_skipped_tests:
kadonotakashi 0:8fdf9a60065b 720 if test.peripherals:
kadonotakashi 0:8fdf9a60065b 721 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 722 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 723 'Peripheral %s test skipped for target %s' %
kadonotakashi 0:8fdf9a60065b 724 (",".join(test.peripherals), target)))
kadonotakashi 0:8fdf9a60065b 725 else:
kadonotakashi 0:8fdf9a60065b 726 print(self.logger.log_line(
kadonotakashi 0:8fdf9a60065b 727 self.logger.LogType.INFO,
kadonotakashi 0:8fdf9a60065b 728 'Test %s skipped for target %s' %
kadonotakashi 0:8fdf9a60065b 729 (test_id, target)))
kadonotakashi 0:8fdf9a60065b 730 continue
kadonotakashi 0:8fdf9a60065b 731
kadonotakashi 0:8fdf9a60065b 732 # The test has made it through all the filters, so add it to the valid tests list
kadonotakashi 0:8fdf9a60065b 733 valid_test_map_keys.append(test_id)
kadonotakashi 0:8fdf9a60065b 734
kadonotakashi 0:8fdf9a60065b 735 return valid_test_map_keys
kadonotakashi 0:8fdf9a60065b 736
kadonotakashi 0:8fdf9a60065b 737 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
kadonotakashi 0:8fdf9a60065b 738 # NOTE: This will not preserve order
kadonotakashi 0:8fdf9a60065b 739 return list(set(all_test_map_keys) - set(valid_test_map_keys))
kadonotakashi 0:8fdf9a60065b 740
kadonotakashi 0:8fdf9a60065b 741 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
kadonotakashi 0:8fdf9a60065b 742 """ Prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 743 table shows text x toolchain test result matrix
kadonotakashi 0:8fdf9a60065b 744 """
kadonotakashi 0:8fdf9a60065b 745 RESULT_INDEX = 0
kadonotakashi 0:8fdf9a60065b 746 TARGET_INDEX = 1
kadonotakashi 0:8fdf9a60065b 747 TOOLCHAIN_INDEX = 2
kadonotakashi 0:8fdf9a60065b 748 TEST_INDEX = 3
kadonotakashi 0:8fdf9a60065b 749 DESC_INDEX = 4
kadonotakashi 0:8fdf9a60065b 750
kadonotakashi 0:8fdf9a60065b 751 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
kadonotakashi 0:8fdf9a60065b 752 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
kadonotakashi 0:8fdf9a60065b 753 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
kadonotakashi 0:8fdf9a60065b 754 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
kadonotakashi 0:8fdf9a60065b 755
kadonotakashi 0:8fdf9a60065b 756 result = "Test summary:\n"
kadonotakashi 0:8fdf9a60065b 757 for target in unique_targets:
kadonotakashi 0:8fdf9a60065b 758 result_dict = {} # test : { toolchain : result }
kadonotakashi 0:8fdf9a60065b 759 unique_target_toolchains = []
kadonotakashi 0:8fdf9a60065b 760 for test in test_summary:
kadonotakashi 0:8fdf9a60065b 761 if test[TARGET_INDEX] == target:
kadonotakashi 0:8fdf9a60065b 762 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
kadonotakashi 0:8fdf9a60065b 763 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
kadonotakashi 0:8fdf9a60065b 764 if test[TEST_INDEX] not in result_dict:
kadonotakashi 0:8fdf9a60065b 765 result_dict[test[TEST_INDEX]] = {}
kadonotakashi 0:8fdf9a60065b 766 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
kadonotakashi 0:8fdf9a60065b 767
kadonotakashi 0:8fdf9a60065b 768 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
kadonotakashi 0:8fdf9a60065b 769 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 770 for col in pt_cols:
kadonotakashi 0:8fdf9a60065b 771 pt.align[col] = "l"
kadonotakashi 0:8fdf9a60065b 772 pt.padding_width = 1 # One space between column edges and contents (default)
kadonotakashi 0:8fdf9a60065b 773
kadonotakashi 0:8fdf9a60065b 774 for test in unique_tests:
kadonotakashi 0:8fdf9a60065b 775 if test in result_dict:
kadonotakashi 0:8fdf9a60065b 776 test_results = result_dict[test]
kadonotakashi 0:8fdf9a60065b 777 if test in unique_test_desc:
kadonotakashi 0:8fdf9a60065b 778 row = [target, test, unique_test_desc[test]]
kadonotakashi 0:8fdf9a60065b 779 for toolchain in unique_toolchains:
kadonotakashi 0:8fdf9a60065b 780 if toolchain in test_results:
kadonotakashi 0:8fdf9a60065b 781 row.append(test_results[toolchain])
kadonotakashi 0:8fdf9a60065b 782 pt.add_row(row)
kadonotakashi 0:8fdf9a60065b 783 result += pt.get_string()
kadonotakashi 0:8fdf9a60065b 784 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
kadonotakashi 0:8fdf9a60065b 785 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
kadonotakashi 0:8fdf9a60065b 786 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
kadonotakashi 0:8fdf9a60065b 787 return result
kadonotakashi 0:8fdf9a60065b 788
kadonotakashi 0:8fdf9a60065b 789 def generate_test_summary(self, test_summary, shuffle_seed=None):
kadonotakashi 0:8fdf9a60065b 790 """ Prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 791 table shows target x test results matrix across
kadonotakashi 0:8fdf9a60065b 792 """
kadonotakashi 0:8fdf9a60065b 793 success_code = 0 # Success code that can be leter returned to
kadonotakashi 0:8fdf9a60065b 794 result = "Test summary:\n"
kadonotakashi 0:8fdf9a60065b 795 # Pretty table package is used to print results
kadonotakashi 0:8fdf9a60065b 796 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
kadonotakashi 0:8fdf9a60065b 797 "Elapsed Time (sec)", "Timeout (sec)", "Loops"], junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 798 pt.align["Result"] = "l" # Left align
kadonotakashi 0:8fdf9a60065b 799 pt.align["Target"] = "l" # Left align
kadonotakashi 0:8fdf9a60065b 800 pt.align["Toolchain"] = "l" # Left align
kadonotakashi 0:8fdf9a60065b 801 pt.align["Test ID"] = "l" # Left align
kadonotakashi 0:8fdf9a60065b 802 pt.align["Test Description"] = "l" # Left align
kadonotakashi 0:8fdf9a60065b 803 pt.padding_width = 1 # One space between column edges and contents (default)
kadonotakashi 0:8fdf9a60065b 804
kadonotakashi 0:8fdf9a60065b 805 result_dict = {self.TEST_RESULT_OK : 0,
kadonotakashi 0:8fdf9a60065b 806 self.TEST_RESULT_FAIL : 0,
kadonotakashi 0:8fdf9a60065b 807 self.TEST_RESULT_ERROR : 0,
kadonotakashi 0:8fdf9a60065b 808 self.TEST_RESULT_UNDEF : 0,
kadonotakashi 0:8fdf9a60065b 809 self.TEST_RESULT_IOERR_COPY : 0,
kadonotakashi 0:8fdf9a60065b 810 self.TEST_RESULT_IOERR_DISK : 0,
kadonotakashi 0:8fdf9a60065b 811 self.TEST_RESULT_IOERR_SERIAL : 0,
kadonotakashi 0:8fdf9a60065b 812 self.TEST_RESULT_NO_IMAGE : 0,
kadonotakashi 0:8fdf9a60065b 813 self.TEST_RESULT_TIMEOUT : 0,
kadonotakashi 0:8fdf9a60065b 814 self.TEST_RESULT_MBED_ASSERT : 0,
kadonotakashi 0:8fdf9a60065b 815 self.TEST_RESULT_BUILD_FAILED : 0,
kadonotakashi 0:8fdf9a60065b 816 self.TEST_RESULT_NOT_SUPPORTED : 0
kadonotakashi 0:8fdf9a60065b 817 }
kadonotakashi 0:8fdf9a60065b 818
kadonotakashi 0:8fdf9a60065b 819 for test in test_summary:
kadonotakashi 0:8fdf9a60065b 820 if test[0] in result_dict:
kadonotakashi 0:8fdf9a60065b 821 result_dict[test[0]] += 1
kadonotakashi 0:8fdf9a60065b 822 pt.add_row(test)
kadonotakashi 0:8fdf9a60065b 823 result += pt.get_string()
kadonotakashi 0:8fdf9a60065b 824 result += "\n"
kadonotakashi 0:8fdf9a60065b 825
kadonotakashi 0:8fdf9a60065b 826 # Print result count
kadonotakashi 0:8fdf9a60065b 827 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()])
kadonotakashi 0:8fdf9a60065b 828 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
kadonotakashi 0:8fdf9a60065b 829 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
kadonotakashi 0:8fdf9a60065b 830 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
kadonotakashi 0:8fdf9a60065b 831 return result
kadonotakashi 0:8fdf9a60065b 832
kadonotakashi 0:8fdf9a60065b 833 def test_loop_list_to_dict(self, test_loops_str):
kadonotakashi 0:8fdf9a60065b 834 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
kadonotakashi 0:8fdf9a60065b 835 """
kadonotakashi 0:8fdf9a60065b 836 result = {}
kadonotakashi 0:8fdf9a60065b 837 if test_loops_str:
kadonotakashi 0:8fdf9a60065b 838 test_loops = test_loops_str
kadonotakashi 0:8fdf9a60065b 839 for test_loop in test_loops:
kadonotakashi 0:8fdf9a60065b 840 test_loop_count = test_loop.split('=')
kadonotakashi 0:8fdf9a60065b 841 if len(test_loop_count) == 2:
kadonotakashi 0:8fdf9a60065b 842 _test_id, _test_loops = test_loop_count
kadonotakashi 0:8fdf9a60065b 843 try:
kadonotakashi 0:8fdf9a60065b 844 _test_loops = int(_test_loops)
kadonotakashi 0:8fdf9a60065b 845 except:
kadonotakashi 0:8fdf9a60065b 846 continue
kadonotakashi 0:8fdf9a60065b 847 result[_test_id] = _test_loops
kadonotakashi 0:8fdf9a60065b 848 return result
kadonotakashi 0:8fdf9a60065b 849
kadonotakashi 0:8fdf9a60065b 850 def get_test_loop_count(self, test_id):
kadonotakashi 0:8fdf9a60065b 851 """ This function returns no. of loops per test (deducted by test_id_.
kadonotakashi 0:8fdf9a60065b 852 If test is not in list of redefined loop counts it will use default value.
kadonotakashi 0:8fdf9a60065b 853 """
kadonotakashi 0:8fdf9a60065b 854 result = self.GLOBAL_LOOPS_COUNT
kadonotakashi 0:8fdf9a60065b 855 if test_id in self.TEST_LOOPS_DICT:
kadonotakashi 0:8fdf9a60065b 856 result = self.TEST_LOOPS_DICT[test_id]
kadonotakashi 0:8fdf9a60065b 857 return result
kadonotakashi 0:8fdf9a60065b 858
kadonotakashi 0:8fdf9a60065b 859 def delete_file(self, file_path):
kadonotakashi 0:8fdf9a60065b 860 """ Remove file from the system
kadonotakashi 0:8fdf9a60065b 861 """
kadonotakashi 0:8fdf9a60065b 862 result = True
kadonotakashi 0:8fdf9a60065b 863 resutl_msg = ""
kadonotakashi 0:8fdf9a60065b 864 try:
kadonotakashi 0:8fdf9a60065b 865 os.remove(file_path)
kadonotakashi 0:8fdf9a60065b 866 except Exception as e:
kadonotakashi 0:8fdf9a60065b 867 resutl_msg = e
kadonotakashi 0:8fdf9a60065b 868 result = False
kadonotakashi 0:8fdf9a60065b 869 return result, resutl_msg
kadonotakashi 0:8fdf9a60065b 870
kadonotakashi 0:8fdf9a60065b 871 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
kadonotakashi 0:8fdf9a60065b 872 """ Test is being invoked for given MUT.
kadonotakashi 0:8fdf9a60065b 873 """
kadonotakashi 0:8fdf9a60065b 874 # Get test information, image and test timeout
kadonotakashi 0:8fdf9a60065b 875 test_id = data['test_id']
kadonotakashi 0:8fdf9a60065b 876 test = TEST_MAP[test_id]
kadonotakashi 0:8fdf9a60065b 877 test_description = TEST_MAP[test_id].get_description()
kadonotakashi 0:8fdf9a60065b 878 image = data["image"]
kadonotakashi 0:8fdf9a60065b 879 duration = data.get("duration", 10)
kadonotakashi 0:8fdf9a60065b 880
kadonotakashi 0:8fdf9a60065b 881 if mut is None:
kadonotakashi 0:8fdf9a60065b 882 print("Error: No Mbed available: MUT[%s]" % data['mcu'])
kadonotakashi 0:8fdf9a60065b 883 return None
kadonotakashi 0:8fdf9a60065b 884
kadonotakashi 0:8fdf9a60065b 885 mcu = mut['mcu']
kadonotakashi 0:8fdf9a60065b 886 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
kadonotakashi 0:8fdf9a60065b 887
kadonotakashi 0:8fdf9a60065b 888 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 889 self.db_logger.reconnect()
kadonotakashi 0:8fdf9a60065b 890
kadonotakashi 0:8fdf9a60065b 891 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
kadonotakashi 0:8fdf9a60065b 892
kadonotakashi 0:8fdf9a60065b 893 # Tests can be looped so test results must be stored for the same test
kadonotakashi 0:8fdf9a60065b 894 test_all_result = []
kadonotakashi 0:8fdf9a60065b 895 # Test results for one test ran few times
kadonotakashi 0:8fdf9a60065b 896 detailed_test_results = {} # { Loop_number: { results ... } }
kadonotakashi 0:8fdf9a60065b 897
kadonotakashi 0:8fdf9a60065b 898 for test_index in range(test_loops):
kadonotakashi 0:8fdf9a60065b 899
kadonotakashi 0:8fdf9a60065b 900 # If mbedls is available and we are auto detecting MUT info,
kadonotakashi 0:8fdf9a60065b 901 # update MUT info (mounting may changed)
kadonotakashi 0:8fdf9a60065b 902 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
kadonotakashi 0:8fdf9a60065b 903 platform_name_filter = [mcu]
kadonotakashi 0:8fdf9a60065b 904 muts_list = {}
kadonotakashi 0:8fdf9a60065b 905 found = False
kadonotakashi 0:8fdf9a60065b 906
kadonotakashi 0:8fdf9a60065b 907 for i in range(0, 60):
kadonotakashi 0:8fdf9a60065b 908 print('Looking for %s with MBEDLS' % mcu)
kadonotakashi 0:8fdf9a60065b 909 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
kadonotakashi 0:8fdf9a60065b 910
kadonotakashi 0:8fdf9a60065b 911 if 1 not in muts_list:
kadonotakashi 0:8fdf9a60065b 912 sleep(3)
kadonotakashi 0:8fdf9a60065b 913 else:
kadonotakashi 0:8fdf9a60065b 914 found = True
kadonotakashi 0:8fdf9a60065b 915 break
kadonotakashi 0:8fdf9a60065b 916
kadonotakashi 0:8fdf9a60065b 917 if not found:
kadonotakashi 0:8fdf9a60065b 918 print("Error: mbed not found with MBEDLS: %s" % data['mcu'])
kadonotakashi 0:8fdf9a60065b 919 return None
kadonotakashi 0:8fdf9a60065b 920 else:
kadonotakashi 0:8fdf9a60065b 921 mut = muts_list[1]
kadonotakashi 0:8fdf9a60065b 922
kadonotakashi 0:8fdf9a60065b 923 disk = mut.get('disk')
kadonotakashi 0:8fdf9a60065b 924 port = mut.get('port')
kadonotakashi 0:8fdf9a60065b 925
kadonotakashi 0:8fdf9a60065b 926 if disk is None or port is None:
kadonotakashi 0:8fdf9a60065b 927 return None
kadonotakashi 0:8fdf9a60065b 928
kadonotakashi 0:8fdf9a60065b 929 target_by_mcu = TARGET_MAP[mut['mcu']]
kadonotakashi 0:8fdf9a60065b 930 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
kadonotakashi 0:8fdf9a60065b 931 # Some extra stuff can be declared in MUTs structure
kadonotakashi 0:8fdf9a60065b 932 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
kadonotakashi 0:8fdf9a60065b 933 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
kadonotakashi 0:8fdf9a60065b 934
kadonotakashi 0:8fdf9a60065b 935 # When the build and test system were separate, this was relative to a
kadonotakashi 0:8fdf9a60065b 936 # base network folder base path: join(NETWORK_BASE_PATH, )
kadonotakashi 0:8fdf9a60065b 937 image_path = image
kadonotakashi 0:8fdf9a60065b 938
kadonotakashi 0:8fdf9a60065b 939 # Host test execution
kadonotakashi 0:8fdf9a60065b 940 start_host_exec_time = time()
kadonotakashi 0:8fdf9a60065b 941
kadonotakashi 0:8fdf9a60065b 942 single_test_result = self.TEST_RESULT_UNDEF # single test run result
kadonotakashi 0:8fdf9a60065b 943 _copy_method = selected_copy_method
kadonotakashi 0:8fdf9a60065b 944
kadonotakashi 0:8fdf9a60065b 945 if not exists(image_path):
kadonotakashi 0:8fdf9a60065b 946 single_test_result = self.TEST_RESULT_NO_IMAGE
kadonotakashi 0:8fdf9a60065b 947 elapsed_time = 0
kadonotakashi 0:8fdf9a60065b 948 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
kadonotakashi 0:8fdf9a60065b 949 print(single_test_output)
kadonotakashi 0:8fdf9a60065b 950 else:
kadonotakashi 0:8fdf9a60065b 951 # Host test execution
kadonotakashi 0:8fdf9a60065b 952 start_host_exec_time = time()
kadonotakashi 0:8fdf9a60065b 953
kadonotakashi 0:8fdf9a60065b 954 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
kadonotakashi 0:8fdf9a60065b 955 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
kadonotakashi 0:8fdf9a60065b 956 host_test_result = self.run_host_test(test.host_test,
kadonotakashi 0:8fdf9a60065b 957 image_path, disk, port, duration,
kadonotakashi 0:8fdf9a60065b 958 micro=target_name,
kadonotakashi 0:8fdf9a60065b 959 verbose=host_test_verbose,
kadonotakashi 0:8fdf9a60065b 960 reset=host_test_reset,
kadonotakashi 0:8fdf9a60065b 961 reset_tout=reset_tout,
kadonotakashi 0:8fdf9a60065b 962 copy_method=selected_copy_method,
kadonotakashi 0:8fdf9a60065b 963 program_cycle_s=target_by_mcu.program_cycle_s)
kadonotakashi 0:8fdf9a60065b 964 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
kadonotakashi 0:8fdf9a60065b 965
kadonotakashi 0:8fdf9a60065b 966 # Store test result
kadonotakashi 0:8fdf9a60065b 967 test_all_result.append(single_test_result)
kadonotakashi 0:8fdf9a60065b 968 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
kadonotakashi 0:8fdf9a60065b 969 elapsed_time = single_testduration # TIme of single test case execution after reset
kadonotakashi 0:8fdf9a60065b 970
kadonotakashi 0:8fdf9a60065b 971 detailed_test_results[test_index] = {
kadonotakashi 0:8fdf9a60065b 972 'result' : single_test_result,
kadonotakashi 0:8fdf9a60065b 973 'output' : single_test_output,
kadonotakashi 0:8fdf9a60065b 974 'target_name' : target_name,
kadonotakashi 0:8fdf9a60065b 975 'target_name_unique' : target_name_unique,
kadonotakashi 0:8fdf9a60065b 976 'toolchain_name' : toolchain_name,
kadonotakashi 0:8fdf9a60065b 977 'id' : test_id,
kadonotakashi 0:8fdf9a60065b 978 'description' : test_description,
kadonotakashi 0:8fdf9a60065b 979 'elapsed_time' : round(elapsed_time, 2),
kadonotakashi 0:8fdf9a60065b 980 'duration' : single_timeout,
kadonotakashi 0:8fdf9a60065b 981 'copy_method' : _copy_method,
kadonotakashi 0:8fdf9a60065b 982 }
kadonotakashi 0:8fdf9a60065b 983
kadonotakashi 0:8fdf9a60065b 984 print(self.print_test_result(
kadonotakashi 0:8fdf9a60065b 985 single_test_result, target_name_unique, toolchain_name, test_id,
kadonotakashi 0:8fdf9a60065b 986 test_description, elapsed_time, single_timeout))
kadonotakashi 0:8fdf9a60065b 987
kadonotakashi 0:8fdf9a60065b 988 # Update database entries for ongoing test
kadonotakashi 0:8fdf9a60065b 989 if self.db_logger and self.db_logger.is_connected():
kadonotakashi 0:8fdf9a60065b 990 test_type = 'SingleTest'
kadonotakashi 0:8fdf9a60065b 991 self.db_logger.insert_test_entry(self.db_logger_build_id,
kadonotakashi 0:8fdf9a60065b 992 target_name,
kadonotakashi 0:8fdf9a60065b 993 toolchain_name,
kadonotakashi 0:8fdf9a60065b 994 test_type,
kadonotakashi 0:8fdf9a60065b 995 test_id,
kadonotakashi 0:8fdf9a60065b 996 single_test_result,
kadonotakashi 0:8fdf9a60065b 997 single_test_output,
kadonotakashi 0:8fdf9a60065b 998 elapsed_time,
kadonotakashi 0:8fdf9a60065b 999 single_timeout,
kadonotakashi 0:8fdf9a60065b 1000 test_index)
kadonotakashi 0:8fdf9a60065b 1001
kadonotakashi 0:8fdf9a60065b 1002 # If we perform waterfall test we test until we get OK and we stop testing
kadonotakashi 0:8fdf9a60065b 1003 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
kadonotakashi 0:8fdf9a60065b 1004 break
kadonotakashi 0:8fdf9a60065b 1005
kadonotakashi 0:8fdf9a60065b 1006 if self.db_logger:
kadonotakashi 0:8fdf9a60065b 1007 self.db_logger.disconnect()
kadonotakashi 0:8fdf9a60065b 1008
kadonotakashi 0:8fdf9a60065b 1009 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
kadonotakashi 0:8fdf9a60065b 1010 target_name_unique,
kadonotakashi 0:8fdf9a60065b 1011 toolchain_name,
kadonotakashi 0:8fdf9a60065b 1012 test_id,
kadonotakashi 0:8fdf9a60065b 1013 test_description,
kadonotakashi 0:8fdf9a60065b 1014 round(elapsed_time, 2),
kadonotakashi 0:8fdf9a60065b 1015 single_timeout,
kadonotakashi 0:8fdf9a60065b 1016 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
kadonotakashi 0:8fdf9a60065b 1017
kadonotakashi 0:8fdf9a60065b 1018 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
kadonotakashi 0:8fdf9a60065b 1019 """ Function determines MUT's mbed disk/port and copies binary to
kadonotakashi 0:8fdf9a60065b 1020 target.
kadonotakashi 0:8fdf9a60065b 1021 """
kadonotakashi 0:8fdf9a60065b 1022 handle_results = []
kadonotakashi 0:8fdf9a60065b 1023 data = json.loads(test_spec)
kadonotakashi 0:8fdf9a60065b 1024
kadonotakashi 0:8fdf9a60065b 1025 # Find a suitable MUT:
kadonotakashi 0:8fdf9a60065b 1026 mut = None
kadonotakashi 0:8fdf9a60065b 1027 for id, m in self.muts.items():
kadonotakashi 0:8fdf9a60065b 1028 if m['mcu'] == data['mcu']:
kadonotakashi 0:8fdf9a60065b 1029 mut = m
kadonotakashi 0:8fdf9a60065b 1030 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
kadonotakashi 0:8fdf9a60065b 1031 handle_results.append(handle_result)
kadonotakashi 0:8fdf9a60065b 1032
kadonotakashi 0:8fdf9a60065b 1033 return handle_results
kadonotakashi 0:8fdf9a60065b 1034
kadonotakashi 0:8fdf9a60065b 1035 def print_test_result(self, test_result, target_name, toolchain_name,
kadonotakashi 0:8fdf9a60065b 1036 test_id, test_description, elapsed_time, duration):
kadonotakashi 0:8fdf9a60065b 1037 """ Use specific convention to print test result and related data
kadonotakashi 0:8fdf9a60065b 1038 """
kadonotakashi 0:8fdf9a60065b 1039 tokens = []
kadonotakashi 0:8fdf9a60065b 1040 tokens.append("TargetTest")
kadonotakashi 0:8fdf9a60065b 1041 tokens.append(target_name)
kadonotakashi 0:8fdf9a60065b 1042 tokens.append(toolchain_name)
kadonotakashi 0:8fdf9a60065b 1043 tokens.append(test_id)
kadonotakashi 0:8fdf9a60065b 1044 tokens.append(test_description)
kadonotakashi 0:8fdf9a60065b 1045 separator = "::"
kadonotakashi 0:8fdf9a60065b 1046 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
kadonotakashi 0:8fdf9a60065b 1047 result = separator.join(tokens) + " [" + test_result +"]" + time_info
kadonotakashi 0:8fdf9a60065b 1048 return Fore.MAGENTA + result + Fore.RESET
kadonotakashi 0:8fdf9a60065b 1049
kadonotakashi 0:8fdf9a60065b 1050 def shape_test_loop_ok_result_count(self, test_all_result):
kadonotakashi 0:8fdf9a60065b 1051 """ Reformats list of results to simple string
kadonotakashi 0:8fdf9a60065b 1052 """
kadonotakashi 0:8fdf9a60065b 1053 test_loop_count = len(test_all_result)
kadonotakashi 0:8fdf9a60065b 1054 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
kadonotakashi 0:8fdf9a60065b 1055 return "%d/%d"% (test_loop_ok_result, test_loop_count)
kadonotakashi 0:8fdf9a60065b 1056
kadonotakashi 0:8fdf9a60065b 1057 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
kadonotakashi 0:8fdf9a60065b 1058 """ Reformats list of results to simple string
kadonotakashi 0:8fdf9a60065b 1059 """
kadonotakashi 0:8fdf9a60065b 1060 result = self.TEST_RESULT_FAIL
kadonotakashi 0:8fdf9a60065b 1061
kadonotakashi 0:8fdf9a60065b 1062 if all(test_all_result[0] == res for res in test_all_result):
kadonotakashi 0:8fdf9a60065b 1063 result = test_all_result[0]
kadonotakashi 0:8fdf9a60065b 1064 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
kadonotakashi 0:8fdf9a60065b 1065 result = self.TEST_RESULT_OK
kadonotakashi 0:8fdf9a60065b 1066
kadonotakashi 0:8fdf9a60065b 1067 return result
kadonotakashi 0:8fdf9a60065b 1068
kadonotakashi 0:8fdf9a60065b 1069 def run_host_test(self, name, image_path, disk, port, duration,
kadonotakashi 0:8fdf9a60065b 1070 micro=None, reset=None, reset_tout=None,
kadonotakashi 0:8fdf9a60065b 1071 verbose=False, copy_method=None, program_cycle_s=None):
kadonotakashi 0:8fdf9a60065b 1072 """ Function creates new process with host test configured with particular test case.
kadonotakashi 0:8fdf9a60065b 1073 Function also is pooling for serial port activity from process to catch all data
kadonotakashi 0:8fdf9a60065b 1074 printed by test runner and host test during test execution
kadonotakashi 0:8fdf9a60065b 1075 """
kadonotakashi 0:8fdf9a60065b 1076
kadonotakashi 0:8fdf9a60065b 1077 def get_char_from_queue(obs):
kadonotakashi 0:8fdf9a60065b 1078 """ Get character from queue safe way
kadonotakashi 0:8fdf9a60065b 1079 """
kadonotakashi 0:8fdf9a60065b 1080 try:
kadonotakashi 0:8fdf9a60065b 1081 c = obs.queue.get(block=True, timeout=0.5)
kadonotakashi 0:8fdf9a60065b 1082 except Empty:
kadonotakashi 0:8fdf9a60065b 1083 c = None
kadonotakashi 0:8fdf9a60065b 1084 return c
kadonotakashi 0:8fdf9a60065b 1085
kadonotakashi 0:8fdf9a60065b 1086 def filter_queue_char(c):
kadonotakashi 0:8fdf9a60065b 1087 """ Filters out non ASCII characters from serial port
kadonotakashi 0:8fdf9a60065b 1088 """
kadonotakashi 0:8fdf9a60065b 1089 if ord(c) not in range(128):
kadonotakashi 0:8fdf9a60065b 1090 c = ' '
kadonotakashi 0:8fdf9a60065b 1091 return c
kadonotakashi 0:8fdf9a60065b 1092
kadonotakashi 0:8fdf9a60065b 1093 def get_test_result(output):
kadonotakashi 0:8fdf9a60065b 1094 """ Parse test 'output' data
kadonotakashi 0:8fdf9a60065b 1095 """
kadonotakashi 0:8fdf9a60065b 1096 result = self.TEST_RESULT_TIMEOUT
kadonotakashi 0:8fdf9a60065b 1097 for line in "".join(output).splitlines():
kadonotakashi 0:8fdf9a60065b 1098 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
kadonotakashi 0:8fdf9a60065b 1099 if search_result and len(search_result.groups()):
kadonotakashi 0:8fdf9a60065b 1100 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
kadonotakashi 0:8fdf9a60065b 1101 break
kadonotakashi 0:8fdf9a60065b 1102 return result
kadonotakashi 0:8fdf9a60065b 1103
kadonotakashi 0:8fdf9a60065b 1104 def get_auto_property_value(property_name, line):
kadonotakashi 0:8fdf9a60065b 1105 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
kadonotakashi 0:8fdf9a60065b 1106 Returns string
kadonotakashi 0:8fdf9a60065b 1107 """
kadonotakashi 0:8fdf9a60065b 1108 result = None
kadonotakashi 0:8fdf9a60065b 1109 if re.search("HOST: Property '%s'"% property_name, line) is not None:
kadonotakashi 0:8fdf9a60065b 1110 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
kadonotakashi 0:8fdf9a60065b 1111 if property is not None and len(property.groups()) == 1:
kadonotakashi 0:8fdf9a60065b 1112 result = property.groups()[0]
kadonotakashi 0:8fdf9a60065b 1113 return result
kadonotakashi 0:8fdf9a60065b 1114
kadonotakashi 0:8fdf9a60065b 1115 cmd = ["python",
kadonotakashi 0:8fdf9a60065b 1116 '%s.py'% name,
kadonotakashi 0:8fdf9a60065b 1117 '-d', disk,
kadonotakashi 0:8fdf9a60065b 1118 '-f', '"%s"'% image_path,
kadonotakashi 0:8fdf9a60065b 1119 '-p', port,
kadonotakashi 0:8fdf9a60065b 1120 '-t', str(duration),
kadonotakashi 0:8fdf9a60065b 1121 '-C', str(program_cycle_s)]
kadonotakashi 0:8fdf9a60065b 1122
kadonotakashi 0:8fdf9a60065b 1123 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
kadonotakashi 0:8fdf9a60065b 1124 cmd += ['--auto']
kadonotakashi 0:8fdf9a60065b 1125
kadonotakashi 0:8fdf9a60065b 1126 # Add extra parameters to host_test
kadonotakashi 0:8fdf9a60065b 1127 if copy_method is not None:
kadonotakashi 0:8fdf9a60065b 1128 cmd += ["-c", copy_method]
kadonotakashi 0:8fdf9a60065b 1129 if micro is not None:
kadonotakashi 0:8fdf9a60065b 1130 cmd += ["-m", micro]
kadonotakashi 0:8fdf9a60065b 1131 if reset is not None:
kadonotakashi 0:8fdf9a60065b 1132 cmd += ["-r", reset]
kadonotakashi 0:8fdf9a60065b 1133 if reset_tout is not None:
kadonotakashi 0:8fdf9a60065b 1134 cmd += ["-R", str(reset_tout)]
kadonotakashi 0:8fdf9a60065b 1135
kadonotakashi 0:8fdf9a60065b 1136 if verbose:
kadonotakashi 0:8fdf9a60065b 1137 print(Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET)
kadonotakashi 0:8fdf9a60065b 1138 print("Test::Output::Start")
kadonotakashi 0:8fdf9a60065b 1139
kadonotakashi 0:8fdf9a60065b 1140 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
kadonotakashi 0:8fdf9a60065b 1141 obs = ProcessObserver(proc)
kadonotakashi 0:8fdf9a60065b 1142 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
kadonotakashi 0:8fdf9a60065b 1143 line = ''
kadonotakashi 0:8fdf9a60065b 1144 output = []
kadonotakashi 0:8fdf9a60065b 1145 start_time = time()
kadonotakashi 0:8fdf9a60065b 1146 while (time() - start_time) < (2 * duration):
kadonotakashi 0:8fdf9a60065b 1147 c = get_char_from_queue(obs)
kadonotakashi 0:8fdf9a60065b 1148 if c:
kadonotakashi 0:8fdf9a60065b 1149 if verbose:
kadonotakashi 0:8fdf9a60065b 1150 sys.stdout.write(c)
kadonotakashi 0:8fdf9a60065b 1151 c = filter_queue_char(c)
kadonotakashi 0:8fdf9a60065b 1152 output.append(c)
kadonotakashi 0:8fdf9a60065b 1153 # Give the mbed under test a way to communicate the end of the test
kadonotakashi 0:8fdf9a60065b 1154 if c in ['\n', '\r']:
kadonotakashi 0:8fdf9a60065b 1155
kadonotakashi 0:8fdf9a60065b 1156 # Checking for auto-detection information from the test about MUT reset moment
kadonotakashi 0:8fdf9a60065b 1157 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
kadonotakashi 0:8fdf9a60065b 1158 # We will update this marker only once to prevent multiple time resets
kadonotakashi 0:8fdf9a60065b 1159 update_once_flag['reset_target'] = True
kadonotakashi 0:8fdf9a60065b 1160 start_time = time()
kadonotakashi 0:8fdf9a60065b 1161
kadonotakashi 0:8fdf9a60065b 1162 # Checking for auto-detection information from the test about timeout
kadonotakashi 0:8fdf9a60065b 1163 auto_timeout_val = get_auto_property_value('timeout', line)
kadonotakashi 0:8fdf9a60065b 1164 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
kadonotakashi 0:8fdf9a60065b 1165 # We will update this marker only once to prevent multiple time resets
kadonotakashi 0:8fdf9a60065b 1166 update_once_flag['timeout'] = True
kadonotakashi 0:8fdf9a60065b 1167 duration = int(auto_timeout_val)
kadonotakashi 0:8fdf9a60065b 1168
kadonotakashi 0:8fdf9a60065b 1169 # Detect mbed assert:
kadonotakashi 0:8fdf9a60065b 1170 if 'mbed assertation failed: ' in line:
kadonotakashi 0:8fdf9a60065b 1171 output.append('{{mbed_assert}}')
kadonotakashi 0:8fdf9a60065b 1172 break
kadonotakashi 0:8fdf9a60065b 1173
kadonotakashi 0:8fdf9a60065b 1174 # Check for test end
kadonotakashi 0:8fdf9a60065b 1175 if '{end}' in line:
kadonotakashi 0:8fdf9a60065b 1176 break
kadonotakashi 0:8fdf9a60065b 1177 line = ''
kadonotakashi 0:8fdf9a60065b 1178 else:
kadonotakashi 0:8fdf9a60065b 1179 line += c
kadonotakashi 0:8fdf9a60065b 1180 end_time = time()
kadonotakashi 0:8fdf9a60065b 1181 testcase_duration = end_time - start_time # Test case duration from reset to {end}
kadonotakashi 0:8fdf9a60065b 1182
kadonotakashi 0:8fdf9a60065b 1183 c = get_char_from_queue(obs)
kadonotakashi 0:8fdf9a60065b 1184
kadonotakashi 0:8fdf9a60065b 1185 if c:
kadonotakashi 0:8fdf9a60065b 1186 if verbose:
kadonotakashi 0:8fdf9a60065b 1187 sys.stdout.write(c)
kadonotakashi 0:8fdf9a60065b 1188 c = filter_queue_char(c)
kadonotakashi 0:8fdf9a60065b 1189 output.append(c)
kadonotakashi 0:8fdf9a60065b 1190
kadonotakashi 0:8fdf9a60065b 1191 if verbose:
kadonotakashi 0:8fdf9a60065b 1192 print("Test::Output::Finish")
kadonotakashi 0:8fdf9a60065b 1193 # Stop test process
kadonotakashi 0:8fdf9a60065b 1194 obs.stop()
kadonotakashi 0:8fdf9a60065b 1195
kadonotakashi 0:8fdf9a60065b 1196 result = get_test_result(output)
kadonotakashi 0:8fdf9a60065b 1197 return (result, "".join(output), testcase_duration, duration)
kadonotakashi 0:8fdf9a60065b 1198
kadonotakashi 0:8fdf9a60065b 1199 def is_peripherals_available(self, target_mcu_name, peripherals=None):
kadonotakashi 0:8fdf9a60065b 1200 """ Checks if specified target should run specific peripheral test case defined in MUTs file
kadonotakashi 0:8fdf9a60065b 1201 """
kadonotakashi 0:8fdf9a60065b 1202 if peripherals is not None:
kadonotakashi 0:8fdf9a60065b 1203 peripherals = set(peripherals)
kadonotakashi 0:8fdf9a60065b 1204 for id, mut in self.muts.items():
kadonotakashi 0:8fdf9a60065b 1205 # Target MCU name check
kadonotakashi 0:8fdf9a60065b 1206 if mut["mcu"] != target_mcu_name:
kadonotakashi 0:8fdf9a60065b 1207 continue
kadonotakashi 0:8fdf9a60065b 1208 # Peripherals check
kadonotakashi 0:8fdf9a60065b 1209 if peripherals is not None:
kadonotakashi 0:8fdf9a60065b 1210 if 'peripherals' not in mut:
kadonotakashi 0:8fdf9a60065b 1211 continue
kadonotakashi 0:8fdf9a60065b 1212 if not peripherals.issubset(set(mut['peripherals'])):
kadonotakashi 0:8fdf9a60065b 1213 continue
kadonotakashi 0:8fdf9a60065b 1214 return True
kadonotakashi 0:8fdf9a60065b 1215 return False
kadonotakashi 0:8fdf9a60065b 1216
kadonotakashi 0:8fdf9a60065b 1217 def shape_test_request(self, mcu, image_path, test_id, duration=10):
kadonotakashi 0:8fdf9a60065b 1218 """ Function prepares JSON structure describing test specification
kadonotakashi 0:8fdf9a60065b 1219 """
kadonotakashi 0:8fdf9a60065b 1220 test_spec = {
kadonotakashi 0:8fdf9a60065b 1221 "mcu": mcu,
kadonotakashi 0:8fdf9a60065b 1222 "image": image_path,
kadonotakashi 0:8fdf9a60065b 1223 "duration": duration,
kadonotakashi 0:8fdf9a60065b 1224 "test_id": test_id,
kadonotakashi 0:8fdf9a60065b 1225 }
kadonotakashi 0:8fdf9a60065b 1226 return json.dumps(test_spec)
kadonotakashi 0:8fdf9a60065b 1227
kadonotakashi 0:8fdf9a60065b 1228
kadonotakashi 0:8fdf9a60065b 1229 def get_unique_value_from_summary(test_summary, index):
kadonotakashi 0:8fdf9a60065b 1230 """ Gets list of unique target names
kadonotakashi 0:8fdf9a60065b 1231 """
kadonotakashi 0:8fdf9a60065b 1232 result = []
kadonotakashi 0:8fdf9a60065b 1233 for test in test_summary:
kadonotakashi 0:8fdf9a60065b 1234 target_name = test[index]
kadonotakashi 0:8fdf9a60065b 1235 if target_name not in result:
kadonotakashi 0:8fdf9a60065b 1236 result.append(target_name)
kadonotakashi 0:8fdf9a60065b 1237 return sorted(result)
kadonotakashi 0:8fdf9a60065b 1238
kadonotakashi 0:8fdf9a60065b 1239
kadonotakashi 0:8fdf9a60065b 1240 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
kadonotakashi 0:8fdf9a60065b 1241 """ Gets list of unique target names and return dictionary
kadonotakashi 0:8fdf9a60065b 1242 """
kadonotakashi 0:8fdf9a60065b 1243 result = {}
kadonotakashi 0:8fdf9a60065b 1244 for test in test_summary:
kadonotakashi 0:8fdf9a60065b 1245 key = test[index_key]
kadonotakashi 0:8fdf9a60065b 1246 val = test[index_val]
kadonotakashi 0:8fdf9a60065b 1247 if key not in result:
kadonotakashi 0:8fdf9a60065b 1248 result[key] = val
kadonotakashi 0:8fdf9a60065b 1249 return result
kadonotakashi 0:8fdf9a60065b 1250
kadonotakashi 0:8fdf9a60065b 1251
kadonotakashi 0:8fdf9a60065b 1252 def show_json_file_format_error(json_spec_filename, line, column):
kadonotakashi 0:8fdf9a60065b 1253 """ Prints JSON broken content
kadonotakashi 0:8fdf9a60065b 1254 """
kadonotakashi 0:8fdf9a60065b 1255 with open(json_spec_filename) as data_file:
kadonotakashi 0:8fdf9a60065b 1256 line_no = 1
kadonotakashi 0:8fdf9a60065b 1257 for json_line in data_file:
kadonotakashi 0:8fdf9a60065b 1258 if line_no + 5 >= line: # Print last few lines before error
kadonotakashi 0:8fdf9a60065b 1259 print('Line %d:\t'%line_no + json_line)
kadonotakashi 0:8fdf9a60065b 1260 if line_no == line:
kadonotakashi 0:8fdf9a60065b 1261 print('%s\t%s^' (' ' * len('Line %d:' % line_no),
kadonotakashi 0:8fdf9a60065b 1262 '-' * (column - 1)))
kadonotakashi 0:8fdf9a60065b 1263 break
kadonotakashi 0:8fdf9a60065b 1264 line_no += 1
kadonotakashi 0:8fdf9a60065b 1265
kadonotakashi 0:8fdf9a60065b 1266
kadonotakashi 0:8fdf9a60065b 1267 def json_format_error_defect_pos(json_error_msg):
kadonotakashi 0:8fdf9a60065b 1268 """ Gets first error line and column in JSON file format.
kadonotakashi 0:8fdf9a60065b 1269 Parsed from exception thrown by json.loads() string
kadonotakashi 0:8fdf9a60065b 1270 """
kadonotakashi 0:8fdf9a60065b 1271 result = None
kadonotakashi 0:8fdf9a60065b 1272 line, column = 0, 0
kadonotakashi 0:8fdf9a60065b 1273 # Line value search
kadonotakashi 0:8fdf9a60065b 1274 line_search = re.search('line [0-9]+', json_error_msg)
kadonotakashi 0:8fdf9a60065b 1275 if line_search is not None:
kadonotakashi 0:8fdf9a60065b 1276 ls = line_search.group().split(' ')
kadonotakashi 0:8fdf9a60065b 1277 if len(ls) == 2:
kadonotakashi 0:8fdf9a60065b 1278 line = int(ls[1])
kadonotakashi 0:8fdf9a60065b 1279 # Column position search
kadonotakashi 0:8fdf9a60065b 1280 column_search = re.search('column [0-9]+', json_error_msg)
kadonotakashi 0:8fdf9a60065b 1281 if column_search is not None:
kadonotakashi 0:8fdf9a60065b 1282 cs = column_search.group().split(' ')
kadonotakashi 0:8fdf9a60065b 1283 if len(cs) == 2:
kadonotakashi 0:8fdf9a60065b 1284 column = int(cs[1])
kadonotakashi 0:8fdf9a60065b 1285 result = [line, column]
kadonotakashi 0:8fdf9a60065b 1286 return result
kadonotakashi 0:8fdf9a60065b 1287
kadonotakashi 0:8fdf9a60065b 1288
kadonotakashi 0:8fdf9a60065b 1289 def get_json_data_from_file(json_spec_filename, verbose=False):
kadonotakashi 0:8fdf9a60065b 1290 """ Loads from file JSON formatted string to data structure
kadonotakashi 0:8fdf9a60065b 1291 """
kadonotakashi 0:8fdf9a60065b 1292 result = None
kadonotakashi 0:8fdf9a60065b 1293 try:
kadonotakashi 0:8fdf9a60065b 1294 with open(json_spec_filename) as data_file:
kadonotakashi 0:8fdf9a60065b 1295 try:
kadonotakashi 0:8fdf9a60065b 1296 result = json.load(data_file)
kadonotakashi 0:8fdf9a60065b 1297 except ValueError as json_error_msg:
kadonotakashi 0:8fdf9a60065b 1298 result = None
kadonotakashi 0:8fdf9a60065b 1299 print('JSON file %s parsing failed. Reason: %s' %
kadonotakashi 0:8fdf9a60065b 1300 (json_spec_filename, json_error_msg))
kadonotakashi 0:8fdf9a60065b 1301 # We can print where error occurred inside JSON file if we can parse exception msg
kadonotakashi 0:8fdf9a60065b 1302 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
kadonotakashi 0:8fdf9a60065b 1303 if json_format_defect_pos is not None:
kadonotakashi 0:8fdf9a60065b 1304 line = json_format_defect_pos[0]
kadonotakashi 0:8fdf9a60065b 1305 column = json_format_defect_pos[1]
kadonotakashi 0:8fdf9a60065b 1306 print()
kadonotakashi 0:8fdf9a60065b 1307 show_json_file_format_error(json_spec_filename, line, column)
kadonotakashi 0:8fdf9a60065b 1308
kadonotakashi 0:8fdf9a60065b 1309 except IOError as fileopen_error_msg:
kadonotakashi 0:8fdf9a60065b 1310 print('JSON file %s not opened. Reason: %s\n'%
kadonotakashi 0:8fdf9a60065b 1311 (json_spec_filename, fileopen_error_msg))
kadonotakashi 0:8fdf9a60065b 1312 if verbose and result:
kadonotakashi 0:8fdf9a60065b 1313 pp = pprint.PrettyPrinter(indent=4)
kadonotakashi 0:8fdf9a60065b 1314 pp.pprint(result)
kadonotakashi 0:8fdf9a60065b 1315 return result
kadonotakashi 0:8fdf9a60065b 1316
kadonotakashi 0:8fdf9a60065b 1317
kadonotakashi 0:8fdf9a60065b 1318 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
kadonotakashi 0:8fdf9a60065b 1319 """ Prints MUTs configuration passed to test script for verboseness
kadonotakashi 0:8fdf9a60065b 1320 """
kadonotakashi 0:8fdf9a60065b 1321 muts_info_cols = []
kadonotakashi 0:8fdf9a60065b 1322 # We need to check all unique properties for each defined MUT
kadonotakashi 0:8fdf9a60065b 1323 for k in json_data:
kadonotakashi 0:8fdf9a60065b 1324 mut_info = json_data[k]
kadonotakashi 0:8fdf9a60065b 1325 for mut_property in mut_info:
kadonotakashi 0:8fdf9a60065b 1326 if mut_property not in muts_info_cols:
kadonotakashi 0:8fdf9a60065b 1327 muts_info_cols.append(mut_property)
kadonotakashi 0:8fdf9a60065b 1328
kadonotakashi 0:8fdf9a60065b 1329 # Prepare pretty table object to display all MUTs
kadonotakashi 0:8fdf9a60065b 1330 pt_cols = ["index"] + muts_info_cols
kadonotakashi 0:8fdf9a60065b 1331 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 1332 for col in pt_cols:
kadonotakashi 0:8fdf9a60065b 1333 pt.align[col] = "l"
kadonotakashi 0:8fdf9a60065b 1334
kadonotakashi 0:8fdf9a60065b 1335 # Add rows to pretty print object
kadonotakashi 0:8fdf9a60065b 1336 for k in json_data:
kadonotakashi 0:8fdf9a60065b 1337 row = [k]
kadonotakashi 0:8fdf9a60065b 1338 mut_info = json_data[k]
kadonotakashi 0:8fdf9a60065b 1339
kadonotakashi 0:8fdf9a60065b 1340 add_row = True
kadonotakashi 0:8fdf9a60065b 1341 if platform_filter and 'mcu' in mut_info:
kadonotakashi 0:8fdf9a60065b 1342 add_row = re.search(platform_filter, mut_info['mcu']) is not None
kadonotakashi 0:8fdf9a60065b 1343 if add_row:
kadonotakashi 0:8fdf9a60065b 1344 for col in muts_info_cols:
kadonotakashi 0:8fdf9a60065b 1345 cell_val = mut_info[col] if col in mut_info else None
kadonotakashi 0:8fdf9a60065b 1346 if isinstance(cell_val, list):
kadonotakashi 0:8fdf9a60065b 1347 cell_val = join_delim.join(cell_val)
kadonotakashi 0:8fdf9a60065b 1348 row.append(cell_val)
kadonotakashi 0:8fdf9a60065b 1349 pt.add_row(row)
kadonotakashi 0:8fdf9a60065b 1350 return pt.get_string()
kadonotakashi 0:8fdf9a60065b 1351
kadonotakashi 0:8fdf9a60065b 1352
kadonotakashi 0:8fdf9a60065b 1353 def print_test_configuration_from_json(json_data, join_delim=", "):
kadonotakashi 0:8fdf9a60065b 1354 """ Prints test specification configuration passed to test script for verboseness
kadonotakashi 0:8fdf9a60065b 1355 """
kadonotakashi 0:8fdf9a60065b 1356 toolchains_info_cols = []
kadonotakashi 0:8fdf9a60065b 1357 # We need to check all toolchains for each device
kadonotakashi 0:8fdf9a60065b 1358 for k in json_data:
kadonotakashi 0:8fdf9a60065b 1359 # k should be 'targets'
kadonotakashi 0:8fdf9a60065b 1360 targets = json_data[k]
kadonotakashi 0:8fdf9a60065b 1361 for target in targets:
kadonotakashi 0:8fdf9a60065b 1362 toolchains = targets[target]
kadonotakashi 0:8fdf9a60065b 1363 for toolchain in toolchains:
kadonotakashi 0:8fdf9a60065b 1364 if toolchain not in toolchains_info_cols:
kadonotakashi 0:8fdf9a60065b 1365 toolchains_info_cols.append(toolchain)
kadonotakashi 0:8fdf9a60065b 1366
kadonotakashi 0:8fdf9a60065b 1367 # Prepare pretty table object to display test specification
kadonotakashi 0:8fdf9a60065b 1368 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
kadonotakashi 0:8fdf9a60065b 1369 pt = PrettyTable(pt_cols, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 1370 for col in pt_cols:
kadonotakashi 0:8fdf9a60065b 1371 pt.align[col] = "l"
kadonotakashi 0:8fdf9a60065b 1372
kadonotakashi 0:8fdf9a60065b 1373 # { target : [conflicted toolchains] }
kadonotakashi 0:8fdf9a60065b 1374 toolchain_conflicts = {}
kadonotakashi 0:8fdf9a60065b 1375 toolchain_path_conflicts = []
kadonotakashi 0:8fdf9a60065b 1376 for k in json_data:
kadonotakashi 0:8fdf9a60065b 1377 # k should be 'targets'
kadonotakashi 0:8fdf9a60065b 1378 targets = json_data[k]
kadonotakashi 0:8fdf9a60065b 1379 for target in targets:
kadonotakashi 0:8fdf9a60065b 1380 target_supported_toolchains = get_target_supported_toolchains(target)
kadonotakashi 0:8fdf9a60065b 1381 if not target_supported_toolchains:
kadonotakashi 0:8fdf9a60065b 1382 target_supported_toolchains = []
kadonotakashi 0:8fdf9a60065b 1383 target_name = target if target in TARGET_MAP else "%s*"% target
kadonotakashi 0:8fdf9a60065b 1384 row = [target_name]
kadonotakashi 0:8fdf9a60065b 1385 toolchains = targets[target]
kadonotakashi 0:8fdf9a60065b 1386
kadonotakashi 0:8fdf9a60065b 1387 for toolchain in sorted(toolchains_info_cols):
kadonotakashi 0:8fdf9a60065b 1388 # Check for conflicts: target vs toolchain
kadonotakashi 0:8fdf9a60065b 1389 conflict = False
kadonotakashi 0:8fdf9a60065b 1390 conflict_path = False
kadonotakashi 0:8fdf9a60065b 1391 if toolchain in toolchains:
kadonotakashi 0:8fdf9a60065b 1392 if toolchain not in target_supported_toolchains:
kadonotakashi 0:8fdf9a60065b 1393 conflict = True
kadonotakashi 0:8fdf9a60065b 1394 if target not in toolchain_conflicts:
kadonotakashi 0:8fdf9a60065b 1395 toolchain_conflicts[target] = []
kadonotakashi 0:8fdf9a60065b 1396 toolchain_conflicts[target].append(toolchain)
kadonotakashi 0:8fdf9a60065b 1397 # Add marker inside table about target usage / conflict
kadonotakashi 0:8fdf9a60065b 1398 cell_val = 'Yes' if toolchain in toolchains else '-'
kadonotakashi 0:8fdf9a60065b 1399 if conflict:
kadonotakashi 0:8fdf9a60065b 1400 cell_val += '*'
kadonotakashi 0:8fdf9a60065b 1401 # Check for conflicts: toolchain vs toolchain path
kadonotakashi 0:8fdf9a60065b 1402 if toolchain in TOOLCHAIN_PATHS:
kadonotakashi 0:8fdf9a60065b 1403 toolchain_path = TOOLCHAIN_PATHS[toolchain]
kadonotakashi 0:8fdf9a60065b 1404 if not os.path.isdir(toolchain_path):
kadonotakashi 0:8fdf9a60065b 1405 conflict_path = True
kadonotakashi 0:8fdf9a60065b 1406 if toolchain not in toolchain_path_conflicts:
kadonotakashi 0:8fdf9a60065b 1407 toolchain_path_conflicts.append(toolchain)
kadonotakashi 0:8fdf9a60065b 1408 if conflict_path:
kadonotakashi 0:8fdf9a60065b 1409 cell_val += '#'
kadonotakashi 0:8fdf9a60065b 1410 row.append(cell_val)
kadonotakashi 0:8fdf9a60065b 1411 pt.add_row(row)
kadonotakashi 0:8fdf9a60065b 1412
kadonotakashi 0:8fdf9a60065b 1413 # generate result string
kadonotakashi 0:8fdf9a60065b 1414 result = pt.get_string() # Test specification table
kadonotakashi 0:8fdf9a60065b 1415 if toolchain_conflicts or toolchain_path_conflicts:
kadonotakashi 0:8fdf9a60065b 1416 result += "\n"
kadonotakashi 0:8fdf9a60065b 1417 result += "Toolchain conflicts:\n"
kadonotakashi 0:8fdf9a60065b 1418 for target in toolchain_conflicts:
kadonotakashi 0:8fdf9a60065b 1419 if target not in TARGET_MAP:
kadonotakashi 0:8fdf9a60065b 1420 result += "\t* Target %s unknown\n"% (target)
kadonotakashi 0:8fdf9a60065b 1421 conflict_target_list = join_delim.join(toolchain_conflicts[target])
kadonotakashi 0:8fdf9a60065b 1422 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
kadonotakashi 0:8fdf9a60065b 1423 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
kadonotakashi 0:8fdf9a60065b 1424
kadonotakashi 0:8fdf9a60065b 1425 for toolchain in toolchain_path_conflicts:
kadonotakashi 0:8fdf9a60065b 1426 # Let's check toolchain configuration
kadonotakashi 0:8fdf9a60065b 1427 if toolchain in TOOLCHAIN_PATHS:
kadonotakashi 0:8fdf9a60065b 1428 toolchain_path = TOOLCHAIN_PATHS[toolchain]
kadonotakashi 0:8fdf9a60065b 1429 if not os.path.isdir(toolchain_path):
kadonotakashi 0:8fdf9a60065b 1430 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
kadonotakashi 0:8fdf9a60065b 1431 return result
kadonotakashi 0:8fdf9a60065b 1432
kadonotakashi 0:8fdf9a60065b 1433
kadonotakashi 0:8fdf9a60065b 1434 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
kadonotakashi 0:8fdf9a60065b 1435 """ Generates table summary with all test cases and additional test cases
kadonotakashi 0:8fdf9a60065b 1436 information using pretty print functionality. Allows test suite user to
kadonotakashi 0:8fdf9a60065b 1437 see test cases
kadonotakashi 0:8fdf9a60065b 1438 """
kadonotakashi 0:8fdf9a60065b 1439 # get all unique test ID prefixes
kadonotakashi 0:8fdf9a60065b 1440 unique_test_id = []
kadonotakashi 0:8fdf9a60065b 1441 for test in TESTS:
kadonotakashi 0:8fdf9a60065b 1442 split = test['id'].split('_')[:-1]
kadonotakashi 0:8fdf9a60065b 1443 test_id_prefix = '_'.join(split)
kadonotakashi 0:8fdf9a60065b 1444 if test_id_prefix not in unique_test_id:
kadonotakashi 0:8fdf9a60065b 1445 unique_test_id.append(test_id_prefix)
kadonotakashi 0:8fdf9a60065b 1446 unique_test_id.sort()
kadonotakashi 0:8fdf9a60065b 1447 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
kadonotakashi 0:8fdf9a60065b 1448 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
kadonotakashi 0:8fdf9a60065b 1449
kadonotakashi 0:8fdf9a60065b 1450 test_properties = ['id',
kadonotakashi 0:8fdf9a60065b 1451 'automated',
kadonotakashi 0:8fdf9a60065b 1452 'description',
kadonotakashi 0:8fdf9a60065b 1453 'peripherals',
kadonotakashi 0:8fdf9a60065b 1454 'host_test',
kadonotakashi 0:8fdf9a60065b 1455 'duration'] if cols is None else cols
kadonotakashi 0:8fdf9a60065b 1456
kadonotakashi 0:8fdf9a60065b 1457 # All tests status table print
kadonotakashi 0:8fdf9a60065b 1458 pt = PrettyTable(test_properties, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 1459 for col in test_properties:
kadonotakashi 0:8fdf9a60065b 1460 pt.align[col] = "l"
kadonotakashi 0:8fdf9a60065b 1461 pt.align['duration'] = "r"
kadonotakashi 0:8fdf9a60065b 1462
kadonotakashi 0:8fdf9a60065b 1463 counter_all = 0
kadonotakashi 0:8fdf9a60065b 1464 counter_automated = 0
kadonotakashi 0:8fdf9a60065b 1465 pt.padding_width = 1 # One space between column edges and contents (default)
kadonotakashi 0:8fdf9a60065b 1466
kadonotakashi 0:8fdf9a60065b 1467 for test_id in sorted(TEST_MAP.keys()):
kadonotakashi 0:8fdf9a60065b 1468 if platform_filter is not None:
kadonotakashi 0:8fdf9a60065b 1469 # FIlter out platforms using regex
kadonotakashi 0:8fdf9a60065b 1470 if re.search(platform_filter, test_id) is None:
kadonotakashi 0:8fdf9a60065b 1471 continue
kadonotakashi 0:8fdf9a60065b 1472 row = []
kadonotakashi 0:8fdf9a60065b 1473 test = TEST_MAP[test_id]
kadonotakashi 0:8fdf9a60065b 1474 split = test_id.split('_')[:-1]
kadonotakashi 0:8fdf9a60065b 1475 test_id_prefix = '_'.join(split)
kadonotakashi 0:8fdf9a60065b 1476
kadonotakashi 0:8fdf9a60065b 1477 for col in test_properties:
kadonotakashi 0:8fdf9a60065b 1478 col_value = test[col]
kadonotakashi 0:8fdf9a60065b 1479 if isinstance(test[col], list):
kadonotakashi 0:8fdf9a60065b 1480 col_value = join_delim.join(test[col])
kadonotakashi 0:8fdf9a60065b 1481 elif test[col] == None:
kadonotakashi 0:8fdf9a60065b 1482 col_value = "-"
kadonotakashi 0:8fdf9a60065b 1483
kadonotakashi 0:8fdf9a60065b 1484 row.append(col_value)
kadonotakashi 0:8fdf9a60065b 1485 if test['automated'] == True:
kadonotakashi 0:8fdf9a60065b 1486 counter_dict_test_id_types[test_id_prefix] += 1
kadonotakashi 0:8fdf9a60065b 1487 counter_automated += 1
kadonotakashi 0:8fdf9a60065b 1488 pt.add_row(row)
kadonotakashi 0:8fdf9a60065b 1489 # Update counters
kadonotakashi 0:8fdf9a60065b 1490 counter_all += 1
kadonotakashi 0:8fdf9a60065b 1491 counter_dict_test_id_types_all[test_id_prefix] += 1
kadonotakashi 0:8fdf9a60065b 1492 result = pt.get_string()
kadonotakashi 0:8fdf9a60065b 1493 result += "\n\n"
kadonotakashi 0:8fdf9a60065b 1494
kadonotakashi 0:8fdf9a60065b 1495 if result_summary and not platform_filter:
kadonotakashi 0:8fdf9a60065b 1496 # Automation result summary
kadonotakashi 0:8fdf9a60065b 1497 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
kadonotakashi 0:8fdf9a60065b 1498 pt = PrettyTable(test_id_cols, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 1499 pt.align['automated'] = "r"
kadonotakashi 0:8fdf9a60065b 1500 pt.align['all'] = "r"
kadonotakashi 0:8fdf9a60065b 1501 pt.align['percent [%]'] = "r"
kadonotakashi 0:8fdf9a60065b 1502
kadonotakashi 0:8fdf9a60065b 1503 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
kadonotakashi 0:8fdf9a60065b 1504 str_progress = progress_bar(percent_progress, 75)
kadonotakashi 0:8fdf9a60065b 1505 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
kadonotakashi 0:8fdf9a60065b 1506 result += "Automation coverage:\n"
kadonotakashi 0:8fdf9a60065b 1507 result += pt.get_string()
kadonotakashi 0:8fdf9a60065b 1508 result += "\n\n"
kadonotakashi 0:8fdf9a60065b 1509
kadonotakashi 0:8fdf9a60065b 1510 # Test automation coverage table print
kadonotakashi 0:8fdf9a60065b 1511 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
kadonotakashi 0:8fdf9a60065b 1512 pt = PrettyTable(test_id_cols, junction_char="|", hrules=HEADER)
kadonotakashi 0:8fdf9a60065b 1513 pt.align['id'] = "l"
kadonotakashi 0:8fdf9a60065b 1514 pt.align['automated'] = "r"
kadonotakashi 0:8fdf9a60065b 1515 pt.align['all'] = "r"
kadonotakashi 0:8fdf9a60065b 1516 pt.align['percent [%]'] = "r"
kadonotakashi 0:8fdf9a60065b 1517 for unique_id in unique_test_id:
kadonotakashi 0:8fdf9a60065b 1518 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
kadonotakashi 0:8fdf9a60065b 1519 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
kadonotakashi 0:8fdf9a60065b 1520 str_progress = progress_bar(percent_progress, 75)
kadonotakashi 0:8fdf9a60065b 1521 row = [unique_id,
kadonotakashi 0:8fdf9a60065b 1522 counter_dict_test_id_types[unique_id],
kadonotakashi 0:8fdf9a60065b 1523 counter_dict_test_id_types_all[unique_id],
kadonotakashi 0:8fdf9a60065b 1524 percent_progress,
kadonotakashi 0:8fdf9a60065b 1525 "[" + str_progress + "]"]
kadonotakashi 0:8fdf9a60065b 1526 pt.add_row(row)
kadonotakashi 0:8fdf9a60065b 1527 result += "Test automation coverage:\n"
kadonotakashi 0:8fdf9a60065b 1528 result += pt.get_string()
kadonotakashi 0:8fdf9a60065b 1529 result += "\n\n"
kadonotakashi 0:8fdf9a60065b 1530 return result
kadonotakashi 0:8fdf9a60065b 1531
kadonotakashi 0:8fdf9a60065b 1532
kadonotakashi 0:8fdf9a60065b 1533 def progress_bar(percent_progress, saturation=0):
kadonotakashi 0:8fdf9a60065b 1534 """ This function creates progress bar with optional simple saturation mark
kadonotakashi 0:8fdf9a60065b 1535 """
kadonotakashi 0:8fdf9a60065b 1536 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
kadonotakashi 0:8fdf9a60065b 1537 str_progress = '#' * step + '.' * int(50 - step)
kadonotakashi 0:8fdf9a60065b 1538 c = '!' if str_progress[38] == '.' else '|'
kadonotakashi 0:8fdf9a60065b 1539 if saturation > 0:
kadonotakashi 0:8fdf9a60065b 1540 saturation = saturation / 2
kadonotakashi 0:8fdf9a60065b 1541 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
kadonotakashi 0:8fdf9a60065b 1542 return str_progress
kadonotakashi 0:8fdf9a60065b 1543
kadonotakashi 0:8fdf9a60065b 1544
kadonotakashi 0:8fdf9a60065b 1545 def singletest_in_cli_mode(single_test):
kadonotakashi 0:8fdf9a60065b 1546 """ Runs SingleTestRunner object in CLI (Command line interface) mode
kadonotakashi 0:8fdf9a60065b 1547
kadonotakashi 0:8fdf9a60065b 1548 @return returns success code (0 == success) for building and running tests
kadonotakashi 0:8fdf9a60065b 1549 """
kadonotakashi 0:8fdf9a60065b 1550 start = time()
kadonotakashi 0:8fdf9a60065b 1551 # Execute tests depending on options and filter applied
kadonotakashi 0:8fdf9a60065b 1552 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
kadonotakashi 0:8fdf9a60065b 1553 elapsed_time = time() - start
kadonotakashi 0:8fdf9a60065b 1554
kadonotakashi 0:8fdf9a60065b 1555 # Human readable summary
kadonotakashi 0:8fdf9a60065b 1556 if not single_test.opts_suppress_summary:
kadonotakashi 0:8fdf9a60065b 1557 # prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 1558 print(single_test.generate_test_summary(test_summary, shuffle_seed))
kadonotakashi 0:8fdf9a60065b 1559 if single_test.opts_test_x_toolchain_summary:
kadonotakashi 0:8fdf9a60065b 1560 # prints well-formed summary with results (SQL table like)
kadonotakashi 0:8fdf9a60065b 1561 # table shows text x toolchain test result matrix
kadonotakashi 0:8fdf9a60065b 1562 print(single_test.generate_test_summary_by_target(test_summary,
kadonotakashi 0:8fdf9a60065b 1563 shuffle_seed))
kadonotakashi 0:8fdf9a60065b 1564
kadonotakashi 0:8fdf9a60065b 1565 print("Completed in %.2f sec" % elapsed_time)
kadonotakashi 0:8fdf9a60065b 1566 print
kadonotakashi 0:8fdf9a60065b 1567 # Write summary of the builds
kadonotakashi 0:8fdf9a60065b 1568
kadonotakashi 0:8fdf9a60065b 1569 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
kadonotakashi 0:8fdf9a60065b 1570 status = print_report_exporter.report(build_report)
kadonotakashi 0:8fdf9a60065b 1571
kadonotakashi 0:8fdf9a60065b 1572 # Store extra reports in files
kadonotakashi 0:8fdf9a60065b 1573 if single_test.opts_report_html_file_name:
kadonotakashi 0:8fdf9a60065b 1574 # Export results in form of HTML report to separate file
kadonotakashi 0:8fdf9a60065b 1575 report_exporter = ReportExporter(ResultExporterType.HTML)
kadonotakashi 0:8fdf9a60065b 1576 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
kadonotakashi 0:8fdf9a60065b 1577 if single_test.opts_report_junit_file_name:
kadonotakashi 0:8fdf9a60065b 1578 # Export results in form of JUnit XML report to separate file
kadonotakashi 0:8fdf9a60065b 1579 report_exporter = ReportExporter(ResultExporterType.JUNIT)
kadonotakashi 0:8fdf9a60065b 1580 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
kadonotakashi 0:8fdf9a60065b 1581 if single_test.opts_report_text_file_name:
kadonotakashi 0:8fdf9a60065b 1582 # Export results in form of a text file
kadonotakashi 0:8fdf9a60065b 1583 report_exporter = ReportExporter(ResultExporterType.TEXT)
kadonotakashi 0:8fdf9a60065b 1584 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
kadonotakashi 0:8fdf9a60065b 1585 if single_test.opts_report_build_file_name:
kadonotakashi 0:8fdf9a60065b 1586 # Export build results as html report to sparate file
kadonotakashi 0:8fdf9a60065b 1587 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
kadonotakashi 0:8fdf9a60065b 1588 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
kadonotakashi 0:8fdf9a60065b 1589
kadonotakashi 0:8fdf9a60065b 1590 # Returns True if no build failures of the test projects or their dependencies
kadonotakashi 0:8fdf9a60065b 1591 return status
kadonotakashi 0:8fdf9a60065b 1592
kadonotakashi 0:8fdf9a60065b 1593 class TestLogger():
kadonotakashi 0:8fdf9a60065b 1594 """ Super-class for logging and printing ongoing events for test suite pass
kadonotakashi 0:8fdf9a60065b 1595 """
kadonotakashi 0:8fdf9a60065b 1596 def __init__(self, store_log=True):
kadonotakashi 0:8fdf9a60065b 1597 """ We can control if logger actually stores log in memory
kadonotakashi 0:8fdf9a60065b 1598 or just handled all log entries immediately
kadonotakashi 0:8fdf9a60065b 1599 """
kadonotakashi 0:8fdf9a60065b 1600 self.log = []
kadonotakashi 0:8fdf9a60065b 1601 self.log_to_file = False
kadonotakashi 0:8fdf9a60065b 1602 self.log_file_name = None
kadonotakashi 0:8fdf9a60065b 1603 self.store_log = store_log
kadonotakashi 0:8fdf9a60065b 1604
kadonotakashi 0:8fdf9a60065b 1605 self.LogType = construct_enum(INFO='Info',
kadonotakashi 0:8fdf9a60065b 1606 WARN='Warning',
kadonotakashi 0:8fdf9a60065b 1607 NOTIF='Notification',
kadonotakashi 0:8fdf9a60065b 1608 ERROR='Error',
kadonotakashi 0:8fdf9a60065b 1609 EXCEPT='Exception')
kadonotakashi 0:8fdf9a60065b 1610
kadonotakashi 0:8fdf9a60065b 1611 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
kadonotakashi 0:8fdf9a60065b 1612 APPEND=2) # Append to existing log file
kadonotakashi 0:8fdf9a60065b 1613
kadonotakashi 0:8fdf9a60065b 1614 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
kadonotakashi 0:8fdf9a60065b 1615 """ Log one line of text
kadonotakashi 0:8fdf9a60065b 1616 """
kadonotakashi 0:8fdf9a60065b 1617 log_timestamp = time()
kadonotakashi 0:8fdf9a60065b 1618 log_entry = {'log_type' : LogType,
kadonotakashi 0:8fdf9a60065b 1619 'log_timestamp' : log_timestamp,
kadonotakashi 0:8fdf9a60065b 1620 'log_line' : log_line,
kadonotakashi 0:8fdf9a60065b 1621 '_future' : None
kadonotakashi 0:8fdf9a60065b 1622 }
kadonotakashi 0:8fdf9a60065b 1623 # Store log in memory
kadonotakashi 0:8fdf9a60065b 1624 if self.store_log:
kadonotakashi 0:8fdf9a60065b 1625 self.log.append(log_entry)
kadonotakashi 0:8fdf9a60065b 1626 return log_entry
kadonotakashi 0:8fdf9a60065b 1627
kadonotakashi 0:8fdf9a60065b 1628
kadonotakashi 0:8fdf9a60065b 1629 class CLITestLogger(TestLogger):
kadonotakashi 0:8fdf9a60065b 1630 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
kadonotakashi 0:8fdf9a60065b 1631 """
kadonotakashi 0:8fdf9a60065b 1632 def __init__(self, store_log=True, file_name=None):
kadonotakashi 0:8fdf9a60065b 1633 TestLogger.__init__(self)
kadonotakashi 0:8fdf9a60065b 1634 self.log_file_name = file_name
kadonotakashi 0:8fdf9a60065b 1635 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
kadonotakashi 0:8fdf9a60065b 1636 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
kadonotakashi 0:8fdf9a60065b 1637
kadonotakashi 0:8fdf9a60065b 1638 def log_print(self, log_entry, timestamp=True):
kadonotakashi 0:8fdf9a60065b 1639 """ Prints on screen formatted log entry
kadonotakashi 0:8fdf9a60065b 1640 """
kadonotakashi 0:8fdf9a60065b 1641 ts = log_entry['log_timestamp']
kadonotakashi 0:8fdf9a60065b 1642 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
kadonotakashi 0:8fdf9a60065b 1643 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
kadonotakashi 0:8fdf9a60065b 1644 return timestamp_str + log_line_str
kadonotakashi 0:8fdf9a60065b 1645
kadonotakashi 0:8fdf9a60065b 1646 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
kadonotakashi 0:8fdf9a60065b 1647 """ Logs line, if log file output was specified log line will be appended
kadonotakashi 0:8fdf9a60065b 1648 at the end of log file
kadonotakashi 0:8fdf9a60065b 1649 """
kadonotakashi 0:8fdf9a60065b 1650 log_entry = TestLogger.log_line(self, LogType, log_line)
kadonotakashi 0:8fdf9a60065b 1651 log_line_str = self.log_print(log_entry, timestamp)
kadonotakashi 0:8fdf9a60065b 1652 if self.log_file_name is not None:
kadonotakashi 0:8fdf9a60065b 1653 try:
kadonotakashi 0:8fdf9a60065b 1654 with open(self.log_file_name, 'a') as f:
kadonotakashi 0:8fdf9a60065b 1655 f.write(log_line_str + line_delim)
kadonotakashi 0:8fdf9a60065b 1656 except IOError:
kadonotakashi 0:8fdf9a60065b 1657 pass
kadonotakashi 0:8fdf9a60065b 1658 return log_line_str
kadonotakashi 0:8fdf9a60065b 1659
kadonotakashi 0:8fdf9a60065b 1660
kadonotakashi 0:8fdf9a60065b 1661 def factory_db_logger(db_url):
kadonotakashi 0:8fdf9a60065b 1662 """ Factory database driver depending on database type supplied in database connection string db_url
kadonotakashi 0:8fdf9a60065b 1663 """
kadonotakashi 0:8fdf9a60065b 1664 if db_url is not None:
kadonotakashi 0:8fdf9a60065b 1665 from tools.test_mysql import MySQLDBAccess
kadonotakashi 0:8fdf9a60065b 1666 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
kadonotakashi 0:8fdf9a60065b 1667 if connection_info is not None:
kadonotakashi 0:8fdf9a60065b 1668 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
kadonotakashi 0:8fdf9a60065b 1669 if db_type == 'mysql':
kadonotakashi 0:8fdf9a60065b 1670 return MySQLDBAccess()
kadonotakashi 0:8fdf9a60065b 1671 return None
kadonotakashi 0:8fdf9a60065b 1672
kadonotakashi 0:8fdf9a60065b 1673
kadonotakashi 0:8fdf9a60065b 1674 def detect_database_verbose(db_url):
kadonotakashi 0:8fdf9a60065b 1675 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
kadonotakashi 0:8fdf9a60065b 1676 """
kadonotakashi 0:8fdf9a60065b 1677 result = BaseDBAccess().parse_db_connection_string(db_url)
kadonotakashi 0:8fdf9a60065b 1678 if result is not None:
kadonotakashi 0:8fdf9a60065b 1679 # Parsing passed
kadonotakashi 0:8fdf9a60065b 1680 (db_type, username, password, host, db_name) = result
kadonotakashi 0:8fdf9a60065b 1681 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
kadonotakashi 0:8fdf9a60065b 1682 # Let's try to connect
kadonotakashi 0:8fdf9a60065b 1683 db_ = factory_db_logger(db_url)
kadonotakashi 0:8fdf9a60065b 1684 if db_ is not None:
kadonotakashi 0:8fdf9a60065b 1685 print("Connecting to database '%s'..." % db_url)
kadonotakashi 0:8fdf9a60065b 1686 db_.connect(host, username, password, db_name)
kadonotakashi 0:8fdf9a60065b 1687 if db_.is_connected():
kadonotakashi 0:8fdf9a60065b 1688 print("ok")
kadonotakashi 0:8fdf9a60065b 1689 print("Detecting database...")
kadonotakashi 0:8fdf9a60065b 1690 print(db_.detect_database(verbose=True))
kadonotakashi 0:8fdf9a60065b 1691 print("Disconnecting...")
kadonotakashi 0:8fdf9a60065b 1692 db_.disconnect()
kadonotakashi 0:8fdf9a60065b 1693 print("done")
kadonotakashi 0:8fdf9a60065b 1694 else:
kadonotakashi 0:8fdf9a60065b 1695 print("Database type '%s' unknown" % db_type)
kadonotakashi 0:8fdf9a60065b 1696 else:
kadonotakashi 0:8fdf9a60065b 1697 print("Parse error: '%s' - DB Url error" % db_url)
kadonotakashi 0:8fdf9a60065b 1698
kadonotakashi 0:8fdf9a60065b 1699
kadonotakashi 0:8fdf9a60065b 1700 def get_module_avail(module_name):
kadonotakashi 0:8fdf9a60065b 1701 """ This function returns True if module_name is already imported module
kadonotakashi 0:8fdf9a60065b 1702 """
kadonotakashi 0:8fdf9a60065b 1703 return module_name in sys.modules.keys()
kadonotakashi 0:8fdf9a60065b 1704
kadonotakashi 0:8fdf9a60065b 1705 def get_autodetected_MUTS_list(platform_name_filter=None):
kadonotakashi 0:8fdf9a60065b 1706 oldError = None
kadonotakashi 0:8fdf9a60065b 1707 if os.name == 'nt':
kadonotakashi 0:8fdf9a60065b 1708 # Disable Windows error box temporarily
kadonotakashi 0:8fdf9a60065b 1709 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
kadonotakashi 0:8fdf9a60065b 1710
kadonotakashi 0:8fdf9a60065b 1711 mbeds = mbed_lstools.create()
kadonotakashi 0:8fdf9a60065b 1712 detect_muts_list = mbeds.list_mbeds()
kadonotakashi 0:8fdf9a60065b 1713
kadonotakashi 0:8fdf9a60065b 1714 if os.name == 'nt':
kadonotakashi 0:8fdf9a60065b 1715 ctypes.windll.kernel32.SetErrorMode(oldError)
kadonotakashi 0:8fdf9a60065b 1716
kadonotakashi 0:8fdf9a60065b 1717 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
kadonotakashi 0:8fdf9a60065b 1718
kadonotakashi 0:8fdf9a60065b 1719 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
kadonotakashi 0:8fdf9a60065b 1720 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
kadonotakashi 0:8fdf9a60065b 1721 If function fails to auto-detect devices it will return empty dictionary.
kadonotakashi 0:8fdf9a60065b 1722
kadonotakashi 0:8fdf9a60065b 1723 if get_module_avail('mbed_lstools'):
kadonotakashi 0:8fdf9a60065b 1724 mbeds = mbed_lstools.create()
kadonotakashi 0:8fdf9a60065b 1725 mbeds_list = mbeds.list_mbeds()
kadonotakashi 0:8fdf9a60065b 1726
kadonotakashi 0:8fdf9a60065b 1727 @param mbeds_list list of mbeds captured from mbed_lstools
kadonotakashi 0:8fdf9a60065b 1728 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
kadonotakashi 0:8fdf9a60065b 1729 """
kadonotakashi 0:8fdf9a60065b 1730 result = {} # Should be in muts_all.json format
kadonotakashi 0:8fdf9a60065b 1731 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
kadonotakashi 0:8fdf9a60065b 1732 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
kadonotakashi 0:8fdf9a60065b 1733 index = 1
kadonotakashi 0:8fdf9a60065b 1734 for mut in mbeds_list:
kadonotakashi 0:8fdf9a60065b 1735 # Filter the MUTS if a filter is specified
kadonotakashi 0:8fdf9a60065b 1736
kadonotakashi 0:8fdf9a60065b 1737 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
kadonotakashi 0:8fdf9a60065b 1738 continue
kadonotakashi 0:8fdf9a60065b 1739
kadonotakashi 0:8fdf9a60065b 1740 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
kadonotakashi 0:8fdf9a60065b 1741 # if not we are creating our own unique value (last few chars from platform's target_id).
kadonotakashi 0:8fdf9a60065b 1742 m = {'mcu': mut['platform_name'],
kadonotakashi 0:8fdf9a60065b 1743 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
kadonotakashi 0:8fdf9a60065b 1744 'port': mut['serial_port'],
kadonotakashi 0:8fdf9a60065b 1745 'disk': mut['mount_point'],
kadonotakashi 0:8fdf9a60065b 1746 'peripherals': [] # No peripheral detection
kadonotakashi 0:8fdf9a60065b 1747 }
kadonotakashi 0:8fdf9a60065b 1748 if index not in result:
kadonotakashi 0:8fdf9a60065b 1749 result[index] = {}
kadonotakashi 0:8fdf9a60065b 1750 result[index] = m
kadonotakashi 0:8fdf9a60065b 1751 index += 1
kadonotakashi 0:8fdf9a60065b 1752 return result
kadonotakashi 0:8fdf9a60065b 1753
kadonotakashi 0:8fdf9a60065b 1754
kadonotakashi 0:8fdf9a60065b 1755 def get_autodetected_TEST_SPEC(mbeds_list,
kadonotakashi 0:8fdf9a60065b 1756 use_default_toolchain=True,
kadonotakashi 0:8fdf9a60065b 1757 use_supported_toolchains=False,
kadonotakashi 0:8fdf9a60065b 1758 toolchain_filter=None,
kadonotakashi 0:8fdf9a60065b 1759 platform_name_filter=None):
kadonotakashi 0:8fdf9a60065b 1760 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
kadonotakashi 0:8fdf9a60065b 1761 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
kadonotakashi 0:8fdf9a60065b 1762
kadonotakashi 0:8fdf9a60065b 1763 use_default_toolchain - if True add default toolchain to test_spec
kadonotakashi 0:8fdf9a60065b 1764 use_supported_toolchains - if True add all supported toolchains to test_spec
kadonotakashi 0:8fdf9a60065b 1765 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
kadonotakashi 0:8fdf9a60065b 1766 """
kadonotakashi 0:8fdf9a60065b 1767 result = {'targets': {} }
kadonotakashi 0:8fdf9a60065b 1768
kadonotakashi 0:8fdf9a60065b 1769 for mut in mbeds_list:
kadonotakashi 0:8fdf9a60065b 1770 mcu = mut['mcu']
kadonotakashi 0:8fdf9a60065b 1771 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
kadonotakashi 0:8fdf9a60065b 1772 if mcu in TARGET_MAP:
kadonotakashi 0:8fdf9a60065b 1773 default_toolchain = TARGET_MAP[mcu].default_toolchain
kadonotakashi 0:8fdf9a60065b 1774 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
kadonotakashi 0:8fdf9a60065b 1775
kadonotakashi 0:8fdf9a60065b 1776 # Decide which toolchains should be added to test specification toolchain pool for each target
kadonotakashi 0:8fdf9a60065b 1777 toolchains = []
kadonotakashi 0:8fdf9a60065b 1778 if use_default_toolchain:
kadonotakashi 0:8fdf9a60065b 1779 toolchains.append(default_toolchain)
kadonotakashi 0:8fdf9a60065b 1780 if use_supported_toolchains:
kadonotakashi 0:8fdf9a60065b 1781 toolchains += supported_toolchains
kadonotakashi 0:8fdf9a60065b 1782 if toolchain_filter is not None:
kadonotakashi 0:8fdf9a60065b 1783 all_toolchains = supported_toolchains + [default_toolchain]
kadonotakashi 0:8fdf9a60065b 1784 for toolchain in toolchain_filter:
kadonotakashi 0:8fdf9a60065b 1785 if toolchain in all_toolchains:
kadonotakashi 0:8fdf9a60065b 1786 toolchains.append(toolchain)
kadonotakashi 0:8fdf9a60065b 1787
kadonotakashi 0:8fdf9a60065b 1788 result['targets'][mcu] = list(set(toolchains))
kadonotakashi 0:8fdf9a60065b 1789 return result
kadonotakashi 0:8fdf9a60065b 1790
kadonotakashi 0:8fdf9a60065b 1791
kadonotakashi 0:8fdf9a60065b 1792 def get_default_test_options_parser():
kadonotakashi 0:8fdf9a60065b 1793 """ Get common test script options used by CLI, web services etc.
kadonotakashi 0:8fdf9a60065b 1794 """
kadonotakashi 0:8fdf9a60065b 1795 parser = argparse.ArgumentParser()
kadonotakashi 0:8fdf9a60065b 1796 parser.add_argument('-i', '--tests',
kadonotakashi 0:8fdf9a60065b 1797 dest='test_spec_filename',
kadonotakashi 0:8fdf9a60065b 1798 metavar="FILE",
kadonotakashi 0:8fdf9a60065b 1799 type=argparse_filestring_type,
kadonotakashi 0:8fdf9a60065b 1800 help='Points to file with test specification')
kadonotakashi 0:8fdf9a60065b 1801
kadonotakashi 0:8fdf9a60065b 1802 parser.add_argument('-M', '--MUTS',
kadonotakashi 0:8fdf9a60065b 1803 dest='muts_spec_filename',
kadonotakashi 0:8fdf9a60065b 1804 metavar="FILE",
kadonotakashi 0:8fdf9a60065b 1805 type=argparse_filestring_type,
kadonotakashi 0:8fdf9a60065b 1806 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
kadonotakashi 0:8fdf9a60065b 1807
kadonotakashi 0:8fdf9a60065b 1808 parser.add_argument("-j", "--jobs",
kadonotakashi 0:8fdf9a60065b 1809 dest='jobs',
kadonotakashi 0:8fdf9a60065b 1810 metavar="NUMBER",
kadonotakashi 0:8fdf9a60065b 1811 type=int,
kadonotakashi 0:8fdf9a60065b 1812 help="Define number of compilation jobs. Default value is 1")
kadonotakashi 0:8fdf9a60065b 1813
kadonotakashi 0:8fdf9a60065b 1814 if get_module_avail('mbed_lstools'):
kadonotakashi 0:8fdf9a60065b 1815 # Additional features available when mbed_lstools is installed on host and imported
kadonotakashi 0:8fdf9a60065b 1816 # mbed_lstools allow users to detect connected to host mbed-enabled devices
kadonotakashi 0:8fdf9a60065b 1817 parser.add_argument('--auto',
kadonotakashi 0:8fdf9a60065b 1818 dest='auto_detect',
kadonotakashi 0:8fdf9a60065b 1819 action="store_true",
kadonotakashi 0:8fdf9a60065b 1820 help='Use mbed-ls module to detect all connected mbed devices')
kadonotakashi 0:8fdf9a60065b 1821
kadonotakashi 0:8fdf9a60065b 1822 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
kadonotakashi 0:8fdf9a60065b 1823 parser.add_argument('--tc',
kadonotakashi 0:8fdf9a60065b 1824 dest='toolchains_filter',
kadonotakashi 0:8fdf9a60065b 1825 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
kadonotakashi 0:8fdf9a60065b 1826 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
kadonotakashi 0:8fdf9a60065b 1827
kadonotakashi 0:8fdf9a60065b 1828 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
kadonotakashi 0:8fdf9a60065b 1829 parser.add_argument('--oper',
kadonotakashi 0:8fdf9a60065b 1830 dest='operability_checks',
kadonotakashi 0:8fdf9a60065b 1831 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
kadonotakashi 0:8fdf9a60065b 1832 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
kadonotakashi 0:8fdf9a60065b 1833
kadonotakashi 0:8fdf9a60065b 1834 parser.add_argument('--clean',
kadonotakashi 0:8fdf9a60065b 1835 dest='clean',
kadonotakashi 0:8fdf9a60065b 1836 action="store_true",
kadonotakashi 0:8fdf9a60065b 1837 help='Clean the build directory')
kadonotakashi 0:8fdf9a60065b 1838
kadonotakashi 0:8fdf9a60065b 1839 parser.add_argument('-P', '--only-peripherals',
kadonotakashi 0:8fdf9a60065b 1840 dest='test_only_peripheral',
kadonotakashi 0:8fdf9a60065b 1841 default=False,
kadonotakashi 0:8fdf9a60065b 1842 action="store_true",
kadonotakashi 0:8fdf9a60065b 1843 help='Test only peripheral declared for MUT and skip common tests')
kadonotakashi 0:8fdf9a60065b 1844
kadonotakashi 0:8fdf9a60065b 1845 parser.add_argument("--profile", dest="profile", action="append",
kadonotakashi 0:8fdf9a60065b 1846 type=argparse_filestring_type,
kadonotakashi 0:8fdf9a60065b 1847 default=[])
kadonotakashi 0:8fdf9a60065b 1848
kadonotakashi 0:8fdf9a60065b 1849 parser.add_argument('-C', '--only-commons',
kadonotakashi 0:8fdf9a60065b 1850 dest='test_only_common',
kadonotakashi 0:8fdf9a60065b 1851 default=False,
kadonotakashi 0:8fdf9a60065b 1852 action="store_true",
kadonotakashi 0:8fdf9a60065b 1853 help='Test only board internals. Skip perpherials tests and perform common tests')
kadonotakashi 0:8fdf9a60065b 1854
kadonotakashi 0:8fdf9a60065b 1855 parser.add_argument('-n', '--test-by-names',
kadonotakashi 0:8fdf9a60065b 1856 dest='test_by_names',
kadonotakashi 0:8fdf9a60065b 1857 type=argparse_many(str),
kadonotakashi 0:8fdf9a60065b 1858 help='Runs only test enumerated it this switch. Use comma to separate test case names')
kadonotakashi 0:8fdf9a60065b 1859
kadonotakashi 0:8fdf9a60065b 1860 parser.add_argument('-p', '--peripheral-by-names',
kadonotakashi 0:8fdf9a60065b 1861 dest='peripheral_by_names',
kadonotakashi 0:8fdf9a60065b 1862 type=argparse_many(str),
kadonotakashi 0:8fdf9a60065b 1863 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
kadonotakashi 0:8fdf9a60065b 1864
kadonotakashi 0:8fdf9a60065b 1865 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
kadonotakashi 0:8fdf9a60065b 1866 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
kadonotakashi 0:8fdf9a60065b 1867
kadonotakashi 0:8fdf9a60065b 1868 parser.add_argument('-c', '--copy-method',
kadonotakashi 0:8fdf9a60065b 1869 dest='copy_method',
kadonotakashi 0:8fdf9a60065b 1870 type=argparse_uppercase_type(copy_methods, "flash method"),
kadonotakashi 0:8fdf9a60065b 1871 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
kadonotakashi 0:8fdf9a60065b 1872
kadonotakashi 0:8fdf9a60065b 1873 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
kadonotakashi 0:8fdf9a60065b 1874 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
kadonotakashi 0:8fdf9a60065b 1875
kadonotakashi 0:8fdf9a60065b 1876 parser.add_argument('-r', '--reset-type',
kadonotakashi 0:8fdf9a60065b 1877 dest='mut_reset_type',
kadonotakashi 0:8fdf9a60065b 1878 default=None,
kadonotakashi 0:8fdf9a60065b 1879 type=argparse_uppercase_type(reset_methods, "reset method"),
kadonotakashi 0:8fdf9a60065b 1880 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
kadonotakashi 0:8fdf9a60065b 1881
kadonotakashi 0:8fdf9a60065b 1882 parser.add_argument('-g', '--goanna-for-tests',
kadonotakashi 0:8fdf9a60065b 1883 dest='goanna_for_tests',
kadonotakashi 0:8fdf9a60065b 1884 action="store_true",
kadonotakashi 0:8fdf9a60065b 1885 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
kadonotakashi 0:8fdf9a60065b 1886
kadonotakashi 0:8fdf9a60065b 1887 parser.add_argument('-G', '--goanna-for-sdk',
kadonotakashi 0:8fdf9a60065b 1888 dest='goanna_for_mbed_sdk',
kadonotakashi 0:8fdf9a60065b 1889 action="store_true",
kadonotakashi 0:8fdf9a60065b 1890 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
kadonotakashi 0:8fdf9a60065b 1891
kadonotakashi 0:8fdf9a60065b 1892 parser.add_argument('-s', '--suppress-summary',
kadonotakashi 0:8fdf9a60065b 1893 dest='suppress_summary',
kadonotakashi 0:8fdf9a60065b 1894 default=False,
kadonotakashi 0:8fdf9a60065b 1895 action="store_true",
kadonotakashi 0:8fdf9a60065b 1896 help='Suppresses display of wellformatted table with test results')
kadonotakashi 0:8fdf9a60065b 1897
kadonotakashi 0:8fdf9a60065b 1898 parser.add_argument('-t', '--test-summary',
kadonotakashi 0:8fdf9a60065b 1899 dest='test_x_toolchain_summary',
kadonotakashi 0:8fdf9a60065b 1900 default=False,
kadonotakashi 0:8fdf9a60065b 1901 action="store_true",
kadonotakashi 0:8fdf9a60065b 1902 help='Displays wellformatted table with test x toolchain test result per target')
kadonotakashi 0:8fdf9a60065b 1903
kadonotakashi 0:8fdf9a60065b 1904 parser.add_argument('-A', '--test-automation-report',
kadonotakashi 0:8fdf9a60065b 1905 dest='test_automation_report',
kadonotakashi 0:8fdf9a60065b 1906 default=False,
kadonotakashi 0:8fdf9a60065b 1907 action="store_true",
kadonotakashi 0:8fdf9a60065b 1908 help='Prints information about all tests and exits')
kadonotakashi 0:8fdf9a60065b 1909
kadonotakashi 0:8fdf9a60065b 1910 parser.add_argument('-R', '--test-case-report',
kadonotakashi 0:8fdf9a60065b 1911 dest='test_case_report',
kadonotakashi 0:8fdf9a60065b 1912 default=False,
kadonotakashi 0:8fdf9a60065b 1913 action="store_true",
kadonotakashi 0:8fdf9a60065b 1914 help='Prints information about all test cases and exits')
kadonotakashi 0:8fdf9a60065b 1915
kadonotakashi 0:8fdf9a60065b 1916 parser.add_argument("-S", "--supported-toolchains",
kadonotakashi 0:8fdf9a60065b 1917 action="store_true",
kadonotakashi 0:8fdf9a60065b 1918 dest="supported_toolchains",
kadonotakashi 0:8fdf9a60065b 1919 default=False,
kadonotakashi 0:8fdf9a60065b 1920 help="Displays supported matrix of MCUs and toolchains")
kadonotakashi 0:8fdf9a60065b 1921
kadonotakashi 0:8fdf9a60065b 1922 parser.add_argument("-O", "--only-build",
kadonotakashi 0:8fdf9a60065b 1923 action="store_true",
kadonotakashi 0:8fdf9a60065b 1924 dest="only_build_tests",
kadonotakashi 0:8fdf9a60065b 1925 default=False,
kadonotakashi 0:8fdf9a60065b 1926 help="Only build tests, skips actual test procedures (flashing etc.)")
kadonotakashi 0:8fdf9a60065b 1927
kadonotakashi 0:8fdf9a60065b 1928 parser.add_argument('--parallel',
kadonotakashi 0:8fdf9a60065b 1929 dest='parallel_test_exec',
kadonotakashi 0:8fdf9a60065b 1930 default=False,
kadonotakashi 0:8fdf9a60065b 1931 action="store_true",
kadonotakashi 0:8fdf9a60065b 1932 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
kadonotakashi 0:8fdf9a60065b 1933
kadonotakashi 0:8fdf9a60065b 1934 parser.add_argument('--config',
kadonotakashi 0:8fdf9a60065b 1935 dest='verbose_test_configuration_only',
kadonotakashi 0:8fdf9a60065b 1936 default=False,
kadonotakashi 0:8fdf9a60065b 1937 action="store_true",
kadonotakashi 0:8fdf9a60065b 1938 help='Displays full test specification and MUTs configration and exits')
kadonotakashi 0:8fdf9a60065b 1939
kadonotakashi 0:8fdf9a60065b 1940 parser.add_argument('--loops',
kadonotakashi 0:8fdf9a60065b 1941 dest='test_loops_list',
kadonotakashi 0:8fdf9a60065b 1942 type=argparse_many(str),
kadonotakashi 0:8fdf9a60065b 1943 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
kadonotakashi 0:8fdf9a60065b 1944
kadonotakashi 0:8fdf9a60065b 1945 parser.add_argument('--global-loops',
kadonotakashi 0:8fdf9a60065b 1946 dest='test_global_loops_value',
kadonotakashi 0:8fdf9a60065b 1947 type=int,
kadonotakashi 0:8fdf9a60065b 1948 help='Set global number of test loops per test. Default value is set 1')
kadonotakashi 0:8fdf9a60065b 1949
kadonotakashi 0:8fdf9a60065b 1950 parser.add_argument('--consolidate-waterfall',
kadonotakashi 0:8fdf9a60065b 1951 dest='consolidate_waterfall_test',
kadonotakashi 0:8fdf9a60065b 1952 default=False,
kadonotakashi 0:8fdf9a60065b 1953 action="store_true",
kadonotakashi 0:8fdf9a60065b 1954 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
kadonotakashi 0:8fdf9a60065b 1955
kadonotakashi 0:8fdf9a60065b 1956 parser.add_argument('-W', '--waterfall',
kadonotakashi 0:8fdf9a60065b 1957 dest='waterfall_test',
kadonotakashi 0:8fdf9a60065b 1958 default=False,
kadonotakashi 0:8fdf9a60065b 1959 action="store_true",
kadonotakashi 0:8fdf9a60065b 1960 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
kadonotakashi 0:8fdf9a60065b 1961
kadonotakashi 0:8fdf9a60065b 1962 parser.add_argument('-N', '--firmware-name',
kadonotakashi 0:8fdf9a60065b 1963 dest='firmware_global_name',
kadonotakashi 0:8fdf9a60065b 1964 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
kadonotakashi 0:8fdf9a60065b 1965
kadonotakashi 0:8fdf9a60065b 1966 parser.add_argument('-u', '--shuffle',
kadonotakashi 0:8fdf9a60065b 1967 dest='shuffle_test_order',
kadonotakashi 0:8fdf9a60065b 1968 default=False,
kadonotakashi 0:8fdf9a60065b 1969 action="store_true",
kadonotakashi 0:8fdf9a60065b 1970 help='Shuffles test execution order')
kadonotakashi 0:8fdf9a60065b 1971
kadonotakashi 0:8fdf9a60065b 1972 parser.add_argument('--shuffle-seed',
kadonotakashi 0:8fdf9a60065b 1973 dest='shuffle_test_seed',
kadonotakashi 0:8fdf9a60065b 1974 default=None,
kadonotakashi 0:8fdf9a60065b 1975 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
kadonotakashi 0:8fdf9a60065b 1976
kadonotakashi 0:8fdf9a60065b 1977 parser.add_argument('-f', '--filter',
kadonotakashi 0:8fdf9a60065b 1978 dest='general_filter_regex',
kadonotakashi 0:8fdf9a60065b 1979 type=argparse_many(str),
kadonotakashi 0:8fdf9a60065b 1980 default=None,
kadonotakashi 0:8fdf9a60065b 1981 help='For some commands you can use filter to filter out results')
kadonotakashi 0:8fdf9a60065b 1982
kadonotakashi 0:8fdf9a60065b 1983 parser.add_argument('--inc-timeout',
kadonotakashi 0:8fdf9a60065b 1984 dest='extend_test_timeout',
kadonotakashi 0:8fdf9a60065b 1985 metavar="NUMBER",
kadonotakashi 0:8fdf9a60065b 1986 type=int,
kadonotakashi 0:8fdf9a60065b 1987 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
kadonotakashi 0:8fdf9a60065b 1988
kadonotakashi 0:8fdf9a60065b 1989 parser.add_argument('--db',
kadonotakashi 0:8fdf9a60065b 1990 dest='db_url',
kadonotakashi 0:8fdf9a60065b 1991 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
kadonotakashi 0:8fdf9a60065b 1992
kadonotakashi 0:8fdf9a60065b 1993 parser.add_argument('-l', '--log',
kadonotakashi 0:8fdf9a60065b 1994 dest='log_file_name',
kadonotakashi 0:8fdf9a60065b 1995 help='Log events to external file (note not all console entries may be visible in log file)')
kadonotakashi 0:8fdf9a60065b 1996
kadonotakashi 0:8fdf9a60065b 1997 parser.add_argument('--report-html',
kadonotakashi 0:8fdf9a60065b 1998 dest='report_html_file_name',
kadonotakashi 0:8fdf9a60065b 1999 help='You can log test suite results in form of HTML report')
kadonotakashi 0:8fdf9a60065b 2000
kadonotakashi 0:8fdf9a60065b 2001 parser.add_argument('--report-junit',
kadonotakashi 0:8fdf9a60065b 2002 dest='report_junit_file_name',
kadonotakashi 0:8fdf9a60065b 2003 help='You can log test suite results in form of JUnit compliant XML report')
kadonotakashi 0:8fdf9a60065b 2004
kadonotakashi 0:8fdf9a60065b 2005 parser.add_argument("--report-build",
kadonotakashi 0:8fdf9a60065b 2006 dest="report_build_file_name",
kadonotakashi 0:8fdf9a60065b 2007 help="Output the build results to a junit xml file")
kadonotakashi 0:8fdf9a60065b 2008
kadonotakashi 0:8fdf9a60065b 2009 parser.add_argument("--report-text",
kadonotakashi 0:8fdf9a60065b 2010 dest="report_text_file_name",
kadonotakashi 0:8fdf9a60065b 2011 help="Output the build results to a text file")
kadonotakashi 0:8fdf9a60065b 2012
kadonotakashi 0:8fdf9a60065b 2013 parser.add_argument('--verbose-skipped',
kadonotakashi 0:8fdf9a60065b 2014 dest='verbose_skipped_tests',
kadonotakashi 0:8fdf9a60065b 2015 default=False,
kadonotakashi 0:8fdf9a60065b 2016 action="store_true",
kadonotakashi 0:8fdf9a60065b 2017 help='Prints some extra information about skipped tests')
kadonotakashi 0:8fdf9a60065b 2018
kadonotakashi 0:8fdf9a60065b 2019 parser.add_argument('-V', '--verbose-test-result',
kadonotakashi 0:8fdf9a60065b 2020 dest='verbose_test_result_only',
kadonotakashi 0:8fdf9a60065b 2021 default=False,
kadonotakashi 0:8fdf9a60065b 2022 action="store_true",
kadonotakashi 0:8fdf9a60065b 2023 help='Prints test serial output')
kadonotakashi 0:8fdf9a60065b 2024
kadonotakashi 0:8fdf9a60065b 2025 parser.add_argument('-v', '--verbose',
kadonotakashi 0:8fdf9a60065b 2026 dest='verbose',
kadonotakashi 0:8fdf9a60065b 2027 default=False,
kadonotakashi 0:8fdf9a60065b 2028 action="store_true",
kadonotakashi 0:8fdf9a60065b 2029 help='Verbose mode (prints some extra information)')
kadonotakashi 0:8fdf9a60065b 2030
kadonotakashi 0:8fdf9a60065b 2031 parser.add_argument('--version',
kadonotakashi 0:8fdf9a60065b 2032 dest='version',
kadonotakashi 0:8fdf9a60065b 2033 default=False,
kadonotakashi 0:8fdf9a60065b 2034 action="store_true",
kadonotakashi 0:8fdf9a60065b 2035 help='Prints script version and exits')
kadonotakashi 0:8fdf9a60065b 2036
kadonotakashi 0:8fdf9a60065b 2037 parser.add_argument('--stats-depth',
kadonotakashi 0:8fdf9a60065b 2038 dest='stats_depth',
kadonotakashi 0:8fdf9a60065b 2039 default=2,
kadonotakashi 0:8fdf9a60065b 2040 type=int,
kadonotakashi 0:8fdf9a60065b 2041 help="Depth level for static memory report")
kadonotakashi 0:8fdf9a60065b 2042 return parser
kadonotakashi 0:8fdf9a60065b 2043
kadonotakashi 0:8fdf9a60065b 2044 def test_path_to_name(path, base):
kadonotakashi 0:8fdf9a60065b 2045 """Change all slashes in a path into hyphens
kadonotakashi 0:8fdf9a60065b 2046 This creates a unique cross-platform test name based on the path
kadonotakashi 0:8fdf9a60065b 2047 This can eventually be overriden by a to-be-determined meta-data mechanism"""
kadonotakashi 0:8fdf9a60065b 2048 name_parts = []
kadonotakashi 0:8fdf9a60065b 2049 head, tail = os.path.split(relpath(path,base))
kadonotakashi 0:8fdf9a60065b 2050 while (tail and tail != "."):
kadonotakashi 0:8fdf9a60065b 2051 name_parts.insert(0, tail)
kadonotakashi 0:8fdf9a60065b 2052 head, tail = os.path.split(head)
kadonotakashi 0:8fdf9a60065b 2053
kadonotakashi 0:8fdf9a60065b 2054 return "-".join(name_parts).lower()
kadonotakashi 0:8fdf9a60065b 2055
kadonotakashi 0:8fdf9a60065b 2056 def get_test_config(config_name, target_name):
kadonotakashi 0:8fdf9a60065b 2057 """Finds the path to a test configuration file
kadonotakashi 0:8fdf9a60065b 2058 config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
kadonotakashi 0:8fdf9a60065b 2059 target_name: name of target to determing if mbed OS interface given is valid
kadonotakashi 0:8fdf9a60065b 2060 returns path to config, will return None if no valid config is found
kadonotakashi 0:8fdf9a60065b 2061 """
kadonotakashi 0:8fdf9a60065b 2062 # If they passed in a full path
kadonotakashi 0:8fdf9a60065b 2063 if exists(config_name):
kadonotakashi 0:8fdf9a60065b 2064 # This is a module config
kadonotakashi 0:8fdf9a60065b 2065 return config_name
kadonotakashi 0:8fdf9a60065b 2066 # Otherwise find the path to configuration file based on mbed OS interface
kadonotakashi 0:8fdf9a60065b 2067 return TestConfig.get_config_path(config_name, target_name)
kadonotakashi 0:8fdf9a60065b 2068
kadonotakashi 0:8fdf9a60065b 2069
kadonotakashi 0:8fdf9a60065b 2070 def find_tests(base_dir, target_name, toolchain_name, icetea, greentea, app_config=None):
kadonotakashi 0:8fdf9a60065b 2071 """ Finds all tests in a directory recursively
kadonotakashi 0:8fdf9a60065b 2072 :param base_dir: path to the directory to scan for tests (ex. 'path/to/project')
kadonotakashi 0:8fdf9a60065b 2073 :param target_name: name of the target to use for scanning (ex. 'K64F')
kadonotakashi 0:8fdf9a60065b 2074 :param toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
kadonotakashi 0:8fdf9a60065b 2075 :param icetea: icetea enabled
kadonotakashi 0:8fdf9a60065b 2076 :param greentea: greentea enabled
kadonotakashi 0:8fdf9a60065b 2077 :param app_config - location of a chosen mbed_app.json file
kadonotakashi 0:8fdf9a60065b 2078
kadonotakashi 0:8fdf9a60065b 2079 returns a dictionary where keys are the test name, and the values are
kadonotakashi 0:8fdf9a60065b 2080 lists of paths needed to biuld the test.
kadonotakashi 0:8fdf9a60065b 2081 """
kadonotakashi 0:8fdf9a60065b 2082
kadonotakashi 0:8fdf9a60065b 2083 # Temporary structure: tests referenced by (name, base, group, case) tuple
kadonotakashi 0:8fdf9a60065b 2084 tests = {}
kadonotakashi 0:8fdf9a60065b 2085 # List of common folders: (predicate function, path) tuple
kadonotakashi 0:8fdf9a60065b 2086 commons = []
kadonotakashi 0:8fdf9a60065b 2087
kadonotakashi 0:8fdf9a60065b 2088 config = Config(target_name, base_dir, app_config)
kadonotakashi 0:8fdf9a60065b 2089
kadonotakashi 0:8fdf9a60065b 2090 # Scan the directory for paths to probe for 'TESTS' folders
kadonotakashi 0:8fdf9a60065b 2091 base_resources = Resources(MockNotifier(), collect_ignores=True)
kadonotakashi 0:8fdf9a60065b 2092 base_resources.scan_with_config(base_dir, config)
kadonotakashi 0:8fdf9a60065b 2093
kadonotakashi 0:8fdf9a60065b 2094 if greentea:
kadonotakashi 0:8fdf9a60065b 2095 dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TESTS']
kadonotakashi 0:8fdf9a60065b 2096 ignoreset = MbedIgnoreSet()
kadonotakashi 0:8fdf9a60065b 2097
kadonotakashi 0:8fdf9a60065b 2098 for directory in dirs:
kadonotakashi 0:8fdf9a60065b 2099 ignorefile = join(directory, IGNORE_FILENAME)
kadonotakashi 0:8fdf9a60065b 2100 if isfile(ignorefile):
kadonotakashi 0:8fdf9a60065b 2101 ignoreset.add_mbedignore(directory, ignorefile)
kadonotakashi 0:8fdf9a60065b 2102 for test_group_directory in os.listdir(directory):
kadonotakashi 0:8fdf9a60065b 2103 grp_dir = join(directory, test_group_directory)
kadonotakashi 0:8fdf9a60065b 2104 if not isdir(grp_dir) or ignoreset.is_ignored(grp_dir):
kadonotakashi 0:8fdf9a60065b 2105 continue
kadonotakashi 0:8fdf9a60065b 2106 grpignorefile = join(grp_dir, IGNORE_FILENAME)
kadonotakashi 0:8fdf9a60065b 2107 if isfile(grpignorefile):
kadonotakashi 0:8fdf9a60065b 2108 ignoreset.add_mbedignore(grp_dir, grpignorefile)
kadonotakashi 0:8fdf9a60065b 2109 for test_case_directory in os.listdir(grp_dir):
kadonotakashi 0:8fdf9a60065b 2110 d = join(directory, test_group_directory, test_case_directory)
kadonotakashi 0:8fdf9a60065b 2111 if not isdir(d) or ignoreset.is_ignored(d):
kadonotakashi 0:8fdf9a60065b 2112 continue
kadonotakashi 0:8fdf9a60065b 2113 special_dirs = ['host_tests', 'COMMON']
kadonotakashi 0:8fdf9a60065b 2114 if test_group_directory not in special_dirs and test_case_directory not in special_dirs:
kadonotakashi 0:8fdf9a60065b 2115 test_name = test_path_to_name(d, base_dir)
kadonotakashi 0:8fdf9a60065b 2116 tests[(test_name, directory, test_group_directory, test_case_directory)] = [d]
kadonotakashi 0:8fdf9a60065b 2117 if test_case_directory == 'COMMON':
kadonotakashi 0:8fdf9a60065b 2118 def predicate(base_pred, group_pred, name_base_group_case):
kadonotakashi 0:8fdf9a60065b 2119 (name, base, group, case) = name_base_group_case
kadonotakashi 0:8fdf9a60065b 2120 return base == base_pred and group == group_pred
kadonotakashi 0:8fdf9a60065b 2121
kadonotakashi 0:8fdf9a60065b 2122 commons.append((functools.partial(predicate, directory, test_group_directory), d))
kadonotakashi 0:8fdf9a60065b 2123 if test_group_directory == 'COMMON':
kadonotakashi 0:8fdf9a60065b 2124 def predicate(base_pred, name_base_group_case):
kadonotakashi 0:8fdf9a60065b 2125 (name, base, group, case) = name_base_group_case
kadonotakashi 0:8fdf9a60065b 2126 return base == base_pred
kadonotakashi 0:8fdf9a60065b 2127
kadonotakashi 0:8fdf9a60065b 2128 commons.append((functools.partial(predicate, directory), grp_dir))
kadonotakashi 0:8fdf9a60065b 2129
kadonotakashi 0:8fdf9a60065b 2130 if icetea:
kadonotakashi 0:8fdf9a60065b 2131 dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TEST_APPS']
kadonotakashi 0:8fdf9a60065b 2132 for directory in dirs:
kadonotakashi 0:8fdf9a60065b 2133 if not isdir(directory):
kadonotakashi 0:8fdf9a60065b 2134 continue
kadonotakashi 0:8fdf9a60065b 2135 for subdir in os.listdir(directory):
kadonotakashi 0:8fdf9a60065b 2136 d = join(directory, subdir)
kadonotakashi 0:8fdf9a60065b 2137 if not isdir(d):
kadonotakashi 0:8fdf9a60065b 2138 continue
kadonotakashi 0:8fdf9a60065b 2139 if 'device' == subdir:
kadonotakashi 0:8fdf9a60065b 2140 for test_dir in os.listdir(d):
kadonotakashi 0:8fdf9a60065b 2141 test_dir_path = join(d, test_dir)
kadonotakashi 0:8fdf9a60065b 2142 test_name = test_path_to_name(test_dir_path, base_dir)
kadonotakashi 0:8fdf9a60065b 2143 tests[(test_name, directory, subdir, test_dir)] = [test_dir_path]
kadonotakashi 0:8fdf9a60065b 2144
kadonotakashi 0:8fdf9a60065b 2145 # Apply common directories
kadonotakashi 0:8fdf9a60065b 2146 for pred, path in commons:
kadonotakashi 0:8fdf9a60065b 2147 for test_identity, test_paths in six.iteritems(tests):
kadonotakashi 0:8fdf9a60065b 2148 if pred(test_identity):
kadonotakashi 0:8fdf9a60065b 2149 test_paths.append(path)
kadonotakashi 0:8fdf9a60065b 2150
kadonotakashi 0:8fdf9a60065b 2151 # Drop identity besides name
kadonotakashi 0:8fdf9a60065b 2152 return {name: paths for (name, _, _, _), paths in six.iteritems(tests)}
kadonotakashi 0:8fdf9a60065b 2153
kadonotakashi 0:8fdf9a60065b 2154
kadonotakashi 0:8fdf9a60065b 2155 def print_tests(tests, format="list", sort=True):
kadonotakashi 0:8fdf9a60065b 2156 """Given a dictionary of tests (as returned from "find_tests"), print them
kadonotakashi 0:8fdf9a60065b 2157 in the specified format"""
kadonotakashi 0:8fdf9a60065b 2158 if format == "list":
kadonotakashi 0:8fdf9a60065b 2159 for test_name in sorted(tests.keys()):
kadonotakashi 0:8fdf9a60065b 2160 test_path = tests[test_name][0]
kadonotakashi 0:8fdf9a60065b 2161 print("Test Case:")
kadonotakashi 0:8fdf9a60065b 2162 print(" Name: %s" % test_name)
kadonotakashi 0:8fdf9a60065b 2163 print(" Path: %s" % test_path)
kadonotakashi 0:8fdf9a60065b 2164 elif format == "json":
kadonotakashi 0:8fdf9a60065b 2165 print(json.dumps({test_name: test_path[0] for test_name, test_paths
kadonotakashi 0:8fdf9a60065b 2166 in tests}, indent=2))
kadonotakashi 0:8fdf9a60065b 2167 else:
kadonotakashi 0:8fdf9a60065b 2168 print("Unknown format '%s'" % format)
kadonotakashi 0:8fdf9a60065b 2169 sys.exit(1)
kadonotakashi 0:8fdf9a60065b 2170
kadonotakashi 0:8fdf9a60065b 2171 def norm_relative_path(path, start):
kadonotakashi 0:8fdf9a60065b 2172 """This function will create a normalized, relative path. It mimics the
kadonotakashi 0:8fdf9a60065b 2173 python os.path.relpath function, but also normalizes a Windows-syle path
kadonotakashi 0:8fdf9a60065b 2174 that use backslashes to a Unix style path that uses forward slashes."""
kadonotakashi 0:8fdf9a60065b 2175 path = os.path.normpath(path)
kadonotakashi 0:8fdf9a60065b 2176 path = os.path.relpath(path, start)
kadonotakashi 0:8fdf9a60065b 2177 path = path.replace("\\", "/")
kadonotakashi 0:8fdf9a60065b 2178 return path
kadonotakashi 0:8fdf9a60065b 2179
kadonotakashi 0:8fdf9a60065b 2180
kadonotakashi 0:8fdf9a60065b 2181 def build_test_worker(*args, **kwargs):
kadonotakashi 0:8fdf9a60065b 2182 """This is a worker function for the parallel building of tests. The `args`
kadonotakashi 0:8fdf9a60065b 2183 and `kwargs` are passed directly to `build_project`. It returns a dictionary
kadonotakashi 0:8fdf9a60065b 2184 with the following structure:
kadonotakashi 0:8fdf9a60065b 2185
kadonotakashi 0:8fdf9a60065b 2186 {
kadonotakashi 0:8fdf9a60065b 2187 'result': `True` if no exceptions were thrown, `False` otherwise
kadonotakashi 0:8fdf9a60065b 2188 'reason': Instance of exception that was thrown on failure
kadonotakashi 0:8fdf9a60065b 2189 'bin_file': Path to the created binary if `build_project` was
kadonotakashi 0:8fdf9a60065b 2190 successful. Not present otherwise
kadonotakashi 0:8fdf9a60065b 2191 'kwargs': The keyword arguments that were passed to `build_project`.
kadonotakashi 0:8fdf9a60065b 2192 This includes arguments that were modified (ex. report)
kadonotakashi 0:8fdf9a60065b 2193 }
kadonotakashi 0:8fdf9a60065b 2194 """
kadonotakashi 0:8fdf9a60065b 2195 bin_file = None
kadonotakashi 0:8fdf9a60065b 2196 ret = {
kadonotakashi 0:8fdf9a60065b 2197 'result': False,
kadonotakashi 0:8fdf9a60065b 2198 'args': args,
kadonotakashi 0:8fdf9a60065b 2199 'kwargs': kwargs
kadonotakashi 0:8fdf9a60065b 2200 }
kadonotakashi 0:8fdf9a60065b 2201
kadonotakashi 0:8fdf9a60065b 2202 # Use parent TOOLCHAIN_PATHS variable
kadonotakashi 0:8fdf9a60065b 2203 for key, value in kwargs['toolchain_paths'].items():
kadonotakashi 0:8fdf9a60065b 2204 TOOLCHAIN_PATHS[key] = value
kadonotakashi 0:8fdf9a60065b 2205
kadonotakashi 0:8fdf9a60065b 2206 del kwargs['toolchain_paths']
kadonotakashi 0:8fdf9a60065b 2207
kadonotakashi 0:8fdf9a60065b 2208 try:
kadonotakashi 0:8fdf9a60065b 2209 bin_file, _ = build_project(*args, **kwargs)
kadonotakashi 0:8fdf9a60065b 2210 ret['result'] = True
kadonotakashi 0:8fdf9a60065b 2211 ret['bin_file'] = bin_file
kadonotakashi 0:8fdf9a60065b 2212 ret['kwargs'] = kwargs
kadonotakashi 0:8fdf9a60065b 2213
kadonotakashi 0:8fdf9a60065b 2214 except NotSupportedException as e:
kadonotakashi 0:8fdf9a60065b 2215 ret['reason'] = e
kadonotakashi 0:8fdf9a60065b 2216 except ToolException as e:
kadonotakashi 0:8fdf9a60065b 2217 ret['reason'] = e
kadonotakashi 0:8fdf9a60065b 2218 except KeyboardInterrupt as e:
kadonotakashi 0:8fdf9a60065b 2219 ret['reason'] = e
kadonotakashi 0:8fdf9a60065b 2220 except:
kadonotakashi 0:8fdf9a60065b 2221 # Print unhandled exceptions here
kadonotakashi 0:8fdf9a60065b 2222 import traceback
kadonotakashi 0:8fdf9a60065b 2223 traceback.print_exc(file=sys.stdout)
kadonotakashi 0:8fdf9a60065b 2224
kadonotakashi 0:8fdf9a60065b 2225 return ret
kadonotakashi 0:8fdf9a60065b 2226
kadonotakashi 0:8fdf9a60065b 2227
kadonotakashi 0:8fdf9a60065b 2228 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
kadonotakashi 0:8fdf9a60065b 2229 clean=False, notify=None, jobs=1, macros=None,
kadonotakashi 0:8fdf9a60065b 2230 silent=False, report=None, properties=None,
kadonotakashi 0:8fdf9a60065b 2231 continue_on_build_fail=False, app_config=None,
kadonotakashi 0:8fdf9a60065b 2232 build_profile=None, stats_depth=None, ignore=None):
kadonotakashi 0:8fdf9a60065b 2233 """Given the data structure from 'find_tests' and the typical build parameters,
kadonotakashi 0:8fdf9a60065b 2234 build all the tests
kadonotakashi 0:8fdf9a60065b 2235
kadonotakashi 0:8fdf9a60065b 2236 Returns a tuple of the build result (True or False) followed by the test
kadonotakashi 0:8fdf9a60065b 2237 build data structure"""
kadonotakashi 0:8fdf9a60065b 2238
kadonotakashi 0:8fdf9a60065b 2239 execution_directory = "."
kadonotakashi 0:8fdf9a60065b 2240 base_path = norm_relative_path(build_path, execution_directory)
kadonotakashi 0:8fdf9a60065b 2241
kadonotakashi 0:8fdf9a60065b 2242 if isinstance(target, Target):
kadonotakashi 0:8fdf9a60065b 2243 target_name = target.name
kadonotakashi 0:8fdf9a60065b 2244 else:
kadonotakashi 0:8fdf9a60065b 2245 target_name = target
kadonotakashi 0:8fdf9a60065b 2246 target = TARGET_MAP[target_name]
kadonotakashi 0:8fdf9a60065b 2247 cfg, _, _ = get_config(base_source_paths, target, app_config=app_config)
kadonotakashi 0:8fdf9a60065b 2248
kadonotakashi 0:8fdf9a60065b 2249 baud_rate = 9600
kadonotakashi 0:8fdf9a60065b 2250 if 'platform.stdio-baud-rate' in cfg:
kadonotakashi 0:8fdf9a60065b 2251 baud_rate = cfg['platform.stdio-baud-rate'].value
kadonotakashi 0:8fdf9a60065b 2252
kadonotakashi 0:8fdf9a60065b 2253 test_build = {
kadonotakashi 0:8fdf9a60065b 2254 "platform": target_name,
kadonotakashi 0:8fdf9a60065b 2255 "toolchain": toolchain_name,
kadonotakashi 0:8fdf9a60065b 2256 "base_path": base_path,
kadonotakashi 0:8fdf9a60065b 2257 "baud_rate": baud_rate,
kadonotakashi 0:8fdf9a60065b 2258 "binary_type": "bootable",
kadonotakashi 0:8fdf9a60065b 2259 "tests": {},
kadonotakashi 0:8fdf9a60065b 2260 "test_apps": {}
kadonotakashi 0:8fdf9a60065b 2261 }
kadonotakashi 0:8fdf9a60065b 2262
kadonotakashi 0:8fdf9a60065b 2263 result = True
kadonotakashi 0:8fdf9a60065b 2264
kadonotakashi 0:8fdf9a60065b 2265 jobs_count = int(jobs if jobs else cpu_count())
kadonotakashi 0:8fdf9a60065b 2266 p = Pool(processes=jobs_count)
kadonotakashi 0:8fdf9a60065b 2267 results = []
kadonotakashi 0:8fdf9a60065b 2268 for test_name, test_paths in tests.items():
kadonotakashi 0:8fdf9a60065b 2269 if not isinstance(test_paths, list):
kadonotakashi 0:8fdf9a60065b 2270 test_paths = [test_paths]
kadonotakashi 0:8fdf9a60065b 2271
kadonotakashi 0:8fdf9a60065b 2272 test_build_path = os.path.join(build_path, test_paths[0])
kadonotakashi 0:8fdf9a60065b 2273 src_paths = base_source_paths + test_paths
kadonotakashi 0:8fdf9a60065b 2274 bin_file = None
kadonotakashi 0:8fdf9a60065b 2275 test_case_folder_name = os.path.basename(test_paths[0])
kadonotakashi 0:8fdf9a60065b 2276
kadonotakashi 0:8fdf9a60065b 2277 args = (src_paths, test_build_path, deepcopy(target), toolchain_name)
kadonotakashi 0:8fdf9a60065b 2278 kwargs = {
kadonotakashi 0:8fdf9a60065b 2279 'jobs': 1,
kadonotakashi 0:8fdf9a60065b 2280 'clean': clean,
kadonotakashi 0:8fdf9a60065b 2281 'macros': macros,
kadonotakashi 0:8fdf9a60065b 2282 'name': test_case_folder_name,
kadonotakashi 0:8fdf9a60065b 2283 'project_id': test_name,
kadonotakashi 0:8fdf9a60065b 2284 'report': report,
kadonotakashi 0:8fdf9a60065b 2285 'properties': properties,
kadonotakashi 0:8fdf9a60065b 2286 'app_config': app_config,
kadonotakashi 0:8fdf9a60065b 2287 'build_profile': build_profile,
kadonotakashi 0:8fdf9a60065b 2288 'toolchain_paths': TOOLCHAIN_PATHS,
kadonotakashi 0:8fdf9a60065b 2289 'stats_depth': stats_depth,
kadonotakashi 0:8fdf9a60065b 2290 'notify': MockNotifier()
kadonotakashi 0:8fdf9a60065b 2291 }
kadonotakashi 0:8fdf9a60065b 2292
kadonotakashi 0:8fdf9a60065b 2293 results.append(p.apply_async(build_test_worker, args, kwargs))
kadonotakashi 0:8fdf9a60065b 2294
kadonotakashi 0:8fdf9a60065b 2295 p.close()
kadonotakashi 0:8fdf9a60065b 2296 result = True
kadonotakashi 0:8fdf9a60065b 2297 itr = 0
kadonotakashi 0:8fdf9a60065b 2298 while len(results):
kadonotakashi 0:8fdf9a60065b 2299 itr += 1
kadonotakashi 0:8fdf9a60065b 2300 if itr > 360000:
kadonotakashi 0:8fdf9a60065b 2301 p.terminate()
kadonotakashi 0:8fdf9a60065b 2302 p.join()
kadonotakashi 0:8fdf9a60065b 2303 raise ToolException("Compile did not finish in 10 minutes")
kadonotakashi 0:8fdf9a60065b 2304 else:
kadonotakashi 0:8fdf9a60065b 2305 sleep(0.01)
kadonotakashi 0:8fdf9a60065b 2306 pending = 0
kadonotakashi 0:8fdf9a60065b 2307 for r in results:
kadonotakashi 0:8fdf9a60065b 2308 if r.ready() is True:
kadonotakashi 0:8fdf9a60065b 2309 try:
kadonotakashi 0:8fdf9a60065b 2310 worker_result = r.get()
kadonotakashi 0:8fdf9a60065b 2311 results.remove(r)
kadonotakashi 0:8fdf9a60065b 2312
kadonotakashi 0:8fdf9a60065b 2313 # Push all deferred notifications out to the actual notifier
kadonotakashi 0:8fdf9a60065b 2314 new_notify = deepcopy(notify)
kadonotakashi 0:8fdf9a60065b 2315 for message in worker_result['kwargs']['notify'].messages:
kadonotakashi 0:8fdf9a60065b 2316 new_notify.notify(message)
kadonotakashi 0:8fdf9a60065b 2317
kadonotakashi 0:8fdf9a60065b 2318 # Take report from the kwargs and merge it into existing report
kadonotakashi 0:8fdf9a60065b 2319 if report:
kadonotakashi 0:8fdf9a60065b 2320 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
kadonotakashi 0:8fdf9a60065b 2321 report_entry[worker_result['kwargs']['project_id'].upper()][0][0]['output'] = new_notify.get_output()
kadonotakashi 0:8fdf9a60065b 2322 for test_key in report_entry.keys():
kadonotakashi 0:8fdf9a60065b 2323 report[target_name][toolchain_name][test_key] = report_entry[test_key]
kadonotakashi 0:8fdf9a60065b 2324
kadonotakashi 0:8fdf9a60065b 2325 # Set the overall result to a failure if a build failure occurred
kadonotakashi 0:8fdf9a60065b 2326 if ('reason' in worker_result and
kadonotakashi 0:8fdf9a60065b 2327 not worker_result['reason'] and
kadonotakashi 0:8fdf9a60065b 2328 not isinstance(worker_result['reason'], NotSupportedException)):
kadonotakashi 0:8fdf9a60065b 2329 result = False
kadonotakashi 0:8fdf9a60065b 2330 break
kadonotakashi 0:8fdf9a60065b 2331
kadonotakashi 0:8fdf9a60065b 2332
kadonotakashi 0:8fdf9a60065b 2333 # Adding binary path to test build result
kadonotakashi 0:8fdf9a60065b 2334 if ('result' in worker_result and
kadonotakashi 0:8fdf9a60065b 2335 worker_result['result'] and
kadonotakashi 0:8fdf9a60065b 2336 'bin_file' in worker_result):
kadonotakashi 0:8fdf9a60065b 2337 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
kadonotakashi 0:8fdf9a60065b 2338
kadonotakashi 0:8fdf9a60065b 2339 test_key = 'test_apps' if 'test_apps-' in worker_result['kwargs']['project_id'] else 'tests'
kadonotakashi 0:8fdf9a60065b 2340 test_build[test_key][worker_result['kwargs']['project_id']] = {
kadonotakashi 0:8fdf9a60065b 2341 "binaries": [
kadonotakashi 0:8fdf9a60065b 2342 {
kadonotakashi 0:8fdf9a60065b 2343 "path": bin_file
kadonotakashi 0:8fdf9a60065b 2344 }
kadonotakashi 0:8fdf9a60065b 2345 ]
kadonotakashi 0:8fdf9a60065b 2346 }
kadonotakashi 0:8fdf9a60065b 2347
kadonotakashi 0:8fdf9a60065b 2348 test_key = worker_result['kwargs']['project_id'].upper()
kadonotakashi 0:8fdf9a60065b 2349 print('Image: %s\n' % bin_file)
kadonotakashi 0:8fdf9a60065b 2350
kadonotakashi 0:8fdf9a60065b 2351 except:
kadonotakashi 0:8fdf9a60065b 2352 if p._taskqueue.queue:
kadonotakashi 0:8fdf9a60065b 2353 p._taskqueue.queue.clear()
kadonotakashi 0:8fdf9a60065b 2354 sleep(0.5)
kadonotakashi 0:8fdf9a60065b 2355 p.terminate()
kadonotakashi 0:8fdf9a60065b 2356 p.join()
kadonotakashi 0:8fdf9a60065b 2357 raise
kadonotakashi 0:8fdf9a60065b 2358 else:
kadonotakashi 0:8fdf9a60065b 2359 pending += 1
kadonotakashi 0:8fdf9a60065b 2360 if pending >= jobs_count:
kadonotakashi 0:8fdf9a60065b 2361 break
kadonotakashi 0:8fdf9a60065b 2362
kadonotakashi 0:8fdf9a60065b 2363 # Break as soon as possible if there is a failure and we are not
kadonotakashi 0:8fdf9a60065b 2364 # continuing on build failures
kadonotakashi 0:8fdf9a60065b 2365 if not result and not continue_on_build_fail:
kadonotakashi 0:8fdf9a60065b 2366 if p._taskqueue.queue:
kadonotakashi 0:8fdf9a60065b 2367 p._taskqueue.queue.clear()
kadonotakashi 0:8fdf9a60065b 2368 sleep(0.5)
kadonotakashi 0:8fdf9a60065b 2369 p.terminate()
kadonotakashi 0:8fdf9a60065b 2370 break
kadonotakashi 0:8fdf9a60065b 2371
kadonotakashi 0:8fdf9a60065b 2372 p.join()
kadonotakashi 0:8fdf9a60065b 2373
kadonotakashi 0:8fdf9a60065b 2374 test_builds = {}
kadonotakashi 0:8fdf9a60065b 2375 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
kadonotakashi 0:8fdf9a60065b 2376
kadonotakashi 0:8fdf9a60065b 2377 return result, test_builds
kadonotakashi 0:8fdf9a60065b 2378
kadonotakashi 0:8fdf9a60065b 2379
kadonotakashi 0:8fdf9a60065b 2380 def test_spec_from_test_builds(test_builds):
kadonotakashi 0:8fdf9a60065b 2381 return {
kadonotakashi 0:8fdf9a60065b 2382 "builds": test_builds
kadonotakashi 0:8fdf9a60065b 2383 }