Nathan Yonkee / Mbed 2 deprecated Nucleo_sinewave_output_copy

init

Dependencies:   mbed

mbed-os/tools/test_api.py@10:46a4cf51ee38, 2018-03-02 (annotated)

Committer:
Nathan Yonkee
Date:
Fri Mar 02 07:16:49 2018 -0700
Revision:
10:46a4cf51ee38
Parent:
9:d58e77ebd769 
remove mbed-os

Who changed what in which revision?

UserRevisionLine numberNew contents of line
Nathan Yonkee 9:d58e77ebd769 1 """
Nathan Yonkee 9:d58e77ebd769 2 mbed SDK
Nathan Yonkee 9:d58e77ebd769 3 Copyright (c) 2011-2014 ARM Limited
Nathan Yonkee 9:d58e77ebd769 4
Nathan Yonkee 9:d58e77ebd769 5 Licensed under the Apache License, Version 2.0 (the "License");
Nathan Yonkee 9:d58e77ebd769 6 you may not use this file except in compliance with the License.
Nathan Yonkee 9:d58e77ebd769 7 You may obtain a copy of the License at
Nathan Yonkee 9:d58e77ebd769 8
Nathan Yonkee 9:d58e77ebd769 9 http://www.apache.org/licenses/LICENSE-2.0
Nathan Yonkee 9:d58e77ebd769 10
Nathan Yonkee 9:d58e77ebd769 11 Unless required by applicable law or agreed to in writing, software
Nathan Yonkee 9:d58e77ebd769 12 distributed under the License is distributed on an "AS IS" BASIS,
Nathan Yonkee 9:d58e77ebd769 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Nathan Yonkee 9:d58e77ebd769 14 See the License for the specific language governing permissions and
Nathan Yonkee 9:d58e77ebd769 15 limitations under the License.
Nathan Yonkee 9:d58e77ebd769 16
Nathan Yonkee 9:d58e77ebd769 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
Nathan Yonkee 9:d58e77ebd769 18 """
Nathan Yonkee 9:d58e77ebd769 19 from __future__ import print_function
Nathan Yonkee 9:d58e77ebd769 20
Nathan Yonkee 9:d58e77ebd769 21 import os
Nathan Yonkee 9:d58e77ebd769 22 import re
Nathan Yonkee 9:d58e77ebd769 23 import sys
Nathan Yonkee 9:d58e77ebd769 24 import json
Nathan Yonkee 9:d58e77ebd769 25 import uuid
Nathan Yonkee 9:d58e77ebd769 26 import pprint
Nathan Yonkee 9:d58e77ebd769 27 import random
Nathan Yonkee 9:d58e77ebd769 28 import argparse
Nathan Yonkee 9:d58e77ebd769 29 import datetime
Nathan Yonkee 9:d58e77ebd769 30 import threading
Nathan Yonkee 9:d58e77ebd769 31 import ctypes
Nathan Yonkee 9:d58e77ebd769 32 import functools
Nathan Yonkee 9:d58e77ebd769 33 from colorama import Fore, Back, Style
Nathan Yonkee 9:d58e77ebd769 34 from prettytable import PrettyTable
Nathan Yonkee 9:d58e77ebd769 35 from copy import copy
Nathan Yonkee 9:d58e77ebd769 36
Nathan Yonkee 9:d58e77ebd769 37 from time import sleep, time
Nathan Yonkee 9:d58e77ebd769 38 try:
Nathan Yonkee 9:d58e77ebd769 39 from Queue import Queue, Empty
Nathan Yonkee 9:d58e77ebd769 40 except ImportError:
Nathan Yonkee 9:d58e77ebd769 41 from queue import Queue, Empty
Nathan Yonkee 9:d58e77ebd769 42 from os.path import join, exists, basename, relpath
Nathan Yonkee 9:d58e77ebd769 43 from threading import Thread, Lock
Nathan Yonkee 9:d58e77ebd769 44 from multiprocessing import Pool, cpu_count
Nathan Yonkee 9:d58e77ebd769 45 from subprocess import Popen, PIPE
Nathan Yonkee 9:d58e77ebd769 46
Nathan Yonkee 9:d58e77ebd769 47 # Imports related to mbed build api
Nathan Yonkee 9:d58e77ebd769 48 from tools.tests import TESTS
Nathan Yonkee 9:d58e77ebd769 49 from tools.tests import TEST_MAP
Nathan Yonkee 9:d58e77ebd769 50 from tools.paths import BUILD_DIR
Nathan Yonkee 9:d58e77ebd769 51 from tools.paths import HOST_TESTS
Nathan Yonkee 9:d58e77ebd769 52 from tools.utils import ToolException
Nathan Yonkee 9:d58e77ebd769 53 from tools.utils import NotSupportedException
Nathan Yonkee 9:d58e77ebd769 54 from tools.utils import construct_enum
Nathan Yonkee 9:d58e77ebd769 55 from tools.memap import MemapParser
Nathan Yonkee 9:d58e77ebd769 56 from tools.targets import TARGET_MAP, Target
Nathan Yonkee 9:d58e77ebd769 57 import tools.test_configs as TestConfig
Nathan Yonkee 9:d58e77ebd769 58 from tools.test_db import BaseDBAccess
Nathan Yonkee 9:d58e77ebd769 59 from tools.build_api import build_project, build_mbed_libs, build_lib
Nathan Yonkee 9:d58e77ebd769 60 from tools.build_api import get_target_supported_toolchains
Nathan Yonkee 9:d58e77ebd769 61 from tools.build_api import write_build_report
Nathan Yonkee 9:d58e77ebd769 62 from tools.build_api import prep_report
Nathan Yonkee 9:d58e77ebd769 63 from tools.build_api import prep_properties
Nathan Yonkee 9:d58e77ebd769 64 from tools.build_api import create_result
Nathan Yonkee 9:d58e77ebd769 65 from tools.build_api import add_result_to_report
Nathan Yonkee 9:d58e77ebd769 66 from tools.build_api import prepare_toolchain
Nathan Yonkee 9:d58e77ebd769 67 from tools.build_api import scan_resources
Nathan Yonkee 9:d58e77ebd769 68 from tools.build_api import get_config
Nathan Yonkee 9:d58e77ebd769 69 from tools.libraries import LIBRARIES, LIBRARY_MAP
Nathan Yonkee 9:d58e77ebd769 70 from tools.options import extract_profile
Nathan Yonkee 9:d58e77ebd769 71 from tools.toolchains import TOOLCHAIN_PATHS
Nathan Yonkee 9:d58e77ebd769 72 from tools.toolchains import TOOLCHAINS
Nathan Yonkee 9:d58e77ebd769 73 from tools.test_exporters import ReportExporter, ResultExporterType
Nathan Yonkee 9:d58e77ebd769 74 from tools.utils import argparse_filestring_type
Nathan Yonkee 9:d58e77ebd769 75 from tools.utils import argparse_uppercase_type
Nathan Yonkee 9:d58e77ebd769 76 from tools.utils import argparse_lowercase_type
Nathan Yonkee 9:d58e77ebd769 77 from tools.utils import argparse_many
Nathan Yonkee 9:d58e77ebd769 78
Nathan Yonkee 9:d58e77ebd769 79 import tools.host_tests.host_tests_plugins as host_tests_plugins
Nathan Yonkee 9:d58e77ebd769 80
Nathan Yonkee 9:d58e77ebd769 81 try:
Nathan Yonkee 9:d58e77ebd769 82 import mbed_lstools
Nathan Yonkee 9:d58e77ebd769 83 from tools.compliance.ioper_runner import get_available_oper_test_scopes
Nathan Yonkee 9:d58e77ebd769 84 except:
Nathan Yonkee 9:d58e77ebd769 85 pass
Nathan Yonkee 9:d58e77ebd769 86
Nathan Yonkee 9:d58e77ebd769 87
Nathan Yonkee 9:d58e77ebd769 88 class ProcessObserver(Thread):
Nathan Yonkee 9:d58e77ebd769 89 def __init__(self, proc):
Nathan Yonkee 9:d58e77ebd769 90 Thread.__init__(self)
Nathan Yonkee 9:d58e77ebd769 91 self.proc = proc
Nathan Yonkee 9:d58e77ebd769 92 self.queue = Queue()
Nathan Yonkee 9:d58e77ebd769 93 self.daemon = True
Nathan Yonkee 9:d58e77ebd769 94 self.active = True
Nathan Yonkee 9:d58e77ebd769 95 self.start()
Nathan Yonkee 9:d58e77ebd769 96
Nathan Yonkee 9:d58e77ebd769 97 def run(self):
Nathan Yonkee 9:d58e77ebd769 98 while self.active:
Nathan Yonkee 9:d58e77ebd769 99 c = self.proc.stdout.read(1)
Nathan Yonkee 9:d58e77ebd769 100 self.queue.put(c)
Nathan Yonkee 9:d58e77ebd769 101
Nathan Yonkee 9:d58e77ebd769 102 def stop(self):
Nathan Yonkee 9:d58e77ebd769 103 self.active = False
Nathan Yonkee 9:d58e77ebd769 104 try:
Nathan Yonkee 9:d58e77ebd769 105 self.proc.terminate()
Nathan Yonkee 9:d58e77ebd769 106 except Exception:
Nathan Yonkee 9:d58e77ebd769 107 pass
Nathan Yonkee 9:d58e77ebd769 108
Nathan Yonkee 9:d58e77ebd769 109
Nathan Yonkee 9:d58e77ebd769 110 class SingleTestExecutor(threading.Thread):
Nathan Yonkee 9:d58e77ebd769 111 """ Example: Single test class in separate thread usage
Nathan Yonkee 9:d58e77ebd769 112 """
Nathan Yonkee 9:d58e77ebd769 113 def __init__(self, single_test):
Nathan Yonkee 9:d58e77ebd769 114 self.single_test = single_test
Nathan Yonkee 9:d58e77ebd769 115 threading.Thread.__init__(self)
Nathan Yonkee 9:d58e77ebd769 116
Nathan Yonkee 9:d58e77ebd769 117 def run(self):
Nathan Yonkee 9:d58e77ebd769 118 start = time()
Nathan Yonkee 9:d58e77ebd769 119 # Execute tests depending on options and filter applied
Nathan Yonkee 9:d58e77ebd769 120 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
Nathan Yonkee 9:d58e77ebd769 121 elapsed_time = time() - start
Nathan Yonkee 9:d58e77ebd769 122
Nathan Yonkee 9:d58e77ebd769 123 # Human readable summary
Nathan Yonkee 9:d58e77ebd769 124 if not self.single_test.opts_suppress_summary:
Nathan Yonkee 9:d58e77ebd769 125 # prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 126 print(self.single_test.generate_test_summary(test_summary,
Nathan Yonkee 9:d58e77ebd769 127 shuffle_seed))
Nathan Yonkee 9:d58e77ebd769 128 if self.single_test.opts_test_x_toolchain_summary:
Nathan Yonkee 9:d58e77ebd769 129 # prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 130 # table shows text x toolchain test result matrix
Nathan Yonkee 9:d58e77ebd769 131 print(self.single_test.generate_test_summary_by_target(
Nathan Yonkee 9:d58e77ebd769 132 test_summary, shuffle_seed))
Nathan Yonkee 9:d58e77ebd769 133 print("Completed in %.2f sec"% (elapsed_time))
Nathan Yonkee 9:d58e77ebd769 134
Nathan Yonkee 9:d58e77ebd769 135
Nathan Yonkee 9:d58e77ebd769 136 class SingleTestRunner(object):
Nathan Yonkee 9:d58e77ebd769 137 """ Object wrapper for single test run which may involve multiple MUTs
Nathan Yonkee 9:d58e77ebd769 138 """
Nathan Yonkee 9:d58e77ebd769 139 RE_DETECT_TESTCASE_RESULT = None
Nathan Yonkee 9:d58e77ebd769 140
Nathan Yonkee 9:d58e77ebd769 141 # Return codes for test script
Nathan Yonkee 9:d58e77ebd769 142 TEST_RESULT_OK = "OK"
Nathan Yonkee 9:d58e77ebd769 143 TEST_RESULT_FAIL = "FAIL"
Nathan Yonkee 9:d58e77ebd769 144 TEST_RESULT_ERROR = "ERROR"
Nathan Yonkee 9:d58e77ebd769 145 TEST_RESULT_UNDEF = "UNDEF"
Nathan Yonkee 9:d58e77ebd769 146 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
Nathan Yonkee 9:d58e77ebd769 147 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
Nathan Yonkee 9:d58e77ebd769 148 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
Nathan Yonkee 9:d58e77ebd769 149 TEST_RESULT_TIMEOUT = "TIMEOUT"
Nathan Yonkee 9:d58e77ebd769 150 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
Nathan Yonkee 9:d58e77ebd769 151 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
Nathan Yonkee 9:d58e77ebd769 152 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
Nathan Yonkee 9:d58e77ebd769 153 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
Nathan Yonkee 9:d58e77ebd769 154
Nathan Yonkee 9:d58e77ebd769 155 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
Nathan Yonkee 9:d58e77ebd769 156 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
Nathan Yonkee 9:d58e77ebd769 157 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
Nathan Yonkee 9:d58e77ebd769 158
Nathan Yonkee 9:d58e77ebd769 159 muts = {} # MUTs descriptor (from external file)
Nathan Yonkee 9:d58e77ebd769 160 test_spec = {} # Test specification (from external file)
Nathan Yonkee 9:d58e77ebd769 161
Nathan Yonkee 9:d58e77ebd769 162 # mbed test suite -> SingleTestRunner
Nathan Yonkee 9:d58e77ebd769 163 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
Nathan Yonkee 9:d58e77ebd769 164 "failure" : TEST_RESULT_FAIL,
Nathan Yonkee 9:d58e77ebd769 165 "error" : TEST_RESULT_ERROR,
Nathan Yonkee 9:d58e77ebd769 166 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
Nathan Yonkee 9:d58e77ebd769 167 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
Nathan Yonkee 9:d58e77ebd769 168 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
Nathan Yonkee 9:d58e77ebd769 169 "timeout" : TEST_RESULT_TIMEOUT,
Nathan Yonkee 9:d58e77ebd769 170 "no_image" : TEST_RESULT_NO_IMAGE,
Nathan Yonkee 9:d58e77ebd769 171 "end" : TEST_RESULT_UNDEF,
Nathan Yonkee 9:d58e77ebd769 172 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
Nathan Yonkee 9:d58e77ebd769 173 "build_failed" : TEST_RESULT_BUILD_FAILED,
Nathan Yonkee 9:d58e77ebd769 174 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
Nathan Yonkee 9:d58e77ebd769 175 }
Nathan Yonkee 9:d58e77ebd769 176
Nathan Yonkee 9:d58e77ebd769 177 def __init__(self,
Nathan Yonkee 9:d58e77ebd769 178 _global_loops_count=1,
Nathan Yonkee 9:d58e77ebd769 179 _test_loops_list=None,
Nathan Yonkee 9:d58e77ebd769 180 _muts={},
Nathan Yonkee 9:d58e77ebd769 181 _clean=False,
Nathan Yonkee 9:d58e77ebd769 182 _parser=None,
Nathan Yonkee 9:d58e77ebd769 183 _opts=None,
Nathan Yonkee 9:d58e77ebd769 184 _opts_db_url=None,
Nathan Yonkee 9:d58e77ebd769 185 _opts_log_file_name=None,
Nathan Yonkee 9:d58e77ebd769 186 _opts_report_html_file_name=None,
Nathan Yonkee 9:d58e77ebd769 187 _opts_report_junit_file_name=None,
Nathan Yonkee 9:d58e77ebd769 188 _opts_report_build_file_name=None,
Nathan Yonkee 9:d58e77ebd769 189 _opts_report_text_file_name=None,
Nathan Yonkee 9:d58e77ebd769 190 _opts_build_report={},
Nathan Yonkee 9:d58e77ebd769 191 _opts_build_properties={},
Nathan Yonkee 9:d58e77ebd769 192 _test_spec={},
Nathan Yonkee 9:d58e77ebd769 193 _opts_goanna_for_mbed_sdk=None,
Nathan Yonkee 9:d58e77ebd769 194 _opts_goanna_for_tests=None,
Nathan Yonkee 9:d58e77ebd769 195 _opts_shuffle_test_order=False,
Nathan Yonkee 9:d58e77ebd769 196 _opts_shuffle_test_seed=None,
Nathan Yonkee 9:d58e77ebd769 197 _opts_test_by_names=None,
Nathan Yonkee 9:d58e77ebd769 198 _opts_peripheral_by_names=None,
Nathan Yonkee 9:d58e77ebd769 199 _opts_test_only_peripheral=False,
Nathan Yonkee 9:d58e77ebd769 200 _opts_test_only_common=False,
Nathan Yonkee 9:d58e77ebd769 201 _opts_verbose_skipped_tests=False,
Nathan Yonkee 9:d58e77ebd769 202 _opts_verbose_test_result_only=False,
Nathan Yonkee 9:d58e77ebd769 203 _opts_verbose=False,
Nathan Yonkee 9:d58e77ebd769 204 _opts_firmware_global_name=None,
Nathan Yonkee 9:d58e77ebd769 205 _opts_only_build_tests=False,
Nathan Yonkee 9:d58e77ebd769 206 _opts_parallel_test_exec=False,
Nathan Yonkee 9:d58e77ebd769 207 _opts_suppress_summary=False,
Nathan Yonkee 9:d58e77ebd769 208 _opts_test_x_toolchain_summary=False,
Nathan Yonkee 9:d58e77ebd769 209 _opts_copy_method=None,
Nathan Yonkee 9:d58e77ebd769 210 _opts_mut_reset_type=None,
Nathan Yonkee 9:d58e77ebd769 211 _opts_jobs=None,
Nathan Yonkee 9:d58e77ebd769 212 _opts_waterfall_test=None,
Nathan Yonkee 9:d58e77ebd769 213 _opts_consolidate_waterfall_test=None,
Nathan Yonkee 9:d58e77ebd769 214 _opts_extend_test_timeout=None,
Nathan Yonkee 9:d58e77ebd769 215 _opts_auto_detect=None,
Nathan Yonkee 9:d58e77ebd769 216 _opts_include_non_automated=False):
Nathan Yonkee 9:d58e77ebd769 217 """ Let's try hard to init this object
Nathan Yonkee 9:d58e77ebd769 218 """
Nathan Yonkee 9:d58e77ebd769 219 from colorama import init
Nathan Yonkee 9:d58e77ebd769 220 init()
Nathan Yonkee 9:d58e77ebd769 221
Nathan Yonkee 9:d58e77ebd769 222 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
Nathan Yonkee 9:d58e77ebd769 223 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
Nathan Yonkee 9:d58e77ebd769 224 # Settings related to test loops counters
Nathan Yonkee 9:d58e77ebd769 225 try:
Nathan Yonkee 9:d58e77ebd769 226 _global_loops_count = int(_global_loops_count)
Nathan Yonkee 9:d58e77ebd769 227 except:
Nathan Yonkee 9:d58e77ebd769 228 _global_loops_count = 1
Nathan Yonkee 9:d58e77ebd769 229 if _global_loops_count < 1:
Nathan Yonkee 9:d58e77ebd769 230 _global_loops_count = 1
Nathan Yonkee 9:d58e77ebd769 231 self.GLOBAL_LOOPS_COUNT = _global_loops_count
Nathan Yonkee 9:d58e77ebd769 232 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
Nathan Yonkee 9:d58e77ebd769 233 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
Nathan Yonkee 9:d58e77ebd769 234
Nathan Yonkee 9:d58e77ebd769 235 self.shuffle_random_seed = 0.0
Nathan Yonkee 9:d58e77ebd769 236 self.SHUFFLE_SEED_ROUND = 10
Nathan Yonkee 9:d58e77ebd769 237
Nathan Yonkee 9:d58e77ebd769 238 # MUT list and test specification storage
Nathan Yonkee 9:d58e77ebd769 239 self.muts = _muts
Nathan Yonkee 9:d58e77ebd769 240 self.test_spec = _test_spec
Nathan Yonkee 9:d58e77ebd769 241
Nathan Yonkee 9:d58e77ebd769 242 # Settings passed e.g. from command line
Nathan Yonkee 9:d58e77ebd769 243 self.opts_db_url = _opts_db_url
Nathan Yonkee 9:d58e77ebd769 244 self.opts_log_file_name = _opts_log_file_name
Nathan Yonkee 9:d58e77ebd769 245 self.opts_report_html_file_name = _opts_report_html_file_name
Nathan Yonkee 9:d58e77ebd769 246 self.opts_report_junit_file_name = _opts_report_junit_file_name
Nathan Yonkee 9:d58e77ebd769 247 self.opts_report_build_file_name = _opts_report_build_file_name
Nathan Yonkee 9:d58e77ebd769 248 self.opts_report_text_file_name = _opts_report_text_file_name
Nathan Yonkee 9:d58e77ebd769 249 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
Nathan Yonkee 9:d58e77ebd769 250 self.opts_goanna_for_tests = _opts_goanna_for_tests
Nathan Yonkee 9:d58e77ebd769 251 self.opts_shuffle_test_order = _opts_shuffle_test_order
Nathan Yonkee 9:d58e77ebd769 252 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
Nathan Yonkee 9:d58e77ebd769 253 self.opts_test_by_names = _opts_test_by_names
Nathan Yonkee 9:d58e77ebd769 254 self.opts_peripheral_by_names = _opts_peripheral_by_names
Nathan Yonkee 9:d58e77ebd769 255 self.opts_test_only_peripheral = _opts_test_only_peripheral
Nathan Yonkee 9:d58e77ebd769 256 self.opts_test_only_common = _opts_test_only_common
Nathan Yonkee 9:d58e77ebd769 257 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
Nathan Yonkee 9:d58e77ebd769 258 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
Nathan Yonkee 9:d58e77ebd769 259 self.opts_verbose = _opts_verbose
Nathan Yonkee 9:d58e77ebd769 260 self.opts_firmware_global_name = _opts_firmware_global_name
Nathan Yonkee 9:d58e77ebd769 261 self.opts_only_build_tests = _opts_only_build_tests
Nathan Yonkee 9:d58e77ebd769 262 self.opts_parallel_test_exec = _opts_parallel_test_exec
Nathan Yonkee 9:d58e77ebd769 263 self.opts_suppress_summary = _opts_suppress_summary
Nathan Yonkee 9:d58e77ebd769 264 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
Nathan Yonkee 9:d58e77ebd769 265 self.opts_copy_method = _opts_copy_method
Nathan Yonkee 9:d58e77ebd769 266 self.opts_mut_reset_type = _opts_mut_reset_type
Nathan Yonkee 9:d58e77ebd769 267 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
Nathan Yonkee 9:d58e77ebd769 268 self.opts_waterfall_test = _opts_waterfall_test
Nathan Yonkee 9:d58e77ebd769 269 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
Nathan Yonkee 9:d58e77ebd769 270 self.opts_extend_test_timeout = _opts_extend_test_timeout
Nathan Yonkee 9:d58e77ebd769 271 self.opts_clean = _clean
Nathan Yonkee 9:d58e77ebd769 272 self.opts_parser = _parser
Nathan Yonkee 9:d58e77ebd769 273 self.opts = _opts
Nathan Yonkee 9:d58e77ebd769 274 self.opts_auto_detect = _opts_auto_detect
Nathan Yonkee 9:d58e77ebd769 275 self.opts_include_non_automated = _opts_include_non_automated
Nathan Yonkee 9:d58e77ebd769 276
Nathan Yonkee 9:d58e77ebd769 277 self.build_report = _opts_build_report
Nathan Yonkee 9:d58e77ebd769 278 self.build_properties = _opts_build_properties
Nathan Yonkee 9:d58e77ebd769 279
Nathan Yonkee 9:d58e77ebd769 280 # File / screen logger initialization
Nathan Yonkee 9:d58e77ebd769 281 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
Nathan Yonkee 9:d58e77ebd769 282
Nathan Yonkee 9:d58e77ebd769 283 # Database related initializations
Nathan Yonkee 9:d58e77ebd769 284 self.db_logger = factory_db_logger(self.opts_db_url)
Nathan Yonkee 9:d58e77ebd769 285 self.db_logger_build_id = None # Build ID (database index of build_id table)
Nathan Yonkee 9:d58e77ebd769 286 # Let's connect to database to set up credentials and confirm database is ready
Nathan Yonkee 9:d58e77ebd769 287 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 288 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
Nathan Yonkee 9:d58e77ebd769 289 if self.db_logger.is_connected():
Nathan Yonkee 9:d58e77ebd769 290 # Get hostname and uname so we can use it as build description
Nathan Yonkee 9:d58e77ebd769 291 # when creating new build_id in external database
Nathan Yonkee 9:d58e77ebd769 292 (_hostname, _uname) = self.db_logger.get_hostname()
Nathan Yonkee 9:d58e77ebd769 293 _host_location = os.path.dirname(os.path.abspath(__file__))
Nathan Yonkee 9:d58e77ebd769 294 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
Nathan Yonkee 9:d58e77ebd769 295 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
Nathan Yonkee 9:d58e77ebd769 296 self.db_logger.disconnect()
Nathan Yonkee 9:d58e77ebd769 297
Nathan Yonkee 9:d58e77ebd769 298 def dump_options(self):
Nathan Yonkee 9:d58e77ebd769 299 """ Function returns data structure with common settings passed to SingelTestRunner
Nathan Yonkee 9:d58e77ebd769 300 It can be used for example to fill _extra fields in database storing test suite single run data
Nathan Yonkee 9:d58e77ebd769 301 Example:
Nathan Yonkee 9:d58e77ebd769 302 data = self.dump_options()
Nathan Yonkee 9:d58e77ebd769 303 or
Nathan Yonkee 9:d58e77ebd769 304 data_str = json.dumps(self.dump_options())
Nathan Yonkee 9:d58e77ebd769 305 """
Nathan Yonkee 9:d58e77ebd769 306 result = {"db_url" : str(self.opts_db_url),
Nathan Yonkee 9:d58e77ebd769 307 "log_file_name" : str(self.opts_log_file_name),
Nathan Yonkee 9:d58e77ebd769 308 "shuffle_test_order" : str(self.opts_shuffle_test_order),
Nathan Yonkee 9:d58e77ebd769 309 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
Nathan Yonkee 9:d58e77ebd769 310 "test_by_names" : str(self.opts_test_by_names),
Nathan Yonkee 9:d58e77ebd769 311 "peripheral_by_names" : str(self.opts_peripheral_by_names),
Nathan Yonkee 9:d58e77ebd769 312 "test_only_peripheral" : str(self.opts_test_only_peripheral),
Nathan Yonkee 9:d58e77ebd769 313 "test_only_common" : str(self.opts_test_only_common),
Nathan Yonkee 9:d58e77ebd769 314 "verbose" : str(self.opts_verbose),
Nathan Yonkee 9:d58e77ebd769 315 "firmware_global_name" : str(self.opts_firmware_global_name),
Nathan Yonkee 9:d58e77ebd769 316 "only_build_tests" : str(self.opts_only_build_tests),
Nathan Yonkee 9:d58e77ebd769 317 "copy_method" : str(self.opts_copy_method),
Nathan Yonkee 9:d58e77ebd769 318 "mut_reset_type" : str(self.opts_mut_reset_type),
Nathan Yonkee 9:d58e77ebd769 319 "jobs" : str(self.opts_jobs),
Nathan Yonkee 9:d58e77ebd769 320 "extend_test_timeout" : str(self.opts_extend_test_timeout),
Nathan Yonkee 9:d58e77ebd769 321 "_dummy" : ''
Nathan Yonkee 9:d58e77ebd769 322 }
Nathan Yonkee 9:d58e77ebd769 323 return result
Nathan Yonkee 9:d58e77ebd769 324
Nathan Yonkee 9:d58e77ebd769 325 def shuffle_random_func(self):
Nathan Yonkee 9:d58e77ebd769 326 return self.shuffle_random_seed
Nathan Yonkee 9:d58e77ebd769 327
Nathan Yonkee 9:d58e77ebd769 328 def is_shuffle_seed_float(self):
Nathan Yonkee 9:d58e77ebd769 329 """ return true if function parameter can be converted to float
Nathan Yonkee 9:d58e77ebd769 330 """
Nathan Yonkee 9:d58e77ebd769 331 result = True
Nathan Yonkee 9:d58e77ebd769 332 try:
Nathan Yonkee 9:d58e77ebd769 333 float(self.shuffle_random_seed)
Nathan Yonkee 9:d58e77ebd769 334 except ValueError:
Nathan Yonkee 9:d58e77ebd769 335 result = False
Nathan Yonkee 9:d58e77ebd769 336 return result
Nathan Yonkee 9:d58e77ebd769 337
Nathan Yonkee 9:d58e77ebd769 338 # This will store target / toolchain specific properties
Nathan Yonkee 9:d58e77ebd769 339 test_suite_properties_ext = {} # target : toolchain
Nathan Yonkee 9:d58e77ebd769 340 # Here we store test results
Nathan Yonkee 9:d58e77ebd769 341 test_summary = []
Nathan Yonkee 9:d58e77ebd769 342 # Here we store test results in extended data structure
Nathan Yonkee 9:d58e77ebd769 343 test_summary_ext = {}
Nathan Yonkee 9:d58e77ebd769 344 execute_thread_slice_lock = Lock()
Nathan Yonkee 9:d58e77ebd769 345
Nathan Yonkee 9:d58e77ebd769 346 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
Nathan Yonkee 9:d58e77ebd769 347 for toolchain in toolchains:
Nathan Yonkee 9:d58e77ebd769 348 tt_id = "%s::%s" % (toolchain, target)
Nathan Yonkee 9:d58e77ebd769 349
Nathan Yonkee 9:d58e77ebd769 350 T = TARGET_MAP[target]
Nathan Yonkee 9:d58e77ebd769 351
Nathan Yonkee 9:d58e77ebd769 352 # print target, toolchain
Nathan Yonkee 9:d58e77ebd769 353 # Test suite properties returned to external tools like CI
Nathan Yonkee 9:d58e77ebd769 354 test_suite_properties = {
Nathan Yonkee 9:d58e77ebd769 355 'jobs': self.opts_jobs,
Nathan Yonkee 9:d58e77ebd769 356 'clean': clean,
Nathan Yonkee 9:d58e77ebd769 357 'target': target,
Nathan Yonkee 9:d58e77ebd769 358 'vendor': T.extra_labels[0],
Nathan Yonkee 9:d58e77ebd769 359 'test_ids': ', '.join(test_ids),
Nathan Yonkee 9:d58e77ebd769 360 'toolchain': toolchain,
Nathan Yonkee 9:d58e77ebd769 361 'shuffle_random_seed': self.shuffle_random_seed
Nathan Yonkee 9:d58e77ebd769 362 }
Nathan Yonkee 9:d58e77ebd769 363
Nathan Yonkee 9:d58e77ebd769 364
Nathan Yonkee 9:d58e77ebd769 365 # print '=== %s::%s ===' % (target, toolchain)
Nathan Yonkee 9:d58e77ebd769 366 # Let's build our test
Nathan Yonkee 9:d58e77ebd769 367 if target not in TARGET_MAP:
Nathan Yonkee 9:d58e77ebd769 368 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 369 self.logger.LogType.NOTIF,
Nathan Yonkee 9:d58e77ebd769 370 'Skipped tests for %s target. Target platform not found' %
Nathan Yonkee 9:d58e77ebd769 371 (target)))
Nathan Yonkee 9:d58e77ebd769 372 continue
Nathan Yonkee 9:d58e77ebd769 373
Nathan Yonkee 9:d58e77ebd769 374 clean_mbed_libs_options = (self.opts_goanna_for_mbed_sdk or
Nathan Yonkee 9:d58e77ebd769 375 self.opts_clean or clean)
Nathan Yonkee 9:d58e77ebd769 376
Nathan Yonkee 9:d58e77ebd769 377 profile = extract_profile(self.opts_parser, self.opts, toolchain)
Nathan Yonkee 9:d58e77ebd769 378 stats_depth = self.opts.stats_depth or 2
Nathan Yonkee 9:d58e77ebd769 379
Nathan Yonkee 9:d58e77ebd769 380 try:
Nathan Yonkee 9:d58e77ebd769 381 build_mbed_libs_result = build_mbed_libs(
Nathan Yonkee 9:d58e77ebd769 382 T, toolchain,
Nathan Yonkee 9:d58e77ebd769 383 clean=clean_mbed_libs_options,
Nathan Yonkee 9:d58e77ebd769 384 verbose=self.opts_verbose,
Nathan Yonkee 9:d58e77ebd769 385 jobs=self.opts_jobs,
Nathan Yonkee 9:d58e77ebd769 386 report=build_report,
Nathan Yonkee 9:d58e77ebd769 387 properties=build_properties,
Nathan Yonkee 9:d58e77ebd769 388 build_profile=profile)
Nathan Yonkee 9:d58e77ebd769 389
Nathan Yonkee 9:d58e77ebd769 390 if not build_mbed_libs_result:
Nathan Yonkee 9:d58e77ebd769 391 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 392 self.logger.LogType.NOTIF,
Nathan Yonkee 9:d58e77ebd769 393 'Skipped tests for %s target. Toolchain %s is not '
Nathan Yonkee 9:d58e77ebd769 394 'supported for this target'% (T.name, toolchain)))
Nathan Yonkee 9:d58e77ebd769 395 continue
Nathan Yonkee 9:d58e77ebd769 396
Nathan Yonkee 9:d58e77ebd769 397 except ToolException:
Nathan Yonkee 9:d58e77ebd769 398 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 399 self.logger.LogType.ERROR,
Nathan Yonkee 9:d58e77ebd769 400 'There were errors while building MBED libs for %s using %s'
Nathan Yonkee 9:d58e77ebd769 401 % (target, toolchain)))
Nathan Yonkee 9:d58e77ebd769 402 continue
Nathan Yonkee 9:d58e77ebd769 403
Nathan Yonkee 9:d58e77ebd769 404 build_dir = join(BUILD_DIR, "test", target, toolchain)
Nathan Yonkee 9:d58e77ebd769 405
Nathan Yonkee 9:d58e77ebd769 406 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
Nathan Yonkee 9:d58e77ebd769 407 test_suite_properties['build_dir'] = build_dir
Nathan Yonkee 9:d58e77ebd769 408 test_suite_properties['skipped'] = []
Nathan Yonkee 9:d58e77ebd769 409
Nathan Yonkee 9:d58e77ebd769 410 # Enumerate through all tests and shuffle test order if requested
Nathan Yonkee 9:d58e77ebd769 411 test_map_keys = sorted(TEST_MAP.keys())
Nathan Yonkee 9:d58e77ebd769 412
Nathan Yonkee 9:d58e77ebd769 413 if self.opts_shuffle_test_order:
Nathan Yonkee 9:d58e77ebd769 414 random.shuffle(test_map_keys, self.shuffle_random_func)
Nathan Yonkee 9:d58e77ebd769 415 # Update database with shuffle seed f applicable
Nathan Yonkee 9:d58e77ebd769 416 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 417 self.db_logger.reconnect();
Nathan Yonkee 9:d58e77ebd769 418 if self.db_logger.is_connected():
Nathan Yonkee 9:d58e77ebd769 419 self.db_logger.update_build_id_info(
Nathan Yonkee 9:d58e77ebd769 420 self.db_logger_build_id,
Nathan Yonkee 9:d58e77ebd769 421 _shuffle_seed=self.shuffle_random_func())
Nathan Yonkee 9:d58e77ebd769 422 self.db_logger.disconnect();
Nathan Yonkee 9:d58e77ebd769 423
Nathan Yonkee 9:d58e77ebd769 424 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 425 self.db_logger.reconnect();
Nathan Yonkee 9:d58e77ebd769 426 if self.db_logger.is_connected():
Nathan Yonkee 9:d58e77ebd769 427 # Update MUTs and Test Specification in database
Nathan Yonkee 9:d58e77ebd769 428 self.db_logger.update_build_id_info(
Nathan Yonkee 9:d58e77ebd769 429 self.db_logger_build_id,
Nathan Yonkee 9:d58e77ebd769 430 _muts=self.muts, _test_spec=self.test_spec)
Nathan Yonkee 9:d58e77ebd769 431 # Update Extra information in database (some options passed to test suite)
Nathan Yonkee 9:d58e77ebd769 432 self.db_logger.update_build_id_info(
Nathan Yonkee 9:d58e77ebd769 433 self.db_logger_build_id,
Nathan Yonkee 9:d58e77ebd769 434 _extra=json.dumps(self.dump_options()))
Nathan Yonkee 9:d58e77ebd769 435 self.db_logger.disconnect();
Nathan Yonkee 9:d58e77ebd769 436
Nathan Yonkee 9:d58e77ebd769 437 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
Nathan Yonkee 9:d58e77ebd769 438 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
Nathan Yonkee 9:d58e77ebd769 439
Nathan Yonkee 9:d58e77ebd769 440 for skipped_test_id in skipped_test_map_keys:
Nathan Yonkee 9:d58e77ebd769 441 test_suite_properties['skipped'].append(skipped_test_id)
Nathan Yonkee 9:d58e77ebd769 442
Nathan Yonkee 9:d58e77ebd769 443
Nathan Yonkee 9:d58e77ebd769 444 # First pass through all tests and determine which libraries need to be built
Nathan Yonkee 9:d58e77ebd769 445 libraries = []
Nathan Yonkee 9:d58e77ebd769 446 for test_id in valid_test_map_keys:
Nathan Yonkee 9:d58e77ebd769 447 test = TEST_MAP[test_id]
Nathan Yonkee 9:d58e77ebd769 448
Nathan Yonkee 9:d58e77ebd769 449 # Detect which lib should be added to test
Nathan Yonkee 9:d58e77ebd769 450 # Some libs have to compiled like RTOS or ETH
Nathan Yonkee 9:d58e77ebd769 451 for lib in LIBRARIES:
Nathan Yonkee 9:d58e77ebd769 452 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
Nathan Yonkee 9:d58e77ebd769 453 libraries.append(lib['id'])
Nathan Yonkee 9:d58e77ebd769 454
Nathan Yonkee 9:d58e77ebd769 455
Nathan Yonkee 9:d58e77ebd769 456 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
Nathan Yonkee 9:d58e77ebd769 457
Nathan Yonkee 9:d58e77ebd769 458 # Build all required libraries
Nathan Yonkee 9:d58e77ebd769 459 for lib_id in libraries:
Nathan Yonkee 9:d58e77ebd769 460 try:
Nathan Yonkee 9:d58e77ebd769 461 build_lib(lib_id,
Nathan Yonkee 9:d58e77ebd769 462 T,
Nathan Yonkee 9:d58e77ebd769 463 toolchain,
Nathan Yonkee 9:d58e77ebd769 464 verbose=self.opts_verbose,
Nathan Yonkee 9:d58e77ebd769 465 clean=clean_mbed_libs_options,
Nathan Yonkee 9:d58e77ebd769 466 jobs=self.opts_jobs,
Nathan Yonkee 9:d58e77ebd769 467 report=build_report,
Nathan Yonkee 9:d58e77ebd769 468 properties=build_properties,
Nathan Yonkee 9:d58e77ebd769 469 build_profile=profile)
Nathan Yonkee 9:d58e77ebd769 470
Nathan Yonkee 9:d58e77ebd769 471 except ToolException:
Nathan Yonkee 9:d58e77ebd769 472 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 473 self.logger.LogType.ERROR,
Nathan Yonkee 9:d58e77ebd769 474 'There were errors while building library %s' % lib_id))
Nathan Yonkee 9:d58e77ebd769 475 continue
Nathan Yonkee 9:d58e77ebd769 476
Nathan Yonkee 9:d58e77ebd769 477
Nathan Yonkee 9:d58e77ebd769 478 for test_id in valid_test_map_keys:
Nathan Yonkee 9:d58e77ebd769 479 test = TEST_MAP[test_id]
Nathan Yonkee 9:d58e77ebd769 480
Nathan Yonkee 9:d58e77ebd769 481 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
Nathan Yonkee 9:d58e77ebd769 482
Nathan Yonkee 9:d58e77ebd769 483 # TODO: move this 2 below loops to separate function
Nathan Yonkee 9:d58e77ebd769 484 INC_DIRS = []
Nathan Yonkee 9:d58e77ebd769 485 for lib_id in libraries:
Nathan Yonkee 9:d58e77ebd769 486 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
Nathan Yonkee 9:d58e77ebd769 487 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
Nathan Yonkee 9:d58e77ebd769 488
Nathan Yonkee 9:d58e77ebd769 489 MACROS = []
Nathan Yonkee 9:d58e77ebd769 490 for lib_id in libraries:
Nathan Yonkee 9:d58e77ebd769 491 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
Nathan Yonkee 9:d58e77ebd769 492 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
Nathan Yonkee 9:d58e77ebd769 493 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
Nathan Yonkee 9:d58e77ebd769 494 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
Nathan Yonkee 9:d58e77ebd769 495 test_uuid = uuid.uuid4()
Nathan Yonkee 9:d58e77ebd769 496 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
Nathan Yonkee 9:d58e77ebd769 497
Nathan Yonkee 9:d58e77ebd769 498 # Prepare extended test results data structure (it can be used to generate detailed test report)
Nathan Yonkee 9:d58e77ebd769 499 if target not in self.test_summary_ext:
Nathan Yonkee 9:d58e77ebd769 500 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
Nathan Yonkee 9:d58e77ebd769 501 if toolchain not in self.test_summary_ext[target]:
Nathan Yonkee 9:d58e77ebd769 502 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
Nathan Yonkee 9:d58e77ebd769 503
Nathan Yonkee 9:d58e77ebd769 504 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
Nathan Yonkee 9:d58e77ebd769 505
Nathan Yonkee 9:d58e77ebd769 506 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
Nathan Yonkee 9:d58e77ebd769 507 try:
Nathan Yonkee 9:d58e77ebd769 508 path = build_project(test.source_dir, join(build_dir, test_id), T,
Nathan Yonkee 9:d58e77ebd769 509 toolchain, test.dependencies, clean=clean_project_options,
Nathan Yonkee 9:d58e77ebd769 510 verbose=self.opts_verbose, name=project_name, macros=MACROS,
Nathan Yonkee 9:d58e77ebd769 511 inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
Nathan Yonkee 9:d58e77ebd769 512 properties=build_properties, project_id=test_id,
Nathan Yonkee 9:d58e77ebd769 513 project_description=test.get_description(),
Nathan Yonkee 9:d58e77ebd769 514 build_profile=profile, stats_depth=stats_depth)
Nathan Yonkee 9:d58e77ebd769 515
Nathan Yonkee 9:d58e77ebd769 516 except Exception as e:
Nathan Yonkee 9:d58e77ebd769 517 project_name_str = project_name if project_name is not None else test_id
Nathan Yonkee 9:d58e77ebd769 518
Nathan Yonkee 9:d58e77ebd769 519
Nathan Yonkee 9:d58e77ebd769 520 test_result = self.TEST_RESULT_FAIL
Nathan Yonkee 9:d58e77ebd769 521
Nathan Yonkee 9:d58e77ebd769 522 if isinstance(e, ToolException):
Nathan Yonkee 9:d58e77ebd769 523 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 524 self.logger.LogType.ERROR,
Nathan Yonkee 9:d58e77ebd769 525 'There were errors while building project %s' %
Nathan Yonkee 9:d58e77ebd769 526 project_name_str))
Nathan Yonkee 9:d58e77ebd769 527 test_result = self.TEST_RESULT_BUILD_FAILED
Nathan Yonkee 9:d58e77ebd769 528 elif isinstance(e, NotSupportedException):
Nathan Yonkee 9:d58e77ebd769 529 print(elf.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 530 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 531 'Project %s is not supported' % project_name_str))
Nathan Yonkee 9:d58e77ebd769 532 test_result = self.TEST_RESULT_NOT_SUPPORTED
Nathan Yonkee 9:d58e77ebd769 533
Nathan Yonkee 9:d58e77ebd769 534
Nathan Yonkee 9:d58e77ebd769 535 # Append test results to global test summary
Nathan Yonkee 9:d58e77ebd769 536 self.test_summary.append(
Nathan Yonkee 9:d58e77ebd769 537 (test_result, target, toolchain, test_id,
Nathan Yonkee 9:d58e77ebd769 538 test.get_description(), 0, 0, '-')
Nathan Yonkee 9:d58e77ebd769 539 )
Nathan Yonkee 9:d58e77ebd769 540
Nathan Yonkee 9:d58e77ebd769 541 # Add detailed test result to test summary structure
Nathan Yonkee 9:d58e77ebd769 542 if test_id not in self.test_summary_ext[target][toolchain]:
Nathan Yonkee 9:d58e77ebd769 543 self.test_summary_ext[target][toolchain][test_id] = []
Nathan Yonkee 9:d58e77ebd769 544
Nathan Yonkee 9:d58e77ebd769 545 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
Nathan Yonkee 9:d58e77ebd769 546 'result' : test_result,
Nathan Yonkee 9:d58e77ebd769 547 'output' : '',
Nathan Yonkee 9:d58e77ebd769 548 'target_name' : target,
Nathan Yonkee 9:d58e77ebd769 549 'target_name_unique': target,
Nathan Yonkee 9:d58e77ebd769 550 'toolchain_name' : toolchain,
Nathan Yonkee 9:d58e77ebd769 551 'id' : test_id,
Nathan Yonkee 9:d58e77ebd769 552 'description' : test.get_description(),
Nathan Yonkee 9:d58e77ebd769 553 'elapsed_time' : 0,
Nathan Yonkee 9:d58e77ebd769 554 'duration' : 0,
Nathan Yonkee 9:d58e77ebd769 555 'copy_method' : None
Nathan Yonkee 9:d58e77ebd769 556 }})
Nathan Yonkee 9:d58e77ebd769 557 continue
Nathan Yonkee 9:d58e77ebd769 558
Nathan Yonkee 9:d58e77ebd769 559 if self.opts_only_build_tests:
Nathan Yonkee 9:d58e77ebd769 560 # With this option we are skipping testing phase
Nathan Yonkee 9:d58e77ebd769 561 continue
Nathan Yonkee 9:d58e77ebd769 562
Nathan Yonkee 9:d58e77ebd769 563 # Test duration can be increased by global value
Nathan Yonkee 9:d58e77ebd769 564 test_duration = test.duration
Nathan Yonkee 9:d58e77ebd769 565 if self.opts_extend_test_timeout is not None:
Nathan Yonkee 9:d58e77ebd769 566 test_duration += self.opts_extend_test_timeout
Nathan Yonkee 9:d58e77ebd769 567
Nathan Yonkee 9:d58e77ebd769 568 # For an automated test the duration act as a timeout after
Nathan Yonkee 9:d58e77ebd769 569 # which the test gets interrupted
Nathan Yonkee 9:d58e77ebd769 570 test_spec = self.shape_test_request(target, path, test_id, test_duration)
Nathan Yonkee 9:d58e77ebd769 571 test_loops = self.get_test_loop_count(test_id)
Nathan Yonkee 9:d58e77ebd769 572
Nathan Yonkee 9:d58e77ebd769 573 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
Nathan Yonkee 9:d58e77ebd769 574 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
Nathan Yonkee 9:d58e77ebd769 575 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
Nathan Yonkee 9:d58e77ebd769 576
Nathan Yonkee 9:d58e77ebd769 577 # read MUTs, test specification and perform tests
Nathan Yonkee 9:d58e77ebd769 578 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
Nathan Yonkee 9:d58e77ebd769 579
Nathan Yonkee 9:d58e77ebd769 580 if handle_results is None:
Nathan Yonkee 9:d58e77ebd769 581 continue
Nathan Yonkee 9:d58e77ebd769 582
Nathan Yonkee 9:d58e77ebd769 583 for handle_result in handle_results:
Nathan Yonkee 9:d58e77ebd769 584 if handle_result:
Nathan Yonkee 9:d58e77ebd769 585 single_test_result, detailed_test_results = handle_result
Nathan Yonkee 9:d58e77ebd769 586 else:
Nathan Yonkee 9:d58e77ebd769 587 continue
Nathan Yonkee 9:d58e77ebd769 588
Nathan Yonkee 9:d58e77ebd769 589 # Append test results to global test summary
Nathan Yonkee 9:d58e77ebd769 590 if single_test_result is not None:
Nathan Yonkee 9:d58e77ebd769 591 self.test_summary.append(single_test_result)
Nathan Yonkee 9:d58e77ebd769 592
Nathan Yonkee 9:d58e77ebd769 593 # Add detailed test result to test summary structure
Nathan Yonkee 9:d58e77ebd769 594 if target not in self.test_summary_ext[target][toolchain]:
Nathan Yonkee 9:d58e77ebd769 595 if test_id not in self.test_summary_ext[target][toolchain]:
Nathan Yonkee 9:d58e77ebd769 596 self.test_summary_ext[target][toolchain][test_id] = []
Nathan Yonkee 9:d58e77ebd769 597
Nathan Yonkee 9:d58e77ebd769 598 append_test_result = detailed_test_results
Nathan Yonkee 9:d58e77ebd769 599
Nathan Yonkee 9:d58e77ebd769 600 # If waterfall and consolidate-waterfall options are enabled,
Nathan Yonkee 9:d58e77ebd769 601 # only include the last test result in the report.
Nathan Yonkee 9:d58e77ebd769 602 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
Nathan Yonkee 9:d58e77ebd769 603 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
Nathan Yonkee 9:d58e77ebd769 604
Nathan Yonkee 9:d58e77ebd769 605 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
Nathan Yonkee 9:d58e77ebd769 606
Nathan Yonkee 9:d58e77ebd769 607 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
Nathan Yonkee 9:d58e77ebd769 608 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
Nathan Yonkee 9:d58e77ebd769 609
Nathan Yonkee 9:d58e77ebd769 610 q.put(target + '_'.join(toolchains))
Nathan Yonkee 9:d58e77ebd769 611 return
Nathan Yonkee 9:d58e77ebd769 612
Nathan Yonkee 9:d58e77ebd769 613 def execute(self):
Nathan Yonkee 9:d58e77ebd769 614 clean = self.test_spec.get('clean', False)
Nathan Yonkee 9:d58e77ebd769 615 test_ids = self.test_spec.get('test_ids', [])
Nathan Yonkee 9:d58e77ebd769 616 q = Queue()
Nathan Yonkee 9:d58e77ebd769 617
Nathan Yonkee 9:d58e77ebd769 618 # Generate seed for shuffle if seed is not provided in
Nathan Yonkee 9:d58e77ebd769 619 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
Nathan Yonkee 9:d58e77ebd769 620 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
Nathan Yonkee 9:d58e77ebd769 621 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
Nathan Yonkee 9:d58e77ebd769 622
Nathan Yonkee 9:d58e77ebd769 623
Nathan Yonkee 9:d58e77ebd769 624 if self.opts_parallel_test_exec:
Nathan Yonkee 9:d58e77ebd769 625 ###################################################################
Nathan Yonkee 9:d58e77ebd769 626 # Experimental, parallel test execution per singletest instance.
Nathan Yonkee 9:d58e77ebd769 627 ###################################################################
Nathan Yonkee 9:d58e77ebd769 628 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
Nathan Yonkee 9:d58e77ebd769 629 # Note: We are building here in parallel for each target separately!
Nathan Yonkee 9:d58e77ebd769 630 # So we are not building the same thing multiple times and compilers
Nathan Yonkee 9:d58e77ebd769 631 # in separate threads do not collide.
Nathan Yonkee 9:d58e77ebd769 632 # Inside execute_thread_slice() function function handle() will be called to
Nathan Yonkee 9:d58e77ebd769 633 # get information about available MUTs (per target).
Nathan Yonkee 9:d58e77ebd769 634 for target, toolchains in self.test_spec['targets'].items():
Nathan Yonkee 9:d58e77ebd769 635 self.test_suite_properties_ext[target] = {}
Nathan Yonkee 9:d58e77ebd769 636 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
Nathan Yonkee 9:d58e77ebd769 637 t.daemon = True
Nathan Yonkee 9:d58e77ebd769 638 t.start()
Nathan Yonkee 9:d58e77ebd769 639 execute_threads.append(t)
Nathan Yonkee 9:d58e77ebd769 640
Nathan Yonkee 9:d58e77ebd769 641 for t in execute_threads:
Nathan Yonkee 9:d58e77ebd769 642 q.get() # t.join() would block some threads because we should not wait in any order for thread end
Nathan Yonkee 9:d58e77ebd769 643 else:
Nathan Yonkee 9:d58e77ebd769 644 # Serialized (not parallel) test execution
Nathan Yonkee 9:d58e77ebd769 645 for target, toolchains in self.test_spec['targets'].items():
Nathan Yonkee 9:d58e77ebd769 646 if target not in self.test_suite_properties_ext:
Nathan Yonkee 9:d58e77ebd769 647 self.test_suite_properties_ext[target] = {}
Nathan Yonkee 9:d58e77ebd769 648
Nathan Yonkee 9:d58e77ebd769 649 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
Nathan Yonkee 9:d58e77ebd769 650 q.get()
Nathan Yonkee 9:d58e77ebd769 651
Nathan Yonkee 9:d58e77ebd769 652 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 653 self.db_logger.reconnect();
Nathan Yonkee 9:d58e77ebd769 654 if self.db_logger.is_connected():
Nathan Yonkee 9:d58e77ebd769 655 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
Nathan Yonkee 9:d58e77ebd769 656 self.db_logger.disconnect();
Nathan Yonkee 9:d58e77ebd769 657
Nathan Yonkee 9:d58e77ebd769 658 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
Nathan Yonkee 9:d58e77ebd769 659
Nathan Yonkee 9:d58e77ebd769 660 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
Nathan Yonkee 9:d58e77ebd769 661 valid_test_map_keys = []
Nathan Yonkee 9:d58e77ebd769 662
Nathan Yonkee 9:d58e77ebd769 663 for test_id in test_map_keys:
Nathan Yonkee 9:d58e77ebd769 664 test = TEST_MAP[test_id]
Nathan Yonkee 9:d58e77ebd769 665 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
Nathan Yonkee 9:d58e77ebd769 666 continue
Nathan Yonkee 9:d58e77ebd769 667
Nathan Yonkee 9:d58e77ebd769 668 if test_ids and test_id not in test_ids:
Nathan Yonkee 9:d58e77ebd769 669 continue
Nathan Yonkee 9:d58e77ebd769 670
Nathan Yonkee 9:d58e77ebd769 671 if self.opts_test_only_peripheral and not test.peripherals:
Nathan Yonkee 9:d58e77ebd769 672 if self.opts_verbose_skipped_tests:
Nathan Yonkee 9:d58e77ebd769 673 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 674 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 675 'Common test skipped for target %s' % target))
Nathan Yonkee 9:d58e77ebd769 676 continue
Nathan Yonkee 9:d58e77ebd769 677
Nathan Yonkee 9:d58e77ebd769 678 if (self.opts_peripheral_by_names and test.peripherals and
Nathan Yonkee 9:d58e77ebd769 679 not any((i in self.opts_peripheral_by_names)
Nathan Yonkee 9:d58e77ebd769 680 for i in test.peripherals)):
Nathan Yonkee 9:d58e77ebd769 681 # We will skip tests not forced with -p option
Nathan Yonkee 9:d58e77ebd769 682 if self.opts_verbose_skipped_tests:
Nathan Yonkee 9:d58e77ebd769 683 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 684 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 685 'Common test skipped for target %s' % target))
Nathan Yonkee 9:d58e77ebd769 686 continue
Nathan Yonkee 9:d58e77ebd769 687
Nathan Yonkee 9:d58e77ebd769 688 if self.opts_test_only_common and test.peripherals:
Nathan Yonkee 9:d58e77ebd769 689 if self.opts_verbose_skipped_tests:
Nathan Yonkee 9:d58e77ebd769 690 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 691 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 692 'Peripheral test skipped for target %s' % target))
Nathan Yonkee 9:d58e77ebd769 693 continue
Nathan Yonkee 9:d58e77ebd769 694
Nathan Yonkee 9:d58e77ebd769 695 if not include_non_automated and not test.automated:
Nathan Yonkee 9:d58e77ebd769 696 if self.opts_verbose_skipped_tests:
Nathan Yonkee 9:d58e77ebd769 697 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 698 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 699 'Non automated test skipped for target %s' % target))
Nathan Yonkee 9:d58e77ebd769 700 continue
Nathan Yonkee 9:d58e77ebd769 701
Nathan Yonkee 9:d58e77ebd769 702 if test.is_supported(target, toolchain):
Nathan Yonkee 9:d58e77ebd769 703 if test.peripherals is None and self.opts_only_build_tests:
Nathan Yonkee 9:d58e77ebd769 704 # When users are using 'build only flag' and test do not have
Nathan Yonkee 9:d58e77ebd769 705 # specified peripherals we can allow test building by default
Nathan Yonkee 9:d58e77ebd769 706 pass
Nathan Yonkee 9:d58e77ebd769 707 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
Nathan Yonkee 9:d58e77ebd769 708 # If we force peripheral with option -p we expect test
Nathan Yonkee 9:d58e77ebd769 709 # to pass even if peripheral is not in MUTs file.
Nathan Yonkee 9:d58e77ebd769 710 pass
Nathan Yonkee 9:d58e77ebd769 711 elif not self.is_peripherals_available(target, test.peripherals):
Nathan Yonkee 9:d58e77ebd769 712 if self.opts_verbose_skipped_tests:
Nathan Yonkee 9:d58e77ebd769 713 if test.peripherals:
Nathan Yonkee 9:d58e77ebd769 714 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 715 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 716 'Peripheral %s test skipped for target %s' %
Nathan Yonkee 9:d58e77ebd769 717 (",".join(test.peripherals), target)))
Nathan Yonkee 9:d58e77ebd769 718 else:
Nathan Yonkee 9:d58e77ebd769 719 print(self.logger.log_line(
Nathan Yonkee 9:d58e77ebd769 720 self.logger.LogType.INFO,
Nathan Yonkee 9:d58e77ebd769 721 'Test %s skipped for target %s' %
Nathan Yonkee 9:d58e77ebd769 722 (test_id, target)))
Nathan Yonkee 9:d58e77ebd769 723 continue
Nathan Yonkee 9:d58e77ebd769 724
Nathan Yonkee 9:d58e77ebd769 725 # The test has made it through all the filters, so add it to the valid tests list
Nathan Yonkee 9:d58e77ebd769 726 valid_test_map_keys.append(test_id)
Nathan Yonkee 9:d58e77ebd769 727
Nathan Yonkee 9:d58e77ebd769 728 return valid_test_map_keys
Nathan Yonkee 9:d58e77ebd769 729
Nathan Yonkee 9:d58e77ebd769 730 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
Nathan Yonkee 9:d58e77ebd769 731 # NOTE: This will not preserve order
Nathan Yonkee 9:d58e77ebd769 732 return list(set(all_test_map_keys) - set(valid_test_map_keys))
Nathan Yonkee 9:d58e77ebd769 733
Nathan Yonkee 9:d58e77ebd769 734 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
Nathan Yonkee 9:d58e77ebd769 735 """ Prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 736 table shows text x toolchain test result matrix
Nathan Yonkee 9:d58e77ebd769 737 """
Nathan Yonkee 9:d58e77ebd769 738 RESULT_INDEX = 0
Nathan Yonkee 9:d58e77ebd769 739 TARGET_INDEX = 1
Nathan Yonkee 9:d58e77ebd769 740 TOOLCHAIN_INDEX = 2
Nathan Yonkee 9:d58e77ebd769 741 TEST_INDEX = 3
Nathan Yonkee 9:d58e77ebd769 742 DESC_INDEX = 4
Nathan Yonkee 9:d58e77ebd769 743
Nathan Yonkee 9:d58e77ebd769 744 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
Nathan Yonkee 9:d58e77ebd769 745 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
Nathan Yonkee 9:d58e77ebd769 746 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
Nathan Yonkee 9:d58e77ebd769 747 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
Nathan Yonkee 9:d58e77ebd769 748
Nathan Yonkee 9:d58e77ebd769 749 result = "Test summary:\n"
Nathan Yonkee 9:d58e77ebd769 750 for target in unique_targets:
Nathan Yonkee 9:d58e77ebd769 751 result_dict = {} # test : { toolchain : result }
Nathan Yonkee 9:d58e77ebd769 752 unique_target_toolchains = []
Nathan Yonkee 9:d58e77ebd769 753 for test in test_summary:
Nathan Yonkee 9:d58e77ebd769 754 if test[TARGET_INDEX] == target:
Nathan Yonkee 9:d58e77ebd769 755 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
Nathan Yonkee 9:d58e77ebd769 756 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
Nathan Yonkee 9:d58e77ebd769 757 if test[TEST_INDEX] not in result_dict:
Nathan Yonkee 9:d58e77ebd769 758 result_dict[test[TEST_INDEX]] = {}
Nathan Yonkee 9:d58e77ebd769 759 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
Nathan Yonkee 9:d58e77ebd769 760
Nathan Yonkee 9:d58e77ebd769 761 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
Nathan Yonkee 9:d58e77ebd769 762 pt = PrettyTable(pt_cols)
Nathan Yonkee 9:d58e77ebd769 763 for col in pt_cols:
Nathan Yonkee 9:d58e77ebd769 764 pt.align[col] = "l"
Nathan Yonkee 9:d58e77ebd769 765 pt.padding_width = 1 # One space between column edges and contents (default)
Nathan Yonkee 9:d58e77ebd769 766
Nathan Yonkee 9:d58e77ebd769 767 for test in unique_tests:
Nathan Yonkee 9:d58e77ebd769 768 if test in result_dict:
Nathan Yonkee 9:d58e77ebd769 769 test_results = result_dict[test]
Nathan Yonkee 9:d58e77ebd769 770 if test in unique_test_desc:
Nathan Yonkee 9:d58e77ebd769 771 row = [target, test, unique_test_desc[test]]
Nathan Yonkee 9:d58e77ebd769 772 for toolchain in unique_toolchains:
Nathan Yonkee 9:d58e77ebd769 773 if toolchain in test_results:
Nathan Yonkee 9:d58e77ebd769 774 row.append(test_results[toolchain])
Nathan Yonkee 9:d58e77ebd769 775 pt.add_row(row)
Nathan Yonkee 9:d58e77ebd769 776 result += pt.get_string()
Nathan Yonkee 9:d58e77ebd769 777 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
Nathan Yonkee 9:d58e77ebd769 778 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
Nathan Yonkee 9:d58e77ebd769 779 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
Nathan Yonkee 9:d58e77ebd769 780 return result
Nathan Yonkee 9:d58e77ebd769 781
Nathan Yonkee 9:d58e77ebd769 782 def generate_test_summary(self, test_summary, shuffle_seed=None):
Nathan Yonkee 9:d58e77ebd769 783 """ Prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 784 table shows target x test results matrix across
Nathan Yonkee 9:d58e77ebd769 785 """
Nathan Yonkee 9:d58e77ebd769 786 success_code = 0 # Success code that can be leter returned to
Nathan Yonkee 9:d58e77ebd769 787 result = "Test summary:\n"
Nathan Yonkee 9:d58e77ebd769 788 # Pretty table package is used to print results
Nathan Yonkee 9:d58e77ebd769 789 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
Nathan Yonkee 9:d58e77ebd769 790 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
Nathan Yonkee 9:d58e77ebd769 791 pt.align["Result"] = "l" # Left align
Nathan Yonkee 9:d58e77ebd769 792 pt.align["Target"] = "l" # Left align
Nathan Yonkee 9:d58e77ebd769 793 pt.align["Toolchain"] = "l" # Left align
Nathan Yonkee 9:d58e77ebd769 794 pt.align["Test ID"] = "l" # Left align
Nathan Yonkee 9:d58e77ebd769 795 pt.align["Test Description"] = "l" # Left align
Nathan Yonkee 9:d58e77ebd769 796 pt.padding_width = 1 # One space between column edges and contents (default)
Nathan Yonkee 9:d58e77ebd769 797
Nathan Yonkee 9:d58e77ebd769 798 result_dict = {self.TEST_RESULT_OK : 0,
Nathan Yonkee 9:d58e77ebd769 799 self.TEST_RESULT_FAIL : 0,
Nathan Yonkee 9:d58e77ebd769 800 self.TEST_RESULT_ERROR : 0,
Nathan Yonkee 9:d58e77ebd769 801 self.TEST_RESULT_UNDEF : 0,
Nathan Yonkee 9:d58e77ebd769 802 self.TEST_RESULT_IOERR_COPY : 0,
Nathan Yonkee 9:d58e77ebd769 803 self.TEST_RESULT_IOERR_DISK : 0,
Nathan Yonkee 9:d58e77ebd769 804 self.TEST_RESULT_IOERR_SERIAL : 0,
Nathan Yonkee 9:d58e77ebd769 805 self.TEST_RESULT_NO_IMAGE : 0,
Nathan Yonkee 9:d58e77ebd769 806 self.TEST_RESULT_TIMEOUT : 0,
Nathan Yonkee 9:d58e77ebd769 807 self.TEST_RESULT_MBED_ASSERT : 0,
Nathan Yonkee 9:d58e77ebd769 808 self.TEST_RESULT_BUILD_FAILED : 0,
Nathan Yonkee 9:d58e77ebd769 809 self.TEST_RESULT_NOT_SUPPORTED : 0
Nathan Yonkee 9:d58e77ebd769 810 }
Nathan Yonkee 9:d58e77ebd769 811
Nathan Yonkee 9:d58e77ebd769 812 for test in test_summary:
Nathan Yonkee 9:d58e77ebd769 813 if test[0] in result_dict:
Nathan Yonkee 9:d58e77ebd769 814 result_dict[test[0]] += 1
Nathan Yonkee 9:d58e77ebd769 815 pt.add_row(test)
Nathan Yonkee 9:d58e77ebd769 816 result += pt.get_string()
Nathan Yonkee 9:d58e77ebd769 817 result += "\n"
Nathan Yonkee 9:d58e77ebd769 818
Nathan Yonkee 9:d58e77ebd769 819 # Print result count
Nathan Yonkee 9:d58e77ebd769 820 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()])
Nathan Yonkee 9:d58e77ebd769 821 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
Nathan Yonkee 9:d58e77ebd769 822 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
Nathan Yonkee 9:d58e77ebd769 823 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
Nathan Yonkee 9:d58e77ebd769 824 return result
Nathan Yonkee 9:d58e77ebd769 825
Nathan Yonkee 9:d58e77ebd769 826 def test_loop_list_to_dict(self, test_loops_str):
Nathan Yonkee 9:d58e77ebd769 827 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
Nathan Yonkee 9:d58e77ebd769 828 """
Nathan Yonkee 9:d58e77ebd769 829 result = {}
Nathan Yonkee 9:d58e77ebd769 830 if test_loops_str:
Nathan Yonkee 9:d58e77ebd769 831 test_loops = test_loops_str
Nathan Yonkee 9:d58e77ebd769 832 for test_loop in test_loops:
Nathan Yonkee 9:d58e77ebd769 833 test_loop_count = test_loop.split('=')
Nathan Yonkee 9:d58e77ebd769 834 if len(test_loop_count) == 2:
Nathan Yonkee 9:d58e77ebd769 835 _test_id, _test_loops = test_loop_count
Nathan Yonkee 9:d58e77ebd769 836 try:
Nathan Yonkee 9:d58e77ebd769 837 _test_loops = int(_test_loops)
Nathan Yonkee 9:d58e77ebd769 838 except:
Nathan Yonkee 9:d58e77ebd769 839 continue
Nathan Yonkee 9:d58e77ebd769 840 result[_test_id] = _test_loops
Nathan Yonkee 9:d58e77ebd769 841 return result
Nathan Yonkee 9:d58e77ebd769 842
Nathan Yonkee 9:d58e77ebd769 843 def get_test_loop_count(self, test_id):
Nathan Yonkee 9:d58e77ebd769 844 """ This function returns no. of loops per test (deducted by test_id_.
Nathan Yonkee 9:d58e77ebd769 845 If test is not in list of redefined loop counts it will use default value.
Nathan Yonkee 9:d58e77ebd769 846 """
Nathan Yonkee 9:d58e77ebd769 847 result = self.GLOBAL_LOOPS_COUNT
Nathan Yonkee 9:d58e77ebd769 848 if test_id in self.TEST_LOOPS_DICT:
Nathan Yonkee 9:d58e77ebd769 849 result = self.TEST_LOOPS_DICT[test_id]
Nathan Yonkee 9:d58e77ebd769 850 return result
Nathan Yonkee 9:d58e77ebd769 851
Nathan Yonkee 9:d58e77ebd769 852 def delete_file(self, file_path):
Nathan Yonkee 9:d58e77ebd769 853 """ Remove file from the system
Nathan Yonkee 9:d58e77ebd769 854 """
Nathan Yonkee 9:d58e77ebd769 855 result = True
Nathan Yonkee 9:d58e77ebd769 856 resutl_msg = ""
Nathan Yonkee 9:d58e77ebd769 857 try:
Nathan Yonkee 9:d58e77ebd769 858 os.remove(file_path)
Nathan Yonkee 9:d58e77ebd769 859 except Exception as e:
Nathan Yonkee 9:d58e77ebd769 860 resutl_msg = e
Nathan Yonkee 9:d58e77ebd769 861 result = False
Nathan Yonkee 9:d58e77ebd769 862 return result, resutl_msg
Nathan Yonkee 9:d58e77ebd769 863
Nathan Yonkee 9:d58e77ebd769 864 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
Nathan Yonkee 9:d58e77ebd769 865 """ Test is being invoked for given MUT.
Nathan Yonkee 9:d58e77ebd769 866 """
Nathan Yonkee 9:d58e77ebd769 867 # Get test information, image and test timeout
Nathan Yonkee 9:d58e77ebd769 868 test_id = data['test_id']
Nathan Yonkee 9:d58e77ebd769 869 test = TEST_MAP[test_id]
Nathan Yonkee 9:d58e77ebd769 870 test_description = TEST_MAP[test_id].get_description()
Nathan Yonkee 9:d58e77ebd769 871 image = data["image"]
Nathan Yonkee 9:d58e77ebd769 872 duration = data.get("duration", 10)
Nathan Yonkee 9:d58e77ebd769 873
Nathan Yonkee 9:d58e77ebd769 874 if mut is None:
Nathan Yonkee 9:d58e77ebd769 875 print("Error: No Mbed available: MUT[%s]" % data['mcu'])
Nathan Yonkee 9:d58e77ebd769 876 return None
Nathan Yonkee 9:d58e77ebd769 877
Nathan Yonkee 9:d58e77ebd769 878 mcu = mut['mcu']
Nathan Yonkee 9:d58e77ebd769 879 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
Nathan Yonkee 9:d58e77ebd769 880
Nathan Yonkee 9:d58e77ebd769 881 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 882 self.db_logger.reconnect()
Nathan Yonkee 9:d58e77ebd769 883
Nathan Yonkee 9:d58e77ebd769 884 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
Nathan Yonkee 9:d58e77ebd769 885
Nathan Yonkee 9:d58e77ebd769 886 # Tests can be looped so test results must be stored for the same test
Nathan Yonkee 9:d58e77ebd769 887 test_all_result = []
Nathan Yonkee 9:d58e77ebd769 888 # Test results for one test ran few times
Nathan Yonkee 9:d58e77ebd769 889 detailed_test_results = {} # { Loop_number: { results ... } }
Nathan Yonkee 9:d58e77ebd769 890
Nathan Yonkee 9:d58e77ebd769 891 for test_index in range(test_loops):
Nathan Yonkee 9:d58e77ebd769 892
Nathan Yonkee 9:d58e77ebd769 893 # If mbedls is available and we are auto detecting MUT info,
Nathan Yonkee 9:d58e77ebd769 894 # update MUT info (mounting may changed)
Nathan Yonkee 9:d58e77ebd769 895 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
Nathan Yonkee 9:d58e77ebd769 896 platform_name_filter = [mcu]
Nathan Yonkee 9:d58e77ebd769 897 muts_list = {}
Nathan Yonkee 9:d58e77ebd769 898 found = False
Nathan Yonkee 9:d58e77ebd769 899
Nathan Yonkee 9:d58e77ebd769 900 for i in range(0, 60):
Nathan Yonkee 9:d58e77ebd769 901 print('Looking for %s with MBEDLS' % mcu)
Nathan Yonkee 9:d58e77ebd769 902 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
Nathan Yonkee 9:d58e77ebd769 903
Nathan Yonkee 9:d58e77ebd769 904 if 1 not in muts_list:
Nathan Yonkee 9:d58e77ebd769 905 sleep(3)
Nathan Yonkee 9:d58e77ebd769 906 else:
Nathan Yonkee 9:d58e77ebd769 907 found = True
Nathan Yonkee 9:d58e77ebd769 908 break
Nathan Yonkee 9:d58e77ebd769 909
Nathan Yonkee 9:d58e77ebd769 910 if not found:
Nathan Yonkee 9:d58e77ebd769 911 print("Error: mbed not found with MBEDLS: %s" % data['mcu'])
Nathan Yonkee 9:d58e77ebd769 912 return None
Nathan Yonkee 9:d58e77ebd769 913 else:
Nathan Yonkee 9:d58e77ebd769 914 mut = muts_list[1]
Nathan Yonkee 9:d58e77ebd769 915
Nathan Yonkee 9:d58e77ebd769 916 disk = mut.get('disk')
Nathan Yonkee 9:d58e77ebd769 917 port = mut.get('port')
Nathan Yonkee 9:d58e77ebd769 918
Nathan Yonkee 9:d58e77ebd769 919 if disk is None or port is None:
Nathan Yonkee 9:d58e77ebd769 920 return None
Nathan Yonkee 9:d58e77ebd769 921
Nathan Yonkee 9:d58e77ebd769 922 target_by_mcu = TARGET_MAP[mut['mcu']]
Nathan Yonkee 9:d58e77ebd769 923 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
Nathan Yonkee 9:d58e77ebd769 924 # Some extra stuff can be declared in MUTs structure
Nathan Yonkee 9:d58e77ebd769 925 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
Nathan Yonkee 9:d58e77ebd769 926 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
Nathan Yonkee 9:d58e77ebd769 927
Nathan Yonkee 9:d58e77ebd769 928 # When the build and test system were separate, this was relative to a
Nathan Yonkee 9:d58e77ebd769 929 # base network folder base path: join(NETWORK_BASE_PATH, )
Nathan Yonkee 9:d58e77ebd769 930 image_path = image
Nathan Yonkee 9:d58e77ebd769 931
Nathan Yonkee 9:d58e77ebd769 932 # Host test execution
Nathan Yonkee 9:d58e77ebd769 933 start_host_exec_time = time()
Nathan Yonkee 9:d58e77ebd769 934
Nathan Yonkee 9:d58e77ebd769 935 single_test_result = self.TEST_RESULT_UNDEF # single test run result
Nathan Yonkee 9:d58e77ebd769 936 _copy_method = selected_copy_method
Nathan Yonkee 9:d58e77ebd769 937
Nathan Yonkee 9:d58e77ebd769 938 if not exists(image_path):
Nathan Yonkee 9:d58e77ebd769 939 single_test_result = self.TEST_RESULT_NO_IMAGE
Nathan Yonkee 9:d58e77ebd769 940 elapsed_time = 0
Nathan Yonkee 9:d58e77ebd769 941 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
Nathan Yonkee 9:d58e77ebd769 942 print(single_test_output)
Nathan Yonkee 9:d58e77ebd769 943 else:
Nathan Yonkee 9:d58e77ebd769 944 # Host test execution
Nathan Yonkee 9:d58e77ebd769 945 start_host_exec_time = time()
Nathan Yonkee 9:d58e77ebd769 946
Nathan Yonkee 9:d58e77ebd769 947 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
Nathan Yonkee 9:d58e77ebd769 948 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
Nathan Yonkee 9:d58e77ebd769 949 host_test_result = self.run_host_test(test.host_test,
Nathan Yonkee 9:d58e77ebd769 950 image_path, disk, port, duration,
Nathan Yonkee 9:d58e77ebd769 951 micro=target_name,
Nathan Yonkee 9:d58e77ebd769 952 verbose=host_test_verbose,
Nathan Yonkee 9:d58e77ebd769 953 reset=host_test_reset,
Nathan Yonkee 9:d58e77ebd769 954 reset_tout=reset_tout,
Nathan Yonkee 9:d58e77ebd769 955 copy_method=selected_copy_method,
Nathan Yonkee 9:d58e77ebd769 956 program_cycle_s=target_by_mcu.program_cycle_s)
Nathan Yonkee 9:d58e77ebd769 957 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
Nathan Yonkee 9:d58e77ebd769 958
Nathan Yonkee 9:d58e77ebd769 959 # Store test result
Nathan Yonkee 9:d58e77ebd769 960 test_all_result.append(single_test_result)
Nathan Yonkee 9:d58e77ebd769 961 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
Nathan Yonkee 9:d58e77ebd769 962 elapsed_time = single_testduration # TIme of single test case execution after reset
Nathan Yonkee 9:d58e77ebd769 963
Nathan Yonkee 9:d58e77ebd769 964 detailed_test_results[test_index] = {
Nathan Yonkee 9:d58e77ebd769 965 'result' : single_test_result,
Nathan Yonkee 9:d58e77ebd769 966 'output' : single_test_output,
Nathan Yonkee 9:d58e77ebd769 967 'target_name' : target_name,
Nathan Yonkee 9:d58e77ebd769 968 'target_name_unique' : target_name_unique,
Nathan Yonkee 9:d58e77ebd769 969 'toolchain_name' : toolchain_name,
Nathan Yonkee 9:d58e77ebd769 970 'id' : test_id,
Nathan Yonkee 9:d58e77ebd769 971 'description' : test_description,
Nathan Yonkee 9:d58e77ebd769 972 'elapsed_time' : round(elapsed_time, 2),
Nathan Yonkee 9:d58e77ebd769 973 'duration' : single_timeout,
Nathan Yonkee 9:d58e77ebd769 974 'copy_method' : _copy_method,
Nathan Yonkee 9:d58e77ebd769 975 }
Nathan Yonkee 9:d58e77ebd769 976
Nathan Yonkee 9:d58e77ebd769 977 print(self.print_test_result(
Nathan Yonkee 9:d58e77ebd769 978 single_test_result, target_name_unique, toolchain_name, test_id,
Nathan Yonkee 9:d58e77ebd769 979 test_description, elapsed_time, single_timeout))
Nathan Yonkee 9:d58e77ebd769 980
Nathan Yonkee 9:d58e77ebd769 981 # Update database entries for ongoing test
Nathan Yonkee 9:d58e77ebd769 982 if self.db_logger and self.db_logger.is_connected():
Nathan Yonkee 9:d58e77ebd769 983 test_type = 'SingleTest'
Nathan Yonkee 9:d58e77ebd769 984 self.db_logger.insert_test_entry(self.db_logger_build_id,
Nathan Yonkee 9:d58e77ebd769 985 target_name,
Nathan Yonkee 9:d58e77ebd769 986 toolchain_name,
Nathan Yonkee 9:d58e77ebd769 987 test_type,
Nathan Yonkee 9:d58e77ebd769 988 test_id,
Nathan Yonkee 9:d58e77ebd769 989 single_test_result,
Nathan Yonkee 9:d58e77ebd769 990 single_test_output,
Nathan Yonkee 9:d58e77ebd769 991 elapsed_time,
Nathan Yonkee 9:d58e77ebd769 992 single_timeout,
Nathan Yonkee 9:d58e77ebd769 993 test_index)
Nathan Yonkee 9:d58e77ebd769 994
Nathan Yonkee 9:d58e77ebd769 995 # If we perform waterfall test we test until we get OK and we stop testing
Nathan Yonkee 9:d58e77ebd769 996 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
Nathan Yonkee 9:d58e77ebd769 997 break
Nathan Yonkee 9:d58e77ebd769 998
Nathan Yonkee 9:d58e77ebd769 999 if self.db_logger:
Nathan Yonkee 9:d58e77ebd769 1000 self.db_logger.disconnect()
Nathan Yonkee 9:d58e77ebd769 1001
Nathan Yonkee 9:d58e77ebd769 1002 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
Nathan Yonkee 9:d58e77ebd769 1003 target_name_unique,
Nathan Yonkee 9:d58e77ebd769 1004 toolchain_name,
Nathan Yonkee 9:d58e77ebd769 1005 test_id,
Nathan Yonkee 9:d58e77ebd769 1006 test_description,
Nathan Yonkee 9:d58e77ebd769 1007 round(elapsed_time, 2),
Nathan Yonkee 9:d58e77ebd769 1008 single_timeout,
Nathan Yonkee 9:d58e77ebd769 1009 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
Nathan Yonkee 9:d58e77ebd769 1010
Nathan Yonkee 9:d58e77ebd769 1011 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
Nathan Yonkee 9:d58e77ebd769 1012 """ Function determines MUT's mbed disk/port and copies binary to
Nathan Yonkee 9:d58e77ebd769 1013 target.
Nathan Yonkee 9:d58e77ebd769 1014 """
Nathan Yonkee 9:d58e77ebd769 1015 handle_results = []
Nathan Yonkee 9:d58e77ebd769 1016 data = json.loads(test_spec)
Nathan Yonkee 9:d58e77ebd769 1017
Nathan Yonkee 9:d58e77ebd769 1018 # Find a suitable MUT:
Nathan Yonkee 9:d58e77ebd769 1019 mut = None
Nathan Yonkee 9:d58e77ebd769 1020 for id, m in self.muts.items():
Nathan Yonkee 9:d58e77ebd769 1021 if m['mcu'] == data['mcu']:
Nathan Yonkee 9:d58e77ebd769 1022 mut = m
Nathan Yonkee 9:d58e77ebd769 1023 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
Nathan Yonkee 9:d58e77ebd769 1024 handle_results.append(handle_result)
Nathan Yonkee 9:d58e77ebd769 1025
Nathan Yonkee 9:d58e77ebd769 1026 return handle_results
Nathan Yonkee 9:d58e77ebd769 1027
Nathan Yonkee 9:d58e77ebd769 1028 def print_test_result(self, test_result, target_name, toolchain_name,
Nathan Yonkee 9:d58e77ebd769 1029 test_id, test_description, elapsed_time, duration):
Nathan Yonkee 9:d58e77ebd769 1030 """ Use specific convention to print test result and related data
Nathan Yonkee 9:d58e77ebd769 1031 """
Nathan Yonkee 9:d58e77ebd769 1032 tokens = []
Nathan Yonkee 9:d58e77ebd769 1033 tokens.append("TargetTest")
Nathan Yonkee 9:d58e77ebd769 1034 tokens.append(target_name)
Nathan Yonkee 9:d58e77ebd769 1035 tokens.append(toolchain_name)
Nathan Yonkee 9:d58e77ebd769 1036 tokens.append(test_id)
Nathan Yonkee 9:d58e77ebd769 1037 tokens.append(test_description)
Nathan Yonkee 9:d58e77ebd769 1038 separator = "::"
Nathan Yonkee 9:d58e77ebd769 1039 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
Nathan Yonkee 9:d58e77ebd769 1040 result = separator.join(tokens) + " [" + test_result +"]" + time_info
Nathan Yonkee 9:d58e77ebd769 1041 return Fore.MAGENTA + result + Fore.RESET
Nathan Yonkee 9:d58e77ebd769 1042
Nathan Yonkee 9:d58e77ebd769 1043 def shape_test_loop_ok_result_count(self, test_all_result):
Nathan Yonkee 9:d58e77ebd769 1044 """ Reformats list of results to simple string
Nathan Yonkee 9:d58e77ebd769 1045 """
Nathan Yonkee 9:d58e77ebd769 1046 test_loop_count = len(test_all_result)
Nathan Yonkee 9:d58e77ebd769 1047 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
Nathan Yonkee 9:d58e77ebd769 1048 return "%d/%d"% (test_loop_ok_result, test_loop_count)
Nathan Yonkee 9:d58e77ebd769 1049
Nathan Yonkee 9:d58e77ebd769 1050 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
Nathan Yonkee 9:d58e77ebd769 1051 """ Reformats list of results to simple string
Nathan Yonkee 9:d58e77ebd769 1052 """
Nathan Yonkee 9:d58e77ebd769 1053 result = self.TEST_RESULT_FAIL
Nathan Yonkee 9:d58e77ebd769 1054
Nathan Yonkee 9:d58e77ebd769 1055 if all(test_all_result[0] == res for res in test_all_result):
Nathan Yonkee 9:d58e77ebd769 1056 result = test_all_result[0]
Nathan Yonkee 9:d58e77ebd769 1057 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
Nathan Yonkee 9:d58e77ebd769 1058 result = self.TEST_RESULT_OK
Nathan Yonkee 9:d58e77ebd769 1059
Nathan Yonkee 9:d58e77ebd769 1060 return result
Nathan Yonkee 9:d58e77ebd769 1061
Nathan Yonkee 9:d58e77ebd769 1062 def run_host_test(self, name, image_path, disk, port, duration,
Nathan Yonkee 9:d58e77ebd769 1063 micro=None, reset=None, reset_tout=None,
Nathan Yonkee 9:d58e77ebd769 1064 verbose=False, copy_method=None, program_cycle_s=None):
Nathan Yonkee 9:d58e77ebd769 1065 """ Function creates new process with host test configured with particular test case.
Nathan Yonkee 9:d58e77ebd769 1066 Function also is pooling for serial port activity from process to catch all data
Nathan Yonkee 9:d58e77ebd769 1067 printed by test runner and host test during test execution
Nathan Yonkee 9:d58e77ebd769 1068 """
Nathan Yonkee 9:d58e77ebd769 1069
Nathan Yonkee 9:d58e77ebd769 1070 def get_char_from_queue(obs):
Nathan Yonkee 9:d58e77ebd769 1071 """ Get character from queue safe way
Nathan Yonkee 9:d58e77ebd769 1072 """
Nathan Yonkee 9:d58e77ebd769 1073 try:
Nathan Yonkee 9:d58e77ebd769 1074 c = obs.queue.get(block=True, timeout=0.5)
Nathan Yonkee 9:d58e77ebd769 1075 except Empty:
Nathan Yonkee 9:d58e77ebd769 1076 c = None
Nathan Yonkee 9:d58e77ebd769 1077 return c
Nathan Yonkee 9:d58e77ebd769 1078
Nathan Yonkee 9:d58e77ebd769 1079 def filter_queue_char(c):
Nathan Yonkee 9:d58e77ebd769 1080 """ Filters out non ASCII characters from serial port
Nathan Yonkee 9:d58e77ebd769 1081 """
Nathan Yonkee 9:d58e77ebd769 1082 if ord(c) not in range(128):
Nathan Yonkee 9:d58e77ebd769 1083 c = ' '
Nathan Yonkee 9:d58e77ebd769 1084 return c
Nathan Yonkee 9:d58e77ebd769 1085
Nathan Yonkee 9:d58e77ebd769 1086 def get_test_result(output):
Nathan Yonkee 9:d58e77ebd769 1087 """ Parse test 'output' data
Nathan Yonkee 9:d58e77ebd769 1088 """
Nathan Yonkee 9:d58e77ebd769 1089 result = self.TEST_RESULT_TIMEOUT
Nathan Yonkee 9:d58e77ebd769 1090 for line in "".join(output).splitlines():
Nathan Yonkee 9:d58e77ebd769 1091 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
Nathan Yonkee 9:d58e77ebd769 1092 if search_result and len(search_result.groups()):
Nathan Yonkee 9:d58e77ebd769 1093 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
Nathan Yonkee 9:d58e77ebd769 1094 break
Nathan Yonkee 9:d58e77ebd769 1095 return result
Nathan Yonkee 9:d58e77ebd769 1096
Nathan Yonkee 9:d58e77ebd769 1097 def get_auto_property_value(property_name, line):
Nathan Yonkee 9:d58e77ebd769 1098 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
Nathan Yonkee 9:d58e77ebd769 1099 Returns string
Nathan Yonkee 9:d58e77ebd769 1100 """
Nathan Yonkee 9:d58e77ebd769 1101 result = None
Nathan Yonkee 9:d58e77ebd769 1102 if re.search("HOST: Property '%s'"% property_name, line) is not None:
Nathan Yonkee 9:d58e77ebd769 1103 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
Nathan Yonkee 9:d58e77ebd769 1104 if property is not None and len(property.groups()) == 1:
Nathan Yonkee 9:d58e77ebd769 1105 result = property.groups()[0]
Nathan Yonkee 9:d58e77ebd769 1106 return result
Nathan Yonkee 9:d58e77ebd769 1107
Nathan Yonkee 9:d58e77ebd769 1108 cmd = ["python",
Nathan Yonkee 9:d58e77ebd769 1109 '%s.py'% name,
Nathan Yonkee 9:d58e77ebd769 1110 '-d', disk,
Nathan Yonkee 9:d58e77ebd769 1111 '-f', '"%s"'% image_path,
Nathan Yonkee 9:d58e77ebd769 1112 '-p', port,
Nathan Yonkee 9:d58e77ebd769 1113 '-t', str(duration),
Nathan Yonkee 9:d58e77ebd769 1114 '-C', str(program_cycle_s)]
Nathan Yonkee 9:d58e77ebd769 1115
Nathan Yonkee 9:d58e77ebd769 1116 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
Nathan Yonkee 9:d58e77ebd769 1117 cmd += ['--auto']
Nathan Yonkee 9:d58e77ebd769 1118
Nathan Yonkee 9:d58e77ebd769 1119 # Add extra parameters to host_test
Nathan Yonkee 9:d58e77ebd769 1120 if copy_method is not None:
Nathan Yonkee 9:d58e77ebd769 1121 cmd += ["-c", copy_method]
Nathan Yonkee 9:d58e77ebd769 1122 if micro is not None:
Nathan Yonkee 9:d58e77ebd769 1123 cmd += ["-m", micro]
Nathan Yonkee 9:d58e77ebd769 1124 if reset is not None:
Nathan Yonkee 9:d58e77ebd769 1125 cmd += ["-r", reset]
Nathan Yonkee 9:d58e77ebd769 1126 if reset_tout is not None:
Nathan Yonkee 9:d58e77ebd769 1127 cmd += ["-R", str(reset_tout)]
Nathan Yonkee 9:d58e77ebd769 1128
Nathan Yonkee 9:d58e77ebd769 1129 if verbose:
Nathan Yonkee 9:d58e77ebd769 1130 print(Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET)
Nathan Yonkee 9:d58e77ebd769 1131 print("Test::Output::Start")
Nathan Yonkee 9:d58e77ebd769 1132
Nathan Yonkee 9:d58e77ebd769 1133 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
Nathan Yonkee 9:d58e77ebd769 1134 obs = ProcessObserver(proc)
Nathan Yonkee 9:d58e77ebd769 1135 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
Nathan Yonkee 9:d58e77ebd769 1136 line = ''
Nathan Yonkee 9:d58e77ebd769 1137 output = []
Nathan Yonkee 9:d58e77ebd769 1138 start_time = time()
Nathan Yonkee 9:d58e77ebd769 1139 while (time() - start_time) < (2 * duration):
Nathan Yonkee 9:d58e77ebd769 1140 c = get_char_from_queue(obs)
Nathan Yonkee 9:d58e77ebd769 1141 if c:
Nathan Yonkee 9:d58e77ebd769 1142 if verbose:
Nathan Yonkee 9:d58e77ebd769 1143 sys.stdout.write(c)
Nathan Yonkee 9:d58e77ebd769 1144 c = filter_queue_char(c)
Nathan Yonkee 9:d58e77ebd769 1145 output.append(c)
Nathan Yonkee 9:d58e77ebd769 1146 # Give the mbed under test a way to communicate the end of the test
Nathan Yonkee 9:d58e77ebd769 1147 if c in ['\n', '\r']:
Nathan Yonkee 9:d58e77ebd769 1148
Nathan Yonkee 9:d58e77ebd769 1149 # Checking for auto-detection information from the test about MUT reset moment
Nathan Yonkee 9:d58e77ebd769 1150 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
Nathan Yonkee 9:d58e77ebd769 1151 # We will update this marker only once to prevent multiple time resets
Nathan Yonkee 9:d58e77ebd769 1152 update_once_flag['reset_target'] = True
Nathan Yonkee 9:d58e77ebd769 1153 start_time = time()
Nathan Yonkee 9:d58e77ebd769 1154
Nathan Yonkee 9:d58e77ebd769 1155 # Checking for auto-detection information from the test about timeout
Nathan Yonkee 9:d58e77ebd769 1156 auto_timeout_val = get_auto_property_value('timeout', line)
Nathan Yonkee 9:d58e77ebd769 1157 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
Nathan Yonkee 9:d58e77ebd769 1158 # We will update this marker only once to prevent multiple time resets
Nathan Yonkee 9:d58e77ebd769 1159 update_once_flag['timeout'] = True
Nathan Yonkee 9:d58e77ebd769 1160 duration = int(auto_timeout_val)
Nathan Yonkee 9:d58e77ebd769 1161
Nathan Yonkee 9:d58e77ebd769 1162 # Detect mbed assert:
Nathan Yonkee 9:d58e77ebd769 1163 if 'mbed assertation failed: ' in line:
Nathan Yonkee 9:d58e77ebd769 1164 output.append('{{mbed_assert}}')
Nathan Yonkee 9:d58e77ebd769 1165 break
Nathan Yonkee 9:d58e77ebd769 1166
Nathan Yonkee 9:d58e77ebd769 1167 # Check for test end
Nathan Yonkee 9:d58e77ebd769 1168 if '{end}' in line:
Nathan Yonkee 9:d58e77ebd769 1169 break
Nathan Yonkee 9:d58e77ebd769 1170 line = ''
Nathan Yonkee 9:d58e77ebd769 1171 else:
Nathan Yonkee 9:d58e77ebd769 1172 line += c
Nathan Yonkee 9:d58e77ebd769 1173 end_time = time()
Nathan Yonkee 9:d58e77ebd769 1174 testcase_duration = end_time - start_time # Test case duration from reset to {end}
Nathan Yonkee 9:d58e77ebd769 1175
Nathan Yonkee 9:d58e77ebd769 1176 c = get_char_from_queue(obs)
Nathan Yonkee 9:d58e77ebd769 1177
Nathan Yonkee 9:d58e77ebd769 1178 if c:
Nathan Yonkee 9:d58e77ebd769 1179 if verbose:
Nathan Yonkee 9:d58e77ebd769 1180 sys.stdout.write(c)
Nathan Yonkee 9:d58e77ebd769 1181 c = filter_queue_char(c)
Nathan Yonkee 9:d58e77ebd769 1182 output.append(c)
Nathan Yonkee 9:d58e77ebd769 1183
Nathan Yonkee 9:d58e77ebd769 1184 if verbose:
Nathan Yonkee 9:d58e77ebd769 1185 print("Test::Output::Finish")
Nathan Yonkee 9:d58e77ebd769 1186 # Stop test process
Nathan Yonkee 9:d58e77ebd769 1187 obs.stop()
Nathan Yonkee 9:d58e77ebd769 1188
Nathan Yonkee 9:d58e77ebd769 1189 result = get_test_result(output)
Nathan Yonkee 9:d58e77ebd769 1190 return (result, "".join(output), testcase_duration, duration)
Nathan Yonkee 9:d58e77ebd769 1191
Nathan Yonkee 9:d58e77ebd769 1192 def is_peripherals_available(self, target_mcu_name, peripherals=None):
Nathan Yonkee 9:d58e77ebd769 1193 """ Checks if specified target should run specific peripheral test case defined in MUTs file
Nathan Yonkee 9:d58e77ebd769 1194 """
Nathan Yonkee 9:d58e77ebd769 1195 if peripherals is not None:
Nathan Yonkee 9:d58e77ebd769 1196 peripherals = set(peripherals)
Nathan Yonkee 9:d58e77ebd769 1197 for id, mut in self.muts.items():
Nathan Yonkee 9:d58e77ebd769 1198 # Target MCU name check
Nathan Yonkee 9:d58e77ebd769 1199 if mut["mcu"] != target_mcu_name:
Nathan Yonkee 9:d58e77ebd769 1200 continue
Nathan Yonkee 9:d58e77ebd769 1201 # Peripherals check
Nathan Yonkee 9:d58e77ebd769 1202 if peripherals is not None:
Nathan Yonkee 9:d58e77ebd769 1203 if 'peripherals' not in mut:
Nathan Yonkee 9:d58e77ebd769 1204 continue
Nathan Yonkee 9:d58e77ebd769 1205 if not peripherals.issubset(set(mut['peripherals'])):
Nathan Yonkee 9:d58e77ebd769 1206 continue
Nathan Yonkee 9:d58e77ebd769 1207 return True
Nathan Yonkee 9:d58e77ebd769 1208 return False
Nathan Yonkee 9:d58e77ebd769 1209
Nathan Yonkee 9:d58e77ebd769 1210 def shape_test_request(self, mcu, image_path, test_id, duration=10):
Nathan Yonkee 9:d58e77ebd769 1211 """ Function prepares JSON structure describing test specification
Nathan Yonkee 9:d58e77ebd769 1212 """
Nathan Yonkee 9:d58e77ebd769 1213 test_spec = {
Nathan Yonkee 9:d58e77ebd769 1214 "mcu": mcu,
Nathan Yonkee 9:d58e77ebd769 1215 "image": image_path,
Nathan Yonkee 9:d58e77ebd769 1216 "duration": duration,
Nathan Yonkee 9:d58e77ebd769 1217 "test_id": test_id,
Nathan Yonkee 9:d58e77ebd769 1218 }
Nathan Yonkee 9:d58e77ebd769 1219 return json.dumps(test_spec)
Nathan Yonkee 9:d58e77ebd769 1220
Nathan Yonkee 9:d58e77ebd769 1221
Nathan Yonkee 9:d58e77ebd769 1222 def get_unique_value_from_summary(test_summary, index):
Nathan Yonkee 9:d58e77ebd769 1223 """ Gets list of unique target names
Nathan Yonkee 9:d58e77ebd769 1224 """
Nathan Yonkee 9:d58e77ebd769 1225 result = []
Nathan Yonkee 9:d58e77ebd769 1226 for test in test_summary:
Nathan Yonkee 9:d58e77ebd769 1227 target_name = test[index]
Nathan Yonkee 9:d58e77ebd769 1228 if target_name not in result:
Nathan Yonkee 9:d58e77ebd769 1229 result.append(target_name)
Nathan Yonkee 9:d58e77ebd769 1230 return sorted(result)
Nathan Yonkee 9:d58e77ebd769 1231
Nathan Yonkee 9:d58e77ebd769 1232
Nathan Yonkee 9:d58e77ebd769 1233 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
Nathan Yonkee 9:d58e77ebd769 1234 """ Gets list of unique target names and return dictionary
Nathan Yonkee 9:d58e77ebd769 1235 """
Nathan Yonkee 9:d58e77ebd769 1236 result = {}
Nathan Yonkee 9:d58e77ebd769 1237 for test in test_summary:
Nathan Yonkee 9:d58e77ebd769 1238 key = test[index_key]
Nathan Yonkee 9:d58e77ebd769 1239 val = test[index_val]
Nathan Yonkee 9:d58e77ebd769 1240 if key not in result:
Nathan Yonkee 9:d58e77ebd769 1241 result[key] = val
Nathan Yonkee 9:d58e77ebd769 1242 return result
Nathan Yonkee 9:d58e77ebd769 1243
Nathan Yonkee 9:d58e77ebd769 1244
Nathan Yonkee 9:d58e77ebd769 1245 def show_json_file_format_error(json_spec_filename, line, column):
Nathan Yonkee 9:d58e77ebd769 1246 """ Prints JSON broken content
Nathan Yonkee 9:d58e77ebd769 1247 """
Nathan Yonkee 9:d58e77ebd769 1248 with open(json_spec_filename) as data_file:
Nathan Yonkee 9:d58e77ebd769 1249 line_no = 1
Nathan Yonkee 9:d58e77ebd769 1250 for json_line in data_file:
Nathan Yonkee 9:d58e77ebd769 1251 if line_no + 5 >= line: # Print last few lines before error
Nathan Yonkee 9:d58e77ebd769 1252 print('Line %d:\t'%line_no + json_line)
Nathan Yonkee 9:d58e77ebd769 1253 if line_no == line:
Nathan Yonkee 9:d58e77ebd769 1254 print('%s\t%s^' (' ' * len('Line %d:' % line_no),
Nathan Yonkee 9:d58e77ebd769 1255 '-' * (column - 1)))
Nathan Yonkee 9:d58e77ebd769 1256 break
Nathan Yonkee 9:d58e77ebd769 1257 line_no += 1
Nathan Yonkee 9:d58e77ebd769 1258
Nathan Yonkee 9:d58e77ebd769 1259
Nathan Yonkee 9:d58e77ebd769 1260 def json_format_error_defect_pos(json_error_msg):
Nathan Yonkee 9:d58e77ebd769 1261 """ Gets first error line and column in JSON file format.
Nathan Yonkee 9:d58e77ebd769 1262 Parsed from exception thrown by json.loads() string
Nathan Yonkee 9:d58e77ebd769 1263 """
Nathan Yonkee 9:d58e77ebd769 1264 result = None
Nathan Yonkee 9:d58e77ebd769 1265 line, column = 0, 0
Nathan Yonkee 9:d58e77ebd769 1266 # Line value search
Nathan Yonkee 9:d58e77ebd769 1267 line_search = re.search('line [0-9]+', json_error_msg)
Nathan Yonkee 9:d58e77ebd769 1268 if line_search is not None:
Nathan Yonkee 9:d58e77ebd769 1269 ls = line_search.group().split(' ')
Nathan Yonkee 9:d58e77ebd769 1270 if len(ls) == 2:
Nathan Yonkee 9:d58e77ebd769 1271 line = int(ls[1])
Nathan Yonkee 9:d58e77ebd769 1272 # Column position search
Nathan Yonkee 9:d58e77ebd769 1273 column_search = re.search('column [0-9]+', json_error_msg)
Nathan Yonkee 9:d58e77ebd769 1274 if column_search is not None:
Nathan Yonkee 9:d58e77ebd769 1275 cs = column_search.group().split(' ')
Nathan Yonkee 9:d58e77ebd769 1276 if len(cs) == 2:
Nathan Yonkee 9:d58e77ebd769 1277 column = int(cs[1])
Nathan Yonkee 9:d58e77ebd769 1278 result = [line, column]
Nathan Yonkee 9:d58e77ebd769 1279 return result
Nathan Yonkee 9:d58e77ebd769 1280
Nathan Yonkee 9:d58e77ebd769 1281
Nathan Yonkee 9:d58e77ebd769 1282 def get_json_data_from_file(json_spec_filename, verbose=False):
Nathan Yonkee 9:d58e77ebd769 1283 """ Loads from file JSON formatted string to data structure
Nathan Yonkee 9:d58e77ebd769 1284 """
Nathan Yonkee 9:d58e77ebd769 1285 result = None
Nathan Yonkee 9:d58e77ebd769 1286 try:
Nathan Yonkee 9:d58e77ebd769 1287 with open(json_spec_filename) as data_file:
Nathan Yonkee 9:d58e77ebd769 1288 try:
Nathan Yonkee 9:d58e77ebd769 1289 result = json.load(data_file)
Nathan Yonkee 9:d58e77ebd769 1290 except ValueError as json_error_msg:
Nathan Yonkee 9:d58e77ebd769 1291 result = None
Nathan Yonkee 9:d58e77ebd769 1292 print('JSON file %s parsing failed. Reason: %s' %
Nathan Yonkee 9:d58e77ebd769 1293 (json_spec_filename, json_error_msg))
Nathan Yonkee 9:d58e77ebd769 1294 # We can print where error occurred inside JSON file if we can parse exception msg
Nathan Yonkee 9:d58e77ebd769 1295 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
Nathan Yonkee 9:d58e77ebd769 1296 if json_format_defect_pos is not None:
Nathan Yonkee 9:d58e77ebd769 1297 line = json_format_defect_pos[0]
Nathan Yonkee 9:d58e77ebd769 1298 column = json_format_defect_pos[1]
Nathan Yonkee 9:d58e77ebd769 1299 print()
Nathan Yonkee 9:d58e77ebd769 1300 show_json_file_format_error(json_spec_filename, line, column)
Nathan Yonkee 9:d58e77ebd769 1301
Nathan Yonkee 9:d58e77ebd769 1302 except IOError as fileopen_error_msg:
Nathan Yonkee 9:d58e77ebd769 1303 print('JSON file %s not opened. Reason: %s\n'%
Nathan Yonkee 9:d58e77ebd769 1304 (json_spec_filename, fileopen_error_msg))
Nathan Yonkee 9:d58e77ebd769 1305 if verbose and result:
Nathan Yonkee 9:d58e77ebd769 1306 pp = pprint.PrettyPrinter(indent=4)
Nathan Yonkee 9:d58e77ebd769 1307 pp.pprint(result)
Nathan Yonkee 9:d58e77ebd769 1308 return result
Nathan Yonkee 9:d58e77ebd769 1309
Nathan Yonkee 9:d58e77ebd769 1310
Nathan Yonkee 9:d58e77ebd769 1311 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
Nathan Yonkee 9:d58e77ebd769 1312 """ Prints MUTs configuration passed to test script for verboseness
Nathan Yonkee 9:d58e77ebd769 1313 """
Nathan Yonkee 9:d58e77ebd769 1314 muts_info_cols = []
Nathan Yonkee 9:d58e77ebd769 1315 # We need to check all unique properties for each defined MUT
Nathan Yonkee 9:d58e77ebd769 1316 for k in json_data:
Nathan Yonkee 9:d58e77ebd769 1317 mut_info = json_data[k]
Nathan Yonkee 9:d58e77ebd769 1318 for mut_property in mut_info:
Nathan Yonkee 9:d58e77ebd769 1319 if mut_property not in muts_info_cols:
Nathan Yonkee 9:d58e77ebd769 1320 muts_info_cols.append(mut_property)
Nathan Yonkee 9:d58e77ebd769 1321
Nathan Yonkee 9:d58e77ebd769 1322 # Prepare pretty table object to display all MUTs
Nathan Yonkee 9:d58e77ebd769 1323 pt_cols = ["index"] + muts_info_cols
Nathan Yonkee 9:d58e77ebd769 1324 pt = PrettyTable(pt_cols)
Nathan Yonkee 9:d58e77ebd769 1325 for col in pt_cols:
Nathan Yonkee 9:d58e77ebd769 1326 pt.align[col] = "l"
Nathan Yonkee 9:d58e77ebd769 1327
Nathan Yonkee 9:d58e77ebd769 1328 # Add rows to pretty print object
Nathan Yonkee 9:d58e77ebd769 1329 for k in json_data:
Nathan Yonkee 9:d58e77ebd769 1330 row = [k]
Nathan Yonkee 9:d58e77ebd769 1331 mut_info = json_data[k]
Nathan Yonkee 9:d58e77ebd769 1332
Nathan Yonkee 9:d58e77ebd769 1333 add_row = True
Nathan Yonkee 9:d58e77ebd769 1334 if platform_filter and 'mcu' in mut_info:
Nathan Yonkee 9:d58e77ebd769 1335 add_row = re.search(platform_filter, mut_info['mcu']) is not None
Nathan Yonkee 9:d58e77ebd769 1336 if add_row:
Nathan Yonkee 9:d58e77ebd769 1337 for col in muts_info_cols:
Nathan Yonkee 9:d58e77ebd769 1338 cell_val = mut_info[col] if col in mut_info else None
Nathan Yonkee 9:d58e77ebd769 1339 if isinstance(cell_val, list):
Nathan Yonkee 9:d58e77ebd769 1340 cell_val = join_delim.join(cell_val)
Nathan Yonkee 9:d58e77ebd769 1341 row.append(cell_val)
Nathan Yonkee 9:d58e77ebd769 1342 pt.add_row(row)
Nathan Yonkee 9:d58e77ebd769 1343 return pt.get_string()
Nathan Yonkee 9:d58e77ebd769 1344
Nathan Yonkee 9:d58e77ebd769 1345
Nathan Yonkee 9:d58e77ebd769 1346 def print_test_configuration_from_json(json_data, join_delim=", "):
Nathan Yonkee 9:d58e77ebd769 1347 """ Prints test specification configuration passed to test script for verboseness
Nathan Yonkee 9:d58e77ebd769 1348 """
Nathan Yonkee 9:d58e77ebd769 1349 toolchains_info_cols = []
Nathan Yonkee 9:d58e77ebd769 1350 # We need to check all toolchains for each device
Nathan Yonkee 9:d58e77ebd769 1351 for k in json_data:
Nathan Yonkee 9:d58e77ebd769 1352 # k should be 'targets'
Nathan Yonkee 9:d58e77ebd769 1353 targets = json_data[k]
Nathan Yonkee 9:d58e77ebd769 1354 for target in targets:
Nathan Yonkee 9:d58e77ebd769 1355 toolchains = targets[target]
Nathan Yonkee 9:d58e77ebd769 1356 for toolchain in toolchains:
Nathan Yonkee 9:d58e77ebd769 1357 if toolchain not in toolchains_info_cols:
Nathan Yonkee 9:d58e77ebd769 1358 toolchains_info_cols.append(toolchain)
Nathan Yonkee 9:d58e77ebd769 1359
Nathan Yonkee 9:d58e77ebd769 1360 # Prepare pretty table object to display test specification
Nathan Yonkee 9:d58e77ebd769 1361 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
Nathan Yonkee 9:d58e77ebd769 1362 pt = PrettyTable(pt_cols)
Nathan Yonkee 9:d58e77ebd769 1363 for col in pt_cols:
Nathan Yonkee 9:d58e77ebd769 1364 pt.align[col] = "l"
Nathan Yonkee 9:d58e77ebd769 1365
Nathan Yonkee 9:d58e77ebd769 1366 # { target : [conflicted toolchains] }
Nathan Yonkee 9:d58e77ebd769 1367 toolchain_conflicts = {}
Nathan Yonkee 9:d58e77ebd769 1368 toolchain_path_conflicts = []
Nathan Yonkee 9:d58e77ebd769 1369 for k in json_data:
Nathan Yonkee 9:d58e77ebd769 1370 # k should be 'targets'
Nathan Yonkee 9:d58e77ebd769 1371 targets = json_data[k]
Nathan Yonkee 9:d58e77ebd769 1372 for target in targets:
Nathan Yonkee 9:d58e77ebd769 1373 target_supported_toolchains = get_target_supported_toolchains(target)
Nathan Yonkee 9:d58e77ebd769 1374 if not target_supported_toolchains:
Nathan Yonkee 9:d58e77ebd769 1375 target_supported_toolchains = []
Nathan Yonkee 9:d58e77ebd769 1376 target_name = target if target in TARGET_MAP else "%s*"% target
Nathan Yonkee 9:d58e77ebd769 1377 row = [target_name]
Nathan Yonkee 9:d58e77ebd769 1378 toolchains = targets[target]
Nathan Yonkee 9:d58e77ebd769 1379
Nathan Yonkee 9:d58e77ebd769 1380 for toolchain in sorted(toolchains_info_cols):
Nathan Yonkee 9:d58e77ebd769 1381 # Check for conflicts: target vs toolchain
Nathan Yonkee 9:d58e77ebd769 1382 conflict = False
Nathan Yonkee 9:d58e77ebd769 1383 conflict_path = False
Nathan Yonkee 9:d58e77ebd769 1384 if toolchain in toolchains:
Nathan Yonkee 9:d58e77ebd769 1385 if toolchain not in target_supported_toolchains:
Nathan Yonkee 9:d58e77ebd769 1386 conflict = True
Nathan Yonkee 9:d58e77ebd769 1387 if target not in toolchain_conflicts:
Nathan Yonkee 9:d58e77ebd769 1388 toolchain_conflicts[target] = []
Nathan Yonkee 9:d58e77ebd769 1389 toolchain_conflicts[target].append(toolchain)
Nathan Yonkee 9:d58e77ebd769 1390 # Add marker inside table about target usage / conflict
Nathan Yonkee 9:d58e77ebd769 1391 cell_val = 'Yes' if toolchain in toolchains else '-'
Nathan Yonkee 9:d58e77ebd769 1392 if conflict:
Nathan Yonkee 9:d58e77ebd769 1393 cell_val += '*'
Nathan Yonkee 9:d58e77ebd769 1394 # Check for conflicts: toolchain vs toolchain path
Nathan Yonkee 9:d58e77ebd769 1395 if toolchain in TOOLCHAIN_PATHS:
Nathan Yonkee 9:d58e77ebd769 1396 toolchain_path = TOOLCHAIN_PATHS[toolchain]
Nathan Yonkee 9:d58e77ebd769 1397 if not os.path.isdir(toolchain_path):
Nathan Yonkee 9:d58e77ebd769 1398 conflict_path = True
Nathan Yonkee 9:d58e77ebd769 1399 if toolchain not in toolchain_path_conflicts:
Nathan Yonkee 9:d58e77ebd769 1400 toolchain_path_conflicts.append(toolchain)
Nathan Yonkee 9:d58e77ebd769 1401 if conflict_path:
Nathan Yonkee 9:d58e77ebd769 1402 cell_val += '#'
Nathan Yonkee 9:d58e77ebd769 1403 row.append(cell_val)
Nathan Yonkee 9:d58e77ebd769 1404 pt.add_row(row)
Nathan Yonkee 9:d58e77ebd769 1405
Nathan Yonkee 9:d58e77ebd769 1406 # generate result string
Nathan Yonkee 9:d58e77ebd769 1407 result = pt.get_string() # Test specification table
Nathan Yonkee 9:d58e77ebd769 1408 if toolchain_conflicts or toolchain_path_conflicts:
Nathan Yonkee 9:d58e77ebd769 1409 result += "\n"
Nathan Yonkee 9:d58e77ebd769 1410 result += "Toolchain conflicts:\n"
Nathan Yonkee 9:d58e77ebd769 1411 for target in toolchain_conflicts:
Nathan Yonkee 9:d58e77ebd769 1412 if target not in TARGET_MAP:
Nathan Yonkee 9:d58e77ebd769 1413 result += "\t* Target %s unknown\n"% (target)
Nathan Yonkee 9:d58e77ebd769 1414 conflict_target_list = join_delim.join(toolchain_conflicts[target])
Nathan Yonkee 9:d58e77ebd769 1415 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
Nathan Yonkee 9:d58e77ebd769 1416 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
Nathan Yonkee 9:d58e77ebd769 1417
Nathan Yonkee 9:d58e77ebd769 1418 for toolchain in toolchain_path_conflicts:
Nathan Yonkee 9:d58e77ebd769 1419 # Let's check toolchain configuration
Nathan Yonkee 9:d58e77ebd769 1420 if toolchain in TOOLCHAIN_PATHS:
Nathan Yonkee 9:d58e77ebd769 1421 toolchain_path = TOOLCHAIN_PATHS[toolchain]
Nathan Yonkee 9:d58e77ebd769 1422 if not os.path.isdir(toolchain_path):
Nathan Yonkee 9:d58e77ebd769 1423 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
Nathan Yonkee 9:d58e77ebd769 1424 return result
Nathan Yonkee 9:d58e77ebd769 1425
Nathan Yonkee 9:d58e77ebd769 1426
Nathan Yonkee 9:d58e77ebd769 1427 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
Nathan Yonkee 9:d58e77ebd769 1428 """ Generates table summary with all test cases and additional test cases
Nathan Yonkee 9:d58e77ebd769 1429 information using pretty print functionality. Allows test suite user to
Nathan Yonkee 9:d58e77ebd769 1430 see test cases
Nathan Yonkee 9:d58e77ebd769 1431 """
Nathan Yonkee 9:d58e77ebd769 1432 # get all unique test ID prefixes
Nathan Yonkee 9:d58e77ebd769 1433 unique_test_id = []
Nathan Yonkee 9:d58e77ebd769 1434 for test in TESTS:
Nathan Yonkee 9:d58e77ebd769 1435 split = test['id'].split('_')[:-1]
Nathan Yonkee 9:d58e77ebd769 1436 test_id_prefix = '_'.join(split)
Nathan Yonkee 9:d58e77ebd769 1437 if test_id_prefix not in unique_test_id:
Nathan Yonkee 9:d58e77ebd769 1438 unique_test_id.append(test_id_prefix)
Nathan Yonkee 9:d58e77ebd769 1439 unique_test_id.sort()
Nathan Yonkee 9:d58e77ebd769 1440 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
Nathan Yonkee 9:d58e77ebd769 1441 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
Nathan Yonkee 9:d58e77ebd769 1442
Nathan Yonkee 9:d58e77ebd769 1443 test_properties = ['id',
Nathan Yonkee 9:d58e77ebd769 1444 'automated',
Nathan Yonkee 9:d58e77ebd769 1445 'description',
Nathan Yonkee 9:d58e77ebd769 1446 'peripherals',
Nathan Yonkee 9:d58e77ebd769 1447 'host_test',
Nathan Yonkee 9:d58e77ebd769 1448 'duration'] if cols is None else cols
Nathan Yonkee 9:d58e77ebd769 1449
Nathan Yonkee 9:d58e77ebd769 1450 # All tests status table print
Nathan Yonkee 9:d58e77ebd769 1451 pt = PrettyTable(test_properties)
Nathan Yonkee 9:d58e77ebd769 1452 for col in test_properties:
Nathan Yonkee 9:d58e77ebd769 1453 pt.align[col] = "l"
Nathan Yonkee 9:d58e77ebd769 1454 pt.align['duration'] = "r"
Nathan Yonkee 9:d58e77ebd769 1455
Nathan Yonkee 9:d58e77ebd769 1456 counter_all = 0
Nathan Yonkee 9:d58e77ebd769 1457 counter_automated = 0
Nathan Yonkee 9:d58e77ebd769 1458 pt.padding_width = 1 # One space between column edges and contents (default)
Nathan Yonkee 9:d58e77ebd769 1459
Nathan Yonkee 9:d58e77ebd769 1460 for test_id in sorted(TEST_MAP.keys()):
Nathan Yonkee 9:d58e77ebd769 1461 if platform_filter is not None:
Nathan Yonkee 9:d58e77ebd769 1462 # FIlter out platforms using regex
Nathan Yonkee 9:d58e77ebd769 1463 if re.search(platform_filter, test_id) is None:
Nathan Yonkee 9:d58e77ebd769 1464 continue
Nathan Yonkee 9:d58e77ebd769 1465 row = []
Nathan Yonkee 9:d58e77ebd769 1466 test = TEST_MAP[test_id]
Nathan Yonkee 9:d58e77ebd769 1467 split = test_id.split('_')[:-1]
Nathan Yonkee 9:d58e77ebd769 1468 test_id_prefix = '_'.join(split)
Nathan Yonkee 9:d58e77ebd769 1469
Nathan Yonkee 9:d58e77ebd769 1470 for col in test_properties:
Nathan Yonkee 9:d58e77ebd769 1471 col_value = test[col]
Nathan Yonkee 9:d58e77ebd769 1472 if isinstance(test[col], list):
Nathan Yonkee 9:d58e77ebd769 1473 col_value = join_delim.join(test[col])
Nathan Yonkee 9:d58e77ebd769 1474 elif test[col] == None:
Nathan Yonkee 9:d58e77ebd769 1475 col_value = "-"
Nathan Yonkee 9:d58e77ebd769 1476
Nathan Yonkee 9:d58e77ebd769 1477 row.append(col_value)
Nathan Yonkee 9:d58e77ebd769 1478 if test['automated'] == True:
Nathan Yonkee 9:d58e77ebd769 1479 counter_dict_test_id_types[test_id_prefix] += 1
Nathan Yonkee 9:d58e77ebd769 1480 counter_automated += 1
Nathan Yonkee 9:d58e77ebd769 1481 pt.add_row(row)
Nathan Yonkee 9:d58e77ebd769 1482 # Update counters
Nathan Yonkee 9:d58e77ebd769 1483 counter_all += 1
Nathan Yonkee 9:d58e77ebd769 1484 counter_dict_test_id_types_all[test_id_prefix] += 1
Nathan Yonkee 9:d58e77ebd769 1485 result = pt.get_string()
Nathan Yonkee 9:d58e77ebd769 1486 result += "\n\n"
Nathan Yonkee 9:d58e77ebd769 1487
Nathan Yonkee 9:d58e77ebd769 1488 if result_summary and not platform_filter:
Nathan Yonkee 9:d58e77ebd769 1489 # Automation result summary
Nathan Yonkee 9:d58e77ebd769 1490 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
Nathan Yonkee 9:d58e77ebd769 1491 pt = PrettyTable(test_id_cols)
Nathan Yonkee 9:d58e77ebd769 1492 pt.align['automated'] = "r"
Nathan Yonkee 9:d58e77ebd769 1493 pt.align['all'] = "r"
Nathan Yonkee 9:d58e77ebd769 1494 pt.align['percent [%]'] = "r"
Nathan Yonkee 9:d58e77ebd769 1495
Nathan Yonkee 9:d58e77ebd769 1496 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
Nathan Yonkee 9:d58e77ebd769 1497 str_progress = progress_bar(percent_progress, 75)
Nathan Yonkee 9:d58e77ebd769 1498 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
Nathan Yonkee 9:d58e77ebd769 1499 result += "Automation coverage:\n"
Nathan Yonkee 9:d58e77ebd769 1500 result += pt.get_string()
Nathan Yonkee 9:d58e77ebd769 1501 result += "\n\n"
Nathan Yonkee 9:d58e77ebd769 1502
Nathan Yonkee 9:d58e77ebd769 1503 # Test automation coverage table print
Nathan Yonkee 9:d58e77ebd769 1504 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
Nathan Yonkee 9:d58e77ebd769 1505 pt = PrettyTable(test_id_cols)
Nathan Yonkee 9:d58e77ebd769 1506 pt.align['id'] = "l"
Nathan Yonkee 9:d58e77ebd769 1507 pt.align['automated'] = "r"
Nathan Yonkee 9:d58e77ebd769 1508 pt.align['all'] = "r"
Nathan Yonkee 9:d58e77ebd769 1509 pt.align['percent [%]'] = "r"
Nathan Yonkee 9:d58e77ebd769 1510 for unique_id in unique_test_id:
Nathan Yonkee 9:d58e77ebd769 1511 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
Nathan Yonkee 9:d58e77ebd769 1512 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
Nathan Yonkee 9:d58e77ebd769 1513 str_progress = progress_bar(percent_progress, 75)
Nathan Yonkee 9:d58e77ebd769 1514 row = [unique_id,
Nathan Yonkee 9:d58e77ebd769 1515 counter_dict_test_id_types[unique_id],
Nathan Yonkee 9:d58e77ebd769 1516 counter_dict_test_id_types_all[unique_id],
Nathan Yonkee 9:d58e77ebd769 1517 percent_progress,
Nathan Yonkee 9:d58e77ebd769 1518 "[" + str_progress + "]"]
Nathan Yonkee 9:d58e77ebd769 1519 pt.add_row(row)
Nathan Yonkee 9:d58e77ebd769 1520 result += "Test automation coverage:\n"
Nathan Yonkee 9:d58e77ebd769 1521 result += pt.get_string()
Nathan Yonkee 9:d58e77ebd769 1522 result += "\n\n"
Nathan Yonkee 9:d58e77ebd769 1523 return result
Nathan Yonkee 9:d58e77ebd769 1524
Nathan Yonkee 9:d58e77ebd769 1525
Nathan Yonkee 9:d58e77ebd769 1526 def progress_bar(percent_progress, saturation=0):
Nathan Yonkee 9:d58e77ebd769 1527 """ This function creates progress bar with optional simple saturation mark
Nathan Yonkee 9:d58e77ebd769 1528 """
Nathan Yonkee 9:d58e77ebd769 1529 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
Nathan Yonkee 9:d58e77ebd769 1530 str_progress = '#' * step + '.' * int(50 - step)
Nathan Yonkee 9:d58e77ebd769 1531 c = '!' if str_progress[38] == '.' else '|'
Nathan Yonkee 9:d58e77ebd769 1532 if saturation > 0:
Nathan Yonkee 9:d58e77ebd769 1533 saturation = saturation / 2
Nathan Yonkee 9:d58e77ebd769 1534 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
Nathan Yonkee 9:d58e77ebd769 1535 return str_progress
Nathan Yonkee 9:d58e77ebd769 1536
Nathan Yonkee 9:d58e77ebd769 1537
Nathan Yonkee 9:d58e77ebd769 1538 def singletest_in_cli_mode(single_test):
Nathan Yonkee 9:d58e77ebd769 1539 """ Runs SingleTestRunner object in CLI (Command line interface) mode
Nathan Yonkee 9:d58e77ebd769 1540
Nathan Yonkee 9:d58e77ebd769 1541 @return returns success code (0 == success) for building and running tests
Nathan Yonkee 9:d58e77ebd769 1542 """
Nathan Yonkee 9:d58e77ebd769 1543 start = time()
Nathan Yonkee 9:d58e77ebd769 1544 # Execute tests depending on options and filter applied
Nathan Yonkee 9:d58e77ebd769 1545 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
Nathan Yonkee 9:d58e77ebd769 1546 elapsed_time = time() - start
Nathan Yonkee 9:d58e77ebd769 1547
Nathan Yonkee 9:d58e77ebd769 1548 # Human readable summary
Nathan Yonkee 9:d58e77ebd769 1549 if not single_test.opts_suppress_summary:
Nathan Yonkee 9:d58e77ebd769 1550 # prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 1551 print(single_test.generate_test_summary(test_summary, shuffle_seed))
Nathan Yonkee 9:d58e77ebd769 1552 if single_test.opts_test_x_toolchain_summary:
Nathan Yonkee 9:d58e77ebd769 1553 # prints well-formed summary with results (SQL table like)
Nathan Yonkee 9:d58e77ebd769 1554 # table shows text x toolchain test result matrix
Nathan Yonkee 9:d58e77ebd769 1555 print(single_test.generate_test_summary_by_target(test_summary,
Nathan Yonkee 9:d58e77ebd769 1556 shuffle_seed))
Nathan Yonkee 9:d58e77ebd769 1557
Nathan Yonkee 9:d58e77ebd769 1558 print("Completed in %.2f sec" % elapsed_time)
Nathan Yonkee 9:d58e77ebd769 1559 print
Nathan Yonkee 9:d58e77ebd769 1560 # Write summary of the builds
Nathan Yonkee 9:d58e77ebd769 1561
Nathan Yonkee 9:d58e77ebd769 1562 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
Nathan Yonkee 9:d58e77ebd769 1563 status = print_report_exporter.report(build_report)
Nathan Yonkee 9:d58e77ebd769 1564
Nathan Yonkee 9:d58e77ebd769 1565 # Store extra reports in files
Nathan Yonkee 9:d58e77ebd769 1566 if single_test.opts_report_html_file_name:
Nathan Yonkee 9:d58e77ebd769 1567 # Export results in form of HTML report to separate file
Nathan Yonkee 9:d58e77ebd769 1568 report_exporter = ReportExporter(ResultExporterType.HTML)
Nathan Yonkee 9:d58e77ebd769 1569 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
Nathan Yonkee 9:d58e77ebd769 1570 if single_test.opts_report_junit_file_name:
Nathan Yonkee 9:d58e77ebd769 1571 # Export results in form of JUnit XML report to separate file
Nathan Yonkee 9:d58e77ebd769 1572 report_exporter = ReportExporter(ResultExporterType.JUNIT)
Nathan Yonkee 9:d58e77ebd769 1573 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
Nathan Yonkee 9:d58e77ebd769 1574 if single_test.opts_report_text_file_name:
Nathan Yonkee 9:d58e77ebd769 1575 # Export results in form of a text file
Nathan Yonkee 9:d58e77ebd769 1576 report_exporter = ReportExporter(ResultExporterType.TEXT)
Nathan Yonkee 9:d58e77ebd769 1577 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
Nathan Yonkee 9:d58e77ebd769 1578 if single_test.opts_report_build_file_name:
Nathan Yonkee 9:d58e77ebd769 1579 # Export build results as html report to sparate file
Nathan Yonkee 9:d58e77ebd769 1580 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
Nathan Yonkee 9:d58e77ebd769 1581 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
Nathan Yonkee 9:d58e77ebd769 1582
Nathan Yonkee 9:d58e77ebd769 1583 # Returns True if no build failures of the test projects or their dependencies
Nathan Yonkee 9:d58e77ebd769 1584 return status
Nathan Yonkee 9:d58e77ebd769 1585
Nathan Yonkee 9:d58e77ebd769 1586 class TestLogger():
Nathan Yonkee 9:d58e77ebd769 1587 """ Super-class for logging and printing ongoing events for test suite pass
Nathan Yonkee 9:d58e77ebd769 1588 """
Nathan Yonkee 9:d58e77ebd769 1589 def __init__(self, store_log=True):
Nathan Yonkee 9:d58e77ebd769 1590 """ We can control if logger actually stores log in memory
Nathan Yonkee 9:d58e77ebd769 1591 or just handled all log entries immediately
Nathan Yonkee 9:d58e77ebd769 1592 """
Nathan Yonkee 9:d58e77ebd769 1593 self.log = []
Nathan Yonkee 9:d58e77ebd769 1594 self.log_to_file = False
Nathan Yonkee 9:d58e77ebd769 1595 self.log_file_name = None
Nathan Yonkee 9:d58e77ebd769 1596 self.store_log = store_log
Nathan Yonkee 9:d58e77ebd769 1597
Nathan Yonkee 9:d58e77ebd769 1598 self.LogType = construct_enum(INFO='Info',
Nathan Yonkee 9:d58e77ebd769 1599 WARN='Warning',
Nathan Yonkee 9:d58e77ebd769 1600 NOTIF='Notification',
Nathan Yonkee 9:d58e77ebd769 1601 ERROR='Error',
Nathan Yonkee 9:d58e77ebd769 1602 EXCEPT='Exception')
Nathan Yonkee 9:d58e77ebd769 1603
Nathan Yonkee 9:d58e77ebd769 1604 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
Nathan Yonkee 9:d58e77ebd769 1605 APPEND=2) # Append to existing log file
Nathan Yonkee 9:d58e77ebd769 1606
Nathan Yonkee 9:d58e77ebd769 1607 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
Nathan Yonkee 9:d58e77ebd769 1608 """ Log one line of text
Nathan Yonkee 9:d58e77ebd769 1609 """
Nathan Yonkee 9:d58e77ebd769 1610 log_timestamp = time()
Nathan Yonkee 9:d58e77ebd769 1611 log_entry = {'log_type' : LogType,
Nathan Yonkee 9:d58e77ebd769 1612 'log_timestamp' : log_timestamp,
Nathan Yonkee 9:d58e77ebd769 1613 'log_line' : log_line,
Nathan Yonkee 9:d58e77ebd769 1614 '_future' : None
Nathan Yonkee 9:d58e77ebd769 1615 }
Nathan Yonkee 9:d58e77ebd769 1616 # Store log in memory
Nathan Yonkee 9:d58e77ebd769 1617 if self.store_log:
Nathan Yonkee 9:d58e77ebd769 1618 self.log.append(log_entry)
Nathan Yonkee 9:d58e77ebd769 1619 return log_entry
Nathan Yonkee 9:d58e77ebd769 1620
Nathan Yonkee 9:d58e77ebd769 1621
Nathan Yonkee 9:d58e77ebd769 1622 class CLITestLogger(TestLogger):
Nathan Yonkee 9:d58e77ebd769 1623 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
Nathan Yonkee 9:d58e77ebd769 1624 """
Nathan Yonkee 9:d58e77ebd769 1625 def __init__(self, store_log=True, file_name=None):
Nathan Yonkee 9:d58e77ebd769 1626 TestLogger.__init__(self)
Nathan Yonkee 9:d58e77ebd769 1627 self.log_file_name = file_name
Nathan Yonkee 9:d58e77ebd769 1628 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
Nathan Yonkee 9:d58e77ebd769 1629 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
Nathan Yonkee 9:d58e77ebd769 1630
Nathan Yonkee 9:d58e77ebd769 1631 def log_print(self, log_entry, timestamp=True):
Nathan Yonkee 9:d58e77ebd769 1632 """ Prints on screen formatted log entry
Nathan Yonkee 9:d58e77ebd769 1633 """
Nathan Yonkee 9:d58e77ebd769 1634 ts = log_entry['log_timestamp']
Nathan Yonkee 9:d58e77ebd769 1635 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
Nathan Yonkee 9:d58e77ebd769 1636 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
Nathan Yonkee 9:d58e77ebd769 1637 return timestamp_str + log_line_str
Nathan Yonkee 9:d58e77ebd769 1638
Nathan Yonkee 9:d58e77ebd769 1639 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
Nathan Yonkee 9:d58e77ebd769 1640 """ Logs line, if log file output was specified log line will be appended
Nathan Yonkee 9:d58e77ebd769 1641 at the end of log file
Nathan Yonkee 9:d58e77ebd769 1642 """
Nathan Yonkee 9:d58e77ebd769 1643 log_entry = TestLogger.log_line(self, LogType, log_line)
Nathan Yonkee 9:d58e77ebd769 1644 log_line_str = self.log_print(log_entry, timestamp)
Nathan Yonkee 9:d58e77ebd769 1645 if self.log_file_name is not None:
Nathan Yonkee 9:d58e77ebd769 1646 try:
Nathan Yonkee 9:d58e77ebd769 1647 with open(self.log_file_name, 'a') as f:
Nathan Yonkee 9:d58e77ebd769 1648 f.write(log_line_str + line_delim)
Nathan Yonkee 9:d58e77ebd769 1649 except IOError:
Nathan Yonkee 9:d58e77ebd769 1650 pass
Nathan Yonkee 9:d58e77ebd769 1651 return log_line_str
Nathan Yonkee 9:d58e77ebd769 1652
Nathan Yonkee 9:d58e77ebd769 1653
Nathan Yonkee 9:d58e77ebd769 1654 def factory_db_logger(db_url):
Nathan Yonkee 9:d58e77ebd769 1655 """ Factory database driver depending on database type supplied in database connection string db_url
Nathan Yonkee 9:d58e77ebd769 1656 """
Nathan Yonkee 9:d58e77ebd769 1657 if db_url is not None:
Nathan Yonkee 9:d58e77ebd769 1658 from tools.test_mysql import MySQLDBAccess
Nathan Yonkee 9:d58e77ebd769 1659 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
Nathan Yonkee 9:d58e77ebd769 1660 if connection_info is not None:
Nathan Yonkee 9:d58e77ebd769 1661 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
Nathan Yonkee 9:d58e77ebd769 1662 if db_type == 'mysql':
Nathan Yonkee 9:d58e77ebd769 1663 return MySQLDBAccess()
Nathan Yonkee 9:d58e77ebd769 1664 return None
Nathan Yonkee 9:d58e77ebd769 1665
Nathan Yonkee 9:d58e77ebd769 1666
Nathan Yonkee 9:d58e77ebd769 1667 def detect_database_verbose(db_url):
Nathan Yonkee 9:d58e77ebd769 1668 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
Nathan Yonkee 9:d58e77ebd769 1669 """
Nathan Yonkee 9:d58e77ebd769 1670 result = BaseDBAccess().parse_db_connection_string(db_url)
Nathan Yonkee 9:d58e77ebd769 1671 if result is not None:
Nathan Yonkee 9:d58e77ebd769 1672 # Parsing passed
Nathan Yonkee 9:d58e77ebd769 1673 (db_type, username, password, host, db_name) = result
Nathan Yonkee 9:d58e77ebd769 1674 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
Nathan Yonkee 9:d58e77ebd769 1675 # Let's try to connect
Nathan Yonkee 9:d58e77ebd769 1676 db_ = factory_db_logger(db_url)
Nathan Yonkee 9:d58e77ebd769 1677 if db_ is not None:
Nathan Yonkee 9:d58e77ebd769 1678 print("Connecting to database '%s'..." % db_url)
Nathan Yonkee 9:d58e77ebd769 1679 db_.connect(host, username, password, db_name)
Nathan Yonkee 9:d58e77ebd769 1680 if db_.is_connected():
Nathan Yonkee 9:d58e77ebd769 1681 print("ok")
Nathan Yonkee 9:d58e77ebd769 1682 print("Detecting database...")
Nathan Yonkee 9:d58e77ebd769 1683 print(db_.detect_database(verbose=True))
Nathan Yonkee 9:d58e77ebd769 1684 print("Disconnecting...")
Nathan Yonkee 9:d58e77ebd769 1685 db_.disconnect()
Nathan Yonkee 9:d58e77ebd769 1686 print("done")
Nathan Yonkee 9:d58e77ebd769 1687 else:
Nathan Yonkee 9:d58e77ebd769 1688 print("Database type '%s' unknown" % db_type)
Nathan Yonkee 9:d58e77ebd769 1689 else:
Nathan Yonkee 9:d58e77ebd769 1690 print("Parse error: '%s' - DB Url error" % db_url)
Nathan Yonkee 9:d58e77ebd769 1691
Nathan Yonkee 9:d58e77ebd769 1692
Nathan Yonkee 9:d58e77ebd769 1693 def get_module_avail(module_name):
Nathan Yonkee 9:d58e77ebd769 1694 """ This function returns True if module_name is already imported module
Nathan Yonkee 9:d58e77ebd769 1695 """
Nathan Yonkee 9:d58e77ebd769 1696 return module_name in sys.modules.keys()
Nathan Yonkee 9:d58e77ebd769 1697
Nathan Yonkee 9:d58e77ebd769 1698 def get_autodetected_MUTS_list(platform_name_filter=None):
Nathan Yonkee 9:d58e77ebd769 1699 oldError = None
Nathan Yonkee 9:d58e77ebd769 1700 if os.name == 'nt':
Nathan Yonkee 9:d58e77ebd769 1701 # Disable Windows error box temporarily
Nathan Yonkee 9:d58e77ebd769 1702 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
Nathan Yonkee 9:d58e77ebd769 1703
Nathan Yonkee 9:d58e77ebd769 1704 mbeds = mbed_lstools.create()
Nathan Yonkee 9:d58e77ebd769 1705 detect_muts_list = mbeds.list_mbeds()
Nathan Yonkee 9:d58e77ebd769 1706
Nathan Yonkee 9:d58e77ebd769 1707 if os.name == 'nt':
Nathan Yonkee 9:d58e77ebd769 1708 ctypes.windll.kernel32.SetErrorMode(oldError)
Nathan Yonkee 9:d58e77ebd769 1709
Nathan Yonkee 9:d58e77ebd769 1710 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
Nathan Yonkee 9:d58e77ebd769 1711
Nathan Yonkee 9:d58e77ebd769 1712 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
Nathan Yonkee 9:d58e77ebd769 1713 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
Nathan Yonkee 9:d58e77ebd769 1714 If function fails to auto-detect devices it will return empty dictionary.
Nathan Yonkee 9:d58e77ebd769 1715
Nathan Yonkee 9:d58e77ebd769 1716 if get_module_avail('mbed_lstools'):
Nathan Yonkee 9:d58e77ebd769 1717 mbeds = mbed_lstools.create()
Nathan Yonkee 9:d58e77ebd769 1718 mbeds_list = mbeds.list_mbeds()
Nathan Yonkee 9:d58e77ebd769 1719
Nathan Yonkee 9:d58e77ebd769 1720 @param mbeds_list list of mbeds captured from mbed_lstools
Nathan Yonkee 9:d58e77ebd769 1721 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
Nathan Yonkee 9:d58e77ebd769 1722 """
Nathan Yonkee 9:d58e77ebd769 1723 result = {} # Should be in muts_all.json format
Nathan Yonkee 9:d58e77ebd769 1724 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
Nathan Yonkee 9:d58e77ebd769 1725 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
Nathan Yonkee 9:d58e77ebd769 1726 index = 1
Nathan Yonkee 9:d58e77ebd769 1727 for mut in mbeds_list:
Nathan Yonkee 9:d58e77ebd769 1728 # Filter the MUTS if a filter is specified
Nathan Yonkee 9:d58e77ebd769 1729
Nathan Yonkee 9:d58e77ebd769 1730 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
Nathan Yonkee 9:d58e77ebd769 1731 continue
Nathan Yonkee 9:d58e77ebd769 1732
Nathan Yonkee 9:d58e77ebd769 1733 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
Nathan Yonkee 9:d58e77ebd769 1734 # if not we are creating our own unique value (last few chars from platform's target_id).
Nathan Yonkee 9:d58e77ebd769 1735 m = {'mcu': mut['platform_name'],
Nathan Yonkee 9:d58e77ebd769 1736 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
Nathan Yonkee 9:d58e77ebd769 1737 'port': mut['serial_port'],
Nathan Yonkee 9:d58e77ebd769 1738 'disk': mut['mount_point'],
Nathan Yonkee 9:d58e77ebd769 1739 'peripherals': [] # No peripheral detection
Nathan Yonkee 9:d58e77ebd769 1740 }
Nathan Yonkee 9:d58e77ebd769 1741 if index not in result:
Nathan Yonkee 9:d58e77ebd769 1742 result[index] = {}
Nathan Yonkee 9:d58e77ebd769 1743 result[index] = m
Nathan Yonkee 9:d58e77ebd769 1744 index += 1
Nathan Yonkee 9:d58e77ebd769 1745 return result
Nathan Yonkee 9:d58e77ebd769 1746
Nathan Yonkee 9:d58e77ebd769 1747
Nathan Yonkee 9:d58e77ebd769 1748 def get_autodetected_TEST_SPEC(mbeds_list,
Nathan Yonkee 9:d58e77ebd769 1749 use_default_toolchain=True,
Nathan Yonkee 9:d58e77ebd769 1750 use_supported_toolchains=False,
Nathan Yonkee 9:d58e77ebd769 1751 toolchain_filter=None,
Nathan Yonkee 9:d58e77ebd769 1752 platform_name_filter=None):
Nathan Yonkee 9:d58e77ebd769 1753 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
Nathan Yonkee 9:d58e77ebd769 1754 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
Nathan Yonkee 9:d58e77ebd769 1755
Nathan Yonkee 9:d58e77ebd769 1756 use_default_toolchain - if True add default toolchain to test_spec
Nathan Yonkee 9:d58e77ebd769 1757 use_supported_toolchains - if True add all supported toolchains to test_spec
Nathan Yonkee 9:d58e77ebd769 1758 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
Nathan Yonkee 9:d58e77ebd769 1759 """
Nathan Yonkee 9:d58e77ebd769 1760 result = {'targets': {} }
Nathan Yonkee 9:d58e77ebd769 1761
Nathan Yonkee 9:d58e77ebd769 1762 for mut in mbeds_list:
Nathan Yonkee 9:d58e77ebd769 1763 mcu = mut['mcu']
Nathan Yonkee 9:d58e77ebd769 1764 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
Nathan Yonkee 9:d58e77ebd769 1765 if mcu in TARGET_MAP:
Nathan Yonkee 9:d58e77ebd769 1766 default_toolchain = TARGET_MAP[mcu].default_toolchain
Nathan Yonkee 9:d58e77ebd769 1767 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
Nathan Yonkee 9:d58e77ebd769 1768
Nathan Yonkee 9:d58e77ebd769 1769 # Decide which toolchains should be added to test specification toolchain pool for each target
Nathan Yonkee 9:d58e77ebd769 1770 toolchains = []
Nathan Yonkee 9:d58e77ebd769 1771 if use_default_toolchain:
Nathan Yonkee 9:d58e77ebd769 1772 toolchains.append(default_toolchain)
Nathan Yonkee 9:d58e77ebd769 1773 if use_supported_toolchains:
Nathan Yonkee 9:d58e77ebd769 1774 toolchains += supported_toolchains
Nathan Yonkee 9:d58e77ebd769 1775 if toolchain_filter is not None:
Nathan Yonkee 9:d58e77ebd769 1776 all_toolchains = supported_toolchains + [default_toolchain]
Nathan Yonkee 9:d58e77ebd769 1777 for toolchain in toolchain_filter:
Nathan Yonkee 9:d58e77ebd769 1778 if toolchain in all_toolchains:
Nathan Yonkee 9:d58e77ebd769 1779 toolchains.append(toolchain)
Nathan Yonkee 9:d58e77ebd769 1780
Nathan Yonkee 9:d58e77ebd769 1781 result['targets'][mcu] = list(set(toolchains))
Nathan Yonkee 9:d58e77ebd769 1782 return result
Nathan Yonkee 9:d58e77ebd769 1783
Nathan Yonkee 9:d58e77ebd769 1784
Nathan Yonkee 9:d58e77ebd769 1785 def get_default_test_options_parser():
Nathan Yonkee 9:d58e77ebd769 1786 """ Get common test script options used by CLI, web services etc.
Nathan Yonkee 9:d58e77ebd769 1787 """
Nathan Yonkee 9:d58e77ebd769 1788 parser = argparse.ArgumentParser()
Nathan Yonkee 9:d58e77ebd769 1789 parser.add_argument('-i', '--tests',
Nathan Yonkee 9:d58e77ebd769 1790 dest='test_spec_filename',
Nathan Yonkee 9:d58e77ebd769 1791 metavar="FILE",
Nathan Yonkee 9:d58e77ebd769 1792 type=argparse_filestring_type,
Nathan Yonkee 9:d58e77ebd769 1793 help='Points to file with test specification')
Nathan Yonkee 9:d58e77ebd769 1794
Nathan Yonkee 9:d58e77ebd769 1795 parser.add_argument('-M', '--MUTS',
Nathan Yonkee 9:d58e77ebd769 1796 dest='muts_spec_filename',
Nathan Yonkee 9:d58e77ebd769 1797 metavar="FILE",
Nathan Yonkee 9:d58e77ebd769 1798 type=argparse_filestring_type,
Nathan Yonkee 9:d58e77ebd769 1799 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
Nathan Yonkee 9:d58e77ebd769 1800
Nathan Yonkee 9:d58e77ebd769 1801 parser.add_argument("-j", "--jobs",
Nathan Yonkee 9:d58e77ebd769 1802 dest='jobs',
Nathan Yonkee 9:d58e77ebd769 1803 metavar="NUMBER",
Nathan Yonkee 9:d58e77ebd769 1804 type=int,
Nathan Yonkee 9:d58e77ebd769 1805 help="Define number of compilation jobs. Default value is 1")
Nathan Yonkee 9:d58e77ebd769 1806
Nathan Yonkee 9:d58e77ebd769 1807 if get_module_avail('mbed_lstools'):
Nathan Yonkee 9:d58e77ebd769 1808 # Additional features available when mbed_lstools is installed on host and imported
Nathan Yonkee 9:d58e77ebd769 1809 # mbed_lstools allow users to detect connected to host mbed-enabled devices
Nathan Yonkee 9:d58e77ebd769 1810 parser.add_argument('--auto',
Nathan Yonkee 9:d58e77ebd769 1811 dest='auto_detect',
Nathan Yonkee 9:d58e77ebd769 1812 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1813 help='Use mbed-ls module to detect all connected mbed devices')
Nathan Yonkee 9:d58e77ebd769 1814
Nathan Yonkee 9:d58e77ebd769 1815 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
Nathan Yonkee 9:d58e77ebd769 1816 parser.add_argument('--tc',
Nathan Yonkee 9:d58e77ebd769 1817 dest='toolchains_filter',
Nathan Yonkee 9:d58e77ebd769 1818 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
Nathan Yonkee 9:d58e77ebd769 1819 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
Nathan Yonkee 9:d58e77ebd769 1820
Nathan Yonkee 9:d58e77ebd769 1821 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
Nathan Yonkee 9:d58e77ebd769 1822 parser.add_argument('--oper',
Nathan Yonkee 9:d58e77ebd769 1823 dest='operability_checks',
Nathan Yonkee 9:d58e77ebd769 1824 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
Nathan Yonkee 9:d58e77ebd769 1825 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
Nathan Yonkee 9:d58e77ebd769 1826
Nathan Yonkee 9:d58e77ebd769 1827 parser.add_argument('--clean',
Nathan Yonkee 9:d58e77ebd769 1828 dest='clean',
Nathan Yonkee 9:d58e77ebd769 1829 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1830 help='Clean the build directory')
Nathan Yonkee 9:d58e77ebd769 1831
Nathan Yonkee 9:d58e77ebd769 1832 parser.add_argument('-P', '--only-peripherals',
Nathan Yonkee 9:d58e77ebd769 1833 dest='test_only_peripheral',
Nathan Yonkee 9:d58e77ebd769 1834 default=False,
Nathan Yonkee 9:d58e77ebd769 1835 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1836 help='Test only peripheral declared for MUT and skip common tests')
Nathan Yonkee 9:d58e77ebd769 1837
Nathan Yonkee 9:d58e77ebd769 1838 parser.add_argument("--profile", dest="profile", action="append",
Nathan Yonkee 9:d58e77ebd769 1839 type=argparse_filestring_type,
Nathan Yonkee 9:d58e77ebd769 1840 default=[])
Nathan Yonkee 9:d58e77ebd769 1841
Nathan Yonkee 9:d58e77ebd769 1842 parser.add_argument('-C', '--only-commons',
Nathan Yonkee 9:d58e77ebd769 1843 dest='test_only_common',
Nathan Yonkee 9:d58e77ebd769 1844 default=False,
Nathan Yonkee 9:d58e77ebd769 1845 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1846 help='Test only board internals. Skip perpherials tests and perform common tests')
Nathan Yonkee 9:d58e77ebd769 1847
Nathan Yonkee 9:d58e77ebd769 1848 parser.add_argument('-n', '--test-by-names',
Nathan Yonkee 9:d58e77ebd769 1849 dest='test_by_names',
Nathan Yonkee 9:d58e77ebd769 1850 type=argparse_many(str),
Nathan Yonkee 9:d58e77ebd769 1851 help='Runs only test enumerated it this switch. Use comma to separate test case names')
Nathan Yonkee 9:d58e77ebd769 1852
Nathan Yonkee 9:d58e77ebd769 1853 parser.add_argument('-p', '--peripheral-by-names',
Nathan Yonkee 9:d58e77ebd769 1854 dest='peripheral_by_names',
Nathan Yonkee 9:d58e77ebd769 1855 type=argparse_many(str),
Nathan Yonkee 9:d58e77ebd769 1856 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
Nathan Yonkee 9:d58e77ebd769 1857
Nathan Yonkee 9:d58e77ebd769 1858 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
Nathan Yonkee 9:d58e77ebd769 1859 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
Nathan Yonkee 9:d58e77ebd769 1860
Nathan Yonkee 9:d58e77ebd769 1861 parser.add_argument('-c', '--copy-method',
Nathan Yonkee 9:d58e77ebd769 1862 dest='copy_method',
Nathan Yonkee 9:d58e77ebd769 1863 type=argparse_uppercase_type(copy_methods, "flash method"),
Nathan Yonkee 9:d58e77ebd769 1864 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
Nathan Yonkee 9:d58e77ebd769 1865
Nathan Yonkee 9:d58e77ebd769 1866 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
Nathan Yonkee 9:d58e77ebd769 1867 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
Nathan Yonkee 9:d58e77ebd769 1868
Nathan Yonkee 9:d58e77ebd769 1869 parser.add_argument('-r', '--reset-type',
Nathan Yonkee 9:d58e77ebd769 1870 dest='mut_reset_type',
Nathan Yonkee 9:d58e77ebd769 1871 default=None,
Nathan Yonkee 9:d58e77ebd769 1872 type=argparse_uppercase_type(reset_methods, "reset method"),
Nathan Yonkee 9:d58e77ebd769 1873 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
Nathan Yonkee 9:d58e77ebd769 1874
Nathan Yonkee 9:d58e77ebd769 1875 parser.add_argument('-g', '--goanna-for-tests',
Nathan Yonkee 9:d58e77ebd769 1876 dest='goanna_for_tests',
Nathan Yonkee 9:d58e77ebd769 1877 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1878 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
Nathan Yonkee 9:d58e77ebd769 1879
Nathan Yonkee 9:d58e77ebd769 1880 parser.add_argument('-G', '--goanna-for-sdk',
Nathan Yonkee 9:d58e77ebd769 1881 dest='goanna_for_mbed_sdk',
Nathan Yonkee 9:d58e77ebd769 1882 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1883 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
Nathan Yonkee 9:d58e77ebd769 1884
Nathan Yonkee 9:d58e77ebd769 1885 parser.add_argument('-s', '--suppress-summary',
Nathan Yonkee 9:d58e77ebd769 1886 dest='suppress_summary',
Nathan Yonkee 9:d58e77ebd769 1887 default=False,
Nathan Yonkee 9:d58e77ebd769 1888 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1889 help='Suppresses display of wellformatted table with test results')
Nathan Yonkee 9:d58e77ebd769 1890
Nathan Yonkee 9:d58e77ebd769 1891 parser.add_argument('-t', '--test-summary',
Nathan Yonkee 9:d58e77ebd769 1892 dest='test_x_toolchain_summary',
Nathan Yonkee 9:d58e77ebd769 1893 default=False,
Nathan Yonkee 9:d58e77ebd769 1894 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1895 help='Displays wellformatted table with test x toolchain test result per target')
Nathan Yonkee 9:d58e77ebd769 1896
Nathan Yonkee 9:d58e77ebd769 1897 parser.add_argument('-A', '--test-automation-report',
Nathan Yonkee 9:d58e77ebd769 1898 dest='test_automation_report',
Nathan Yonkee 9:d58e77ebd769 1899 default=False,
Nathan Yonkee 9:d58e77ebd769 1900 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1901 help='Prints information about all tests and exits')
Nathan Yonkee 9:d58e77ebd769 1902
Nathan Yonkee 9:d58e77ebd769 1903 parser.add_argument('-R', '--test-case-report',
Nathan Yonkee 9:d58e77ebd769 1904 dest='test_case_report',
Nathan Yonkee 9:d58e77ebd769 1905 default=False,
Nathan Yonkee 9:d58e77ebd769 1906 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1907 help='Prints information about all test cases and exits')
Nathan Yonkee 9:d58e77ebd769 1908
Nathan Yonkee 9:d58e77ebd769 1909 parser.add_argument("-S", "--supported-toolchains",
Nathan Yonkee 9:d58e77ebd769 1910 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1911 dest="supported_toolchains",
Nathan Yonkee 9:d58e77ebd769 1912 default=False,
Nathan Yonkee 9:d58e77ebd769 1913 help="Displays supported matrix of MCUs and toolchains")
Nathan Yonkee 9:d58e77ebd769 1914
Nathan Yonkee 9:d58e77ebd769 1915 parser.add_argument("-O", "--only-build",
Nathan Yonkee 9:d58e77ebd769 1916 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1917 dest="only_build_tests",
Nathan Yonkee 9:d58e77ebd769 1918 default=False,
Nathan Yonkee 9:d58e77ebd769 1919 help="Only build tests, skips actual test procedures (flashing etc.)")
Nathan Yonkee 9:d58e77ebd769 1920
Nathan Yonkee 9:d58e77ebd769 1921 parser.add_argument('--parallel',
Nathan Yonkee 9:d58e77ebd769 1922 dest='parallel_test_exec',
Nathan Yonkee 9:d58e77ebd769 1923 default=False,
Nathan Yonkee 9:d58e77ebd769 1924 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1925 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
Nathan Yonkee 9:d58e77ebd769 1926
Nathan Yonkee 9:d58e77ebd769 1927 parser.add_argument('--config',
Nathan Yonkee 9:d58e77ebd769 1928 dest='verbose_test_configuration_only',
Nathan Yonkee 9:d58e77ebd769 1929 default=False,
Nathan Yonkee 9:d58e77ebd769 1930 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1931 help='Displays full test specification and MUTs configration and exits')
Nathan Yonkee 9:d58e77ebd769 1932
Nathan Yonkee 9:d58e77ebd769 1933 parser.add_argument('--loops',
Nathan Yonkee 9:d58e77ebd769 1934 dest='test_loops_list',
Nathan Yonkee 9:d58e77ebd769 1935 type=argparse_many(str),
Nathan Yonkee 9:d58e77ebd769 1936 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
Nathan Yonkee 9:d58e77ebd769 1937
Nathan Yonkee 9:d58e77ebd769 1938 parser.add_argument('--global-loops',
Nathan Yonkee 9:d58e77ebd769 1939 dest='test_global_loops_value',
Nathan Yonkee 9:d58e77ebd769 1940 type=int,
Nathan Yonkee 9:d58e77ebd769 1941 help='Set global number of test loops per test. Default value is set 1')
Nathan Yonkee 9:d58e77ebd769 1942
Nathan Yonkee 9:d58e77ebd769 1943 parser.add_argument('--consolidate-waterfall',
Nathan Yonkee 9:d58e77ebd769 1944 dest='consolidate_waterfall_test',
Nathan Yonkee 9:d58e77ebd769 1945 default=False,
Nathan Yonkee 9:d58e77ebd769 1946 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1947 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
Nathan Yonkee 9:d58e77ebd769 1948
Nathan Yonkee 9:d58e77ebd769 1949 parser.add_argument('-W', '--waterfall',
Nathan Yonkee 9:d58e77ebd769 1950 dest='waterfall_test',
Nathan Yonkee 9:d58e77ebd769 1951 default=False,
Nathan Yonkee 9:d58e77ebd769 1952 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1953 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
Nathan Yonkee 9:d58e77ebd769 1954
Nathan Yonkee 9:d58e77ebd769 1955 parser.add_argument('-N', '--firmware-name',
Nathan Yonkee 9:d58e77ebd769 1956 dest='firmware_global_name',
Nathan Yonkee 9:d58e77ebd769 1957 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
Nathan Yonkee 9:d58e77ebd769 1958
Nathan Yonkee 9:d58e77ebd769 1959 parser.add_argument('-u', '--shuffle',
Nathan Yonkee 9:d58e77ebd769 1960 dest='shuffle_test_order',
Nathan Yonkee 9:d58e77ebd769 1961 default=False,
Nathan Yonkee 9:d58e77ebd769 1962 action="store_true",
Nathan Yonkee 9:d58e77ebd769 1963 help='Shuffles test execution order')
Nathan Yonkee 9:d58e77ebd769 1964
Nathan Yonkee 9:d58e77ebd769 1965 parser.add_argument('--shuffle-seed',
Nathan Yonkee 9:d58e77ebd769 1966 dest='shuffle_test_seed',
Nathan Yonkee 9:d58e77ebd769 1967 default=None,
Nathan Yonkee 9:d58e77ebd769 1968 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
Nathan Yonkee 9:d58e77ebd769 1969
Nathan Yonkee 9:d58e77ebd769 1970 parser.add_argument('-f', '--filter',
Nathan Yonkee 9:d58e77ebd769 1971 dest='general_filter_regex',
Nathan Yonkee 9:d58e77ebd769 1972 type=argparse_many(str),
Nathan Yonkee 9:d58e77ebd769 1973 default=None,
Nathan Yonkee 9:d58e77ebd769 1974 help='For some commands you can use filter to filter out results')
Nathan Yonkee 9:d58e77ebd769 1975
Nathan Yonkee 9:d58e77ebd769 1976 parser.add_argument('--inc-timeout',
Nathan Yonkee 9:d58e77ebd769 1977 dest='extend_test_timeout',
Nathan Yonkee 9:d58e77ebd769 1978 metavar="NUMBER",
Nathan Yonkee 9:d58e77ebd769 1979 type=int,
Nathan Yonkee 9:d58e77ebd769 1980 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
Nathan Yonkee 9:d58e77ebd769 1981
Nathan Yonkee 9:d58e77ebd769 1982 parser.add_argument('--db',
Nathan Yonkee 9:d58e77ebd769 1983 dest='db_url',
Nathan Yonkee 9:d58e77ebd769 1984 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
Nathan Yonkee 9:d58e77ebd769 1985
Nathan Yonkee 9:d58e77ebd769 1986 parser.add_argument('-l', '--log',
Nathan Yonkee 9:d58e77ebd769 1987 dest='log_file_name',
Nathan Yonkee 9:d58e77ebd769 1988 help='Log events to external file (note not all console entries may be visible in log file)')
Nathan Yonkee 9:d58e77ebd769 1989
Nathan Yonkee 9:d58e77ebd769 1990 parser.add_argument('--report-html',
Nathan Yonkee 9:d58e77ebd769 1991 dest='report_html_file_name',
Nathan Yonkee 9:d58e77ebd769 1992 help='You can log test suite results in form of HTML report')
Nathan Yonkee 9:d58e77ebd769 1993
Nathan Yonkee 9:d58e77ebd769 1994 parser.add_argument('--report-junit',
Nathan Yonkee 9:d58e77ebd769 1995 dest='report_junit_file_name',
Nathan Yonkee 9:d58e77ebd769 1996 help='You can log test suite results in form of JUnit compliant XML report')
Nathan Yonkee 9:d58e77ebd769 1997
Nathan Yonkee 9:d58e77ebd769 1998 parser.add_argument("--report-build",
Nathan Yonkee 9:d58e77ebd769 1999 dest="report_build_file_name",
Nathan Yonkee 9:d58e77ebd769 2000 help="Output the build results to a junit xml file")
Nathan Yonkee 9:d58e77ebd769 2001
Nathan Yonkee 9:d58e77ebd769 2002 parser.add_argument("--report-text",
Nathan Yonkee 9:d58e77ebd769 2003 dest="report_text_file_name",
Nathan Yonkee 9:d58e77ebd769 2004 help="Output the build results to a text file")
Nathan Yonkee 9:d58e77ebd769 2005
Nathan Yonkee 9:d58e77ebd769 2006 parser.add_argument('--verbose-skipped',
Nathan Yonkee 9:d58e77ebd769 2007 dest='verbose_skipped_tests',
Nathan Yonkee 9:d58e77ebd769 2008 default=False,
Nathan Yonkee 9:d58e77ebd769 2009 action="store_true",
Nathan Yonkee 9:d58e77ebd769 2010 help='Prints some extra information about skipped tests')
Nathan Yonkee 9:d58e77ebd769 2011
Nathan Yonkee 9:d58e77ebd769 2012 parser.add_argument('-V', '--verbose-test-result',
Nathan Yonkee 9:d58e77ebd769 2013 dest='verbose_test_result_only',
Nathan Yonkee 9:d58e77ebd769 2014 default=False,
Nathan Yonkee 9:d58e77ebd769 2015 action="store_true",
Nathan Yonkee 9:d58e77ebd769 2016 help='Prints test serial output')
Nathan Yonkee 9:d58e77ebd769 2017
Nathan Yonkee 9:d58e77ebd769 2018 parser.add_argument('-v', '--verbose',
Nathan Yonkee 9:d58e77ebd769 2019 dest='verbose',
Nathan Yonkee 9:d58e77ebd769 2020 default=False,
Nathan Yonkee 9:d58e77ebd769 2021 action="store_true",
Nathan Yonkee 9:d58e77ebd769 2022 help='Verbose mode (prints some extra information)')
Nathan Yonkee 9:d58e77ebd769 2023
Nathan Yonkee 9:d58e77ebd769 2024 parser.add_argument('--version',
Nathan Yonkee 9:d58e77ebd769 2025 dest='version',
Nathan Yonkee 9:d58e77ebd769 2026 default=False,
Nathan Yonkee 9:d58e77ebd769 2027 action="store_true",
Nathan Yonkee 9:d58e77ebd769 2028 help='Prints script version and exits')
Nathan Yonkee 9:d58e77ebd769 2029
Nathan Yonkee 9:d58e77ebd769 2030 parser.add_argument('--stats-depth',
Nathan Yonkee 9:d58e77ebd769 2031 dest='stats_depth',
Nathan Yonkee 9:d58e77ebd769 2032 default=2,
Nathan Yonkee 9:d58e77ebd769 2033 type=int,
Nathan Yonkee 9:d58e77ebd769 2034 help="Depth level for static memory report")
Nathan Yonkee 9:d58e77ebd769 2035 return parser
Nathan Yonkee 9:d58e77ebd769 2036
Nathan Yonkee 9:d58e77ebd769 2037 def test_path_to_name(path, base):
Nathan Yonkee 9:d58e77ebd769 2038 """Change all slashes in a path into hyphens
Nathan Yonkee 9:d58e77ebd769 2039 This creates a unique cross-platform test name based on the path
Nathan Yonkee 9:d58e77ebd769 2040 This can eventually be overriden by a to-be-determined meta-data mechanism"""
Nathan Yonkee 9:d58e77ebd769 2041 name_parts = []
Nathan Yonkee 9:d58e77ebd769 2042 head, tail = os.path.split(relpath(path,base))
Nathan Yonkee 9:d58e77ebd769 2043 while (tail and tail != "."):
Nathan Yonkee 9:d58e77ebd769 2044 name_parts.insert(0, tail)
Nathan Yonkee 9:d58e77ebd769 2045 head, tail = os.path.split(head)
Nathan Yonkee 9:d58e77ebd769 2046
Nathan Yonkee 9:d58e77ebd769 2047 return "-".join(name_parts).lower()
Nathan Yonkee 9:d58e77ebd769 2048
Nathan Yonkee 9:d58e77ebd769 2049 def get_test_config(config_name, target_name):
Nathan Yonkee 9:d58e77ebd769 2050 """Finds the path to a test configuration file
Nathan Yonkee 9:d58e77ebd769 2051 config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
Nathan Yonkee 9:d58e77ebd769 2052 target_name: name of target to determing if mbed OS interface given is valid
Nathan Yonkee 9:d58e77ebd769 2053 returns path to config, will return None if no valid config is found
Nathan Yonkee 9:d58e77ebd769 2054 """
Nathan Yonkee 9:d58e77ebd769 2055 # If they passed in a full path
Nathan Yonkee 9:d58e77ebd769 2056 if exists(config_name):
Nathan Yonkee 9:d58e77ebd769 2057 # This is a module config
Nathan Yonkee 9:d58e77ebd769 2058 return config_name
Nathan Yonkee 9:d58e77ebd769 2059 # Otherwise find the path to configuration file based on mbed OS interface
Nathan Yonkee 9:d58e77ebd769 2060 return TestConfig.get_config_path(config_name, target_name)
Nathan Yonkee 9:d58e77ebd769 2061
Nathan Yonkee 9:d58e77ebd769 2062 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
Nathan Yonkee 9:d58e77ebd769 2063 """ Finds all tests in a directory recursively
Nathan Yonkee 9:d58e77ebd769 2064 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
Nathan Yonkee 9:d58e77ebd769 2065 target_name: name of the target to use for scanning (ex. 'K64F')
Nathan Yonkee 9:d58e77ebd769 2066 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
Nathan Yonkee 9:d58e77ebd769 2067 options: Compile options to pass to the toolchain (ex. ['debug-info'])
Nathan Yonkee 9:d58e77ebd769 2068 app_config - location of a chosen mbed_app.json file
Nathan Yonkee 9:d58e77ebd769 2069
Nathan Yonkee 9:d58e77ebd769 2070 returns a dictionary where keys are the test name, and the values are
Nathan Yonkee 9:d58e77ebd769 2071 lists of paths needed to biuld the test.
Nathan Yonkee 9:d58e77ebd769 2072 """
Nathan Yonkee 9:d58e77ebd769 2073
Nathan Yonkee 9:d58e77ebd769 2074 # Temporary structure: tests referenced by (name, base, group, case) tuple
Nathan Yonkee 9:d58e77ebd769 2075 tests = {}
Nathan Yonkee 9:d58e77ebd769 2076 # List of common folders: (predicate function, path) tuple
Nathan Yonkee 9:d58e77ebd769 2077 commons = []
Nathan Yonkee 9:d58e77ebd769 2078
Nathan Yonkee 9:d58e77ebd769 2079 # Prepare the toolchain
Nathan Yonkee 9:d58e77ebd769 2080 toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
Nathan Yonkee 9:d58e77ebd769 2081 silent=True, app_config=app_config)
Nathan Yonkee 9:d58e77ebd769 2082
Nathan Yonkee 9:d58e77ebd769 2083 # Scan the directory for paths to probe for 'TESTS' folders
Nathan Yonkee 9:d58e77ebd769 2084 base_resources = scan_resources([base_dir], toolchain)
Nathan Yonkee 9:d58e77ebd769 2085
Nathan Yonkee 9:d58e77ebd769 2086 dirs = base_resources.inc_dirs
Nathan Yonkee 9:d58e77ebd769 2087 for directory in dirs:
Nathan Yonkee 9:d58e77ebd769 2088 subdirs = os.listdir(directory)
Nathan Yonkee 9:d58e77ebd769 2089
Nathan Yonkee 9:d58e77ebd769 2090 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
Nathan Yonkee 9:d58e77ebd769 2091 if 'TESTS' in subdirs:
Nathan Yonkee 9:d58e77ebd769 2092 walk_base_dir = join(directory, 'TESTS')
Nathan Yonkee 9:d58e77ebd769 2093 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
Nathan Yonkee 9:d58e77ebd769 2094
Nathan Yonkee 9:d58e77ebd769 2095 # Loop through all subdirectories
Nathan Yonkee 9:d58e77ebd769 2096 for d in test_resources.inc_dirs:
Nathan Yonkee 9:d58e77ebd769 2097
Nathan Yonkee 9:d58e77ebd769 2098 # If the test case folder is not called 'host_tests' or 'COMMON' and it is
Nathan Yonkee 9:d58e77ebd769 2099 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
Nathan Yonkee 9:d58e77ebd769 2100 # then add it to the tests
Nathan Yonkee 9:d58e77ebd769 2101 relative_path = relpath(d, walk_base_dir)
Nathan Yonkee 9:d58e77ebd769 2102 relative_path_parts = os.path.normpath(relative_path).split(os.sep)
Nathan Yonkee 9:d58e77ebd769 2103 if len(relative_path_parts) == 2:
Nathan Yonkee 9:d58e77ebd769 2104 test_group_directory_path, test_case_directory = os.path.split(d)
Nathan Yonkee 9:d58e77ebd769 2105 test_group_directory = os.path.basename(test_group_directory_path)
Nathan Yonkee 9:d58e77ebd769 2106
Nathan Yonkee 9:d58e77ebd769 2107 # Check to make sure discoverd folder is not in a host test directory or common directory
Nathan Yonkee 9:d58e77ebd769 2108 special_dirs = ['host_tests', 'COMMON']
Nathan Yonkee 9:d58e77ebd769 2109 if test_group_directory not in special_dirs and test_case_directory not in special_dirs:
Nathan Yonkee 9:d58e77ebd769 2110 test_name = test_path_to_name(d, base_dir)
Nathan Yonkee 9:d58e77ebd769 2111 tests[(test_name, walk_base_dir, test_group_directory, test_case_directory)] = [d]
Nathan Yonkee 9:d58e77ebd769 2112
Nathan Yonkee 9:d58e77ebd769 2113 # Also find any COMMON paths, we'll add these later once we find all the base tests
Nathan Yonkee 9:d58e77ebd769 2114 if 'COMMON' in relative_path_parts:
Nathan Yonkee 9:d58e77ebd769 2115 if relative_path_parts[0] != 'COMMON':
Nathan Yonkee 9:d58e77ebd769 2116 def predicate(base_pred, group_pred, (name, base, group, case)):
Nathan Yonkee 9:d58e77ebd769 2117 return base == base_pred and group == group_pred
Nathan Yonkee 9:d58e77ebd769 2118 commons.append((functools.partial(predicate, walk_base_dir, relative_path_parts[0]), d))
Nathan Yonkee 9:d58e77ebd769 2119 else:
Nathan Yonkee 9:d58e77ebd769 2120 def predicate(base_pred, (name, base, group, case)):
Nathan Yonkee 9:d58e77ebd769 2121 return base == base_pred
Nathan Yonkee 9:d58e77ebd769 2122 commons.append((functools.partial(predicate, walk_base_dir), d))
Nathan Yonkee 9:d58e77ebd769 2123
Nathan Yonkee 9:d58e77ebd769 2124 # Apply common directories
Nathan Yonkee 9:d58e77ebd769 2125 for pred, path in commons:
Nathan Yonkee 9:d58e77ebd769 2126 for test_identity, test_paths in tests.iteritems():
Nathan Yonkee 9:d58e77ebd769 2127 if pred(test_identity):
Nathan Yonkee 9:d58e77ebd769 2128 test_paths.append(path)
Nathan Yonkee 9:d58e77ebd769 2129
Nathan Yonkee 9:d58e77ebd769 2130 # Drop identity besides name
Nathan Yonkee 9:d58e77ebd769 2131 return {name: paths for (name, _, _, _), paths in tests.iteritems()}
Nathan Yonkee 9:d58e77ebd769 2132
Nathan Yonkee 9:d58e77ebd769 2133 def print_tests(tests, format="list", sort=True):
Nathan Yonkee 9:d58e77ebd769 2134 """Given a dictionary of tests (as returned from "find_tests"), print them
Nathan Yonkee 9:d58e77ebd769 2135 in the specified format"""
Nathan Yonkee 9:d58e77ebd769 2136 if format == "list":
Nathan Yonkee 9:d58e77ebd769 2137 for test_name in sorted(tests.keys()):
Nathan Yonkee 9:d58e77ebd769 2138 test_path = tests[test_name][0]
Nathan Yonkee 9:d58e77ebd769 2139 print("Test Case:")
Nathan Yonkee 9:d58e77ebd769 2140 print(" Name: %s" % test_name)
Nathan Yonkee 9:d58e77ebd769 2141 print(" Path: %s" % test_path)
Nathan Yonkee 9:d58e77ebd769 2142 elif format == "json":
Nathan Yonkee 9:d58e77ebd769 2143 print(json.dumps({test_name: test_path[0] for test_name, test_paths
Nathan Yonkee 9:d58e77ebd769 2144 in tests}, indent=2))
Nathan Yonkee 9:d58e77ebd769 2145 else:
Nathan Yonkee 9:d58e77ebd769 2146 print("Unknown format '%s'" % format)
Nathan Yonkee 9:d58e77ebd769 2147 sys.exit(1)
Nathan Yonkee 9:d58e77ebd769 2148
Nathan Yonkee 9:d58e77ebd769 2149 def norm_relative_path(path, start):
Nathan Yonkee 9:d58e77ebd769 2150 """This function will create a normalized, relative path. It mimics the
Nathan Yonkee 9:d58e77ebd769 2151 python os.path.relpath function, but also normalizes a Windows-syle path
Nathan Yonkee 9:d58e77ebd769 2152 that use backslashes to a Unix style path that uses forward slashes."""
Nathan Yonkee 9:d58e77ebd769 2153 path = os.path.normpath(path)
Nathan Yonkee 9:d58e77ebd769 2154 path = os.path.relpath(path, start)
Nathan Yonkee 9:d58e77ebd769 2155 path = path.replace("\\", "/")
Nathan Yonkee 9:d58e77ebd769 2156 return path
Nathan Yonkee 9:d58e77ebd769 2157
Nathan Yonkee 9:d58e77ebd769 2158
Nathan Yonkee 9:d58e77ebd769 2159 def build_test_worker(*args, **kwargs):
Nathan Yonkee 9:d58e77ebd769 2160 """This is a worker function for the parallel building of tests. The `args`
Nathan Yonkee 9:d58e77ebd769 2161 and `kwargs` are passed directly to `build_project`. It returns a dictionary
Nathan Yonkee 9:d58e77ebd769 2162 with the following structure:
Nathan Yonkee 9:d58e77ebd769 2163
Nathan Yonkee 9:d58e77ebd769 2164 {
Nathan Yonkee 9:d58e77ebd769 2165 'result': `True` if no exceptions were thrown, `False` otherwise
Nathan Yonkee 9:d58e77ebd769 2166 'reason': Instance of exception that was thrown on failure
Nathan Yonkee 9:d58e77ebd769 2167 'bin_file': Path to the created binary if `build_project` was
Nathan Yonkee 9:d58e77ebd769 2168 successful. Not present otherwise
Nathan Yonkee 9:d58e77ebd769 2169 'kwargs': The keyword arguments that were passed to `build_project`.
Nathan Yonkee 9:d58e77ebd769 2170 This includes arguments that were modified (ex. report)
Nathan Yonkee 9:d58e77ebd769 2171 }
Nathan Yonkee 9:d58e77ebd769 2172 """
Nathan Yonkee 9:d58e77ebd769 2173 bin_file = None
Nathan Yonkee 9:d58e77ebd769 2174 ret = {
Nathan Yonkee 9:d58e77ebd769 2175 'result': False,
Nathan Yonkee 9:d58e77ebd769 2176 'args': args,
Nathan Yonkee 9:d58e77ebd769 2177 'kwargs': kwargs
Nathan Yonkee 9:d58e77ebd769 2178 }
Nathan Yonkee 9:d58e77ebd769 2179
Nathan Yonkee 9:d58e77ebd769 2180 # Use parent TOOLCHAIN_PATHS variable
Nathan Yonkee 9:d58e77ebd769 2181 for key, value in kwargs['toolchain_paths'].items():
Nathan Yonkee 9:d58e77ebd769 2182 TOOLCHAIN_PATHS[key] = value
Nathan Yonkee 9:d58e77ebd769 2183
Nathan Yonkee 9:d58e77ebd769 2184 del kwargs['toolchain_paths']
Nathan Yonkee 9:d58e77ebd769 2185
Nathan Yonkee 9:d58e77ebd769 2186 try:
Nathan Yonkee 9:d58e77ebd769 2187 bin_file = build_project(*args, **kwargs)
Nathan Yonkee 9:d58e77ebd769 2188 ret['result'] = True
Nathan Yonkee 9:d58e77ebd769 2189 ret['bin_file'] = bin_file
Nathan Yonkee 9:d58e77ebd769 2190 ret['kwargs'] = kwargs
Nathan Yonkee 9:d58e77ebd769 2191
Nathan Yonkee 9:d58e77ebd769 2192 except NotSupportedException as e:
Nathan Yonkee 9:d58e77ebd769 2193 ret['reason'] = e
Nathan Yonkee 9:d58e77ebd769 2194 except ToolException as e:
Nathan Yonkee 9:d58e77ebd769 2195 ret['reason'] = e
Nathan Yonkee 9:d58e77ebd769 2196 except KeyboardInterrupt as e:
Nathan Yonkee 9:d58e77ebd769 2197 ret['reason'] = e
Nathan Yonkee 9:d58e77ebd769 2198 except:
Nathan Yonkee 9:d58e77ebd769 2199 # Print unhandled exceptions here
Nathan Yonkee 9:d58e77ebd769 2200 import traceback
Nathan Yonkee 9:d58e77ebd769 2201 traceback.print_exc(file=sys.stdout)
Nathan Yonkee 9:d58e77ebd769 2202
Nathan Yonkee 9:d58e77ebd769 2203 return ret
Nathan Yonkee 9:d58e77ebd769 2204
Nathan Yonkee 9:d58e77ebd769 2205
Nathan Yonkee 9:d58e77ebd769 2206 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
Nathan Yonkee 9:d58e77ebd769 2207 clean=False, notify=None, verbose=False, jobs=1, macros=None,
Nathan Yonkee 9:d58e77ebd769 2208 silent=False, report=None, properties=None,
Nathan Yonkee 9:d58e77ebd769 2209 continue_on_build_fail=False, app_config=None,
Nathan Yonkee 9:d58e77ebd769 2210 build_profile=None, stats_depth=None):
Nathan Yonkee 9:d58e77ebd769 2211 """Given the data structure from 'find_tests' and the typical build parameters,
Nathan Yonkee 9:d58e77ebd769 2212 build all the tests
Nathan Yonkee 9:d58e77ebd769 2213
Nathan Yonkee 9:d58e77ebd769 2214 Returns a tuple of the build result (True or False) followed by the test
Nathan Yonkee 9:d58e77ebd769 2215 build data structure"""
Nathan Yonkee 9:d58e77ebd769 2216
Nathan Yonkee 9:d58e77ebd769 2217 execution_directory = "."
Nathan Yonkee 9:d58e77ebd769 2218 base_path = norm_relative_path(build_path, execution_directory)
Nathan Yonkee 9:d58e77ebd769 2219
Nathan Yonkee 9:d58e77ebd769 2220 target_name = target.name if isinstance(target, Target) else target
Nathan Yonkee 9:d58e77ebd769 2221 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
Nathan Yonkee 9:d58e77ebd769 2222
Nathan Yonkee 9:d58e77ebd769 2223 baud_rate = 9600
Nathan Yonkee 9:d58e77ebd769 2224 if 'platform.stdio-baud-rate' in cfg:
Nathan Yonkee 9:d58e77ebd769 2225 baud_rate = cfg['platform.stdio-baud-rate'].value
Nathan Yonkee 9:d58e77ebd769 2226
Nathan Yonkee 9:d58e77ebd769 2227 test_build = {
Nathan Yonkee 9:d58e77ebd769 2228 "platform": target_name,
Nathan Yonkee 9:d58e77ebd769 2229 "toolchain": toolchain_name,
Nathan Yonkee 9:d58e77ebd769 2230 "base_path": base_path,
Nathan Yonkee 9:d58e77ebd769 2231 "baud_rate": baud_rate,
Nathan Yonkee 9:d58e77ebd769 2232 "binary_type": "bootable",
Nathan Yonkee 9:d58e77ebd769 2233 "tests": {}
Nathan Yonkee 9:d58e77ebd769 2234 }
Nathan Yonkee 9:d58e77ebd769 2235
Nathan Yonkee 9:d58e77ebd769 2236 result = True
Nathan Yonkee 9:d58e77ebd769 2237
Nathan Yonkee 9:d58e77ebd769 2238 jobs_count = int(jobs if jobs else cpu_count())
Nathan Yonkee 9:d58e77ebd769 2239 p = Pool(processes=jobs_count)
Nathan Yonkee 9:d58e77ebd769 2240 results = []
Nathan Yonkee 9:d58e77ebd769 2241 for test_name, test_paths in tests.items():
Nathan Yonkee 9:d58e77ebd769 2242 if not isinstance(test_paths, list):
Nathan Yonkee 9:d58e77ebd769 2243 test_paths = [test_paths]
Nathan Yonkee 9:d58e77ebd769 2244
Nathan Yonkee 9:d58e77ebd769 2245 test_build_path = os.path.join(build_path, test_paths[0])
Nathan Yonkee 9:d58e77ebd769 2246 src_paths = base_source_paths + test_paths
Nathan Yonkee 9:d58e77ebd769 2247 bin_file = None
Nathan Yonkee 9:d58e77ebd769 2248 test_case_folder_name = os.path.basename(test_paths[0])
Nathan Yonkee 9:d58e77ebd769 2249
Nathan Yonkee 9:d58e77ebd769 2250 args = (src_paths, test_build_path, target, toolchain_name)
Nathan Yonkee 9:d58e77ebd769 2251 kwargs = {
Nathan Yonkee 9:d58e77ebd769 2252 'jobs': 1,
Nathan Yonkee 9:d58e77ebd769 2253 'clean': clean,
Nathan Yonkee 9:d58e77ebd769 2254 'macros': macros,
Nathan Yonkee 9:d58e77ebd769 2255 'name': test_case_folder_name,
Nathan Yonkee 9:d58e77ebd769 2256 'project_id': test_name,
Nathan Yonkee 9:d58e77ebd769 2257 'report': report,
Nathan Yonkee 9:d58e77ebd769 2258 'properties': properties,
Nathan Yonkee 9:d58e77ebd769 2259 'verbose': verbose,
Nathan Yonkee 9:d58e77ebd769 2260 'app_config': app_config,
Nathan Yonkee 9:d58e77ebd769 2261 'build_profile': build_profile,
Nathan Yonkee 9:d58e77ebd769 2262 'silent': True,
Nathan Yonkee 9:d58e77ebd769 2263 'toolchain_paths': TOOLCHAIN_PATHS,
Nathan Yonkee 9:d58e77ebd769 2264 'stats_depth': stats_depth
Nathan Yonkee 9:d58e77ebd769 2265 }
Nathan Yonkee 9:d58e77ebd769 2266
Nathan Yonkee 9:d58e77ebd769 2267 results.append(p.apply_async(build_test_worker, args, kwargs))
Nathan Yonkee 9:d58e77ebd769 2268
Nathan Yonkee 9:d58e77ebd769 2269 p.close()
Nathan Yonkee 9:d58e77ebd769 2270 result = True
Nathan Yonkee 9:d58e77ebd769 2271 itr = 0
Nathan Yonkee 9:d58e77ebd769 2272 while len(results):
Nathan Yonkee 9:d58e77ebd769 2273 itr += 1
Nathan Yonkee 9:d58e77ebd769 2274 if itr > 360000:
Nathan Yonkee 9:d58e77ebd769 2275 p.terminate()
Nathan Yonkee 9:d58e77ebd769 2276 p.join()
Nathan Yonkee 9:d58e77ebd769 2277 raise ToolException("Compile did not finish in 10 minutes")
Nathan Yonkee 9:d58e77ebd769 2278 else:
Nathan Yonkee 9:d58e77ebd769 2279 sleep(0.01)
Nathan Yonkee 9:d58e77ebd769 2280 pending = 0
Nathan Yonkee 9:d58e77ebd769 2281 for r in results:
Nathan Yonkee 9:d58e77ebd769 2282 if r.ready() is True:
Nathan Yonkee 9:d58e77ebd769 2283 try:
Nathan Yonkee 9:d58e77ebd769 2284 worker_result = r.get()
Nathan Yonkee 9:d58e77ebd769 2285 results.remove(r)
Nathan Yonkee 9:d58e77ebd769 2286
Nathan Yonkee 9:d58e77ebd769 2287 # Take report from the kwargs and merge it into existing report
Nathan Yonkee 9:d58e77ebd769 2288 if report:
Nathan Yonkee 9:d58e77ebd769 2289 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
Nathan Yonkee 9:d58e77ebd769 2290 for test_key in report_entry.keys():
Nathan Yonkee 9:d58e77ebd769 2291 report[target_name][toolchain_name][test_key] = report_entry[test_key]
Nathan Yonkee 9:d58e77ebd769 2292
Nathan Yonkee 9:d58e77ebd769 2293 # Set the overall result to a failure if a build failure occurred
Nathan Yonkee 9:d58e77ebd769 2294 if ('reason' in worker_result and
Nathan Yonkee 9:d58e77ebd769 2295 not worker_result['reason'] and
Nathan Yonkee 9:d58e77ebd769 2296 not isinstance(worker_result['reason'], NotSupportedException)):
Nathan Yonkee 9:d58e77ebd769 2297 result = False
Nathan Yonkee 9:d58e77ebd769 2298 break
Nathan Yonkee 9:d58e77ebd769 2299
Nathan Yonkee 9:d58e77ebd769 2300 # Adding binary path to test build result
Nathan Yonkee 9:d58e77ebd769 2301 if ('result' in worker_result and
Nathan Yonkee 9:d58e77ebd769 2302 worker_result['result'] and
Nathan Yonkee 9:d58e77ebd769 2303 'bin_file' in worker_result):
Nathan Yonkee 9:d58e77ebd769 2304 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
Nathan Yonkee 9:d58e77ebd769 2305
Nathan Yonkee 9:d58e77ebd769 2306 test_build['tests'][worker_result['kwargs']['project_id']] = {
Nathan Yonkee 9:d58e77ebd769 2307 "binaries": [
Nathan Yonkee 9:d58e77ebd769 2308 {
Nathan Yonkee 9:d58e77ebd769 2309 "path": bin_file
Nathan Yonkee 9:d58e77ebd769 2310 }
Nathan Yonkee 9:d58e77ebd769 2311 ]
Nathan Yonkee 9:d58e77ebd769 2312 }
Nathan Yonkee 9:d58e77ebd769 2313
Nathan Yonkee 9:d58e77ebd769 2314 test_key = worker_result['kwargs']['project_id'].upper()
Nathan Yonkee 9:d58e77ebd769 2315 if report:
Nathan Yonkee 9:d58e77ebd769 2316 print(report[target_name][toolchain_name][test_key][0][0]['output'].rstrip())
Nathan Yonkee 9:d58e77ebd769 2317 print('Image: %s\n' % bin_file)
Nathan Yonkee 9:d58e77ebd769 2318
Nathan Yonkee 9:d58e77ebd769 2319 except:
Nathan Yonkee 9:d58e77ebd769 2320 if p._taskqueue.queue:
Nathan Yonkee 9:d58e77ebd769 2321 p._taskqueue.queue.clear()
Nathan Yonkee 9:d58e77ebd769 2322 sleep(0.5)
Nathan Yonkee 9:d58e77ebd769 2323 p.terminate()
Nathan Yonkee 9:d58e77ebd769 2324 p.join()
Nathan Yonkee 9:d58e77ebd769 2325 raise
Nathan Yonkee 9:d58e77ebd769 2326 else:
Nathan Yonkee 9:d58e77ebd769 2327 pending += 1
Nathan Yonkee 9:d58e77ebd769 2328 if pending >= jobs_count:
Nathan Yonkee 9:d58e77ebd769 2329 break
Nathan Yonkee 9:d58e77ebd769 2330
Nathan Yonkee 9:d58e77ebd769 2331 # Break as soon as possible if there is a failure and we are not
Nathan Yonkee 9:d58e77ebd769 2332 # continuing on build failures
Nathan Yonkee 9:d58e77ebd769 2333 if not result and not continue_on_build_fail:
Nathan Yonkee 9:d58e77ebd769 2334 if p._taskqueue.queue:
Nathan Yonkee 9:d58e77ebd769 2335 p._taskqueue.queue.clear()
Nathan Yonkee 9:d58e77ebd769 2336 sleep(0.5)
Nathan Yonkee 9:d58e77ebd769 2337 p.terminate()
Nathan Yonkee 9:d58e77ebd769 2338 break
Nathan Yonkee 9:d58e77ebd769 2339
Nathan Yonkee 9:d58e77ebd769 2340 p.join()
Nathan Yonkee 9:d58e77ebd769 2341
Nathan Yonkee 9:d58e77ebd769 2342 test_builds = {}
Nathan Yonkee 9:d58e77ebd769 2343 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
Nathan Yonkee 9:d58e77ebd769 2344
Nathan Yonkee 9:d58e77ebd769 2345 return result, test_builds
Nathan Yonkee 9:d58e77ebd769 2346
Nathan Yonkee 9:d58e77ebd769 2347
Nathan Yonkee 9:d58e77ebd769 2348 def test_spec_from_test_builds(test_builds):
Nathan Yonkee 9:d58e77ebd769 2349 return {
Nathan Yonkee 9:d58e77ebd769 2350 "builds": test_builds
Nathan Yonkee 9:d58e77ebd769 2351 }