ON Semiconductor / mbed-os

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Dependents:   mbed-TFT-example-NCS36510 mbed-Accelerometer-example-NCS36510 mbed-Accelerometer-example-NCS36510

tools/test_api.py@1:f30bdcd2b33b, 2017-02-27 (annotated)

Committer:
jacobjohnson
Date:
Mon Feb 27 17:45:05 2017 +0000
Revision:
1:f30bdcd2b33b
Parent:
0:098463de4c5d 
changed the inputscale from 1 to 7 in analogin_api.c.  This will need to be changed later, and accessed from the main level, but for now this allows the  adc to read a value from 0 to 3.7V, instead of just up to 1V.;

Who changed what in which revision?

UserRevisionLine numberNew contents of line
group-onsemi 0:098463de4c5d 1 """
group-onsemi 0:098463de4c5d 2 mbed SDK
group-onsemi 0:098463de4c5d 3 Copyright (c) 2011-2014 ARM Limited
group-onsemi 0:098463de4c5d 4
group-onsemi 0:098463de4c5d 5 Licensed under the Apache License, Version 2.0 (the "License");
group-onsemi 0:098463de4c5d 6 you may not use this file except in compliance with the License.
group-onsemi 0:098463de4c5d 7 You may obtain a copy of the License at
group-onsemi 0:098463de4c5d 8
group-onsemi 0:098463de4c5d 9 http://www.apache.org/licenses/LICENSE-2.0
group-onsemi 0:098463de4c5d 10
group-onsemi 0:098463de4c5d 11 Unless required by applicable law or agreed to in writing, software
group-onsemi 0:098463de4c5d 12 distributed under the License is distributed on an "AS IS" BASIS,
group-onsemi 0:098463de4c5d 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
group-onsemi 0:098463de4c5d 14 See the License for the specific language governing permissions and
group-onsemi 0:098463de4c5d 15 limitations under the License.
group-onsemi 0:098463de4c5d 16
group-onsemi 0:098463de4c5d 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
group-onsemi 0:098463de4c5d 18 """
group-onsemi 0:098463de4c5d 19
group-onsemi 0:098463de4c5d 20 import os
group-onsemi 0:098463de4c5d 21 import re
group-onsemi 0:098463de4c5d 22 import sys
group-onsemi 0:098463de4c5d 23 import json
group-onsemi 0:098463de4c5d 24 import uuid
group-onsemi 0:098463de4c5d 25 import pprint
group-onsemi 0:098463de4c5d 26 import random
group-onsemi 0:098463de4c5d 27 import argparse
group-onsemi 0:098463de4c5d 28 import datetime
group-onsemi 0:098463de4c5d 29 import threading
group-onsemi 0:098463de4c5d 30 import ctypes
group-onsemi 0:098463de4c5d 31 from types import ListType
group-onsemi 0:098463de4c5d 32 from colorama import Fore, Back, Style
group-onsemi 0:098463de4c5d 33 from prettytable import PrettyTable
group-onsemi 0:098463de4c5d 34 from copy import copy
group-onsemi 0:098463de4c5d 35
group-onsemi 0:098463de4c5d 36 from time import sleep, time
group-onsemi 0:098463de4c5d 37 from Queue import Queue, Empty
group-onsemi 0:098463de4c5d 38 from os.path import join, exists, basename, relpath
group-onsemi 0:098463de4c5d 39 from threading import Thread, Lock
group-onsemi 0:098463de4c5d 40 from multiprocessing import Pool, cpu_count
group-onsemi 0:098463de4c5d 41 from subprocess import Popen, PIPE
group-onsemi 0:098463de4c5d 42
group-onsemi 0:098463de4c5d 43 # Imports related to mbed build api
group-onsemi 0:098463de4c5d 44 from tools.tests import TESTS
group-onsemi 0:098463de4c5d 45 from tools.tests import TEST_MAP
group-onsemi 0:098463de4c5d 46 from tools.paths import BUILD_DIR
group-onsemi 0:098463de4c5d 47 from tools.paths import HOST_TESTS
group-onsemi 0:098463de4c5d 48 from tools.utils import ToolException
group-onsemi 0:098463de4c5d 49 from tools.utils import NotSupportedException
group-onsemi 0:098463de4c5d 50 from tools.utils import construct_enum
group-onsemi 0:098463de4c5d 51 from tools.memap import MemapParser
group-onsemi 0:098463de4c5d 52 from tools.targets import TARGET_MAP
group-onsemi 0:098463de4c5d 53 from tools.test_db import BaseDBAccess
group-onsemi 0:098463de4c5d 54 from tools.build_api import build_project, build_mbed_libs, build_lib
group-onsemi 0:098463de4c5d 55 from tools.build_api import get_target_supported_toolchains
group-onsemi 0:098463de4c5d 56 from tools.build_api import write_build_report
group-onsemi 0:098463de4c5d 57 from tools.build_api import prep_report
group-onsemi 0:098463de4c5d 58 from tools.build_api import prep_properties
group-onsemi 0:098463de4c5d 59 from tools.build_api import create_result
group-onsemi 0:098463de4c5d 60 from tools.build_api import add_result_to_report
group-onsemi 0:098463de4c5d 61 from tools.build_api import prepare_toolchain
group-onsemi 0:098463de4c5d 62 from tools.build_api import scan_resources
group-onsemi 0:098463de4c5d 63 from tools.build_api import get_config
group-onsemi 0:098463de4c5d 64 from tools.libraries import LIBRARIES, LIBRARY_MAP
group-onsemi 0:098463de4c5d 65 from tools.options import extract_profile
group-onsemi 0:098463de4c5d 66 from tools.toolchains import TOOLCHAIN_PATHS
group-onsemi 0:098463de4c5d 67 from tools.toolchains import TOOLCHAINS
group-onsemi 0:098463de4c5d 68 from tools.test_exporters import ReportExporter, ResultExporterType
group-onsemi 0:098463de4c5d 69 from tools.utils import argparse_filestring_type
group-onsemi 0:098463de4c5d 70 from tools.utils import argparse_uppercase_type
group-onsemi 0:098463de4c5d 71 from tools.utils import argparse_lowercase_type
group-onsemi 0:098463de4c5d 72 from tools.utils import argparse_many
group-onsemi 0:098463de4c5d 73 from tools.utils import get_path_depth
group-onsemi 0:098463de4c5d 74
group-onsemi 0:098463de4c5d 75 import tools.host_tests.host_tests_plugins as host_tests_plugins
group-onsemi 0:098463de4c5d 76
group-onsemi 0:098463de4c5d 77 try:
group-onsemi 0:098463de4c5d 78 import mbed_lstools
group-onsemi 0:098463de4c5d 79 from tools.compliance.ioper_runner import get_available_oper_test_scopes
group-onsemi 0:098463de4c5d 80 except:
group-onsemi 0:098463de4c5d 81 pass
group-onsemi 0:098463de4c5d 82
group-onsemi 0:098463de4c5d 83
group-onsemi 0:098463de4c5d 84 class ProcessObserver(Thread):
group-onsemi 0:098463de4c5d 85 def __init__(self, proc):
group-onsemi 0:098463de4c5d 86 Thread.__init__(self)
group-onsemi 0:098463de4c5d 87 self.proc = proc
group-onsemi 0:098463de4c5d 88 self.queue = Queue()
group-onsemi 0:098463de4c5d 89 self.daemon = True
group-onsemi 0:098463de4c5d 90 self.active = True
group-onsemi 0:098463de4c5d 91 self.start()
group-onsemi 0:098463de4c5d 92
group-onsemi 0:098463de4c5d 93 def run(self):
group-onsemi 0:098463de4c5d 94 while self.active:
group-onsemi 0:098463de4c5d 95 c = self.proc.stdout.read(1)
group-onsemi 0:098463de4c5d 96 self.queue.put(c)
group-onsemi 0:098463de4c5d 97
group-onsemi 0:098463de4c5d 98 def stop(self):
group-onsemi 0:098463de4c5d 99 self.active = False
group-onsemi 0:098463de4c5d 100 try:
group-onsemi 0:098463de4c5d 101 self.proc.terminate()
group-onsemi 0:098463de4c5d 102 except Exception, _:
group-onsemi 0:098463de4c5d 103 pass
group-onsemi 0:098463de4c5d 104
group-onsemi 0:098463de4c5d 105
group-onsemi 0:098463de4c5d 106 class SingleTestExecutor(threading.Thread):
group-onsemi 0:098463de4c5d 107 """ Example: Single test class in separate thread usage
group-onsemi 0:098463de4c5d 108 """
group-onsemi 0:098463de4c5d 109 def __init__(self, single_test):
group-onsemi 0:098463de4c5d 110 self.single_test = single_test
group-onsemi 0:098463de4c5d 111 threading.Thread.__init__(self)
group-onsemi 0:098463de4c5d 112
group-onsemi 0:098463de4c5d 113 def run(self):
group-onsemi 0:098463de4c5d 114 start = time()
group-onsemi 0:098463de4c5d 115 # Execute tests depending on options and filter applied
group-onsemi 0:098463de4c5d 116 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
group-onsemi 0:098463de4c5d 117 elapsed_time = time() - start
group-onsemi 0:098463de4c5d 118
group-onsemi 0:098463de4c5d 119 # Human readable summary
group-onsemi 0:098463de4c5d 120 if not self.single_test.opts_suppress_summary:
group-onsemi 0:098463de4c5d 121 # prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 122 print self.single_test.generate_test_summary(test_summary, shuffle_seed)
group-onsemi 0:098463de4c5d 123 if self.single_test.opts_test_x_toolchain_summary:
group-onsemi 0:098463de4c5d 124 # prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 125 # table shows text x toolchain test result matrix
group-onsemi 0:098463de4c5d 126 print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
group-onsemi 0:098463de4c5d 127 print "Completed in %.2f sec"% (elapsed_time)
group-onsemi 0:098463de4c5d 128
group-onsemi 0:098463de4c5d 129
group-onsemi 0:098463de4c5d 130 class SingleTestRunner(object):
group-onsemi 0:098463de4c5d 131 """ Object wrapper for single test run which may involve multiple MUTs
group-onsemi 0:098463de4c5d 132 """
group-onsemi 0:098463de4c5d 133 RE_DETECT_TESTCASE_RESULT = None
group-onsemi 0:098463de4c5d 134
group-onsemi 0:098463de4c5d 135 # Return codes for test script
group-onsemi 0:098463de4c5d 136 TEST_RESULT_OK = "OK"
group-onsemi 0:098463de4c5d 137 TEST_RESULT_FAIL = "FAIL"
group-onsemi 0:098463de4c5d 138 TEST_RESULT_ERROR = "ERROR"
group-onsemi 0:098463de4c5d 139 TEST_RESULT_UNDEF = "UNDEF"
group-onsemi 0:098463de4c5d 140 TEST_RESULT_IOERR_COPY = "IOERR_COPY"
group-onsemi 0:098463de4c5d 141 TEST_RESULT_IOERR_DISK = "IOERR_DISK"
group-onsemi 0:098463de4c5d 142 TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
group-onsemi 0:098463de4c5d 143 TEST_RESULT_TIMEOUT = "TIMEOUT"
group-onsemi 0:098463de4c5d 144 TEST_RESULT_NO_IMAGE = "NO_IMAGE"
group-onsemi 0:098463de4c5d 145 TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
group-onsemi 0:098463de4c5d 146 TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
group-onsemi 0:098463de4c5d 147 TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
group-onsemi 0:098463de4c5d 148
group-onsemi 0:098463de4c5d 149 GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
group-onsemi 0:098463de4c5d 150 TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
group-onsemi 0:098463de4c5d 151 TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
group-onsemi 0:098463de4c5d 152
group-onsemi 0:098463de4c5d 153 muts = {} # MUTs descriptor (from external file)
group-onsemi 0:098463de4c5d 154 test_spec = {} # Test specification (from external file)
group-onsemi 0:098463de4c5d 155
group-onsemi 0:098463de4c5d 156 # mbed test suite -> SingleTestRunner
group-onsemi 0:098463de4c5d 157 TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
group-onsemi 0:098463de4c5d 158 "failure" : TEST_RESULT_FAIL,
group-onsemi 0:098463de4c5d 159 "error" : TEST_RESULT_ERROR,
group-onsemi 0:098463de4c5d 160 "ioerr_copy" : TEST_RESULT_IOERR_COPY,
group-onsemi 0:098463de4c5d 161 "ioerr_disk" : TEST_RESULT_IOERR_DISK,
group-onsemi 0:098463de4c5d 162 "ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
group-onsemi 0:098463de4c5d 163 "timeout" : TEST_RESULT_TIMEOUT,
group-onsemi 0:098463de4c5d 164 "no_image" : TEST_RESULT_NO_IMAGE,
group-onsemi 0:098463de4c5d 165 "end" : TEST_RESULT_UNDEF,
group-onsemi 0:098463de4c5d 166 "mbed_assert" : TEST_RESULT_MBED_ASSERT,
group-onsemi 0:098463de4c5d 167 "build_failed" : TEST_RESULT_BUILD_FAILED,
group-onsemi 0:098463de4c5d 168 "not_supproted" : TEST_RESULT_NOT_SUPPORTED
group-onsemi 0:098463de4c5d 169 }
group-onsemi 0:098463de4c5d 170
group-onsemi 0:098463de4c5d 171 def __init__(self,
group-onsemi 0:098463de4c5d 172 _global_loops_count=1,
group-onsemi 0:098463de4c5d 173 _test_loops_list=None,
group-onsemi 0:098463de4c5d 174 _muts={},
group-onsemi 0:098463de4c5d 175 _clean=False,
group-onsemi 0:098463de4c5d 176 _parser=None,
group-onsemi 0:098463de4c5d 177 _opts=None,
group-onsemi 0:098463de4c5d 178 _opts_db_url=None,
group-onsemi 0:098463de4c5d 179 _opts_log_file_name=None,
group-onsemi 0:098463de4c5d 180 _opts_report_html_file_name=None,
group-onsemi 0:098463de4c5d 181 _opts_report_junit_file_name=None,
group-onsemi 0:098463de4c5d 182 _opts_report_build_file_name=None,
group-onsemi 0:098463de4c5d 183 _opts_report_text_file_name=None,
group-onsemi 0:098463de4c5d 184 _opts_build_report={},
group-onsemi 0:098463de4c5d 185 _opts_build_properties={},
group-onsemi 0:098463de4c5d 186 _test_spec={},
group-onsemi 0:098463de4c5d 187 _opts_goanna_for_mbed_sdk=None,
group-onsemi 0:098463de4c5d 188 _opts_goanna_for_tests=None,
group-onsemi 0:098463de4c5d 189 _opts_shuffle_test_order=False,
group-onsemi 0:098463de4c5d 190 _opts_shuffle_test_seed=None,
group-onsemi 0:098463de4c5d 191 _opts_test_by_names=None,
group-onsemi 0:098463de4c5d 192 _opts_peripheral_by_names=None,
group-onsemi 0:098463de4c5d 193 _opts_test_only_peripheral=False,
group-onsemi 0:098463de4c5d 194 _opts_test_only_common=False,
group-onsemi 0:098463de4c5d 195 _opts_verbose_skipped_tests=False,
group-onsemi 0:098463de4c5d 196 _opts_verbose_test_result_only=False,
group-onsemi 0:098463de4c5d 197 _opts_verbose=False,
group-onsemi 0:098463de4c5d 198 _opts_firmware_global_name=None,
group-onsemi 0:098463de4c5d 199 _opts_only_build_tests=False,
group-onsemi 0:098463de4c5d 200 _opts_parallel_test_exec=False,
group-onsemi 0:098463de4c5d 201 _opts_suppress_summary=False,
group-onsemi 0:098463de4c5d 202 _opts_test_x_toolchain_summary=False,
group-onsemi 0:098463de4c5d 203 _opts_copy_method=None,
group-onsemi 0:098463de4c5d 204 _opts_mut_reset_type=None,
group-onsemi 0:098463de4c5d 205 _opts_jobs=None,
group-onsemi 0:098463de4c5d 206 _opts_waterfall_test=None,
group-onsemi 0:098463de4c5d 207 _opts_consolidate_waterfall_test=None,
group-onsemi 0:098463de4c5d 208 _opts_extend_test_timeout=None,
group-onsemi 0:098463de4c5d 209 _opts_auto_detect=None,
group-onsemi 0:098463de4c5d 210 _opts_include_non_automated=False):
group-onsemi 0:098463de4c5d 211 """ Let's try hard to init this object
group-onsemi 0:098463de4c5d 212 """
group-onsemi 0:098463de4c5d 213 from colorama import init
group-onsemi 0:098463de4c5d 214 init()
group-onsemi 0:098463de4c5d 215
group-onsemi 0:098463de4c5d 216 PATTERN = "\\{(" + "|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
group-onsemi 0:098463de4c5d 217 self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
group-onsemi 0:098463de4c5d 218 # Settings related to test loops counters
group-onsemi 0:098463de4c5d 219 try:
group-onsemi 0:098463de4c5d 220 _global_loops_count = int(_global_loops_count)
group-onsemi 0:098463de4c5d 221 except:
group-onsemi 0:098463de4c5d 222 _global_loops_count = 1
group-onsemi 0:098463de4c5d 223 if _global_loops_count < 1:
group-onsemi 0:098463de4c5d 224 _global_loops_count = 1
group-onsemi 0:098463de4c5d 225 self.GLOBAL_LOOPS_COUNT = _global_loops_count
group-onsemi 0:098463de4c5d 226 self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
group-onsemi 0:098463de4c5d 227 self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
group-onsemi 0:098463de4c5d 228
group-onsemi 0:098463de4c5d 229 self.shuffle_random_seed = 0.0
group-onsemi 0:098463de4c5d 230 self.SHUFFLE_SEED_ROUND = 10
group-onsemi 0:098463de4c5d 231
group-onsemi 0:098463de4c5d 232 # MUT list and test specification storage
group-onsemi 0:098463de4c5d 233 self.muts = _muts
group-onsemi 0:098463de4c5d 234 self.test_spec = _test_spec
group-onsemi 0:098463de4c5d 235
group-onsemi 0:098463de4c5d 236 # Settings passed e.g. from command line
group-onsemi 0:098463de4c5d 237 self.opts_db_url = _opts_db_url
group-onsemi 0:098463de4c5d 238 self.opts_log_file_name = _opts_log_file_name
group-onsemi 0:098463de4c5d 239 self.opts_report_html_file_name = _opts_report_html_file_name
group-onsemi 0:098463de4c5d 240 self.opts_report_junit_file_name = _opts_report_junit_file_name
group-onsemi 0:098463de4c5d 241 self.opts_report_build_file_name = _opts_report_build_file_name
group-onsemi 0:098463de4c5d 242 self.opts_report_text_file_name = _opts_report_text_file_name
group-onsemi 0:098463de4c5d 243 self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
group-onsemi 0:098463de4c5d 244 self.opts_goanna_for_tests = _opts_goanna_for_tests
group-onsemi 0:098463de4c5d 245 self.opts_shuffle_test_order = _opts_shuffle_test_order
group-onsemi 0:098463de4c5d 246 self.opts_shuffle_test_seed = _opts_shuffle_test_seed
group-onsemi 0:098463de4c5d 247 self.opts_test_by_names = _opts_test_by_names
group-onsemi 0:098463de4c5d 248 self.opts_peripheral_by_names = _opts_peripheral_by_names
group-onsemi 0:098463de4c5d 249 self.opts_test_only_peripheral = _opts_test_only_peripheral
group-onsemi 0:098463de4c5d 250 self.opts_test_only_common = _opts_test_only_common
group-onsemi 0:098463de4c5d 251 self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
group-onsemi 0:098463de4c5d 252 self.opts_verbose_test_result_only = _opts_verbose_test_result_only
group-onsemi 0:098463de4c5d 253 self.opts_verbose = _opts_verbose
group-onsemi 0:098463de4c5d 254 self.opts_firmware_global_name = _opts_firmware_global_name
group-onsemi 0:098463de4c5d 255 self.opts_only_build_tests = _opts_only_build_tests
group-onsemi 0:098463de4c5d 256 self.opts_parallel_test_exec = _opts_parallel_test_exec
group-onsemi 0:098463de4c5d 257 self.opts_suppress_summary = _opts_suppress_summary
group-onsemi 0:098463de4c5d 258 self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
group-onsemi 0:098463de4c5d 259 self.opts_copy_method = _opts_copy_method
group-onsemi 0:098463de4c5d 260 self.opts_mut_reset_type = _opts_mut_reset_type
group-onsemi 0:098463de4c5d 261 self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
group-onsemi 0:098463de4c5d 262 self.opts_waterfall_test = _opts_waterfall_test
group-onsemi 0:098463de4c5d 263 self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
group-onsemi 0:098463de4c5d 264 self.opts_extend_test_timeout = _opts_extend_test_timeout
group-onsemi 0:098463de4c5d 265 self.opts_clean = _clean
group-onsemi 0:098463de4c5d 266 self.opts_parser = _parser
group-onsemi 0:098463de4c5d 267 self.opts = _opts
group-onsemi 0:098463de4c5d 268 self.opts_auto_detect = _opts_auto_detect
group-onsemi 0:098463de4c5d 269 self.opts_include_non_automated = _opts_include_non_automated
group-onsemi 0:098463de4c5d 270
group-onsemi 0:098463de4c5d 271 self.build_report = _opts_build_report
group-onsemi 0:098463de4c5d 272 self.build_properties = _opts_build_properties
group-onsemi 0:098463de4c5d 273
group-onsemi 0:098463de4c5d 274 # File / screen logger initialization
group-onsemi 0:098463de4c5d 275 self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
group-onsemi 0:098463de4c5d 276
group-onsemi 0:098463de4c5d 277 # Database related initializations
group-onsemi 0:098463de4c5d 278 self.db_logger = factory_db_logger(self.opts_db_url)
group-onsemi 0:098463de4c5d 279 self.db_logger_build_id = None # Build ID (database index of build_id table)
group-onsemi 0:098463de4c5d 280 # Let's connect to database to set up credentials and confirm database is ready
group-onsemi 0:098463de4c5d 281 if self.db_logger:
group-onsemi 0:098463de4c5d 282 self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
group-onsemi 0:098463de4c5d 283 if self.db_logger.is_connected():
group-onsemi 0:098463de4c5d 284 # Get hostname and uname so we can use it as build description
group-onsemi 0:098463de4c5d 285 # when creating new build_id in external database
group-onsemi 0:098463de4c5d 286 (_hostname, _uname) = self.db_logger.get_hostname()
group-onsemi 0:098463de4c5d 287 _host_location = os.path.dirname(os.path.abspath(__file__))
group-onsemi 0:098463de4c5d 288 build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
group-onsemi 0:098463de4c5d 289 self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
group-onsemi 0:098463de4c5d 290 self.db_logger.disconnect()
group-onsemi 0:098463de4c5d 291
group-onsemi 0:098463de4c5d 292 def dump_options(self):
group-onsemi 0:098463de4c5d 293 """ Function returns data structure with common settings passed to SingelTestRunner
group-onsemi 0:098463de4c5d 294 It can be used for example to fill _extra fields in database storing test suite single run data
group-onsemi 0:098463de4c5d 295 Example:
group-onsemi 0:098463de4c5d 296 data = self.dump_options()
group-onsemi 0:098463de4c5d 297 or
group-onsemi 0:098463de4c5d 298 data_str = json.dumps(self.dump_options())
group-onsemi 0:098463de4c5d 299 """
group-onsemi 0:098463de4c5d 300 result = {"db_url" : str(self.opts_db_url),
group-onsemi 0:098463de4c5d 301 "log_file_name" : str(self.opts_log_file_name),
group-onsemi 0:098463de4c5d 302 "shuffle_test_order" : str(self.opts_shuffle_test_order),
group-onsemi 0:098463de4c5d 303 "shuffle_test_seed" : str(self.opts_shuffle_test_seed),
group-onsemi 0:098463de4c5d 304 "test_by_names" : str(self.opts_test_by_names),
group-onsemi 0:098463de4c5d 305 "peripheral_by_names" : str(self.opts_peripheral_by_names),
group-onsemi 0:098463de4c5d 306 "test_only_peripheral" : str(self.opts_test_only_peripheral),
group-onsemi 0:098463de4c5d 307 "test_only_common" : str(self.opts_test_only_common),
group-onsemi 0:098463de4c5d 308 "verbose" : str(self.opts_verbose),
group-onsemi 0:098463de4c5d 309 "firmware_global_name" : str(self.opts_firmware_global_name),
group-onsemi 0:098463de4c5d 310 "only_build_tests" : str(self.opts_only_build_tests),
group-onsemi 0:098463de4c5d 311 "copy_method" : str(self.opts_copy_method),
group-onsemi 0:098463de4c5d 312 "mut_reset_type" : str(self.opts_mut_reset_type),
group-onsemi 0:098463de4c5d 313 "jobs" : str(self.opts_jobs),
group-onsemi 0:098463de4c5d 314 "extend_test_timeout" : str(self.opts_extend_test_timeout),
group-onsemi 0:098463de4c5d 315 "_dummy" : ''
group-onsemi 0:098463de4c5d 316 }
group-onsemi 0:098463de4c5d 317 return result
group-onsemi 0:098463de4c5d 318
group-onsemi 0:098463de4c5d 319 def shuffle_random_func(self):
group-onsemi 0:098463de4c5d 320 return self.shuffle_random_seed
group-onsemi 0:098463de4c5d 321
group-onsemi 0:098463de4c5d 322 def is_shuffle_seed_float(self):
group-onsemi 0:098463de4c5d 323 """ return true if function parameter can be converted to float
group-onsemi 0:098463de4c5d 324 """
group-onsemi 0:098463de4c5d 325 result = True
group-onsemi 0:098463de4c5d 326 try:
group-onsemi 0:098463de4c5d 327 float(self.shuffle_random_seed)
group-onsemi 0:098463de4c5d 328 except ValueError:
group-onsemi 0:098463de4c5d 329 result = False
group-onsemi 0:098463de4c5d 330 return result
group-onsemi 0:098463de4c5d 331
group-onsemi 0:098463de4c5d 332 # This will store target / toolchain specific properties
group-onsemi 0:098463de4c5d 333 test_suite_properties_ext = {} # target : toolchain
group-onsemi 0:098463de4c5d 334 # Here we store test results
group-onsemi 0:098463de4c5d 335 test_summary = []
group-onsemi 0:098463de4c5d 336 # Here we store test results in extended data structure
group-onsemi 0:098463de4c5d 337 test_summary_ext = {}
group-onsemi 0:098463de4c5d 338 execute_thread_slice_lock = Lock()
group-onsemi 0:098463de4c5d 339
group-onsemi 0:098463de4c5d 340 def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
group-onsemi 0:098463de4c5d 341 for toolchain in toolchains:
group-onsemi 0:098463de4c5d 342 tt_id = "%s::%s" % (toolchain, target)
group-onsemi 0:098463de4c5d 343
group-onsemi 0:098463de4c5d 344 T = TARGET_MAP[target]
group-onsemi 0:098463de4c5d 345
group-onsemi 0:098463de4c5d 346 # print target, toolchain
group-onsemi 0:098463de4c5d 347 # Test suite properties returned to external tools like CI
group-onsemi 0:098463de4c5d 348 test_suite_properties = {
group-onsemi 0:098463de4c5d 349 'jobs': self.opts_jobs,
group-onsemi 0:098463de4c5d 350 'clean': clean,
group-onsemi 0:098463de4c5d 351 'target': target,
group-onsemi 0:098463de4c5d 352 'vendor': T.extra_labels[0],
group-onsemi 0:098463de4c5d 353 'test_ids': ', '.join(test_ids),
group-onsemi 0:098463de4c5d 354 'toolchain': toolchain,
group-onsemi 0:098463de4c5d 355 'shuffle_random_seed': self.shuffle_random_seed
group-onsemi 0:098463de4c5d 356 }
group-onsemi 0:098463de4c5d 357
group-onsemi 0:098463de4c5d 358
group-onsemi 0:098463de4c5d 359 # print '=== %s::%s ===' % (target, toolchain)
group-onsemi 0:098463de4c5d 360 # Let's build our test
group-onsemi 0:098463de4c5d 361 if target not in TARGET_MAP:
group-onsemi 0:098463de4c5d 362 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
group-onsemi 0:098463de4c5d 363 continue
group-onsemi 0:098463de4c5d 364
group-onsemi 0:098463de4c5d 365 clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
group-onsemi 0:098463de4c5d 366
group-onsemi 0:098463de4c5d 367 profile = extract_profile(self.opts_parser, self.opts, toolchain)
group-onsemi 0:098463de4c5d 368
group-onsemi 0:098463de4c5d 369
group-onsemi 0:098463de4c5d 370 try:
group-onsemi 0:098463de4c5d 371 build_mbed_libs_result = build_mbed_libs(T,
group-onsemi 0:098463de4c5d 372 toolchain,
group-onsemi 0:098463de4c5d 373 clean=clean_mbed_libs_options,
group-onsemi 0:098463de4c5d 374 verbose=self.opts_verbose,
group-onsemi 0:098463de4c5d 375 jobs=self.opts_jobs,
group-onsemi 0:098463de4c5d 376 report=build_report,
group-onsemi 0:098463de4c5d 377 properties=build_properties,
group-onsemi 0:098463de4c5d 378 build_profile=profile)
group-onsemi 0:098463de4c5d 379
group-onsemi 0:098463de4c5d 380 if not build_mbed_libs_result:
group-onsemi 0:098463de4c5d 381 print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
group-onsemi 0:098463de4c5d 382 continue
group-onsemi 0:098463de4c5d 383
group-onsemi 0:098463de4c5d 384 except ToolException:
group-onsemi 0:098463de4c5d 385 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
group-onsemi 0:098463de4c5d 386 continue
group-onsemi 0:098463de4c5d 387
group-onsemi 0:098463de4c5d 388 build_dir = join(BUILD_DIR, "test", target, toolchain)
group-onsemi 0:098463de4c5d 389
group-onsemi 0:098463de4c5d 390 test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
group-onsemi 0:098463de4c5d 391 test_suite_properties['build_dir'] = build_dir
group-onsemi 0:098463de4c5d 392 test_suite_properties['skipped'] = []
group-onsemi 0:098463de4c5d 393
group-onsemi 0:098463de4c5d 394 # Enumerate through all tests and shuffle test order if requested
group-onsemi 0:098463de4c5d 395 test_map_keys = sorted(TEST_MAP.keys())
group-onsemi 0:098463de4c5d 396
group-onsemi 0:098463de4c5d 397 if self.opts_shuffle_test_order:
group-onsemi 0:098463de4c5d 398 random.shuffle(test_map_keys, self.shuffle_random_func)
group-onsemi 0:098463de4c5d 399 # Update database with shuffle seed f applicable
group-onsemi 0:098463de4c5d 400 if self.db_logger:
group-onsemi 0:098463de4c5d 401 self.db_logger.reconnect();
group-onsemi 0:098463de4c5d 402 if self.db_logger.is_connected():
group-onsemi 0:098463de4c5d 403 self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
group-onsemi 0:098463de4c5d 404 self.db_logger.disconnect();
group-onsemi 0:098463de4c5d 405
group-onsemi 0:098463de4c5d 406 if self.db_logger:
group-onsemi 0:098463de4c5d 407 self.db_logger.reconnect();
group-onsemi 0:098463de4c5d 408 if self.db_logger.is_connected():
group-onsemi 0:098463de4c5d 409 # Update MUTs and Test Specification in database
group-onsemi 0:098463de4c5d 410 self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
group-onsemi 0:098463de4c5d 411 # Update Extra information in database (some options passed to test suite)
group-onsemi 0:098463de4c5d 412 self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
group-onsemi 0:098463de4c5d 413 self.db_logger.disconnect();
group-onsemi 0:098463de4c5d 414
group-onsemi 0:098463de4c5d 415 valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
group-onsemi 0:098463de4c5d 416 skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
group-onsemi 0:098463de4c5d 417
group-onsemi 0:098463de4c5d 418 for skipped_test_id in skipped_test_map_keys:
group-onsemi 0:098463de4c5d 419 test_suite_properties['skipped'].append(skipped_test_id)
group-onsemi 0:098463de4c5d 420
group-onsemi 0:098463de4c5d 421
group-onsemi 0:098463de4c5d 422 # First pass through all tests and determine which libraries need to be built
group-onsemi 0:098463de4c5d 423 libraries = []
group-onsemi 0:098463de4c5d 424 for test_id in valid_test_map_keys:
group-onsemi 0:098463de4c5d 425 test = TEST_MAP[test_id]
group-onsemi 0:098463de4c5d 426
group-onsemi 0:098463de4c5d 427 # Detect which lib should be added to test
group-onsemi 0:098463de4c5d 428 # Some libs have to compiled like RTOS or ETH
group-onsemi 0:098463de4c5d 429 for lib in LIBRARIES:
group-onsemi 0:098463de4c5d 430 if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
group-onsemi 0:098463de4c5d 431 libraries.append(lib['id'])
group-onsemi 0:098463de4c5d 432
group-onsemi 0:098463de4c5d 433
group-onsemi 0:098463de4c5d 434 clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
group-onsemi 0:098463de4c5d 435
group-onsemi 0:098463de4c5d 436 # Build all required libraries
group-onsemi 0:098463de4c5d 437 for lib_id in libraries:
group-onsemi 0:098463de4c5d 438 try:
group-onsemi 0:098463de4c5d 439 build_lib(lib_id,
group-onsemi 0:098463de4c5d 440 T,
group-onsemi 0:098463de4c5d 441 toolchain,
group-onsemi 0:098463de4c5d 442 verbose=self.opts_verbose,
group-onsemi 0:098463de4c5d 443 clean=clean_mbed_libs_options,
group-onsemi 0:098463de4c5d 444 jobs=self.opts_jobs,
group-onsemi 0:098463de4c5d 445 report=build_report,
group-onsemi 0:098463de4c5d 446 properties=build_properties,
group-onsemi 0:098463de4c5d 447 build_profile=profile)
group-onsemi 0:098463de4c5d 448
group-onsemi 0:098463de4c5d 449 except ToolException:
group-onsemi 0:098463de4c5d 450 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
group-onsemi 0:098463de4c5d 451 continue
group-onsemi 0:098463de4c5d 452
group-onsemi 0:098463de4c5d 453
group-onsemi 0:098463de4c5d 454 for test_id in valid_test_map_keys:
group-onsemi 0:098463de4c5d 455 test = TEST_MAP[test_id]
group-onsemi 0:098463de4c5d 456
group-onsemi 0:098463de4c5d 457 test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
group-onsemi 0:098463de4c5d 458
group-onsemi 0:098463de4c5d 459 # TODO: move this 2 below loops to separate function
group-onsemi 0:098463de4c5d 460 INC_DIRS = []
group-onsemi 0:098463de4c5d 461 for lib_id in libraries:
group-onsemi 0:098463de4c5d 462 if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
group-onsemi 0:098463de4c5d 463 INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
group-onsemi 0:098463de4c5d 464
group-onsemi 0:098463de4c5d 465 MACROS = []
group-onsemi 0:098463de4c5d 466 for lib_id in libraries:
group-onsemi 0:098463de4c5d 467 if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
group-onsemi 0:098463de4c5d 468 MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
group-onsemi 0:098463de4c5d 469 MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
group-onsemi 0:098463de4c5d 470 MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
group-onsemi 0:098463de4c5d 471 test_uuid = uuid.uuid4()
group-onsemi 0:098463de4c5d 472 MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
group-onsemi 0:098463de4c5d 473
group-onsemi 0:098463de4c5d 474 # Prepare extended test results data structure (it can be used to generate detailed test report)
group-onsemi 0:098463de4c5d 475 if target not in self.test_summary_ext:
group-onsemi 0:098463de4c5d 476 self.test_summary_ext[target] = {} # test_summary_ext : toolchain
group-onsemi 0:098463de4c5d 477 if toolchain not in self.test_summary_ext[target]:
group-onsemi 0:098463de4c5d 478 self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
group-onsemi 0:098463de4c5d 479
group-onsemi 0:098463de4c5d 480 tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
group-onsemi 0:098463de4c5d 481
group-onsemi 0:098463de4c5d 482 project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
group-onsemi 0:098463de4c5d 483 try:
group-onsemi 0:098463de4c5d 484 path = build_project(test.source_dir,
group-onsemi 0:098463de4c5d 485 join(build_dir, test_id),
group-onsemi 0:098463de4c5d 486 T,
group-onsemi 0:098463de4c5d 487 toolchain,
group-onsemi 0:098463de4c5d 488 test.dependencies,
group-onsemi 0:098463de4c5d 489 clean=clean_project_options,
group-onsemi 0:098463de4c5d 490 verbose=self.opts_verbose,
group-onsemi 0:098463de4c5d 491 name=project_name,
group-onsemi 0:098463de4c5d 492 macros=MACROS,
group-onsemi 0:098463de4c5d 493 inc_dirs=INC_DIRS,
group-onsemi 0:098463de4c5d 494 jobs=self.opts_jobs,
group-onsemi 0:098463de4c5d 495 report=build_report,
group-onsemi 0:098463de4c5d 496 properties=build_properties,
group-onsemi 0:098463de4c5d 497 project_id=test_id,
group-onsemi 0:098463de4c5d 498 project_description=test.get_description(),
group-onsemi 0:098463de4c5d 499 build_profile=profile)
group-onsemi 0:098463de4c5d 500
group-onsemi 0:098463de4c5d 501 except Exception, e:
group-onsemi 0:098463de4c5d 502 project_name_str = project_name if project_name is not None else test_id
group-onsemi 0:098463de4c5d 503
group-onsemi 0:098463de4c5d 504
group-onsemi 0:098463de4c5d 505 test_result = self.TEST_RESULT_FAIL
group-onsemi 0:098463de4c5d 506
group-onsemi 0:098463de4c5d 507 if isinstance(e, ToolException):
group-onsemi 0:098463de4c5d 508 print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
group-onsemi 0:098463de4c5d 509 test_result = self.TEST_RESULT_BUILD_FAILED
group-onsemi 0:098463de4c5d 510 elif isinstance(e, NotSupportedException):
group-onsemi 0:098463de4c5d 511 print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
group-onsemi 0:098463de4c5d 512 test_result = self.TEST_RESULT_NOT_SUPPORTED
group-onsemi 0:098463de4c5d 513
group-onsemi 0:098463de4c5d 514
group-onsemi 0:098463de4c5d 515 # Append test results to global test summary
group-onsemi 0:098463de4c5d 516 self.test_summary.append(
group-onsemi 0:098463de4c5d 517 (test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
group-onsemi 0:098463de4c5d 518 )
group-onsemi 0:098463de4c5d 519
group-onsemi 0:098463de4c5d 520 # Add detailed test result to test summary structure
group-onsemi 0:098463de4c5d 521 if test_id not in self.test_summary_ext[target][toolchain]:
group-onsemi 0:098463de4c5d 522 self.test_summary_ext[target][toolchain][test_id] = []
group-onsemi 0:098463de4c5d 523
group-onsemi 0:098463de4c5d 524 self.test_summary_ext[target][toolchain][test_id].append({ 0: {
group-onsemi 0:098463de4c5d 525 'result' : test_result,
group-onsemi 0:098463de4c5d 526 'output' : '',
group-onsemi 0:098463de4c5d 527 'target_name' : target,
group-onsemi 0:098463de4c5d 528 'target_name_unique': target,
group-onsemi 0:098463de4c5d 529 'toolchain_name' : toolchain,
group-onsemi 0:098463de4c5d 530 'id' : test_id,
group-onsemi 0:098463de4c5d 531 'description' : test.get_description(),
group-onsemi 0:098463de4c5d 532 'elapsed_time' : 0,
group-onsemi 0:098463de4c5d 533 'duration' : 0,
group-onsemi 0:098463de4c5d 534 'copy_method' : None
group-onsemi 0:098463de4c5d 535 }})
group-onsemi 0:098463de4c5d 536 continue
group-onsemi 0:098463de4c5d 537
group-onsemi 0:098463de4c5d 538 if self.opts_only_build_tests:
group-onsemi 0:098463de4c5d 539 # With this option we are skipping testing phase
group-onsemi 0:098463de4c5d 540 continue
group-onsemi 0:098463de4c5d 541
group-onsemi 0:098463de4c5d 542 # Test duration can be increased by global value
group-onsemi 0:098463de4c5d 543 test_duration = test.duration
group-onsemi 0:098463de4c5d 544 if self.opts_extend_test_timeout is not None:
group-onsemi 0:098463de4c5d 545 test_duration += self.opts_extend_test_timeout
group-onsemi 0:098463de4c5d 546
group-onsemi 0:098463de4c5d 547 # For an automated test the duration act as a timeout after
group-onsemi 0:098463de4c5d 548 # which the test gets interrupted
group-onsemi 0:098463de4c5d 549 test_spec = self.shape_test_request(target, path, test_id, test_duration)
group-onsemi 0:098463de4c5d 550 test_loops = self.get_test_loop_count(test_id)
group-onsemi 0:098463de4c5d 551
group-onsemi 0:098463de4c5d 552 test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
group-onsemi 0:098463de4c5d 553 test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
group-onsemi 0:098463de4c5d 554 test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
group-onsemi 0:098463de4c5d 555
group-onsemi 0:098463de4c5d 556 # read MUTs, test specification and perform tests
group-onsemi 0:098463de4c5d 557 handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
group-onsemi 0:098463de4c5d 558
group-onsemi 0:098463de4c5d 559 if handle_results is None:
group-onsemi 0:098463de4c5d 560 continue
group-onsemi 0:098463de4c5d 561
group-onsemi 0:098463de4c5d 562 for handle_result in handle_results:
group-onsemi 0:098463de4c5d 563 if handle_result:
group-onsemi 0:098463de4c5d 564 single_test_result, detailed_test_results = handle_result
group-onsemi 0:098463de4c5d 565 else:
group-onsemi 0:098463de4c5d 566 continue
group-onsemi 0:098463de4c5d 567
group-onsemi 0:098463de4c5d 568 # Append test results to global test summary
group-onsemi 0:098463de4c5d 569 if single_test_result is not None:
group-onsemi 0:098463de4c5d 570 self.test_summary.append(single_test_result)
group-onsemi 0:098463de4c5d 571
group-onsemi 0:098463de4c5d 572 # Add detailed test result to test summary structure
group-onsemi 0:098463de4c5d 573 if target not in self.test_summary_ext[target][toolchain]:
group-onsemi 0:098463de4c5d 574 if test_id not in self.test_summary_ext[target][toolchain]:
group-onsemi 0:098463de4c5d 575 self.test_summary_ext[target][toolchain][test_id] = []
group-onsemi 0:098463de4c5d 576
group-onsemi 0:098463de4c5d 577 append_test_result = detailed_test_results
group-onsemi 0:098463de4c5d 578
group-onsemi 0:098463de4c5d 579 # If waterfall and consolidate-waterfall options are enabled,
group-onsemi 0:098463de4c5d 580 # only include the last test result in the report.
group-onsemi 0:098463de4c5d 581 if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
group-onsemi 0:098463de4c5d 582 append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
group-onsemi 0:098463de4c5d 583
group-onsemi 0:098463de4c5d 584 self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
group-onsemi 0:098463de4c5d 585
group-onsemi 0:098463de4c5d 586 test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
group-onsemi 0:098463de4c5d 587 self.test_suite_properties_ext[target][toolchain] = test_suite_properties
group-onsemi 0:098463de4c5d 588
group-onsemi 0:098463de4c5d 589 q.put(target + '_'.join(toolchains))
group-onsemi 0:098463de4c5d 590 return
group-onsemi 0:098463de4c5d 591
group-onsemi 0:098463de4c5d 592 def execute(self):
group-onsemi 0:098463de4c5d 593 clean = self.test_spec.get('clean', False)
group-onsemi 0:098463de4c5d 594 test_ids = self.test_spec.get('test_ids', [])
group-onsemi 0:098463de4c5d 595 q = Queue()
group-onsemi 0:098463de4c5d 596
group-onsemi 0:098463de4c5d 597 # Generate seed for shuffle if seed is not provided in
group-onsemi 0:098463de4c5d 598 self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
group-onsemi 0:098463de4c5d 599 if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
group-onsemi 0:098463de4c5d 600 self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
group-onsemi 0:098463de4c5d 601
group-onsemi 0:098463de4c5d 602
group-onsemi 0:098463de4c5d 603 if self.opts_parallel_test_exec:
group-onsemi 0:098463de4c5d 604 ###################################################################
group-onsemi 0:098463de4c5d 605 # Experimental, parallel test execution per singletest instance.
group-onsemi 0:098463de4c5d 606 ###################################################################
group-onsemi 0:098463de4c5d 607 execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
group-onsemi 0:098463de4c5d 608 # Note: We are building here in parallel for each target separately!
group-onsemi 0:098463de4c5d 609 # So we are not building the same thing multiple times and compilers
group-onsemi 0:098463de4c5d 610 # in separate threads do not collide.
group-onsemi 0:098463de4c5d 611 # Inside execute_thread_slice() function function handle() will be called to
group-onsemi 0:098463de4c5d 612 # get information about available MUTs (per target).
group-onsemi 0:098463de4c5d 613 for target, toolchains in self.test_spec['targets'].iteritems():
group-onsemi 0:098463de4c5d 614 self.test_suite_properties_ext[target] = {}
group-onsemi 0:098463de4c5d 615 t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
group-onsemi 0:098463de4c5d 616 t.daemon = True
group-onsemi 0:098463de4c5d 617 t.start()
group-onsemi 0:098463de4c5d 618 execute_threads.append(t)
group-onsemi 0:098463de4c5d 619
group-onsemi 0:098463de4c5d 620 for t in execute_threads:
group-onsemi 0:098463de4c5d 621 q.get() # t.join() would block some threads because we should not wait in any order for thread end
group-onsemi 0:098463de4c5d 622 else:
group-onsemi 0:098463de4c5d 623 # Serialized (not parallel) test execution
group-onsemi 0:098463de4c5d 624 for target, toolchains in self.test_spec['targets'].iteritems():
group-onsemi 0:098463de4c5d 625 if target not in self.test_suite_properties_ext:
group-onsemi 0:098463de4c5d 626 self.test_suite_properties_ext[target] = {}
group-onsemi 0:098463de4c5d 627
group-onsemi 0:098463de4c5d 628 self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
group-onsemi 0:098463de4c5d 629 q.get()
group-onsemi 0:098463de4c5d 630
group-onsemi 0:098463de4c5d 631 if self.db_logger:
group-onsemi 0:098463de4c5d 632 self.db_logger.reconnect();
group-onsemi 0:098463de4c5d 633 if self.db_logger.is_connected():
group-onsemi 0:098463de4c5d 634 self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
group-onsemi 0:098463de4c5d 635 self.db_logger.disconnect();
group-onsemi 0:098463de4c5d 636
group-onsemi 0:098463de4c5d 637 return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
group-onsemi 0:098463de4c5d 638
group-onsemi 0:098463de4c5d 639 def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
group-onsemi 0:098463de4c5d 640 valid_test_map_keys = []
group-onsemi 0:098463de4c5d 641
group-onsemi 0:098463de4c5d 642 for test_id in test_map_keys:
group-onsemi 0:098463de4c5d 643 test = TEST_MAP[test_id]
group-onsemi 0:098463de4c5d 644 if self.opts_test_by_names and test_id not in self.opts_test_by_names:
group-onsemi 0:098463de4c5d 645 continue
group-onsemi 0:098463de4c5d 646
group-onsemi 0:098463de4c5d 647 if test_ids and test_id not in test_ids:
group-onsemi 0:098463de4c5d 648 continue
group-onsemi 0:098463de4c5d 649
group-onsemi 0:098463de4c5d 650 if self.opts_test_only_peripheral and not test.peripherals:
group-onsemi 0:098463de4c5d 651 if self.opts_verbose_skipped_tests:
group-onsemi 0:098463de4c5d 652 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
group-onsemi 0:098463de4c5d 653 continue
group-onsemi 0:098463de4c5d 654
group-onsemi 0:098463de4c5d 655 if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
group-onsemi 0:098463de4c5d 656 # We will skip tests not forced with -p option
group-onsemi 0:098463de4c5d 657 if self.opts_verbose_skipped_tests:
group-onsemi 0:098463de4c5d 658 print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
group-onsemi 0:098463de4c5d 659 continue
group-onsemi 0:098463de4c5d 660
group-onsemi 0:098463de4c5d 661 if self.opts_test_only_common and test.peripherals:
group-onsemi 0:098463de4c5d 662 if self.opts_verbose_skipped_tests:
group-onsemi 0:098463de4c5d 663 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
group-onsemi 0:098463de4c5d 664 continue
group-onsemi 0:098463de4c5d 665
group-onsemi 0:098463de4c5d 666 if not include_non_automated and not test.automated:
group-onsemi 0:098463de4c5d 667 if self.opts_verbose_skipped_tests:
group-onsemi 0:098463de4c5d 668 print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
group-onsemi 0:098463de4c5d 669 continue
group-onsemi 0:098463de4c5d 670
group-onsemi 0:098463de4c5d 671 if test.is_supported(target, toolchain):
group-onsemi 0:098463de4c5d 672 if test.peripherals is None and self.opts_only_build_tests:
group-onsemi 0:098463de4c5d 673 # When users are using 'build only flag' and test do not have
group-onsemi 0:098463de4c5d 674 # specified peripherals we can allow test building by default
group-onsemi 0:098463de4c5d 675 pass
group-onsemi 0:098463de4c5d 676 elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
group-onsemi 0:098463de4c5d 677 # If we force peripheral with option -p we expect test
group-onsemi 0:098463de4c5d 678 # to pass even if peripheral is not in MUTs file.
group-onsemi 0:098463de4c5d 679 pass
group-onsemi 0:098463de4c5d 680 elif not self.is_peripherals_available(target, test.peripherals):
group-onsemi 0:098463de4c5d 681 if self.opts_verbose_skipped_tests:
group-onsemi 0:098463de4c5d 682 if test.peripherals:
group-onsemi 0:098463de4c5d 683 print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
group-onsemi 0:098463de4c5d 684 else:
group-onsemi 0:098463de4c5d 685 print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
group-onsemi 0:098463de4c5d 686 continue
group-onsemi 0:098463de4c5d 687
group-onsemi 0:098463de4c5d 688 # The test has made it through all the filters, so add it to the valid tests list
group-onsemi 0:098463de4c5d 689 valid_test_map_keys.append(test_id)
group-onsemi 0:098463de4c5d 690
group-onsemi 0:098463de4c5d 691 return valid_test_map_keys
group-onsemi 0:098463de4c5d 692
group-onsemi 0:098463de4c5d 693 def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
group-onsemi 0:098463de4c5d 694 # NOTE: This will not preserve order
group-onsemi 0:098463de4c5d 695 return list(set(all_test_map_keys) - set(valid_test_map_keys))
group-onsemi 0:098463de4c5d 696
group-onsemi 0:098463de4c5d 697 def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
group-onsemi 0:098463de4c5d 698 """ Prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 699 table shows text x toolchain test result matrix
group-onsemi 0:098463de4c5d 700 """
group-onsemi 0:098463de4c5d 701 RESULT_INDEX = 0
group-onsemi 0:098463de4c5d 702 TARGET_INDEX = 1
group-onsemi 0:098463de4c5d 703 TOOLCHAIN_INDEX = 2
group-onsemi 0:098463de4c5d 704 TEST_INDEX = 3
group-onsemi 0:098463de4c5d 705 DESC_INDEX = 4
group-onsemi 0:098463de4c5d 706
group-onsemi 0:098463de4c5d 707 unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
group-onsemi 0:098463de4c5d 708 unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
group-onsemi 0:098463de4c5d 709 unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
group-onsemi 0:098463de4c5d 710 unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
group-onsemi 0:098463de4c5d 711
group-onsemi 0:098463de4c5d 712 result = "Test summary:\n"
group-onsemi 0:098463de4c5d 713 for target in unique_targets:
group-onsemi 0:098463de4c5d 714 result_dict = {} # test : { toolchain : result }
group-onsemi 0:098463de4c5d 715 unique_target_toolchains = []
group-onsemi 0:098463de4c5d 716 for test in test_summary:
group-onsemi 0:098463de4c5d 717 if test[TARGET_INDEX] == target:
group-onsemi 0:098463de4c5d 718 if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
group-onsemi 0:098463de4c5d 719 unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
group-onsemi 0:098463de4c5d 720 if test[TEST_INDEX] not in result_dict:
group-onsemi 0:098463de4c5d 721 result_dict[test[TEST_INDEX]] = {}
group-onsemi 0:098463de4c5d 722 result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
group-onsemi 0:098463de4c5d 723
group-onsemi 0:098463de4c5d 724 pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
group-onsemi 0:098463de4c5d 725 pt = PrettyTable(pt_cols)
group-onsemi 0:098463de4c5d 726 for col in pt_cols:
group-onsemi 0:098463de4c5d 727 pt.align[col] = "l"
group-onsemi 0:098463de4c5d 728 pt.padding_width = 1 # One space between column edges and contents (default)
group-onsemi 0:098463de4c5d 729
group-onsemi 0:098463de4c5d 730 for test in unique_tests:
group-onsemi 0:098463de4c5d 731 if test in result_dict:
group-onsemi 0:098463de4c5d 732 test_results = result_dict[test]
group-onsemi 0:098463de4c5d 733 if test in unique_test_desc:
group-onsemi 0:098463de4c5d 734 row = [target, test, unique_test_desc[test]]
group-onsemi 0:098463de4c5d 735 for toolchain in unique_toolchains:
group-onsemi 0:098463de4c5d 736 if toolchain in test_results:
group-onsemi 0:098463de4c5d 737 row.append(test_results[toolchain])
group-onsemi 0:098463de4c5d 738 pt.add_row(row)
group-onsemi 0:098463de4c5d 739 result += pt.get_string()
group-onsemi 0:098463de4c5d 740 shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
group-onsemi 0:098463de4c5d 741 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
group-onsemi 0:098463de4c5d 742 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
group-onsemi 0:098463de4c5d 743 return result
group-onsemi 0:098463de4c5d 744
group-onsemi 0:098463de4c5d 745 def generate_test_summary(self, test_summary, shuffle_seed=None):
group-onsemi 0:098463de4c5d 746 """ Prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 747 table shows target x test results matrix across
group-onsemi 0:098463de4c5d 748 """
group-onsemi 0:098463de4c5d 749 success_code = 0 # Success code that can be leter returned to
group-onsemi 0:098463de4c5d 750 result = "Test summary:\n"
group-onsemi 0:098463de4c5d 751 # Pretty table package is used to print results
group-onsemi 0:098463de4c5d 752 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
group-onsemi 0:098463de4c5d 753 "Elapsed Time (sec)", "Timeout (sec)", "Loops"])
group-onsemi 0:098463de4c5d 754 pt.align["Result"] = "l" # Left align
group-onsemi 0:098463de4c5d 755 pt.align["Target"] = "l" # Left align
group-onsemi 0:098463de4c5d 756 pt.align["Toolchain"] = "l" # Left align
group-onsemi 0:098463de4c5d 757 pt.align["Test ID"] = "l" # Left align
group-onsemi 0:098463de4c5d 758 pt.align["Test Description"] = "l" # Left align
group-onsemi 0:098463de4c5d 759 pt.padding_width = 1 # One space between column edges and contents (default)
group-onsemi 0:098463de4c5d 760
group-onsemi 0:098463de4c5d 761 result_dict = {self.TEST_RESULT_OK : 0,
group-onsemi 0:098463de4c5d 762 self.TEST_RESULT_FAIL : 0,
group-onsemi 0:098463de4c5d 763 self.TEST_RESULT_ERROR : 0,
group-onsemi 0:098463de4c5d 764 self.TEST_RESULT_UNDEF : 0,
group-onsemi 0:098463de4c5d 765 self.TEST_RESULT_IOERR_COPY : 0,
group-onsemi 0:098463de4c5d 766 self.TEST_RESULT_IOERR_DISK : 0,
group-onsemi 0:098463de4c5d 767 self.TEST_RESULT_IOERR_SERIAL : 0,
group-onsemi 0:098463de4c5d 768 self.TEST_RESULT_NO_IMAGE : 0,
group-onsemi 0:098463de4c5d 769 self.TEST_RESULT_TIMEOUT : 0,
group-onsemi 0:098463de4c5d 770 self.TEST_RESULT_MBED_ASSERT : 0,
group-onsemi 0:098463de4c5d 771 self.TEST_RESULT_BUILD_FAILED : 0,
group-onsemi 0:098463de4c5d 772 self.TEST_RESULT_NOT_SUPPORTED : 0
group-onsemi 0:098463de4c5d 773 }
group-onsemi 0:098463de4c5d 774
group-onsemi 0:098463de4c5d 775 for test in test_summary:
group-onsemi 0:098463de4c5d 776 if test[0] in result_dict:
group-onsemi 0:098463de4c5d 777 result_dict[test[0]] += 1
group-onsemi 0:098463de4c5d 778 pt.add_row(test)
group-onsemi 0:098463de4c5d 779 result += pt.get_string()
group-onsemi 0:098463de4c5d 780 result += "\n"
group-onsemi 0:098463de4c5d 781
group-onsemi 0:098463de4c5d 782 # Print result count
group-onsemi 0:098463de4c5d 783 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
group-onsemi 0:098463de4c5d 784 shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
group-onsemi 0:098463de4c5d 785 shuffle_seed if shuffle_seed else self.shuffle_random_seed)
group-onsemi 0:098463de4c5d 786 result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
group-onsemi 0:098463de4c5d 787 return result
group-onsemi 0:098463de4c5d 788
group-onsemi 0:098463de4c5d 789 def test_loop_list_to_dict(self, test_loops_str):
group-onsemi 0:098463de4c5d 790 """ Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
group-onsemi 0:098463de4c5d 791 """
group-onsemi 0:098463de4c5d 792 result = {}
group-onsemi 0:098463de4c5d 793 if test_loops_str:
group-onsemi 0:098463de4c5d 794 test_loops = test_loops_str
group-onsemi 0:098463de4c5d 795 for test_loop in test_loops:
group-onsemi 0:098463de4c5d 796 test_loop_count = test_loop.split('=')
group-onsemi 0:098463de4c5d 797 if len(test_loop_count) == 2:
group-onsemi 0:098463de4c5d 798 _test_id, _test_loops = test_loop_count
group-onsemi 0:098463de4c5d 799 try:
group-onsemi 0:098463de4c5d 800 _test_loops = int(_test_loops)
group-onsemi 0:098463de4c5d 801 except:
group-onsemi 0:098463de4c5d 802 continue
group-onsemi 0:098463de4c5d 803 result[_test_id] = _test_loops
group-onsemi 0:098463de4c5d 804 return result
group-onsemi 0:098463de4c5d 805
group-onsemi 0:098463de4c5d 806 def get_test_loop_count(self, test_id):
group-onsemi 0:098463de4c5d 807 """ This function returns no. of loops per test (deducted by test_id_.
group-onsemi 0:098463de4c5d 808 If test is not in list of redefined loop counts it will use default value.
group-onsemi 0:098463de4c5d 809 """
group-onsemi 0:098463de4c5d 810 result = self.GLOBAL_LOOPS_COUNT
group-onsemi 0:098463de4c5d 811 if test_id in self.TEST_LOOPS_DICT:
group-onsemi 0:098463de4c5d 812 result = self.TEST_LOOPS_DICT[test_id]
group-onsemi 0:098463de4c5d 813 return result
group-onsemi 0:098463de4c5d 814
group-onsemi 0:098463de4c5d 815 def delete_file(self, file_path):
group-onsemi 0:098463de4c5d 816 """ Remove file from the system
group-onsemi 0:098463de4c5d 817 """
group-onsemi 0:098463de4c5d 818 result = True
group-onsemi 0:098463de4c5d 819 resutl_msg = ""
group-onsemi 0:098463de4c5d 820 try:
group-onsemi 0:098463de4c5d 821 os.remove(file_path)
group-onsemi 0:098463de4c5d 822 except Exception, e:
group-onsemi 0:098463de4c5d 823 resutl_msg = e
group-onsemi 0:098463de4c5d 824 result = False
group-onsemi 0:098463de4c5d 825 return result, resutl_msg
group-onsemi 0:098463de4c5d 826
group-onsemi 0:098463de4c5d 827 def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
group-onsemi 0:098463de4c5d 828 """ Test is being invoked for given MUT.
group-onsemi 0:098463de4c5d 829 """
group-onsemi 0:098463de4c5d 830 # Get test information, image and test timeout
group-onsemi 0:098463de4c5d 831 test_id = data['test_id']
group-onsemi 0:098463de4c5d 832 test = TEST_MAP[test_id]
group-onsemi 0:098463de4c5d 833 test_description = TEST_MAP[test_id].get_description()
group-onsemi 0:098463de4c5d 834 image = data["image"]
group-onsemi 0:098463de4c5d 835 duration = data.get("duration", 10)
group-onsemi 0:098463de4c5d 836
group-onsemi 0:098463de4c5d 837 if mut is None:
group-onsemi 0:098463de4c5d 838 print "Error: No Mbed available: MUT[%s]" % data['mcu']
group-onsemi 0:098463de4c5d 839 return None
group-onsemi 0:098463de4c5d 840
group-onsemi 0:098463de4c5d 841 mcu = mut['mcu']
group-onsemi 0:098463de4c5d 842 copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
group-onsemi 0:098463de4c5d 843
group-onsemi 0:098463de4c5d 844 if self.db_logger:
group-onsemi 0:098463de4c5d 845 self.db_logger.reconnect()
group-onsemi 0:098463de4c5d 846
group-onsemi 0:098463de4c5d 847 selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
group-onsemi 0:098463de4c5d 848
group-onsemi 0:098463de4c5d 849 # Tests can be looped so test results must be stored for the same test
group-onsemi 0:098463de4c5d 850 test_all_result = []
group-onsemi 0:098463de4c5d 851 # Test results for one test ran few times
group-onsemi 0:098463de4c5d 852 detailed_test_results = {} # { Loop_number: { results ... } }
group-onsemi 0:098463de4c5d 853
group-onsemi 0:098463de4c5d 854 for test_index in range(test_loops):
group-onsemi 0:098463de4c5d 855
group-onsemi 0:098463de4c5d 856 # If mbedls is available and we are auto detecting MUT info,
group-onsemi 0:098463de4c5d 857 # update MUT info (mounting may changed)
group-onsemi 0:098463de4c5d 858 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
group-onsemi 0:098463de4c5d 859 platform_name_filter = [mcu]
group-onsemi 0:098463de4c5d 860 muts_list = {}
group-onsemi 0:098463de4c5d 861 found = False
group-onsemi 0:098463de4c5d 862
group-onsemi 0:098463de4c5d 863 for i in range(0, 60):
group-onsemi 0:098463de4c5d 864 print('Looking for %s with MBEDLS' % mcu)
group-onsemi 0:098463de4c5d 865 muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
group-onsemi 0:098463de4c5d 866
group-onsemi 0:098463de4c5d 867 if 1 not in muts_list:
group-onsemi 0:098463de4c5d 868 sleep(3)
group-onsemi 0:098463de4c5d 869 else:
group-onsemi 0:098463de4c5d 870 found = True
group-onsemi 0:098463de4c5d 871 break
group-onsemi 0:098463de4c5d 872
group-onsemi 0:098463de4c5d 873 if not found:
group-onsemi 0:098463de4c5d 874 print "Error: mbed not found with MBEDLS: %s" % data['mcu']
group-onsemi 0:098463de4c5d 875 return None
group-onsemi 0:098463de4c5d 876 else:
group-onsemi 0:098463de4c5d 877 mut = muts_list[1]
group-onsemi 0:098463de4c5d 878
group-onsemi 0:098463de4c5d 879 disk = mut.get('disk')
group-onsemi 0:098463de4c5d 880 port = mut.get('port')
group-onsemi 0:098463de4c5d 881
group-onsemi 0:098463de4c5d 882 if disk is None or port is None:
group-onsemi 0:098463de4c5d 883 return None
group-onsemi 0:098463de4c5d 884
group-onsemi 0:098463de4c5d 885 target_by_mcu = TARGET_MAP[mut['mcu']]
group-onsemi 0:098463de4c5d 886 target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
group-onsemi 0:098463de4c5d 887 # Some extra stuff can be declared in MUTs structure
group-onsemi 0:098463de4c5d 888 reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
group-onsemi 0:098463de4c5d 889 reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
group-onsemi 0:098463de4c5d 890
group-onsemi 0:098463de4c5d 891 # When the build and test system were separate, this was relative to a
group-onsemi 0:098463de4c5d 892 # base network folder base path: join(NETWORK_BASE_PATH, )
group-onsemi 0:098463de4c5d 893 image_path = image
group-onsemi 0:098463de4c5d 894
group-onsemi 0:098463de4c5d 895 # Host test execution
group-onsemi 0:098463de4c5d 896 start_host_exec_time = time()
group-onsemi 0:098463de4c5d 897
group-onsemi 0:098463de4c5d 898 single_test_result = self.TEST_RESULT_UNDEF # single test run result
group-onsemi 0:098463de4c5d 899 _copy_method = selected_copy_method
group-onsemi 0:098463de4c5d 900
group-onsemi 0:098463de4c5d 901 if not exists(image_path):
group-onsemi 0:098463de4c5d 902 single_test_result = self.TEST_RESULT_NO_IMAGE
group-onsemi 0:098463de4c5d 903 elapsed_time = 0
group-onsemi 0:098463de4c5d 904 single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
group-onsemi 0:098463de4c5d 905 print single_test_output
group-onsemi 0:098463de4c5d 906 else:
group-onsemi 0:098463de4c5d 907 # Host test execution
group-onsemi 0:098463de4c5d 908 start_host_exec_time = time()
group-onsemi 0:098463de4c5d 909
group-onsemi 0:098463de4c5d 910 host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
group-onsemi 0:098463de4c5d 911 host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
group-onsemi 0:098463de4c5d 912 host_test_result = self.run_host_test(test.host_test,
group-onsemi 0:098463de4c5d 913 image_path, disk, port, duration,
group-onsemi 0:098463de4c5d 914 micro=target_name,
group-onsemi 0:098463de4c5d 915 verbose=host_test_verbose,
group-onsemi 0:098463de4c5d 916 reset=host_test_reset,
group-onsemi 0:098463de4c5d 917 reset_tout=reset_tout,
group-onsemi 0:098463de4c5d 918 copy_method=selected_copy_method,
group-onsemi 0:098463de4c5d 919 program_cycle_s=target_by_mcu.program_cycle_s)
group-onsemi 0:098463de4c5d 920 single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
group-onsemi 0:098463de4c5d 921
group-onsemi 0:098463de4c5d 922 # Store test result
group-onsemi 0:098463de4c5d 923 test_all_result.append(single_test_result)
group-onsemi 0:098463de4c5d 924 total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
group-onsemi 0:098463de4c5d 925 elapsed_time = single_testduration # TIme of single test case execution after reset
group-onsemi 0:098463de4c5d 926
group-onsemi 0:098463de4c5d 927 detailed_test_results[test_index] = {
group-onsemi 0:098463de4c5d 928 'result' : single_test_result,
group-onsemi 0:098463de4c5d 929 'output' : single_test_output,
group-onsemi 0:098463de4c5d 930 'target_name' : target_name,
group-onsemi 0:098463de4c5d 931 'target_name_unique' : target_name_unique,
group-onsemi 0:098463de4c5d 932 'toolchain_name' : toolchain_name,
group-onsemi 0:098463de4c5d 933 'id' : test_id,
group-onsemi 0:098463de4c5d 934 'description' : test_description,
group-onsemi 0:098463de4c5d 935 'elapsed_time' : round(elapsed_time, 2),
group-onsemi 0:098463de4c5d 936 'duration' : single_timeout,
group-onsemi 0:098463de4c5d 937 'copy_method' : _copy_method,
group-onsemi 0:098463de4c5d 938 }
group-onsemi 0:098463de4c5d 939
group-onsemi 0:098463de4c5d 940 print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
group-onsemi 0:098463de4c5d 941 test_id, test_description, elapsed_time, single_timeout)
group-onsemi 0:098463de4c5d 942
group-onsemi 0:098463de4c5d 943 # Update database entries for ongoing test
group-onsemi 0:098463de4c5d 944 if self.db_logger and self.db_logger.is_connected():
group-onsemi 0:098463de4c5d 945 test_type = 'SingleTest'
group-onsemi 0:098463de4c5d 946 self.db_logger.insert_test_entry(self.db_logger_build_id,
group-onsemi 0:098463de4c5d 947 target_name,
group-onsemi 0:098463de4c5d 948 toolchain_name,
group-onsemi 0:098463de4c5d 949 test_type,
group-onsemi 0:098463de4c5d 950 test_id,
group-onsemi 0:098463de4c5d 951 single_test_result,
group-onsemi 0:098463de4c5d 952 single_test_output,
group-onsemi 0:098463de4c5d 953 elapsed_time,
group-onsemi 0:098463de4c5d 954 single_timeout,
group-onsemi 0:098463de4c5d 955 test_index)
group-onsemi 0:098463de4c5d 956
group-onsemi 0:098463de4c5d 957 # If we perform waterfall test we test until we get OK and we stop testing
group-onsemi 0:098463de4c5d 958 if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
group-onsemi 0:098463de4c5d 959 break
group-onsemi 0:098463de4c5d 960
group-onsemi 0:098463de4c5d 961 if self.db_logger:
group-onsemi 0:098463de4c5d 962 self.db_logger.disconnect()
group-onsemi 0:098463de4c5d 963
group-onsemi 0:098463de4c5d 964 return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
group-onsemi 0:098463de4c5d 965 target_name_unique,
group-onsemi 0:098463de4c5d 966 toolchain_name,
group-onsemi 0:098463de4c5d 967 test_id,
group-onsemi 0:098463de4c5d 968 test_description,
group-onsemi 0:098463de4c5d 969 round(elapsed_time, 2),
group-onsemi 0:098463de4c5d 970 single_timeout,
group-onsemi 0:098463de4c5d 971 self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
group-onsemi 0:098463de4c5d 972
group-onsemi 0:098463de4c5d 973 def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
group-onsemi 0:098463de4c5d 974 """ Function determines MUT's mbed disk/port and copies binary to
group-onsemi 0:098463de4c5d 975 target.
group-onsemi 0:098463de4c5d 976 """
group-onsemi 0:098463de4c5d 977 handle_results = []
group-onsemi 0:098463de4c5d 978 data = json.loads(test_spec)
group-onsemi 0:098463de4c5d 979
group-onsemi 0:098463de4c5d 980 # Find a suitable MUT:
group-onsemi 0:098463de4c5d 981 mut = None
group-onsemi 0:098463de4c5d 982 for id, m in self.muts.iteritems():
group-onsemi 0:098463de4c5d 983 if m['mcu'] == data['mcu']:
group-onsemi 0:098463de4c5d 984 mut = m
group-onsemi 0:098463de4c5d 985 handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
group-onsemi 0:098463de4c5d 986 handle_results.append(handle_result)
group-onsemi 0:098463de4c5d 987
group-onsemi 0:098463de4c5d 988 return handle_results
group-onsemi 0:098463de4c5d 989
group-onsemi 0:098463de4c5d 990 def print_test_result(self, test_result, target_name, toolchain_name,
group-onsemi 0:098463de4c5d 991 test_id, test_description, elapsed_time, duration):
group-onsemi 0:098463de4c5d 992 """ Use specific convention to print test result and related data
group-onsemi 0:098463de4c5d 993 """
group-onsemi 0:098463de4c5d 994 tokens = []
group-onsemi 0:098463de4c5d 995 tokens.append("TargetTest")
group-onsemi 0:098463de4c5d 996 tokens.append(target_name)
group-onsemi 0:098463de4c5d 997 tokens.append(toolchain_name)
group-onsemi 0:098463de4c5d 998 tokens.append(test_id)
group-onsemi 0:098463de4c5d 999 tokens.append(test_description)
group-onsemi 0:098463de4c5d 1000 separator = "::"
group-onsemi 0:098463de4c5d 1001 time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
group-onsemi 0:098463de4c5d 1002 result = separator.join(tokens) + " [" + test_result +"]" + time_info
group-onsemi 0:098463de4c5d 1003 return Fore.MAGENTA + result + Fore.RESET
group-onsemi 0:098463de4c5d 1004
group-onsemi 0:098463de4c5d 1005 def shape_test_loop_ok_result_count(self, test_all_result):
group-onsemi 0:098463de4c5d 1006 """ Reformats list of results to simple string
group-onsemi 0:098463de4c5d 1007 """
group-onsemi 0:098463de4c5d 1008 test_loop_count = len(test_all_result)
group-onsemi 0:098463de4c5d 1009 test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
group-onsemi 0:098463de4c5d 1010 return "%d/%d"% (test_loop_ok_result, test_loop_count)
group-onsemi 0:098463de4c5d 1011
group-onsemi 0:098463de4c5d 1012 def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
group-onsemi 0:098463de4c5d 1013 """ Reformats list of results to simple string
group-onsemi 0:098463de4c5d 1014 """
group-onsemi 0:098463de4c5d 1015 result = self.TEST_RESULT_FAIL
group-onsemi 0:098463de4c5d 1016
group-onsemi 0:098463de4c5d 1017 if all(test_all_result[0] == res for res in test_all_result):
group-onsemi 0:098463de4c5d 1018 result = test_all_result[0]
group-onsemi 0:098463de4c5d 1019 elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
group-onsemi 0:098463de4c5d 1020 result = self.TEST_RESULT_OK
group-onsemi 0:098463de4c5d 1021
group-onsemi 0:098463de4c5d 1022 return result
group-onsemi 0:098463de4c5d 1023
group-onsemi 0:098463de4c5d 1024 def run_host_test(self, name, image_path, disk, port, duration,
group-onsemi 0:098463de4c5d 1025 micro=None, reset=None, reset_tout=None,
group-onsemi 0:098463de4c5d 1026 verbose=False, copy_method=None, program_cycle_s=None):
group-onsemi 0:098463de4c5d 1027 """ Function creates new process with host test configured with particular test case.
group-onsemi 0:098463de4c5d 1028 Function also is pooling for serial port activity from process to catch all data
group-onsemi 0:098463de4c5d 1029 printed by test runner and host test during test execution
group-onsemi 0:098463de4c5d 1030 """
group-onsemi 0:098463de4c5d 1031
group-onsemi 0:098463de4c5d 1032 def get_char_from_queue(obs):
group-onsemi 0:098463de4c5d 1033 """ Get character from queue safe way
group-onsemi 0:098463de4c5d 1034 """
group-onsemi 0:098463de4c5d 1035 try:
group-onsemi 0:098463de4c5d 1036 c = obs.queue.get(block=True, timeout=0.5)
group-onsemi 0:098463de4c5d 1037 except Empty, _:
group-onsemi 0:098463de4c5d 1038 c = None
group-onsemi 0:098463de4c5d 1039 return c
group-onsemi 0:098463de4c5d 1040
group-onsemi 0:098463de4c5d 1041 def filter_queue_char(c):
group-onsemi 0:098463de4c5d 1042 """ Filters out non ASCII characters from serial port
group-onsemi 0:098463de4c5d 1043 """
group-onsemi 0:098463de4c5d 1044 if ord(c) not in range(128):
group-onsemi 0:098463de4c5d 1045 c = ' '
group-onsemi 0:098463de4c5d 1046 return c
group-onsemi 0:098463de4c5d 1047
group-onsemi 0:098463de4c5d 1048 def get_test_result(output):
group-onsemi 0:098463de4c5d 1049 """ Parse test 'output' data
group-onsemi 0:098463de4c5d 1050 """
group-onsemi 0:098463de4c5d 1051 result = self.TEST_RESULT_TIMEOUT
group-onsemi 0:098463de4c5d 1052 for line in "".join(output).splitlines():
group-onsemi 0:098463de4c5d 1053 search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
group-onsemi 0:098463de4c5d 1054 if search_result and len(search_result.groups()):
group-onsemi 0:098463de4c5d 1055 result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
group-onsemi 0:098463de4c5d 1056 break
group-onsemi 0:098463de4c5d 1057 return result
group-onsemi 0:098463de4c5d 1058
group-onsemi 0:098463de4c5d 1059 def get_auto_property_value(property_name, line):
group-onsemi 0:098463de4c5d 1060 """ Scans auto detection line from MUT and returns scanned parameter 'property_name'
group-onsemi 0:098463de4c5d 1061 Returns string
group-onsemi 0:098463de4c5d 1062 """
group-onsemi 0:098463de4c5d 1063 result = None
group-onsemi 0:098463de4c5d 1064 if re.search("HOST: Property '%s'"% property_name, line) is not None:
group-onsemi 0:098463de4c5d 1065 property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
group-onsemi 0:098463de4c5d 1066 if property is not None and len(property.groups()) == 1:
group-onsemi 0:098463de4c5d 1067 result = property.groups()[0]
group-onsemi 0:098463de4c5d 1068 return result
group-onsemi 0:098463de4c5d 1069
group-onsemi 0:098463de4c5d 1070 # print "{%s} port:%s disk:%s" % (name, port, disk),
group-onsemi 0:098463de4c5d 1071 cmd = ["python",
group-onsemi 0:098463de4c5d 1072 '%s.py'% name,
group-onsemi 0:098463de4c5d 1073 '-d', disk,
group-onsemi 0:098463de4c5d 1074 '-f', '"%s"'% image_path,
group-onsemi 0:098463de4c5d 1075 '-p', port,
group-onsemi 0:098463de4c5d 1076 '-t', str(duration),
group-onsemi 0:098463de4c5d 1077 '-C', str(program_cycle_s)]
group-onsemi 0:098463de4c5d 1078
group-onsemi 0:098463de4c5d 1079 if get_module_avail('mbed_lstools') and self.opts_auto_detect:
group-onsemi 0:098463de4c5d 1080 cmd += ['--auto']
group-onsemi 0:098463de4c5d 1081
group-onsemi 0:098463de4c5d 1082 # Add extra parameters to host_test
group-onsemi 0:098463de4c5d 1083 if copy_method is not None:
group-onsemi 0:098463de4c5d 1084 cmd += ["-c", copy_method]
group-onsemi 0:098463de4c5d 1085 if micro is not None:
group-onsemi 0:098463de4c5d 1086 cmd += ["-m", micro]
group-onsemi 0:098463de4c5d 1087 if reset is not None:
group-onsemi 0:098463de4c5d 1088 cmd += ["-r", reset]
group-onsemi 0:098463de4c5d 1089 if reset_tout is not None:
group-onsemi 0:098463de4c5d 1090 cmd += ["-R", str(reset_tout)]
group-onsemi 0:098463de4c5d 1091
group-onsemi 0:098463de4c5d 1092 if verbose:
group-onsemi 0:098463de4c5d 1093 print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
group-onsemi 0:098463de4c5d 1094 print "Test::Output::Start"
group-onsemi 0:098463de4c5d 1095
group-onsemi 0:098463de4c5d 1096 proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
group-onsemi 0:098463de4c5d 1097 obs = ProcessObserver(proc)
group-onsemi 0:098463de4c5d 1098 update_once_flag = {} # Stores flags checking if some auto-parameter was already set
group-onsemi 0:098463de4c5d 1099 line = ''
group-onsemi 0:098463de4c5d 1100 output = []
group-onsemi 0:098463de4c5d 1101 start_time = time()
group-onsemi 0:098463de4c5d 1102 while (time() - start_time) < (2 * duration):
group-onsemi 0:098463de4c5d 1103 c = get_char_from_queue(obs)
group-onsemi 0:098463de4c5d 1104 if c:
group-onsemi 0:098463de4c5d 1105 if verbose:
group-onsemi 0:098463de4c5d 1106 sys.stdout.write(c)
group-onsemi 0:098463de4c5d 1107 c = filter_queue_char(c)
group-onsemi 0:098463de4c5d 1108 output.append(c)
group-onsemi 0:098463de4c5d 1109 # Give the mbed under test a way to communicate the end of the test
group-onsemi 0:098463de4c5d 1110 if c in ['\n', '\r']:
group-onsemi 0:098463de4c5d 1111
group-onsemi 0:098463de4c5d 1112 # Checking for auto-detection information from the test about MUT reset moment
group-onsemi 0:098463de4c5d 1113 if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
group-onsemi 0:098463de4c5d 1114 # We will update this marker only once to prevent multiple time resets
group-onsemi 0:098463de4c5d 1115 update_once_flag['reset_target'] = True
group-onsemi 0:098463de4c5d 1116 start_time = time()
group-onsemi 0:098463de4c5d 1117
group-onsemi 0:098463de4c5d 1118 # Checking for auto-detection information from the test about timeout
group-onsemi 0:098463de4c5d 1119 auto_timeout_val = get_auto_property_value('timeout', line)
group-onsemi 0:098463de4c5d 1120 if 'timeout' not in update_once_flag and auto_timeout_val is not None:
group-onsemi 0:098463de4c5d 1121 # We will update this marker only once to prevent multiple time resets
group-onsemi 0:098463de4c5d 1122 update_once_flag['timeout'] = True
group-onsemi 0:098463de4c5d 1123 duration = int(auto_timeout_val)
group-onsemi 0:098463de4c5d 1124
group-onsemi 0:098463de4c5d 1125 # Detect mbed assert:
group-onsemi 0:098463de4c5d 1126 if 'mbed assertation failed: ' in line:
group-onsemi 0:098463de4c5d 1127 output.append('{{mbed_assert}}')
group-onsemi 0:098463de4c5d 1128 break
group-onsemi 0:098463de4c5d 1129
group-onsemi 0:098463de4c5d 1130 # Check for test end
group-onsemi 0:098463de4c5d 1131 if '{end}' in line:
group-onsemi 0:098463de4c5d 1132 break
group-onsemi 0:098463de4c5d 1133 line = ''
group-onsemi 0:098463de4c5d 1134 else:
group-onsemi 0:098463de4c5d 1135 line += c
group-onsemi 0:098463de4c5d 1136 end_time = time()
group-onsemi 0:098463de4c5d 1137 testcase_duration = end_time - start_time # Test case duration from reset to {end}
group-onsemi 0:098463de4c5d 1138
group-onsemi 0:098463de4c5d 1139 c = get_char_from_queue(obs)
group-onsemi 0:098463de4c5d 1140
group-onsemi 0:098463de4c5d 1141 if c:
group-onsemi 0:098463de4c5d 1142 if verbose:
group-onsemi 0:098463de4c5d 1143 sys.stdout.write(c)
group-onsemi 0:098463de4c5d 1144 c = filter_queue_char(c)
group-onsemi 0:098463de4c5d 1145 output.append(c)
group-onsemi 0:098463de4c5d 1146
group-onsemi 0:098463de4c5d 1147 if verbose:
group-onsemi 0:098463de4c5d 1148 print "Test::Output::Finish"
group-onsemi 0:098463de4c5d 1149 # Stop test process
group-onsemi 0:098463de4c5d 1150 obs.stop()
group-onsemi 0:098463de4c5d 1151
group-onsemi 0:098463de4c5d 1152 result = get_test_result(output)
group-onsemi 0:098463de4c5d 1153 return (result, "".join(output), testcase_duration, duration)
group-onsemi 0:098463de4c5d 1154
group-onsemi 0:098463de4c5d 1155 def is_peripherals_available(self, target_mcu_name, peripherals=None):
group-onsemi 0:098463de4c5d 1156 """ Checks if specified target should run specific peripheral test case defined in MUTs file
group-onsemi 0:098463de4c5d 1157 """
group-onsemi 0:098463de4c5d 1158 if peripherals is not None:
group-onsemi 0:098463de4c5d 1159 peripherals = set(peripherals)
group-onsemi 0:098463de4c5d 1160 for id, mut in self.muts.iteritems():
group-onsemi 0:098463de4c5d 1161 # Target MCU name check
group-onsemi 0:098463de4c5d 1162 if mut["mcu"] != target_mcu_name:
group-onsemi 0:098463de4c5d 1163 continue
group-onsemi 0:098463de4c5d 1164 # Peripherals check
group-onsemi 0:098463de4c5d 1165 if peripherals is not None:
group-onsemi 0:098463de4c5d 1166 if 'peripherals' not in mut:
group-onsemi 0:098463de4c5d 1167 continue
group-onsemi 0:098463de4c5d 1168 if not peripherals.issubset(set(mut['peripherals'])):
group-onsemi 0:098463de4c5d 1169 continue
group-onsemi 0:098463de4c5d 1170 return True
group-onsemi 0:098463de4c5d 1171 return False
group-onsemi 0:098463de4c5d 1172
group-onsemi 0:098463de4c5d 1173 def shape_test_request(self, mcu, image_path, test_id, duration=10):
group-onsemi 0:098463de4c5d 1174 """ Function prepares JSON structure describing test specification
group-onsemi 0:098463de4c5d 1175 """
group-onsemi 0:098463de4c5d 1176 test_spec = {
group-onsemi 0:098463de4c5d 1177 "mcu": mcu,
group-onsemi 0:098463de4c5d 1178 "image": image_path,
group-onsemi 0:098463de4c5d 1179 "duration": duration,
group-onsemi 0:098463de4c5d 1180 "test_id": test_id,
group-onsemi 0:098463de4c5d 1181 }
group-onsemi 0:098463de4c5d 1182 return json.dumps(test_spec)
group-onsemi 0:098463de4c5d 1183
group-onsemi 0:098463de4c5d 1184
group-onsemi 0:098463de4c5d 1185 def get_unique_value_from_summary(test_summary, index):
group-onsemi 0:098463de4c5d 1186 """ Gets list of unique target names
group-onsemi 0:098463de4c5d 1187 """
group-onsemi 0:098463de4c5d 1188 result = []
group-onsemi 0:098463de4c5d 1189 for test in test_summary:
group-onsemi 0:098463de4c5d 1190 target_name = test[index]
group-onsemi 0:098463de4c5d 1191 if target_name not in result:
group-onsemi 0:098463de4c5d 1192 result.append(target_name)
group-onsemi 0:098463de4c5d 1193 return sorted(result)
group-onsemi 0:098463de4c5d 1194
group-onsemi 0:098463de4c5d 1195
group-onsemi 0:098463de4c5d 1196 def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
group-onsemi 0:098463de4c5d 1197 """ Gets list of unique target names and return dictionary
group-onsemi 0:098463de4c5d 1198 """
group-onsemi 0:098463de4c5d 1199 result = {}
group-onsemi 0:098463de4c5d 1200 for test in test_summary:
group-onsemi 0:098463de4c5d 1201 key = test[index_key]
group-onsemi 0:098463de4c5d 1202 val = test[index_val]
group-onsemi 0:098463de4c5d 1203 if key not in result:
group-onsemi 0:098463de4c5d 1204 result[key] = val
group-onsemi 0:098463de4c5d 1205 return result
group-onsemi 0:098463de4c5d 1206
group-onsemi 0:098463de4c5d 1207
group-onsemi 0:098463de4c5d 1208 def show_json_file_format_error(json_spec_filename, line, column):
group-onsemi 0:098463de4c5d 1209 """ Prints JSON broken content
group-onsemi 0:098463de4c5d 1210 """
group-onsemi 0:098463de4c5d 1211 with open(json_spec_filename) as data_file:
group-onsemi 0:098463de4c5d 1212 line_no = 1
group-onsemi 0:098463de4c5d 1213 for json_line in data_file:
group-onsemi 0:098463de4c5d 1214 if line_no + 5 >= line: # Print last few lines before error
group-onsemi 0:098463de4c5d 1215 print 'Line %d:\t'%line_no + json_line, # Prints line
group-onsemi 0:098463de4c5d 1216 if line_no == line:
group-onsemi 0:098463de4c5d 1217 print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
group-onsemi 0:098463de4c5d 1218 break
group-onsemi 0:098463de4c5d 1219 line_no += 1
group-onsemi 0:098463de4c5d 1220
group-onsemi 0:098463de4c5d 1221
group-onsemi 0:098463de4c5d 1222 def json_format_error_defect_pos(json_error_msg):
group-onsemi 0:098463de4c5d 1223 """ Gets first error line and column in JSON file format.
group-onsemi 0:098463de4c5d 1224 Parsed from exception thrown by json.loads() string
group-onsemi 0:098463de4c5d 1225 """
group-onsemi 0:098463de4c5d 1226 result = None
group-onsemi 0:098463de4c5d 1227 line, column = 0, 0
group-onsemi 0:098463de4c5d 1228 # Line value search
group-onsemi 0:098463de4c5d 1229 line_search = re.search('line [0-9]+', json_error_msg)
group-onsemi 0:098463de4c5d 1230 if line_search is not None:
group-onsemi 0:098463de4c5d 1231 ls = line_search.group().split(' ')
group-onsemi 0:098463de4c5d 1232 if len(ls) == 2:
group-onsemi 0:098463de4c5d 1233 line = int(ls[1])
group-onsemi 0:098463de4c5d 1234 # Column position search
group-onsemi 0:098463de4c5d 1235 column_search = re.search('column [0-9]+', json_error_msg)
group-onsemi 0:098463de4c5d 1236 if column_search is not None:
group-onsemi 0:098463de4c5d 1237 cs = column_search.group().split(' ')
group-onsemi 0:098463de4c5d 1238 if len(cs) == 2:
group-onsemi 0:098463de4c5d 1239 column = int(cs[1])
group-onsemi 0:098463de4c5d 1240 result = [line, column]
group-onsemi 0:098463de4c5d 1241 return result
group-onsemi 0:098463de4c5d 1242
group-onsemi 0:098463de4c5d 1243
group-onsemi 0:098463de4c5d 1244 def get_json_data_from_file(json_spec_filename, verbose=False):
group-onsemi 0:098463de4c5d 1245 """ Loads from file JSON formatted string to data structure
group-onsemi 0:098463de4c5d 1246 """
group-onsemi 0:098463de4c5d 1247 result = None
group-onsemi 0:098463de4c5d 1248 try:
group-onsemi 0:098463de4c5d 1249 with open(json_spec_filename) as data_file:
group-onsemi 0:098463de4c5d 1250 try:
group-onsemi 0:098463de4c5d 1251 result = json.load(data_file)
group-onsemi 0:098463de4c5d 1252 except ValueError as json_error_msg:
group-onsemi 0:098463de4c5d 1253 result = None
group-onsemi 0:098463de4c5d 1254 print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
group-onsemi 0:098463de4c5d 1255 # We can print where error occurred inside JSON file if we can parse exception msg
group-onsemi 0:098463de4c5d 1256 json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
group-onsemi 0:098463de4c5d 1257 if json_format_defect_pos is not None:
group-onsemi 0:098463de4c5d 1258 line = json_format_defect_pos[0]
group-onsemi 0:098463de4c5d 1259 column = json_format_defect_pos[1]
group-onsemi 0:098463de4c5d 1260 print
group-onsemi 0:098463de4c5d 1261 show_json_file_format_error(json_spec_filename, line, column)
group-onsemi 0:098463de4c5d 1262
group-onsemi 0:098463de4c5d 1263 except IOError as fileopen_error_msg:
group-onsemi 0:098463de4c5d 1264 print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
group-onsemi 0:098463de4c5d 1265 print
group-onsemi 0:098463de4c5d 1266 if verbose and result:
group-onsemi 0:098463de4c5d 1267 pp = pprint.PrettyPrinter(indent=4)
group-onsemi 0:098463de4c5d 1268 pp.pprint(result)
group-onsemi 0:098463de4c5d 1269 return result
group-onsemi 0:098463de4c5d 1270
group-onsemi 0:098463de4c5d 1271
group-onsemi 0:098463de4c5d 1272 def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
group-onsemi 0:098463de4c5d 1273 """ Prints MUTs configuration passed to test script for verboseness
group-onsemi 0:098463de4c5d 1274 """
group-onsemi 0:098463de4c5d 1275 muts_info_cols = []
group-onsemi 0:098463de4c5d 1276 # We need to check all unique properties for each defined MUT
group-onsemi 0:098463de4c5d 1277 for k in json_data:
group-onsemi 0:098463de4c5d 1278 mut_info = json_data[k]
group-onsemi 0:098463de4c5d 1279 for mut_property in mut_info:
group-onsemi 0:098463de4c5d 1280 if mut_property not in muts_info_cols:
group-onsemi 0:098463de4c5d 1281 muts_info_cols.append(mut_property)
group-onsemi 0:098463de4c5d 1282
group-onsemi 0:098463de4c5d 1283 # Prepare pretty table object to display all MUTs
group-onsemi 0:098463de4c5d 1284 pt_cols = ["index"] + muts_info_cols
group-onsemi 0:098463de4c5d 1285 pt = PrettyTable(pt_cols)
group-onsemi 0:098463de4c5d 1286 for col in pt_cols:
group-onsemi 0:098463de4c5d 1287 pt.align[col] = "l"
group-onsemi 0:098463de4c5d 1288
group-onsemi 0:098463de4c5d 1289 # Add rows to pretty print object
group-onsemi 0:098463de4c5d 1290 for k in json_data:
group-onsemi 0:098463de4c5d 1291 row = [k]
group-onsemi 0:098463de4c5d 1292 mut_info = json_data[k]
group-onsemi 0:098463de4c5d 1293
group-onsemi 0:098463de4c5d 1294 add_row = True
group-onsemi 0:098463de4c5d 1295 if platform_filter and 'mcu' in mut_info:
group-onsemi 0:098463de4c5d 1296 add_row = re.search(platform_filter, mut_info['mcu']) is not None
group-onsemi 0:098463de4c5d 1297 if add_row:
group-onsemi 0:098463de4c5d 1298 for col in muts_info_cols:
group-onsemi 0:098463de4c5d 1299 cell_val = mut_info[col] if col in mut_info else None
group-onsemi 0:098463de4c5d 1300 if type(cell_val) == ListType:
group-onsemi 0:098463de4c5d 1301 cell_val = join_delim.join(cell_val)
group-onsemi 0:098463de4c5d 1302 row.append(cell_val)
group-onsemi 0:098463de4c5d 1303 pt.add_row(row)
group-onsemi 0:098463de4c5d 1304 return pt.get_string()
group-onsemi 0:098463de4c5d 1305
group-onsemi 0:098463de4c5d 1306
group-onsemi 0:098463de4c5d 1307 def print_test_configuration_from_json(json_data, join_delim=", "):
group-onsemi 0:098463de4c5d 1308 """ Prints test specification configuration passed to test script for verboseness
group-onsemi 0:098463de4c5d 1309 """
group-onsemi 0:098463de4c5d 1310 toolchains_info_cols = []
group-onsemi 0:098463de4c5d 1311 # We need to check all toolchains for each device
group-onsemi 0:098463de4c5d 1312 for k in json_data:
group-onsemi 0:098463de4c5d 1313 # k should be 'targets'
group-onsemi 0:098463de4c5d 1314 targets = json_data[k]
group-onsemi 0:098463de4c5d 1315 for target in targets:
group-onsemi 0:098463de4c5d 1316 toolchains = targets[target]
group-onsemi 0:098463de4c5d 1317 for toolchain in toolchains:
group-onsemi 0:098463de4c5d 1318 if toolchain not in toolchains_info_cols:
group-onsemi 0:098463de4c5d 1319 toolchains_info_cols.append(toolchain)
group-onsemi 0:098463de4c5d 1320
group-onsemi 0:098463de4c5d 1321 # Prepare pretty table object to display test specification
group-onsemi 0:098463de4c5d 1322 pt_cols = ["mcu"] + sorted(toolchains_info_cols)
group-onsemi 0:098463de4c5d 1323 pt = PrettyTable(pt_cols)
group-onsemi 0:098463de4c5d 1324 for col in pt_cols:
group-onsemi 0:098463de4c5d 1325 pt.align[col] = "l"
group-onsemi 0:098463de4c5d 1326
group-onsemi 0:098463de4c5d 1327 # { target : [conflicted toolchains] }
group-onsemi 0:098463de4c5d 1328 toolchain_conflicts = {}
group-onsemi 0:098463de4c5d 1329 toolchain_path_conflicts = []
group-onsemi 0:098463de4c5d 1330 for k in json_data:
group-onsemi 0:098463de4c5d 1331 # k should be 'targets'
group-onsemi 0:098463de4c5d 1332 targets = json_data[k]
group-onsemi 0:098463de4c5d 1333 for target in targets:
group-onsemi 0:098463de4c5d 1334 target_supported_toolchains = get_target_supported_toolchains(target)
group-onsemi 0:098463de4c5d 1335 if not target_supported_toolchains:
group-onsemi 0:098463de4c5d 1336 target_supported_toolchains = []
group-onsemi 0:098463de4c5d 1337 target_name = target if target in TARGET_MAP else "%s*"% target
group-onsemi 0:098463de4c5d 1338 row = [target_name]
group-onsemi 0:098463de4c5d 1339 toolchains = targets[target]
group-onsemi 0:098463de4c5d 1340
group-onsemi 0:098463de4c5d 1341 for toolchain in sorted(toolchains_info_cols):
group-onsemi 0:098463de4c5d 1342 # Check for conflicts: target vs toolchain
group-onsemi 0:098463de4c5d 1343 conflict = False
group-onsemi 0:098463de4c5d 1344 conflict_path = False
group-onsemi 0:098463de4c5d 1345 if toolchain in toolchains:
group-onsemi 0:098463de4c5d 1346 if toolchain not in target_supported_toolchains:
group-onsemi 0:098463de4c5d 1347 conflict = True
group-onsemi 0:098463de4c5d 1348 if target not in toolchain_conflicts:
group-onsemi 0:098463de4c5d 1349 toolchain_conflicts[target] = []
group-onsemi 0:098463de4c5d 1350 toolchain_conflicts[target].append(toolchain)
group-onsemi 0:098463de4c5d 1351 # Add marker inside table about target usage / conflict
group-onsemi 0:098463de4c5d 1352 cell_val = 'Yes' if toolchain in toolchains else '-'
group-onsemi 0:098463de4c5d 1353 if conflict:
group-onsemi 0:098463de4c5d 1354 cell_val += '*'
group-onsemi 0:098463de4c5d 1355 # Check for conflicts: toolchain vs toolchain path
group-onsemi 0:098463de4c5d 1356 if toolchain in TOOLCHAIN_PATHS:
group-onsemi 0:098463de4c5d 1357 toolchain_path = TOOLCHAIN_PATHS[toolchain]
group-onsemi 0:098463de4c5d 1358 if not os.path.isdir(toolchain_path):
group-onsemi 0:098463de4c5d 1359 conflict_path = True
group-onsemi 0:098463de4c5d 1360 if toolchain not in toolchain_path_conflicts:
group-onsemi 0:098463de4c5d 1361 toolchain_path_conflicts.append(toolchain)
group-onsemi 0:098463de4c5d 1362 if conflict_path:
group-onsemi 0:098463de4c5d 1363 cell_val += '#'
group-onsemi 0:098463de4c5d 1364 row.append(cell_val)
group-onsemi 0:098463de4c5d 1365 pt.add_row(row)
group-onsemi 0:098463de4c5d 1366
group-onsemi 0:098463de4c5d 1367 # generate result string
group-onsemi 0:098463de4c5d 1368 result = pt.get_string() # Test specification table
group-onsemi 0:098463de4c5d 1369 if toolchain_conflicts or toolchain_path_conflicts:
group-onsemi 0:098463de4c5d 1370 result += "\n"
group-onsemi 0:098463de4c5d 1371 result += "Toolchain conflicts:\n"
group-onsemi 0:098463de4c5d 1372 for target in toolchain_conflicts:
group-onsemi 0:098463de4c5d 1373 if target not in TARGET_MAP:
group-onsemi 0:098463de4c5d 1374 result += "\t* Target %s unknown\n"% (target)
group-onsemi 0:098463de4c5d 1375 conflict_target_list = join_delim.join(toolchain_conflicts[target])
group-onsemi 0:098463de4c5d 1376 sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
group-onsemi 0:098463de4c5d 1377 result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
group-onsemi 0:098463de4c5d 1378
group-onsemi 0:098463de4c5d 1379 for toolchain in toolchain_path_conflicts:
group-onsemi 0:098463de4c5d 1380 # Let's check toolchain configuration
group-onsemi 0:098463de4c5d 1381 if toolchain in TOOLCHAIN_PATHS:
group-onsemi 0:098463de4c5d 1382 toolchain_path = TOOLCHAIN_PATHS[toolchain]
group-onsemi 0:098463de4c5d 1383 if not os.path.isdir(toolchain_path):
group-onsemi 0:098463de4c5d 1384 result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
group-onsemi 0:098463de4c5d 1385 return result
group-onsemi 0:098463de4c5d 1386
group-onsemi 0:098463de4c5d 1387
group-onsemi 0:098463de4c5d 1388 def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
group-onsemi 0:098463de4c5d 1389 """ Generates table summary with all test cases and additional test cases
group-onsemi 0:098463de4c5d 1390 information using pretty print functionality. Allows test suite user to
group-onsemi 0:098463de4c5d 1391 see test cases
group-onsemi 0:098463de4c5d 1392 """
group-onsemi 0:098463de4c5d 1393 # get all unique test ID prefixes
group-onsemi 0:098463de4c5d 1394 unique_test_id = []
group-onsemi 0:098463de4c5d 1395 for test in TESTS:
group-onsemi 0:098463de4c5d 1396 split = test['id'].split('_')[:-1]
group-onsemi 0:098463de4c5d 1397 test_id_prefix = '_'.join(split)
group-onsemi 0:098463de4c5d 1398 if test_id_prefix not in unique_test_id:
group-onsemi 0:098463de4c5d 1399 unique_test_id.append(test_id_prefix)
group-onsemi 0:098463de4c5d 1400 unique_test_id.sort()
group-onsemi 0:098463de4c5d 1401 counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
group-onsemi 0:098463de4c5d 1402 counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
group-onsemi 0:098463de4c5d 1403
group-onsemi 0:098463de4c5d 1404 test_properties = ['id',
group-onsemi 0:098463de4c5d 1405 'automated',
group-onsemi 0:098463de4c5d 1406 'description',
group-onsemi 0:098463de4c5d 1407 'peripherals',
group-onsemi 0:098463de4c5d 1408 'host_test',
group-onsemi 0:098463de4c5d 1409 'duration'] if cols is None else cols
group-onsemi 0:098463de4c5d 1410
group-onsemi 0:098463de4c5d 1411 # All tests status table print
group-onsemi 0:098463de4c5d 1412 pt = PrettyTable(test_properties)
group-onsemi 0:098463de4c5d 1413 for col in test_properties:
group-onsemi 0:098463de4c5d 1414 pt.align[col] = "l"
group-onsemi 0:098463de4c5d 1415 pt.align['duration'] = "r"
group-onsemi 0:098463de4c5d 1416
group-onsemi 0:098463de4c5d 1417 counter_all = 0
group-onsemi 0:098463de4c5d 1418 counter_automated = 0
group-onsemi 0:098463de4c5d 1419 pt.padding_width = 1 # One space between column edges and contents (default)
group-onsemi 0:098463de4c5d 1420
group-onsemi 0:098463de4c5d 1421 for test_id in sorted(TEST_MAP.keys()):
group-onsemi 0:098463de4c5d 1422 if platform_filter is not None:
group-onsemi 0:098463de4c5d 1423 # FIlter out platforms using regex
group-onsemi 0:098463de4c5d 1424 if re.search(platform_filter, test_id) is None:
group-onsemi 0:098463de4c5d 1425 continue
group-onsemi 0:098463de4c5d 1426 row = []
group-onsemi 0:098463de4c5d 1427 test = TEST_MAP[test_id]
group-onsemi 0:098463de4c5d 1428 split = test_id.split('_')[:-1]
group-onsemi 0:098463de4c5d 1429 test_id_prefix = '_'.join(split)
group-onsemi 0:098463de4c5d 1430
group-onsemi 0:098463de4c5d 1431 for col in test_properties:
group-onsemi 0:098463de4c5d 1432 col_value = test[col]
group-onsemi 0:098463de4c5d 1433 if type(test[col]) == ListType:
group-onsemi 0:098463de4c5d 1434 col_value = join_delim.join(test[col])
group-onsemi 0:098463de4c5d 1435 elif test[col] == None:
group-onsemi 0:098463de4c5d 1436 col_value = "-"
group-onsemi 0:098463de4c5d 1437
group-onsemi 0:098463de4c5d 1438 row.append(col_value)
group-onsemi 0:098463de4c5d 1439 if test['automated'] == True:
group-onsemi 0:098463de4c5d 1440 counter_dict_test_id_types[test_id_prefix] += 1
group-onsemi 0:098463de4c5d 1441 counter_automated += 1
group-onsemi 0:098463de4c5d 1442 pt.add_row(row)
group-onsemi 0:098463de4c5d 1443 # Update counters
group-onsemi 0:098463de4c5d 1444 counter_all += 1
group-onsemi 0:098463de4c5d 1445 counter_dict_test_id_types_all[test_id_prefix] += 1
group-onsemi 0:098463de4c5d 1446 result = pt.get_string()
group-onsemi 0:098463de4c5d 1447 result += "\n\n"
group-onsemi 0:098463de4c5d 1448
group-onsemi 0:098463de4c5d 1449 if result_summary and not platform_filter:
group-onsemi 0:098463de4c5d 1450 # Automation result summary
group-onsemi 0:098463de4c5d 1451 test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
group-onsemi 0:098463de4c5d 1452 pt = PrettyTable(test_id_cols)
group-onsemi 0:098463de4c5d 1453 pt.align['automated'] = "r"
group-onsemi 0:098463de4c5d 1454 pt.align['all'] = "r"
group-onsemi 0:098463de4c5d 1455 pt.align['percent [%]'] = "r"
group-onsemi 0:098463de4c5d 1456
group-onsemi 0:098463de4c5d 1457 percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
group-onsemi 0:098463de4c5d 1458 str_progress = progress_bar(percent_progress, 75)
group-onsemi 0:098463de4c5d 1459 pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
group-onsemi 0:098463de4c5d 1460 result += "Automation coverage:\n"
group-onsemi 0:098463de4c5d 1461 result += pt.get_string()
group-onsemi 0:098463de4c5d 1462 result += "\n\n"
group-onsemi 0:098463de4c5d 1463
group-onsemi 0:098463de4c5d 1464 # Test automation coverage table print
group-onsemi 0:098463de4c5d 1465 test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
group-onsemi 0:098463de4c5d 1466 pt = PrettyTable(test_id_cols)
group-onsemi 0:098463de4c5d 1467 pt.align['id'] = "l"
group-onsemi 0:098463de4c5d 1468 pt.align['automated'] = "r"
group-onsemi 0:098463de4c5d 1469 pt.align['all'] = "r"
group-onsemi 0:098463de4c5d 1470 pt.align['percent [%]'] = "r"
group-onsemi 0:098463de4c5d 1471 for unique_id in unique_test_id:
group-onsemi 0:098463de4c5d 1472 # print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
group-onsemi 0:098463de4c5d 1473 percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
group-onsemi 0:098463de4c5d 1474 str_progress = progress_bar(percent_progress, 75)
group-onsemi 0:098463de4c5d 1475 row = [unique_id,
group-onsemi 0:098463de4c5d 1476 counter_dict_test_id_types[unique_id],
group-onsemi 0:098463de4c5d 1477 counter_dict_test_id_types_all[unique_id],
group-onsemi 0:098463de4c5d 1478 percent_progress,
group-onsemi 0:098463de4c5d 1479 "[" + str_progress + "]"]
group-onsemi 0:098463de4c5d 1480 pt.add_row(row)
group-onsemi 0:098463de4c5d 1481 result += "Test automation coverage:\n"
group-onsemi 0:098463de4c5d 1482 result += pt.get_string()
group-onsemi 0:098463de4c5d 1483 result += "\n\n"
group-onsemi 0:098463de4c5d 1484 return result
group-onsemi 0:098463de4c5d 1485
group-onsemi 0:098463de4c5d 1486
group-onsemi 0:098463de4c5d 1487 def progress_bar(percent_progress, saturation=0):
group-onsemi 0:098463de4c5d 1488 """ This function creates progress bar with optional simple saturation mark
group-onsemi 0:098463de4c5d 1489 """
group-onsemi 0:098463de4c5d 1490 step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
group-onsemi 0:098463de4c5d 1491 str_progress = '#' * step + '.' * int(50 - step)
group-onsemi 0:098463de4c5d 1492 c = '!' if str_progress[38] == '.' else '|'
group-onsemi 0:098463de4c5d 1493 if saturation > 0:
group-onsemi 0:098463de4c5d 1494 saturation = saturation / 2
group-onsemi 0:098463de4c5d 1495 str_progress = str_progress[:saturation] + c + str_progress[saturation:]
group-onsemi 0:098463de4c5d 1496 return str_progress
group-onsemi 0:098463de4c5d 1497
group-onsemi 0:098463de4c5d 1498
group-onsemi 0:098463de4c5d 1499 def singletest_in_cli_mode(single_test):
group-onsemi 0:098463de4c5d 1500 """ Runs SingleTestRunner object in CLI (Command line interface) mode
group-onsemi 0:098463de4c5d 1501
group-onsemi 0:098463de4c5d 1502 @return returns success code (0 == success) for building and running tests
group-onsemi 0:098463de4c5d 1503 """
group-onsemi 0:098463de4c5d 1504 start = time()
group-onsemi 0:098463de4c5d 1505 # Execute tests depending on options and filter applied
group-onsemi 0:098463de4c5d 1506 test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
group-onsemi 0:098463de4c5d 1507 elapsed_time = time() - start
group-onsemi 0:098463de4c5d 1508
group-onsemi 0:098463de4c5d 1509 # Human readable summary
group-onsemi 0:098463de4c5d 1510 if not single_test.opts_suppress_summary:
group-onsemi 0:098463de4c5d 1511 # prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 1512 print single_test.generate_test_summary(test_summary, shuffle_seed)
group-onsemi 0:098463de4c5d 1513 if single_test.opts_test_x_toolchain_summary:
group-onsemi 0:098463de4c5d 1514 # prints well-formed summary with results (SQL table like)
group-onsemi 0:098463de4c5d 1515 # table shows text x toolchain test result matrix
group-onsemi 0:098463de4c5d 1516 print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
group-onsemi 0:098463de4c5d 1517
group-onsemi 0:098463de4c5d 1518 print "Completed in %.2f sec"% (elapsed_time)
group-onsemi 0:098463de4c5d 1519 print
group-onsemi 0:098463de4c5d 1520 # Write summary of the builds
group-onsemi 0:098463de4c5d 1521
group-onsemi 0:098463de4c5d 1522 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
group-onsemi 0:098463de4c5d 1523 status = print_report_exporter.report(build_report)
group-onsemi 0:098463de4c5d 1524
group-onsemi 0:098463de4c5d 1525 # Store extra reports in files
group-onsemi 0:098463de4c5d 1526 if single_test.opts_report_html_file_name:
group-onsemi 0:098463de4c5d 1527 # Export results in form of HTML report to separate file
group-onsemi 0:098463de4c5d 1528 report_exporter = ReportExporter(ResultExporterType.HTML)
group-onsemi 0:098463de4c5d 1529 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
group-onsemi 0:098463de4c5d 1530 if single_test.opts_report_junit_file_name:
group-onsemi 0:098463de4c5d 1531 # Export results in form of JUnit XML report to separate file
group-onsemi 0:098463de4c5d 1532 report_exporter = ReportExporter(ResultExporterType.JUNIT)
group-onsemi 0:098463de4c5d 1533 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
group-onsemi 0:098463de4c5d 1534 if single_test.opts_report_text_file_name:
group-onsemi 0:098463de4c5d 1535 # Export results in form of a text file
group-onsemi 0:098463de4c5d 1536 report_exporter = ReportExporter(ResultExporterType.TEXT)
group-onsemi 0:098463de4c5d 1537 report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
group-onsemi 0:098463de4c5d 1538 if single_test.opts_report_build_file_name:
group-onsemi 0:098463de4c5d 1539 # Export build results as html report to sparate file
group-onsemi 0:098463de4c5d 1540 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
group-onsemi 0:098463de4c5d 1541 report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
group-onsemi 0:098463de4c5d 1542
group-onsemi 0:098463de4c5d 1543 # Returns True if no build failures of the test projects or their dependencies
group-onsemi 0:098463de4c5d 1544 return status
group-onsemi 0:098463de4c5d 1545
group-onsemi 0:098463de4c5d 1546 class TestLogger():
group-onsemi 0:098463de4c5d 1547 """ Super-class for logging and printing ongoing events for test suite pass
group-onsemi 0:098463de4c5d 1548 """
group-onsemi 0:098463de4c5d 1549 def __init__(self, store_log=True):
group-onsemi 0:098463de4c5d 1550 """ We can control if logger actually stores log in memory
group-onsemi 0:098463de4c5d 1551 or just handled all log entries immediately
group-onsemi 0:098463de4c5d 1552 """
group-onsemi 0:098463de4c5d 1553 self.log = []
group-onsemi 0:098463de4c5d 1554 self.log_to_file = False
group-onsemi 0:098463de4c5d 1555 self.log_file_name = None
group-onsemi 0:098463de4c5d 1556 self.store_log = store_log
group-onsemi 0:098463de4c5d 1557
group-onsemi 0:098463de4c5d 1558 self.LogType = construct_enum(INFO='Info',
group-onsemi 0:098463de4c5d 1559 WARN='Warning',
group-onsemi 0:098463de4c5d 1560 NOTIF='Notification',
group-onsemi 0:098463de4c5d 1561 ERROR='Error',
group-onsemi 0:098463de4c5d 1562 EXCEPT='Exception')
group-onsemi 0:098463de4c5d 1563
group-onsemi 0:098463de4c5d 1564 self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
group-onsemi 0:098463de4c5d 1565 APPEND=2) # Append to existing log file
group-onsemi 0:098463de4c5d 1566
group-onsemi 0:098463de4c5d 1567 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
group-onsemi 0:098463de4c5d 1568 """ Log one line of text
group-onsemi 0:098463de4c5d 1569 """
group-onsemi 0:098463de4c5d 1570 log_timestamp = time()
group-onsemi 0:098463de4c5d 1571 log_entry = {'log_type' : LogType,
group-onsemi 0:098463de4c5d 1572 'log_timestamp' : log_timestamp,
group-onsemi 0:098463de4c5d 1573 'log_line' : log_line,
group-onsemi 0:098463de4c5d 1574 '_future' : None
group-onsemi 0:098463de4c5d 1575 }
group-onsemi 0:098463de4c5d 1576 # Store log in memory
group-onsemi 0:098463de4c5d 1577 if self.store_log:
group-onsemi 0:098463de4c5d 1578 self.log.append(log_entry)
group-onsemi 0:098463de4c5d 1579 return log_entry
group-onsemi 0:098463de4c5d 1580
group-onsemi 0:098463de4c5d 1581
group-onsemi 0:098463de4c5d 1582 class CLITestLogger(TestLogger):
group-onsemi 0:098463de4c5d 1583 """ Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
group-onsemi 0:098463de4c5d 1584 """
group-onsemi 0:098463de4c5d 1585 def __init__(self, store_log=True, file_name=None):
group-onsemi 0:098463de4c5d 1586 TestLogger.__init__(self)
group-onsemi 0:098463de4c5d 1587 self.log_file_name = file_name
group-onsemi 0:098463de4c5d 1588 #self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
group-onsemi 0:098463de4c5d 1589 self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
group-onsemi 0:098463de4c5d 1590
group-onsemi 0:098463de4c5d 1591 def log_print(self, log_entry, timestamp=True):
group-onsemi 0:098463de4c5d 1592 """ Prints on screen formatted log entry
group-onsemi 0:098463de4c5d 1593 """
group-onsemi 0:098463de4c5d 1594 ts = log_entry['log_timestamp']
group-onsemi 0:098463de4c5d 1595 timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
group-onsemi 0:098463de4c5d 1596 log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
group-onsemi 0:098463de4c5d 1597 return timestamp_str + log_line_str
group-onsemi 0:098463de4c5d 1598
group-onsemi 0:098463de4c5d 1599 def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
group-onsemi 0:098463de4c5d 1600 """ Logs line, if log file output was specified log line will be appended
group-onsemi 0:098463de4c5d 1601 at the end of log file
group-onsemi 0:098463de4c5d 1602 """
group-onsemi 0:098463de4c5d 1603 log_entry = TestLogger.log_line(self, LogType, log_line)
group-onsemi 0:098463de4c5d 1604 log_line_str = self.log_print(log_entry, timestamp)
group-onsemi 0:098463de4c5d 1605 if self.log_file_name is not None:
group-onsemi 0:098463de4c5d 1606 try:
group-onsemi 0:098463de4c5d 1607 with open(self.log_file_name, 'a') as f:
group-onsemi 0:098463de4c5d 1608 f.write(log_line_str + line_delim)
group-onsemi 0:098463de4c5d 1609 except IOError:
group-onsemi 0:098463de4c5d 1610 pass
group-onsemi 0:098463de4c5d 1611 return log_line_str
group-onsemi 0:098463de4c5d 1612
group-onsemi 0:098463de4c5d 1613
group-onsemi 0:098463de4c5d 1614 def factory_db_logger(db_url):
group-onsemi 0:098463de4c5d 1615 """ Factory database driver depending on database type supplied in database connection string db_url
group-onsemi 0:098463de4c5d 1616 """
group-onsemi 0:098463de4c5d 1617 if db_url is not None:
group-onsemi 0:098463de4c5d 1618 from tools.test_mysql import MySQLDBAccess
group-onsemi 0:098463de4c5d 1619 connection_info = BaseDBAccess().parse_db_connection_string(db_url)
group-onsemi 0:098463de4c5d 1620 if connection_info is not None:
group-onsemi 0:098463de4c5d 1621 (db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
group-onsemi 0:098463de4c5d 1622 if db_type == 'mysql':
group-onsemi 0:098463de4c5d 1623 return MySQLDBAccess()
group-onsemi 0:098463de4c5d 1624 return None
group-onsemi 0:098463de4c5d 1625
group-onsemi 0:098463de4c5d 1626
group-onsemi 0:098463de4c5d 1627 def detect_database_verbose(db_url):
group-onsemi 0:098463de4c5d 1628 """ uses verbose mode (prints) database detection sequence to check it database connection string is valid
group-onsemi 0:098463de4c5d 1629 """
group-onsemi 0:098463de4c5d 1630 result = BaseDBAccess().parse_db_connection_string(db_url)
group-onsemi 0:098463de4c5d 1631 if result is not None:
group-onsemi 0:098463de4c5d 1632 # Parsing passed
group-onsemi 0:098463de4c5d 1633 (db_type, username, password, host, db_name) = result
group-onsemi 0:098463de4c5d 1634 #print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
group-onsemi 0:098463de4c5d 1635 # Let's try to connect
group-onsemi 0:098463de4c5d 1636 db_ = factory_db_logger(db_url)
group-onsemi 0:098463de4c5d 1637 if db_ is not None:
group-onsemi 0:098463de4c5d 1638 print "Connecting to database '%s'..."% db_url,
group-onsemi 0:098463de4c5d 1639 db_.connect(host, username, password, db_name)
group-onsemi 0:098463de4c5d 1640 if db_.is_connected():
group-onsemi 0:098463de4c5d 1641 print "ok"
group-onsemi 0:098463de4c5d 1642 print "Detecting database..."
group-onsemi 0:098463de4c5d 1643 print db_.detect_database(verbose=True)
group-onsemi 0:098463de4c5d 1644 print "Disconnecting...",
group-onsemi 0:098463de4c5d 1645 db_.disconnect()
group-onsemi 0:098463de4c5d 1646 print "done"
group-onsemi 0:098463de4c5d 1647 else:
group-onsemi 0:098463de4c5d 1648 print "Database type '%s' unknown"% db_type
group-onsemi 0:098463de4c5d 1649 else:
group-onsemi 0:098463de4c5d 1650 print "Parse error: '%s' - DB Url error"% (db_url)
group-onsemi 0:098463de4c5d 1651
group-onsemi 0:098463de4c5d 1652
group-onsemi 0:098463de4c5d 1653 def get_module_avail(module_name):
group-onsemi 0:098463de4c5d 1654 """ This function returns True if module_name is already impored module
group-onsemi 0:098463de4c5d 1655 """
group-onsemi 0:098463de4c5d 1656 return module_name in sys.modules.keys()
group-onsemi 0:098463de4c5d 1657
group-onsemi 0:098463de4c5d 1658
group-onsemi 0:098463de4c5d 1659 def get_autodetected_MUTS_list(platform_name_filter=None):
group-onsemi 0:098463de4c5d 1660 oldError = None
group-onsemi 0:098463de4c5d 1661 if os.name == 'nt':
group-onsemi 0:098463de4c5d 1662 # Disable Windows error box temporarily
group-onsemi 0:098463de4c5d 1663 oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
group-onsemi 0:098463de4c5d 1664
group-onsemi 0:098463de4c5d 1665 mbeds = mbed_lstools.create()
group-onsemi 0:098463de4c5d 1666 detect_muts_list = mbeds.list_mbeds()
group-onsemi 0:098463de4c5d 1667
group-onsemi 0:098463de4c5d 1668 if os.name == 'nt':
group-onsemi 0:098463de4c5d 1669 ctypes.windll.kernel32.SetErrorMode(oldError)
group-onsemi 0:098463de4c5d 1670
group-onsemi 0:098463de4c5d 1671 return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
group-onsemi 0:098463de4c5d 1672
group-onsemi 0:098463de4c5d 1673 def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
group-onsemi 0:098463de4c5d 1674 """ Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
group-onsemi 0:098463de4c5d 1675 If function fails to auto-detect devices it will return empty dictionary.
group-onsemi 0:098463de4c5d 1676
group-onsemi 0:098463de4c5d 1677 if get_module_avail('mbed_lstools'):
group-onsemi 0:098463de4c5d 1678 mbeds = mbed_lstools.create()
group-onsemi 0:098463de4c5d 1679 mbeds_list = mbeds.list_mbeds()
group-onsemi 0:098463de4c5d 1680
group-onsemi 0:098463de4c5d 1681 @param mbeds_list list of mbeds captured from mbed_lstools
group-onsemi 0:098463de4c5d 1682 @param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
group-onsemi 0:098463de4c5d 1683 """
group-onsemi 0:098463de4c5d 1684 result = {} # Should be in muts_all.json format
group-onsemi 0:098463de4c5d 1685 # Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
group-onsemi 0:098463de4c5d 1686 # mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
group-onsemi 0:098463de4c5d 1687 index = 1
group-onsemi 0:098463de4c5d 1688 for mut in mbeds_list:
group-onsemi 0:098463de4c5d 1689 # Filter the MUTS if a filter is specified
group-onsemi 0:098463de4c5d 1690
group-onsemi 0:098463de4c5d 1691 if platform_name_filter and not mut['platform_name'] in platform_name_filter:
group-onsemi 0:098463de4c5d 1692 continue
group-onsemi 0:098463de4c5d 1693
group-onsemi 0:098463de4c5d 1694 # For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
group-onsemi 0:098463de4c5d 1695 # if not we are creating our own unique value (last few chars from platform's target_id).
group-onsemi 0:098463de4c5d 1696 m = {'mcu': mut['platform_name'],
group-onsemi 0:098463de4c5d 1697 'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
group-onsemi 0:098463de4c5d 1698 'port': mut['serial_port'],
group-onsemi 0:098463de4c5d 1699 'disk': mut['mount_point'],
group-onsemi 0:098463de4c5d 1700 'peripherals': [] # No peripheral detection
group-onsemi 0:098463de4c5d 1701 }
group-onsemi 0:098463de4c5d 1702 if index not in result:
group-onsemi 0:098463de4c5d 1703 result[index] = {}
group-onsemi 0:098463de4c5d 1704 result[index] = m
group-onsemi 0:098463de4c5d 1705 index += 1
group-onsemi 0:098463de4c5d 1706 return result
group-onsemi 0:098463de4c5d 1707
group-onsemi 0:098463de4c5d 1708
group-onsemi 0:098463de4c5d 1709 def get_autodetected_TEST_SPEC(mbeds_list,
group-onsemi 0:098463de4c5d 1710 use_default_toolchain=True,
group-onsemi 0:098463de4c5d 1711 use_supported_toolchains=False,
group-onsemi 0:098463de4c5d 1712 toolchain_filter=None,
group-onsemi 0:098463de4c5d 1713 platform_name_filter=None):
group-onsemi 0:098463de4c5d 1714 """ Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
group-onsemi 0:098463de4c5d 1715 If function fails to auto-detect devices it will return empty 'targets' test_spec description.
group-onsemi 0:098463de4c5d 1716
group-onsemi 0:098463de4c5d 1717 use_default_toolchain - if True add default toolchain to test_spec
group-onsemi 0:098463de4c5d 1718 use_supported_toolchains - if True add all supported toolchains to test_spec
group-onsemi 0:098463de4c5d 1719 toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
group-onsemi 0:098463de4c5d 1720 """
group-onsemi 0:098463de4c5d 1721 result = {'targets': {} }
group-onsemi 0:098463de4c5d 1722
group-onsemi 0:098463de4c5d 1723 for mut in mbeds_list:
group-onsemi 0:098463de4c5d 1724 mcu = mut['mcu']
group-onsemi 0:098463de4c5d 1725 if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
group-onsemi 0:098463de4c5d 1726 if mcu in TARGET_MAP:
group-onsemi 0:098463de4c5d 1727 default_toolchain = TARGET_MAP[mcu].default_toolchain
group-onsemi 0:098463de4c5d 1728 supported_toolchains = TARGET_MAP[mcu].supported_toolchains
group-onsemi 0:098463de4c5d 1729
group-onsemi 0:098463de4c5d 1730 # Decide which toolchains should be added to test specification toolchain pool for each target
group-onsemi 0:098463de4c5d 1731 toolchains = []
group-onsemi 0:098463de4c5d 1732 if use_default_toolchain:
group-onsemi 0:098463de4c5d 1733 toolchains.append(default_toolchain)
group-onsemi 0:098463de4c5d 1734 if use_supported_toolchains:
group-onsemi 0:098463de4c5d 1735 toolchains += supported_toolchains
group-onsemi 0:098463de4c5d 1736 if toolchain_filter is not None:
group-onsemi 0:098463de4c5d 1737 all_toolchains = supported_toolchains + [default_toolchain]
group-onsemi 0:098463de4c5d 1738 for toolchain in toolchain_filter:
group-onsemi 0:098463de4c5d 1739 if toolchain in all_toolchains:
group-onsemi 0:098463de4c5d 1740 toolchains.append(toolchain)
group-onsemi 0:098463de4c5d 1741
group-onsemi 0:098463de4c5d 1742 result['targets'][mcu] = list(set(toolchains))
group-onsemi 0:098463de4c5d 1743 return result
group-onsemi 0:098463de4c5d 1744
group-onsemi 0:098463de4c5d 1745
group-onsemi 0:098463de4c5d 1746 def get_default_test_options_parser():
group-onsemi 0:098463de4c5d 1747 """ Get common test script options used by CLI, web services etc.
group-onsemi 0:098463de4c5d 1748 """
group-onsemi 0:098463de4c5d 1749 parser = argparse.ArgumentParser()
group-onsemi 0:098463de4c5d 1750 parser.add_argument('-i', '--tests',
group-onsemi 0:098463de4c5d 1751 dest='test_spec_filename',
group-onsemi 0:098463de4c5d 1752 metavar="FILE",
group-onsemi 0:098463de4c5d 1753 type=argparse_filestring_type,
group-onsemi 0:098463de4c5d 1754 help='Points to file with test specification')
group-onsemi 0:098463de4c5d 1755
group-onsemi 0:098463de4c5d 1756 parser.add_argument('-M', '--MUTS',
group-onsemi 0:098463de4c5d 1757 dest='muts_spec_filename',
group-onsemi 0:098463de4c5d 1758 metavar="FILE",
group-onsemi 0:098463de4c5d 1759 type=argparse_filestring_type,
group-onsemi 0:098463de4c5d 1760 help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
group-onsemi 0:098463de4c5d 1761
group-onsemi 0:098463de4c5d 1762 parser.add_argument("-j", "--jobs",
group-onsemi 0:098463de4c5d 1763 dest='jobs',
group-onsemi 0:098463de4c5d 1764 metavar="NUMBER",
group-onsemi 0:098463de4c5d 1765 type=int,
group-onsemi 0:098463de4c5d 1766 help="Define number of compilation jobs. Default value is 1")
group-onsemi 0:098463de4c5d 1767
group-onsemi 0:098463de4c5d 1768 if get_module_avail('mbed_lstools'):
group-onsemi 0:098463de4c5d 1769 # Additional features available when mbed_lstools is installed on host and imported
group-onsemi 0:098463de4c5d 1770 # mbed_lstools allow users to detect connected to host mbed-enabled devices
group-onsemi 0:098463de4c5d 1771 parser.add_argument('--auto',
group-onsemi 0:098463de4c5d 1772 dest='auto_detect',
group-onsemi 0:098463de4c5d 1773 action="store_true",
group-onsemi 0:098463de4c5d 1774 help='Use mbed-ls module to detect all connected mbed devices')
group-onsemi 0:098463de4c5d 1775
group-onsemi 0:098463de4c5d 1776 toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
group-onsemi 0:098463de4c5d 1777 parser.add_argument('--tc',
group-onsemi 0:098463de4c5d 1778 dest='toolchains_filter',
group-onsemi 0:098463de4c5d 1779 type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
group-onsemi 0:098463de4c5d 1780 help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
group-onsemi 0:098463de4c5d 1781
group-onsemi 0:098463de4c5d 1782 test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
group-onsemi 0:098463de4c5d 1783 parser.add_argument('--oper',
group-onsemi 0:098463de4c5d 1784 dest='operability_checks',
group-onsemi 0:098463de4c5d 1785 type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
group-onsemi 0:098463de4c5d 1786 help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
group-onsemi 0:098463de4c5d 1787
group-onsemi 0:098463de4c5d 1788 parser.add_argument('--clean',
group-onsemi 0:098463de4c5d 1789 dest='clean',
group-onsemi 0:098463de4c5d 1790 action="store_true",
group-onsemi 0:098463de4c5d 1791 help='Clean the build directory')
group-onsemi 0:098463de4c5d 1792
group-onsemi 0:098463de4c5d 1793 parser.add_argument('-P', '--only-peripherals',
group-onsemi 0:098463de4c5d 1794 dest='test_only_peripheral',
group-onsemi 0:098463de4c5d 1795 default=False,
group-onsemi 0:098463de4c5d 1796 action="store_true",
group-onsemi 0:098463de4c5d 1797 help='Test only peripheral declared for MUT and skip common tests')
group-onsemi 0:098463de4c5d 1798
group-onsemi 0:098463de4c5d 1799 parser.add_argument("--profile", dest="profile", action="append",
group-onsemi 0:098463de4c5d 1800 type=argparse_filestring_type,
group-onsemi 0:098463de4c5d 1801 default=[])
group-onsemi 0:098463de4c5d 1802
group-onsemi 0:098463de4c5d 1803 parser.add_argument('-C', '--only-commons',
group-onsemi 0:098463de4c5d 1804 dest='test_only_common',
group-onsemi 0:098463de4c5d 1805 default=False,
group-onsemi 0:098463de4c5d 1806 action="store_true",
group-onsemi 0:098463de4c5d 1807 help='Test only board internals. Skip perpherials tests and perform common tests')
group-onsemi 0:098463de4c5d 1808
group-onsemi 0:098463de4c5d 1809 parser.add_argument('-n', '--test-by-names',
group-onsemi 0:098463de4c5d 1810 dest='test_by_names',
group-onsemi 0:098463de4c5d 1811 type=argparse_many(str),
group-onsemi 0:098463de4c5d 1812 help='Runs only test enumerated it this switch. Use comma to separate test case names')
group-onsemi 0:098463de4c5d 1813
group-onsemi 0:098463de4c5d 1814 parser.add_argument('-p', '--peripheral-by-names',
group-onsemi 0:098463de4c5d 1815 dest='peripheral_by_names',
group-onsemi 0:098463de4c5d 1816 type=argparse_many(str),
group-onsemi 0:098463de4c5d 1817 help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
group-onsemi 0:098463de4c5d 1818
group-onsemi 0:098463de4c5d 1819 copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
group-onsemi 0:098463de4c5d 1820 copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
group-onsemi 0:098463de4c5d 1821
group-onsemi 0:098463de4c5d 1822 parser.add_argument('-c', '--copy-method',
group-onsemi 0:098463de4c5d 1823 dest='copy_method',
group-onsemi 0:098463de4c5d 1824 type=argparse_uppercase_type(copy_methods, "flash method"),
group-onsemi 0:098463de4c5d 1825 help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
group-onsemi 0:098463de4c5d 1826
group-onsemi 0:098463de4c5d 1827 reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
group-onsemi 0:098463de4c5d 1828 reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
group-onsemi 0:098463de4c5d 1829
group-onsemi 0:098463de4c5d 1830 parser.add_argument('-r', '--reset-type',
group-onsemi 0:098463de4c5d 1831 dest='mut_reset_type',
group-onsemi 0:098463de4c5d 1832 default=None,
group-onsemi 0:098463de4c5d 1833 type=argparse_uppercase_type(reset_methods, "reset method"),
group-onsemi 0:098463de4c5d 1834 help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
group-onsemi 0:098463de4c5d 1835
group-onsemi 0:098463de4c5d 1836 parser.add_argument('-g', '--goanna-for-tests',
group-onsemi 0:098463de4c5d 1837 dest='goanna_for_tests',
group-onsemi 0:098463de4c5d 1838 action="store_true",
group-onsemi 0:098463de4c5d 1839 help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
group-onsemi 0:098463de4c5d 1840
group-onsemi 0:098463de4c5d 1841 parser.add_argument('-G', '--goanna-for-sdk',
group-onsemi 0:098463de4c5d 1842 dest='goanna_for_mbed_sdk',
group-onsemi 0:098463de4c5d 1843 action="store_true",
group-onsemi 0:098463de4c5d 1844 help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
group-onsemi 0:098463de4c5d 1845
group-onsemi 0:098463de4c5d 1846 parser.add_argument('-s', '--suppress-summary',
group-onsemi 0:098463de4c5d 1847 dest='suppress_summary',
group-onsemi 0:098463de4c5d 1848 default=False,
group-onsemi 0:098463de4c5d 1849 action="store_true",
group-onsemi 0:098463de4c5d 1850 help='Suppresses display of wellformatted table with test results')
group-onsemi 0:098463de4c5d 1851
group-onsemi 0:098463de4c5d 1852 parser.add_argument('-t', '--test-summary',
group-onsemi 0:098463de4c5d 1853 dest='test_x_toolchain_summary',
group-onsemi 0:098463de4c5d 1854 default=False,
group-onsemi 0:098463de4c5d 1855 action="store_true",
group-onsemi 0:098463de4c5d 1856 help='Displays wellformatted table with test x toolchain test result per target')
group-onsemi 0:098463de4c5d 1857
group-onsemi 0:098463de4c5d 1858 parser.add_argument('-A', '--test-automation-report',
group-onsemi 0:098463de4c5d 1859 dest='test_automation_report',
group-onsemi 0:098463de4c5d 1860 default=False,
group-onsemi 0:098463de4c5d 1861 action="store_true",
group-onsemi 0:098463de4c5d 1862 help='Prints information about all tests and exits')
group-onsemi 0:098463de4c5d 1863
group-onsemi 0:098463de4c5d 1864 parser.add_argument('-R', '--test-case-report',
group-onsemi 0:098463de4c5d 1865 dest='test_case_report',
group-onsemi 0:098463de4c5d 1866 default=False,
group-onsemi 0:098463de4c5d 1867 action="store_true",
group-onsemi 0:098463de4c5d 1868 help='Prints information about all test cases and exits')
group-onsemi 0:098463de4c5d 1869
group-onsemi 0:098463de4c5d 1870 parser.add_argument("-S", "--supported-toolchains",
group-onsemi 0:098463de4c5d 1871 action="store_true",
group-onsemi 0:098463de4c5d 1872 dest="supported_toolchains",
group-onsemi 0:098463de4c5d 1873 default=False,
group-onsemi 0:098463de4c5d 1874 help="Displays supported matrix of MCUs and toolchains")
group-onsemi 0:098463de4c5d 1875
group-onsemi 0:098463de4c5d 1876 parser.add_argument("-O", "--only-build",
group-onsemi 0:098463de4c5d 1877 action="store_true",
group-onsemi 0:098463de4c5d 1878 dest="only_build_tests",
group-onsemi 0:098463de4c5d 1879 default=False,
group-onsemi 0:098463de4c5d 1880 help="Only build tests, skips actual test procedures (flashing etc.)")
group-onsemi 0:098463de4c5d 1881
group-onsemi 0:098463de4c5d 1882 parser.add_argument('--parallel',
group-onsemi 0:098463de4c5d 1883 dest='parallel_test_exec',
group-onsemi 0:098463de4c5d 1884 default=False,
group-onsemi 0:098463de4c5d 1885 action="store_true",
group-onsemi 0:098463de4c5d 1886 help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
group-onsemi 0:098463de4c5d 1887
group-onsemi 0:098463de4c5d 1888 parser.add_argument('--config',
group-onsemi 0:098463de4c5d 1889 dest='verbose_test_configuration_only',
group-onsemi 0:098463de4c5d 1890 default=False,
group-onsemi 0:098463de4c5d 1891 action="store_true",
group-onsemi 0:098463de4c5d 1892 help='Displays full test specification and MUTs configration and exits')
group-onsemi 0:098463de4c5d 1893
group-onsemi 0:098463de4c5d 1894 parser.add_argument('--loops',
group-onsemi 0:098463de4c5d 1895 dest='test_loops_list',
group-onsemi 0:098463de4c5d 1896 type=argparse_many(str),
group-onsemi 0:098463de4c5d 1897 help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
group-onsemi 0:098463de4c5d 1898
group-onsemi 0:098463de4c5d 1899 parser.add_argument('--global-loops',
group-onsemi 0:098463de4c5d 1900 dest='test_global_loops_value',
group-onsemi 0:098463de4c5d 1901 type=int,
group-onsemi 0:098463de4c5d 1902 help='Set global number of test loops per test. Default value is set 1')
group-onsemi 0:098463de4c5d 1903
group-onsemi 0:098463de4c5d 1904 parser.add_argument('--consolidate-waterfall',
group-onsemi 0:098463de4c5d 1905 dest='consolidate_waterfall_test',
group-onsemi 0:098463de4c5d 1906 default=False,
group-onsemi 0:098463de4c5d 1907 action="store_true",
group-onsemi 0:098463de4c5d 1908 help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
group-onsemi 0:098463de4c5d 1909
group-onsemi 0:098463de4c5d 1910 parser.add_argument('-W', '--waterfall',
group-onsemi 0:098463de4c5d 1911 dest='waterfall_test',
group-onsemi 0:098463de4c5d 1912 default=False,
group-onsemi 0:098463de4c5d 1913 action="store_true",
group-onsemi 0:098463de4c5d 1914 help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
group-onsemi 0:098463de4c5d 1915
group-onsemi 0:098463de4c5d 1916 parser.add_argument('-N', '--firmware-name',
group-onsemi 0:098463de4c5d 1917 dest='firmware_global_name',
group-onsemi 0:098463de4c5d 1918 help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
group-onsemi 0:098463de4c5d 1919
group-onsemi 0:098463de4c5d 1920 parser.add_argument('-u', '--shuffle',
group-onsemi 0:098463de4c5d 1921 dest='shuffle_test_order',
group-onsemi 0:098463de4c5d 1922 default=False,
group-onsemi 0:098463de4c5d 1923 action="store_true",
group-onsemi 0:098463de4c5d 1924 help='Shuffles test execution order')
group-onsemi 0:098463de4c5d 1925
group-onsemi 0:098463de4c5d 1926 parser.add_argument('--shuffle-seed',
group-onsemi 0:098463de4c5d 1927 dest='shuffle_test_seed',
group-onsemi 0:098463de4c5d 1928 default=None,
group-onsemi 0:098463de4c5d 1929 help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
group-onsemi 0:098463de4c5d 1930
group-onsemi 0:098463de4c5d 1931 parser.add_argument('-f', '--filter',
group-onsemi 0:098463de4c5d 1932 dest='general_filter_regex',
group-onsemi 0:098463de4c5d 1933 type=argparse_many(str),
group-onsemi 0:098463de4c5d 1934 default=None,
group-onsemi 0:098463de4c5d 1935 help='For some commands you can use filter to filter out results')
group-onsemi 0:098463de4c5d 1936
group-onsemi 0:098463de4c5d 1937 parser.add_argument('--inc-timeout',
group-onsemi 0:098463de4c5d 1938 dest='extend_test_timeout',
group-onsemi 0:098463de4c5d 1939 metavar="NUMBER",
group-onsemi 0:098463de4c5d 1940 type=int,
group-onsemi 0:098463de4c5d 1941 help='You can increase global timeout for each test by specifying additional test timeout in seconds')
group-onsemi 0:098463de4c5d 1942
group-onsemi 0:098463de4c5d 1943 parser.add_argument('--db',
group-onsemi 0:098463de4c5d 1944 dest='db_url',
group-onsemi 0:098463de4c5d 1945 help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
group-onsemi 0:098463de4c5d 1946
group-onsemi 0:098463de4c5d 1947 parser.add_argument('-l', '--log',
group-onsemi 0:098463de4c5d 1948 dest='log_file_name',
group-onsemi 0:098463de4c5d 1949 help='Log events to external file (note not all console entries may be visible in log file)')
group-onsemi 0:098463de4c5d 1950
group-onsemi 0:098463de4c5d 1951 parser.add_argument('--report-html',
group-onsemi 0:098463de4c5d 1952 dest='report_html_file_name',
group-onsemi 0:098463de4c5d 1953 help='You can log test suite results in form of HTML report')
group-onsemi 0:098463de4c5d 1954
group-onsemi 0:098463de4c5d 1955 parser.add_argument('--report-junit',
group-onsemi 0:098463de4c5d 1956 dest='report_junit_file_name',
group-onsemi 0:098463de4c5d 1957 help='You can log test suite results in form of JUnit compliant XML report')
group-onsemi 0:098463de4c5d 1958
group-onsemi 0:098463de4c5d 1959 parser.add_argument("--report-build",
group-onsemi 0:098463de4c5d 1960 dest="report_build_file_name",
group-onsemi 0:098463de4c5d 1961 help="Output the build results to a junit xml file")
group-onsemi 0:098463de4c5d 1962
group-onsemi 0:098463de4c5d 1963 parser.add_argument("--report-text",
group-onsemi 0:098463de4c5d 1964 dest="report_text_file_name",
group-onsemi 0:098463de4c5d 1965 help="Output the build results to a text file")
group-onsemi 0:098463de4c5d 1966
group-onsemi 0:098463de4c5d 1967 parser.add_argument('--verbose-skipped',
group-onsemi 0:098463de4c5d 1968 dest='verbose_skipped_tests',
group-onsemi 0:098463de4c5d 1969 default=False,
group-onsemi 0:098463de4c5d 1970 action="store_true",
group-onsemi 0:098463de4c5d 1971 help='Prints some extra information about skipped tests')
group-onsemi 0:098463de4c5d 1972
group-onsemi 0:098463de4c5d 1973 parser.add_argument('-V', '--verbose-test-result',
group-onsemi 0:098463de4c5d 1974 dest='verbose_test_result_only',
group-onsemi 0:098463de4c5d 1975 default=False,
group-onsemi 0:098463de4c5d 1976 action="store_true",
group-onsemi 0:098463de4c5d 1977 help='Prints test serial output')
group-onsemi 0:098463de4c5d 1978
group-onsemi 0:098463de4c5d 1979 parser.add_argument('-v', '--verbose',
group-onsemi 0:098463de4c5d 1980 dest='verbose',
group-onsemi 0:098463de4c5d 1981 default=False,
group-onsemi 0:098463de4c5d 1982 action="store_true",
group-onsemi 0:098463de4c5d 1983 help='Verbose mode (prints some extra information)')
group-onsemi 0:098463de4c5d 1984
group-onsemi 0:098463de4c5d 1985 parser.add_argument('--version',
group-onsemi 0:098463de4c5d 1986 dest='version',
group-onsemi 0:098463de4c5d 1987 default=False,
group-onsemi 0:098463de4c5d 1988 action="store_true",
group-onsemi 0:098463de4c5d 1989 help='Prints script version and exits')
group-onsemi 0:098463de4c5d 1990 return parser
group-onsemi 0:098463de4c5d 1991
group-onsemi 0:098463de4c5d 1992 def test_path_to_name(path, base):
group-onsemi 0:098463de4c5d 1993 """Change all slashes in a path into hyphens
group-onsemi 0:098463de4c5d 1994 This creates a unique cross-platform test name based on the path
group-onsemi 0:098463de4c5d 1995 This can eventually be overriden by a to-be-determined meta-data mechanism"""
group-onsemi 0:098463de4c5d 1996 name_parts = []
group-onsemi 0:098463de4c5d 1997 head, tail = os.path.split(relpath(path,base))
group-onsemi 0:098463de4c5d 1998 while (tail and tail != "."):
group-onsemi 0:098463de4c5d 1999 name_parts.insert(0, tail)
group-onsemi 0:098463de4c5d 2000 head, tail = os.path.split(head)
group-onsemi 0:098463de4c5d 2001
group-onsemi 0:098463de4c5d 2002 return "-".join(name_parts).lower()
group-onsemi 0:098463de4c5d 2003
group-onsemi 0:098463de4c5d 2004 def find_tests(base_dir, target_name, toolchain_name, app_config=None):
group-onsemi 0:098463de4c5d 2005 """ Finds all tests in a directory recursively
group-onsemi 0:098463de4c5d 2006 base_dir: path to the directory to scan for tests (ex. 'path/to/project')
group-onsemi 0:098463de4c5d 2007 target_name: name of the target to use for scanning (ex. 'K64F')
group-onsemi 0:098463de4c5d 2008 toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
group-onsemi 0:098463de4c5d 2009 options: Compile options to pass to the toolchain (ex. ['debug-info'])
group-onsemi 0:098463de4c5d 2010 app_config - location of a chosen mbed_app.json file
group-onsemi 0:098463de4c5d 2011 """
group-onsemi 0:098463de4c5d 2012
group-onsemi 0:098463de4c5d 2013 tests = {}
group-onsemi 0:098463de4c5d 2014
group-onsemi 0:098463de4c5d 2015 # Prepare the toolchain
group-onsemi 0:098463de4c5d 2016 toolchain = prepare_toolchain([base_dir], target_name, toolchain_name,
group-onsemi 0:098463de4c5d 2017 silent=True, app_config=app_config)
group-onsemi 0:098463de4c5d 2018
group-onsemi 0:098463de4c5d 2019 # Scan the directory for paths to probe for 'TESTS' folders
group-onsemi 0:098463de4c5d 2020 base_resources = scan_resources([base_dir], toolchain)
group-onsemi 0:098463de4c5d 2021
group-onsemi 0:098463de4c5d 2022 dirs = base_resources.inc_dirs
group-onsemi 0:098463de4c5d 2023 for directory in dirs:
group-onsemi 0:098463de4c5d 2024 subdirs = os.listdir(directory)
group-onsemi 0:098463de4c5d 2025
group-onsemi 0:098463de4c5d 2026 # If the directory contains a subdirectory called 'TESTS', scan it for test cases
group-onsemi 0:098463de4c5d 2027 if 'TESTS' in subdirs:
group-onsemi 0:098463de4c5d 2028 walk_base_dir = join(directory, 'TESTS')
group-onsemi 0:098463de4c5d 2029 test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
group-onsemi 0:098463de4c5d 2030
group-onsemi 0:098463de4c5d 2031 # Loop through all subdirectories
group-onsemi 0:098463de4c5d 2032 for d in test_resources.inc_dirs:
group-onsemi 0:098463de4c5d 2033
group-onsemi 0:098463de4c5d 2034 # If the test case folder is not called 'host_tests' and it is
group-onsemi 0:098463de4c5d 2035 # located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
group-onsemi 0:098463de4c5d 2036 # then add it to the tests
group-onsemi 0:098463de4c5d 2037 path_depth = get_path_depth(relpath(d, walk_base_dir))
group-onsemi 0:098463de4c5d 2038 if path_depth == 2:
group-onsemi 0:098463de4c5d 2039 test_group_directory_path, test_case_directory = os.path.split(d)
group-onsemi 0:098463de4c5d 2040 test_group_directory = os.path.basename(test_group_directory_path)
group-onsemi 0:098463de4c5d 2041
group-onsemi 0:098463de4c5d 2042 # Check to make sure discoverd folder is not in a host test directory
group-onsemi 0:098463de4c5d 2043 if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
group-onsemi 0:098463de4c5d 2044 test_name = test_path_to_name(d, base_dir)
group-onsemi 0:098463de4c5d 2045 tests[test_name] = d
group-onsemi 0:098463de4c5d 2046
group-onsemi 0:098463de4c5d 2047 return tests
group-onsemi 0:098463de4c5d 2048
group-onsemi 0:098463de4c5d 2049 def print_tests(tests, format="list", sort=True):
group-onsemi 0:098463de4c5d 2050 """Given a dictionary of tests (as returned from "find_tests"), print them
group-onsemi 0:098463de4c5d 2051 in the specified format"""
group-onsemi 0:098463de4c5d 2052 if format == "list":
group-onsemi 0:098463de4c5d 2053 for test_name in sorted(tests.keys()):
group-onsemi 0:098463de4c5d 2054 test_path = tests[test_name]
group-onsemi 0:098463de4c5d 2055 print "Test Case:"
group-onsemi 0:098463de4c5d 2056 print " Name: %s" % test_name
group-onsemi 0:098463de4c5d 2057 print " Path: %s" % test_path
group-onsemi 0:098463de4c5d 2058 elif format == "json":
group-onsemi 0:098463de4c5d 2059 print json.dumps(tests, indent=2)
group-onsemi 0:098463de4c5d 2060 else:
group-onsemi 0:098463de4c5d 2061 print "Unknown format '%s'" % format
group-onsemi 0:098463de4c5d 2062 sys.exit(1)
group-onsemi 0:098463de4c5d 2063
group-onsemi 0:098463de4c5d 2064 def norm_relative_path(path, start):
group-onsemi 0:098463de4c5d 2065 """This function will create a normalized, relative path. It mimics the
group-onsemi 0:098463de4c5d 2066 python os.path.relpath function, but also normalizes a Windows-syle path
group-onsemi 0:098463de4c5d 2067 that use backslashes to a Unix style path that uses forward slashes."""
group-onsemi 0:098463de4c5d 2068 path = os.path.normpath(path)
group-onsemi 0:098463de4c5d 2069 path = os.path.relpath(path, start)
group-onsemi 0:098463de4c5d 2070 path = path.replace("\\", "/")
group-onsemi 0:098463de4c5d 2071 return path
group-onsemi 0:098463de4c5d 2072
group-onsemi 0:098463de4c5d 2073
group-onsemi 0:098463de4c5d 2074 def build_test_worker(*args, **kwargs):
group-onsemi 0:098463de4c5d 2075 """This is a worker function for the parallel building of tests. The `args`
group-onsemi 0:098463de4c5d 2076 and `kwargs` are passed directly to `build_project`. It returns a dictionary
group-onsemi 0:098463de4c5d 2077 with the following structure:
group-onsemi 0:098463de4c5d 2078
group-onsemi 0:098463de4c5d 2079 {
group-onsemi 0:098463de4c5d 2080 'result': `True` if no exceptions were thrown, `False` otherwise
group-onsemi 0:098463de4c5d 2081 'reason': Instance of exception that was thrown on failure
group-onsemi 0:098463de4c5d 2082 'bin_file': Path to the created binary if `build_project` was
group-onsemi 0:098463de4c5d 2083 successful. Not present otherwise
group-onsemi 0:098463de4c5d 2084 'kwargs': The keyword arguments that were passed to `build_project`.
group-onsemi 0:098463de4c5d 2085 This includes arguments that were modified (ex. report)
group-onsemi 0:098463de4c5d 2086 }
group-onsemi 0:098463de4c5d 2087 """
group-onsemi 0:098463de4c5d 2088 bin_file = None
group-onsemi 0:098463de4c5d 2089 ret = {
group-onsemi 0:098463de4c5d 2090 'result': False,
group-onsemi 0:098463de4c5d 2091 'args': args,
group-onsemi 0:098463de4c5d 2092 'kwargs': kwargs
group-onsemi 0:098463de4c5d 2093 }
group-onsemi 0:098463de4c5d 2094
group-onsemi 0:098463de4c5d 2095 # Use parent TOOLCHAIN_PATHS variable
group-onsemi 0:098463de4c5d 2096 for key, value in kwargs['toolchain_paths'].iteritems():
group-onsemi 0:098463de4c5d 2097 TOOLCHAIN_PATHS[key] = value
group-onsemi 0:098463de4c5d 2098
group-onsemi 0:098463de4c5d 2099 del kwargs['toolchain_paths']
group-onsemi 0:098463de4c5d 2100
group-onsemi 0:098463de4c5d 2101 try:
group-onsemi 0:098463de4c5d 2102 bin_file = build_project(*args, **kwargs)
group-onsemi 0:098463de4c5d 2103 ret['result'] = True
group-onsemi 0:098463de4c5d 2104 ret['bin_file'] = bin_file
group-onsemi 0:098463de4c5d 2105 ret['kwargs'] = kwargs
group-onsemi 0:098463de4c5d 2106
group-onsemi 0:098463de4c5d 2107 except NotSupportedException, e:
group-onsemi 0:098463de4c5d 2108 ret['reason'] = e
group-onsemi 0:098463de4c5d 2109 except ToolException, e:
group-onsemi 0:098463de4c5d 2110 ret['reason'] = e
group-onsemi 0:098463de4c5d 2111 except KeyboardInterrupt, e:
group-onsemi 0:098463de4c5d 2112 ret['reason'] = e
group-onsemi 0:098463de4c5d 2113 except:
group-onsemi 0:098463de4c5d 2114 # Print unhandled exceptions here
group-onsemi 0:098463de4c5d 2115 import traceback
group-onsemi 0:098463de4c5d 2116 traceback.print_exc(file=sys.stdout)
group-onsemi 0:098463de4c5d 2117
group-onsemi 0:098463de4c5d 2118 return ret
group-onsemi 0:098463de4c5d 2119
group-onsemi 0:098463de4c5d 2120
group-onsemi 0:098463de4c5d 2121 def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
group-onsemi 0:098463de4c5d 2122 clean=False, notify=None, verbose=False, jobs=1, macros=None,
group-onsemi 0:098463de4c5d 2123 silent=False, report=None, properties=None,
group-onsemi 0:098463de4c5d 2124 continue_on_build_fail=False, app_config=None,
group-onsemi 0:098463de4c5d 2125 build_profile=None):
group-onsemi 0:098463de4c5d 2126 """Given the data structure from 'find_tests' and the typical build parameters,
group-onsemi 0:098463de4c5d 2127 build all the tests
group-onsemi 0:098463de4c5d 2128
group-onsemi 0:098463de4c5d 2129 Returns a tuple of the build result (True or False) followed by the test
group-onsemi 0:098463de4c5d 2130 build data structure"""
group-onsemi 0:098463de4c5d 2131
group-onsemi 0:098463de4c5d 2132 execution_directory = "."
group-onsemi 0:098463de4c5d 2133 base_path = norm_relative_path(build_path, execution_directory)
group-onsemi 0:098463de4c5d 2134
group-onsemi 0:098463de4c5d 2135 target_name = target if isinstance(target, str) else target.name
group-onsemi 0:098463de4c5d 2136 cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
group-onsemi 0:098463de4c5d 2137
group-onsemi 0:098463de4c5d 2138 baud_rate = 9600
group-onsemi 0:098463de4c5d 2139 if 'platform.stdio-baud-rate' in cfg:
group-onsemi 0:098463de4c5d 2140 baud_rate = cfg['platform.stdio-baud-rate'].value
group-onsemi 0:098463de4c5d 2141
group-onsemi 0:098463de4c5d 2142 test_build = {
group-onsemi 0:098463de4c5d 2143 "platform": target_name,
group-onsemi 0:098463de4c5d 2144 "toolchain": toolchain_name,
group-onsemi 0:098463de4c5d 2145 "base_path": base_path,
group-onsemi 0:098463de4c5d 2146 "baud_rate": baud_rate,
group-onsemi 0:098463de4c5d 2147 "binary_type": "bootable",
group-onsemi 0:098463de4c5d 2148 "tests": {}
group-onsemi 0:098463de4c5d 2149 }
group-onsemi 0:098463de4c5d 2150
group-onsemi 0:098463de4c5d 2151 result = True
group-onsemi 0:098463de4c5d 2152
group-onsemi 0:098463de4c5d 2153 jobs_count = int(jobs if jobs else cpu_count())
group-onsemi 0:098463de4c5d 2154 p = Pool(processes=jobs_count)
group-onsemi 0:098463de4c5d 2155 results = []
group-onsemi 0:098463de4c5d 2156 for test_name, test_path in tests.iteritems():
group-onsemi 0:098463de4c5d 2157 test_build_path = os.path.join(build_path, test_path)
group-onsemi 0:098463de4c5d 2158 src_path = base_source_paths + [test_path]
group-onsemi 0:098463de4c5d 2159 bin_file = None
group-onsemi 0:098463de4c5d 2160 test_case_folder_name = os.path.basename(test_path)
group-onsemi 0:098463de4c5d 2161
group-onsemi 0:098463de4c5d 2162 args = (src_path, test_build_path, target, toolchain_name)
group-onsemi 0:098463de4c5d 2163 kwargs = {
group-onsemi 0:098463de4c5d 2164 'jobs': 1,
group-onsemi 0:098463de4c5d 2165 'clean': clean,
group-onsemi 0:098463de4c5d 2166 'macros': macros,
group-onsemi 0:098463de4c5d 2167 'name': test_case_folder_name,
group-onsemi 0:098463de4c5d 2168 'project_id': test_name,
group-onsemi 0:098463de4c5d 2169 'report': report,
group-onsemi 0:098463de4c5d 2170 'properties': properties,
group-onsemi 0:098463de4c5d 2171 'verbose': verbose,
group-onsemi 0:098463de4c5d 2172 'app_config': app_config,
group-onsemi 0:098463de4c5d 2173 'build_profile': build_profile,
group-onsemi 0:098463de4c5d 2174 'silent': True,
group-onsemi 0:098463de4c5d 2175 'toolchain_paths': TOOLCHAIN_PATHS
group-onsemi 0:098463de4c5d 2176 }
group-onsemi 0:098463de4c5d 2177
group-onsemi 0:098463de4c5d 2178 results.append(p.apply_async(build_test_worker, args, kwargs))
group-onsemi 0:098463de4c5d 2179
group-onsemi 0:098463de4c5d 2180 p.close()
group-onsemi 0:098463de4c5d 2181 result = True
group-onsemi 0:098463de4c5d 2182 itr = 0
group-onsemi 0:098463de4c5d 2183 while len(results):
group-onsemi 0:098463de4c5d 2184 itr += 1
group-onsemi 0:098463de4c5d 2185 if itr > 360000:
group-onsemi 0:098463de4c5d 2186 p.terminate()
group-onsemi 0:098463de4c5d 2187 p.join()
group-onsemi 0:098463de4c5d 2188 raise ToolException("Compile did not finish in 10 minutes")
group-onsemi 0:098463de4c5d 2189 else:
group-onsemi 0:098463de4c5d 2190 sleep(0.01)
group-onsemi 0:098463de4c5d 2191 pending = 0
group-onsemi 0:098463de4c5d 2192 for r in results:
group-onsemi 0:098463de4c5d 2193 if r.ready() is True:
group-onsemi 0:098463de4c5d 2194 try:
group-onsemi 0:098463de4c5d 2195 worker_result = r.get()
group-onsemi 0:098463de4c5d 2196 results.remove(r)
group-onsemi 0:098463de4c5d 2197
group-onsemi 0:098463de4c5d 2198 # Take report from the kwargs and merge it into existing report
group-onsemi 0:098463de4c5d 2199 report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
group-onsemi 0:098463de4c5d 2200 for test_key in report_entry.keys():
group-onsemi 0:098463de4c5d 2201 report[target_name][toolchain_name][test_key] = report_entry[test_key]
group-onsemi 0:098463de4c5d 2202
group-onsemi 0:098463de4c5d 2203 # Set the overall result to a failure if a build failure occurred
group-onsemi 0:098463de4c5d 2204 if not worker_result['result'] and not isinstance(worker_result['reason'], NotSupportedException):
group-onsemi 0:098463de4c5d 2205 result = False
group-onsemi 0:098463de4c5d 2206 break
group-onsemi 0:098463de4c5d 2207
group-onsemi 0:098463de4c5d 2208 # Adding binary path to test build result
group-onsemi 0:098463de4c5d 2209 if worker_result['result'] and 'bin_file' in worker_result:
group-onsemi 0:098463de4c5d 2210 bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
group-onsemi 0:098463de4c5d 2211
group-onsemi 0:098463de4c5d 2212 test_build['tests'][worker_result['kwargs']['project_id']] = {
group-onsemi 0:098463de4c5d 2213 "binaries": [
group-onsemi 0:098463de4c5d 2214 {
group-onsemi 0:098463de4c5d 2215 "path": bin_file
group-onsemi 0:098463de4c5d 2216 }
group-onsemi 0:098463de4c5d 2217 ]
group-onsemi 0:098463de4c5d 2218 }
group-onsemi 0:098463de4c5d 2219
group-onsemi 0:098463de4c5d 2220 test_key = worker_result['kwargs']['project_id'].upper()
group-onsemi 0:098463de4c5d 2221 print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
group-onsemi 0:098463de4c5d 2222 print 'Image: %s\n' % bin_file
group-onsemi 0:098463de4c5d 2223
group-onsemi 0:098463de4c5d 2224 except:
group-onsemi 0:098463de4c5d 2225 if p._taskqueue.queue:
group-onsemi 0:098463de4c5d 2226 p._taskqueue.queue.clear()
group-onsemi 0:098463de4c5d 2227 sleep(0.5)
group-onsemi 0:098463de4c5d 2228 p.terminate()
group-onsemi 0:098463de4c5d 2229 p.join()
group-onsemi 0:098463de4c5d 2230 raise
group-onsemi 0:098463de4c5d 2231 else:
group-onsemi 0:098463de4c5d 2232 pending += 1
group-onsemi 0:098463de4c5d 2233 if pending >= jobs_count:
group-onsemi 0:098463de4c5d 2234 break
group-onsemi 0:098463de4c5d 2235
group-onsemi 0:098463de4c5d 2236 # Break as soon as possible if there is a failure and we are not
group-onsemi 0:098463de4c5d 2237 # continuing on build failures
group-onsemi 0:098463de4c5d 2238 if not result and not continue_on_build_fail:
group-onsemi 0:098463de4c5d 2239 if p._taskqueue.queue:
group-onsemi 0:098463de4c5d 2240 p._taskqueue.queue.clear()
group-onsemi 0:098463de4c5d 2241 sleep(0.5)
group-onsemi 0:098463de4c5d 2242 p.terminate()
group-onsemi 0:098463de4c5d 2243 break
group-onsemi 0:098463de4c5d 2244
group-onsemi 0:098463de4c5d 2245 p.join()
group-onsemi 0:098463de4c5d 2246
group-onsemi 0:098463de4c5d 2247 test_builds = {}
group-onsemi 0:098463de4c5d 2248 test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
group-onsemi 0:098463de4c5d 2249
group-onsemi 0:098463de4c5d 2250 return result, test_builds
group-onsemi 0:098463de4c5d 2251
group-onsemi 0:098463de4c5d 2252
group-onsemi 0:098463de4c5d 2253 def test_spec_from_test_builds(test_builds):
group-onsemi 0:098463de4c5d 2254 return {
group-onsemi 0:098463de4c5d 2255 "builds": test_builds
group-onsemi 0:098463de4c5d 2256 }