BaBoRo1 | Mbed

BA » Code » BaBoRo1

mbed-os/tools/test_api.py@4:75df35ef4fb6, 2018-03-30 (annotated)

Committer:: borlanic
Date:: Fri Mar 30 14:07:05 2018 +0000
Revision:: 4:75df35ef4fb6
Parent:: 0:380207fcb5c1

commentar

Who changed what in which revision?

User	Revision	Line number	New contents of line
borlanic	0:380207fcb5c1	1	"""
borlanic	0:380207fcb5c1	2	mbed SDK
borlanic	0:380207fcb5c1	3	Copyright (c) 2011-2014 ARM Limited
borlanic	0:380207fcb5c1	4
borlanic	0:380207fcb5c1	5	Licensed under the Apache License, Version 2.0 (the "License");
borlanic	0:380207fcb5c1	6	you may not use this file except in compliance with the License.
borlanic	0:380207fcb5c1	7	You may obtain a copy of the License at
borlanic	0:380207fcb5c1	8
borlanic	0:380207fcb5c1	9	http://www.apache.org/licenses/LICENSE-2.0
borlanic	0:380207fcb5c1	10
borlanic	0:380207fcb5c1	11	Unless required by applicable law or agreed to in writing, software
borlanic	0:380207fcb5c1	12	distributed under the License is distributed on an "AS IS" BASIS,
borlanic	0:380207fcb5c1	13	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
borlanic	0:380207fcb5c1	14	See the License for the specific language governing permissions and
borlanic	0:380207fcb5c1	15	limitations under the License.
borlanic	0:380207fcb5c1	16
borlanic	0:380207fcb5c1	17	Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
borlanic	0:380207fcb5c1	18	"""
borlanic	0:380207fcb5c1	19	from __future__ import print_function
borlanic	0:380207fcb5c1	20
borlanic	0:380207fcb5c1	21	import os
borlanic	0:380207fcb5c1	22	import re
borlanic	0:380207fcb5c1	23	import sys
borlanic	0:380207fcb5c1	24	import json
borlanic	0:380207fcb5c1	25	import uuid
borlanic	0:380207fcb5c1	26	import pprint
borlanic	0:380207fcb5c1	27	import random
borlanic	0:380207fcb5c1	28	import argparse
borlanic	0:380207fcb5c1	29	import datetime
borlanic	0:380207fcb5c1	30	import threading
borlanic	0:380207fcb5c1	31	import ctypes
borlanic	0:380207fcb5c1	32	import functools
borlanic	0:380207fcb5c1	33	from colorama import Fore, Back, Style
borlanic	0:380207fcb5c1	34	from prettytable import PrettyTable
borlanic	0:380207fcb5c1	35	from copy import copy
borlanic	0:380207fcb5c1	36
borlanic	0:380207fcb5c1	37	from time import sleep, time
borlanic	0:380207fcb5c1	38	try:
borlanic	0:380207fcb5c1	39	from Queue import Queue, Empty
borlanic	0:380207fcb5c1	40	except ImportError:
borlanic	0:380207fcb5c1	41	from queue import Queue, Empty
borlanic	0:380207fcb5c1	42	from os.path import join, exists, basename, relpath
borlanic	0:380207fcb5c1	43	from threading import Thread, Lock
borlanic	0:380207fcb5c1	44	from multiprocessing import Pool, cpu_count
borlanic	0:380207fcb5c1	45	from subprocess import Popen, PIPE
borlanic	0:380207fcb5c1	46
borlanic	0:380207fcb5c1	47	# Imports related to mbed build api
borlanic	0:380207fcb5c1	48	from tools.tests import TESTS
borlanic	0:380207fcb5c1	49	from tools.tests import TEST_MAP
borlanic	0:380207fcb5c1	50	from tools.paths import BUILD_DIR
borlanic	0:380207fcb5c1	51	from tools.paths import HOST_TESTS
borlanic	0:380207fcb5c1	52	from tools.utils import ToolException
borlanic	0:380207fcb5c1	53	from tools.utils import NotSupportedException
borlanic	0:380207fcb5c1	54	from tools.utils import construct_enum
borlanic	0:380207fcb5c1	55	from tools.memap import MemapParser
borlanic	0:380207fcb5c1	56	from tools.targets import TARGET_MAP, Target
borlanic	0:380207fcb5c1	57	import tools.test_configs as TestConfig
borlanic	0:380207fcb5c1	58	from tools.test_db import BaseDBAccess
borlanic	0:380207fcb5c1	59	from tools.build_api import build_project, build_mbed_libs, build_lib
borlanic	0:380207fcb5c1	60	from tools.build_api import get_target_supported_toolchains
borlanic	0:380207fcb5c1	61	from tools.build_api import write_build_report
borlanic	0:380207fcb5c1	62	from tools.build_api import prep_report
borlanic	0:380207fcb5c1	63	from tools.build_api import prep_properties
borlanic	0:380207fcb5c1	64	from tools.build_api import create_result
borlanic	0:380207fcb5c1	65	from tools.build_api import add_result_to_report
borlanic	0:380207fcb5c1	66	from tools.build_api import prepare_toolchain
borlanic	0:380207fcb5c1	67	from tools.build_api import scan_resources
borlanic	0:380207fcb5c1	68	from tools.build_api import get_config
borlanic	0:380207fcb5c1	69	from tools.libraries import LIBRARIES, LIBRARY_MAP
borlanic	0:380207fcb5c1	70	from tools.options import extract_profile
borlanic	0:380207fcb5c1	71	from tools.toolchains import TOOLCHAIN_PATHS
borlanic	0:380207fcb5c1	72	from tools.toolchains import TOOLCHAINS
borlanic	0:380207fcb5c1	73	from tools.test_exporters import ReportExporter, ResultExporterType
borlanic	0:380207fcb5c1	74	from tools.utils import argparse_filestring_type
borlanic	0:380207fcb5c1	75	from tools.utils import argparse_uppercase_type
borlanic	0:380207fcb5c1	76	from tools.utils import argparse_lowercase_type
borlanic	0:380207fcb5c1	77	from tools.utils import argparse_many
borlanic	0:380207fcb5c1	78
borlanic	0:380207fcb5c1	79	import tools.host_tests.host_tests_plugins as host_tests_plugins
borlanic	0:380207fcb5c1	80
borlanic	0:380207fcb5c1	81	try:
borlanic	0:380207fcb5c1	82	import mbed_lstools
borlanic	0:380207fcb5c1	83	from tools.compliance.ioper_runner import get_available_oper_test_scopes
borlanic	0:380207fcb5c1	84	except:
borlanic	0:380207fcb5c1	85	pass
borlanic	0:380207fcb5c1	86
borlanic	0:380207fcb5c1	87
borlanic	0:380207fcb5c1	88	class ProcessObserver(Thread):
borlanic	0:380207fcb5c1	89	def __init__(self, proc):
borlanic	0:380207fcb5c1	90	Thread.__init__(self)
borlanic	0:380207fcb5c1	91	self.proc = proc
borlanic	0:380207fcb5c1	92	self.queue = Queue()
borlanic	0:380207fcb5c1	93	self.daemon = True
borlanic	0:380207fcb5c1	94	self.active = True
borlanic	0:380207fcb5c1	95	self.start()
borlanic	0:380207fcb5c1	96
borlanic	0:380207fcb5c1	97	def run(self):
borlanic	0:380207fcb5c1	98	while self.active:
borlanic	0:380207fcb5c1	99	c = self.proc.stdout.read(1)
borlanic	0:380207fcb5c1	100	self.queue.put(c)
borlanic	0:380207fcb5c1	101
borlanic	0:380207fcb5c1	102	def stop(self):
borlanic	0:380207fcb5c1	103	self.active = False
borlanic	0:380207fcb5c1	104	try:
borlanic	0:380207fcb5c1	105	self.proc.terminate()
borlanic	0:380207fcb5c1	106	except Exception:
borlanic	0:380207fcb5c1	107	pass
borlanic	0:380207fcb5c1	108
borlanic	0:380207fcb5c1	109
borlanic	0:380207fcb5c1	110	class SingleTestExecutor(threading.Thread):
borlanic	0:380207fcb5c1	111	""" Example: Single test class in separate thread usage
borlanic	0:380207fcb5c1	112	"""
borlanic	0:380207fcb5c1	113	def __init__(self, single_test):
borlanic	0:380207fcb5c1	114	self.single_test = single_test
borlanic	0:380207fcb5c1	115	threading.Thread.__init__(self)
borlanic	0:380207fcb5c1	116
borlanic	0:380207fcb5c1	117	def run(self):
borlanic	0:380207fcb5c1	118	start = time()
borlanic	0:380207fcb5c1	119	# Execute tests depending on options and filter applied
borlanic	0:380207fcb5c1	120	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
borlanic	0:380207fcb5c1	121	elapsed_time = time() - start
borlanic	0:380207fcb5c1	122
borlanic	0:380207fcb5c1	123	# Human readable summary
borlanic	0:380207fcb5c1	124	if not self.single_test.opts_suppress_summary:
borlanic	0:380207fcb5c1	125	# prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	126	print(self.single_test.generate_test_summary(test_summary,
borlanic	0:380207fcb5c1	127	shuffle_seed))
borlanic	0:380207fcb5c1	128	if self.single_test.opts_test_x_toolchain_summary:
borlanic	0:380207fcb5c1	129	# prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	130	# table shows text x toolchain test result matrix
borlanic	0:380207fcb5c1	131	print(self.single_test.generate_test_summary_by_target(
borlanic	0:380207fcb5c1	132	test_summary, shuffle_seed))
borlanic	0:380207fcb5c1	133	print("Completed in %.2f sec"% (elapsed_time))
borlanic	0:380207fcb5c1	134
borlanic	0:380207fcb5c1	135
borlanic	0:380207fcb5c1	136	class SingleTestRunner(object):
borlanic	0:380207fcb5c1	137	""" Object wrapper for single test run which may involve multiple MUTs
borlanic	0:380207fcb5c1	138	"""
borlanic	0:380207fcb5c1	139	RE_DETECT_TESTCASE_RESULT = None
borlanic	0:380207fcb5c1	140
borlanic	0:380207fcb5c1	141	# Return codes for test script
borlanic	0:380207fcb5c1	142	TEST_RESULT_OK = "OK"
borlanic	0:380207fcb5c1	143	TEST_RESULT_FAIL = "FAIL"
borlanic	0:380207fcb5c1	144	TEST_RESULT_ERROR = "ERROR"
borlanic	0:380207fcb5c1	145	TEST_RESULT_UNDEF = "UNDEF"
borlanic	0:380207fcb5c1	146	TEST_RESULT_IOERR_COPY = "IOERR_COPY"
borlanic	0:380207fcb5c1	147	TEST_RESULT_IOERR_DISK = "IOERR_DISK"
borlanic	0:380207fcb5c1	148	TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
borlanic	0:380207fcb5c1	149	TEST_RESULT_TIMEOUT = "TIMEOUT"
borlanic	0:380207fcb5c1	150	TEST_RESULT_NO_IMAGE = "NO_IMAGE"
borlanic	0:380207fcb5c1	151	TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
borlanic	0:380207fcb5c1	152	TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
borlanic	0:380207fcb5c1	153	TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
borlanic	0:380207fcb5c1	154
borlanic	0:380207fcb5c1	155	GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
borlanic	0:380207fcb5c1	156	TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
borlanic	0:380207fcb5c1	157	TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
borlanic	0:380207fcb5c1	158
borlanic	0:380207fcb5c1	159	muts = {} # MUTs descriptor (from external file)
borlanic	0:380207fcb5c1	160	test_spec = {} # Test specification (from external file)
borlanic	0:380207fcb5c1	161
borlanic	0:380207fcb5c1	162	# mbed test suite -> SingleTestRunner
borlanic	0:380207fcb5c1	163	TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
borlanic	0:380207fcb5c1	164	"failure" : TEST_RESULT_FAIL,
borlanic	0:380207fcb5c1	165	"error" : TEST_RESULT_ERROR,
borlanic	0:380207fcb5c1	166	"ioerr_copy" : TEST_RESULT_IOERR_COPY,
borlanic	0:380207fcb5c1	167	"ioerr_disk" : TEST_RESULT_IOERR_DISK,
borlanic	0:380207fcb5c1	168	"ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
borlanic	0:380207fcb5c1	169	"timeout" : TEST_RESULT_TIMEOUT,
borlanic	0:380207fcb5c1	170	"no_image" : TEST_RESULT_NO_IMAGE,
borlanic	0:380207fcb5c1	171	"end" : TEST_RESULT_UNDEF,
borlanic	0:380207fcb5c1	172	"mbed_assert" : TEST_RESULT_MBED_ASSERT,
borlanic	0:380207fcb5c1	173	"build_failed" : TEST_RESULT_BUILD_FAILED,
borlanic	0:380207fcb5c1	174	"not_supproted" : TEST_RESULT_NOT_SUPPORTED
borlanic	0:380207fcb5c1	175	}
borlanic	0:380207fcb5c1	176
borlanic	0:380207fcb5c1	177	def __init__(self,
borlanic	0:380207fcb5c1	178	_global_loops_count=1,
borlanic	0:380207fcb5c1	179	_test_loops_list=None,
borlanic	0:380207fcb5c1	180	_muts={},
borlanic	0:380207fcb5c1	181	_clean=False,
borlanic	0:380207fcb5c1	182	_parser=None,
borlanic	0:380207fcb5c1	183	_opts=None,
borlanic	0:380207fcb5c1	184	_opts_db_url=None,
borlanic	0:380207fcb5c1	185	_opts_log_file_name=None,
borlanic	0:380207fcb5c1	186	_opts_report_html_file_name=None,
borlanic	0:380207fcb5c1	187	_opts_report_junit_file_name=None,
borlanic	0:380207fcb5c1	188	_opts_report_build_file_name=None,
borlanic	0:380207fcb5c1	189	_opts_report_text_file_name=None,
borlanic	0:380207fcb5c1	190	_opts_build_report={},
borlanic	0:380207fcb5c1	191	_opts_build_properties={},
borlanic	0:380207fcb5c1	192	_test_spec={},
borlanic	0:380207fcb5c1	193	_opts_goanna_for_mbed_sdk=None,
borlanic	0:380207fcb5c1	194	_opts_goanna_for_tests=None,
borlanic	0:380207fcb5c1	195	_opts_shuffle_test_order=False,
borlanic	0:380207fcb5c1	196	_opts_shuffle_test_seed=None,
borlanic	0:380207fcb5c1	197	_opts_test_by_names=None,
borlanic	0:380207fcb5c1	198	_opts_peripheral_by_names=None,
borlanic	0:380207fcb5c1	199	_opts_test_only_peripheral=False,
borlanic	0:380207fcb5c1	200	_opts_test_only_common=False,
borlanic	0:380207fcb5c1	201	_opts_verbose_skipped_tests=False,
borlanic	0:380207fcb5c1	202	_opts_verbose_test_result_only=False,
borlanic	0:380207fcb5c1	203	_opts_verbose=False,
borlanic	0:380207fcb5c1	204	_opts_firmware_global_name=None,
borlanic	0:380207fcb5c1	205	_opts_only_build_tests=False,
borlanic	0:380207fcb5c1	206	_opts_parallel_test_exec=False,
borlanic	0:380207fcb5c1	207	_opts_suppress_summary=False,
borlanic	0:380207fcb5c1	208	_opts_test_x_toolchain_summary=False,
borlanic	0:380207fcb5c1	209	_opts_copy_method=None,
borlanic	0:380207fcb5c1	210	_opts_mut_reset_type=None,
borlanic	0:380207fcb5c1	211	_opts_jobs=None,
borlanic	0:380207fcb5c1	212	_opts_waterfall_test=None,
borlanic	0:380207fcb5c1	213	_opts_consolidate_waterfall_test=None,
borlanic	0:380207fcb5c1	214	_opts_extend_test_timeout=None,
borlanic	0:380207fcb5c1	215	_opts_auto_detect=None,
borlanic	0:380207fcb5c1	216	_opts_include_non_automated=False):
borlanic	0:380207fcb5c1	217	""" Let's try hard to init this object
borlanic	0:380207fcb5c1	218	"""
borlanic	0:380207fcb5c1	219	from colorama import init
borlanic	0:380207fcb5c1	220	init()
borlanic	0:380207fcb5c1	221
borlanic	0:380207fcb5c1	222	PATTERN = "\\{(" + "\|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
borlanic	0:380207fcb5c1	223	self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
borlanic	0:380207fcb5c1	224	# Settings related to test loops counters
borlanic	0:380207fcb5c1	225	try:
borlanic	0:380207fcb5c1	226	_global_loops_count = int(_global_loops_count)
borlanic	0:380207fcb5c1	227	except:
borlanic	0:380207fcb5c1	228	_global_loops_count = 1
borlanic	0:380207fcb5c1	229	if _global_loops_count < 1:
borlanic	0:380207fcb5c1	230	_global_loops_count = 1
borlanic	0:380207fcb5c1	231	self.GLOBAL_LOOPS_COUNT = _global_loops_count
borlanic	0:380207fcb5c1	232	self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
borlanic	0:380207fcb5c1	233	self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
borlanic	0:380207fcb5c1	234
borlanic	0:380207fcb5c1	235	self.shuffle_random_seed = 0.0
borlanic	0:380207fcb5c1	236	self.SHUFFLE_SEED_ROUND = 10
borlanic	0:380207fcb5c1	237
borlanic	0:380207fcb5c1	238	# MUT list and test specification storage
borlanic	0:380207fcb5c1	239	self.muts = _muts
borlanic	0:380207fcb5c1	240	self.test_spec = _test_spec
borlanic	0:380207fcb5c1	241
borlanic	0:380207fcb5c1	242	# Settings passed e.g. from command line
borlanic	0:380207fcb5c1	243	self.opts_db_url = _opts_db_url
borlanic	0:380207fcb5c1	244	self.opts_log_file_name = _opts_log_file_name
borlanic	0:380207fcb5c1	245	self.opts_report_html_file_name = _opts_report_html_file_name
borlanic	0:380207fcb5c1	246	self.opts_report_junit_file_name = _opts_report_junit_file_name
borlanic	0:380207fcb5c1	247	self.opts_report_build_file_name = _opts_report_build_file_name
borlanic	0:380207fcb5c1	248	self.opts_report_text_file_name = _opts_report_text_file_name
borlanic	0:380207fcb5c1	249	self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
borlanic	0:380207fcb5c1	250	self.opts_goanna_for_tests = _opts_goanna_for_tests
borlanic	0:380207fcb5c1	251	self.opts_shuffle_test_order = _opts_shuffle_test_order
borlanic	0:380207fcb5c1	252	self.opts_shuffle_test_seed = _opts_shuffle_test_seed
borlanic	0:380207fcb5c1	253	self.opts_test_by_names = _opts_test_by_names
borlanic	0:380207fcb5c1	254	self.opts_peripheral_by_names = _opts_peripheral_by_names
borlanic	0:380207fcb5c1	255	self.opts_test_only_peripheral = _opts_test_only_peripheral
borlanic	0:380207fcb5c1	256	self.opts_test_only_common = _opts_test_only_common
borlanic	0:380207fcb5c1	257	self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
borlanic	0:380207fcb5c1	258	self.opts_verbose_test_result_only = _opts_verbose_test_result_only
borlanic	0:380207fcb5c1	259	self.opts_verbose = _opts_verbose
borlanic	0:380207fcb5c1	260	self.opts_firmware_global_name = _opts_firmware_global_name
borlanic	0:380207fcb5c1	261	self.opts_only_build_tests = _opts_only_build_tests
borlanic	0:380207fcb5c1	262	self.opts_parallel_test_exec = _opts_parallel_test_exec
borlanic	0:380207fcb5c1	263	self.opts_suppress_summary = _opts_suppress_summary
borlanic	0:380207fcb5c1	264	self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
borlanic	0:380207fcb5c1	265	self.opts_copy_method = _opts_copy_method
borlanic	0:380207fcb5c1	266	self.opts_mut_reset_type = _opts_mut_reset_type
borlanic	0:380207fcb5c1	267	self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
borlanic	0:380207fcb5c1	268	self.opts_waterfall_test = _opts_waterfall_test
borlanic	0:380207fcb5c1	269	self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
borlanic	0:380207fcb5c1	270	self.opts_extend_test_timeout = _opts_extend_test_timeout
borlanic	0:380207fcb5c1	271	self.opts_clean = _clean
borlanic	0:380207fcb5c1	272	self.opts_parser = _parser
borlanic	0:380207fcb5c1	273	self.opts = _opts
borlanic	0:380207fcb5c1	274	self.opts_auto_detect = _opts_auto_detect
borlanic	0:380207fcb5c1	275	self.opts_include_non_automated = _opts_include_non_automated
borlanic	0:380207fcb5c1	276
borlanic	0:380207fcb5c1	277	self.build_report = _opts_build_report
borlanic	0:380207fcb5c1	278	self.build_properties = _opts_build_properties
borlanic	0:380207fcb5c1	279
borlanic	0:380207fcb5c1	280	# File / screen logger initialization
borlanic	0:380207fcb5c1	281	self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
borlanic	0:380207fcb5c1	282
borlanic	0:380207fcb5c1	283	# Database related initializations
borlanic	0:380207fcb5c1	284	self.db_logger = factory_db_logger(self.opts_db_url)
borlanic	0:380207fcb5c1	285	self.db_logger_build_id = None # Build ID (database index of build_id table)
borlanic	0:380207fcb5c1	286	# Let's connect to database to set up credentials and confirm database is ready
borlanic	0:380207fcb5c1	287	if self.db_logger:
borlanic	0:380207fcb5c1	288	self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
borlanic	0:380207fcb5c1	289	if self.db_logger.is_connected():
borlanic	0:380207fcb5c1	290	# Get hostname and uname so we can use it as build description
borlanic	0:380207fcb5c1	291	# when creating new build_id in external database
borlanic	0:380207fcb5c1	292	(_hostname, _uname) = self.db_logger.get_hostname()
borlanic	0:380207fcb5c1	293	_host_location = os.path.dirname(os.path.abspath(__file__))
borlanic	0:380207fcb5c1	294	build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
borlanic	0:380207fcb5c1	295	self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
borlanic	0:380207fcb5c1	296	self.db_logger.disconnect()
borlanic	0:380207fcb5c1	297
borlanic	0:380207fcb5c1	298	def dump_options(self):
borlanic	0:380207fcb5c1	299	""" Function returns data structure with common settings passed to SingelTestRunner
borlanic	0:380207fcb5c1	300	It can be used for example to fill _extra fields in database storing test suite single run data
borlanic	0:380207fcb5c1	301	Example:
borlanic	0:380207fcb5c1	302	data = self.dump_options()
borlanic	0:380207fcb5c1	303	or
borlanic	0:380207fcb5c1	304	data_str = json.dumps(self.dump_options())
borlanic	0:380207fcb5c1	305	"""
borlanic	0:380207fcb5c1	306	result = {"db_url" : str(self.opts_db_url),
borlanic	0:380207fcb5c1	307	"log_file_name" : str(self.opts_log_file_name),
borlanic	0:380207fcb5c1	308	"shuffle_test_order" : str(self.opts_shuffle_test_order),
borlanic	0:380207fcb5c1	309	"shuffle_test_seed" : str(self.opts_shuffle_test_seed),
borlanic	0:380207fcb5c1	310	"test_by_names" : str(self.opts_test_by_names),
borlanic	0:380207fcb5c1	311	"peripheral_by_names" : str(self.opts_peripheral_by_names),
borlanic	0:380207fcb5c1	312	"test_only_peripheral" : str(self.opts_test_only_peripheral),
borlanic	0:380207fcb5c1	313	"test_only_common" : str(self.opts_test_only_common),
borlanic	0:380207fcb5c1	314	"verbose" : str(self.opts_verbose),
borlanic	0:380207fcb5c1	315	"firmware_global_name" : str(self.opts_firmware_global_name),
borlanic	0:380207fcb5c1	316	"only_build_tests" : str(self.opts_only_build_tests),
borlanic	0:380207fcb5c1	317	"copy_method" : str(self.opts_copy_method),
borlanic	0:380207fcb5c1	318	"mut_reset_type" : str(self.opts_mut_reset_type),
borlanic	0:380207fcb5c1	319	"jobs" : str(self.opts_jobs),
borlanic	0:380207fcb5c1	320	"extend_test_timeout" : str(self.opts_extend_test_timeout),
borlanic	0:380207fcb5c1	321	"_dummy" : ''
borlanic	0:380207fcb5c1	322	}
borlanic	0:380207fcb5c1	323	return result
borlanic	0:380207fcb5c1	324
borlanic	0:380207fcb5c1	325	def shuffle_random_func(self):
borlanic	0:380207fcb5c1	326	return self.shuffle_random_seed
borlanic	0:380207fcb5c1	327
borlanic	0:380207fcb5c1	328	def is_shuffle_seed_float(self):
borlanic	0:380207fcb5c1	329	""" return true if function parameter can be converted to float
borlanic	0:380207fcb5c1	330	"""
borlanic	0:380207fcb5c1	331	result = True
borlanic	0:380207fcb5c1	332	try:
borlanic	0:380207fcb5c1	333	float(self.shuffle_random_seed)
borlanic	0:380207fcb5c1	334	except ValueError:
borlanic	0:380207fcb5c1	335	result = False
borlanic	0:380207fcb5c1	336	return result
borlanic	0:380207fcb5c1	337
borlanic	0:380207fcb5c1	338	# This will store target / toolchain specific properties
borlanic	0:380207fcb5c1	339	test_suite_properties_ext = {} # target : toolchain
borlanic	0:380207fcb5c1	340	# Here we store test results
borlanic	0:380207fcb5c1	341	test_summary = []
borlanic	0:380207fcb5c1	342	# Here we store test results in extended data structure
borlanic	0:380207fcb5c1	343	test_summary_ext = {}
borlanic	0:380207fcb5c1	344	execute_thread_slice_lock = Lock()
borlanic	0:380207fcb5c1	345
borlanic	0:380207fcb5c1	346	def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
borlanic	0:380207fcb5c1	347	for toolchain in toolchains:
borlanic	0:380207fcb5c1	348	tt_id = "%s::%s" % (toolchain, target)
borlanic	0:380207fcb5c1	349
borlanic	0:380207fcb5c1	350	T = TARGET_MAP[target]
borlanic	0:380207fcb5c1	351
borlanic	0:380207fcb5c1	352	# print target, toolchain
borlanic	0:380207fcb5c1	353	# Test suite properties returned to external tools like CI
borlanic	0:380207fcb5c1	354	test_suite_properties = {
borlanic	0:380207fcb5c1	355	'jobs': self.opts_jobs,
borlanic	0:380207fcb5c1	356	'clean': clean,
borlanic	0:380207fcb5c1	357	'target': target,
borlanic	0:380207fcb5c1	358	'vendor': T.extra_labels[0],
borlanic	0:380207fcb5c1	359	'test_ids': ', '.join(test_ids),
borlanic	0:380207fcb5c1	360	'toolchain': toolchain,
borlanic	0:380207fcb5c1	361	'shuffle_random_seed': self.shuffle_random_seed
borlanic	0:380207fcb5c1	362	}
borlanic	0:380207fcb5c1	363
borlanic	0:380207fcb5c1	364
borlanic	0:380207fcb5c1	365	# print '=== %s::%s ===' % (target, toolchain)
borlanic	0:380207fcb5c1	366	# Let's build our test
borlanic	0:380207fcb5c1	367	if target not in TARGET_MAP:
borlanic	0:380207fcb5c1	368	print(self.logger.log_line(
borlanic	0:380207fcb5c1	369	self.logger.LogType.NOTIF,
borlanic	0:380207fcb5c1	370	'Skipped tests for %s target. Target platform not found' %
borlanic	0:380207fcb5c1	371	(target)))
borlanic	0:380207fcb5c1	372	continue
borlanic	0:380207fcb5c1	373
borlanic	0:380207fcb5c1	374	clean_mbed_libs_options = (self.opts_goanna_for_mbed_sdk or
borlanic	0:380207fcb5c1	375	self.opts_clean or clean)
borlanic	0:380207fcb5c1	376
borlanic	0:380207fcb5c1	377	profile = extract_profile(self.opts_parser, self.opts, toolchain)
borlanic	0:380207fcb5c1	378	stats_depth = self.opts.stats_depth or 2
borlanic	0:380207fcb5c1	379
borlanic	0:380207fcb5c1	380	try:
borlanic	0:380207fcb5c1	381	build_mbed_libs_result = build_mbed_libs(
borlanic	0:380207fcb5c1	382	T, toolchain,
borlanic	0:380207fcb5c1	383	clean=clean_mbed_libs_options,
borlanic	0:380207fcb5c1	384	verbose=self.opts_verbose,
borlanic	0:380207fcb5c1	385	jobs=self.opts_jobs,
borlanic	0:380207fcb5c1	386	report=build_report,
borlanic	0:380207fcb5c1	387	properties=build_properties,
borlanic	0:380207fcb5c1	388	build_profile=profile)
borlanic	0:380207fcb5c1	389
borlanic	0:380207fcb5c1	390	if not build_mbed_libs_result:
borlanic	0:380207fcb5c1	391	print(self.logger.log_line(
borlanic	0:380207fcb5c1	392	self.logger.LogType.NOTIF,
borlanic	0:380207fcb5c1	393	'Skipped tests for %s target. Toolchain %s is not '
borlanic	0:380207fcb5c1	394	'supported for this target'% (T.name, toolchain)))
borlanic	0:380207fcb5c1	395	continue
borlanic	0:380207fcb5c1	396
borlanic	0:380207fcb5c1	397	except ToolException:
borlanic	0:380207fcb5c1	398	print(self.logger.log_line(
borlanic	0:380207fcb5c1	399	self.logger.LogType.ERROR,
borlanic	0:380207fcb5c1	400	'There were errors while building MBED libs for %s using %s'
borlanic	0:380207fcb5c1	401	% (target, toolchain)))
borlanic	0:380207fcb5c1	402	continue
borlanic	0:380207fcb5c1	403
borlanic	0:380207fcb5c1	404	build_dir = join(BUILD_DIR, "test", target, toolchain)
borlanic	0:380207fcb5c1	405
borlanic	0:380207fcb5c1	406	test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
borlanic	0:380207fcb5c1	407	test_suite_properties['build_dir'] = build_dir
borlanic	0:380207fcb5c1	408	test_suite_properties['skipped'] = []
borlanic	0:380207fcb5c1	409
borlanic	0:380207fcb5c1	410	# Enumerate through all tests and shuffle test order if requested
borlanic	0:380207fcb5c1	411	test_map_keys = sorted(TEST_MAP.keys())
borlanic	0:380207fcb5c1	412
borlanic	0:380207fcb5c1	413	if self.opts_shuffle_test_order:
borlanic	0:380207fcb5c1	414	random.shuffle(test_map_keys, self.shuffle_random_func)
borlanic	0:380207fcb5c1	415	# Update database with shuffle seed f applicable
borlanic	0:380207fcb5c1	416	if self.db_logger:
borlanic	0:380207fcb5c1	417	self.db_logger.reconnect();
borlanic	0:380207fcb5c1	418	if self.db_logger.is_connected():
borlanic	0:380207fcb5c1	419	self.db_logger.update_build_id_info(
borlanic	0:380207fcb5c1	420	self.db_logger_build_id,
borlanic	0:380207fcb5c1	421	_shuffle_seed=self.shuffle_random_func())
borlanic	0:380207fcb5c1	422	self.db_logger.disconnect();
borlanic	0:380207fcb5c1	423
borlanic	0:380207fcb5c1	424	if self.db_logger:
borlanic	0:380207fcb5c1	425	self.db_logger.reconnect();
borlanic	0:380207fcb5c1	426	if self.db_logger.is_connected():
borlanic	0:380207fcb5c1	427	# Update MUTs and Test Specification in database
borlanic	0:380207fcb5c1	428	self.db_logger.update_build_id_info(
borlanic	0:380207fcb5c1	429	self.db_logger_build_id,
borlanic	0:380207fcb5c1	430	_muts=self.muts, _test_spec=self.test_spec)
borlanic	0:380207fcb5c1	431	# Update Extra information in database (some options passed to test suite)
borlanic	0:380207fcb5c1	432	self.db_logger.update_build_id_info(
borlanic	0:380207fcb5c1	433	self.db_logger_build_id,
borlanic	0:380207fcb5c1	434	_extra=json.dumps(self.dump_options()))
borlanic	0:380207fcb5c1	435	self.db_logger.disconnect();
borlanic	0:380207fcb5c1	436
borlanic	0:380207fcb5c1	437	valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
borlanic	0:380207fcb5c1	438	skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
borlanic	0:380207fcb5c1	439
borlanic	0:380207fcb5c1	440	for skipped_test_id in skipped_test_map_keys:
borlanic	0:380207fcb5c1	441	test_suite_properties['skipped'].append(skipped_test_id)
borlanic	0:380207fcb5c1	442
borlanic	0:380207fcb5c1	443
borlanic	0:380207fcb5c1	444	# First pass through all tests and determine which libraries need to be built
borlanic	0:380207fcb5c1	445	libraries = []
borlanic	0:380207fcb5c1	446	for test_id in valid_test_map_keys:
borlanic	0:380207fcb5c1	447	test = TEST_MAP[test_id]
borlanic	0:380207fcb5c1	448
borlanic	0:380207fcb5c1	449	# Detect which lib should be added to test
borlanic	0:380207fcb5c1	450	# Some libs have to compiled like RTOS or ETH
borlanic	0:380207fcb5c1	451	for lib in LIBRARIES:
borlanic	0:380207fcb5c1	452	if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
borlanic	0:380207fcb5c1	453	libraries.append(lib['id'])
borlanic	0:380207fcb5c1	454
borlanic	0:380207fcb5c1	455
borlanic	0:380207fcb5c1	456	clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
borlanic	0:380207fcb5c1	457
borlanic	0:380207fcb5c1	458	# Build all required libraries
borlanic	0:380207fcb5c1	459	for lib_id in libraries:
borlanic	0:380207fcb5c1	460	try:
borlanic	0:380207fcb5c1	461	build_lib(lib_id,
borlanic	0:380207fcb5c1	462	T,
borlanic	0:380207fcb5c1	463	toolchain,
borlanic	0:380207fcb5c1	464	verbose=self.opts_verbose,
borlanic	0:380207fcb5c1	465	clean=clean_mbed_libs_options,
borlanic	0:380207fcb5c1	466	jobs=self.opts_jobs,
borlanic	0:380207fcb5c1	467	report=build_report,
borlanic	0:380207fcb5c1	468	properties=build_properties,
borlanic	0:380207fcb5c1	469	build_profile=profile)
borlanic	0:380207fcb5c1	470
borlanic	0:380207fcb5c1	471	except ToolException:
borlanic	0:380207fcb5c1	472	print(self.logger.log_line(
borlanic	0:380207fcb5c1	473	self.logger.LogType.ERROR,
borlanic	0:380207fcb5c1	474	'There were errors while building library %s' % lib_id))
borlanic	0:380207fcb5c1	475	continue
borlanic	0:380207fcb5c1	476
borlanic	0:380207fcb5c1	477
borlanic	0:380207fcb5c1	478	for test_id in valid_test_map_keys:
borlanic	0:380207fcb5c1	479	test = TEST_MAP[test_id]
borlanic	0:380207fcb5c1	480
borlanic	0:380207fcb5c1	481	test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
borlanic	0:380207fcb5c1	482
borlanic	0:380207fcb5c1	483	# TODO: move this 2 below loops to separate function
borlanic	0:380207fcb5c1	484	INC_DIRS = []
borlanic	0:380207fcb5c1	485	for lib_id in libraries:
borlanic	0:380207fcb5c1	486	if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
borlanic	0:380207fcb5c1	487	INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
borlanic	0:380207fcb5c1	488
borlanic	0:380207fcb5c1	489	MACROS = []
borlanic	0:380207fcb5c1	490	for lib_id in libraries:
borlanic	0:380207fcb5c1	491	if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
borlanic	0:380207fcb5c1	492	MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
borlanic	0:380207fcb5c1	493	MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
borlanic	0:380207fcb5c1	494	MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
borlanic	0:380207fcb5c1	495	test_uuid = uuid.uuid4()
borlanic	0:380207fcb5c1	496	MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
borlanic	0:380207fcb5c1	497
borlanic	0:380207fcb5c1	498	# Prepare extended test results data structure (it can be used to generate detailed test report)
borlanic	0:380207fcb5c1	499	if target not in self.test_summary_ext:
borlanic	0:380207fcb5c1	500	self.test_summary_ext[target] = {} # test_summary_ext : toolchain
borlanic	0:380207fcb5c1	501	if toolchain not in self.test_summary_ext[target]:
borlanic	0:380207fcb5c1	502	self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
borlanic	0:380207fcb5c1	503
borlanic	0:380207fcb5c1	504	tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
borlanic	0:380207fcb5c1	505
borlanic	0:380207fcb5c1	506	project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
borlanic	0:380207fcb5c1	507	try:
borlanic	0:380207fcb5c1	508	path = build_project(test.source_dir, join(build_dir, test_id), T,
borlanic	0:380207fcb5c1	509	toolchain, test.dependencies, clean=clean_project_options,
borlanic	0:380207fcb5c1	510	verbose=self.opts_verbose, name=project_name, macros=MACROS,
borlanic	0:380207fcb5c1	511	inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
borlanic	0:380207fcb5c1	512	properties=build_properties, project_id=test_id,
borlanic	0:380207fcb5c1	513	project_description=test.get_description(),
borlanic	0:380207fcb5c1	514	build_profile=profile, stats_depth=stats_depth)
borlanic	0:380207fcb5c1	515
borlanic	0:380207fcb5c1	516	except Exception as e:
borlanic	0:380207fcb5c1	517	project_name_str = project_name if project_name is not None else test_id
borlanic	0:380207fcb5c1	518
borlanic	0:380207fcb5c1	519
borlanic	0:380207fcb5c1	520	test_result = self.TEST_RESULT_FAIL
borlanic	0:380207fcb5c1	521
borlanic	0:380207fcb5c1	522	if isinstance(e, ToolException):
borlanic	0:380207fcb5c1	523	print(self.logger.log_line(
borlanic	0:380207fcb5c1	524	self.logger.LogType.ERROR,
borlanic	0:380207fcb5c1	525	'There were errors while building project %s' %
borlanic	0:380207fcb5c1	526	project_name_str))
borlanic	0:380207fcb5c1	527	test_result = self.TEST_RESULT_BUILD_FAILED
borlanic	0:380207fcb5c1	528	elif isinstance(e, NotSupportedException):
borlanic	0:380207fcb5c1	529	print(elf.logger.log_line(
borlanic	0:380207fcb5c1	530	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	531	'Project %s is not supported' % project_name_str))
borlanic	0:380207fcb5c1	532	test_result = self.TEST_RESULT_NOT_SUPPORTED
borlanic	0:380207fcb5c1	533
borlanic	0:380207fcb5c1	534
borlanic	0:380207fcb5c1	535	# Append test results to global test summary
borlanic	0:380207fcb5c1	536	self.test_summary.append(
borlanic	0:380207fcb5c1	537	(test_result, target, toolchain, test_id,
borlanic	0:380207fcb5c1	538	test.get_description(), 0, 0, '-')
borlanic	0:380207fcb5c1	539	)
borlanic	0:380207fcb5c1	540
borlanic	0:380207fcb5c1	541	# Add detailed test result to test summary structure
borlanic	0:380207fcb5c1	542	if test_id not in self.test_summary_ext[target][toolchain]:
borlanic	0:380207fcb5c1	543	self.test_summary_ext[target][toolchain][test_id] = []
borlanic	0:380207fcb5c1	544
borlanic	0:380207fcb5c1	545	self.test_summary_ext[target][toolchain][test_id].append({ 0: {
borlanic	0:380207fcb5c1	546	'result' : test_result,
borlanic	0:380207fcb5c1	547	'output' : '',
borlanic	0:380207fcb5c1	548	'target_name' : target,
borlanic	0:380207fcb5c1	549	'target_name_unique': target,
borlanic	0:380207fcb5c1	550	'toolchain_name' : toolchain,
borlanic	0:380207fcb5c1	551	'id' : test_id,
borlanic	0:380207fcb5c1	552	'description' : test.get_description(),
borlanic	0:380207fcb5c1	553	'elapsed_time' : 0,
borlanic	0:380207fcb5c1	554	'duration' : 0,
borlanic	0:380207fcb5c1	555	'copy_method' : None
borlanic	0:380207fcb5c1	556	}})
borlanic	0:380207fcb5c1	557	continue
borlanic	0:380207fcb5c1	558
borlanic	0:380207fcb5c1	559	if self.opts_only_build_tests:
borlanic	0:380207fcb5c1	560	# With this option we are skipping testing phase
borlanic	0:380207fcb5c1	561	continue
borlanic	0:380207fcb5c1	562
borlanic	0:380207fcb5c1	563	# Test duration can be increased by global value
borlanic	0:380207fcb5c1	564	test_duration = test.duration
borlanic	0:380207fcb5c1	565	if self.opts_extend_test_timeout is not None:
borlanic	0:380207fcb5c1	566	test_duration += self.opts_extend_test_timeout
borlanic	0:380207fcb5c1	567
borlanic	0:380207fcb5c1	568	# For an automated test the duration act as a timeout after
borlanic	0:380207fcb5c1	569	# which the test gets interrupted
borlanic	0:380207fcb5c1	570	test_spec = self.shape_test_request(target, path, test_id, test_duration)
borlanic	0:380207fcb5c1	571	test_loops = self.get_test_loop_count(test_id)
borlanic	0:380207fcb5c1	572
borlanic	0:380207fcb5c1	573	test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
borlanic	0:380207fcb5c1	574	test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
borlanic	0:380207fcb5c1	575	test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
borlanic	0:380207fcb5c1	576
borlanic	0:380207fcb5c1	577	# read MUTs, test specification and perform tests
borlanic	0:380207fcb5c1	578	handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
borlanic	0:380207fcb5c1	579
borlanic	0:380207fcb5c1	580	if handle_results is None:
borlanic	0:380207fcb5c1	581	continue
borlanic	0:380207fcb5c1	582
borlanic	0:380207fcb5c1	583	for handle_result in handle_results:
borlanic	0:380207fcb5c1	584	if handle_result:
borlanic	0:380207fcb5c1	585	single_test_result, detailed_test_results = handle_result
borlanic	0:380207fcb5c1	586	else:
borlanic	0:380207fcb5c1	587	continue
borlanic	0:380207fcb5c1	588
borlanic	0:380207fcb5c1	589	# Append test results to global test summary
borlanic	0:380207fcb5c1	590	if single_test_result is not None:
borlanic	0:380207fcb5c1	591	self.test_summary.append(single_test_result)
borlanic	0:380207fcb5c1	592
borlanic	0:380207fcb5c1	593	# Add detailed test result to test summary structure
borlanic	0:380207fcb5c1	594	if target not in self.test_summary_ext[target][toolchain]:
borlanic	0:380207fcb5c1	595	if test_id not in self.test_summary_ext[target][toolchain]:
borlanic	0:380207fcb5c1	596	self.test_summary_ext[target][toolchain][test_id] = []
borlanic	0:380207fcb5c1	597
borlanic	0:380207fcb5c1	598	append_test_result = detailed_test_results
borlanic	0:380207fcb5c1	599
borlanic	0:380207fcb5c1	600	# If waterfall and consolidate-waterfall options are enabled,
borlanic	0:380207fcb5c1	601	# only include the last test result in the report.
borlanic	0:380207fcb5c1	602	if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
borlanic	0:380207fcb5c1	603	append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
borlanic	0:380207fcb5c1	604
borlanic	0:380207fcb5c1	605	self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
borlanic	0:380207fcb5c1	606
borlanic	0:380207fcb5c1	607	test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
borlanic	0:380207fcb5c1	608	self.test_suite_properties_ext[target][toolchain] = test_suite_properties
borlanic	0:380207fcb5c1	609
borlanic	0:380207fcb5c1	610	q.put(target + '_'.join(toolchains))
borlanic	0:380207fcb5c1	611	return
borlanic	0:380207fcb5c1	612
borlanic	0:380207fcb5c1	613	def execute(self):
borlanic	0:380207fcb5c1	614	clean = self.test_spec.get('clean', False)
borlanic	0:380207fcb5c1	615	test_ids = self.test_spec.get('test_ids', [])
borlanic	0:380207fcb5c1	616	q = Queue()
borlanic	0:380207fcb5c1	617
borlanic	0:380207fcb5c1	618	# Generate seed for shuffle if seed is not provided in
borlanic	0:380207fcb5c1	619	self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
borlanic	0:380207fcb5c1	620	if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
borlanic	0:380207fcb5c1	621	self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
borlanic	0:380207fcb5c1	622
borlanic	0:380207fcb5c1	623
borlanic	0:380207fcb5c1	624	if self.opts_parallel_test_exec:
borlanic	0:380207fcb5c1	625	###################################################################
borlanic	0:380207fcb5c1	626	# Experimental, parallel test execution per singletest instance.
borlanic	0:380207fcb5c1	627	###################################################################
borlanic	0:380207fcb5c1	628	execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
borlanic	0:380207fcb5c1	629	# Note: We are building here in parallel for each target separately!
borlanic	0:380207fcb5c1	630	# So we are not building the same thing multiple times and compilers
borlanic	0:380207fcb5c1	631	# in separate threads do not collide.
borlanic	0:380207fcb5c1	632	# Inside execute_thread_slice() function function handle() will be called to
borlanic	0:380207fcb5c1	633	# get information about available MUTs (per target).
borlanic	0:380207fcb5c1	634	for target, toolchains in self.test_spec['targets'].items():
borlanic	0:380207fcb5c1	635	self.test_suite_properties_ext[target] = {}
borlanic	0:380207fcb5c1	636	t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
borlanic	0:380207fcb5c1	637	t.daemon = True
borlanic	0:380207fcb5c1	638	t.start()
borlanic	0:380207fcb5c1	639	execute_threads.append(t)
borlanic	0:380207fcb5c1	640
borlanic	0:380207fcb5c1	641	for t in execute_threads:
borlanic	0:380207fcb5c1	642	q.get() # t.join() would block some threads because we should not wait in any order for thread end
borlanic	0:380207fcb5c1	643	else:
borlanic	0:380207fcb5c1	644	# Serialized (not parallel) test execution
borlanic	0:380207fcb5c1	645	for target, toolchains in self.test_spec['targets'].items():
borlanic	0:380207fcb5c1	646	if target not in self.test_suite_properties_ext:
borlanic	0:380207fcb5c1	647	self.test_suite_properties_ext[target] = {}
borlanic	0:380207fcb5c1	648
borlanic	0:380207fcb5c1	649	self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
borlanic	0:380207fcb5c1	650	q.get()
borlanic	0:380207fcb5c1	651
borlanic	0:380207fcb5c1	652	if self.db_logger:
borlanic	0:380207fcb5c1	653	self.db_logger.reconnect();
borlanic	0:380207fcb5c1	654	if self.db_logger.is_connected():
borlanic	0:380207fcb5c1	655	self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
borlanic	0:380207fcb5c1	656	self.db_logger.disconnect();
borlanic	0:380207fcb5c1	657
borlanic	0:380207fcb5c1	658	return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
borlanic	0:380207fcb5c1	659
borlanic	0:380207fcb5c1	660	def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
borlanic	0:380207fcb5c1	661	valid_test_map_keys = []
borlanic	0:380207fcb5c1	662
borlanic	0:380207fcb5c1	663	for test_id in test_map_keys:
borlanic	0:380207fcb5c1	664	test = TEST_MAP[test_id]
borlanic	0:380207fcb5c1	665	if self.opts_test_by_names and test_id not in self.opts_test_by_names:
borlanic	0:380207fcb5c1	666	continue
borlanic	0:380207fcb5c1	667
borlanic	0:380207fcb5c1	668	if test_ids and test_id not in test_ids:
borlanic	0:380207fcb5c1	669	continue
borlanic	0:380207fcb5c1	670
borlanic	0:380207fcb5c1	671	if self.opts_test_only_peripheral and not test.peripherals:
borlanic	0:380207fcb5c1	672	if self.opts_verbose_skipped_tests:
borlanic	0:380207fcb5c1	673	print(self.logger.log_line(
borlanic	0:380207fcb5c1	674	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	675	'Common test skipped for target %s' % target))
borlanic	0:380207fcb5c1	676	continue
borlanic	0:380207fcb5c1	677
borlanic	0:380207fcb5c1	678	if (self.opts_peripheral_by_names and test.peripherals and
borlanic	0:380207fcb5c1	679	not any((i in self.opts_peripheral_by_names)
borlanic	0:380207fcb5c1	680	for i in test.peripherals)):
borlanic	0:380207fcb5c1	681	# We will skip tests not forced with -p option
borlanic	0:380207fcb5c1	682	if self.opts_verbose_skipped_tests:
borlanic	0:380207fcb5c1	683	print(self.logger.log_line(
borlanic	0:380207fcb5c1	684	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	685	'Common test skipped for target %s' % target))
borlanic	0:380207fcb5c1	686	continue
borlanic	0:380207fcb5c1	687
borlanic	0:380207fcb5c1	688	if self.opts_test_only_common and test.peripherals:
borlanic	0:380207fcb5c1	689	if self.opts_verbose_skipped_tests:
borlanic	0:380207fcb5c1	690	print(self.logger.log_line(
borlanic	0:380207fcb5c1	691	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	692	'Peripheral test skipped for target %s' % target))
borlanic	0:380207fcb5c1	693	continue
borlanic	0:380207fcb5c1	694
borlanic	0:380207fcb5c1	695	if not include_non_automated and not test.automated:
borlanic	0:380207fcb5c1	696	if self.opts_verbose_skipped_tests:
borlanic	0:380207fcb5c1	697	print(self.logger.log_line(
borlanic	0:380207fcb5c1	698	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	699	'Non automated test skipped for target %s' % target))
borlanic	0:380207fcb5c1	700	continue
borlanic	0:380207fcb5c1	701
borlanic	0:380207fcb5c1	702	if test.is_supported(target, toolchain):
borlanic	0:380207fcb5c1	703	if test.peripherals is None and self.opts_only_build_tests:
borlanic	0:380207fcb5c1	704	# When users are using 'build only flag' and test do not have
borlanic	0:380207fcb5c1	705	# specified peripherals we can allow test building by default
borlanic	0:380207fcb5c1	706	pass
borlanic	0:380207fcb5c1	707	elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
borlanic	0:380207fcb5c1	708	# If we force peripheral with option -p we expect test
borlanic	0:380207fcb5c1	709	# to pass even if peripheral is not in MUTs file.
borlanic	0:380207fcb5c1	710	pass
borlanic	0:380207fcb5c1	711	elif not self.is_peripherals_available(target, test.peripherals):
borlanic	0:380207fcb5c1	712	if self.opts_verbose_skipped_tests:
borlanic	0:380207fcb5c1	713	if test.peripherals:
borlanic	0:380207fcb5c1	714	print(self.logger.log_line(
borlanic	0:380207fcb5c1	715	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	716	'Peripheral %s test skipped for target %s' %
borlanic	0:380207fcb5c1	717	(",".join(test.peripherals), target)))
borlanic	0:380207fcb5c1	718	else:
borlanic	0:380207fcb5c1	719	print(self.logger.log_line(
borlanic	0:380207fcb5c1	720	self.logger.LogType.INFO,
borlanic	0:380207fcb5c1	721	'Test %s skipped for target %s' %
borlanic	0:380207fcb5c1	722	(test_id, target)))
borlanic	0:380207fcb5c1	723	continue
borlanic	0:380207fcb5c1	724
borlanic	0:380207fcb5c1	725	# The test has made it through all the filters, so add it to the valid tests list
borlanic	0:380207fcb5c1	726	valid_test_map_keys.append(test_id)
borlanic	0:380207fcb5c1	727
borlanic	0:380207fcb5c1	728	return valid_test_map_keys
borlanic	0:380207fcb5c1	729
borlanic	0:380207fcb5c1	730	def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
borlanic	0:380207fcb5c1	731	# NOTE: This will not preserve order
borlanic	0:380207fcb5c1	732	return list(set(all_test_map_keys) - set(valid_test_map_keys))
borlanic	0:380207fcb5c1	733
borlanic	0:380207fcb5c1	734	def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
borlanic	0:380207fcb5c1	735	""" Prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	736	table shows text x toolchain test result matrix
borlanic	0:380207fcb5c1	737	"""
borlanic	0:380207fcb5c1	738	RESULT_INDEX = 0
borlanic	0:380207fcb5c1	739	TARGET_INDEX = 1
borlanic	0:380207fcb5c1	740	TOOLCHAIN_INDEX = 2
borlanic	0:380207fcb5c1	741	TEST_INDEX = 3
borlanic	0:380207fcb5c1	742	DESC_INDEX = 4
borlanic	0:380207fcb5c1	743
borlanic	0:380207fcb5c1	744	unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
borlanic	0:380207fcb5c1	745	unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
borlanic	0:380207fcb5c1	746	unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
borlanic	0:380207fcb5c1	747	unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
borlanic	0:380207fcb5c1	748
borlanic	0:380207fcb5c1	749	result = "Test summary:\n"
borlanic	0:380207fcb5c1	750	for target in unique_targets:
borlanic	0:380207fcb5c1	751	result_dict = {} # test : { toolchain : result }
borlanic	0:380207fcb5c1	752	unique_target_toolchains = []
borlanic	0:380207fcb5c1	753	for test in test_summary:
borlanic	0:380207fcb5c1	754	if test[TARGET_INDEX] == target:
borlanic	0:380207fcb5c1	755	if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
borlanic	0:380207fcb5c1	756	unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
borlanic	0:380207fcb5c1	757	if test[TEST_INDEX] not in result_dict:
borlanic	0:380207fcb5c1	758	result_dict[test[TEST_INDEX]] = {}
borlanic	0:380207fcb5c1	759	result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
borlanic	0:380207fcb5c1	760
borlanic	0:380207fcb5c1	761	pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
borlanic	0:380207fcb5c1	762	pt = PrettyTable(pt_cols)
borlanic	0:380207fcb5c1	763	for col in pt_cols:
borlanic	0:380207fcb5c1	764	pt.align[col] = "l"
borlanic	0:380207fcb5c1	765	pt.padding_width = 1 # One space between column edges and contents (default)
borlanic	0:380207fcb5c1	766
borlanic	0:380207fcb5c1	767	for test in unique_tests:
borlanic	0:380207fcb5c1	768	if test in result_dict:
borlanic	0:380207fcb5c1	769	test_results = result_dict[test]
borlanic	0:380207fcb5c1	770	if test in unique_test_desc:
borlanic	0:380207fcb5c1	771	row = [target, test, unique_test_desc[test]]
borlanic	0:380207fcb5c1	772	for toolchain in unique_toolchains:
borlanic	0:380207fcb5c1	773	if toolchain in test_results:
borlanic	0:380207fcb5c1	774	row.append(test_results[toolchain])
borlanic	0:380207fcb5c1	775	pt.add_row(row)
borlanic	0:380207fcb5c1	776	result += pt.get_string()
borlanic	0:380207fcb5c1	777	shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
borlanic	0:380207fcb5c1	778	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
borlanic	0:380207fcb5c1	779	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
borlanic	0:380207fcb5c1	780	return result
borlanic	0:380207fcb5c1	781
borlanic	0:380207fcb5c1	782	def generate_test_summary(self, test_summary, shuffle_seed=None):
borlanic	0:380207fcb5c1	783	""" Prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	784	table shows target x test results matrix across
borlanic	0:380207fcb5c1	785	"""
borlanic	0:380207fcb5c1	786	success_code = 0 # Success code that can be leter returned to
borlanic	0:380207fcb5c1	787	result = "Test summary:\n"
borlanic	0:380207fcb5c1	788	# Pretty table package is used to print results
borlanic	0:380207fcb5c1	789	pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
borlanic	0:380207fcb5c1	790	"Elapsed Time (sec)", "Timeout (sec)", "Loops"])
borlanic	0:380207fcb5c1	791	pt.align["Result"] = "l" # Left align
borlanic	0:380207fcb5c1	792	pt.align["Target"] = "l" # Left align
borlanic	0:380207fcb5c1	793	pt.align["Toolchain"] = "l" # Left align
borlanic	0:380207fcb5c1	794	pt.align["Test ID"] = "l" # Left align
borlanic	0:380207fcb5c1	795	pt.align["Test Description"] = "l" # Left align
borlanic	0:380207fcb5c1	796	pt.padding_width = 1 # One space between column edges and contents (default)
borlanic	0:380207fcb5c1	797
borlanic	0:380207fcb5c1	798	result_dict = {self.TEST_RESULT_OK : 0,
borlanic	0:380207fcb5c1	799	self.TEST_RESULT_FAIL : 0,
borlanic	0:380207fcb5c1	800	self.TEST_RESULT_ERROR : 0,
borlanic	0:380207fcb5c1	801	self.TEST_RESULT_UNDEF : 0,
borlanic	0:380207fcb5c1	802	self.TEST_RESULT_IOERR_COPY : 0,
borlanic	0:380207fcb5c1	803	self.TEST_RESULT_IOERR_DISK : 0,
borlanic	0:380207fcb5c1	804	self.TEST_RESULT_IOERR_SERIAL : 0,
borlanic	0:380207fcb5c1	805	self.TEST_RESULT_NO_IMAGE : 0,
borlanic	0:380207fcb5c1	806	self.TEST_RESULT_TIMEOUT : 0,
borlanic	0:380207fcb5c1	807	self.TEST_RESULT_MBED_ASSERT : 0,
borlanic	0:380207fcb5c1	808	self.TEST_RESULT_BUILD_FAILED : 0,
borlanic	0:380207fcb5c1	809	self.TEST_RESULT_NOT_SUPPORTED : 0
borlanic	0:380207fcb5c1	810	}
borlanic	0:380207fcb5c1	811
borlanic	0:380207fcb5c1	812	for test in test_summary:
borlanic	0:380207fcb5c1	813	if test[0] in result_dict:
borlanic	0:380207fcb5c1	814	result_dict[test[0]] += 1
borlanic	0:380207fcb5c1	815	pt.add_row(test)
borlanic	0:380207fcb5c1	816	result += pt.get_string()
borlanic	0:380207fcb5c1	817	result += "\n"
borlanic	0:380207fcb5c1	818
borlanic	0:380207fcb5c1	819	# Print result count
borlanic	0:380207fcb5c1	820	result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()])
borlanic	0:380207fcb5c1	821	shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
borlanic	0:380207fcb5c1	822	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
borlanic	0:380207fcb5c1	823	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
borlanic	0:380207fcb5c1	824	return result
borlanic	0:380207fcb5c1	825
borlanic	0:380207fcb5c1	826	def test_loop_list_to_dict(self, test_loops_str):
borlanic	0:380207fcb5c1	827	""" Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
borlanic	0:380207fcb5c1	828	"""
borlanic	0:380207fcb5c1	829	result = {}
borlanic	0:380207fcb5c1	830	if test_loops_str:
borlanic	0:380207fcb5c1	831	test_loops = test_loops_str
borlanic	0:380207fcb5c1	832	for test_loop in test_loops:
borlanic	0:380207fcb5c1	833	test_loop_count = test_loop.split('=')
borlanic	0:380207fcb5c1	834	if len(test_loop_count) == 2:
borlanic	0:380207fcb5c1	835	_test_id, _test_loops = test_loop_count
borlanic	0:380207fcb5c1	836	try:
borlanic	0:380207fcb5c1	837	_test_loops = int(_test_loops)
borlanic	0:380207fcb5c1	838	except:
borlanic	0:380207fcb5c1	839	continue
borlanic	0:380207fcb5c1	840	result[_test_id] = _test_loops
borlanic	0:380207fcb5c1	841	return result
borlanic	0:380207fcb5c1	842
borlanic	0:380207fcb5c1	843	def get_test_loop_count(self, test_id):
borlanic	0:380207fcb5c1	844	""" This function returns no. of loops per test (deducted by test_id_.
borlanic	0:380207fcb5c1	845	If test is not in list of redefined loop counts it will use default value.
borlanic	0:380207fcb5c1	846	"""
borlanic	0:380207fcb5c1	847	result = self.GLOBAL_LOOPS_COUNT
borlanic	0:380207fcb5c1	848	if test_id in self.TEST_LOOPS_DICT:
borlanic	0:380207fcb5c1	849	result = self.TEST_LOOPS_DICT[test_id]
borlanic	0:380207fcb5c1	850	return result
borlanic	0:380207fcb5c1	851
borlanic	0:380207fcb5c1	852	def delete_file(self, file_path):
borlanic	0:380207fcb5c1	853	""" Remove file from the system
borlanic	0:380207fcb5c1	854	"""
borlanic	0:380207fcb5c1	855	result = True
borlanic	0:380207fcb5c1	856	resutl_msg = ""
borlanic	0:380207fcb5c1	857	try:
borlanic	0:380207fcb5c1	858	os.remove(file_path)
borlanic	0:380207fcb5c1	859	except Exception as e:
borlanic	0:380207fcb5c1	860	resutl_msg = e
borlanic	0:380207fcb5c1	861	result = False
borlanic	0:380207fcb5c1	862	return result, resutl_msg
borlanic	0:380207fcb5c1	863
borlanic	0:380207fcb5c1	864	def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
borlanic	0:380207fcb5c1	865	""" Test is being invoked for given MUT.
borlanic	0:380207fcb5c1	866	"""
borlanic	0:380207fcb5c1	867	# Get test information, image and test timeout
borlanic	0:380207fcb5c1	868	test_id = data['test_id']
borlanic	0:380207fcb5c1	869	test = TEST_MAP[test_id]
borlanic	0:380207fcb5c1	870	test_description = TEST_MAP[test_id].get_description()
borlanic	0:380207fcb5c1	871	image = data["image"]
borlanic	0:380207fcb5c1	872	duration = data.get("duration", 10)
borlanic	0:380207fcb5c1	873
borlanic	0:380207fcb5c1	874	if mut is None:
borlanic	0:380207fcb5c1	875	print("Error: No Mbed available: MUT[%s]" % data['mcu'])
borlanic	0:380207fcb5c1	876	return None
borlanic	0:380207fcb5c1	877
borlanic	0:380207fcb5c1	878	mcu = mut['mcu']
borlanic	0:380207fcb5c1	879	copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
borlanic	0:380207fcb5c1	880
borlanic	0:380207fcb5c1	881	if self.db_logger:
borlanic	0:380207fcb5c1	882	self.db_logger.reconnect()
borlanic	0:380207fcb5c1	883
borlanic	0:380207fcb5c1	884	selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
borlanic	0:380207fcb5c1	885
borlanic	0:380207fcb5c1	886	# Tests can be looped so test results must be stored for the same test
borlanic	0:380207fcb5c1	887	test_all_result = []
borlanic	0:380207fcb5c1	888	# Test results for one test ran few times
borlanic	0:380207fcb5c1	889	detailed_test_results = {} # { Loop_number: { results ... } }
borlanic	0:380207fcb5c1	890
borlanic	0:380207fcb5c1	891	for test_index in range(test_loops):
borlanic	0:380207fcb5c1	892
borlanic	0:380207fcb5c1	893	# If mbedls is available and we are auto detecting MUT info,
borlanic	0:380207fcb5c1	894	# update MUT info (mounting may changed)
borlanic	0:380207fcb5c1	895	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
borlanic	0:380207fcb5c1	896	platform_name_filter = [mcu]
borlanic	0:380207fcb5c1	897	muts_list = {}
borlanic	0:380207fcb5c1	898	found = False
borlanic	0:380207fcb5c1	899
borlanic	0:380207fcb5c1	900	for i in range(0, 60):
borlanic	0:380207fcb5c1	901	print('Looking for %s with MBEDLS' % mcu)
borlanic	0:380207fcb5c1	902	muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
borlanic	0:380207fcb5c1	903
borlanic	0:380207fcb5c1	904	if 1 not in muts_list:
borlanic	0:380207fcb5c1	905	sleep(3)
borlanic	0:380207fcb5c1	906	else:
borlanic	0:380207fcb5c1	907	found = True
borlanic	0:380207fcb5c1	908	break
borlanic	0:380207fcb5c1	909
borlanic	0:380207fcb5c1	910	if not found:
borlanic	0:380207fcb5c1	911	print("Error: mbed not found with MBEDLS: %s" % data['mcu'])
borlanic	0:380207fcb5c1	912	return None
borlanic	0:380207fcb5c1	913	else:
borlanic	0:380207fcb5c1	914	mut = muts_list[1]
borlanic	0:380207fcb5c1	915
borlanic	0:380207fcb5c1	916	disk = mut.get('disk')
borlanic	0:380207fcb5c1	917	port = mut.get('port')
borlanic	0:380207fcb5c1	918
borlanic	0:380207fcb5c1	919	if disk is None or port is None:
borlanic	0:380207fcb5c1	920	return None
borlanic	0:380207fcb5c1	921
borlanic	0:380207fcb5c1	922	target_by_mcu = TARGET_MAP[mut['mcu']]
borlanic	0:380207fcb5c1	923	target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
borlanic	0:380207fcb5c1	924	# Some extra stuff can be declared in MUTs structure
borlanic	0:380207fcb5c1	925	reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
borlanic	0:380207fcb5c1	926	reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
borlanic	0:380207fcb5c1	927
borlanic	0:380207fcb5c1	928	# When the build and test system were separate, this was relative to a
borlanic	0:380207fcb5c1	929	# base network folder base path: join(NETWORK_BASE_PATH, )
borlanic	0:380207fcb5c1	930	image_path = image
borlanic	0:380207fcb5c1	931
borlanic	0:380207fcb5c1	932	# Host test execution
borlanic	0:380207fcb5c1	933	start_host_exec_time = time()
borlanic	0:380207fcb5c1	934
borlanic	0:380207fcb5c1	935	single_test_result = self.TEST_RESULT_UNDEF # single test run result
borlanic	0:380207fcb5c1	936	_copy_method = selected_copy_method
borlanic	0:380207fcb5c1	937
borlanic	0:380207fcb5c1	938	if not exists(image_path):
borlanic	0:380207fcb5c1	939	single_test_result = self.TEST_RESULT_NO_IMAGE
borlanic	0:380207fcb5c1	940	elapsed_time = 0
borlanic	0:380207fcb5c1	941	single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
borlanic	0:380207fcb5c1	942	print(single_test_output)
borlanic	0:380207fcb5c1	943	else:
borlanic	0:380207fcb5c1	944	# Host test execution
borlanic	0:380207fcb5c1	945	start_host_exec_time = time()
borlanic	0:380207fcb5c1	946
borlanic	0:380207fcb5c1	947	host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
borlanic	0:380207fcb5c1	948	host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
borlanic	0:380207fcb5c1	949	host_test_result = self.run_host_test(test.host_test,
borlanic	0:380207fcb5c1	950	image_path, disk, port, duration,
borlanic	0:380207fcb5c1	951	micro=target_name,
borlanic	0:380207fcb5c1	952	verbose=host_test_verbose,
borlanic	0:380207fcb5c1	953	reset=host_test_reset,
borlanic	0:380207fcb5c1	954	reset_tout=reset_tout,
borlanic	0:380207fcb5c1	955	copy_method=selected_copy_method,
borlanic	0:380207fcb5c1	956	program_cycle_s=target_by_mcu.program_cycle_s)
borlanic	0:380207fcb5c1	957	single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
borlanic	0:380207fcb5c1	958
borlanic	0:380207fcb5c1	959	# Store test result
borlanic	0:380207fcb5c1	960	test_all_result.append(single_test_result)
borlanic	0:380207fcb5c1	961	total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
borlanic	0:380207fcb5c1	962	elapsed_time = single_testduration # TIme of single test case execution after reset
borlanic	0:380207fcb5c1	963
borlanic	0:380207fcb5c1	964	detailed_test_results[test_index] = {
borlanic	0:380207fcb5c1	965	'result' : single_test_result,
borlanic	0:380207fcb5c1	966	'output' : single_test_output,
borlanic	0:380207fcb5c1	967	'target_name' : target_name,
borlanic	0:380207fcb5c1	968	'target_name_unique' : target_name_unique,
borlanic	0:380207fcb5c1	969	'toolchain_name' : toolchain_name,
borlanic	0:380207fcb5c1	970	'id' : test_id,
borlanic	0:380207fcb5c1	971	'description' : test_description,
borlanic	0:380207fcb5c1	972	'elapsed_time' : round(elapsed_time, 2),
borlanic	0:380207fcb5c1	973	'duration' : single_timeout,
borlanic	0:380207fcb5c1	974	'copy_method' : _copy_method,
borlanic	0:380207fcb5c1	975	}
borlanic	0:380207fcb5c1	976
borlanic	0:380207fcb5c1	977	print(self.print_test_result(
borlanic	0:380207fcb5c1	978	single_test_result, target_name_unique, toolchain_name, test_id,
borlanic	0:380207fcb5c1	979	test_description, elapsed_time, single_timeout))
borlanic	0:380207fcb5c1	980
borlanic	0:380207fcb5c1	981	# Update database entries for ongoing test
borlanic	0:380207fcb5c1	982	if self.db_logger and self.db_logger.is_connected():
borlanic	0:380207fcb5c1	983	test_type = 'SingleTest'
borlanic	0:380207fcb5c1	984	self.db_logger.insert_test_entry(self.db_logger_build_id,
borlanic	0:380207fcb5c1	985	target_name,
borlanic	0:380207fcb5c1	986	toolchain_name,
borlanic	0:380207fcb5c1	987	test_type,
borlanic	0:380207fcb5c1	988	test_id,
borlanic	0:380207fcb5c1	989	single_test_result,
borlanic	0:380207fcb5c1	990	single_test_output,
borlanic	0:380207fcb5c1	991	elapsed_time,
borlanic	0:380207fcb5c1	992	single_timeout,
borlanic	0:380207fcb5c1	993	test_index)
borlanic	0:380207fcb5c1	994
borlanic	0:380207fcb5c1	995	# If we perform waterfall test we test until we get OK and we stop testing
borlanic	0:380207fcb5c1	996	if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
borlanic	0:380207fcb5c1	997	break
borlanic	0:380207fcb5c1	998
borlanic	0:380207fcb5c1	999	if self.db_logger:
borlanic	0:380207fcb5c1	1000	self.db_logger.disconnect()
borlanic	0:380207fcb5c1	1001
borlanic	0:380207fcb5c1	1002	return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
borlanic	0:380207fcb5c1	1003	target_name_unique,
borlanic	0:380207fcb5c1	1004	toolchain_name,
borlanic	0:380207fcb5c1	1005	test_id,
borlanic	0:380207fcb5c1	1006	test_description,
borlanic	0:380207fcb5c1	1007	round(elapsed_time, 2),
borlanic	0:380207fcb5c1	1008	single_timeout,
borlanic	0:380207fcb5c1	1009	self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
borlanic	0:380207fcb5c1	1010
borlanic	0:380207fcb5c1	1011	def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
borlanic	0:380207fcb5c1	1012	""" Function determines MUT's mbed disk/port and copies binary to
borlanic	0:380207fcb5c1	1013	target.
borlanic	0:380207fcb5c1	1014	"""
borlanic	0:380207fcb5c1	1015	handle_results = []
borlanic	0:380207fcb5c1	1016	data = json.loads(test_spec)
borlanic	0:380207fcb5c1	1017
borlanic	0:380207fcb5c1	1018	# Find a suitable MUT:
borlanic	0:380207fcb5c1	1019	mut = None
borlanic	0:380207fcb5c1	1020	for id, m in self.muts.items():
borlanic	0:380207fcb5c1	1021	if m['mcu'] == data['mcu']:
borlanic	0:380207fcb5c1	1022	mut = m
borlanic	0:380207fcb5c1	1023	handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
borlanic	0:380207fcb5c1	1024	handle_results.append(handle_result)
borlanic	0:380207fcb5c1	1025
borlanic	0:380207fcb5c1	1026	return handle_results
borlanic	0:380207fcb5c1	1027
borlanic	0:380207fcb5c1	1028	def print_test_result(self, test_result, target_name, toolchain_name,
borlanic	0:380207fcb5c1	1029	test_id, test_description, elapsed_time, duration):
borlanic	0:380207fcb5c1	1030	""" Use specific convention to print test result and related data
borlanic	0:380207fcb5c1	1031	"""
borlanic	0:380207fcb5c1	1032	tokens = []
borlanic	0:380207fcb5c1	1033	tokens.append("TargetTest")
borlanic	0:380207fcb5c1	1034	tokens.append(target_name)
borlanic	0:380207fcb5c1	1035	tokens.append(toolchain_name)
borlanic	0:380207fcb5c1	1036	tokens.append(test_id)
borlanic	0:380207fcb5c1	1037	tokens.append(test_description)
borlanic	0:380207fcb5c1	1038	separator = "::"
borlanic	0:380207fcb5c1	1039	time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
borlanic	0:380207fcb5c1	1040	result = separator.join(tokens) + " [" + test_result +"]" + time_info
borlanic	0:380207fcb5c1	1041	return Fore.MAGENTA + result + Fore.RESET
borlanic	0:380207fcb5c1	1042
borlanic	0:380207fcb5c1	1043	def shape_test_loop_ok_result_count(self, test_all_result):
borlanic	0:380207fcb5c1	1044	""" Reformats list of results to simple string
borlanic	0:380207fcb5c1	1045	"""
borlanic	0:380207fcb5c1	1046	test_loop_count = len(test_all_result)
borlanic	0:380207fcb5c1	1047	test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
borlanic	0:380207fcb5c1	1048	return "%d/%d"% (test_loop_ok_result, test_loop_count)
borlanic	0:380207fcb5c1	1049
borlanic	0:380207fcb5c1	1050	def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
borlanic	0:380207fcb5c1	1051	""" Reformats list of results to simple string
borlanic	0:380207fcb5c1	1052	"""
borlanic	0:380207fcb5c1	1053	result = self.TEST_RESULT_FAIL
borlanic	0:380207fcb5c1	1054
borlanic	0:380207fcb5c1	1055	if all(test_all_result[0] == res for res in test_all_result):
borlanic	0:380207fcb5c1	1056	result = test_all_result[0]
borlanic	0:380207fcb5c1	1057	elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
borlanic	0:380207fcb5c1	1058	result = self.TEST_RESULT_OK
borlanic	0:380207fcb5c1	1059
borlanic	0:380207fcb5c1	1060	return result
borlanic	0:380207fcb5c1	1061
borlanic	0:380207fcb5c1	1062	def run_host_test(self, name, image_path, disk, port, duration,
borlanic	0:380207fcb5c1	1063	micro=None, reset=None, reset_tout=None,
borlanic	0:380207fcb5c1	1064	verbose=False, copy_method=None, program_cycle_s=None):
borlanic	0:380207fcb5c1	1065	""" Function creates new process with host test configured with particular test case.
borlanic	0:380207fcb5c1	1066	Function also is pooling for serial port activity from process to catch all data
borlanic	0:380207fcb5c1	1067	printed by test runner and host test during test execution
borlanic	0:380207fcb5c1	1068	"""
borlanic	0:380207fcb5c1	1069
borlanic	0:380207fcb5c1	1070	def get_char_from_queue(obs):
borlanic	0:380207fcb5c1	1071	""" Get character from queue safe way
borlanic	0:380207fcb5c1	1072	"""
borlanic	0:380207fcb5c1	1073	try:
borlanic	0:380207fcb5c1	1074	c = obs.queue.get(block=True, timeout=0.5)
borlanic	0:380207fcb5c1	1075	except Empty:
borlanic	0:380207fcb5c1	1076	c = None
borlanic	0:380207fcb5c1	1077	return c
borlanic	0:380207fcb5c1	1078
borlanic	0:380207fcb5c1	1079	def filter_queue_char(c):
borlanic	0:380207fcb5c1	1080	""" Filters out non ASCII characters from serial port
borlanic	0:380207fcb5c1	1081	"""
borlanic	0:380207fcb5c1	1082	if ord(c) not in range(128):
borlanic	0:380207fcb5c1	1083	c = ' '
borlanic	0:380207fcb5c1	1084	return c
borlanic	0:380207fcb5c1	1085
borlanic	0:380207fcb5c1	1086	def get_test_result(output):
borlanic	0:380207fcb5c1	1087	""" Parse test 'output' data
borlanic	0:380207fcb5c1	1088	"""
borlanic	0:380207fcb5c1	1089	result = self.TEST_RESULT_TIMEOUT
borlanic	0:380207fcb5c1	1090	for line in "".join(output).splitlines():
borlanic	0:380207fcb5c1	1091	search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
borlanic	0:380207fcb5c1	1092	if search_result and len(search_result.groups()):
borlanic	0:380207fcb5c1	1093	result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
borlanic	0:380207fcb5c1	1094	break
borlanic	0:380207fcb5c1	1095	return result
borlanic	0:380207fcb5c1	1096
borlanic	0:380207fcb5c1	1097	def get_auto_property_value(property_name, line):
borlanic	0:380207fcb5c1	1098	""" Scans auto detection line from MUT and returns scanned parameter 'property_name'
borlanic	0:380207fcb5c1	1099	Returns string
borlanic	0:380207fcb5c1	1100	"""
borlanic	0:380207fcb5c1	1101	result = None
borlanic	0:380207fcb5c1	1102	if re.search("HOST: Property '%s'"% property_name, line) is not None:
borlanic	0:380207fcb5c1	1103	property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
borlanic	0:380207fcb5c1	1104	if property is not None and len(property.groups()) == 1:
borlanic	0:380207fcb5c1	1105	result = property.groups()[0]
borlanic	0:380207fcb5c1	1106	return result
borlanic	0:380207fcb5c1	1107
borlanic	0:380207fcb5c1	1108	cmd = ["python",
borlanic	0:380207fcb5c1	1109	'%s.py'% name,
borlanic	0:380207fcb5c1	1110	'-d', disk,
borlanic	0:380207fcb5c1	1111	'-f', '"%s"'% image_path,
borlanic	0:380207fcb5c1	1112	'-p', port,
borlanic	0:380207fcb5c1	1113	'-t', str(duration),
borlanic	0:380207fcb5c1	1114	'-C', str(program_cycle_s)]
borlanic	0:380207fcb5c1	1115
borlanic	0:380207fcb5c1	1116	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
borlanic	0:380207fcb5c1	1117	cmd += ['--auto']
borlanic	0:380207fcb5c1	1118
borlanic	0:380207fcb5c1	1119	# Add extra parameters to host_test
borlanic	0:380207fcb5c1	1120	if copy_method is not None:
borlanic	0:380207fcb5c1	1121	cmd += ["-c", copy_method]
borlanic	0:380207fcb5c1	1122	if micro is not None:
borlanic	0:380207fcb5c1	1123	cmd += ["-m", micro]
borlanic	0:380207fcb5c1	1124	if reset is not None:
borlanic	0:380207fcb5c1	1125	cmd += ["-r", reset]
borlanic	0:380207fcb5c1	1126	if reset_tout is not None:
borlanic	0:380207fcb5c1	1127	cmd += ["-R", str(reset_tout)]
borlanic	0:380207fcb5c1	1128
borlanic	0:380207fcb5c1	1129	if verbose:
borlanic	0:380207fcb5c1	1130	print(Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET)
borlanic	0:380207fcb5c1	1131	print("Test::Output::Start")
borlanic	0:380207fcb5c1	1132
borlanic	0:380207fcb5c1	1133	proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
borlanic	0:380207fcb5c1	1134	obs = ProcessObserver(proc)
borlanic	0:380207fcb5c1	1135	update_once_flag = {} # Stores flags checking if some auto-parameter was already set
borlanic	0:380207fcb5c1	1136	line = ''
borlanic	0:380207fcb5c1	1137	output = []
borlanic	0:380207fcb5c1	1138	start_time = time()
borlanic	0:380207fcb5c1	1139	while (time() - start_time) < (2 * duration):
borlanic	0:380207fcb5c1	1140	c = get_char_from_queue(obs)
borlanic	0:380207fcb5c1	1141	if c:
borlanic	0:380207fcb5c1	1142	if verbose:
borlanic	0:380207fcb5c1	1143	sys.stdout.write(c)
borlanic	0:380207fcb5c1	1144	c = filter_queue_char(c)
borlanic	0:380207fcb5c1	1145	output.append(c)
borlanic	0:380207fcb5c1	1146	# Give the mbed under test a way to communicate the end of the test
borlanic	0:380207fcb5c1	1147	if c in ['\n', '\r']:
borlanic	0:380207fcb5c1	1148
borlanic	0:380207fcb5c1	1149	# Checking for auto-detection information from the test about MUT reset moment
borlanic	0:380207fcb5c1	1150	if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
borlanic	0:380207fcb5c1	1151	# We will update this marker only once to prevent multiple time resets
borlanic	0:380207fcb5c1	1152	update_once_flag['reset_target'] = True
borlanic	0:380207fcb5c1	1153	start_time = time()
borlanic	0:380207fcb5c1	1154
borlanic	0:380207fcb5c1	1155	# Checking for auto-detection information from the test about timeout
borlanic	0:380207fcb5c1	1156	auto_timeout_val = get_auto_property_value('timeout', line)
borlanic	0:380207fcb5c1	1157	if 'timeout' not in update_once_flag and auto_timeout_val is not None:
borlanic	0:380207fcb5c1	1158	# We will update this marker only once to prevent multiple time resets
borlanic	0:380207fcb5c1	1159	update_once_flag['timeout'] = True
borlanic	0:380207fcb5c1	1160	duration = int(auto_timeout_val)
borlanic	0:380207fcb5c1	1161
borlanic	0:380207fcb5c1	1162	# Detect mbed assert:
borlanic	0:380207fcb5c1	1163	if 'mbed assertation failed: ' in line:
borlanic	0:380207fcb5c1	1164	output.append('{{mbed_assert}}')
borlanic	0:380207fcb5c1	1165	break
borlanic	0:380207fcb5c1	1166
borlanic	0:380207fcb5c1	1167	# Check for test end
borlanic	0:380207fcb5c1	1168	if '{end}' in line:
borlanic	0:380207fcb5c1	1169	break
borlanic	0:380207fcb5c1	1170	line = ''
borlanic	0:380207fcb5c1	1171	else:
borlanic	0:380207fcb5c1	1172	line += c
borlanic	0:380207fcb5c1	1173	end_time = time()
borlanic	0:380207fcb5c1	1174	testcase_duration = end_time - start_time # Test case duration from reset to {end}
borlanic	0:380207fcb5c1	1175
borlanic	0:380207fcb5c1	1176	c = get_char_from_queue(obs)
borlanic	0:380207fcb5c1	1177
borlanic	0:380207fcb5c1	1178	if c:
borlanic	0:380207fcb5c1	1179	if verbose:
borlanic	0:380207fcb5c1	1180	sys.stdout.write(c)
borlanic	0:380207fcb5c1	1181	c = filter_queue_char(c)
borlanic	0:380207fcb5c1	1182	output.append(c)
borlanic	0:380207fcb5c1	1183
borlanic	0:380207fcb5c1	1184	if verbose:
borlanic	0:380207fcb5c1	1185	print("Test::Output::Finish")
borlanic	0:380207fcb5c1	1186	# Stop test process
borlanic	0:380207fcb5c1	1187	obs.stop()
borlanic	0:380207fcb5c1	1188
borlanic	0:380207fcb5c1	1189	result = get_test_result(output)
borlanic	0:380207fcb5c1	1190	return (result, "".join(output), testcase_duration, duration)
borlanic	0:380207fcb5c1	1191
borlanic	0:380207fcb5c1	1192	def is_peripherals_available(self, target_mcu_name, peripherals=None):
borlanic	0:380207fcb5c1	1193	""" Checks if specified target should run specific peripheral test case defined in MUTs file
borlanic	0:380207fcb5c1	1194	"""
borlanic	0:380207fcb5c1	1195	if peripherals is not None:
borlanic	0:380207fcb5c1	1196	peripherals = set(peripherals)
borlanic	0:380207fcb5c1	1197	for id, mut in self.muts.items():
borlanic	0:380207fcb5c1	1198	# Target MCU name check
borlanic	0:380207fcb5c1	1199	if mut["mcu"] != target_mcu_name:
borlanic	0:380207fcb5c1	1200	continue
borlanic	0:380207fcb5c1	1201	# Peripherals check
borlanic	0:380207fcb5c1	1202	if peripherals is not None:
borlanic	0:380207fcb5c1	1203	if 'peripherals' not in mut:
borlanic	0:380207fcb5c1	1204	continue
borlanic	0:380207fcb5c1	1205	if not peripherals.issubset(set(mut['peripherals'])):
borlanic	0:380207fcb5c1	1206	continue
borlanic	0:380207fcb5c1	1207	return True
borlanic	0:380207fcb5c1	1208	return False
borlanic	0:380207fcb5c1	1209
borlanic	0:380207fcb5c1	1210	def shape_test_request(self, mcu, image_path, test_id, duration=10):
borlanic	0:380207fcb5c1	1211	""" Function prepares JSON structure describing test specification
borlanic	0:380207fcb5c1	1212	"""
borlanic	0:380207fcb5c1	1213	test_spec = {
borlanic	0:380207fcb5c1	1214	"mcu": mcu,
borlanic	0:380207fcb5c1	1215	"image": image_path,
borlanic	0:380207fcb5c1	1216	"duration": duration,
borlanic	0:380207fcb5c1	1217	"test_id": test_id,
borlanic	0:380207fcb5c1	1218	}
borlanic	0:380207fcb5c1	1219	return json.dumps(test_spec)
borlanic	0:380207fcb5c1	1220
borlanic	0:380207fcb5c1	1221
borlanic	0:380207fcb5c1	1222	def get_unique_value_from_summary(test_summary, index):
borlanic	0:380207fcb5c1	1223	""" Gets list of unique target names
borlanic	0:380207fcb5c1	1224	"""
borlanic	0:380207fcb5c1	1225	result = []
borlanic	0:380207fcb5c1	1226	for test in test_summary:
borlanic	0:380207fcb5c1	1227	target_name = test[index]
borlanic	0:380207fcb5c1	1228	if target_name not in result:
borlanic	0:380207fcb5c1	1229	result.append(target_name)
borlanic	0:380207fcb5c1	1230	return sorted(result)
borlanic	0:380207fcb5c1	1231
borlanic	0:380207fcb5c1	1232
borlanic	0:380207fcb5c1	1233	def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
borlanic	0:380207fcb5c1	1234	""" Gets list of unique target names and return dictionary
borlanic	0:380207fcb5c1	1235	"""
borlanic	0:380207fcb5c1	1236	result = {}
borlanic	0:380207fcb5c1	1237	for test in test_summary:
borlanic	0:380207fcb5c1	1238	key = test[index_key]
borlanic	0:380207fcb5c1	1239	val = test[index_val]
borlanic	0:380207fcb5c1	1240	if key not in result:
borlanic	0:380207fcb5c1	1241	result[key] = val
borlanic	0:380207fcb5c1	1242	return result
borlanic	0:380207fcb5c1	1243
borlanic	0:380207fcb5c1	1244
borlanic	0:380207fcb5c1	1245	def show_json_file_format_error(json_spec_filename, line, column):
borlanic	0:380207fcb5c1	1246	""" Prints JSON broken content
borlanic	0:380207fcb5c1	1247	"""
borlanic	0:380207fcb5c1	1248	with open(json_spec_filename) as data_file:
borlanic	0:380207fcb5c1	1249	line_no = 1
borlanic	0:380207fcb5c1	1250	for json_line in data_file:
borlanic	0:380207fcb5c1	1251	if line_no + 5 >= line: # Print last few lines before error
borlanic	0:380207fcb5c1	1252	print('Line %d:\t'%line_no + json_line)
borlanic	0:380207fcb5c1	1253	if line_no == line:
borlanic	0:380207fcb5c1	1254	print('%s\t%s^' (' ' * len('Line %d:' % line_no),
borlanic	0:380207fcb5c1	1255	'-' * (column - 1)))
borlanic	0:380207fcb5c1	1256	break
borlanic	0:380207fcb5c1	1257	line_no += 1
borlanic	0:380207fcb5c1	1258
borlanic	0:380207fcb5c1	1259
borlanic	0:380207fcb5c1	1260	def json_format_error_defect_pos(json_error_msg):
borlanic	0:380207fcb5c1	1261	""" Gets first error line and column in JSON file format.
borlanic	0:380207fcb5c1	1262	Parsed from exception thrown by json.loads() string
borlanic	0:380207fcb5c1	1263	"""
borlanic	0:380207fcb5c1	1264	result = None
borlanic	0:380207fcb5c1	1265	line, column = 0, 0
borlanic	0:380207fcb5c1	1266	# Line value search
borlanic	0:380207fcb5c1	1267	line_search = re.search('line [0-9]+', json_error_msg)
borlanic	0:380207fcb5c1	1268	if line_search is not None:
borlanic	0:380207fcb5c1	1269	ls = line_search.group().split(' ')
borlanic	0:380207fcb5c1	1270	if len(ls) == 2:
borlanic	0:380207fcb5c1	1271	line = int(ls[1])
borlanic	0:380207fcb5c1	1272	# Column position search
borlanic	0:380207fcb5c1	1273	column_search = re.search('column [0-9]+', json_error_msg)
borlanic	0:380207fcb5c1	1274	if column_search is not None:
borlanic	0:380207fcb5c1	1275	cs = column_search.group().split(' ')
borlanic	0:380207fcb5c1	1276	if len(cs) == 2:
borlanic	0:380207fcb5c1	1277	column = int(cs[1])
borlanic	0:380207fcb5c1	1278	result = [line, column]
borlanic	0:380207fcb5c1	1279	return result
borlanic	0:380207fcb5c1	1280
borlanic	0:380207fcb5c1	1281
borlanic	0:380207fcb5c1	1282	def get_json_data_from_file(json_spec_filename, verbose=False):
borlanic	0:380207fcb5c1	1283	""" Loads from file JSON formatted string to data structure
borlanic	0:380207fcb5c1	1284	"""
borlanic	0:380207fcb5c1	1285	result = None
borlanic	0:380207fcb5c1	1286	try:
borlanic	0:380207fcb5c1	1287	with open(json_spec_filename) as data_file:
borlanic	0:380207fcb5c1	1288	try:
borlanic	0:380207fcb5c1	1289	result = json.load(data_file)
borlanic	0:380207fcb5c1	1290	except ValueError as json_error_msg:
borlanic	0:380207fcb5c1	1291	result = None
borlanic	0:380207fcb5c1	1292	print('JSON file %s parsing failed. Reason: %s' %
borlanic	0:380207fcb5c1	1293	(json_spec_filename, json_error_msg))
borlanic	0:380207fcb5c1	1294	# We can print where error occurred inside JSON file if we can parse exception msg
borlanic	0:380207fcb5c1	1295	json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
borlanic	0:380207fcb5c1	1296	if json_format_defect_pos is not None:
borlanic	0:380207fcb5c1	1297	line = json_format_defect_pos[0]
borlanic	0:380207fcb5c1	1298	column = json_format_defect_pos[1]
borlanic	0:380207fcb5c1	1299	print()
borlanic	0:380207fcb5c1	1300	show_json_file_format_error(json_spec_filename, line, column)
borlanic	0:380207fcb5c1	1301
borlanic	0:380207fcb5c1	1302	except IOError as fileopen_error_msg:
borlanic	0:380207fcb5c1	1303	print('JSON file %s not opened. Reason: %s\n'%
borlanic	0:380207fcb5c1	1304	(json_spec_filename, fileopen_error_msg))
borlanic	0:380207fcb5c1	1305	if verbose and result:
borlanic	0:380207fcb5c1	1306	pp = pprint.PrettyPrinter(indent=4)
borlanic	0:380207fcb5c1	1307	pp.pprint(result)
borlanic	0:380207fcb5c1	1308	return result
borlanic	0:380207fcb5c1	1309
borlanic	0:380207fcb5c1	1310
borlanic	0:380207fcb5c1	1311	def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
borlanic	0:380207fcb5c1	1312	""" Prints MUTs configuration passed to test script for verboseness
borlanic	0:380207fcb5c1	1313	"""
borlanic	0:380207fcb5c1	1314	muts_info_cols = []
borlanic	0:380207fcb5c1	1315	# We need to check all unique properties for each defined MUT
borlanic	0:380207fcb5c1	1316	for k in json_data:
borlanic	0:380207fcb5c1	1317	mut_info = json_data[k]
borlanic	0:380207fcb5c1	1318	for mut_property in mut_info:
borlanic	0:380207fcb5c1	1319	if mut_property not in muts_info_cols:
borlanic	0:380207fcb5c1	1320	muts_info_cols.append(mut_property)
borlanic	0:380207fcb5c1	1321
borlanic	0:380207fcb5c1	1322	# Prepare pretty table object to display all MUTs
borlanic	0:380207fcb5c1	1323	pt_cols = ["index"] + muts_info_cols
borlanic	0:380207fcb5c1	1324	pt = PrettyTable(pt_cols)
borlanic	0:380207fcb5c1	1325	for col in pt_cols:
borlanic	0:380207fcb5c1	1326	pt.align[col] = "l"
borlanic	0:380207fcb5c1	1327
borlanic	0:380207fcb5c1	1328	# Add rows to pretty print object
borlanic	0:380207fcb5c1	1329	for k in json_data:
borlanic	0:380207fcb5c1	1330	row = [k]
borlanic	0:380207fcb5c1	1331	mut_info = json_data[k]
borlanic	0:380207fcb5c1	1332
borlanic	0:380207fcb5c1	1333	add_row = True
borlanic	0:380207fcb5c1	1334	if platform_filter and 'mcu' in mut_info:
borlanic	0:380207fcb5c1	1335	add_row = re.search(platform_filter, mut_info['mcu']) is not None
borlanic	0:380207fcb5c1	1336	if add_row:
borlanic	0:380207fcb5c1	1337	for col in muts_info_cols:
borlanic	0:380207fcb5c1	1338	cell_val = mut_info[col] if col in mut_info else None
borlanic	0:380207fcb5c1	1339	if isinstance(cell_val, list):
borlanic	0:380207fcb5c1	1340	cell_val = join_delim.join(cell_val)
borlanic	0:380207fcb5c1	1341	row.append(cell_val)
borlanic	0:380207fcb5c1	1342	pt.add_row(row)
borlanic	0:380207fcb5c1	1343	return pt.get_string()
borlanic	0:380207fcb5c1	1344
borlanic	0:380207fcb5c1	1345
borlanic	0:380207fcb5c1	1346	def print_test_configuration_from_json(json_data, join_delim=", "):
borlanic	0:380207fcb5c1	1347	""" Prints test specification configuration passed to test script for verboseness
borlanic	0:380207fcb5c1	1348	"""
borlanic	0:380207fcb5c1	1349	toolchains_info_cols = []
borlanic	0:380207fcb5c1	1350	# We need to check all toolchains for each device
borlanic	0:380207fcb5c1	1351	for k in json_data:
borlanic	0:380207fcb5c1	1352	# k should be 'targets'
borlanic	0:380207fcb5c1	1353	targets = json_data[k]
borlanic	0:380207fcb5c1	1354	for target in targets:
borlanic	0:380207fcb5c1	1355	toolchains = targets[target]
borlanic	0:380207fcb5c1	1356	for toolchain in toolchains:
borlanic	0:380207fcb5c1	1357	if toolchain not in toolchains_info_cols:
borlanic	0:380207fcb5c1	1358	toolchains_info_cols.append(toolchain)
borlanic	0:380207fcb5c1	1359
borlanic	0:380207fcb5c1	1360	# Prepare pretty table object to display test specification
borlanic	0:380207fcb5c1	1361	pt_cols = ["mcu"] + sorted(toolchains_info_cols)
borlanic	0:380207fcb5c1	1362	pt = PrettyTable(pt_cols)
borlanic	0:380207fcb5c1	1363	for col in pt_cols:
borlanic	0:380207fcb5c1	1364	pt.align[col] = "l"
borlanic	0:380207fcb5c1	1365
borlanic	0:380207fcb5c1	1366	# { target : [conflicted toolchains] }
borlanic	0:380207fcb5c1	1367	toolchain_conflicts = {}
borlanic	0:380207fcb5c1	1368	toolchain_path_conflicts = []
borlanic	0:380207fcb5c1	1369	for k in json_data:
borlanic	0:380207fcb5c1	1370	# k should be 'targets'
borlanic	0:380207fcb5c1	1371	targets = json_data[k]
borlanic	0:380207fcb5c1	1372	for target in targets:
borlanic	0:380207fcb5c1	1373	target_supported_toolchains = get_target_supported_toolchains(target)
borlanic	0:380207fcb5c1	1374	if not target_supported_toolchains:
borlanic	0:380207fcb5c1	1375	target_supported_toolchains = []
borlanic	0:380207fcb5c1	1376	target_name = target if target in TARGET_MAP else "%s*"% target
borlanic	0:380207fcb5c1	1377	row = [target_name]
borlanic	0:380207fcb5c1	1378	toolchains = targets[target]
borlanic	0:380207fcb5c1	1379
borlanic	0:380207fcb5c1	1380	for toolchain in sorted(toolchains_info_cols):
borlanic	0:380207fcb5c1	1381	# Check for conflicts: target vs toolchain
borlanic	0:380207fcb5c1	1382	conflict = False
borlanic	0:380207fcb5c1	1383	conflict_path = False
borlanic	0:380207fcb5c1	1384	if toolchain in toolchains:
borlanic	0:380207fcb5c1	1385	if toolchain not in target_supported_toolchains:
borlanic	0:380207fcb5c1	1386	conflict = True
borlanic	0:380207fcb5c1	1387	if target not in toolchain_conflicts:
borlanic	0:380207fcb5c1	1388	toolchain_conflicts[target] = []
borlanic	0:380207fcb5c1	1389	toolchain_conflicts[target].append(toolchain)
borlanic	0:380207fcb5c1	1390	# Add marker inside table about target usage / conflict
borlanic	0:380207fcb5c1	1391	cell_val = 'Yes' if toolchain in toolchains else '-'
borlanic	0:380207fcb5c1	1392	if conflict:
borlanic	0:380207fcb5c1	1393	cell_val += '*'
borlanic	0:380207fcb5c1	1394	# Check for conflicts: toolchain vs toolchain path
borlanic	0:380207fcb5c1	1395	if toolchain in TOOLCHAIN_PATHS:
borlanic	0:380207fcb5c1	1396	toolchain_path = TOOLCHAIN_PATHS[toolchain]
borlanic	0:380207fcb5c1	1397	if not os.path.isdir(toolchain_path):
borlanic	0:380207fcb5c1	1398	conflict_path = True
borlanic	0:380207fcb5c1	1399	if toolchain not in toolchain_path_conflicts:
borlanic	0:380207fcb5c1	1400	toolchain_path_conflicts.append(toolchain)
borlanic	0:380207fcb5c1	1401	if conflict_path:
borlanic	0:380207fcb5c1	1402	cell_val += '#'
borlanic	0:380207fcb5c1	1403	row.append(cell_val)
borlanic	0:380207fcb5c1	1404	pt.add_row(row)
borlanic	0:380207fcb5c1	1405
borlanic	0:380207fcb5c1	1406	# generate result string
borlanic	0:380207fcb5c1	1407	result = pt.get_string() # Test specification table
borlanic	0:380207fcb5c1	1408	if toolchain_conflicts or toolchain_path_conflicts:
borlanic	0:380207fcb5c1	1409	result += "\n"
borlanic	0:380207fcb5c1	1410	result += "Toolchain conflicts:\n"
borlanic	0:380207fcb5c1	1411	for target in toolchain_conflicts:
borlanic	0:380207fcb5c1	1412	if target not in TARGET_MAP:
borlanic	0:380207fcb5c1	1413	result += "\t* Target %s unknown\n"% (target)
borlanic	0:380207fcb5c1	1414	conflict_target_list = join_delim.join(toolchain_conflicts[target])
borlanic	0:380207fcb5c1	1415	sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
borlanic	0:380207fcb5c1	1416	result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
borlanic	0:380207fcb5c1	1417
borlanic	0:380207fcb5c1	1418	for toolchain in toolchain_path_conflicts:
borlanic	0:380207fcb5c1	1419	# Let's check toolchain configuration
borlanic	0:380207fcb5c1	1420	if toolchain in TOOLCHAIN_PATHS:
borlanic	0:380207fcb5c1	1421	toolchain_path = TOOLCHAIN_PATHS[toolchain]
borlanic	0:380207fcb5c1	1422	if not os.path.isdir(toolchain_path):
borlanic	0:380207fcb5c1	1423	result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
borlanic	0:380207fcb5c1	1424	return result
borlanic	0:380207fcb5c1	1425
borlanic	0:380207fcb5c1	1426
borlanic	0:380207fcb5c1	1427	def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
borlanic	0:380207fcb5c1	1428	""" Generates table summary with all test cases and additional test cases
borlanic	0:380207fcb5c1	1429	information using pretty print functionality. Allows test suite user to
borlanic	0:380207fcb5c1	1430	see test cases
borlanic	0:380207fcb5c1	1431	"""
borlanic	0:380207fcb5c1	1432	# get all unique test ID prefixes
borlanic	0:380207fcb5c1	1433	unique_test_id = []
borlanic	0:380207fcb5c1	1434	for test in TESTS:
borlanic	0:380207fcb5c1	1435	split = test['id'].split('_')[:-1]
borlanic	0:380207fcb5c1	1436	test_id_prefix = '_'.join(split)
borlanic	0:380207fcb5c1	1437	if test_id_prefix not in unique_test_id:
borlanic	0:380207fcb5c1	1438	unique_test_id.append(test_id_prefix)
borlanic	0:380207fcb5c1	1439	unique_test_id.sort()
borlanic	0:380207fcb5c1	1440	counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
borlanic	0:380207fcb5c1	1441	counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
borlanic	0:380207fcb5c1	1442
borlanic	0:380207fcb5c1	1443	test_properties = ['id',
borlanic	0:380207fcb5c1	1444	'automated',
borlanic	0:380207fcb5c1	1445	'description',
borlanic	0:380207fcb5c1	1446	'peripherals',
borlanic	0:380207fcb5c1	1447	'host_test',
borlanic	0:380207fcb5c1	1448	'duration'] if cols is None else cols
borlanic	0:380207fcb5c1	1449
borlanic	0:380207fcb5c1	1450	# All tests status table print
borlanic	0:380207fcb5c1	1451	pt = PrettyTable(test_properties)
borlanic	0:380207fcb5c1	1452	for col in test_properties:
borlanic	0:380207fcb5c1	1453	pt.align[col] = "l"
borlanic	0:380207fcb5c1	1454	pt.align['duration'] = "r"
borlanic	0:380207fcb5c1	1455
borlanic	0:380207fcb5c1	1456	counter_all = 0
borlanic	0:380207fcb5c1	1457	counter_automated = 0
borlanic	0:380207fcb5c1	1458	pt.padding_width = 1 # One space between column edges and contents (default)
borlanic	0:380207fcb5c1	1459
borlanic	0:380207fcb5c1	1460	for test_id in sorted(TEST_MAP.keys()):
borlanic	0:380207fcb5c1	1461	if platform_filter is not None:
borlanic	0:380207fcb5c1	1462	# FIlter out platforms using regex
borlanic	0:380207fcb5c1	1463	if re.search(platform_filter, test_id) is None:
borlanic	0:380207fcb5c1	1464	continue
borlanic	0:380207fcb5c1	1465	row = []
borlanic	0:380207fcb5c1	1466	test = TEST_MAP[test_id]
borlanic	0:380207fcb5c1	1467	split = test_id.split('_')[:-1]
borlanic	0:380207fcb5c1	1468	test_id_prefix = '_'.join(split)
borlanic	0:380207fcb5c1	1469
borlanic	0:380207fcb5c1	1470	for col in test_properties:
borlanic	0:380207fcb5c1	1471	col_value = test[col]
borlanic	0:380207fcb5c1	1472	if isinstance(test[col], list):
borlanic	0:380207fcb5c1	1473	col_value = join_delim.join(test[col])
borlanic	0:380207fcb5c1	1474	elif test[col] == None:
borlanic	0:380207fcb5c1	1475	col_value = "-"
borlanic	0:380207fcb5c1	1476
borlanic	0:380207fcb5c1	1477	row.append(col_value)
borlanic	0:380207fcb5c1	1478	if test['automated'] == True:
borlanic	0:380207fcb5c1	1479	counter_dict_test_id_types[test_id_prefix] += 1
borlanic	0:380207fcb5c1	1480	counter_automated += 1
borlanic	0:380207fcb5c1	1481	pt.add_row(row)
borlanic	0:380207fcb5c1	1482	# Update counters
borlanic	0:380207fcb5c1	1483	counter_all += 1
borlanic	0:380207fcb5c1	1484	counter_dict_test_id_types_all[test_id_prefix] += 1
borlanic	0:380207fcb5c1	1485	result = pt.get_string()
borlanic	0:380207fcb5c1	1486	result += "\n\n"
borlanic	0:380207fcb5c1	1487
borlanic	0:380207fcb5c1	1488	if result_summary and not platform_filter:
borlanic	0:380207fcb5c1	1489	# Automation result summary
borlanic	0:380207fcb5c1	1490	test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
borlanic	0:380207fcb5c1	1491	pt = PrettyTable(test_id_cols)
borlanic	0:380207fcb5c1	1492	pt.align['automated'] = "r"
borlanic	0:380207fcb5c1	1493	pt.align['all'] = "r"
borlanic	0:380207fcb5c1	1494	pt.align['percent [%]'] = "r"
borlanic	0:380207fcb5c1	1495
borlanic	0:380207fcb5c1	1496	percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
borlanic	0:380207fcb5c1	1497	str_progress = progress_bar(percent_progress, 75)
borlanic	0:380207fcb5c1	1498	pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
borlanic	0:380207fcb5c1	1499	result += "Automation coverage:\n"
borlanic	0:380207fcb5c1	1500	result += pt.get_string()
borlanic	0:380207fcb5c1	1501	result += "\n\n"
borlanic	0:380207fcb5c1	1502
borlanic	0:380207fcb5c1	1503	# Test automation coverage table print
borlanic	0:380207fcb5c1	1504	test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
borlanic	0:380207fcb5c1	1505	pt = PrettyTable(test_id_cols)
borlanic	0:380207fcb5c1	1506	pt.align['id'] = "l"
borlanic	0:380207fcb5c1	1507	pt.align['automated'] = "r"
borlanic	0:380207fcb5c1	1508	pt.align['all'] = "r"
borlanic	0:380207fcb5c1	1509	pt.align['percent [%]'] = "r"
borlanic	0:380207fcb5c1	1510	for unique_id in unique_test_id:
borlanic	0:380207fcb5c1	1511	# print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
borlanic	0:380207fcb5c1	1512	percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
borlanic	0:380207fcb5c1	1513	str_progress = progress_bar(percent_progress, 75)
borlanic	0:380207fcb5c1	1514	row = [unique_id,
borlanic	0:380207fcb5c1	1515	counter_dict_test_id_types[unique_id],
borlanic	0:380207fcb5c1	1516	counter_dict_test_id_types_all[unique_id],
borlanic	0:380207fcb5c1	1517	percent_progress,
borlanic	0:380207fcb5c1	1518	"[" + str_progress + "]"]
borlanic	0:380207fcb5c1	1519	pt.add_row(row)
borlanic	0:380207fcb5c1	1520	result += "Test automation coverage:\n"
borlanic	0:380207fcb5c1	1521	result += pt.get_string()
borlanic	0:380207fcb5c1	1522	result += "\n\n"
borlanic	0:380207fcb5c1	1523	return result
borlanic	0:380207fcb5c1	1524
borlanic	0:380207fcb5c1	1525
borlanic	0:380207fcb5c1	1526	def progress_bar(percent_progress, saturation=0):
borlanic	0:380207fcb5c1	1527	""" This function creates progress bar with optional simple saturation mark
borlanic	0:380207fcb5c1	1528	"""
borlanic	0:380207fcb5c1	1529	step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
borlanic	0:380207fcb5c1	1530	str_progress = '#' * step + '.' * int(50 - step)
borlanic	0:380207fcb5c1	1531	c = '!' if str_progress[38] == '.' else '\|'
borlanic	0:380207fcb5c1	1532	if saturation > 0:
borlanic	0:380207fcb5c1	1533	saturation = saturation / 2
borlanic	0:380207fcb5c1	1534	str_progress = str_progress[:saturation] + c + str_progress[saturation:]
borlanic	0:380207fcb5c1	1535	return str_progress
borlanic	0:380207fcb5c1	1536
borlanic	0:380207fcb5c1	1537
borlanic	0:380207fcb5c1	1538	def singletest_in_cli_mode(single_test):
borlanic	0:380207fcb5c1	1539	""" Runs SingleTestRunner object in CLI (Command line interface) mode
borlanic	0:380207fcb5c1	1540
borlanic	0:380207fcb5c1	1541	@return returns success code (0 == success) for building and running tests
borlanic	0:380207fcb5c1	1542	"""
borlanic	0:380207fcb5c1	1543	start = time()
borlanic	0:380207fcb5c1	1544	# Execute tests depending on options and filter applied
borlanic	0:380207fcb5c1	1545	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
borlanic	0:380207fcb5c1	1546	elapsed_time = time() - start
borlanic	0:380207fcb5c1	1547
borlanic	0:380207fcb5c1	1548	# Human readable summary
borlanic	0:380207fcb5c1	1549	if not single_test.opts_suppress_summary:
borlanic	0:380207fcb5c1	1550	# prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	1551	print(single_test.generate_test_summary(test_summary, shuffle_seed))
borlanic	0:380207fcb5c1	1552	if single_test.opts_test_x_toolchain_summary:
borlanic	0:380207fcb5c1	1553	# prints well-formed summary with results (SQL table like)
borlanic	0:380207fcb5c1	1554	# table shows text x toolchain test result matrix
borlanic	0:380207fcb5c1	1555	print(single_test.generate_test_summary_by_target(test_summary,
borlanic	0:380207fcb5c1	1556	shuffle_seed))
borlanic	0:380207fcb5c1	1557
borlanic	0:380207fcb5c1	1558	print("Completed in %.2f sec" % elapsed_time)
borlanic	0:380207fcb5c1	1559	print
borlanic	0:380207fcb5c1	1560	# Write summary of the builds
borlanic	0:380207fcb5c1	1561
borlanic	0:380207fcb5c1	1562	print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
borlanic	0:380207fcb5c1	1563	status = print_report_exporter.report(build_report)
borlanic	0:380207fcb5c1	1564
borlanic	0:380207fcb5c1	1565	# Store extra reports in files
borlanic	0:380207fcb5c1	1566	if single_test.opts_report_html_file_name:
borlanic	0:380207fcb5c1	1567	# Export results in form of HTML report to separate file
borlanic	0:380207fcb5c1	1568	report_exporter = ReportExporter(ResultExporterType.HTML)
borlanic	0:380207fcb5c1	1569	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
borlanic	0:380207fcb5c1	1570	if single_test.opts_report_junit_file_name:
borlanic	0:380207fcb5c1	1571	# Export results in form of JUnit XML report to separate file
borlanic	0:380207fcb5c1	1572	report_exporter = ReportExporter(ResultExporterType.JUNIT)
borlanic	0:380207fcb5c1	1573	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
borlanic	0:380207fcb5c1	1574	if single_test.opts_report_text_file_name:
borlanic	0:380207fcb5c1	1575	# Export results in form of a text file
borlanic	0:380207fcb5c1	1576	report_exporter = ReportExporter(ResultExporterType.TEXT)
borlanic	0:380207fcb5c1	1577	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
borlanic	0:380207fcb5c1	1578	if single_test.opts_report_build_file_name:
borlanic	0:380207fcb5c1	1579	# Export build results as html report to sparate file
borlanic	0:380207fcb5c1	1580	report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
borlanic	0:380207fcb5c1	1581	report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
borlanic	0:380207fcb5c1	1582
borlanic	0:380207fcb5c1	1583	# Returns True if no build failures of the test projects or their dependencies
borlanic	0:380207fcb5c1	1584	return status
borlanic	0:380207fcb5c1	1585
borlanic	0:380207fcb5c1	1586	class TestLogger():
borlanic	0:380207fcb5c1	1587	""" Super-class for logging and printing ongoing events for test suite pass
borlanic	0:380207fcb5c1	1588	"""
borlanic	0:380207fcb5c1	1589	def __init__(self, store_log=True):
borlanic	0:380207fcb5c1	1590	""" We can control if logger actually stores log in memory
borlanic	0:380207fcb5c1	1591	or just handled all log entries immediately
borlanic	0:380207fcb5c1	1592	"""
borlanic	0:380207fcb5c1	1593	self.log = []
borlanic	0:380207fcb5c1	1594	self.log_to_file = False
borlanic	0:380207fcb5c1	1595	self.log_file_name = None
borlanic	0:380207fcb5c1	1596	self.store_log = store_log
borlanic	0:380207fcb5c1	1597
borlanic	0:380207fcb5c1	1598	self.LogType = construct_enum(INFO='Info',
borlanic	0:380207fcb5c1	1599	WARN='Warning',
borlanic	0:380207fcb5c1	1600	NOTIF='Notification',
borlanic	0:380207fcb5c1	1601	ERROR='Error',
borlanic	0:380207fcb5c1	1602	EXCEPT='Exception')
borlanic	0:380207fcb5c1	1603
borlanic	0:380207fcb5c1	1604	self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
borlanic	0:380207fcb5c1	1605	APPEND=2) # Append to existing log file
borlanic	0:380207fcb5c1	1606
borlanic	0:380207fcb5c1	1607	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
borlanic	0:380207fcb5c1	1608	""" Log one line of text
borlanic	0:380207fcb5c1	1609	"""
borlanic	0:380207fcb5c1	1610	log_timestamp = time()
borlanic	0:380207fcb5c1	1611	log_entry = {'log_type' : LogType,
borlanic	0:380207fcb5c1	1612	'log_timestamp' : log_timestamp,
borlanic	0:380207fcb5c1	1613	'log_line' : log_line,
borlanic	0:380207fcb5c1	1614	'_future' : None
borlanic	0:380207fcb5c1	1615	}
borlanic	0:380207fcb5c1	1616	# Store log in memory
borlanic	0:380207fcb5c1	1617	if self.store_log:
borlanic	0:380207fcb5c1	1618	self.log.append(log_entry)
borlanic	0:380207fcb5c1	1619	return log_entry
borlanic	0:380207fcb5c1	1620
borlanic	0:380207fcb5c1	1621
borlanic	0:380207fcb5c1	1622	class CLITestLogger(TestLogger):
borlanic	0:380207fcb5c1	1623	""" Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
borlanic	0:380207fcb5c1	1624	"""
borlanic	0:380207fcb5c1	1625	def __init__(self, store_log=True, file_name=None):
borlanic	0:380207fcb5c1	1626	TestLogger.__init__(self)
borlanic	0:380207fcb5c1	1627	self.log_file_name = file_name
borlanic	0:380207fcb5c1	1628	#self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
borlanic	0:380207fcb5c1	1629	self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
borlanic	0:380207fcb5c1	1630
borlanic	0:380207fcb5c1	1631	def log_print(self, log_entry, timestamp=True):
borlanic	0:380207fcb5c1	1632	""" Prints on screen formatted log entry
borlanic	0:380207fcb5c1	1633	"""
borlanic	0:380207fcb5c1	1634	ts = log_entry['log_timestamp']
borlanic	0:380207fcb5c1	1635	timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
borlanic	0:380207fcb5c1	1636	log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
borlanic	0:380207fcb5c1	1637	return timestamp_str + log_line_str
borlanic	0:380207fcb5c1	1638
borlanic	0:380207fcb5c1	1639	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
borlanic	0:380207fcb5c1	1640	""" Logs line, if log file output was specified log line will be appended
borlanic	0:380207fcb5c1	1641	at the end of log file
borlanic	0:380207fcb5c1	1642	"""
borlanic	0:380207fcb5c1	1643	log_entry = TestLogger.log_line(self, LogType, log_line)
borlanic	0:380207fcb5c1	1644	log_line_str = self.log_print(log_entry, timestamp)
borlanic	0:380207fcb5c1	1645	if self.log_file_name is not None:
borlanic	0:380207fcb5c1	1646	try:
borlanic	0:380207fcb5c1	1647	with open(self.log_file_name, 'a') as f:
borlanic	0:380207fcb5c1	1648	f.write(log_line_str + line_delim)
borlanic	0:380207fcb5c1	1649	except IOError:
borlanic	0:380207fcb5c1	1650	pass
borlanic	0:380207fcb5c1	1651	return log_line_str
borlanic	0:380207fcb5c1	1652
borlanic	0:380207fcb5c1	1653
borlanic	0:380207fcb5c1	1654	def factory_db_logger(db_url):
borlanic	0:380207fcb5c1	1655	""" Factory database driver depending on database type supplied in database connection string db_url
borlanic	0:380207fcb5c1	1656	"""
borlanic	0:380207fcb5c1	1657	if db_url is not None:
borlanic	0:380207fcb5c1	1658	from tools.test_mysql import MySQLDBAccess
borlanic	0:380207fcb5c1	1659	connection_info = BaseDBAccess().parse_db_connection_string(db_url)
borlanic	0:380207fcb5c1	1660	if connection_info is not None:
borlanic	0:380207fcb5c1	1661	(db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
borlanic	0:380207fcb5c1	1662	if db_type == 'mysql':
borlanic	0:380207fcb5c1	1663	return MySQLDBAccess()
borlanic	0:380207fcb5c1	1664	return None
borlanic	0:380207fcb5c1	1665
borlanic	0:380207fcb5c1	1666
borlanic	0:380207fcb5c1	1667	def detect_database_verbose(db_url):
borlanic	0:380207fcb5c1	1668	""" uses verbose mode (prints) database detection sequence to check it database connection string is valid
borlanic	0:380207fcb5c1	1669	"""
borlanic	0:380207fcb5c1	1670	result = BaseDBAccess().parse_db_connection_string(db_url)
borlanic	0:380207fcb5c1	1671	if result is not None:
borlanic	0:380207fcb5c1	1672	# Parsing passed
borlanic	0:380207fcb5c1	1673	(db_type, username, password, host, db_name) = result
borlanic	0:380207fcb5c1	1674	#print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
borlanic	0:380207fcb5c1	1675	# Let's try to connect
borlanic	0:380207fcb5c1	1676	db_ = factory_db_logger(db_url)
borlanic	0:380207fcb5c1	1677	if db_ is not None:
borlanic	0:380207fcb5c1	1678	print("Connecting to database '%s'..." % db_url)
borlanic	0:380207fcb5c1	1679	db_.connect(host, username, password, db_name)
borlanic	0:380207fcb5c1	1680	if db_.is_connected():
borlanic	0:380207fcb5c1	1681	print("ok")
borlanic	0:380207fcb5c1	1682	print("Detecting database...")
borlanic	0:380207fcb5c1	1683	print(db_.detect_database(verbose=True))
borlanic	0:380207fcb5c1	1684	print("Disconnecting...")
borlanic	0:380207fcb5c1	1685	db_.disconnect()
borlanic	0:380207fcb5c1	1686	print("done")
borlanic	0:380207fcb5c1	1687	else:
borlanic	0:380207fcb5c1	1688	print("Database type '%s' unknown" % db_type)
borlanic	0:380207fcb5c1	1689	else:
borlanic	0:380207fcb5c1	1690	print("Parse error: '%s' - DB Url error" % db_url)
borlanic	0:380207fcb5c1	1691
borlanic	0:380207fcb5c1	1692
borlanic	0:380207fcb5c1	1693	def get_module_avail(module_name):
borlanic	0:380207fcb5c1	1694	""" This function returns True if module_name is already imported module
borlanic	0:380207fcb5c1	1695	"""
borlanic	0:380207fcb5c1	1696	return module_name in sys.modules.keys()
borlanic	0:380207fcb5c1	1697
borlanic	0:380207fcb5c1	1698	def get_autodetected_MUTS_list(platform_name_filter=None):
borlanic	0:380207fcb5c1	1699	oldError = None
borlanic	0:380207fcb5c1	1700	if os.name == 'nt':
borlanic	0:380207fcb5c1	1701	# Disable Windows error box temporarily
borlanic	0:380207fcb5c1	1702	oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
borlanic	0:380207fcb5c1	1703
borlanic	0:380207fcb5c1	1704	mbeds = mbed_lstools.create()
borlanic	0:380207fcb5c1	1705	detect_muts_list = mbeds.list_mbeds()
borlanic	0:380207fcb5c1	1706
borlanic	0:380207fcb5c1	1707	if os.name == 'nt':
borlanic	0:380207fcb5c1	1708	ctypes.windll.kernel32.SetErrorMode(oldError)
borlanic	0:380207fcb5c1	1709
borlanic	0:380207fcb5c1	1710	return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
borlanic	0:380207fcb5c1	1711
borlanic	0:380207fcb5c1	1712	def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
borlanic	0:380207fcb5c1	1713	""" Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
borlanic	0:380207fcb5c1	1714	If function fails to auto-detect devices it will return empty dictionary.
borlanic	0:380207fcb5c1	1715
borlanic	0:380207fcb5c1	1716	if get_module_avail('mbed_lstools'):
borlanic	0:380207fcb5c1	1717	mbeds = mbed_lstools.create()
borlanic	0:380207fcb5c1	1718	mbeds_list = mbeds.list_mbeds()
borlanic	0:380207fcb5c1	1719
borlanic	0:380207fcb5c1	1720	@param mbeds_list list of mbeds captured from mbed_lstools
borlanic	0:380207fcb5c1	1721	@param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
borlanic	0:380207fcb5c1	1722	"""
borlanic	0:380207fcb5c1	1723	result = {} # Should be in muts_all.json format
borlanic	0:380207fcb5c1	1724	# Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
borlanic	0:380207fcb5c1	1725	# mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
borlanic	0:380207fcb5c1	1726	index = 1
borlanic	0:380207fcb5c1	1727	for mut in mbeds_list:
borlanic	0:380207fcb5c1	1728	# Filter the MUTS if a filter is specified
borlanic	0:380207fcb5c1	1729
borlanic	0:380207fcb5c1	1730	if platform_name_filter and not mut['platform_name'] in platform_name_filter:
borlanic	0:380207fcb5c1	1731	continue
borlanic	0:380207fcb5c1	1732
borlanic	0:380207fcb5c1	1733	# For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
borlanic	0:380207fcb5c1	1734	# if not we are creating our own unique value (last few chars from platform's target_id).
borlanic	0:380207fcb5c1	1735	m = {'mcu': mut['platform_name'],
borlanic	0:380207fcb5c1	1736	'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
borlanic	0:380207fcb5c1	1737	'port': mut['serial_port'],
borlanic	0:380207fcb5c1	1738	'disk': mut['mount_point'],
borlanic	0:380207fcb5c1	1739	'peripherals': [] # No peripheral detection
borlanic	0:380207fcb5c1	1740	}
borlanic	0:380207fcb5c1	1741	if index not in result:
borlanic	0:380207fcb5c1	1742	result[index] = {}
borlanic	0:380207fcb5c1	1743	result[index] = m
borlanic	0:380207fcb5c1	1744	index += 1
borlanic	0:380207fcb5c1	1745	return result
borlanic	0:380207fcb5c1	1746
borlanic	0:380207fcb5c1	1747
borlanic	0:380207fcb5c1	1748	def get_autodetected_TEST_SPEC(mbeds_list,
borlanic	0:380207fcb5c1	1749	use_default_toolchain=True,
borlanic	0:380207fcb5c1	1750	use_supported_toolchains=False,
borlanic	0:380207fcb5c1	1751	toolchain_filter=None,
borlanic	0:380207fcb5c1	1752	platform_name_filter=None):
borlanic	0:380207fcb5c1	1753	""" Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
borlanic	0:380207fcb5c1	1754	If function fails to auto-detect devices it will return empty 'targets' test_spec description.
borlanic	0:380207fcb5c1	1755
borlanic	0:380207fcb5c1	1756	use_default_toolchain - if True add default toolchain to test_spec
borlanic	0:380207fcb5c1	1757	use_supported_toolchains - if True add all supported toolchains to test_spec
borlanic	0:380207fcb5c1	1758	toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
borlanic	0:380207fcb5c1	1759	"""
borlanic	0:380207fcb5c1	1760	result = {'targets': {} }
borlanic	0:380207fcb5c1	1761
borlanic	0:380207fcb5c1	1762	for mut in mbeds_list:
borlanic	0:380207fcb5c1	1763	mcu = mut['mcu']
borlanic	0:380207fcb5c1	1764	if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
borlanic	0:380207fcb5c1	1765	if mcu in TARGET_MAP:
borlanic	0:380207fcb5c1	1766	default_toolchain = TARGET_MAP[mcu].default_toolchain
borlanic	0:380207fcb5c1	1767	supported_toolchains = TARGET_MAP[mcu].supported_toolchains
borlanic	0:380207fcb5c1	1768
borlanic	0:380207fcb5c1	1769	# Decide which toolchains should be added to test specification toolchain pool for each target
borlanic	0:380207fcb5c1	1770	toolchains = []
borlanic	0:380207fcb5c1	1771	if use_default_toolchain:
borlanic	0:380207fcb5c1	1772	toolchains.append(default_toolchain)
borlanic	0:380207fcb5c1	1773	if use_supported_toolchains:
borlanic	0:380207fcb5c1	1774	toolchains += supported_toolchains
borlanic	0:380207fcb5c1	1775	if toolchain_filter is not None:
borlanic	0:380207fcb5c1	1776	all_toolchains = supported_toolchains + [default_toolchain]
borlanic	0:380207fcb5c1	1777	for toolchain in toolchain_filter:
borlanic	0:380207fcb5c1	1778	if toolchain in all_toolchains:
borlanic	0:380207fcb5c1	1779	toolchains.append(toolchain)
borlanic	0:380207fcb5c1	1780
borlanic	0:380207fcb5c1	1781	result['targets'][mcu] = list(set(toolchains))
borlanic	0:380207fcb5c1	1782	return result
borlanic	0:380207fcb5c1	1783
borlanic	0:380207fcb5c1	1784
borlanic	0:380207fcb5c1	1785	def get_default_test_options_parser():
borlanic	0:380207fcb5c1	1786	""" Get common test script options used by CLI, web services etc.
borlanic	0:380207fcb5c1	1787	"""
borlanic	0:380207fcb5c1	1788	parser = argparse.ArgumentParser()
borlanic	0:380207fcb5c1	1789	parser.add_argument('-i', '--tests',
borlanic	0:380207fcb5c1	1790	dest='test_spec_filename',
borlanic	0:380207fcb5c1	1791	metavar="FILE",
borlanic	0:380207fcb5c1	1792	type=argparse_filestring_type,
borlanic	0:380207fcb5c1	1793	help='Points to file with test specification')
borlanic	0:380207fcb5c1	1794
borlanic	0:380207fcb5c1	1795	parser.add_argument('-M', '--MUTS',
borlanic	0:380207fcb5c1	1796	dest='muts_spec_filename',
borlanic	0:380207fcb5c1	1797	metavar="FILE",
borlanic	0:380207fcb5c1	1798	type=argparse_filestring_type,
borlanic	0:380207fcb5c1	1799	help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
borlanic	0:380207fcb5c1	1800
borlanic	0:380207fcb5c1	1801	parser.add_argument("-j", "--jobs",
borlanic	0:380207fcb5c1	1802	dest='jobs',
borlanic	0:380207fcb5c1	1803	metavar="NUMBER",
borlanic	0:380207fcb5c1	1804	type=int,
borlanic	0:380207fcb5c1	1805	help="Define number of compilation jobs. Default value is 1")
borlanic	0:380207fcb5c1	1806
borlanic	0:380207fcb5c1	1807	if get_module_avail('mbed_lstools'):
borlanic	0:380207fcb5c1	1808	# Additional features available when mbed_lstools is installed on host and imported
borlanic	0:380207fcb5c1	1809	# mbed_lstools allow users to detect connected to host mbed-enabled devices
borlanic	0:380207fcb5c1	1810	parser.add_argument('--auto',
borlanic	0:380207fcb5c1	1811	dest='auto_detect',
borlanic	0:380207fcb5c1	1812	action="store_true",
borlanic	0:380207fcb5c1	1813	help='Use mbed-ls module to detect all connected mbed devices')
borlanic	0:380207fcb5c1	1814
borlanic	0:380207fcb5c1	1815	toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
borlanic	0:380207fcb5c1	1816	parser.add_argument('--tc',
borlanic	0:380207fcb5c1	1817	dest='toolchains_filter',
borlanic	0:380207fcb5c1	1818	type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
borlanic	0:380207fcb5c1	1819	help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
borlanic	0:380207fcb5c1	1820
borlanic	0:380207fcb5c1	1821	test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
borlanic	0:380207fcb5c1	1822	parser.add_argument('--oper',
borlanic	0:380207fcb5c1	1823	dest='operability_checks',
borlanic	0:380207fcb5c1	1824	type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
borlanic	0:380207fcb5c1	1825	help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
borlanic	0:380207fcb5c1	1826
borlanic	0:380207fcb5c1	1827	parser.add_argument('--clean',
borlanic	0:380207fcb5c1	1828	dest='clean',
borlanic	0:380207fcb5c1	1829	action="store_true",
borlanic	0:380207fcb5c1	1830	help='Clean the build directory')
borlanic	0:380207fcb5c1	1831
borlanic	0:380207fcb5c1	1832	parser.add_argument('-P', '--only-peripherals',
borlanic	0:380207fcb5c1	1833	dest='test_only_peripheral',
borlanic	0:380207fcb5c1	1834	default=False,
borlanic	0:380207fcb5c1	1835	action="store_true",
borlanic	0:380207fcb5c1	1836	help='Test only peripheral declared for MUT and skip common tests')
borlanic	0:380207fcb5c1	1837
borlanic	0:380207fcb5c1	1838	parser.add_argument("--profile", dest="profile", action="append",
borlanic	0:380207fcb5c1	1839	type=argparse_filestring_type,
borlanic	0:380207fcb5c1	1840	default=[])
borlanic	0:380207fcb5c1	1841
borlanic	0:380207fcb5c1	1842	parser.add_argument('-C', '--only-commons',
borlanic	0:380207fcb5c1	1843	dest='test_only_common',
borlanic	0:380207fcb5c1	1844	default=False,
borlanic	0:380207fcb5c1	1845	action="store_true",
borlanic	0:380207fcb5c1	1846	help='Test only board internals. Skip perpherials tests and perform common tests')
borlanic	0:380207fcb5c1	1847
borlanic	0:380207fcb5c1	1848	parser.add_argument('-n', '--test-by-names',
borlanic	0:380207fcb5c1	1849	dest='test_by_names',
borlanic	0:380207fcb5c1	1850	type=argparse_many(str),
borlanic	0:380207fcb5c1	1851	help='Runs only test enumerated it this switch. Use comma to separate test case names')
borlanic	0:380207fcb5c1	1852
borlanic	0:380207fcb5c1	1853	parser.add_argument('-p', '--peripheral-by-names',
borlanic	0:380207fcb5c1	1854	dest='peripheral_by_names',
borlanic	0:380207fcb5c1	1855	type=argparse_many(str),
borlanic	0:380207fcb5c1	1856	help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
borlanic	0:380207fcb5c1	1857
borlanic	0:380207fcb5c1	1858	copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
borlanic	0:380207fcb5c1	1859	copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
borlanic	0:380207fcb5c1	1860
borlanic	0:380207fcb5c1	1861	parser.add_argument('-c', '--copy-method',
borlanic	0:380207fcb5c1	1862	dest='copy_method',
borlanic	0:380207fcb5c1	1863	type=argparse_uppercase_type(copy_methods, "flash method"),
borlanic	0:380207fcb5c1	1864	help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
borlanic	0:380207fcb5c1	1865
borlanic	0:380207fcb5c1	1866	reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
borlanic	0:380207fcb5c1	1867	reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
borlanic	0:380207fcb5c1	1868
borlanic	0:380207fcb5c1	1869	parser.add_argument('-r', '--reset-type',
borlanic	0:380207fcb5c1	1870	dest='mut_reset_type',
borlanic	0:380207fcb5c1	1871	default=None,
borlanic	0:380207fcb5c1	1872	type=argparse_uppercase_type(reset_methods, "reset method"),
borlanic	0:380207fcb5c1	1873	help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
borlanic	0:380207fcb5c1	1874
borlanic	0:380207fcb5c1	1875	parser.add_argument('-g', '--goanna-for-tests',
borlanic	0:380207fcb5c1	1876	dest='goanna_for_tests',
borlanic	0:380207fcb5c1	1877	action="store_true",
borlanic	0:380207fcb5c1	1878	help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
borlanic	0:380207fcb5c1	1879
borlanic	0:380207fcb5c1	1880	parser.add_argument('-G', '--goanna-for-sdk',
borlanic	0:380207fcb5c1	1881	dest='goanna_for_mbed_sdk',
borlanic	0:380207fcb5c1	1882	action="store_true",
borlanic	0:380207fcb5c1	1883	help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
borlanic	0:380207fcb5c1	1884
borlanic	0:380207fcb5c1	1885	parser.add_argument('-s', '--suppress-summary',
borlanic	0:380207fcb5c1	1886	dest='suppress_summary',
borlanic	0:380207fcb5c1	1887	default=False,
borlanic	0:380207fcb5c1	1888	action="store_true",
borlanic	0:380207fcb5c1	1889	help='Suppresses display of wellformatted table with test results')
borlanic	0:380207fcb5c1	1890
borlanic	0:380207fcb5c1	1891	parser.add_argument('-t', '--test-summary',
borlanic	0:380207fcb5c1	1892	dest='test_x_toolchain_summary',
borlanic	0:380207fcb5c1	1893	default=False,
borlanic	0:380207fcb5c1	1894	action="store_true",
borlanic	0:380207fcb5c1	1895	help='Displays wellformatted table with test x toolchain test result per target')
borlanic	0:380207fcb5c1	1896
borlanic	0:380207fcb5c1	1897	parser.add_argument('-A', '--test-automation-report',
borlanic	0:380207fcb5c1	1898	dest='test_automation_report',
borlanic	0:380207fcb5c1	1899	default=False,
borlanic	0:380207fcb5c1	1900	action="store_true",
borlanic	0:380207fcb5c1	1901	help='Prints information about all tests and exits')
borlanic	0:380207fcb5c1	1902
borlanic	0:380207fcb5c1	1903	parser.add_argument('-R', '--test-case-report',
borlanic	0:380207fcb5c1	1904	dest='test_case_report',
borlanic	0:380207fcb5c1	1905	default=False,
borlanic	0:380207fcb5c1	1906	action="store_true",
borlanic	0:380207fcb5c1	1907	help='Prints information about all test cases and exits')
borlanic	0:380207fcb5c1	1908
borlanic	0:380207fcb5c1	1909	parser.add_argument("-S", "--supported-toolchains",
borlanic	0:380207fcb5c1	1910	action="store_true",
borlanic	0:380207fcb5c1	1911	dest="supported_toolchains",
borlanic	0:380207fcb5c1	1912	default=False,
borlanic	0:380207fcb5c1	1913	help="Displays supported matrix of MCUs and toolchains")
borlanic	0:380207fcb5c1	1914
borlanic	0:380207fcb5c1	1915	parser.add_argument("-O", "--only-build",
borlanic	0:380207fcb5c1	1916	action="store_true",
borlanic	0:380207fcb5c1	1917	dest="only_build_tests",
borlanic	0:380207fcb5c1	1918	default=False,
borlanic	0:380207fcb5c1	1919	help="Only build tests, skips actual test procedures (flashing etc.)")
borlanic	0:380207fcb5c1	1920
borlanic	0:380207fcb5c1	1921	parser.add_argument('--parallel',
borlanic	0:380207fcb5c1	1922	dest='parallel_test_exec',
borlanic	0:380207fcb5c1	1923	default=False,
borlanic	0:380207fcb5c1	1924	action="store_true",
borlanic	0:380207fcb5c1	1925	help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
borlanic	0:380207fcb5c1	1926
borlanic	0:380207fcb5c1	1927	parser.add_argument('--config',
borlanic	0:380207fcb5c1	1928	dest='verbose_test_configuration_only',
borlanic	0:380207fcb5c1	1929	default=False,
borlanic	0:380207fcb5c1	1930	action="store_true",
borlanic	0:380207fcb5c1	1931	help='Displays full test specification and MUTs configration and exits')
borlanic	0:380207fcb5c1	1932
borlanic	0:380207fcb5c1	1933	parser.add_argument('--loops',
borlanic	0:380207fcb5c1	1934	dest='test_loops_list',
borlanic	0:380207fcb5c1	1935	type=argparse_many(str),
borlanic	0:380207fcb5c1	1936	help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
borlanic	0:380207fcb5c1	1937
borlanic	0:380207fcb5c1	1938	parser.add_argument('--global-loops',
borlanic	0:380207fcb5c1	1939	dest='test_global_loops_value',
borlanic	0:380207fcb5c1	1940	type=int,
borlanic	0:380207fcb5c1	1941	help='Set global number of test loops per test. Default value is set 1')
borlanic	0:380207fcb5c1	1942
borlanic	0:380207fcb5c1	1943	parser.add_argument('--consolidate-waterfall',
borlanic	0:380207fcb5c1	1944	dest='consolidate_waterfall_test',
borlanic	0:380207fcb5c1	1945	default=False,
borlanic	0:380207fcb5c1	1946	action="store_true",
borlanic	0:380207fcb5c1	1947	help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
borlanic	0:380207fcb5c1	1948
borlanic	0:380207fcb5c1	1949	parser.add_argument('-W', '--waterfall',
borlanic	0:380207fcb5c1	1950	dest='waterfall_test',
borlanic	0:380207fcb5c1	1951	default=False,
borlanic	0:380207fcb5c1	1952	action="store_true",
borlanic	0:380207fcb5c1	1953	help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
borlanic	0:380207fcb5c1	1954
borlanic	0:380207fcb5c1	1955	parser.add_argument('-N', '--firmware-name',
borlanic	0:380207fcb5c1	1956	dest='firmware_global_name',
borlanic	0:380207fcb5c1	1957	help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
borlanic	0:380207fcb5c1	1958
borlanic	0:380207fcb5c1	1959	parser.add_argument('-u', '--shuffle',
borlanic	0:380207fcb5c1	1960	dest='shuffle_test_order',
borlanic	0:380207fcb5c1	1961	default=False,
borlanic	0:380207fcb5c1	1962	action="store_true",
borlanic	0:380207fcb5c1	1963	help='Shuffles test execution order')
borlanic	0:380207fcb5c1	1964
borlanic	0:380207fcb5c1	1965	parser.add_argument('--shuffle-seed',
borlanic	0:380207fcb5c1	1966	dest='shuffle_test_seed',
borlanic	0:380207fcb5c1	1967	default=None,
borlanic	0:380207fcb5c1	1968	help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
borlanic	0:380207fcb5c1	1969
borlanic	0:380207fcb5c1	1970	parser.add_argument('-f', '--filter',
borlanic	0:380207fcb5c1	1971	dest='general_filter_regex',
borlanic	0:380207fcb5c1	1972	type=argparse_many(str),
borlanic	0:380207fcb5c1	1973	default=None,
borlanic	0:380207fcb5c1	1974	help='For some commands you can use filter to filter out results')
borlanic	0:380207fcb5c1	1975
borlanic	0:380207fcb5c1	1976	parser.add_argument('--inc-timeout',
borlanic	0:380207fcb5c1	1977	dest='extend_test_timeout',
borlanic	0:380207fcb5c1	1978	metavar="NUMBER",
borlanic	0:380207fcb5c1	1979	type=int,
borlanic	0:380207fcb5c1	1980	help='You can increase global timeout for each test by specifying additional test timeout in seconds')
borlanic	0:380207fcb5c1	1981
borlanic	0:380207fcb5c1	1982	parser.add_argument('--db',
borlanic	0:380207fcb5c1	1983	dest='db_url',
borlanic	0:380207fcb5c1	1984	help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
borlanic	0:380207fcb5c1	1985
borlanic	0:380207fcb5c1	1986	parser.add_argument('-l', '--log',
borlanic	0:380207fcb5c1	1987	dest='log_file_name',
borlanic	0:380207fcb5c1	1988	help='Log events to external file (note not all console entries may be visible in log file)')
borlanic	0:380207fcb5c1	1989
borlanic	0:380207fcb5c1	1990	parser.add_argument('--report-html',
borlanic	0:380207fcb5c1	1991	dest='report_html_file_name',
borlanic	0:380207fcb5c1	1992	help='You can log test suite results in form of HTML report')
borlanic	0:380207fcb5c1	1993
borlanic	0:380207fcb5c1	1994	parser.add_argument('--report-junit',
borlanic	0:380207fcb5c1	1995	dest='report_junit_file_name',
borlanic	0:380207fcb5c1	1996	help='You can log test suite results in form of JUnit compliant XML report')
borlanic	0:380207fcb5c1	1997
borlanic	0:380207fcb5c1	1998	parser.add_argument("--report-build",
borlanic	0:380207fcb5c1	1999	dest="report_build_file_name",
borlanic	0:380207fcb5c1	2000	help="Output the build results to a junit xml file")
borlanic	0:380207fcb5c1	2001
borlanic	0:380207fcb5c1	2002	parser.add_argument("--report-text",
borlanic	0:380207fcb5c1	2003	dest="report_text_file_name",
borlanic	0:380207fcb5c1	2004	help="Output the build results to a text file")
borlanic	0:380207fcb5c1	2005
borlanic	0:380207fcb5c1	2006	parser.add_argument('--verbose-skipped',
borlanic	0:380207fcb5c1	2007	dest='verbose_skipped_tests',
borlanic	0:380207fcb5c1	2008	default=False,
borlanic	0:380207fcb5c1	2009	action="store_true",
borlanic	0:380207fcb5c1	2010	help='Prints some extra information about skipped tests')
borlanic	0:380207fcb5c1	2011
borlanic	0:380207fcb5c1	2012	parser.add_argument('-V', '--verbose-test-result',
borlanic	0:380207fcb5c1	2013	dest='verbose_test_result_only',
borlanic	0:380207fcb5c1	2014	default=False,
borlanic	0:380207fcb5c1	2015	action="store_true",
borlanic	0:380207fcb5c1	2016	help='Prints test serial output')
borlanic	0:380207fcb5c1	2017
borlanic	0:380207fcb5c1	2018	parser.add_argument('-v', '--verbose',
borlanic	0:380207fcb5c1	2019	dest='verbose',
borlanic	0:380207fcb5c1	2020	default=False,
borlanic	0:380207fcb5c1	2021	action="store_true",
borlanic	0:380207fcb5c1	2022	help='Verbose mode (prints some extra information)')
borlanic	0:380207fcb5c1	2023
borlanic	0:380207fcb5c1	2024	parser.add_argument('--version',
borlanic	0:380207fcb5c1	2025	dest='version',
borlanic	0:380207fcb5c1	2026	default=False,
borlanic	0:380207fcb5c1	2027	action="store_true",
borlanic	0:380207fcb5c1	2028	help='Prints script version and exits')
borlanic	0:380207fcb5c1	2029
borlanic	0:380207fcb5c1	2030	parser.add_argument('--stats-depth',
borlanic	0:380207fcb5c1	2031	dest='stats_depth',
borlanic	0:380207fcb5c1	2032	default=2,
borlanic	0:380207fcb5c1	2033	type=int,
borlanic	0:380207fcb5c1	2034	help="Depth level for static memory report")
borlanic	0:380207fcb5c1	2035	return parser
borlanic	0:380207fcb5c1	2036
borlanic	0:380207fcb5c1	2037	def test_path_to_name(path, base):
borlanic	0:380207fcb5c1	2038	"""Change all slashes in a path into hyphens
borlanic	0:380207fcb5c1	2039	This creates a unique cross-platform test name based on the path
borlanic	0:380207fcb5c1	2040	This can eventually be overriden by a to-be-determined meta-data mechanism"""
borlanic	0:380207fcb5c1	2041	name_parts = []
borlanic	0:380207fcb5c1	2042	head, tail = os.path.split(relpath(path,base))
borlanic	0:380207fcb5c1	2043	while (tail and tail != "."):
borlanic	0:380207fcb5c1	2044	name_parts.insert(0, tail)
borlanic	0:380207fcb5c1	2045	head, tail = os.path.split(head)
borlanic	0:380207fcb5c1	2046
borlanic	0:380207fcb5c1	2047	return "-".join(name_parts).lower()
borlanic	0:380207fcb5c1	2048
borlanic	0:380207fcb5c1	2049	def get_test_config(config_name, target_name):
borlanic	0:380207fcb5c1	2050	"""Finds the path to a test configuration file
borlanic	0:380207fcb5c1	2051	config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
borlanic	0:380207fcb5c1	2052	target_name: name of target to determing if mbed OS interface given is valid
borlanic	0:380207fcb5c1	2053	returns path to config, will return None if no valid config is found
borlanic	0:380207fcb5c1	2054	"""
borlanic	0:380207fcb5c1	2055	# If they passed in a full path
borlanic	0:380207fcb5c1	2056	if exists(config_name):
borlanic	0:380207fcb5c1	2057	# This is a module config
borlanic	0:380207fcb5c1	2058	return config_name
borlanic	0:380207fcb5c1	2059	# Otherwise find the path to configuration file based on mbed OS interface
borlanic	0:380207fcb5c1	2060	return TestConfig.get_config_path(config_name, target_name)
borlanic	0:380207fcb5c1	2061
borlanic	0:380207fcb5c1	2062	def find_tests(base_dir, target_name, toolchain_name, app_config=None):
borlanic	0:380207fcb5c1	2063	""" Finds all tests in a directory recursively
borlanic	0:380207fcb5c1	2064	base_dir: path to the directory to scan for tests (ex. 'path/to/project')
borlanic	0:380207fcb5c1	2065	target_name: name of the target to use for scanning (ex. 'K64F')
borlanic	0:380207fcb5c1	2066	toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
borlanic	0:380207fcb5c1	2067	options: Compile options to pass to the toolchain (ex. ['debug-info'])
borlanic	0:380207fcb5c1	2068	app_config - location of a chosen mbed_app.json file
borlanic	0:380207fcb5c1	2069
borlanic	0:380207fcb5c1	2070	returns a dictionary where keys are the test name, and the values are
borlanic	0:380207fcb5c1	2071	lists of paths needed to biuld the test.
borlanic	0:380207fcb5c1	2072	"""
borlanic	0:380207fcb5c1	2073
borlanic	0:380207fcb5c1	2074	# Temporary structure: tests referenced by (name, base, group, case) tuple
borlanic	0:380207fcb5c1	2075	tests = {}
borlanic	0:380207fcb5c1	2076	# List of common folders: (predicate function, path) tuple
borlanic	0:380207fcb5c1	2077	commons = []
borlanic	0:380207fcb5c1	2078
borlanic	0:380207fcb5c1	2079	# Prepare the toolchain
borlanic	0:380207fcb5c1	2080	toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
borlanic	0:380207fcb5c1	2081	silent=True, app_config=app_config)
borlanic	0:380207fcb5c1	2082
borlanic	0:380207fcb5c1	2083	# Scan the directory for paths to probe for 'TESTS' folders
borlanic	0:380207fcb5c1	2084	base_resources = scan_resources([base_dir], toolchain)
borlanic	0:380207fcb5c1	2085
borlanic	0:380207fcb5c1	2086	dirs = base_resources.inc_dirs
borlanic	0:380207fcb5c1	2087	for directory in dirs:
borlanic	0:380207fcb5c1	2088	subdirs = os.listdir(directory)
borlanic	0:380207fcb5c1	2089
borlanic	0:380207fcb5c1	2090	# If the directory contains a subdirectory called 'TESTS', scan it for test cases
borlanic	0:380207fcb5c1	2091	if 'TESTS' in subdirs:
borlanic	0:380207fcb5c1	2092	walk_base_dir = join(directory, 'TESTS')
borlanic	0:380207fcb5c1	2093	test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
borlanic	0:380207fcb5c1	2094
borlanic	0:380207fcb5c1	2095	# Loop through all subdirectories
borlanic	0:380207fcb5c1	2096	for d in test_resources.inc_dirs:
borlanic	0:380207fcb5c1	2097
borlanic	0:380207fcb5c1	2098	# If the test case folder is not called 'host_tests' or 'COMMON' and it is
borlanic	0:380207fcb5c1	2099	# located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
borlanic	0:380207fcb5c1	2100	# then add it to the tests
borlanic	0:380207fcb5c1	2101	relative_path = relpath(d, walk_base_dir)
borlanic	0:380207fcb5c1	2102	relative_path_parts = os.path.normpath(relative_path).split(os.sep)
borlanic	0:380207fcb5c1	2103	if len(relative_path_parts) == 2:
borlanic	0:380207fcb5c1	2104	test_group_directory_path, test_case_directory = os.path.split(d)
borlanic	0:380207fcb5c1	2105	test_group_directory = os.path.basename(test_group_directory_path)
borlanic	0:380207fcb5c1	2106
borlanic	0:380207fcb5c1	2107	# Check to make sure discoverd folder is not in a host test directory or common directory
borlanic	0:380207fcb5c1	2108	special_dirs = ['host_tests', 'COMMON']
borlanic	0:380207fcb5c1	2109	if test_group_directory not in special_dirs and test_case_directory not in special_dirs:
borlanic	0:380207fcb5c1	2110	test_name = test_path_to_name(d, base_dir)
borlanic	0:380207fcb5c1	2111	tests[(test_name, walk_base_dir, test_group_directory, test_case_directory)] = [d]
borlanic	0:380207fcb5c1	2112
borlanic	0:380207fcb5c1	2113	# Also find any COMMON paths, we'll add these later once we find all the base tests
borlanic	0:380207fcb5c1	2114	if 'COMMON' in relative_path_parts:
borlanic	0:380207fcb5c1	2115	if relative_path_parts[0] != 'COMMON':
borlanic	0:380207fcb5c1	2116	def predicate(base_pred, group_pred, (name, base, group, case)):
borlanic	0:380207fcb5c1	2117	return base == base_pred and group == group_pred
borlanic	0:380207fcb5c1	2118	commons.append((functools.partial(predicate, walk_base_dir, relative_path_parts[0]), d))
borlanic	0:380207fcb5c1	2119	else:
borlanic	0:380207fcb5c1	2120	def predicate(base_pred, (name, base, group, case)):
borlanic	0:380207fcb5c1	2121	return base == base_pred
borlanic	0:380207fcb5c1	2122	commons.append((functools.partial(predicate, walk_base_dir), d))
borlanic	0:380207fcb5c1	2123
borlanic	0:380207fcb5c1	2124	# Apply common directories
borlanic	0:380207fcb5c1	2125	for pred, path in commons:
borlanic	0:380207fcb5c1	2126	for test_identity, test_paths in tests.iteritems():
borlanic	0:380207fcb5c1	2127	if pred(test_identity):
borlanic	0:380207fcb5c1	2128	test_paths.append(path)
borlanic	0:380207fcb5c1	2129
borlanic	0:380207fcb5c1	2130	# Drop identity besides name
borlanic	0:380207fcb5c1	2131	return {name: paths for (name, _, _, _), paths in tests.iteritems()}
borlanic	0:380207fcb5c1	2132
borlanic	0:380207fcb5c1	2133	def print_tests(tests, format="list", sort=True):
borlanic	0:380207fcb5c1	2134	"""Given a dictionary of tests (as returned from "find_tests"), print them
borlanic	0:380207fcb5c1	2135	in the specified format"""
borlanic	0:380207fcb5c1	2136	if format == "list":
borlanic	0:380207fcb5c1	2137	for test_name in sorted(tests.keys()):
borlanic	0:380207fcb5c1	2138	test_path = tests[test_name][0]
borlanic	0:380207fcb5c1	2139	print("Test Case:")
borlanic	0:380207fcb5c1	2140	print(" Name: %s" % test_name)
borlanic	0:380207fcb5c1	2141	print(" Path: %s" % test_path)
borlanic	0:380207fcb5c1	2142	elif format == "json":
borlanic	0:380207fcb5c1	2143	print(json.dumps({test_name: test_path[0] for test_name, test_paths
borlanic	0:380207fcb5c1	2144	in tests}, indent=2))
borlanic	0:380207fcb5c1	2145	else:
borlanic	0:380207fcb5c1	2146	print("Unknown format '%s'" % format)
borlanic	0:380207fcb5c1	2147	sys.exit(1)
borlanic	0:380207fcb5c1	2148
borlanic	0:380207fcb5c1	2149	def norm_relative_path(path, start):
borlanic	0:380207fcb5c1	2150	"""This function will create a normalized, relative path. It mimics the
borlanic	0:380207fcb5c1	2151	python os.path.relpath function, but also normalizes a Windows-syle path
borlanic	0:380207fcb5c1	2152	that use backslashes to a Unix style path that uses forward slashes."""
borlanic	0:380207fcb5c1	2153	path = os.path.normpath(path)
borlanic	0:380207fcb5c1	2154	path = os.path.relpath(path, start)
borlanic	0:380207fcb5c1	2155	path = path.replace("\\", "/")
borlanic	0:380207fcb5c1	2156	return path
borlanic	0:380207fcb5c1	2157
borlanic	0:380207fcb5c1	2158
borlanic	0:380207fcb5c1	2159	def build_test_worker(args, *kwargs):
borlanic	0:380207fcb5c1	2160	"""This is a worker function for the parallel building of tests. The `args`
borlanic	0:380207fcb5c1	2161	and `kwargs` are passed directly to `build_project`. It returns a dictionary
borlanic	0:380207fcb5c1	2162	with the following structure:
borlanic	0:380207fcb5c1	2163
borlanic	0:380207fcb5c1	2164	{
borlanic	0:380207fcb5c1	2165	'result': `True` if no exceptions were thrown, `False` otherwise
borlanic	0:380207fcb5c1	2166	'reason': Instance of exception that was thrown on failure
borlanic	0:380207fcb5c1	2167	'bin_file': Path to the created binary if `build_project` was
borlanic	0:380207fcb5c1	2168	successful. Not present otherwise
borlanic	0:380207fcb5c1	2169	'kwargs': The keyword arguments that were passed to `build_project`.
borlanic	0:380207fcb5c1	2170	This includes arguments that were modified (ex. report)
borlanic	0:380207fcb5c1	2171	}
borlanic	0:380207fcb5c1	2172	"""
borlanic	0:380207fcb5c1	2173	bin_file = None
borlanic	0:380207fcb5c1	2174	ret = {
borlanic	0:380207fcb5c1	2175	'result': False,
borlanic	0:380207fcb5c1	2176	'args': args,
borlanic	0:380207fcb5c1	2177	'kwargs': kwargs
borlanic	0:380207fcb5c1	2178	}
borlanic	0:380207fcb5c1	2179
borlanic	0:380207fcb5c1	2180	# Use parent TOOLCHAIN_PATHS variable
borlanic	0:380207fcb5c1	2181	for key, value in kwargs['toolchain_paths'].items():
borlanic	0:380207fcb5c1	2182	TOOLCHAIN_PATHS[key] = value
borlanic	0:380207fcb5c1	2183
borlanic	0:380207fcb5c1	2184	del kwargs['toolchain_paths']
borlanic	0:380207fcb5c1	2185
borlanic	0:380207fcb5c1	2186	try:
borlanic	0:380207fcb5c1	2187	bin_file = build_project(args, *kwargs)
borlanic	0:380207fcb5c1	2188	ret['result'] = True
borlanic	0:380207fcb5c1	2189	ret['bin_file'] = bin_file
borlanic	0:380207fcb5c1	2190	ret['kwargs'] = kwargs
borlanic	0:380207fcb5c1	2191
borlanic	0:380207fcb5c1	2192	except NotSupportedException as e:
borlanic	0:380207fcb5c1	2193	ret['reason'] = e
borlanic	0:380207fcb5c1	2194	except ToolException as e:
borlanic	0:380207fcb5c1	2195	ret['reason'] = e
borlanic	0:380207fcb5c1	2196	except KeyboardInterrupt as e:
borlanic	0:380207fcb5c1	2197	ret['reason'] = e
borlanic	0:380207fcb5c1	2198	except:
borlanic	0:380207fcb5c1	2199	# Print unhandled exceptions here
borlanic	0:380207fcb5c1	2200	import traceback
borlanic	0:380207fcb5c1	2201	traceback.print_exc(file=sys.stdout)
borlanic	0:380207fcb5c1	2202
borlanic	0:380207fcb5c1	2203	return ret
borlanic	0:380207fcb5c1	2204
borlanic	0:380207fcb5c1	2205
borlanic	0:380207fcb5c1	2206	def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
borlanic	0:380207fcb5c1	2207	clean=False, notify=None, verbose=False, jobs=1, macros=None,
borlanic	0:380207fcb5c1	2208	silent=False, report=None, properties=None,
borlanic	0:380207fcb5c1	2209	continue_on_build_fail=False, app_config=None,
borlanic	0:380207fcb5c1	2210	build_profile=None, stats_depth=None):
borlanic	0:380207fcb5c1	2211	"""Given the data structure from 'find_tests' and the typical build parameters,
borlanic	0:380207fcb5c1	2212	build all the tests
borlanic	0:380207fcb5c1	2213
borlanic	0:380207fcb5c1	2214	Returns a tuple of the build result (True or False) followed by the test
borlanic	0:380207fcb5c1	2215	build data structure"""
borlanic	0:380207fcb5c1	2216
borlanic	0:380207fcb5c1	2217	execution_directory = "."
borlanic	0:380207fcb5c1	2218	base_path = norm_relative_path(build_path, execution_directory)
borlanic	0:380207fcb5c1	2219
borlanic	0:380207fcb5c1	2220	target_name = target.name if isinstance(target, Target) else target
borlanic	0:380207fcb5c1	2221	cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
borlanic	0:380207fcb5c1	2222
borlanic	0:380207fcb5c1	2223	baud_rate = 9600
borlanic	0:380207fcb5c1	2224	if 'platform.stdio-baud-rate' in cfg:
borlanic	0:380207fcb5c1	2225	baud_rate = cfg['platform.stdio-baud-rate'].value
borlanic	0:380207fcb5c1	2226
borlanic	0:380207fcb5c1	2227	test_build = {
borlanic	0:380207fcb5c1	2228	"platform": target_name,
borlanic	0:380207fcb5c1	2229	"toolchain": toolchain_name,
borlanic	0:380207fcb5c1	2230	"base_path": base_path,
borlanic	0:380207fcb5c1	2231	"baud_rate": baud_rate,
borlanic	0:380207fcb5c1	2232	"binary_type": "bootable",
borlanic	0:380207fcb5c1	2233	"tests": {}
borlanic	0:380207fcb5c1	2234	}
borlanic	0:380207fcb5c1	2235
borlanic	0:380207fcb5c1	2236	result = True
borlanic	0:380207fcb5c1	2237
borlanic	0:380207fcb5c1	2238	jobs_count = int(jobs if jobs else cpu_count())
borlanic	0:380207fcb5c1	2239	p = Pool(processes=jobs_count)
borlanic	0:380207fcb5c1	2240	results = []
borlanic	0:380207fcb5c1	2241	for test_name, test_paths in tests.items():
borlanic	0:380207fcb5c1	2242	if not isinstance(test_paths, list):
borlanic	0:380207fcb5c1	2243	test_paths = [test_paths]
borlanic	0:380207fcb5c1	2244
borlanic	0:380207fcb5c1	2245	test_build_path = os.path.join(build_path, test_paths[0])
borlanic	0:380207fcb5c1	2246	src_paths = base_source_paths + test_paths
borlanic	0:380207fcb5c1	2247	bin_file = None
borlanic	0:380207fcb5c1	2248	test_case_folder_name = os.path.basename(test_paths[0])
borlanic	0:380207fcb5c1	2249
borlanic	0:380207fcb5c1	2250	args = (src_paths, test_build_path, target, toolchain_name)
borlanic	0:380207fcb5c1	2251	kwargs = {
borlanic	0:380207fcb5c1	2252	'jobs': 1,
borlanic	0:380207fcb5c1	2253	'clean': clean,
borlanic	0:380207fcb5c1	2254	'macros': macros,
borlanic	0:380207fcb5c1	2255	'name': test_case_folder_name,
borlanic	0:380207fcb5c1	2256	'project_id': test_name,
borlanic	0:380207fcb5c1	2257	'report': report,
borlanic	0:380207fcb5c1	2258	'properties': properties,
borlanic	0:380207fcb5c1	2259	'verbose': verbose,
borlanic	0:380207fcb5c1	2260	'app_config': app_config,
borlanic	0:380207fcb5c1	2261	'build_profile': build_profile,
borlanic	0:380207fcb5c1	2262	'silent': True,
borlanic	0:380207fcb5c1	2263	'toolchain_paths': TOOLCHAIN_PATHS,
borlanic	0:380207fcb5c1	2264	'stats_depth': stats_depth
borlanic	0:380207fcb5c1	2265	}
borlanic	0:380207fcb5c1	2266
borlanic	0:380207fcb5c1	2267	results.append(p.apply_async(build_test_worker, args, kwargs))
borlanic	0:380207fcb5c1	2268
borlanic	0:380207fcb5c1	2269	p.close()
borlanic	0:380207fcb5c1	2270	result = True
borlanic	0:380207fcb5c1	2271	itr = 0
borlanic	0:380207fcb5c1	2272	while len(results):
borlanic	0:380207fcb5c1	2273	itr += 1
borlanic	0:380207fcb5c1	2274	if itr > 360000:
borlanic	0:380207fcb5c1	2275	p.terminate()
borlanic	0:380207fcb5c1	2276	p.join()
borlanic	0:380207fcb5c1	2277	raise ToolException("Compile did not finish in 10 minutes")
borlanic	0:380207fcb5c1	2278	else:
borlanic	0:380207fcb5c1	2279	sleep(0.01)
borlanic	0:380207fcb5c1	2280	pending = 0
borlanic	0:380207fcb5c1	2281	for r in results:
borlanic	0:380207fcb5c1	2282	if r.ready() is True:
borlanic	0:380207fcb5c1	2283	try:
borlanic	0:380207fcb5c1	2284	worker_result = r.get()
borlanic	0:380207fcb5c1	2285	results.remove(r)
borlanic	0:380207fcb5c1	2286
borlanic	0:380207fcb5c1	2287	# Take report from the kwargs and merge it into existing report
borlanic	0:380207fcb5c1	2288	if report:
borlanic	0:380207fcb5c1	2289	report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
borlanic	0:380207fcb5c1	2290	for test_key in report_entry.keys():
borlanic	0:380207fcb5c1	2291	report[target_name][toolchain_name][test_key] = report_entry[test_key]
borlanic	0:380207fcb5c1	2292
borlanic	0:380207fcb5c1	2293	# Set the overall result to a failure if a build failure occurred
borlanic	0:380207fcb5c1	2294	if ('reason' in worker_result and
borlanic	0:380207fcb5c1	2295	not worker_result['reason'] and
borlanic	0:380207fcb5c1	2296	not isinstance(worker_result['reason'], NotSupportedException)):
borlanic	0:380207fcb5c1	2297	result = False
borlanic	0:380207fcb5c1	2298	break
borlanic	0:380207fcb5c1	2299
borlanic	0:380207fcb5c1	2300	# Adding binary path to test build result
borlanic	0:380207fcb5c1	2301	if ('result' in worker_result and
borlanic	0:380207fcb5c1	2302	worker_result['result'] and
borlanic	0:380207fcb5c1	2303	'bin_file' in worker_result):
borlanic	0:380207fcb5c1	2304	bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
borlanic	0:380207fcb5c1	2305
borlanic	0:380207fcb5c1	2306	test_build['tests'][worker_result['kwargs']['project_id']] = {
borlanic	0:380207fcb5c1	2307	"binaries": [
borlanic	0:380207fcb5c1	2308	{
borlanic	0:380207fcb5c1	2309	"path": bin_file
borlanic	0:380207fcb5c1	2310	}
borlanic	0:380207fcb5c1	2311	]
borlanic	0:380207fcb5c1	2312	}
borlanic	0:380207fcb5c1	2313
borlanic	0:380207fcb5c1	2314	test_key = worker_result['kwargs']['project_id'].upper()
borlanic	0:380207fcb5c1	2315	if report:
borlanic	0:380207fcb5c1	2316	print(report[target_name][toolchain_name][test_key][0][0]['output'].rstrip())
borlanic	0:380207fcb5c1	2317	print('Image: %s\n' % bin_file)
borlanic	0:380207fcb5c1	2318
borlanic	0:380207fcb5c1	2319	except:
borlanic	0:380207fcb5c1	2320	if p._taskqueue.queue:
borlanic	0:380207fcb5c1	2321	p._taskqueue.queue.clear()
borlanic	0:380207fcb5c1	2322	sleep(0.5)
borlanic	0:380207fcb5c1	2323	p.terminate()
borlanic	0:380207fcb5c1	2324	p.join()
borlanic	0:380207fcb5c1	2325	raise
borlanic	0:380207fcb5c1	2326	else:
borlanic	0:380207fcb5c1	2327	pending += 1
borlanic	0:380207fcb5c1	2328	if pending >= jobs_count:
borlanic	0:380207fcb5c1	2329	break
borlanic	0:380207fcb5c1	2330
borlanic	0:380207fcb5c1	2331	# Break as soon as possible if there is a failure and we are not
borlanic	0:380207fcb5c1	2332	# continuing on build failures
borlanic	0:380207fcb5c1	2333	if not result and not continue_on_build_fail:
borlanic	0:380207fcb5c1	2334	if p._taskqueue.queue:
borlanic	0:380207fcb5c1	2335	p._taskqueue.queue.clear()
borlanic	0:380207fcb5c1	2336	sleep(0.5)
borlanic	0:380207fcb5c1	2337	p.terminate()
borlanic	0:380207fcb5c1	2338	break
borlanic	0:380207fcb5c1	2339
borlanic	0:380207fcb5c1	2340	p.join()
borlanic	0:380207fcb5c1	2341
borlanic	0:380207fcb5c1	2342	test_builds = {}
borlanic	0:380207fcb5c1	2343	test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
borlanic	0:380207fcb5c1	2344
borlanic	0:380207fcb5c1	2345	return result, test_builds
borlanic	0:380207fcb5c1	2346
borlanic	0:380207fcb5c1	2347
borlanic	0:380207fcb5c1	2348	def test_spec_from_test_builds(test_builds):
borlanic	0:380207fcb5c1	2349	return {
borlanic	0:380207fcb5c1	2350	"builds": test_builds
borlanic	0:380207fcb5c1	2351	}

Repository toolbox

Repository details

Type:	Program
Mbed OS support:	Mbed OS
Created:	29 Mar 2018
Imports:	22
Forks:	0
Commits:	5
Dependents:	0
Dependencies:	0
Followers:	6

Important changes to repositories hosted on mbed.com

mbed-os/tools/test_api.py@4:75df35ef4fb6, 2018-03-30 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Important changes to repositories hosted on mbed.com

mbed-os/tools/test_api.py@4:75df35ef4fb6, 2018-03-30 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Access Warning