mbed-sdk-tools - Clone of official tools

Users » andersblomdell » Code » mbed-sdk-tools

Clone of official tools

test_api.py@47:21ae3e5a7128, 2021-02-04 (annotated)

Committer:: Anders Blomdell
Date:: Thu Feb 04 17:17:13 2021 +0100
Revision:: 47:21ae3e5a7128
Parent:: 43:2a7da56ebd24

Add a few normpath calls

Who changed what in which revision?

User	Revision	Line number	New contents of line
screamer	0:66f3b5499f7f	1	"""
screamer	0:66f3b5499f7f	2	mbed SDK
screamer	0:66f3b5499f7f	3	Copyright (c) 2011-2014 ARM Limited
screamer	0:66f3b5499f7f	4
screamer	0:66f3b5499f7f	5	Licensed under the Apache License, Version 2.0 (the "License");
screamer	0:66f3b5499f7f	6	you may not use this file except in compliance with the License.
screamer	0:66f3b5499f7f	7	You may obtain a copy of the License at
screamer	0:66f3b5499f7f	8
screamer	0:66f3b5499f7f	9	http://www.apache.org/licenses/LICENSE-2.0
screamer	0:66f3b5499f7f	10
screamer	0:66f3b5499f7f	11	Unless required by applicable law or agreed to in writing, software
screamer	0:66f3b5499f7f	12	distributed under the License is distributed on an "AS IS" BASIS,
screamer	0:66f3b5499f7f	13	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer	0:66f3b5499f7f	14	See the License for the specific language governing permissions and
screamer	0:66f3b5499f7f	15	limitations under the License.
screamer	0:66f3b5499f7f	16
screamer	0:66f3b5499f7f	17	Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
screamer	0:66f3b5499f7f	18	"""
theotherjimmy	43:2a7da56ebd24	19	from __future__ import print_function
theotherjimmy	43:2a7da56ebd24	20	import six
screamer	0:66f3b5499f7f	21
screamer	0:66f3b5499f7f	22	import os
screamer	0:66f3b5499f7f	23	import re
screamer	0:66f3b5499f7f	24	import sys
screamer	0:66f3b5499f7f	25	import json
screamer	0:66f3b5499f7f	26	import uuid
screamer	0:66f3b5499f7f	27	import pprint
screamer	0:66f3b5499f7f	28	import random
screamer	22:9e85236d8716	29	import argparse
screamer	0:66f3b5499f7f	30	import datetime
screamer	0:66f3b5499f7f	31	import threading
screamer	0:66f3b5499f7f	32	import ctypes
theotherjimmy	43:2a7da56ebd24	33	import functools
screamer	0:66f3b5499f7f	34	from colorama import Fore, Back, Style
theotherjimmy	43:2a7da56ebd24	35	from prettytable import PrettyTable, HEADER
theotherjimmy	43:2a7da56ebd24	36	from copy import copy, deepcopy
screamer	0:66f3b5499f7f	37
screamer	0:66f3b5499f7f	38	from time import sleep, time
theotherjimmy	43:2a7da56ebd24	39	try:
theotherjimmy	43:2a7da56ebd24	40	from Queue import Queue, Empty
theotherjimmy	43:2a7da56ebd24	41	except ImportError:
theotherjimmy	43:2a7da56ebd24	42	from queue import Queue, Empty
theotherjimmy	43:2a7da56ebd24	43	from os.path import join, exists, basename, relpath, isdir, isfile
screamer	0:66f3b5499f7f	44	from threading import Thread, Lock
The Other Jimmy	31:8ea194f6145b	45	from multiprocessing import Pool, cpu_count
screamer	0:66f3b5499f7f	46	from subprocess import Popen, PIPE
screamer	0:66f3b5499f7f	47
screamer	0:66f3b5499f7f	48	# Imports related to mbed build api
screamer	0:66f3b5499f7f	49	from tools.tests import TESTS
screamer	0:66f3b5499f7f	50	from tools.tests import TEST_MAP
screamer	0:66f3b5499f7f	51	from tools.paths import BUILD_DIR
screamer	0:66f3b5499f7f	52	from tools.paths import HOST_TESTS
screamer	0:66f3b5499f7f	53	from tools.utils import ToolException
screamer	0:66f3b5499f7f	54	from tools.utils import NotSupportedException
screamer	0:66f3b5499f7f	55	from tools.utils import construct_enum
screamer	22:9e85236d8716	56	from tools.memap import MemapParser
theotherjimmy	43:2a7da56ebd24	57	from tools.targets import TARGET_MAP, Target
theotherjimmy	43:2a7da56ebd24	58	from tools.config import Config
theotherjimmy	41:2a77626a4c21	59	import tools.test_configs as TestConfig
screamer	0:66f3b5499f7f	60	from tools.test_db import BaseDBAccess
screamer	0:66f3b5499f7f	61	from tools.build_api import build_project, build_mbed_libs, build_lib
screamer	0:66f3b5499f7f	62	from tools.build_api import get_target_supported_toolchains
screamer	0:66f3b5499f7f	63	from tools.build_api import write_build_report
screamer	0:66f3b5499f7f	64	from tools.build_api import prep_report
screamer	0:66f3b5499f7f	65	from tools.build_api import prep_properties
screamer	0:66f3b5499f7f	66	from tools.build_api import create_result
screamer	0:66f3b5499f7f	67	from tools.build_api import add_result_to_report
screamer	24:25bff2709c20	68	from tools.build_api import prepare_toolchain
The Other Jimmy	31:8ea194f6145b	69	from tools.build_api import get_config
theotherjimmy	43:2a7da56ebd24	70	from tools.resources import Resources, MbedIgnoreSet, IGNORE_FILENAME
screamer	0:66f3b5499f7f	71	from tools.libraries import LIBRARIES, LIBRARY_MAP
The Other Jimmy	31:8ea194f6145b	72	from tools.options import extract_profile
screamer	20:835f6355470d	73	from tools.toolchains import TOOLCHAIN_PATHS
screamer	22:9e85236d8716	74	from tools.toolchains import TOOLCHAINS
screamer	0:66f3b5499f7f	75	from tools.test_exporters import ReportExporter, ResultExporterType
screamer	22:9e85236d8716	76	from tools.utils import argparse_filestring_type
screamer	22:9e85236d8716	77	from tools.utils import argparse_uppercase_type
screamer	22:9e85236d8716	78	from tools.utils import argparse_lowercase_type
screamer	22:9e85236d8716	79	from tools.utils import argparse_many
theotherjimmy	43:2a7da56ebd24	80	from tools.notifier.mock import MockNotifier
theotherjimmy	43:2a7da56ebd24	81	from tools.notifier.term import TerminalNotifier
screamer	0:66f3b5499f7f	82
screamer	0:66f3b5499f7f	83	import tools.host_tests.host_tests_plugins as host_tests_plugins
screamer	0:66f3b5499f7f	84
screamer	0:66f3b5499f7f	85	try:
screamer	0:66f3b5499f7f	86	import mbed_lstools
screamer	0:66f3b5499f7f	87	from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer	0:66f3b5499f7f	88	except:
screamer	0:66f3b5499f7f	89	pass
screamer	0:66f3b5499f7f	90
screamer	0:66f3b5499f7f	91
screamer	0:66f3b5499f7f	92	class ProcessObserver(Thread):
screamer	0:66f3b5499f7f	93	def __init__(self, proc):
screamer	0:66f3b5499f7f	94	Thread.__init__(self)
screamer	0:66f3b5499f7f	95	self.proc = proc
screamer	0:66f3b5499f7f	96	self.queue = Queue()
screamer	0:66f3b5499f7f	97	self.daemon = True
screamer	0:66f3b5499f7f	98	self.active = True
screamer	0:66f3b5499f7f	99	self.start()
screamer	0:66f3b5499f7f	100
screamer	0:66f3b5499f7f	101	def run(self):
screamer	0:66f3b5499f7f	102	while self.active:
screamer	0:66f3b5499f7f	103	c = self.proc.stdout.read(1)
screamer	0:66f3b5499f7f	104	self.queue.put(c)
screamer	0:66f3b5499f7f	105
screamer	0:66f3b5499f7f	106	def stop(self):
screamer	0:66f3b5499f7f	107	self.active = False
screamer	0:66f3b5499f7f	108	try:
screamer	0:66f3b5499f7f	109	self.proc.terminate()
theotherjimmy	43:2a7da56ebd24	110	except Exception:
screamer	0:66f3b5499f7f	111	pass
screamer	0:66f3b5499f7f	112
screamer	0:66f3b5499f7f	113
screamer	0:66f3b5499f7f	114	class SingleTestExecutor(threading.Thread):
screamer	0:66f3b5499f7f	115	""" Example: Single test class in separate thread usage
screamer	0:66f3b5499f7f	116	"""
screamer	0:66f3b5499f7f	117	def __init__(self, single_test):
screamer	0:66f3b5499f7f	118	self.single_test = single_test
screamer	0:66f3b5499f7f	119	threading.Thread.__init__(self)
screamer	0:66f3b5499f7f	120
screamer	0:66f3b5499f7f	121	def run(self):
screamer	0:66f3b5499f7f	122	start = time()
screamer	0:66f3b5499f7f	123	# Execute tests depending on options and filter applied
screamer	0:66f3b5499f7f	124	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
screamer	0:66f3b5499f7f	125	elapsed_time = time() - start
screamer	0:66f3b5499f7f	126
screamer	0:66f3b5499f7f	127	# Human readable summary
screamer	0:66f3b5499f7f	128	if not self.single_test.opts_suppress_summary:
screamer	0:66f3b5499f7f	129	# prints well-formed summary with results (SQL table like)
theotherjimmy	43:2a7da56ebd24	130	print(self.single_test.generate_test_summary(test_summary,
theotherjimmy	43:2a7da56ebd24	131	shuffle_seed))
screamer	0:66f3b5499f7f	132	if self.single_test.opts_test_x_toolchain_summary:
screamer	0:66f3b5499f7f	133	# prints well-formed summary with results (SQL table like)
screamer	0:66f3b5499f7f	134	# table shows text x toolchain test result matrix
theotherjimmy	43:2a7da56ebd24	135	print(self.single_test.generate_test_summary_by_target(
theotherjimmy	43:2a7da56ebd24	136	test_summary, shuffle_seed))
theotherjimmy	43:2a7da56ebd24	137	print("Completed in %.2f sec"% (elapsed_time))
screamer	0:66f3b5499f7f	138
screamer	0:66f3b5499f7f	139
screamer	0:66f3b5499f7f	140	class SingleTestRunner(object):
screamer	0:66f3b5499f7f	141	""" Object wrapper for single test run which may involve multiple MUTs
screamer	0:66f3b5499f7f	142	"""
screamer	0:66f3b5499f7f	143	RE_DETECT_TESTCASE_RESULT = None
screamer	0:66f3b5499f7f	144
screamer	0:66f3b5499f7f	145	# Return codes for test script
screamer	0:66f3b5499f7f	146	TEST_RESULT_OK = "OK"
screamer	0:66f3b5499f7f	147	TEST_RESULT_FAIL = "FAIL"
screamer	0:66f3b5499f7f	148	TEST_RESULT_ERROR = "ERROR"
screamer	0:66f3b5499f7f	149	TEST_RESULT_UNDEF = "UNDEF"
screamer	0:66f3b5499f7f	150	TEST_RESULT_IOERR_COPY = "IOERR_COPY"
screamer	0:66f3b5499f7f	151	TEST_RESULT_IOERR_DISK = "IOERR_DISK"
screamer	0:66f3b5499f7f	152	TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
screamer	0:66f3b5499f7f	153	TEST_RESULT_TIMEOUT = "TIMEOUT"
screamer	0:66f3b5499f7f	154	TEST_RESULT_NO_IMAGE = "NO_IMAGE"
screamer	0:66f3b5499f7f	155	TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
screamer	0:66f3b5499f7f	156	TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
screamer	0:66f3b5499f7f	157	TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
screamer	0:66f3b5499f7f	158
screamer	0:66f3b5499f7f	159	GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
screamer	0:66f3b5499f7f	160	TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
screamer	0:66f3b5499f7f	161	TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
screamer	0:66f3b5499f7f	162
screamer	0:66f3b5499f7f	163	muts = {} # MUTs descriptor (from external file)
screamer	0:66f3b5499f7f	164	test_spec = {} # Test specification (from external file)
screamer	0:66f3b5499f7f	165
screamer	0:66f3b5499f7f	166	# mbed test suite -> SingleTestRunner
screamer	0:66f3b5499f7f	167	TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
screamer	0:66f3b5499f7f	168	"failure" : TEST_RESULT_FAIL,
screamer	0:66f3b5499f7f	169	"error" : TEST_RESULT_ERROR,
screamer	0:66f3b5499f7f	170	"ioerr_copy" : TEST_RESULT_IOERR_COPY,
screamer	0:66f3b5499f7f	171	"ioerr_disk" : TEST_RESULT_IOERR_DISK,
screamer	0:66f3b5499f7f	172	"ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
screamer	0:66f3b5499f7f	173	"timeout" : TEST_RESULT_TIMEOUT,
screamer	0:66f3b5499f7f	174	"no_image" : TEST_RESULT_NO_IMAGE,
screamer	0:66f3b5499f7f	175	"end" : TEST_RESULT_UNDEF,
screamer	0:66f3b5499f7f	176	"mbed_assert" : TEST_RESULT_MBED_ASSERT,
screamer	0:66f3b5499f7f	177	"build_failed" : TEST_RESULT_BUILD_FAILED,
screamer	0:66f3b5499f7f	178	"not_supproted" : TEST_RESULT_NOT_SUPPORTED
screamer	0:66f3b5499f7f	179	}
screamer	0:66f3b5499f7f	180
screamer	0:66f3b5499f7f	181	def __init__(self,
screamer	0:66f3b5499f7f	182	_global_loops_count=1,
screamer	0:66f3b5499f7f	183	_test_loops_list=None,
screamer	0:66f3b5499f7f	184	_muts={},
screamer	0:66f3b5499f7f	185	_clean=False,
The Other Jimmy	31:8ea194f6145b	186	_parser=None,
The Other Jimmy	31:8ea194f6145b	187	_opts=None,
screamer	0:66f3b5499f7f	188	_opts_db_url=None,
screamer	0:66f3b5499f7f	189	_opts_log_file_name=None,
screamer	0:66f3b5499f7f	190	_opts_report_html_file_name=None,
screamer	0:66f3b5499f7f	191	_opts_report_junit_file_name=None,
screamer	0:66f3b5499f7f	192	_opts_report_build_file_name=None,
screamer	22:9e85236d8716	193	_opts_report_text_file_name=None,
screamer	0:66f3b5499f7f	194	_opts_build_report={},
screamer	0:66f3b5499f7f	195	_opts_build_properties={},
screamer	0:66f3b5499f7f	196	_test_spec={},
screamer	0:66f3b5499f7f	197	_opts_goanna_for_mbed_sdk=None,
screamer	0:66f3b5499f7f	198	_opts_goanna_for_tests=None,
screamer	0:66f3b5499f7f	199	_opts_shuffle_test_order=False,
screamer	0:66f3b5499f7f	200	_opts_shuffle_test_seed=None,
screamer	0:66f3b5499f7f	201	_opts_test_by_names=None,
screamer	0:66f3b5499f7f	202	_opts_peripheral_by_names=None,
screamer	0:66f3b5499f7f	203	_opts_test_only_peripheral=False,
screamer	0:66f3b5499f7f	204	_opts_test_only_common=False,
screamer	0:66f3b5499f7f	205	_opts_verbose_skipped_tests=False,
screamer	0:66f3b5499f7f	206	_opts_verbose_test_result_only=False,
screamer	0:66f3b5499f7f	207	_opts_verbose=False,
screamer	0:66f3b5499f7f	208	_opts_firmware_global_name=None,
screamer	0:66f3b5499f7f	209	_opts_only_build_tests=False,
screamer	0:66f3b5499f7f	210	_opts_parallel_test_exec=False,
screamer	0:66f3b5499f7f	211	_opts_suppress_summary=False,
screamer	0:66f3b5499f7f	212	_opts_test_x_toolchain_summary=False,
screamer	0:66f3b5499f7f	213	_opts_copy_method=None,
screamer	0:66f3b5499f7f	214	_opts_mut_reset_type=None,
screamer	0:66f3b5499f7f	215	_opts_jobs=None,
screamer	0:66f3b5499f7f	216	_opts_waterfall_test=None,
screamer	0:66f3b5499f7f	217	_opts_consolidate_waterfall_test=None,
screamer	0:66f3b5499f7f	218	_opts_extend_test_timeout=None,
screamer	0:66f3b5499f7f	219	_opts_auto_detect=None,
screamer	0:66f3b5499f7f	220	_opts_include_non_automated=False):
screamer	0:66f3b5499f7f	221	""" Let's try hard to init this object
screamer	0:66f3b5499f7f	222	"""
screamer	0:66f3b5499f7f	223	from colorama import init
screamer	0:66f3b5499f7f	224	init()
screamer	0:66f3b5499f7f	225
screamer	0:66f3b5499f7f	226	PATTERN = "\\{(" + "\|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
screamer	0:66f3b5499f7f	227	self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
screamer	0:66f3b5499f7f	228	# Settings related to test loops counters
screamer	0:66f3b5499f7f	229	try:
screamer	0:66f3b5499f7f	230	_global_loops_count = int(_global_loops_count)
screamer	0:66f3b5499f7f	231	except:
screamer	0:66f3b5499f7f	232	_global_loops_count = 1
screamer	0:66f3b5499f7f	233	if _global_loops_count < 1:
screamer	0:66f3b5499f7f	234	_global_loops_count = 1
screamer	0:66f3b5499f7f	235	self.GLOBAL_LOOPS_COUNT = _global_loops_count
screamer	0:66f3b5499f7f	236	self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
screamer	0:66f3b5499f7f	237	self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
screamer	0:66f3b5499f7f	238
screamer	0:66f3b5499f7f	239	self.shuffle_random_seed = 0.0
screamer	0:66f3b5499f7f	240	self.SHUFFLE_SEED_ROUND = 10
screamer	0:66f3b5499f7f	241
screamer	0:66f3b5499f7f	242	# MUT list and test specification storage
screamer	0:66f3b5499f7f	243	self.muts = _muts
screamer	0:66f3b5499f7f	244	self.test_spec = _test_spec
screamer	0:66f3b5499f7f	245
screamer	0:66f3b5499f7f	246	# Settings passed e.g. from command line
screamer	0:66f3b5499f7f	247	self.opts_db_url = _opts_db_url
screamer	0:66f3b5499f7f	248	self.opts_log_file_name = _opts_log_file_name
screamer	0:66f3b5499f7f	249	self.opts_report_html_file_name = _opts_report_html_file_name
screamer	0:66f3b5499f7f	250	self.opts_report_junit_file_name = _opts_report_junit_file_name
screamer	0:66f3b5499f7f	251	self.opts_report_build_file_name = _opts_report_build_file_name
screamer	22:9e85236d8716	252	self.opts_report_text_file_name = _opts_report_text_file_name
screamer	0:66f3b5499f7f	253	self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
screamer	0:66f3b5499f7f	254	self.opts_goanna_for_tests = _opts_goanna_for_tests
screamer	0:66f3b5499f7f	255	self.opts_shuffle_test_order = _opts_shuffle_test_order
screamer	0:66f3b5499f7f	256	self.opts_shuffle_test_seed = _opts_shuffle_test_seed
screamer	0:66f3b5499f7f	257	self.opts_test_by_names = _opts_test_by_names
screamer	0:66f3b5499f7f	258	self.opts_peripheral_by_names = _opts_peripheral_by_names
screamer	0:66f3b5499f7f	259	self.opts_test_only_peripheral = _opts_test_only_peripheral
screamer	0:66f3b5499f7f	260	self.opts_test_only_common = _opts_test_only_common
screamer	0:66f3b5499f7f	261	self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
screamer	0:66f3b5499f7f	262	self.opts_verbose_test_result_only = _opts_verbose_test_result_only
screamer	0:66f3b5499f7f	263	self.opts_verbose = _opts_verbose
screamer	0:66f3b5499f7f	264	self.opts_firmware_global_name = _opts_firmware_global_name
screamer	0:66f3b5499f7f	265	self.opts_only_build_tests = _opts_only_build_tests
screamer	0:66f3b5499f7f	266	self.opts_parallel_test_exec = _opts_parallel_test_exec
screamer	0:66f3b5499f7f	267	self.opts_suppress_summary = _opts_suppress_summary
screamer	0:66f3b5499f7f	268	self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
screamer	0:66f3b5499f7f	269	self.opts_copy_method = _opts_copy_method
screamer	0:66f3b5499f7f	270	self.opts_mut_reset_type = _opts_mut_reset_type
screamer	0:66f3b5499f7f	271	self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
screamer	0:66f3b5499f7f	272	self.opts_waterfall_test = _opts_waterfall_test
screamer	0:66f3b5499f7f	273	self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
screamer	0:66f3b5499f7f	274	self.opts_extend_test_timeout = _opts_extend_test_timeout
screamer	0:66f3b5499f7f	275	self.opts_clean = _clean
The Other Jimmy	31:8ea194f6145b	276	self.opts_parser = _parser
The Other Jimmy	31:8ea194f6145b	277	self.opts = _opts
screamer	0:66f3b5499f7f	278	self.opts_auto_detect = _opts_auto_detect
screamer	0:66f3b5499f7f	279	self.opts_include_non_automated = _opts_include_non_automated
screamer	0:66f3b5499f7f	280
screamer	0:66f3b5499f7f	281	self.build_report = _opts_build_report
screamer	0:66f3b5499f7f	282	self.build_properties = _opts_build_properties
screamer	0:66f3b5499f7f	283
screamer	0:66f3b5499f7f	284	# File / screen logger initialization
screamer	0:66f3b5499f7f	285	self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
screamer	0:66f3b5499f7f	286
screamer	0:66f3b5499f7f	287	# Database related initializations
screamer	0:66f3b5499f7f	288	self.db_logger = factory_db_logger(self.opts_db_url)
screamer	0:66f3b5499f7f	289	self.db_logger_build_id = None # Build ID (database index of build_id table)
screamer	0:66f3b5499f7f	290	# Let's connect to database to set up credentials and confirm database is ready
screamer	0:66f3b5499f7f	291	if self.db_logger:
screamer	0:66f3b5499f7f	292	self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
screamer	0:66f3b5499f7f	293	if self.db_logger.is_connected():
screamer	0:66f3b5499f7f	294	# Get hostname and uname so we can use it as build description
screamer	0:66f3b5499f7f	295	# when creating new build_id in external database
screamer	0:66f3b5499f7f	296	(_hostname, _uname) = self.db_logger.get_hostname()
screamer	0:66f3b5499f7f	297	_host_location = os.path.dirname(os.path.abspath(__file__))
screamer	0:66f3b5499f7f	298	build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
screamer	0:66f3b5499f7f	299	self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
screamer	0:66f3b5499f7f	300	self.db_logger.disconnect()
screamer	0:66f3b5499f7f	301
screamer	0:66f3b5499f7f	302	def dump_options(self):
screamer	0:66f3b5499f7f	303	""" Function returns data structure with common settings passed to SingelTestRunner
screamer	0:66f3b5499f7f	304	It can be used for example to fill _extra fields in database storing test suite single run data
screamer	0:66f3b5499f7f	305	Example:
screamer	0:66f3b5499f7f	306	data = self.dump_options()
screamer	0:66f3b5499f7f	307	or
screamer	0:66f3b5499f7f	308	data_str = json.dumps(self.dump_options())
screamer	0:66f3b5499f7f	309	"""
screamer	0:66f3b5499f7f	310	result = {"db_url" : str(self.opts_db_url),
screamer	0:66f3b5499f7f	311	"log_file_name" : str(self.opts_log_file_name),
screamer	0:66f3b5499f7f	312	"shuffle_test_order" : str(self.opts_shuffle_test_order),
screamer	0:66f3b5499f7f	313	"shuffle_test_seed" : str(self.opts_shuffle_test_seed),
screamer	0:66f3b5499f7f	314	"test_by_names" : str(self.opts_test_by_names),
screamer	0:66f3b5499f7f	315	"peripheral_by_names" : str(self.opts_peripheral_by_names),
screamer	0:66f3b5499f7f	316	"test_only_peripheral" : str(self.opts_test_only_peripheral),
screamer	0:66f3b5499f7f	317	"test_only_common" : str(self.opts_test_only_common),
screamer	0:66f3b5499f7f	318	"verbose" : str(self.opts_verbose),
screamer	0:66f3b5499f7f	319	"firmware_global_name" : str(self.opts_firmware_global_name),
screamer	0:66f3b5499f7f	320	"only_build_tests" : str(self.opts_only_build_tests),
screamer	0:66f3b5499f7f	321	"copy_method" : str(self.opts_copy_method),
screamer	0:66f3b5499f7f	322	"mut_reset_type" : str(self.opts_mut_reset_type),
screamer	0:66f3b5499f7f	323	"jobs" : str(self.opts_jobs),
screamer	0:66f3b5499f7f	324	"extend_test_timeout" : str(self.opts_extend_test_timeout),
screamer	0:66f3b5499f7f	325	"_dummy" : ''
screamer	0:66f3b5499f7f	326	}
screamer	0:66f3b5499f7f	327	return result
screamer	0:66f3b5499f7f	328
screamer	0:66f3b5499f7f	329	def shuffle_random_func(self):
screamer	0:66f3b5499f7f	330	return self.shuffle_random_seed
screamer	0:66f3b5499f7f	331
screamer	0:66f3b5499f7f	332	def is_shuffle_seed_float(self):
screamer	0:66f3b5499f7f	333	""" return true if function parameter can be converted to float
screamer	0:66f3b5499f7f	334	"""
screamer	0:66f3b5499f7f	335	result = True
screamer	0:66f3b5499f7f	336	try:
screamer	0:66f3b5499f7f	337	float(self.shuffle_random_seed)
screamer	0:66f3b5499f7f	338	except ValueError:
screamer	0:66f3b5499f7f	339	result = False
screamer	0:66f3b5499f7f	340	return result
screamer	0:66f3b5499f7f	341
screamer	0:66f3b5499f7f	342	# This will store target / toolchain specific properties
screamer	0:66f3b5499f7f	343	test_suite_properties_ext = {} # target : toolchain
screamer	0:66f3b5499f7f	344	# Here we store test results
screamer	0:66f3b5499f7f	345	test_summary = []
screamer	0:66f3b5499f7f	346	# Here we store test results in extended data structure
screamer	0:66f3b5499f7f	347	test_summary_ext = {}
screamer	0:66f3b5499f7f	348	execute_thread_slice_lock = Lock()
screamer	0:66f3b5499f7f	349
screamer	0:66f3b5499f7f	350	def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
screamer	0:66f3b5499f7f	351	for toolchain in toolchains:
screamer	0:66f3b5499f7f	352	tt_id = "%s::%s" % (toolchain, target)
screamer	0:66f3b5499f7f	353
screamer	0:66f3b5499f7f	354	T = TARGET_MAP[target]
screamer	0:66f3b5499f7f	355
screamer	0:66f3b5499f7f	356	# print target, toolchain
screamer	0:66f3b5499f7f	357	# Test suite properties returned to external tools like CI
screamer	0:66f3b5499f7f	358	test_suite_properties = {
screamer	0:66f3b5499f7f	359	'jobs': self.opts_jobs,
screamer	0:66f3b5499f7f	360	'clean': clean,
screamer	0:66f3b5499f7f	361	'target': target,
screamer	0:66f3b5499f7f	362	'vendor': T.extra_labels[0],
screamer	0:66f3b5499f7f	363	'test_ids': ', '.join(test_ids),
screamer	0:66f3b5499f7f	364	'toolchain': toolchain,
screamer	0:66f3b5499f7f	365	'shuffle_random_seed': self.shuffle_random_seed
screamer	0:66f3b5499f7f	366	}
screamer	0:66f3b5499f7f	367
screamer	0:66f3b5499f7f	368
screamer	0:66f3b5499f7f	369	# print '=== %s::%s ===' % (target, toolchain)
screamer	0:66f3b5499f7f	370	# Let's build our test
screamer	0:66f3b5499f7f	371	if target not in TARGET_MAP:
theotherjimmy	43:2a7da56ebd24	372	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	373	self.logger.LogType.NOTIF,
theotherjimmy	43:2a7da56ebd24	374	'Skipped tests for %s target. Target platform not found' %
theotherjimmy	43:2a7da56ebd24	375	(target)))
screamer	0:66f3b5499f7f	376	continue
screamer	0:66f3b5499f7f	377
theotherjimmy	43:2a7da56ebd24	378	clean_mbed_libs_options = (self.opts_goanna_for_mbed_sdk or
theotherjimmy	43:2a7da56ebd24	379	self.opts_clean or clean)
screamer	0:66f3b5499f7f	380
The Other Jimmy	31:8ea194f6145b	381	profile = extract_profile(self.opts_parser, self.opts, toolchain)
theotherjimmy	40:7d3fa6b99b2b	382	stats_depth = self.opts.stats_depth or 2
The Other Jimmy	31:8ea194f6145b	383
screamer	0:66f3b5499f7f	384	try:
theotherjimmy	43:2a7da56ebd24	385	build_mbed_libs_result = build_mbed_libs(
theotherjimmy	43:2a7da56ebd24	386	T, toolchain,
theotherjimmy	43:2a7da56ebd24	387	clean=clean_mbed_libs_options,
theotherjimmy	43:2a7da56ebd24	388	jobs=self.opts_jobs,
theotherjimmy	43:2a7da56ebd24	389	report=build_report,
theotherjimmy	43:2a7da56ebd24	390	properties=build_properties,
theotherjimmy	43:2a7da56ebd24	391	build_profile=profile,
theotherjimmy	43:2a7da56ebd24	392	notify=TerminalNotifier())
screamer	0:66f3b5499f7f	393
screamer	0:66f3b5499f7f	394	if not build_mbed_libs_result:
theotherjimmy	43:2a7da56ebd24	395	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	396	self.logger.LogType.NOTIF,
theotherjimmy	43:2a7da56ebd24	397	'Skipped tests for %s target. Toolchain %s is not '
theotherjimmy	43:2a7da56ebd24	398	'supported for this target'% (T.name, toolchain)))
screamer	0:66f3b5499f7f	399	continue
screamer	0:66f3b5499f7f	400
screamer	0:66f3b5499f7f	401	except ToolException:
theotherjimmy	43:2a7da56ebd24	402	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	403	self.logger.LogType.ERROR,
theotherjimmy	43:2a7da56ebd24	404	'There were errors while building MBED libs for %s using %s'
theotherjimmy	43:2a7da56ebd24	405	% (target, toolchain)))
screamer	0:66f3b5499f7f	406	continue
screamer	0:66f3b5499f7f	407
screamer	0:66f3b5499f7f	408	build_dir = join(BUILD_DIR, "test", target, toolchain)
screamer	0:66f3b5499f7f	409
screamer	0:66f3b5499f7f	410	test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
screamer	0:66f3b5499f7f	411	test_suite_properties['build_dir'] = build_dir
screamer	0:66f3b5499f7f	412	test_suite_properties['skipped'] = []
screamer	0:66f3b5499f7f	413
screamer	0:66f3b5499f7f	414	# Enumerate through all tests and shuffle test order if requested
screamer	0:66f3b5499f7f	415	test_map_keys = sorted(TEST_MAP.keys())
screamer	0:66f3b5499f7f	416
screamer	0:66f3b5499f7f	417	if self.opts_shuffle_test_order:
screamer	0:66f3b5499f7f	418	random.shuffle(test_map_keys, self.shuffle_random_func)
screamer	0:66f3b5499f7f	419	# Update database with shuffle seed f applicable
screamer	0:66f3b5499f7f	420	if self.db_logger:
screamer	0:66f3b5499f7f	421	self.db_logger.reconnect();
screamer	0:66f3b5499f7f	422	if self.db_logger.is_connected():
theotherjimmy	43:2a7da56ebd24	423	self.db_logger.update_build_id_info(
theotherjimmy	43:2a7da56ebd24	424	self.db_logger_build_id,
theotherjimmy	43:2a7da56ebd24	425	_shuffle_seed=self.shuffle_random_func())
screamer	0:66f3b5499f7f	426	self.db_logger.disconnect();
screamer	0:66f3b5499f7f	427
screamer	0:66f3b5499f7f	428	if self.db_logger:
screamer	0:66f3b5499f7f	429	self.db_logger.reconnect();
screamer	0:66f3b5499f7f	430	if self.db_logger.is_connected():
screamer	0:66f3b5499f7f	431	# Update MUTs and Test Specification in database
theotherjimmy	43:2a7da56ebd24	432	self.db_logger.update_build_id_info(
theotherjimmy	43:2a7da56ebd24	433	self.db_logger_build_id,
theotherjimmy	43:2a7da56ebd24	434	_muts=self.muts, _test_spec=self.test_spec)
screamer	0:66f3b5499f7f	435	# Update Extra information in database (some options passed to test suite)
theotherjimmy	43:2a7da56ebd24	436	self.db_logger.update_build_id_info(
theotherjimmy	43:2a7da56ebd24	437	self.db_logger_build_id,
theotherjimmy	43:2a7da56ebd24	438	_extra=json.dumps(self.dump_options()))
screamer	0:66f3b5499f7f	439	self.db_logger.disconnect();
screamer	0:66f3b5499f7f	440
screamer	0:66f3b5499f7f	441	valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
screamer	0:66f3b5499f7f	442	skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
screamer	0:66f3b5499f7f	443
screamer	0:66f3b5499f7f	444	for skipped_test_id in skipped_test_map_keys:
screamer	0:66f3b5499f7f	445	test_suite_properties['skipped'].append(skipped_test_id)
screamer	0:66f3b5499f7f	446
screamer	0:66f3b5499f7f	447
screamer	0:66f3b5499f7f	448	# First pass through all tests and determine which libraries need to be built
screamer	0:66f3b5499f7f	449	libraries = []
screamer	0:66f3b5499f7f	450	for test_id in valid_test_map_keys:
screamer	0:66f3b5499f7f	451	test = TEST_MAP[test_id]
screamer	0:66f3b5499f7f	452
screamer	0:66f3b5499f7f	453	# Detect which lib should be added to test
screamer	0:66f3b5499f7f	454	# Some libs have to compiled like RTOS or ETH
screamer	0:66f3b5499f7f	455	for lib in LIBRARIES:
screamer	0:66f3b5499f7f	456	if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
screamer	0:66f3b5499f7f	457	libraries.append(lib['id'])
screamer	0:66f3b5499f7f	458
screamer	0:66f3b5499f7f	459
screamer	0:66f3b5499f7f	460	clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
screamer	0:66f3b5499f7f	461
screamer	0:66f3b5499f7f	462	# Build all required libraries
screamer	0:66f3b5499f7f	463	for lib_id in libraries:
screamer	0:66f3b5499f7f	464	try:
screamer	0:66f3b5499f7f	465	build_lib(lib_id,
screamer	0:66f3b5499f7f	466	T,
screamer	0:66f3b5499f7f	467	toolchain,
screamer	0:66f3b5499f7f	468	clean=clean_mbed_libs_options,
screamer	0:66f3b5499f7f	469	jobs=self.opts_jobs,
screamer	0:66f3b5499f7f	470	report=build_report,
The Other Jimmy	31:8ea194f6145b	471	properties=build_properties,
theotherjimmy	43:2a7da56ebd24	472	build_profile=profile,
theotherjimmy	43:2a7da56ebd24	473	notify=TerminalNotifier())
screamer	0:66f3b5499f7f	474
screamer	0:66f3b5499f7f	475	except ToolException:
theotherjimmy	43:2a7da56ebd24	476	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	477	self.logger.LogType.ERROR,
theotherjimmy	43:2a7da56ebd24	478	'There were errors while building library %s' % lib_id))
screamer	0:66f3b5499f7f	479	continue
screamer	0:66f3b5499f7f	480
screamer	0:66f3b5499f7f	481
screamer	0:66f3b5499f7f	482	for test_id in valid_test_map_keys:
screamer	0:66f3b5499f7f	483	test = TEST_MAP[test_id]
screamer	0:66f3b5499f7f	484
screamer	0:66f3b5499f7f	485	test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
screamer	0:66f3b5499f7f	486
screamer	0:66f3b5499f7f	487	# TODO: move this 2 below loops to separate function
screamer	0:66f3b5499f7f	488	INC_DIRS = []
screamer	0:66f3b5499f7f	489	for lib_id in libraries:
screamer	0:66f3b5499f7f	490	if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
screamer	0:66f3b5499f7f	491	INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
screamer	0:66f3b5499f7f	492
screamer	0:66f3b5499f7f	493	MACROS = []
screamer	0:66f3b5499f7f	494	for lib_id in libraries:
screamer	0:66f3b5499f7f	495	if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
screamer	0:66f3b5499f7f	496	MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
screamer	0:66f3b5499f7f	497	MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
screamer	0:66f3b5499f7f	498	MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
screamer	0:66f3b5499f7f	499	test_uuid = uuid.uuid4()
screamer	0:66f3b5499f7f	500	MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
screamer	0:66f3b5499f7f	501
screamer	0:66f3b5499f7f	502	# Prepare extended test results data structure (it can be used to generate detailed test report)
screamer	0:66f3b5499f7f	503	if target not in self.test_summary_ext:
screamer	0:66f3b5499f7f	504	self.test_summary_ext[target] = {} # test_summary_ext : toolchain
screamer	0:66f3b5499f7f	505	if toolchain not in self.test_summary_ext[target]:
screamer	0:66f3b5499f7f	506	self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
screamer	0:66f3b5499f7f	507
screamer	0:66f3b5499f7f	508	tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
screamer	0:66f3b5499f7f	509
screamer	0:66f3b5499f7f	510	project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
screamer	0:66f3b5499f7f	511	try:
theotherjimmy	43:2a7da56ebd24	512	path = build_project(
theotherjimmy	43:2a7da56ebd24	513	test.source_dir, join(build_dir, test_id), T,
theotherjimmy	40:7d3fa6b99b2b	514	toolchain, test.dependencies, clean=clean_project_options,
theotherjimmy	43:2a7da56ebd24	515	name=project_name, macros=MACROS,
theotherjimmy	40:7d3fa6b99b2b	516	inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
theotherjimmy	40:7d3fa6b99b2b	517	properties=build_properties, project_id=test_id,
theotherjimmy	40:7d3fa6b99b2b	518	project_description=test.get_description(),
theotherjimmy	43:2a7da56ebd24	519	build_profile=profile, stats_depth=stats_depth,
theotherjimmy	43:2a7da56ebd24	520	notify=TerminalNotifier(),
theotherjimmy	43:2a7da56ebd24	521	)
screamer	0:66f3b5499f7f	522
theotherjimmy	43:2a7da56ebd24	523	except Exception as e:
screamer	0:66f3b5499f7f	524	project_name_str = project_name if project_name is not None else test_id
screamer	0:66f3b5499f7f	525
screamer	0:66f3b5499f7f	526
screamer	0:66f3b5499f7f	527	test_result = self.TEST_RESULT_FAIL
screamer	0:66f3b5499f7f	528
screamer	0:66f3b5499f7f	529	if isinstance(e, ToolException):
theotherjimmy	43:2a7da56ebd24	530	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	531	self.logger.LogType.ERROR,
theotherjimmy	43:2a7da56ebd24	532	'There were errors while building project %s' %
theotherjimmy	43:2a7da56ebd24	533	project_name_str))
screamer	0:66f3b5499f7f	534	test_result = self.TEST_RESULT_BUILD_FAILED
screamer	0:66f3b5499f7f	535	elif isinstance(e, NotSupportedException):
theotherjimmy	43:2a7da56ebd24	536	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	537	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	538	'Project %s is not supported' % project_name_str))
screamer	0:66f3b5499f7f	539	test_result = self.TEST_RESULT_NOT_SUPPORTED
screamer	0:66f3b5499f7f	540
screamer	0:66f3b5499f7f	541
screamer	0:66f3b5499f7f	542	# Append test results to global test summary
screamer	0:66f3b5499f7f	543	self.test_summary.append(
theotherjimmy	43:2a7da56ebd24	544	(test_result, target, toolchain, test_id,
theotherjimmy	43:2a7da56ebd24	545	test.get_description(), 0, 0, '-')
screamer	0:66f3b5499f7f	546	)
screamer	0:66f3b5499f7f	547
screamer	0:66f3b5499f7f	548	# Add detailed test result to test summary structure
screamer	0:66f3b5499f7f	549	if test_id not in self.test_summary_ext[target][toolchain]:
screamer	0:66f3b5499f7f	550	self.test_summary_ext[target][toolchain][test_id] = []
screamer	0:66f3b5499f7f	551
screamer	0:66f3b5499f7f	552	self.test_summary_ext[target][toolchain][test_id].append({ 0: {
screamer	0:66f3b5499f7f	553	'result' : test_result,
screamer	0:66f3b5499f7f	554	'output' : '',
screamer	0:66f3b5499f7f	555	'target_name' : target,
screamer	0:66f3b5499f7f	556	'target_name_unique': target,
screamer	0:66f3b5499f7f	557	'toolchain_name' : toolchain,
screamer	0:66f3b5499f7f	558	'id' : test_id,
screamer	0:66f3b5499f7f	559	'description' : test.get_description(),
screamer	0:66f3b5499f7f	560	'elapsed_time' : 0,
screamer	0:66f3b5499f7f	561	'duration' : 0,
screamer	0:66f3b5499f7f	562	'copy_method' : None
screamer	0:66f3b5499f7f	563	}})
screamer	0:66f3b5499f7f	564	continue
screamer	0:66f3b5499f7f	565
screamer	0:66f3b5499f7f	566	if self.opts_only_build_tests:
screamer	0:66f3b5499f7f	567	# With this option we are skipping testing phase
screamer	0:66f3b5499f7f	568	continue
screamer	0:66f3b5499f7f	569
screamer	0:66f3b5499f7f	570	# Test duration can be increased by global value
screamer	0:66f3b5499f7f	571	test_duration = test.duration
screamer	0:66f3b5499f7f	572	if self.opts_extend_test_timeout is not None:
screamer	0:66f3b5499f7f	573	test_duration += self.opts_extend_test_timeout
screamer	0:66f3b5499f7f	574
screamer	0:66f3b5499f7f	575	# For an automated test the duration act as a timeout after
screamer	0:66f3b5499f7f	576	# which the test gets interrupted
screamer	0:66f3b5499f7f	577	test_spec = self.shape_test_request(target, path, test_id, test_duration)
screamer	0:66f3b5499f7f	578	test_loops = self.get_test_loop_count(test_id)
screamer	0:66f3b5499f7f	579
screamer	0:66f3b5499f7f	580	test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
screamer	0:66f3b5499f7f	581	test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
screamer	0:66f3b5499f7f	582	test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
screamer	0:66f3b5499f7f	583
screamer	0:66f3b5499f7f	584	# read MUTs, test specification and perform tests
screamer	0:66f3b5499f7f	585	handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
screamer	0:66f3b5499f7f	586
screamer	0:66f3b5499f7f	587	if handle_results is None:
screamer	0:66f3b5499f7f	588	continue
screamer	0:66f3b5499f7f	589
screamer	0:66f3b5499f7f	590	for handle_result in handle_results:
screamer	0:66f3b5499f7f	591	if handle_result:
screamer	0:66f3b5499f7f	592	single_test_result, detailed_test_results = handle_result
screamer	0:66f3b5499f7f	593	else:
screamer	0:66f3b5499f7f	594	continue
screamer	0:66f3b5499f7f	595
screamer	0:66f3b5499f7f	596	# Append test results to global test summary
screamer	0:66f3b5499f7f	597	if single_test_result is not None:
screamer	0:66f3b5499f7f	598	self.test_summary.append(single_test_result)
screamer	0:66f3b5499f7f	599
screamer	0:66f3b5499f7f	600	# Add detailed test result to test summary structure
screamer	0:66f3b5499f7f	601	if target not in self.test_summary_ext[target][toolchain]:
screamer	0:66f3b5499f7f	602	if test_id not in self.test_summary_ext[target][toolchain]:
screamer	0:66f3b5499f7f	603	self.test_summary_ext[target][toolchain][test_id] = []
screamer	0:66f3b5499f7f	604
screamer	0:66f3b5499f7f	605	append_test_result = detailed_test_results
screamer	0:66f3b5499f7f	606
screamer	0:66f3b5499f7f	607	# If waterfall and consolidate-waterfall options are enabled,
screamer	0:66f3b5499f7f	608	# only include the last test result in the report.
screamer	0:66f3b5499f7f	609	if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
screamer	0:66f3b5499f7f	610	append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
screamer	0:66f3b5499f7f	611
screamer	0:66f3b5499f7f	612	self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
screamer	0:66f3b5499f7f	613
screamer	0:66f3b5499f7f	614	test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
screamer	0:66f3b5499f7f	615	self.test_suite_properties_ext[target][toolchain] = test_suite_properties
screamer	0:66f3b5499f7f	616
screamer	0:66f3b5499f7f	617	q.put(target + '_'.join(toolchains))
screamer	0:66f3b5499f7f	618	return
screamer	0:66f3b5499f7f	619
screamer	0:66f3b5499f7f	620	def execute(self):
screamer	0:66f3b5499f7f	621	clean = self.test_spec.get('clean', False)
screamer	0:66f3b5499f7f	622	test_ids = self.test_spec.get('test_ids', [])
screamer	0:66f3b5499f7f	623	q = Queue()
screamer	0:66f3b5499f7f	624
screamer	0:66f3b5499f7f	625	# Generate seed for shuffle if seed is not provided in
screamer	0:66f3b5499f7f	626	self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
screamer	0:66f3b5499f7f	627	if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
screamer	0:66f3b5499f7f	628	self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
screamer	0:66f3b5499f7f	629
screamer	0:66f3b5499f7f	630
screamer	0:66f3b5499f7f	631	if self.opts_parallel_test_exec:
screamer	0:66f3b5499f7f	632	###################################################################
screamer	0:66f3b5499f7f	633	# Experimental, parallel test execution per singletest instance.
screamer	0:66f3b5499f7f	634	###################################################################
screamer	0:66f3b5499f7f	635	execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
screamer	0:66f3b5499f7f	636	# Note: We are building here in parallel for each target separately!
screamer	0:66f3b5499f7f	637	# So we are not building the same thing multiple times and compilers
screamer	0:66f3b5499f7f	638	# in separate threads do not collide.
screamer	0:66f3b5499f7f	639	# Inside execute_thread_slice() function function handle() will be called to
screamer	0:66f3b5499f7f	640	# get information about available MUTs (per target).
theotherjimmy	43:2a7da56ebd24	641	for target, toolchains in self.test_spec['targets'].items():
screamer	0:66f3b5499f7f	642	self.test_suite_properties_ext[target] = {}
screamer	0:66f3b5499f7f	643	t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
screamer	0:66f3b5499f7f	644	t.daemon = True
screamer	0:66f3b5499f7f	645	t.start()
screamer	0:66f3b5499f7f	646	execute_threads.append(t)
screamer	0:66f3b5499f7f	647
screamer	0:66f3b5499f7f	648	for t in execute_threads:
screamer	0:66f3b5499f7f	649	q.get() # t.join() would block some threads because we should not wait in any order for thread end
screamer	0:66f3b5499f7f	650	else:
screamer	0:66f3b5499f7f	651	# Serialized (not parallel) test execution
theotherjimmy	43:2a7da56ebd24	652	for target, toolchains in self.test_spec['targets'].items():
screamer	0:66f3b5499f7f	653	if target not in self.test_suite_properties_ext:
screamer	0:66f3b5499f7f	654	self.test_suite_properties_ext[target] = {}
screamer	0:66f3b5499f7f	655
screamer	0:66f3b5499f7f	656	self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
screamer	0:66f3b5499f7f	657	q.get()
screamer	0:66f3b5499f7f	658
screamer	0:66f3b5499f7f	659	if self.db_logger:
screamer	0:66f3b5499f7f	660	self.db_logger.reconnect();
screamer	0:66f3b5499f7f	661	if self.db_logger.is_connected():
screamer	0:66f3b5499f7f	662	self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
screamer	0:66f3b5499f7f	663	self.db_logger.disconnect();
screamer	0:66f3b5499f7f	664
screamer	0:66f3b5499f7f	665	return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
screamer	0:66f3b5499f7f	666
screamer	0:66f3b5499f7f	667	def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
screamer	0:66f3b5499f7f	668	valid_test_map_keys = []
screamer	0:66f3b5499f7f	669
screamer	0:66f3b5499f7f	670	for test_id in test_map_keys:
screamer	0:66f3b5499f7f	671	test = TEST_MAP[test_id]
screamer	22:9e85236d8716	672	if self.opts_test_by_names and test_id not in self.opts_test_by_names:
screamer	0:66f3b5499f7f	673	continue
screamer	0:66f3b5499f7f	674
screamer	0:66f3b5499f7f	675	if test_ids and test_id not in test_ids:
screamer	0:66f3b5499f7f	676	continue
screamer	0:66f3b5499f7f	677
screamer	0:66f3b5499f7f	678	if self.opts_test_only_peripheral and not test.peripherals:
screamer	0:66f3b5499f7f	679	if self.opts_verbose_skipped_tests:
theotherjimmy	43:2a7da56ebd24	680	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	681	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	682	'Common test skipped for target %s' % target))
screamer	0:66f3b5499f7f	683	continue
screamer	0:66f3b5499f7f	684
theotherjimmy	43:2a7da56ebd24	685	if (self.opts_peripheral_by_names and test.peripherals and
theotherjimmy	43:2a7da56ebd24	686	not any((i in self.opts_peripheral_by_names)
theotherjimmy	43:2a7da56ebd24	687	for i in test.peripherals)):
screamer	0:66f3b5499f7f	688	# We will skip tests not forced with -p option
screamer	0:66f3b5499f7f	689	if self.opts_verbose_skipped_tests:
theotherjimmy	43:2a7da56ebd24	690	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	691	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	692	'Common test skipped for target %s' % target))
screamer	0:66f3b5499f7f	693	continue
screamer	0:66f3b5499f7f	694
screamer	0:66f3b5499f7f	695	if self.opts_test_only_common and test.peripherals:
screamer	0:66f3b5499f7f	696	if self.opts_verbose_skipped_tests:
theotherjimmy	43:2a7da56ebd24	697	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	698	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	699	'Peripheral test skipped for target %s' % target))
screamer	0:66f3b5499f7f	700	continue
screamer	0:66f3b5499f7f	701
screamer	0:66f3b5499f7f	702	if not include_non_automated and not test.automated:
screamer	0:66f3b5499f7f	703	if self.opts_verbose_skipped_tests:
theotherjimmy	43:2a7da56ebd24	704	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	705	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	706	'Non automated test skipped for target %s' % target))
screamer	0:66f3b5499f7f	707	continue
screamer	0:66f3b5499f7f	708
screamer	0:66f3b5499f7f	709	if test.is_supported(target, toolchain):
screamer	0:66f3b5499f7f	710	if test.peripherals is None and self.opts_only_build_tests:
screamer	0:66f3b5499f7f	711	# When users are using 'build only flag' and test do not have
screamer	0:66f3b5499f7f	712	# specified peripherals we can allow test building by default
screamer	0:66f3b5499f7f	713	pass
screamer	22:9e85236d8716	714	elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
screamer	0:66f3b5499f7f	715	# If we force peripheral with option -p we expect test
screamer	0:66f3b5499f7f	716	# to pass even if peripheral is not in MUTs file.
screamer	0:66f3b5499f7f	717	pass
screamer	0:66f3b5499f7f	718	elif not self.is_peripherals_available(target, test.peripherals):
screamer	0:66f3b5499f7f	719	if self.opts_verbose_skipped_tests:
screamer	0:66f3b5499f7f	720	if test.peripherals:
theotherjimmy	43:2a7da56ebd24	721	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	722	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	723	'Peripheral %s test skipped for target %s' %
theotherjimmy	43:2a7da56ebd24	724	(",".join(test.peripherals), target)))
screamer	0:66f3b5499f7f	725	else:
theotherjimmy	43:2a7da56ebd24	726	print(self.logger.log_line(
theotherjimmy	43:2a7da56ebd24	727	self.logger.LogType.INFO,
theotherjimmy	43:2a7da56ebd24	728	'Test %s skipped for target %s' %
theotherjimmy	43:2a7da56ebd24	729	(test_id, target)))
screamer	0:66f3b5499f7f	730	continue
screamer	0:66f3b5499f7f	731
screamer	0:66f3b5499f7f	732	# The test has made it through all the filters, so add it to the valid tests list
screamer	0:66f3b5499f7f	733	valid_test_map_keys.append(test_id)
screamer	0:66f3b5499f7f	734
screamer	0:66f3b5499f7f	735	return valid_test_map_keys
screamer	0:66f3b5499f7f	736
screamer	0:66f3b5499f7f	737	def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
screamer	0:66f3b5499f7f	738	# NOTE: This will not preserve order
screamer	0:66f3b5499f7f	739	return list(set(all_test_map_keys) - set(valid_test_map_keys))
screamer	0:66f3b5499f7f	740
screamer	0:66f3b5499f7f	741	def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
screamer	0:66f3b5499f7f	742	""" Prints well-formed summary with results (SQL table like)
screamer	0:66f3b5499f7f	743	table shows text x toolchain test result matrix
screamer	0:66f3b5499f7f	744	"""
screamer	0:66f3b5499f7f	745	RESULT_INDEX = 0
screamer	0:66f3b5499f7f	746	TARGET_INDEX = 1
screamer	0:66f3b5499f7f	747	TOOLCHAIN_INDEX = 2
screamer	0:66f3b5499f7f	748	TEST_INDEX = 3
screamer	0:66f3b5499f7f	749	DESC_INDEX = 4
screamer	0:66f3b5499f7f	750
screamer	0:66f3b5499f7f	751	unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
screamer	0:66f3b5499f7f	752	unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
screamer	0:66f3b5499f7f	753	unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
screamer	0:66f3b5499f7f	754	unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
screamer	0:66f3b5499f7f	755
screamer	0:66f3b5499f7f	756	result = "Test summary:\n"
screamer	0:66f3b5499f7f	757	for target in unique_targets:
screamer	0:66f3b5499f7f	758	result_dict = {} # test : { toolchain : result }
screamer	0:66f3b5499f7f	759	unique_target_toolchains = []
screamer	0:66f3b5499f7f	760	for test in test_summary:
screamer	0:66f3b5499f7f	761	if test[TARGET_INDEX] == target:
screamer	0:66f3b5499f7f	762	if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
screamer	0:66f3b5499f7f	763	unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
screamer	0:66f3b5499f7f	764	if test[TEST_INDEX] not in result_dict:
screamer	0:66f3b5499f7f	765	result_dict[test[TEST_INDEX]] = {}
screamer	0:66f3b5499f7f	766	result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
screamer	0:66f3b5499f7f	767
screamer	0:66f3b5499f7f	768	pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
theotherjimmy	43:2a7da56ebd24	769	pt = PrettyTable(pt_cols, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	770	for col in pt_cols:
screamer	0:66f3b5499f7f	771	pt.align[col] = "l"
screamer	0:66f3b5499f7f	772	pt.padding_width = 1 # One space between column edges and contents (default)
screamer	0:66f3b5499f7f	773
screamer	0:66f3b5499f7f	774	for test in unique_tests:
screamer	0:66f3b5499f7f	775	if test in result_dict:
screamer	0:66f3b5499f7f	776	test_results = result_dict[test]
screamer	0:66f3b5499f7f	777	if test in unique_test_desc:
screamer	0:66f3b5499f7f	778	row = [target, test, unique_test_desc[test]]
screamer	0:66f3b5499f7f	779	for toolchain in unique_toolchains:
screamer	0:66f3b5499f7f	780	if toolchain in test_results:
screamer	0:66f3b5499f7f	781	row.append(test_results[toolchain])
screamer	0:66f3b5499f7f	782	pt.add_row(row)
screamer	0:66f3b5499f7f	783	result += pt.get_string()
screamer	0:66f3b5499f7f	784	shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
screamer	0:66f3b5499f7f	785	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer	0:66f3b5499f7f	786	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer	0:66f3b5499f7f	787	return result
screamer	0:66f3b5499f7f	788
screamer	0:66f3b5499f7f	789	def generate_test_summary(self, test_summary, shuffle_seed=None):
screamer	0:66f3b5499f7f	790	""" Prints well-formed summary with results (SQL table like)
screamer	0:66f3b5499f7f	791	table shows target x test results matrix across
screamer	0:66f3b5499f7f	792	"""
screamer	0:66f3b5499f7f	793	success_code = 0 # Success code that can be leter returned to
screamer	0:66f3b5499f7f	794	result = "Test summary:\n"
screamer	0:66f3b5499f7f	795	# Pretty table package is used to print results
screamer	0:66f3b5499f7f	796	pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
theotherjimmy	43:2a7da56ebd24	797	"Elapsed Time (sec)", "Timeout (sec)", "Loops"], junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	798	pt.align["Result"] = "l" # Left align
screamer	0:66f3b5499f7f	799	pt.align["Target"] = "l" # Left align
screamer	0:66f3b5499f7f	800	pt.align["Toolchain"] = "l" # Left align
screamer	0:66f3b5499f7f	801	pt.align["Test ID"] = "l" # Left align
screamer	0:66f3b5499f7f	802	pt.align["Test Description"] = "l" # Left align
screamer	0:66f3b5499f7f	803	pt.padding_width = 1 # One space between column edges and contents (default)
screamer	0:66f3b5499f7f	804
screamer	0:66f3b5499f7f	805	result_dict = {self.TEST_RESULT_OK : 0,
screamer	0:66f3b5499f7f	806	self.TEST_RESULT_FAIL : 0,
screamer	0:66f3b5499f7f	807	self.TEST_RESULT_ERROR : 0,
screamer	0:66f3b5499f7f	808	self.TEST_RESULT_UNDEF : 0,
screamer	0:66f3b5499f7f	809	self.TEST_RESULT_IOERR_COPY : 0,
screamer	0:66f3b5499f7f	810	self.TEST_RESULT_IOERR_DISK : 0,
screamer	0:66f3b5499f7f	811	self.TEST_RESULT_IOERR_SERIAL : 0,
screamer	0:66f3b5499f7f	812	self.TEST_RESULT_NO_IMAGE : 0,
screamer	0:66f3b5499f7f	813	self.TEST_RESULT_TIMEOUT : 0,
screamer	0:66f3b5499f7f	814	self.TEST_RESULT_MBED_ASSERT : 0,
screamer	0:66f3b5499f7f	815	self.TEST_RESULT_BUILD_FAILED : 0,
screamer	0:66f3b5499f7f	816	self.TEST_RESULT_NOT_SUPPORTED : 0
screamer	0:66f3b5499f7f	817	}
screamer	0:66f3b5499f7f	818
screamer	0:66f3b5499f7f	819	for test in test_summary:
screamer	0:66f3b5499f7f	820	if test[0] in result_dict:
screamer	0:66f3b5499f7f	821	result_dict[test[0]] += 1
screamer	0:66f3b5499f7f	822	pt.add_row(test)
screamer	0:66f3b5499f7f	823	result += pt.get_string()
screamer	0:66f3b5499f7f	824	result += "\n"
screamer	0:66f3b5499f7f	825
screamer	0:66f3b5499f7f	826	# Print result count
theotherjimmy	43:2a7da56ebd24	827	result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()])
screamer	0:66f3b5499f7f	828	shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
screamer	0:66f3b5499f7f	829	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
screamer	0:66f3b5499f7f	830	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
screamer	0:66f3b5499f7f	831	return result
screamer	0:66f3b5499f7f	832
screamer	0:66f3b5499f7f	833	def test_loop_list_to_dict(self, test_loops_str):
screamer	0:66f3b5499f7f	834	""" Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
screamer	0:66f3b5499f7f	835	"""
screamer	0:66f3b5499f7f	836	result = {}
screamer	0:66f3b5499f7f	837	if test_loops_str:
screamer	22:9e85236d8716	838	test_loops = test_loops_str
screamer	0:66f3b5499f7f	839	for test_loop in test_loops:
screamer	0:66f3b5499f7f	840	test_loop_count = test_loop.split('=')
screamer	0:66f3b5499f7f	841	if len(test_loop_count) == 2:
screamer	0:66f3b5499f7f	842	_test_id, _test_loops = test_loop_count
screamer	0:66f3b5499f7f	843	try:
screamer	0:66f3b5499f7f	844	_test_loops = int(_test_loops)
screamer	0:66f3b5499f7f	845	except:
screamer	0:66f3b5499f7f	846	continue
screamer	0:66f3b5499f7f	847	result[_test_id] = _test_loops
screamer	0:66f3b5499f7f	848	return result
screamer	0:66f3b5499f7f	849
screamer	0:66f3b5499f7f	850	def get_test_loop_count(self, test_id):
screamer	0:66f3b5499f7f	851	""" This function returns no. of loops per test (deducted by test_id_.
screamer	0:66f3b5499f7f	852	If test is not in list of redefined loop counts it will use default value.
screamer	0:66f3b5499f7f	853	"""
screamer	0:66f3b5499f7f	854	result = self.GLOBAL_LOOPS_COUNT
screamer	0:66f3b5499f7f	855	if test_id in self.TEST_LOOPS_DICT:
screamer	0:66f3b5499f7f	856	result = self.TEST_LOOPS_DICT[test_id]
screamer	0:66f3b5499f7f	857	return result
screamer	0:66f3b5499f7f	858
screamer	0:66f3b5499f7f	859	def delete_file(self, file_path):
screamer	0:66f3b5499f7f	860	""" Remove file from the system
screamer	0:66f3b5499f7f	861	"""
screamer	0:66f3b5499f7f	862	result = True
screamer	0:66f3b5499f7f	863	resutl_msg = ""
screamer	0:66f3b5499f7f	864	try:
screamer	0:66f3b5499f7f	865	os.remove(file_path)
theotherjimmy	43:2a7da56ebd24	866	except Exception as e:
screamer	0:66f3b5499f7f	867	resutl_msg = e
screamer	0:66f3b5499f7f	868	result = False
screamer	0:66f3b5499f7f	869	return result, resutl_msg
screamer	0:66f3b5499f7f	870
screamer	0:66f3b5499f7f	871	def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
screamer	0:66f3b5499f7f	872	""" Test is being invoked for given MUT.
screamer	0:66f3b5499f7f	873	"""
screamer	0:66f3b5499f7f	874	# Get test information, image and test timeout
screamer	0:66f3b5499f7f	875	test_id = data['test_id']
screamer	0:66f3b5499f7f	876	test = TEST_MAP[test_id]
screamer	0:66f3b5499f7f	877	test_description = TEST_MAP[test_id].get_description()
screamer	0:66f3b5499f7f	878	image = data["image"]
screamer	0:66f3b5499f7f	879	duration = data.get("duration", 10)
screamer	0:66f3b5499f7f	880
screamer	0:66f3b5499f7f	881	if mut is None:
theotherjimmy	43:2a7da56ebd24	882	print("Error: No Mbed available: MUT[%s]" % data['mcu'])
screamer	0:66f3b5499f7f	883	return None
screamer	0:66f3b5499f7f	884
screamer	0:66f3b5499f7f	885	mcu = mut['mcu']
screamer	0:66f3b5499f7f	886	copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
screamer	0:66f3b5499f7f	887
screamer	0:66f3b5499f7f	888	if self.db_logger:
screamer	0:66f3b5499f7f	889	self.db_logger.reconnect()
screamer	0:66f3b5499f7f	890
screamer	0:66f3b5499f7f	891	selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
screamer	0:66f3b5499f7f	892
screamer	0:66f3b5499f7f	893	# Tests can be looped so test results must be stored for the same test
screamer	0:66f3b5499f7f	894	test_all_result = []
screamer	0:66f3b5499f7f	895	# Test results for one test ran few times
screamer	0:66f3b5499f7f	896	detailed_test_results = {} # { Loop_number: { results ... } }
screamer	0:66f3b5499f7f	897
screamer	0:66f3b5499f7f	898	for test_index in range(test_loops):
screamer	0:66f3b5499f7f	899
screamer	0:66f3b5499f7f	900	# If mbedls is available and we are auto detecting MUT info,
screamer	0:66f3b5499f7f	901	# update MUT info (mounting may changed)
screamer	0:66f3b5499f7f	902	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer	0:66f3b5499f7f	903	platform_name_filter = [mcu]
screamer	0:66f3b5499f7f	904	muts_list = {}
screamer	0:66f3b5499f7f	905	found = False
screamer	0:66f3b5499f7f	906
screamer	0:66f3b5499f7f	907	for i in range(0, 60):
screamer	0:66f3b5499f7f	908	print('Looking for %s with MBEDLS' % mcu)
screamer	0:66f3b5499f7f	909	muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
screamer	0:66f3b5499f7f	910
screamer	0:66f3b5499f7f	911	if 1 not in muts_list:
screamer	0:66f3b5499f7f	912	sleep(3)
screamer	0:66f3b5499f7f	913	else:
screamer	0:66f3b5499f7f	914	found = True
screamer	0:66f3b5499f7f	915	break
screamer	0:66f3b5499f7f	916
screamer	0:66f3b5499f7f	917	if not found:
theotherjimmy	43:2a7da56ebd24	918	print("Error: mbed not found with MBEDLS: %s" % data['mcu'])
screamer	0:66f3b5499f7f	919	return None
screamer	0:66f3b5499f7f	920	else:
screamer	0:66f3b5499f7f	921	mut = muts_list[1]
screamer	0:66f3b5499f7f	922
screamer	0:66f3b5499f7f	923	disk = mut.get('disk')
screamer	0:66f3b5499f7f	924	port = mut.get('port')
screamer	0:66f3b5499f7f	925
screamer	0:66f3b5499f7f	926	if disk is None or port is None:
screamer	0:66f3b5499f7f	927	return None
screamer	0:66f3b5499f7f	928
screamer	0:66f3b5499f7f	929	target_by_mcu = TARGET_MAP[mut['mcu']]
screamer	0:66f3b5499f7f	930	target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
screamer	0:66f3b5499f7f	931	# Some extra stuff can be declared in MUTs structure
screamer	0:66f3b5499f7f	932	reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
screamer	0:66f3b5499f7f	933	reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
screamer	0:66f3b5499f7f	934
screamer	0:66f3b5499f7f	935	# When the build and test system were separate, this was relative to a
screamer	0:66f3b5499f7f	936	# base network folder base path: join(NETWORK_BASE_PATH, )
screamer	0:66f3b5499f7f	937	image_path = image
screamer	0:66f3b5499f7f	938
screamer	0:66f3b5499f7f	939	# Host test execution
screamer	0:66f3b5499f7f	940	start_host_exec_time = time()
screamer	0:66f3b5499f7f	941
screamer	0:66f3b5499f7f	942	single_test_result = self.TEST_RESULT_UNDEF # single test run result
screamer	0:66f3b5499f7f	943	_copy_method = selected_copy_method
screamer	0:66f3b5499f7f	944
screamer	0:66f3b5499f7f	945	if not exists(image_path):
screamer	0:66f3b5499f7f	946	single_test_result = self.TEST_RESULT_NO_IMAGE
screamer	0:66f3b5499f7f	947	elapsed_time = 0
screamer	0:66f3b5499f7f	948	single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
theotherjimmy	43:2a7da56ebd24	949	print(single_test_output)
screamer	0:66f3b5499f7f	950	else:
screamer	0:66f3b5499f7f	951	# Host test execution
screamer	0:66f3b5499f7f	952	start_host_exec_time = time()
screamer	0:66f3b5499f7f	953
screamer	0:66f3b5499f7f	954	host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
screamer	0:66f3b5499f7f	955	host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
screamer	0:66f3b5499f7f	956	host_test_result = self.run_host_test(test.host_test,
screamer	0:66f3b5499f7f	957	image_path, disk, port, duration,
screamer	0:66f3b5499f7f	958	micro=target_name,
screamer	0:66f3b5499f7f	959	verbose=host_test_verbose,
screamer	0:66f3b5499f7f	960	reset=host_test_reset,
screamer	0:66f3b5499f7f	961	reset_tout=reset_tout,
screamer	0:66f3b5499f7f	962	copy_method=selected_copy_method,
screamer	13:ab47a20b66f0	963	program_cycle_s=target_by_mcu.program_cycle_s)
screamer	0:66f3b5499f7f	964	single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
screamer	0:66f3b5499f7f	965
screamer	0:66f3b5499f7f	966	# Store test result
screamer	0:66f3b5499f7f	967	test_all_result.append(single_test_result)
screamer	0:66f3b5499f7f	968	total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
screamer	0:66f3b5499f7f	969	elapsed_time = single_testduration # TIme of single test case execution after reset
screamer	0:66f3b5499f7f	970
screamer	0:66f3b5499f7f	971	detailed_test_results[test_index] = {
screamer	0:66f3b5499f7f	972	'result' : single_test_result,
screamer	0:66f3b5499f7f	973	'output' : single_test_output,
screamer	0:66f3b5499f7f	974	'target_name' : target_name,
screamer	0:66f3b5499f7f	975	'target_name_unique' : target_name_unique,
screamer	0:66f3b5499f7f	976	'toolchain_name' : toolchain_name,
screamer	0:66f3b5499f7f	977	'id' : test_id,
screamer	0:66f3b5499f7f	978	'description' : test_description,
screamer	0:66f3b5499f7f	979	'elapsed_time' : round(elapsed_time, 2),
screamer	0:66f3b5499f7f	980	'duration' : single_timeout,
screamer	0:66f3b5499f7f	981	'copy_method' : _copy_method,
screamer	0:66f3b5499f7f	982	}
screamer	0:66f3b5499f7f	983
theotherjimmy	43:2a7da56ebd24	984	print(self.print_test_result(
theotherjimmy	43:2a7da56ebd24	985	single_test_result, target_name_unique, toolchain_name, test_id,
theotherjimmy	43:2a7da56ebd24	986	test_description, elapsed_time, single_timeout))
screamer	0:66f3b5499f7f	987
screamer	0:66f3b5499f7f	988	# Update database entries for ongoing test
screamer	0:66f3b5499f7f	989	if self.db_logger and self.db_logger.is_connected():
screamer	0:66f3b5499f7f	990	test_type = 'SingleTest'
screamer	0:66f3b5499f7f	991	self.db_logger.insert_test_entry(self.db_logger_build_id,
screamer	0:66f3b5499f7f	992	target_name,
screamer	0:66f3b5499f7f	993	toolchain_name,
screamer	0:66f3b5499f7f	994	test_type,
screamer	0:66f3b5499f7f	995	test_id,
screamer	0:66f3b5499f7f	996	single_test_result,
screamer	0:66f3b5499f7f	997	single_test_output,
screamer	0:66f3b5499f7f	998	elapsed_time,
screamer	0:66f3b5499f7f	999	single_timeout,
screamer	0:66f3b5499f7f	1000	test_index)
screamer	0:66f3b5499f7f	1001
screamer	0:66f3b5499f7f	1002	# If we perform waterfall test we test until we get OK and we stop testing
screamer	0:66f3b5499f7f	1003	if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
screamer	0:66f3b5499f7f	1004	break
screamer	0:66f3b5499f7f	1005
screamer	0:66f3b5499f7f	1006	if self.db_logger:
screamer	0:66f3b5499f7f	1007	self.db_logger.disconnect()
screamer	0:66f3b5499f7f	1008
screamer	0:66f3b5499f7f	1009	return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
screamer	0:66f3b5499f7f	1010	target_name_unique,
screamer	0:66f3b5499f7f	1011	toolchain_name,
screamer	0:66f3b5499f7f	1012	test_id,
screamer	0:66f3b5499f7f	1013	test_description,
screamer	0:66f3b5499f7f	1014	round(elapsed_time, 2),
screamer	0:66f3b5499f7f	1015	single_timeout,
screamer	0:66f3b5499f7f	1016	self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
screamer	0:66f3b5499f7f	1017
screamer	0:66f3b5499f7f	1018	def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
screamer	0:66f3b5499f7f	1019	""" Function determines MUT's mbed disk/port and copies binary to
screamer	0:66f3b5499f7f	1020	target.
screamer	0:66f3b5499f7f	1021	"""
screamer	0:66f3b5499f7f	1022	handle_results = []
screamer	0:66f3b5499f7f	1023	data = json.loads(test_spec)
screamer	0:66f3b5499f7f	1024
screamer	0:66f3b5499f7f	1025	# Find a suitable MUT:
screamer	0:66f3b5499f7f	1026	mut = None
theotherjimmy	43:2a7da56ebd24	1027	for id, m in self.muts.items():
screamer	0:66f3b5499f7f	1028	if m['mcu'] == data['mcu']:
screamer	0:66f3b5499f7f	1029	mut = m
screamer	0:66f3b5499f7f	1030	handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
screamer	0:66f3b5499f7f	1031	handle_results.append(handle_result)
screamer	0:66f3b5499f7f	1032
screamer	0:66f3b5499f7f	1033	return handle_results
screamer	0:66f3b5499f7f	1034
screamer	0:66f3b5499f7f	1035	def print_test_result(self, test_result, target_name, toolchain_name,
screamer	0:66f3b5499f7f	1036	test_id, test_description, elapsed_time, duration):
screamer	0:66f3b5499f7f	1037	""" Use specific convention to print test result and related data
screamer	0:66f3b5499f7f	1038	"""
screamer	0:66f3b5499f7f	1039	tokens = []
screamer	0:66f3b5499f7f	1040	tokens.append("TargetTest")
screamer	0:66f3b5499f7f	1041	tokens.append(target_name)
screamer	0:66f3b5499f7f	1042	tokens.append(toolchain_name)
screamer	0:66f3b5499f7f	1043	tokens.append(test_id)
screamer	0:66f3b5499f7f	1044	tokens.append(test_description)
screamer	0:66f3b5499f7f	1045	separator = "::"
screamer	0:66f3b5499f7f	1046	time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
screamer	0:66f3b5499f7f	1047	result = separator.join(tokens) + " [" + test_result +"]" + time_info
screamer	0:66f3b5499f7f	1048	return Fore.MAGENTA + result + Fore.RESET
screamer	0:66f3b5499f7f	1049
screamer	0:66f3b5499f7f	1050	def shape_test_loop_ok_result_count(self, test_all_result):
screamer	0:66f3b5499f7f	1051	""" Reformats list of results to simple string
screamer	0:66f3b5499f7f	1052	"""
screamer	0:66f3b5499f7f	1053	test_loop_count = len(test_all_result)
screamer	0:66f3b5499f7f	1054	test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
screamer	0:66f3b5499f7f	1055	return "%d/%d"% (test_loop_ok_result, test_loop_count)
screamer	0:66f3b5499f7f	1056
screamer	0:66f3b5499f7f	1057	def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
screamer	0:66f3b5499f7f	1058	""" Reformats list of results to simple string
screamer	0:66f3b5499f7f	1059	"""
screamer	0:66f3b5499f7f	1060	result = self.TEST_RESULT_FAIL
screamer	0:66f3b5499f7f	1061
screamer	0:66f3b5499f7f	1062	if all(test_all_result[0] == res for res in test_all_result):
screamer	0:66f3b5499f7f	1063	result = test_all_result[0]
screamer	0:66f3b5499f7f	1064	elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
screamer	0:66f3b5499f7f	1065	result = self.TEST_RESULT_OK
screamer	0:66f3b5499f7f	1066
screamer	0:66f3b5499f7f	1067	return result
screamer	0:66f3b5499f7f	1068
screamer	0:66f3b5499f7f	1069	def run_host_test(self, name, image_path, disk, port, duration,
screamer	0:66f3b5499f7f	1070	micro=None, reset=None, reset_tout=None,
screamer	0:66f3b5499f7f	1071	verbose=False, copy_method=None, program_cycle_s=None):
screamer	0:66f3b5499f7f	1072	""" Function creates new process with host test configured with particular test case.
screamer	0:66f3b5499f7f	1073	Function also is pooling for serial port activity from process to catch all data
screamer	0:66f3b5499f7f	1074	printed by test runner and host test during test execution
screamer	0:66f3b5499f7f	1075	"""
screamer	0:66f3b5499f7f	1076
screamer	0:66f3b5499f7f	1077	def get_char_from_queue(obs):
screamer	0:66f3b5499f7f	1078	""" Get character from queue safe way
screamer	0:66f3b5499f7f	1079	"""
screamer	0:66f3b5499f7f	1080	try:
screamer	0:66f3b5499f7f	1081	c = obs.queue.get(block=True, timeout=0.5)
theotherjimmy	43:2a7da56ebd24	1082	except Empty:
screamer	0:66f3b5499f7f	1083	c = None
screamer	0:66f3b5499f7f	1084	return c
screamer	0:66f3b5499f7f	1085
screamer	0:66f3b5499f7f	1086	def filter_queue_char(c):
screamer	0:66f3b5499f7f	1087	""" Filters out non ASCII characters from serial port
screamer	0:66f3b5499f7f	1088	"""
screamer	0:66f3b5499f7f	1089	if ord(c) not in range(128):
screamer	0:66f3b5499f7f	1090	c = ' '
screamer	0:66f3b5499f7f	1091	return c
screamer	0:66f3b5499f7f	1092
screamer	0:66f3b5499f7f	1093	def get_test_result(output):
screamer	0:66f3b5499f7f	1094	""" Parse test 'output' data
screamer	0:66f3b5499f7f	1095	"""
screamer	0:66f3b5499f7f	1096	result = self.TEST_RESULT_TIMEOUT
screamer	0:66f3b5499f7f	1097	for line in "".join(output).splitlines():
screamer	0:66f3b5499f7f	1098	search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
screamer	0:66f3b5499f7f	1099	if search_result and len(search_result.groups()):
screamer	0:66f3b5499f7f	1100	result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
screamer	0:66f3b5499f7f	1101	break
screamer	0:66f3b5499f7f	1102	return result
screamer	0:66f3b5499f7f	1103
screamer	0:66f3b5499f7f	1104	def get_auto_property_value(property_name, line):
screamer	0:66f3b5499f7f	1105	""" Scans auto detection line from MUT and returns scanned parameter 'property_name'
screamer	0:66f3b5499f7f	1106	Returns string
screamer	0:66f3b5499f7f	1107	"""
screamer	0:66f3b5499f7f	1108	result = None
screamer	0:66f3b5499f7f	1109	if re.search("HOST: Property '%s'"% property_name, line) is not None:
screamer	0:66f3b5499f7f	1110	property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
screamer	0:66f3b5499f7f	1111	if property is not None and len(property.groups()) == 1:
screamer	0:66f3b5499f7f	1112	result = property.groups()[0]
screamer	0:66f3b5499f7f	1113	return result
screamer	0:66f3b5499f7f	1114
screamer	0:66f3b5499f7f	1115	cmd = ["python",
screamer	0:66f3b5499f7f	1116	'%s.py'% name,
screamer	0:66f3b5499f7f	1117	'-d', disk,
screamer	0:66f3b5499f7f	1118	'-f', '"%s"'% image_path,
screamer	0:66f3b5499f7f	1119	'-p', port,
screamer	0:66f3b5499f7f	1120	'-t', str(duration),
screamer	0:66f3b5499f7f	1121	'-C', str(program_cycle_s)]
screamer	0:66f3b5499f7f	1122
screamer	0:66f3b5499f7f	1123	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
screamer	0:66f3b5499f7f	1124	cmd += ['--auto']
screamer	0:66f3b5499f7f	1125
screamer	0:66f3b5499f7f	1126	# Add extra parameters to host_test
screamer	0:66f3b5499f7f	1127	if copy_method is not None:
screamer	0:66f3b5499f7f	1128	cmd += ["-c", copy_method]
screamer	0:66f3b5499f7f	1129	if micro is not None:
screamer	0:66f3b5499f7f	1130	cmd += ["-m", micro]
screamer	0:66f3b5499f7f	1131	if reset is not None:
screamer	0:66f3b5499f7f	1132	cmd += ["-r", reset]
screamer	0:66f3b5499f7f	1133	if reset_tout is not None:
screamer	0:66f3b5499f7f	1134	cmd += ["-R", str(reset_tout)]
screamer	0:66f3b5499f7f	1135
screamer	0:66f3b5499f7f	1136	if verbose:
theotherjimmy	43:2a7da56ebd24	1137	print(Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET)
theotherjimmy	43:2a7da56ebd24	1138	print("Test::Output::Start")
screamer	0:66f3b5499f7f	1139
screamer	0:66f3b5499f7f	1140	proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
screamer	0:66f3b5499f7f	1141	obs = ProcessObserver(proc)
screamer	0:66f3b5499f7f	1142	update_once_flag = {} # Stores flags checking if some auto-parameter was already set
screamer	0:66f3b5499f7f	1143	line = ''
screamer	0:66f3b5499f7f	1144	output = []
screamer	0:66f3b5499f7f	1145	start_time = time()
screamer	0:66f3b5499f7f	1146	while (time() - start_time) < (2 * duration):
screamer	0:66f3b5499f7f	1147	c = get_char_from_queue(obs)
screamer	0:66f3b5499f7f	1148	if c:
screamer	0:66f3b5499f7f	1149	if verbose:
screamer	0:66f3b5499f7f	1150	sys.stdout.write(c)
screamer	0:66f3b5499f7f	1151	c = filter_queue_char(c)
screamer	0:66f3b5499f7f	1152	output.append(c)
screamer	0:66f3b5499f7f	1153	# Give the mbed under test a way to communicate the end of the test
screamer	0:66f3b5499f7f	1154	if c in ['\n', '\r']:
screamer	0:66f3b5499f7f	1155
screamer	0:66f3b5499f7f	1156	# Checking for auto-detection information from the test about MUT reset moment
screamer	0:66f3b5499f7f	1157	if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
screamer	0:66f3b5499f7f	1158	# We will update this marker only once to prevent multiple time resets
screamer	0:66f3b5499f7f	1159	update_once_flag['reset_target'] = True
screamer	0:66f3b5499f7f	1160	start_time = time()
screamer	0:66f3b5499f7f	1161
screamer	0:66f3b5499f7f	1162	# Checking for auto-detection information from the test about timeout
screamer	0:66f3b5499f7f	1163	auto_timeout_val = get_auto_property_value('timeout', line)
screamer	0:66f3b5499f7f	1164	if 'timeout' not in update_once_flag and auto_timeout_val is not None:
screamer	0:66f3b5499f7f	1165	# We will update this marker only once to prevent multiple time resets
screamer	0:66f3b5499f7f	1166	update_once_flag['timeout'] = True
screamer	0:66f3b5499f7f	1167	duration = int(auto_timeout_val)
screamer	0:66f3b5499f7f	1168
screamer	0:66f3b5499f7f	1169	# Detect mbed assert:
screamer	0:66f3b5499f7f	1170	if 'mbed assertation failed: ' in line:
screamer	0:66f3b5499f7f	1171	output.append('{{mbed_assert}}')
screamer	0:66f3b5499f7f	1172	break
screamer	0:66f3b5499f7f	1173
screamer	0:66f3b5499f7f	1174	# Check for test end
screamer	0:66f3b5499f7f	1175	if '{end}' in line:
screamer	0:66f3b5499f7f	1176	break
screamer	0:66f3b5499f7f	1177	line = ''
screamer	0:66f3b5499f7f	1178	else:
screamer	0:66f3b5499f7f	1179	line += c
screamer	0:66f3b5499f7f	1180	end_time = time()
screamer	0:66f3b5499f7f	1181	testcase_duration = end_time - start_time # Test case duration from reset to {end}
screamer	0:66f3b5499f7f	1182
screamer	0:66f3b5499f7f	1183	c = get_char_from_queue(obs)
screamer	0:66f3b5499f7f	1184
screamer	0:66f3b5499f7f	1185	if c:
screamer	0:66f3b5499f7f	1186	if verbose:
screamer	0:66f3b5499f7f	1187	sys.stdout.write(c)
screamer	0:66f3b5499f7f	1188	c = filter_queue_char(c)
screamer	0:66f3b5499f7f	1189	output.append(c)
screamer	0:66f3b5499f7f	1190
screamer	0:66f3b5499f7f	1191	if verbose:
theotherjimmy	43:2a7da56ebd24	1192	print("Test::Output::Finish")
screamer	0:66f3b5499f7f	1193	# Stop test process
screamer	0:66f3b5499f7f	1194	obs.stop()
screamer	0:66f3b5499f7f	1195
screamer	0:66f3b5499f7f	1196	result = get_test_result(output)
screamer	0:66f3b5499f7f	1197	return (result, "".join(output), testcase_duration, duration)
screamer	0:66f3b5499f7f	1198
screamer	0:66f3b5499f7f	1199	def is_peripherals_available(self, target_mcu_name, peripherals=None):
screamer	0:66f3b5499f7f	1200	""" Checks if specified target should run specific peripheral test case defined in MUTs file
screamer	0:66f3b5499f7f	1201	"""
screamer	0:66f3b5499f7f	1202	if peripherals is not None:
screamer	0:66f3b5499f7f	1203	peripherals = set(peripherals)
theotherjimmy	43:2a7da56ebd24	1204	for id, mut in self.muts.items():
screamer	0:66f3b5499f7f	1205	# Target MCU name check
screamer	0:66f3b5499f7f	1206	if mut["mcu"] != target_mcu_name:
screamer	0:66f3b5499f7f	1207	continue
screamer	0:66f3b5499f7f	1208	# Peripherals check
screamer	0:66f3b5499f7f	1209	if peripherals is not None:
screamer	0:66f3b5499f7f	1210	if 'peripherals' not in mut:
screamer	0:66f3b5499f7f	1211	continue
screamer	0:66f3b5499f7f	1212	if not peripherals.issubset(set(mut['peripherals'])):
screamer	0:66f3b5499f7f	1213	continue
screamer	0:66f3b5499f7f	1214	return True
screamer	0:66f3b5499f7f	1215	return False
screamer	0:66f3b5499f7f	1216
screamer	0:66f3b5499f7f	1217	def shape_test_request(self, mcu, image_path, test_id, duration=10):
screamer	0:66f3b5499f7f	1218	""" Function prepares JSON structure describing test specification
screamer	0:66f3b5499f7f	1219	"""
screamer	0:66f3b5499f7f	1220	test_spec = {
screamer	0:66f3b5499f7f	1221	"mcu": mcu,
screamer	0:66f3b5499f7f	1222	"image": image_path,
screamer	0:66f3b5499f7f	1223	"duration": duration,
screamer	0:66f3b5499f7f	1224	"test_id": test_id,
screamer	0:66f3b5499f7f	1225	}
screamer	0:66f3b5499f7f	1226	return json.dumps(test_spec)
screamer	0:66f3b5499f7f	1227
screamer	0:66f3b5499f7f	1228
screamer	0:66f3b5499f7f	1229	def get_unique_value_from_summary(test_summary, index):
screamer	0:66f3b5499f7f	1230	""" Gets list of unique target names
screamer	0:66f3b5499f7f	1231	"""
screamer	0:66f3b5499f7f	1232	result = []
screamer	0:66f3b5499f7f	1233	for test in test_summary:
screamer	0:66f3b5499f7f	1234	target_name = test[index]
screamer	0:66f3b5499f7f	1235	if target_name not in result:
screamer	0:66f3b5499f7f	1236	result.append(target_name)
screamer	0:66f3b5499f7f	1237	return sorted(result)
screamer	0:66f3b5499f7f	1238
screamer	0:66f3b5499f7f	1239
screamer	0:66f3b5499f7f	1240	def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
screamer	0:66f3b5499f7f	1241	""" Gets list of unique target names and return dictionary
screamer	0:66f3b5499f7f	1242	"""
screamer	0:66f3b5499f7f	1243	result = {}
screamer	0:66f3b5499f7f	1244	for test in test_summary:
screamer	0:66f3b5499f7f	1245	key = test[index_key]
screamer	0:66f3b5499f7f	1246	val = test[index_val]
screamer	0:66f3b5499f7f	1247	if key not in result:
screamer	0:66f3b5499f7f	1248	result[key] = val
screamer	0:66f3b5499f7f	1249	return result
screamer	0:66f3b5499f7f	1250
screamer	0:66f3b5499f7f	1251
screamer	0:66f3b5499f7f	1252	def show_json_file_format_error(json_spec_filename, line, column):
screamer	0:66f3b5499f7f	1253	""" Prints JSON broken content
screamer	0:66f3b5499f7f	1254	"""
screamer	0:66f3b5499f7f	1255	with open(json_spec_filename) as data_file:
screamer	0:66f3b5499f7f	1256	line_no = 1
screamer	0:66f3b5499f7f	1257	for json_line in data_file:
screamer	0:66f3b5499f7f	1258	if line_no + 5 >= line: # Print last few lines before error
theotherjimmy	43:2a7da56ebd24	1259	print('Line %d:\t'%line_no + json_line)
screamer	0:66f3b5499f7f	1260	if line_no == line:
theotherjimmy	43:2a7da56ebd24	1261	print('%s\t%s^' (' ' * len('Line %d:' % line_no),
theotherjimmy	43:2a7da56ebd24	1262	'-' * (column - 1)))
screamer	0:66f3b5499f7f	1263	break
screamer	0:66f3b5499f7f	1264	line_no += 1
screamer	0:66f3b5499f7f	1265
screamer	0:66f3b5499f7f	1266
screamer	0:66f3b5499f7f	1267	def json_format_error_defect_pos(json_error_msg):
screamer	0:66f3b5499f7f	1268	""" Gets first error line and column in JSON file format.
screamer	0:66f3b5499f7f	1269	Parsed from exception thrown by json.loads() string
screamer	0:66f3b5499f7f	1270	"""
screamer	0:66f3b5499f7f	1271	result = None
screamer	0:66f3b5499f7f	1272	line, column = 0, 0
screamer	0:66f3b5499f7f	1273	# Line value search
screamer	0:66f3b5499f7f	1274	line_search = re.search('line [0-9]+', json_error_msg)
screamer	0:66f3b5499f7f	1275	if line_search is not None:
screamer	0:66f3b5499f7f	1276	ls = line_search.group().split(' ')
screamer	0:66f3b5499f7f	1277	if len(ls) == 2:
screamer	0:66f3b5499f7f	1278	line = int(ls[1])
screamer	0:66f3b5499f7f	1279	# Column position search
screamer	0:66f3b5499f7f	1280	column_search = re.search('column [0-9]+', json_error_msg)
screamer	0:66f3b5499f7f	1281	if column_search is not None:
screamer	0:66f3b5499f7f	1282	cs = column_search.group().split(' ')
screamer	0:66f3b5499f7f	1283	if len(cs) == 2:
screamer	0:66f3b5499f7f	1284	column = int(cs[1])
screamer	0:66f3b5499f7f	1285	result = [line, column]
screamer	0:66f3b5499f7f	1286	return result
screamer	0:66f3b5499f7f	1287
screamer	0:66f3b5499f7f	1288
screamer	0:66f3b5499f7f	1289	def get_json_data_from_file(json_spec_filename, verbose=False):
screamer	0:66f3b5499f7f	1290	""" Loads from file JSON formatted string to data structure
screamer	0:66f3b5499f7f	1291	"""
screamer	0:66f3b5499f7f	1292	result = None
screamer	0:66f3b5499f7f	1293	try:
screamer	0:66f3b5499f7f	1294	with open(json_spec_filename) as data_file:
screamer	0:66f3b5499f7f	1295	try:
screamer	0:66f3b5499f7f	1296	result = json.load(data_file)
screamer	0:66f3b5499f7f	1297	except ValueError as json_error_msg:
screamer	0:66f3b5499f7f	1298	result = None
theotherjimmy	43:2a7da56ebd24	1299	print('JSON file %s parsing failed. Reason: %s' %
theotherjimmy	43:2a7da56ebd24	1300	(json_spec_filename, json_error_msg))
screamer	0:66f3b5499f7f	1301	# We can print where error occurred inside JSON file if we can parse exception msg
screamer	0:66f3b5499f7f	1302	json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
screamer	0:66f3b5499f7f	1303	if json_format_defect_pos is not None:
screamer	0:66f3b5499f7f	1304	line = json_format_defect_pos[0]
screamer	0:66f3b5499f7f	1305	column = json_format_defect_pos[1]
theotherjimmy	43:2a7da56ebd24	1306	print()
screamer	0:66f3b5499f7f	1307	show_json_file_format_error(json_spec_filename, line, column)
screamer	0:66f3b5499f7f	1308
screamer	0:66f3b5499f7f	1309	except IOError as fileopen_error_msg:
theotherjimmy	43:2a7da56ebd24	1310	print('JSON file %s not opened. Reason: %s\n'%
theotherjimmy	43:2a7da56ebd24	1311	(json_spec_filename, fileopen_error_msg))
screamer	0:66f3b5499f7f	1312	if verbose and result:
screamer	0:66f3b5499f7f	1313	pp = pprint.PrettyPrinter(indent=4)
screamer	0:66f3b5499f7f	1314	pp.pprint(result)
screamer	0:66f3b5499f7f	1315	return result
screamer	0:66f3b5499f7f	1316
screamer	0:66f3b5499f7f	1317
screamer	0:66f3b5499f7f	1318	def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
screamer	0:66f3b5499f7f	1319	""" Prints MUTs configuration passed to test script for verboseness
screamer	0:66f3b5499f7f	1320	"""
screamer	0:66f3b5499f7f	1321	muts_info_cols = []
screamer	0:66f3b5499f7f	1322	# We need to check all unique properties for each defined MUT
screamer	0:66f3b5499f7f	1323	for k in json_data:
screamer	0:66f3b5499f7f	1324	mut_info = json_data[k]
screamer	0:66f3b5499f7f	1325	for mut_property in mut_info:
screamer	0:66f3b5499f7f	1326	if mut_property not in muts_info_cols:
screamer	0:66f3b5499f7f	1327	muts_info_cols.append(mut_property)
screamer	0:66f3b5499f7f	1328
screamer	0:66f3b5499f7f	1329	# Prepare pretty table object to display all MUTs
screamer	0:66f3b5499f7f	1330	pt_cols = ["index"] + muts_info_cols
theotherjimmy	43:2a7da56ebd24	1331	pt = PrettyTable(pt_cols, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	1332	for col in pt_cols:
screamer	0:66f3b5499f7f	1333	pt.align[col] = "l"
screamer	0:66f3b5499f7f	1334
screamer	0:66f3b5499f7f	1335	# Add rows to pretty print object
screamer	0:66f3b5499f7f	1336	for k in json_data:
screamer	0:66f3b5499f7f	1337	row = [k]
screamer	0:66f3b5499f7f	1338	mut_info = json_data[k]
screamer	0:66f3b5499f7f	1339
screamer	0:66f3b5499f7f	1340	add_row = True
screamer	0:66f3b5499f7f	1341	if platform_filter and 'mcu' in mut_info:
screamer	0:66f3b5499f7f	1342	add_row = re.search(platform_filter, mut_info['mcu']) is not None
screamer	0:66f3b5499f7f	1343	if add_row:
screamer	0:66f3b5499f7f	1344	for col in muts_info_cols:
screamer	0:66f3b5499f7f	1345	cell_val = mut_info[col] if col in mut_info else None
theotherjimmy	43:2a7da56ebd24	1346	if isinstance(cell_val, list):
screamer	0:66f3b5499f7f	1347	cell_val = join_delim.join(cell_val)
screamer	0:66f3b5499f7f	1348	row.append(cell_val)
screamer	0:66f3b5499f7f	1349	pt.add_row(row)
screamer	0:66f3b5499f7f	1350	return pt.get_string()
screamer	0:66f3b5499f7f	1351
screamer	0:66f3b5499f7f	1352
screamer	0:66f3b5499f7f	1353	def print_test_configuration_from_json(json_data, join_delim=", "):
screamer	0:66f3b5499f7f	1354	""" Prints test specification configuration passed to test script for verboseness
screamer	0:66f3b5499f7f	1355	"""
screamer	0:66f3b5499f7f	1356	toolchains_info_cols = []
screamer	0:66f3b5499f7f	1357	# We need to check all toolchains for each device
screamer	0:66f3b5499f7f	1358	for k in json_data:
screamer	0:66f3b5499f7f	1359	# k should be 'targets'
screamer	0:66f3b5499f7f	1360	targets = json_data[k]
screamer	0:66f3b5499f7f	1361	for target in targets:
screamer	0:66f3b5499f7f	1362	toolchains = targets[target]
screamer	0:66f3b5499f7f	1363	for toolchain in toolchains:
screamer	0:66f3b5499f7f	1364	if toolchain not in toolchains_info_cols:
screamer	0:66f3b5499f7f	1365	toolchains_info_cols.append(toolchain)
screamer	0:66f3b5499f7f	1366
screamer	0:66f3b5499f7f	1367	# Prepare pretty table object to display test specification
screamer	0:66f3b5499f7f	1368	pt_cols = ["mcu"] + sorted(toolchains_info_cols)
theotherjimmy	43:2a7da56ebd24	1369	pt = PrettyTable(pt_cols, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	1370	for col in pt_cols:
screamer	0:66f3b5499f7f	1371	pt.align[col] = "l"
screamer	0:66f3b5499f7f	1372
screamer	0:66f3b5499f7f	1373	# { target : [conflicted toolchains] }
screamer	0:66f3b5499f7f	1374	toolchain_conflicts = {}
screamer	0:66f3b5499f7f	1375	toolchain_path_conflicts = []
screamer	0:66f3b5499f7f	1376	for k in json_data:
screamer	0:66f3b5499f7f	1377	# k should be 'targets'
screamer	0:66f3b5499f7f	1378	targets = json_data[k]
screamer	0:66f3b5499f7f	1379	for target in targets:
screamer	0:66f3b5499f7f	1380	target_supported_toolchains = get_target_supported_toolchains(target)
screamer	0:66f3b5499f7f	1381	if not target_supported_toolchains:
screamer	0:66f3b5499f7f	1382	target_supported_toolchains = []
screamer	0:66f3b5499f7f	1383	target_name = target if target in TARGET_MAP else "%s*"% target
screamer	0:66f3b5499f7f	1384	row = [target_name]
screamer	0:66f3b5499f7f	1385	toolchains = targets[target]
screamer	0:66f3b5499f7f	1386
screamer	0:66f3b5499f7f	1387	for toolchain in sorted(toolchains_info_cols):
screamer	0:66f3b5499f7f	1388	# Check for conflicts: target vs toolchain
screamer	0:66f3b5499f7f	1389	conflict = False
screamer	0:66f3b5499f7f	1390	conflict_path = False
screamer	0:66f3b5499f7f	1391	if toolchain in toolchains:
screamer	0:66f3b5499f7f	1392	if toolchain not in target_supported_toolchains:
screamer	0:66f3b5499f7f	1393	conflict = True
screamer	0:66f3b5499f7f	1394	if target not in toolchain_conflicts:
screamer	0:66f3b5499f7f	1395	toolchain_conflicts[target] = []
screamer	0:66f3b5499f7f	1396	toolchain_conflicts[target].append(toolchain)
screamer	0:66f3b5499f7f	1397	# Add marker inside table about target usage / conflict
screamer	0:66f3b5499f7f	1398	cell_val = 'Yes' if toolchain in toolchains else '-'
screamer	0:66f3b5499f7f	1399	if conflict:
screamer	0:66f3b5499f7f	1400	cell_val += '*'
screamer	0:66f3b5499f7f	1401	# Check for conflicts: toolchain vs toolchain path
screamer	20:835f6355470d	1402	if toolchain in TOOLCHAIN_PATHS:
screamer	20:835f6355470d	1403	toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer	0:66f3b5499f7f	1404	if not os.path.isdir(toolchain_path):
screamer	0:66f3b5499f7f	1405	conflict_path = True
screamer	0:66f3b5499f7f	1406	if toolchain not in toolchain_path_conflicts:
screamer	0:66f3b5499f7f	1407	toolchain_path_conflicts.append(toolchain)
screamer	0:66f3b5499f7f	1408	if conflict_path:
screamer	0:66f3b5499f7f	1409	cell_val += '#'
screamer	0:66f3b5499f7f	1410	row.append(cell_val)
screamer	0:66f3b5499f7f	1411	pt.add_row(row)
screamer	0:66f3b5499f7f	1412
screamer	0:66f3b5499f7f	1413	# generate result string
screamer	0:66f3b5499f7f	1414	result = pt.get_string() # Test specification table
screamer	0:66f3b5499f7f	1415	if toolchain_conflicts or toolchain_path_conflicts:
screamer	0:66f3b5499f7f	1416	result += "\n"
screamer	0:66f3b5499f7f	1417	result += "Toolchain conflicts:\n"
screamer	0:66f3b5499f7f	1418	for target in toolchain_conflicts:
screamer	0:66f3b5499f7f	1419	if target not in TARGET_MAP:
screamer	0:66f3b5499f7f	1420	result += "\t* Target %s unknown\n"% (target)
screamer	0:66f3b5499f7f	1421	conflict_target_list = join_delim.join(toolchain_conflicts[target])
screamer	0:66f3b5499f7f	1422	sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
screamer	0:66f3b5499f7f	1423	result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
screamer	0:66f3b5499f7f	1424
screamer	0:66f3b5499f7f	1425	for toolchain in toolchain_path_conflicts:
screamer	0:66f3b5499f7f	1426	# Let's check toolchain configuration
screamer	20:835f6355470d	1427	if toolchain in TOOLCHAIN_PATHS:
screamer	20:835f6355470d	1428	toolchain_path = TOOLCHAIN_PATHS[toolchain]
screamer	0:66f3b5499f7f	1429	if not os.path.isdir(toolchain_path):
screamer	0:66f3b5499f7f	1430	result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
screamer	0:66f3b5499f7f	1431	return result
screamer	0:66f3b5499f7f	1432
screamer	0:66f3b5499f7f	1433
screamer	0:66f3b5499f7f	1434	def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
screamer	0:66f3b5499f7f	1435	""" Generates table summary with all test cases and additional test cases
screamer	0:66f3b5499f7f	1436	information using pretty print functionality. Allows test suite user to
screamer	0:66f3b5499f7f	1437	see test cases
screamer	0:66f3b5499f7f	1438	"""
screamer	0:66f3b5499f7f	1439	# get all unique test ID prefixes
screamer	0:66f3b5499f7f	1440	unique_test_id = []
screamer	0:66f3b5499f7f	1441	for test in TESTS:
screamer	0:66f3b5499f7f	1442	split = test['id'].split('_')[:-1]
screamer	0:66f3b5499f7f	1443	test_id_prefix = '_'.join(split)
screamer	0:66f3b5499f7f	1444	if test_id_prefix not in unique_test_id:
screamer	0:66f3b5499f7f	1445	unique_test_id.append(test_id_prefix)
screamer	0:66f3b5499f7f	1446	unique_test_id.sort()
screamer	0:66f3b5499f7f	1447	counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
screamer	0:66f3b5499f7f	1448	counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
screamer	0:66f3b5499f7f	1449
screamer	0:66f3b5499f7f	1450	test_properties = ['id',
screamer	0:66f3b5499f7f	1451	'automated',
screamer	0:66f3b5499f7f	1452	'description',
screamer	0:66f3b5499f7f	1453	'peripherals',
screamer	0:66f3b5499f7f	1454	'host_test',
screamer	0:66f3b5499f7f	1455	'duration'] if cols is None else cols
screamer	0:66f3b5499f7f	1456
screamer	0:66f3b5499f7f	1457	# All tests status table print
theotherjimmy	43:2a7da56ebd24	1458	pt = PrettyTable(test_properties, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	1459	for col in test_properties:
screamer	0:66f3b5499f7f	1460	pt.align[col] = "l"
screamer	0:66f3b5499f7f	1461	pt.align['duration'] = "r"
screamer	0:66f3b5499f7f	1462
screamer	0:66f3b5499f7f	1463	counter_all = 0
screamer	0:66f3b5499f7f	1464	counter_automated = 0
screamer	0:66f3b5499f7f	1465	pt.padding_width = 1 # One space between column edges and contents (default)
screamer	0:66f3b5499f7f	1466
screamer	0:66f3b5499f7f	1467	for test_id in sorted(TEST_MAP.keys()):
screamer	0:66f3b5499f7f	1468	if platform_filter is not None:
screamer	0:66f3b5499f7f	1469	# FIlter out platforms using regex
screamer	0:66f3b5499f7f	1470	if re.search(platform_filter, test_id) is None:
screamer	0:66f3b5499f7f	1471	continue
screamer	0:66f3b5499f7f	1472	row = []
screamer	0:66f3b5499f7f	1473	test = TEST_MAP[test_id]
screamer	0:66f3b5499f7f	1474	split = test_id.split('_')[:-1]
screamer	0:66f3b5499f7f	1475	test_id_prefix = '_'.join(split)
screamer	0:66f3b5499f7f	1476
screamer	0:66f3b5499f7f	1477	for col in test_properties:
screamer	0:66f3b5499f7f	1478	col_value = test[col]
theotherjimmy	43:2a7da56ebd24	1479	if isinstance(test[col], list):
screamer	0:66f3b5499f7f	1480	col_value = join_delim.join(test[col])
screamer	0:66f3b5499f7f	1481	elif test[col] == None:
screamer	0:66f3b5499f7f	1482	col_value = "-"
screamer	0:66f3b5499f7f	1483
screamer	0:66f3b5499f7f	1484	row.append(col_value)
screamer	0:66f3b5499f7f	1485	if test['automated'] == True:
screamer	0:66f3b5499f7f	1486	counter_dict_test_id_types[test_id_prefix] += 1
screamer	0:66f3b5499f7f	1487	counter_automated += 1
screamer	0:66f3b5499f7f	1488	pt.add_row(row)
screamer	0:66f3b5499f7f	1489	# Update counters
screamer	0:66f3b5499f7f	1490	counter_all += 1
screamer	0:66f3b5499f7f	1491	counter_dict_test_id_types_all[test_id_prefix] += 1
screamer	0:66f3b5499f7f	1492	result = pt.get_string()
screamer	0:66f3b5499f7f	1493	result += "\n\n"
screamer	0:66f3b5499f7f	1494
screamer	0:66f3b5499f7f	1495	if result_summary and not platform_filter:
screamer	0:66f3b5499f7f	1496	# Automation result summary
screamer	0:66f3b5499f7f	1497	test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
theotherjimmy	43:2a7da56ebd24	1498	pt = PrettyTable(test_id_cols, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	1499	pt.align['automated'] = "r"
screamer	0:66f3b5499f7f	1500	pt.align['all'] = "r"
screamer	0:66f3b5499f7f	1501	pt.align['percent [%]'] = "r"
screamer	0:66f3b5499f7f	1502
screamer	0:66f3b5499f7f	1503	percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
screamer	0:66f3b5499f7f	1504	str_progress = progress_bar(percent_progress, 75)
screamer	0:66f3b5499f7f	1505	pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
screamer	0:66f3b5499f7f	1506	result += "Automation coverage:\n"
screamer	0:66f3b5499f7f	1507	result += pt.get_string()
screamer	0:66f3b5499f7f	1508	result += "\n\n"
screamer	0:66f3b5499f7f	1509
screamer	0:66f3b5499f7f	1510	# Test automation coverage table print
screamer	0:66f3b5499f7f	1511	test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
theotherjimmy	43:2a7da56ebd24	1512	pt = PrettyTable(test_id_cols, junction_char="\|", hrules=HEADER)
screamer	0:66f3b5499f7f	1513	pt.align['id'] = "l"
screamer	0:66f3b5499f7f	1514	pt.align['automated'] = "r"
screamer	0:66f3b5499f7f	1515	pt.align['all'] = "r"
screamer	0:66f3b5499f7f	1516	pt.align['percent [%]'] = "r"
screamer	0:66f3b5499f7f	1517	for unique_id in unique_test_id:
screamer	0:66f3b5499f7f	1518	# print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
screamer	0:66f3b5499f7f	1519	percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
screamer	0:66f3b5499f7f	1520	str_progress = progress_bar(percent_progress, 75)
screamer	0:66f3b5499f7f	1521	row = [unique_id,
screamer	0:66f3b5499f7f	1522	counter_dict_test_id_types[unique_id],
screamer	0:66f3b5499f7f	1523	counter_dict_test_id_types_all[unique_id],
screamer	0:66f3b5499f7f	1524	percent_progress,
screamer	0:66f3b5499f7f	1525	"[" + str_progress + "]"]
screamer	0:66f3b5499f7f	1526	pt.add_row(row)
screamer	0:66f3b5499f7f	1527	result += "Test automation coverage:\n"
screamer	0:66f3b5499f7f	1528	result += pt.get_string()
screamer	0:66f3b5499f7f	1529	result += "\n\n"
screamer	0:66f3b5499f7f	1530	return result
screamer	0:66f3b5499f7f	1531
screamer	0:66f3b5499f7f	1532
screamer	0:66f3b5499f7f	1533	def progress_bar(percent_progress, saturation=0):
screamer	0:66f3b5499f7f	1534	""" This function creates progress bar with optional simple saturation mark
screamer	0:66f3b5499f7f	1535	"""
screamer	0:66f3b5499f7f	1536	step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
screamer	0:66f3b5499f7f	1537	str_progress = '#' * step + '.' * int(50 - step)
screamer	0:66f3b5499f7f	1538	c = '!' if str_progress[38] == '.' else '\|'
screamer	0:66f3b5499f7f	1539	if saturation > 0:
screamer	0:66f3b5499f7f	1540	saturation = saturation / 2
screamer	0:66f3b5499f7f	1541	str_progress = str_progress[:saturation] + c + str_progress[saturation:]
screamer	0:66f3b5499f7f	1542	return str_progress
screamer	0:66f3b5499f7f	1543
screamer	0:66f3b5499f7f	1544
screamer	0:66f3b5499f7f	1545	def singletest_in_cli_mode(single_test):
screamer	0:66f3b5499f7f	1546	""" Runs SingleTestRunner object in CLI (Command line interface) mode
screamer	0:66f3b5499f7f	1547
screamer	0:66f3b5499f7f	1548	@return returns success code (0 == success) for building and running tests
screamer	0:66f3b5499f7f	1549	"""
screamer	0:66f3b5499f7f	1550	start = time()
screamer	0:66f3b5499f7f	1551	# Execute tests depending on options and filter applied
screamer	0:66f3b5499f7f	1552	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
screamer	0:66f3b5499f7f	1553	elapsed_time = time() - start
screamer	0:66f3b5499f7f	1554
screamer	0:66f3b5499f7f	1555	# Human readable summary
screamer	0:66f3b5499f7f	1556	if not single_test.opts_suppress_summary:
screamer	0:66f3b5499f7f	1557	# prints well-formed summary with results (SQL table like)
theotherjimmy	43:2a7da56ebd24	1558	print(single_test.generate_test_summary(test_summary, shuffle_seed))
screamer	0:66f3b5499f7f	1559	if single_test.opts_test_x_toolchain_summary:
screamer	0:66f3b5499f7f	1560	# prints well-formed summary with results (SQL table like)
screamer	0:66f3b5499f7f	1561	# table shows text x toolchain test result matrix
theotherjimmy	43:2a7da56ebd24	1562	print(single_test.generate_test_summary_by_target(test_summary,
theotherjimmy	43:2a7da56ebd24	1563	shuffle_seed))
screamer	0:66f3b5499f7f	1564
theotherjimmy	43:2a7da56ebd24	1565	print("Completed in %.2f sec" % elapsed_time)
screamer	0:66f3b5499f7f	1566	print
screamer	0:66f3b5499f7f	1567	# Write summary of the builds
screamer	0:66f3b5499f7f	1568
screamer	0:66f3b5499f7f	1569	print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer	0:66f3b5499f7f	1570	status = print_report_exporter.report(build_report)
screamer	0:66f3b5499f7f	1571
screamer	0:66f3b5499f7f	1572	# Store extra reports in files
screamer	0:66f3b5499f7f	1573	if single_test.opts_report_html_file_name:
screamer	0:66f3b5499f7f	1574	# Export results in form of HTML report to separate file
screamer	0:66f3b5499f7f	1575	report_exporter = ReportExporter(ResultExporterType.HTML)
screamer	0:66f3b5499f7f	1576	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
screamer	0:66f3b5499f7f	1577	if single_test.opts_report_junit_file_name:
screamer	0:66f3b5499f7f	1578	# Export results in form of JUnit XML report to separate file
screamer	0:66f3b5499f7f	1579	report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer	0:66f3b5499f7f	1580	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
screamer	22:9e85236d8716	1581	if single_test.opts_report_text_file_name:
screamer	22:9e85236d8716	1582	# Export results in form of a text file
screamer	22:9e85236d8716	1583	report_exporter = ReportExporter(ResultExporterType.TEXT)
screamer	22:9e85236d8716	1584	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
screamer	0:66f3b5499f7f	1585	if single_test.opts_report_build_file_name:
screamer	0:66f3b5499f7f	1586	# Export build results as html report to sparate file
screamer	0:66f3b5499f7f	1587	report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer	0:66f3b5499f7f	1588	report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
screamer	0:66f3b5499f7f	1589
screamer	0:66f3b5499f7f	1590	# Returns True if no build failures of the test projects or their dependencies
screamer	0:66f3b5499f7f	1591	return status
screamer	0:66f3b5499f7f	1592
screamer	0:66f3b5499f7f	1593	class TestLogger():
screamer	0:66f3b5499f7f	1594	""" Super-class for logging and printing ongoing events for test suite pass
screamer	0:66f3b5499f7f	1595	"""
screamer	0:66f3b5499f7f	1596	def __init__(self, store_log=True):
screamer	0:66f3b5499f7f	1597	""" We can control if logger actually stores log in memory
screamer	0:66f3b5499f7f	1598	or just handled all log entries immediately
screamer	0:66f3b5499f7f	1599	"""
screamer	0:66f3b5499f7f	1600	self.log = []
screamer	0:66f3b5499f7f	1601	self.log_to_file = False
screamer	0:66f3b5499f7f	1602	self.log_file_name = None
screamer	0:66f3b5499f7f	1603	self.store_log = store_log
screamer	0:66f3b5499f7f	1604
screamer	0:66f3b5499f7f	1605	self.LogType = construct_enum(INFO='Info',
screamer	0:66f3b5499f7f	1606	WARN='Warning',
screamer	0:66f3b5499f7f	1607	NOTIF='Notification',
screamer	0:66f3b5499f7f	1608	ERROR='Error',
screamer	0:66f3b5499f7f	1609	EXCEPT='Exception')
screamer	0:66f3b5499f7f	1610
screamer	0:66f3b5499f7f	1611	self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
screamer	0:66f3b5499f7f	1612	APPEND=2) # Append to existing log file
screamer	0:66f3b5499f7f	1613
screamer	0:66f3b5499f7f	1614	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer	0:66f3b5499f7f	1615	""" Log one line of text
screamer	0:66f3b5499f7f	1616	"""
screamer	0:66f3b5499f7f	1617	log_timestamp = time()
screamer	0:66f3b5499f7f	1618	log_entry = {'log_type' : LogType,
screamer	0:66f3b5499f7f	1619	'log_timestamp' : log_timestamp,
screamer	0:66f3b5499f7f	1620	'log_line' : log_line,
screamer	0:66f3b5499f7f	1621	'_future' : None
screamer	0:66f3b5499f7f	1622	}
screamer	0:66f3b5499f7f	1623	# Store log in memory
screamer	0:66f3b5499f7f	1624	if self.store_log:
screamer	0:66f3b5499f7f	1625	self.log.append(log_entry)
screamer	0:66f3b5499f7f	1626	return log_entry
screamer	0:66f3b5499f7f	1627
screamer	0:66f3b5499f7f	1628
screamer	0:66f3b5499f7f	1629	class CLITestLogger(TestLogger):
screamer	0:66f3b5499f7f	1630	""" Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
screamer	0:66f3b5499f7f	1631	"""
screamer	0:66f3b5499f7f	1632	def __init__(self, store_log=True, file_name=None):
screamer	0:66f3b5499f7f	1633	TestLogger.__init__(self)
screamer	0:66f3b5499f7f	1634	self.log_file_name = file_name
screamer	0:66f3b5499f7f	1635	#self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
screamer	0:66f3b5499f7f	1636	self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
screamer	0:66f3b5499f7f	1637
screamer	0:66f3b5499f7f	1638	def log_print(self, log_entry, timestamp=True):
screamer	0:66f3b5499f7f	1639	""" Prints on screen formatted log entry
screamer	0:66f3b5499f7f	1640	"""
screamer	0:66f3b5499f7f	1641	ts = log_entry['log_timestamp']
screamer	0:66f3b5499f7f	1642	timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
screamer	0:66f3b5499f7f	1643	log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
screamer	0:66f3b5499f7f	1644	return timestamp_str + log_line_str
screamer	0:66f3b5499f7f	1645
screamer	0:66f3b5499f7f	1646	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
screamer	0:66f3b5499f7f	1647	""" Logs line, if log file output was specified log line will be appended
screamer	0:66f3b5499f7f	1648	at the end of log file
screamer	0:66f3b5499f7f	1649	"""
screamer	0:66f3b5499f7f	1650	log_entry = TestLogger.log_line(self, LogType, log_line)
screamer	0:66f3b5499f7f	1651	log_line_str = self.log_print(log_entry, timestamp)
screamer	0:66f3b5499f7f	1652	if self.log_file_name is not None:
screamer	0:66f3b5499f7f	1653	try:
screamer	0:66f3b5499f7f	1654	with open(self.log_file_name, 'a') as f:
screamer	0:66f3b5499f7f	1655	f.write(log_line_str + line_delim)
screamer	0:66f3b5499f7f	1656	except IOError:
screamer	0:66f3b5499f7f	1657	pass
screamer	0:66f3b5499f7f	1658	return log_line_str
screamer	0:66f3b5499f7f	1659
screamer	0:66f3b5499f7f	1660
screamer	0:66f3b5499f7f	1661	def factory_db_logger(db_url):
screamer	0:66f3b5499f7f	1662	""" Factory database driver depending on database type supplied in database connection string db_url
screamer	0:66f3b5499f7f	1663	"""
screamer	0:66f3b5499f7f	1664	if db_url is not None:
screamer	0:66f3b5499f7f	1665	from tools.test_mysql import MySQLDBAccess
screamer	0:66f3b5499f7f	1666	connection_info = BaseDBAccess().parse_db_connection_string(db_url)
screamer	0:66f3b5499f7f	1667	if connection_info is not None:
screamer	0:66f3b5499f7f	1668	(db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
screamer	0:66f3b5499f7f	1669	if db_type == 'mysql':
screamer	0:66f3b5499f7f	1670	return MySQLDBAccess()
screamer	0:66f3b5499f7f	1671	return None
screamer	0:66f3b5499f7f	1672
screamer	0:66f3b5499f7f	1673
screamer	0:66f3b5499f7f	1674	def detect_database_verbose(db_url):
screamer	0:66f3b5499f7f	1675	""" uses verbose mode (prints) database detection sequence to check it database connection string is valid
screamer	0:66f3b5499f7f	1676	"""
screamer	0:66f3b5499f7f	1677	result = BaseDBAccess().parse_db_connection_string(db_url)
screamer	0:66f3b5499f7f	1678	if result is not None:
screamer	0:66f3b5499f7f	1679	# Parsing passed
screamer	0:66f3b5499f7f	1680	(db_type, username, password, host, db_name) = result
screamer	0:66f3b5499f7f	1681	#print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
screamer	0:66f3b5499f7f	1682	# Let's try to connect
screamer	0:66f3b5499f7f	1683	db_ = factory_db_logger(db_url)
screamer	0:66f3b5499f7f	1684	if db_ is not None:
theotherjimmy	43:2a7da56ebd24	1685	print("Connecting to database '%s'..." % db_url)
screamer	0:66f3b5499f7f	1686	db_.connect(host, username, password, db_name)
screamer	0:66f3b5499f7f	1687	if db_.is_connected():
theotherjimmy	43:2a7da56ebd24	1688	print("ok")
theotherjimmy	43:2a7da56ebd24	1689	print("Detecting database...")
theotherjimmy	43:2a7da56ebd24	1690	print(db_.detect_database(verbose=True))
theotherjimmy	43:2a7da56ebd24	1691	print("Disconnecting...")
screamer	0:66f3b5499f7f	1692	db_.disconnect()
theotherjimmy	43:2a7da56ebd24	1693	print("done")
screamer	0:66f3b5499f7f	1694	else:
theotherjimmy	43:2a7da56ebd24	1695	print("Database type '%s' unknown" % db_type)
screamer	0:66f3b5499f7f	1696	else:
theotherjimmy	43:2a7da56ebd24	1697	print("Parse error: '%s' - DB Url error" % db_url)
screamer	0:66f3b5499f7f	1698
screamer	0:66f3b5499f7f	1699
screamer	0:66f3b5499f7f	1700	def get_module_avail(module_name):
theotherjimmy	41:2a77626a4c21	1701	""" This function returns True if module_name is already imported module
screamer	0:66f3b5499f7f	1702	"""
screamer	0:66f3b5499f7f	1703	return module_name in sys.modules.keys()
screamer	0:66f3b5499f7f	1704
screamer	0:66f3b5499f7f	1705	def get_autodetected_MUTS_list(platform_name_filter=None):
screamer	0:66f3b5499f7f	1706	oldError = None
screamer	0:66f3b5499f7f	1707	if os.name == 'nt':
screamer	0:66f3b5499f7f	1708	# Disable Windows error box temporarily
screamer	0:66f3b5499f7f	1709	oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
screamer	0:66f3b5499f7f	1710
screamer	0:66f3b5499f7f	1711	mbeds = mbed_lstools.create()
screamer	0:66f3b5499f7f	1712	detect_muts_list = mbeds.list_mbeds()
screamer	0:66f3b5499f7f	1713
screamer	0:66f3b5499f7f	1714	if os.name == 'nt':
screamer	0:66f3b5499f7f	1715	ctypes.windll.kernel32.SetErrorMode(oldError)
screamer	0:66f3b5499f7f	1716
screamer	0:66f3b5499f7f	1717	return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
screamer	0:66f3b5499f7f	1718
screamer	0:66f3b5499f7f	1719	def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
screamer	0:66f3b5499f7f	1720	""" Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
screamer	0:66f3b5499f7f	1721	If function fails to auto-detect devices it will return empty dictionary.
screamer	0:66f3b5499f7f	1722
screamer	0:66f3b5499f7f	1723	if get_module_avail('mbed_lstools'):
screamer	0:66f3b5499f7f	1724	mbeds = mbed_lstools.create()
screamer	0:66f3b5499f7f	1725	mbeds_list = mbeds.list_mbeds()
screamer	0:66f3b5499f7f	1726
screamer	0:66f3b5499f7f	1727	@param mbeds_list list of mbeds captured from mbed_lstools
screamer	0:66f3b5499f7f	1728	@param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
screamer	0:66f3b5499f7f	1729	"""
screamer	0:66f3b5499f7f	1730	result = {} # Should be in muts_all.json format
screamer	0:66f3b5499f7f	1731	# Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
screamer	0:66f3b5499f7f	1732	# mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
screamer	0:66f3b5499f7f	1733	index = 1
screamer	0:66f3b5499f7f	1734	for mut in mbeds_list:
screamer	0:66f3b5499f7f	1735	# Filter the MUTS if a filter is specified
screamer	0:66f3b5499f7f	1736
screamer	0:66f3b5499f7f	1737	if platform_name_filter and not mut['platform_name'] in platform_name_filter:
screamer	0:66f3b5499f7f	1738	continue
screamer	0:66f3b5499f7f	1739
screamer	0:66f3b5499f7f	1740	# For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
screamer	0:66f3b5499f7f	1741	# if not we are creating our own unique value (last few chars from platform's target_id).
screamer	0:66f3b5499f7f	1742	m = {'mcu': mut['platform_name'],
screamer	0:66f3b5499f7f	1743	'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
screamer	0:66f3b5499f7f	1744	'port': mut['serial_port'],
screamer	0:66f3b5499f7f	1745	'disk': mut['mount_point'],
screamer	0:66f3b5499f7f	1746	'peripherals': [] # No peripheral detection
screamer	0:66f3b5499f7f	1747	}
screamer	0:66f3b5499f7f	1748	if index not in result:
screamer	0:66f3b5499f7f	1749	result[index] = {}
screamer	0:66f3b5499f7f	1750	result[index] = m
screamer	0:66f3b5499f7f	1751	index += 1
screamer	0:66f3b5499f7f	1752	return result
screamer	0:66f3b5499f7f	1753
screamer	0:66f3b5499f7f	1754
screamer	0:66f3b5499f7f	1755	def get_autodetected_TEST_SPEC(mbeds_list,
screamer	0:66f3b5499f7f	1756	use_default_toolchain=True,
screamer	0:66f3b5499f7f	1757	use_supported_toolchains=False,
screamer	0:66f3b5499f7f	1758	toolchain_filter=None,
screamer	0:66f3b5499f7f	1759	platform_name_filter=None):
screamer	0:66f3b5499f7f	1760	""" Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
screamer	0:66f3b5499f7f	1761	If function fails to auto-detect devices it will return empty 'targets' test_spec description.
screamer	0:66f3b5499f7f	1762
screamer	0:66f3b5499f7f	1763	use_default_toolchain - if True add default toolchain to test_spec
screamer	0:66f3b5499f7f	1764	use_supported_toolchains - if True add all supported toolchains to test_spec
screamer	0:66f3b5499f7f	1765	toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
screamer	0:66f3b5499f7f	1766	"""
screamer	0:66f3b5499f7f	1767	result = {'targets': {} }
screamer	0:66f3b5499f7f	1768
screamer	0:66f3b5499f7f	1769	for mut in mbeds_list:
screamer	0:66f3b5499f7f	1770	mcu = mut['mcu']
screamer	0:66f3b5499f7f	1771	if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
screamer	0:66f3b5499f7f	1772	if mcu in TARGET_MAP:
screamer	0:66f3b5499f7f	1773	default_toolchain = TARGET_MAP[mcu].default_toolchain
screamer	0:66f3b5499f7f	1774	supported_toolchains = TARGET_MAP[mcu].supported_toolchains
screamer	0:66f3b5499f7f	1775
screamer	0:66f3b5499f7f	1776	# Decide which toolchains should be added to test specification toolchain pool for each target
screamer	0:66f3b5499f7f	1777	toolchains = []
screamer	0:66f3b5499f7f	1778	if use_default_toolchain:
screamer	0:66f3b5499f7f	1779	toolchains.append(default_toolchain)
screamer	0:66f3b5499f7f	1780	if use_supported_toolchains:
screamer	0:66f3b5499f7f	1781	toolchains += supported_toolchains
screamer	0:66f3b5499f7f	1782	if toolchain_filter is not None:
screamer	0:66f3b5499f7f	1783	all_toolchains = supported_toolchains + [default_toolchain]
screamer	22:9e85236d8716	1784	for toolchain in toolchain_filter:
screamer	0:66f3b5499f7f	1785	if toolchain in all_toolchains:
screamer	0:66f3b5499f7f	1786	toolchains.append(toolchain)
screamer	0:66f3b5499f7f	1787
screamer	0:66f3b5499f7f	1788	result['targets'][mcu] = list(set(toolchains))
screamer	0:66f3b5499f7f	1789	return result
screamer	0:66f3b5499f7f	1790
screamer	0:66f3b5499f7f	1791
screamer	0:66f3b5499f7f	1792	def get_default_test_options_parser():
screamer	0:66f3b5499f7f	1793	""" Get common test script options used by CLI, web services etc.
screamer	0:66f3b5499f7f	1794	"""
screamer	22:9e85236d8716	1795	parser = argparse.ArgumentParser()
screamer	22:9e85236d8716	1796	parser.add_argument('-i', '--tests',
screamer	22:9e85236d8716	1797	dest='test_spec_filename',
screamer	22:9e85236d8716	1798	metavar="FILE",
screamer	22:9e85236d8716	1799	type=argparse_filestring_type,
screamer	22:9e85236d8716	1800	help='Points to file with test specification')
screamer	0:66f3b5499f7f	1801
screamer	22:9e85236d8716	1802	parser.add_argument('-M', '--MUTS',
screamer	22:9e85236d8716	1803	dest='muts_spec_filename',
screamer	22:9e85236d8716	1804	metavar="FILE",
screamer	22:9e85236d8716	1805	type=argparse_filestring_type,
screamer	22:9e85236d8716	1806	help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
screamer	0:66f3b5499f7f	1807
screamer	22:9e85236d8716	1808	parser.add_argument("-j", "--jobs",
screamer	22:9e85236d8716	1809	dest='jobs',
screamer	22:9e85236d8716	1810	metavar="NUMBER",
screamer	22:9e85236d8716	1811	type=int,
screamer	22:9e85236d8716	1812	help="Define number of compilation jobs. Default value is 1")
screamer	0:66f3b5499f7f	1813
screamer	0:66f3b5499f7f	1814	if get_module_avail('mbed_lstools'):
screamer	0:66f3b5499f7f	1815	# Additional features available when mbed_lstools is installed on host and imported
screamer	0:66f3b5499f7f	1816	# mbed_lstools allow users to detect connected to host mbed-enabled devices
screamer	22:9e85236d8716	1817	parser.add_argument('--auto',
screamer	22:9e85236d8716	1818	dest='auto_detect',
screamer	22:9e85236d8716	1819	action="store_true",
screamer	22:9e85236d8716	1820	help='Use mbed-ls module to detect all connected mbed devices')
screamer	0:66f3b5499f7f	1821
screamer	22:9e85236d8716	1822	toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
screamer	22:9e85236d8716	1823	parser.add_argument('--tc',
screamer	22:9e85236d8716	1824	dest='toolchains_filter',
screamer	22:9e85236d8716	1825	type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
screamer	22:9e85236d8716	1826	help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
screamer	0:66f3b5499f7f	1827
screamer	0:66f3b5499f7f	1828	test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
screamer	22:9e85236d8716	1829	parser.add_argument('--oper',
screamer	22:9e85236d8716	1830	dest='operability_checks',
screamer	22:9e85236d8716	1831	type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
screamer	22:9e85236d8716	1832	help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
screamer	0:66f3b5499f7f	1833
screamer	22:9e85236d8716	1834	parser.add_argument('--clean',
screamer	22:9e85236d8716	1835	dest='clean',
screamer	22:9e85236d8716	1836	action="store_true",
screamer	22:9e85236d8716	1837	help='Clean the build directory')
screamer	0:66f3b5499f7f	1838
screamer	22:9e85236d8716	1839	parser.add_argument('-P', '--only-peripherals',
screamer	22:9e85236d8716	1840	dest='test_only_peripheral',
screamer	22:9e85236d8716	1841	default=False,
screamer	22:9e85236d8716	1842	action="store_true",
screamer	22:9e85236d8716	1843	help='Test only peripheral declared for MUT and skip common tests')
screamer	0:66f3b5499f7f	1844
The Other Jimmy	31:8ea194f6145b	1845	parser.add_argument("--profile", dest="profile", action="append",
The Other Jimmy	31:8ea194f6145b	1846	type=argparse_filestring_type,
The Other Jimmy	31:8ea194f6145b	1847	default=[])
The Other Jimmy	31:8ea194f6145b	1848
screamer	22:9e85236d8716	1849	parser.add_argument('-C', '--only-commons',
screamer	22:9e85236d8716	1850	dest='test_only_common',
screamer	22:9e85236d8716	1851	default=False,
screamer	22:9e85236d8716	1852	action="store_true",
screamer	22:9e85236d8716	1853	help='Test only board internals. Skip perpherials tests and perform common tests')
screamer	0:66f3b5499f7f	1854
screamer	22:9e85236d8716	1855	parser.add_argument('-n', '--test-by-names',
screamer	22:9e85236d8716	1856	dest='test_by_names',
screamer	22:9e85236d8716	1857	type=argparse_many(str),
screamer	22:9e85236d8716	1858	help='Runs only test enumerated it this switch. Use comma to separate test case names')
screamer	0:66f3b5499f7f	1859
screamer	22:9e85236d8716	1860	parser.add_argument('-p', '--peripheral-by-names',
screamer	0:66f3b5499f7f	1861	dest='peripheral_by_names',
screamer	22:9e85236d8716	1862	type=argparse_many(str),
screamer	0:66f3b5499f7f	1863	help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
screamer	0:66f3b5499f7f	1864
screamer	0:66f3b5499f7f	1865	copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
screamer	0:66f3b5499f7f	1866	copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
screamer	0:66f3b5499f7f	1867
screamer	22:9e85236d8716	1868	parser.add_argument('-c', '--copy-method',
screamer	22:9e85236d8716	1869	dest='copy_method',
screamer	22:9e85236d8716	1870	type=argparse_uppercase_type(copy_methods, "flash method"),
screamer	22:9e85236d8716	1871	help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
screamer	0:66f3b5499f7f	1872
screamer	0:66f3b5499f7f	1873	reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
screamer	0:66f3b5499f7f	1874	reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
screamer	0:66f3b5499f7f	1875
screamer	22:9e85236d8716	1876	parser.add_argument('-r', '--reset-type',
screamer	22:9e85236d8716	1877	dest='mut_reset_type',
screamer	22:9e85236d8716	1878	default=None,
screamer	22:9e85236d8716	1879	type=argparse_uppercase_type(reset_methods, "reset method"),
screamer	22:9e85236d8716	1880	help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
screamer	0:66f3b5499f7f	1881
screamer	22:9e85236d8716	1882	parser.add_argument('-g', '--goanna-for-tests',
screamer	22:9e85236d8716	1883	dest='goanna_for_tests',
screamer	22:9e85236d8716	1884	action="store_true",
screamer	22:9e85236d8716	1885	help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
screamer	0:66f3b5499f7f	1886
screamer	22:9e85236d8716	1887	parser.add_argument('-G', '--goanna-for-sdk',
screamer	22:9e85236d8716	1888	dest='goanna_for_mbed_sdk',
screamer	22:9e85236d8716	1889	action="store_true",
screamer	22:9e85236d8716	1890	help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
screamer	0:66f3b5499f7f	1891
screamer	22:9e85236d8716	1892	parser.add_argument('-s', '--suppress-summary',
screamer	22:9e85236d8716	1893	dest='suppress_summary',
screamer	22:9e85236d8716	1894	default=False,
screamer	22:9e85236d8716	1895	action="store_true",
screamer	22:9e85236d8716	1896	help='Suppresses display of wellformatted table with test results')
screamer	22:9e85236d8716	1897
screamer	22:9e85236d8716	1898	parser.add_argument('-t', '--test-summary',
screamer	22:9e85236d8716	1899	dest='test_x_toolchain_summary',
screamer	22:9e85236d8716	1900	default=False,
screamer	22:9e85236d8716	1901	action="store_true",
screamer	22:9e85236d8716	1902	help='Displays wellformatted table with test x toolchain test result per target')
screamer	0:66f3b5499f7f	1903
screamer	22:9e85236d8716	1904	parser.add_argument('-A', '--test-automation-report',
screamer	22:9e85236d8716	1905	dest='test_automation_report',
screamer	22:9e85236d8716	1906	default=False,
screamer	22:9e85236d8716	1907	action="store_true",
screamer	22:9e85236d8716	1908	help='Prints information about all tests and exits')
screamer	0:66f3b5499f7f	1909
screamer	22:9e85236d8716	1910	parser.add_argument('-R', '--test-case-report',
screamer	22:9e85236d8716	1911	dest='test_case_report',
screamer	22:9e85236d8716	1912	default=False,
screamer	22:9e85236d8716	1913	action="store_true",
screamer	22:9e85236d8716	1914	help='Prints information about all test cases and exits')
screamer	0:66f3b5499f7f	1915
screamer	22:9e85236d8716	1916	parser.add_argument("-S", "--supported-toolchains",
screamer	22:9e85236d8716	1917	action="store_true",
screamer	22:9e85236d8716	1918	dest="supported_toolchains",
screamer	22:9e85236d8716	1919	default=False,
screamer	22:9e85236d8716	1920	help="Displays supported matrix of MCUs and toolchains")
screamer	0:66f3b5499f7f	1921
screamer	22:9e85236d8716	1922	parser.add_argument("-O", "--only-build",
screamer	22:9e85236d8716	1923	action="store_true",
screamer	22:9e85236d8716	1924	dest="only_build_tests",
screamer	22:9e85236d8716	1925	default=False,
screamer	22:9e85236d8716	1926	help="Only build tests, skips actual test procedures (flashing etc.)")
screamer	0:66f3b5499f7f	1927
screamer	22:9e85236d8716	1928	parser.add_argument('--parallel',
screamer	22:9e85236d8716	1929	dest='parallel_test_exec',
screamer	22:9e85236d8716	1930	default=False,
screamer	22:9e85236d8716	1931	action="store_true",
screamer	22:9e85236d8716	1932	help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
screamer	0:66f3b5499f7f	1933
screamer	22:9e85236d8716	1934	parser.add_argument('--config',
screamer	22:9e85236d8716	1935	dest='verbose_test_configuration_only',
screamer	22:9e85236d8716	1936	default=False,
screamer	22:9e85236d8716	1937	action="store_true",
screamer	22:9e85236d8716	1938	help='Displays full test specification and MUTs configration and exits')
screamer	22:9e85236d8716	1939
screamer	22:9e85236d8716	1940	parser.add_argument('--loops',
screamer	22:9e85236d8716	1941	dest='test_loops_list',
screamer	22:9e85236d8716	1942	type=argparse_many(str),
screamer	22:9e85236d8716	1943	help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
screamer	0:66f3b5499f7f	1944
screamer	22:9e85236d8716	1945	parser.add_argument('--global-loops',
screamer	22:9e85236d8716	1946	dest='test_global_loops_value',
screamer	22:9e85236d8716	1947	type=int,
screamer	22:9e85236d8716	1948	help='Set global number of test loops per test. Default value is set 1')
screamer	0:66f3b5499f7f	1949
screamer	22:9e85236d8716	1950	parser.add_argument('--consolidate-waterfall',
screamer	22:9e85236d8716	1951	dest='consolidate_waterfall_test',
screamer	22:9e85236d8716	1952	default=False,
screamer	22:9e85236d8716	1953	action="store_true",
screamer	22:9e85236d8716	1954	help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
screamer	0:66f3b5499f7f	1955
screamer	22:9e85236d8716	1956	parser.add_argument('-W', '--waterfall',
screamer	22:9e85236d8716	1957	dest='waterfall_test',
screamer	22:9e85236d8716	1958	default=False,
screamer	22:9e85236d8716	1959	action="store_true",
screamer	22:9e85236d8716	1960	help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
screamer	0:66f3b5499f7f	1961
screamer	22:9e85236d8716	1962	parser.add_argument('-N', '--firmware-name',
screamer	22:9e85236d8716	1963	dest='firmware_global_name',
screamer	22:9e85236d8716	1964	help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
screamer	0:66f3b5499f7f	1965
screamer	22:9e85236d8716	1966	parser.add_argument('-u', '--shuffle',
screamer	22:9e85236d8716	1967	dest='shuffle_test_order',
screamer	22:9e85236d8716	1968	default=False,
screamer	22:9e85236d8716	1969	action="store_true",
screamer	22:9e85236d8716	1970	help='Shuffles test execution order')
screamer	0:66f3b5499f7f	1971
screamer	22:9e85236d8716	1972	parser.add_argument('--shuffle-seed',
screamer	22:9e85236d8716	1973	dest='shuffle_test_seed',
screamer	22:9e85236d8716	1974	default=None,
screamer	22:9e85236d8716	1975	help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
screamer	0:66f3b5499f7f	1976
screamer	22:9e85236d8716	1977	parser.add_argument('-f', '--filter',
screamer	22:9e85236d8716	1978	dest='general_filter_regex',
screamer	22:9e85236d8716	1979	type=argparse_many(str),
screamer	22:9e85236d8716	1980	default=None,
screamer	22:9e85236d8716	1981	help='For some commands you can use filter to filter out results')
screamer	0:66f3b5499f7f	1982
screamer	22:9e85236d8716	1983	parser.add_argument('--inc-timeout',
screamer	22:9e85236d8716	1984	dest='extend_test_timeout',
screamer	22:9e85236d8716	1985	metavar="NUMBER",
screamer	22:9e85236d8716	1986	type=int,
screamer	22:9e85236d8716	1987	help='You can increase global timeout for each test by specifying additional test timeout in seconds')
screamer	22:9e85236d8716	1988
screamer	22:9e85236d8716	1989	parser.add_argument('--db',
screamer	22:9e85236d8716	1990	dest='db_url',
screamer	22:9e85236d8716	1991	help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
screamer	0:66f3b5499f7f	1992
screamer	22:9e85236d8716	1993	parser.add_argument('-l', '--log',
screamer	22:9e85236d8716	1994	dest='log_file_name',
screamer	22:9e85236d8716	1995	help='Log events to external file (note not all console entries may be visible in log file)')
screamer	0:66f3b5499f7f	1996
screamer	22:9e85236d8716	1997	parser.add_argument('--report-html',
screamer	22:9e85236d8716	1998	dest='report_html_file_name',
screamer	22:9e85236d8716	1999	help='You can log test suite results in form of HTML report')
screamer	0:66f3b5499f7f	2000
screamer	22:9e85236d8716	2001	parser.add_argument('--report-junit',
screamer	22:9e85236d8716	2002	dest='report_junit_file_name',
screamer	22:9e85236d8716	2003	help='You can log test suite results in form of JUnit compliant XML report')
screamer	0:66f3b5499f7f	2004
screamer	22:9e85236d8716	2005	parser.add_argument("--report-build",
screamer	22:9e85236d8716	2006	dest="report_build_file_name",
screamer	22:9e85236d8716	2007	help="Output the build results to a junit xml file")
screamer	0:66f3b5499f7f	2008
screamer	22:9e85236d8716	2009	parser.add_argument("--report-text",
screamer	22:9e85236d8716	2010	dest="report_text_file_name",
screamer	22:9e85236d8716	2011	help="Output the build results to a text file")
screamer	0:66f3b5499f7f	2012
screamer	22:9e85236d8716	2013	parser.add_argument('--verbose-skipped',
screamer	22:9e85236d8716	2014	dest='verbose_skipped_tests',
screamer	22:9e85236d8716	2015	default=False,
screamer	22:9e85236d8716	2016	action="store_true",
screamer	22:9e85236d8716	2017	help='Prints some extra information about skipped tests')
screamer	0:66f3b5499f7f	2018
screamer	22:9e85236d8716	2019	parser.add_argument('-V', '--verbose-test-result',
screamer	22:9e85236d8716	2020	dest='verbose_test_result_only',
screamer	22:9e85236d8716	2021	default=False,
screamer	22:9e85236d8716	2022	action="store_true",
screamer	22:9e85236d8716	2023	help='Prints test serial output')
screamer	0:66f3b5499f7f	2024
screamer	22:9e85236d8716	2025	parser.add_argument('-v', '--verbose',
screamer	22:9e85236d8716	2026	dest='verbose',
screamer	22:9e85236d8716	2027	default=False,
screamer	22:9e85236d8716	2028	action="store_true",
screamer	22:9e85236d8716	2029	help='Verbose mode (prints some extra information)')
screamer	0:66f3b5499f7f	2030
screamer	22:9e85236d8716	2031	parser.add_argument('--version',
screamer	22:9e85236d8716	2032	dest='version',
screamer	22:9e85236d8716	2033	default=False,
screamer	22:9e85236d8716	2034	action="store_true",
screamer	22:9e85236d8716	2035	help='Prints script version and exits')
theotherjimmy	40:7d3fa6b99b2b	2036
theotherjimmy	40:7d3fa6b99b2b	2037	parser.add_argument('--stats-depth',
theotherjimmy	40:7d3fa6b99b2b	2038	dest='stats_depth',
theotherjimmy	40:7d3fa6b99b2b	2039	default=2,
theotherjimmy	40:7d3fa6b99b2b	2040	type=int,
theotherjimmy	40:7d3fa6b99b2b	2041	help="Depth level for static memory report")
screamer	0:66f3b5499f7f	2042	return parser
screamer	0:66f3b5499f7f	2043
The Other Jimmy	31:8ea194f6145b	2044	def test_path_to_name(path, base):
screamer	0:66f3b5499f7f	2045	"""Change all slashes in a path into hyphens
screamer	0:66f3b5499f7f	2046	This creates a unique cross-platform test name based on the path
screamer	0:66f3b5499f7f	2047	This can eventually be overriden by a to-be-determined meta-data mechanism"""
screamer	0:66f3b5499f7f	2048	name_parts = []
The Other Jimmy	31:8ea194f6145b	2049	head, tail = os.path.split(relpath(path,base))
screamer	0:66f3b5499f7f	2050	while (tail and tail != "."):
screamer	0:66f3b5499f7f	2051	name_parts.insert(0, tail)
screamer	0:66f3b5499f7f	2052	head, tail = os.path.split(head)
screamer	22:9e85236d8716	2053
screamer	13:ab47a20b66f0	2054	return "-".join(name_parts).lower()
screamer	0:66f3b5499f7f	2055
theotherjimmy	41:2a77626a4c21	2056	def get_test_config(config_name, target_name):
theotherjimmy	41:2a77626a4c21	2057	"""Finds the path to a test configuration file
theotherjimmy	41:2a77626a4c21	2058	config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
theotherjimmy	41:2a77626a4c21	2059	target_name: name of target to determing if mbed OS interface given is valid
theotherjimmy	43:2a7da56ebd24	2060	returns path to config, will return None if no valid config is found
theotherjimmy	41:2a77626a4c21	2061	"""
theotherjimmy	41:2a77626a4c21	2062	# If they passed in a full path
theotherjimmy	41:2a77626a4c21	2063	if exists(config_name):
theotherjimmy	41:2a77626a4c21	2064	# This is a module config
theotherjimmy	41:2a77626a4c21	2065	return config_name
theotherjimmy	41:2a77626a4c21	2066	# Otherwise find the path to configuration file based on mbed OS interface
theotherjimmy	41:2a77626a4c21	2067	return TestConfig.get_config_path(config_name, target_name)
theotherjimmy	41:2a77626a4c21	2068
theotherjimmy	43:2a7da56ebd24	2069
theotherjimmy	43:2a7da56ebd24	2070	def find_tests(base_dir, target_name, toolchain_name, icetea, greentea, app_config=None):
screamer	24:25bff2709c20	2071	""" Finds all tests in a directory recursively
theotherjimmy	43:2a7da56ebd24	2072	:param base_dir: path to the directory to scan for tests (ex. 'path/to/project')
theotherjimmy	43:2a7da56ebd24	2073	:param target_name: name of the target to use for scanning (ex. 'K64F')
theotherjimmy	43:2a7da56ebd24	2074	:param toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
theotherjimmy	43:2a7da56ebd24	2075	:param icetea: icetea enabled
theotherjimmy	43:2a7da56ebd24	2076	:param greentea: greentea enabled
theotherjimmy	43:2a7da56ebd24	2077	:param app_config - location of a chosen mbed_app.json file
theotherjimmy	43:2a7da56ebd24	2078
theotherjimmy	43:2a7da56ebd24	2079	returns a dictionary where keys are the test name, and the values are
theotherjimmy	43:2a7da56ebd24	2080	lists of paths needed to biuld the test.
screamer	24:25bff2709c20	2081	"""
screamer	24:25bff2709c20	2082
theotherjimmy	43:2a7da56ebd24	2083	# Temporary structure: tests referenced by (name, base, group, case) tuple
screamer	24:25bff2709c20	2084	tests = {}
theotherjimmy	43:2a7da56ebd24	2085	# List of common folders: (predicate function, path) tuple
theotherjimmy	43:2a7da56ebd24	2086	commons = []
screamer	24:25bff2709c20	2087
theotherjimmy	43:2a7da56ebd24	2088	config = Config(target_name, base_dir, app_config)
screamer	24:25bff2709c20	2089
screamer	24:25bff2709c20	2090	# Scan the directory for paths to probe for 'TESTS' folders
theotherjimmy	43:2a7da56ebd24	2091	base_resources = Resources(MockNotifier(), collect_ignores=True)
theotherjimmy	43:2a7da56ebd24	2092	base_resources.scan_with_config(base_dir, config)
screamer	24:25bff2709c20	2093
theotherjimmy	43:2a7da56ebd24	2094	if greentea:
theotherjimmy	43:2a7da56ebd24	2095	dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TESTS']
theotherjimmy	43:2a7da56ebd24	2096	ignoreset = MbedIgnoreSet()
screamer	22:9e85236d8716	2097
theotherjimmy	43:2a7da56ebd24	2098	for directory in dirs:
theotherjimmy	43:2a7da56ebd24	2099	ignorefile = join(directory, IGNORE_FILENAME)
theotherjimmy	43:2a7da56ebd24	2100	if isfile(ignorefile):
theotherjimmy	43:2a7da56ebd24	2101	ignoreset.add_mbedignore(directory, ignorefile)
theotherjimmy	43:2a7da56ebd24	2102	for test_group_directory in os.listdir(directory):
theotherjimmy	43:2a7da56ebd24	2103	grp_dir = join(directory, test_group_directory)
theotherjimmy	43:2a7da56ebd24	2104	if not isdir(grp_dir) or ignoreset.is_ignored(grp_dir):
theotherjimmy	43:2a7da56ebd24	2105	continue
theotherjimmy	43:2a7da56ebd24	2106	grpignorefile = join(grp_dir, IGNORE_FILENAME)
theotherjimmy	43:2a7da56ebd24	2107	if isfile(grpignorefile):
theotherjimmy	43:2a7da56ebd24	2108	ignoreset.add_mbedignore(grp_dir, grpignorefile)
theotherjimmy	43:2a7da56ebd24	2109	for test_case_directory in os.listdir(grp_dir):
theotherjimmy	43:2a7da56ebd24	2110	d = join(directory, test_group_directory, test_case_directory)
theotherjimmy	43:2a7da56ebd24	2111	if not isdir(d) or ignoreset.is_ignored(d):
theotherjimmy	43:2a7da56ebd24	2112	continue
theotherjimmy	43:2a7da56ebd24	2113	special_dirs = ['host_tests', 'COMMON']
theotherjimmy	43:2a7da56ebd24	2114	if test_group_directory not in special_dirs and test_case_directory not in special_dirs:
theotherjimmy	43:2a7da56ebd24	2115	test_name = test_path_to_name(d, base_dir)
theotherjimmy	43:2a7da56ebd24	2116	tests[(test_name, directory, test_group_directory, test_case_directory)] = [d]
theotherjimmy	43:2a7da56ebd24	2117	if test_case_directory == 'COMMON':
theotherjimmy	43:2a7da56ebd24	2118	def predicate(base_pred, group_pred, name_base_group_case):
theotherjimmy	43:2a7da56ebd24	2119	(name, base, group, case) = name_base_group_case
theotherjimmy	43:2a7da56ebd24	2120	return base == base_pred and group == group_pred
screamer	22:9e85236d8716	2121
theotherjimmy	43:2a7da56ebd24	2122	commons.append((functools.partial(predicate, directory, test_group_directory), d))
theotherjimmy	43:2a7da56ebd24	2123	if test_group_directory == 'COMMON':
theotherjimmy	43:2a7da56ebd24	2124	def predicate(base_pred, name_base_group_case):
theotherjimmy	43:2a7da56ebd24	2125	(name, base, group, case) = name_base_group_case
theotherjimmy	43:2a7da56ebd24	2126	return base == base_pred
theotherjimmy	43:2a7da56ebd24	2127
theotherjimmy	43:2a7da56ebd24	2128	commons.append((functools.partial(predicate, directory), grp_dir))
theotherjimmy	40:7d3fa6b99b2b	2129
theotherjimmy	43:2a7da56ebd24	2130	if icetea:
theotherjimmy	43:2a7da56ebd24	2131	dirs = [d for d in base_resources.ignored_dirs if basename(d) == 'TEST_APPS']
theotherjimmy	43:2a7da56ebd24	2132	for directory in dirs:
theotherjimmy	43:2a7da56ebd24	2133	if not isdir(directory):
theotherjimmy	43:2a7da56ebd24	2134	continue
theotherjimmy	43:2a7da56ebd24	2135	for subdir in os.listdir(directory):
theotherjimmy	43:2a7da56ebd24	2136	d = join(directory, subdir)
theotherjimmy	43:2a7da56ebd24	2137	if not isdir(d):
theotherjimmy	43:2a7da56ebd24	2138	continue
theotherjimmy	43:2a7da56ebd24	2139	if 'device' == subdir:
theotherjimmy	43:2a7da56ebd24	2140	for test_dir in os.listdir(d):
theotherjimmy	43:2a7da56ebd24	2141	test_dir_path = join(d, test_dir)
theotherjimmy	43:2a7da56ebd24	2142	test_name = test_path_to_name(test_dir_path, base_dir)
theotherjimmy	43:2a7da56ebd24	2143	tests[(test_name, directory, subdir, test_dir)] = [test_dir_path]
screamer	22:9e85236d8716	2144
theotherjimmy	43:2a7da56ebd24	2145	# Apply common directories
theotherjimmy	43:2a7da56ebd24	2146	for pred, path in commons:
theotherjimmy	43:2a7da56ebd24	2147	for test_identity, test_paths in six.iteritems(tests):
theotherjimmy	43:2a7da56ebd24	2148	if pred(test_identity):
theotherjimmy	43:2a7da56ebd24	2149	test_paths.append(path)
theotherjimmy	43:2a7da56ebd24	2150
theotherjimmy	43:2a7da56ebd24	2151	# Drop identity besides name
theotherjimmy	43:2a7da56ebd24	2152	return {name: paths for (name, _, _, _), paths in six.iteritems(tests)}
theotherjimmy	43:2a7da56ebd24	2153
screamer	22:9e85236d8716	2154
screamer	13:ab47a20b66f0	2155	def print_tests(tests, format="list", sort=True):
screamer	0:66f3b5499f7f	2156	"""Given a dictionary of tests (as returned from "find_tests"), print them
screamer	0:66f3b5499f7f	2157	in the specified format"""
screamer	0:66f3b5499f7f	2158	if format == "list":
screamer	13:ab47a20b66f0	2159	for test_name in sorted(tests.keys()):
theotherjimmy	43:2a7da56ebd24	2160	test_path = tests[test_name][0]
theotherjimmy	43:2a7da56ebd24	2161	print("Test Case:")
theotherjimmy	43:2a7da56ebd24	2162	print(" Name: %s" % test_name)
theotherjimmy	43:2a7da56ebd24	2163	print(" Path: %s" % test_path)
screamer	0:66f3b5499f7f	2164	elif format == "json":
theotherjimmy	43:2a7da56ebd24	2165	print(json.dumps({test_name: test_path[0] for test_name, test_paths
theotherjimmy	43:2a7da56ebd24	2166	in tests}, indent=2))
screamer	0:66f3b5499f7f	2167	else:
theotherjimmy	43:2a7da56ebd24	2168	print("Unknown format '%s'" % format)
screamer	0:66f3b5499f7f	2169	sys.exit(1)
screamer	0:66f3b5499f7f	2170
screamer	13:ab47a20b66f0	2171	def norm_relative_path(path, start):
screamer	13:ab47a20b66f0	2172	"""This function will create a normalized, relative path. It mimics the
screamer	13:ab47a20b66f0	2173	python os.path.relpath function, but also normalizes a Windows-syle path
screamer	13:ab47a20b66f0	2174	that use backslashes to a Unix style path that uses forward slashes."""
screamer	13:ab47a20b66f0	2175	path = os.path.normpath(path)
screamer	13:ab47a20b66f0	2176	path = os.path.relpath(path, start)
screamer	13:ab47a20b66f0	2177	path = path.replace("\\", "/")
screamer	13:ab47a20b66f0	2178	return path
screamer	13:ab47a20b66f0	2179
The Other Jimmy	31:8ea194f6145b	2180
The Other Jimmy	31:8ea194f6145b	2181	def build_test_worker(args, *kwargs):
The Other Jimmy	31:8ea194f6145b	2182	"""This is a worker function for the parallel building of tests. The `args`
The Other Jimmy	31:8ea194f6145b	2183	and `kwargs` are passed directly to `build_project`. It returns a dictionary
The Other Jimmy	31:8ea194f6145b	2184	with the following structure:
The Other Jimmy	31:8ea194f6145b	2185
The Other Jimmy	31:8ea194f6145b	2186	{
The Other Jimmy	31:8ea194f6145b	2187	'result': `True` if no exceptions were thrown, `False` otherwise
The Other Jimmy	31:8ea194f6145b	2188	'reason': Instance of exception that was thrown on failure
The Other Jimmy	31:8ea194f6145b	2189	'bin_file': Path to the created binary if `build_project` was
The Other Jimmy	31:8ea194f6145b	2190	successful. Not present otherwise
The Other Jimmy	31:8ea194f6145b	2191	'kwargs': The keyword arguments that were passed to `build_project`.
The Other Jimmy	31:8ea194f6145b	2192	This includes arguments that were modified (ex. report)
The Other Jimmy	31:8ea194f6145b	2193	}
The Other Jimmy	31:8ea194f6145b	2194	"""
The Other Jimmy	31:8ea194f6145b	2195	bin_file = None
The Other Jimmy	31:8ea194f6145b	2196	ret = {
The Other Jimmy	31:8ea194f6145b	2197	'result': False,
The Other Jimmy	31:8ea194f6145b	2198	'args': args,
The Other Jimmy	31:8ea194f6145b	2199	'kwargs': kwargs
The Other Jimmy	31:8ea194f6145b	2200	}
The Other Jimmy	31:8ea194f6145b	2201
The Other Jimmy	31:8ea194f6145b	2202	# Use parent TOOLCHAIN_PATHS variable
theotherjimmy	43:2a7da56ebd24	2203	for key, value in kwargs['toolchain_paths'].items():
The Other Jimmy	31:8ea194f6145b	2204	TOOLCHAIN_PATHS[key] = value
The Other Jimmy	31:8ea194f6145b	2205
The Other Jimmy	31:8ea194f6145b	2206	del kwargs['toolchain_paths']
The Other Jimmy	31:8ea194f6145b	2207
The Other Jimmy	31:8ea194f6145b	2208	try:
theotherjimmy	43:2a7da56ebd24	2209	bin_file, _ = build_project(args, *kwargs)
The Other Jimmy	31:8ea194f6145b	2210	ret['result'] = True
The Other Jimmy	31:8ea194f6145b	2211	ret['bin_file'] = bin_file
The Other Jimmy	31:8ea194f6145b	2212	ret['kwargs'] = kwargs
The Other Jimmy	31:8ea194f6145b	2213
theotherjimmy	43:2a7da56ebd24	2214	except NotSupportedException as e:
The Other Jimmy	31:8ea194f6145b	2215	ret['reason'] = e
theotherjimmy	43:2a7da56ebd24	2216	except ToolException as e:
The Other Jimmy	31:8ea194f6145b	2217	ret['reason'] = e
theotherjimmy	43:2a7da56ebd24	2218	except KeyboardInterrupt as e:
The Other Jimmy	31:8ea194f6145b	2219	ret['reason'] = e
The Other Jimmy	31:8ea194f6145b	2220	except:
The Other Jimmy	31:8ea194f6145b	2221	# Print unhandled exceptions here
The Other Jimmy	31:8ea194f6145b	2222	import traceback
The Other Jimmy	31:8ea194f6145b	2223	traceback.print_exc(file=sys.stdout)
The Other Jimmy	31:8ea194f6145b	2224
The Other Jimmy	31:8ea194f6145b	2225	return ret
The Other Jimmy	31:8ea194f6145b	2226
The Other Jimmy	31:8ea194f6145b	2227
screamer	0:66f3b5499f7f	2228	def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
theotherjimmy	43:2a7da56ebd24	2229	clean=False, notify=None, jobs=1, macros=None,
The Other Jimmy	31:8ea194f6145b	2230	silent=False, report=None, properties=None,
The Other Jimmy	31:8ea194f6145b	2231	continue_on_build_fail=False, app_config=None,
theotherjimmy	43:2a7da56ebd24	2232	build_profile=None, stats_depth=None, ignore=None):
screamer	0:66f3b5499f7f	2233	"""Given the data structure from 'find_tests' and the typical build parameters,
screamer	13:ab47a20b66f0	2234	build all the tests
screamer	22:9e85236d8716	2235
screamer	13:ab47a20b66f0	2236	Returns a tuple of the build result (True or False) followed by the test
screamer	13:ab47a20b66f0	2237	build data structure"""
screamer	22:9e85236d8716	2238
screamer	13:ab47a20b66f0	2239	execution_directory = "."
screamer	13:ab47a20b66f0	2240	base_path = norm_relative_path(build_path, execution_directory)
screamer	13:ab47a20b66f0	2241
theotherjimmy	43:2a7da56ebd24	2242	if isinstance(target, Target):
theotherjimmy	43:2a7da56ebd24	2243	target_name = target.name
theotherjimmy	43:2a7da56ebd24	2244	else:
theotherjimmy	43:2a7da56ebd24	2245	target_name = target
theotherjimmy	43:2a7da56ebd24	2246	target = TARGET_MAP[target_name]
theotherjimmy	43:2a7da56ebd24	2247	cfg, _, _ = get_config(base_source_paths, target, app_config=app_config)
The Other Jimmy	31:8ea194f6145b	2248
The Other Jimmy	31:8ea194f6145b	2249	baud_rate = 9600
The Other Jimmy	31:8ea194f6145b	2250	if 'platform.stdio-baud-rate' in cfg:
The Other Jimmy	31:8ea194f6145b	2251	baud_rate = cfg['platform.stdio-baud-rate'].value
The Other Jimmy	31:8ea194f6145b	2252
screamer	0:66f3b5499f7f	2253	test_build = {
screamer	13:ab47a20b66f0	2254	"platform": target_name,
screamer	0:66f3b5499f7f	2255	"toolchain": toolchain_name,
screamer	13:ab47a20b66f0	2256	"base_path": base_path,
The Other Jimmy	31:8ea194f6145b	2257	"baud_rate": baud_rate,
screamer	0:66f3b5499f7f	2258	"binary_type": "bootable",
theotherjimmy	43:2a7da56ebd24	2259	"tests": {},
theotherjimmy	43:2a7da56ebd24	2260	"test_apps": {}
screamer	0:66f3b5499f7f	2261	}
screamer	22:9e85236d8716	2262
screamer	13:ab47a20b66f0	2263	result = True
screamer	22:9e85236d8716	2264
The Other Jimmy	31:8ea194f6145b	2265	jobs_count = int(jobs if jobs else cpu_count())
The Other Jimmy	31:8ea194f6145b	2266	p = Pool(processes=jobs_count)
The Other Jimmy	31:8ea194f6145b	2267	results = []
theotherjimmy	43:2a7da56ebd24	2268	for test_name, test_paths in tests.items():
theotherjimmy	43:2a7da56ebd24	2269	if not isinstance(test_paths, list):
theotherjimmy	43:2a7da56ebd24	2270	test_paths = [test_paths]
theotherjimmy	43:2a7da56ebd24	2271
theotherjimmy	43:2a7da56ebd24	2272	test_build_path = os.path.join(build_path, test_paths[0])
theotherjimmy	43:2a7da56ebd24	2273	src_paths = base_source_paths + test_paths
screamer	13:ab47a20b66f0	2274	bin_file = None
theotherjimmy	43:2a7da56ebd24	2275	test_case_folder_name = os.path.basename(test_paths[0])
theotherjimmy	40:7d3fa6b99b2b	2276
theotherjimmy	43:2a7da56ebd24	2277	args = (src_paths, test_build_path, deepcopy(target), toolchain_name)
The Other Jimmy	31:8ea194f6145b	2278	kwargs = {
The Other Jimmy	31:8ea194f6145b	2279	'jobs': 1,
The Other Jimmy	31:8ea194f6145b	2280	'clean': clean,
The Other Jimmy	31:8ea194f6145b	2281	'macros': macros,
The Other Jimmy	31:8ea194f6145b	2282	'name': test_case_folder_name,
The Other Jimmy	31:8ea194f6145b	2283	'project_id': test_name,
The Other Jimmy	31:8ea194f6145b	2284	'report': report,
The Other Jimmy	31:8ea194f6145b	2285	'properties': properties,
The Other Jimmy	31:8ea194f6145b	2286	'app_config': app_config,
The Other Jimmy	31:8ea194f6145b	2287	'build_profile': build_profile,
theotherjimmy	40:7d3fa6b99b2b	2288	'toolchain_paths': TOOLCHAIN_PATHS,
theotherjimmy	43:2a7da56ebd24	2289	'stats_depth': stats_depth,
theotherjimmy	43:2a7da56ebd24	2290	'notify': MockNotifier()
The Other Jimmy	31:8ea194f6145b	2291	}
theotherjimmy	40:7d3fa6b99b2b	2292
The Other Jimmy	31:8ea194f6145b	2293	results.append(p.apply_async(build_test_worker, args, kwargs))
The Other Jimmy	31:8ea194f6145b	2294
The Other Jimmy	31:8ea194f6145b	2295	p.close()
The Other Jimmy	31:8ea194f6145b	2296	result = True
The Other Jimmy	31:8ea194f6145b	2297	itr = 0
The Other Jimmy	31:8ea194f6145b	2298	while len(results):
The Other Jimmy	31:8ea194f6145b	2299	itr += 1
The Other Jimmy	31:8ea194f6145b	2300	if itr > 360000:
The Other Jimmy	31:8ea194f6145b	2301	p.terminate()
The Other Jimmy	31:8ea194f6145b	2302	p.join()
The Other Jimmy	31:8ea194f6145b	2303	raise ToolException("Compile did not finish in 10 minutes")
The Other Jimmy	31:8ea194f6145b	2304	else:
The Other Jimmy	31:8ea194f6145b	2305	sleep(0.01)
The Other Jimmy	31:8ea194f6145b	2306	pending = 0
The Other Jimmy	31:8ea194f6145b	2307	for r in results:
The Other Jimmy	31:8ea194f6145b	2308	if r.ready() is True:
The Other Jimmy	31:8ea194f6145b	2309	try:
The Other Jimmy	31:8ea194f6145b	2310	worker_result = r.get()
The Other Jimmy	31:8ea194f6145b	2311	results.remove(r)
screamer	13:ab47a20b66f0	2312
theotherjimmy	43:2a7da56ebd24	2313	# Push all deferred notifications out to the actual notifier
theotherjimmy	43:2a7da56ebd24	2314	new_notify = deepcopy(notify)
theotherjimmy	43:2a7da56ebd24	2315	for message in worker_result['kwargs']['notify'].messages:
theotherjimmy	43:2a7da56ebd24	2316	new_notify.notify(message)
theotherjimmy	43:2a7da56ebd24	2317
The Other Jimmy	31:8ea194f6145b	2318	# Take report from the kwargs and merge it into existing report
theotherjimmy	40:7d3fa6b99b2b	2319	if report:
theotherjimmy	40:7d3fa6b99b2b	2320	report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
theotherjimmy	43:2a7da56ebd24	2321	report_entry[worker_result['kwargs']['project_id'].upper()][0][0]['output'] = new_notify.get_output()
theotherjimmy	40:7d3fa6b99b2b	2322	for test_key in report_entry.keys():
theotherjimmy	40:7d3fa6b99b2b	2323	report[target_name][toolchain_name][test_key] = report_entry[test_key]
theotherjimmy	40:7d3fa6b99b2b	2324
The Other Jimmy	31:8ea194f6145b	2325	# Set the overall result to a failure if a build failure occurred
The Other Jimmy	35:da9c89f8be7d	2326	if ('reason' in worker_result and
The Other Jimmy	35:da9c89f8be7d	2327	not worker_result['reason'] and
The Other Jimmy	35:da9c89f8be7d	2328	not isinstance(worker_result['reason'], NotSupportedException)):
The Other Jimmy	31:8ea194f6145b	2329	result = False
The Other Jimmy	31:8ea194f6145b	2330	break
The Other Jimmy	31:8ea194f6145b	2331
theotherjimmy	43:2a7da56ebd24	2332
The Other Jimmy	31:8ea194f6145b	2333	# Adding binary path to test build result
The Other Jimmy	35:da9c89f8be7d	2334	if ('result' in worker_result and
The Other Jimmy	35:da9c89f8be7d	2335	worker_result['result'] and
The Other Jimmy	35:da9c89f8be7d	2336	'bin_file' in worker_result):
The Other Jimmy	31:8ea194f6145b	2337	bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
screamer	22:9e85236d8716	2338
theotherjimmy	43:2a7da56ebd24	2339	test_key = 'test_apps' if 'test_apps-' in worker_result['kwargs']['project_id'] else 'tests'
theotherjimmy	43:2a7da56ebd24	2340	test_build[test_key][worker_result['kwargs']['project_id']] = {
The Other Jimmy	31:8ea194f6145b	2341	"binaries": [
The Other Jimmy	31:8ea194f6145b	2342	{
The Other Jimmy	31:8ea194f6145b	2343	"path": bin_file
The Other Jimmy	31:8ea194f6145b	2344	}
The Other Jimmy	31:8ea194f6145b	2345	]
The Other Jimmy	31:8ea194f6145b	2346	}
The Other Jimmy	31:8ea194f6145b	2347
The Other Jimmy	31:8ea194f6145b	2348	test_key = worker_result['kwargs']['project_id'].upper()
theotherjimmy	43:2a7da56ebd24	2349	print('Image: %s\n' % bin_file)
The Other Jimmy	31:8ea194f6145b	2350
The Other Jimmy	31:8ea194f6145b	2351	except:
The Other Jimmy	31:8ea194f6145b	2352	if p._taskqueue.queue:
The Other Jimmy	31:8ea194f6145b	2353	p._taskqueue.queue.clear()
The Other Jimmy	31:8ea194f6145b	2354	sleep(0.5)
The Other Jimmy	31:8ea194f6145b	2355	p.terminate()
The Other Jimmy	31:8ea194f6145b	2356	p.join()
The Other Jimmy	31:8ea194f6145b	2357	raise
screamer	13:ab47a20b66f0	2358	else:
The Other Jimmy	31:8ea194f6145b	2359	pending += 1
The Other Jimmy	31:8ea194f6145b	2360	if pending >= jobs_count:
The Other Jimmy	31:8ea194f6145b	2361	break
screamer	22:9e85236d8716	2362
The Other Jimmy	31:8ea194f6145b	2363	# Break as soon as possible if there is a failure and we are not
The Other Jimmy	31:8ea194f6145b	2364	# continuing on build failures
The Other Jimmy	31:8ea194f6145b	2365	if not result and not continue_on_build_fail:
The Other Jimmy	31:8ea194f6145b	2366	if p._taskqueue.queue:
The Other Jimmy	31:8ea194f6145b	2367	p._taskqueue.queue.clear()
The Other Jimmy	31:8ea194f6145b	2368	sleep(0.5)
The Other Jimmy	31:8ea194f6145b	2369	p.terminate()
The Other Jimmy	31:8ea194f6145b	2370	break
screamer	22:9e85236d8716	2371
The Other Jimmy	31:8ea194f6145b	2372	p.join()
screamer	22:9e85236d8716	2373
screamer	0:66f3b5499f7f	2374	test_builds = {}
screamer	13:ab47a20b66f0	2375	test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
screamer	22:9e85236d8716	2376
screamer	13:ab47a20b66f0	2377	return result, test_builds
screamer	22:9e85236d8716	2378
screamer	0:66f3b5499f7f	2379
screamer	13:ab47a20b66f0	2380	def test_spec_from_test_builds(test_builds):
screamer	0:66f3b5499f7f	2381	return {
screamer	0:66f3b5499f7f	2382	"builds": test_builds
theotherjimmy	43:2a7da56ebd24	2383	}

Repository toolbox

Export to desktop IDE

Repository details

Type:	Library
Created:	04 Feb 2021
Imports:	8
Forks:	0
Commits:	48
Dependents:	0
Dependencies:	0
Followers:	2

test_api.py@47:21ae3e5a7128, 2021-02-04 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Access Warning