mbed-os - mbed-os

EL4121 Embedded System » Code » mbed-os

mbed-os

Dependents: cobaLCDJoyMotor_Thread odometry_omni_3roda_v3 odometry_omni_3roda_v1 odometry_omni_3roda_v2 ... more

tools/test_api.py@0:b74591d5ab33, 2017-12-11 (annotated)

Committer:: be_bryan
Date:: Mon Dec 11 17:54:04 2017 +0000
Revision:: 0:b74591d5ab33

motor ++

Who changed what in which revision?

User	Revision	Line number	New contents of line
be_bryan	0:b74591d5ab33	1	"""
be_bryan	0:b74591d5ab33	2	mbed SDK
be_bryan	0:b74591d5ab33	3	Copyright (c) 2011-2014 ARM Limited
be_bryan	0:b74591d5ab33	4
be_bryan	0:b74591d5ab33	5	Licensed under the Apache License, Version 2.0 (the "License");
be_bryan	0:b74591d5ab33	6	you may not use this file except in compliance with the License.
be_bryan	0:b74591d5ab33	7	You may obtain a copy of the License at
be_bryan	0:b74591d5ab33	8
be_bryan	0:b74591d5ab33	9	http://www.apache.org/licenses/LICENSE-2.0
be_bryan	0:b74591d5ab33	10
be_bryan	0:b74591d5ab33	11	Unless required by applicable law or agreed to in writing, software
be_bryan	0:b74591d5ab33	12	distributed under the License is distributed on an "AS IS" BASIS,
be_bryan	0:b74591d5ab33	13	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
be_bryan	0:b74591d5ab33	14	See the License for the specific language governing permissions and
be_bryan	0:b74591d5ab33	15	limitations under the License.
be_bryan	0:b74591d5ab33	16
be_bryan	0:b74591d5ab33	17	Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
be_bryan	0:b74591d5ab33	18	"""
be_bryan	0:b74591d5ab33	19
be_bryan	0:b74591d5ab33	20	import os
be_bryan	0:b74591d5ab33	21	import re
be_bryan	0:b74591d5ab33	22	import sys
be_bryan	0:b74591d5ab33	23	import json
be_bryan	0:b74591d5ab33	24	import uuid
be_bryan	0:b74591d5ab33	25	import pprint
be_bryan	0:b74591d5ab33	26	import random
be_bryan	0:b74591d5ab33	27	import argparse
be_bryan	0:b74591d5ab33	28	import datetime
be_bryan	0:b74591d5ab33	29	import threading
be_bryan	0:b74591d5ab33	30	import ctypes
be_bryan	0:b74591d5ab33	31	from types import ListType
be_bryan	0:b74591d5ab33	32	from colorama import Fore, Back, Style
be_bryan	0:b74591d5ab33	33	from prettytable import PrettyTable
be_bryan	0:b74591d5ab33	34	from copy import copy
be_bryan	0:b74591d5ab33	35
be_bryan	0:b74591d5ab33	36	from time import sleep, time
be_bryan	0:b74591d5ab33	37	from Queue import Queue, Empty
be_bryan	0:b74591d5ab33	38	from os.path import join, exists, basename, relpath
be_bryan	0:b74591d5ab33	39	from threading import Thread, Lock
be_bryan	0:b74591d5ab33	40	from multiprocessing import Pool, cpu_count
be_bryan	0:b74591d5ab33	41	from subprocess import Popen, PIPE
be_bryan	0:b74591d5ab33	42
be_bryan	0:b74591d5ab33	43	# Imports related to mbed build api
be_bryan	0:b74591d5ab33	44	from tools.tests import TESTS
be_bryan	0:b74591d5ab33	45	from tools.tests import TEST_MAP
be_bryan	0:b74591d5ab33	46	from tools.paths import BUILD_DIR
be_bryan	0:b74591d5ab33	47	from tools.paths import HOST_TESTS
be_bryan	0:b74591d5ab33	48	from tools.utils import ToolException
be_bryan	0:b74591d5ab33	49	from tools.utils import NotSupportedException
be_bryan	0:b74591d5ab33	50	from tools.utils import construct_enum
be_bryan	0:b74591d5ab33	51	from tools.memap import MemapParser
be_bryan	0:b74591d5ab33	52	from tools.targets import TARGET_MAP
be_bryan	0:b74591d5ab33	53	import tools.test_configs as TestConfig
be_bryan	0:b74591d5ab33	54	from tools.test_db import BaseDBAccess
be_bryan	0:b74591d5ab33	55	from tools.build_api import build_project, build_mbed_libs, build_lib
be_bryan	0:b74591d5ab33	56	from tools.build_api import get_target_supported_toolchains
be_bryan	0:b74591d5ab33	57	from tools.build_api import write_build_report
be_bryan	0:b74591d5ab33	58	from tools.build_api import prep_report
be_bryan	0:b74591d5ab33	59	from tools.build_api import prep_properties
be_bryan	0:b74591d5ab33	60	from tools.build_api import create_result
be_bryan	0:b74591d5ab33	61	from tools.build_api import add_result_to_report
be_bryan	0:b74591d5ab33	62	from tools.build_api import prepare_toolchain
be_bryan	0:b74591d5ab33	63	from tools.build_api import scan_resources
be_bryan	0:b74591d5ab33	64	from tools.build_api import get_config
be_bryan	0:b74591d5ab33	65	from tools.libraries import LIBRARIES, LIBRARY_MAP
be_bryan	0:b74591d5ab33	66	from tools.options import extract_profile
be_bryan	0:b74591d5ab33	67	from tools.toolchains import TOOLCHAIN_PATHS
be_bryan	0:b74591d5ab33	68	from tools.toolchains import TOOLCHAINS
be_bryan	0:b74591d5ab33	69	from tools.test_exporters import ReportExporter, ResultExporterType
be_bryan	0:b74591d5ab33	70	from tools.utils import argparse_filestring_type
be_bryan	0:b74591d5ab33	71	from tools.utils import argparse_uppercase_type
be_bryan	0:b74591d5ab33	72	from tools.utils import argparse_lowercase_type
be_bryan	0:b74591d5ab33	73	from tools.utils import argparse_many
be_bryan	0:b74591d5ab33	74	from tools.utils import get_path_depth
be_bryan	0:b74591d5ab33	75
be_bryan	0:b74591d5ab33	76	import tools.host_tests.host_tests_plugins as host_tests_plugins
be_bryan	0:b74591d5ab33	77
be_bryan	0:b74591d5ab33	78	try:
be_bryan	0:b74591d5ab33	79	import mbed_lstools
be_bryan	0:b74591d5ab33	80	from tools.compliance.ioper_runner import get_available_oper_test_scopes
be_bryan	0:b74591d5ab33	81	except:
be_bryan	0:b74591d5ab33	82	pass
be_bryan	0:b74591d5ab33	83
be_bryan	0:b74591d5ab33	84
be_bryan	0:b74591d5ab33	85	class ProcessObserver(Thread):
be_bryan	0:b74591d5ab33	86	def __init__(self, proc):
be_bryan	0:b74591d5ab33	87	Thread.__init__(self)
be_bryan	0:b74591d5ab33	88	self.proc = proc
be_bryan	0:b74591d5ab33	89	self.queue = Queue()
be_bryan	0:b74591d5ab33	90	self.daemon = True
be_bryan	0:b74591d5ab33	91	self.active = True
be_bryan	0:b74591d5ab33	92	self.start()
be_bryan	0:b74591d5ab33	93
be_bryan	0:b74591d5ab33	94	def run(self):
be_bryan	0:b74591d5ab33	95	while self.active:
be_bryan	0:b74591d5ab33	96	c = self.proc.stdout.read(1)
be_bryan	0:b74591d5ab33	97	self.queue.put(c)
be_bryan	0:b74591d5ab33	98
be_bryan	0:b74591d5ab33	99	def stop(self):
be_bryan	0:b74591d5ab33	100	self.active = False
be_bryan	0:b74591d5ab33	101	try:
be_bryan	0:b74591d5ab33	102	self.proc.terminate()
be_bryan	0:b74591d5ab33	103	except Exception, _:
be_bryan	0:b74591d5ab33	104	pass
be_bryan	0:b74591d5ab33	105
be_bryan	0:b74591d5ab33	106
be_bryan	0:b74591d5ab33	107	class SingleTestExecutor(threading.Thread):
be_bryan	0:b74591d5ab33	108	""" Example: Single test class in separate thread usage
be_bryan	0:b74591d5ab33	109	"""
be_bryan	0:b74591d5ab33	110	def __init__(self, single_test):
be_bryan	0:b74591d5ab33	111	self.single_test = single_test
be_bryan	0:b74591d5ab33	112	threading.Thread.__init__(self)
be_bryan	0:b74591d5ab33	113
be_bryan	0:b74591d5ab33	114	def run(self):
be_bryan	0:b74591d5ab33	115	start = time()
be_bryan	0:b74591d5ab33	116	# Execute tests depending on options and filter applied
be_bryan	0:b74591d5ab33	117	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext = self.single_test.execute()
be_bryan	0:b74591d5ab33	118	elapsed_time = time() - start
be_bryan	0:b74591d5ab33	119
be_bryan	0:b74591d5ab33	120	# Human readable summary
be_bryan	0:b74591d5ab33	121	if not self.single_test.opts_suppress_summary:
be_bryan	0:b74591d5ab33	122	# prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	123	print self.single_test.generate_test_summary(test_summary, shuffle_seed)
be_bryan	0:b74591d5ab33	124	if self.single_test.opts_test_x_toolchain_summary:
be_bryan	0:b74591d5ab33	125	# prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	126	# table shows text x toolchain test result matrix
be_bryan	0:b74591d5ab33	127	print self.single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
be_bryan	0:b74591d5ab33	128	print "Completed in %.2f sec"% (elapsed_time)
be_bryan	0:b74591d5ab33	129
be_bryan	0:b74591d5ab33	130
be_bryan	0:b74591d5ab33	131	class SingleTestRunner(object):
be_bryan	0:b74591d5ab33	132	""" Object wrapper for single test run which may involve multiple MUTs
be_bryan	0:b74591d5ab33	133	"""
be_bryan	0:b74591d5ab33	134	RE_DETECT_TESTCASE_RESULT = None
be_bryan	0:b74591d5ab33	135
be_bryan	0:b74591d5ab33	136	# Return codes for test script
be_bryan	0:b74591d5ab33	137	TEST_RESULT_OK = "OK"
be_bryan	0:b74591d5ab33	138	TEST_RESULT_FAIL = "FAIL"
be_bryan	0:b74591d5ab33	139	TEST_RESULT_ERROR = "ERROR"
be_bryan	0:b74591d5ab33	140	TEST_RESULT_UNDEF = "UNDEF"
be_bryan	0:b74591d5ab33	141	TEST_RESULT_IOERR_COPY = "IOERR_COPY"
be_bryan	0:b74591d5ab33	142	TEST_RESULT_IOERR_DISK = "IOERR_DISK"
be_bryan	0:b74591d5ab33	143	TEST_RESULT_IOERR_SERIAL = "IOERR_SERIAL"
be_bryan	0:b74591d5ab33	144	TEST_RESULT_TIMEOUT = "TIMEOUT"
be_bryan	0:b74591d5ab33	145	TEST_RESULT_NO_IMAGE = "NO_IMAGE"
be_bryan	0:b74591d5ab33	146	TEST_RESULT_MBED_ASSERT = "MBED_ASSERT"
be_bryan	0:b74591d5ab33	147	TEST_RESULT_BUILD_FAILED = "BUILD_FAILED"
be_bryan	0:b74591d5ab33	148	TEST_RESULT_NOT_SUPPORTED = "NOT_SUPPORTED"
be_bryan	0:b74591d5ab33	149
be_bryan	0:b74591d5ab33	150	GLOBAL_LOOPS_COUNT = 1 # How many times each test should be repeated
be_bryan	0:b74591d5ab33	151	TEST_LOOPS_LIST = [] # We redefine no.of loops per test_id
be_bryan	0:b74591d5ab33	152	TEST_LOOPS_DICT = {} # TEST_LOOPS_LIST in dict format: { test_id : test_loop_count}
be_bryan	0:b74591d5ab33	153
be_bryan	0:b74591d5ab33	154	muts = {} # MUTs descriptor (from external file)
be_bryan	0:b74591d5ab33	155	test_spec = {} # Test specification (from external file)
be_bryan	0:b74591d5ab33	156
be_bryan	0:b74591d5ab33	157	# mbed test suite -> SingleTestRunner
be_bryan	0:b74591d5ab33	158	TEST_RESULT_MAPPING = {"success" : TEST_RESULT_OK,
be_bryan	0:b74591d5ab33	159	"failure" : TEST_RESULT_FAIL,
be_bryan	0:b74591d5ab33	160	"error" : TEST_RESULT_ERROR,
be_bryan	0:b74591d5ab33	161	"ioerr_copy" : TEST_RESULT_IOERR_COPY,
be_bryan	0:b74591d5ab33	162	"ioerr_disk" : TEST_RESULT_IOERR_DISK,
be_bryan	0:b74591d5ab33	163	"ioerr_serial" : TEST_RESULT_IOERR_SERIAL,
be_bryan	0:b74591d5ab33	164	"timeout" : TEST_RESULT_TIMEOUT,
be_bryan	0:b74591d5ab33	165	"no_image" : TEST_RESULT_NO_IMAGE,
be_bryan	0:b74591d5ab33	166	"end" : TEST_RESULT_UNDEF,
be_bryan	0:b74591d5ab33	167	"mbed_assert" : TEST_RESULT_MBED_ASSERT,
be_bryan	0:b74591d5ab33	168	"build_failed" : TEST_RESULT_BUILD_FAILED,
be_bryan	0:b74591d5ab33	169	"not_supproted" : TEST_RESULT_NOT_SUPPORTED
be_bryan	0:b74591d5ab33	170	}
be_bryan	0:b74591d5ab33	171
be_bryan	0:b74591d5ab33	172	def __init__(self,
be_bryan	0:b74591d5ab33	173	_global_loops_count=1,
be_bryan	0:b74591d5ab33	174	_test_loops_list=None,
be_bryan	0:b74591d5ab33	175	_muts={},
be_bryan	0:b74591d5ab33	176	_clean=False,
be_bryan	0:b74591d5ab33	177	_parser=None,
be_bryan	0:b74591d5ab33	178	_opts=None,
be_bryan	0:b74591d5ab33	179	_opts_db_url=None,
be_bryan	0:b74591d5ab33	180	_opts_log_file_name=None,
be_bryan	0:b74591d5ab33	181	_opts_report_html_file_name=None,
be_bryan	0:b74591d5ab33	182	_opts_report_junit_file_name=None,
be_bryan	0:b74591d5ab33	183	_opts_report_build_file_name=None,
be_bryan	0:b74591d5ab33	184	_opts_report_text_file_name=None,
be_bryan	0:b74591d5ab33	185	_opts_build_report={},
be_bryan	0:b74591d5ab33	186	_opts_build_properties={},
be_bryan	0:b74591d5ab33	187	_test_spec={},
be_bryan	0:b74591d5ab33	188	_opts_goanna_for_mbed_sdk=None,
be_bryan	0:b74591d5ab33	189	_opts_goanna_for_tests=None,
be_bryan	0:b74591d5ab33	190	_opts_shuffle_test_order=False,
be_bryan	0:b74591d5ab33	191	_opts_shuffle_test_seed=None,
be_bryan	0:b74591d5ab33	192	_opts_test_by_names=None,
be_bryan	0:b74591d5ab33	193	_opts_peripheral_by_names=None,
be_bryan	0:b74591d5ab33	194	_opts_test_only_peripheral=False,
be_bryan	0:b74591d5ab33	195	_opts_test_only_common=False,
be_bryan	0:b74591d5ab33	196	_opts_verbose_skipped_tests=False,
be_bryan	0:b74591d5ab33	197	_opts_verbose_test_result_only=False,
be_bryan	0:b74591d5ab33	198	_opts_verbose=False,
be_bryan	0:b74591d5ab33	199	_opts_firmware_global_name=None,
be_bryan	0:b74591d5ab33	200	_opts_only_build_tests=False,
be_bryan	0:b74591d5ab33	201	_opts_parallel_test_exec=False,
be_bryan	0:b74591d5ab33	202	_opts_suppress_summary=False,
be_bryan	0:b74591d5ab33	203	_opts_test_x_toolchain_summary=False,
be_bryan	0:b74591d5ab33	204	_opts_copy_method=None,
be_bryan	0:b74591d5ab33	205	_opts_mut_reset_type=None,
be_bryan	0:b74591d5ab33	206	_opts_jobs=None,
be_bryan	0:b74591d5ab33	207	_opts_waterfall_test=None,
be_bryan	0:b74591d5ab33	208	_opts_consolidate_waterfall_test=None,
be_bryan	0:b74591d5ab33	209	_opts_extend_test_timeout=None,
be_bryan	0:b74591d5ab33	210	_opts_auto_detect=None,
be_bryan	0:b74591d5ab33	211	_opts_include_non_automated=False):
be_bryan	0:b74591d5ab33	212	""" Let's try hard to init this object
be_bryan	0:b74591d5ab33	213	"""
be_bryan	0:b74591d5ab33	214	from colorama import init
be_bryan	0:b74591d5ab33	215	init()
be_bryan	0:b74591d5ab33	216
be_bryan	0:b74591d5ab33	217	PATTERN = "\\{(" + "\|".join(self.TEST_RESULT_MAPPING.keys()) + ")\\}"
be_bryan	0:b74591d5ab33	218	self.RE_DETECT_TESTCASE_RESULT = re.compile(PATTERN)
be_bryan	0:b74591d5ab33	219	# Settings related to test loops counters
be_bryan	0:b74591d5ab33	220	try:
be_bryan	0:b74591d5ab33	221	_global_loops_count = int(_global_loops_count)
be_bryan	0:b74591d5ab33	222	except:
be_bryan	0:b74591d5ab33	223	_global_loops_count = 1
be_bryan	0:b74591d5ab33	224	if _global_loops_count < 1:
be_bryan	0:b74591d5ab33	225	_global_loops_count = 1
be_bryan	0:b74591d5ab33	226	self.GLOBAL_LOOPS_COUNT = _global_loops_count
be_bryan	0:b74591d5ab33	227	self.TEST_LOOPS_LIST = _test_loops_list if _test_loops_list else []
be_bryan	0:b74591d5ab33	228	self.TEST_LOOPS_DICT = self.test_loop_list_to_dict(_test_loops_list)
be_bryan	0:b74591d5ab33	229
be_bryan	0:b74591d5ab33	230	self.shuffle_random_seed = 0.0
be_bryan	0:b74591d5ab33	231	self.SHUFFLE_SEED_ROUND = 10
be_bryan	0:b74591d5ab33	232
be_bryan	0:b74591d5ab33	233	# MUT list and test specification storage
be_bryan	0:b74591d5ab33	234	self.muts = _muts
be_bryan	0:b74591d5ab33	235	self.test_spec = _test_spec
be_bryan	0:b74591d5ab33	236
be_bryan	0:b74591d5ab33	237	# Settings passed e.g. from command line
be_bryan	0:b74591d5ab33	238	self.opts_db_url = _opts_db_url
be_bryan	0:b74591d5ab33	239	self.opts_log_file_name = _opts_log_file_name
be_bryan	0:b74591d5ab33	240	self.opts_report_html_file_name = _opts_report_html_file_name
be_bryan	0:b74591d5ab33	241	self.opts_report_junit_file_name = _opts_report_junit_file_name
be_bryan	0:b74591d5ab33	242	self.opts_report_build_file_name = _opts_report_build_file_name
be_bryan	0:b74591d5ab33	243	self.opts_report_text_file_name = _opts_report_text_file_name
be_bryan	0:b74591d5ab33	244	self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
be_bryan	0:b74591d5ab33	245	self.opts_goanna_for_tests = _opts_goanna_for_tests
be_bryan	0:b74591d5ab33	246	self.opts_shuffle_test_order = _opts_shuffle_test_order
be_bryan	0:b74591d5ab33	247	self.opts_shuffle_test_seed = _opts_shuffle_test_seed
be_bryan	0:b74591d5ab33	248	self.opts_test_by_names = _opts_test_by_names
be_bryan	0:b74591d5ab33	249	self.opts_peripheral_by_names = _opts_peripheral_by_names
be_bryan	0:b74591d5ab33	250	self.opts_test_only_peripheral = _opts_test_only_peripheral
be_bryan	0:b74591d5ab33	251	self.opts_test_only_common = _opts_test_only_common
be_bryan	0:b74591d5ab33	252	self.opts_verbose_skipped_tests = _opts_verbose_skipped_tests
be_bryan	0:b74591d5ab33	253	self.opts_verbose_test_result_only = _opts_verbose_test_result_only
be_bryan	0:b74591d5ab33	254	self.opts_verbose = _opts_verbose
be_bryan	0:b74591d5ab33	255	self.opts_firmware_global_name = _opts_firmware_global_name
be_bryan	0:b74591d5ab33	256	self.opts_only_build_tests = _opts_only_build_tests
be_bryan	0:b74591d5ab33	257	self.opts_parallel_test_exec = _opts_parallel_test_exec
be_bryan	0:b74591d5ab33	258	self.opts_suppress_summary = _opts_suppress_summary
be_bryan	0:b74591d5ab33	259	self.opts_test_x_toolchain_summary = _opts_test_x_toolchain_summary
be_bryan	0:b74591d5ab33	260	self.opts_copy_method = _opts_copy_method
be_bryan	0:b74591d5ab33	261	self.opts_mut_reset_type = _opts_mut_reset_type
be_bryan	0:b74591d5ab33	262	self.opts_jobs = _opts_jobs if _opts_jobs is not None else 1
be_bryan	0:b74591d5ab33	263	self.opts_waterfall_test = _opts_waterfall_test
be_bryan	0:b74591d5ab33	264	self.opts_consolidate_waterfall_test = _opts_consolidate_waterfall_test
be_bryan	0:b74591d5ab33	265	self.opts_extend_test_timeout = _opts_extend_test_timeout
be_bryan	0:b74591d5ab33	266	self.opts_clean = _clean
be_bryan	0:b74591d5ab33	267	self.opts_parser = _parser
be_bryan	0:b74591d5ab33	268	self.opts = _opts
be_bryan	0:b74591d5ab33	269	self.opts_auto_detect = _opts_auto_detect
be_bryan	0:b74591d5ab33	270	self.opts_include_non_automated = _opts_include_non_automated
be_bryan	0:b74591d5ab33	271
be_bryan	0:b74591d5ab33	272	self.build_report = _opts_build_report
be_bryan	0:b74591d5ab33	273	self.build_properties = _opts_build_properties
be_bryan	0:b74591d5ab33	274
be_bryan	0:b74591d5ab33	275	# File / screen logger initialization
be_bryan	0:b74591d5ab33	276	self.logger = CLITestLogger(file_name=self.opts_log_file_name) # Default test logger
be_bryan	0:b74591d5ab33	277
be_bryan	0:b74591d5ab33	278	# Database related initializations
be_bryan	0:b74591d5ab33	279	self.db_logger = factory_db_logger(self.opts_db_url)
be_bryan	0:b74591d5ab33	280	self.db_logger_build_id = None # Build ID (database index of build_id table)
be_bryan	0:b74591d5ab33	281	# Let's connect to database to set up credentials and confirm database is ready
be_bryan	0:b74591d5ab33	282	if self.db_logger:
be_bryan	0:b74591d5ab33	283	self.db_logger.connect_url(self.opts_db_url) # Save db access info inside db_logger object
be_bryan	0:b74591d5ab33	284	if self.db_logger.is_connected():
be_bryan	0:b74591d5ab33	285	# Get hostname and uname so we can use it as build description
be_bryan	0:b74591d5ab33	286	# when creating new build_id in external database
be_bryan	0:b74591d5ab33	287	(_hostname, _uname) = self.db_logger.get_hostname()
be_bryan	0:b74591d5ab33	288	_host_location = os.path.dirname(os.path.abspath(__file__))
be_bryan	0:b74591d5ab33	289	build_id_type = None if self.opts_only_build_tests is None else self.db_logger.BUILD_ID_TYPE_BUILD_ONLY
be_bryan	0:b74591d5ab33	290	self.db_logger_build_id = self.db_logger.get_next_build_id(_hostname, desc=_uname, location=_host_location, type=build_id_type)
be_bryan	0:b74591d5ab33	291	self.db_logger.disconnect()
be_bryan	0:b74591d5ab33	292
be_bryan	0:b74591d5ab33	293	def dump_options(self):
be_bryan	0:b74591d5ab33	294	""" Function returns data structure with common settings passed to SingelTestRunner
be_bryan	0:b74591d5ab33	295	It can be used for example to fill _extra fields in database storing test suite single run data
be_bryan	0:b74591d5ab33	296	Example:
be_bryan	0:b74591d5ab33	297	data = self.dump_options()
be_bryan	0:b74591d5ab33	298	or
be_bryan	0:b74591d5ab33	299	data_str = json.dumps(self.dump_options())
be_bryan	0:b74591d5ab33	300	"""
be_bryan	0:b74591d5ab33	301	result = {"db_url" : str(self.opts_db_url),
be_bryan	0:b74591d5ab33	302	"log_file_name" : str(self.opts_log_file_name),
be_bryan	0:b74591d5ab33	303	"shuffle_test_order" : str(self.opts_shuffle_test_order),
be_bryan	0:b74591d5ab33	304	"shuffle_test_seed" : str(self.opts_shuffle_test_seed),
be_bryan	0:b74591d5ab33	305	"test_by_names" : str(self.opts_test_by_names),
be_bryan	0:b74591d5ab33	306	"peripheral_by_names" : str(self.opts_peripheral_by_names),
be_bryan	0:b74591d5ab33	307	"test_only_peripheral" : str(self.opts_test_only_peripheral),
be_bryan	0:b74591d5ab33	308	"test_only_common" : str(self.opts_test_only_common),
be_bryan	0:b74591d5ab33	309	"verbose" : str(self.opts_verbose),
be_bryan	0:b74591d5ab33	310	"firmware_global_name" : str(self.opts_firmware_global_name),
be_bryan	0:b74591d5ab33	311	"only_build_tests" : str(self.opts_only_build_tests),
be_bryan	0:b74591d5ab33	312	"copy_method" : str(self.opts_copy_method),
be_bryan	0:b74591d5ab33	313	"mut_reset_type" : str(self.opts_mut_reset_type),
be_bryan	0:b74591d5ab33	314	"jobs" : str(self.opts_jobs),
be_bryan	0:b74591d5ab33	315	"extend_test_timeout" : str(self.opts_extend_test_timeout),
be_bryan	0:b74591d5ab33	316	"_dummy" : ''
be_bryan	0:b74591d5ab33	317	}
be_bryan	0:b74591d5ab33	318	return result
be_bryan	0:b74591d5ab33	319
be_bryan	0:b74591d5ab33	320	def shuffle_random_func(self):
be_bryan	0:b74591d5ab33	321	return self.shuffle_random_seed
be_bryan	0:b74591d5ab33	322
be_bryan	0:b74591d5ab33	323	def is_shuffle_seed_float(self):
be_bryan	0:b74591d5ab33	324	""" return true if function parameter can be converted to float
be_bryan	0:b74591d5ab33	325	"""
be_bryan	0:b74591d5ab33	326	result = True
be_bryan	0:b74591d5ab33	327	try:
be_bryan	0:b74591d5ab33	328	float(self.shuffle_random_seed)
be_bryan	0:b74591d5ab33	329	except ValueError:
be_bryan	0:b74591d5ab33	330	result = False
be_bryan	0:b74591d5ab33	331	return result
be_bryan	0:b74591d5ab33	332
be_bryan	0:b74591d5ab33	333	# This will store target / toolchain specific properties
be_bryan	0:b74591d5ab33	334	test_suite_properties_ext = {} # target : toolchain
be_bryan	0:b74591d5ab33	335	# Here we store test results
be_bryan	0:b74591d5ab33	336	test_summary = []
be_bryan	0:b74591d5ab33	337	# Here we store test results in extended data structure
be_bryan	0:b74591d5ab33	338	test_summary_ext = {}
be_bryan	0:b74591d5ab33	339	execute_thread_slice_lock = Lock()
be_bryan	0:b74591d5ab33	340
be_bryan	0:b74591d5ab33	341	def execute_thread_slice(self, q, target, toolchains, clean, test_ids, build_report, build_properties):
be_bryan	0:b74591d5ab33	342	for toolchain in toolchains:
be_bryan	0:b74591d5ab33	343	tt_id = "%s::%s" % (toolchain, target)
be_bryan	0:b74591d5ab33	344
be_bryan	0:b74591d5ab33	345	T = TARGET_MAP[target]
be_bryan	0:b74591d5ab33	346
be_bryan	0:b74591d5ab33	347	# print target, toolchain
be_bryan	0:b74591d5ab33	348	# Test suite properties returned to external tools like CI
be_bryan	0:b74591d5ab33	349	test_suite_properties = {
be_bryan	0:b74591d5ab33	350	'jobs': self.opts_jobs,
be_bryan	0:b74591d5ab33	351	'clean': clean,
be_bryan	0:b74591d5ab33	352	'target': target,
be_bryan	0:b74591d5ab33	353	'vendor': T.extra_labels[0],
be_bryan	0:b74591d5ab33	354	'test_ids': ', '.join(test_ids),
be_bryan	0:b74591d5ab33	355	'toolchain': toolchain,
be_bryan	0:b74591d5ab33	356	'shuffle_random_seed': self.shuffle_random_seed
be_bryan	0:b74591d5ab33	357	}
be_bryan	0:b74591d5ab33	358
be_bryan	0:b74591d5ab33	359
be_bryan	0:b74591d5ab33	360	# print '=== %s::%s ===' % (target, toolchain)
be_bryan	0:b74591d5ab33	361	# Let's build our test
be_bryan	0:b74591d5ab33	362	if target not in TARGET_MAP:
be_bryan	0:b74591d5ab33	363	print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Target platform not found'% (target))
be_bryan	0:b74591d5ab33	364	continue
be_bryan	0:b74591d5ab33	365
be_bryan	0:b74591d5ab33	366	clean_mbed_libs_options = True if self.opts_goanna_for_mbed_sdk or clean or self.opts_clean else None
be_bryan	0:b74591d5ab33	367
be_bryan	0:b74591d5ab33	368	profile = extract_profile(self.opts_parser, self.opts, toolchain)
be_bryan	0:b74591d5ab33	369	stats_depth = self.opts.stats_depth or 2
be_bryan	0:b74591d5ab33	370
be_bryan	0:b74591d5ab33	371
be_bryan	0:b74591d5ab33	372	try:
be_bryan	0:b74591d5ab33	373	build_mbed_libs_result = build_mbed_libs(T,
be_bryan	0:b74591d5ab33	374	toolchain,
be_bryan	0:b74591d5ab33	375	clean=clean_mbed_libs_options,
be_bryan	0:b74591d5ab33	376	verbose=self.opts_verbose,
be_bryan	0:b74591d5ab33	377	jobs=self.opts_jobs,
be_bryan	0:b74591d5ab33	378	report=build_report,
be_bryan	0:b74591d5ab33	379	properties=build_properties,
be_bryan	0:b74591d5ab33	380	build_profile=profile)
be_bryan	0:b74591d5ab33	381
be_bryan	0:b74591d5ab33	382	if not build_mbed_libs_result:
be_bryan	0:b74591d5ab33	383	print self.logger.log_line(self.logger.LogType.NOTIF, 'Skipped tests for %s target. Toolchain %s is not yet supported for this target'% (T.name, toolchain))
be_bryan	0:b74591d5ab33	384	continue
be_bryan	0:b74591d5ab33	385
be_bryan	0:b74591d5ab33	386	except ToolException:
be_bryan	0:b74591d5ab33	387	print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building MBED libs for %s using %s'% (target, toolchain))
be_bryan	0:b74591d5ab33	388	continue
be_bryan	0:b74591d5ab33	389
be_bryan	0:b74591d5ab33	390	build_dir = join(BUILD_DIR, "test", target, toolchain)
be_bryan	0:b74591d5ab33	391
be_bryan	0:b74591d5ab33	392	test_suite_properties['build_mbed_libs_result'] = build_mbed_libs_result
be_bryan	0:b74591d5ab33	393	test_suite_properties['build_dir'] = build_dir
be_bryan	0:b74591d5ab33	394	test_suite_properties['skipped'] = []
be_bryan	0:b74591d5ab33	395
be_bryan	0:b74591d5ab33	396	# Enumerate through all tests and shuffle test order if requested
be_bryan	0:b74591d5ab33	397	test_map_keys = sorted(TEST_MAP.keys())
be_bryan	0:b74591d5ab33	398
be_bryan	0:b74591d5ab33	399	if self.opts_shuffle_test_order:
be_bryan	0:b74591d5ab33	400	random.shuffle(test_map_keys, self.shuffle_random_func)
be_bryan	0:b74591d5ab33	401	# Update database with shuffle seed f applicable
be_bryan	0:b74591d5ab33	402	if self.db_logger:
be_bryan	0:b74591d5ab33	403	self.db_logger.reconnect();
be_bryan	0:b74591d5ab33	404	if self.db_logger.is_connected():
be_bryan	0:b74591d5ab33	405	self.db_logger.update_build_id_info(self.db_logger_build_id, _shuffle_seed=self.shuffle_random_func())
be_bryan	0:b74591d5ab33	406	self.db_logger.disconnect();
be_bryan	0:b74591d5ab33	407
be_bryan	0:b74591d5ab33	408	if self.db_logger:
be_bryan	0:b74591d5ab33	409	self.db_logger.reconnect();
be_bryan	0:b74591d5ab33	410	if self.db_logger.is_connected():
be_bryan	0:b74591d5ab33	411	# Update MUTs and Test Specification in database
be_bryan	0:b74591d5ab33	412	self.db_logger.update_build_id_info(self.db_logger_build_id, _muts=self.muts, _test_spec=self.test_spec)
be_bryan	0:b74591d5ab33	413	# Update Extra information in database (some options passed to test suite)
be_bryan	0:b74591d5ab33	414	self.db_logger.update_build_id_info(self.db_logger_build_id, _extra=json.dumps(self.dump_options()))
be_bryan	0:b74591d5ab33	415	self.db_logger.disconnect();
be_bryan	0:b74591d5ab33	416
be_bryan	0:b74591d5ab33	417	valid_test_map_keys = self.get_valid_tests(test_map_keys, target, toolchain, test_ids, self.opts_include_non_automated)
be_bryan	0:b74591d5ab33	418	skipped_test_map_keys = self.get_skipped_tests(test_map_keys, valid_test_map_keys)
be_bryan	0:b74591d5ab33	419
be_bryan	0:b74591d5ab33	420	for skipped_test_id in skipped_test_map_keys:
be_bryan	0:b74591d5ab33	421	test_suite_properties['skipped'].append(skipped_test_id)
be_bryan	0:b74591d5ab33	422
be_bryan	0:b74591d5ab33	423
be_bryan	0:b74591d5ab33	424	# First pass through all tests and determine which libraries need to be built
be_bryan	0:b74591d5ab33	425	libraries = []
be_bryan	0:b74591d5ab33	426	for test_id in valid_test_map_keys:
be_bryan	0:b74591d5ab33	427	test = TEST_MAP[test_id]
be_bryan	0:b74591d5ab33	428
be_bryan	0:b74591d5ab33	429	# Detect which lib should be added to test
be_bryan	0:b74591d5ab33	430	# Some libs have to compiled like RTOS or ETH
be_bryan	0:b74591d5ab33	431	for lib in LIBRARIES:
be_bryan	0:b74591d5ab33	432	if lib['build_dir'] in test.dependencies and lib['id'] not in libraries:
be_bryan	0:b74591d5ab33	433	libraries.append(lib['id'])
be_bryan	0:b74591d5ab33	434
be_bryan	0:b74591d5ab33	435
be_bryan	0:b74591d5ab33	436	clean_project_options = True if self.opts_goanna_for_tests or clean or self.opts_clean else None
be_bryan	0:b74591d5ab33	437
be_bryan	0:b74591d5ab33	438	# Build all required libraries
be_bryan	0:b74591d5ab33	439	for lib_id in libraries:
be_bryan	0:b74591d5ab33	440	try:
be_bryan	0:b74591d5ab33	441	build_lib(lib_id,
be_bryan	0:b74591d5ab33	442	T,
be_bryan	0:b74591d5ab33	443	toolchain,
be_bryan	0:b74591d5ab33	444	verbose=self.opts_verbose,
be_bryan	0:b74591d5ab33	445	clean=clean_mbed_libs_options,
be_bryan	0:b74591d5ab33	446	jobs=self.opts_jobs,
be_bryan	0:b74591d5ab33	447	report=build_report,
be_bryan	0:b74591d5ab33	448	properties=build_properties,
be_bryan	0:b74591d5ab33	449	build_profile=profile)
be_bryan	0:b74591d5ab33	450
be_bryan	0:b74591d5ab33	451	except ToolException:
be_bryan	0:b74591d5ab33	452	print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building library %s'% (lib_id))
be_bryan	0:b74591d5ab33	453	continue
be_bryan	0:b74591d5ab33	454
be_bryan	0:b74591d5ab33	455
be_bryan	0:b74591d5ab33	456	for test_id in valid_test_map_keys:
be_bryan	0:b74591d5ab33	457	test = TEST_MAP[test_id]
be_bryan	0:b74591d5ab33	458
be_bryan	0:b74591d5ab33	459	test_suite_properties['test.libs.%s.%s.%s'% (target, toolchain, test_id)] = ', '.join(libraries)
be_bryan	0:b74591d5ab33	460
be_bryan	0:b74591d5ab33	461	# TODO: move this 2 below loops to separate function
be_bryan	0:b74591d5ab33	462	INC_DIRS = []
be_bryan	0:b74591d5ab33	463	for lib_id in libraries:
be_bryan	0:b74591d5ab33	464	if 'inc_dirs_ext' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['inc_dirs_ext']:
be_bryan	0:b74591d5ab33	465	INC_DIRS.extend(LIBRARY_MAP[lib_id]['inc_dirs_ext'])
be_bryan	0:b74591d5ab33	466
be_bryan	0:b74591d5ab33	467	MACROS = []
be_bryan	0:b74591d5ab33	468	for lib_id in libraries:
be_bryan	0:b74591d5ab33	469	if 'macros' in LIBRARY_MAP[lib_id] and LIBRARY_MAP[lib_id]['macros']:
be_bryan	0:b74591d5ab33	470	MACROS.extend(LIBRARY_MAP[lib_id]['macros'])
be_bryan	0:b74591d5ab33	471	MACROS.append('TEST_SUITE_TARGET_NAME="%s"'% target)
be_bryan	0:b74591d5ab33	472	MACROS.append('TEST_SUITE_TEST_ID="%s"'% test_id)
be_bryan	0:b74591d5ab33	473	test_uuid = uuid.uuid4()
be_bryan	0:b74591d5ab33	474	MACROS.append('TEST_SUITE_UUID="%s"'% str(test_uuid))
be_bryan	0:b74591d5ab33	475
be_bryan	0:b74591d5ab33	476	# Prepare extended test results data structure (it can be used to generate detailed test report)
be_bryan	0:b74591d5ab33	477	if target not in self.test_summary_ext:
be_bryan	0:b74591d5ab33	478	self.test_summary_ext[target] = {} # test_summary_ext : toolchain
be_bryan	0:b74591d5ab33	479	if toolchain not in self.test_summary_ext[target]:
be_bryan	0:b74591d5ab33	480	self.test_summary_ext[target][toolchain] = {} # test_summary_ext : toolchain : target
be_bryan	0:b74591d5ab33	481
be_bryan	0:b74591d5ab33	482	tt_test_id = "%s::%s::%s" % (toolchain, target, test_id) # For logging only
be_bryan	0:b74591d5ab33	483
be_bryan	0:b74591d5ab33	484	project_name = self.opts_firmware_global_name if self.opts_firmware_global_name else None
be_bryan	0:b74591d5ab33	485	try:
be_bryan	0:b74591d5ab33	486	path = build_project(test.source_dir, join(build_dir, test_id), T,
be_bryan	0:b74591d5ab33	487	toolchain, test.dependencies, clean=clean_project_options,
be_bryan	0:b74591d5ab33	488	verbose=self.opts_verbose, name=project_name, macros=MACROS,
be_bryan	0:b74591d5ab33	489	inc_dirs=INC_DIRS, jobs=self.opts_jobs, report=build_report,
be_bryan	0:b74591d5ab33	490	properties=build_properties, project_id=test_id,
be_bryan	0:b74591d5ab33	491	project_description=test.get_description(),
be_bryan	0:b74591d5ab33	492	build_profile=profile, stats_depth=stats_depth)
be_bryan	0:b74591d5ab33	493
be_bryan	0:b74591d5ab33	494	except Exception, e:
be_bryan	0:b74591d5ab33	495	project_name_str = project_name if project_name is not None else test_id
be_bryan	0:b74591d5ab33	496
be_bryan	0:b74591d5ab33	497
be_bryan	0:b74591d5ab33	498	test_result = self.TEST_RESULT_FAIL
be_bryan	0:b74591d5ab33	499
be_bryan	0:b74591d5ab33	500	if isinstance(e, ToolException):
be_bryan	0:b74591d5ab33	501	print self.logger.log_line(self.logger.LogType.ERROR, 'There were errors while building project %s'% (project_name_str))
be_bryan	0:b74591d5ab33	502	test_result = self.TEST_RESULT_BUILD_FAILED
be_bryan	0:b74591d5ab33	503	elif isinstance(e, NotSupportedException):
be_bryan	0:b74591d5ab33	504	print self.logger.log_line(self.logger.LogType.INFO, 'The project %s is not supported'% (project_name_str))
be_bryan	0:b74591d5ab33	505	test_result = self.TEST_RESULT_NOT_SUPPORTED
be_bryan	0:b74591d5ab33	506
be_bryan	0:b74591d5ab33	507
be_bryan	0:b74591d5ab33	508	# Append test results to global test summary
be_bryan	0:b74591d5ab33	509	self.test_summary.append(
be_bryan	0:b74591d5ab33	510	(test_result, target, toolchain, test_id, test.get_description(), 0, 0, '-')
be_bryan	0:b74591d5ab33	511	)
be_bryan	0:b74591d5ab33	512
be_bryan	0:b74591d5ab33	513	# Add detailed test result to test summary structure
be_bryan	0:b74591d5ab33	514	if test_id not in self.test_summary_ext[target][toolchain]:
be_bryan	0:b74591d5ab33	515	self.test_summary_ext[target][toolchain][test_id] = []
be_bryan	0:b74591d5ab33	516
be_bryan	0:b74591d5ab33	517	self.test_summary_ext[target][toolchain][test_id].append({ 0: {
be_bryan	0:b74591d5ab33	518	'result' : test_result,
be_bryan	0:b74591d5ab33	519	'output' : '',
be_bryan	0:b74591d5ab33	520	'target_name' : target,
be_bryan	0:b74591d5ab33	521	'target_name_unique': target,
be_bryan	0:b74591d5ab33	522	'toolchain_name' : toolchain,
be_bryan	0:b74591d5ab33	523	'id' : test_id,
be_bryan	0:b74591d5ab33	524	'description' : test.get_description(),
be_bryan	0:b74591d5ab33	525	'elapsed_time' : 0,
be_bryan	0:b74591d5ab33	526	'duration' : 0,
be_bryan	0:b74591d5ab33	527	'copy_method' : None
be_bryan	0:b74591d5ab33	528	}})
be_bryan	0:b74591d5ab33	529	continue
be_bryan	0:b74591d5ab33	530
be_bryan	0:b74591d5ab33	531	if self.opts_only_build_tests:
be_bryan	0:b74591d5ab33	532	# With this option we are skipping testing phase
be_bryan	0:b74591d5ab33	533	continue
be_bryan	0:b74591d5ab33	534
be_bryan	0:b74591d5ab33	535	# Test duration can be increased by global value
be_bryan	0:b74591d5ab33	536	test_duration = test.duration
be_bryan	0:b74591d5ab33	537	if self.opts_extend_test_timeout is not None:
be_bryan	0:b74591d5ab33	538	test_duration += self.opts_extend_test_timeout
be_bryan	0:b74591d5ab33	539
be_bryan	0:b74591d5ab33	540	# For an automated test the duration act as a timeout after
be_bryan	0:b74591d5ab33	541	# which the test gets interrupted
be_bryan	0:b74591d5ab33	542	test_spec = self.shape_test_request(target, path, test_id, test_duration)
be_bryan	0:b74591d5ab33	543	test_loops = self.get_test_loop_count(test_id)
be_bryan	0:b74591d5ab33	544
be_bryan	0:b74591d5ab33	545	test_suite_properties['test.duration.%s.%s.%s'% (target, toolchain, test_id)] = test_duration
be_bryan	0:b74591d5ab33	546	test_suite_properties['test.loops.%s.%s.%s'% (target, toolchain, test_id)] = test_loops
be_bryan	0:b74591d5ab33	547	test_suite_properties['test.path.%s.%s.%s'% (target, toolchain, test_id)] = path
be_bryan	0:b74591d5ab33	548
be_bryan	0:b74591d5ab33	549	# read MUTs, test specification and perform tests
be_bryan	0:b74591d5ab33	550	handle_results = self.handle(test_spec, target, toolchain, test_loops=test_loops)
be_bryan	0:b74591d5ab33	551
be_bryan	0:b74591d5ab33	552	if handle_results is None:
be_bryan	0:b74591d5ab33	553	continue
be_bryan	0:b74591d5ab33	554
be_bryan	0:b74591d5ab33	555	for handle_result in handle_results:
be_bryan	0:b74591d5ab33	556	if handle_result:
be_bryan	0:b74591d5ab33	557	single_test_result, detailed_test_results = handle_result
be_bryan	0:b74591d5ab33	558	else:
be_bryan	0:b74591d5ab33	559	continue
be_bryan	0:b74591d5ab33	560
be_bryan	0:b74591d5ab33	561	# Append test results to global test summary
be_bryan	0:b74591d5ab33	562	if single_test_result is not None:
be_bryan	0:b74591d5ab33	563	self.test_summary.append(single_test_result)
be_bryan	0:b74591d5ab33	564
be_bryan	0:b74591d5ab33	565	# Add detailed test result to test summary structure
be_bryan	0:b74591d5ab33	566	if target not in self.test_summary_ext[target][toolchain]:
be_bryan	0:b74591d5ab33	567	if test_id not in self.test_summary_ext[target][toolchain]:
be_bryan	0:b74591d5ab33	568	self.test_summary_ext[target][toolchain][test_id] = []
be_bryan	0:b74591d5ab33	569
be_bryan	0:b74591d5ab33	570	append_test_result = detailed_test_results
be_bryan	0:b74591d5ab33	571
be_bryan	0:b74591d5ab33	572	# If waterfall and consolidate-waterfall options are enabled,
be_bryan	0:b74591d5ab33	573	# only include the last test result in the report.
be_bryan	0:b74591d5ab33	574	if self.opts_waterfall_test and self.opts_consolidate_waterfall_test:
be_bryan	0:b74591d5ab33	575	append_test_result = {0: detailed_test_results[len(detailed_test_results) - 1]}
be_bryan	0:b74591d5ab33	576
be_bryan	0:b74591d5ab33	577	self.test_summary_ext[target][toolchain][test_id].append(append_test_result)
be_bryan	0:b74591d5ab33	578
be_bryan	0:b74591d5ab33	579	test_suite_properties['skipped'] = ', '.join(test_suite_properties['skipped'])
be_bryan	0:b74591d5ab33	580	self.test_suite_properties_ext[target][toolchain] = test_suite_properties
be_bryan	0:b74591d5ab33	581
be_bryan	0:b74591d5ab33	582	q.put(target + '_'.join(toolchains))
be_bryan	0:b74591d5ab33	583	return
be_bryan	0:b74591d5ab33	584
be_bryan	0:b74591d5ab33	585	def execute(self):
be_bryan	0:b74591d5ab33	586	clean = self.test_spec.get('clean', False)
be_bryan	0:b74591d5ab33	587	test_ids = self.test_spec.get('test_ids', [])
be_bryan	0:b74591d5ab33	588	q = Queue()
be_bryan	0:b74591d5ab33	589
be_bryan	0:b74591d5ab33	590	# Generate seed for shuffle if seed is not provided in
be_bryan	0:b74591d5ab33	591	self.shuffle_random_seed = round(random.random(), self.SHUFFLE_SEED_ROUND)
be_bryan	0:b74591d5ab33	592	if self.opts_shuffle_test_seed is not None and self.is_shuffle_seed_float():
be_bryan	0:b74591d5ab33	593	self.shuffle_random_seed = round(float(self.opts_shuffle_test_seed), self.SHUFFLE_SEED_ROUND)
be_bryan	0:b74591d5ab33	594
be_bryan	0:b74591d5ab33	595
be_bryan	0:b74591d5ab33	596	if self.opts_parallel_test_exec:
be_bryan	0:b74591d5ab33	597	###################################################################
be_bryan	0:b74591d5ab33	598	# Experimental, parallel test execution per singletest instance.
be_bryan	0:b74591d5ab33	599	###################################################################
be_bryan	0:b74591d5ab33	600	execute_threads = [] # Threads used to build mbed SDL, libs, test cases and execute tests
be_bryan	0:b74591d5ab33	601	# Note: We are building here in parallel for each target separately!
be_bryan	0:b74591d5ab33	602	# So we are not building the same thing multiple times and compilers
be_bryan	0:b74591d5ab33	603	# in separate threads do not collide.
be_bryan	0:b74591d5ab33	604	# Inside execute_thread_slice() function function handle() will be called to
be_bryan	0:b74591d5ab33	605	# get information about available MUTs (per target).
be_bryan	0:b74591d5ab33	606	for target, toolchains in self.test_spec['targets'].iteritems():
be_bryan	0:b74591d5ab33	607	self.test_suite_properties_ext[target] = {}
be_bryan	0:b74591d5ab33	608	t = threading.Thread(target=self.execute_thread_slice, args = (q, target, toolchains, clean, test_ids, self.build_report, self.build_properties))
be_bryan	0:b74591d5ab33	609	t.daemon = True
be_bryan	0:b74591d5ab33	610	t.start()
be_bryan	0:b74591d5ab33	611	execute_threads.append(t)
be_bryan	0:b74591d5ab33	612
be_bryan	0:b74591d5ab33	613	for t in execute_threads:
be_bryan	0:b74591d5ab33	614	q.get() # t.join() would block some threads because we should not wait in any order for thread end
be_bryan	0:b74591d5ab33	615	else:
be_bryan	0:b74591d5ab33	616	# Serialized (not parallel) test execution
be_bryan	0:b74591d5ab33	617	for target, toolchains in self.test_spec['targets'].iteritems():
be_bryan	0:b74591d5ab33	618	if target not in self.test_suite_properties_ext:
be_bryan	0:b74591d5ab33	619	self.test_suite_properties_ext[target] = {}
be_bryan	0:b74591d5ab33	620
be_bryan	0:b74591d5ab33	621	self.execute_thread_slice(q, target, toolchains, clean, test_ids, self.build_report, self.build_properties)
be_bryan	0:b74591d5ab33	622	q.get()
be_bryan	0:b74591d5ab33	623
be_bryan	0:b74591d5ab33	624	if self.db_logger:
be_bryan	0:b74591d5ab33	625	self.db_logger.reconnect();
be_bryan	0:b74591d5ab33	626	if self.db_logger.is_connected():
be_bryan	0:b74591d5ab33	627	self.db_logger.update_build_id_info(self.db_logger_build_id, _status_fk=self.db_logger.BUILD_ID_STATUS_COMPLETED)
be_bryan	0:b74591d5ab33	628	self.db_logger.disconnect();
be_bryan	0:b74591d5ab33	629
be_bryan	0:b74591d5ab33	630	return self.test_summary, self.shuffle_random_seed, self.test_summary_ext, self.test_suite_properties_ext, self.build_report, self.build_properties
be_bryan	0:b74591d5ab33	631
be_bryan	0:b74591d5ab33	632	def get_valid_tests(self, test_map_keys, target, toolchain, test_ids, include_non_automated):
be_bryan	0:b74591d5ab33	633	valid_test_map_keys = []
be_bryan	0:b74591d5ab33	634
be_bryan	0:b74591d5ab33	635	for test_id in test_map_keys:
be_bryan	0:b74591d5ab33	636	test = TEST_MAP[test_id]
be_bryan	0:b74591d5ab33	637	if self.opts_test_by_names and test_id not in self.opts_test_by_names:
be_bryan	0:b74591d5ab33	638	continue
be_bryan	0:b74591d5ab33	639
be_bryan	0:b74591d5ab33	640	if test_ids and test_id not in test_ids:
be_bryan	0:b74591d5ab33	641	continue
be_bryan	0:b74591d5ab33	642
be_bryan	0:b74591d5ab33	643	if self.opts_test_only_peripheral and not test.peripherals:
be_bryan	0:b74591d5ab33	644	if self.opts_verbose_skipped_tests:
be_bryan	0:b74591d5ab33	645	print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
be_bryan	0:b74591d5ab33	646	continue
be_bryan	0:b74591d5ab33	647
be_bryan	0:b74591d5ab33	648	if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
be_bryan	0:b74591d5ab33	649	# We will skip tests not forced with -p option
be_bryan	0:b74591d5ab33	650	if self.opts_verbose_skipped_tests:
be_bryan	0:b74591d5ab33	651	print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
be_bryan	0:b74591d5ab33	652	continue
be_bryan	0:b74591d5ab33	653
be_bryan	0:b74591d5ab33	654	if self.opts_test_only_common and test.peripherals:
be_bryan	0:b74591d5ab33	655	if self.opts_verbose_skipped_tests:
be_bryan	0:b74591d5ab33	656	print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral test skipped for target %s'% (target))
be_bryan	0:b74591d5ab33	657	continue
be_bryan	0:b74591d5ab33	658
be_bryan	0:b74591d5ab33	659	if not include_non_automated and not test.automated:
be_bryan	0:b74591d5ab33	660	if self.opts_verbose_skipped_tests:
be_bryan	0:b74591d5ab33	661	print self.logger.log_line(self.logger.LogType.INFO, 'Non automated test skipped for target %s'% (target))
be_bryan	0:b74591d5ab33	662	continue
be_bryan	0:b74591d5ab33	663
be_bryan	0:b74591d5ab33	664	if test.is_supported(target, toolchain):
be_bryan	0:b74591d5ab33	665	if test.peripherals is None and self.opts_only_build_tests:
be_bryan	0:b74591d5ab33	666	# When users are using 'build only flag' and test do not have
be_bryan	0:b74591d5ab33	667	# specified peripherals we can allow test building by default
be_bryan	0:b74591d5ab33	668	pass
be_bryan	0:b74591d5ab33	669	elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
be_bryan	0:b74591d5ab33	670	# If we force peripheral with option -p we expect test
be_bryan	0:b74591d5ab33	671	# to pass even if peripheral is not in MUTs file.
be_bryan	0:b74591d5ab33	672	pass
be_bryan	0:b74591d5ab33	673	elif not self.is_peripherals_available(target, test.peripherals):
be_bryan	0:b74591d5ab33	674	if self.opts_verbose_skipped_tests:
be_bryan	0:b74591d5ab33	675	if test.peripherals:
be_bryan	0:b74591d5ab33	676	print self.logger.log_line(self.logger.LogType.INFO, 'Peripheral %s test skipped for target %s'% (",".join(test.peripherals), target))
be_bryan	0:b74591d5ab33	677	else:
be_bryan	0:b74591d5ab33	678	print self.logger.log_line(self.logger.LogType.INFO, 'Test %s skipped for target %s'% (test_id, target))
be_bryan	0:b74591d5ab33	679	continue
be_bryan	0:b74591d5ab33	680
be_bryan	0:b74591d5ab33	681	# The test has made it through all the filters, so add it to the valid tests list
be_bryan	0:b74591d5ab33	682	valid_test_map_keys.append(test_id)
be_bryan	0:b74591d5ab33	683
be_bryan	0:b74591d5ab33	684	return valid_test_map_keys
be_bryan	0:b74591d5ab33	685
be_bryan	0:b74591d5ab33	686	def get_skipped_tests(self, all_test_map_keys, valid_test_map_keys):
be_bryan	0:b74591d5ab33	687	# NOTE: This will not preserve order
be_bryan	0:b74591d5ab33	688	return list(set(all_test_map_keys) - set(valid_test_map_keys))
be_bryan	0:b74591d5ab33	689
be_bryan	0:b74591d5ab33	690	def generate_test_summary_by_target(self, test_summary, shuffle_seed=None):
be_bryan	0:b74591d5ab33	691	""" Prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	692	table shows text x toolchain test result matrix
be_bryan	0:b74591d5ab33	693	"""
be_bryan	0:b74591d5ab33	694	RESULT_INDEX = 0
be_bryan	0:b74591d5ab33	695	TARGET_INDEX = 1
be_bryan	0:b74591d5ab33	696	TOOLCHAIN_INDEX = 2
be_bryan	0:b74591d5ab33	697	TEST_INDEX = 3
be_bryan	0:b74591d5ab33	698	DESC_INDEX = 4
be_bryan	0:b74591d5ab33	699
be_bryan	0:b74591d5ab33	700	unique_targets = get_unique_value_from_summary(test_summary, TARGET_INDEX)
be_bryan	0:b74591d5ab33	701	unique_tests = get_unique_value_from_summary(test_summary, TEST_INDEX)
be_bryan	0:b74591d5ab33	702	unique_test_desc = get_unique_value_from_summary_ext(test_summary, TEST_INDEX, DESC_INDEX)
be_bryan	0:b74591d5ab33	703	unique_toolchains = get_unique_value_from_summary(test_summary, TOOLCHAIN_INDEX)
be_bryan	0:b74591d5ab33	704
be_bryan	0:b74591d5ab33	705	result = "Test summary:\n"
be_bryan	0:b74591d5ab33	706	for target in unique_targets:
be_bryan	0:b74591d5ab33	707	result_dict = {} # test : { toolchain : result }
be_bryan	0:b74591d5ab33	708	unique_target_toolchains = []
be_bryan	0:b74591d5ab33	709	for test in test_summary:
be_bryan	0:b74591d5ab33	710	if test[TARGET_INDEX] == target:
be_bryan	0:b74591d5ab33	711	if test[TOOLCHAIN_INDEX] not in unique_target_toolchains:
be_bryan	0:b74591d5ab33	712	unique_target_toolchains.append(test[TOOLCHAIN_INDEX])
be_bryan	0:b74591d5ab33	713	if test[TEST_INDEX] not in result_dict:
be_bryan	0:b74591d5ab33	714	result_dict[test[TEST_INDEX]] = {}
be_bryan	0:b74591d5ab33	715	result_dict[test[TEST_INDEX]][test[TOOLCHAIN_INDEX]] = test[RESULT_INDEX]
be_bryan	0:b74591d5ab33	716
be_bryan	0:b74591d5ab33	717	pt_cols = ["Target", "Test ID", "Test Description"] + unique_target_toolchains
be_bryan	0:b74591d5ab33	718	pt = PrettyTable(pt_cols)
be_bryan	0:b74591d5ab33	719	for col in pt_cols:
be_bryan	0:b74591d5ab33	720	pt.align[col] = "l"
be_bryan	0:b74591d5ab33	721	pt.padding_width = 1 # One space between column edges and contents (default)
be_bryan	0:b74591d5ab33	722
be_bryan	0:b74591d5ab33	723	for test in unique_tests:
be_bryan	0:b74591d5ab33	724	if test in result_dict:
be_bryan	0:b74591d5ab33	725	test_results = result_dict[test]
be_bryan	0:b74591d5ab33	726	if test in unique_test_desc:
be_bryan	0:b74591d5ab33	727	row = [target, test, unique_test_desc[test]]
be_bryan	0:b74591d5ab33	728	for toolchain in unique_toolchains:
be_bryan	0:b74591d5ab33	729	if toolchain in test_results:
be_bryan	0:b74591d5ab33	730	row.append(test_results[toolchain])
be_bryan	0:b74591d5ab33	731	pt.add_row(row)
be_bryan	0:b74591d5ab33	732	result += pt.get_string()
be_bryan	0:b74591d5ab33	733	shuffle_seed_text = "Shuffle Seed: %.*f"% (self.SHUFFLE_SEED_ROUND,
be_bryan	0:b74591d5ab33	734	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
be_bryan	0:b74591d5ab33	735	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
be_bryan	0:b74591d5ab33	736	return result
be_bryan	0:b74591d5ab33	737
be_bryan	0:b74591d5ab33	738	def generate_test_summary(self, test_summary, shuffle_seed=None):
be_bryan	0:b74591d5ab33	739	""" Prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	740	table shows target x test results matrix across
be_bryan	0:b74591d5ab33	741	"""
be_bryan	0:b74591d5ab33	742	success_code = 0 # Success code that can be leter returned to
be_bryan	0:b74591d5ab33	743	result = "Test summary:\n"
be_bryan	0:b74591d5ab33	744	# Pretty table package is used to print results
be_bryan	0:b74591d5ab33	745	pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
be_bryan	0:b74591d5ab33	746	"Elapsed Time (sec)", "Timeout (sec)", "Loops"])
be_bryan	0:b74591d5ab33	747	pt.align["Result"] = "l" # Left align
be_bryan	0:b74591d5ab33	748	pt.align["Target"] = "l" # Left align
be_bryan	0:b74591d5ab33	749	pt.align["Toolchain"] = "l" # Left align
be_bryan	0:b74591d5ab33	750	pt.align["Test ID"] = "l" # Left align
be_bryan	0:b74591d5ab33	751	pt.align["Test Description"] = "l" # Left align
be_bryan	0:b74591d5ab33	752	pt.padding_width = 1 # One space between column edges and contents (default)
be_bryan	0:b74591d5ab33	753
be_bryan	0:b74591d5ab33	754	result_dict = {self.TEST_RESULT_OK : 0,
be_bryan	0:b74591d5ab33	755	self.TEST_RESULT_FAIL : 0,
be_bryan	0:b74591d5ab33	756	self.TEST_RESULT_ERROR : 0,
be_bryan	0:b74591d5ab33	757	self.TEST_RESULT_UNDEF : 0,
be_bryan	0:b74591d5ab33	758	self.TEST_RESULT_IOERR_COPY : 0,
be_bryan	0:b74591d5ab33	759	self.TEST_RESULT_IOERR_DISK : 0,
be_bryan	0:b74591d5ab33	760	self.TEST_RESULT_IOERR_SERIAL : 0,
be_bryan	0:b74591d5ab33	761	self.TEST_RESULT_NO_IMAGE : 0,
be_bryan	0:b74591d5ab33	762	self.TEST_RESULT_TIMEOUT : 0,
be_bryan	0:b74591d5ab33	763	self.TEST_RESULT_MBED_ASSERT : 0,
be_bryan	0:b74591d5ab33	764	self.TEST_RESULT_BUILD_FAILED : 0,
be_bryan	0:b74591d5ab33	765	self.TEST_RESULT_NOT_SUPPORTED : 0
be_bryan	0:b74591d5ab33	766	}
be_bryan	0:b74591d5ab33	767
be_bryan	0:b74591d5ab33	768	for test in test_summary:
be_bryan	0:b74591d5ab33	769	if test[0] in result_dict:
be_bryan	0:b74591d5ab33	770	result_dict[test[0]] += 1
be_bryan	0:b74591d5ab33	771	pt.add_row(test)
be_bryan	0:b74591d5ab33	772	result += pt.get_string()
be_bryan	0:b74591d5ab33	773	result += "\n"
be_bryan	0:b74591d5ab33	774
be_bryan	0:b74591d5ab33	775	# Print result count
be_bryan	0:b74591d5ab33	776	result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
be_bryan	0:b74591d5ab33	777	shuffle_seed_text = "Shuffle Seed: %.*f\n"% (self.SHUFFLE_SEED_ROUND,
be_bryan	0:b74591d5ab33	778	shuffle_seed if shuffle_seed else self.shuffle_random_seed)
be_bryan	0:b74591d5ab33	779	result += "\n%s"% (shuffle_seed_text if self.opts_shuffle_test_order else '')
be_bryan	0:b74591d5ab33	780	return result
be_bryan	0:b74591d5ab33	781
be_bryan	0:b74591d5ab33	782	def test_loop_list_to_dict(self, test_loops_str):
be_bryan	0:b74591d5ab33	783	""" Transforms test_id=X,test_id=X,test_id=X into dictionary {test_id : test_id_loops_count}
be_bryan	0:b74591d5ab33	784	"""
be_bryan	0:b74591d5ab33	785	result = {}
be_bryan	0:b74591d5ab33	786	if test_loops_str:
be_bryan	0:b74591d5ab33	787	test_loops = test_loops_str
be_bryan	0:b74591d5ab33	788	for test_loop in test_loops:
be_bryan	0:b74591d5ab33	789	test_loop_count = test_loop.split('=')
be_bryan	0:b74591d5ab33	790	if len(test_loop_count) == 2:
be_bryan	0:b74591d5ab33	791	_test_id, _test_loops = test_loop_count
be_bryan	0:b74591d5ab33	792	try:
be_bryan	0:b74591d5ab33	793	_test_loops = int(_test_loops)
be_bryan	0:b74591d5ab33	794	except:
be_bryan	0:b74591d5ab33	795	continue
be_bryan	0:b74591d5ab33	796	result[_test_id] = _test_loops
be_bryan	0:b74591d5ab33	797	return result
be_bryan	0:b74591d5ab33	798
be_bryan	0:b74591d5ab33	799	def get_test_loop_count(self, test_id):
be_bryan	0:b74591d5ab33	800	""" This function returns no. of loops per test (deducted by test_id_.
be_bryan	0:b74591d5ab33	801	If test is not in list of redefined loop counts it will use default value.
be_bryan	0:b74591d5ab33	802	"""
be_bryan	0:b74591d5ab33	803	result = self.GLOBAL_LOOPS_COUNT
be_bryan	0:b74591d5ab33	804	if test_id in self.TEST_LOOPS_DICT:
be_bryan	0:b74591d5ab33	805	result = self.TEST_LOOPS_DICT[test_id]
be_bryan	0:b74591d5ab33	806	return result
be_bryan	0:b74591d5ab33	807
be_bryan	0:b74591d5ab33	808	def delete_file(self, file_path):
be_bryan	0:b74591d5ab33	809	""" Remove file from the system
be_bryan	0:b74591d5ab33	810	"""
be_bryan	0:b74591d5ab33	811	result = True
be_bryan	0:b74591d5ab33	812	resutl_msg = ""
be_bryan	0:b74591d5ab33	813	try:
be_bryan	0:b74591d5ab33	814	os.remove(file_path)
be_bryan	0:b74591d5ab33	815	except Exception, e:
be_bryan	0:b74591d5ab33	816	resutl_msg = e
be_bryan	0:b74591d5ab33	817	result = False
be_bryan	0:b74591d5ab33	818	return result, resutl_msg
be_bryan	0:b74591d5ab33	819
be_bryan	0:b74591d5ab33	820	def handle_mut(self, mut, data, target_name, toolchain_name, test_loops=1):
be_bryan	0:b74591d5ab33	821	""" Test is being invoked for given MUT.
be_bryan	0:b74591d5ab33	822	"""
be_bryan	0:b74591d5ab33	823	# Get test information, image and test timeout
be_bryan	0:b74591d5ab33	824	test_id = data['test_id']
be_bryan	0:b74591d5ab33	825	test = TEST_MAP[test_id]
be_bryan	0:b74591d5ab33	826	test_description = TEST_MAP[test_id].get_description()
be_bryan	0:b74591d5ab33	827	image = data["image"]
be_bryan	0:b74591d5ab33	828	duration = data.get("duration", 10)
be_bryan	0:b74591d5ab33	829
be_bryan	0:b74591d5ab33	830	if mut is None:
be_bryan	0:b74591d5ab33	831	print "Error: No Mbed available: MUT[%s]" % data['mcu']
be_bryan	0:b74591d5ab33	832	return None
be_bryan	0:b74591d5ab33	833
be_bryan	0:b74591d5ab33	834	mcu = mut['mcu']
be_bryan	0:b74591d5ab33	835	copy_method = mut.get('copy_method') # Available board configuration selection e.g. core selection etc.
be_bryan	0:b74591d5ab33	836
be_bryan	0:b74591d5ab33	837	if self.db_logger:
be_bryan	0:b74591d5ab33	838	self.db_logger.reconnect()
be_bryan	0:b74591d5ab33	839
be_bryan	0:b74591d5ab33	840	selected_copy_method = self.opts_copy_method if copy_method is None else copy_method
be_bryan	0:b74591d5ab33	841
be_bryan	0:b74591d5ab33	842	# Tests can be looped so test results must be stored for the same test
be_bryan	0:b74591d5ab33	843	test_all_result = []
be_bryan	0:b74591d5ab33	844	# Test results for one test ran few times
be_bryan	0:b74591d5ab33	845	detailed_test_results = {} # { Loop_number: { results ... } }
be_bryan	0:b74591d5ab33	846
be_bryan	0:b74591d5ab33	847	for test_index in range(test_loops):
be_bryan	0:b74591d5ab33	848
be_bryan	0:b74591d5ab33	849	# If mbedls is available and we are auto detecting MUT info,
be_bryan	0:b74591d5ab33	850	# update MUT info (mounting may changed)
be_bryan	0:b74591d5ab33	851	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
be_bryan	0:b74591d5ab33	852	platform_name_filter = [mcu]
be_bryan	0:b74591d5ab33	853	muts_list = {}
be_bryan	0:b74591d5ab33	854	found = False
be_bryan	0:b74591d5ab33	855
be_bryan	0:b74591d5ab33	856	for i in range(0, 60):
be_bryan	0:b74591d5ab33	857	print('Looking for %s with MBEDLS' % mcu)
be_bryan	0:b74591d5ab33	858	muts_list = get_autodetected_MUTS_list(platform_name_filter=platform_name_filter)
be_bryan	0:b74591d5ab33	859
be_bryan	0:b74591d5ab33	860	if 1 not in muts_list:
be_bryan	0:b74591d5ab33	861	sleep(3)
be_bryan	0:b74591d5ab33	862	else:
be_bryan	0:b74591d5ab33	863	found = True
be_bryan	0:b74591d5ab33	864	break
be_bryan	0:b74591d5ab33	865
be_bryan	0:b74591d5ab33	866	if not found:
be_bryan	0:b74591d5ab33	867	print "Error: mbed not found with MBEDLS: %s" % data['mcu']
be_bryan	0:b74591d5ab33	868	return None
be_bryan	0:b74591d5ab33	869	else:
be_bryan	0:b74591d5ab33	870	mut = muts_list[1]
be_bryan	0:b74591d5ab33	871
be_bryan	0:b74591d5ab33	872	disk = mut.get('disk')
be_bryan	0:b74591d5ab33	873	port = mut.get('port')
be_bryan	0:b74591d5ab33	874
be_bryan	0:b74591d5ab33	875	if disk is None or port is None:
be_bryan	0:b74591d5ab33	876	return None
be_bryan	0:b74591d5ab33	877
be_bryan	0:b74591d5ab33	878	target_by_mcu = TARGET_MAP[mut['mcu']]
be_bryan	0:b74591d5ab33	879	target_name_unique = mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu']
be_bryan	0:b74591d5ab33	880	# Some extra stuff can be declared in MUTs structure
be_bryan	0:b74591d5ab33	881	reset_type = mut.get('reset_type') # reboot.txt, reset.txt, shutdown.txt
be_bryan	0:b74591d5ab33	882	reset_tout = mut.get('reset_tout') # COPY_IMAGE -> RESET_PROC -> SLEEP(RESET_TOUT)
be_bryan	0:b74591d5ab33	883
be_bryan	0:b74591d5ab33	884	# When the build and test system were separate, this was relative to a
be_bryan	0:b74591d5ab33	885	# base network folder base path: join(NETWORK_BASE_PATH, )
be_bryan	0:b74591d5ab33	886	image_path = image
be_bryan	0:b74591d5ab33	887
be_bryan	0:b74591d5ab33	888	# Host test execution
be_bryan	0:b74591d5ab33	889	start_host_exec_time = time()
be_bryan	0:b74591d5ab33	890
be_bryan	0:b74591d5ab33	891	single_test_result = self.TEST_RESULT_UNDEF # single test run result
be_bryan	0:b74591d5ab33	892	_copy_method = selected_copy_method
be_bryan	0:b74591d5ab33	893
be_bryan	0:b74591d5ab33	894	if not exists(image_path):
be_bryan	0:b74591d5ab33	895	single_test_result = self.TEST_RESULT_NO_IMAGE
be_bryan	0:b74591d5ab33	896	elapsed_time = 0
be_bryan	0:b74591d5ab33	897	single_test_output = self.logger.log_line(self.logger.LogType.ERROR, 'Image file does not exist: %s'% image_path)
be_bryan	0:b74591d5ab33	898	print single_test_output
be_bryan	0:b74591d5ab33	899	else:
be_bryan	0:b74591d5ab33	900	# Host test execution
be_bryan	0:b74591d5ab33	901	start_host_exec_time = time()
be_bryan	0:b74591d5ab33	902
be_bryan	0:b74591d5ab33	903	host_test_verbose = self.opts_verbose_test_result_only or self.opts_verbose
be_bryan	0:b74591d5ab33	904	host_test_reset = self.opts_mut_reset_type if reset_type is None else reset_type
be_bryan	0:b74591d5ab33	905	host_test_result = self.run_host_test(test.host_test,
be_bryan	0:b74591d5ab33	906	image_path, disk, port, duration,
be_bryan	0:b74591d5ab33	907	micro=target_name,
be_bryan	0:b74591d5ab33	908	verbose=host_test_verbose,
be_bryan	0:b74591d5ab33	909	reset=host_test_reset,
be_bryan	0:b74591d5ab33	910	reset_tout=reset_tout,
be_bryan	0:b74591d5ab33	911	copy_method=selected_copy_method,
be_bryan	0:b74591d5ab33	912	program_cycle_s=target_by_mcu.program_cycle_s)
be_bryan	0:b74591d5ab33	913	single_test_result, single_test_output, single_testduration, single_timeout = host_test_result
be_bryan	0:b74591d5ab33	914
be_bryan	0:b74591d5ab33	915	# Store test result
be_bryan	0:b74591d5ab33	916	test_all_result.append(single_test_result)
be_bryan	0:b74591d5ab33	917	total_elapsed_time = time() - start_host_exec_time # Test time with copy (flashing) / reset
be_bryan	0:b74591d5ab33	918	elapsed_time = single_testduration # TIme of single test case execution after reset
be_bryan	0:b74591d5ab33	919
be_bryan	0:b74591d5ab33	920	detailed_test_results[test_index] = {
be_bryan	0:b74591d5ab33	921	'result' : single_test_result,
be_bryan	0:b74591d5ab33	922	'output' : single_test_output,
be_bryan	0:b74591d5ab33	923	'target_name' : target_name,
be_bryan	0:b74591d5ab33	924	'target_name_unique' : target_name_unique,
be_bryan	0:b74591d5ab33	925	'toolchain_name' : toolchain_name,
be_bryan	0:b74591d5ab33	926	'id' : test_id,
be_bryan	0:b74591d5ab33	927	'description' : test_description,
be_bryan	0:b74591d5ab33	928	'elapsed_time' : round(elapsed_time, 2),
be_bryan	0:b74591d5ab33	929	'duration' : single_timeout,
be_bryan	0:b74591d5ab33	930	'copy_method' : _copy_method,
be_bryan	0:b74591d5ab33	931	}
be_bryan	0:b74591d5ab33	932
be_bryan	0:b74591d5ab33	933	print self.print_test_result(single_test_result, target_name_unique, toolchain_name,
be_bryan	0:b74591d5ab33	934	test_id, test_description, elapsed_time, single_timeout)
be_bryan	0:b74591d5ab33	935
be_bryan	0:b74591d5ab33	936	# Update database entries for ongoing test
be_bryan	0:b74591d5ab33	937	if self.db_logger and self.db_logger.is_connected():
be_bryan	0:b74591d5ab33	938	test_type = 'SingleTest'
be_bryan	0:b74591d5ab33	939	self.db_logger.insert_test_entry(self.db_logger_build_id,
be_bryan	0:b74591d5ab33	940	target_name,
be_bryan	0:b74591d5ab33	941	toolchain_name,
be_bryan	0:b74591d5ab33	942	test_type,
be_bryan	0:b74591d5ab33	943	test_id,
be_bryan	0:b74591d5ab33	944	single_test_result,
be_bryan	0:b74591d5ab33	945	single_test_output,
be_bryan	0:b74591d5ab33	946	elapsed_time,
be_bryan	0:b74591d5ab33	947	single_timeout,
be_bryan	0:b74591d5ab33	948	test_index)
be_bryan	0:b74591d5ab33	949
be_bryan	0:b74591d5ab33	950	# If we perform waterfall test we test until we get OK and we stop testing
be_bryan	0:b74591d5ab33	951	if self.opts_waterfall_test and single_test_result == self.TEST_RESULT_OK:
be_bryan	0:b74591d5ab33	952	break
be_bryan	0:b74591d5ab33	953
be_bryan	0:b74591d5ab33	954	if self.db_logger:
be_bryan	0:b74591d5ab33	955	self.db_logger.disconnect()
be_bryan	0:b74591d5ab33	956
be_bryan	0:b74591d5ab33	957	return (self.shape_global_test_loop_result(test_all_result, self.opts_waterfall_test and self.opts_consolidate_waterfall_test),
be_bryan	0:b74591d5ab33	958	target_name_unique,
be_bryan	0:b74591d5ab33	959	toolchain_name,
be_bryan	0:b74591d5ab33	960	test_id,
be_bryan	0:b74591d5ab33	961	test_description,
be_bryan	0:b74591d5ab33	962	round(elapsed_time, 2),
be_bryan	0:b74591d5ab33	963	single_timeout,
be_bryan	0:b74591d5ab33	964	self.shape_test_loop_ok_result_count(test_all_result)), detailed_test_results
be_bryan	0:b74591d5ab33	965
be_bryan	0:b74591d5ab33	966	def handle(self, test_spec, target_name, toolchain_name, test_loops=1):
be_bryan	0:b74591d5ab33	967	""" Function determines MUT's mbed disk/port and copies binary to
be_bryan	0:b74591d5ab33	968	target.
be_bryan	0:b74591d5ab33	969	"""
be_bryan	0:b74591d5ab33	970	handle_results = []
be_bryan	0:b74591d5ab33	971	data = json.loads(test_spec)
be_bryan	0:b74591d5ab33	972
be_bryan	0:b74591d5ab33	973	# Find a suitable MUT:
be_bryan	0:b74591d5ab33	974	mut = None
be_bryan	0:b74591d5ab33	975	for id, m in self.muts.iteritems():
be_bryan	0:b74591d5ab33	976	if m['mcu'] == data['mcu']:
be_bryan	0:b74591d5ab33	977	mut = m
be_bryan	0:b74591d5ab33	978	handle_result = self.handle_mut(mut, data, target_name, toolchain_name, test_loops=test_loops)
be_bryan	0:b74591d5ab33	979	handle_results.append(handle_result)
be_bryan	0:b74591d5ab33	980
be_bryan	0:b74591d5ab33	981	return handle_results
be_bryan	0:b74591d5ab33	982
be_bryan	0:b74591d5ab33	983	def print_test_result(self, test_result, target_name, toolchain_name,
be_bryan	0:b74591d5ab33	984	test_id, test_description, elapsed_time, duration):
be_bryan	0:b74591d5ab33	985	""" Use specific convention to print test result and related data
be_bryan	0:b74591d5ab33	986	"""
be_bryan	0:b74591d5ab33	987	tokens = []
be_bryan	0:b74591d5ab33	988	tokens.append("TargetTest")
be_bryan	0:b74591d5ab33	989	tokens.append(target_name)
be_bryan	0:b74591d5ab33	990	tokens.append(toolchain_name)
be_bryan	0:b74591d5ab33	991	tokens.append(test_id)
be_bryan	0:b74591d5ab33	992	tokens.append(test_description)
be_bryan	0:b74591d5ab33	993	separator = "::"
be_bryan	0:b74591d5ab33	994	time_info = " in %.2f of %d sec" % (round(elapsed_time, 2), duration)
be_bryan	0:b74591d5ab33	995	result = separator.join(tokens) + " [" + test_result +"]" + time_info
be_bryan	0:b74591d5ab33	996	return Fore.MAGENTA + result + Fore.RESET
be_bryan	0:b74591d5ab33	997
be_bryan	0:b74591d5ab33	998	def shape_test_loop_ok_result_count(self, test_all_result):
be_bryan	0:b74591d5ab33	999	""" Reformats list of results to simple string
be_bryan	0:b74591d5ab33	1000	"""
be_bryan	0:b74591d5ab33	1001	test_loop_count = len(test_all_result)
be_bryan	0:b74591d5ab33	1002	test_loop_ok_result = test_all_result.count(self.TEST_RESULT_OK)
be_bryan	0:b74591d5ab33	1003	return "%d/%d"% (test_loop_ok_result, test_loop_count)
be_bryan	0:b74591d5ab33	1004
be_bryan	0:b74591d5ab33	1005	def shape_global_test_loop_result(self, test_all_result, waterfall_and_consolidate):
be_bryan	0:b74591d5ab33	1006	""" Reformats list of results to simple string
be_bryan	0:b74591d5ab33	1007	"""
be_bryan	0:b74591d5ab33	1008	result = self.TEST_RESULT_FAIL
be_bryan	0:b74591d5ab33	1009
be_bryan	0:b74591d5ab33	1010	if all(test_all_result[0] == res for res in test_all_result):
be_bryan	0:b74591d5ab33	1011	result = test_all_result[0]
be_bryan	0:b74591d5ab33	1012	elif waterfall_and_consolidate and any(res == self.TEST_RESULT_OK for res in test_all_result):
be_bryan	0:b74591d5ab33	1013	result = self.TEST_RESULT_OK
be_bryan	0:b74591d5ab33	1014
be_bryan	0:b74591d5ab33	1015	return result
be_bryan	0:b74591d5ab33	1016
be_bryan	0:b74591d5ab33	1017	def run_host_test(self, name, image_path, disk, port, duration,
be_bryan	0:b74591d5ab33	1018	micro=None, reset=None, reset_tout=None,
be_bryan	0:b74591d5ab33	1019	verbose=False, copy_method=None, program_cycle_s=None):
be_bryan	0:b74591d5ab33	1020	""" Function creates new process with host test configured with particular test case.
be_bryan	0:b74591d5ab33	1021	Function also is pooling for serial port activity from process to catch all data
be_bryan	0:b74591d5ab33	1022	printed by test runner and host test during test execution
be_bryan	0:b74591d5ab33	1023	"""
be_bryan	0:b74591d5ab33	1024
be_bryan	0:b74591d5ab33	1025	def get_char_from_queue(obs):
be_bryan	0:b74591d5ab33	1026	""" Get character from queue safe way
be_bryan	0:b74591d5ab33	1027	"""
be_bryan	0:b74591d5ab33	1028	try:
be_bryan	0:b74591d5ab33	1029	c = obs.queue.get(block=True, timeout=0.5)
be_bryan	0:b74591d5ab33	1030	except Empty, _:
be_bryan	0:b74591d5ab33	1031	c = None
be_bryan	0:b74591d5ab33	1032	return c
be_bryan	0:b74591d5ab33	1033
be_bryan	0:b74591d5ab33	1034	def filter_queue_char(c):
be_bryan	0:b74591d5ab33	1035	""" Filters out non ASCII characters from serial port
be_bryan	0:b74591d5ab33	1036	"""
be_bryan	0:b74591d5ab33	1037	if ord(c) not in range(128):
be_bryan	0:b74591d5ab33	1038	c = ' '
be_bryan	0:b74591d5ab33	1039	return c
be_bryan	0:b74591d5ab33	1040
be_bryan	0:b74591d5ab33	1041	def get_test_result(output):
be_bryan	0:b74591d5ab33	1042	""" Parse test 'output' data
be_bryan	0:b74591d5ab33	1043	"""
be_bryan	0:b74591d5ab33	1044	result = self.TEST_RESULT_TIMEOUT
be_bryan	0:b74591d5ab33	1045	for line in "".join(output).splitlines():
be_bryan	0:b74591d5ab33	1046	search_result = self.RE_DETECT_TESTCASE_RESULT.search(line)
be_bryan	0:b74591d5ab33	1047	if search_result and len(search_result.groups()):
be_bryan	0:b74591d5ab33	1048	result = self.TEST_RESULT_MAPPING[search_result.groups(0)[0]]
be_bryan	0:b74591d5ab33	1049	break
be_bryan	0:b74591d5ab33	1050	return result
be_bryan	0:b74591d5ab33	1051
be_bryan	0:b74591d5ab33	1052	def get_auto_property_value(property_name, line):
be_bryan	0:b74591d5ab33	1053	""" Scans auto detection line from MUT and returns scanned parameter 'property_name'
be_bryan	0:b74591d5ab33	1054	Returns string
be_bryan	0:b74591d5ab33	1055	"""
be_bryan	0:b74591d5ab33	1056	result = None
be_bryan	0:b74591d5ab33	1057	if re.search("HOST: Property '%s'"% property_name, line) is not None:
be_bryan	0:b74591d5ab33	1058	property = re.search("HOST: Property '%s' = '([\w\d _]+)'"% property_name, line)
be_bryan	0:b74591d5ab33	1059	if property is not None and len(property.groups()) == 1:
be_bryan	0:b74591d5ab33	1060	result = property.groups()[0]
be_bryan	0:b74591d5ab33	1061	return result
be_bryan	0:b74591d5ab33	1062
be_bryan	0:b74591d5ab33	1063	# print "{%s} port:%s disk:%s" % (name, port, disk),
be_bryan	0:b74591d5ab33	1064	cmd = ["python",
be_bryan	0:b74591d5ab33	1065	'%s.py'% name,
be_bryan	0:b74591d5ab33	1066	'-d', disk,
be_bryan	0:b74591d5ab33	1067	'-f', '"%s"'% image_path,
be_bryan	0:b74591d5ab33	1068	'-p', port,
be_bryan	0:b74591d5ab33	1069	'-t', str(duration),
be_bryan	0:b74591d5ab33	1070	'-C', str(program_cycle_s)]
be_bryan	0:b74591d5ab33	1071
be_bryan	0:b74591d5ab33	1072	if get_module_avail('mbed_lstools') and self.opts_auto_detect:
be_bryan	0:b74591d5ab33	1073	cmd += ['--auto']
be_bryan	0:b74591d5ab33	1074
be_bryan	0:b74591d5ab33	1075	# Add extra parameters to host_test
be_bryan	0:b74591d5ab33	1076	if copy_method is not None:
be_bryan	0:b74591d5ab33	1077	cmd += ["-c", copy_method]
be_bryan	0:b74591d5ab33	1078	if micro is not None:
be_bryan	0:b74591d5ab33	1079	cmd += ["-m", micro]
be_bryan	0:b74591d5ab33	1080	if reset is not None:
be_bryan	0:b74591d5ab33	1081	cmd += ["-r", reset]
be_bryan	0:b74591d5ab33	1082	if reset_tout is not None:
be_bryan	0:b74591d5ab33	1083	cmd += ["-R", str(reset_tout)]
be_bryan	0:b74591d5ab33	1084
be_bryan	0:b74591d5ab33	1085	if verbose:
be_bryan	0:b74591d5ab33	1086	print Fore.MAGENTA + "Executing '" + " ".join(cmd) + "'" + Fore.RESET
be_bryan	0:b74591d5ab33	1087	print "Test::Output::Start"
be_bryan	0:b74591d5ab33	1088
be_bryan	0:b74591d5ab33	1089	proc = Popen(cmd, stdout=PIPE, cwd=HOST_TESTS)
be_bryan	0:b74591d5ab33	1090	obs = ProcessObserver(proc)
be_bryan	0:b74591d5ab33	1091	update_once_flag = {} # Stores flags checking if some auto-parameter was already set
be_bryan	0:b74591d5ab33	1092	line = ''
be_bryan	0:b74591d5ab33	1093	output = []
be_bryan	0:b74591d5ab33	1094	start_time = time()
be_bryan	0:b74591d5ab33	1095	while (time() - start_time) < (2 * duration):
be_bryan	0:b74591d5ab33	1096	c = get_char_from_queue(obs)
be_bryan	0:b74591d5ab33	1097	if c:
be_bryan	0:b74591d5ab33	1098	if verbose:
be_bryan	0:b74591d5ab33	1099	sys.stdout.write(c)
be_bryan	0:b74591d5ab33	1100	c = filter_queue_char(c)
be_bryan	0:b74591d5ab33	1101	output.append(c)
be_bryan	0:b74591d5ab33	1102	# Give the mbed under test a way to communicate the end of the test
be_bryan	0:b74591d5ab33	1103	if c in ['\n', '\r']:
be_bryan	0:b74591d5ab33	1104
be_bryan	0:b74591d5ab33	1105	# Checking for auto-detection information from the test about MUT reset moment
be_bryan	0:b74591d5ab33	1106	if 'reset_target' not in update_once_flag and "HOST: Reset target..." in line:
be_bryan	0:b74591d5ab33	1107	# We will update this marker only once to prevent multiple time resets
be_bryan	0:b74591d5ab33	1108	update_once_flag['reset_target'] = True
be_bryan	0:b74591d5ab33	1109	start_time = time()
be_bryan	0:b74591d5ab33	1110
be_bryan	0:b74591d5ab33	1111	# Checking for auto-detection information from the test about timeout
be_bryan	0:b74591d5ab33	1112	auto_timeout_val = get_auto_property_value('timeout', line)
be_bryan	0:b74591d5ab33	1113	if 'timeout' not in update_once_flag and auto_timeout_val is not None:
be_bryan	0:b74591d5ab33	1114	# We will update this marker only once to prevent multiple time resets
be_bryan	0:b74591d5ab33	1115	update_once_flag['timeout'] = True
be_bryan	0:b74591d5ab33	1116	duration = int(auto_timeout_val)
be_bryan	0:b74591d5ab33	1117
be_bryan	0:b74591d5ab33	1118	# Detect mbed assert:
be_bryan	0:b74591d5ab33	1119	if 'mbed assertation failed: ' in line:
be_bryan	0:b74591d5ab33	1120	output.append('{{mbed_assert}}')
be_bryan	0:b74591d5ab33	1121	break
be_bryan	0:b74591d5ab33	1122
be_bryan	0:b74591d5ab33	1123	# Check for test end
be_bryan	0:b74591d5ab33	1124	if '{end}' in line:
be_bryan	0:b74591d5ab33	1125	break
be_bryan	0:b74591d5ab33	1126	line = ''
be_bryan	0:b74591d5ab33	1127	else:
be_bryan	0:b74591d5ab33	1128	line += c
be_bryan	0:b74591d5ab33	1129	end_time = time()
be_bryan	0:b74591d5ab33	1130	testcase_duration = end_time - start_time # Test case duration from reset to {end}
be_bryan	0:b74591d5ab33	1131
be_bryan	0:b74591d5ab33	1132	c = get_char_from_queue(obs)
be_bryan	0:b74591d5ab33	1133
be_bryan	0:b74591d5ab33	1134	if c:
be_bryan	0:b74591d5ab33	1135	if verbose:
be_bryan	0:b74591d5ab33	1136	sys.stdout.write(c)
be_bryan	0:b74591d5ab33	1137	c = filter_queue_char(c)
be_bryan	0:b74591d5ab33	1138	output.append(c)
be_bryan	0:b74591d5ab33	1139
be_bryan	0:b74591d5ab33	1140	if verbose:
be_bryan	0:b74591d5ab33	1141	print "Test::Output::Finish"
be_bryan	0:b74591d5ab33	1142	# Stop test process
be_bryan	0:b74591d5ab33	1143	obs.stop()
be_bryan	0:b74591d5ab33	1144
be_bryan	0:b74591d5ab33	1145	result = get_test_result(output)
be_bryan	0:b74591d5ab33	1146	return (result, "".join(output), testcase_duration, duration)
be_bryan	0:b74591d5ab33	1147
be_bryan	0:b74591d5ab33	1148	def is_peripherals_available(self, target_mcu_name, peripherals=None):
be_bryan	0:b74591d5ab33	1149	""" Checks if specified target should run specific peripheral test case defined in MUTs file
be_bryan	0:b74591d5ab33	1150	"""
be_bryan	0:b74591d5ab33	1151	if peripherals is not None:
be_bryan	0:b74591d5ab33	1152	peripherals = set(peripherals)
be_bryan	0:b74591d5ab33	1153	for id, mut in self.muts.iteritems():
be_bryan	0:b74591d5ab33	1154	# Target MCU name check
be_bryan	0:b74591d5ab33	1155	if mut["mcu"] != target_mcu_name:
be_bryan	0:b74591d5ab33	1156	continue
be_bryan	0:b74591d5ab33	1157	# Peripherals check
be_bryan	0:b74591d5ab33	1158	if peripherals is not None:
be_bryan	0:b74591d5ab33	1159	if 'peripherals' not in mut:
be_bryan	0:b74591d5ab33	1160	continue
be_bryan	0:b74591d5ab33	1161	if not peripherals.issubset(set(mut['peripherals'])):
be_bryan	0:b74591d5ab33	1162	continue
be_bryan	0:b74591d5ab33	1163	return True
be_bryan	0:b74591d5ab33	1164	return False
be_bryan	0:b74591d5ab33	1165
be_bryan	0:b74591d5ab33	1166	def shape_test_request(self, mcu, image_path, test_id, duration=10):
be_bryan	0:b74591d5ab33	1167	""" Function prepares JSON structure describing test specification
be_bryan	0:b74591d5ab33	1168	"""
be_bryan	0:b74591d5ab33	1169	test_spec = {
be_bryan	0:b74591d5ab33	1170	"mcu": mcu,
be_bryan	0:b74591d5ab33	1171	"image": image_path,
be_bryan	0:b74591d5ab33	1172	"duration": duration,
be_bryan	0:b74591d5ab33	1173	"test_id": test_id,
be_bryan	0:b74591d5ab33	1174	}
be_bryan	0:b74591d5ab33	1175	return json.dumps(test_spec)
be_bryan	0:b74591d5ab33	1176
be_bryan	0:b74591d5ab33	1177
be_bryan	0:b74591d5ab33	1178	def get_unique_value_from_summary(test_summary, index):
be_bryan	0:b74591d5ab33	1179	""" Gets list of unique target names
be_bryan	0:b74591d5ab33	1180	"""
be_bryan	0:b74591d5ab33	1181	result = []
be_bryan	0:b74591d5ab33	1182	for test in test_summary:
be_bryan	0:b74591d5ab33	1183	target_name = test[index]
be_bryan	0:b74591d5ab33	1184	if target_name not in result:
be_bryan	0:b74591d5ab33	1185	result.append(target_name)
be_bryan	0:b74591d5ab33	1186	return sorted(result)
be_bryan	0:b74591d5ab33	1187
be_bryan	0:b74591d5ab33	1188
be_bryan	0:b74591d5ab33	1189	def get_unique_value_from_summary_ext(test_summary, index_key, index_val):
be_bryan	0:b74591d5ab33	1190	""" Gets list of unique target names and return dictionary
be_bryan	0:b74591d5ab33	1191	"""
be_bryan	0:b74591d5ab33	1192	result = {}
be_bryan	0:b74591d5ab33	1193	for test in test_summary:
be_bryan	0:b74591d5ab33	1194	key = test[index_key]
be_bryan	0:b74591d5ab33	1195	val = test[index_val]
be_bryan	0:b74591d5ab33	1196	if key not in result:
be_bryan	0:b74591d5ab33	1197	result[key] = val
be_bryan	0:b74591d5ab33	1198	return result
be_bryan	0:b74591d5ab33	1199
be_bryan	0:b74591d5ab33	1200
be_bryan	0:b74591d5ab33	1201	def show_json_file_format_error(json_spec_filename, line, column):
be_bryan	0:b74591d5ab33	1202	""" Prints JSON broken content
be_bryan	0:b74591d5ab33	1203	"""
be_bryan	0:b74591d5ab33	1204	with open(json_spec_filename) as data_file:
be_bryan	0:b74591d5ab33	1205	line_no = 1
be_bryan	0:b74591d5ab33	1206	for json_line in data_file:
be_bryan	0:b74591d5ab33	1207	if line_no + 5 >= line: # Print last few lines before error
be_bryan	0:b74591d5ab33	1208	print 'Line %d:\t'%line_no + json_line, # Prints line
be_bryan	0:b74591d5ab33	1209	if line_no == line:
be_bryan	0:b74591d5ab33	1210	print ' ' * len('Line %d:'%line_no) + '\t', '-' * (column-1) + '^'
be_bryan	0:b74591d5ab33	1211	break
be_bryan	0:b74591d5ab33	1212	line_no += 1
be_bryan	0:b74591d5ab33	1213
be_bryan	0:b74591d5ab33	1214
be_bryan	0:b74591d5ab33	1215	def json_format_error_defect_pos(json_error_msg):
be_bryan	0:b74591d5ab33	1216	""" Gets first error line and column in JSON file format.
be_bryan	0:b74591d5ab33	1217	Parsed from exception thrown by json.loads() string
be_bryan	0:b74591d5ab33	1218	"""
be_bryan	0:b74591d5ab33	1219	result = None
be_bryan	0:b74591d5ab33	1220	line, column = 0, 0
be_bryan	0:b74591d5ab33	1221	# Line value search
be_bryan	0:b74591d5ab33	1222	line_search = re.search('line [0-9]+', json_error_msg)
be_bryan	0:b74591d5ab33	1223	if line_search is not None:
be_bryan	0:b74591d5ab33	1224	ls = line_search.group().split(' ')
be_bryan	0:b74591d5ab33	1225	if len(ls) == 2:
be_bryan	0:b74591d5ab33	1226	line = int(ls[1])
be_bryan	0:b74591d5ab33	1227	# Column position search
be_bryan	0:b74591d5ab33	1228	column_search = re.search('column [0-9]+', json_error_msg)
be_bryan	0:b74591d5ab33	1229	if column_search is not None:
be_bryan	0:b74591d5ab33	1230	cs = column_search.group().split(' ')
be_bryan	0:b74591d5ab33	1231	if len(cs) == 2:
be_bryan	0:b74591d5ab33	1232	column = int(cs[1])
be_bryan	0:b74591d5ab33	1233	result = [line, column]
be_bryan	0:b74591d5ab33	1234	return result
be_bryan	0:b74591d5ab33	1235
be_bryan	0:b74591d5ab33	1236
be_bryan	0:b74591d5ab33	1237	def get_json_data_from_file(json_spec_filename, verbose=False):
be_bryan	0:b74591d5ab33	1238	""" Loads from file JSON formatted string to data structure
be_bryan	0:b74591d5ab33	1239	"""
be_bryan	0:b74591d5ab33	1240	result = None
be_bryan	0:b74591d5ab33	1241	try:
be_bryan	0:b74591d5ab33	1242	with open(json_spec_filename) as data_file:
be_bryan	0:b74591d5ab33	1243	try:
be_bryan	0:b74591d5ab33	1244	result = json.load(data_file)
be_bryan	0:b74591d5ab33	1245	except ValueError as json_error_msg:
be_bryan	0:b74591d5ab33	1246	result = None
be_bryan	0:b74591d5ab33	1247	print 'JSON file %s parsing failed. Reason: %s' % (json_spec_filename, json_error_msg)
be_bryan	0:b74591d5ab33	1248	# We can print where error occurred inside JSON file if we can parse exception msg
be_bryan	0:b74591d5ab33	1249	json_format_defect_pos = json_format_error_defect_pos(str(json_error_msg))
be_bryan	0:b74591d5ab33	1250	if json_format_defect_pos is not None:
be_bryan	0:b74591d5ab33	1251	line = json_format_defect_pos[0]
be_bryan	0:b74591d5ab33	1252	column = json_format_defect_pos[1]
be_bryan	0:b74591d5ab33	1253	print
be_bryan	0:b74591d5ab33	1254	show_json_file_format_error(json_spec_filename, line, column)
be_bryan	0:b74591d5ab33	1255
be_bryan	0:b74591d5ab33	1256	except IOError as fileopen_error_msg:
be_bryan	0:b74591d5ab33	1257	print 'JSON file %s not opened. Reason: %s'% (json_spec_filename, fileopen_error_msg)
be_bryan	0:b74591d5ab33	1258	print
be_bryan	0:b74591d5ab33	1259	if verbose and result:
be_bryan	0:b74591d5ab33	1260	pp = pprint.PrettyPrinter(indent=4)
be_bryan	0:b74591d5ab33	1261	pp.pprint(result)
be_bryan	0:b74591d5ab33	1262	return result
be_bryan	0:b74591d5ab33	1263
be_bryan	0:b74591d5ab33	1264
be_bryan	0:b74591d5ab33	1265	def print_muts_configuration_from_json(json_data, join_delim=", ", platform_filter=None):
be_bryan	0:b74591d5ab33	1266	""" Prints MUTs configuration passed to test script for verboseness
be_bryan	0:b74591d5ab33	1267	"""
be_bryan	0:b74591d5ab33	1268	muts_info_cols = []
be_bryan	0:b74591d5ab33	1269	# We need to check all unique properties for each defined MUT
be_bryan	0:b74591d5ab33	1270	for k in json_data:
be_bryan	0:b74591d5ab33	1271	mut_info = json_data[k]
be_bryan	0:b74591d5ab33	1272	for mut_property in mut_info:
be_bryan	0:b74591d5ab33	1273	if mut_property not in muts_info_cols:
be_bryan	0:b74591d5ab33	1274	muts_info_cols.append(mut_property)
be_bryan	0:b74591d5ab33	1275
be_bryan	0:b74591d5ab33	1276	# Prepare pretty table object to display all MUTs
be_bryan	0:b74591d5ab33	1277	pt_cols = ["index"] + muts_info_cols
be_bryan	0:b74591d5ab33	1278	pt = PrettyTable(pt_cols)
be_bryan	0:b74591d5ab33	1279	for col in pt_cols:
be_bryan	0:b74591d5ab33	1280	pt.align[col] = "l"
be_bryan	0:b74591d5ab33	1281
be_bryan	0:b74591d5ab33	1282	# Add rows to pretty print object
be_bryan	0:b74591d5ab33	1283	for k in json_data:
be_bryan	0:b74591d5ab33	1284	row = [k]
be_bryan	0:b74591d5ab33	1285	mut_info = json_data[k]
be_bryan	0:b74591d5ab33	1286
be_bryan	0:b74591d5ab33	1287	add_row = True
be_bryan	0:b74591d5ab33	1288	if platform_filter and 'mcu' in mut_info:
be_bryan	0:b74591d5ab33	1289	add_row = re.search(platform_filter, mut_info['mcu']) is not None
be_bryan	0:b74591d5ab33	1290	if add_row:
be_bryan	0:b74591d5ab33	1291	for col in muts_info_cols:
be_bryan	0:b74591d5ab33	1292	cell_val = mut_info[col] if col in mut_info else None
be_bryan	0:b74591d5ab33	1293	if type(cell_val) == ListType:
be_bryan	0:b74591d5ab33	1294	cell_val = join_delim.join(cell_val)
be_bryan	0:b74591d5ab33	1295	row.append(cell_val)
be_bryan	0:b74591d5ab33	1296	pt.add_row(row)
be_bryan	0:b74591d5ab33	1297	return pt.get_string()
be_bryan	0:b74591d5ab33	1298
be_bryan	0:b74591d5ab33	1299
be_bryan	0:b74591d5ab33	1300	def print_test_configuration_from_json(json_data, join_delim=", "):
be_bryan	0:b74591d5ab33	1301	""" Prints test specification configuration passed to test script for verboseness
be_bryan	0:b74591d5ab33	1302	"""
be_bryan	0:b74591d5ab33	1303	toolchains_info_cols = []
be_bryan	0:b74591d5ab33	1304	# We need to check all toolchains for each device
be_bryan	0:b74591d5ab33	1305	for k in json_data:
be_bryan	0:b74591d5ab33	1306	# k should be 'targets'
be_bryan	0:b74591d5ab33	1307	targets = json_data[k]
be_bryan	0:b74591d5ab33	1308	for target in targets:
be_bryan	0:b74591d5ab33	1309	toolchains = targets[target]
be_bryan	0:b74591d5ab33	1310	for toolchain in toolchains:
be_bryan	0:b74591d5ab33	1311	if toolchain not in toolchains_info_cols:
be_bryan	0:b74591d5ab33	1312	toolchains_info_cols.append(toolchain)
be_bryan	0:b74591d5ab33	1313
be_bryan	0:b74591d5ab33	1314	# Prepare pretty table object to display test specification
be_bryan	0:b74591d5ab33	1315	pt_cols = ["mcu"] + sorted(toolchains_info_cols)
be_bryan	0:b74591d5ab33	1316	pt = PrettyTable(pt_cols)
be_bryan	0:b74591d5ab33	1317	for col in pt_cols:
be_bryan	0:b74591d5ab33	1318	pt.align[col] = "l"
be_bryan	0:b74591d5ab33	1319
be_bryan	0:b74591d5ab33	1320	# { target : [conflicted toolchains] }
be_bryan	0:b74591d5ab33	1321	toolchain_conflicts = {}
be_bryan	0:b74591d5ab33	1322	toolchain_path_conflicts = []
be_bryan	0:b74591d5ab33	1323	for k in json_data:
be_bryan	0:b74591d5ab33	1324	# k should be 'targets'
be_bryan	0:b74591d5ab33	1325	targets = json_data[k]
be_bryan	0:b74591d5ab33	1326	for target in targets:
be_bryan	0:b74591d5ab33	1327	target_supported_toolchains = get_target_supported_toolchains(target)
be_bryan	0:b74591d5ab33	1328	if not target_supported_toolchains:
be_bryan	0:b74591d5ab33	1329	target_supported_toolchains = []
be_bryan	0:b74591d5ab33	1330	target_name = target if target in TARGET_MAP else "%s*"% target
be_bryan	0:b74591d5ab33	1331	row = [target_name]
be_bryan	0:b74591d5ab33	1332	toolchains = targets[target]
be_bryan	0:b74591d5ab33	1333
be_bryan	0:b74591d5ab33	1334	for toolchain in sorted(toolchains_info_cols):
be_bryan	0:b74591d5ab33	1335	# Check for conflicts: target vs toolchain
be_bryan	0:b74591d5ab33	1336	conflict = False
be_bryan	0:b74591d5ab33	1337	conflict_path = False
be_bryan	0:b74591d5ab33	1338	if toolchain in toolchains:
be_bryan	0:b74591d5ab33	1339	if toolchain not in target_supported_toolchains:
be_bryan	0:b74591d5ab33	1340	conflict = True
be_bryan	0:b74591d5ab33	1341	if target not in toolchain_conflicts:
be_bryan	0:b74591d5ab33	1342	toolchain_conflicts[target] = []
be_bryan	0:b74591d5ab33	1343	toolchain_conflicts[target].append(toolchain)
be_bryan	0:b74591d5ab33	1344	# Add marker inside table about target usage / conflict
be_bryan	0:b74591d5ab33	1345	cell_val = 'Yes' if toolchain in toolchains else '-'
be_bryan	0:b74591d5ab33	1346	if conflict:
be_bryan	0:b74591d5ab33	1347	cell_val += '*'
be_bryan	0:b74591d5ab33	1348	# Check for conflicts: toolchain vs toolchain path
be_bryan	0:b74591d5ab33	1349	if toolchain in TOOLCHAIN_PATHS:
be_bryan	0:b74591d5ab33	1350	toolchain_path = TOOLCHAIN_PATHS[toolchain]
be_bryan	0:b74591d5ab33	1351	if not os.path.isdir(toolchain_path):
be_bryan	0:b74591d5ab33	1352	conflict_path = True
be_bryan	0:b74591d5ab33	1353	if toolchain not in toolchain_path_conflicts:
be_bryan	0:b74591d5ab33	1354	toolchain_path_conflicts.append(toolchain)
be_bryan	0:b74591d5ab33	1355	if conflict_path:
be_bryan	0:b74591d5ab33	1356	cell_val += '#'
be_bryan	0:b74591d5ab33	1357	row.append(cell_val)
be_bryan	0:b74591d5ab33	1358	pt.add_row(row)
be_bryan	0:b74591d5ab33	1359
be_bryan	0:b74591d5ab33	1360	# generate result string
be_bryan	0:b74591d5ab33	1361	result = pt.get_string() # Test specification table
be_bryan	0:b74591d5ab33	1362	if toolchain_conflicts or toolchain_path_conflicts:
be_bryan	0:b74591d5ab33	1363	result += "\n"
be_bryan	0:b74591d5ab33	1364	result += "Toolchain conflicts:\n"
be_bryan	0:b74591d5ab33	1365	for target in toolchain_conflicts:
be_bryan	0:b74591d5ab33	1366	if target not in TARGET_MAP:
be_bryan	0:b74591d5ab33	1367	result += "\t* Target %s unknown\n"% (target)
be_bryan	0:b74591d5ab33	1368	conflict_target_list = join_delim.join(toolchain_conflicts[target])
be_bryan	0:b74591d5ab33	1369	sufix = 's' if len(toolchain_conflicts[target]) > 1 else ''
be_bryan	0:b74591d5ab33	1370	result += "\t* Target %s does not support %s toolchain%s\n"% (target, conflict_target_list, sufix)
be_bryan	0:b74591d5ab33	1371
be_bryan	0:b74591d5ab33	1372	for toolchain in toolchain_path_conflicts:
be_bryan	0:b74591d5ab33	1373	# Let's check toolchain configuration
be_bryan	0:b74591d5ab33	1374	if toolchain in TOOLCHAIN_PATHS:
be_bryan	0:b74591d5ab33	1375	toolchain_path = TOOLCHAIN_PATHS[toolchain]
be_bryan	0:b74591d5ab33	1376	if not os.path.isdir(toolchain_path):
be_bryan	0:b74591d5ab33	1377	result += "\t# Toolchain %s path not found: %s\n"% (toolchain, toolchain_path)
be_bryan	0:b74591d5ab33	1378	return result
be_bryan	0:b74591d5ab33	1379
be_bryan	0:b74591d5ab33	1380
be_bryan	0:b74591d5ab33	1381	def get_avail_tests_summary_table(cols=None, result_summary=True, join_delim=',',platform_filter=None):
be_bryan	0:b74591d5ab33	1382	""" Generates table summary with all test cases and additional test cases
be_bryan	0:b74591d5ab33	1383	information using pretty print functionality. Allows test suite user to
be_bryan	0:b74591d5ab33	1384	see test cases
be_bryan	0:b74591d5ab33	1385	"""
be_bryan	0:b74591d5ab33	1386	# get all unique test ID prefixes
be_bryan	0:b74591d5ab33	1387	unique_test_id = []
be_bryan	0:b74591d5ab33	1388	for test in TESTS:
be_bryan	0:b74591d5ab33	1389	split = test['id'].split('_')[:-1]
be_bryan	0:b74591d5ab33	1390	test_id_prefix = '_'.join(split)
be_bryan	0:b74591d5ab33	1391	if test_id_prefix not in unique_test_id:
be_bryan	0:b74591d5ab33	1392	unique_test_id.append(test_id_prefix)
be_bryan	0:b74591d5ab33	1393	unique_test_id.sort()
be_bryan	0:b74591d5ab33	1394	counter_dict_test_id_types = dict((t, 0) for t in unique_test_id)
be_bryan	0:b74591d5ab33	1395	counter_dict_test_id_types_all = dict((t, 0) for t in unique_test_id)
be_bryan	0:b74591d5ab33	1396
be_bryan	0:b74591d5ab33	1397	test_properties = ['id',
be_bryan	0:b74591d5ab33	1398	'automated',
be_bryan	0:b74591d5ab33	1399	'description',
be_bryan	0:b74591d5ab33	1400	'peripherals',
be_bryan	0:b74591d5ab33	1401	'host_test',
be_bryan	0:b74591d5ab33	1402	'duration'] if cols is None else cols
be_bryan	0:b74591d5ab33	1403
be_bryan	0:b74591d5ab33	1404	# All tests status table print
be_bryan	0:b74591d5ab33	1405	pt = PrettyTable(test_properties)
be_bryan	0:b74591d5ab33	1406	for col in test_properties:
be_bryan	0:b74591d5ab33	1407	pt.align[col] = "l"
be_bryan	0:b74591d5ab33	1408	pt.align['duration'] = "r"
be_bryan	0:b74591d5ab33	1409
be_bryan	0:b74591d5ab33	1410	counter_all = 0
be_bryan	0:b74591d5ab33	1411	counter_automated = 0
be_bryan	0:b74591d5ab33	1412	pt.padding_width = 1 # One space between column edges and contents (default)
be_bryan	0:b74591d5ab33	1413
be_bryan	0:b74591d5ab33	1414	for test_id in sorted(TEST_MAP.keys()):
be_bryan	0:b74591d5ab33	1415	if platform_filter is not None:
be_bryan	0:b74591d5ab33	1416	# FIlter out platforms using regex
be_bryan	0:b74591d5ab33	1417	if re.search(platform_filter, test_id) is None:
be_bryan	0:b74591d5ab33	1418	continue
be_bryan	0:b74591d5ab33	1419	row = []
be_bryan	0:b74591d5ab33	1420	test = TEST_MAP[test_id]
be_bryan	0:b74591d5ab33	1421	split = test_id.split('_')[:-1]
be_bryan	0:b74591d5ab33	1422	test_id_prefix = '_'.join(split)
be_bryan	0:b74591d5ab33	1423
be_bryan	0:b74591d5ab33	1424	for col in test_properties:
be_bryan	0:b74591d5ab33	1425	col_value = test[col]
be_bryan	0:b74591d5ab33	1426	if type(test[col]) == ListType:
be_bryan	0:b74591d5ab33	1427	col_value = join_delim.join(test[col])
be_bryan	0:b74591d5ab33	1428	elif test[col] == None:
be_bryan	0:b74591d5ab33	1429	col_value = "-"
be_bryan	0:b74591d5ab33	1430
be_bryan	0:b74591d5ab33	1431	row.append(col_value)
be_bryan	0:b74591d5ab33	1432	if test['automated'] == True:
be_bryan	0:b74591d5ab33	1433	counter_dict_test_id_types[test_id_prefix] += 1
be_bryan	0:b74591d5ab33	1434	counter_automated += 1
be_bryan	0:b74591d5ab33	1435	pt.add_row(row)
be_bryan	0:b74591d5ab33	1436	# Update counters
be_bryan	0:b74591d5ab33	1437	counter_all += 1
be_bryan	0:b74591d5ab33	1438	counter_dict_test_id_types_all[test_id_prefix] += 1
be_bryan	0:b74591d5ab33	1439	result = pt.get_string()
be_bryan	0:b74591d5ab33	1440	result += "\n\n"
be_bryan	0:b74591d5ab33	1441
be_bryan	0:b74591d5ab33	1442	if result_summary and not platform_filter:
be_bryan	0:b74591d5ab33	1443	# Automation result summary
be_bryan	0:b74591d5ab33	1444	test_id_cols = ['automated', 'all', 'percent [%]', 'progress']
be_bryan	0:b74591d5ab33	1445	pt = PrettyTable(test_id_cols)
be_bryan	0:b74591d5ab33	1446	pt.align['automated'] = "r"
be_bryan	0:b74591d5ab33	1447	pt.align['all'] = "r"
be_bryan	0:b74591d5ab33	1448	pt.align['percent [%]'] = "r"
be_bryan	0:b74591d5ab33	1449
be_bryan	0:b74591d5ab33	1450	percent_progress = round(100.0 * counter_automated / float(counter_all), 1)
be_bryan	0:b74591d5ab33	1451	str_progress = progress_bar(percent_progress, 75)
be_bryan	0:b74591d5ab33	1452	pt.add_row([counter_automated, counter_all, percent_progress, str_progress])
be_bryan	0:b74591d5ab33	1453	result += "Automation coverage:\n"
be_bryan	0:b74591d5ab33	1454	result += pt.get_string()
be_bryan	0:b74591d5ab33	1455	result += "\n\n"
be_bryan	0:b74591d5ab33	1456
be_bryan	0:b74591d5ab33	1457	# Test automation coverage table print
be_bryan	0:b74591d5ab33	1458	test_id_cols = ['id', 'automated', 'all', 'percent [%]', 'progress']
be_bryan	0:b74591d5ab33	1459	pt = PrettyTable(test_id_cols)
be_bryan	0:b74591d5ab33	1460	pt.align['id'] = "l"
be_bryan	0:b74591d5ab33	1461	pt.align['automated'] = "r"
be_bryan	0:b74591d5ab33	1462	pt.align['all'] = "r"
be_bryan	0:b74591d5ab33	1463	pt.align['percent [%]'] = "r"
be_bryan	0:b74591d5ab33	1464	for unique_id in unique_test_id:
be_bryan	0:b74591d5ab33	1465	# print "\t\t%s: %d / %d" % (unique_id, counter_dict_test_id_types[unique_id], counter_dict_test_id_types_all[unique_id])
be_bryan	0:b74591d5ab33	1466	percent_progress = round(100.0 * counter_dict_test_id_types[unique_id] / float(counter_dict_test_id_types_all[unique_id]), 1)
be_bryan	0:b74591d5ab33	1467	str_progress = progress_bar(percent_progress, 75)
be_bryan	0:b74591d5ab33	1468	row = [unique_id,
be_bryan	0:b74591d5ab33	1469	counter_dict_test_id_types[unique_id],
be_bryan	0:b74591d5ab33	1470	counter_dict_test_id_types_all[unique_id],
be_bryan	0:b74591d5ab33	1471	percent_progress,
be_bryan	0:b74591d5ab33	1472	"[" + str_progress + "]"]
be_bryan	0:b74591d5ab33	1473	pt.add_row(row)
be_bryan	0:b74591d5ab33	1474	result += "Test automation coverage:\n"
be_bryan	0:b74591d5ab33	1475	result += pt.get_string()
be_bryan	0:b74591d5ab33	1476	result += "\n\n"
be_bryan	0:b74591d5ab33	1477	return result
be_bryan	0:b74591d5ab33	1478
be_bryan	0:b74591d5ab33	1479
be_bryan	0:b74591d5ab33	1480	def progress_bar(percent_progress, saturation=0):
be_bryan	0:b74591d5ab33	1481	""" This function creates progress bar with optional simple saturation mark
be_bryan	0:b74591d5ab33	1482	"""
be_bryan	0:b74591d5ab33	1483	step = int(percent_progress / 2) # Scale by to (scale: 1 - 50)
be_bryan	0:b74591d5ab33	1484	str_progress = '#' * step + '.' * int(50 - step)
be_bryan	0:b74591d5ab33	1485	c = '!' if str_progress[38] == '.' else '\|'
be_bryan	0:b74591d5ab33	1486	if saturation > 0:
be_bryan	0:b74591d5ab33	1487	saturation = saturation / 2
be_bryan	0:b74591d5ab33	1488	str_progress = str_progress[:saturation] + c + str_progress[saturation:]
be_bryan	0:b74591d5ab33	1489	return str_progress
be_bryan	0:b74591d5ab33	1490
be_bryan	0:b74591d5ab33	1491
be_bryan	0:b74591d5ab33	1492	def singletest_in_cli_mode(single_test):
be_bryan	0:b74591d5ab33	1493	""" Runs SingleTestRunner object in CLI (Command line interface) mode
be_bryan	0:b74591d5ab33	1494
be_bryan	0:b74591d5ab33	1495	@return returns success code (0 == success) for building and running tests
be_bryan	0:b74591d5ab33	1496	"""
be_bryan	0:b74591d5ab33	1497	start = time()
be_bryan	0:b74591d5ab33	1498	# Execute tests depending on options and filter applied
be_bryan	0:b74591d5ab33	1499	test_summary, shuffle_seed, test_summary_ext, test_suite_properties_ext, build_report, build_properties = single_test.execute()
be_bryan	0:b74591d5ab33	1500	elapsed_time = time() - start
be_bryan	0:b74591d5ab33	1501
be_bryan	0:b74591d5ab33	1502	# Human readable summary
be_bryan	0:b74591d5ab33	1503	if not single_test.opts_suppress_summary:
be_bryan	0:b74591d5ab33	1504	# prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	1505	print single_test.generate_test_summary(test_summary, shuffle_seed)
be_bryan	0:b74591d5ab33	1506	if single_test.opts_test_x_toolchain_summary:
be_bryan	0:b74591d5ab33	1507	# prints well-formed summary with results (SQL table like)
be_bryan	0:b74591d5ab33	1508	# table shows text x toolchain test result matrix
be_bryan	0:b74591d5ab33	1509	print single_test.generate_test_summary_by_target(test_summary, shuffle_seed)
be_bryan	0:b74591d5ab33	1510
be_bryan	0:b74591d5ab33	1511	print "Completed in %.2f sec"% (elapsed_time)
be_bryan	0:b74591d5ab33	1512	print
be_bryan	0:b74591d5ab33	1513	# Write summary of the builds
be_bryan	0:b74591d5ab33	1514
be_bryan	0:b74591d5ab33	1515	print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
be_bryan	0:b74591d5ab33	1516	status = print_report_exporter.report(build_report)
be_bryan	0:b74591d5ab33	1517
be_bryan	0:b74591d5ab33	1518	# Store extra reports in files
be_bryan	0:b74591d5ab33	1519	if single_test.opts_report_html_file_name:
be_bryan	0:b74591d5ab33	1520	# Export results in form of HTML report to separate file
be_bryan	0:b74591d5ab33	1521	report_exporter = ReportExporter(ResultExporterType.HTML)
be_bryan	0:b74591d5ab33	1522	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_html_file_name, test_suite_properties=test_suite_properties_ext)
be_bryan	0:b74591d5ab33	1523	if single_test.opts_report_junit_file_name:
be_bryan	0:b74591d5ab33	1524	# Export results in form of JUnit XML report to separate file
be_bryan	0:b74591d5ab33	1525	report_exporter = ReportExporter(ResultExporterType.JUNIT)
be_bryan	0:b74591d5ab33	1526	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
be_bryan	0:b74591d5ab33	1527	if single_test.opts_report_text_file_name:
be_bryan	0:b74591d5ab33	1528	# Export results in form of a text file
be_bryan	0:b74591d5ab33	1529	report_exporter = ReportExporter(ResultExporterType.TEXT)
be_bryan	0:b74591d5ab33	1530	report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
be_bryan	0:b74591d5ab33	1531	if single_test.opts_report_build_file_name:
be_bryan	0:b74591d5ab33	1532	# Export build results as html report to sparate file
be_bryan	0:b74591d5ab33	1533	report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
be_bryan	0:b74591d5ab33	1534	report_exporter.report_to_file(build_report, single_test.opts_report_build_file_name, test_suite_properties=build_properties)
be_bryan	0:b74591d5ab33	1535
be_bryan	0:b74591d5ab33	1536	# Returns True if no build failures of the test projects or their dependencies
be_bryan	0:b74591d5ab33	1537	return status
be_bryan	0:b74591d5ab33	1538
be_bryan	0:b74591d5ab33	1539	class TestLogger():
be_bryan	0:b74591d5ab33	1540	""" Super-class for logging and printing ongoing events for test suite pass
be_bryan	0:b74591d5ab33	1541	"""
be_bryan	0:b74591d5ab33	1542	def __init__(self, store_log=True):
be_bryan	0:b74591d5ab33	1543	""" We can control if logger actually stores log in memory
be_bryan	0:b74591d5ab33	1544	or just handled all log entries immediately
be_bryan	0:b74591d5ab33	1545	"""
be_bryan	0:b74591d5ab33	1546	self.log = []
be_bryan	0:b74591d5ab33	1547	self.log_to_file = False
be_bryan	0:b74591d5ab33	1548	self.log_file_name = None
be_bryan	0:b74591d5ab33	1549	self.store_log = store_log
be_bryan	0:b74591d5ab33	1550
be_bryan	0:b74591d5ab33	1551	self.LogType = construct_enum(INFO='Info',
be_bryan	0:b74591d5ab33	1552	WARN='Warning',
be_bryan	0:b74591d5ab33	1553	NOTIF='Notification',
be_bryan	0:b74591d5ab33	1554	ERROR='Error',
be_bryan	0:b74591d5ab33	1555	EXCEPT='Exception')
be_bryan	0:b74591d5ab33	1556
be_bryan	0:b74591d5ab33	1557	self.LogToFileAttr = construct_enum(CREATE=1, # Create or overwrite existing log file
be_bryan	0:b74591d5ab33	1558	APPEND=2) # Append to existing log file
be_bryan	0:b74591d5ab33	1559
be_bryan	0:b74591d5ab33	1560	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
be_bryan	0:b74591d5ab33	1561	""" Log one line of text
be_bryan	0:b74591d5ab33	1562	"""
be_bryan	0:b74591d5ab33	1563	log_timestamp = time()
be_bryan	0:b74591d5ab33	1564	log_entry = {'log_type' : LogType,
be_bryan	0:b74591d5ab33	1565	'log_timestamp' : log_timestamp,
be_bryan	0:b74591d5ab33	1566	'log_line' : log_line,
be_bryan	0:b74591d5ab33	1567	'_future' : None
be_bryan	0:b74591d5ab33	1568	}
be_bryan	0:b74591d5ab33	1569	# Store log in memory
be_bryan	0:b74591d5ab33	1570	if self.store_log:
be_bryan	0:b74591d5ab33	1571	self.log.append(log_entry)
be_bryan	0:b74591d5ab33	1572	return log_entry
be_bryan	0:b74591d5ab33	1573
be_bryan	0:b74591d5ab33	1574
be_bryan	0:b74591d5ab33	1575	class CLITestLogger(TestLogger):
be_bryan	0:b74591d5ab33	1576	""" Logger used with CLI (Command line interface) test suite. Logs on screen and to file if needed
be_bryan	0:b74591d5ab33	1577	"""
be_bryan	0:b74591d5ab33	1578	def __init__(self, store_log=True, file_name=None):
be_bryan	0:b74591d5ab33	1579	TestLogger.__init__(self)
be_bryan	0:b74591d5ab33	1580	self.log_file_name = file_name
be_bryan	0:b74591d5ab33	1581	#self.TIMESTAMP_FORMAT = '%y-%m-%d %H:%M:%S' # Full date and time
be_bryan	0:b74591d5ab33	1582	self.TIMESTAMP_FORMAT = '%H:%M:%S' # Time only
be_bryan	0:b74591d5ab33	1583
be_bryan	0:b74591d5ab33	1584	def log_print(self, log_entry, timestamp=True):
be_bryan	0:b74591d5ab33	1585	""" Prints on screen formatted log entry
be_bryan	0:b74591d5ab33	1586	"""
be_bryan	0:b74591d5ab33	1587	ts = log_entry['log_timestamp']
be_bryan	0:b74591d5ab33	1588	timestamp_str = datetime.datetime.fromtimestamp(ts).strftime("[%s] "% self.TIMESTAMP_FORMAT) if timestamp else ''
be_bryan	0:b74591d5ab33	1589	log_line_str = "%(log_type)s: %(log_line)s"% (log_entry)
be_bryan	0:b74591d5ab33	1590	return timestamp_str + log_line_str
be_bryan	0:b74591d5ab33	1591
be_bryan	0:b74591d5ab33	1592	def log_line(self, LogType, log_line, timestamp=True, line_delim='\n'):
be_bryan	0:b74591d5ab33	1593	""" Logs line, if log file output was specified log line will be appended
be_bryan	0:b74591d5ab33	1594	at the end of log file
be_bryan	0:b74591d5ab33	1595	"""
be_bryan	0:b74591d5ab33	1596	log_entry = TestLogger.log_line(self, LogType, log_line)
be_bryan	0:b74591d5ab33	1597	log_line_str = self.log_print(log_entry, timestamp)
be_bryan	0:b74591d5ab33	1598	if self.log_file_name is not None:
be_bryan	0:b74591d5ab33	1599	try:
be_bryan	0:b74591d5ab33	1600	with open(self.log_file_name, 'a') as f:
be_bryan	0:b74591d5ab33	1601	f.write(log_line_str + line_delim)
be_bryan	0:b74591d5ab33	1602	except IOError:
be_bryan	0:b74591d5ab33	1603	pass
be_bryan	0:b74591d5ab33	1604	return log_line_str
be_bryan	0:b74591d5ab33	1605
be_bryan	0:b74591d5ab33	1606
be_bryan	0:b74591d5ab33	1607	def factory_db_logger(db_url):
be_bryan	0:b74591d5ab33	1608	""" Factory database driver depending on database type supplied in database connection string db_url
be_bryan	0:b74591d5ab33	1609	"""
be_bryan	0:b74591d5ab33	1610	if db_url is not None:
be_bryan	0:b74591d5ab33	1611	from tools.test_mysql import MySQLDBAccess
be_bryan	0:b74591d5ab33	1612	connection_info = BaseDBAccess().parse_db_connection_string(db_url)
be_bryan	0:b74591d5ab33	1613	if connection_info is not None:
be_bryan	0:b74591d5ab33	1614	(db_type, username, password, host, db_name) = BaseDBAccess().parse_db_connection_string(db_url)
be_bryan	0:b74591d5ab33	1615	if db_type == 'mysql':
be_bryan	0:b74591d5ab33	1616	return MySQLDBAccess()
be_bryan	0:b74591d5ab33	1617	return None
be_bryan	0:b74591d5ab33	1618
be_bryan	0:b74591d5ab33	1619
be_bryan	0:b74591d5ab33	1620	def detect_database_verbose(db_url):
be_bryan	0:b74591d5ab33	1621	""" uses verbose mode (prints) database detection sequence to check it database connection string is valid
be_bryan	0:b74591d5ab33	1622	"""
be_bryan	0:b74591d5ab33	1623	result = BaseDBAccess().parse_db_connection_string(db_url)
be_bryan	0:b74591d5ab33	1624	if result is not None:
be_bryan	0:b74591d5ab33	1625	# Parsing passed
be_bryan	0:b74591d5ab33	1626	(db_type, username, password, host, db_name) = result
be_bryan	0:b74591d5ab33	1627	#print "DB type '%s', user name '%s', password '%s', host '%s', db name '%s'"% result
be_bryan	0:b74591d5ab33	1628	# Let's try to connect
be_bryan	0:b74591d5ab33	1629	db_ = factory_db_logger(db_url)
be_bryan	0:b74591d5ab33	1630	if db_ is not None:
be_bryan	0:b74591d5ab33	1631	print "Connecting to database '%s'..."% db_url,
be_bryan	0:b74591d5ab33	1632	db_.connect(host, username, password, db_name)
be_bryan	0:b74591d5ab33	1633	if db_.is_connected():
be_bryan	0:b74591d5ab33	1634	print "ok"
be_bryan	0:b74591d5ab33	1635	print "Detecting database..."
be_bryan	0:b74591d5ab33	1636	print db_.detect_database(verbose=True)
be_bryan	0:b74591d5ab33	1637	print "Disconnecting...",
be_bryan	0:b74591d5ab33	1638	db_.disconnect()
be_bryan	0:b74591d5ab33	1639	print "done"
be_bryan	0:b74591d5ab33	1640	else:
be_bryan	0:b74591d5ab33	1641	print "Database type '%s' unknown"% db_type
be_bryan	0:b74591d5ab33	1642	else:
be_bryan	0:b74591d5ab33	1643	print "Parse error: '%s' - DB Url error"% (db_url)
be_bryan	0:b74591d5ab33	1644
be_bryan	0:b74591d5ab33	1645
be_bryan	0:b74591d5ab33	1646	def get_module_avail(module_name):
be_bryan	0:b74591d5ab33	1647	""" This function returns True if module_name is already imported module
be_bryan	0:b74591d5ab33	1648	"""
be_bryan	0:b74591d5ab33	1649	return module_name in sys.modules.keys()
be_bryan	0:b74591d5ab33	1650
be_bryan	0:b74591d5ab33	1651	def get_autodetected_MUTS_list(platform_name_filter=None):
be_bryan	0:b74591d5ab33	1652	oldError = None
be_bryan	0:b74591d5ab33	1653	if os.name == 'nt':
be_bryan	0:b74591d5ab33	1654	# Disable Windows error box temporarily
be_bryan	0:b74591d5ab33	1655	oldError = ctypes.windll.kernel32.SetErrorMode(1) #note that SEM_FAILCRITICALERRORS = 1
be_bryan	0:b74591d5ab33	1656
be_bryan	0:b74591d5ab33	1657	mbeds = mbed_lstools.create()
be_bryan	0:b74591d5ab33	1658	detect_muts_list = mbeds.list_mbeds()
be_bryan	0:b74591d5ab33	1659
be_bryan	0:b74591d5ab33	1660	if os.name == 'nt':
be_bryan	0:b74591d5ab33	1661	ctypes.windll.kernel32.SetErrorMode(oldError)
be_bryan	0:b74591d5ab33	1662
be_bryan	0:b74591d5ab33	1663	return get_autodetected_MUTS(detect_muts_list, platform_name_filter=platform_name_filter)
be_bryan	0:b74591d5ab33	1664
be_bryan	0:b74591d5ab33	1665	def get_autodetected_MUTS(mbeds_list, platform_name_filter=None):
be_bryan	0:b74591d5ab33	1666	""" Function detects all connected to host mbed-enabled devices and generates artificial MUTS file.
be_bryan	0:b74591d5ab33	1667	If function fails to auto-detect devices it will return empty dictionary.
be_bryan	0:b74591d5ab33	1668
be_bryan	0:b74591d5ab33	1669	if get_module_avail('mbed_lstools'):
be_bryan	0:b74591d5ab33	1670	mbeds = mbed_lstools.create()
be_bryan	0:b74591d5ab33	1671	mbeds_list = mbeds.list_mbeds()
be_bryan	0:b74591d5ab33	1672
be_bryan	0:b74591d5ab33	1673	@param mbeds_list list of mbeds captured from mbed_lstools
be_bryan	0:b74591d5ab33	1674	@param platform_name You can filter 'platform_name' with list of filtered targets from 'platform_name_filter'
be_bryan	0:b74591d5ab33	1675	"""
be_bryan	0:b74591d5ab33	1676	result = {} # Should be in muts_all.json format
be_bryan	0:b74591d5ab33	1677	# Align mbeds_list from mbed_lstools to MUT file format (JSON dictionary with muts)
be_bryan	0:b74591d5ab33	1678	# mbeds_list = [{'platform_name': 'NUCLEO_F302R8', 'mount_point': 'E:', 'target_id': '07050200623B61125D5EF72A', 'serial_port': u'COM34'}]
be_bryan	0:b74591d5ab33	1679	index = 1
be_bryan	0:b74591d5ab33	1680	for mut in mbeds_list:
be_bryan	0:b74591d5ab33	1681	# Filter the MUTS if a filter is specified
be_bryan	0:b74591d5ab33	1682
be_bryan	0:b74591d5ab33	1683	if platform_name_filter and not mut['platform_name'] in platform_name_filter:
be_bryan	0:b74591d5ab33	1684	continue
be_bryan	0:b74591d5ab33	1685
be_bryan	0:b74591d5ab33	1686	# For mcu_unique - we are assigning 'platform_name_unique' value from mbedls output (if its existing)
be_bryan	0:b74591d5ab33	1687	# if not we are creating our own unique value (last few chars from platform's target_id).
be_bryan	0:b74591d5ab33	1688	m = {'mcu': mut['platform_name'],
be_bryan	0:b74591d5ab33	1689	'mcu_unique' : mut['platform_name_unique'] if 'platform_name_unique' in mut else "%s[%s]" % (mut['platform_name'], mut['target_id'][-4:]),
be_bryan	0:b74591d5ab33	1690	'port': mut['serial_port'],
be_bryan	0:b74591d5ab33	1691	'disk': mut['mount_point'],
be_bryan	0:b74591d5ab33	1692	'peripherals': [] # No peripheral detection
be_bryan	0:b74591d5ab33	1693	}
be_bryan	0:b74591d5ab33	1694	if index not in result:
be_bryan	0:b74591d5ab33	1695	result[index] = {}
be_bryan	0:b74591d5ab33	1696	result[index] = m
be_bryan	0:b74591d5ab33	1697	index += 1
be_bryan	0:b74591d5ab33	1698	return result
be_bryan	0:b74591d5ab33	1699
be_bryan	0:b74591d5ab33	1700
be_bryan	0:b74591d5ab33	1701	def get_autodetected_TEST_SPEC(mbeds_list,
be_bryan	0:b74591d5ab33	1702	use_default_toolchain=True,
be_bryan	0:b74591d5ab33	1703	use_supported_toolchains=False,
be_bryan	0:b74591d5ab33	1704	toolchain_filter=None,
be_bryan	0:b74591d5ab33	1705	platform_name_filter=None):
be_bryan	0:b74591d5ab33	1706	""" Function detects all connected to host mbed-enabled devices and generates artificial test_spec file.
be_bryan	0:b74591d5ab33	1707	If function fails to auto-detect devices it will return empty 'targets' test_spec description.
be_bryan	0:b74591d5ab33	1708
be_bryan	0:b74591d5ab33	1709	use_default_toolchain - if True add default toolchain to test_spec
be_bryan	0:b74591d5ab33	1710	use_supported_toolchains - if True add all supported toolchains to test_spec
be_bryan	0:b74591d5ab33	1711	toolchain_filter - if [...list of toolchains...] add from all toolchains only those in filter to test_spec
be_bryan	0:b74591d5ab33	1712	"""
be_bryan	0:b74591d5ab33	1713	result = {'targets': {} }
be_bryan	0:b74591d5ab33	1714
be_bryan	0:b74591d5ab33	1715	for mut in mbeds_list:
be_bryan	0:b74591d5ab33	1716	mcu = mut['mcu']
be_bryan	0:b74591d5ab33	1717	if platform_name_filter is None or (platform_name_filter and mut['mcu'] in platform_name_filter):
be_bryan	0:b74591d5ab33	1718	if mcu in TARGET_MAP:
be_bryan	0:b74591d5ab33	1719	default_toolchain = TARGET_MAP[mcu].default_toolchain
be_bryan	0:b74591d5ab33	1720	supported_toolchains = TARGET_MAP[mcu].supported_toolchains
be_bryan	0:b74591d5ab33	1721
be_bryan	0:b74591d5ab33	1722	# Decide which toolchains should be added to test specification toolchain pool for each target
be_bryan	0:b74591d5ab33	1723	toolchains = []
be_bryan	0:b74591d5ab33	1724	if use_default_toolchain:
be_bryan	0:b74591d5ab33	1725	toolchains.append(default_toolchain)
be_bryan	0:b74591d5ab33	1726	if use_supported_toolchains:
be_bryan	0:b74591d5ab33	1727	toolchains += supported_toolchains
be_bryan	0:b74591d5ab33	1728	if toolchain_filter is not None:
be_bryan	0:b74591d5ab33	1729	all_toolchains = supported_toolchains + [default_toolchain]
be_bryan	0:b74591d5ab33	1730	for toolchain in toolchain_filter:
be_bryan	0:b74591d5ab33	1731	if toolchain in all_toolchains:
be_bryan	0:b74591d5ab33	1732	toolchains.append(toolchain)
be_bryan	0:b74591d5ab33	1733
be_bryan	0:b74591d5ab33	1734	result['targets'][mcu] = list(set(toolchains))
be_bryan	0:b74591d5ab33	1735	return result
be_bryan	0:b74591d5ab33	1736
be_bryan	0:b74591d5ab33	1737
be_bryan	0:b74591d5ab33	1738	def get_default_test_options_parser():
be_bryan	0:b74591d5ab33	1739	""" Get common test script options used by CLI, web services etc.
be_bryan	0:b74591d5ab33	1740	"""
be_bryan	0:b74591d5ab33	1741	parser = argparse.ArgumentParser()
be_bryan	0:b74591d5ab33	1742	parser.add_argument('-i', '--tests',
be_bryan	0:b74591d5ab33	1743	dest='test_spec_filename',
be_bryan	0:b74591d5ab33	1744	metavar="FILE",
be_bryan	0:b74591d5ab33	1745	type=argparse_filestring_type,
be_bryan	0:b74591d5ab33	1746	help='Points to file with test specification')
be_bryan	0:b74591d5ab33	1747
be_bryan	0:b74591d5ab33	1748	parser.add_argument('-M', '--MUTS',
be_bryan	0:b74591d5ab33	1749	dest='muts_spec_filename',
be_bryan	0:b74591d5ab33	1750	metavar="FILE",
be_bryan	0:b74591d5ab33	1751	type=argparse_filestring_type,
be_bryan	0:b74591d5ab33	1752	help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
be_bryan	0:b74591d5ab33	1753
be_bryan	0:b74591d5ab33	1754	parser.add_argument("-j", "--jobs",
be_bryan	0:b74591d5ab33	1755	dest='jobs',
be_bryan	0:b74591d5ab33	1756	metavar="NUMBER",
be_bryan	0:b74591d5ab33	1757	type=int,
be_bryan	0:b74591d5ab33	1758	help="Define number of compilation jobs. Default value is 1")
be_bryan	0:b74591d5ab33	1759
be_bryan	0:b74591d5ab33	1760	if get_module_avail('mbed_lstools'):
be_bryan	0:b74591d5ab33	1761	# Additional features available when mbed_lstools is installed on host and imported
be_bryan	0:b74591d5ab33	1762	# mbed_lstools allow users to detect connected to host mbed-enabled devices
be_bryan	0:b74591d5ab33	1763	parser.add_argument('--auto',
be_bryan	0:b74591d5ab33	1764	dest='auto_detect',
be_bryan	0:b74591d5ab33	1765	action="store_true",
be_bryan	0:b74591d5ab33	1766	help='Use mbed-ls module to detect all connected mbed devices')
be_bryan	0:b74591d5ab33	1767
be_bryan	0:b74591d5ab33	1768	toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
be_bryan	0:b74591d5ab33	1769	parser.add_argument('--tc',
be_bryan	0:b74591d5ab33	1770	dest='toolchains_filter',
be_bryan	0:b74591d5ab33	1771	type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
be_bryan	0:b74591d5ab33	1772	help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
be_bryan	0:b74591d5ab33	1773
be_bryan	0:b74591d5ab33	1774	test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
be_bryan	0:b74591d5ab33	1775	parser.add_argument('--oper',
be_bryan	0:b74591d5ab33	1776	dest='operability_checks',
be_bryan	0:b74591d5ab33	1777	type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
be_bryan	0:b74591d5ab33	1778	help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
be_bryan	0:b74591d5ab33	1779
be_bryan	0:b74591d5ab33	1780	parser.add_argument('--clean',
be_bryan	0:b74591d5ab33	1781	dest='clean',
be_bryan	0:b74591d5ab33	1782	action="store_true",
be_bryan	0:b74591d5ab33	1783	help='Clean the build directory')
be_bryan	0:b74591d5ab33	1784
be_bryan	0:b74591d5ab33	1785	parser.add_argument('-P', '--only-peripherals',
be_bryan	0:b74591d5ab33	1786	dest='test_only_peripheral',
be_bryan	0:b74591d5ab33	1787	default=False,
be_bryan	0:b74591d5ab33	1788	action="store_true",
be_bryan	0:b74591d5ab33	1789	help='Test only peripheral declared for MUT and skip common tests')
be_bryan	0:b74591d5ab33	1790
be_bryan	0:b74591d5ab33	1791	parser.add_argument("--profile", dest="profile", action="append",
be_bryan	0:b74591d5ab33	1792	type=argparse_filestring_type,
be_bryan	0:b74591d5ab33	1793	default=[])
be_bryan	0:b74591d5ab33	1794
be_bryan	0:b74591d5ab33	1795	parser.add_argument('-C', '--only-commons',
be_bryan	0:b74591d5ab33	1796	dest='test_only_common',
be_bryan	0:b74591d5ab33	1797	default=False,
be_bryan	0:b74591d5ab33	1798	action="store_true",
be_bryan	0:b74591d5ab33	1799	help='Test only board internals. Skip perpherials tests and perform common tests')
be_bryan	0:b74591d5ab33	1800
be_bryan	0:b74591d5ab33	1801	parser.add_argument('-n', '--test-by-names',
be_bryan	0:b74591d5ab33	1802	dest='test_by_names',
be_bryan	0:b74591d5ab33	1803	type=argparse_many(str),
be_bryan	0:b74591d5ab33	1804	help='Runs only test enumerated it this switch. Use comma to separate test case names')
be_bryan	0:b74591d5ab33	1805
be_bryan	0:b74591d5ab33	1806	parser.add_argument('-p', '--peripheral-by-names',
be_bryan	0:b74591d5ab33	1807	dest='peripheral_by_names',
be_bryan	0:b74591d5ab33	1808	type=argparse_many(str),
be_bryan	0:b74591d5ab33	1809	help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
be_bryan	0:b74591d5ab33	1810
be_bryan	0:b74591d5ab33	1811	copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
be_bryan	0:b74591d5ab33	1812	copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
be_bryan	0:b74591d5ab33	1813
be_bryan	0:b74591d5ab33	1814	parser.add_argument('-c', '--copy-method',
be_bryan	0:b74591d5ab33	1815	dest='copy_method',
be_bryan	0:b74591d5ab33	1816	type=argparse_uppercase_type(copy_methods, "flash method"),
be_bryan	0:b74591d5ab33	1817	help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
be_bryan	0:b74591d5ab33	1818
be_bryan	0:b74591d5ab33	1819	reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
be_bryan	0:b74591d5ab33	1820	reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
be_bryan	0:b74591d5ab33	1821
be_bryan	0:b74591d5ab33	1822	parser.add_argument('-r', '--reset-type',
be_bryan	0:b74591d5ab33	1823	dest='mut_reset_type',
be_bryan	0:b74591d5ab33	1824	default=None,
be_bryan	0:b74591d5ab33	1825	type=argparse_uppercase_type(reset_methods, "reset method"),
be_bryan	0:b74591d5ab33	1826	help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
be_bryan	0:b74591d5ab33	1827
be_bryan	0:b74591d5ab33	1828	parser.add_argument('-g', '--goanna-for-tests',
be_bryan	0:b74591d5ab33	1829	dest='goanna_for_tests',
be_bryan	0:b74591d5ab33	1830	action="store_true",
be_bryan	0:b74591d5ab33	1831	help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
be_bryan	0:b74591d5ab33	1832
be_bryan	0:b74591d5ab33	1833	parser.add_argument('-G', '--goanna-for-sdk',
be_bryan	0:b74591d5ab33	1834	dest='goanna_for_mbed_sdk',
be_bryan	0:b74591d5ab33	1835	action="store_true",
be_bryan	0:b74591d5ab33	1836	help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
be_bryan	0:b74591d5ab33	1837
be_bryan	0:b74591d5ab33	1838	parser.add_argument('-s', '--suppress-summary',
be_bryan	0:b74591d5ab33	1839	dest='suppress_summary',
be_bryan	0:b74591d5ab33	1840	default=False,
be_bryan	0:b74591d5ab33	1841	action="store_true",
be_bryan	0:b74591d5ab33	1842	help='Suppresses display of wellformatted table with test results')
be_bryan	0:b74591d5ab33	1843
be_bryan	0:b74591d5ab33	1844	parser.add_argument('-t', '--test-summary',
be_bryan	0:b74591d5ab33	1845	dest='test_x_toolchain_summary',
be_bryan	0:b74591d5ab33	1846	default=False,
be_bryan	0:b74591d5ab33	1847	action="store_true",
be_bryan	0:b74591d5ab33	1848	help='Displays wellformatted table with test x toolchain test result per target')
be_bryan	0:b74591d5ab33	1849
be_bryan	0:b74591d5ab33	1850	parser.add_argument('-A', '--test-automation-report',
be_bryan	0:b74591d5ab33	1851	dest='test_automation_report',
be_bryan	0:b74591d5ab33	1852	default=False,
be_bryan	0:b74591d5ab33	1853	action="store_true",
be_bryan	0:b74591d5ab33	1854	help='Prints information about all tests and exits')
be_bryan	0:b74591d5ab33	1855
be_bryan	0:b74591d5ab33	1856	parser.add_argument('-R', '--test-case-report',
be_bryan	0:b74591d5ab33	1857	dest='test_case_report',
be_bryan	0:b74591d5ab33	1858	default=False,
be_bryan	0:b74591d5ab33	1859	action="store_true",
be_bryan	0:b74591d5ab33	1860	help='Prints information about all test cases and exits')
be_bryan	0:b74591d5ab33	1861
be_bryan	0:b74591d5ab33	1862	parser.add_argument("-S", "--supported-toolchains",
be_bryan	0:b74591d5ab33	1863	action="store_true",
be_bryan	0:b74591d5ab33	1864	dest="supported_toolchains",
be_bryan	0:b74591d5ab33	1865	default=False,
be_bryan	0:b74591d5ab33	1866	help="Displays supported matrix of MCUs and toolchains")
be_bryan	0:b74591d5ab33	1867
be_bryan	0:b74591d5ab33	1868	parser.add_argument("-O", "--only-build",
be_bryan	0:b74591d5ab33	1869	action="store_true",
be_bryan	0:b74591d5ab33	1870	dest="only_build_tests",
be_bryan	0:b74591d5ab33	1871	default=False,
be_bryan	0:b74591d5ab33	1872	help="Only build tests, skips actual test procedures (flashing etc.)")
be_bryan	0:b74591d5ab33	1873
be_bryan	0:b74591d5ab33	1874	parser.add_argument('--parallel',
be_bryan	0:b74591d5ab33	1875	dest='parallel_test_exec',
be_bryan	0:b74591d5ab33	1876	default=False,
be_bryan	0:b74591d5ab33	1877	action="store_true",
be_bryan	0:b74591d5ab33	1878	help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
be_bryan	0:b74591d5ab33	1879
be_bryan	0:b74591d5ab33	1880	parser.add_argument('--config',
be_bryan	0:b74591d5ab33	1881	dest='verbose_test_configuration_only',
be_bryan	0:b74591d5ab33	1882	default=False,
be_bryan	0:b74591d5ab33	1883	action="store_true",
be_bryan	0:b74591d5ab33	1884	help='Displays full test specification and MUTs configration and exits')
be_bryan	0:b74591d5ab33	1885
be_bryan	0:b74591d5ab33	1886	parser.add_argument('--loops',
be_bryan	0:b74591d5ab33	1887	dest='test_loops_list',
be_bryan	0:b74591d5ab33	1888	type=argparse_many(str),
be_bryan	0:b74591d5ab33	1889	help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
be_bryan	0:b74591d5ab33	1890
be_bryan	0:b74591d5ab33	1891	parser.add_argument('--global-loops',
be_bryan	0:b74591d5ab33	1892	dest='test_global_loops_value',
be_bryan	0:b74591d5ab33	1893	type=int,
be_bryan	0:b74591d5ab33	1894	help='Set global number of test loops per test. Default value is set 1')
be_bryan	0:b74591d5ab33	1895
be_bryan	0:b74591d5ab33	1896	parser.add_argument('--consolidate-waterfall',
be_bryan	0:b74591d5ab33	1897	dest='consolidate_waterfall_test',
be_bryan	0:b74591d5ab33	1898	default=False,
be_bryan	0:b74591d5ab33	1899	action="store_true",
be_bryan	0:b74591d5ab33	1900	help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
be_bryan	0:b74591d5ab33	1901
be_bryan	0:b74591d5ab33	1902	parser.add_argument('-W', '--waterfall',
be_bryan	0:b74591d5ab33	1903	dest='waterfall_test',
be_bryan	0:b74591d5ab33	1904	default=False,
be_bryan	0:b74591d5ab33	1905	action="store_true",
be_bryan	0:b74591d5ab33	1906	help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
be_bryan	0:b74591d5ab33	1907
be_bryan	0:b74591d5ab33	1908	parser.add_argument('-N', '--firmware-name',
be_bryan	0:b74591d5ab33	1909	dest='firmware_global_name',
be_bryan	0:b74591d5ab33	1910	help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
be_bryan	0:b74591d5ab33	1911
be_bryan	0:b74591d5ab33	1912	parser.add_argument('-u', '--shuffle',
be_bryan	0:b74591d5ab33	1913	dest='shuffle_test_order',
be_bryan	0:b74591d5ab33	1914	default=False,
be_bryan	0:b74591d5ab33	1915	action="store_true",
be_bryan	0:b74591d5ab33	1916	help='Shuffles test execution order')
be_bryan	0:b74591d5ab33	1917
be_bryan	0:b74591d5ab33	1918	parser.add_argument('--shuffle-seed',
be_bryan	0:b74591d5ab33	1919	dest='shuffle_test_seed',
be_bryan	0:b74591d5ab33	1920	default=None,
be_bryan	0:b74591d5ab33	1921	help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
be_bryan	0:b74591d5ab33	1922
be_bryan	0:b74591d5ab33	1923	parser.add_argument('-f', '--filter',
be_bryan	0:b74591d5ab33	1924	dest='general_filter_regex',
be_bryan	0:b74591d5ab33	1925	type=argparse_many(str),
be_bryan	0:b74591d5ab33	1926	default=None,
be_bryan	0:b74591d5ab33	1927	help='For some commands you can use filter to filter out results')
be_bryan	0:b74591d5ab33	1928
be_bryan	0:b74591d5ab33	1929	parser.add_argument('--inc-timeout',
be_bryan	0:b74591d5ab33	1930	dest='extend_test_timeout',
be_bryan	0:b74591d5ab33	1931	metavar="NUMBER",
be_bryan	0:b74591d5ab33	1932	type=int,
be_bryan	0:b74591d5ab33	1933	help='You can increase global timeout for each test by specifying additional test timeout in seconds')
be_bryan	0:b74591d5ab33	1934
be_bryan	0:b74591d5ab33	1935	parser.add_argument('--db',
be_bryan	0:b74591d5ab33	1936	dest='db_url',
be_bryan	0:b74591d5ab33	1937	help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
be_bryan	0:b74591d5ab33	1938
be_bryan	0:b74591d5ab33	1939	parser.add_argument('-l', '--log',
be_bryan	0:b74591d5ab33	1940	dest='log_file_name',
be_bryan	0:b74591d5ab33	1941	help='Log events to external file (note not all console entries may be visible in log file)')
be_bryan	0:b74591d5ab33	1942
be_bryan	0:b74591d5ab33	1943	parser.add_argument('--report-html',
be_bryan	0:b74591d5ab33	1944	dest='report_html_file_name',
be_bryan	0:b74591d5ab33	1945	help='You can log test suite results in form of HTML report')
be_bryan	0:b74591d5ab33	1946
be_bryan	0:b74591d5ab33	1947	parser.add_argument('--report-junit',
be_bryan	0:b74591d5ab33	1948	dest='report_junit_file_name',
be_bryan	0:b74591d5ab33	1949	help='You can log test suite results in form of JUnit compliant XML report')
be_bryan	0:b74591d5ab33	1950
be_bryan	0:b74591d5ab33	1951	parser.add_argument("--report-build",
be_bryan	0:b74591d5ab33	1952	dest="report_build_file_name",
be_bryan	0:b74591d5ab33	1953	help="Output the build results to a junit xml file")
be_bryan	0:b74591d5ab33	1954
be_bryan	0:b74591d5ab33	1955	parser.add_argument("--report-text",
be_bryan	0:b74591d5ab33	1956	dest="report_text_file_name",
be_bryan	0:b74591d5ab33	1957	help="Output the build results to a text file")
be_bryan	0:b74591d5ab33	1958
be_bryan	0:b74591d5ab33	1959	parser.add_argument('--verbose-skipped',
be_bryan	0:b74591d5ab33	1960	dest='verbose_skipped_tests',
be_bryan	0:b74591d5ab33	1961	default=False,
be_bryan	0:b74591d5ab33	1962	action="store_true",
be_bryan	0:b74591d5ab33	1963	help='Prints some extra information about skipped tests')
be_bryan	0:b74591d5ab33	1964
be_bryan	0:b74591d5ab33	1965	parser.add_argument('-V', '--verbose-test-result',
be_bryan	0:b74591d5ab33	1966	dest='verbose_test_result_only',
be_bryan	0:b74591d5ab33	1967	default=False,
be_bryan	0:b74591d5ab33	1968	action="store_true",
be_bryan	0:b74591d5ab33	1969	help='Prints test serial output')
be_bryan	0:b74591d5ab33	1970
be_bryan	0:b74591d5ab33	1971	parser.add_argument('-v', '--verbose',
be_bryan	0:b74591d5ab33	1972	dest='verbose',
be_bryan	0:b74591d5ab33	1973	default=False,
be_bryan	0:b74591d5ab33	1974	action="store_true",
be_bryan	0:b74591d5ab33	1975	help='Verbose mode (prints some extra information)')
be_bryan	0:b74591d5ab33	1976
be_bryan	0:b74591d5ab33	1977	parser.add_argument('--version',
be_bryan	0:b74591d5ab33	1978	dest='version',
be_bryan	0:b74591d5ab33	1979	default=False,
be_bryan	0:b74591d5ab33	1980	action="store_true",
be_bryan	0:b74591d5ab33	1981	help='Prints script version and exits')
be_bryan	0:b74591d5ab33	1982
be_bryan	0:b74591d5ab33	1983	parser.add_argument('--stats-depth',
be_bryan	0:b74591d5ab33	1984	dest='stats_depth',
be_bryan	0:b74591d5ab33	1985	default=2,
be_bryan	0:b74591d5ab33	1986	type=int,
be_bryan	0:b74591d5ab33	1987	help="Depth level for static memory report")
be_bryan	0:b74591d5ab33	1988	return parser
be_bryan	0:b74591d5ab33	1989
be_bryan	0:b74591d5ab33	1990	def test_path_to_name(path, base):
be_bryan	0:b74591d5ab33	1991	"""Change all slashes in a path into hyphens
be_bryan	0:b74591d5ab33	1992	This creates a unique cross-platform test name based on the path
be_bryan	0:b74591d5ab33	1993	This can eventually be overriden by a to-be-determined meta-data mechanism"""
be_bryan	0:b74591d5ab33	1994	name_parts = []
be_bryan	0:b74591d5ab33	1995	head, tail = os.path.split(relpath(path,base))
be_bryan	0:b74591d5ab33	1996	while (tail and tail != "."):
be_bryan	0:b74591d5ab33	1997	name_parts.insert(0, tail)
be_bryan	0:b74591d5ab33	1998	head, tail = os.path.split(head)
be_bryan	0:b74591d5ab33	1999
be_bryan	0:b74591d5ab33	2000	return "-".join(name_parts).lower()
be_bryan	0:b74591d5ab33	2001
be_bryan	0:b74591d5ab33	2002	def get_test_config(config_name, target_name):
be_bryan	0:b74591d5ab33	2003	"""Finds the path to a test configuration file
be_bryan	0:b74591d5ab33	2004	config_name: path to a custom configuration file OR mbed OS interface "ethernet, wifi_odin, etc"
be_bryan	0:b74591d5ab33	2005	target_name: name of target to determing if mbed OS interface given is valid
be_bryan	0:b74591d5ab33	2006	returns path to config, will return None if no valid config is found
be_bryan	0:b74591d5ab33	2007	"""
be_bryan	0:b74591d5ab33	2008	# If they passed in a full path
be_bryan	0:b74591d5ab33	2009	if exists(config_name):
be_bryan	0:b74591d5ab33	2010	# This is a module config
be_bryan	0:b74591d5ab33	2011	return config_name
be_bryan	0:b74591d5ab33	2012	# Otherwise find the path to configuration file based on mbed OS interface
be_bryan	0:b74591d5ab33	2013	return TestConfig.get_config_path(config_name, target_name)
be_bryan	0:b74591d5ab33	2014
be_bryan	0:b74591d5ab33	2015	def find_tests(base_dir, target_name, toolchain_name, app_config=None):
be_bryan	0:b74591d5ab33	2016	""" Finds all tests in a directory recursively
be_bryan	0:b74591d5ab33	2017	base_dir: path to the directory to scan for tests (ex. 'path/to/project')
be_bryan	0:b74591d5ab33	2018	target_name: name of the target to use for scanning (ex. 'K64F')
be_bryan	0:b74591d5ab33	2019	toolchain_name: name of the toolchain to use for scanning (ex. 'GCC_ARM')
be_bryan	0:b74591d5ab33	2020	options: Compile options to pass to the toolchain (ex. ['debug-info'])
be_bryan	0:b74591d5ab33	2021	app_config - location of a chosen mbed_app.json file
be_bryan	0:b74591d5ab33	2022	"""
be_bryan	0:b74591d5ab33	2023
be_bryan	0:b74591d5ab33	2024	tests = {}
be_bryan	0:b74591d5ab33	2025
be_bryan	0:b74591d5ab33	2026	# Prepare the toolchain
be_bryan	0:b74591d5ab33	2027	toolchain = prepare_toolchain([base_dir], None, target_name, toolchain_name,
be_bryan	0:b74591d5ab33	2028	silent=True, app_config=app_config)
be_bryan	0:b74591d5ab33	2029
be_bryan	0:b74591d5ab33	2030	# Scan the directory for paths to probe for 'TESTS' folders
be_bryan	0:b74591d5ab33	2031	base_resources = scan_resources([base_dir], toolchain)
be_bryan	0:b74591d5ab33	2032
be_bryan	0:b74591d5ab33	2033	dirs = base_resources.inc_dirs
be_bryan	0:b74591d5ab33	2034	for directory in dirs:
be_bryan	0:b74591d5ab33	2035	subdirs = os.listdir(directory)
be_bryan	0:b74591d5ab33	2036
be_bryan	0:b74591d5ab33	2037	# If the directory contains a subdirectory called 'TESTS', scan it for test cases
be_bryan	0:b74591d5ab33	2038	if 'TESTS' in subdirs:
be_bryan	0:b74591d5ab33	2039	walk_base_dir = join(directory, 'TESTS')
be_bryan	0:b74591d5ab33	2040	test_resources = toolchain.scan_resources(walk_base_dir, base_path=base_dir)
be_bryan	0:b74591d5ab33	2041
be_bryan	0:b74591d5ab33	2042	# Loop through all subdirectories
be_bryan	0:b74591d5ab33	2043	for d in test_resources.inc_dirs:
be_bryan	0:b74591d5ab33	2044
be_bryan	0:b74591d5ab33	2045	# If the test case folder is not called 'host_tests' and it is
be_bryan	0:b74591d5ab33	2046	# located two folders down from the main 'TESTS' folder (ex. TESTS/testgroup/testcase)
be_bryan	0:b74591d5ab33	2047	# then add it to the tests
be_bryan	0:b74591d5ab33	2048	path_depth = get_path_depth(relpath(d, walk_base_dir))
be_bryan	0:b74591d5ab33	2049	if path_depth == 2:
be_bryan	0:b74591d5ab33	2050	test_group_directory_path, test_case_directory = os.path.split(d)
be_bryan	0:b74591d5ab33	2051	test_group_directory = os.path.basename(test_group_directory_path)
be_bryan	0:b74591d5ab33	2052
be_bryan	0:b74591d5ab33	2053	# Check to make sure discoverd folder is not in a host test directory
be_bryan	0:b74591d5ab33	2054	if test_case_directory != 'host_tests' and test_group_directory != 'host_tests':
be_bryan	0:b74591d5ab33	2055	test_name = test_path_to_name(d, base_dir)
be_bryan	0:b74591d5ab33	2056	tests[test_name] = d
be_bryan	0:b74591d5ab33	2057
be_bryan	0:b74591d5ab33	2058	return tests
be_bryan	0:b74591d5ab33	2059
be_bryan	0:b74591d5ab33	2060	def print_tests(tests, format="list", sort=True):
be_bryan	0:b74591d5ab33	2061	"""Given a dictionary of tests (as returned from "find_tests"), print them
be_bryan	0:b74591d5ab33	2062	in the specified format"""
be_bryan	0:b74591d5ab33	2063	if format == "list":
be_bryan	0:b74591d5ab33	2064	for test_name in sorted(tests.keys()):
be_bryan	0:b74591d5ab33	2065	test_path = tests[test_name]
be_bryan	0:b74591d5ab33	2066	print "Test Case:"
be_bryan	0:b74591d5ab33	2067	print " Name: %s" % test_name
be_bryan	0:b74591d5ab33	2068	print " Path: %s" % test_path
be_bryan	0:b74591d5ab33	2069	elif format == "json":
be_bryan	0:b74591d5ab33	2070	print json.dumps(tests, indent=2)
be_bryan	0:b74591d5ab33	2071	else:
be_bryan	0:b74591d5ab33	2072	print "Unknown format '%s'" % format
be_bryan	0:b74591d5ab33	2073	sys.exit(1)
be_bryan	0:b74591d5ab33	2074
be_bryan	0:b74591d5ab33	2075	def norm_relative_path(path, start):
be_bryan	0:b74591d5ab33	2076	"""This function will create a normalized, relative path. It mimics the
be_bryan	0:b74591d5ab33	2077	python os.path.relpath function, but also normalizes a Windows-syle path
be_bryan	0:b74591d5ab33	2078	that use backslashes to a Unix style path that uses forward slashes."""
be_bryan	0:b74591d5ab33	2079	path = os.path.normpath(path)
be_bryan	0:b74591d5ab33	2080	path = os.path.relpath(path, start)
be_bryan	0:b74591d5ab33	2081	path = path.replace("\\", "/")
be_bryan	0:b74591d5ab33	2082	return path
be_bryan	0:b74591d5ab33	2083
be_bryan	0:b74591d5ab33	2084
be_bryan	0:b74591d5ab33	2085	def build_test_worker(args, *kwargs):
be_bryan	0:b74591d5ab33	2086	"""This is a worker function for the parallel building of tests. The `args`
be_bryan	0:b74591d5ab33	2087	and `kwargs` are passed directly to `build_project`. It returns a dictionary
be_bryan	0:b74591d5ab33	2088	with the following structure:
be_bryan	0:b74591d5ab33	2089
be_bryan	0:b74591d5ab33	2090	{
be_bryan	0:b74591d5ab33	2091	'result': `True` if no exceptions were thrown, `False` otherwise
be_bryan	0:b74591d5ab33	2092	'reason': Instance of exception that was thrown on failure
be_bryan	0:b74591d5ab33	2093	'bin_file': Path to the created binary if `build_project` was
be_bryan	0:b74591d5ab33	2094	successful. Not present otherwise
be_bryan	0:b74591d5ab33	2095	'kwargs': The keyword arguments that were passed to `build_project`.
be_bryan	0:b74591d5ab33	2096	This includes arguments that were modified (ex. report)
be_bryan	0:b74591d5ab33	2097	}
be_bryan	0:b74591d5ab33	2098	"""
be_bryan	0:b74591d5ab33	2099	bin_file = None
be_bryan	0:b74591d5ab33	2100	ret = {
be_bryan	0:b74591d5ab33	2101	'result': False,
be_bryan	0:b74591d5ab33	2102	'args': args,
be_bryan	0:b74591d5ab33	2103	'kwargs': kwargs
be_bryan	0:b74591d5ab33	2104	}
be_bryan	0:b74591d5ab33	2105
be_bryan	0:b74591d5ab33	2106	# Use parent TOOLCHAIN_PATHS variable
be_bryan	0:b74591d5ab33	2107	for key, value in kwargs['toolchain_paths'].iteritems():
be_bryan	0:b74591d5ab33	2108	TOOLCHAIN_PATHS[key] = value
be_bryan	0:b74591d5ab33	2109
be_bryan	0:b74591d5ab33	2110	del kwargs['toolchain_paths']
be_bryan	0:b74591d5ab33	2111
be_bryan	0:b74591d5ab33	2112	try:
be_bryan	0:b74591d5ab33	2113	bin_file = build_project(args, *kwargs)
be_bryan	0:b74591d5ab33	2114	ret['result'] = True
be_bryan	0:b74591d5ab33	2115	ret['bin_file'] = bin_file
be_bryan	0:b74591d5ab33	2116	ret['kwargs'] = kwargs
be_bryan	0:b74591d5ab33	2117
be_bryan	0:b74591d5ab33	2118	except NotSupportedException, e:
be_bryan	0:b74591d5ab33	2119	ret['reason'] = e
be_bryan	0:b74591d5ab33	2120	except ToolException, e:
be_bryan	0:b74591d5ab33	2121	ret['reason'] = e
be_bryan	0:b74591d5ab33	2122	except KeyboardInterrupt, e:
be_bryan	0:b74591d5ab33	2123	ret['reason'] = e
be_bryan	0:b74591d5ab33	2124	except:
be_bryan	0:b74591d5ab33	2125	# Print unhandled exceptions here
be_bryan	0:b74591d5ab33	2126	import traceback
be_bryan	0:b74591d5ab33	2127	traceback.print_exc(file=sys.stdout)
be_bryan	0:b74591d5ab33	2128
be_bryan	0:b74591d5ab33	2129	return ret
be_bryan	0:b74591d5ab33	2130
be_bryan	0:b74591d5ab33	2131
be_bryan	0:b74591d5ab33	2132	def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
be_bryan	0:b74591d5ab33	2133	clean=False, notify=None, verbose=False, jobs=1, macros=None,
be_bryan	0:b74591d5ab33	2134	silent=False, report=None, properties=None,
be_bryan	0:b74591d5ab33	2135	continue_on_build_fail=False, app_config=None,
be_bryan	0:b74591d5ab33	2136	build_profile=None, stats_depth=None):
be_bryan	0:b74591d5ab33	2137	"""Given the data structure from 'find_tests' and the typical build parameters,
be_bryan	0:b74591d5ab33	2138	build all the tests
be_bryan	0:b74591d5ab33	2139
be_bryan	0:b74591d5ab33	2140	Returns a tuple of the build result (True or False) followed by the test
be_bryan	0:b74591d5ab33	2141	build data structure"""
be_bryan	0:b74591d5ab33	2142
be_bryan	0:b74591d5ab33	2143	execution_directory = "."
be_bryan	0:b74591d5ab33	2144	base_path = norm_relative_path(build_path, execution_directory)
be_bryan	0:b74591d5ab33	2145
be_bryan	0:b74591d5ab33	2146	target_name = target if isinstance(target, str) else target.name
be_bryan	0:b74591d5ab33	2147	cfg, _, _ = get_config(base_source_paths, target_name, toolchain_name)
be_bryan	0:b74591d5ab33	2148
be_bryan	0:b74591d5ab33	2149	baud_rate = 9600
be_bryan	0:b74591d5ab33	2150	if 'platform.stdio-baud-rate' in cfg:
be_bryan	0:b74591d5ab33	2151	baud_rate = cfg['platform.stdio-baud-rate'].value
be_bryan	0:b74591d5ab33	2152
be_bryan	0:b74591d5ab33	2153	test_build = {
be_bryan	0:b74591d5ab33	2154	"platform": target_name,
be_bryan	0:b74591d5ab33	2155	"toolchain": toolchain_name,
be_bryan	0:b74591d5ab33	2156	"base_path": base_path,
be_bryan	0:b74591d5ab33	2157	"baud_rate": baud_rate,
be_bryan	0:b74591d5ab33	2158	"binary_type": "bootable",
be_bryan	0:b74591d5ab33	2159	"tests": {}
be_bryan	0:b74591d5ab33	2160	}
be_bryan	0:b74591d5ab33	2161
be_bryan	0:b74591d5ab33	2162	result = True
be_bryan	0:b74591d5ab33	2163
be_bryan	0:b74591d5ab33	2164	jobs_count = int(jobs if jobs else cpu_count())
be_bryan	0:b74591d5ab33	2165	p = Pool(processes=jobs_count)
be_bryan	0:b74591d5ab33	2166	results = []
be_bryan	0:b74591d5ab33	2167	for test_name, test_path in tests.iteritems():
be_bryan	0:b74591d5ab33	2168	test_build_path = os.path.join(build_path, test_path)
be_bryan	0:b74591d5ab33	2169	src_path = base_source_paths + [test_path]
be_bryan	0:b74591d5ab33	2170	bin_file = None
be_bryan	0:b74591d5ab33	2171	test_case_folder_name = os.path.basename(test_path)
be_bryan	0:b74591d5ab33	2172
be_bryan	0:b74591d5ab33	2173	args = (src_path, test_build_path, target, toolchain_name)
be_bryan	0:b74591d5ab33	2174	kwargs = {
be_bryan	0:b74591d5ab33	2175	'jobs': 1,
be_bryan	0:b74591d5ab33	2176	'clean': clean,
be_bryan	0:b74591d5ab33	2177	'macros': macros,
be_bryan	0:b74591d5ab33	2178	'name': test_case_folder_name,
be_bryan	0:b74591d5ab33	2179	'project_id': test_name,
be_bryan	0:b74591d5ab33	2180	'report': report,
be_bryan	0:b74591d5ab33	2181	'properties': properties,
be_bryan	0:b74591d5ab33	2182	'verbose': verbose,
be_bryan	0:b74591d5ab33	2183	'app_config': app_config,
be_bryan	0:b74591d5ab33	2184	'build_profile': build_profile,
be_bryan	0:b74591d5ab33	2185	'silent': True,
be_bryan	0:b74591d5ab33	2186	'toolchain_paths': TOOLCHAIN_PATHS,
be_bryan	0:b74591d5ab33	2187	'stats_depth': stats_depth
be_bryan	0:b74591d5ab33	2188	}
be_bryan	0:b74591d5ab33	2189
be_bryan	0:b74591d5ab33	2190	results.append(p.apply_async(build_test_worker, args, kwargs))
be_bryan	0:b74591d5ab33	2191
be_bryan	0:b74591d5ab33	2192	p.close()
be_bryan	0:b74591d5ab33	2193	result = True
be_bryan	0:b74591d5ab33	2194	itr = 0
be_bryan	0:b74591d5ab33	2195	while len(results):
be_bryan	0:b74591d5ab33	2196	itr += 1
be_bryan	0:b74591d5ab33	2197	if itr > 360000:
be_bryan	0:b74591d5ab33	2198	p.terminate()
be_bryan	0:b74591d5ab33	2199	p.join()
be_bryan	0:b74591d5ab33	2200	raise ToolException("Compile did not finish in 10 minutes")
be_bryan	0:b74591d5ab33	2201	else:
be_bryan	0:b74591d5ab33	2202	sleep(0.01)
be_bryan	0:b74591d5ab33	2203	pending = 0
be_bryan	0:b74591d5ab33	2204	for r in results:
be_bryan	0:b74591d5ab33	2205	if r.ready() is True:
be_bryan	0:b74591d5ab33	2206	try:
be_bryan	0:b74591d5ab33	2207	worker_result = r.get()
be_bryan	0:b74591d5ab33	2208	results.remove(r)
be_bryan	0:b74591d5ab33	2209
be_bryan	0:b74591d5ab33	2210	# Take report from the kwargs and merge it into existing report
be_bryan	0:b74591d5ab33	2211	if report:
be_bryan	0:b74591d5ab33	2212	report_entry = worker_result['kwargs']['report'][target_name][toolchain_name]
be_bryan	0:b74591d5ab33	2213	for test_key in report_entry.keys():
be_bryan	0:b74591d5ab33	2214	report[target_name][toolchain_name][test_key] = report_entry[test_key]
be_bryan	0:b74591d5ab33	2215
be_bryan	0:b74591d5ab33	2216	# Set the overall result to a failure if a build failure occurred
be_bryan	0:b74591d5ab33	2217	if ('reason' in worker_result and
be_bryan	0:b74591d5ab33	2218	not worker_result['reason'] and
be_bryan	0:b74591d5ab33	2219	not isinstance(worker_result['reason'], NotSupportedException)):
be_bryan	0:b74591d5ab33	2220	result = False
be_bryan	0:b74591d5ab33	2221	break
be_bryan	0:b74591d5ab33	2222
be_bryan	0:b74591d5ab33	2223	# Adding binary path to test build result
be_bryan	0:b74591d5ab33	2224	if ('result' in worker_result and
be_bryan	0:b74591d5ab33	2225	worker_result['result'] and
be_bryan	0:b74591d5ab33	2226	'bin_file' in worker_result):
be_bryan	0:b74591d5ab33	2227	bin_file = norm_relative_path(worker_result['bin_file'], execution_directory)
be_bryan	0:b74591d5ab33	2228
be_bryan	0:b74591d5ab33	2229	test_build['tests'][worker_result['kwargs']['project_id']] = {
be_bryan	0:b74591d5ab33	2230	"binaries": [
be_bryan	0:b74591d5ab33	2231	{
be_bryan	0:b74591d5ab33	2232	"path": bin_file
be_bryan	0:b74591d5ab33	2233	}
be_bryan	0:b74591d5ab33	2234	]
be_bryan	0:b74591d5ab33	2235	}
be_bryan	0:b74591d5ab33	2236
be_bryan	0:b74591d5ab33	2237	test_key = worker_result['kwargs']['project_id'].upper()
be_bryan	0:b74591d5ab33	2238	if report:
be_bryan	0:b74591d5ab33	2239	print report[target_name][toolchain_name][test_key][0][0]['output'].rstrip()
be_bryan	0:b74591d5ab33	2240	print 'Image: %s\n' % bin_file
be_bryan	0:b74591d5ab33	2241
be_bryan	0:b74591d5ab33	2242	except:
be_bryan	0:b74591d5ab33	2243	if p._taskqueue.queue:
be_bryan	0:b74591d5ab33	2244	p._taskqueue.queue.clear()
be_bryan	0:b74591d5ab33	2245	sleep(0.5)
be_bryan	0:b74591d5ab33	2246	p.terminate()
be_bryan	0:b74591d5ab33	2247	p.join()
be_bryan	0:b74591d5ab33	2248	raise
be_bryan	0:b74591d5ab33	2249	else:
be_bryan	0:b74591d5ab33	2250	pending += 1
be_bryan	0:b74591d5ab33	2251	if pending >= jobs_count:
be_bryan	0:b74591d5ab33	2252	break
be_bryan	0:b74591d5ab33	2253
be_bryan	0:b74591d5ab33	2254	# Break as soon as possible if there is a failure and we are not
be_bryan	0:b74591d5ab33	2255	# continuing on build failures
be_bryan	0:b74591d5ab33	2256	if not result and not continue_on_build_fail:
be_bryan	0:b74591d5ab33	2257	if p._taskqueue.queue:
be_bryan	0:b74591d5ab33	2258	p._taskqueue.queue.clear()
be_bryan	0:b74591d5ab33	2259	sleep(0.5)
be_bryan	0:b74591d5ab33	2260	p.terminate()
be_bryan	0:b74591d5ab33	2261	break
be_bryan	0:b74591d5ab33	2262
be_bryan	0:b74591d5ab33	2263	p.join()
be_bryan	0:b74591d5ab33	2264
be_bryan	0:b74591d5ab33	2265	test_builds = {}
be_bryan	0:b74591d5ab33	2266	test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
be_bryan	0:b74591d5ab33	2267
be_bryan	0:b74591d5ab33	2268	return result, test_builds
be_bryan	0:b74591d5ab33	2269
be_bryan	0:b74591d5ab33	2270
be_bryan	0:b74591d5ab33	2271	def test_spec_from_test_builds(test_builds):
be_bryan	0:b74591d5ab33	2272	return {
be_bryan	0:b74591d5ab33	2273	"builds": test_builds
be_bryan	0:b74591d5ab33	2274	}

Repository toolbox

Export to desktop IDE

Repository details

Type:	Library
Created:	12 Dec 2017
Imports:	1018
Forks:	0
Commits:	1
Dependents:	14
Dependencies:	0
Followers:	23

tools/test_api.py@0:b74591d5ab33, 2017-12-11 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Access Warning