Anders Blomdell
Clone of official tools
Diff: test_api.py
- Revision:
- 22:9e85236d8716
- Parent:
- 20:835f6355470d
- Child:
- 24:25bff2709c20
--- a/test_api.py Fri Jul 15 22:58:15 2016 +0100
+++ b/test_api.py Sat Jul 16 00:34:03 2016 +0100
@@ -24,7 +24,7 @@
import uuid
import pprint
import random
-import optparse
+import argparse
import datetime
import threading
import ctypes
@@ -46,6 +46,7 @@
from tools.utils import ToolException
from tools.utils import NotSupportedException
from tools.utils import construct_enum
+from tools.memap import MemapParser
from tools.targets import TARGET_MAP
from tools.test_db import BaseDBAccess
from tools.build_api import build_project, build_mbed_libs, build_lib
@@ -58,7 +59,12 @@
from tools.build_api import scan_for_source_paths
from tools.libraries import LIBRARIES, LIBRARY_MAP
from tools.toolchains import TOOLCHAIN_PATHS
+from tools.toolchains import TOOLCHAINS
from tools.test_exporters import ReportExporter, ResultExporterType
+from tools.utils import argparse_filestring_type
+from tools.utils import argparse_uppercase_type
+from tools.utils import argparse_lowercase_type
+from tools.utils import argparse_many
import tools.host_tests.host_tests_plugins as host_tests_plugins
@@ -166,6 +172,7 @@
_opts_report_html_file_name=None,
_opts_report_junit_file_name=None,
_opts_report_build_file_name=None,
+ _opts_report_text_file_name=None,
_opts_build_report={},
_opts_build_properties={},
_test_spec={},
@@ -224,6 +231,7 @@
self.opts_report_html_file_name = _opts_report_html_file_name
self.opts_report_junit_file_name = _opts_report_junit_file_name
self.opts_report_build_file_name = _opts_report_build_file_name
+ self.opts_report_text_file_name = _opts_report_text_file_name
self.opts_goanna_for_mbed_sdk = _opts_goanna_for_mbed_sdk
self.opts_goanna_for_tests = _opts_goanna_for_tests
self.opts_shuffle_test_order = _opts_shuffle_test_order
@@ -623,7 +631,7 @@
for test_id in test_map_keys:
test = TEST_MAP[test_id]
- if self.opts_test_by_names and test_id not in self.opts_test_by_names.split(','):
+ if self.opts_test_by_names and test_id not in self.opts_test_by_names:
continue
if test_ids and test_id not in test_ids:
@@ -634,7 +642,7 @@
print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
continue
- if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names.split(',')]):
+ if self.opts_peripheral_by_names and test.peripherals and not len([i for i in test.peripherals if i in self.opts_peripheral_by_names]):
# We will skip tests not forced with -p option
if self.opts_verbose_skipped_tests:
print self.logger.log_line(self.logger.LogType.INFO, 'Common test skipped for target %s'% (target))
@@ -655,7 +663,7 @@
# When users are using 'build only flag' and test do not have
# specified peripherals we can allow test building by default
pass
- elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names.split(','):
+ elif self.opts_peripheral_by_names and test_id not in self.opts_peripheral_by_names:
# If we force peripheral with option -p we expect test
# to pass even if peripheral is not in MUTs file.
pass
@@ -773,7 +781,7 @@
"""
result = {}
if test_loops_str:
- test_loops = test_loops_str.split(',')
+ test_loops = test_loops_str
for test_loop in test_loops:
test_loop_count = test_loop.split('=')
if len(test_loop_count) == 2:
@@ -1513,6 +1521,10 @@
# Export results in form of JUnit XML report to separate file
report_exporter = ReportExporter(ResultExporterType.JUNIT)
report_exporter.report_to_file(test_summary_ext, single_test.opts_report_junit_file_name, test_suite_properties=test_suite_properties_ext)
+ if single_test.opts_report_text_file_name:
+ # Export results in form of a text file
+ report_exporter = ReportExporter(ResultExporterType.TEXT)
+ report_exporter.report_to_file(test_summary_ext, single_test.opts_report_text_file_name, test_suite_properties=test_suite_properties_ext)
if single_test.opts_report_build_file_name:
# Export build results as html report to sparate file
report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
@@ -1713,7 +1725,7 @@
toolchains += supported_toolchains
if toolchain_filter is not None:
all_toolchains = supported_toolchains + [default_toolchain]
- for toolchain in toolchain_filter.split(','):
+ for toolchain in toolchain_filter:
if toolchain in all_toolchains:
toolchains.append(toolchain)
@@ -1724,231 +1736,243 @@
def get_default_test_options_parser():
""" Get common test script options used by CLI, web services etc.
"""
- parser = optparse.OptionParser()
- parser.add_option('-i', '--tests',
- dest='test_spec_filename',
- metavar="FILE",
- help='Points to file with test specification')
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-i', '--tests',
+ dest='test_spec_filename',
+ metavar="FILE",
+ type=argparse_filestring_type,
+ help='Points to file with test specification')
- parser.add_option('-M', '--MUTS',
- dest='muts_spec_filename',
- metavar="FILE",
- help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
+ parser.add_argument('-M', '--MUTS',
+ dest='muts_spec_filename',
+ metavar="FILE",
+ type=argparse_filestring_type,
+ help='Points to file with MUTs specification (overwrites settings.py and private_settings.py)')
- parser.add_option("-j", "--jobs",
- dest='jobs',
- metavar="NUMBER",
- type="int",
- help="Define number of compilation jobs. Default value is 1")
+ parser.add_argument("-j", "--jobs",
+ dest='jobs',
+ metavar="NUMBER",
+ type=int,
+ help="Define number of compilation jobs. Default value is 1")
if get_module_avail('mbed_lstools'):
# Additional features available when mbed_lstools is installed on host and imported
# mbed_lstools allow users to detect connected to host mbed-enabled devices
- parser.add_option('', '--auto',
- dest='auto_detect',
- metavar=False,
- action="store_true",
- help='Use mbed-ls module to detect all connected mbed devices')
+ parser.add_argument('--auto',
+ dest='auto_detect',
+ action="store_true",
+ help='Use mbed-ls module to detect all connected mbed devices')
- parser.add_option('', '--tc',
- dest='toolchains_filter',
- help="Toolchain filter for --auto option. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
+ toolchain_list = list(TOOLCHAINS) + ["DEFAULT", "ALL"]
+ parser.add_argument('--tc',
+ dest='toolchains_filter',
+ type=argparse_many(argparse_uppercase_type(toolchain_list, "toolchains")),
+ help="Toolchain filter for --auto argument. Use toolchains names separated by comma, 'default' or 'all' to select toolchains")
test_scopes = ','.join(["'%s'" % n for n in get_available_oper_test_scopes()])
- parser.add_option('', '--oper',
- dest='operability_checks',
- help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
+ parser.add_argument('--oper',
+ dest='operability_checks',
+ type=argparse_lowercase_type(get_available_oper_test_scopes(), "scopes"),
+ help='Perform interoperability tests between host and connected mbed devices. Available test scopes are: %s' % test_scopes)
- parser.add_option('', '--clean',
- dest='clean',
- metavar=False,
- action="store_true",
- help='Clean the build directory')
+ parser.add_argument('--clean',
+ dest='clean',
+ action="store_true",
+ help='Clean the build directory')
- parser.add_option('-P', '--only-peripherals',
- dest='test_only_peripheral',
- default=False,
- action="store_true",
- help='Test only peripheral declared for MUT and skip common tests')
+ parser.add_argument('-P', '--only-peripherals',
+ dest='test_only_peripheral',
+ default=False,
+ action="store_true",
+ help='Test only peripheral declared for MUT and skip common tests')
- parser.add_option('-C', '--only-commons',
- dest='test_only_common',
- default=False,
- action="store_true",
- help='Test only board internals. Skip perpherials tests and perform common tests')
+ parser.add_argument('-C', '--only-commons',
+ dest='test_only_common',
+ default=False,
+ action="store_true",
+ help='Test only board internals. Skip perpherials tests and perform common tests')
- parser.add_option('-n', '--test-by-names',
- dest='test_by_names',
- help='Runs only test enumerated it this switch. Use comma to separate test case names')
+ parser.add_argument('-n', '--test-by-names',
+ dest='test_by_names',
+ type=argparse_many(str),
+ help='Runs only test enumerated it this switch. Use comma to separate test case names')
- parser.add_option('-p', '--peripheral-by-names',
+ parser.add_argument('-p', '--peripheral-by-names',
dest='peripheral_by_names',
+ type=argparse_many(str),
help='Forces discovery of particular peripherals. Use comma to separate peripheral names')
copy_methods = host_tests_plugins.get_plugin_caps('CopyMethod')
copy_methods_str = "Plugin support: " + ', '.join(copy_methods)
- parser.add_option('-c', '--copy-method',
- dest='copy_method',
- help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
+ parser.add_argument('-c', '--copy-method',
+ dest='copy_method',
+ type=argparse_uppercase_type(copy_methods, "flash method"),
+ help="Select binary copy (flash) method. Default is Python's shutil.copy() method. %s"% copy_methods_str)
reset_methods = host_tests_plugins.get_plugin_caps('ResetMethod')
reset_methods_str = "Plugin support: " + ', '.join(reset_methods)
- parser.add_option('-r', '--reset-type',
- dest='mut_reset_type',
- default=None,
- help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
+ parser.add_argument('-r', '--reset-type',
+ dest='mut_reset_type',
+ default=None,
+ type=argparse_uppercase_type(reset_methods, "reset method"),
+ help='Extra reset method used to reset MUT by host test script. %s'% reset_methods_str)
- parser.add_option('-g', '--goanna-for-tests',
- dest='goanna_for_tests',
- metavar=False,
- action="store_true",
- help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
+ parser.add_argument('-g', '--goanna-for-tests',
+ dest='goanna_for_tests',
+ action="store_true",
+ help='Run Goanna static analyse tool for tests. (Project will be rebuilded)')
- parser.add_option('-G', '--goanna-for-sdk',
- dest='goanna_for_mbed_sdk',
- metavar=False,
- action="store_true",
- help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
+ parser.add_argument('-G', '--goanna-for-sdk',
+ dest='goanna_for_mbed_sdk',
+ action="store_true",
+ help='Run Goanna static analyse tool for mbed SDK (Project will be rebuilded)')
- parser.add_option('-s', '--suppress-summary',
- dest='suppress_summary',
- default=False,
- action="store_true",
- help='Suppresses display of wellformatted table with test results')
+ parser.add_argument('-s', '--suppress-summary',
+ dest='suppress_summary',
+ default=False,
+ action="store_true",
+ help='Suppresses display of wellformatted table with test results')
+
+ parser.add_argument('-t', '--test-summary',
+ dest='test_x_toolchain_summary',
+ default=False,
+ action="store_true",
+ help='Displays wellformatted table with test x toolchain test result per target')
- parser.add_option('-t', '--test-summary',
- dest='test_x_toolchain_summary',
- default=False,
- action="store_true",
- help='Displays wellformatted table with test x toolchain test result per target')
+ parser.add_argument('-A', '--test-automation-report',
+ dest='test_automation_report',
+ default=False,
+ action="store_true",
+ help='Prints information about all tests and exits')
- parser.add_option('-A', '--test-automation-report',
- dest='test_automation_report',
- default=False,
- action="store_true",
- help='Prints information about all tests and exits')
+ parser.add_argument('-R', '--test-case-report',
+ dest='test_case_report',
+ default=False,
+ action="store_true",
+ help='Prints information about all test cases and exits')
- parser.add_option('-R', '--test-case-report',
- dest='test_case_report',
- default=False,
- action="store_true",
- help='Prints information about all test cases and exits')
+ parser.add_argument("-S", "--supported-toolchains",
+ action="store_true",
+ dest="supported_toolchains",
+ default=False,
+ help="Displays supported matrix of MCUs and toolchains")
- parser.add_option("-S", "--supported-toolchains",
- action="store_true",
- dest="supported_toolchains",
- default=False,
- help="Displays supported matrix of MCUs and toolchains")
+ parser.add_argument("-O", "--only-build",
+ action="store_true",
+ dest="only_build_tests",
+ default=False,
+ help="Only build tests, skips actual test procedures (flashing etc.)")
- parser.add_option("-O", "--only-build",
- action="store_true",
- dest="only_build_tests",
- default=False,
- help="Only build tests, skips actual test procedures (flashing etc.)")
+ parser.add_argument('--parallel',
+ dest='parallel_test_exec',
+ default=False,
+ action="store_true",
+ help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
- parser.add_option('', '--parallel',
- dest='parallel_test_exec',
- default=False,
- action="store_true",
- help='Experimental, you execute test runners for connected to your host MUTs in parallel (speeds up test result collection)')
+ parser.add_argument('--config',
+ dest='verbose_test_configuration_only',
+ default=False,
+ action="store_true",
+ help='Displays full test specification and MUTs configration and exits')
+
+ parser.add_argument('--loops',
+ dest='test_loops_list',
+ type=argparse_many(str),
+ help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
- parser.add_option('', '--config',
- dest='verbose_test_configuration_only',
- default=False,
- action="store_true",
- help='Displays full test specification and MUTs configration and exits')
+ parser.add_argument('--global-loops',
+ dest='test_global_loops_value',
+ type=int,
+ help='Set global number of test loops per test. Default value is set 1')
- parser.add_option('', '--loops',
- dest='test_loops_list',
- help='Set no. of loops per test. Format: TEST_1=1,TEST_2=2,TEST_3=3')
-
- parser.add_option('', '--global-loops',
- dest='test_global_loops_value',
- help='Set global number of test loops per test. Default value is set 1')
+ parser.add_argument('--consolidate-waterfall',
+ dest='consolidate_waterfall_test',
+ default=False,
+ action="store_true",
+ help='Used with --waterfall argument. Adds only one test to report reflecting outcome of waterfall test.')
- parser.add_option('', '--consolidate-waterfall',
- dest='consolidate_waterfall_test',
- default=False,
- action="store_true",
- help='Used with --waterfall option. Adds only one test to report reflecting outcome of waterfall test.')
+ parser.add_argument('-W', '--waterfall',
+ dest='waterfall_test',
+ default=False,
+ action="store_true",
+ help='Used with --loops or --global-loops arguments. Tests until OK result occurs and assumes test passed')
- parser.add_option('-W', '--waterfall',
- dest='waterfall_test',
- default=False,
- action="store_true",
- help='Used with --loops or --global-loops options. Tests until OK result occurs and assumes test passed')
+ parser.add_argument('-N', '--firmware-name',
+ dest='firmware_global_name',
+ help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
- parser.add_option('-N', '--firmware-name',
- dest='firmware_global_name',
- help='Set global name for all produced projects. Note, proper file extension will be added by buid scripts')
+ parser.add_argument('-u', '--shuffle',
+ dest='shuffle_test_order',
+ default=False,
+ action="store_true",
+ help='Shuffles test execution order')
- parser.add_option('-u', '--shuffle',
- dest='shuffle_test_order',
- default=False,
- action="store_true",
- help='Shuffles test execution order')
-
- parser.add_option('', '--shuffle-seed',
- dest='shuffle_test_seed',
- default=None,
- help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
+ parser.add_argument('--shuffle-seed',
+ dest='shuffle_test_seed',
+ default=None,
+ help='Shuffle seed (If you want to reproduce your shuffle order please use seed provided in test summary)')
- parser.add_option('-f', '--filter',
- dest='general_filter_regex',
- default=None,
- help='For some commands you can use filter to filter out results')
+ parser.add_argument('-f', '--filter',
+ dest='general_filter_regex',
+ type=argparse_many(str),
+ default=None,
+ help='For some commands you can use filter to filter out results')
- parser.add_option('', '--inc-timeout',
- dest='extend_test_timeout',
- metavar="NUMBER",
- type="int",
- help='You can increase global timeout for each test by specifying additional test timeout in seconds')
+ parser.add_argument('--inc-timeout',
+ dest='extend_test_timeout',
+ metavar="NUMBER",
+ type=int,
+ help='You can increase global timeout for each test by specifying additional test timeout in seconds')
+
+ parser.add_argument('--db',
+ dest='db_url',
+ help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
- parser.add_option('', '--db',
- dest='db_url',
- help='This specifies what database test suite uses to store its state. To pass DB connection info use database connection string. Example: \'mysql://username:password@127.0.0.1/db_name\'')
+ parser.add_argument('-l', '--log',
+ dest='log_file_name',
+ help='Log events to external file (note not all console entries may be visible in log file)')
- parser.add_option('-l', '--log',
- dest='log_file_name',
- help='Log events to external file (note not all console entries may be visible in log file)')
+ parser.add_argument('--report-html',
+ dest='report_html_file_name',
+ help='You can log test suite results in form of HTML report')
- parser.add_option('', '--report-html',
- dest='report_html_file_name',
- help='You can log test suite results in form of HTML report')
+ parser.add_argument('--report-junit',
+ dest='report_junit_file_name',
+ help='You can log test suite results in form of JUnit compliant XML report')
- parser.add_option('', '--report-junit',
- dest='report_junit_file_name',
- help='You can log test suite results in form of JUnit compliant XML report')
+ parser.add_argument("--report-build",
+ dest="report_build_file_name",
+ help="Output the build results to a junit xml file")
- parser.add_option("", "--report-build",
- dest="report_build_file_name",
- help="Output the build results to a junit xml file")
+ parser.add_argument("--report-text",
+ dest="report_text_file_name",
+ help="Output the build results to a text file")
- parser.add_option('', '--verbose-skipped',
- dest='verbose_skipped_tests',
- default=False,
- action="store_true",
- help='Prints some extra information about skipped tests')
+ parser.add_argument('--verbose-skipped',
+ dest='verbose_skipped_tests',
+ default=False,
+ action="store_true",
+ help='Prints some extra information about skipped tests')
- parser.add_option('-V', '--verbose-test-result',
- dest='verbose_test_result_only',
- default=False,
- action="store_true",
- help='Prints test serial output')
+ parser.add_argument('-V', '--verbose-test-result',
+ dest='verbose_test_result_only',
+ default=False,
+ action="store_true",
+ help='Prints test serial output')
- parser.add_option('-v', '--verbose',
- dest='verbose',
- default=False,
- action="store_true",
- help='Verbose mode (prints some extra information)')
+ parser.add_argument('-v', '--verbose',
+ dest='verbose',
+ default=False,
+ action="store_true",
+ help='Verbose mode (prints some extra information)')
- parser.add_option('', '--version',
- dest='version',
- default=False,
- action="store_true",
- help='Prints script version and exits')
+ parser.add_argument('--version',
+ dest='version',
+ default=False,
+ action="store_true",
+ help='Prints script version and exits')
return parser
def test_path_to_name(path):
@@ -1960,12 +1984,12 @@
while (tail and tail != "."):
name_parts.insert(0, tail)
head, tail = os.path.split(head)
-
+
return "-".join(name_parts).lower()
def find_tests(base_dir):
"""Given any directory, walk through the subdirectories and find all tests"""
-
+
def find_test_in_directory(directory, tests_path):
"""Given a 'TESTS' directory, return a dictionary of test names and test paths.
The formate of the dictionary is {"test-name": "./path/to/test"}"""
@@ -1979,20 +2003,20 @@
"name": test_path_to_name(directory),
"path": directory
}
-
+
return test
tests_path = 'TESTS'
tests = {}
dirs = scan_for_source_paths(base_dir)
-
+
for directory in dirs:
test = find_test_in_directory(directory, tests_path)
if test:
tests[test['name']] = test['path']
-
+
return tests
-
+
def print_tests(tests, format="list", sort=True):
"""Given a dictionary of tests (as returned from "find_tests"), print them
in the specified format"""
@@ -2023,12 +2047,11 @@
continue_on_build_fail=False):
"""Given the data structure from 'find_tests' and the typical build parameters,
build all the tests
-
+
Returns a tuple of the build result (True or False) followed by the test
build data structure"""
-
+
execution_directory = "."
-
base_path = norm_relative_path(build_path, execution_directory)
target_name = target if isinstance(target, str) else target.name
@@ -2041,9 +2064,10 @@
"binary_type": "bootable",
"tests": {}
}
-
+
result = True
-
+
+ map_outputs_total = list()
for test_name, test_path in tests.iteritems():
test_build_path = os.path.join(build_path, test_path)
src_path = base_source_paths + [test_path]
@@ -2062,21 +2086,21 @@
except Exception, e:
if not isinstance(e, NotSupportedException):
result = False
-
+
if continue_on_build_fail:
continue
else:
break
-
+
# If a clean build was carried out last time, disable it for the next build.
# Otherwise the previously built test will be deleted.
if clean:
clean = False
-
+
# Normalize the path
if bin_file:
bin_file = norm_relative_path(bin_file, execution_directory)
-
+
test_build['tests'][test_name] = {
"binaries": [
{
@@ -2084,15 +2108,15 @@
}
]
}
-
+
print 'Image: %s'% bin_file
-
+
test_builds = {}
test_builds["%s-%s" % (target_name, toolchain_name)] = test_build
-
+
return result, test_builds
-
+
def test_spec_from_test_builds(test_builds):
return {