the other jimmy / mbed-sdk-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-sdk-tools by mbed official

singletest.py@32:8ea194f6145b, 2017-01-04 (annotated)

Committer:
The Other Jimmy
Date:
Wed Jan 04 11:58:24 2017 -0600
Revision:
32:8ea194f6145b
Parent:
22:9e85236d8716 
Update tools to follow mbed-os tools release 5.3.1

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 #!/usr/bin/env python2
screamer 0:66f3b5499f7f 2
screamer 0:66f3b5499f7f 3 """
screamer 0:66f3b5499f7f 4 mbed SDK
screamer 0:66f3b5499f7f 5 Copyright (c) 2011-2014 ARM Limited
screamer 0:66f3b5499f7f 6
screamer 0:66f3b5499f7f 7 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 8 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 9 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 12
screamer 0:66f3b5499f7f 13 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 14 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 16 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 17 limitations under the License.
screamer 0:66f3b5499f7f 18
screamer 0:66f3b5499f7f 19 Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
screamer 0:66f3b5499f7f 20 """
screamer 0:66f3b5499f7f 21
screamer 0:66f3b5499f7f 22 """
screamer 0:66f3b5499f7f 23 File format example: test_spec.json:
screamer 0:66f3b5499f7f 24 {
screamer 0:66f3b5499f7f 25 "targets": {
screamer 0:66f3b5499f7f 26 "KL46Z": ["ARM", "GCC_ARM"],
screamer 0:66f3b5499f7f 27 "LPC1768": ["ARM", "GCC_ARM", "GCC_CR", "IAR"],
screamer 0:66f3b5499f7f 28 "LPC11U24": ["uARM"],
screamer 0:66f3b5499f7f 29 "NRF51822": ["ARM"]
screamer 0:66f3b5499f7f 30 }
screamer 0:66f3b5499f7f 31 }
screamer 0:66f3b5499f7f 32
screamer 0:66f3b5499f7f 33 File format example: muts_all.json:
screamer 0:66f3b5499f7f 34 {
screamer 0:66f3b5499f7f 35 "1" : {"mcu": "LPC1768",
screamer 0:66f3b5499f7f 36 "port":"COM4",
screamer 0:66f3b5499f7f 37 "disk":"J:\\",
screamer 0:66f3b5499f7f 38 "peripherals": ["TMP102", "digital_loop", "port_loop", "analog_loop", "SD"]
screamer 0:66f3b5499f7f 39 },
screamer 0:66f3b5499f7f 40
screamer 0:66f3b5499f7f 41 "2" : {"mcu": "KL25Z",
screamer 0:66f3b5499f7f 42 "port":"COM7",
screamer 0:66f3b5499f7f 43 "disk":"G:\\",
screamer 0:66f3b5499f7f 44 "peripherals": ["digital_loop", "port_loop", "analog_loop"]
screamer 0:66f3b5499f7f 45 }
screamer 0:66f3b5499f7f 46 }
screamer 0:66f3b5499f7f 47 """
screamer 0:66f3b5499f7f 48
screamer 0:66f3b5499f7f 49
screamer 0:66f3b5499f7f 50 # Be sure that the tools directory is in the search path
screamer 0:66f3b5499f7f 51 import sys
screamer 0:66f3b5499f7f 52 from os.path import join, abspath, dirname
screamer 0:66f3b5499f7f 53
screamer 0:66f3b5499f7f 54 ROOT = abspath(join(dirname(__file__), ".."))
screamer 0:66f3b5499f7f 55 sys.path.insert(0, ROOT)
screamer 0:66f3b5499f7f 56
screamer 0:66f3b5499f7f 57
screamer 0:66f3b5499f7f 58 # Check: Extra modules which are required by core test suite
screamer 0:66f3b5499f7f 59 from tools.utils import check_required_modules
screamer 0:66f3b5499f7f 60 check_required_modules(['prettytable', 'serial'])
screamer 0:66f3b5499f7f 61
screamer 0:66f3b5499f7f 62 # Imports related to mbed build api
screamer 0:66f3b5499f7f 63 from tools.build_api import mcu_toolchain_matrix
screamer 0:66f3b5499f7f 64
screamer 0:66f3b5499f7f 65 # Imports from TEST API
screamer 0:66f3b5499f7f 66 from tools.test_api import SingleTestRunner
screamer 0:66f3b5499f7f 67 from tools.test_api import singletest_in_cli_mode
screamer 0:66f3b5499f7f 68 from tools.test_api import detect_database_verbose
screamer 0:66f3b5499f7f 69 from tools.test_api import get_json_data_from_file
screamer 0:66f3b5499f7f 70 from tools.test_api import get_avail_tests_summary_table
screamer 0:66f3b5499f7f 71 from tools.test_api import get_default_test_options_parser
screamer 0:66f3b5499f7f 72 from tools.test_api import print_muts_configuration_from_json
screamer 0:66f3b5499f7f 73 from tools.test_api import print_test_configuration_from_json
screamer 0:66f3b5499f7f 74 from tools.test_api import get_autodetected_MUTS_list
screamer 0:66f3b5499f7f 75 from tools.test_api import get_autodetected_TEST_SPEC
screamer 0:66f3b5499f7f 76 from tools.test_api import get_module_avail
screamer 0:66f3b5499f7f 77 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 0:66f3b5499f7f 78
screamer 0:66f3b5499f7f 79
screamer 0:66f3b5499f7f 80 # Importing extra modules which can be not installed but if available they can extend test suite functionality
screamer 0:66f3b5499f7f 81 try:
screamer 0:66f3b5499f7f 82 import mbed_lstools
screamer 0:66f3b5499f7f 83 from tools.compliance.ioper_runner import IOperTestRunner
screamer 0:66f3b5499f7f 84 from tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:66f3b5499f7f 85 except:
screamer 0:66f3b5499f7f 86 pass
screamer 0:66f3b5499f7f 87
screamer 0:66f3b5499f7f 88 def get_version():
screamer 0:66f3b5499f7f 89 """ Returns test script version
screamer 0:66f3b5499f7f 90 """
screamer 0:66f3b5499f7f 91 single_test_version_major = 1
screamer 0:66f3b5499f7f 92 single_test_version_minor = 5
screamer 0:66f3b5499f7f 93 return (single_test_version_major, single_test_version_minor)
screamer 0:66f3b5499f7f 94
screamer 0:66f3b5499f7f 95
screamer 0:66f3b5499f7f 96 if __name__ == '__main__':
screamer 0:66f3b5499f7f 97 # Command line options
screamer 0:66f3b5499f7f 98 parser = get_default_test_options_parser()
screamer 0:66f3b5499f7f 99
screamer 0:66f3b5499f7f 100 parser.description = """This script allows you to run mbed defined test cases for particular MCU(s) and corresponding toolchain(s)."""
screamer 0:66f3b5499f7f 101 parser.epilog = """Example: singletest.py -i test_spec.json -M muts_all.json"""
screamer 0:66f3b5499f7f 102
screamer 22:9e85236d8716 103 opts = parser.parse_args()
screamer 0:66f3b5499f7f 104
screamer 0:66f3b5499f7f 105 # Print scrip version
screamer 0:66f3b5499f7f 106 if opts.version:
screamer 0:66f3b5499f7f 107 print parser.description
screamer 0:66f3b5499f7f 108 print parser.epilog
screamer 0:66f3b5499f7f 109 print "Version %d.%d"% get_version()
screamer 0:66f3b5499f7f 110 exit(0)
screamer 0:66f3b5499f7f 111
screamer 0:66f3b5499f7f 112 if opts.db_url and opts.verbose_test_configuration_only:
screamer 0:66f3b5499f7f 113 detect_database_verbose(opts.db_url)
screamer 0:66f3b5499f7f 114 exit(0)
screamer 0:66f3b5499f7f 115
screamer 0:66f3b5499f7f 116 # Print summary / information about automation test status
screamer 0:66f3b5499f7f 117 if opts.test_automation_report:
screamer 0:66f3b5499f7f 118 print get_avail_tests_summary_table(platform_filter=opts.general_filter_regex)
screamer 0:66f3b5499f7f 119 exit(0)
screamer 0:66f3b5499f7f 120
screamer 0:66f3b5499f7f 121 # Print summary / information about automation test status
screamer 0:66f3b5499f7f 122 if opts.test_case_report:
screamer 0:66f3b5499f7f 123 test_case_report_cols = ['id',
screamer 0:66f3b5499f7f 124 'automated',
screamer 0:66f3b5499f7f 125 'description',
screamer 0:66f3b5499f7f 126 'peripherals',
screamer 0:66f3b5499f7f 127 'host_test',
screamer 0:66f3b5499f7f 128 'duration',
screamer 0:66f3b5499f7f 129 'source_dir']
screamer 0:66f3b5499f7f 130 print get_avail_tests_summary_table(cols=test_case_report_cols,
screamer 0:66f3b5499f7f 131 result_summary=False,
screamer 0:66f3b5499f7f 132 join_delim='\n',
screamer 0:66f3b5499f7f 133 platform_filter=opts.general_filter_regex)
screamer 0:66f3b5499f7f 134 exit(0)
screamer 0:66f3b5499f7f 135
screamer 0:66f3b5499f7f 136 # Only prints matrix of supported toolchains
screamer 0:66f3b5499f7f 137 if opts.supported_toolchains:
screamer 0:66f3b5499f7f 138 print mcu_toolchain_matrix(platform_filter=opts.general_filter_regex)
screamer 0:66f3b5499f7f 139 exit(0)
screamer 0:66f3b5499f7f 140
screamer 0:66f3b5499f7f 141 test_spec = None
screamer 0:66f3b5499f7f 142 MUTs = None
screamer 0:66f3b5499f7f 143
screamer 0:66f3b5499f7f 144 if hasattr(opts, 'auto_detect') and opts.auto_detect:
screamer 0:66f3b5499f7f 145 # If auto_detect attribute is present, we assume other auto-detection
screamer 0:66f3b5499f7f 146 # parameters like 'toolchains_filter' are also set.
screamer 0:66f3b5499f7f 147 print "MBEDLS: Detecting connected mbed-enabled devices... "
screamer 0:66f3b5499f7f 148
screamer 0:66f3b5499f7f 149 MUTs = get_autodetected_MUTS_list()
screamer 0:66f3b5499f7f 150
screamer 0:66f3b5499f7f 151 for mut in MUTs.values():
screamer 0:66f3b5499f7f 152 print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu'],
screamer 0:66f3b5499f7f 153 mut['port'],
screamer 0:66f3b5499f7f 154 mut['disk'])
screamer 0:66f3b5499f7f 155
screamer 0:66f3b5499f7f 156 # Set up parameters for test specification filter function (we need to set toolchains per target here)
screamer 22:9e85236d8716 157 use_default_toolchain = 'default' in opts.toolchains_filter if opts.toolchains_filter is not None else True
screamer 22:9e85236d8716 158 use_supported_toolchains = 'all' in opts.toolchains_filter if opts.toolchains_filter is not None else False
screamer 0:66f3b5499f7f 159 toolchain_filter = opts.toolchains_filter
screamer 22:9e85236d8716 160 platform_name_filter = opts.general_filter_regex if opts.general_filter_regex is not None else opts.general_filter_regex
screamer 0:66f3b5499f7f 161 # Test specification with information about each target and associated toolchain
screamer 0:66f3b5499f7f 162 test_spec = get_autodetected_TEST_SPEC(MUTs.values(),
screamer 0:66f3b5499f7f 163 use_default_toolchain=use_default_toolchain,
screamer 0:66f3b5499f7f 164 use_supported_toolchains=use_supported_toolchains,
screamer 0:66f3b5499f7f 165 toolchain_filter=toolchain_filter,
screamer 0:66f3b5499f7f 166 platform_name_filter=platform_name_filter)
screamer 0:66f3b5499f7f 167 else:
screamer 0:66f3b5499f7f 168 # Open file with test specification
screamer 0:66f3b5499f7f 169 # test_spec_filename tells script which targets and their toolchain(s)
screamer 0:66f3b5499f7f 170 # should be covered by the test scenario
screamer 0:66f3b5499f7f 171 opts.auto_detect = False
screamer 0:66f3b5499f7f 172 test_spec = get_json_data_from_file(opts.test_spec_filename) if opts.test_spec_filename else None
screamer 0:66f3b5499f7f 173 if test_spec is None:
screamer 0:66f3b5499f7f 174 if not opts.test_spec_filename:
screamer 0:66f3b5499f7f 175 parser.print_help()
screamer 0:66f3b5499f7f 176 exit(-1)
screamer 0:66f3b5499f7f 177
screamer 0:66f3b5499f7f 178 # Get extra MUTs if applicable
screamer 0:66f3b5499f7f 179 MUTs = get_json_data_from_file(opts.muts_spec_filename) if opts.muts_spec_filename else None
screamer 0:66f3b5499f7f 180
screamer 0:66f3b5499f7f 181 if MUTs is None:
screamer 0:66f3b5499f7f 182 if not opts.muts_spec_filename:
screamer 0:66f3b5499f7f 183 parser.print_help()
screamer 0:66f3b5499f7f 184 exit(-1)
screamer 0:66f3b5499f7f 185
screamer 0:66f3b5499f7f 186 if opts.verbose_test_configuration_only:
screamer 0:66f3b5499f7f 187 print "MUTs configuration in %s:" % ('auto-detected' if opts.auto_detect else opts.muts_spec_filename)
screamer 0:66f3b5499f7f 188 if MUTs:
screamer 0:66f3b5499f7f 189 print print_muts_configuration_from_json(MUTs, platform_filter=opts.general_filter_regex)
screamer 0:66f3b5499f7f 190 print
screamer 0:66f3b5499f7f 191 print "Test specification in %s:" % ('auto-detected' if opts.auto_detect else opts.test_spec_filename)
screamer 0:66f3b5499f7f 192 if test_spec:
screamer 0:66f3b5499f7f 193 print print_test_configuration_from_json(test_spec)
screamer 0:66f3b5499f7f 194 exit(0)
screamer 0:66f3b5499f7f 195
screamer 0:66f3b5499f7f 196 if get_module_avail('mbed_lstools'):
screamer 0:66f3b5499f7f 197 if opts.operability_checks:
screamer 0:66f3b5499f7f 198 # Check if test scope is valid and run tests
screamer 0:66f3b5499f7f 199 test_scope = get_available_oper_test_scopes()
screamer 0:66f3b5499f7f 200 if opts.operability_checks in test_scope:
screamer 0:66f3b5499f7f 201 tests = IOperTestRunner(scope=opts.operability_checks)
screamer 0:66f3b5499f7f 202 test_results = tests.run()
screamer 0:66f3b5499f7f 203
screamer 0:66f3b5499f7f 204 # Export results in form of JUnit XML report to separate file
screamer 0:66f3b5499f7f 205 if opts.report_junit_file_name:
screamer 0:66f3b5499f7f 206 report_exporter = ReportExporter(ResultExporterType.JUNIT_OPER)
screamer 0:66f3b5499f7f 207 report_exporter.report_to_file(test_results, opts.report_junit_file_name)
screamer 0:66f3b5499f7f 208 else:
screamer 0:66f3b5499f7f 209 print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks)
screamer 0:66f3b5499f7f 210 print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope]))
screamer 0:66f3b5499f7f 211
screamer 0:66f3b5499f7f 212 exit(0)
screamer 0:66f3b5499f7f 213
screamer 0:66f3b5499f7f 214 # Verbose test specification and MUTs configuration
screamer 0:66f3b5499f7f 215 if MUTs and opts.verbose:
screamer 0:66f3b5499f7f 216 print print_muts_configuration_from_json(MUTs)
screamer 0:66f3b5499f7f 217 if test_spec and opts.verbose:
screamer 0:66f3b5499f7f 218 print print_test_configuration_from_json(test_spec)
screamer 0:66f3b5499f7f 219
screamer 0:66f3b5499f7f 220 if opts.only_build_tests:
screamer 0:66f3b5499f7f 221 # We are skipping testing phase, and suppress summary
screamer 0:66f3b5499f7f 222 opts.suppress_summary = True
screamer 0:66f3b5499f7f 223
screamer 0:66f3b5499f7f 224 single_test = SingleTestRunner(_global_loops_count=opts.test_global_loops_value,
screamer 0:66f3b5499f7f 225 _test_loops_list=opts.test_loops_list,
screamer 0:66f3b5499f7f 226 _muts=MUTs,
screamer 0:66f3b5499f7f 227 _clean=opts.clean,
The Other Jimmy 32:8ea194f6145b 228 _parser=parser,
The Other Jimmy 32:8ea194f6145b 229 _opts=opts,
screamer 0:66f3b5499f7f 230 _opts_db_url=opts.db_url,
screamer 0:66f3b5499f7f 231 _opts_log_file_name=opts.log_file_name,
screamer 0:66f3b5499f7f 232 _opts_report_html_file_name=opts.report_html_file_name,
screamer 0:66f3b5499f7f 233 _opts_report_junit_file_name=opts.report_junit_file_name,
screamer 0:66f3b5499f7f 234 _opts_report_build_file_name=opts.report_build_file_name,
screamer 22:9e85236d8716 235 _opts_report_text_file_name=opts.report_text_file_name,
screamer 0:66f3b5499f7f 236 _test_spec=test_spec,
screamer 0:66f3b5499f7f 237 _opts_goanna_for_mbed_sdk=opts.goanna_for_mbed_sdk,
screamer 0:66f3b5499f7f 238 _opts_goanna_for_tests=opts.goanna_for_tests,
screamer 0:66f3b5499f7f 239 _opts_shuffle_test_order=opts.shuffle_test_order,
screamer 0:66f3b5499f7f 240 _opts_shuffle_test_seed=opts.shuffle_test_seed,
screamer 0:66f3b5499f7f 241 _opts_test_by_names=opts.test_by_names,
screamer 0:66f3b5499f7f 242 _opts_peripheral_by_names=opts.peripheral_by_names,
screamer 0:66f3b5499f7f 243 _opts_test_only_peripheral=opts.test_only_peripheral,
screamer 0:66f3b5499f7f 244 _opts_test_only_common=opts.test_only_common,
screamer 0:66f3b5499f7f 245 _opts_verbose_skipped_tests=opts.verbose_skipped_tests,
screamer 0:66f3b5499f7f 246 _opts_verbose_test_result_only=opts.verbose_test_result_only,
screamer 0:66f3b5499f7f 247 _opts_verbose=opts.verbose,
screamer 0:66f3b5499f7f 248 _opts_firmware_global_name=opts.firmware_global_name,
screamer 0:66f3b5499f7f 249 _opts_only_build_tests=opts.only_build_tests,
screamer 0:66f3b5499f7f 250 _opts_parallel_test_exec=opts.parallel_test_exec,
screamer 0:66f3b5499f7f 251 _opts_suppress_summary=opts.suppress_summary,
screamer 0:66f3b5499f7f 252 _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary,
screamer 0:66f3b5499f7f 253 _opts_copy_method=opts.copy_method,
screamer 0:66f3b5499f7f 254 _opts_mut_reset_type=opts.mut_reset_type,
screamer 0:66f3b5499f7f 255 _opts_jobs=opts.jobs,
screamer 0:66f3b5499f7f 256 _opts_waterfall_test=opts.waterfall_test,
screamer 0:66f3b5499f7f 257 _opts_consolidate_waterfall_test=opts.consolidate_waterfall_test,
screamer 0:66f3b5499f7f 258 _opts_extend_test_timeout=opts.extend_test_timeout,
screamer 0:66f3b5499f7f 259 _opts_auto_detect=opts.auto_detect)
screamer 0:66f3b5499f7f 260
screamer 0:66f3b5499f7f 261 # Runs test suite in CLI mode
screamer 0:66f3b5499f7f 262 if (singletest_in_cli_mode(single_test)):
screamer 0:66f3b5499f7f 263 exit(0)
screamer 0:66f3b5499f7f 264 else:
screamer 0:66f3b5499f7f 265 exit(-1)