Brian Daniels / mbed-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-tools by Morpheus

singletest.py@0:4a2e5f0422d6, 2016-03-30 (annotated)

Committer:
screamer
Date:
Wed Mar 30 16:23:44 2016 +0000
Revision:
0:4a2e5f0422d6
Child:
2:5f044fef8f98
Initial revision

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:4a2e5f0422d6 1 #!/usr/bin/env python2
screamer 0:4a2e5f0422d6 2
screamer 0:4a2e5f0422d6 3 """
screamer 0:4a2e5f0422d6 4 mbed SDK
screamer 0:4a2e5f0422d6 5 Copyright (c) 2011-2014 ARM Limited
screamer 0:4a2e5f0422d6 6
screamer 0:4a2e5f0422d6 7 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:4a2e5f0422d6 8 you may not use this file except in compliance with the License.
screamer 0:4a2e5f0422d6 9 You may obtain a copy of the License at
screamer 0:4a2e5f0422d6 10
screamer 0:4a2e5f0422d6 11 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:4a2e5f0422d6 12
screamer 0:4a2e5f0422d6 13 Unless required by applicable law or agreed to in writing, software
screamer 0:4a2e5f0422d6 14 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:4a2e5f0422d6 15 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:4a2e5f0422d6 16 See the License for the specific language governing permissions and
screamer 0:4a2e5f0422d6 17 limitations under the License.
screamer 0:4a2e5f0422d6 18
screamer 0:4a2e5f0422d6 19 Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
screamer 0:4a2e5f0422d6 20 """
screamer 0:4a2e5f0422d6 21
screamer 0:4a2e5f0422d6 22 """
screamer 0:4a2e5f0422d6 23 File format example: test_spec.json:
screamer 0:4a2e5f0422d6 24 {
screamer 0:4a2e5f0422d6 25 "targets": {
screamer 0:4a2e5f0422d6 26 "KL46Z": ["ARM", "GCC_ARM"],
screamer 0:4a2e5f0422d6 27 "LPC1768": ["ARM", "GCC_ARM", "GCC_CR", "IAR"],
screamer 0:4a2e5f0422d6 28 "LPC11U24": ["uARM"],
screamer 0:4a2e5f0422d6 29 "NRF51822": ["ARM"]
screamer 0:4a2e5f0422d6 30 }
screamer 0:4a2e5f0422d6 31 }
screamer 0:4a2e5f0422d6 32
screamer 0:4a2e5f0422d6 33 File format example: muts_all.json:
screamer 0:4a2e5f0422d6 34 {
screamer 0:4a2e5f0422d6 35 "1" : {"mcu": "LPC1768",
screamer 0:4a2e5f0422d6 36 "port":"COM4",
screamer 0:4a2e5f0422d6 37 "disk":"J:\\",
screamer 0:4a2e5f0422d6 38 "peripherals": ["TMP102", "digital_loop", "port_loop", "analog_loop", "SD"]
screamer 0:4a2e5f0422d6 39 },
screamer 0:4a2e5f0422d6 40
screamer 0:4a2e5f0422d6 41 "2" : {"mcu": "KL25Z",
screamer 0:4a2e5f0422d6 42 "port":"COM7",
screamer 0:4a2e5f0422d6 43 "disk":"G:\\",
screamer 0:4a2e5f0422d6 44 "peripherals": ["digital_loop", "port_loop", "analog_loop"]
screamer 0:4a2e5f0422d6 45 }
screamer 0:4a2e5f0422d6 46 }
screamer 0:4a2e5f0422d6 47 """
screamer 0:4a2e5f0422d6 48
screamer 0:4a2e5f0422d6 49
screamer 0:4a2e5f0422d6 50 # Be sure that the tools directory is in the search path
screamer 0:4a2e5f0422d6 51 import sys
screamer 0:4a2e5f0422d6 52 from os.path import join, abspath, dirname
screamer 0:4a2e5f0422d6 53
screamer 0:4a2e5f0422d6 54 ROOT = abspath(join(dirname(__file__), ".."))
screamer 0:4a2e5f0422d6 55 sys.path.insert(0, ROOT)
screamer 0:4a2e5f0422d6 56
screamer 0:4a2e5f0422d6 57
screamer 0:4a2e5f0422d6 58 # Check: Extra modules which are required by core test suite
screamer 0:4a2e5f0422d6 59 from workspace_tools.utils import check_required_modules
screamer 0:4a2e5f0422d6 60 check_required_modules(['prettytable', 'serial'])
screamer 0:4a2e5f0422d6 61
screamer 0:4a2e5f0422d6 62 # Imports related to mbed build api
screamer 0:4a2e5f0422d6 63 from workspace_tools.build_api import mcu_toolchain_matrix
screamer 0:4a2e5f0422d6 64
screamer 0:4a2e5f0422d6 65 # Imports from TEST API
screamer 0:4a2e5f0422d6 66 from workspace_tools.test_api import SingleTestRunner
screamer 0:4a2e5f0422d6 67 from workspace_tools.test_api import singletest_in_cli_mode
screamer 0:4a2e5f0422d6 68 from workspace_tools.test_api import detect_database_verbose
screamer 0:4a2e5f0422d6 69 from workspace_tools.test_api import get_json_data_from_file
screamer 0:4a2e5f0422d6 70 from workspace_tools.test_api import get_avail_tests_summary_table
screamer 0:4a2e5f0422d6 71 from workspace_tools.test_api import get_default_test_options_parser
screamer 0:4a2e5f0422d6 72 from workspace_tools.test_api import print_muts_configuration_from_json
screamer 0:4a2e5f0422d6 73 from workspace_tools.test_api import print_test_configuration_from_json
screamer 0:4a2e5f0422d6 74 from workspace_tools.test_api import get_autodetected_MUTS_list
screamer 0:4a2e5f0422d6 75 from workspace_tools.test_api import get_autodetected_TEST_SPEC
screamer 0:4a2e5f0422d6 76 from workspace_tools.test_api import get_module_avail
screamer 0:4a2e5f0422d6 77 from workspace_tools.test_exporters import ReportExporter, ResultExporterType
screamer 0:4a2e5f0422d6 78
screamer 0:4a2e5f0422d6 79
screamer 0:4a2e5f0422d6 80 # Importing extra modules which can be not installed but if available they can extend test suite functionality
screamer 0:4a2e5f0422d6 81 try:
screamer 0:4a2e5f0422d6 82 import mbed_lstools
screamer 0:4a2e5f0422d6 83 from workspace_tools.compliance.ioper_runner import IOperTestRunner
screamer 0:4a2e5f0422d6 84 from workspace_tools.compliance.ioper_runner import get_available_oper_test_scopes
screamer 0:4a2e5f0422d6 85 except:
screamer 0:4a2e5f0422d6 86 pass
screamer 0:4a2e5f0422d6 87
screamer 0:4a2e5f0422d6 88 def get_version():
screamer 0:4a2e5f0422d6 89 """ Returns test script version
screamer 0:4a2e5f0422d6 90 """
screamer 0:4a2e5f0422d6 91 single_test_version_major = 1
screamer 0:4a2e5f0422d6 92 single_test_version_minor = 5
screamer 0:4a2e5f0422d6 93 return (single_test_version_major, single_test_version_minor)
screamer 0:4a2e5f0422d6 94
screamer 0:4a2e5f0422d6 95
screamer 0:4a2e5f0422d6 96 if __name__ == '__main__':
screamer 0:4a2e5f0422d6 97 # Command line options
screamer 0:4a2e5f0422d6 98 parser = get_default_test_options_parser()
screamer 0:4a2e5f0422d6 99
screamer 0:4a2e5f0422d6 100 parser.description = """This script allows you to run mbed defined test cases for particular MCU(s) and corresponding toolchain(s)."""
screamer 0:4a2e5f0422d6 101 parser.epilog = """Example: singletest.py -i test_spec.json -M muts_all.json"""
screamer 0:4a2e5f0422d6 102
screamer 0:4a2e5f0422d6 103 (opts, args) = parser.parse_args()
screamer 0:4a2e5f0422d6 104
screamer 0:4a2e5f0422d6 105 # Print scrip version
screamer 0:4a2e5f0422d6 106 if opts.version:
screamer 0:4a2e5f0422d6 107 print parser.description
screamer 0:4a2e5f0422d6 108 print parser.epilog
screamer 0:4a2e5f0422d6 109 print "Version %d.%d"% get_version()
screamer 0:4a2e5f0422d6 110 exit(0)
screamer 0:4a2e5f0422d6 111
screamer 0:4a2e5f0422d6 112 if opts.db_url and opts.verbose_test_configuration_only:
screamer 0:4a2e5f0422d6 113 detect_database_verbose(opts.db_url)
screamer 0:4a2e5f0422d6 114 exit(0)
screamer 0:4a2e5f0422d6 115
screamer 0:4a2e5f0422d6 116 # Print summary / information about automation test status
screamer 0:4a2e5f0422d6 117 if opts.test_automation_report:
screamer 0:4a2e5f0422d6 118 print get_avail_tests_summary_table(platform_filter=opts.general_filter_regex)
screamer 0:4a2e5f0422d6 119 exit(0)
screamer 0:4a2e5f0422d6 120
screamer 0:4a2e5f0422d6 121 # Print summary / information about automation test status
screamer 0:4a2e5f0422d6 122 if opts.test_case_report:
screamer 0:4a2e5f0422d6 123 test_case_report_cols = ['id',
screamer 0:4a2e5f0422d6 124 'automated',
screamer 0:4a2e5f0422d6 125 'description',
screamer 0:4a2e5f0422d6 126 'peripherals',
screamer 0:4a2e5f0422d6 127 'host_test',
screamer 0:4a2e5f0422d6 128 'duration',
screamer 0:4a2e5f0422d6 129 'source_dir']
screamer 0:4a2e5f0422d6 130 print get_avail_tests_summary_table(cols=test_case_report_cols,
screamer 0:4a2e5f0422d6 131 result_summary=False,
screamer 0:4a2e5f0422d6 132 join_delim='\n',
screamer 0:4a2e5f0422d6 133 platform_filter=opts.general_filter_regex)
screamer 0:4a2e5f0422d6 134 exit(0)
screamer 0:4a2e5f0422d6 135
screamer 0:4a2e5f0422d6 136 # Only prints matrix of supported toolchains
screamer 0:4a2e5f0422d6 137 if opts.supported_toolchains:
screamer 0:4a2e5f0422d6 138 print mcu_toolchain_matrix(platform_filter=opts.general_filter_regex)
screamer 0:4a2e5f0422d6 139 exit(0)
screamer 0:4a2e5f0422d6 140
screamer 0:4a2e5f0422d6 141 test_spec = None
screamer 0:4a2e5f0422d6 142 MUTs = None
screamer 0:4a2e5f0422d6 143
screamer 0:4a2e5f0422d6 144 if hasattr(opts, 'auto_detect') and opts.auto_detect:
screamer 0:4a2e5f0422d6 145 # If auto_detect attribute is present, we assume other auto-detection
screamer 0:4a2e5f0422d6 146 # parameters like 'toolchains_filter' are also set.
screamer 0:4a2e5f0422d6 147 print "MBEDLS: Detecting connected mbed-enabled devices... "
screamer 0:4a2e5f0422d6 148
screamer 0:4a2e5f0422d6 149 MUTs = get_autodetected_MUTS_list()
screamer 0:4a2e5f0422d6 150
screamer 0:4a2e5f0422d6 151 for mut in MUTs.values():
screamer 0:4a2e5f0422d6 152 print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu'],
screamer 0:4a2e5f0422d6 153 mut['port'],
screamer 0:4a2e5f0422d6 154 mut['disk'])
screamer 0:4a2e5f0422d6 155
screamer 0:4a2e5f0422d6 156 # Set up parameters for test specification filter function (we need to set toolchains per target here)
screamer 0:4a2e5f0422d6 157 use_default_toolchain = 'default' in opts.toolchains_filter.split(',') if opts.toolchains_filter is not None else True
screamer 0:4a2e5f0422d6 158 use_supported_toolchains = 'all' in opts.toolchains_filter.split(',') if opts.toolchains_filter is not None else False
screamer 0:4a2e5f0422d6 159 toolchain_filter = opts.toolchains_filter
screamer 0:4a2e5f0422d6 160 platform_name_filter = opts.general_filter_regex.split(',') if opts.general_filter_regex is not None else opts.general_filter_regex
screamer 0:4a2e5f0422d6 161 # Test specification with information about each target and associated toolchain
screamer 0:4a2e5f0422d6 162 test_spec = get_autodetected_TEST_SPEC(MUTs.values(),
screamer 0:4a2e5f0422d6 163 use_default_toolchain=use_default_toolchain,
screamer 0:4a2e5f0422d6 164 use_supported_toolchains=use_supported_toolchains,
screamer 0:4a2e5f0422d6 165 toolchain_filter=toolchain_filter,
screamer 0:4a2e5f0422d6 166 platform_name_filter=platform_name_filter)
screamer 0:4a2e5f0422d6 167 else:
screamer 0:4a2e5f0422d6 168 # Open file with test specification
screamer 0:4a2e5f0422d6 169 # test_spec_filename tells script which targets and their toolchain(s)
screamer 0:4a2e5f0422d6 170 # should be covered by the test scenario
screamer 0:4a2e5f0422d6 171 opts.auto_detect = False
screamer 0:4a2e5f0422d6 172 test_spec = get_json_data_from_file(opts.test_spec_filename) if opts.test_spec_filename else None
screamer 0:4a2e5f0422d6 173 if test_spec is None:
screamer 0:4a2e5f0422d6 174 if not opts.test_spec_filename:
screamer 0:4a2e5f0422d6 175 parser.print_help()
screamer 0:4a2e5f0422d6 176 exit(-1)
screamer 0:4a2e5f0422d6 177
screamer 0:4a2e5f0422d6 178 # Get extra MUTs if applicable
screamer 0:4a2e5f0422d6 179 MUTs = get_json_data_from_file(opts.muts_spec_filename) if opts.muts_spec_filename else None
screamer 0:4a2e5f0422d6 180
screamer 0:4a2e5f0422d6 181 if MUTs is None:
screamer 0:4a2e5f0422d6 182 if not opts.muts_spec_filename:
screamer 0:4a2e5f0422d6 183 parser.print_help()
screamer 0:4a2e5f0422d6 184 exit(-1)
screamer 0:4a2e5f0422d6 185
screamer 0:4a2e5f0422d6 186 if opts.verbose_test_configuration_only:
screamer 0:4a2e5f0422d6 187 print "MUTs configuration in %s:" % ('auto-detected' if opts.auto_detect else opts.muts_spec_filename)
screamer 0:4a2e5f0422d6 188 if MUTs:
screamer 0:4a2e5f0422d6 189 print print_muts_configuration_from_json(MUTs, platform_filter=opts.general_filter_regex)
screamer 0:4a2e5f0422d6 190 print
screamer 0:4a2e5f0422d6 191 print "Test specification in %s:" % ('auto-detected' if opts.auto_detect else opts.test_spec_filename)
screamer 0:4a2e5f0422d6 192 if test_spec:
screamer 0:4a2e5f0422d6 193 print print_test_configuration_from_json(test_spec)
screamer 0:4a2e5f0422d6 194 exit(0)
screamer 0:4a2e5f0422d6 195
screamer 0:4a2e5f0422d6 196 if get_module_avail('mbed_lstools'):
screamer 0:4a2e5f0422d6 197 if opts.operability_checks:
screamer 0:4a2e5f0422d6 198 # Check if test scope is valid and run tests
screamer 0:4a2e5f0422d6 199 test_scope = get_available_oper_test_scopes()
screamer 0:4a2e5f0422d6 200 if opts.operability_checks in test_scope:
screamer 0:4a2e5f0422d6 201 tests = IOperTestRunner(scope=opts.operability_checks)
screamer 0:4a2e5f0422d6 202 test_results = tests.run()
screamer 0:4a2e5f0422d6 203
screamer 0:4a2e5f0422d6 204 # Export results in form of JUnit XML report to separate file
screamer 0:4a2e5f0422d6 205 if opts.report_junit_file_name:
screamer 0:4a2e5f0422d6 206 report_exporter = ReportExporter(ResultExporterType.JUNIT_OPER)
screamer 0:4a2e5f0422d6 207 report_exporter.report_to_file(test_results, opts.report_junit_file_name)
screamer 0:4a2e5f0422d6 208 else:
screamer 0:4a2e5f0422d6 209 print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks)
screamer 0:4a2e5f0422d6 210 print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope]))
screamer 0:4a2e5f0422d6 211
screamer 0:4a2e5f0422d6 212 exit(0)
screamer 0:4a2e5f0422d6 213
screamer 0:4a2e5f0422d6 214 # Verbose test specification and MUTs configuration
screamer 0:4a2e5f0422d6 215 if MUTs and opts.verbose:
screamer 0:4a2e5f0422d6 216 print print_muts_configuration_from_json(MUTs)
screamer 0:4a2e5f0422d6 217 if test_spec and opts.verbose:
screamer 0:4a2e5f0422d6 218 print print_test_configuration_from_json(test_spec)
screamer 0:4a2e5f0422d6 219
screamer 0:4a2e5f0422d6 220 if opts.only_build_tests:
screamer 0:4a2e5f0422d6 221 # We are skipping testing phase, and suppress summary
screamer 0:4a2e5f0422d6 222 opts.suppress_summary = True
screamer 0:4a2e5f0422d6 223
screamer 0:4a2e5f0422d6 224 single_test = SingleTestRunner(_global_loops_count=opts.test_global_loops_value,
screamer 0:4a2e5f0422d6 225 _test_loops_list=opts.test_loops_list,
screamer 0:4a2e5f0422d6 226 _muts=MUTs,
screamer 0:4a2e5f0422d6 227 _clean=opts.clean,
screamer 0:4a2e5f0422d6 228 _opts_db_url=opts.db_url,
screamer 0:4a2e5f0422d6 229 _opts_log_file_name=opts.log_file_name,
screamer 0:4a2e5f0422d6 230 _opts_report_html_file_name=opts.report_html_file_name,
screamer 0:4a2e5f0422d6 231 _opts_report_junit_file_name=opts.report_junit_file_name,
screamer 0:4a2e5f0422d6 232 _opts_report_build_file_name=opts.report_build_file_name,
screamer 0:4a2e5f0422d6 233 _test_spec=test_spec,
screamer 0:4a2e5f0422d6 234 _opts_goanna_for_mbed_sdk=opts.goanna_for_mbed_sdk,
screamer 0:4a2e5f0422d6 235 _opts_goanna_for_tests=opts.goanna_for_tests,
screamer 0:4a2e5f0422d6 236 _opts_shuffle_test_order=opts.shuffle_test_order,
screamer 0:4a2e5f0422d6 237 _opts_shuffle_test_seed=opts.shuffle_test_seed,
screamer 0:4a2e5f0422d6 238 _opts_test_by_names=opts.test_by_names,
screamer 0:4a2e5f0422d6 239 _opts_peripheral_by_names=opts.peripheral_by_names,
screamer 0:4a2e5f0422d6 240 _opts_test_only_peripheral=opts.test_only_peripheral,
screamer 0:4a2e5f0422d6 241 _opts_test_only_common=opts.test_only_common,
screamer 0:4a2e5f0422d6 242 _opts_verbose_skipped_tests=opts.verbose_skipped_tests,
screamer 0:4a2e5f0422d6 243 _opts_verbose_test_result_only=opts.verbose_test_result_only,
screamer 0:4a2e5f0422d6 244 _opts_verbose=opts.verbose,
screamer 0:4a2e5f0422d6 245 _opts_firmware_global_name=opts.firmware_global_name,
screamer 0:4a2e5f0422d6 246 _opts_only_build_tests=opts.only_build_tests,
screamer 0:4a2e5f0422d6 247 _opts_parallel_test_exec=opts.parallel_test_exec,
screamer 0:4a2e5f0422d6 248 _opts_suppress_summary=opts.suppress_summary,
screamer 0:4a2e5f0422d6 249 _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary,
screamer 0:4a2e5f0422d6 250 _opts_copy_method=opts.copy_method,
screamer 0:4a2e5f0422d6 251 _opts_mut_reset_type=opts.mut_reset_type,
screamer 0:4a2e5f0422d6 252 _opts_jobs=opts.jobs,
screamer 0:4a2e5f0422d6 253 _opts_waterfall_test=opts.waterfall_test,
screamer 0:4a2e5f0422d6 254 _opts_consolidate_waterfall_test=opts.consolidate_waterfall_test,
screamer 0:4a2e5f0422d6 255 _opts_extend_test_timeout=opts.extend_test_timeout,
screamer 0:4a2e5f0422d6 256 _opts_auto_detect=opts.auto_detect)
screamer 0:4a2e5f0422d6 257
screamer 0:4a2e5f0422d6 258 # Runs test suite in CLI mode
screamer 0:4a2e5f0422d6 259 if (singletest_in_cli_mode(single_test)):
screamer 0:4a2e5f0422d6 260 exit(0)
screamer 0:4a2e5f0422d6 261 else:
screamer 0:4a2e5f0422d6 262 exit(-1)