Anders Blomdell / mbed-sdk-tools

Clone of official tools

test.py@41:2a77626a4c21, 2017-10-25 (annotated)

Committer:
theotherjimmy
Date:
Wed Oct 25 14:46:50 2017 -0500
Revision:
41:2a77626a4c21
Parent:
40:7d3fa6b99b2b
Child:
43:2a7da56ebd24
Update to track Mbed OS 5.6.3

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 #! /usr/bin/env python2
screamer 0:66f3b5499f7f 2 """
screamer 0:66f3b5499f7f 3 mbed SDK
screamer 0:66f3b5499f7f 4 Copyright (c) 2011-2013 ARM Limited
screamer 0:66f3b5499f7f 5
screamer 0:66f3b5499f7f 6 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 7 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 8 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 9
screamer 0:66f3b5499f7f 10 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 11
screamer 0:66f3b5499f7f 12 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 13 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 15 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 16 limitations under the License.
screamer 0:66f3b5499f7f 17
screamer 0:66f3b5499f7f 18
screamer 0:66f3b5499f7f 19 TEST BUILD & RUN
screamer 0:66f3b5499f7f 20 """
screamer 0:66f3b5499f7f 21 import sys
screamer 0:66f3b5499f7f 22 import os
screamer 0:66f3b5499f7f 23 import json
screamer 13:ab47a20b66f0 24 import fnmatch
screamer 0:66f3b5499f7f 25
screamer 0:66f3b5499f7f 26 ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
screamer 0:66f3b5499f7f 27 sys.path.insert(0, ROOT)
screamer 0:66f3b5499f7f 28
The Other Jimmy 31:8ea194f6145b 29 from tools.config import ConfigException
theotherjimmy 41:2a77626a4c21 30 from tools.test_api import test_path_to_name, find_tests, get_test_config, print_tests, build_tests, test_spec_from_test_builds
theotherjimmy 41:2a77626a4c21 31 import tools.test_configs as TestConfig
The Other Jimmy 38:399953da035d 32 from tools.options import get_default_options_parser, extract_profile, extract_mcus
screamer 0:66f3b5499f7f 33 from tools.build_api import build_project, build_library
screamer 29:1210849dba19 34 from tools.build_api import print_build_memory_usage
The Other Jimmy 36:96847d42f010 35 from tools.build_api import merge_build_data
screamer 0:66f3b5499f7f 36 from tools.targets import TARGET_MAP
screamer 24:25bff2709c20 37 from tools.utils import mkdir, ToolException, NotSupportedException, args_error
screamer 13:ab47a20b66f0 38 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 22:9e85236d8716 39 from utils import argparse_filestring_type, argparse_lowercase_type, argparse_many
screamer 24:25bff2709c20 40 from utils import argparse_dir_not_parent
The Other Jimmy 31:8ea194f6145b 41 from tools.toolchains import mbedToolchain, TOOLCHAIN_PATHS, TOOLCHAIN_CLASSES
screamer 22:9e85236d8716 42 from tools.settings import CLI_COLOR_MAP
screamer 0:66f3b5499f7f 43
screamer 0:66f3b5499f7f 44 if __name__ == '__main__':
screamer 0:66f3b5499f7f 45 try:
screamer 0:66f3b5499f7f 46 # Parse Options
The Other Jimmy 31:8ea194f6145b 47 parser = get_default_options_parser(add_app_config=True)
theotherjimmy 40:7d3fa6b99b2b 48
screamer 22:9e85236d8716 49 parser.add_argument("-D",
screamer 13:ab47a20b66f0 50 action="append",
screamer 13:ab47a20b66f0 51 dest="macros",
screamer 13:ab47a20b66f0 52 help="Add a macro definition")
theotherjimmy 40:7d3fa6b99b2b 53
screamer 22:9e85236d8716 54 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 55 type=int,
screamer 0:66f3b5499f7f 56 dest="jobs",
screamer 13:ab47a20b66f0 57 default=0,
screamer 13:ab47a20b66f0 58 help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
screamer 0:66f3b5499f7f 59
screamer 22:9e85236d8716 60 parser.add_argument("--source", dest="source_dir",
screamer 22:9e85236d8716 61 type=argparse_filestring_type,
screamer 22:9e85236d8716 62 default=None, help="The source (input) directory (for sources other than tests). Defaults to current directory.", action="append")
screamer 0:66f3b5499f7f 63
screamer 24:25bff2709c20 64 parser.add_argument("--build", dest="build_dir", type=argparse_dir_not_parent(ROOT),
screamer 0:66f3b5499f7f 65 default=None, help="The build (output) directory")
screamer 0:66f3b5499f7f 66
screamer 22:9e85236d8716 67 parser.add_argument("-l", "--list", action="store_true", dest="list",
screamer 0:66f3b5499f7f 68 default=False, help="List (recursively) available tests in order and exit")
screamer 0:66f3b5499f7f 69
screamer 22:9e85236d8716 70 parser.add_argument("-p", "--paths", dest="paths",
screamer 22:9e85236d8716 71 type=argparse_many(argparse_filestring_type),
screamer 0:66f3b5499f7f 72 default=None, help="Limit the tests to those within the specified comma separated list of paths")
screamer 22:9e85236d8716 73
screamer 0:66f3b5499f7f 74 format_choices = ["list", "json"]
screamer 0:66f3b5499f7f 75 format_default_choice = "list"
screamer 0:66f3b5499f7f 76 format_help = "Change the format in which tests are listed. Choices include: %s. Default: %s" % (", ".join(format_choices), format_default_choice)
screamer 22:9e85236d8716 77 parser.add_argument("-f", "--format", dest="format",
screamer 22:9e85236d8716 78 type=argparse_lowercase_type(format_choices, "format"),
screamer 22:9e85236d8716 79 default=format_default_choice, help=format_help)
theotherjimmy 40:7d3fa6b99b2b 80
screamer 22:9e85236d8716 81 parser.add_argument("--continue-on-build-fail", action="store_true", dest="continue_on_build_fail",
screamer 13:ab47a20b66f0 82 default=None, help="Continue trying to build all tests if a build failure occurs")
screamer 13:ab47a20b66f0 83
screamer 22:9e85236d8716 84 #TODO validate the names instead of just passing through str
screamer 22:9e85236d8716 85 parser.add_argument("-n", "--names", dest="names", type=argparse_many(str),
screamer 0:66f3b5499f7f 86 default=None, help="Limit the tests to a comma separated list of names")
theotherjimmy 40:7d3fa6b99b2b 87
theotherjimmy 41:2a77626a4c21 88 parser.add_argument("--test-config", dest="test_config", type=str,
theotherjimmy 41:2a77626a4c21 89 default=None, help="Test config for a module")
theotherjimmy 41:2a77626a4c21 90
screamer 22:9e85236d8716 91 parser.add_argument("--test-spec", dest="test_spec",
screamer 0:66f3b5499f7f 92 default=None, help="Destination path for a test spec file that can be used by the Greentea automated test tool")
theotherjimmy 40:7d3fa6b99b2b 93
screamer 22:9e85236d8716 94 parser.add_argument("--build-report-junit", dest="build_report_junit",
screamer 13:ab47a20b66f0 95 default=None, help="Destination path for a build report in the JUnit xml format")
The Other Jimmy 36:96847d42f010 96 parser.add_argument("--build-data",
The Other Jimmy 36:96847d42f010 97 dest="build_data",
The Other Jimmy 36:96847d42f010 98 default=None,
The Other Jimmy 36:96847d42f010 99 help="Dump build_data to this file")
theotherjimmy 40:7d3fa6b99b2b 100
screamer 22:9e85236d8716 101 parser.add_argument("-v", "--verbose",
screamer 0:66f3b5499f7f 102 action="store_true",
screamer 0:66f3b5499f7f 103 dest="verbose",
screamer 0:66f3b5499f7f 104 default=False,
screamer 0:66f3b5499f7f 105 help="Verbose diagnostic output")
screamer 0:66f3b5499f7f 106
theotherjimmy 40:7d3fa6b99b2b 107 parser.add_argument("--stats-depth",
theotherjimmy 40:7d3fa6b99b2b 108 type=int,
theotherjimmy 40:7d3fa6b99b2b 109 dest="stats_depth",
theotherjimmy 40:7d3fa6b99b2b 110 default=2,
theotherjimmy 40:7d3fa6b99b2b 111 help="Depth level for static memory report")
theotherjimmy 40:7d3fa6b99b2b 112
screamer 22:9e85236d8716 113 options = parser.parse_args()
screamer 0:66f3b5499f7f 114
screamer 22:9e85236d8716 115 # Filter tests by path if specified
screamer 0:66f3b5499f7f 116 if options.paths:
screamer 22:9e85236d8716 117 all_paths = options.paths
screamer 0:66f3b5499f7f 118 else:
screamer 0:66f3b5499f7f 119 all_paths = ["."]
screamer 22:9e85236d8716 120
screamer 0:66f3b5499f7f 121 all_tests = {}
screamer 0:66f3b5499f7f 122 tests = {}
screamer 22:9e85236d8716 123
screamer 24:25bff2709c20 124 # Target
screamer 24:25bff2709c20 125 if options.mcu is None :
The Other Jimmy 31:8ea194f6145b 126 args_error(parser, "argument -m/--mcu is required")
The Other Jimmy 38:399953da035d 127 mcu = extract_mcus(parser, options)[0]
screamer 24:25bff2709c20 128
screamer 24:25bff2709c20 129 # Toolchain
screamer 24:25bff2709c20 130 if options.tool is None:
The Other Jimmy 31:8ea194f6145b 131 args_error(parser, "argument -t/--tool is required")
screamer 24:25bff2709c20 132 toolchain = options.tool[0]
screamer 24:25bff2709c20 133
The Other Jimmy 31:8ea194f6145b 134 if not TOOLCHAIN_CLASSES[toolchain].check_executable():
The Other Jimmy 31:8ea194f6145b 135 search_path = TOOLCHAIN_PATHS[toolchain] or "No path set"
The Other Jimmy 31:8ea194f6145b 136 args_error(parser, "Could not find executable for %s.\n"
The Other Jimmy 31:8ea194f6145b 137 "Currently set search path: %s"
The Other Jimmy 31:8ea194f6145b 138 % (toolchain, search_path))
The Other Jimmy 31:8ea194f6145b 139
theotherjimmy 41:2a77626a4c21 140 # Assign config file. Precedence: test_config>app_config
theotherjimmy 41:2a77626a4c21 141 # TODO: merge configs if both given
theotherjimmy 41:2a77626a4c21 142 if options.test_config:
theotherjimmy 41:2a77626a4c21 143 config = get_test_config(options.test_config, mcu)
theotherjimmy 41:2a77626a4c21 144 if not config:
theotherjimmy 41:2a77626a4c21 145 args_error(parser, "argument --test-config contains invalid path or identifier")
theotherjimmy 41:2a77626a4c21 146 elif not options.app_config:
theotherjimmy 41:2a77626a4c21 147 config = TestConfig.get_default_config(mcu)
theotherjimmy 41:2a77626a4c21 148 else:
theotherjimmy 41:2a77626a4c21 149 config = options.app_config
theotherjimmy 41:2a77626a4c21 150
screamer 0:66f3b5499f7f 151 # Find all tests in the relevant paths
screamer 0:66f3b5499f7f 152 for path in all_paths:
theotherjimmy 40:7d3fa6b99b2b 153 all_tests.update(find_tests(path, mcu, toolchain,
theotherjimmy 41:2a77626a4c21 154 app_config=config))
screamer 22:9e85236d8716 155
screamer 0:66f3b5499f7f 156 # Filter tests by name if specified
screamer 0:66f3b5499f7f 157 if options.names:
screamer 22:9e85236d8716 158 all_names = options.names
screamer 13:ab47a20b66f0 159 all_names = [x.lower() for x in all_names]
screamer 22:9e85236d8716 160
screamer 0:66f3b5499f7f 161 for name in all_names:
screamer 13:ab47a20b66f0 162 if any(fnmatch.fnmatch(testname, name) for testname in all_tests):
screamer 13:ab47a20b66f0 163 for testname, test in all_tests.items():
screamer 13:ab47a20b66f0 164 if fnmatch.fnmatch(testname, name):
screamer 13:ab47a20b66f0 165 tests[testname] = test
screamer 0:66f3b5499f7f 166 else:
screamer 0:66f3b5499f7f 167 print "[Warning] Test with name '%s' was not found in the available tests" % (name)
screamer 0:66f3b5499f7f 168 else:
screamer 0:66f3b5499f7f 169 tests = all_tests
screamer 0:66f3b5499f7f 170
screamer 22:9e85236d8716 171 if options.color:
screamer 22:9e85236d8716 172 # This import happens late to prevent initializing colorization when we don't need it
screamer 22:9e85236d8716 173 import colorize
screamer 22:9e85236d8716 174 if options.verbose:
screamer 22:9e85236d8716 175 notify = mbedToolchain.print_notify_verbose
screamer 22:9e85236d8716 176 else:
screamer 22:9e85236d8716 177 notify = mbedToolchain.print_notify
screamer 22:9e85236d8716 178 notify = colorize.print_in_color_notifier(CLI_COLOR_MAP, notify)
screamer 22:9e85236d8716 179 else:
screamer 22:9e85236d8716 180 notify = None
screamer 22:9e85236d8716 181
screamer 0:66f3b5499f7f 182 if options.list:
screamer 0:66f3b5499f7f 183 # Print available tests in order and exit
screamer 0:66f3b5499f7f 184 print_tests(tests, options.format)
screamer 13:ab47a20b66f0 185 sys.exit(0)
screamer 0:66f3b5499f7f 186 else:
screamer 0:66f3b5499f7f 187 # Build all tests
screamer 0:66f3b5499f7f 188 if not options.build_dir:
The Other Jimmy 31:8ea194f6145b 189 args_error(parser, "argument --build is required")
screamer 22:9e85236d8716 190
screamer 0:66f3b5499f7f 191 base_source_paths = options.source_dir
screamer 22:9e85236d8716 192
screamer 0:66f3b5499f7f 193 # Default base source path is the current directory
screamer 0:66f3b5499f7f 194 if not base_source_paths:
screamer 0:66f3b5499f7f 195 base_source_paths = ['.']
theotherjimmy 40:7d3fa6b99b2b 196
screamer 13:ab47a20b66f0 197 build_report = {}
screamer 13:ab47a20b66f0 198 build_properties = {}
screamer 13:ab47a20b66f0 199
screamer 13:ab47a20b66f0 200 library_build_success = False
The Other Jimmy 31:8ea194f6145b 201 profile = extract_profile(parser, options, toolchain)
screamer 13:ab47a20b66f0 202 try:
screamer 13:ab47a20b66f0 203 # Build sources
The Other Jimmy 36:96847d42f010 204 build_library(base_source_paths, options.build_dir, mcu,
The Other Jimmy 36:96847d42f010 205 toolchain, jobs=options.jobs,
The Other Jimmy 36:96847d42f010 206 clean=options.clean, report=build_report,
The Other Jimmy 36:96847d42f010 207 properties=build_properties, name="mbed-build",
The Other Jimmy 36:96847d42f010 208 macros=options.macros, verbose=options.verbose,
The Other Jimmy 36:96847d42f010 209 notify=notify, archive=False,
theotherjimmy 41:2a77626a4c21 210 app_config=config,
The Other Jimmy 31:8ea194f6145b 211 build_profile=profile)
screamer 22:9e85236d8716 212
screamer 13:ab47a20b66f0 213 library_build_success = True
screamer 13:ab47a20b66f0 214 except ToolException, e:
screamer 13:ab47a20b66f0 215 # ToolException output is handled by the build log
screamer 13:ab47a20b66f0 216 pass
screamer 13:ab47a20b66f0 217 except NotSupportedException, e:
screamer 13:ab47a20b66f0 218 # NotSupportedException is handled by the build log
screamer 13:ab47a20b66f0 219 pass
screamer 13:ab47a20b66f0 220 except Exception, e:
screamer 13:ab47a20b66f0 221 # Some other exception occurred, print the error message
screamer 13:ab47a20b66f0 222 print e
screamer 22:9e85236d8716 223
screamer 13:ab47a20b66f0 224 if not library_build_success:
screamer 13:ab47a20b66f0 225 print "Failed to build library"
screamer 13:ab47a20b66f0 226 else:
screamer 13:ab47a20b66f0 227 # Build all the tests
The Other Jimmy 31:8ea194f6145b 228
screamer 24:25bff2709c20 229 test_build_success, test_build = build_tests(tests, [options.build_dir], options.build_dir, mcu, toolchain,
screamer 13:ab47a20b66f0 230 clean=options.clean,
screamer 13:ab47a20b66f0 231 report=build_report,
screamer 13:ab47a20b66f0 232 properties=build_properties,
screamer 13:ab47a20b66f0 233 macros=options.macros,
screamer 13:ab47a20b66f0 234 verbose=options.verbose,
screamer 22:9e85236d8716 235 notify=notify,
screamer 13:ab47a20b66f0 236 jobs=options.jobs,
The Other Jimmy 31:8ea194f6145b 237 continue_on_build_fail=options.continue_on_build_fail,
theotherjimmy 41:2a77626a4c21 238 app_config=config,
theotherjimmy 40:7d3fa6b99b2b 239 build_profile=profile,
theotherjimmy 40:7d3fa6b99b2b 240 stats_depth=options.stats_depth)
screamer 22:9e85236d8716 241
screamer 13:ab47a20b66f0 242 # If a path to a test spec is provided, write it to a file
screamer 13:ab47a20b66f0 243 if options.test_spec:
screamer 13:ab47a20b66f0 244 test_spec_data = test_spec_from_test_builds(test_build)
screamer 22:9e85236d8716 245
screamer 13:ab47a20b66f0 246 # Create the target dir for the test spec if necessary
screamer 13:ab47a20b66f0 247 # mkdir will not create the dir if it already exists
screamer 13:ab47a20b66f0 248 test_spec_dir = os.path.dirname(options.test_spec)
screamer 13:ab47a20b66f0 249 if test_spec_dir:
screamer 13:ab47a20b66f0 250 mkdir(test_spec_dir)
screamer 22:9e85236d8716 251
screamer 13:ab47a20b66f0 252 try:
screamer 13:ab47a20b66f0 253 with open(options.test_spec, 'w') as f:
screamer 13:ab47a20b66f0 254 f.write(json.dumps(test_spec_data, indent=2))
screamer 13:ab47a20b66f0 255 except IOError, e:
screamer 13:ab47a20b66f0 256 print "[ERROR] Error writing test spec to file"
screamer 13:ab47a20b66f0 257 print e
screamer 22:9e85236d8716 258
screamer 13:ab47a20b66f0 259 # If a path to a JUnit build report spec is provided, write it to a file
screamer 13:ab47a20b66f0 260 if options.build_report_junit:
screamer 13:ab47a20b66f0 261 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 13:ab47a20b66f0 262 report_exporter.report_to_file(build_report, options.build_report_junit, test_suite_properties=build_properties)
screamer 22:9e85236d8716 263
screamer 22:9e85236d8716 264 # Print memory map summary on screen
screamer 22:9e85236d8716 265 if build_report:
screamer 22:9e85236d8716 266 print
screamer 29:1210849dba19 267 print print_build_memory_usage(build_report)
screamer 22:9e85236d8716 268
screamer 13:ab47a20b66f0 269 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 13:ab47a20b66f0 270 status = print_report_exporter.report(build_report)
The Other Jimmy 36:96847d42f010 271 if options.build_data:
The Other Jimmy 36:96847d42f010 272 merge_build_data(options.build_data, build_report, "test")
screamer 22:9e85236d8716 273
screamer 13:ab47a20b66f0 274 if status:
screamer 13:ab47a20b66f0 275 sys.exit(0)
screamer 13:ab47a20b66f0 276 else:
screamer 13:ab47a20b66f0 277 sys.exit(1)
screamer 0:66f3b5499f7f 278
screamer 0:66f3b5499f7f 279 except KeyboardInterrupt, e:
screamer 0:66f3b5499f7f 280 print "\n[CTRL+c] exit"
The Other Jimmy 31:8ea194f6145b 281 except ConfigException, e:
The Other Jimmy 31:8ea194f6145b 282 # Catching ConfigException here to prevent a traceback
The Other Jimmy 31:8ea194f6145b 283 print "[ERROR] %s" % str(e)
screamer 0:66f3b5499f7f 284 except Exception,e:
screamer 0:66f3b5499f7f 285 import traceback
screamer 0:66f3b5499f7f 286 traceback.print_exc(file=sys.stdout)
screamer 0:66f3b5499f7f 287 print "[ERROR] %s" % str(e)
screamer 0:66f3b5499f7f 288 sys.exit(1)