Anders Blomdell / mbed-sdk-tools

Clone of official tools

test.py@43:2a7da56ebd24, 2018-09-25 (annotated)

Committer:
theotherjimmy
Date:
Tue Sep 25 13:43:09 2018 -0500
Revision:
43:2a7da56ebd24
Parent:
41:2a77626a4c21 
Release 5.10.0

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 #! /usr/bin/env python2
screamer 0:66f3b5499f7f 2 """
screamer 0:66f3b5499f7f 3 mbed SDK
screamer 0:66f3b5499f7f 4 Copyright (c) 2011-2013 ARM Limited
screamer 0:66f3b5499f7f 5
screamer 0:66f3b5499f7f 6 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 7 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 8 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 9
screamer 0:66f3b5499f7f 10 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 11
screamer 0:66f3b5499f7f 12 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 13 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 15 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 16 limitations under the License.
screamer 0:66f3b5499f7f 17
screamer 0:66f3b5499f7f 18
theotherjimmy 43:2a7da56ebd24 19 TEST BUILD
screamer 0:66f3b5499f7f 20 """
theotherjimmy 43:2a7da56ebd24 21 from __future__ import print_function, division, absolute_import
screamer 0:66f3b5499f7f 22 import sys
screamer 0:66f3b5499f7f 23 import os
screamer 13:ab47a20b66f0 24 import fnmatch
screamer 0:66f3b5499f7f 25
screamer 0:66f3b5499f7f 26 ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
screamer 0:66f3b5499f7f 27 sys.path.insert(0, ROOT)
screamer 0:66f3b5499f7f 28
theotherjimmy 43:2a7da56ebd24 29 from tools.config import ConfigException, Config
theotherjimmy 43:2a7da56ebd24 30 from tools.test_configs import get_default_config
The Other Jimmy 31:8ea194f6145b 31 from tools.config import ConfigException
theotherjimmy 43:2a7da56ebd24 32 from tools.test_api import find_tests, get_test_config, print_tests, build_tests, test_spec_from_test_builds
theotherjimmy 41:2a77626a4c21 33 import tools.test_configs as TestConfig
The Other Jimmy 38:399953da035d 34 from tools.options import get_default_options_parser, extract_profile, extract_mcus
screamer 0:66f3b5499f7f 35 from tools.build_api import build_project, build_library
screamer 29:1210849dba19 36 from tools.build_api import print_build_memory_usage
The Other Jimmy 36:96847d42f010 37 from tools.build_api import merge_build_data
screamer 0:66f3b5499f7f 38 from tools.targets import TARGET_MAP
theotherjimmy 43:2a7da56ebd24 39 from tools.notifier.term import TerminalNotifier
theotherjimmy 43:2a7da56ebd24 40 from tools.utils import mkdir, ToolException, NotSupportedException, args_error, write_json_to_file
screamer 13:ab47a20b66f0 41 from tools.test_exporters import ReportExporter, ResultExporterType
theotherjimmy 43:2a7da56ebd24 42 from tools.utils import argparse_filestring_type, argparse_lowercase_type, argparse_many
theotherjimmy 43:2a7da56ebd24 43 from tools.utils import argparse_dir_not_parent
The Other Jimmy 31:8ea194f6145b 44 from tools.toolchains import mbedToolchain, TOOLCHAIN_PATHS, TOOLCHAIN_CLASSES
screamer 22:9e85236d8716 45 from tools.settings import CLI_COLOR_MAP
screamer 0:66f3b5499f7f 46
screamer 0:66f3b5499f7f 47 if __name__ == '__main__':
screamer 0:66f3b5499f7f 48 try:
screamer 0:66f3b5499f7f 49 # Parse Options
The Other Jimmy 31:8ea194f6145b 50 parser = get_default_options_parser(add_app_config=True)
theotherjimmy 40:7d3fa6b99b2b 51
screamer 22:9e85236d8716 52 parser.add_argument("-D",
screamer 13:ab47a20b66f0 53 action="append",
screamer 13:ab47a20b66f0 54 dest="macros",
screamer 13:ab47a20b66f0 55 help="Add a macro definition")
theotherjimmy 40:7d3fa6b99b2b 56
screamer 22:9e85236d8716 57 parser.add_argument("-j", "--jobs",
screamer 22:9e85236d8716 58 type=int,
screamer 0:66f3b5499f7f 59 dest="jobs",
screamer 13:ab47a20b66f0 60 default=0,
screamer 13:ab47a20b66f0 61 help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
screamer 0:66f3b5499f7f 62
screamer 22:9e85236d8716 63 parser.add_argument("--source", dest="source_dir",
screamer 22:9e85236d8716 64 type=argparse_filestring_type,
screamer 22:9e85236d8716 65 default=None, help="The source (input) directory (for sources other than tests). Defaults to current directory.", action="append")
screamer 0:66f3b5499f7f 66
screamer 24:25bff2709c20 67 parser.add_argument("--build", dest="build_dir", type=argparse_dir_not_parent(ROOT),
screamer 0:66f3b5499f7f 68 default=None, help="The build (output) directory")
screamer 0:66f3b5499f7f 69
screamer 22:9e85236d8716 70 parser.add_argument("-l", "--list", action="store_true", dest="list",
screamer 0:66f3b5499f7f 71 default=False, help="List (recursively) available tests in order and exit")
screamer 0:66f3b5499f7f 72
screamer 22:9e85236d8716 73 parser.add_argument("-p", "--paths", dest="paths",
screamer 22:9e85236d8716 74 type=argparse_many(argparse_filestring_type),
screamer 0:66f3b5499f7f 75 default=None, help="Limit the tests to those within the specified comma separated list of paths")
screamer 22:9e85236d8716 76
screamer 0:66f3b5499f7f 77 format_choices = ["list", "json"]
screamer 0:66f3b5499f7f 78 format_default_choice = "list"
screamer 0:66f3b5499f7f 79 format_help = "Change the format in which tests are listed. Choices include: %s. Default: %s" % (", ".join(format_choices), format_default_choice)
screamer 22:9e85236d8716 80 parser.add_argument("-f", "--format", dest="format",
screamer 22:9e85236d8716 81 type=argparse_lowercase_type(format_choices, "format"),
screamer 22:9e85236d8716 82 default=format_default_choice, help=format_help)
theotherjimmy 40:7d3fa6b99b2b 83
screamer 22:9e85236d8716 84 parser.add_argument("--continue-on-build-fail", action="store_true", dest="continue_on_build_fail",
screamer 13:ab47a20b66f0 85 default=None, help="Continue trying to build all tests if a build failure occurs")
screamer 13:ab47a20b66f0 86
screamer 22:9e85236d8716 87 #TODO validate the names instead of just passing through str
screamer 22:9e85236d8716 88 parser.add_argument("-n", "--names", dest="names", type=argparse_many(str),
screamer 0:66f3b5499f7f 89 default=None, help="Limit the tests to a comma separated list of names")
theotherjimmy 40:7d3fa6b99b2b 90
theotherjimmy 41:2a77626a4c21 91 parser.add_argument("--test-config", dest="test_config", type=str,
theotherjimmy 41:2a77626a4c21 92 default=None, help="Test config for a module")
theotherjimmy 41:2a77626a4c21 93
screamer 22:9e85236d8716 94 parser.add_argument("--test-spec", dest="test_spec",
screamer 0:66f3b5499f7f 95 default=None, help="Destination path for a test spec file that can be used by the Greentea automated test tool")
theotherjimmy 40:7d3fa6b99b2b 96
screamer 22:9e85236d8716 97 parser.add_argument("--build-report-junit", dest="build_report_junit",
screamer 13:ab47a20b66f0 98 default=None, help="Destination path for a build report in the JUnit xml format")
The Other Jimmy 36:96847d42f010 99 parser.add_argument("--build-data",
The Other Jimmy 36:96847d42f010 100 dest="build_data",
The Other Jimmy 36:96847d42f010 101 default=None,
The Other Jimmy 36:96847d42f010 102 help="Dump build_data to this file")
theotherjimmy 40:7d3fa6b99b2b 103
screamer 22:9e85236d8716 104 parser.add_argument("-v", "--verbose",
screamer 0:66f3b5499f7f 105 action="store_true",
screamer 0:66f3b5499f7f 106 dest="verbose",
screamer 0:66f3b5499f7f 107 default=False,
screamer 0:66f3b5499f7f 108 help="Verbose diagnostic output")
screamer 0:66f3b5499f7f 109
theotherjimmy 40:7d3fa6b99b2b 110 parser.add_argument("--stats-depth",
theotherjimmy 40:7d3fa6b99b2b 111 type=int,
theotherjimmy 40:7d3fa6b99b2b 112 dest="stats_depth",
theotherjimmy 40:7d3fa6b99b2b 113 default=2,
theotherjimmy 40:7d3fa6b99b2b 114 help="Depth level for static memory report")
theotherjimmy 43:2a7da56ebd24 115 parser.add_argument("--ignore", dest="ignore", type=argparse_many(str),
theotherjimmy 43:2a7da56ebd24 116 default=None, help="Comma separated list of patterns to add to mbedignore (eg. ./main.cpp)")
theotherjimmy 43:2a7da56ebd24 117 parser.add_argument("--icetea",
theotherjimmy 43:2a7da56ebd24 118 action="store_true",
theotherjimmy 43:2a7da56ebd24 119 dest="icetea",
theotherjimmy 43:2a7da56ebd24 120 default=False,
theotherjimmy 43:2a7da56ebd24 121 help="Only icetea tests")
theotherjimmy 43:2a7da56ebd24 122
theotherjimmy 43:2a7da56ebd24 123 parser.add_argument("--greentea",
theotherjimmy 43:2a7da56ebd24 124 action="store_true",
theotherjimmy 43:2a7da56ebd24 125 dest="greentea",
theotherjimmy 43:2a7da56ebd24 126 default=False,
theotherjimmy 43:2a7da56ebd24 127 help="Only greentea tests")
theotherjimmy 40:7d3fa6b99b2b 128
screamer 22:9e85236d8716 129 options = parser.parse_args()
screamer 0:66f3b5499f7f 130
screamer 22:9e85236d8716 131 # Filter tests by path if specified
screamer 0:66f3b5499f7f 132 if options.paths:
screamer 22:9e85236d8716 133 all_paths = options.paths
screamer 0:66f3b5499f7f 134 else:
screamer 0:66f3b5499f7f 135 all_paths = ["."]
screamer 22:9e85236d8716 136
screamer 0:66f3b5499f7f 137 all_tests = {}
screamer 0:66f3b5499f7f 138 tests = {}
screamer 22:9e85236d8716 139
theotherjimmy 43:2a7da56ebd24 140 # As default both test tools are enabled
theotherjimmy 43:2a7da56ebd24 141 if not (options.greentea or options.icetea):
theotherjimmy 43:2a7da56ebd24 142 options.greentea = True
theotherjimmy 43:2a7da56ebd24 143 options.icetea = True
theotherjimmy 43:2a7da56ebd24 144
screamer 24:25bff2709c20 145 # Target
theotherjimmy 43:2a7da56ebd24 146 if options.mcu is None:
The Other Jimmy 31:8ea194f6145b 147 args_error(parser, "argument -m/--mcu is required")
The Other Jimmy 38:399953da035d 148 mcu = extract_mcus(parser, options)[0]
screamer 24:25bff2709c20 149
screamer 24:25bff2709c20 150 # Toolchain
screamer 24:25bff2709c20 151 if options.tool is None:
The Other Jimmy 31:8ea194f6145b 152 args_error(parser, "argument -t/--tool is required")
screamer 24:25bff2709c20 153 toolchain = options.tool[0]
screamer 24:25bff2709c20 154
The Other Jimmy 31:8ea194f6145b 155 if not TOOLCHAIN_CLASSES[toolchain].check_executable():
The Other Jimmy 31:8ea194f6145b 156 search_path = TOOLCHAIN_PATHS[toolchain] or "No path set"
The Other Jimmy 31:8ea194f6145b 157 args_error(parser, "Could not find executable for %s.\n"
The Other Jimmy 31:8ea194f6145b 158 "Currently set search path: %s"
The Other Jimmy 31:8ea194f6145b 159 % (toolchain, search_path))
The Other Jimmy 31:8ea194f6145b 160
theotherjimmy 41:2a77626a4c21 161 # Assign config file. Precedence: test_config>app_config
theotherjimmy 41:2a77626a4c21 162 # TODO: merge configs if both given
theotherjimmy 41:2a77626a4c21 163 if options.test_config:
theotherjimmy 41:2a77626a4c21 164 config = get_test_config(options.test_config, mcu)
theotherjimmy 41:2a77626a4c21 165 if not config:
theotherjimmy 41:2a77626a4c21 166 args_error(parser, "argument --test-config contains invalid path or identifier")
theotherjimmy 43:2a7da56ebd24 167 elif options.app_config:
theotherjimmy 43:2a7da56ebd24 168 config = options.app_config
theotherjimmy 41:2a77626a4c21 169 else:
theotherjimmy 43:2a7da56ebd24 170 config = Config.find_app_config(options.source_dir)
theotherjimmy 43:2a7da56ebd24 171
theotherjimmy 43:2a7da56ebd24 172 if not config:
theotherjimmy 43:2a7da56ebd24 173 config = get_default_config(options.source_dir or ['.'], mcu)
theotherjimmy 43:2a7da56ebd24 174
theotherjimmy 41:2a77626a4c21 175
screamer 0:66f3b5499f7f 176 # Find all tests in the relevant paths
screamer 0:66f3b5499f7f 177 for path in all_paths:
theotherjimmy 43:2a7da56ebd24 178 all_tests.update(find_tests(
theotherjimmy 43:2a7da56ebd24 179 base_dir=path,
theotherjimmy 43:2a7da56ebd24 180 target_name=mcu,
theotherjimmy 43:2a7da56ebd24 181 toolchain_name=toolchain,
theotherjimmy 43:2a7da56ebd24 182 icetea=options.icetea,
theotherjimmy 43:2a7da56ebd24 183 greentea=options.greentea,
theotherjimmy 43:2a7da56ebd24 184 app_config=config))
screamer 22:9e85236d8716 185
screamer 0:66f3b5499f7f 186 # Filter tests by name if specified
screamer 0:66f3b5499f7f 187 if options.names:
screamer 22:9e85236d8716 188 all_names = options.names
screamer 13:ab47a20b66f0 189 all_names = [x.lower() for x in all_names]
screamer 22:9e85236d8716 190
screamer 0:66f3b5499f7f 191 for name in all_names:
screamer 13:ab47a20b66f0 192 if any(fnmatch.fnmatch(testname, name) for testname in all_tests):
screamer 13:ab47a20b66f0 193 for testname, test in all_tests.items():
screamer 13:ab47a20b66f0 194 if fnmatch.fnmatch(testname, name):
screamer 13:ab47a20b66f0 195 tests[testname] = test
screamer 0:66f3b5499f7f 196 else:
theotherjimmy 43:2a7da56ebd24 197 print("[Warning] Test with name '%s' was not found in the "
theotherjimmy 43:2a7da56ebd24 198 "available tests" % (name))
screamer 0:66f3b5499f7f 199 else:
screamer 0:66f3b5499f7f 200 tests = all_tests
screamer 0:66f3b5499f7f 201
screamer 22:9e85236d8716 202
screamer 0:66f3b5499f7f 203 if options.list:
screamer 0:66f3b5499f7f 204 # Print available tests in order and exit
screamer 0:66f3b5499f7f 205 print_tests(tests, options.format)
screamer 13:ab47a20b66f0 206 sys.exit(0)
screamer 0:66f3b5499f7f 207 else:
screamer 0:66f3b5499f7f 208 # Build all tests
screamer 0:66f3b5499f7f 209 if not options.build_dir:
The Other Jimmy 31:8ea194f6145b 210 args_error(parser, "argument --build is required")
screamer 22:9e85236d8716 211
screamer 0:66f3b5499f7f 212 base_source_paths = options.source_dir
screamer 22:9e85236d8716 213
screamer 0:66f3b5499f7f 214 # Default base source path is the current directory
screamer 0:66f3b5499f7f 215 if not base_source_paths:
screamer 0:66f3b5499f7f 216 base_source_paths = ['.']
theotherjimmy 40:7d3fa6b99b2b 217
screamer 13:ab47a20b66f0 218 build_report = {}
screamer 13:ab47a20b66f0 219 build_properties = {}
screamer 13:ab47a20b66f0 220
screamer 13:ab47a20b66f0 221 library_build_success = False
The Other Jimmy 31:8ea194f6145b 222 profile = extract_profile(parser, options, toolchain)
screamer 13:ab47a20b66f0 223 try:
screamer 13:ab47a20b66f0 224 # Build sources
theotherjimmy 43:2a7da56ebd24 225 notify = TerminalNotifier(options.verbose)
The Other Jimmy 36:96847d42f010 226 build_library(base_source_paths, options.build_dir, mcu,
The Other Jimmy 36:96847d42f010 227 toolchain, jobs=options.jobs,
The Other Jimmy 36:96847d42f010 228 clean=options.clean, report=build_report,
The Other Jimmy 36:96847d42f010 229 properties=build_properties, name="mbed-build",
theotherjimmy 43:2a7da56ebd24 230 macros=options.macros,
The Other Jimmy 36:96847d42f010 231 notify=notify, archive=False,
theotherjimmy 41:2a77626a4c21 232 app_config=config,
theotherjimmy 43:2a7da56ebd24 233 build_profile=profile,
theotherjimmy 43:2a7da56ebd24 234 ignore=options.ignore)
screamer 22:9e85236d8716 235
screamer 13:ab47a20b66f0 236 library_build_success = True
theotherjimmy 43:2a7da56ebd24 237 except ToolException as e:
screamer 13:ab47a20b66f0 238 # ToolException output is handled by the build log
screamer 13:ab47a20b66f0 239 pass
theotherjimmy 43:2a7da56ebd24 240 except NotSupportedException as e:
screamer 13:ab47a20b66f0 241 # NotSupportedException is handled by the build log
screamer 13:ab47a20b66f0 242 pass
theotherjimmy 43:2a7da56ebd24 243 except Exception as e:
theotherjimmy 43:2a7da56ebd24 244 if options.verbose:
theotherjimmy 43:2a7da56ebd24 245 import traceback
theotherjimmy 43:2a7da56ebd24 246 traceback.print_exc()
screamer 13:ab47a20b66f0 247 # Some other exception occurred, print the error message
theotherjimmy 43:2a7da56ebd24 248 print(e)
screamer 22:9e85236d8716 249
screamer 13:ab47a20b66f0 250 if not library_build_success:
theotherjimmy 43:2a7da56ebd24 251 print("Failed to build library")
screamer 13:ab47a20b66f0 252 else:
screamer 13:ab47a20b66f0 253 # Build all the tests
theotherjimmy 43:2a7da56ebd24 254 notify = TerminalNotifier(options.verbose)
theotherjimmy 43:2a7da56ebd24 255 test_build_success, test_build = build_tests(
theotherjimmy 43:2a7da56ebd24 256 tests,
theotherjimmy 43:2a7da56ebd24 257 [os.path.relpath(options.build_dir)],
theotherjimmy 43:2a7da56ebd24 258 options.build_dir,
theotherjimmy 43:2a7da56ebd24 259 mcu,
theotherjimmy 43:2a7da56ebd24 260 toolchain,
theotherjimmy 43:2a7da56ebd24 261 clean=options.clean,
theotherjimmy 43:2a7da56ebd24 262 report=build_report,
theotherjimmy 43:2a7da56ebd24 263 properties=build_properties,
theotherjimmy 43:2a7da56ebd24 264 macros=options.macros,
theotherjimmy 43:2a7da56ebd24 265 notify=notify,
theotherjimmy 43:2a7da56ebd24 266 jobs=options.jobs,
theotherjimmy 43:2a7da56ebd24 267 continue_on_build_fail=options.continue_on_build_fail,
theotherjimmy 43:2a7da56ebd24 268 app_config=config,
theotherjimmy 43:2a7da56ebd24 269 build_profile=profile,
theotherjimmy 43:2a7da56ebd24 270 stats_depth=options.stats_depth,
theotherjimmy 43:2a7da56ebd24 271 ignore=options.ignore)
screamer 22:9e85236d8716 272
screamer 13:ab47a20b66f0 273 # If a path to a test spec is provided, write it to a file
screamer 13:ab47a20b66f0 274 if options.test_spec:
theotherjimmy 43:2a7da56ebd24 275 write_json_to_file(test_spec_from_test_builds(test_build), options.test_spec)
screamer 22:9e85236d8716 276
screamer 13:ab47a20b66f0 277 # If a path to a JUnit build report spec is provided, write it to a file
screamer 13:ab47a20b66f0 278 if options.build_report_junit:
screamer 13:ab47a20b66f0 279 report_exporter = ReportExporter(ResultExporterType.JUNIT, package="build")
screamer 13:ab47a20b66f0 280 report_exporter.report_to_file(build_report, options.build_report_junit, test_suite_properties=build_properties)
screamer 22:9e85236d8716 281
screamer 22:9e85236d8716 282 # Print memory map summary on screen
screamer 22:9e85236d8716 283 if build_report:
screamer 22:9e85236d8716 284 print
theotherjimmy 43:2a7da56ebd24 285 print(print_build_memory_usage(build_report))
screamer 22:9e85236d8716 286
screamer 13:ab47a20b66f0 287 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 13:ab47a20b66f0 288 status = print_report_exporter.report(build_report)
The Other Jimmy 36:96847d42f010 289 if options.build_data:
The Other Jimmy 36:96847d42f010 290 merge_build_data(options.build_data, build_report, "test")
screamer 22:9e85236d8716 291
screamer 13:ab47a20b66f0 292 if status:
screamer 13:ab47a20b66f0 293 sys.exit(0)
screamer 13:ab47a20b66f0 294 else:
screamer 13:ab47a20b66f0 295 sys.exit(1)
screamer 0:66f3b5499f7f 296
theotherjimmy 43:2a7da56ebd24 297 except KeyboardInterrupt as e:
theotherjimmy 43:2a7da56ebd24 298 print("\n[CTRL+c] exit")
theotherjimmy 43:2a7da56ebd24 299 except ConfigException as e:
The Other Jimmy 31:8ea194f6145b 300 # Catching ConfigException here to prevent a traceback
theotherjimmy 43:2a7da56ebd24 301 print("[ERROR] %s" % str(e))
theotherjimmy 43:2a7da56ebd24 302 except Exception as e:
screamer 0:66f3b5499f7f 303 import traceback
screamer 0:66f3b5499f7f 304 traceback.print_exc(file=sys.stdout)
theotherjimmy 43:2a7da56ebd24 305 print("[ERROR] %s" % str(e))
screamer 0:66f3b5499f7f 306 sys.exit(1)
theotherjimmy 43:2a7da56ebd24 307