the other jimmy / mbed-sdk-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-sdk-tools by mbed official

build_everything.py@13:ab47a20b66f0, 2016-07-14 (annotated)

Committer:
screamer
Date:
Thu Jul 14 20:21:19 2016 +0100
Revision:
13:ab47a20b66f0
Child:
29:1210849dba19
Apply latest tools

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 13:ab47a20b66f0 1 #! /usr/bin/env python
screamer 13:ab47a20b66f0 2 """
screamer 13:ab47a20b66f0 3 mbed SDK
screamer 13:ab47a20b66f0 4 Copyright (c) 2011-2013 ARM Limited
screamer 13:ab47a20b66f0 5
screamer 13:ab47a20b66f0 6 Licensed under the Apache License, Version 2.0 (the "License");
screamer 13:ab47a20b66f0 7 you may not use this file except in compliance with the License.
screamer 13:ab47a20b66f0 8 You may obtain a copy of the License at
screamer 13:ab47a20b66f0 9
screamer 13:ab47a20b66f0 10 http://www.apache.org/licenses/LICENSE-2.0
screamer 13:ab47a20b66f0 11
screamer 13:ab47a20b66f0 12 Unless required by applicable law or agreed to in writing, software
screamer 13:ab47a20b66f0 13 distributed under the License is distributed on an "AS IS" BASIS,
screamer 13:ab47a20b66f0 14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 13:ab47a20b66f0 15 See the License for the specific language governing permissions and
screamer 13:ab47a20b66f0 16 limitations under the License.
screamer 13:ab47a20b66f0 17 """
screamer 13:ab47a20b66f0 18 import sys
screamer 13:ab47a20b66f0 19 from time import time
screamer 13:ab47a20b66f0 20 from os.path import join, abspath, dirname, normpath
screamer 13:ab47a20b66f0 21 from optparse import OptionParser
screamer 13:ab47a20b66f0 22 import json
screamer 13:ab47a20b66f0 23
screamer 13:ab47a20b66f0 24 # Be sure that the tools directory is in the search path
screamer 13:ab47a20b66f0 25 ROOT = abspath(join(dirname(__file__), ".."))
screamer 13:ab47a20b66f0 26 sys.path.insert(0, ROOT)
screamer 13:ab47a20b66f0 27
screamer 13:ab47a20b66f0 28 from tools.build_api import build_library
screamer 13:ab47a20b66f0 29 from tools.build_api import write_build_report
screamer 13:ab47a20b66f0 30 from tools.targets import TARGET_MAP, TARGET_NAMES
screamer 13:ab47a20b66f0 31 from tools.toolchains import TOOLCHAINS
screamer 13:ab47a20b66f0 32 from tools.test_exporters import ReportExporter, ResultExporterType
screamer 13:ab47a20b66f0 33 from tools.test_api import find_tests, build_tests, test_spec_from_test_builds
screamer 13:ab47a20b66f0 34 from tools.build_release import OFFICIAL_MBED_LIBRARY_BUILD
screamer 13:ab47a20b66f0 35
screamer 13:ab47a20b66f0 36 if __name__ == '__main__':
screamer 13:ab47a20b66f0 37 try:
screamer 13:ab47a20b66f0 38 parser = OptionParser()
screamer 13:ab47a20b66f0 39
screamer 13:ab47a20b66f0 40 parser.add_option("--source", dest="source_dir",
screamer 13:ab47a20b66f0 41 default=None, help="The source (input) directory (for sources other than tests). Defaults to current directory.", action="append")
screamer 13:ab47a20b66f0 42
screamer 13:ab47a20b66f0 43 parser.add_option("--build", dest="build_dir",
screamer 13:ab47a20b66f0 44 default=None, help="The build (output) directory")
screamer 13:ab47a20b66f0 45
screamer 13:ab47a20b66f0 46 parser.add_option('-c', '--clean',
screamer 13:ab47a20b66f0 47 dest='clean',
screamer 13:ab47a20b66f0 48 metavar=False,
screamer 13:ab47a20b66f0 49 action="store_true",
screamer 13:ab47a20b66f0 50 help='Clean the build directory')
screamer 13:ab47a20b66f0 51
screamer 13:ab47a20b66f0 52 parser.add_option('-a', '--all', dest="all", default=False, action="store_true",
screamer 13:ab47a20b66f0 53 help="Build every target (including unofficial targets) and with each of the supported toolchains")
screamer 13:ab47a20b66f0 54
screamer 13:ab47a20b66f0 55 parser.add_option('-o', '--official', dest="official_only", default=False, action="store_true",
screamer 13:ab47a20b66f0 56 help="Build using only the official toolchain for each target")
screamer 13:ab47a20b66f0 57
screamer 13:ab47a20b66f0 58 parser.add_option("-D", "",
screamer 13:ab47a20b66f0 59 action="append",
screamer 13:ab47a20b66f0 60 dest="macros",
screamer 13:ab47a20b66f0 61 help="Add a macro definition")
screamer 13:ab47a20b66f0 62
screamer 13:ab47a20b66f0 63 parser.add_option("-j", "--jobs", type="int", dest="jobs",
screamer 13:ab47a20b66f0 64 default=0, help="Number of concurrent jobs. Default: 0/auto (based on host machine's number of CPUs)")
screamer 13:ab47a20b66f0 65
screamer 13:ab47a20b66f0 66 parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
screamer 13:ab47a20b66f0 67 default=False, help="Verbose diagnostic output")
screamer 13:ab47a20b66f0 68
screamer 13:ab47a20b66f0 69 parser.add_option("-t", "--toolchains", dest="toolchains", help="Use toolchains names separated by comma")
screamer 13:ab47a20b66f0 70
screamer 13:ab47a20b66f0 71 parser.add_option("-p", "--platforms", dest="platforms", default="", help="Build only for the platform namesseparated by comma")
screamer 13:ab47a20b66f0 72
screamer 13:ab47a20b66f0 73 parser.add_option("", "--config", action="store_true", dest="list_config",
screamer 13:ab47a20b66f0 74 default=False, help="List the platforms and toolchains in the release in JSON")
screamer 13:ab47a20b66f0 75
screamer 13:ab47a20b66f0 76 parser.add_option("", "--test-spec", dest="test_spec",
screamer 13:ab47a20b66f0 77 default=None, help="Destination path for a test spec file that can be used by the Greentea automated test tool")
screamer 13:ab47a20b66f0 78
screamer 13:ab47a20b66f0 79 parser.add_option("", "--build-report-junit", dest="build_report_junit", help="Output the build results to an junit xml file")
screamer 13:ab47a20b66f0 80
screamer 13:ab47a20b66f0 81 parser.add_option("--continue-on-build-fail", action="store_true", dest="continue_on_build_fail",
screamer 13:ab47a20b66f0 82 default=False, help="Continue trying to build all tests if a build failure occurs")
screamer 13:ab47a20b66f0 83
screamer 13:ab47a20b66f0 84 options, args = parser.parse_args()
screamer 13:ab47a20b66f0 85
screamer 13:ab47a20b66f0 86 # Get set of valid targets
screamer 13:ab47a20b66f0 87 all_platforms = set(TARGET_NAMES)
screamer 13:ab47a20b66f0 88 bad_platforms = set()
screamer 13:ab47a20b66f0 89 platforms = set()
screamer 13:ab47a20b66f0 90 if options.platforms != "":
screamer 13:ab47a20b66f0 91 platforms = set(options.platforms.split(","))
screamer 13:ab47a20b66f0 92 bad_platforms = platforms.difference(all_platforms)
screamer 13:ab47a20b66f0 93 platforms = platforms.intersection(all_platforms)
screamer 13:ab47a20b66f0 94 elif options.all:
screamer 13:ab47a20b66f0 95 platforms = all_platforms
screamer 13:ab47a20b66f0 96 else:
screamer 13:ab47a20b66f0 97 platforms = set(x[0] for x in OFFICIAL_MBED_LIBRARY_BUILD)
screamer 13:ab47a20b66f0 98 bad_platforms = platforms.difference(all_platforms)
screamer 13:ab47a20b66f0 99 platforms = platforms.intersection(all_platforms)
screamer 13:ab47a20b66f0 100
screamer 13:ab47a20b66f0 101 for bad_platform in bad_platforms:
screamer 13:ab47a20b66f0 102 print "Platform '%s' is not a valid platform. Skipping." % bad_platform
screamer 13:ab47a20b66f0 103
screamer 13:ab47a20b66f0 104 if options.platforms:
screamer 13:ab47a20b66f0 105 print "Limiting build to the following platforms: %s" % ",".join(platforms)
screamer 13:ab47a20b66f0 106
screamer 13:ab47a20b66f0 107 # Get set of valid toolchains
screamer 13:ab47a20b66f0 108 all_toolchains = set(TOOLCHAINS)
screamer 13:ab47a20b66f0 109 bad_toolchains = set()
screamer 13:ab47a20b66f0 110 toolchains = set()
screamer 13:ab47a20b66f0 111
screamer 13:ab47a20b66f0 112 if options.toolchains:
screamer 13:ab47a20b66f0 113 toolchains = set(options.toolchains.split(","))
screamer 13:ab47a20b66f0 114 bad_toolchains = toolchains.difference(all_toolchains)
screamer 13:ab47a20b66f0 115 toolchains = toolchains.intersection(all_toolchains)
screamer 13:ab47a20b66f0 116 else:
screamer 13:ab47a20b66f0 117 toolchains = all_toolchains
screamer 13:ab47a20b66f0 118
screamer 13:ab47a20b66f0 119 for bad_toolchain in bad_toolchains:
screamer 13:ab47a20b66f0 120 print "Toolchain '%s' is not a valid toolchain. Skipping." % bad_toolchain
screamer 13:ab47a20b66f0 121
screamer 13:ab47a20b66f0 122 if options.toolchains:
screamer 13:ab47a20b66f0 123 print "Limiting build to the following toolchains: %s" % ",".join(toolchains)
screamer 13:ab47a20b66f0 124
screamer 13:ab47a20b66f0 125 build_config = {}
screamer 13:ab47a20b66f0 126
screamer 13:ab47a20b66f0 127 for platform in platforms:
screamer 13:ab47a20b66f0 128 target = TARGET_MAP[platform]
screamer 13:ab47a20b66f0 129
screamer 13:ab47a20b66f0 130 if options.official_only:
screamer 13:ab47a20b66f0 131 default_toolchain = getattr(target, 'default_toolchain', 'ARM')
screamer 13:ab47a20b66f0 132 build_config[platform] = list(toolchains.intersection(set([default_toolchain])))
screamer 13:ab47a20b66f0 133 else:
screamer 13:ab47a20b66f0 134 build_config[platform] = list(toolchains.intersection(set(target.supported_toolchains)))
screamer 13:ab47a20b66f0 135
screamer 13:ab47a20b66f0 136 if options.list_config:
screamer 13:ab47a20b66f0 137 print json.dumps(build_config, indent=4)
screamer 13:ab47a20b66f0 138 sys.exit(0)
screamer 13:ab47a20b66f0 139
screamer 13:ab47a20b66f0 140 # Ensure build directory is set
screamer 13:ab47a20b66f0 141 if not options.build_dir:
screamer 13:ab47a20b66f0 142 print "[ERROR] You must specify a build path"
screamer 13:ab47a20b66f0 143 sys.exit(1)
screamer 13:ab47a20b66f0 144
screamer 13:ab47a20b66f0 145 # Default base source path is the current directory
screamer 13:ab47a20b66f0 146 base_source_paths = options.source_dir
screamer 13:ab47a20b66f0 147 if not base_source_paths:
screamer 13:ab47a20b66f0 148 base_source_paths = ['.']
screamer 13:ab47a20b66f0 149
screamer 13:ab47a20b66f0 150 all_tests = find_tests(base_source_paths[0])
screamer 13:ab47a20b66f0 151
screamer 13:ab47a20b66f0 152 start = time()
screamer 13:ab47a20b66f0 153 build_report = {}
screamer 13:ab47a20b66f0 154 build_properties = {}
screamer 13:ab47a20b66f0 155 test_builds = {}
screamer 13:ab47a20b66f0 156 total_build_success = True
screamer 13:ab47a20b66f0 157
screamer 13:ab47a20b66f0 158 for target_name, target_toolchains in build_config.iteritems():
screamer 13:ab47a20b66f0 159 target = TARGET_MAP[target_name]
screamer 13:ab47a20b66f0 160
screamer 13:ab47a20b66f0 161 for target_toolchain in target_toolchains:
screamer 13:ab47a20b66f0 162 library_build_success = True
screamer 13:ab47a20b66f0 163
screamer 13:ab47a20b66f0 164 try:
screamer 13:ab47a20b66f0 165 build_directory = join(options.build_dir, target_name, target_toolchain)
screamer 13:ab47a20b66f0 166 # Build sources
screamer 13:ab47a20b66f0 167 build_library(base_source_paths, build_directory, target, target_toolchain,
screamer 13:ab47a20b66f0 168 jobs=options.jobs,
screamer 13:ab47a20b66f0 169 clean=options.clean,
screamer 13:ab47a20b66f0 170 report=build_report,
screamer 13:ab47a20b66f0 171 properties=build_properties,
screamer 13:ab47a20b66f0 172 name="mbed-os",
screamer 13:ab47a20b66f0 173 macros=options.macros,
screamer 13:ab47a20b66f0 174 verbose=options.verbose,
screamer 13:ab47a20b66f0 175 archive=False)
screamer 13:ab47a20b66f0 176 except Exception, e:
screamer 13:ab47a20b66f0 177 library_build_success = False
screamer 13:ab47a20b66f0 178 print "Failed to build library"
screamer 13:ab47a20b66f0 179 print e
screamer 13:ab47a20b66f0 180
screamer 13:ab47a20b66f0 181 if options.continue_on_build_fail or library_build_success:
screamer 13:ab47a20b66f0 182 # Build all the tests
screamer 13:ab47a20b66f0 183 test_build_success, test_build = build_tests(all_tests, [build_directory], build_directory, target, target_toolchain,
screamer 13:ab47a20b66f0 184 clean=options.clean,
screamer 13:ab47a20b66f0 185 report=build_report,
screamer 13:ab47a20b66f0 186 properties=build_properties,
screamer 13:ab47a20b66f0 187 macros=options.macros,
screamer 13:ab47a20b66f0 188 verbose=options.verbose,
screamer 13:ab47a20b66f0 189 jobs=options.jobs,
screamer 13:ab47a20b66f0 190 continue_on_build_fail=options.continue_on_build_fail)
screamer 13:ab47a20b66f0 191
screamer 13:ab47a20b66f0 192 if not test_build_success:
screamer 13:ab47a20b66f0 193 total_build_success = False
screamer 13:ab47a20b66f0 194 print "Failed to build some tests, check build log for details"
screamer 13:ab47a20b66f0 195
screamer 13:ab47a20b66f0 196 test_builds.update(test_build)
screamer 13:ab47a20b66f0 197 else:
screamer 13:ab47a20b66f0 198 total_build_success = False
screamer 13:ab47a20b66f0 199 break
screamer 13:ab47a20b66f0 200
screamer 13:ab47a20b66f0 201 # If a path to a test spec is provided, write it to a file
screamer 13:ab47a20b66f0 202 if options.test_spec:
screamer 13:ab47a20b66f0 203 test_spec_data = test_spec_from_test_builds(test_builds)
screamer 13:ab47a20b66f0 204
screamer 13:ab47a20b66f0 205 # Create the target dir for the test spec if necessary
screamer 13:ab47a20b66f0 206 # mkdir will not create the dir if it already exists
screamer 13:ab47a20b66f0 207 test_spec_dir = dirname(options.test_spec)
screamer 13:ab47a20b66f0 208 if test_spec_dir:
screamer 13:ab47a20b66f0 209 mkdir(test_spec_dir)
screamer 13:ab47a20b66f0 210
screamer 13:ab47a20b66f0 211 try:
screamer 13:ab47a20b66f0 212 with open(options.test_spec, 'w') as f:
screamer 13:ab47a20b66f0 213 f.write(json.dumps(test_spec_data, indent=2))
screamer 13:ab47a20b66f0 214 except IOError, e:
screamer 13:ab47a20b66f0 215 print "[ERROR] Error writing test spec to file"
screamer 13:ab47a20b66f0 216 print e
screamer 13:ab47a20b66f0 217
screamer 13:ab47a20b66f0 218 # If a path to a JUnit build report spec is provided, write it to a file
screamer 13:ab47a20b66f0 219 if options.build_report_junit:
screamer 13:ab47a20b66f0 220 report_exporter = ReportExporter(ResultExporterType.JUNIT)
screamer 13:ab47a20b66f0 221 report_exporter.report_to_file(build_report, options.build_report_junit, test_suite_properties=build_properties)
screamer 13:ab47a20b66f0 222
screamer 13:ab47a20b66f0 223 print "\n\nCompleted in: (%.2f)s" % (time() - start)
screamer 13:ab47a20b66f0 224
screamer 13:ab47a20b66f0 225 print_report_exporter = ReportExporter(ResultExporterType.PRINT, package="build")
screamer 13:ab47a20b66f0 226 status = print_report_exporter.report(build_report)
screamer 13:ab47a20b66f0 227
screamer 13:ab47a20b66f0 228 if status:
screamer 13:ab47a20b66f0 229 sys.exit(0)
screamer 13:ab47a20b66f0 230 else:
screamer 13:ab47a20b66f0 231 sys.exit(1)
screamer 13:ab47a20b66f0 232
screamer 13:ab47a20b66f0 233 except KeyboardInterrupt, e:
screamer 13:ab47a20b66f0 234 print "\n[CTRL+c] exit"
screamer 13:ab47a20b66f0 235 except Exception,e:
screamer 13:ab47a20b66f0 236 import traceback
screamer 13:ab47a20b66f0 237 traceback.print_exc(file=sys.stdout)
screamer 13:ab47a20b66f0 238 print "[ERROR] %s" % str(e)
screamer 13:ab47a20b66f0 239 sys.exit(1)