EL4121 Embedded System / mbed-os

mbed-os

Dependents:   cobaLCDJoyMotor_Thread odometry_omni_3roda_v3 odometry_omni_3roda_v1 odometry_omni_3roda_v2 ... more

tools/check_release.py@0:b74591d5ab33, 2017-12-11 (annotated)

Committer:
be_bryan
Date:
Mon Dec 11 17:54:04 2017 +0000
Revision:
0:b74591d5ab33
motor ++

Who changed what in which revision?

UserRevisionLine numberNew contents of line
be_bryan 0:b74591d5ab33 1 # Script to check a new mbed 2 release by compiling a set of specified test apps
be_bryan 0:b74591d5ab33 2 # for all currently supported platforms. Each test app must include an mbed library.
be_bryan 0:b74591d5ab33 3 # This can either be the pre-compiled version 'mbed' or the source version 'mbed-dev'.
be_bryan 0:b74591d5ab33 4 #
be_bryan 0:b74591d5ab33 5 # Setup:
be_bryan 0:b74591d5ab33 6 # 1. Set up your global .hgrc file
be_bryan 0:b74591d5ab33 7 #
be_bryan 0:b74591d5ab33 8 # If you don't already have a .hgrc file in your $HOME directory, create one there.
be_bryan 0:b74591d5ab33 9 # Then add the following section:
be_bryan 0:b74591d5ab33 10 #
be_bryan 0:b74591d5ab33 11 # [auth]
be_bryan 0:b74591d5ab33 12 # x.prefix = *
be_bryan 0:b74591d5ab33 13 # x.username = <put your mbed org username here>
be_bryan 0:b74591d5ab33 14 # x.password = <put your mbed org password here>
be_bryan 0:b74591d5ab33 15 #
be_bryan 0:b74591d5ab33 16 # This has 2 purposes, the first means you don't get prompted for your password
be_bryan 0:b74591d5ab33 17 # whenever you run hg commands on the commandline. The second is that this script
be_bryan 0:b74591d5ab33 18 # reads these details in order to fully automate the Mercurial commands.
be_bryan 0:b74591d5ab33 19 #
be_bryan 0:b74591d5ab33 20 # Edit "check_release.json". This has the following structure:
be_bryan 0:b74591d5ab33 21 #{
be_bryan 0:b74591d5ab33 22 # "config" : {
be_bryan 0:b74591d5ab33 23 # "mbed_repo_path" : "C:/Users/annbri01/Work/Mercurial"
be_bryan 0:b74591d5ab33 24 # },
be_bryan 0:b74591d5ab33 25 # "test_list" : [
be_bryan 0:b74591d5ab33 26 # {
be_bryan 0:b74591d5ab33 27 # "name" : "test_compile_mbed_lib",
be_bryan 0:b74591d5ab33 28 # "lib" : "mbed"
be_bryan 0:b74591d5ab33 29 # },
be_bryan 0:b74591d5ab33 30 # {
be_bryan 0:b74591d5ab33 31 # "name" : "test_compile_mbed_dev",
be_bryan 0:b74591d5ab33 32 # "lib" : "mbed-dev"
be_bryan 0:b74591d5ab33 33 # }
be_bryan 0:b74591d5ab33 34 # ],
be_bryan 0:b74591d5ab33 35 # "target_list" : []
be_bryan 0:b74591d5ab33 36 #}
be_bryan 0:b74591d5ab33 37 #
be_bryan 0:b74591d5ab33 38 # The mbed_repo_path field should be changed to point to where your local
be_bryan 0:b74591d5ab33 39 # working directory is for Mercurial repositories.
be_bryan 0:b74591d5ab33 40 # For each test app you wish to run, add an entry to the test list. The example
be_bryan 0:b74591d5ab33 41 # above has 2 test apps
be_bryan 0:b74591d5ab33 42 # "test_compile_mbed_lib" and "test_compile_mbed_dev"
be_bryan 0:b74591d5ab33 43 # The lib field in each says which type of mbed 2 library the app contains.
be_bryan 0:b74591d5ab33 44 # These test apps MUST be available as repos in the user's online Mercurial area.
be_bryan 0:b74591d5ab33 45 # The target_list allows the user to override the set of targets/platforms used
be_bryan 0:b74591d5ab33 46 # for the compilation.
be_bryan 0:b74591d5ab33 47 # E.g to just compile for 2 targets, K64F and K22F :
be_bryan 0:b74591d5ab33 48 # "target_list" : ["K64F", "K22F"]
be_bryan 0:b74591d5ab33 49 #
be_bryan 0:b74591d5ab33 50 # Run the script from the mbed-os directory as follows:
be_bryan 0:b74591d5ab33 51 # > python tools/check_release.py
be_bryan 0:b74591d5ab33 52 #
be_bryan 0:b74591d5ab33 53 # It will look for local clones of the test app repos. If they don't exist
be_bryan 0:b74591d5ab33 54 # it will clone them. It will then read the latest versions of mbed and mbed-dev
be_bryan 0:b74591d5ab33 55 # (an assumption is made that both of these are already cloned in your Mercurial area).
be_bryan 0:b74591d5ab33 56 # The lib files within the test apps are then updated to the corresponding version in
be_bryan 0:b74591d5ab33 57 # the associated lib itself. The test apps are then committed and pushed back to the users
be_bryan 0:b74591d5ab33 58 # fork.
be_bryan 0:b74591d5ab33 59 # The test apps will then be compiled for all supported targets and a % result output at
be_bryan 0:b74591d5ab33 60 # the end.
be_bryan 0:b74591d5ab33 61 #
be_bryan 0:b74591d5ab33 62 # Uses the online compiler API at https://mbed.org/handbook/Compile-API
be_bryan 0:b74591d5ab33 63 # Based on the example from https://mbed.org/teams/mbed/code/mbed-API-helper/
be_bryan 0:b74591d5ab33 64
be_bryan 0:b74591d5ab33 65
be_bryan 0:b74591d5ab33 66 import os, getpass, sys, json, time, requests, logging
be_bryan 0:b74591d5ab33 67 from os.path import dirname, abspath, basename, join
be_bryan 0:b74591d5ab33 68 import argparse
be_bryan 0:b74591d5ab33 69 import subprocess
be_bryan 0:b74591d5ab33 70 import re
be_bryan 0:b74591d5ab33 71 import hglib
be_bryan 0:b74591d5ab33 72 import argparse
be_bryan 0:b74591d5ab33 73
be_bryan 0:b74591d5ab33 74 # Be sure that the tools directory is in the search path
be_bryan 0:b74591d5ab33 75 ROOT = abspath(join(dirname(__file__), ".."))
be_bryan 0:b74591d5ab33 76 sys.path.insert(0, ROOT)
be_bryan 0:b74591d5ab33 77
be_bryan 0:b74591d5ab33 78 from tools.build_api import get_mbed_official_release
be_bryan 0:b74591d5ab33 79
be_bryan 0:b74591d5ab33 80 OFFICIAL_MBED_LIBRARY_BUILD = get_mbed_official_release('2')
be_bryan 0:b74591d5ab33 81
be_bryan 0:b74591d5ab33 82 def get_compilation_failure(messages):
be_bryan 0:b74591d5ab33 83 """ Reads the json formatted 'messages' and checks for compilation errors.
be_bryan 0:b74591d5ab33 84 If there is a genuine compilation error then there should be a new
be_bryan 0:b74591d5ab33 85 message containing a severity field = Error and an accompanying message
be_bryan 0:b74591d5ab33 86 with the compile error text. Any other combination is considered an
be_bryan 0:b74591d5ab33 87 internal compile engine failure
be_bryan 0:b74591d5ab33 88 Args:
be_bryan 0:b74591d5ab33 89 messages - json formatted text returned by the online compiler API.
be_bryan 0:b74591d5ab33 90
be_bryan 0:b74591d5ab33 91 Returns:
be_bryan 0:b74591d5ab33 92 Either "Error" or "Internal" to indicate an actual compilation error or an
be_bryan 0:b74591d5ab33 93 internal IDE API fault.
be_bryan 0:b74591d5ab33 94
be_bryan 0:b74591d5ab33 95 """
be_bryan 0:b74591d5ab33 96 for m in messages:
be_bryan 0:b74591d5ab33 97 # Get message text if it exists
be_bryan 0:b74591d5ab33 98 try:
be_bryan 0:b74591d5ab33 99 message = m['message']
be_bryan 0:b74591d5ab33 100 message = message + "\n"
be_bryan 0:b74591d5ab33 101 except KeyError:
be_bryan 0:b74591d5ab33 102 # Skip this message as it has no 'message' field
be_bryan 0:b74591d5ab33 103 continue
be_bryan 0:b74591d5ab33 104
be_bryan 0:b74591d5ab33 105 # Get type of message text
be_bryan 0:b74591d5ab33 106 try:
be_bryan 0:b74591d5ab33 107 msg_type = m['type']
be_bryan 0:b74591d5ab33 108 except KeyError:
be_bryan 0:b74591d5ab33 109 # Skip this message as it has no 'type' field
be_bryan 0:b74591d5ab33 110 continue
be_bryan 0:b74591d5ab33 111
be_bryan 0:b74591d5ab33 112 if msg_type == 'error' or msg_type == 'tool_error':
be_bryan 0:b74591d5ab33 113 rel_log.error(message)
be_bryan 0:b74591d5ab33 114 return "Error"
be_bryan 0:b74591d5ab33 115 else:
be_bryan 0:b74591d5ab33 116 rel_log.debug(message)
be_bryan 0:b74591d5ab33 117
be_bryan 0:b74591d5ab33 118 return "Internal"
be_bryan 0:b74591d5ab33 119
be_bryan 0:b74591d5ab33 120 def invoke_api(payload, url, auth, polls, begin="start/"):
be_bryan 0:b74591d5ab33 121 """ Sends an API command request to the online IDE. Waits for a task completed
be_bryan 0:b74591d5ab33 122 response before returning the results.
be_bryan 0:b74591d5ab33 123
be_bryan 0:b74591d5ab33 124 Args:
be_bryan 0:b74591d5ab33 125 payload - Configuration parameters to be passed to the API
be_bryan 0:b74591d5ab33 126 url - THe URL for the online compiler API
be_bryan 0:b74591d5ab33 127 auth - Tuple containing authentication credentials
be_bryan 0:b74591d5ab33 128 polls - Number of times to poll for results
be_bryan 0:b74591d5ab33 129 begin - Default value = "start/", start command to be appended to URL
be_bryan 0:b74591d5ab33 130
be_bryan 0:b74591d5ab33 131 Returns:
be_bryan 0:b74591d5ab33 132 result - True/False indicating the success/failure of the compilation
be_bryan 0:b74591d5ab33 133 fail_type - the failure text if the compilation failed, else None
be_bryan 0:b74591d5ab33 134 """
be_bryan 0:b74591d5ab33 135
be_bryan 0:b74591d5ab33 136 # send task to api
be_bryan 0:b74591d5ab33 137 rel_log.debug(url + begin + "| data: " + str(payload))
be_bryan 0:b74591d5ab33 138 r = requests.post(url + begin, data=payload, auth=auth)
be_bryan 0:b74591d5ab33 139 rel_log.debug(r.request.body)
be_bryan 0:b74591d5ab33 140
be_bryan 0:b74591d5ab33 141 if r.status_code != 200:
be_bryan 0:b74591d5ab33 142 rel_log.error("HTTP code %d reported.", r.status_code)
be_bryan 0:b74591d5ab33 143 return False, "Internal"
be_bryan 0:b74591d5ab33 144
be_bryan 0:b74591d5ab33 145 response = r.json()
be_bryan 0:b74591d5ab33 146 rel_log.debug(response)
be_bryan 0:b74591d5ab33 147 uuid = response['result']['data']['task_id']
be_bryan 0:b74591d5ab33 148 rel_log.debug("Task accepted and given ID: %s", uuid)
be_bryan 0:b74591d5ab33 149 result = False
be_bryan 0:b74591d5ab33 150 fail_type = None
be_bryan 0:b74591d5ab33 151
be_bryan 0:b74591d5ab33 152 # It currently seems to take the onlide IDE API ~30s to process the compile
be_bryan 0:b74591d5ab33 153 # request and provide a response. Set the poll time to half that in case it
be_bryan 0:b74591d5ab33 154 # does manage to compile quicker.
be_bryan 0:b74591d5ab33 155 poll_delay = 15
be_bryan 0:b74591d5ab33 156 rel_log.debug("Running with a poll for response delay of: %ss", poll_delay)
be_bryan 0:b74591d5ab33 157
be_bryan 0:b74591d5ab33 158 # poll for output
be_bryan 0:b74591d5ab33 159 for check in range(polls):
be_bryan 0:b74591d5ab33 160 time.sleep(poll_delay)
be_bryan 0:b74591d5ab33 161
be_bryan 0:b74591d5ab33 162 try:
be_bryan 0:b74591d5ab33 163 r = requests.get(url + "output/%s" % uuid, auth=auth)
be_bryan 0:b74591d5ab33 164
be_bryan 0:b74591d5ab33 165 except ConnectionError:
be_bryan 0:b74591d5ab33 166 return "Internal"
be_bryan 0:b74591d5ab33 167
be_bryan 0:b74591d5ab33 168 response = r.json()
be_bryan 0:b74591d5ab33 169
be_bryan 0:b74591d5ab33 170 data = response['result']['data']
be_bryan 0:b74591d5ab33 171 if data['task_complete']:
be_bryan 0:b74591d5ab33 172 # Task completed. Now determine the result. Should be one of :
be_bryan 0:b74591d5ab33 173 # 1) Successful compilation
be_bryan 0:b74591d5ab33 174 # 2) Failed compilation with an error message
be_bryan 0:b74591d5ab33 175 # 3) Internal failure of the online compiler
be_bryan 0:b74591d5ab33 176 result = bool(data['compilation_success'])
be_bryan 0:b74591d5ab33 177 if result:
be_bryan 0:b74591d5ab33 178 rel_log.info("COMPILATION SUCCESSFUL\n")
be_bryan 0:b74591d5ab33 179 else:
be_bryan 0:b74591d5ab33 180 # Did this fail due to a genuine compilation error or a failue of
be_bryan 0:b74591d5ab33 181 # the api itself ?
be_bryan 0:b74591d5ab33 182 rel_log.info("COMPILATION FAILURE\n")
be_bryan 0:b74591d5ab33 183 fail_type = get_compilation_failure(data['new_messages'])
be_bryan 0:b74591d5ab33 184 break
be_bryan 0:b74591d5ab33 185 else:
be_bryan 0:b74591d5ab33 186 rel_log.info("COMPILATION FAILURE\n")
be_bryan 0:b74591d5ab33 187
be_bryan 0:b74591d5ab33 188 if not result and fail_type == None:
be_bryan 0:b74591d5ab33 189 fail_type = "Internal"
be_bryan 0:b74591d5ab33 190
be_bryan 0:b74591d5ab33 191 return result, fail_type
be_bryan 0:b74591d5ab33 192
be_bryan 0:b74591d5ab33 193
be_bryan 0:b74591d5ab33 194 def build_repo(target, program, user, pw, polls=25,
be_bryan 0:b74591d5ab33 195 url="https://developer.mbed.org/api/v2/tasks/compiler/"):
be_bryan 0:b74591d5ab33 196 """ Wrapper for sending an API command request to the online IDE. Sends a
be_bryan 0:b74591d5ab33 197 build request.
be_bryan 0:b74591d5ab33 198
be_bryan 0:b74591d5ab33 199 Args:
be_bryan 0:b74591d5ab33 200 target - Target to be built
be_bryan 0:b74591d5ab33 201 program - Test program to build
be_bryan 0:b74591d5ab33 202 user - mbed username
be_bryan 0:b74591d5ab33 203 pw - mbed password
be_bryan 0:b74591d5ab33 204 polls - Number of times to poll for results
be_bryan 0:b74591d5ab33 205 url - THe URL for the online compiler API
be_bryan 0:b74591d5ab33 206
be_bryan 0:b74591d5ab33 207 Returns:
be_bryan 0:b74591d5ab33 208 result - True/False indicating the success/failure of the compilation
be_bryan 0:b74591d5ab33 209 fail_type - the failure text if the compilation failed, else None
be_bryan 0:b74591d5ab33 210 """
be_bryan 0:b74591d5ab33 211 payload = {'clean':True, 'target':target, 'program':program}
be_bryan 0:b74591d5ab33 212 auth = (user, pw)
be_bryan 0:b74591d5ab33 213 return invoke_api(payload, url, auth, polls)
be_bryan 0:b74591d5ab33 214
be_bryan 0:b74591d5ab33 215 def run_cmd(command, exit_on_failure=False):
be_bryan 0:b74591d5ab33 216 """ Passes a command to the system and returns a True/False result once the
be_bryan 0:b74591d5ab33 217 command has been executed, indicating success/failure. Commands are passed
be_bryan 0:b74591d5ab33 218 as a list of tokens.
be_bryan 0:b74591d5ab33 219 E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
be_bryan 0:b74591d5ab33 220
be_bryan 0:b74591d5ab33 221 Args:
be_bryan 0:b74591d5ab33 222 command - system command as a list of tokens
be_bryan 0:b74591d5ab33 223 exit_on_failure - If True exit the program on failure (default = False)
be_bryan 0:b74591d5ab33 224
be_bryan 0:b74591d5ab33 225 Returns:
be_bryan 0:b74591d5ab33 226 result - True/False indicating the success/failure of the command
be_bryan 0:b74591d5ab33 227 """
be_bryan 0:b74591d5ab33 228 rel_log.debug('[Exec] %s', ' '.join(command))
be_bryan 0:b74591d5ab33 229 return_code = subprocess.call(command, shell=True)
be_bryan 0:b74591d5ab33 230
be_bryan 0:b74591d5ab33 231 if return_code:
be_bryan 0:b74591d5ab33 232 rel_log.warning("The command '%s' failed with return code: %s",
be_bryan 0:b74591d5ab33 233 (' '.join(command), return_code))
be_bryan 0:b74591d5ab33 234 if exit_on_failure:
be_bryan 0:b74591d5ab33 235 sys.exit(1)
be_bryan 0:b74591d5ab33 236
be_bryan 0:b74591d5ab33 237 return return_code
be_bryan 0:b74591d5ab33 238
be_bryan 0:b74591d5ab33 239 def run_cmd_with_output(command, exit_on_failure=False):
be_bryan 0:b74591d5ab33 240 """ Passes a command to the system and returns a True/False result once the
be_bryan 0:b74591d5ab33 241 command has been executed, indicating success/failure. If the command was
be_bryan 0:b74591d5ab33 242 successful then the output from the command is returned to the caller.
be_bryan 0:b74591d5ab33 243 Commands are passed as a list of tokens.
be_bryan 0:b74591d5ab33 244 E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
be_bryan 0:b74591d5ab33 245
be_bryan 0:b74591d5ab33 246 Args:
be_bryan 0:b74591d5ab33 247 command - system command as a list of tokens
be_bryan 0:b74591d5ab33 248 exit_on_failure - If True exit the program on failure (default = False)
be_bryan 0:b74591d5ab33 249
be_bryan 0:b74591d5ab33 250 Returns:
be_bryan 0:b74591d5ab33 251 result - True/False indicating the success/failure of the command
be_bryan 0:b74591d5ab33 252 output - The output of the command if it was successful, else empty string
be_bryan 0:b74591d5ab33 253 """
be_bryan 0:b74591d5ab33 254 rel_log.debug('[Exec] %s', ' '.join(command))
be_bryan 0:b74591d5ab33 255 returncode = 0
be_bryan 0:b74591d5ab33 256 output = ""
be_bryan 0:b74591d5ab33 257 try:
be_bryan 0:b74591d5ab33 258 output = subprocess.check_output(command, shell=True)
be_bryan 0:b74591d5ab33 259 except subprocess.CalledProcessError as e:
be_bryan 0:b74591d5ab33 260 rel_log.warning("The command '%s' failed with return code: %s",
be_bryan 0:b74591d5ab33 261 (' '.join(command), e.returncode))
be_bryan 0:b74591d5ab33 262 returncode = e.returncode
be_bryan 0:b74591d5ab33 263 if exit_on_failure:
be_bryan 0:b74591d5ab33 264 sys.exit(1)
be_bryan 0:b74591d5ab33 265 return returncode, output
be_bryan 0:b74591d5ab33 266
be_bryan 0:b74591d5ab33 267 def upgrade_test_repo(test, user, library, ref, repo_path):
be_bryan 0:b74591d5ab33 268 """ Upgrades a local version of a test repo to the latest version of its
be_bryan 0:b74591d5ab33 269 embedded library.
be_bryan 0:b74591d5ab33 270 If the test repo is not present in the user area specified in the json
be_bryan 0:b74591d5ab33 271 config file, then it will first be cloned.
be_bryan 0:b74591d5ab33 272 Args:
be_bryan 0:b74591d5ab33 273 test - Mercurial test repo name
be_bryan 0:b74591d5ab33 274 user - Mercurial user name
be_bryan 0:b74591d5ab33 275 library - library name
be_bryan 0:b74591d5ab33 276 ref - SHA corresponding to the latest version of the library
be_bryan 0:b74591d5ab33 277 repo_path - path to the user's repo area
be_bryan 0:b74591d5ab33 278
be_bryan 0:b74591d5ab33 279 Returns:
be_bryan 0:b74591d5ab33 280 updated - True if library was updated, False otherwise
be_bryan 0:b74591d5ab33 281 """
be_bryan 0:b74591d5ab33 282 rel_log.info("Updating test repo: '%s' to SHA: %s", test, ref)
be_bryan 0:b74591d5ab33 283 cwd = os.getcwd()
be_bryan 0:b74591d5ab33 284
be_bryan 0:b74591d5ab33 285 repo = "https://" + user + '@developer.mbed.org/users/' + user + '/code/' + test
be_bryan 0:b74591d5ab33 286
be_bryan 0:b74591d5ab33 287 # Clone the repo if it doesn't already exist
be_bryan 0:b74591d5ab33 288 path = abspath(repo_path + '/' + test)
be_bryan 0:b74591d5ab33 289 if not os.path.exists(path):
be_bryan 0:b74591d5ab33 290 rel_log.info("Test repo doesn't exist, cloning...")
be_bryan 0:b74591d5ab33 291 os.chdir(abspath(repo_path))
be_bryan 0:b74591d5ab33 292 clone_cmd = ['hg', 'clone', repo]
be_bryan 0:b74591d5ab33 293 run_cmd(clone_cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 294
be_bryan 0:b74591d5ab33 295 os.chdir(path)
be_bryan 0:b74591d5ab33 296
be_bryan 0:b74591d5ab33 297 client = hglib.open(path)
be_bryan 0:b74591d5ab33 298
be_bryan 0:b74591d5ab33 299 lib_file = library + '.lib'
be_bryan 0:b74591d5ab33 300 if os.path.isfile(lib_file):
be_bryan 0:b74591d5ab33 301 # Rename command will fail on some OS's if the target file already exist,
be_bryan 0:b74591d5ab33 302 # so ensure if it does, it is deleted first.
be_bryan 0:b74591d5ab33 303 bak_file = library + '_bak'
be_bryan 0:b74591d5ab33 304 if os.path.isfile(bak_file):
be_bryan 0:b74591d5ab33 305 os.remove(bak_file)
be_bryan 0:b74591d5ab33 306
be_bryan 0:b74591d5ab33 307 os.rename(lib_file, bak_file)
be_bryan 0:b74591d5ab33 308 else:
be_bryan 0:b74591d5ab33 309 rel_log.error("Failure to backup lib file prior to updating.")
be_bryan 0:b74591d5ab33 310 return False
be_bryan 0:b74591d5ab33 311
be_bryan 0:b74591d5ab33 312 # mbed 2 style lib file contains one line with the following format
be_bryan 0:b74591d5ab33 313 # e.g. https://developer.mbed.org/users/<user>/code/mbed-dev/#156823d33999
be_bryan 0:b74591d5ab33 314 exp = 'https://developer.mbed.org/users/' + user + '/code/' + library + '/#[A-Za-z0-9]+'
be_bryan 0:b74591d5ab33 315 lib_re = re.compile(exp)
be_bryan 0:b74591d5ab33 316 updated = False
be_bryan 0:b74591d5ab33 317
be_bryan 0:b74591d5ab33 318 # Scan through mbed-os.lib line by line, looking for lib version and update
be_bryan 0:b74591d5ab33 319 # it if found
be_bryan 0:b74591d5ab33 320 with open(bak_file, 'r') as ip, open(lib_file, 'w') as op:
be_bryan 0:b74591d5ab33 321 for line in ip:
be_bryan 0:b74591d5ab33 322
be_bryan 0:b74591d5ab33 323 opline = line
be_bryan 0:b74591d5ab33 324
be_bryan 0:b74591d5ab33 325 regexp = lib_re.match(line)
be_bryan 0:b74591d5ab33 326 if regexp:
be_bryan 0:b74591d5ab33 327 opline = 'https://developer.mbed.org/users/' + user + '/code/' + library + '/#' + ref
be_bryan 0:b74591d5ab33 328 updated = True
be_bryan 0:b74591d5ab33 329
be_bryan 0:b74591d5ab33 330 op.write(opline)
be_bryan 0:b74591d5ab33 331
be_bryan 0:b74591d5ab33 332 if updated:
be_bryan 0:b74591d5ab33 333
be_bryan 0:b74591d5ab33 334 # Setup the default commit message
be_bryan 0:b74591d5ab33 335 commit_message = '"Updating ' + library + ' to ' + ref + '"'
be_bryan 0:b74591d5ab33 336
be_bryan 0:b74591d5ab33 337 # Setup and run the commit command. Need to use the rawcommand in the hglib
be_bryan 0:b74591d5ab33 338 # for this in order to pass the string value to the -m option. run_cmd using
be_bryan 0:b74591d5ab33 339 # subprocess does not like this syntax.
be_bryan 0:b74591d5ab33 340 try:
be_bryan 0:b74591d5ab33 341 client.rawcommand(['commit','-m '+commit_message, lib_file])
be_bryan 0:b74591d5ab33 342
be_bryan 0:b74591d5ab33 343 cmd = ['hg', 'push', '-f', repo]
be_bryan 0:b74591d5ab33 344 run_cmd(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 345
be_bryan 0:b74591d5ab33 346 except:
be_bryan 0:b74591d5ab33 347 rel_log.info("Lib file already up to date and thus nothing to commit")
be_bryan 0:b74591d5ab33 348
be_bryan 0:b74591d5ab33 349 os.chdir(cwd)
be_bryan 0:b74591d5ab33 350 return updated
be_bryan 0:b74591d5ab33 351
be_bryan 0:b74591d5ab33 352 def get_sha(repo_path, library):
be_bryan 0:b74591d5ab33 353 """ Gets the latest SHA for the library specified. The library is assumed to be
be_bryan 0:b74591d5ab33 354 located at the repo_path. If a SHA cannot be obtained this script will exit.
be_bryan 0:b74591d5ab33 355
be_bryan 0:b74591d5ab33 356 Args:
be_bryan 0:b74591d5ab33 357 library - library name
be_bryan 0:b74591d5ab33 358 repo_path - path to the user's repo area
be_bryan 0:b74591d5ab33 359
be_bryan 0:b74591d5ab33 360 Returns:
be_bryan 0:b74591d5ab33 361 sha - last commit SHA
be_bryan 0:b74591d5ab33 362 """
be_bryan 0:b74591d5ab33 363 cwd = os.getcwd()
be_bryan 0:b74591d5ab33 364 sha = None
be_bryan 0:b74591d5ab33 365 os.chdir(abspath(repo_path + '/' + library))
be_bryan 0:b74591d5ab33 366
be_bryan 0:b74591d5ab33 367 cmd = ['hg', 'log', '-l', '1']
be_bryan 0:b74591d5ab33 368 ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 369
be_bryan 0:b74591d5ab33 370 # Output should contain a 4 line string of the form:
be_bryan 0:b74591d5ab33 371 # changeset: 135:176b8275d35d
be_bryan 0:b74591d5ab33 372 # tag: tip
be_bryan 0:b74591d5ab33 373 # user: <>
be_bryan 0:b74591d5ab33 374 # date: Thu Feb 02 16:02:30 2017 +0000
be_bryan 0:b74591d5ab33 375 # summary: Release 135 of the mbed library
be_bryan 0:b74591d5ab33 376 # All we want is the changeset string after version number
be_bryan 0:b74591d5ab33 377
be_bryan 0:b74591d5ab33 378 lines = output.split('\n')
be_bryan 0:b74591d5ab33 379 fields = lines[0].split(':')
be_bryan 0:b74591d5ab33 380 sha = fields[2]
be_bryan 0:b74591d5ab33 381
be_bryan 0:b74591d5ab33 382 os.chdir(cwd)
be_bryan 0:b74591d5ab33 383 return sha
be_bryan 0:b74591d5ab33 384
be_bryan 0:b74591d5ab33 385 def get_latest_library_versions(repo_path):
be_bryan 0:b74591d5ab33 386 """ Returns the latest library versions (SHAs) for 'mbed' and 'mbed-dev'.
be_bryan 0:b74591d5ab33 387 If the SHAs cannot be obtained this script will exit.
be_bryan 0:b74591d5ab33 388
be_bryan 0:b74591d5ab33 389 Args:
be_bryan 0:b74591d5ab33 390 repo_path - path to the user's repo area
be_bryan 0:b74591d5ab33 391
be_bryan 0:b74591d5ab33 392 Returns:
be_bryan 0:b74591d5ab33 393 mbed - last commit SHA for mbed library
be_bryan 0:b74591d5ab33 394 mbed_dev - last commit SHA for mbed_dev library
be_bryan 0:b74591d5ab33 395
be_bryan 0:b74591d5ab33 396 """
be_bryan 0:b74591d5ab33 397
be_bryan 0:b74591d5ab33 398 mbed = get_sha(repo_path, 'mbed')
be_bryan 0:b74591d5ab33 399 mbed_dev = get_sha(repo_path, 'mbed-dev')
be_bryan 0:b74591d5ab33 400
be_bryan 0:b74591d5ab33 401 return mbed, mbed_dev
be_bryan 0:b74591d5ab33 402
be_bryan 0:b74591d5ab33 403 def log_results(lst, title):
be_bryan 0:b74591d5ab33 404 if len(lst) == 0:
be_bryan 0:b74591d5ab33 405 rel_log.info("%s - None", title)
be_bryan 0:b74591d5ab33 406 else:
be_bryan 0:b74591d5ab33 407 for entry in lst:
be_bryan 0:b74591d5ab33 408 rel_log.info("%s - Test: %s, Target: %s", title, entry[0], entry[1])
be_bryan 0:b74591d5ab33 409
be_bryan 0:b74591d5ab33 410
be_bryan 0:b74591d5ab33 411 if __name__ == '__main__':
be_bryan 0:b74591d5ab33 412
be_bryan 0:b74591d5ab33 413 parser = argparse.ArgumentParser(description=__doc__,
be_bryan 0:b74591d5ab33 414 formatter_class=argparse.RawDescriptionHelpFormatter)
be_bryan 0:b74591d5ab33 415 parser.add_argument('-l', '--log-level',
be_bryan 0:b74591d5ab33 416 help="Level for providing logging output",
be_bryan 0:b74591d5ab33 417 default='INFO')
be_bryan 0:b74591d5ab33 418 args = parser.parse_args()
be_bryan 0:b74591d5ab33 419
be_bryan 0:b74591d5ab33 420 default = getattr(logging, 'INFO')
be_bryan 0:b74591d5ab33 421 level = getattr(logging, args.log_level.upper(), default)
be_bryan 0:b74591d5ab33 422
be_bryan 0:b74591d5ab33 423 # Set logging level
be_bryan 0:b74591d5ab33 424 logging.basicConfig(level=level)
be_bryan 0:b74591d5ab33 425 rel_log = logging.getLogger("check-release")
be_bryan 0:b74591d5ab33 426
be_bryan 0:b74591d5ab33 427 # Read configuration data
be_bryan 0:b74591d5ab33 428 with open(os.path.join(os.path.dirname(__file__), "check_release.json")) as config:
be_bryan 0:b74591d5ab33 429 json_data = json.load(config)
be_bryan 0:b74591d5ab33 430
be_bryan 0:b74591d5ab33 431 supported_targets = []
be_bryan 0:b74591d5ab33 432
be_bryan 0:b74591d5ab33 433 if len(json_data["target_list"]) > 0:
be_bryan 0:b74591d5ab33 434 # Compile user supplied subset of targets
be_bryan 0:b74591d5ab33 435 supported_targets = json_data["target_list"]
be_bryan 0:b74591d5ab33 436 else:
be_bryan 0:b74591d5ab33 437 # Get a list of the officially supported mbed-os 2 targets
be_bryan 0:b74591d5ab33 438 for tgt in OFFICIAL_MBED_LIBRARY_BUILD:
be_bryan 0:b74591d5ab33 439 supported_targets.append(tgt[0])
be_bryan 0:b74591d5ab33 440
be_bryan 0:b74591d5ab33 441 ignore_list = []
be_bryan 0:b74591d5ab33 442
be_bryan 0:b74591d5ab33 443 if len(json_data["ignore_list"]) > 0:
be_bryan 0:b74591d5ab33 444 # List of tuples of (test, target) to be ignored in this test
be_bryan 0:b74591d5ab33 445 ignore_list = json_data["ignore_list"]
be_bryan 0:b74591d5ab33 446
be_bryan 0:b74591d5ab33 447 config = json_data["config"]
be_bryan 0:b74591d5ab33 448 test_list = json_data["test_list"]
be_bryan 0:b74591d5ab33 449 repo_path = config["mbed_repo_path"]
be_bryan 0:b74591d5ab33 450 tests = []
be_bryan 0:b74591d5ab33 451
be_bryan 0:b74591d5ab33 452 # get username
be_bryan 0:b74591d5ab33 453 cmd = ['hg', 'config', 'auth.x.username']
be_bryan 0:b74591d5ab33 454 ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 455 output = output.split('\n')
be_bryan 0:b74591d5ab33 456 user = output[0]
be_bryan 0:b74591d5ab33 457
be_bryan 0:b74591d5ab33 458 # get password
be_bryan 0:b74591d5ab33 459 cmd = ['hg', 'config', 'auth.x.password']
be_bryan 0:b74591d5ab33 460 ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 461 output = output.split('\n')
be_bryan 0:b74591d5ab33 462 password = output[0]
be_bryan 0:b74591d5ab33 463
be_bryan 0:b74591d5ab33 464 mbed, mbed_dev = get_latest_library_versions(repo_path)
be_bryan 0:b74591d5ab33 465
be_bryan 0:b74591d5ab33 466 if not mbed or not mbed_dev:
be_bryan 0:b74591d5ab33 467 rel_log.error("Could not obtain latest versions of library files!!")
be_bryan 0:b74591d5ab33 468 exit(1)
be_bryan 0:b74591d5ab33 469
be_bryan 0:b74591d5ab33 470 rel_log.info("Latest mbed lib version = %s", mbed)
be_bryan 0:b74591d5ab33 471 rel_log.info("Latest mbed-dev lib version = %s", mbed_dev)
be_bryan 0:b74591d5ab33 472
be_bryan 0:b74591d5ab33 473 # First update test repos to latest versions of their embedded libraries
be_bryan 0:b74591d5ab33 474 for test in test_list:
be_bryan 0:b74591d5ab33 475 tests.append(test['name'])
be_bryan 0:b74591d5ab33 476 upgrade_test_repo(test['name'], user, test['lib'],
be_bryan 0:b74591d5ab33 477 mbed if test['lib'] == "mbed" else mbed_dev,
be_bryan 0:b74591d5ab33 478 repo_path)
be_bryan 0:b74591d5ab33 479
be_bryan 0:b74591d5ab33 480 total = len(supported_targets) * len(tests)
be_bryan 0:b74591d5ab33 481 current = 0
be_bryan 0:b74591d5ab33 482 retries = 10
be_bryan 0:b74591d5ab33 483 passes = 0
be_bryan 0:b74591d5ab33 484 failures = []
be_bryan 0:b74591d5ab33 485 skipped = []
be_bryan 0:b74591d5ab33 486
be_bryan 0:b74591d5ab33 487 # Compile each test for each supported target
be_bryan 0:b74591d5ab33 488 for test in tests:
be_bryan 0:b74591d5ab33 489 for target in supported_targets:
be_bryan 0:b74591d5ab33 490
be_bryan 0:b74591d5ab33 491 combo = [test, target]
be_bryan 0:b74591d5ab33 492
be_bryan 0:b74591d5ab33 493 if combo in ignore_list:
be_bryan 0:b74591d5ab33 494 rel_log.info("SKIPPING TEST: %s, TARGET: %s", test, target)
be_bryan 0:b74591d5ab33 495 total -= 1
be_bryan 0:b74591d5ab33 496 skipped.append(combo)
be_bryan 0:b74591d5ab33 497 continue
be_bryan 0:b74591d5ab33 498
be_bryan 0:b74591d5ab33 499 current += 1
be_bryan 0:b74591d5ab33 500 for retry in range(0, retries):
be_bryan 0:b74591d5ab33 501 rel_log.info("COMPILING (%d/%d): TEST %s, TARGET: %s , attempt %u\n", current, total, test, target, retry)
be_bryan 0:b74591d5ab33 502 result, mesg = build_repo(target, test, user, password)
be_bryan 0:b74591d5ab33 503 if not result:
be_bryan 0:b74591d5ab33 504 if mesg == 'Internal':
be_bryan 0:b74591d5ab33 505 # Internal compiler error thus retry
be_bryan 0:b74591d5ab33 506 continue
be_bryan 0:b74591d5ab33 507 else:
be_bryan 0:b74591d5ab33 508 # Actual error thus move on to next compilation
be_bryan 0:b74591d5ab33 509 failures.append(combo)
be_bryan 0:b74591d5ab33 510 break
be_bryan 0:b74591d5ab33 511
be_bryan 0:b74591d5ab33 512 passes += (int)(result)
be_bryan 0:b74591d5ab33 513 break
be_bryan 0:b74591d5ab33 514 else:
be_bryan 0:b74591d5ab33 515 rel_log.error("Compilation failed due to internal errors.")
be_bryan 0:b74591d5ab33 516 rel_log.error("Skipping test/target combination.")
be_bryan 0:b74591d5ab33 517 total -= 1
be_bryan 0:b74591d5ab33 518 skipped.append(combo)
be_bryan 0:b74591d5ab33 519
be_bryan 0:b74591d5ab33 520 rel_log.info(" SUMMARY OF COMPILATION RESULTS")
be_bryan 0:b74591d5ab33 521 rel_log.info(" ------------------------------")
be_bryan 0:b74591d5ab33 522 rel_log.info(" NUMBER OF TEST APPS: %d, NUMBER OF TARGETS: %d",
be_bryan 0:b74591d5ab33 523 len(tests), len(supported_targets))
be_bryan 0:b74591d5ab33 524 log_results(failures, " FAILED")
be_bryan 0:b74591d5ab33 525 log_results(skipped, " SKIPPED")
be_bryan 0:b74591d5ab33 526
be_bryan 0:b74591d5ab33 527 # Output a % pass rate, indicate a failure if not 100% successful
be_bryan 0:b74591d5ab33 528 pass_rate = (float(passes) / float(total)) * 100.0
be_bryan 0:b74591d5ab33 529 rel_log.info(" PASS RATE %.1f %%\n", pass_rate)
be_bryan 0:b74591d5ab33 530 sys.exit(not (pass_rate == 100))