BA / Mbed OS BaBoRo_test2

Backup 1

mbed-os/tools/test/examples/update.py@0:02dd72d1d465, 2018-04-24 (annotated)

Committer:
borlanic
Date:
Tue Apr 24 11:45:18 2018 +0000
Revision:
0:02dd72d1d465
BaBoRo_test2 - backup 1

Who changed what in which revision?

UserRevisionLine numberNew contents of line
borlanic 0:02dd72d1d465 1 #!/usr/bin/env python
borlanic 0:02dd72d1d465 2
borlanic 0:02dd72d1d465 3 # This script is used to update the version of mbed-os used within a specified set of example
borlanic 0:02dd72d1d465 4 # applications. The list of examples to be updated lives in the examples.json file and is
borlanic 0:02dd72d1d465 5 # shared with the examples.py script. Logging is used to provide varying levels of output
borlanic 0:02dd72d1d465 6 # during execution.
borlanic 0:02dd72d1d465 7 #
borlanic 0:02dd72d1d465 8 # There are two modes that can be used:
borlanic 0:02dd72d1d465 9 # 1) Update the ARMmbed/master branch of the specified example
borlanic 0:02dd72d1d465 10 #
borlanic 0:02dd72d1d465 11 # This is done by updating a user fork of the example and then raising a pull request
borlanic 0:02dd72d1d465 12 # against ARMmbed/master.
borlanic 0:02dd72d1d465 13 #
borlanic 0:02dd72d1d465 14 # 2) Update a different ARMmbed branch of the specified example
borlanic 0:02dd72d1d465 15 #
borlanic 0:02dd72d1d465 16 # A branch to update is specified. If it doesn't already exist then it is first created.
borlanic 0:02dd72d1d465 17 # This branch will be updated and the change automatically pushed. The new branch will
borlanic 0:02dd72d1d465 18 # be created from the specified source branch.
borlanic 0:02dd72d1d465 19 #
borlanic 0:02dd72d1d465 20 # The modes are controlled via configuration data in the json file.
borlanic 0:02dd72d1d465 21 # E.g.
borlanic 0:02dd72d1d465 22 #
borlanic 0:02dd72d1d465 23 # "update-config" : {
borlanic 0:02dd72d1d465 24 # "help" : "Update each example repo with a version of mbed-os identified by the tag",
borlanic 0:02dd72d1d465 25 # "via-fork" : {
borlanic 0:02dd72d1d465 26 # "help" : "-f cmd line option. Update a fork",
borlanic 0:02dd72d1d465 27 # "github-user" : "adbridge"
borlanic 0:02dd72d1d465 28 # },
borlanic 0:02dd72d1d465 29 # "via-branch" : {
borlanic 0:02dd72d1d465 30 # "help" : "-b cmd line option. Update dst branch, created from src branch",
borlanic 0:02dd72d1d465 31 # "src-branch" : "mbed-os-5.5.0-rc1-oob",
borlanic 0:02dd72d1d465 32 # "dst-branch" : "mbed-os-5.5.0-rc2-oob"
borlanic 0:02dd72d1d465 33 # },
borlanic 0:02dd72d1d465 34 # "tag" : "mbed-os-5.5.0-rc2"
borlanic 0:02dd72d1d465 35 #
borlanic 0:02dd72d1d465 36 #
borlanic 0:02dd72d1d465 37 # Command usage:
borlanic 0:02dd72d1d465 38 #
borlanic 0:02dd72d1d465 39 # update.py -c <config file> - T <github_token> -f -b -s
borlanic 0:02dd72d1d465 40 #
borlanic 0:02dd72d1d465 41 # Where:
borlanic 0:02dd72d1d465 42 # -c <config file> - Optional path to an examples file.
borlanic 0:02dd72d1d465 43 # If not proved the default is 'examples.json'
borlanic 0:02dd72d1d465 44 # -T <github_token> - GitHub token for secure access (required)
borlanic 0:02dd72d1d465 45 # -f - Update forked repos. This will use the 'github-user' parameter in
borlanic 0:02dd72d1d465 46 # the 'via-fork' section.
borlanic 0:02dd72d1d465 47 # -b - Update branched repos. This will use the "src-branch" and
borlanic 0:02dd72d1d465 48 # "dst-branch" parameters in the 'via-branch' section. The destination
borlanic 0:02dd72d1d465 49 # branch is created from the source branch (if it doesn't already exist).
borlanic 0:02dd72d1d465 50 # -s - Show the status of any pull requests with a tag matching that in the
borlanic 0:02dd72d1d465 51 # json config file
borlanic 0:02dd72d1d465 52 #
borlanic 0:02dd72d1d465 53 # The options -f, -b and -s are mutually exlusive. Only one can be specified.
borlanic 0:02dd72d1d465 54 #
borlanic 0:02dd72d1d465 55 #
borlanic 0:02dd72d1d465 56
borlanic 0:02dd72d1d465 57 import os
borlanic 0:02dd72d1d465 58 from os.path import dirname, abspath, basename, join
borlanic 0:02dd72d1d465 59 import sys
borlanic 0:02dd72d1d465 60 import logging
borlanic 0:02dd72d1d465 61 import argparse
borlanic 0:02dd72d1d465 62 import json
borlanic 0:02dd72d1d465 63 import subprocess
borlanic 0:02dd72d1d465 64 import shutil
borlanic 0:02dd72d1d465 65 import stat
borlanic 0:02dd72d1d465 66 import re
borlanic 0:02dd72d1d465 67 from github import Github, GithubException
borlanic 0:02dd72d1d465 68 from jinja2 import FileSystemLoader, StrictUndefined
borlanic 0:02dd72d1d465 69 from jinja2.environment import Environment
borlanic 0:02dd72d1d465 70
borlanic 0:02dd72d1d465 71 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
borlanic 0:02dd72d1d465 72 sys.path.insert(0, ROOT)
borlanic 0:02dd72d1d465 73
borlanic 0:02dd72d1d465 74 import examples_lib as lib
borlanic 0:02dd72d1d465 75 from examples_lib import SUPPORTED_TOOLCHAINS
borlanic 0:02dd72d1d465 76
borlanic 0:02dd72d1d465 77 userlog = logging.getLogger("Update")
borlanic 0:02dd72d1d465 78
borlanic 0:02dd72d1d465 79 # Set logging level
borlanic 0:02dd72d1d465 80 userlog.setLevel(logging.DEBUG)
borlanic 0:02dd72d1d465 81
borlanic 0:02dd72d1d465 82 # Everything is output to the log file
borlanic 0:02dd72d1d465 83 logfile = os.path.join(os.getcwd(), 'update.log')
borlanic 0:02dd72d1d465 84 fh = logging.FileHandler(logfile)
borlanic 0:02dd72d1d465 85 fh.setLevel(logging.DEBUG)
borlanic 0:02dd72d1d465 86
borlanic 0:02dd72d1d465 87 # create console handler with a higher log level
borlanic 0:02dd72d1d465 88 ch = logging.StreamHandler()
borlanic 0:02dd72d1d465 89 ch.setLevel(logging.INFO)
borlanic 0:02dd72d1d465 90
borlanic 0:02dd72d1d465 91 formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
borlanic 0:02dd72d1d465 92 ch.setFormatter(formatter)
borlanic 0:02dd72d1d465 93 fh.setFormatter(formatter)
borlanic 0:02dd72d1d465 94
borlanic 0:02dd72d1d465 95 # add the handlers to the logger
borlanic 0:02dd72d1d465 96 userlog.addHandler(fh)
borlanic 0:02dd72d1d465 97 userlog.addHandler(ch)
borlanic 0:02dd72d1d465 98
borlanic 0:02dd72d1d465 99 def run_cmd(command, exit_on_failure=False):
borlanic 0:02dd72d1d465 100 """ Run a system command returning a status result
borlanic 0:02dd72d1d465 101
borlanic 0:02dd72d1d465 102 This is just a wrapper for the run_cmd_with_output() function, but
borlanic 0:02dd72d1d465 103 only returns the status of the call.
borlanic 0:02dd72d1d465 104
borlanic 0:02dd72d1d465 105 Args:
borlanic 0:02dd72d1d465 106 command - system command as a list of tokens
borlanic 0:02dd72d1d465 107 exit_on_failure - If True exit the program on failure (default = False)
borlanic 0:02dd72d1d465 108
borlanic 0:02dd72d1d465 109 Returns:
borlanic 0:02dd72d1d465 110 return_code - True/False indicating the success/failure of the command
borlanic 0:02dd72d1d465 111 """
borlanic 0:02dd72d1d465 112 return_code, _ = run_cmd_with_output(command, exit_on_failure)
borlanic 0:02dd72d1d465 113 return return_code
borlanic 0:02dd72d1d465 114
borlanic 0:02dd72d1d465 115 def run_cmd_with_output(command, exit_on_failure=False):
borlanic 0:02dd72d1d465 116 """ Run a system command returning a status result and any command output
borlanic 0:02dd72d1d465 117
borlanic 0:02dd72d1d465 118 Passes a command to the system and returns a True/False result once the
borlanic 0:02dd72d1d465 119 command has been executed, indicating success/failure. If the command was
borlanic 0:02dd72d1d465 120 successful then the output from the command is returned to the caller.
borlanic 0:02dd72d1d465 121 Commands are passed as a string.
borlanic 0:02dd72d1d465 122 E.g. The command 'git remote -v' would be passed in as "git remote -v"
borlanic 0:02dd72d1d465 123
borlanic 0:02dd72d1d465 124 Args:
borlanic 0:02dd72d1d465 125 command - system command as a string
borlanic 0:02dd72d1d465 126 exit_on_failure - If True exit the program on failure (default = False)
borlanic 0:02dd72d1d465 127
borlanic 0:02dd72d1d465 128 Returns:
borlanic 0:02dd72d1d465 129 return_code - True/False indicating the success/failure of the command
borlanic 0:02dd72d1d465 130 output - The output of the command if it was successful, else empty string
borlanic 0:02dd72d1d465 131 """
borlanic 0:02dd72d1d465 132 text = '[Exec] ' + command
borlanic 0:02dd72d1d465 133 userlog.debug(text)
borlanic 0:02dd72d1d465 134 returncode = 0
borlanic 0:02dd72d1d465 135 output = ""
borlanic 0:02dd72d1d465 136 try:
borlanic 0:02dd72d1d465 137 output = subprocess.check_output(command, shell=True)
borlanic 0:02dd72d1d465 138 except subprocess.CalledProcessError as e:
borlanic 0:02dd72d1d465 139 text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
borlanic 0:02dd72d1d465 140 userlog.warning(text)
borlanic 0:02dd72d1d465 141 returncode = e.returncode
borlanic 0:02dd72d1d465 142 if exit_on_failure:
borlanic 0:02dd72d1d465 143 sys.exit(1)
borlanic 0:02dd72d1d465 144 return returncode, output
borlanic 0:02dd72d1d465 145
borlanic 0:02dd72d1d465 146
borlanic 0:02dd72d1d465 147 def rmtree_readonly(directory):
borlanic 0:02dd72d1d465 148 """ Deletes a readonly directory tree.
borlanic 0:02dd72d1d465 149
borlanic 0:02dd72d1d465 150 Args:
borlanic 0:02dd72d1d465 151 directory - tree to delete
borlanic 0:02dd72d1d465 152 """
borlanic 0:02dd72d1d465 153 def remove_readonly(func, path, _):
borlanic 0:02dd72d1d465 154 os.chmod(path, stat.S_IWRITE)
borlanic 0:02dd72d1d465 155 func(path)
borlanic 0:02dd72d1d465 156
borlanic 0:02dd72d1d465 157 shutil.rmtree(directory, onerror=remove_readonly)
borlanic 0:02dd72d1d465 158
borlanic 0:02dd72d1d465 159 def find_all_examples(path):
borlanic 0:02dd72d1d465 160 """ Search the path for examples
borlanic 0:02dd72d1d465 161
borlanic 0:02dd72d1d465 162 Description:
borlanic 0:02dd72d1d465 163
borlanic 0:02dd72d1d465 164 Searches the path specified for sub-example folders, ie those containing an
borlanic 0:02dd72d1d465 165 mbed-os.lib file. If found adds the path to the sub-example to a list which is
borlanic 0:02dd72d1d465 166 then returned.
borlanic 0:02dd72d1d465 167
borlanic 0:02dd72d1d465 168 Args:
borlanic 0:02dd72d1d465 169 path - path to search.
borlanic 0:02dd72d1d465 170 examples - (returned) list of paths to example directories.
borlanic 0:02dd72d1d465 171
borlanic 0:02dd72d1d465 172 """
borlanic 0:02dd72d1d465 173 examples = []
borlanic 0:02dd72d1d465 174 for root, dirs, files in os.walk(path):
borlanic 0:02dd72d1d465 175 if 'mbed-os.lib' in files:
borlanic 0:02dd72d1d465 176 examples += [root]
borlanic 0:02dd72d1d465 177
borlanic 0:02dd72d1d465 178 return examples
borlanic 0:02dd72d1d465 179
borlanic 0:02dd72d1d465 180 def upgrade_single_example(example, tag, directory, ref):
borlanic 0:02dd72d1d465 181 """ Update the mbed-os version for a single example
borlanic 0:02dd72d1d465 182
borlanic 0:02dd72d1d465 183 Description:
borlanic 0:02dd72d1d465 184
borlanic 0:02dd72d1d465 185 Updates the mbed-os.lib file in the example specified to correspond to the
borlanic 0:02dd72d1d465 186 version specified by the GitHub tag supplied. Also deals with
borlanic 0:02dd72d1d465 187 multiple sub-examples in the GitHub repo, updating them in the same way.
borlanic 0:02dd72d1d465 188
borlanic 0:02dd72d1d465 189 Args:
borlanic 0:02dd72d1d465 190 example - json example object containing the GitHub repo to update.
borlanic 0:02dd72d1d465 191 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
borlanic 0:02dd72d1d465 192 directory - directory path for the example.
borlanic 0:02dd72d1d465 193 ref - SHA corresponding to the supplied tag
borlanic 0:02dd72d1d465 194 returns - True if the upgrade was successful, False otherwise.
borlanic 0:02dd72d1d465 195
borlanic 0:02dd72d1d465 196 """
borlanic 0:02dd72d1d465 197 cwd = os.getcwd()
borlanic 0:02dd72d1d465 198 os.chdir(directory)
borlanic 0:02dd72d1d465 199
borlanic 0:02dd72d1d465 200 return_code = False
borlanic 0:02dd72d1d465 201
borlanic 0:02dd72d1d465 202 if os.path.isfile("mbed-os.lib"):
borlanic 0:02dd72d1d465 203 # Rename command will fail on some OS's if the target file already exist,
borlanic 0:02dd72d1d465 204 # so ensure if it does, it is deleted first.
borlanic 0:02dd72d1d465 205 if os.path.isfile("mbed-os.lib_bak"):
borlanic 0:02dd72d1d465 206 os.remove("mbed-os.lib_bak")
borlanic 0:02dd72d1d465 207
borlanic 0:02dd72d1d465 208 os.rename("mbed-os.lib", "mbed-os.lib_bak")
borlanic 0:02dd72d1d465 209 else:
borlanic 0:02dd72d1d465 210 userlog.error("Failed to backup mbed-os.lib prior to updating.")
borlanic 0:02dd72d1d465 211 return False
borlanic 0:02dd72d1d465 212
borlanic 0:02dd72d1d465 213 # mbed-os.lib file contains one line with the following format
borlanic 0:02dd72d1d465 214 # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
borlanic 0:02dd72d1d465 215 lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
borlanic 0:02dd72d1d465 216 updated = False
borlanic 0:02dd72d1d465 217
borlanic 0:02dd72d1d465 218 # Scan through mbed-os.lib line by line
borlanic 0:02dd72d1d465 219 with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
borlanic 0:02dd72d1d465 220 for line in ip:
borlanic 0:02dd72d1d465 221
borlanic 0:02dd72d1d465 222 opline = line
borlanic 0:02dd72d1d465 223
borlanic 0:02dd72d1d465 224 regexp = lib_re.match(line)
borlanic 0:02dd72d1d465 225 if regexp:
borlanic 0:02dd72d1d465 226 opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
borlanic 0:02dd72d1d465 227 updated = True
borlanic 0:02dd72d1d465 228
borlanic 0:02dd72d1d465 229 op.write(opline)
borlanic 0:02dd72d1d465 230
borlanic 0:02dd72d1d465 231 if updated:
borlanic 0:02dd72d1d465 232 # Setup and run the git add command
borlanic 0:02dd72d1d465 233 cmd = "git add mbed-os.lib"
borlanic 0:02dd72d1d465 234 return_code = run_cmd(cmd)
borlanic 0:02dd72d1d465 235
borlanic 0:02dd72d1d465 236 os.chdir(cwd)
borlanic 0:02dd72d1d465 237 return not return_code
borlanic 0:02dd72d1d465 238
borlanic 0:02dd72d1d465 239 def prepare_fork(arm_example):
borlanic 0:02dd72d1d465 240 """ Synchronises a cloned fork to ensure it is up to date with the original.
borlanic 0:02dd72d1d465 241
borlanic 0:02dd72d1d465 242 Description:
borlanic 0:02dd72d1d465 243
borlanic 0:02dd72d1d465 244 This function sets a fork of an ARMmbed repo to be up to date with the
borlanic 0:02dd72d1d465 245 repo it was forked from. It does this by hard resetting to the ARMmbed
borlanic 0:02dd72d1d465 246 master branch.
borlanic 0:02dd72d1d465 247
borlanic 0:02dd72d1d465 248 Args:
borlanic 0:02dd72d1d465 249 arm_example - Full GitHub repo path for original example
borlanic 0:02dd72d1d465 250
borlanic 0:02dd72d1d465 251 """
borlanic 0:02dd72d1d465 252
borlanic 0:02dd72d1d465 253 logstr = "In: " + os.getcwd()
borlanic 0:02dd72d1d465 254 userlog.debug(logstr)
borlanic 0:02dd72d1d465 255
borlanic 0:02dd72d1d465 256 for cmd in ["git remote add armmbed " + str(arm_example),
borlanic 0:02dd72d1d465 257 "git fetch armmbed",
borlanic 0:02dd72d1d465 258 "git reset --hard armmbed/master",
borlanic 0:02dd72d1d465 259 "git push -f origin"]:
borlanic 0:02dd72d1d465 260 run_cmd(cmd, exit_on_failure=True)
borlanic 0:02dd72d1d465 261
borlanic 0:02dd72d1d465 262 def prepare_branch(src, dst):
borlanic 0:02dd72d1d465 263 """ Set up at branch ready for use in updating examples
borlanic 0:02dd72d1d465 264
borlanic 0:02dd72d1d465 265 Description:
borlanic 0:02dd72d1d465 266
borlanic 0:02dd72d1d465 267 This function checks whether or not the supplied dst branch exists.
borlanic 0:02dd72d1d465 268 If it does not, the branch is created from the src and pushed to the origin.
borlanic 0:02dd72d1d465 269 The branch is then switched to.
borlanic 0:02dd72d1d465 270
borlanic 0:02dd72d1d465 271 Args:
borlanic 0:02dd72d1d465 272 src - branch to create the dst branch from
borlanic 0:02dd72d1d465 273 dst - branch to update
borlanic 0:02dd72d1d465 274
borlanic 0:02dd72d1d465 275 """
borlanic 0:02dd72d1d465 276
borlanic 0:02dd72d1d465 277 userlog.debug("Preparing branch: %s", dst)
borlanic 0:02dd72d1d465 278
borlanic 0:02dd72d1d465 279 # Check if branch already exists or not.
borlanic 0:02dd72d1d465 280 # We can use the 'git branch -r' command. This returns all the remote branches for
borlanic 0:02dd72d1d465 281 # the current repo.
borlanic 0:02dd72d1d465 282 # The output consists of a list of lines of the form:
borlanic 0:02dd72d1d465 283 # origin/<branch>
borlanic 0:02dd72d1d465 284 # From these we need to extract just the branch names to a list and then check if
borlanic 0:02dd72d1d465 285 # the specified dst exists in that list
borlanic 0:02dd72d1d465 286 branches = []
borlanic 0:02dd72d1d465 287 cmd = "git branch -r"
borlanic 0:02dd72d1d465 288 _, output = run_cmd_with_output(cmd, exit_on_failure=True)
borlanic 0:02dd72d1d465 289
borlanic 0:02dd72d1d465 290 branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]
borlanic 0:02dd72d1d465 291
borlanic 0:02dd72d1d465 292 if not dst in branches:
borlanic 0:02dd72d1d465 293
borlanic 0:02dd72d1d465 294 # OOB branch does not exist thus create it, first ensuring we are on
borlanic 0:02dd72d1d465 295 # the src branch and then check it out
borlanic 0:02dd72d1d465 296
borlanic 0:02dd72d1d465 297 for cmd in ["git checkout " + str(src),
borlanic 0:02dd72d1d465 298 "git checkout -b " + str(dst),
borlanic 0:02dd72d1d465 299 "git push -u origin " + str(dst)]:
borlanic 0:02dd72d1d465 300
borlanic 0:02dd72d1d465 301 run_cmd(cmd, exit_on_failure=True)
borlanic 0:02dd72d1d465 302
borlanic 0:02dd72d1d465 303 else:
borlanic 0:02dd72d1d465 304 cmd = "git checkout " + str(dst)
borlanic 0:02dd72d1d465 305 run_cmd(cmd, exit_on_failure=True)
borlanic 0:02dd72d1d465 306
borlanic 0:02dd72d1d465 307 def upgrade_example(github, example, tag, ref, user, src, dst, template):
borlanic 0:02dd72d1d465 308 """ Upgrade all versions of mbed-os.lib found in the specified example repo
borlanic 0:02dd72d1d465 309
borlanic 0:02dd72d1d465 310 Description:
borlanic 0:02dd72d1d465 311
borlanic 0:02dd72d1d465 312 Clone a version of the example specified and upgrade all versions of
borlanic 0:02dd72d1d465 313 mbed-os.lib found within its tree. The version cloned and how it
borlanic 0:02dd72d1d465 314 is upgraded depends on the user, src and dst settings.
borlanic 0:02dd72d1d465 315 1) user == None
borlanic 0:02dd72d1d465 316 The destination branch will be updated with the version of mbed-os
borlanic 0:02dd72d1d465 317 idenfied by the tag. If the destination branch does not exist then it
borlanic 0:02dd72d1d465 318 will be created from the source branch.
borlanic 0:02dd72d1d465 319
borlanic 0:02dd72d1d465 320 2) user != None
borlanic 0:02dd72d1d465 321 The master branch of a fork of the example will be updated with the
borlanic 0:02dd72d1d465 322 version of mbed-os identified by the tag.
borlanic 0:02dd72d1d465 323
borlanic 0:02dd72d1d465 324 Args:
borlanic 0:02dd72d1d465 325 github - GitHub instance to allow internal git commands to be run
borlanic 0:02dd72d1d465 326 example - json example object containing the GitHub repo to update.
borlanic 0:02dd72d1d465 327 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
borlanic 0:02dd72d1d465 328 ref - SHA corresponding to the tag
borlanic 0:02dd72d1d465 329 user - GitHub user name
borlanic 0:02dd72d1d465 330 src - branch to create the dst branch from
borlanic 0:02dd72d1d465 331 dst - branch to update
borlanic 0:02dd72d1d465 332
borlanic 0:02dd72d1d465 333 returns True if the upgrade was successful, False otherwise
borlanic 0:02dd72d1d465 334 """
borlanic 0:02dd72d1d465 335
borlanic 0:02dd72d1d465 336 # If a user has not been specified then branch update will be used and thus
borlanic 0:02dd72d1d465 337 # the git user will be ARMmbed.
borlanic 0:02dd72d1d465 338 if not user:
borlanic 0:02dd72d1d465 339 user = 'ARMmbed'
borlanic 0:02dd72d1d465 340
borlanic 0:02dd72d1d465 341 ret = False
borlanic 0:02dd72d1d465 342 userlog.info("Updating example '%s'", example['name'])
borlanic 0:02dd72d1d465 343 userlog.debug("User: %s", user)
borlanic 0:02dd72d1d465 344 userlog.debug("Src branch: %s", (src or "None"))
borlanic 0:02dd72d1d465 345 userlog.debug("Dst branch: %s", (dst or "None"))
borlanic 0:02dd72d1d465 346
borlanic 0:02dd72d1d465 347 cwd = os.getcwd()
borlanic 0:02dd72d1d465 348
borlanic 0:02dd72d1d465 349 update_repo = "https://github.com/" + user + '/' + example['name']
borlanic 0:02dd72d1d465 350 userlog.debug("Update repository: %s", update_repo)
borlanic 0:02dd72d1d465 351
borlanic 0:02dd72d1d465 352 # Clone the example repo
borlanic 0:02dd72d1d465 353 clone_cmd = "git clone " + str(update_repo)
borlanic 0:02dd72d1d465 354 return_code = run_cmd(clone_cmd)
borlanic 0:02dd72d1d465 355
borlanic 0:02dd72d1d465 356 if not return_code:
borlanic 0:02dd72d1d465 357
borlanic 0:02dd72d1d465 358 # Find all examples
borlanic 0:02dd72d1d465 359 example_directories = find_all_examples(example['name'])
borlanic 0:02dd72d1d465 360
borlanic 0:02dd72d1d465 361 os.chdir(example['name'])
borlanic 0:02dd72d1d465 362
borlanic 0:02dd72d1d465 363 # If the user is ARMmbed then a branch is used.
borlanic 0:02dd72d1d465 364 if user == 'ARMmbed':
borlanic 0:02dd72d1d465 365 prepare_branch(src, dst)
borlanic 0:02dd72d1d465 366 else:
borlanic 0:02dd72d1d465 367 prepare_fork(example['github'])
borlanic 0:02dd72d1d465 368
borlanic 0:02dd72d1d465 369 for example_directory in example_directories:
borlanic 0:02dd72d1d465 370 if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
borlanic 0:02dd72d1d465 371 os.chdir(cwd)
borlanic 0:02dd72d1d465 372 return False
borlanic 0:02dd72d1d465 373
borlanic 0:02dd72d1d465 374 # Setup and run the commit command
borlanic 0:02dd72d1d465 375 commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
borlanic 0:02dd72d1d465 376 return_code = run_cmd(commit_cmd)
borlanic 0:02dd72d1d465 377 if not return_code:
borlanic 0:02dd72d1d465 378
borlanic 0:02dd72d1d465 379 # Setup and run the push command
borlanic 0:02dd72d1d465 380 push_cmd = "git push origin"
borlanic 0:02dd72d1d465 381 return_code = run_cmd(push_cmd)
borlanic 0:02dd72d1d465 382
borlanic 0:02dd72d1d465 383 if not return_code:
borlanic 0:02dd72d1d465 384 # If the user is not ARMmbed then a fork is being used
borlanic 0:02dd72d1d465 385 if user != 'ARMmbed':
borlanic 0:02dd72d1d465 386
borlanic 0:02dd72d1d465 387 upstream_repo = 'ARMmbed/'+ example['name']
borlanic 0:02dd72d1d465 388 userlog.debug("Upstream repository: %s", upstream_repo)
borlanic 0:02dd72d1d465 389 # Check access to mbed-os repo
borlanic 0:02dd72d1d465 390 try:
borlanic 0:02dd72d1d465 391 repo = github.get_repo(upstream_repo, False)
borlanic 0:02dd72d1d465 392
borlanic 0:02dd72d1d465 393 except:
borlanic 0:02dd72d1d465 394 userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
borlanic 0:02dd72d1d465 395 return False
borlanic 0:02dd72d1d465 396
borlanic 0:02dd72d1d465 397 jinja_loader = FileSystemLoader(template)
borlanic 0:02dd72d1d465 398 jinja_environment = Environment(loader=jinja_loader,
borlanic 0:02dd72d1d465 399 undefined=StrictUndefined)
borlanic 0:02dd72d1d465 400 pr_body = jinja_environment.get_template("pr.tmpl").render(tag=tag)
borlanic 0:02dd72d1d465 401
borlanic 0:02dd72d1d465 402 # Raise a PR from release-candidate to master
borlanic 0:02dd72d1d465 403 user_fork = user + ':master'
borlanic 0:02dd72d1d465 404 try:
borlanic 0:02dd72d1d465 405 pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=pr_body)
borlanic 0:02dd72d1d465 406 ret = True
borlanic 0:02dd72d1d465 407 except GithubException as e:
borlanic 0:02dd72d1d465 408 # Default to False
borlanic 0:02dd72d1d465 409 userlog.error("Pull request creation failed with error: %s", e)
borlanic 0:02dd72d1d465 410 else:
borlanic 0:02dd72d1d465 411 ret = True
borlanic 0:02dd72d1d465 412 else:
borlanic 0:02dd72d1d465 413 userlog.error("Git push command failed.")
borlanic 0:02dd72d1d465 414 else:
borlanic 0:02dd72d1d465 415 userlog.error("Git commit command failed.")
borlanic 0:02dd72d1d465 416 else:
borlanic 0:02dd72d1d465 417 userlog.error("Git clone %s failed", update_repo)
borlanic 0:02dd72d1d465 418
borlanic 0:02dd72d1d465 419 os.chdir(cwd)
borlanic 0:02dd72d1d465 420 return ret
borlanic 0:02dd72d1d465 421
borlanic 0:02dd72d1d465 422 def create_work_directory(path):
borlanic 0:02dd72d1d465 423 """ Create a new directory specified in 'path', overwrite if the directory already
borlanic 0:02dd72d1d465 424 exists.
borlanic 0:02dd72d1d465 425
borlanic 0:02dd72d1d465 426 Args:
borlanic 0:02dd72d1d465 427 path - directory path to be created.
borlanic 0:02dd72d1d465 428
borlanic 0:02dd72d1d465 429 """
borlanic 0:02dd72d1d465 430 if os.path.exists(path):
borlanic 0:02dd72d1d465 431 userlog.info("'%s' directory already exists. Deleting...", path)
borlanic 0:02dd72d1d465 432 rmtree_readonly(path)
borlanic 0:02dd72d1d465 433
borlanic 0:02dd72d1d465 434 os.makedirs(path)
borlanic 0:02dd72d1d465 435
borlanic 0:02dd72d1d465 436 def check_update_status(examples, github, tag):
borlanic 0:02dd72d1d465 437 """ Check the status of previously raised update pull requests
borlanic 0:02dd72d1d465 438
borlanic 0:02dd72d1d465 439 Args:
borlanic 0:02dd72d1d465 440 examples - list of examples which should have had PRs raised against them.
borlanic 0:02dd72d1d465 441 github - github rest API instance
borlanic 0:02dd72d1d465 442 tag - release tag used for the update
borlanic 0:02dd72d1d465 443
borlanic 0:02dd72d1d465 444 """
borlanic 0:02dd72d1d465 445
borlanic 0:02dd72d1d465 446 for example in examples:
borlanic 0:02dd72d1d465 447
borlanic 0:02dd72d1d465 448 repo_name = ''.join(['ARMmbed/', example['name']])
borlanic 0:02dd72d1d465 449 try:
borlanic 0:02dd72d1d465 450 repo = github.get_repo(repo_name, False)
borlanic 0:02dd72d1d465 451
borlanic 0:02dd72d1d465 452 except Exception as exc:
borlanic 0:02dd72d1d465 453 text = "Cannot access: " + str(repo_name)
borlanic 0:02dd72d1d465 454 userlog.error(text)
borlanic 0:02dd72d1d465 455 userlog.exception(exc)
borlanic 0:02dd72d1d465 456 sys.exit(1)
borlanic 0:02dd72d1d465 457
borlanic 0:02dd72d1d465 458 # Create the full repository filter component
borlanic 0:02dd72d1d465 459 org_str = ''.join(['repo:ARMmbed/', example['name']])
borlanic 0:02dd72d1d465 460 filt = ' '.join([org_str, 'is:pr', tag])
borlanic 0:02dd72d1d465 461 merged = False
borlanic 0:02dd72d1d465 462
borlanic 0:02dd72d1d465 463 issues = github.search_issues(query=(filt))
borlanic 0:02dd72d1d465 464 pr_list = [repo.get_pull(issue.number) for issue in issues]
borlanic 0:02dd72d1d465 465
borlanic 0:02dd72d1d465 466 # Should only be one matching PR but just in case, go through paginated list
borlanic 0:02dd72d1d465 467 for pr in pr_list:
borlanic 0:02dd72d1d465 468 if pr.merged:
borlanic 0:02dd72d1d465 469 userlog.info("%s - '%s': MERGED", example['name'], pr.title)
borlanic 0:02dd72d1d465 470 elif pr.state == 'open':
borlanic 0:02dd72d1d465 471 userlog.info("%s - '%s': PENDING", example['name'], pr.title)
borlanic 0:02dd72d1d465 472 elif pr.state == 'closed':
borlanic 0:02dd72d1d465 473 userlog.info("%s - '%s': CLOSED NOT MERGED", example['name'], pr.title)
borlanic 0:02dd72d1d465 474 else:
borlanic 0:02dd72d1d465 475 userlog.error("%s: Cannot find a pull request for %s", example['name'], tag)
borlanic 0:02dd72d1d465 476
borlanic 0:02dd72d1d465 477 if __name__ == '__main__':
borlanic 0:02dd72d1d465 478
borlanic 0:02dd72d1d465 479 parser = argparse.ArgumentParser(description=__doc__,
borlanic 0:02dd72d1d465 480 formatter_class=argparse.RawDescriptionHelpFormatter)
borlanic 0:02dd72d1d465 481 parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
borlanic 0:02dd72d1d465 482 parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
borlanic 0:02dd72d1d465 483
borlanic 0:02dd72d1d465 484 exclusive = parser.add_mutually_exclusive_group(required=True)
borlanic 0:02dd72d1d465 485 exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
borlanic 0:02dd72d1d465 486 exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true')
borlanic 0:02dd72d1d465 487 exclusive.add_argument('-s', '--status', help="Show examples update status", action='store_true')
borlanic 0:02dd72d1d465 488
borlanic 0:02dd72d1d465 489 args = parser.parse_args()
borlanic 0:02dd72d1d465 490
borlanic 0:02dd72d1d465 491 # Load the config file
borlanic 0:02dd72d1d465 492 with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
borlanic 0:02dd72d1d465 493 if not config:
borlanic 0:02dd72d1d465 494 userlog.error("Failed to load config file '%s'", args.config_file)
borlanic 0:02dd72d1d465 495 sys.exit(1)
borlanic 0:02dd72d1d465 496 json_data = json.load(config)
borlanic 0:02dd72d1d465 497
borlanic 0:02dd72d1d465 498
borlanic 0:02dd72d1d465 499 github = Github(args.github_token)
borlanic 0:02dd72d1d465 500 config = json_data['update-config']
borlanic 0:02dd72d1d465 501 tag = config['tag']
borlanic 0:02dd72d1d465 502
borlanic 0:02dd72d1d465 503 user = None
borlanic 0:02dd72d1d465 504 src = "master"
borlanic 0:02dd72d1d465 505 dst = None
borlanic 0:02dd72d1d465 506
borlanic 0:02dd72d1d465 507 if args.status:
borlanic 0:02dd72d1d465 508
borlanic 0:02dd72d1d465 509 # This option should only be called after an update has been performed
borlanic 0:02dd72d1d465 510 check_update_status(json_data['examples'], github, tag)
borlanic 0:02dd72d1d465 511 exit(0)
borlanic 0:02dd72d1d465 512
borlanic 0:02dd72d1d465 513 # Create working directory
borlanic 0:02dd72d1d465 514 create_work_directory('examples')
borlanic 0:02dd72d1d465 515
borlanic 0:02dd72d1d465 516 if args.fork:
borlanic 0:02dd72d1d465 517 user = config['via-fork']['github-user']
borlanic 0:02dd72d1d465 518 elif args.branch:
borlanic 0:02dd72d1d465 519 src = config['via-branch']['src-branch']
borlanic 0:02dd72d1d465 520 dst = config['via-branch']['dst-branch']
borlanic 0:02dd72d1d465 521 else:
borlanic 0:02dd72d1d465 522 userlog.error("Must specify either -f or -b command line option")
borlanic 0:02dd72d1d465 523 exit(1)
borlanic 0:02dd72d1d465 524
borlanic 0:02dd72d1d465 525 # Get the github sha corresponding to the specified mbed-os tag
borlanic 0:02dd72d1d465 526 cmd = "git rev-list -1 " + tag
borlanic 0:02dd72d1d465 527 return_code, ref = run_cmd_with_output(cmd)
borlanic 0:02dd72d1d465 528
borlanic 0:02dd72d1d465 529 if return_code:
borlanic 0:02dd72d1d465 530 userlog.error("Could not obtain SHA for tag: %s", tag)
borlanic 0:02dd72d1d465 531 sys.exit(1)
borlanic 0:02dd72d1d465 532
borlanic 0:02dd72d1d465 533 # Loop through the examples
borlanic 0:02dd72d1d465 534 failures = []
borlanic 0:02dd72d1d465 535 successes = []
borlanic 0:02dd72d1d465 536 results = {}
borlanic 0:02dd72d1d465 537 template = dirname(abspath(__file__))
borlanic 0:02dd72d1d465 538
borlanic 0:02dd72d1d465 539 os.chdir('examples')
borlanic 0:02dd72d1d465 540
borlanic 0:02dd72d1d465 541 for example in json_data['examples']:
borlanic 0:02dd72d1d465 542 # Determine if this example should be updated and if so update any found
borlanic 0:02dd72d1d465 543 # mbed-os.lib files.
borlanic 0:02dd72d1d465 544
borlanic 0:02dd72d1d465 545 result = upgrade_example(github, example, tag, ref, user, src, dst, template)
borlanic 0:02dd72d1d465 546
borlanic 0:02dd72d1d465 547 if result:
borlanic 0:02dd72d1d465 548 successes += [example['name']]
borlanic 0:02dd72d1d465 549 else:
borlanic 0:02dd72d1d465 550 failures += [example['name']]
borlanic 0:02dd72d1d465 551
borlanic 0:02dd72d1d465 552 os.chdir('../')
borlanic 0:02dd72d1d465 553
borlanic 0:02dd72d1d465 554 # Finish the script and report the results
borlanic 0:02dd72d1d465 555 userlog.info("Finished updating examples")
borlanic 0:02dd72d1d465 556 if successes:
borlanic 0:02dd72d1d465 557 for success in successes:
borlanic 0:02dd72d1d465 558 userlog.info(" SUCCEEDED: %s", success)
borlanic 0:02dd72d1d465 559
borlanic 0:02dd72d1d465 560 if failures:
borlanic 0:02dd72d1d465 561 for fail in failures:
borlanic 0:02dd72d1d465 562 userlog.info(" FAILED: %s", fail)