EL4121 Embedded System / mbed-os

mbed-os

Dependents:   cobaLCDJoyMotor_Thread odometry_omni_3roda_v3 odometry_omni_3roda_v1 odometry_omni_3roda_v2 ... more

tools/test/examples/update.py@0:b74591d5ab33, 2017-12-11 (annotated)

Committer:
be_bryan
Date:
Mon Dec 11 17:54:04 2017 +0000
Revision:
0:b74591d5ab33
motor ++

Who changed what in which revision?

UserRevisionLine numberNew contents of line
be_bryan 0:b74591d5ab33 1 #!/usr/bin/env python
be_bryan 0:b74591d5ab33 2
be_bryan 0:b74591d5ab33 3 # This script is used to update the version of mbed-os used within a specified set of example
be_bryan 0:b74591d5ab33 4 # applications. The list of examples to be updated lives in the examples.json file and is
be_bryan 0:b74591d5ab33 5 # shared with the examples.py script. Logging is used to provide varying levels of output
be_bryan 0:b74591d5ab33 6 # during execution.
be_bryan 0:b74591d5ab33 7 #
be_bryan 0:b74591d5ab33 8 # There are two modes that can be used:
be_bryan 0:b74591d5ab33 9 # 1) Update the ARMmbed/master branch of the specified example
be_bryan 0:b74591d5ab33 10 #
be_bryan 0:b74591d5ab33 11 # This is done by updating a user fork of the example and then raising a pull request
be_bryan 0:b74591d5ab33 12 # against ARMmbed/master.
be_bryan 0:b74591d5ab33 13 #
be_bryan 0:b74591d5ab33 14 # 2) Update a different ARMmbed branch of the specified example
be_bryan 0:b74591d5ab33 15 #
be_bryan 0:b74591d5ab33 16 # A branch to update is specified. If it doesn't already exist then it is first created.
be_bryan 0:b74591d5ab33 17 # This branch will be updated and the change automatically pushed. The new branch will
be_bryan 0:b74591d5ab33 18 # be created from the specified source branch.
be_bryan 0:b74591d5ab33 19 #
be_bryan 0:b74591d5ab33 20 # The modes are controlled via configuration data in the json file.
be_bryan 0:b74591d5ab33 21 # E.g.
be_bryan 0:b74591d5ab33 22 #
be_bryan 0:b74591d5ab33 23 # "update-config" : {
be_bryan 0:b74591d5ab33 24 # "help" : "Update each example repo with a version of mbed-os identified by the tag",
be_bryan 0:b74591d5ab33 25 # "via-fork" : {
be_bryan 0:b74591d5ab33 26 # "help" : "-f cmd line option. Update a fork",
be_bryan 0:b74591d5ab33 27 # "github-user" : "adbridge"
be_bryan 0:b74591d5ab33 28 # },
be_bryan 0:b74591d5ab33 29 # "via-branch" : {
be_bryan 0:b74591d5ab33 30 # "help" : "-b cmd line option. Update dst branch, created from src branch",
be_bryan 0:b74591d5ab33 31 # "src-branch" : "mbed-os-5.5.0-rc1-oob",
be_bryan 0:b74591d5ab33 32 # "dst-branch" : "mbed-os-5.5.0-rc2-oob"
be_bryan 0:b74591d5ab33 33 # },
be_bryan 0:b74591d5ab33 34 # "tag" : "mbed-os-5.5.0-rc2"
be_bryan 0:b74591d5ab33 35 #
be_bryan 0:b74591d5ab33 36 #
be_bryan 0:b74591d5ab33 37 # Command usage:
be_bryan 0:b74591d5ab33 38 #
be_bryan 0:b74591d5ab33 39 # update.py -c <config file> - T <github_token> -f -b -s
be_bryan 0:b74591d5ab33 40 #
be_bryan 0:b74591d5ab33 41 # Where:
be_bryan 0:b74591d5ab33 42 # -c <config file> - Optional path to an examples file.
be_bryan 0:b74591d5ab33 43 # If not proved the default is 'examples.json'
be_bryan 0:b74591d5ab33 44 # -T <github_token> - GitHub token for secure access (required)
be_bryan 0:b74591d5ab33 45 # -f - Update forked repos. This will use the 'github-user' parameter in
be_bryan 0:b74591d5ab33 46 # the 'via-fork' section.
be_bryan 0:b74591d5ab33 47 # -b - Update branched repos. This will use the "src-branch" and
be_bryan 0:b74591d5ab33 48 # "dst-branch" parameters in the 'via-branch' section. The destination
be_bryan 0:b74591d5ab33 49 # branch is created from the source branch (if it doesn't already exist).
be_bryan 0:b74591d5ab33 50 # -s - Show the status of any pull requests with a tag matching that in the
be_bryan 0:b74591d5ab33 51 # json config file
be_bryan 0:b74591d5ab33 52 #
be_bryan 0:b74591d5ab33 53 # The options -f, -b and -s are mutually exlusive. Only one can be specified.
be_bryan 0:b74591d5ab33 54 #
be_bryan 0:b74591d5ab33 55 #
be_bryan 0:b74591d5ab33 56
be_bryan 0:b74591d5ab33 57 import os
be_bryan 0:b74591d5ab33 58 from os.path import dirname, abspath, basename, join
be_bryan 0:b74591d5ab33 59 import sys
be_bryan 0:b74591d5ab33 60 import logging
be_bryan 0:b74591d5ab33 61 import argparse
be_bryan 0:b74591d5ab33 62 import json
be_bryan 0:b74591d5ab33 63 import subprocess
be_bryan 0:b74591d5ab33 64 import shutil
be_bryan 0:b74591d5ab33 65 import stat
be_bryan 0:b74591d5ab33 66 import re
be_bryan 0:b74591d5ab33 67 from github import Github, GithubException
be_bryan 0:b74591d5ab33 68 from jinja2 import FileSystemLoader, StrictUndefined
be_bryan 0:b74591d5ab33 69 from jinja2.environment import Environment
be_bryan 0:b74591d5ab33 70
be_bryan 0:b74591d5ab33 71 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
be_bryan 0:b74591d5ab33 72 sys.path.insert(0, ROOT)
be_bryan 0:b74591d5ab33 73
be_bryan 0:b74591d5ab33 74 import examples_lib as lib
be_bryan 0:b74591d5ab33 75 from examples_lib import SUPPORTED_TOOLCHAINS
be_bryan 0:b74591d5ab33 76
be_bryan 0:b74591d5ab33 77 userlog = logging.getLogger("Update")
be_bryan 0:b74591d5ab33 78
be_bryan 0:b74591d5ab33 79 # Set logging level
be_bryan 0:b74591d5ab33 80 userlog.setLevel(logging.DEBUG)
be_bryan 0:b74591d5ab33 81
be_bryan 0:b74591d5ab33 82 # Everything is output to the log file
be_bryan 0:b74591d5ab33 83 logfile = os.path.join(os.getcwd(), 'update.log')
be_bryan 0:b74591d5ab33 84 fh = logging.FileHandler(logfile)
be_bryan 0:b74591d5ab33 85 fh.setLevel(logging.DEBUG)
be_bryan 0:b74591d5ab33 86
be_bryan 0:b74591d5ab33 87 # create console handler with a higher log level
be_bryan 0:b74591d5ab33 88 ch = logging.StreamHandler()
be_bryan 0:b74591d5ab33 89 ch.setLevel(logging.INFO)
be_bryan 0:b74591d5ab33 90
be_bryan 0:b74591d5ab33 91 formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
be_bryan 0:b74591d5ab33 92 ch.setFormatter(formatter)
be_bryan 0:b74591d5ab33 93 fh.setFormatter(formatter)
be_bryan 0:b74591d5ab33 94
be_bryan 0:b74591d5ab33 95 # add the handlers to the logger
be_bryan 0:b74591d5ab33 96 userlog.addHandler(fh)
be_bryan 0:b74591d5ab33 97 userlog.addHandler(ch)
be_bryan 0:b74591d5ab33 98
be_bryan 0:b74591d5ab33 99 def run_cmd(command, exit_on_failure=False):
be_bryan 0:b74591d5ab33 100 """ Run a system command returning a status result
be_bryan 0:b74591d5ab33 101
be_bryan 0:b74591d5ab33 102 This is just a wrapper for the run_cmd_with_output() function, but
be_bryan 0:b74591d5ab33 103 only returns the status of the call.
be_bryan 0:b74591d5ab33 104
be_bryan 0:b74591d5ab33 105 Args:
be_bryan 0:b74591d5ab33 106 command - system command as a list of tokens
be_bryan 0:b74591d5ab33 107 exit_on_failure - If True exit the program on failure (default = False)
be_bryan 0:b74591d5ab33 108
be_bryan 0:b74591d5ab33 109 Returns:
be_bryan 0:b74591d5ab33 110 return_code - True/False indicating the success/failure of the command
be_bryan 0:b74591d5ab33 111 """
be_bryan 0:b74591d5ab33 112 return_code, _ = run_cmd_with_output(command, exit_on_failure)
be_bryan 0:b74591d5ab33 113 return return_code
be_bryan 0:b74591d5ab33 114
be_bryan 0:b74591d5ab33 115 def run_cmd_with_output(command, exit_on_failure=False):
be_bryan 0:b74591d5ab33 116 """ Run a system command returning a status result and any command output
be_bryan 0:b74591d5ab33 117
be_bryan 0:b74591d5ab33 118 Passes a command to the system and returns a True/False result once the
be_bryan 0:b74591d5ab33 119 command has been executed, indicating success/failure. If the command was
be_bryan 0:b74591d5ab33 120 successful then the output from the command is returned to the caller.
be_bryan 0:b74591d5ab33 121 Commands are passed as a string.
be_bryan 0:b74591d5ab33 122 E.g. The command 'git remote -v' would be passed in as "git remote -v"
be_bryan 0:b74591d5ab33 123
be_bryan 0:b74591d5ab33 124 Args:
be_bryan 0:b74591d5ab33 125 command - system command as a string
be_bryan 0:b74591d5ab33 126 exit_on_failure - If True exit the program on failure (default = False)
be_bryan 0:b74591d5ab33 127
be_bryan 0:b74591d5ab33 128 Returns:
be_bryan 0:b74591d5ab33 129 return_code - True/False indicating the success/failure of the command
be_bryan 0:b74591d5ab33 130 output - The output of the command if it was successful, else empty string
be_bryan 0:b74591d5ab33 131 """
be_bryan 0:b74591d5ab33 132 text = '[Exec] ' + command
be_bryan 0:b74591d5ab33 133 userlog.debug(text)
be_bryan 0:b74591d5ab33 134 returncode = 0
be_bryan 0:b74591d5ab33 135 output = ""
be_bryan 0:b74591d5ab33 136 try:
be_bryan 0:b74591d5ab33 137 output = subprocess.check_output(command, shell=True)
be_bryan 0:b74591d5ab33 138 except subprocess.CalledProcessError as e:
be_bryan 0:b74591d5ab33 139 text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
be_bryan 0:b74591d5ab33 140 userlog.warning(text)
be_bryan 0:b74591d5ab33 141 returncode = e.returncode
be_bryan 0:b74591d5ab33 142 if exit_on_failure:
be_bryan 0:b74591d5ab33 143 sys.exit(1)
be_bryan 0:b74591d5ab33 144 return returncode, output
be_bryan 0:b74591d5ab33 145
be_bryan 0:b74591d5ab33 146
be_bryan 0:b74591d5ab33 147 def rmtree_readonly(directory):
be_bryan 0:b74591d5ab33 148 """ Deletes a readonly directory tree.
be_bryan 0:b74591d5ab33 149
be_bryan 0:b74591d5ab33 150 Args:
be_bryan 0:b74591d5ab33 151 directory - tree to delete
be_bryan 0:b74591d5ab33 152 """
be_bryan 0:b74591d5ab33 153 def remove_readonly(func, path, _):
be_bryan 0:b74591d5ab33 154 os.chmod(path, stat.S_IWRITE)
be_bryan 0:b74591d5ab33 155 func(path)
be_bryan 0:b74591d5ab33 156
be_bryan 0:b74591d5ab33 157 shutil.rmtree(directory, onerror=remove_readonly)
be_bryan 0:b74591d5ab33 158
be_bryan 0:b74591d5ab33 159 def find_all_examples(path):
be_bryan 0:b74591d5ab33 160 """ Search the path for examples
be_bryan 0:b74591d5ab33 161
be_bryan 0:b74591d5ab33 162 Description:
be_bryan 0:b74591d5ab33 163
be_bryan 0:b74591d5ab33 164 Searches the path specified for sub-example folders, ie those containing an
be_bryan 0:b74591d5ab33 165 mbed-os.lib file. If found adds the path to the sub-example to a list which is
be_bryan 0:b74591d5ab33 166 then returned.
be_bryan 0:b74591d5ab33 167
be_bryan 0:b74591d5ab33 168 Args:
be_bryan 0:b74591d5ab33 169 path - path to search.
be_bryan 0:b74591d5ab33 170 examples - (returned) list of paths to example directories.
be_bryan 0:b74591d5ab33 171
be_bryan 0:b74591d5ab33 172 """
be_bryan 0:b74591d5ab33 173 examples = []
be_bryan 0:b74591d5ab33 174 for root, dirs, files in os.walk(path):
be_bryan 0:b74591d5ab33 175 if 'mbed-os.lib' in files:
be_bryan 0:b74591d5ab33 176 examples += [root]
be_bryan 0:b74591d5ab33 177
be_bryan 0:b74591d5ab33 178 return examples
be_bryan 0:b74591d5ab33 179
be_bryan 0:b74591d5ab33 180 def upgrade_single_example(example, tag, directory, ref):
be_bryan 0:b74591d5ab33 181 """ Update the mbed-os version for a single example
be_bryan 0:b74591d5ab33 182
be_bryan 0:b74591d5ab33 183 Description:
be_bryan 0:b74591d5ab33 184
be_bryan 0:b74591d5ab33 185 Updates the mbed-os.lib file in the example specified to correspond to the
be_bryan 0:b74591d5ab33 186 version specified by the GitHub tag supplied. Also deals with
be_bryan 0:b74591d5ab33 187 multiple sub-examples in the GitHub repo, updating them in the same way.
be_bryan 0:b74591d5ab33 188
be_bryan 0:b74591d5ab33 189 Args:
be_bryan 0:b74591d5ab33 190 example - json example object containing the GitHub repo to update.
be_bryan 0:b74591d5ab33 191 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
be_bryan 0:b74591d5ab33 192 directory - directory path for the example.
be_bryan 0:b74591d5ab33 193 ref - SHA corresponding to the supplied tag
be_bryan 0:b74591d5ab33 194 returns - True if the upgrade was successful, False otherwise.
be_bryan 0:b74591d5ab33 195
be_bryan 0:b74591d5ab33 196 """
be_bryan 0:b74591d5ab33 197 cwd = os.getcwd()
be_bryan 0:b74591d5ab33 198 os.chdir(directory)
be_bryan 0:b74591d5ab33 199
be_bryan 0:b74591d5ab33 200 return_code = False
be_bryan 0:b74591d5ab33 201
be_bryan 0:b74591d5ab33 202 if os.path.isfile("mbed-os.lib"):
be_bryan 0:b74591d5ab33 203 # Rename command will fail on some OS's if the target file already exist,
be_bryan 0:b74591d5ab33 204 # so ensure if it does, it is deleted first.
be_bryan 0:b74591d5ab33 205 if os.path.isfile("mbed-os.lib_bak"):
be_bryan 0:b74591d5ab33 206 os.remove("mbed-os.lib_bak")
be_bryan 0:b74591d5ab33 207
be_bryan 0:b74591d5ab33 208 os.rename("mbed-os.lib", "mbed-os.lib_bak")
be_bryan 0:b74591d5ab33 209 else:
be_bryan 0:b74591d5ab33 210 userlog.error("Failed to backup mbed-os.lib prior to updating.")
be_bryan 0:b74591d5ab33 211 return False
be_bryan 0:b74591d5ab33 212
be_bryan 0:b74591d5ab33 213 # mbed-os.lib file contains one line with the following format
be_bryan 0:b74591d5ab33 214 # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
be_bryan 0:b74591d5ab33 215 lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
be_bryan 0:b74591d5ab33 216 updated = False
be_bryan 0:b74591d5ab33 217
be_bryan 0:b74591d5ab33 218 # Scan through mbed-os.lib line by line
be_bryan 0:b74591d5ab33 219 with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
be_bryan 0:b74591d5ab33 220 for line in ip:
be_bryan 0:b74591d5ab33 221
be_bryan 0:b74591d5ab33 222 opline = line
be_bryan 0:b74591d5ab33 223
be_bryan 0:b74591d5ab33 224 regexp = lib_re.match(line)
be_bryan 0:b74591d5ab33 225 if regexp:
be_bryan 0:b74591d5ab33 226 opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
be_bryan 0:b74591d5ab33 227 updated = True
be_bryan 0:b74591d5ab33 228
be_bryan 0:b74591d5ab33 229 op.write(opline)
be_bryan 0:b74591d5ab33 230
be_bryan 0:b74591d5ab33 231 if updated:
be_bryan 0:b74591d5ab33 232 # Setup and run the git add command
be_bryan 0:b74591d5ab33 233 cmd = "git add mbed-os.lib"
be_bryan 0:b74591d5ab33 234 return_code = run_cmd(cmd)
be_bryan 0:b74591d5ab33 235
be_bryan 0:b74591d5ab33 236 os.chdir(cwd)
be_bryan 0:b74591d5ab33 237 return not return_code
be_bryan 0:b74591d5ab33 238
be_bryan 0:b74591d5ab33 239 def prepare_fork(arm_example):
be_bryan 0:b74591d5ab33 240 """ Synchronises a cloned fork to ensure it is up to date with the original.
be_bryan 0:b74591d5ab33 241
be_bryan 0:b74591d5ab33 242 Description:
be_bryan 0:b74591d5ab33 243
be_bryan 0:b74591d5ab33 244 This function sets a fork of an ARMmbed repo to be up to date with the
be_bryan 0:b74591d5ab33 245 repo it was forked from. It does this by hard resetting to the ARMmbed
be_bryan 0:b74591d5ab33 246 master branch.
be_bryan 0:b74591d5ab33 247
be_bryan 0:b74591d5ab33 248 Args:
be_bryan 0:b74591d5ab33 249 arm_example - Full GitHub repo path for original example
be_bryan 0:b74591d5ab33 250
be_bryan 0:b74591d5ab33 251 """
be_bryan 0:b74591d5ab33 252
be_bryan 0:b74591d5ab33 253 logstr = "In: " + os.getcwd()
be_bryan 0:b74591d5ab33 254 userlog.debug(logstr)
be_bryan 0:b74591d5ab33 255
be_bryan 0:b74591d5ab33 256 for cmd in ["git remote add armmbed " + str(arm_example),
be_bryan 0:b74591d5ab33 257 "git fetch armmbed",
be_bryan 0:b74591d5ab33 258 "git reset --hard armmbed/master",
be_bryan 0:b74591d5ab33 259 "git push -f origin"]:
be_bryan 0:b74591d5ab33 260 run_cmd(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 261
be_bryan 0:b74591d5ab33 262 def prepare_branch(src, dst):
be_bryan 0:b74591d5ab33 263 """ Set up at branch ready for use in updating examples
be_bryan 0:b74591d5ab33 264
be_bryan 0:b74591d5ab33 265 Description:
be_bryan 0:b74591d5ab33 266
be_bryan 0:b74591d5ab33 267 This function checks whether or not the supplied dst branch exists.
be_bryan 0:b74591d5ab33 268 If it does not, the branch is created from the src and pushed to the origin.
be_bryan 0:b74591d5ab33 269 The branch is then switched to.
be_bryan 0:b74591d5ab33 270
be_bryan 0:b74591d5ab33 271 Args:
be_bryan 0:b74591d5ab33 272 src - branch to create the dst branch from
be_bryan 0:b74591d5ab33 273 dst - branch to update
be_bryan 0:b74591d5ab33 274
be_bryan 0:b74591d5ab33 275 """
be_bryan 0:b74591d5ab33 276
be_bryan 0:b74591d5ab33 277 userlog.debug("Preparing branch: %s", dst)
be_bryan 0:b74591d5ab33 278
be_bryan 0:b74591d5ab33 279 # Check if branch already exists or not.
be_bryan 0:b74591d5ab33 280 # We can use the 'git branch -r' command. This returns all the remote branches for
be_bryan 0:b74591d5ab33 281 # the current repo.
be_bryan 0:b74591d5ab33 282 # The output consists of a list of lines of the form:
be_bryan 0:b74591d5ab33 283 # origin/<branch>
be_bryan 0:b74591d5ab33 284 # From these we need to extract just the branch names to a list and then check if
be_bryan 0:b74591d5ab33 285 # the specified dst exists in that list
be_bryan 0:b74591d5ab33 286 branches = []
be_bryan 0:b74591d5ab33 287 cmd = "git branch -r"
be_bryan 0:b74591d5ab33 288 _, output = run_cmd_with_output(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 289
be_bryan 0:b74591d5ab33 290 branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]
be_bryan 0:b74591d5ab33 291
be_bryan 0:b74591d5ab33 292 if not dst in branches:
be_bryan 0:b74591d5ab33 293
be_bryan 0:b74591d5ab33 294 # OOB branch does not exist thus create it, first ensuring we are on
be_bryan 0:b74591d5ab33 295 # the src branch and then check it out
be_bryan 0:b74591d5ab33 296
be_bryan 0:b74591d5ab33 297 for cmd in ["git checkout " + str(src),
be_bryan 0:b74591d5ab33 298 "git checkout -b " + str(dst),
be_bryan 0:b74591d5ab33 299 "git push -u origin " + str(dst)]:
be_bryan 0:b74591d5ab33 300
be_bryan 0:b74591d5ab33 301 run_cmd(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 302
be_bryan 0:b74591d5ab33 303 else:
be_bryan 0:b74591d5ab33 304 cmd = "git checkout " + str(dst)
be_bryan 0:b74591d5ab33 305 run_cmd(cmd, exit_on_failure=True)
be_bryan 0:b74591d5ab33 306
be_bryan 0:b74591d5ab33 307 def upgrade_example(github, example, tag, ref, user, src, dst, template):
be_bryan 0:b74591d5ab33 308 """ Upgrade all versions of mbed-os.lib found in the specified example repo
be_bryan 0:b74591d5ab33 309
be_bryan 0:b74591d5ab33 310 Description:
be_bryan 0:b74591d5ab33 311
be_bryan 0:b74591d5ab33 312 Clone a version of the example specified and upgrade all versions of
be_bryan 0:b74591d5ab33 313 mbed-os.lib found within its tree. The version cloned and how it
be_bryan 0:b74591d5ab33 314 is upgraded depends on the user, src and dst settings.
be_bryan 0:b74591d5ab33 315 1) user == None
be_bryan 0:b74591d5ab33 316 The destination branch will be updated with the version of mbed-os
be_bryan 0:b74591d5ab33 317 idenfied by the tag. If the destination branch does not exist then it
be_bryan 0:b74591d5ab33 318 will be created from the source branch.
be_bryan 0:b74591d5ab33 319
be_bryan 0:b74591d5ab33 320 2) user != None
be_bryan 0:b74591d5ab33 321 The master branch of a fork of the example will be updated with the
be_bryan 0:b74591d5ab33 322 version of mbed-os identified by the tag.
be_bryan 0:b74591d5ab33 323
be_bryan 0:b74591d5ab33 324 Args:
be_bryan 0:b74591d5ab33 325 github - GitHub instance to allow internal git commands to be run
be_bryan 0:b74591d5ab33 326 example - json example object containing the GitHub repo to update.
be_bryan 0:b74591d5ab33 327 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
be_bryan 0:b74591d5ab33 328 ref - SHA corresponding to the tag
be_bryan 0:b74591d5ab33 329 user - GitHub user name
be_bryan 0:b74591d5ab33 330 src - branch to create the dst branch from
be_bryan 0:b74591d5ab33 331 dst - branch to update
be_bryan 0:b74591d5ab33 332
be_bryan 0:b74591d5ab33 333 returns True if the upgrade was successful, False otherwise
be_bryan 0:b74591d5ab33 334 """
be_bryan 0:b74591d5ab33 335
be_bryan 0:b74591d5ab33 336 # If a user has not been specified then branch update will be used and thus
be_bryan 0:b74591d5ab33 337 # the git user will be ARMmbed.
be_bryan 0:b74591d5ab33 338 if not user:
be_bryan 0:b74591d5ab33 339 user = 'ARMmbed'
be_bryan 0:b74591d5ab33 340
be_bryan 0:b74591d5ab33 341 ret = False
be_bryan 0:b74591d5ab33 342 userlog.info("Updating example '%s'", example['name'])
be_bryan 0:b74591d5ab33 343 userlog.debug("User: %s", user)
be_bryan 0:b74591d5ab33 344 userlog.debug("Src branch: %s", (src or "None"))
be_bryan 0:b74591d5ab33 345 userlog.debug("Dst branch: %s", (dst or "None"))
be_bryan 0:b74591d5ab33 346
be_bryan 0:b74591d5ab33 347 cwd = os.getcwd()
be_bryan 0:b74591d5ab33 348
be_bryan 0:b74591d5ab33 349 update_repo = "https://github.com/" + user + '/' + example['name']
be_bryan 0:b74591d5ab33 350 userlog.debug("Update repository: %s", update_repo)
be_bryan 0:b74591d5ab33 351
be_bryan 0:b74591d5ab33 352 # Clone the example repo
be_bryan 0:b74591d5ab33 353 clone_cmd = "git clone " + str(update_repo)
be_bryan 0:b74591d5ab33 354 return_code = run_cmd(clone_cmd)
be_bryan 0:b74591d5ab33 355
be_bryan 0:b74591d5ab33 356 if not return_code:
be_bryan 0:b74591d5ab33 357
be_bryan 0:b74591d5ab33 358 # Find all examples
be_bryan 0:b74591d5ab33 359 example_directories = find_all_examples(example['name'])
be_bryan 0:b74591d5ab33 360
be_bryan 0:b74591d5ab33 361 os.chdir(example['name'])
be_bryan 0:b74591d5ab33 362
be_bryan 0:b74591d5ab33 363 # If the user is ARMmbed then a branch is used.
be_bryan 0:b74591d5ab33 364 if user == 'ARMmbed':
be_bryan 0:b74591d5ab33 365 prepare_branch(src, dst)
be_bryan 0:b74591d5ab33 366 else:
be_bryan 0:b74591d5ab33 367 prepare_fork(example['github'])
be_bryan 0:b74591d5ab33 368
be_bryan 0:b74591d5ab33 369 for example_directory in example_directories:
be_bryan 0:b74591d5ab33 370 if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
be_bryan 0:b74591d5ab33 371 os.chdir(cwd)
be_bryan 0:b74591d5ab33 372 return False
be_bryan 0:b74591d5ab33 373
be_bryan 0:b74591d5ab33 374 # Setup and run the commit command
be_bryan 0:b74591d5ab33 375 commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
be_bryan 0:b74591d5ab33 376 return_code = run_cmd(commit_cmd)
be_bryan 0:b74591d5ab33 377 if not return_code:
be_bryan 0:b74591d5ab33 378
be_bryan 0:b74591d5ab33 379 # Setup and run the push command
be_bryan 0:b74591d5ab33 380 push_cmd = "git push origin"
be_bryan 0:b74591d5ab33 381 return_code = run_cmd(push_cmd)
be_bryan 0:b74591d5ab33 382
be_bryan 0:b74591d5ab33 383 if not return_code:
be_bryan 0:b74591d5ab33 384 # If the user is not ARMmbed then a fork is being used
be_bryan 0:b74591d5ab33 385 if user != 'ARMmbed':
be_bryan 0:b74591d5ab33 386
be_bryan 0:b74591d5ab33 387 upstream_repo = 'ARMmbed/'+ example['name']
be_bryan 0:b74591d5ab33 388 userlog.debug("Upstream repository: %s", upstream_repo)
be_bryan 0:b74591d5ab33 389 # Check access to mbed-os repo
be_bryan 0:b74591d5ab33 390 try:
be_bryan 0:b74591d5ab33 391 repo = github.get_repo(upstream_repo, False)
be_bryan 0:b74591d5ab33 392
be_bryan 0:b74591d5ab33 393 except:
be_bryan 0:b74591d5ab33 394 userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
be_bryan 0:b74591d5ab33 395 return False
be_bryan 0:b74591d5ab33 396
be_bryan 0:b74591d5ab33 397 jinja_loader = FileSystemLoader(template)
be_bryan 0:b74591d5ab33 398 jinja_environment = Environment(loader=jinja_loader,
be_bryan 0:b74591d5ab33 399 undefined=StrictUndefined)
be_bryan 0:b74591d5ab33 400 pr_body = jinja_environment.get_template("pr.tmpl").render(tag=tag)
be_bryan 0:b74591d5ab33 401
be_bryan 0:b74591d5ab33 402 # Raise a PR from release-candidate to master
be_bryan 0:b74591d5ab33 403 user_fork = user + ':master'
be_bryan 0:b74591d5ab33 404 try:
be_bryan 0:b74591d5ab33 405 pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=pr_body)
be_bryan 0:b74591d5ab33 406 ret = True
be_bryan 0:b74591d5ab33 407 except GithubException as e:
be_bryan 0:b74591d5ab33 408 # Default to False
be_bryan 0:b74591d5ab33 409 userlog.error("Pull request creation failed with error: %s", e)
be_bryan 0:b74591d5ab33 410 else:
be_bryan 0:b74591d5ab33 411 ret = True
be_bryan 0:b74591d5ab33 412 else:
be_bryan 0:b74591d5ab33 413 userlog.error("Git push command failed.")
be_bryan 0:b74591d5ab33 414 else:
be_bryan 0:b74591d5ab33 415 userlog.error("Git commit command failed.")
be_bryan 0:b74591d5ab33 416 else:
be_bryan 0:b74591d5ab33 417 userlog.error("Git clone %s failed", update_repo)
be_bryan 0:b74591d5ab33 418
be_bryan 0:b74591d5ab33 419 os.chdir(cwd)
be_bryan 0:b74591d5ab33 420 return ret
be_bryan 0:b74591d5ab33 421
be_bryan 0:b74591d5ab33 422 def create_work_directory(path):
be_bryan 0:b74591d5ab33 423 """ Create a new directory specified in 'path', overwrite if the directory already
be_bryan 0:b74591d5ab33 424 exists.
be_bryan 0:b74591d5ab33 425
be_bryan 0:b74591d5ab33 426 Args:
be_bryan 0:b74591d5ab33 427 path - directory path to be created.
be_bryan 0:b74591d5ab33 428
be_bryan 0:b74591d5ab33 429 """
be_bryan 0:b74591d5ab33 430 if os.path.exists(path):
be_bryan 0:b74591d5ab33 431 userlog.info("'%s' directory already exists. Deleting...", path)
be_bryan 0:b74591d5ab33 432 rmtree_readonly(path)
be_bryan 0:b74591d5ab33 433
be_bryan 0:b74591d5ab33 434 os.makedirs(path)
be_bryan 0:b74591d5ab33 435
be_bryan 0:b74591d5ab33 436 def check_update_status(examples, github, tag):
be_bryan 0:b74591d5ab33 437 """ Check the status of previously raised update pull requests
be_bryan 0:b74591d5ab33 438
be_bryan 0:b74591d5ab33 439 Args:
be_bryan 0:b74591d5ab33 440 examples - list of examples which should have had PRs raised against them.
be_bryan 0:b74591d5ab33 441 github - github rest API instance
be_bryan 0:b74591d5ab33 442 tag - release tag used for the update
be_bryan 0:b74591d5ab33 443
be_bryan 0:b74591d5ab33 444 """
be_bryan 0:b74591d5ab33 445
be_bryan 0:b74591d5ab33 446 for example in examples:
be_bryan 0:b74591d5ab33 447
be_bryan 0:b74591d5ab33 448 repo_name = ''.join(['ARMmbed/', example['name']])
be_bryan 0:b74591d5ab33 449 try:
be_bryan 0:b74591d5ab33 450 repo = github.get_repo(repo_name, False)
be_bryan 0:b74591d5ab33 451
be_bryan 0:b74591d5ab33 452 except Exception as exc:
be_bryan 0:b74591d5ab33 453 text = "Cannot access: " + str(repo_name)
be_bryan 0:b74591d5ab33 454 userlog.error(text)
be_bryan 0:b74591d5ab33 455 userlog.exception(exc)
be_bryan 0:b74591d5ab33 456 sys.exit(1)
be_bryan 0:b74591d5ab33 457
be_bryan 0:b74591d5ab33 458 # Create the full repository filter component
be_bryan 0:b74591d5ab33 459 org_str = ''.join(['repo:ARMmbed/', example['name']])
be_bryan 0:b74591d5ab33 460 filt = ' '.join([org_str, 'is:pr', tag])
be_bryan 0:b74591d5ab33 461 merged = False
be_bryan 0:b74591d5ab33 462
be_bryan 0:b74591d5ab33 463 issues = github.search_issues(query=(filt))
be_bryan 0:b74591d5ab33 464 pr_list = [repo.get_pull(issue.number) for issue in issues]
be_bryan 0:b74591d5ab33 465
be_bryan 0:b74591d5ab33 466 # Should only be one matching PR but just in case, go through paginated list
be_bryan 0:b74591d5ab33 467 for pr in pr_list:
be_bryan 0:b74591d5ab33 468 if pr.merged:
be_bryan 0:b74591d5ab33 469 userlog.info("%s - '%s': MERGED", example['name'], pr.title)
be_bryan 0:b74591d5ab33 470 elif pr.state == 'open':
be_bryan 0:b74591d5ab33 471 userlog.info("%s - '%s': PENDING", example['name'], pr.title)
be_bryan 0:b74591d5ab33 472 elif pr.state == 'closed':
be_bryan 0:b74591d5ab33 473 userlog.info("%s - '%s': CLOSED NOT MERGED", example['name'], pr.title)
be_bryan 0:b74591d5ab33 474 else:
be_bryan 0:b74591d5ab33 475 userlog.error("%s: Cannot find a pull request for %s", example['name'], tag)
be_bryan 0:b74591d5ab33 476
be_bryan 0:b74591d5ab33 477 if __name__ == '__main__':
be_bryan 0:b74591d5ab33 478
be_bryan 0:b74591d5ab33 479 parser = argparse.ArgumentParser(description=__doc__,
be_bryan 0:b74591d5ab33 480 formatter_class=argparse.RawDescriptionHelpFormatter)
be_bryan 0:b74591d5ab33 481 parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
be_bryan 0:b74591d5ab33 482 parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
be_bryan 0:b74591d5ab33 483
be_bryan 0:b74591d5ab33 484 exclusive = parser.add_mutually_exclusive_group(required=True)
be_bryan 0:b74591d5ab33 485 exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
be_bryan 0:b74591d5ab33 486 exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true')
be_bryan 0:b74591d5ab33 487 exclusive.add_argument('-s', '--status', help="Show examples update status", action='store_true')
be_bryan 0:b74591d5ab33 488
be_bryan 0:b74591d5ab33 489 args = parser.parse_args()
be_bryan 0:b74591d5ab33 490
be_bryan 0:b74591d5ab33 491 # Load the config file
be_bryan 0:b74591d5ab33 492 with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
be_bryan 0:b74591d5ab33 493 if not config:
be_bryan 0:b74591d5ab33 494 userlog.error("Failed to load config file '%s'", args.config_file)
be_bryan 0:b74591d5ab33 495 sys.exit(1)
be_bryan 0:b74591d5ab33 496 json_data = json.load(config)
be_bryan 0:b74591d5ab33 497
be_bryan 0:b74591d5ab33 498
be_bryan 0:b74591d5ab33 499 github = Github(args.github_token)
be_bryan 0:b74591d5ab33 500 config = json_data['update-config']
be_bryan 0:b74591d5ab33 501 tag = config['tag']
be_bryan 0:b74591d5ab33 502
be_bryan 0:b74591d5ab33 503 user = None
be_bryan 0:b74591d5ab33 504 src = "master"
be_bryan 0:b74591d5ab33 505 dst = None
be_bryan 0:b74591d5ab33 506
be_bryan 0:b74591d5ab33 507 if args.status:
be_bryan 0:b74591d5ab33 508
be_bryan 0:b74591d5ab33 509 # This option should only be called after an update has been performed
be_bryan 0:b74591d5ab33 510 check_update_status(json_data['examples'], github, tag)
be_bryan 0:b74591d5ab33 511 exit(0)
be_bryan 0:b74591d5ab33 512
be_bryan 0:b74591d5ab33 513 # Create working directory
be_bryan 0:b74591d5ab33 514 create_work_directory('examples')
be_bryan 0:b74591d5ab33 515
be_bryan 0:b74591d5ab33 516 if args.fork:
be_bryan 0:b74591d5ab33 517 user = config['via-fork']['github-user']
be_bryan 0:b74591d5ab33 518 elif args.branch:
be_bryan 0:b74591d5ab33 519 src = config['via-branch']['src-branch']
be_bryan 0:b74591d5ab33 520 dst = config['via-branch']['dst-branch']
be_bryan 0:b74591d5ab33 521 else:
be_bryan 0:b74591d5ab33 522 userlog.error("Must specify either -f or -b command line option")
be_bryan 0:b74591d5ab33 523 exit(1)
be_bryan 0:b74591d5ab33 524
be_bryan 0:b74591d5ab33 525 # Get the github sha corresponding to the specified mbed-os tag
be_bryan 0:b74591d5ab33 526 cmd = "git rev-list -1 " + tag
be_bryan 0:b74591d5ab33 527 return_code, ref = run_cmd_with_output(cmd)
be_bryan 0:b74591d5ab33 528
be_bryan 0:b74591d5ab33 529 if return_code:
be_bryan 0:b74591d5ab33 530 userlog.error("Could not obtain SHA for tag: %s", tag)
be_bryan 0:b74591d5ab33 531 sys.exit(1)
be_bryan 0:b74591d5ab33 532
be_bryan 0:b74591d5ab33 533 # Loop through the examples
be_bryan 0:b74591d5ab33 534 failures = []
be_bryan 0:b74591d5ab33 535 successes = []
be_bryan 0:b74591d5ab33 536 results = {}
be_bryan 0:b74591d5ab33 537 template = dirname(abspath(__file__))
be_bryan 0:b74591d5ab33 538
be_bryan 0:b74591d5ab33 539 os.chdir('examples')
be_bryan 0:b74591d5ab33 540
be_bryan 0:b74591d5ab33 541 for example in json_data['examples']:
be_bryan 0:b74591d5ab33 542 # Determine if this example should be updated and if so update any found
be_bryan 0:b74591d5ab33 543 # mbed-os.lib files.
be_bryan 0:b74591d5ab33 544
be_bryan 0:b74591d5ab33 545 result = upgrade_example(github, example, tag, ref, user, src, dst, template)
be_bryan 0:b74591d5ab33 546
be_bryan 0:b74591d5ab33 547 if result:
be_bryan 0:b74591d5ab33 548 successes += [example['name']]
be_bryan 0:b74591d5ab33 549 else:
be_bryan 0:b74591d5ab33 550 failures += [example['name']]
be_bryan 0:b74591d5ab33 551
be_bryan 0:b74591d5ab33 552 os.chdir('../')
be_bryan 0:b74591d5ab33 553
be_bryan 0:b74591d5ab33 554 # Finish the script and report the results
be_bryan 0:b74591d5ab33 555 userlog.info("Finished updating examples")
be_bryan 0:b74591d5ab33 556 if successes:
be_bryan 0:b74591d5ab33 557 for success in successes:
be_bryan 0:b74591d5ab33 558 userlog.info(" SUCCEEDED: %s", success)
be_bryan 0:b74591d5ab33 559
be_bryan 0:b74591d5ab33 560 if failures:
be_bryan 0:b74591d5ab33 561 for fail in failures:
be_bryan 0:b74591d5ab33 562 userlog.info(" FAILED: %s", fail)