Marco Zecchini / Mbed OS Example_RTOS

Rtos API example

mbed-os/tools/test/examples/update.py@0:9fca2b23d0ba, 2019-02-23 (annotated)

Committer:
marcozecchini
Date:
Sat Feb 23 12:13:36 2019 +0000
Revision:
0:9fca2b23d0ba
final commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
marcozecchini 0:9fca2b23d0ba 1 #!/usr/bin/env python
marcozecchini 0:9fca2b23d0ba 2
marcozecchini 0:9fca2b23d0ba 3 # This script is used to update the version of mbed-os used within a specified set of example
marcozecchini 0:9fca2b23d0ba 4 # applications. The list of examples to be updated lives in the examples.json file and is
marcozecchini 0:9fca2b23d0ba 5 # shared with the examples.py script. Logging is used to provide varying levels of output
marcozecchini 0:9fca2b23d0ba 6 # during execution.
marcozecchini 0:9fca2b23d0ba 7 #
marcozecchini 0:9fca2b23d0ba 8 # There are two modes that can be used:
marcozecchini 0:9fca2b23d0ba 9 # 1) Update the ARMmbed/master branch of the specified example
marcozecchini 0:9fca2b23d0ba 10 #
marcozecchini 0:9fca2b23d0ba 11 # This is done by updating a user fork of the example and then raising a pull request
marcozecchini 0:9fca2b23d0ba 12 # against ARMmbed/master.
marcozecchini 0:9fca2b23d0ba 13 #
marcozecchini 0:9fca2b23d0ba 14 # 2) Update a different ARMmbed branch of the specified example
marcozecchini 0:9fca2b23d0ba 15 #
marcozecchini 0:9fca2b23d0ba 16 # A branch to update is specified. If it doesn't already exist then it is first created.
marcozecchini 0:9fca2b23d0ba 17 # This branch will be updated and the change automatically pushed. The new branch will
marcozecchini 0:9fca2b23d0ba 18 # be created from the specified source branch.
marcozecchini 0:9fca2b23d0ba 19 #
marcozecchini 0:9fca2b23d0ba 20 # The modes are controlled via configuration data in the json file.
marcozecchini 0:9fca2b23d0ba 21 # E.g.
marcozecchini 0:9fca2b23d0ba 22 #
marcozecchini 0:9fca2b23d0ba 23 # "update-config" : {
marcozecchini 0:9fca2b23d0ba 24 # "help" : "Update each example repo with a version of mbed-os identified by the tag",
marcozecchini 0:9fca2b23d0ba 25 # "via-fork" : {
marcozecchini 0:9fca2b23d0ba 26 # "help" : "-f cmd line option. Update a fork",
marcozecchini 0:9fca2b23d0ba 27 # "github-user" : "adbridge"
marcozecchini 0:9fca2b23d0ba 28 # },
marcozecchini 0:9fca2b23d0ba 29 # "via-branch" : {
marcozecchini 0:9fca2b23d0ba 30 # "help" : "-b cmd line option. Update dst branch, created from src branch",
marcozecchini 0:9fca2b23d0ba 31 # "src-branch" : "mbed-os-5.5.0-rc1-oob",
marcozecchini 0:9fca2b23d0ba 32 # "dst-branch" : "mbed-os-5.5.0-rc2-oob"
marcozecchini 0:9fca2b23d0ba 33 # },
marcozecchini 0:9fca2b23d0ba 34 # "tag" : "mbed-os-5.5.0-rc2"
marcozecchini 0:9fca2b23d0ba 35 #
marcozecchini 0:9fca2b23d0ba 36 #
marcozecchini 0:9fca2b23d0ba 37 # Command usage:
marcozecchini 0:9fca2b23d0ba 38 #
marcozecchini 0:9fca2b23d0ba 39 # update.py -c <config file> - T <github_token> -f -b -s
marcozecchini 0:9fca2b23d0ba 40 #
marcozecchini 0:9fca2b23d0ba 41 # Where:
marcozecchini 0:9fca2b23d0ba 42 # -c <config file> - Optional path to an examples file.
marcozecchini 0:9fca2b23d0ba 43 # If not proved the default is 'examples.json'
marcozecchini 0:9fca2b23d0ba 44 # -T <github_token> - GitHub token for secure access (required)
marcozecchini 0:9fca2b23d0ba 45 # -f - Update forked repos. This will use the 'github-user' parameter in
marcozecchini 0:9fca2b23d0ba 46 # the 'via-fork' section.
marcozecchini 0:9fca2b23d0ba 47 # -b - Update branched repos. This will use the "src-branch" and
marcozecchini 0:9fca2b23d0ba 48 # "dst-branch" parameters in the 'via-branch' section. The destination
marcozecchini 0:9fca2b23d0ba 49 # branch is created from the source branch (if it doesn't already exist).
marcozecchini 0:9fca2b23d0ba 50 # -s - Show the status of any pull requests with a tag matching that in the
marcozecchini 0:9fca2b23d0ba 51 # json config file
marcozecchini 0:9fca2b23d0ba 52 #
marcozecchini 0:9fca2b23d0ba 53 # The options -f, -b and -s are mutually exlusive. Only one can be specified.
marcozecchini 0:9fca2b23d0ba 54 #
marcozecchini 0:9fca2b23d0ba 55 #
marcozecchini 0:9fca2b23d0ba 56
marcozecchini 0:9fca2b23d0ba 57 import os
marcozecchini 0:9fca2b23d0ba 58 from os.path import dirname, abspath, basename, join
marcozecchini 0:9fca2b23d0ba 59 import sys
marcozecchini 0:9fca2b23d0ba 60 import logging
marcozecchini 0:9fca2b23d0ba 61 import argparse
marcozecchini 0:9fca2b23d0ba 62 import json
marcozecchini 0:9fca2b23d0ba 63 import subprocess
marcozecchini 0:9fca2b23d0ba 64 import shutil
marcozecchini 0:9fca2b23d0ba 65 import stat
marcozecchini 0:9fca2b23d0ba 66 import re
marcozecchini 0:9fca2b23d0ba 67 from github import Github, GithubException
marcozecchini 0:9fca2b23d0ba 68 from jinja2 import FileSystemLoader, StrictUndefined
marcozecchini 0:9fca2b23d0ba 69 from jinja2.environment import Environment
marcozecchini 0:9fca2b23d0ba 70
marcozecchini 0:9fca2b23d0ba 71 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
marcozecchini 0:9fca2b23d0ba 72 sys.path.insert(0, ROOT)
marcozecchini 0:9fca2b23d0ba 73
marcozecchini 0:9fca2b23d0ba 74 import examples_lib as lib
marcozecchini 0:9fca2b23d0ba 75 from examples_lib import SUPPORTED_TOOLCHAINS
marcozecchini 0:9fca2b23d0ba 76
marcozecchini 0:9fca2b23d0ba 77 userlog = logging.getLogger("Update")
marcozecchini 0:9fca2b23d0ba 78
marcozecchini 0:9fca2b23d0ba 79 # Set logging level
marcozecchini 0:9fca2b23d0ba 80 userlog.setLevel(logging.DEBUG)
marcozecchini 0:9fca2b23d0ba 81
marcozecchini 0:9fca2b23d0ba 82 # Everything is output to the log file
marcozecchini 0:9fca2b23d0ba 83 logfile = os.path.join(os.getcwd(), 'update.log')
marcozecchini 0:9fca2b23d0ba 84 fh = logging.FileHandler(logfile)
marcozecchini 0:9fca2b23d0ba 85 fh.setLevel(logging.DEBUG)
marcozecchini 0:9fca2b23d0ba 86
marcozecchini 0:9fca2b23d0ba 87 # create console handler with a higher log level
marcozecchini 0:9fca2b23d0ba 88 ch = logging.StreamHandler()
marcozecchini 0:9fca2b23d0ba 89 ch.setLevel(logging.INFO)
marcozecchini 0:9fca2b23d0ba 90
marcozecchini 0:9fca2b23d0ba 91 formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
marcozecchini 0:9fca2b23d0ba 92 ch.setFormatter(formatter)
marcozecchini 0:9fca2b23d0ba 93 fh.setFormatter(formatter)
marcozecchini 0:9fca2b23d0ba 94
marcozecchini 0:9fca2b23d0ba 95 # add the handlers to the logger
marcozecchini 0:9fca2b23d0ba 96 userlog.addHandler(fh)
marcozecchini 0:9fca2b23d0ba 97 userlog.addHandler(ch)
marcozecchini 0:9fca2b23d0ba 98
marcozecchini 0:9fca2b23d0ba 99 def run_cmd(command, exit_on_failure=False):
marcozecchini 0:9fca2b23d0ba 100 """ Run a system command returning a status result
marcozecchini 0:9fca2b23d0ba 101
marcozecchini 0:9fca2b23d0ba 102 This is just a wrapper for the run_cmd_with_output() function, but
marcozecchini 0:9fca2b23d0ba 103 only returns the status of the call.
marcozecchini 0:9fca2b23d0ba 104
marcozecchini 0:9fca2b23d0ba 105 Args:
marcozecchini 0:9fca2b23d0ba 106 command - system command as a list of tokens
marcozecchini 0:9fca2b23d0ba 107 exit_on_failure - If True exit the program on failure (default = False)
marcozecchini 0:9fca2b23d0ba 108
marcozecchini 0:9fca2b23d0ba 109 Returns:
marcozecchini 0:9fca2b23d0ba 110 return_code - True/False indicating the success/failure of the command
marcozecchini 0:9fca2b23d0ba 111 """
marcozecchini 0:9fca2b23d0ba 112 return_code, _ = run_cmd_with_output(command, exit_on_failure)
marcozecchini 0:9fca2b23d0ba 113 return return_code
marcozecchini 0:9fca2b23d0ba 114
marcozecchini 0:9fca2b23d0ba 115 def run_cmd_with_output(command, exit_on_failure=False):
marcozecchini 0:9fca2b23d0ba 116 """ Run a system command returning a status result and any command output
marcozecchini 0:9fca2b23d0ba 117
marcozecchini 0:9fca2b23d0ba 118 Passes a command to the system and returns a True/False result once the
marcozecchini 0:9fca2b23d0ba 119 command has been executed, indicating success/failure. If the command was
marcozecchini 0:9fca2b23d0ba 120 successful then the output from the command is returned to the caller.
marcozecchini 0:9fca2b23d0ba 121 Commands are passed as a string.
marcozecchini 0:9fca2b23d0ba 122 E.g. The command 'git remote -v' would be passed in as "git remote -v"
marcozecchini 0:9fca2b23d0ba 123
marcozecchini 0:9fca2b23d0ba 124 Args:
marcozecchini 0:9fca2b23d0ba 125 command - system command as a string
marcozecchini 0:9fca2b23d0ba 126 exit_on_failure - If True exit the program on failure (default = False)
marcozecchini 0:9fca2b23d0ba 127
marcozecchini 0:9fca2b23d0ba 128 Returns:
marcozecchini 0:9fca2b23d0ba 129 return_code - True/False indicating the success/failure of the command
marcozecchini 0:9fca2b23d0ba 130 output - The output of the command if it was successful, else empty string
marcozecchini 0:9fca2b23d0ba 131 """
marcozecchini 0:9fca2b23d0ba 132 text = '[Exec] ' + command
marcozecchini 0:9fca2b23d0ba 133 userlog.debug(text)
marcozecchini 0:9fca2b23d0ba 134 returncode = 0
marcozecchini 0:9fca2b23d0ba 135 output = ""
marcozecchini 0:9fca2b23d0ba 136 try:
marcozecchini 0:9fca2b23d0ba 137 output = subprocess.check_output(command, shell=True)
marcozecchini 0:9fca2b23d0ba 138 except subprocess.CalledProcessError as e:
marcozecchini 0:9fca2b23d0ba 139 text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
marcozecchini 0:9fca2b23d0ba 140 userlog.warning(text)
marcozecchini 0:9fca2b23d0ba 141 returncode = e.returncode
marcozecchini 0:9fca2b23d0ba 142 if exit_on_failure:
marcozecchini 0:9fca2b23d0ba 143 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 144 return returncode, output
marcozecchini 0:9fca2b23d0ba 145
marcozecchini 0:9fca2b23d0ba 146
marcozecchini 0:9fca2b23d0ba 147 def rmtree_readonly(directory):
marcozecchini 0:9fca2b23d0ba 148 """ Deletes a readonly directory tree.
marcozecchini 0:9fca2b23d0ba 149
marcozecchini 0:9fca2b23d0ba 150 Args:
marcozecchini 0:9fca2b23d0ba 151 directory - tree to delete
marcozecchini 0:9fca2b23d0ba 152 """
marcozecchini 0:9fca2b23d0ba 153 def remove_readonly(func, path, _):
marcozecchini 0:9fca2b23d0ba 154 os.chmod(path, stat.S_IWRITE)
marcozecchini 0:9fca2b23d0ba 155 func(path)
marcozecchini 0:9fca2b23d0ba 156
marcozecchini 0:9fca2b23d0ba 157 shutil.rmtree(directory, onerror=remove_readonly)
marcozecchini 0:9fca2b23d0ba 158
marcozecchini 0:9fca2b23d0ba 159 def find_all_examples(path):
marcozecchini 0:9fca2b23d0ba 160 """ Search the path for examples
marcozecchini 0:9fca2b23d0ba 161
marcozecchini 0:9fca2b23d0ba 162 Description:
marcozecchini 0:9fca2b23d0ba 163
marcozecchini 0:9fca2b23d0ba 164 Searches the path specified for sub-example folders, ie those containing an
marcozecchini 0:9fca2b23d0ba 165 mbed-os.lib file. If found adds the path to the sub-example to a list which is
marcozecchini 0:9fca2b23d0ba 166 then returned.
marcozecchini 0:9fca2b23d0ba 167
marcozecchini 0:9fca2b23d0ba 168 Args:
marcozecchini 0:9fca2b23d0ba 169 path - path to search.
marcozecchini 0:9fca2b23d0ba 170 examples - (returned) list of paths to example directories.
marcozecchini 0:9fca2b23d0ba 171
marcozecchini 0:9fca2b23d0ba 172 """
marcozecchini 0:9fca2b23d0ba 173 examples = []
marcozecchini 0:9fca2b23d0ba 174 for root, dirs, files in os.walk(path):
marcozecchini 0:9fca2b23d0ba 175 if 'mbed-os.lib' in files:
marcozecchini 0:9fca2b23d0ba 176 examples += [root]
marcozecchini 0:9fca2b23d0ba 177
marcozecchini 0:9fca2b23d0ba 178 return examples
marcozecchini 0:9fca2b23d0ba 179
marcozecchini 0:9fca2b23d0ba 180 def upgrade_single_example(example, tag, directory, ref):
marcozecchini 0:9fca2b23d0ba 181 """ Update the mbed-os version for a single example
marcozecchini 0:9fca2b23d0ba 182
marcozecchini 0:9fca2b23d0ba 183 Description:
marcozecchini 0:9fca2b23d0ba 184
marcozecchini 0:9fca2b23d0ba 185 Updates the mbed-os.lib file in the example specified to correspond to the
marcozecchini 0:9fca2b23d0ba 186 version specified by the GitHub tag supplied. Also deals with
marcozecchini 0:9fca2b23d0ba 187 multiple sub-examples in the GitHub repo, updating them in the same way.
marcozecchini 0:9fca2b23d0ba 188
marcozecchini 0:9fca2b23d0ba 189 Args:
marcozecchini 0:9fca2b23d0ba 190 example - json example object containing the GitHub repo to update.
marcozecchini 0:9fca2b23d0ba 191 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
marcozecchini 0:9fca2b23d0ba 192 directory - directory path for the example.
marcozecchini 0:9fca2b23d0ba 193 ref - SHA corresponding to the supplied tag
marcozecchini 0:9fca2b23d0ba 194 returns - True if the upgrade was successful, False otherwise.
marcozecchini 0:9fca2b23d0ba 195
marcozecchini 0:9fca2b23d0ba 196 """
marcozecchini 0:9fca2b23d0ba 197 cwd = os.getcwd()
marcozecchini 0:9fca2b23d0ba 198 os.chdir(directory)
marcozecchini 0:9fca2b23d0ba 199
marcozecchini 0:9fca2b23d0ba 200 return_code = False
marcozecchini 0:9fca2b23d0ba 201
marcozecchini 0:9fca2b23d0ba 202 if os.path.isfile("mbed-os.lib"):
marcozecchini 0:9fca2b23d0ba 203 # Rename command will fail on some OS's if the target file already exist,
marcozecchini 0:9fca2b23d0ba 204 # so ensure if it does, it is deleted first.
marcozecchini 0:9fca2b23d0ba 205 if os.path.isfile("mbed-os.lib_bak"):
marcozecchini 0:9fca2b23d0ba 206 os.remove("mbed-os.lib_bak")
marcozecchini 0:9fca2b23d0ba 207
marcozecchini 0:9fca2b23d0ba 208 os.rename("mbed-os.lib", "mbed-os.lib_bak")
marcozecchini 0:9fca2b23d0ba 209 else:
marcozecchini 0:9fca2b23d0ba 210 userlog.error("Failed to backup mbed-os.lib prior to updating.")
marcozecchini 0:9fca2b23d0ba 211 return False
marcozecchini 0:9fca2b23d0ba 212
marcozecchini 0:9fca2b23d0ba 213 # mbed-os.lib file contains one line with the following format
marcozecchini 0:9fca2b23d0ba 214 # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
marcozecchini 0:9fca2b23d0ba 215 lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
marcozecchini 0:9fca2b23d0ba 216 updated = False
marcozecchini 0:9fca2b23d0ba 217
marcozecchini 0:9fca2b23d0ba 218 # Scan through mbed-os.lib line by line
marcozecchini 0:9fca2b23d0ba 219 with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
marcozecchini 0:9fca2b23d0ba 220 for line in ip:
marcozecchini 0:9fca2b23d0ba 221
marcozecchini 0:9fca2b23d0ba 222 opline = line
marcozecchini 0:9fca2b23d0ba 223
marcozecchini 0:9fca2b23d0ba 224 regexp = lib_re.match(line)
marcozecchini 0:9fca2b23d0ba 225 if regexp:
marcozecchini 0:9fca2b23d0ba 226 opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
marcozecchini 0:9fca2b23d0ba 227 updated = True
marcozecchini 0:9fca2b23d0ba 228
marcozecchini 0:9fca2b23d0ba 229 op.write(opline)
marcozecchini 0:9fca2b23d0ba 230
marcozecchini 0:9fca2b23d0ba 231 if updated:
marcozecchini 0:9fca2b23d0ba 232 # Setup and run the git add command
marcozecchini 0:9fca2b23d0ba 233 cmd = "git add mbed-os.lib"
marcozecchini 0:9fca2b23d0ba 234 return_code = run_cmd(cmd)
marcozecchini 0:9fca2b23d0ba 235
marcozecchini 0:9fca2b23d0ba 236 os.chdir(cwd)
marcozecchini 0:9fca2b23d0ba 237 return not return_code
marcozecchini 0:9fca2b23d0ba 238
marcozecchini 0:9fca2b23d0ba 239 def prepare_fork(arm_example):
marcozecchini 0:9fca2b23d0ba 240 """ Synchronises a cloned fork to ensure it is up to date with the original.
marcozecchini 0:9fca2b23d0ba 241
marcozecchini 0:9fca2b23d0ba 242 Description:
marcozecchini 0:9fca2b23d0ba 243
marcozecchini 0:9fca2b23d0ba 244 This function sets a fork of an ARMmbed repo to be up to date with the
marcozecchini 0:9fca2b23d0ba 245 repo it was forked from. It does this by hard resetting to the ARMmbed
marcozecchini 0:9fca2b23d0ba 246 master branch.
marcozecchini 0:9fca2b23d0ba 247
marcozecchini 0:9fca2b23d0ba 248 Args:
marcozecchini 0:9fca2b23d0ba 249 arm_example - Full GitHub repo path for original example
marcozecchini 0:9fca2b23d0ba 250
marcozecchini 0:9fca2b23d0ba 251 """
marcozecchini 0:9fca2b23d0ba 252
marcozecchini 0:9fca2b23d0ba 253 logstr = "In: " + os.getcwd()
marcozecchini 0:9fca2b23d0ba 254 userlog.debug(logstr)
marcozecchini 0:9fca2b23d0ba 255
marcozecchini 0:9fca2b23d0ba 256 for cmd in ["git remote add armmbed " + str(arm_example),
marcozecchini 0:9fca2b23d0ba 257 "git fetch armmbed",
marcozecchini 0:9fca2b23d0ba 258 "git reset --hard armmbed/master",
marcozecchini 0:9fca2b23d0ba 259 "git push -f origin"]:
marcozecchini 0:9fca2b23d0ba 260 run_cmd(cmd, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 261
marcozecchini 0:9fca2b23d0ba 262 def prepare_branch(src, dst):
marcozecchini 0:9fca2b23d0ba 263 """ Set up at branch ready for use in updating examples
marcozecchini 0:9fca2b23d0ba 264
marcozecchini 0:9fca2b23d0ba 265 Description:
marcozecchini 0:9fca2b23d0ba 266
marcozecchini 0:9fca2b23d0ba 267 This function checks whether or not the supplied dst branch exists.
marcozecchini 0:9fca2b23d0ba 268 If it does not, the branch is created from the src and pushed to the origin.
marcozecchini 0:9fca2b23d0ba 269 The branch is then switched to.
marcozecchini 0:9fca2b23d0ba 270
marcozecchini 0:9fca2b23d0ba 271 Args:
marcozecchini 0:9fca2b23d0ba 272 src - branch to create the dst branch from
marcozecchini 0:9fca2b23d0ba 273 dst - branch to update
marcozecchini 0:9fca2b23d0ba 274
marcozecchini 0:9fca2b23d0ba 275 """
marcozecchini 0:9fca2b23d0ba 276
marcozecchini 0:9fca2b23d0ba 277 userlog.debug("Preparing branch: %s", dst)
marcozecchini 0:9fca2b23d0ba 278
marcozecchini 0:9fca2b23d0ba 279 # Check if branch already exists or not.
marcozecchini 0:9fca2b23d0ba 280 # We can use the 'git branch -r' command. This returns all the remote branches for
marcozecchini 0:9fca2b23d0ba 281 # the current repo.
marcozecchini 0:9fca2b23d0ba 282 # The output consists of a list of lines of the form:
marcozecchini 0:9fca2b23d0ba 283 # origin/<branch>
marcozecchini 0:9fca2b23d0ba 284 # From these we need to extract just the branch names to a list and then check if
marcozecchini 0:9fca2b23d0ba 285 # the specified dst exists in that list
marcozecchini 0:9fca2b23d0ba 286 branches = []
marcozecchini 0:9fca2b23d0ba 287 cmd = "git branch -r"
marcozecchini 0:9fca2b23d0ba 288 _, output = run_cmd_with_output(cmd, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 289
marcozecchini 0:9fca2b23d0ba 290 branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]
marcozecchini 0:9fca2b23d0ba 291
marcozecchini 0:9fca2b23d0ba 292 if not dst in branches:
marcozecchini 0:9fca2b23d0ba 293
marcozecchini 0:9fca2b23d0ba 294 # OOB branch does not exist thus create it, first ensuring we are on
marcozecchini 0:9fca2b23d0ba 295 # the src branch and then check it out
marcozecchini 0:9fca2b23d0ba 296
marcozecchini 0:9fca2b23d0ba 297 for cmd in ["git checkout " + str(src),
marcozecchini 0:9fca2b23d0ba 298 "git checkout -b " + str(dst),
marcozecchini 0:9fca2b23d0ba 299 "git push -u origin " + str(dst)]:
marcozecchini 0:9fca2b23d0ba 300
marcozecchini 0:9fca2b23d0ba 301 run_cmd(cmd, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 302
marcozecchini 0:9fca2b23d0ba 303 else:
marcozecchini 0:9fca2b23d0ba 304 cmd = "git checkout " + str(dst)
marcozecchini 0:9fca2b23d0ba 305 run_cmd(cmd, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 306
marcozecchini 0:9fca2b23d0ba 307 def upgrade_example(github, example, tag, ref, user, src, dst, template):
marcozecchini 0:9fca2b23d0ba 308 """ Upgrade all versions of mbed-os.lib found in the specified example repo
marcozecchini 0:9fca2b23d0ba 309
marcozecchini 0:9fca2b23d0ba 310 Description:
marcozecchini 0:9fca2b23d0ba 311
marcozecchini 0:9fca2b23d0ba 312 Clone a version of the example specified and upgrade all versions of
marcozecchini 0:9fca2b23d0ba 313 mbed-os.lib found within its tree. The version cloned and how it
marcozecchini 0:9fca2b23d0ba 314 is upgraded depends on the user, src and dst settings.
marcozecchini 0:9fca2b23d0ba 315 1) user == None
marcozecchini 0:9fca2b23d0ba 316 The destination branch will be updated with the version of mbed-os
marcozecchini 0:9fca2b23d0ba 317 idenfied by the tag. If the destination branch does not exist then it
marcozecchini 0:9fca2b23d0ba 318 will be created from the source branch.
marcozecchini 0:9fca2b23d0ba 319
marcozecchini 0:9fca2b23d0ba 320 2) user != None
marcozecchini 0:9fca2b23d0ba 321 The master branch of a fork of the example will be updated with the
marcozecchini 0:9fca2b23d0ba 322 version of mbed-os identified by the tag.
marcozecchini 0:9fca2b23d0ba 323
marcozecchini 0:9fca2b23d0ba 324 Args:
marcozecchini 0:9fca2b23d0ba 325 github - GitHub instance to allow internal git commands to be run
marcozecchini 0:9fca2b23d0ba 326 example - json example object containing the GitHub repo to update.
marcozecchini 0:9fca2b23d0ba 327 tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
marcozecchini 0:9fca2b23d0ba 328 ref - SHA corresponding to the tag
marcozecchini 0:9fca2b23d0ba 329 user - GitHub user name
marcozecchini 0:9fca2b23d0ba 330 src - branch to create the dst branch from
marcozecchini 0:9fca2b23d0ba 331 dst - branch to update
marcozecchini 0:9fca2b23d0ba 332
marcozecchini 0:9fca2b23d0ba 333 returns True if the upgrade was successful, False otherwise
marcozecchini 0:9fca2b23d0ba 334 """
marcozecchini 0:9fca2b23d0ba 335
marcozecchini 0:9fca2b23d0ba 336 # If a user has not been specified then branch update will be used and thus
marcozecchini 0:9fca2b23d0ba 337 # the git user will be ARMmbed.
marcozecchini 0:9fca2b23d0ba 338 if not user:
marcozecchini 0:9fca2b23d0ba 339 user = 'ARMmbed'
marcozecchini 0:9fca2b23d0ba 340
marcozecchini 0:9fca2b23d0ba 341 ret = False
marcozecchini 0:9fca2b23d0ba 342 userlog.info("Updating example '%s'", example['name'])
marcozecchini 0:9fca2b23d0ba 343 userlog.debug("User: %s", user)
marcozecchini 0:9fca2b23d0ba 344 userlog.debug("Src branch: %s", (src or "None"))
marcozecchini 0:9fca2b23d0ba 345 userlog.debug("Dst branch: %s", (dst or "None"))
marcozecchini 0:9fca2b23d0ba 346
marcozecchini 0:9fca2b23d0ba 347 cwd = os.getcwd()
marcozecchini 0:9fca2b23d0ba 348
marcozecchini 0:9fca2b23d0ba 349 update_repo = "https://github.com/" + user + '/' + example['name']
marcozecchini 0:9fca2b23d0ba 350 userlog.debug("Update repository: %s", update_repo)
marcozecchini 0:9fca2b23d0ba 351
marcozecchini 0:9fca2b23d0ba 352 # Clone the example repo
marcozecchini 0:9fca2b23d0ba 353 clone_cmd = "git clone " + str(update_repo)
marcozecchini 0:9fca2b23d0ba 354 return_code = run_cmd(clone_cmd)
marcozecchini 0:9fca2b23d0ba 355
marcozecchini 0:9fca2b23d0ba 356 if not return_code:
marcozecchini 0:9fca2b23d0ba 357
marcozecchini 0:9fca2b23d0ba 358 # Find all examples
marcozecchini 0:9fca2b23d0ba 359 example_directories = find_all_examples(example['name'])
marcozecchini 0:9fca2b23d0ba 360
marcozecchini 0:9fca2b23d0ba 361 os.chdir(example['name'])
marcozecchini 0:9fca2b23d0ba 362
marcozecchini 0:9fca2b23d0ba 363 # If the user is ARMmbed then a branch is used.
marcozecchini 0:9fca2b23d0ba 364 if user == 'ARMmbed':
marcozecchini 0:9fca2b23d0ba 365 prepare_branch(src, dst)
marcozecchini 0:9fca2b23d0ba 366 else:
marcozecchini 0:9fca2b23d0ba 367 prepare_fork(example['github'])
marcozecchini 0:9fca2b23d0ba 368
marcozecchini 0:9fca2b23d0ba 369 for example_directory in example_directories:
marcozecchini 0:9fca2b23d0ba 370 if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
marcozecchini 0:9fca2b23d0ba 371 os.chdir(cwd)
marcozecchini 0:9fca2b23d0ba 372 return False
marcozecchini 0:9fca2b23d0ba 373
marcozecchini 0:9fca2b23d0ba 374 # Setup and run the commit command
marcozecchini 0:9fca2b23d0ba 375 commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
marcozecchini 0:9fca2b23d0ba 376 return_code = run_cmd(commit_cmd)
marcozecchini 0:9fca2b23d0ba 377 if not return_code:
marcozecchini 0:9fca2b23d0ba 378
marcozecchini 0:9fca2b23d0ba 379 # Setup and run the push command
marcozecchini 0:9fca2b23d0ba 380 push_cmd = "git push origin"
marcozecchini 0:9fca2b23d0ba 381 return_code = run_cmd(push_cmd)
marcozecchini 0:9fca2b23d0ba 382
marcozecchini 0:9fca2b23d0ba 383 if not return_code:
marcozecchini 0:9fca2b23d0ba 384 # If the user is not ARMmbed then a fork is being used
marcozecchini 0:9fca2b23d0ba 385 if user != 'ARMmbed':
marcozecchini 0:9fca2b23d0ba 386
marcozecchini 0:9fca2b23d0ba 387 upstream_repo = 'ARMmbed/'+ example['name']
marcozecchini 0:9fca2b23d0ba 388 userlog.debug("Upstream repository: %s", upstream_repo)
marcozecchini 0:9fca2b23d0ba 389 # Check access to mbed-os repo
marcozecchini 0:9fca2b23d0ba 390 try:
marcozecchini 0:9fca2b23d0ba 391 repo = github.get_repo(upstream_repo, False)
marcozecchini 0:9fca2b23d0ba 392
marcozecchini 0:9fca2b23d0ba 393 except:
marcozecchini 0:9fca2b23d0ba 394 userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
marcozecchini 0:9fca2b23d0ba 395 return False
marcozecchini 0:9fca2b23d0ba 396
marcozecchini 0:9fca2b23d0ba 397 jinja_loader = FileSystemLoader(template)
marcozecchini 0:9fca2b23d0ba 398 jinja_environment = Environment(loader=jinja_loader,
marcozecchini 0:9fca2b23d0ba 399 undefined=StrictUndefined)
marcozecchini 0:9fca2b23d0ba 400 pr_body = jinja_environment.get_template("pr.tmpl").render(tag=tag)
marcozecchini 0:9fca2b23d0ba 401
marcozecchini 0:9fca2b23d0ba 402 # Raise a PR from release-candidate to master
marcozecchini 0:9fca2b23d0ba 403 user_fork = user + ':master'
marcozecchini 0:9fca2b23d0ba 404 try:
marcozecchini 0:9fca2b23d0ba 405 pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=pr_body)
marcozecchini 0:9fca2b23d0ba 406 ret = True
marcozecchini 0:9fca2b23d0ba 407 except GithubException as e:
marcozecchini 0:9fca2b23d0ba 408 # Default to False
marcozecchini 0:9fca2b23d0ba 409 userlog.error("Pull request creation failed with error: %s", e)
marcozecchini 0:9fca2b23d0ba 410 else:
marcozecchini 0:9fca2b23d0ba 411 ret = True
marcozecchini 0:9fca2b23d0ba 412 else:
marcozecchini 0:9fca2b23d0ba 413 userlog.error("Git push command failed.")
marcozecchini 0:9fca2b23d0ba 414 else:
marcozecchini 0:9fca2b23d0ba 415 userlog.error("Git commit command failed.")
marcozecchini 0:9fca2b23d0ba 416 else:
marcozecchini 0:9fca2b23d0ba 417 userlog.error("Git clone %s failed", update_repo)
marcozecchini 0:9fca2b23d0ba 418
marcozecchini 0:9fca2b23d0ba 419 os.chdir(cwd)
marcozecchini 0:9fca2b23d0ba 420 return ret
marcozecchini 0:9fca2b23d0ba 421
marcozecchini 0:9fca2b23d0ba 422 def create_work_directory(path):
marcozecchini 0:9fca2b23d0ba 423 """ Create a new directory specified in 'path', overwrite if the directory already
marcozecchini 0:9fca2b23d0ba 424 exists.
marcozecchini 0:9fca2b23d0ba 425
marcozecchini 0:9fca2b23d0ba 426 Args:
marcozecchini 0:9fca2b23d0ba 427 path - directory path to be created.
marcozecchini 0:9fca2b23d0ba 428
marcozecchini 0:9fca2b23d0ba 429 """
marcozecchini 0:9fca2b23d0ba 430 if os.path.exists(path):
marcozecchini 0:9fca2b23d0ba 431 userlog.info("'%s' directory already exists. Deleting...", path)
marcozecchini 0:9fca2b23d0ba 432 rmtree_readonly(path)
marcozecchini 0:9fca2b23d0ba 433
marcozecchini 0:9fca2b23d0ba 434 os.makedirs(path)
marcozecchini 0:9fca2b23d0ba 435
marcozecchini 0:9fca2b23d0ba 436 def check_update_status(examples, github, tag):
marcozecchini 0:9fca2b23d0ba 437 """ Check the status of previously raised update pull requests
marcozecchini 0:9fca2b23d0ba 438
marcozecchini 0:9fca2b23d0ba 439 Args:
marcozecchini 0:9fca2b23d0ba 440 examples - list of examples which should have had PRs raised against them.
marcozecchini 0:9fca2b23d0ba 441 github - github rest API instance
marcozecchini 0:9fca2b23d0ba 442 tag - release tag used for the update
marcozecchini 0:9fca2b23d0ba 443
marcozecchini 0:9fca2b23d0ba 444 """
marcozecchini 0:9fca2b23d0ba 445
marcozecchini 0:9fca2b23d0ba 446 for example in examples:
marcozecchini 0:9fca2b23d0ba 447
marcozecchini 0:9fca2b23d0ba 448 repo_name = ''.join(['ARMmbed/', example['name']])
marcozecchini 0:9fca2b23d0ba 449 try:
marcozecchini 0:9fca2b23d0ba 450 repo = github.get_repo(repo_name, False)
marcozecchini 0:9fca2b23d0ba 451
marcozecchini 0:9fca2b23d0ba 452 except Exception as exc:
marcozecchini 0:9fca2b23d0ba 453 text = "Cannot access: " + str(repo_name)
marcozecchini 0:9fca2b23d0ba 454 userlog.error(text)
marcozecchini 0:9fca2b23d0ba 455 userlog.exception(exc)
marcozecchini 0:9fca2b23d0ba 456 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 457
marcozecchini 0:9fca2b23d0ba 458 # Create the full repository filter component
marcozecchini 0:9fca2b23d0ba 459 org_str = ''.join(['repo:ARMmbed/', example['name']])
marcozecchini 0:9fca2b23d0ba 460 filt = ' '.join([org_str, 'is:pr', tag])
marcozecchini 0:9fca2b23d0ba 461 merged = False
marcozecchini 0:9fca2b23d0ba 462
marcozecchini 0:9fca2b23d0ba 463 issues = github.search_issues(query=(filt))
marcozecchini 0:9fca2b23d0ba 464 pr_list = [repo.get_pull(issue.number) for issue in issues]
marcozecchini 0:9fca2b23d0ba 465
marcozecchini 0:9fca2b23d0ba 466 # Should only be one matching PR but just in case, go through paginated list
marcozecchini 0:9fca2b23d0ba 467 for pr in pr_list:
marcozecchini 0:9fca2b23d0ba 468 if pr.merged:
marcozecchini 0:9fca2b23d0ba 469 userlog.info("%s - '%s': MERGED", example['name'], pr.title)
marcozecchini 0:9fca2b23d0ba 470 elif pr.state == 'open':
marcozecchini 0:9fca2b23d0ba 471 userlog.info("%s - '%s': PENDING", example['name'], pr.title)
marcozecchini 0:9fca2b23d0ba 472 elif pr.state == 'closed':
marcozecchini 0:9fca2b23d0ba 473 userlog.info("%s - '%s': CLOSED NOT MERGED", example['name'], pr.title)
marcozecchini 0:9fca2b23d0ba 474 else:
marcozecchini 0:9fca2b23d0ba 475 userlog.error("%s: Cannot find a pull request for %s", example['name'], tag)
marcozecchini 0:9fca2b23d0ba 476
marcozecchini 0:9fca2b23d0ba 477 if __name__ == '__main__':
marcozecchini 0:9fca2b23d0ba 478
marcozecchini 0:9fca2b23d0ba 479 parser = argparse.ArgumentParser(description=__doc__,
marcozecchini 0:9fca2b23d0ba 480 formatter_class=argparse.RawDescriptionHelpFormatter)
marcozecchini 0:9fca2b23d0ba 481 parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
marcozecchini 0:9fca2b23d0ba 482 parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
marcozecchini 0:9fca2b23d0ba 483
marcozecchini 0:9fca2b23d0ba 484 exclusive = parser.add_mutually_exclusive_group(required=True)
marcozecchini 0:9fca2b23d0ba 485 exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
marcozecchini 0:9fca2b23d0ba 486 exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true')
marcozecchini 0:9fca2b23d0ba 487 exclusive.add_argument('-s', '--status', help="Show examples update status", action='store_true')
marcozecchini 0:9fca2b23d0ba 488
marcozecchini 0:9fca2b23d0ba 489 args = parser.parse_args()
marcozecchini 0:9fca2b23d0ba 490
marcozecchini 0:9fca2b23d0ba 491 # Load the config file
marcozecchini 0:9fca2b23d0ba 492 with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
marcozecchini 0:9fca2b23d0ba 493 if not config:
marcozecchini 0:9fca2b23d0ba 494 userlog.error("Failed to load config file '%s'", args.config_file)
marcozecchini 0:9fca2b23d0ba 495 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 496 json_data = json.load(config)
marcozecchini 0:9fca2b23d0ba 497
marcozecchini 0:9fca2b23d0ba 498
marcozecchini 0:9fca2b23d0ba 499 github = Github(args.github_token)
marcozecchini 0:9fca2b23d0ba 500 config = json_data['update-config']
marcozecchini 0:9fca2b23d0ba 501 tag = config['tag']
marcozecchini 0:9fca2b23d0ba 502
marcozecchini 0:9fca2b23d0ba 503 user = None
marcozecchini 0:9fca2b23d0ba 504 src = "master"
marcozecchini 0:9fca2b23d0ba 505 dst = None
marcozecchini 0:9fca2b23d0ba 506
marcozecchini 0:9fca2b23d0ba 507 if args.status:
marcozecchini 0:9fca2b23d0ba 508
marcozecchini 0:9fca2b23d0ba 509 # This option should only be called after an update has been performed
marcozecchini 0:9fca2b23d0ba 510 check_update_status(json_data['examples'], github, tag)
marcozecchini 0:9fca2b23d0ba 511 exit(0)
marcozecchini 0:9fca2b23d0ba 512
marcozecchini 0:9fca2b23d0ba 513 # Create working directory
marcozecchini 0:9fca2b23d0ba 514 create_work_directory('examples')
marcozecchini 0:9fca2b23d0ba 515
marcozecchini 0:9fca2b23d0ba 516 if args.fork:
marcozecchini 0:9fca2b23d0ba 517 user = config['via-fork']['github-user']
marcozecchini 0:9fca2b23d0ba 518 elif args.branch:
marcozecchini 0:9fca2b23d0ba 519 src = config['via-branch']['src-branch']
marcozecchini 0:9fca2b23d0ba 520 dst = config['via-branch']['dst-branch']
marcozecchini 0:9fca2b23d0ba 521 else:
marcozecchini 0:9fca2b23d0ba 522 userlog.error("Must specify either -f or -b command line option")
marcozecchini 0:9fca2b23d0ba 523 exit(1)
marcozecchini 0:9fca2b23d0ba 524
marcozecchini 0:9fca2b23d0ba 525 # Get the github sha corresponding to the specified mbed-os tag
marcozecchini 0:9fca2b23d0ba 526 cmd = "git rev-list -1 " + tag
marcozecchini 0:9fca2b23d0ba 527 return_code, ref = run_cmd_with_output(cmd)
marcozecchini 0:9fca2b23d0ba 528
marcozecchini 0:9fca2b23d0ba 529 if return_code:
marcozecchini 0:9fca2b23d0ba 530 userlog.error("Could not obtain SHA for tag: %s", tag)
marcozecchini 0:9fca2b23d0ba 531 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 532
marcozecchini 0:9fca2b23d0ba 533 # Loop through the examples
marcozecchini 0:9fca2b23d0ba 534 failures = []
marcozecchini 0:9fca2b23d0ba 535 successes = []
marcozecchini 0:9fca2b23d0ba 536 results = {}
marcozecchini 0:9fca2b23d0ba 537 template = dirname(abspath(__file__))
marcozecchini 0:9fca2b23d0ba 538
marcozecchini 0:9fca2b23d0ba 539 os.chdir('examples')
marcozecchini 0:9fca2b23d0ba 540
marcozecchini 0:9fca2b23d0ba 541 for example in json_data['examples']:
marcozecchini 0:9fca2b23d0ba 542 # Determine if this example should be updated and if so update any found
marcozecchini 0:9fca2b23d0ba 543 # mbed-os.lib files.
marcozecchini 0:9fca2b23d0ba 544
marcozecchini 0:9fca2b23d0ba 545 result = upgrade_example(github, example, tag, ref, user, src, dst, template)
marcozecchini 0:9fca2b23d0ba 546
marcozecchini 0:9fca2b23d0ba 547 if result:
marcozecchini 0:9fca2b23d0ba 548 successes += [example['name']]
marcozecchini 0:9fca2b23d0ba 549 else:
marcozecchini 0:9fca2b23d0ba 550 failures += [example['name']]
marcozecchini 0:9fca2b23d0ba 551
marcozecchini 0:9fca2b23d0ba 552 os.chdir('../')
marcozecchini 0:9fca2b23d0ba 553
marcozecchini 0:9fca2b23d0ba 554 # Finish the script and report the results
marcozecchini 0:9fca2b23d0ba 555 userlog.info("Finished updating examples")
marcozecchini 0:9fca2b23d0ba 556 if successes:
marcozecchini 0:9fca2b23d0ba 557 for success in successes:
marcozecchini 0:9fca2b23d0ba 558 userlog.info(" SUCCEEDED: %s", success)
marcozecchini 0:9fca2b23d0ba 559
marcozecchini 0:9fca2b23d0ba 560 if failures:
marcozecchini 0:9fca2b23d0ba 561 for fail in failures:
marcozecchini 0:9fca2b23d0ba 562 userlog.info(" FAILED: %s", fail)