update.py

00001 #!/usr/bin/env python
00002 
00003 # This script is used to update the version of mbed-os used within a specified set of example
00004 # applications. The list of examples to be updated lives in the examples.json file and is 
00005 # shared with the examples.py script. Logging is used to provide varying levels of output
00006 # during execution.
00007 #
00008 # There are two modes that can be used:
00009 # 1) Update the ARMmbed/master branch of the specified example
00010 # 
00011 #    This is done by updating a user fork of the example and then raising a pull request
00012 #    against ARMmbed/master.
00013 #
00014 # 2) Update a different ARMmbed branch of the specified example
00015 #
00016 #    A branch to update is specified. If it doesn't already exist then it is first created.
00017 #    This branch will be updated and the change automatically pushed. The new branch will 
00018 #    be created from the specified source branch.
00019 #
00020 # The modes are controlled via configuration data in the json file.
00021 # E.g.
00022 # 
00023 #   "update-config" : {
00024 #      "help" : "Update each example repo with a version of mbed-os identified by the tag",
00025 #      "via-fork" : {
00026 #        "help" : "-f cmd line option. Update a fork",
00027 #        "github-user" : "adbridge"          
00028 #      },
00029 #      "via-branch" : {
00030 #        "help" : "-b cmd line option. Update dst branch, created from src branch",
00031 #        "src-branch" : "mbed-os-5.5.0-rc1-oob",
00032 #        "dst-branch" : "mbed-os-5.5.0-rc2-oob"    
00033 #      },
00034 #      "tag" : "mbed-os-5.5.0-rc2"
00035 #
00036 #
00037 # Command usage:
00038 #
00039 # update.py -c <config file> - T <github_token> -f -b -s
00040 #
00041 # Where:
00042 # -c <config file> - Optional path to an examples file.
00043 #                    If not proved the default is 'examples.json'
00044 # -T <github_token> - GitHub token for secure access (required)
00045 # -f                 - Update forked repos. This will use the 'github-user' parameter in
00046 #                      the 'via-fork' section.
00047 # -b                 - Update branched repos. This will use the "src-branch" and 
00048 #                      "dst-branch" parameters in the 'via-branch' section. The destination
00049 #                      branch is created from the source branch (if it doesn't already exist).
00050 # -s               - Show the status of any pull requests with a tag matching that in the 
00051 #                    json config file
00052 # 
00053 # The options -f, -b and -s are mutually exlusive. Only one can be specified.
00054 #
00055 #
00056 
00057 import os
00058 from os.path import dirname, abspath, basename, join
00059 import sys
00060 import logging
00061 import argparse
00062 import json
00063 import subprocess
00064 import shutil
00065 import stat
00066 import re
00067 from github import Github, GithubException
00068 from jinja2 import FileSystemLoader, StrictUndefined
00069 from jinja2.environment import Environment
00070 
00071 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00072 sys.path.insert(0, ROOT)
00073 
00074 import examples_lib as lib
00075 from examples_lib import SUPPORTED_TOOLCHAINS
00076 
00077 userlog = logging.getLogger("Update")
00078 
00079 # Set logging level
00080 userlog.setLevel(logging.DEBUG)
00081 
00082 # Everything is output to the log file
00083 logfile = os.path.join(os.getcwd(), 'update.log')
00084 fh = logging.FileHandler(logfile)
00085 fh.setLevel(logging.DEBUG)
00086 
00087 # create console handler with a higher log level
00088 ch = logging.StreamHandler()
00089 ch.setLevel(logging.INFO)
00090 
00091 formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s')
00092 ch.setFormatter(formatter)
00093 fh.setFormatter(formatter)    
00094 
00095 # add the handlers to the logger
00096 userlog.addHandler(fh)
00097 userlog.addHandler(ch)
00098 
00099 def run_cmd(command, exit_on_failure=False):
00100     """ Run a system command returning a status result
00101     
00102     This is just a wrapper for the run_cmd_with_output() function, but
00103     only returns the status of the call.
00104     
00105     Args:
00106     command - system command as a list of tokens
00107     exit_on_failure - If True exit the program on failure (default = False)
00108     
00109     Returns:
00110     return_code - True/False indicating the success/failure of the command
00111     """
00112     return_code, _ = run_cmd_with_output(command, exit_on_failure)       
00113     return return_code
00114 
00115 def run_cmd_with_output(command, exit_on_failure=False):
00116     """ Run a system command returning a status result and any command output
00117     
00118     Passes a command to the system and returns a True/False result once the 
00119     command has been executed, indicating success/failure. If the command was 
00120     successful then the output from the command is returned to the caller.
00121     Commands are passed as a string. 
00122     E.g. The command 'git remote -v' would be passed in as "git remote -v"
00123 
00124     Args:
00125     command - system command as a string
00126     exit_on_failure - If True exit the program on failure (default = False)
00127     
00128     Returns:
00129     return_code - True/False indicating the success/failure of the command
00130     output - The output of the command if it was successful, else empty string
00131     """
00132     text = '[Exec] ' + command
00133     userlog.debug(text)
00134     returncode = 0
00135     output = ""
00136     try:
00137         output = subprocess.check_output(command, shell=True)
00138     except subprocess.CalledProcessError as e:
00139         text = "The command " + str(command) + "failed with return code: " + str(e.returncode)
00140         userlog.warning(text)
00141         returncode = e.returncode
00142         if exit_on_failure:
00143             sys.exit(1)
00144     return returncode, output
00145 
00146 
00147 def rmtree_readonly(directory):
00148     """ Deletes a readonly directory tree.
00149         
00150     Args:
00151     directory - tree to delete
00152     """
00153     def remove_readonly(func, path, _):
00154         os.chmod(path, stat.S_IWRITE)
00155         func(path)
00156 
00157     shutil.rmtree(directory, onerror=remove_readonly)
00158 
00159 def find_all_examples(path):
00160     """ Search the path for examples
00161     
00162     Description:
00163     
00164     Searches the path specified for sub-example folders, ie those containing an
00165     mbed-os.lib file. If found adds the path to the sub-example to a list which is 
00166     then returned.
00167         
00168     Args:
00169     path - path to search.
00170     examples - (returned) list of paths to example directories.
00171 
00172     """
00173     examples = []
00174     for root, dirs, files in os.walk(path):
00175         if 'mbed-os.lib' in files:
00176             examples += [root]
00177     
00178     return examples
00179 
00180 def upgrade_single_example(example, tag, directory, ref):
00181     """ Update the mbed-os version for a single example
00182     
00183     Description:
00184     
00185     Updates the mbed-os.lib file in the example specified to correspond to the 
00186     version specified by the GitHub tag supplied. Also deals with 
00187     multiple sub-examples in the GitHub repo, updating them in the same way.
00188         
00189     Args:
00190     example - json example object containing the GitHub repo to update.
00191     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00192     directory - directory path for the example.
00193     ref - SHA corresponding to the supplied tag
00194     returns - True if the upgrade was successful, False otherwise.
00195     
00196     """
00197     cwd = os.getcwd()
00198     os.chdir(directory)
00199     
00200     return_code = False
00201     
00202     if os.path.isfile("mbed-os.lib"):
00203         # Rename command will fail on some OS's if the target file already exist,
00204         # so ensure if it does, it is deleted first.
00205         if os.path.isfile("mbed-os.lib_bak"):
00206             os.remove("mbed-os.lib_bak")
00207         
00208         os.rename("mbed-os.lib", "mbed-os.lib_bak")
00209     else:
00210         userlog.error("Failed to backup mbed-os.lib prior to updating.")
00211         return False
00212     
00213     # mbed-os.lib file contains one line with the following format
00214     # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
00215     lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
00216     updated = False
00217 
00218     # Scan through mbed-os.lib line by line
00219     with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
00220         for line in ip:
00221 
00222             opline = line
00223             
00224             regexp = lib_re.match(line)
00225             if regexp:
00226                 opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
00227                 updated = True
00228     
00229             op.write(opline)
00230 
00231     if updated:
00232         # Setup and run the git add command
00233         cmd = "git add mbed-os.lib"
00234         return_code = run_cmd(cmd)
00235 
00236     os.chdir(cwd)
00237     return not return_code
00238 
00239 def prepare_fork(arm_example):
00240     """ Synchronises a cloned fork to ensure it is up to date with the original. 
00241 
00242     Description:
00243     
00244     This function sets a fork of an ARMmbed repo to be up to date with the 
00245     repo it was forked from. It does this by hard resetting to the ARMmbed
00246     master branch.    
00247         
00248     Args:
00249     arm_example - Full GitHub repo path for original example 
00250     
00251     """
00252 
00253     logstr = "In: " + os.getcwd()
00254     userlog.debug(logstr)
00255 
00256     for cmd in ["git remote add armmbed " + str(arm_example),
00257                 "git fetch armmbed",
00258                 "git reset --hard armmbed/master",
00259                 "git push -f origin"]:
00260         run_cmd(cmd, exit_on_failure=True)
00261 
00262 def prepare_branch(src, dst):
00263     """ Set up at branch ready for use in updating examples 
00264         
00265     Description:
00266     
00267     This function checks whether or not the supplied dst branch exists.
00268     If it does not, the branch is created from the src and pushed to the origin.
00269     The branch is then switched to.
00270     
00271     Args:
00272     src - branch to create the dst branch from
00273     dst - branch to update
00274     
00275     """
00276 
00277     userlog.debug("Preparing branch: %s", dst)
00278 
00279     # Check if branch already exists or not.
00280     # We can use the 'git branch -r' command. This returns all the remote branches for
00281     # the current repo.
00282     # The output consists of a list of lines of the form:
00283     # origin/<branch>
00284     # From these we need to extract just the branch names to a list and then check if 
00285     # the specified dst exists in that list
00286     branches = []
00287     cmd = "git branch -r"
00288     _, output = run_cmd_with_output(cmd, exit_on_failure=True)
00289 
00290     branches = [line.split('/')[1] for line in output.split('\n') if 'origin' in line and not '->' in line]
00291   
00292     if not dst in branches:
00293         
00294         # OOB branch does not exist thus create it, first ensuring we are on 
00295         # the src branch and then check it out
00296 
00297         for cmd in ["git checkout " + str(src),
00298                     "git checkout -b " + str(dst),
00299                     "git push -u origin " + str(dst)]:
00300 
00301             run_cmd(cmd, exit_on_failure=True)
00302 
00303     else: 
00304         cmd = "git checkout " + str(dst)
00305         run_cmd(cmd, exit_on_failure=True)
00306    
00307 def upgrade_example(github, example, tag, ref, user, src, dst, template):
00308     """ Upgrade all versions of mbed-os.lib found in the specified example repo
00309     
00310     Description:
00311     
00312     Clone a version of the example specified and upgrade all versions of 
00313     mbed-os.lib found within its tree.  The version cloned and how it 
00314     is upgraded depends on the user, src and dst settings.
00315     1) user == None 
00316        The destination branch will be updated with the version of mbed-os 
00317        idenfied by the tag. If the destination branch does not exist then it
00318        will be created from the source branch.
00319 
00320     2) user != None 
00321        The master branch of a fork of the example will be updated with the 
00322        version of mbed-os identified by the tag.
00323             
00324     Args:
00325     github - GitHub instance to allow internal git commands to be run
00326     example - json example object containing the GitHub repo to update.
00327     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00328     ref - SHA corresponding to the tag
00329     user - GitHub user name 
00330     src - branch to create the dst branch from
00331     dst - branch to update
00332     
00333     returns True if the upgrade was successful, False otherwise
00334     """
00335     
00336     # If a user has not been specified then branch update will be used and thus
00337     # the git user will be ARMmbed.        
00338     if not user:
00339         user = 'ARMmbed'
00340                 
00341     ret = False
00342     userlog.info("Updating example '%s'", example['name'])
00343     userlog.debug("User: %s", user)
00344     userlog.debug("Src branch: %s", (src or "None"))
00345     userlog.debug("Dst branch: %s", (dst or "None"))
00346 
00347     cwd = os.getcwd()
00348 
00349     update_repo = "https://github.com/" + user + '/' + example['name'] 
00350     userlog.debug("Update repository: %s", update_repo)
00351 
00352     # Clone the example repo
00353     clone_cmd = "git clone " +  str(update_repo)
00354     return_code = run_cmd(clone_cmd)
00355     
00356     if not return_code:
00357     
00358         # Find all examples
00359         example_directories = find_all_examples(example['name'])
00360         
00361         os.chdir(example['name'])
00362         
00363         # If the user is ARMmbed then a branch is used.
00364         if user == 'ARMmbed':
00365             prepare_branch(src, dst)    
00366         else:          
00367             prepare_fork(example['github'])
00368             
00369         for example_directory in example_directories:
00370             if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
00371                 os.chdir(cwd)
00372                 return False
00373 
00374         # Setup and run the commit command
00375         commit_cmd = "git commit -m \"Updating mbed-os to " + tag + "\""
00376         return_code = run_cmd(commit_cmd)
00377         if not return_code:
00378 
00379             # Setup and run the push command
00380             push_cmd = "git push origin"
00381             return_code = run_cmd(push_cmd)
00382             
00383             if not return_code:
00384                 # If the user is not ARMmbed then a fork is being used
00385                 if user != 'ARMmbed': 
00386                     
00387                     upstream_repo = 'ARMmbed/'+ example['name']
00388                     userlog.debug("Upstream repository: %s", upstream_repo)
00389                     # Check access to mbed-os repo
00390                     try:
00391                         repo = github.get_repo(upstream_repo, False)
00392 
00393                     except:
00394                         userlog.error("Upstream repo: %s, does not exist - skipping", upstream_repo)
00395                         return False
00396            
00397                     jinja_loader = FileSystemLoader(template)
00398                     jinja_environment = Environment(loader=jinja_loader,
00399                                                     undefined=StrictUndefined)
00400                     pr_body = jinja_environment.get_template("pr.tmpl").render(tag=tag)
00401 
00402                     # Raise a PR from release-candidate to master
00403                     user_fork = user + ':master' 
00404                     try:
00405                         pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=pr_body)
00406                         ret = True
00407                     except GithubException as e:
00408                         # Default to False
00409                          userlog.error("Pull request creation failed with error: %s", e)
00410                 else:
00411                     ret = True                    
00412             else:
00413                 userlog.error("Git push command failed.")
00414         else:
00415             userlog.error("Git commit command failed.")
00416     else:
00417         userlog.error("Git clone %s failed", update_repo)
00418 
00419     os.chdir(cwd)
00420     return ret
00421 
00422 def create_work_directory(path):
00423     """ Create a new directory specified in 'path', overwrite if the directory already 
00424         exists.
00425         
00426     Args:
00427     path - directory path to be created. 
00428     
00429     """
00430     if os.path.exists(path):
00431         userlog.info("'%s' directory already exists. Deleting...", path)
00432         rmtree_readonly(path)
00433     
00434     os.makedirs(path)
00435 
00436 def check_update_status(examples, github, tag):
00437     """ Check the status of previously raised update pull requests
00438     
00439     Args:
00440     examples - list of examples which should have had PRs raised against them. 
00441     github - github rest API instance
00442     tag - release tag used for the update
00443     
00444     """
00445 
00446     for example in examples:
00447 
00448         repo_name = ''.join(['ARMmbed/', example['name']])
00449         try:
00450             repo = github.get_repo(repo_name, False)
00451 
00452         except Exception as exc:
00453             text = "Cannot access: " + str(repo_name)
00454             userlog.error(text)
00455             userlog.exception(exc)
00456             sys.exit(1)
00457 
00458         # Create the full repository filter component
00459         org_str = ''.join(['repo:ARMmbed/', example['name']])
00460         filt = ' '.join([org_str, 'is:pr', tag])        
00461         merged = False
00462 
00463         issues = github.search_issues(query=(filt))
00464         pr_list = [repo.get_pull(issue.number) for issue in issues]
00465 
00466         # Should only be one matching PR but just in case, go through paginated list  
00467         for pr in pr_list:
00468             if pr.merged:
00469                 userlog.info("%s - '%s': MERGED", example['name'], pr.title)
00470             elif pr.state == 'open':
00471                 userlog.info("%s - '%s': PENDING", example['name'], pr.title)
00472             elif pr.state == 'closed':
00473                 userlog.info("%s - '%s': CLOSED NOT MERGED", example['name'], pr.title)
00474             else:
00475                 userlog.error("%s: Cannot find a pull request for %s", example['name'], tag)
00476 
00477 if __name__ == '__main__':
00478 
00479     parser = argparse.ArgumentParser(description=__doc__,
00480                                      formatter_class=argparse.RawDescriptionHelpFormatter)
00481     parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
00482     parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
00483     
00484     exclusive = parser.add_mutually_exclusive_group(required=True)
00485     exclusive.add_argument('-f', '--fork', help="Update a fork", action='store_true')
00486     exclusive.add_argument('-b', '--branch', help="Update a branch", action='store_true')
00487     exclusive.add_argument('-s', '--status', help="Show examples update status", action='store_true')
00488     
00489     args = parser.parse_args()
00490 
00491     # Load the config file
00492     with open(os.path.join(os.path.dirname(__file__), args.config_file)) as config:
00493         if not config:
00494             userlog.error("Failed to load config file '%s'", args.config_file)
00495             sys.exit(1)
00496         json_data = json.load(config)
00497         
00498 
00499     github = Github(args.github_token)
00500     config = json_data['update-config']
00501     tag = config['tag']
00502 
00503     user = None
00504     src = "master"
00505     dst = None
00506 
00507     if args.status:
00508         
00509         # This option should only be called after an update has been performed
00510         check_update_status(json_data['examples'], github, tag)
00511         exit(0)
00512 
00513     # Create working directory
00514     create_work_directory('examples')
00515 
00516     if args.fork:
00517         user = config['via-fork']['github-user']
00518     elif args.branch:
00519         src = config['via-branch']['src-branch']
00520         dst = config['via-branch']['dst-branch']
00521     else:
00522         userlog.error("Must specify either -f or -b command line option")
00523         exit(1)    
00524 
00525     # Get the github sha corresponding to the specified mbed-os tag
00526     cmd = "git rev-list -1 " + tag
00527     return_code, ref = run_cmd_with_output(cmd) 
00528 
00529     if return_code:
00530         userlog.error("Could not obtain SHA for tag: %s",  tag)
00531         sys.exit(1)
00532 
00533     # Loop through the examples
00534     failures = []
00535     successes = []
00536     results = {}
00537     template = dirname(abspath(__file__))
00538 
00539     os.chdir('examples')
00540 
00541     for example in json_data['examples']:
00542         # Determine if this example should be updated and if so update any found 
00543         # mbed-os.lib files. 
00544         
00545         result = upgrade_example(github, example, tag, ref, user, src, dst, template)
00546             
00547         if result:
00548             successes += [example['name']]
00549         else:
00550             failures += [example['name']]
00551     
00552     os.chdir('../')
00553     
00554     # Finish the script and report the results
00555     userlog.info("Finished updating examples")
00556     if successes:
00557         for success in successes:
00558             userlog.info(" SUCCEEDED: %s", success)
00559     
00560     if failures:
00561         for fail in failures:
00562             userlog.info(" FAILED: %s", fail)