update.py

00001 #!/usr/bin/env python
00002 
00003 import os
00004 from os.path import dirname, abspath, basename
00005 import sys
00006 import argparse
00007 import json
00008 import subprocess
00009 import shutil
00010 import stat
00011 import re
00012 from github import Github, GithubException
00013 
00014 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00015 sys.path.insert(0, ROOT)
00016 
00017 import examples_lib as lib
00018 from examples_lib import SUPPORTED_TOOLCHAINS
00019 
00020 def run_cmd(command, print_warning_on_fail=True):
00021     """ Takes the command specified and runs it in a sub-process, obtaining the return code.
00022         
00023     Args:
00024     command - command to run, provided as a list of individual fields which are combined into a 
00025               single command before passing to the sub-process call.
00026     return_code - result of the command.
00027 
00028     """
00029     print('[Exec] %s' % ' '.join(command))
00030     return_code = subprocess.call(command)
00031     
00032     if return_code:
00033         print("The command '%s' failed with return code: %s" % (' '.join(command), return_code))
00034         print("Ignoring and moving on to the next example")
00035     
00036     return return_code
00037     
00038 def run_cmd_with_output(command, print_warning_on_fail=True):
00039     """ Takes the command specified and runs it in a sub-process, obtaining the return code 
00040         and the returned output.
00041         
00042     Args:
00043     command - command to run, provided as a list of individual fields which are combined into a 
00044               single command before passing to the sub-process call.
00045     return_code - result of the command.
00046     output - the output of the command
00047 
00048     """
00049     print('[Exec] %s' % ' '.join(command))
00050     returncode = 0
00051     output = None
00052     try:
00053         output = subprocess.check_output(command)
00054     except subprocess.CalledProcessError as e:
00055         print("The command '%s' failed with return code: %s" % (' '.join(command), e.returncode))
00056         returncode = e.returncode
00057     return returncode, output
00058 
00059 def rmtree_readonly(directory):
00060     """ Deletes a readonly directory tree.
00061         
00062     Args:
00063     directory - tree to delete
00064     """
00065     def remove_readonly(func, path, _):
00066         os.chmod(path, stat.S_IWRITE)
00067         func(path)
00068 
00069     shutil.rmtree(directory, onerror=remove_readonly)
00070 
00071 def find_all_examples(path):
00072     """ Searches the path specified for sub-example folders, ie those containing an
00073         mbed-os.lib file. If found adds the path to the sub-example to a list which is 
00074         then returned.
00075         
00076     Args:
00077     path - path to search.
00078     examples - (returned) list of paths to example directories.
00079 
00080     """
00081     examples = []
00082     for root, dirs, files in os.walk(path):
00083         if 'mbed-os.lib' in files:
00084             examples += [root]
00085     
00086     return examples
00087 
00088 def upgrade_single_example(example, tag, directory, ref):
00089     """ Updates the mbed-os.lib file in the example specified to correspond to the 
00090         version specified by the GitHub tag supplied. Also deals with 
00091         multiple sub-examples in the GitHub repo, updating them in the same way.
00092         
00093     Args:
00094     example - json example object containing the GitHub repo to update.
00095     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00096     directory - directory path for the example.
00097     ref - SHA corresponding to the supplied tag
00098     returns - True if the upgrade was successful, False otherwise.
00099     
00100     """
00101     cwd = os.getcwd()
00102     os.chdir(directory)
00103     
00104     return_code = False
00105     
00106     if os.path.isfile("mbed-os.lib"):
00107         # Rename command will fail on some OS's if the target file already exist,
00108         # so ensure if it does, it is deleted first.
00109         if os.path.isfile("mbed-os.lib_bak"):
00110             os.remove("mbed-os.lib_bak")
00111         
00112         os.rename("mbed-os.lib", "mbed-os.lib_bak")
00113     else:
00114         print("!! Error trying to backup mbed-os.lib prior to updating.")
00115         return False
00116     
00117     # mbed-os.lib file contains one line with the following format
00118     # e.g. https://github.com/ARMmbed/mbed-os/#0789928ee7f2db08a419fa4a032fffd9bd477aa7
00119     lib_re = re.compile('https://github.com/ARMmbed/mbed-os/#[A-Za-z0-9]+')
00120     updated = False
00121 
00122     # Scan through mbed-os.lib line by line
00123     with open('mbed-os.lib_bak', 'r') as ip, open('mbed-os.lib', 'w') as op:
00124         for line in ip:
00125 
00126             opline = line
00127             
00128             regexp = lib_re.match(line)
00129             if regexp:
00130                 opline = 'https://github.com/ARMmbed/mbed-os/#' + ref
00131                 updated = True
00132     
00133             op.write(opline)
00134 
00135     if updated:
00136         # Setup and run the git add command
00137         cmd = ['git', 'add', 'mbed-os.lib']
00138         return_code = run_cmd(cmd)
00139 
00140     os.chdir(cwd)
00141     return not return_code
00142 
00143 def prepare_fork(arm_example):
00144     """ Synchronises a cloned fork to ensure it is up to date with the original. 
00145         
00146     Args:
00147     arm_example - Full GitHub repo path for original example 
00148     ret - True if the fork was synchronised successfully, False otherwise
00149     
00150     """
00151 
00152     print "In " + os.getcwd()
00153 
00154     for cmd in [['git', 'remote', 'add', 'armmbed', arm_example],
00155                 ['git', 'fetch', 'armmbed'],
00156                 ['git', 'reset', '--hard', 'armmbed/master'],
00157                 ['git', 'push', '-f', 'origin']]:
00158         if run_cmd(cmd):
00159             print("preparation of the fork failed!")
00160             return False
00161     return True
00162 
00163 
00164 def upgrade_example(github, example, tag, user, ref):
00165     """ Clone a fork of the example specified.
00166         Ensures the fork is up to date with the original and then and updates the associated 
00167         mbed-os.lib file on that fork to correspond to the version specified by the GitHub tag supplied. 
00168         Also deals with multiple sub-examples in the GitHub repo, updating them in the same way.
00169         The updates are pushed to the forked repo.
00170         Finally a PR is raised against the original example repo for the changes.
00171         
00172     Args:
00173     github - GitHub instance to allow internal git commands to be run
00174     example - json example object containing the GitHub repo to update.
00175     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00176     user - GitHub user name
00177     ref - SHA corresponding to the tag
00178     
00179     """
00180     ret = False
00181     print("\nUpdating example '%s'" % example['name'])
00182     cwd = os.getcwd()
00183 
00184     full_repo_name = 'ARMmbed/'+ example['name']
00185     fork = "https://github.com/" + user + '/' + example['name'] 
00186 
00187     # Check access to mbed-os repo
00188     try:
00189         repo = github.get_repo(full_repo_name, False)
00190 
00191     except:
00192         print("\t\t!! Repo does not exist - skipping\n")
00193         return False
00194 
00195 
00196     # Clone the forked example repo
00197     clone_cmd = ['git', 'clone', fork]
00198     return_code = run_cmd(clone_cmd)
00199     
00200     if not return_code:
00201     
00202         # Find all examples
00203         example_directories = find_all_examples(example['name'])
00204         
00205         os.chdir(example['name'])
00206         
00207         # checkout and synchronise the release-candidate branch
00208         prepare_fork(example['github'])
00209         
00210         for example_directory in example_directories:
00211             if not upgrade_single_example(example, tag, os.path.relpath(example_directory, example['name']), ref):
00212                 os.chdir(cwd)
00213                 return False
00214 
00215         # Setup the default commit message
00216         commit_message = 'Updating mbed-os to ' + tag 
00217 
00218         # Setup and run the commit command
00219         commit_cmd = ['git', 'commit', '-m', commit_message]
00220         return_code = run_cmd(commit_cmd)
00221         if not return_code:
00222 
00223             # Setup and run the push command
00224             push_cmd = ['git', 'push', 'origin']
00225             return_code = run_cmd(push_cmd)
00226             
00227             if not return_code:
00228                 body = "Please test/merge this PR and then tag Master with " + tag
00229                 # Raise a PR from release-candidate to master
00230                 user_fork = user + ':master' 
00231                 try:
00232                     pr = repo.create_pull(title='Updating mbed-os to ' + tag, head=user_fork, base='master', body=body)
00233                     ret = True
00234                 except GithubException as e:
00235                     # Default to False
00236                     print("Creation of Pull Request from release-candidate to master failed with the following error!")
00237                     print e
00238             else:
00239                 print("!!! Git push command failed.")
00240         else:
00241             print("!!! Git commit command failed.")
00242     else:
00243         print("!!! Could not clone user fork %s\n" % fork)
00244 
00245 
00246     os.chdir(cwd)
00247     return ret
00248 
00249 def create_work_directory(path):
00250     """ Create a new directory specified in 'path', overwrite if the directory already 
00251         exists.
00252         
00253     Args:
00254     path - directory path to be created. 
00255     
00256     """
00257     if os.path.exists(path):
00258         print("'%s' directory already exists. Deleting..." % path)
00259         rmtree_readonly(path)
00260     
00261     os.makedirs(path)
00262 
00263 def test_compile(config, tag):
00264     """ For each example repo identified in the config json object, clone, update mbed-os to
00265         the specified tag and then compile for all supported toolchains.
00266         
00267     Args:
00268     config - the json object imported from the file. 
00269     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00270     results - summary of compilation results. 
00271     
00272     """
00273     # Create work directories
00274     create_work_directory('test_compile')
00275     
00276     # Loop through the examples
00277     results = {}
00278     os.chdir('test_compile')
00279 
00280     lib.source_repos(config)    
00281     lib.update_mbedos_version(config, tag)
00282     results = lib.compile_repos(config, SUPPORTED_TOOLCHAINS)
00283     os.chdir("..")
00284 
00285     return results
00286     
00287 
00288 def main(arguments):
00289     """ Will update any mbed-os.lib files found in the example list specified by the config file.
00290         If no config file is specified the default 'examples.json' is used. 
00291         The update is done by cloning a fork of each example (the fork must be present in the 
00292         github account specified by the github user parameter). The fork is searched for any
00293         mbed-os.lib files and each one found is updated with the SHA corresponding to the supplied
00294         github tag. A pull request is then made from the fork to the original example repo.
00295         
00296     Args:
00297     tag - tag to update the mbed-os.lib to. E.g. mbed-os-5.3.1
00298     github_token - Pre-authorised token to allow github access
00299     github_user - github username whose account contains the example forks
00300     config_file - optional parameter to specify a list of examples
00301 
00302     """
00303 
00304     parser = argparse.ArgumentParser(description=__doc__,
00305                                      formatter_class=argparse.RawDescriptionHelpFormatter)
00306     parser.add_argument('tag', help="mbed-os tag to which all examples will be updated")
00307     parser.add_argument('-c', '--config_file', help="Path to the configuration file (default is 'examples.json')", default='examples.json')
00308     parser.add_argument('-T', '--github_token', help="GitHub token for secure access")
00309     parser.add_argument('-U', '--github_user', help="GitHub user for forked repos")
00310     
00311     args = parser.parse_args(arguments)
00312 
00313     cfg = os.path.join(os.path.dirname(__file__), args.config_file)
00314     
00315     # Load the config file
00316     config = json.load(open(os.path.join(os.path.dirname(__file__),
00317                              args.config_file)))
00318     
00319     if not config:
00320         print("Failed to load config file '%s'" % args.config_file)
00321         sys.exit(1)
00322     
00323     # Create working directory
00324     create_work_directory('examples')
00325 
00326     github = Github(args.github_token)
00327 
00328     # Get the github sha corresponding to the specified mbed-os tag
00329     cmd = ['git', 'rev-list', '-1', args.tag]
00330     return_code, ref = run_cmd_with_output(cmd) 
00331 
00332     if return_code:
00333         print("Could not obtain SHA for tag: %s\n" % args.tag)
00334         sys.exit(1)
00335 
00336     # Loop through the examples
00337     failures = []
00338     successes = []
00339     results = {}
00340     os.chdir('examples')
00341     
00342     for example in config['examples']:
00343         # Determine if this example should be updated and if so update any found 
00344         # mbed-os.lib files.
00345 
00346         if upgrade_example(github, example, args.tag, args.github_user, ref):
00347             successes += [example['name']]
00348         else:
00349             failures += [example['name']]
00350     
00351     os.chdir('../')
00352     
00353     # Finish the script and report the results
00354     print(os.linesep + os.linesep +'Finished updating examples!' + os.linesep)
00355     
00356     if successes:
00357         print('\nThe following examples updated successfully:')
00358         for success in successes:
00359             print('    - %s' % success)
00360     
00361     if failures:
00362         print('\nThe following examples were not updated:')
00363         for fail in failures:
00364             print('    - %s' % fail)
00365 
00366 if __name__ == '__main__':
00367     sys.exit(main(sys.argv[1:]))