synch.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 
00017 
00018 One repository to update them all
00019 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
00020 care of updating them all.
00021 """
00022 import sys
00023 from copy import copy
00024 from os import walk, remove, makedirs, getcwd, rmdir, listdir
00025 from os.path import join, abspath, dirname, relpath, exists, isfile, normpath, isdir
00026 from shutil import copyfile
00027 from optparse import OptionParser
00028 import re
00029 import string
00030 
00031 ROOT = abspath(join(dirname(__file__), ".."))
00032 sys.path.insert(0, ROOT)
00033 
00034 from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
00035 from tools.paths import *
00036 from tools.utils import run_cmd
00037 
00038 MBED_URL = "mbed.org"
00039 MBED_USER = "mbed_official"
00040 
00041 changed = []
00042 push_remote = True
00043 quiet = False
00044 commit_msg = ''
00045 
00046 # Code that does have a mirror in the mbed SDK
00047 # Tuple data: (repo_name, list_of_code_dirs, [team])
00048 # team is optional - if not specified, the code is published under mbed_official
00049 OFFICIAL_CODE = {"mbed-dev" : ["cmsis", "drivers", "hal", "platform", "targets", "mbed.h"]}
00050 
00051 
00052 # A list of regular expressions that will be checked against each directory
00053 # name and skipped if they match.
00054 IGNORE_DIRS = (
00055 )
00056 
00057 IGNORE_FILES = (
00058     'COPYING',
00059     '\.md',
00060     "\.lib",
00061     "\.bld"
00062 )
00063 
00064 def ignore_path(name, reg_exps):
00065     for r in reg_exps:
00066         if re.search(r, name):
00067             return True
00068     return False
00069 
00070 class MbedRepository:
00071     @staticmethod
00072     def run_and_print(command, cwd):
00073         stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
00074         print(stdout)
00075 
00076     def __init__(self, name):
00077         self.name = name
00078         self.path = join(MBED_ORG_PATH, name)
00079         self.url = "http://" + MBED_URL + "/users/" + MBED_ORG_USER + "/code/%s/"
00080 
00081         if not exists(self.path):
00082             # Checkout code
00083             if not exists(MBED_ORG_PATH):
00084                 makedirs(MBED_ORG_PATH)
00085 
00086             self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
00087 
00088         else:
00089             # Update
00090             self.run_and_print(['hg', 'pull'], cwd=self.path)
00091             self.run_and_print(['hg', 'update'], cwd=self.path)
00092 
00093     def publish(self):
00094         # The maintainer has to evaluate the changes first and explicitly accept them
00095         self.run_and_print(['hg', 'addremove'], cwd=self.path)
00096         stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
00097         if stdout == '':
00098             print "No changes"
00099             return False
00100         print stdout
00101         if quiet:
00102             commit = 'Y'
00103         else:
00104             commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
00105         if commit == 'Y':
00106             args = ['hg', 'commit', '-u', MBED_ORG_USER]
00107             
00108             
00109             # NOTE commit_msg should always come from the relevant mbed 2 release text
00110             if commit_msg:
00111                 args = args + ['-m', commit_msg]
00112             self.run_and_print(args, cwd=self.path)
00113             if push_remote:
00114                 self.run_and_print(['hg', 'push'], cwd=self.path)
00115         return True
00116 
00117 # Check if a file is a text file or a binary file
00118 # Taken from http://code.activestate.com/recipes/173220/
00119 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
00120 _null_trans = string.maketrans("", "")
00121 def is_text_file(filename):
00122     block_size = 1024
00123     def istext(s):
00124         if "\0" in s:
00125             return 0
00126 
00127         if not s:  # Empty files are considered text
00128             return 1
00129 
00130         # Get the non-text characters (maps a character to itself then
00131         # use the 'remove' option to get rid of the text characters.)
00132         t = s.translate(_null_trans, text_characters)
00133 
00134         # If more than 30% non-text characters, then
00135         # this is considered a binary file
00136         if float(len(t))/len(s) > 0.30:
00137             return 0
00138         return 1
00139     with open(filename) as f:
00140         res = istext(f.read(block_size))
00141     return res
00142 
00143 # Return the line ending type for the given file ('cr' or 'crlf')
00144 def get_line_endings(f):
00145   examine_size = 1024
00146   try:
00147     tf = open(f, "rb")
00148     lines, ncrlf = tf.readlines(examine_size), 0
00149     tf.close()
00150     for l in lines:
00151       if l.endswith("\r\n"):
00152         ncrlf = ncrlf + 1
00153     return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
00154   except:
00155     return 'cr'
00156 
00157 # Copy file to destination, but preserve destination line endings if possible
00158 # This prevents very annoying issues with huge diffs that appear because of
00159 # differences in line endings
00160 def copy_with_line_endings(sdk_file, repo_file):
00161     if not isfile(repo_file):
00162         copyfile(sdk_file, repo_file)
00163         return
00164     is_text = is_text_file(repo_file)
00165     if is_text:
00166         sdk_le = get_line_endings(sdk_file)
00167         repo_le = get_line_endings(repo_file)
00168     if not is_text or sdk_le == repo_le:
00169         copyfile(sdk_file, repo_file)
00170     else:
00171         print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
00172         f = open(sdk_file, "rb")
00173         data = f.read()
00174         f.close()
00175         f = open(repo_file, "wb")
00176         data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
00177         f.write(data)
00178         f.close()
00179 
00180 def visit_files(path, visit):
00181     for root, dirs, files in walk(path):    
00182         # Ignore hidden directories
00183         for d in copy(dirs):
00184             full = join(root, d)
00185             if d.startswith('.'):
00186                 dirs.remove(d)
00187             if ignore_path(full, IGNORE_DIRS):
00188                 print "Skipping '%s'" % full
00189                 dirs.remove(d)
00190 
00191         for file in files:
00192             if ignore_path(file, IGNORE_FILES):
00193                 continue
00194 
00195             visit(join(root, file))
00196 
00197 def visit_dirs(path, visit):
00198 
00199     for root, dirs, files in walk(path, topdown=False):            
00200         for d in dirs:
00201             full = join(root, d)
00202             
00203             # We don't want to remove the .hg directory
00204             if not '.hg' in full:
00205                 visit(full)
00206 
00207 
00208 def update_repo(repo_name, sdk_paths, lib=False):
00209     repo = MbedRepository(repo_name)
00210     
00211     # copy files from mbed SDK to mbed_official repository
00212     def visit_mbed_sdk(sdk_file):
00213 
00214         # Source files structure is different for the compiled binary lib 
00215         # compared to the mbed-dev sources
00216         if lib:
00217             repo_file = join(repo.path, relpath(sdk_file, sdk_path))            
00218         else:
00219             repo_file = join(repo.path, sdk_file)
00220         repo_dir = dirname(repo_file)
00221         if not exists(repo_dir):
00222             print("CREATING: %s" % repo_dir)
00223             makedirs(repo_dir)
00224 
00225         copy_with_line_endings(sdk_file, repo_file)
00226 
00227     # Go through each path specified in the mbed structure 
00228     for sdk_path in sdk_paths:
00229 
00230         if isfile(sdk_path):
00231             # Single file so just copy directly across
00232             visit_mbed_sdk(sdk_path)
00233         else:    
00234             visit_files(sdk_path, visit_mbed_sdk)
00235 
00236     def sdk_remove(repo_path):
00237         
00238         print("REMOVING: %s" % repo_path)
00239         
00240         # Check if this is an empty directory or a file before determining how to 
00241         # delete it. As this function should only be called with a directory list
00242         # after being called with a file list, the directory should automatically
00243         # be either valid or empty .
00244         if isfile(repo_path):
00245             remove(repo_path)
00246         elif isdir(repo_path) and not listdir(repo_path):
00247             rmdir(repo_path)
00248         else:
00249             print("ERROR: %s is not empty, please remove manually." % repo_path)
00250             print listdir(repo_path)
00251             exit(1)
00252 
00253     # remove repository files that do not exist in the mbed SDK
00254     def visit_lib_repo(repo_path):
00255         for sdk_path in sdk_paths:
00256             sdk_file = join(sdk_path, relpath(repo_path, repo.path))
00257             if not exists(sdk_file):
00258                 sdk_remove(repo_path)
00259 
00260     # remove repository files that do not exist in the mbed SDK source
00261     def visit_repo(repo_path):
00262 
00263         # work out equivalent sdk path from repo file
00264         sdk_path = join(getcwd(), relpath(repo_path, repo.path))
00265 
00266         if not exists(sdk_path):
00267             sdk_remove(repo_path)
00268 
00269     # Go through each path specified in the mbed structure
00270     # Check if there are any files in any of those paths that are no longer part of the SDK
00271 
00272     if lib:
00273         visit_files(repo.path, visit_lib_repo)
00274         # Now do the same for directories that may need to be removed. This needs to be done
00275         # bottom up to ensure any lower nested directories can be deleted first
00276         visit_dirs(repo.path, visit_lib_repo)
00277         
00278     else:
00279         visit_files(repo.path, visit_repo)
00280 
00281         # Now do the same for directories that may need to be removed. This needs to be done
00282         # bottom up to ensure any lower nested directories can be deleted first
00283         visit_dirs(repo.path, visit_repo)
00284     
00285     if repo.publish():
00286         changed.append(repo_name)
00287 
00288 
00289 def update_code(repositories):
00290     for repo_name in repositories.keys():
00291         sdk_dirs = repositories[repo_name]
00292         print '\n=== Updating "%s" ===' % repo_name
00293         update_repo(repo_name, sdk_dirs)
00294 
00295 
00296 def update_mbed():
00297     update_repo("mbed", [join(BUILD_DIR, "mbed")], lib=True)
00298 
00299 def do_sync(options):
00300     global push_remote, quiet, commit_msg, changed
00301 
00302     push_remote = not options.nopush
00303     quiet = options.quiet
00304     commit_msg = options.msg
00305     changed = []
00306 
00307     if options.code:
00308         update_code(OFFICIAL_CODE)
00309 
00310     if options.mbed:
00311         update_mbed()
00312 
00313     if changed:
00314         print "Repositories with changes:", changed
00315 
00316     return changed
00317 
00318 if __name__ == '__main__':
00319     parser = OptionParser()
00320 
00321     parser.add_option("-c", "--code",
00322                   action="store_true",  default=False,
00323                   help="Update the mbed_official code")
00324 
00325     parser.add_option("-m", "--mbed",
00326                   action="store_true",  default=False,
00327                   help="Release a build of the mbed library")
00328 
00329     parser.add_option("-n", "--nopush",
00330                   action="store_true", default=False,
00331                   help="Commit the changes locally only, don't push them")
00332 
00333     parser.add_option("", "--commit_message",
00334                   action="store", type="string", default='', dest='msg',
00335                   help="Commit message to use for all the commits")
00336 
00337     parser.add_option("-q", "--quiet",
00338                   action="store_true", default=False,
00339                   help="Don't ask for confirmation before commiting or pushing")
00340 
00341     (options, args) = parser.parse_args()
00342 
00343     do_sync(options)
00344