synch.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 
00017 
00018 One repository to update them all
00019 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
00020 care of updating them all.
00021 """
00022 import sys
00023 from copy import copy
00024 from os import walk, remove, makedirs
00025 from os.path import join, abspath, dirname, relpath, exists, isfile
00026 from shutil import copyfile
00027 from optparse import OptionParser
00028 import re
00029 import string
00030 
00031 ROOT = abspath(join(dirname(__file__), ".."))
00032 sys.path.insert(0, ROOT)
00033 
00034 from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
00035 from tools.paths import LIB_DIR
00036 from tools.utils import run_cmd
00037 
00038 MBED_URL = "mbed.org"
00039 MBED_USER = "mbed_official"
00040 
00041 changed = []
00042 push_remote = True
00043 quiet = False
00044 commit_msg = ''
00045 
00046 # Code that does have a mirror in the mbed SDK
00047 # Tuple data: (repo_name, list_of_code_dirs, [team])
00048 # team is optional - if not specified, the code is published under mbed_official
00049 OFFICIAL_CODE = (
00050     ("mbed-dev" , "mbed"),
00051     ("mbed-rtos", "rtos"),
00052     ("mbed-dsp" , "dsp"),
00053     ("mbed-rpc" , "rpc"),
00054 
00055     ("lwip"    , "net/lwip/lwip"),
00056     ("lwip-sys", "net/lwip/lwip-sys"),
00057     ("Socket"  , "net/lwip/Socket"),
00058 
00059     ("lwip-eth"         , "net/eth/lwip-eth"),
00060     ("EthernetInterface", "net/eth/EthernetInterface"),
00061 
00062     ("USBDevice", "USBDevice"),
00063     ("USBHost"  , "USBHost"),
00064 
00065     ("CellularModem", "net/cellular/CellularModem"),
00066     ("CellularUSBModem", "net/cellular/CellularUSBModem"),
00067     ("UbloxUSBModem", "net/cellular/UbloxUSBModem"),
00068     ("UbloxModemHTTPClientTest", ["tests/net/cellular/http/common", "tests/net/cellular/http/ubloxusb"]),
00069     ("UbloxModemSMSTest", ["tests/net/cellular/sms/common", "tests/net/cellular/sms/ubloxusb"]),
00070     ("FATFileSystem", "fs/fat", "mbed-official"),
00071 )
00072 
00073 
00074 # Code that does have dependencies to libraries should point to
00075 # the latest revision. By default, they point to a specific revision.
00076 CODE_WITH_DEPENDENCIES = (
00077     # Libraries
00078     "EthernetInterface",
00079 
00080     # RTOS Examples
00081     "rtos_basic",
00082     "rtos_isr",
00083     "rtos_mail",
00084     "rtos_mutex",
00085     "rtos_queue",
00086     "rtos_semaphore",
00087     "rtos_signals",
00088     "rtos_timer",
00089 
00090     # Net Examples
00091     "TCPEchoClient",
00092     "TCPEchoServer",
00093     "TCPSocket_HelloWorld",
00094     "UDPSocket_HelloWorld",
00095     "UDPEchoClient",
00096     "UDPEchoServer",
00097     "BroadcastReceive",
00098     "BroadcastSend",
00099 
00100     # mbed sources
00101     "mbed-src-program",
00102 )
00103 
00104 # A list of regular expressions that will be checked against each directory
00105 # name and skipped if they match.
00106 IGNORE_DIRS = (
00107 )
00108 
00109 IGNORE_FILES = (
00110     'COPYING',
00111     '\.md',
00112     "\.lib",
00113     "\.bld"
00114 )
00115 
00116 def ignore_path(name, reg_exps):
00117     for r in reg_exps:
00118         if re.search(r, name):
00119             return True
00120     return False
00121 
00122 class MbedRepository:
00123     @staticmethod
00124     def run_and_print(command, cwd):
00125         stdout, _, _ = run_cmd(command, wd=cwd, redirect=True)
00126         print(stdout)
00127 
00128     def __init__(self, name, team = None):
00129         self.name = name
00130         self.path = join(MBED_ORG_PATH, name)
00131         if team is None:
00132             self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
00133         else:
00134             self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
00135         if not exists(self.path):
00136             # Checkout code
00137             if not exists(MBED_ORG_PATH):
00138                 makedirs(MBED_ORG_PATH)
00139 
00140             self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
00141 
00142         else:
00143             # Update
00144             self.run_and_print(['hg', 'pull'], cwd=self.path)
00145             self.run_and_print(['hg', 'update'], cwd=self.path)
00146 
00147     def publish(self):
00148         # The maintainer has to evaluate the changes first and explicitly accept them
00149         self.run_and_print(['hg', 'addremove'], cwd=self.path)
00150         stdout, _, _ = run_cmd(['hg', 'status'], wd=self.path)
00151         if stdout == '':
00152             print "No changes"
00153             return False
00154         print stdout
00155         if quiet:
00156             commit = 'Y'
00157         else:
00158             commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
00159         if commit == 'Y':
00160             args = ['hg', 'commit', '-u', MBED_ORG_USER]
00161             if commit_msg:
00162                 args = args + ['-m', commit_msg]
00163             self.run_and_print(args, cwd=self.path)
00164             if push_remote:
00165                 self.run_and_print(['hg', 'push'], cwd=self.path)
00166         return True
00167 
00168 # Check if a file is a text file or a binary file
00169 # Taken from http://code.activestate.com/recipes/173220/
00170 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
00171 _null_trans = string.maketrans("", "")
00172 def is_text_file(filename):
00173     block_size = 1024
00174     def istext(s):
00175         if "\0" in s:
00176             return 0
00177 
00178         if not s:  # Empty files are considered text
00179             return 1
00180 
00181         # Get the non-text characters (maps a character to itself then
00182         # use the 'remove' option to get rid of the text characters.)
00183         t = s.translate(_null_trans, text_characters)
00184 
00185         # If more than 30% non-text characters, then
00186         # this is considered a binary file
00187         if float(len(t))/len(s) > 0.30:
00188             return 0
00189         return 1
00190     with open(filename) as f:
00191         res = istext(f.read(block_size))
00192     return res
00193 
00194 # Return the line ending type for the given file ('cr' or 'crlf')
00195 def get_line_endings(f):
00196   examine_size = 1024
00197   try:
00198     tf = open(f, "rb")
00199     lines, ncrlf = tf.readlines(examine_size), 0
00200     tf.close()
00201     for l in lines:
00202       if l.endswith("\r\n"):
00203         ncrlf = ncrlf + 1
00204     return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
00205   except:
00206     return 'cr'
00207 
00208 # Copy file to destination, but preserve destination line endings if possible
00209 # This prevents very annoying issues with huge diffs that appear because of
00210 # differences in line endings
00211 def copy_with_line_endings(sdk_file, repo_file):
00212     if not isfile(repo_file):
00213         copyfile(sdk_file, repo_file)
00214         return
00215     is_text = is_text_file(repo_file)
00216     if is_text:
00217         sdk_le = get_line_endings(sdk_file)
00218         repo_le = get_line_endings(repo_file)
00219     if not is_text or sdk_le == repo_le:
00220         copyfile(sdk_file, repo_file)
00221     else:
00222         print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
00223         f = open(sdk_file, "rb")
00224         data = f.read()
00225         f.close()
00226         f = open(repo_file, "wb")
00227         data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
00228         f.write(data)
00229         f.close()
00230 
00231 def visit_files(path, visit):
00232     for root, dirs, files in walk(path):
00233         # Ignore hidden directories
00234         for d in copy(dirs):
00235             full = join(root, d)
00236             if d.startswith('.'):
00237                 dirs.remove(d)
00238             if ignore_path(full, IGNORE_DIRS):
00239                 print "Skipping '%s'" % full
00240                 dirs.remove(d)
00241 
00242         for file in files:
00243             if ignore_path(file, IGNORE_FILES):
00244                 continue
00245 
00246             visit(join(root, file))
00247 
00248 
00249 def update_repo(repo_name, sdk_paths, team_name):
00250     repo = MbedRepository(repo_name, team_name)
00251     # copy files from mbed SDK to mbed_official repository
00252     def visit_mbed_sdk(sdk_file):
00253         repo_file = join(repo.path, relpath(sdk_file, sdk_path))
00254 
00255         repo_dir = dirname(repo_file)
00256         if not exists(repo_dir):
00257             makedirs(repo_dir)
00258 
00259         copy_with_line_endings(sdk_file, repo_file)
00260     for sdk_path in sdk_paths:
00261         visit_files(sdk_path, visit_mbed_sdk)
00262 
00263     # remove repository files that do not exist in the mbed SDK
00264     def visit_repo(repo_file):
00265         for sdk_path in sdk_paths:
00266             sdk_file = join(sdk_path, relpath(repo_file, repo.path))
00267             if exists(sdk_file):
00268                 break
00269         else:
00270             remove(repo_file)
00271             print "remove: %s" % repo_file
00272     visit_files(repo.path, visit_repo)
00273 
00274     if repo.publish():
00275         changed.append(repo_name)
00276 
00277 
00278 def update_code(repositories):
00279     for r in repositories:
00280         repo_name, sdk_dir = r[0], r[1]
00281         team_name = r[2] if len(r) == 3 else None
00282         print '\n=== Updating "%s" ===' % repo_name
00283         sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
00284         sdk_path = [join(LIB_DIR, d) for d in sdk_dirs]
00285         update_repo(repo_name, sdk_path, team_name)
00286 
00287 def update_single_repo(repo):
00288     repos = [r for r in OFFICIAL_CODE if r[0] == repo]
00289     if not repos:
00290         print "Repository '%s' not found" % repo
00291     else:
00292         update_code(repos)
00293 
00294 def update_dependencies(repositories):
00295     for repo_name in repositories:
00296         print '\n=== Updating "%s" ===' % repo_name
00297         repo = MbedRepository(repo_name)
00298 
00299         # point to the latest libraries
00300         def visit_repo(repo_file):
00301             with open(repo_file, "r") as f:
00302                 url = f.read()
00303             with open(repo_file, "w") as f:
00304                 f.write(url[:(url.rindex('/')+1)])
00305         visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)
00306 
00307         if repo.publish():
00308             changed.append(repo_name)
00309 
00310 
00311 def update_mbed():
00312     update_repo("mbed", [join(BUILD_DIR, "mbed")], None)
00313 
00314 def do_sync(options):
00315     global push_remote, quiet, commit_msg, changed
00316 
00317     push_remote = not options.nopush
00318     quiet = options.quiet
00319     commit_msg = options.msg
00320     chnaged = []
00321 
00322     if options.code:
00323         update_code(OFFICIAL_CODE)
00324 
00325     if options.dependencies:
00326         update_dependencies(CODE_WITH_DEPENDENCIES)
00327 
00328     if options.mbed:
00329         update_mbed()
00330 
00331     if options.repo:
00332         update_single_repo(options.repo)
00333 
00334     if changed:
00335         print "Repositories with changes:", changed
00336 
00337     return changed
00338 
00339 if __name__ == '__main__':
00340     parser = OptionParser()
00341 
00342     parser.add_option("-c", "--code",
00343                   action="store_true",  default=False,
00344                   help="Update the mbed_official code")
00345 
00346     parser.add_option("-d", "--dependencies",
00347                   action="store_true",  default=False,
00348                   help="Update the mbed_official code dependencies")
00349 
00350     parser.add_option("-m", "--mbed",
00351                   action="store_true",  default=False,
00352                   help="Release a build of the mbed library")
00353 
00354     parser.add_option("-n", "--nopush",
00355                   action="store_true", default=False,
00356                   help="Commit the changes locally only, don't push them")
00357 
00358     parser.add_option("", "--commit_message",
00359                   action="store", type="string", default='', dest='msg',
00360                   help="Commit message to use for all the commits")
00361 
00362     parser.add_option("-r", "--repository",
00363                   action="store", type="string", default='', dest='repo',
00364                   help="Synchronize only the given repository")
00365 
00366     parser.add_option("-q", "--quiet",
00367                   action="store_true", default=False,
00368                   help="Don't ask for confirmation before commiting or pushing")
00369 
00370     (options, args) = parser.parse_args()
00371 
00372     do_sync(options)
00373