synch.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 
00017 
00018 One repository to update them all
00019 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
00020 care of updating them all.
00021 """
00022 import sys
00023 from copy import copy
00024 from os import walk, remove, makedirs
00025 from os.path import join, abspath, dirname, relpath, exists, isfile
00026 from shutil import copyfile
00027 from optparse import OptionParser
00028 import re
00029 import string
00030 
00031 ROOT = abspath(join(dirname(__file__), ".."))
00032 sys.path.insert(0, ROOT)
00033 
00034 from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
00035 from tools.paths import *
00036 from tools.utils import run_cmd
00037 
00038 MBED_URL = "mbed.org"
00039 MBED_USER = "mbed_official"
00040 
00041 changed = []
00042 push_remote = True
00043 quiet = False
00044 commit_msg = ''
00045 
00046 # Code that does have a mirror in the mbed SDK
00047 # Tuple data: (repo_name, list_of_code_dirs, [team])
00048 # team is optional - if not specified, the code is published under mbed_official
00049 OFFICIAL_CODE = (
00050     ("mbed-dev" , MBED_BASE),
00051     ("mbed-rtos", RTOS),
00052     ("mbed-dsp" , DSP),
00053     ("mbed-rpc" , MBED_RPC),
00054 
00055     ("lwip"    , LWIP_SOURCES+"/lwip"),
00056     ("lwip-sys", LWIP_SOURCES+"/lwip-sys"),
00057     ("Socket"  , LWIP_SOURCES+"/Socket"),
00058 
00059     ("lwip-eth"         , ETH_SOURCES+"/lwip-eth"),
00060     ("EthernetInterface", ETH_SOURCES+"/EthernetInterface"),
00061 
00062     ("USBDevice", USB),
00063     ("USBHost"  , USB_HOST),
00064 
00065     ("CellularModem", CELLULAR_SOURCES),
00066     ("CellularUSBModem", CELLULAR_USB_SOURCES),
00067     ("UbloxUSBModem", UBLOX_SOURCES),
00068     ("UbloxModemHTTPClientTest", [TEST_DIR+"/net/cellular/http/common", TEST_DIR+"/net/cellular/http/ubloxusb"]),
00069     ("UbloxModemSMSTest", [TEST_DIR+"/net/cellular/sms/common", TEST_DIR+"/net/cellular/sms/ubloxusb"]),
00070     ("FATFileSystem", FAT_FS, "mbed-official"),
00071 )
00072 
00073 
00074 # Code that does have dependencies to libraries should point to
00075 # the latest revision. By default, they point to a specific revision.
00076 CODE_WITH_DEPENDENCIES = (
00077     # Libraries
00078     "EthernetInterface",
00079 
00080     # RTOS Examples
00081     "rtos_basic",
00082     "rtos_isr",
00083     "rtos_mail",
00084     "rtos_mutex",
00085     "rtos_queue",
00086     "rtos_semaphore",
00087     "rtos_signals",
00088     "rtos_timer",
00089 
00090     # Net Examples
00091     "TCPEchoClient",
00092     "TCPEchoServer",
00093     "TCPSocket_HelloWorld",
00094     "UDPSocket_HelloWorld",
00095     "UDPEchoClient",
00096     "UDPEchoServer",
00097     "BroadcastReceive",
00098     "BroadcastSend",
00099 
00100     # mbed sources
00101     "mbed-src-program",
00102 )
00103 
00104 # A list of regular expressions that will be checked against each directory
00105 # name and skipped if they match.
00106 IGNORE_DIRS = (
00107 )
00108 
00109 IGNORE_FILES = (
00110     'COPYING',
00111     '\.md',
00112     "\.lib",
00113     "\.bld"
00114 )
00115 
00116 def ignore_path(name, reg_exps):
00117     for r in reg_exps:
00118         if re.search(r, name):
00119             return True
00120     return False
00121 
00122 class MbedRepository:
00123     @staticmethod
00124     def run_and_print(command, cwd):
00125         stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
00126         print(stdout)
00127 
00128     def __init__(self, name, team = None):
00129         self.name = name
00130         self.path = join(MBED_ORG_PATH, name)
00131         if team is None:
00132             self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
00133         else:
00134             self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
00135         if not exists(self.path):
00136             # Checkout code
00137             if not exists(MBED_ORG_PATH):
00138                 makedirs(MBED_ORG_PATH)
00139 
00140             self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
00141 
00142         else:
00143             # Update
00144             self.run_and_print(['hg', 'pull'], cwd=self.path)
00145             self.run_and_print(['hg', 'update'], cwd=self.path)
00146 
00147     def publish(self):
00148         # The maintainer has to evaluate the changes first and explicitly accept them
00149         self.run_and_print(['hg', 'addremove'], cwd=self.path)
00150         stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
00151         if stdout == '':
00152             print "No changes"
00153             return False
00154         print stdout
00155         if quiet:
00156             commit = 'Y'
00157         else:
00158             commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
00159         if commit == 'Y':
00160             args = ['hg', 'commit', '-u', MBED_ORG_USER]
00161             if commit_msg:
00162                 args = args + ['-m', commit_msg]
00163             self.run_and_print(args, cwd=self.path)
00164             if push_remote:
00165                 self.run_and_print(['hg', 'push'], cwd=self.path)
00166         return True
00167 
00168 # Check if a file is a text file or a binary file
00169 # Taken from http://code.activestate.com/recipes/173220/
00170 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
00171 _null_trans = string.maketrans("", "")
00172 def is_text_file(filename):
00173     block_size = 1024
00174     def istext(s):
00175         if "\0" in s:
00176             return 0
00177 
00178         if not s:  # Empty files are considered text
00179             return 1
00180 
00181         # Get the non-text characters (maps a character to itself then
00182         # use the 'remove' option to get rid of the text characters.)
00183         t = s.translate(_null_trans, text_characters)
00184 
00185         # If more than 30% non-text characters, then
00186         # this is considered a binary file
00187         if float(len(t))/len(s) > 0.30:
00188             return 0
00189         return 1
00190     with open(filename) as f:
00191         res = istext(f.read(block_size))
00192     return res
00193 
00194 # Return the line ending type for the given file ('cr' or 'crlf')
00195 def get_line_endings(f):
00196   examine_size = 1024
00197   try:
00198     tf = open(f, "rb")
00199     lines, ncrlf = tf.readlines(examine_size), 0
00200     tf.close()
00201     for l in lines:
00202       if l.endswith("\r\n"):
00203         ncrlf = ncrlf + 1
00204     return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
00205   except:
00206     return 'cr'
00207 
00208 # Copy file to destination, but preserve destination line endings if possible
00209 # This prevents very annoying issues with huge diffs that appear because of
00210 # differences in line endings
00211 def copy_with_line_endings(sdk_file, repo_file):
00212     if not isfile(repo_file):
00213         copyfile(sdk_file, repo_file)
00214         return
00215     is_text = is_text_file(repo_file)
00216     if is_text:
00217         sdk_le = get_line_endings(sdk_file)
00218         repo_le = get_line_endings(repo_file)
00219     if not is_text or sdk_le == repo_le:
00220         copyfile(sdk_file, repo_file)
00221     else:
00222         print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
00223         f = open(sdk_file, "rb")
00224         data = f.read()
00225         f.close()
00226         f = open(repo_file, "wb")
00227         data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
00228         f.write(data)
00229         f.close()
00230 
00231 def visit_files(path, visit):
00232     for root, dirs, files in walk(path):
00233         # Ignore hidden directories
00234         for d in copy(dirs):
00235             full = join(root, d)
00236             if d.startswith('.'):
00237                 dirs.remove(d)
00238             if ignore_path(full, IGNORE_DIRS):
00239                 print "Skipping '%s'" % full
00240                 dirs.remove(d)
00241 
00242         for file in files:
00243             if ignore_path(file, IGNORE_FILES):
00244                 continue
00245 
00246             visit(join(root, file))
00247 
00248 
00249 def update_repo(repo_name, sdk_paths, team_name):
00250     repo = MbedRepository(repo_name, team_name)
00251     # copy files from mbed SDK to mbed_official repository
00252     def visit_mbed_sdk(sdk_file):
00253         repo_file = join(repo.path, relpath(sdk_file, sdk_path))
00254 
00255         repo_dir = dirname(repo_file)
00256         if not exists(repo_dir):
00257             makedirs(repo_dir)
00258 
00259         copy_with_line_endings(sdk_file, repo_file)
00260     for sdk_path in sdk_paths:
00261         visit_files(sdk_path, visit_mbed_sdk)
00262 
00263     # remove repository files that do not exist in the mbed SDK
00264     def visit_repo(repo_file):
00265         for sdk_path in sdk_paths:
00266             sdk_file = join(sdk_path, relpath(repo_file, repo.path))
00267             if exists(sdk_file):
00268                 break
00269         else:
00270             remove(repo_file)
00271             print "remove: %s" % repo_file
00272     visit_files(repo.path, visit_repo)
00273 
00274     if repo.publish():
00275         changed.append(repo_name)
00276 
00277 
00278 def update_code(repositories):
00279     for r in repositories:
00280         repo_name, sdk_dir = r[0], r[1]
00281         team_name = r[2] if len(r) == 3 else None
00282         print '\n=== Updating "%s" ===' % repo_name
00283         sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
00284         update_repo(repo_name, sdk_dirs, team_name)
00285 
00286 def update_single_repo(repo):
00287     repos = [r for r in OFFICIAL_CODE if r[0] == repo]
00288     if not repos:
00289         print "Repository '%s' not found" % repo
00290     else:
00291         update_code(repos)
00292 
00293 def update_dependencies(repositories):
00294     for repo_name in repositories:
00295         print '\n=== Updating "%s" ===' % repo_name
00296         repo = MbedRepository(repo_name)
00297 
00298         # point to the latest libraries
00299         def visit_repo(repo_file):
00300             with open(repo_file, "r") as f:
00301                 url = f.read()
00302             with open(repo_file, "w") as f:
00303                 f.write(url[:(url.rindex('/')+1)])
00304         visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)
00305 
00306         if repo.publish():
00307             changed.append(repo_name)
00308 
00309 
00310 def update_mbed():
00311     update_repo("mbed", [join(BUILD_DIR, "mbed")], None)
00312 
00313 def do_sync(options):
00314     global push_remote, quiet, commit_msg, changed
00315 
00316     push_remote = not options.nopush
00317     quiet = options.quiet
00318     commit_msg = options.msg
00319     chnaged = []
00320 
00321     if options.code:
00322         update_code(OFFICIAL_CODE)
00323 
00324     if options.dependencies:
00325         update_dependencies(CODE_WITH_DEPENDENCIES)
00326 
00327     if options.mbed:
00328         update_mbed()
00329 
00330     if options.repo:
00331         update_single_repo(options.repo)
00332 
00333     if changed:
00334         print "Repositories with changes:", changed
00335 
00336     return changed
00337 
00338 if __name__ == '__main__':
00339     parser = OptionParser()
00340 
00341     parser.add_option("-c", "--code",
00342                   action="store_true",  default=False,
00343                   help="Update the mbed_official code")
00344 
00345     parser.add_option("-d", "--dependencies",
00346                   action="store_true",  default=False,
00347                   help="Update the mbed_official code dependencies")
00348 
00349     parser.add_option("-m", "--mbed",
00350                   action="store_true",  default=False,
00351                   help="Release a build of the mbed library")
00352 
00353     parser.add_option("-n", "--nopush",
00354                   action="store_true", default=False,
00355                   help="Commit the changes locally only, don't push them")
00356 
00357     parser.add_option("", "--commit_message",
00358                   action="store", type="string", default='', dest='msg',
00359                   help="Commit message to use for all the commits")
00360 
00361     parser.add_option("-r", "--repository",
00362                   action="store", type="string", default='', dest='repo',
00363                   help="Synchronize only the given repository")
00364 
00365     parser.add_option("-q", "--quiet",
00366                   action="store_true", default=False,
00367                   help="Don't ask for confirmation before commiting or pushing")
00368 
00369     (options, args) = parser.parse_args()
00370 
00371     do_sync(options)
00372