File content as of revision 0:fbdae7e6d805:

"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.


One repository to update them all
On mbed.org the mbed SDK is split up in multiple repositories, this script takes
care of updating them all.
"""
import sys
from copy import copy
from os import walk, remove, makedirs, getcwd, rmdir, listdir
from os.path import join, abspath, dirname, relpath, exists, isfile, normpath, isdir
from shutil import copyfile
from optparse import OptionParser
import re
import string

ROOT = abspath(join(dirname(__file__), ".."))
sys.path.insert(0, ROOT)

from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
from tools.paths import *
from tools.utils import run_cmd

MBED_URL = "mbed.org"
MBED_USER = "mbed_official"

changed = []
push_remote = True
quiet = False
commit_msg = ''

# Code that does have a mirror in the mbed SDK
# Tuple data: (repo_name, list_of_code_dirs, [team])
# team is optional - if not specified, the code is published under mbed_official
OFFICIAL_CODE = {"mbed-dev" : ["cmsis", "drivers", "hal", "platform", "targets", "mbed.h"]}


# A list of regular expressions that will be checked against each directory
# name and skipped if they match.
IGNORE_DIRS = (
)

IGNORE_FILES = (
    'COPYING',
    '\.md',
    "\.lib",
    "\.bld"
)

def ignore_path(name, reg_exps):
    for r in reg_exps:
        if re.search(r, name):
            return True
    return False

class MbedRepository:
    @staticmethod
    def run_and_print(command, cwd):
        stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
        print(stdout)

    def __init__(self, name):
        self.name = name
        self.path = join(MBED_ORG_PATH, name)
        self.url = "http://" + MBED_URL + "/users/" + MBED_ORG_USER + "/code/%s/"

        if not exists(self.path):
            # Checkout code
            if not exists(MBED_ORG_PATH):
                makedirs(MBED_ORG_PATH)

            self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)

        else:
            # Update
            self.run_and_print(['hg', 'pull'], cwd=self.path)
            self.run_and_print(['hg', 'update'], cwd=self.path)

    def publish(self):
        # The maintainer has to evaluate the changes first and explicitly accept them
        self.run_and_print(['hg', 'addremove'], cwd=self.path)
        stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
        if stdout == '':
            print "No changes"
            return False
        print stdout
        if quiet:
            commit = 'Y'
        else:
            commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
        if commit == 'Y':
            args = ['hg', 'commit', '-u', MBED_ORG_USER]
            
            
            # NOTE commit_msg should always come from the relevant mbed 2 release text
            if commit_msg:
                args = args + ['-m', commit_msg]
            self.run_and_print(args, cwd=self.path)
            if push_remote:
                self.run_and_print(['hg', 'push'], cwd=self.path)
        return True

# Check if a file is a text file or a binary file
# Taken from http://code.activestate.com/recipes/173220/
text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
_null_trans = string.maketrans("", "")
def is_text_file(filename):
    block_size = 1024
    def istext(s):
        if "\0" in s:
            return 0

        if not s:  # Empty files are considered text
            return 1

        # Get the non-text characters (maps a character to itself then
        # use the 'remove' option to get rid of the text characters.)
        t = s.translate(_null_trans, text_characters)

        # If more than 30% non-text characters, then
        # this is considered a binary file
        if float(len(t))/len(s) > 0.30:
            return 0
        return 1
    with open(filename) as f:
        res = istext(f.read(block_size))
    return res

# Return the line ending type for the given file ('cr' or 'crlf')
def get_line_endings(f):
  examine_size = 1024
  try:
    tf = open(f, "rb")
    lines, ncrlf = tf.readlines(examine_size), 0
    tf.close()
    for l in lines:
      if l.endswith("\r\n"):
        ncrlf = ncrlf + 1
    return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
  except:
    return 'cr'

# Copy file to destination, but preserve destination line endings if possible
# This prevents very annoying issues with huge diffs that appear because of
# differences in line endings
def copy_with_line_endings(sdk_file, repo_file):
    if not isfile(repo_file):
        copyfile(sdk_file, repo_file)
        return
    is_text = is_text_file(repo_file)
    if is_text:
        sdk_le = get_line_endings(sdk_file)
        repo_le = get_line_endings(repo_file)
    if not is_text or sdk_le == repo_le:
        copyfile(sdk_file, repo_file)
    else:
        print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
        f = open(sdk_file, "rb")
        data = f.read()
        f.close()
        f = open(repo_file, "wb")
        data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
        f.write(data)
        f.close()

def visit_files(path, visit):
    for root, dirs, files in walk(path):    
        # Ignore hidden directories
        for d in copy(dirs):
            full = join(root, d)
            if d.startswith('.'):
                dirs.remove(d)
            if ignore_path(full, IGNORE_DIRS):
                print "Skipping '%s'" % full
                dirs.remove(d)

        for file in files:
            if ignore_path(file, IGNORE_FILES):
                continue

            visit(join(root, file))

def visit_dirs(path, visit):

    for root, dirs, files in walk(path, topdown=False):            
        for d in dirs:
            full = join(root, d)
            
            # We don't want to remove the .hg directory
            if not '.hg' in full:
                visit(full)


def update_repo(repo_name, sdk_paths, lib=False):
    repo = MbedRepository(repo_name)
    
    # copy files from mbed SDK to mbed_official repository
    def visit_mbed_sdk(sdk_file):

        # Source files structure is different for the compiled binary lib 
        # compared to the mbed-dev sources
        if lib:
            repo_file = join(repo.path, relpath(sdk_file, sdk_path))            
        else:
            repo_file = join(repo.path, sdk_file)
        repo_dir = dirname(repo_file)
        if not exists(repo_dir):
            print("CREATING: %s" % repo_dir)
            makedirs(repo_dir)

        copy_with_line_endings(sdk_file, repo_file)

    # Go through each path specified in the mbed structure 
    for sdk_path in sdk_paths:

        if isfile(sdk_path):
            # Single file so just copy directly across
            visit_mbed_sdk(sdk_path)
        else:    
            visit_files(sdk_path, visit_mbed_sdk)

    def sdk_remove(repo_path):
        
        print("REMOVING: %s" % repo_path)
        
        # Check if this is an empty directory or a file before determining how to 
        # delete it. As this function should only be called with a directory list
        # after being called with a file list, the directory should automatically
        # be either valid or empty .
        if isfile(repo_path):
            remove(repo_path)
        elif isdir(repo_path) and not listdir(repo_path):
            rmdir(repo_path)
        else:
            print("ERROR: %s is not empty, please remove manually." % repo_path)
            print listdir(repo_path)
            exit(1)

    # remove repository files that do not exist in the mbed SDK
    def visit_lib_repo(repo_path):
        for sdk_path in sdk_paths:
            sdk_file = join(sdk_path, relpath(repo_path, repo.path))
            if not exists(sdk_file):
                sdk_remove(repo_path)

    # remove repository files that do not exist in the mbed SDK source
    def visit_repo(repo_path):

        # work out equivalent sdk path from repo file
        sdk_path = join(getcwd(), relpath(repo_path, repo.path))

        if not exists(sdk_path):
            sdk_remove(repo_path)

    # Go through each path specified in the mbed structure
    # Check if there are any files in any of those paths that are no longer part of the SDK

    if lib:
        visit_files(repo.path, visit_lib_repo)
        # Now do the same for directories that may need to be removed. This needs to be done
        # bottom up to ensure any lower nested directories can be deleted first
        visit_dirs(repo.path, visit_lib_repo)
        
    else:
        visit_files(repo.path, visit_repo)

        # Now do the same for directories that may need to be removed. This needs to be done
        # bottom up to ensure any lower nested directories can be deleted first
        visit_dirs(repo.path, visit_repo)
    
    if repo.publish():
        changed.append(repo_name)


def update_code(repositories):
    for repo_name in repositories.keys():
        sdk_dirs = repositories[repo_name]
        print '\n=== Updating "%s" ===' % repo_name
        update_repo(repo_name, sdk_dirs)


def update_mbed():
    update_repo("mbed", [join(BUILD_DIR, "mbed")], lib=True)

def do_sync(options):
    global push_remote, quiet, commit_msg, changed

    push_remote = not options.nopush
    quiet = options.quiet
    commit_msg = options.msg
    changed = []

    if options.code:
        update_code(OFFICIAL_CODE)

    if options.mbed:
        update_mbed()

    if changed:
        print "Repositories with changes:", changed

    return changed

if __name__ == '__main__':
    parser = OptionParser()

    parser.add_option("-c", "--code",
                  action="store_true",  default=False,
                  help="Update the mbed_official code")

    parser.add_option("-m", "--mbed",
                  action="store_true",  default=False,
                  help="Release a build of the mbed library")

    parser.add_option("-n", "--nopush",
                  action="store_true", default=False,
                  help="Commit the changes locally only, don't push them")

    parser.add_option("", "--commit_message",
                  action="store", type="string", default='', dest='msg',
                  help="Commit message to use for all the commits")

    parser.add_option("-q", "--quiet",
                  action="store_true", default=False,
                  help="Don't ask for confirmation before commiting or pushing")

    (options, args) = parser.parse_args()

    do_sync(options)