Daiki Kato / mbed-os-lychee

mbed-os for GR-LYCHEE

Dependents:   mbed-os-example-blinky-gr-lychee GR-Boads_Camera_sample GR-Boards_Audio_Recoder GR-Boads_Camera_DisplayApp ... more

tools/synch.py@0:f782d9c66c49, 2018-02-02 (annotated)

Committer:
dkato
Date:
Fri Feb 02 05:42:23 2018 +0000
Revision:
0:f782d9c66c49
mbed-os for GR-LYCHEE

Who changed what in which revision?

UserRevisionLine numberNew contents of line
dkato 0:f782d9c66c49 1 """
dkato 0:f782d9c66c49 2 mbed SDK
dkato 0:f782d9c66c49 3 Copyright (c) 2011-2013 ARM Limited
dkato 0:f782d9c66c49 4
dkato 0:f782d9c66c49 5 Licensed under the Apache License, Version 2.0 (the "License");
dkato 0:f782d9c66c49 6 you may not use this file except in compliance with the License.
dkato 0:f782d9c66c49 7 You may obtain a copy of the License at
dkato 0:f782d9c66c49 8
dkato 0:f782d9c66c49 9 http://www.apache.org/licenses/LICENSE-2.0
dkato 0:f782d9c66c49 10
dkato 0:f782d9c66c49 11 Unless required by applicable law or agreed to in writing, software
dkato 0:f782d9c66c49 12 distributed under the License is distributed on an "AS IS" BASIS,
dkato 0:f782d9c66c49 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
dkato 0:f782d9c66c49 14 See the License for the specific language governing permissions and
dkato 0:f782d9c66c49 15 limitations under the License.
dkato 0:f782d9c66c49 16
dkato 0:f782d9c66c49 17
dkato 0:f782d9c66c49 18 One repository to update them all
dkato 0:f782d9c66c49 19 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
dkato 0:f782d9c66c49 20 care of updating them all.
dkato 0:f782d9c66c49 21 """
dkato 0:f782d9c66c49 22 import sys
dkato 0:f782d9c66c49 23 from copy import copy
dkato 0:f782d9c66c49 24 from os import walk, remove, makedirs, getcwd, rmdir, listdir
dkato 0:f782d9c66c49 25 from os.path import join, abspath, dirname, relpath, exists, isfile, normpath, isdir
dkato 0:f782d9c66c49 26 from shutil import copyfile
dkato 0:f782d9c66c49 27 from optparse import OptionParser
dkato 0:f782d9c66c49 28 import re
dkato 0:f782d9c66c49 29 import string
dkato 0:f782d9c66c49 30
dkato 0:f782d9c66c49 31 ROOT = abspath(join(dirname(__file__), ".."))
dkato 0:f782d9c66c49 32 sys.path.insert(0, ROOT)
dkato 0:f782d9c66c49 33
dkato 0:f782d9c66c49 34 from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
dkato 0:f782d9c66c49 35 from tools.paths import *
dkato 0:f782d9c66c49 36 from tools.utils import run_cmd
dkato 0:f782d9c66c49 37
dkato 0:f782d9c66c49 38 MBED_URL = "mbed.org"
dkato 0:f782d9c66c49 39 MBED_USER = "mbed_official"
dkato 0:f782d9c66c49 40
dkato 0:f782d9c66c49 41 changed = []
dkato 0:f782d9c66c49 42 push_remote = True
dkato 0:f782d9c66c49 43 quiet = False
dkato 0:f782d9c66c49 44 commit_msg = ''
dkato 0:f782d9c66c49 45
dkato 0:f782d9c66c49 46 # Code that does have a mirror in the mbed SDK
dkato 0:f782d9c66c49 47 # Tuple data: (repo_name, list_of_code_dirs, [team])
dkato 0:f782d9c66c49 48 # team is optional - if not specified, the code is published under mbed_official
dkato 0:f782d9c66c49 49 OFFICIAL_CODE = {"mbed-dev" : ["cmsis", "drivers", "hal", "platform", "targets", "mbed.h"]}
dkato 0:f782d9c66c49 50
dkato 0:f782d9c66c49 51
dkato 0:f782d9c66c49 52 # A list of regular expressions that will be checked against each directory
dkato 0:f782d9c66c49 53 # name and skipped if they match.
dkato 0:f782d9c66c49 54 IGNORE_DIRS = (
dkato 0:f782d9c66c49 55 )
dkato 0:f782d9c66c49 56
dkato 0:f782d9c66c49 57 IGNORE_FILES = (
dkato 0:f782d9c66c49 58 'COPYING',
dkato 0:f782d9c66c49 59 '\.md',
dkato 0:f782d9c66c49 60 "\.lib",
dkato 0:f782d9c66c49 61 "\.bld"
dkato 0:f782d9c66c49 62 )
dkato 0:f782d9c66c49 63
dkato 0:f782d9c66c49 64 def ignore_path(name, reg_exps):
dkato 0:f782d9c66c49 65 for r in reg_exps:
dkato 0:f782d9c66c49 66 if re.search(r, name):
dkato 0:f782d9c66c49 67 return True
dkato 0:f782d9c66c49 68 return False
dkato 0:f782d9c66c49 69
dkato 0:f782d9c66c49 70 class MbedRepository:
dkato 0:f782d9c66c49 71 @staticmethod
dkato 0:f782d9c66c49 72 def run_and_print(command, cwd):
dkato 0:f782d9c66c49 73 stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
dkato 0:f782d9c66c49 74 print(stdout)
dkato 0:f782d9c66c49 75
dkato 0:f782d9c66c49 76 def __init__(self, name):
dkato 0:f782d9c66c49 77 self.name = name
dkato 0:f782d9c66c49 78 self.path = join(MBED_ORG_PATH, name)
dkato 0:f782d9c66c49 79 self.url = "http://" + MBED_URL + "/users/" + MBED_ORG_USER + "/code/%s/"
dkato 0:f782d9c66c49 80
dkato 0:f782d9c66c49 81 if not exists(self.path):
dkato 0:f782d9c66c49 82 # Checkout code
dkato 0:f782d9c66c49 83 if not exists(MBED_ORG_PATH):
dkato 0:f782d9c66c49 84 makedirs(MBED_ORG_PATH)
dkato 0:f782d9c66c49 85
dkato 0:f782d9c66c49 86 self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
dkato 0:f782d9c66c49 87
dkato 0:f782d9c66c49 88 else:
dkato 0:f782d9c66c49 89 # Update
dkato 0:f782d9c66c49 90 self.run_and_print(['hg', 'pull'], cwd=self.path)
dkato 0:f782d9c66c49 91 self.run_and_print(['hg', 'update'], cwd=self.path)
dkato 0:f782d9c66c49 92
dkato 0:f782d9c66c49 93 def publish(self):
dkato 0:f782d9c66c49 94 # The maintainer has to evaluate the changes first and explicitly accept them
dkato 0:f782d9c66c49 95 self.run_and_print(['hg', 'addremove'], cwd=self.path)
dkato 0:f782d9c66c49 96 stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
dkato 0:f782d9c66c49 97 if stdout == '':
dkato 0:f782d9c66c49 98 print "No changes"
dkato 0:f782d9c66c49 99 return False
dkato 0:f782d9c66c49 100 print stdout
dkato 0:f782d9c66c49 101 if quiet:
dkato 0:f782d9c66c49 102 commit = 'Y'
dkato 0:f782d9c66c49 103 else:
dkato 0:f782d9c66c49 104 commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
dkato 0:f782d9c66c49 105 if commit == 'Y':
dkato 0:f782d9c66c49 106 args = ['hg', 'commit', '-u', MBED_ORG_USER]
dkato 0:f782d9c66c49 107
dkato 0:f782d9c66c49 108
dkato 0:f782d9c66c49 109 # NOTE commit_msg should always come from the relevant mbed 2 release text
dkato 0:f782d9c66c49 110 if commit_msg:
dkato 0:f782d9c66c49 111 args = args + ['-m', commit_msg]
dkato 0:f782d9c66c49 112 self.run_and_print(args, cwd=self.path)
dkato 0:f782d9c66c49 113 if push_remote:
dkato 0:f782d9c66c49 114 self.run_and_print(['hg', 'push'], cwd=self.path)
dkato 0:f782d9c66c49 115 return True
dkato 0:f782d9c66c49 116
dkato 0:f782d9c66c49 117 # Check if a file is a text file or a binary file
dkato 0:f782d9c66c49 118 # Taken from http://code.activestate.com/recipes/173220/
dkato 0:f782d9c66c49 119 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
dkato 0:f782d9c66c49 120 _null_trans = string.maketrans("", "")
dkato 0:f782d9c66c49 121 def is_text_file(filename):
dkato 0:f782d9c66c49 122 block_size = 1024
dkato 0:f782d9c66c49 123 def istext(s):
dkato 0:f782d9c66c49 124 if "\0" in s:
dkato 0:f782d9c66c49 125 return 0
dkato 0:f782d9c66c49 126
dkato 0:f782d9c66c49 127 if not s: # Empty files are considered text
dkato 0:f782d9c66c49 128 return 1
dkato 0:f782d9c66c49 129
dkato 0:f782d9c66c49 130 # Get the non-text characters (maps a character to itself then
dkato 0:f782d9c66c49 131 # use the 'remove' option to get rid of the text characters.)
dkato 0:f782d9c66c49 132 t = s.translate(_null_trans, text_characters)
dkato 0:f782d9c66c49 133
dkato 0:f782d9c66c49 134 # If more than 30% non-text characters, then
dkato 0:f782d9c66c49 135 # this is considered a binary file
dkato 0:f782d9c66c49 136 if float(len(t))/len(s) > 0.30:
dkato 0:f782d9c66c49 137 return 0
dkato 0:f782d9c66c49 138 return 1
dkato 0:f782d9c66c49 139 with open(filename) as f:
dkato 0:f782d9c66c49 140 res = istext(f.read(block_size))
dkato 0:f782d9c66c49 141 return res
dkato 0:f782d9c66c49 142
dkato 0:f782d9c66c49 143 # Return the line ending type for the given file ('cr' or 'crlf')
dkato 0:f782d9c66c49 144 def get_line_endings(f):
dkato 0:f782d9c66c49 145 examine_size = 1024
dkato 0:f782d9c66c49 146 try:
dkato 0:f782d9c66c49 147 tf = open(f, "rb")
dkato 0:f782d9c66c49 148 lines, ncrlf = tf.readlines(examine_size), 0
dkato 0:f782d9c66c49 149 tf.close()
dkato 0:f782d9c66c49 150 for l in lines:
dkato 0:f782d9c66c49 151 if l.endswith("\r\n"):
dkato 0:f782d9c66c49 152 ncrlf = ncrlf + 1
dkato 0:f782d9c66c49 153 return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
dkato 0:f782d9c66c49 154 except:
dkato 0:f782d9c66c49 155 return 'cr'
dkato 0:f782d9c66c49 156
dkato 0:f782d9c66c49 157 # Copy file to destination, but preserve destination line endings if possible
dkato 0:f782d9c66c49 158 # This prevents very annoying issues with huge diffs that appear because of
dkato 0:f782d9c66c49 159 # differences in line endings
dkato 0:f782d9c66c49 160 def copy_with_line_endings(sdk_file, repo_file):
dkato 0:f782d9c66c49 161 if not isfile(repo_file):
dkato 0:f782d9c66c49 162 copyfile(sdk_file, repo_file)
dkato 0:f782d9c66c49 163 return
dkato 0:f782d9c66c49 164 is_text = is_text_file(repo_file)
dkato 0:f782d9c66c49 165 if is_text:
dkato 0:f782d9c66c49 166 sdk_le = get_line_endings(sdk_file)
dkato 0:f782d9c66c49 167 repo_le = get_line_endings(repo_file)
dkato 0:f782d9c66c49 168 if not is_text or sdk_le == repo_le:
dkato 0:f782d9c66c49 169 copyfile(sdk_file, repo_file)
dkato 0:f782d9c66c49 170 else:
dkato 0:f782d9c66c49 171 print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
dkato 0:f782d9c66c49 172 f = open(sdk_file, "rb")
dkato 0:f782d9c66c49 173 data = f.read()
dkato 0:f782d9c66c49 174 f.close()
dkato 0:f782d9c66c49 175 f = open(repo_file, "wb")
dkato 0:f782d9c66c49 176 data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
dkato 0:f782d9c66c49 177 f.write(data)
dkato 0:f782d9c66c49 178 f.close()
dkato 0:f782d9c66c49 179
dkato 0:f782d9c66c49 180 def visit_files(path, visit):
dkato 0:f782d9c66c49 181 for root, dirs, files in walk(path):
dkato 0:f782d9c66c49 182 # Ignore hidden directories
dkato 0:f782d9c66c49 183 for d in copy(dirs):
dkato 0:f782d9c66c49 184 full = join(root, d)
dkato 0:f782d9c66c49 185 if d.startswith('.'):
dkato 0:f782d9c66c49 186 dirs.remove(d)
dkato 0:f782d9c66c49 187 if ignore_path(full, IGNORE_DIRS):
dkato 0:f782d9c66c49 188 print "Skipping '%s'" % full
dkato 0:f782d9c66c49 189 dirs.remove(d)
dkato 0:f782d9c66c49 190
dkato 0:f782d9c66c49 191 for file in files:
dkato 0:f782d9c66c49 192 if ignore_path(file, IGNORE_FILES):
dkato 0:f782d9c66c49 193 continue
dkato 0:f782d9c66c49 194
dkato 0:f782d9c66c49 195 visit(join(root, file))
dkato 0:f782d9c66c49 196
dkato 0:f782d9c66c49 197 def visit_dirs(path, visit):
dkato 0:f782d9c66c49 198
dkato 0:f782d9c66c49 199 for root, dirs, files in walk(path, topdown=False):
dkato 0:f782d9c66c49 200 for d in dirs:
dkato 0:f782d9c66c49 201 full = join(root, d)
dkato 0:f782d9c66c49 202
dkato 0:f782d9c66c49 203 # We don't want to remove the .hg directory
dkato 0:f782d9c66c49 204 if not '.hg' in full:
dkato 0:f782d9c66c49 205 visit(full)
dkato 0:f782d9c66c49 206
dkato 0:f782d9c66c49 207
dkato 0:f782d9c66c49 208 def update_repo(repo_name, sdk_paths, lib=False):
dkato 0:f782d9c66c49 209 repo = MbedRepository(repo_name)
dkato 0:f782d9c66c49 210
dkato 0:f782d9c66c49 211 # copy files from mbed SDK to mbed_official repository
dkato 0:f782d9c66c49 212 def visit_mbed_sdk(sdk_file):
dkato 0:f782d9c66c49 213
dkato 0:f782d9c66c49 214 # Source files structure is different for the compiled binary lib
dkato 0:f782d9c66c49 215 # compared to the mbed-dev sources
dkato 0:f782d9c66c49 216 if lib:
dkato 0:f782d9c66c49 217 repo_file = join(repo.path, relpath(sdk_file, sdk_path))
dkato 0:f782d9c66c49 218 else:
dkato 0:f782d9c66c49 219 repo_file = join(repo.path, sdk_file)
dkato 0:f782d9c66c49 220 repo_dir = dirname(repo_file)
dkato 0:f782d9c66c49 221 if not exists(repo_dir):
dkato 0:f782d9c66c49 222 print("CREATING: %s" % repo_dir)
dkato 0:f782d9c66c49 223 makedirs(repo_dir)
dkato 0:f782d9c66c49 224
dkato 0:f782d9c66c49 225 copy_with_line_endings(sdk_file, repo_file)
dkato 0:f782d9c66c49 226
dkato 0:f782d9c66c49 227 # Go through each path specified in the mbed structure
dkato 0:f782d9c66c49 228 for sdk_path in sdk_paths:
dkato 0:f782d9c66c49 229
dkato 0:f782d9c66c49 230 if isfile(sdk_path):
dkato 0:f782d9c66c49 231 # Single file so just copy directly across
dkato 0:f782d9c66c49 232 visit_mbed_sdk(sdk_path)
dkato 0:f782d9c66c49 233 else:
dkato 0:f782d9c66c49 234 visit_files(sdk_path, visit_mbed_sdk)
dkato 0:f782d9c66c49 235
dkato 0:f782d9c66c49 236 def sdk_remove(repo_path):
dkato 0:f782d9c66c49 237
dkato 0:f782d9c66c49 238 print("REMOVING: %s" % repo_path)
dkato 0:f782d9c66c49 239
dkato 0:f782d9c66c49 240 # Check if this is an empty directory or a file before determining how to
dkato 0:f782d9c66c49 241 # delete it. As this function should only be called with a directory list
dkato 0:f782d9c66c49 242 # after being called with a file list, the directory should automatically
dkato 0:f782d9c66c49 243 # be either valid or empty .
dkato 0:f782d9c66c49 244 if isfile(repo_path):
dkato 0:f782d9c66c49 245 remove(repo_path)
dkato 0:f782d9c66c49 246 elif isdir(repo_path) and not listdir(repo_path):
dkato 0:f782d9c66c49 247 rmdir(repo_path)
dkato 0:f782d9c66c49 248 else:
dkato 0:f782d9c66c49 249 print("ERROR: %s is not empty, please remove manually." % repo_path)
dkato 0:f782d9c66c49 250 print listdir(repo_path)
dkato 0:f782d9c66c49 251 exit(1)
dkato 0:f782d9c66c49 252
dkato 0:f782d9c66c49 253 # remove repository files that do not exist in the mbed SDK
dkato 0:f782d9c66c49 254 def visit_lib_repo(repo_path):
dkato 0:f782d9c66c49 255 for sdk_path in sdk_paths:
dkato 0:f782d9c66c49 256 sdk_file = join(sdk_path, relpath(repo_path, repo.path))
dkato 0:f782d9c66c49 257 if not exists(sdk_file):
dkato 0:f782d9c66c49 258 sdk_remove(repo_path)
dkato 0:f782d9c66c49 259
dkato 0:f782d9c66c49 260 # remove repository files that do not exist in the mbed SDK source
dkato 0:f782d9c66c49 261 def visit_repo(repo_path):
dkato 0:f782d9c66c49 262
dkato 0:f782d9c66c49 263 # work out equivalent sdk path from repo file
dkato 0:f782d9c66c49 264 sdk_path = join(getcwd(), relpath(repo_path, repo.path))
dkato 0:f782d9c66c49 265
dkato 0:f782d9c66c49 266 if not exists(sdk_path):
dkato 0:f782d9c66c49 267 sdk_remove(repo_path)
dkato 0:f782d9c66c49 268
dkato 0:f782d9c66c49 269 # Go through each path specified in the mbed structure
dkato 0:f782d9c66c49 270 # Check if there are any files in any of those paths that are no longer part of the SDK
dkato 0:f782d9c66c49 271
dkato 0:f782d9c66c49 272 if lib:
dkato 0:f782d9c66c49 273 visit_files(repo.path, visit_lib_repo)
dkato 0:f782d9c66c49 274 # Now do the same for directories that may need to be removed. This needs to be done
dkato 0:f782d9c66c49 275 # bottom up to ensure any lower nested directories can be deleted first
dkato 0:f782d9c66c49 276 visit_dirs(repo.path, visit_lib_repo)
dkato 0:f782d9c66c49 277
dkato 0:f782d9c66c49 278 else:
dkato 0:f782d9c66c49 279 visit_files(repo.path, visit_repo)
dkato 0:f782d9c66c49 280
dkato 0:f782d9c66c49 281 # Now do the same for directories that may need to be removed. This needs to be done
dkato 0:f782d9c66c49 282 # bottom up to ensure any lower nested directories can be deleted first
dkato 0:f782d9c66c49 283 visit_dirs(repo.path, visit_repo)
dkato 0:f782d9c66c49 284
dkato 0:f782d9c66c49 285 if repo.publish():
dkato 0:f782d9c66c49 286 changed.append(repo_name)
dkato 0:f782d9c66c49 287
dkato 0:f782d9c66c49 288
dkato 0:f782d9c66c49 289 def update_code(repositories):
dkato 0:f782d9c66c49 290 for repo_name in repositories.keys():
dkato 0:f782d9c66c49 291 sdk_dirs = repositories[repo_name]
dkato 0:f782d9c66c49 292 print '\n=== Updating "%s" ===' % repo_name
dkato 0:f782d9c66c49 293 update_repo(repo_name, sdk_dirs)
dkato 0:f782d9c66c49 294
dkato 0:f782d9c66c49 295
dkato 0:f782d9c66c49 296 def update_mbed():
dkato 0:f782d9c66c49 297 update_repo("mbed", [join(BUILD_DIR, "mbed")], lib=True)
dkato 0:f782d9c66c49 298
dkato 0:f782d9c66c49 299 def do_sync(options):
dkato 0:f782d9c66c49 300 global push_remote, quiet, commit_msg, changed
dkato 0:f782d9c66c49 301
dkato 0:f782d9c66c49 302 push_remote = not options.nopush
dkato 0:f782d9c66c49 303 quiet = options.quiet
dkato 0:f782d9c66c49 304 commit_msg = options.msg
dkato 0:f782d9c66c49 305 changed = []
dkato 0:f782d9c66c49 306
dkato 0:f782d9c66c49 307 if options.code:
dkato 0:f782d9c66c49 308 update_code(OFFICIAL_CODE)
dkato 0:f782d9c66c49 309
dkato 0:f782d9c66c49 310 if options.mbed:
dkato 0:f782d9c66c49 311 update_mbed()
dkato 0:f782d9c66c49 312
dkato 0:f782d9c66c49 313 if changed:
dkato 0:f782d9c66c49 314 print "Repositories with changes:", changed
dkato 0:f782d9c66c49 315
dkato 0:f782d9c66c49 316 return changed
dkato 0:f782d9c66c49 317
dkato 0:f782d9c66c49 318 if __name__ == '__main__':
dkato 0:f782d9c66c49 319 parser = OptionParser()
dkato 0:f782d9c66c49 320
dkato 0:f782d9c66c49 321 parser.add_option("-c", "--code",
dkato 0:f782d9c66c49 322 action="store_true", default=False,
dkato 0:f782d9c66c49 323 help="Update the mbed_official code")
dkato 0:f782d9c66c49 324
dkato 0:f782d9c66c49 325 parser.add_option("-m", "--mbed",
dkato 0:f782d9c66c49 326 action="store_true", default=False,
dkato 0:f782d9c66c49 327 help="Release a build of the mbed library")
dkato 0:f782d9c66c49 328
dkato 0:f782d9c66c49 329 parser.add_option("-n", "--nopush",
dkato 0:f782d9c66c49 330 action="store_true", default=False,
dkato 0:f782d9c66c49 331 help="Commit the changes locally only, don't push them")
dkato 0:f782d9c66c49 332
dkato 0:f782d9c66c49 333 parser.add_option("", "--commit_message",
dkato 0:f782d9c66c49 334 action="store", type="string", default='', dest='msg',
dkato 0:f782d9c66c49 335 help="Commit message to use for all the commits")
dkato 0:f782d9c66c49 336
dkato 0:f782d9c66c49 337 parser.add_option("-q", "--quiet",
dkato 0:f782d9c66c49 338 action="store_true", default=False,
dkato 0:f782d9c66c49 339 help="Don't ask for confirmation before commiting or pushing")
dkato 0:f782d9c66c49 340
dkato 0:f782d9c66c49 341 (options, args) = parser.parse_args()
dkato 0:f782d9c66c49 342
dkato 0:f782d9c66c49 343 do_sync(options)
dkato 0:f782d9c66c49 344