Lee Kai Xuan / mbed-os

mbed-os

Fork of mbed-os by erkin yucel

tools/synch.py@1:3deb71413561, 2017-07-20 (annotated)

Committer:
xuaner
Date:
Thu Jul 20 14:26:57 2017 +0000
Revision:
1:3deb71413561
Parent:
0:f269e3021894 
mbed_os

Who changed what in which revision?

UserRevisionLine numberNew contents of line
elessair 0:f269e3021894 1 """
elessair 0:f269e3021894 2 mbed SDK
elessair 0:f269e3021894 3 Copyright (c) 2011-2013 ARM Limited
elessair 0:f269e3021894 4
elessair 0:f269e3021894 5 Licensed under the Apache License, Version 2.0 (the "License");
elessair 0:f269e3021894 6 you may not use this file except in compliance with the License.
elessair 0:f269e3021894 7 You may obtain a copy of the License at
elessair 0:f269e3021894 8
elessair 0:f269e3021894 9 http://www.apache.org/licenses/LICENSE-2.0
elessair 0:f269e3021894 10
elessair 0:f269e3021894 11 Unless required by applicable law or agreed to in writing, software
elessair 0:f269e3021894 12 distributed under the License is distributed on an "AS IS" BASIS,
elessair 0:f269e3021894 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
elessair 0:f269e3021894 14 See the License for the specific language governing permissions and
elessair 0:f269e3021894 15 limitations under the License.
elessair 0:f269e3021894 16
elessair 0:f269e3021894 17
elessair 0:f269e3021894 18 One repository to update them all
elessair 0:f269e3021894 19 On mbed.org the mbed SDK is split up in multiple repositories, this script takes
elessair 0:f269e3021894 20 care of updating them all.
elessair 0:f269e3021894 21 """
elessair 0:f269e3021894 22 import sys
elessair 0:f269e3021894 23 from copy import copy
elessair 0:f269e3021894 24 from os import walk, remove, makedirs
elessair 0:f269e3021894 25 from os.path import join, abspath, dirname, relpath, exists, isfile
elessair 0:f269e3021894 26 from shutil import copyfile
elessair 0:f269e3021894 27 from optparse import OptionParser
elessair 0:f269e3021894 28 import re
elessair 0:f269e3021894 29 import string
elessair 0:f269e3021894 30
elessair 0:f269e3021894 31 ROOT = abspath(join(dirname(__file__), ".."))
elessair 0:f269e3021894 32 sys.path.insert(0, ROOT)
elessair 0:f269e3021894 33
elessair 0:f269e3021894 34 from tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
elessair 0:f269e3021894 35 from tools.paths import *
elessair 0:f269e3021894 36 from tools.utils import run_cmd
elessair 0:f269e3021894 37
elessair 0:f269e3021894 38 MBED_URL = "mbed.org"
elessair 0:f269e3021894 39 MBED_USER = "mbed_official"
elessair 0:f269e3021894 40
elessair 0:f269e3021894 41 changed = []
elessair 0:f269e3021894 42 push_remote = True
elessair 0:f269e3021894 43 quiet = False
elessair 0:f269e3021894 44 commit_msg = ''
elessair 0:f269e3021894 45
elessair 0:f269e3021894 46 # Code that does have a mirror in the mbed SDK
elessair 0:f269e3021894 47 # Tuple data: (repo_name, list_of_code_dirs, [team])
elessair 0:f269e3021894 48 # team is optional - if not specified, the code is published under mbed_official
elessair 0:f269e3021894 49 OFFICIAL_CODE = (
elessair 0:f269e3021894 50 ("mbed-dev" , [MBED_DRIVERS, MBED_PLATFORM, MBED_HAL]),
elessair 0:f269e3021894 51 ("mbed-rtos", RTOS),
elessair 0:f269e3021894 52 ("mbed-dsp" , DSP),
elessair 0:f269e3021894 53 ("mbed-rpc" , MBED_RPC),
elessair 0:f269e3021894 54
elessair 0:f269e3021894 55 ("lwip" , LWIP_SOURCES+"/lwip"),
elessair 0:f269e3021894 56 ("lwip-sys", LWIP_SOURCES+"/lwip-sys"),
elessair 0:f269e3021894 57 ("Socket" , LWIP_SOURCES+"/Socket"),
elessair 0:f269e3021894 58
elessair 0:f269e3021894 59 ("lwip-eth" , ETH_SOURCES+"/lwip-eth"),
elessair 0:f269e3021894 60 ("EthernetInterface", ETH_SOURCES+"/EthernetInterface"),
elessair 0:f269e3021894 61
elessair 0:f269e3021894 62 ("USBDevice", USB),
elessair 0:f269e3021894 63 ("USBHost" , USB_HOST),
elessair 0:f269e3021894 64
elessair 0:f269e3021894 65 ("CellularModem", CELLULAR_SOURCES),
elessair 0:f269e3021894 66 ("CellularUSBModem", CELLULAR_USB_SOURCES),
elessair 0:f269e3021894 67 ("UbloxUSBModem", UBLOX_SOURCES),
elessair 0:f269e3021894 68 ("UbloxModemHTTPClientTest", [TEST_DIR+"/net/cellular/http/common", TEST_DIR+"/net/cellular/http/ubloxusb"]),
elessair 0:f269e3021894 69 ("UbloxModemSMSTest", [TEST_DIR+"/net/cellular/sms/common", TEST_DIR+"/net/cellular/sms/ubloxusb"]),
elessair 0:f269e3021894 70 ("FATFileSystem", FAT_FS, "mbed-official"),
elessair 0:f269e3021894 71 )
elessair 0:f269e3021894 72
elessair 0:f269e3021894 73
elessair 0:f269e3021894 74 # Code that does have dependencies to libraries should point to
elessair 0:f269e3021894 75 # the latest revision. By default, they point to a specific revision.
elessair 0:f269e3021894 76 CODE_WITH_DEPENDENCIES = (
elessair 0:f269e3021894 77 # Libraries
elessair 0:f269e3021894 78 "EthernetInterface",
elessair 0:f269e3021894 79
elessair 0:f269e3021894 80 # RTOS Examples
elessair 0:f269e3021894 81 "rtos_basic",
elessair 0:f269e3021894 82 "rtos_isr",
elessair 0:f269e3021894 83 "rtos_mail",
elessair 0:f269e3021894 84 "rtos_mutex",
elessair 0:f269e3021894 85 "rtos_queue",
elessair 0:f269e3021894 86 "rtos_semaphore",
elessair 0:f269e3021894 87 "rtos_signals",
elessair 0:f269e3021894 88 "rtos_timer",
elessair 0:f269e3021894 89
elessair 0:f269e3021894 90 # Net Examples
elessair 0:f269e3021894 91 "TCPEchoClient",
elessair 0:f269e3021894 92 "TCPEchoServer",
elessair 0:f269e3021894 93 "TCPSocket_HelloWorld",
elessair 0:f269e3021894 94 "UDPSocket_HelloWorld",
elessair 0:f269e3021894 95 "UDPEchoClient",
elessair 0:f269e3021894 96 "UDPEchoServer",
elessair 0:f269e3021894 97 "BroadcastReceive",
elessair 0:f269e3021894 98 "BroadcastSend",
elessair 0:f269e3021894 99
elessair 0:f269e3021894 100 # mbed sources
elessair 0:f269e3021894 101 "mbed-src-program",
elessair 0:f269e3021894 102 )
elessair 0:f269e3021894 103
elessair 0:f269e3021894 104 # A list of regular expressions that will be checked against each directory
elessair 0:f269e3021894 105 # name and skipped if they match.
elessair 0:f269e3021894 106 IGNORE_DIRS = (
elessair 0:f269e3021894 107 )
elessair 0:f269e3021894 108
elessair 0:f269e3021894 109 IGNORE_FILES = (
elessair 0:f269e3021894 110 'COPYING',
elessair 0:f269e3021894 111 '\.md',
elessair 0:f269e3021894 112 "\.lib",
elessair 0:f269e3021894 113 "\.bld"
elessair 0:f269e3021894 114 )
elessair 0:f269e3021894 115
elessair 0:f269e3021894 116 def ignore_path(name, reg_exps):
elessair 0:f269e3021894 117 for r in reg_exps:
elessair 0:f269e3021894 118 if re.search(r, name):
elessair 0:f269e3021894 119 return True
elessair 0:f269e3021894 120 return False
elessair 0:f269e3021894 121
elessair 0:f269e3021894 122 class MbedRepository:
elessair 0:f269e3021894 123 @staticmethod
elessair 0:f269e3021894 124 def run_and_print(command, cwd):
elessair 0:f269e3021894 125 stdout, _, _ = run_cmd(command, work_dir=cwd, redirect=True)
elessair 0:f269e3021894 126 print(stdout)
elessair 0:f269e3021894 127
elessair 0:f269e3021894 128 def __init__(self, name, team = None):
elessair 0:f269e3021894 129 self.name = name
elessair 0:f269e3021894 130 self.path = join(MBED_ORG_PATH, name)
elessair 0:f269e3021894 131 if team is None:
elessair 0:f269e3021894 132 self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
elessair 0:f269e3021894 133 else:
elessair 0:f269e3021894 134 self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
elessair 0:f269e3021894 135 if not exists(self.path):
elessair 0:f269e3021894 136 # Checkout code
elessair 0:f269e3021894 137 if not exists(MBED_ORG_PATH):
elessair 0:f269e3021894 138 makedirs(MBED_ORG_PATH)
elessair 0:f269e3021894 139
elessair 0:f269e3021894 140 self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)
elessair 0:f269e3021894 141
elessair 0:f269e3021894 142 else:
elessair 0:f269e3021894 143 # Update
elessair 0:f269e3021894 144 self.run_and_print(['hg', 'pull'], cwd=self.path)
elessair 0:f269e3021894 145 self.run_and_print(['hg', 'update'], cwd=self.path)
elessair 0:f269e3021894 146
elessair 0:f269e3021894 147 def publish(self):
elessair 0:f269e3021894 148 # The maintainer has to evaluate the changes first and explicitly accept them
elessair 0:f269e3021894 149 self.run_and_print(['hg', 'addremove'], cwd=self.path)
elessair 0:f269e3021894 150 stdout, _, _ = run_cmd(['hg', 'status'], work_dir=self.path)
elessair 0:f269e3021894 151 if stdout == '':
elessair 0:f269e3021894 152 print "No changes"
elessair 0:f269e3021894 153 return False
elessair 0:f269e3021894 154 print stdout
elessair 0:f269e3021894 155 if quiet:
elessair 0:f269e3021894 156 commit = 'Y'
elessair 0:f269e3021894 157 else:
elessair 0:f269e3021894 158 commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
elessair 0:f269e3021894 159 if commit == 'Y':
elessair 0:f269e3021894 160 args = ['hg', 'commit', '-u', MBED_ORG_USER]
elessair 0:f269e3021894 161 if commit_msg:
elessair 0:f269e3021894 162 args = args + ['-m', commit_msg]
elessair 0:f269e3021894 163 self.run_and_print(args, cwd=self.path)
elessair 0:f269e3021894 164 if push_remote:
elessair 0:f269e3021894 165 self.run_and_print(['hg', 'push'], cwd=self.path)
elessair 0:f269e3021894 166 return True
elessair 0:f269e3021894 167
elessair 0:f269e3021894 168 # Check if a file is a text file or a binary file
elessair 0:f269e3021894 169 # Taken from http://code.activestate.com/recipes/173220/
elessair 0:f269e3021894 170 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
elessair 0:f269e3021894 171 _null_trans = string.maketrans("", "")
elessair 0:f269e3021894 172 def is_text_file(filename):
elessair 0:f269e3021894 173 block_size = 1024
elessair 0:f269e3021894 174 def istext(s):
elessair 0:f269e3021894 175 if "\0" in s:
elessair 0:f269e3021894 176 return 0
elessair 0:f269e3021894 177
elessair 0:f269e3021894 178 if not s: # Empty files are considered text
elessair 0:f269e3021894 179 return 1
elessair 0:f269e3021894 180
elessair 0:f269e3021894 181 # Get the non-text characters (maps a character to itself then
elessair 0:f269e3021894 182 # use the 'remove' option to get rid of the text characters.)
elessair 0:f269e3021894 183 t = s.translate(_null_trans, text_characters)
elessair 0:f269e3021894 184
elessair 0:f269e3021894 185 # If more than 30% non-text characters, then
elessair 0:f269e3021894 186 # this is considered a binary file
elessair 0:f269e3021894 187 if float(len(t))/len(s) > 0.30:
elessair 0:f269e3021894 188 return 0
elessair 0:f269e3021894 189 return 1
elessair 0:f269e3021894 190 with open(filename) as f:
elessair 0:f269e3021894 191 res = istext(f.read(block_size))
elessair 0:f269e3021894 192 return res
elessair 0:f269e3021894 193
elessair 0:f269e3021894 194 # Return the line ending type for the given file ('cr' or 'crlf')
elessair 0:f269e3021894 195 def get_line_endings(f):
elessair 0:f269e3021894 196 examine_size = 1024
elessair 0:f269e3021894 197 try:
elessair 0:f269e3021894 198 tf = open(f, "rb")
elessair 0:f269e3021894 199 lines, ncrlf = tf.readlines(examine_size), 0
elessair 0:f269e3021894 200 tf.close()
elessair 0:f269e3021894 201 for l in lines:
elessair 0:f269e3021894 202 if l.endswith("\r\n"):
elessair 0:f269e3021894 203 ncrlf = ncrlf + 1
elessair 0:f269e3021894 204 return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
elessair 0:f269e3021894 205 except:
elessair 0:f269e3021894 206 return 'cr'
elessair 0:f269e3021894 207
elessair 0:f269e3021894 208 # Copy file to destination, but preserve destination line endings if possible
elessair 0:f269e3021894 209 # This prevents very annoying issues with huge diffs that appear because of
elessair 0:f269e3021894 210 # differences in line endings
elessair 0:f269e3021894 211 def copy_with_line_endings(sdk_file, repo_file):
elessair 0:f269e3021894 212 if not isfile(repo_file):
elessair 0:f269e3021894 213 copyfile(sdk_file, repo_file)
elessair 0:f269e3021894 214 return
elessair 0:f269e3021894 215 is_text = is_text_file(repo_file)
elessair 0:f269e3021894 216 if is_text:
elessair 0:f269e3021894 217 sdk_le = get_line_endings(sdk_file)
elessair 0:f269e3021894 218 repo_le = get_line_endings(repo_file)
elessair 0:f269e3021894 219 if not is_text or sdk_le == repo_le:
elessair 0:f269e3021894 220 copyfile(sdk_file, repo_file)
elessair 0:f269e3021894 221 else:
elessair 0:f269e3021894 222 print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
elessair 0:f269e3021894 223 f = open(sdk_file, "rb")
elessair 0:f269e3021894 224 data = f.read()
elessair 0:f269e3021894 225 f.close()
elessair 0:f269e3021894 226 f = open(repo_file, "wb")
elessair 0:f269e3021894 227 data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
elessair 0:f269e3021894 228 f.write(data)
elessair 0:f269e3021894 229 f.close()
elessair 0:f269e3021894 230
elessair 0:f269e3021894 231 def visit_files(path, visit):
elessair 0:f269e3021894 232 for root, dirs, files in walk(path):
elessair 0:f269e3021894 233 # Ignore hidden directories
elessair 0:f269e3021894 234 for d in copy(dirs):
elessair 0:f269e3021894 235 full = join(root, d)
elessair 0:f269e3021894 236 if d.startswith('.'):
elessair 0:f269e3021894 237 dirs.remove(d)
elessair 0:f269e3021894 238 if ignore_path(full, IGNORE_DIRS):
elessair 0:f269e3021894 239 print "Skipping '%s'" % full
elessair 0:f269e3021894 240 dirs.remove(d)
elessair 0:f269e3021894 241
elessair 0:f269e3021894 242 for file in files:
elessair 0:f269e3021894 243 if ignore_path(file, IGNORE_FILES):
elessair 0:f269e3021894 244 continue
elessair 0:f269e3021894 245
elessair 0:f269e3021894 246 visit(join(root, file))
elessair 0:f269e3021894 247
elessair 0:f269e3021894 248
elessair 0:f269e3021894 249 def update_repo(repo_name, sdk_paths, team_name):
elessair 0:f269e3021894 250 repo = MbedRepository(repo_name, team_name)
elessair 0:f269e3021894 251 # copy files from mbed SDK to mbed_official repository
elessair 0:f269e3021894 252 def visit_mbed_sdk(sdk_file):
elessair 0:f269e3021894 253 repo_file = join(repo.path, relpath(sdk_file, sdk_path))
elessair 0:f269e3021894 254
elessair 0:f269e3021894 255 repo_dir = dirname(repo_file)
elessair 0:f269e3021894 256 if not exists(repo_dir):
elessair 0:f269e3021894 257 makedirs(repo_dir)
elessair 0:f269e3021894 258
elessair 0:f269e3021894 259 copy_with_line_endings(sdk_file, repo_file)
elessair 0:f269e3021894 260 for sdk_path in sdk_paths:
elessair 0:f269e3021894 261 visit_files(sdk_path, visit_mbed_sdk)
elessair 0:f269e3021894 262
elessair 0:f269e3021894 263 # remove repository files that do not exist in the mbed SDK
elessair 0:f269e3021894 264 def visit_repo(repo_file):
elessair 0:f269e3021894 265 for sdk_path in sdk_paths:
elessair 0:f269e3021894 266 sdk_file = join(sdk_path, relpath(repo_file, repo.path))
elessair 0:f269e3021894 267 if exists(sdk_file):
elessair 0:f269e3021894 268 break
elessair 0:f269e3021894 269 else:
elessair 0:f269e3021894 270 remove(repo_file)
elessair 0:f269e3021894 271 print "remove: %s" % repo_file
elessair 0:f269e3021894 272 visit_files(repo.path, visit_repo)
elessair 0:f269e3021894 273
elessair 0:f269e3021894 274 if repo.publish():
elessair 0:f269e3021894 275 changed.append(repo_name)
elessair 0:f269e3021894 276
elessair 0:f269e3021894 277
elessair 0:f269e3021894 278 def update_code(repositories):
elessair 0:f269e3021894 279 for r in repositories:
elessair 0:f269e3021894 280 repo_name, sdk_dir = r[0], r[1]
elessair 0:f269e3021894 281 team_name = r[2] if len(r) == 3 else None
elessair 0:f269e3021894 282 print '\n=== Updating "%s" ===' % repo_name
elessair 0:f269e3021894 283 sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
elessair 0:f269e3021894 284 update_repo(repo_name, sdk_dirs, team_name)
elessair 0:f269e3021894 285
elessair 0:f269e3021894 286 def update_single_repo(repo):
elessair 0:f269e3021894 287 repos = [r for r in OFFICIAL_CODE if r[0] == repo]
elessair 0:f269e3021894 288 if not repos:
elessair 0:f269e3021894 289 print "Repository '%s' not found" % repo
elessair 0:f269e3021894 290 else:
elessair 0:f269e3021894 291 update_code(repos)
elessair 0:f269e3021894 292
elessair 0:f269e3021894 293 def update_dependencies(repositories):
elessair 0:f269e3021894 294 for repo_name in repositories:
elessair 0:f269e3021894 295 print '\n=== Updating "%s" ===' % repo_name
elessair 0:f269e3021894 296 repo = MbedRepository(repo_name)
elessair 0:f269e3021894 297
elessair 0:f269e3021894 298 # point to the latest libraries
elessair 0:f269e3021894 299 def visit_repo(repo_file):
elessair 0:f269e3021894 300 with open(repo_file, "r") as f:
elessair 0:f269e3021894 301 url = f.read()
elessair 0:f269e3021894 302 with open(repo_file, "w") as f:
elessair 0:f269e3021894 303 f.write(url[:(url.rindex('/')+1)])
elessair 0:f269e3021894 304 visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)
elessair 0:f269e3021894 305
elessair 0:f269e3021894 306 if repo.publish():
elessair 0:f269e3021894 307 changed.append(repo_name)
elessair 0:f269e3021894 308
elessair 0:f269e3021894 309
elessair 0:f269e3021894 310 def update_mbed():
elessair 0:f269e3021894 311 update_repo("mbed", [join(BUILD_DIR, "mbed")], None)
elessair 0:f269e3021894 312
elessair 0:f269e3021894 313 def do_sync(options):
elessair 0:f269e3021894 314 global push_remote, quiet, commit_msg, changed
elessair 0:f269e3021894 315
elessair 0:f269e3021894 316 push_remote = not options.nopush
elessair 0:f269e3021894 317 quiet = options.quiet
elessair 0:f269e3021894 318 commit_msg = options.msg
elessair 0:f269e3021894 319 chnaged = []
elessair 0:f269e3021894 320
elessair 0:f269e3021894 321 if options.code:
elessair 0:f269e3021894 322 update_code(OFFICIAL_CODE)
elessair 0:f269e3021894 323
elessair 0:f269e3021894 324 if options.dependencies:
elessair 0:f269e3021894 325 update_dependencies(CODE_WITH_DEPENDENCIES)
elessair 0:f269e3021894 326
elessair 0:f269e3021894 327 if options.mbed:
elessair 0:f269e3021894 328 update_mbed()
elessair 0:f269e3021894 329
elessair 0:f269e3021894 330 if options.repo:
elessair 0:f269e3021894 331 update_single_repo(options.repo)
elessair 0:f269e3021894 332
elessair 0:f269e3021894 333 if changed:
elessair 0:f269e3021894 334 print "Repositories with changes:", changed
elessair 0:f269e3021894 335
elessair 0:f269e3021894 336 return changed
elessair 0:f269e3021894 337
elessair 0:f269e3021894 338 if __name__ == '__main__':
elessair 0:f269e3021894 339 parser = OptionParser()
elessair 0:f269e3021894 340
elessair 0:f269e3021894 341 parser.add_option("-c", "--code",
elessair 0:f269e3021894 342 action="store_true", default=False,
elessair 0:f269e3021894 343 help="Update the mbed_official code")
elessair 0:f269e3021894 344
elessair 0:f269e3021894 345 parser.add_option("-d", "--dependencies",
elessair 0:f269e3021894 346 action="store_true", default=False,
elessair 0:f269e3021894 347 help="Update the mbed_official code dependencies")
elessair 0:f269e3021894 348
elessair 0:f269e3021894 349 parser.add_option("-m", "--mbed",
elessair 0:f269e3021894 350 action="store_true", default=False,
elessair 0:f269e3021894 351 help="Release a build of the mbed library")
elessair 0:f269e3021894 352
elessair 0:f269e3021894 353 parser.add_option("-n", "--nopush",
elessair 0:f269e3021894 354 action="store_true", default=False,
elessair 0:f269e3021894 355 help="Commit the changes locally only, don't push them")
elessair 0:f269e3021894 356
elessair 0:f269e3021894 357 parser.add_option("", "--commit_message",
elessair 0:f269e3021894 358 action="store", type="string", default='', dest='msg',
elessair 0:f269e3021894 359 help="Commit message to use for all the commits")
elessair 0:f269e3021894 360
elessair 0:f269e3021894 361 parser.add_option("-r", "--repository",
elessair 0:f269e3021894 362 action="store", type="string", default='', dest='repo',
elessair 0:f269e3021894 363 help="Synchronize only the given repository")
elessair 0:f269e3021894 364
elessair 0:f269e3021894 365 parser.add_option("-q", "--quiet",
elessair 0:f269e3021894 366 action="store_true", default=False,
elessair 0:f269e3021894 367 help="Don't ask for confirmation before commiting or pushing")
elessair 0:f269e3021894 368
elessair 0:f269e3021894 369 (options, args) = parser.parse_args()
elessair 0:f269e3021894 370
elessair 0:f269e3021894 371 do_sync(options)
elessair 0:f269e3021894 372