Anders Blomdell / mbed-sdk-tools

Clone of official tools

utils.py@24:25bff2709c20, 2016-08-01 (annotated)

Committer:
screamer
Date:
Mon Aug 01 09:10:17 2016 +0100
Revision:
24:25bff2709c20
Parent:
22:9e85236d8716
Child:
29:1210849dba19
Major update to tools from ARMmbed/mbed-os

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """
screamer 0:66f3b5499f7f 2 mbed SDK
screamer 0:66f3b5499f7f 3 Copyright (c) 2011-2013 ARM Limited
screamer 0:66f3b5499f7f 4
screamer 0:66f3b5499f7f 5 Licensed under the Apache License, Version 2.0 (the "License");
screamer 0:66f3b5499f7f 6 you may not use this file except in compliance with the License.
screamer 0:66f3b5499f7f 7 You may obtain a copy of the License at
screamer 0:66f3b5499f7f 8
screamer 0:66f3b5499f7f 9 http://www.apache.org/licenses/LICENSE-2.0
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 Unless required by applicable law or agreed to in writing, software
screamer 0:66f3b5499f7f 12 distributed under the License is distributed on an "AS IS" BASIS,
screamer 0:66f3b5499f7f 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
screamer 0:66f3b5499f7f 14 See the License for the specific language governing permissions and
screamer 0:66f3b5499f7f 15 limitations under the License.
screamer 0:66f3b5499f7f 16 """
screamer 0:66f3b5499f7f 17 import sys
screamer 0:66f3b5499f7f 18 import inspect
screamer 0:66f3b5499f7f 19 import os
screamer 22:9e85236d8716 20 import argparse
screamer 22:9e85236d8716 21 import math
screamer 0:66f3b5499f7f 22 from os import listdir, remove, makedirs
screamer 0:66f3b5499f7f 23 from shutil import copyfile
screamer 24:25bff2709c20 24 from os.path import isdir, join, exists, split, relpath, splitext, abspath, commonprefix, normpath
screamer 0:66f3b5499f7f 25 from subprocess import Popen, PIPE, STDOUT, call
screamer 7:5af61d55adbe 26 import json
screamer 7:5af61d55adbe 27 from collections import OrderedDict
screamer 19:3604ee113e2d 28 import logging
screamer 0:66f3b5499f7f 29
screamer 21:4fdf0dd04f6f 30 def compile_worker(job):
screamer 21:4fdf0dd04f6f 31 results = []
screamer 21:4fdf0dd04f6f 32 for command in job['commands']:
screamer 21:4fdf0dd04f6f 33 try:
screamer 21:4fdf0dd04f6f 34 _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'], chroot=job['chroot'])
screamer 21:4fdf0dd04f6f 35 except KeyboardInterrupt as e:
screamer 21:4fdf0dd04f6f 36 raise ToolException
screamer 21:4fdf0dd04f6f 37
screamer 21:4fdf0dd04f6f 38 results.append({
screamer 21:4fdf0dd04f6f 39 'code': _rc,
screamer 21:4fdf0dd04f6f 40 'output': _stderr,
screamer 21:4fdf0dd04f6f 41 'command': command
screamer 21:4fdf0dd04f6f 42 })
screamer 21:4fdf0dd04f6f 43
screamer 21:4fdf0dd04f6f 44 return {
screamer 21:4fdf0dd04f6f 45 'source': job['source'],
screamer 21:4fdf0dd04f6f 46 'object': job['object'],
screamer 21:4fdf0dd04f6f 47 'commands': job['commands'],
screamer 21:4fdf0dd04f6f 48 'results': results
screamer 21:4fdf0dd04f6f 49 }
screamer 21:4fdf0dd04f6f 50
screamer 0:66f3b5499f7f 51 def cmd(l, check=True, verbose=False, shell=False, cwd=None):
screamer 0:66f3b5499f7f 52 text = l if shell else ' '.join(l)
screamer 0:66f3b5499f7f 53 if verbose:
screamer 0:66f3b5499f7f 54 print text
screamer 0:66f3b5499f7f 55 rc = call(l, shell=shell, cwd=cwd)
screamer 0:66f3b5499f7f 56 if check and rc != 0:
screamer 0:66f3b5499f7f 57 raise Exception('ERROR %d: "%s"' % (rc, text))
screamer 0:66f3b5499f7f 58
screamer 0:66f3b5499f7f 59
screamer 17:04753e1e329d 60 def run_cmd(command, work_dir=None, chroot=None, redirect=False):
screamer 17:04753e1e329d 61 if chroot:
screamer 17:04753e1e329d 62 # Conventions managed by the web team for the mbed.org build system
screamer 17:04753e1e329d 63 chroot_cmd = [
screamer 17:04753e1e329d 64 '/usr/sbin/chroot', '--userspec=33:33', chroot
screamer 17:04753e1e329d 65 ]
screamer 17:04753e1e329d 66 for c in command:
screamer 17:04753e1e329d 67 chroot_cmd += [c.replace(chroot, '')]
screamer 17:04753e1e329d 68
screamer 17:04753e1e329d 69 logging.debug("Running command %s"%' '.join(chroot_cmd))
screamer 17:04753e1e329d 70 command = chroot_cmd
screamer 21:4fdf0dd04f6f 71 work_dir = None
screamer 17:04753e1e329d 72
screamer 0:66f3b5499f7f 73 try:
screamer 17:04753e1e329d 74 p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=work_dir)
screamer 0:66f3b5499f7f 75 _stdout, _stderr = p.communicate()
screamer 13:ab47a20b66f0 76 except OSError as e:
screamer 0:66f3b5499f7f 77 print "[OS ERROR] Command: "+(' '.join(command))
screamer 0:66f3b5499f7f 78 raise
screamer 13:ab47a20b66f0 79
screamer 0:66f3b5499f7f 80 return _stdout, _stderr, p.returncode
screamer 0:66f3b5499f7f 81
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83 def run_cmd_ext(command):
screamer 0:66f3b5499f7f 84 assert is_cmd_valid(command[0])
screamer 0:66f3b5499f7f 85 p = Popen(command, stdout=PIPE, stderr=PIPE)
screamer 0:66f3b5499f7f 86 _stdout, _stderr = p.communicate()
screamer 0:66f3b5499f7f 87 return _stdout, _stderr, p.returncode
screamer 0:66f3b5499f7f 88
screamer 0:66f3b5499f7f 89
screamer 0:66f3b5499f7f 90 def is_cmd_valid(cmd):
screamer 0:66f3b5499f7f 91 caller = get_caller_name()
screamer 0:66f3b5499f7f 92 abspath = find_cmd_abspath(cmd)
screamer 0:66f3b5499f7f 93 if not abspath:
screamer 0:66f3b5499f7f 94 error("%s: Command '%s' can't be found" % (caller, cmd))
screamer 0:66f3b5499f7f 95 if not is_exec(abspath):
screamer 0:66f3b5499f7f 96 error("%s: Command '%s' resolves to file '%s' which is not executable" % (caller, cmd, abspath))
screamer 0:66f3b5499f7f 97 return True
screamer 0:66f3b5499f7f 98
screamer 0:66f3b5499f7f 99
screamer 0:66f3b5499f7f 100 def is_exec(path):
screamer 0:66f3b5499f7f 101 return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
screamer 0:66f3b5499f7f 102
screamer 0:66f3b5499f7f 103
screamer 0:66f3b5499f7f 104 def find_cmd_abspath(cmd):
screamer 0:66f3b5499f7f 105 """ Returns the absolute path to a command.
screamer 0:66f3b5499f7f 106 None is returned if no absolute path was found.
screamer 0:66f3b5499f7f 107 """
screamer 0:66f3b5499f7f 108 if exists(cmd) or exists(cmd + '.exe'):
screamer 0:66f3b5499f7f 109 return os.path.abspath(cmd)
screamer 0:66f3b5499f7f 110 if not 'PATH' in os.environ:
screamer 0:66f3b5499f7f 111 raise Exception("Can't find command path for current platform ('%s')" % sys.platform)
screamer 0:66f3b5499f7f 112 PATH=os.environ['PATH']
screamer 0:66f3b5499f7f 113 for path in PATH.split(os.pathsep):
screamer 0:66f3b5499f7f 114 abspath = '%s/%s' % (path, cmd)
screamer 0:66f3b5499f7f 115 if exists(abspath) or exists(abspath + '.exe'):
screamer 0:66f3b5499f7f 116 return abspath
screamer 0:66f3b5499f7f 117
screamer 0:66f3b5499f7f 118
screamer 0:66f3b5499f7f 119 def mkdir(path):
screamer 0:66f3b5499f7f 120 if not exists(path):
screamer 0:66f3b5499f7f 121 makedirs(path)
screamer 0:66f3b5499f7f 122
screamer 0:66f3b5499f7f 123
screamer 0:66f3b5499f7f 124 def copy_file(src, dst):
screamer 0:66f3b5499f7f 125 """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
screamer 0:66f3b5499f7f 126 permissions (this was causing errors with directories mounted with samba)
screamer 0:66f3b5499f7f 127 """
screamer 0:66f3b5499f7f 128 if isdir(dst):
screamer 0:66f3b5499f7f 129 _, file = split(src)
screamer 0:66f3b5499f7f 130 dst = join(dst, file)
screamer 0:66f3b5499f7f 131 copyfile(src, dst)
screamer 0:66f3b5499f7f 132
screamer 0:66f3b5499f7f 133
screamer 0:66f3b5499f7f 134 def delete_dir_files(dir):
screamer 0:66f3b5499f7f 135 if not exists(dir):
screamer 0:66f3b5499f7f 136 return
screamer 0:66f3b5499f7f 137
screamer 0:66f3b5499f7f 138 for f in listdir(dir):
screamer 0:66f3b5499f7f 139 file = join(dir, f)
screamer 0:66f3b5499f7f 140 if not isdir(file):
screamer 0:66f3b5499f7f 141 remove(file)
screamer 0:66f3b5499f7f 142
screamer 0:66f3b5499f7f 143
screamer 0:66f3b5499f7f 144 def get_caller_name(steps=2):
screamer 0:66f3b5499f7f 145 """
screamer 0:66f3b5499f7f 146 When called inside a function, it returns the name
screamer 0:66f3b5499f7f 147 of the caller of that function.
screamer 0:66f3b5499f7f 148 """
screamer 0:66f3b5499f7f 149 return inspect.stack()[steps][3]
screamer 0:66f3b5499f7f 150
screamer 0:66f3b5499f7f 151
screamer 0:66f3b5499f7f 152 def error(msg):
screamer 0:66f3b5499f7f 153 print("ERROR: %s" % msg)
screamer 0:66f3b5499f7f 154 sys.exit(1)
screamer 0:66f3b5499f7f 155
screamer 0:66f3b5499f7f 156
screamer 0:66f3b5499f7f 157 def rel_path(path, base, dot=False):
screamer 0:66f3b5499f7f 158 p = relpath(path, base)
screamer 0:66f3b5499f7f 159 if dot and not p.startswith('.'):
screamer 0:66f3b5499f7f 160 p = './' + p
screamer 0:66f3b5499f7f 161 return p
screamer 0:66f3b5499f7f 162
screamer 0:66f3b5499f7f 163
screamer 0:66f3b5499f7f 164 class ToolException(Exception):
screamer 0:66f3b5499f7f 165 pass
screamer 0:66f3b5499f7f 166
screamer 0:66f3b5499f7f 167 class NotSupportedException(Exception):
screamer 0:66f3b5499f7f 168 pass
screamer 0:66f3b5499f7f 169
screamer 24:25bff2709c20 170 class InvalidReleaseTargetException(Exception):
screamer 24:25bff2709c20 171 pass
screamer 24:25bff2709c20 172
screamer 0:66f3b5499f7f 173 def split_path(path):
screamer 0:66f3b5499f7f 174 base, file = split(path)
screamer 0:66f3b5499f7f 175 name, ext = splitext(file)
screamer 0:66f3b5499f7f 176 return base, name, ext
screamer 0:66f3b5499f7f 177
screamer 0:66f3b5499f7f 178
screamer 24:25bff2709c20 179 def get_path_depth(path):
screamer 24:25bff2709c20 180 """ Given a path, return the number of directory levels present.
screamer 24:25bff2709c20 181 This roughly translates to the number of path separators (os.sep) + 1.
screamer 24:25bff2709c20 182 Ex. Given "path/to/dir", this would return 3
screamer 24:25bff2709c20 183 Special cases: "." and "/" return 0
screamer 24:25bff2709c20 184 """
screamer 24:25bff2709c20 185 normalized_path = normpath(path)
screamer 24:25bff2709c20 186 path_depth = 0
screamer 24:25bff2709c20 187 head, tail = split(normalized_path)
screamer 24:25bff2709c20 188
screamer 24:25bff2709c20 189 while(tail and tail != '.'):
screamer 24:25bff2709c20 190 path_depth += 1
screamer 24:25bff2709c20 191 head, tail = split(head)
screamer 24:25bff2709c20 192
screamer 24:25bff2709c20 193 return path_depth
screamer 24:25bff2709c20 194
screamer 24:25bff2709c20 195
screamer 0:66f3b5499f7f 196 def args_error(parser, message):
screamer 0:66f3b5499f7f 197 print "\n\n%s\n\n" % message
screamer 0:66f3b5499f7f 198 parser.print_help()
screamer 0:66f3b5499f7f 199 sys.exit()
screamer 0:66f3b5499f7f 200
screamer 0:66f3b5499f7f 201
screamer 0:66f3b5499f7f 202 def construct_enum(**enums):
screamer 0:66f3b5499f7f 203 """ Create your own pseudo-enums """
screamer 0:66f3b5499f7f 204 return type('Enum', (), enums)
screamer 0:66f3b5499f7f 205
screamer 0:66f3b5499f7f 206
screamer 0:66f3b5499f7f 207 def check_required_modules(required_modules, verbose=True):
screamer 0:66f3b5499f7f 208 """ Function checks for Python modules which should be "importable" (installed)
screamer 0:66f3b5499f7f 209 before test suite can be used.
screamer 0:66f3b5499f7f 210 @return returns True if all modules are installed already
screamer 0:66f3b5499f7f 211 """
screamer 0:66f3b5499f7f 212 import imp
screamer 0:66f3b5499f7f 213 not_installed_modules = []
screamer 0:66f3b5499f7f 214 for module_name in required_modules:
screamer 0:66f3b5499f7f 215 try:
screamer 0:66f3b5499f7f 216 imp.find_module(module_name)
screamer 0:66f3b5499f7f 217 except ImportError as e:
screamer 0:66f3b5499f7f 218 # We also test against a rare case: module is an egg file
screamer 0:66f3b5499f7f 219 try:
screamer 0:66f3b5499f7f 220 __import__(module_name)
screamer 0:66f3b5499f7f 221 except ImportError as e:
screamer 0:66f3b5499f7f 222 not_installed_modules.append(module_name)
screamer 0:66f3b5499f7f 223 if verbose:
screamer 0:66f3b5499f7f 224 print "Error: %s" % e
screamer 0:66f3b5499f7f 225
screamer 0:66f3b5499f7f 226 if verbose:
screamer 0:66f3b5499f7f 227 if not_installed_modules:
screamer 0:66f3b5499f7f 228 print "Warning: Module(s) %s not installed. Please install required module(s) before using this script."% (', '.join(not_installed_modules))
screamer 0:66f3b5499f7f 229
screamer 0:66f3b5499f7f 230 if not_installed_modules:
screamer 0:66f3b5499f7f 231 return False
screamer 0:66f3b5499f7f 232 else:
screamer 0:66f3b5499f7f 233 return True
screamer 7:5af61d55adbe 234
screamer 7:5af61d55adbe 235 # Utility function: traverse a dictionary and change all the strings in the dictionary to
screamer 7:5af61d55adbe 236 # ASCII from Unicode. Useful when reading ASCII JSON data, because the JSON decoder always
screamer 7:5af61d55adbe 237 # returns Unicode string.
screamer 7:5af61d55adbe 238 # Based on http://stackoverflow.com/a/13105359
screamer 7:5af61d55adbe 239 def dict_to_ascii(input):
screamer 7:5af61d55adbe 240 if isinstance(input, dict):
screamer 7:5af61d55adbe 241 return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value)) for key, value in input.iteritems()])
screamer 7:5af61d55adbe 242 elif isinstance(input, list):
screamer 7:5af61d55adbe 243 return [dict_to_ascii(element) for element in input]
screamer 7:5af61d55adbe 244 elif isinstance(input, unicode):
screamer 7:5af61d55adbe 245 return input.encode('ascii')
screamer 7:5af61d55adbe 246 else:
screamer 7:5af61d55adbe 247 return input
screamer 7:5af61d55adbe 248
screamer 7:5af61d55adbe 249 # Read a JSON file and return its Python representation, transforming all the strings from Unicode
screamer 7:5af61d55adbe 250 # to ASCII. The order of keys in the JSON file is preserved.
screamer 7:5af61d55adbe 251 def json_file_to_dict(fname):
screamer 22:9e85236d8716 252 try:
screamer 22:9e85236d8716 253 with open(fname, "rt") as f:
screamer 22:9e85236d8716 254 return dict_to_ascii(json.load(f, object_pairs_hook=OrderedDict))
screamer 22:9e85236d8716 255 except (ValueError, IOError):
screamer 22:9e85236d8716 256 sys.stderr.write("Error parsing '%s':\n" % fname)
screamer 22:9e85236d8716 257 raise
screamer 22:9e85236d8716 258
screamer 22:9e85236d8716 259 # Wowza, double closure
screamer 22:9e85236d8716 260 def argparse_type(casedness, prefer_hyphen=False) :
screamer 22:9e85236d8716 261 def middle(list, type_name):
screamer 22:9e85236d8716 262 # validate that an argument passed in (as string) is a member of the list of possible
screamer 22:9e85236d8716 263 # arguments. Offer a suggestion if the case of the string, or the hyphens/underscores
screamer 22:9e85236d8716 264 # do not match the expected style of the argument.
screamer 22:9e85236d8716 265 def parse_type(string):
screamer 22:9e85236d8716 266 if prefer_hyphen: newstring = casedness(string).replace("_","-")
screamer 22:9e85236d8716 267 else: newstring = casedness(string).replace("-","_")
screamer 22:9e85236d8716 268 if string in list:
screamer 22:9e85236d8716 269 return string
screamer 22:9e85236d8716 270 elif string not in list and newstring in list:
screamer 22:9e85236d8716 271 raise argparse.ArgumentTypeError("{0} is not a supported {1}. Did you mean {2}?".format(string, type_name, newstring))
screamer 22:9e85236d8716 272 else:
screamer 22:9e85236d8716 273 raise argparse.ArgumentTypeError("{0} is not a supported {1}. Supported {1}s are:\n{2}".format(string, type_name, columnate(list)))
screamer 22:9e85236d8716 274 return parse_type
screamer 22:9e85236d8716 275 return middle
screamer 22:9e85236d8716 276
screamer 22:9e85236d8716 277 # short cuts for the argparse_type versions
screamer 22:9e85236d8716 278 argparse_uppercase_type = argparse_type(str.upper, False)
screamer 22:9e85236d8716 279 argparse_lowercase_type = argparse_type(str.lower, False)
screamer 22:9e85236d8716 280 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
screamer 22:9e85236d8716 281 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
screamer 22:9e85236d8716 282
screamer 22:9e85236d8716 283 def argparse_force_type(case):
screamer 22:9e85236d8716 284 def middle(list, type_name):
screamer 22:9e85236d8716 285 # validate that an argument passed in (as string) is a member of the list of possible
screamer 22:9e85236d8716 286 # arguments after converting it's case. Offer a suggestion if the hyphens/underscores
screamer 22:9e85236d8716 287 # do not match the expected style of the argument.
screamer 22:9e85236d8716 288 def parse_type(string):
screamer 22:9e85236d8716 289 for option in list:
screamer 22:9e85236d8716 290 if case(string) == case(option):
screamer 22:9e85236d8716 291 return option
screamer 22:9e85236d8716 292 raise argparse.ArgumentTypeError("{0} is not a supported {1}. Supported {1}s are:\n{2}".format(string, type_name, columnate(list)))
screamer 22:9e85236d8716 293 return parse_type
screamer 22:9e85236d8716 294 return middle
screamer 22:9e85236d8716 295
screamer 22:9e85236d8716 296 # these two types convert the case of their arguments _before_ validation
screamer 22:9e85236d8716 297 argparse_force_uppercase_type = argparse_force_type(str.upper)
screamer 22:9e85236d8716 298 argparse_force_lowercase_type = argparse_force_type(str.lower)
screamer 22:9e85236d8716 299
screamer 22:9e85236d8716 300 # An argument parser combinator that takes in an argument parser and creates a new parser that
screamer 22:9e85236d8716 301 # accepts a comma separated list of the same thing.
screamer 22:9e85236d8716 302 def argparse_many(fn):
screamer 22:9e85236d8716 303 def wrap(string):
screamer 22:9e85236d8716 304 return [fn(s) for s in string.split(",")]
screamer 22:9e85236d8716 305 return wrap
screamer 22:9e85236d8716 306
screamer 22:9e85236d8716 307 # An argument parser that verifies that a string passed in corresponds to a file
screamer 22:9e85236d8716 308 def argparse_filestring_type(string) :
screamer 22:9e85236d8716 309 if exists(string) :
screamer 22:9e85236d8716 310 return string
screamer 22:9e85236d8716 311 else :
screamer 22:9e85236d8716 312 raise argparse.ArgumentTypeError("{0}"" does not exist in the filesystem.".format(string))
screamer 22:9e85236d8716 313
screamer 22:9e85236d8716 314 # render a list of strings as a in a bunch of columns
screamer 22:9e85236d8716 315 def columnate(strings, seperator=", ", chars=80):
screamer 22:9e85236d8716 316 col_width = max(len(s) for s in strings)
screamer 22:9e85236d8716 317 total_width = col_width + len(seperator)
screamer 22:9e85236d8716 318 columns = math.floor(chars / total_width)
screamer 22:9e85236d8716 319 output = ""
screamer 22:9e85236d8716 320 for i, s in zip(range(len(strings)), strings):
screamer 22:9e85236d8716 321 append = s
screamer 22:9e85236d8716 322 if i != len(strings) - 1:
screamer 22:9e85236d8716 323 append += seperator
screamer 22:9e85236d8716 324 if i % columns == columns - 1:
screamer 22:9e85236d8716 325 append += "\n"
screamer 22:9e85236d8716 326 else:
screamer 22:9e85236d8716 327 append = append.ljust(total_width)
screamer 22:9e85236d8716 328 output += append
screamer 22:9e85236d8716 329 return output
screamer 24:25bff2709c20 330
screamer 24:25bff2709c20 331 # fail if argument provided is a parent of the specified directory
screamer 24:25bff2709c20 332 def argparse_dir_not_parent(other):
screamer 24:25bff2709c20 333 def parse_type(not_parent):
screamer 24:25bff2709c20 334 abs_other = abspath(other)
screamer 24:25bff2709c20 335 abs_not_parent = abspath(not_parent)
screamer 24:25bff2709c20 336 if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
screamer 24:25bff2709c20 337 raise argparse.ArgumentTypeError("{0} may not be a parent directory of {1}".format(not_parent, other))
screamer 24:25bff2709c20 338 else:
screamer 24:25bff2709c20 339 return not_parent
screamer 24:25bff2709c20 340 return parse_type