utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 import sys
00019 import inspect
00020 import os
00021 import argparse
00022 import math
00023 from os import listdir, remove, makedirs
00024 from shutil import copyfile
00025 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00026 from os.path import commonprefix, normpath, dirname
00027 from subprocess import Popen, PIPE, STDOUT, call
00028 from math import ceil
00029 import json
00030 from collections import OrderedDict
00031 import logging
00032 from intelhex import IntelHex
00033 
00034 try:
00035     unicode
00036 except NameError:
00037     unicode = str
00038 
00039 def remove_if_in(lst, thing):
00040     if thing in lst:
00041         lst.remove(thing)
00042 
00043 def compile_worker (job):
00044     """Standard task runner used for compiling
00045 
00046     Positional argumets:
00047     job - a dict containing a list of commands and the remaining arguments
00048           to run_cmd
00049     """
00050     results = []
00051     for command in job['commands']:
00052         try:
00053             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00054                                       chroot=job['chroot'])
00055         except KeyboardInterrupt:
00056             raise ToolException
00057 
00058         results.append({
00059             'code': _rc,
00060             'output': _stderr,
00061             'command': command
00062         })
00063 
00064     return {
00065         'source': job['source'],
00066         'object': job['object'],
00067         'commands': job['commands'],
00068         'results': results
00069     }
00070 
00071 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00072     """A wrapper to run a command as a blocking job"""
00073     text = command if shell else ' '.join(command)
00074     if verbose:
00075         print(text)
00076     return_code = call(command, shell=shell, cwd=cwd)
00077     if check and return_code != 0:
00078         raise Exception('ERROR %d: "%s"' % (return_code, text))
00079 
00080 
00081 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00082     """Run a command in the foreground
00083 
00084     Positional arguments:
00085     command - the command to run
00086 
00087     Keyword arguments:
00088     work_dir - the working directory to run the command in
00089     chroot - the chroot to run the command in
00090     redirect - redirect the stderr to a pipe to be used later
00091     """
00092     if chroot:
00093         # Conventions managed by the web team for the mbed.org build system
00094         chroot_cmd = [
00095             '/usr/sbin/chroot', '--userspec=33:33', chroot
00096         ]
00097         for element in command:
00098             chroot_cmd += [element.replace(chroot, '')]
00099 
00100         logging.debug("Running command %s", ' '.join(chroot_cmd))
00101         command = chroot_cmd
00102         work_dir = None
00103 
00104     try:
00105         process = Popen(command, stdout=PIPE,
00106                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00107         _stdout, _stderr = process.communicate()
00108     except OSError:
00109         print("[OS ERROR] Command: "+(' '.join(command)))
00110         raise
00111 
00112     return _stdout, _stderr, process.returncode
00113 
00114 
00115 def run_cmd_ext (command):
00116     """ A version of run command that checks if the command exists befor running
00117 
00118     Positional arguments:
00119     command - the command line you are trying to invoke
00120     """
00121     assert is_cmd_valid(command[0])
00122     process = Popen(command, stdout=PIPE, stderr=PIPE)
00123     _stdout, _stderr = process.communicate()
00124     return _stdout, _stderr, process.returncode
00125 
00126 
00127 def is_cmd_valid (command):
00128     """ Verify that a command exists and is executable
00129 
00130     Positional arguments:
00131     command - the command to check
00132     """
00133     caller = get_caller_name()
00134     cmd_path = find_cmd_abspath(command)
00135     if not cmd_path:
00136         error("%s: Command '%s' can't be found" % (caller, command))
00137     if not is_exec(cmd_path):
00138         error("%s: Command '%s' resolves to file '%s' which is not executable"
00139               % (caller, command, cmd_path))
00140     return True
00141 
00142 
00143 def is_exec (path):
00144     """A simple check to verify that a path to an executable exists
00145 
00146     Positional arguments:
00147     path - the executable
00148     """
00149     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00150 
00151 
00152 def find_cmd_abspath (command):
00153     """ Returns the absolute path to a command.
00154         None is returned if no absolute path was found.
00155 
00156     Positional arguhments:
00157     command - the command to find the path of
00158     """
00159     if exists(command) or exists(command + '.exe'):
00160         return os.path.abspath(command)
00161     if not 'PATH' in os.environ:
00162         raise Exception("Can't find command path for current platform ('%s')"
00163                         % sys.platform)
00164     path_env = os.environ['PATH']
00165     for path in path_env.split(os.pathsep):
00166         cmd_path = '%s/%s' % (path, command)
00167         if exists(cmd_path) or exists(cmd_path + '.exe'):
00168             return cmd_path
00169 
00170 
00171 def mkdir (path):
00172     """ a wrapped makedirs that only tries to create a directory if it does not
00173     exist already
00174 
00175     Positional arguments:
00176     path - the path to maybe create
00177     """
00178     if not exists(path):
00179         makedirs(path)
00180 
00181 
00182 def write_json_to_file (json_data, file_name):
00183     """
00184     Write json content in file
00185     :param json_data:
00186     :param file_name:
00187     :return:
00188     """
00189     # Create the target dir for file if necessary
00190     test_spec_dir = os.path.dirname(file_name)
00191 
00192     if test_spec_dir:
00193         mkdir(test_spec_dir)
00194 
00195     try:
00196         with open(file_name, 'w') as f:
00197             f.write(json.dumps(json_data, indent=2))
00198     except IOError as e:
00199         print("[ERROR] Error writing test spec to file")
00200         print(e)
00201 
00202 
00203 def copy_file (src, dst):
00204     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00205     permissions (this was causing errors with directories mounted with samba)
00206 
00207     Positional arguments:
00208     src - the source of the copy operation
00209     dst - the destination of the copy operation
00210     """
00211     if isdir(dst):
00212         _, base = split(src)
00213         dst = join(dst, base)
00214     copyfile(src, dst)
00215 
00216 
00217 def delete_dir_files (directory):
00218     """ A function that does rm -rf
00219 
00220     Positional arguments:
00221     directory - the directory to remove
00222     """
00223     if not exists(directory):
00224         return
00225 
00226     for element in listdir(directory):
00227         to_remove = join(directory, element)
00228         if not isdir(to_remove):
00229             remove(to_remove)
00230 
00231 
00232 def get_caller_name (steps=2):
00233     """
00234     When called inside a function, it returns the name
00235     of the caller of that function.
00236 
00237     Keyword arguments:
00238     steps - the number of steps up the stack the calling function is
00239     """
00240     return inspect.stack()[steps][3]
00241 
00242 
00243 def error (msg):
00244     """Fatal error, abort hard
00245 
00246     Positional arguments:
00247     msg - the message to print before crashing
00248     """
00249     print("ERROR: %s" % msg)
00250     sys.exit(1)
00251 
00252 
00253 def rel_path (path, base, dot=False):
00254     """Relative path calculation that optionaly always starts with a dot
00255 
00256     Positional arguments:
00257     path - the path to make relative
00258     base - what to make the path relative to
00259 
00260     Keyword arguments:
00261     dot - if True, the path will always start with a './'
00262     """
00263     final_path = relpath(path, base)
00264     if dot and not final_path.startswith('.'):
00265         final_path = './' + final_path
00266     return final_path
00267 
00268 
00269 class ToolException (Exception):
00270     """A class representing an exception throw by the tools"""
00271     pass
00272 
00273 class NotSupportedException(Exception):
00274     """A class a toolchain not supporting a particular target"""
00275     pass
00276 
00277 class InvalidReleaseTargetException(Exception):
00278     pass
00279 
00280 def split_path (path):
00281     """spilt a file name into it's directory name, base name, and extension
00282 
00283     Positional arguments:
00284     path - the file name to split
00285     """
00286     base, has_ext = split(path)
00287     name, ext = splitext(has_ext)
00288     return base, name, ext
00289 
00290 
00291 def get_path_depth (path):
00292     """ Given a path, return the number of directory levels present.
00293         This roughly translates to the number of path separators (os.sep) + 1.
00294         Ex. Given "path/to/dir", this would return 3
00295         Special cases: "." and "/" return 0
00296 
00297     Positional arguments:
00298     path - the path to calculate the depth of
00299     """
00300     normalized_path = normpath(path)
00301     path_depth = 0
00302     head, tail = split(normalized_path)
00303 
00304     while tail and tail != '.':
00305         path_depth += 1
00306         head, tail = split(head)
00307 
00308     return path_depth
00309 
00310 
00311 def args_error (parser, message):
00312     """Abort with an error that was generated by the arguments to a CLI program
00313 
00314     Positional arguments:
00315     parser - the ArgumentParser object that parsed the command line
00316     message - what went wrong
00317     """
00318     parser.error(message)
00319     sys.exit(2)
00320 
00321 
00322 def construct_enum (**enums):
00323     """ Create your own pseudo-enums
00324 
00325     Keyword arguments:
00326     * - a member of the Enum you are creating and it's value
00327     """
00328     return type('Enum', (), enums)
00329 
00330 
00331 def check_required_modules (required_modules, verbose=True):
00332     """ Function checks for Python modules which should be "importable"
00333         before test suite can be used.
00334         @return returns True if all modules are installed already
00335     """
00336     import imp
00337     not_installed_modules = []
00338     for module_name in required_modules:
00339         try:
00340             imp.find_module(module_name)
00341         except ImportError:
00342             # We also test against a rare case: module is an egg file
00343             try:
00344                 __import__(module_name)
00345             except ImportError as exc:
00346                 not_installed_modules.append(module_name)
00347                 if verbose:
00348                     print("Error: %s" % exc)
00349 
00350     if verbose:
00351         if not_installed_modules:
00352             print("Warning: Module(s) %s not installed. Please install "
00353                   "required module(s) before using this script."
00354                   % (', '.join(not_installed_modules)))
00355 
00356     if not_installed_modules:
00357         return False
00358     else:
00359         return True
00360 
00361 def json_file_to_dict (fname):
00362     """ Read a JSON file and return its Python representation, transforming all
00363     the strings from Unicode to ASCII. The order of keys in the JSON file is
00364     preserved.
00365 
00366     Positional arguments:
00367     fname - the name of the file to parse
00368     """
00369     try:
00370         with open(fname, "r") as file_obj:
00371             return json.loads(file_obj.read().encode('ascii', 'ignore'),
00372                               object_pairs_hook=OrderedDict)
00373     except (ValueError, IOError):
00374         sys.stderr.write("Error parsing '%s':\n" % fname)
00375         raise
00376 
00377 # Wowza, double closure
00378 def argparse_type(casedness, prefer_hyphen=False):
00379     def middle(lst, type_name):
00380         def parse_type(string):
00381             """ validate that an argument passed in (as string) is a member of
00382             the list of possible arguments. Offer a suggestion if the case of
00383             the string, or the hyphens/underscores do not match the expected
00384             style of the argument.
00385             """
00386             if not isinstance(string, unicode):
00387                 string = string.decode()
00388             if prefer_hyphen:
00389                 newstring = casedness(string).replace("_", "-")
00390             else:
00391                 newstring = casedness(string).replace("-", "_")
00392             if string in lst:
00393                 return string
00394             elif string not in lst and newstring in lst:
00395                 raise argparse.ArgumentTypeError(
00396                     "{0} is not a supported {1}. Did you mean {2}?".format(
00397                         string, type_name, newstring))
00398             else:
00399                 raise argparse.ArgumentTypeError(
00400                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00401                     format(string, type_name, columnate(lst)))
00402         return parse_type
00403     return middle
00404 
00405 # short cuts for the argparse_type versions
00406 argparse_uppercase_type = argparse_type(unicode.upper, False)
00407 argparse_lowercase_type = argparse_type(unicode.lower, False)
00408 argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True)
00409 argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True)
00410 
00411 def argparse_force_type (case):
00412     """ validate that an argument passed in (as string) is a member of the list
00413     of possible arguments after converting it's case.
00414     """
00415     def middle(lst, type_name):
00416         """ The parser type generator"""
00417         if not isinstance(lst[0], unicode):
00418             lst = [o.decode() for o in lst]
00419         def parse_type(string):
00420             """ The parser type"""
00421             if not isinstance(string, unicode):
00422                 string = string.decode()
00423             for option in lst:
00424                 if case(string) == case(option):
00425                     return option
00426             raise argparse.ArgumentTypeError(
00427                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00428                 format(string, type_name, columnate(lst)))
00429         return parse_type
00430     return middle
00431 
00432 # these two types convert the case of their arguments _before_ validation
00433 argparse_force_uppercase_type = argparse_force_type(unicode.upper)
00434 argparse_force_lowercase_type = argparse_force_type(unicode.lower)
00435 
00436 def argparse_many (func):
00437     """ An argument parser combinator that takes in an argument parser and
00438     creates a new parser that accepts a comma separated list of the same thing.
00439     """
00440     def wrap(string):
00441         """ The actual parser"""
00442         return [func(s) for s in string.split(",")]
00443     return wrap
00444 
00445 def argparse_filestring_type (string):
00446     """ An argument parser that verifies that a string passed in corresponds
00447     to a file"""
00448     if exists(string):
00449         return string
00450     else:
00451         raise argparse.ArgumentTypeError(
00452             "{0}"" does not exist in the filesystem.".format(string))
00453 
00454 def argparse_profile_filestring_type (string):
00455     """ An argument parser that verifies that a string passed in is either
00456     absolute path or a file name (expanded to
00457     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00458     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00459     if exists(string):
00460         return string
00461     elif exists(fpath):
00462         return fpath
00463     else:
00464         raise argparse.ArgumentTypeError(
00465             "{0} does not exist in the filesystem.".format(string))
00466 
00467 def columnate (strings, separator=", ", chars=80):
00468     """ render a list of strings as a in a bunch of columns
00469 
00470     Positional arguments:
00471     strings - the strings to columnate
00472 
00473     Keyword arguments;
00474     separator - the separation between the columns
00475     chars - the maximum with of a row
00476     """
00477     col_width = max(len(s) for s in strings)
00478     total_width = col_width + len(separator)
00479     columns = math.floor(chars / total_width)
00480     output = ""
00481     for i, string in zip(range(len(strings)), strings):
00482         append = string
00483         if i != len(strings) - 1:
00484             append += separator
00485         if i % columns == columns - 1:
00486             append += "\n"
00487         else:
00488             append = append.ljust(total_width)
00489         output += append
00490     return output
00491 
00492 def argparse_dir_not_parent (other):
00493     """fail if argument provided is a parent of the specified directory"""
00494     def parse_type(not_parent):
00495         """The parser type"""
00496         abs_other = abspath(other)
00497         abs_not_parent = abspath(not_parent)
00498         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00499             raise argparse.ArgumentTypeError(
00500                 "{0} may not be a parent directory of {1}".format(
00501                     not_parent, other))
00502         else:
00503             return not_parent
00504     return parse_type
00505 
00506 def argparse_deprecate (replacement_message):
00507     """fail if argument is provided with deprecation warning"""
00508     def parse_type(_):
00509         """The parser type"""
00510         raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
00511     return parse_type
00512 
00513 def print_large_string (large_string):
00514     """ Breaks a string up into smaller pieces before print them
00515 
00516     This is a limitation within Windows, as detailed here:
00517     https://bugs.python.org/issue11395
00518 
00519     Positional arguments:
00520     large_string - the large string to print
00521     """
00522     string_limit = 1000
00523     large_string_len = len(large_string)
00524     num_parts = int(ceil(float(large_string_len) / float(string_limit)))
00525     for string_part in range(num_parts):
00526         start_index = string_part * string_limit
00527         if string_part == num_parts - 1:
00528             sys.stdout.write(large_string[start_index:])
00529         else:
00530             sys.stdout.write(large_string[start_index:
00531                                           start_index + string_limit])
00532     sys.stdout.write("\n")
00533 
00534 def intelhex_offset (filename, offset):
00535     """Load a hex or bin file at a particular offset"""
00536     _, inteltype = splitext(filename)
00537     ih = IntelHex()
00538     if inteltype == ".bin":
00539         ih.loadbin(filename, offset=offset)
00540     elif inteltype == ".hex":
00541         ih.loadhex(filename)
00542     else:
00543         raise ToolException("File %s does not have a known binary file type"
00544                             % filename)
00545     return ih
00546 
00547 
00548 def integer (maybe_string, base):
00549     """Make an integer of a number or a string"""
00550     if isinstance(maybe_string, int):
00551         return maybe_string
00552     else:
00553         return int(maybe_string, base)