utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 import sys
00019 import inspect
00020 import os
00021 import argparse
00022 import math
00023 from os import listdir, remove, makedirs
00024 from shutil import copyfile
00025 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00026 from os.path import commonprefix, normpath, dirname
00027 from subprocess import Popen, PIPE, STDOUT, call
00028 from math import ceil
00029 import json
00030 from collections import OrderedDict
00031 import logging
00032 from intelhex import IntelHex
00033 
00034 try:
00035     unicode
00036 except NameError:
00037     unicode = str
00038 
00039 def remove_if_in(lst, thing):
00040     if thing in lst:
00041         lst.remove(thing)
00042 
00043 def compile_worker (job):
00044     """Standard task runner used for compiling
00045 
00046     Positional argumets:
00047     job - a dict containing a list of commands and the remaining arguments
00048           to run_cmd
00049     """
00050     results = []
00051     for command in job['commands']:
00052         try:
00053             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00054                                       chroot=job['chroot'])
00055         except KeyboardInterrupt:
00056             raise ToolException
00057 
00058         results.append({
00059             'code': _rc,
00060             'output': _stderr,
00061             'command': command
00062         })
00063 
00064     return {
00065         'source': job['source'],
00066         'object': job['object'],
00067         'commands': job['commands'],
00068         'results': results
00069     }
00070 
00071 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00072     """A wrapper to run a command as a blocking job"""
00073     text = command if shell else ' '.join(command)
00074     if verbose:
00075         print(text)
00076     return_code = call(command, shell=shell, cwd=cwd)
00077     if check and return_code != 0:
00078         raise Exception('ERROR %d: "%s"' % (return_code, text))
00079 
00080 
00081 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00082     """Run a command in the foreground
00083 
00084     Positional arguments:
00085     command - the command to run
00086 
00087     Keyword arguments:
00088     work_dir - the working directory to run the command in
00089     chroot - the chroot to run the command in
00090     redirect - redirect the stderr to a pipe to be used later
00091     """
00092     if chroot:
00093         # Conventions managed by the web team for the mbed.org build system
00094         chroot_cmd = [
00095             '/usr/sbin/chroot', '--userspec=33:33', chroot
00096         ]
00097         for element in command:
00098             chroot_cmd += [element.replace(chroot, '')]
00099 
00100         logging.debug("Running command %s", ' '.join(chroot_cmd))
00101         command = chroot_cmd
00102         work_dir = None
00103 
00104     try:
00105         process = Popen(command, stdout=PIPE,
00106                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00107         _stdout, _stderr = process.communicate()
00108     except OSError:
00109         print("[OS ERROR] Command: "+(' '.join(command)))
00110         raise
00111 
00112     return _stdout, _stderr, process.returncode
00113 
00114 
00115 def run_cmd_ext (command):
00116     """ A version of run command that checks if the command exists befor running
00117 
00118     Positional arguments:
00119     command - the command line you are trying to invoke
00120     """
00121     assert is_cmd_valid(command[0])
00122     process = Popen(command, stdout=PIPE, stderr=PIPE)
00123     _stdout, _stderr = process.communicate()
00124     return _stdout, _stderr, process.returncode
00125 
00126 
00127 def is_cmd_valid (command):
00128     """ Verify that a command exists and is executable
00129 
00130     Positional arguments:
00131     command - the command to check
00132     """
00133     caller = get_caller_name()
00134     cmd_path = find_cmd_abspath(command)
00135     if not cmd_path:
00136         error("%s: Command '%s' can't be found" % (caller, command))
00137     if not is_exec(cmd_path):
00138         error("%s: Command '%s' resolves to file '%s' which is not executable"
00139               % (caller, command, cmd_path))
00140     return True
00141 
00142 
00143 def is_exec (path):
00144     """A simple check to verify that a path to an executable exists
00145 
00146     Positional arguments:
00147     path - the executable
00148     """
00149     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00150 
00151 
00152 def find_cmd_abspath (command):
00153     """ Returns the absolute path to a command.
00154         None is returned if no absolute path was found.
00155 
00156     Positional arguhments:
00157     command - the command to find the path of
00158     """
00159     if exists(command) or exists(command + '.exe'):
00160         return os.path.abspath(command)
00161     if not 'PATH' in os.environ:
00162         raise Exception("Can't find command path for current platform ('%s')"
00163                         % sys.platform)
00164     path_env = os.environ['PATH']
00165     for path in path_env.split(os.pathsep):
00166         cmd_path = '%s/%s' % (path, command)
00167         if exists(cmd_path) or exists(cmd_path + '.exe'):
00168             return cmd_path
00169 
00170 
00171 def mkdir (path):
00172     """ a wrapped makedirs that only tries to create a directory if it does not
00173     exist already
00174 
00175     Positional arguments:
00176     path - the path to maybe create
00177     """
00178     if not exists(path):
00179         makedirs(path)
00180 
00181 
00182 def copy_file (src, dst):
00183     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00184     permissions (this was causing errors with directories mounted with samba)
00185 
00186     Positional arguments:
00187     src - the source of the copy operation
00188     dst - the destination of the copy operation
00189     """
00190     if isdir(dst):
00191         _, base = split(src)
00192         dst = join(dst, base)
00193     copyfile(src, dst)
00194 
00195 
00196 def delete_dir_files (directory):
00197     """ A function that does rm -rf
00198 
00199     Positional arguments:
00200     directory - the directory to remove
00201     """
00202     if not exists(directory):
00203         return
00204 
00205     for element in listdir(directory):
00206         to_remove = join(directory, element)
00207         if not isdir(to_remove):
00208             remove(to_remove)
00209 
00210 
00211 def get_caller_name (steps=2):
00212     """
00213     When called inside a function, it returns the name
00214     of the caller of that function.
00215 
00216     Keyword arguments:
00217     steps - the number of steps up the stack the calling function is
00218     """
00219     return inspect.stack()[steps][3]
00220 
00221 
00222 def error (msg):
00223     """Fatal error, abort hard
00224 
00225     Positional arguments:
00226     msg - the message to print before crashing
00227     """
00228     print("ERROR: %s" % msg)
00229     sys.exit(1)
00230 
00231 
00232 def rel_path (path, base, dot=False):
00233     """Relative path calculation that optionaly always starts with a dot
00234 
00235     Positional arguments:
00236     path - the path to make relative
00237     base - what to make the path relative to
00238 
00239     Keyword arguments:
00240     dot - if True, the path will always start with a './'
00241     """
00242     final_path = relpath(path, base)
00243     if dot and not final_path.startswith('.'):
00244         final_path = './' + final_path
00245     return final_path
00246 
00247 
00248 class ToolException (Exception):
00249     """A class representing an exception throw by the tools"""
00250     pass
00251 
00252 class NotSupportedException(Exception):
00253     """A class a toolchain not supporting a particular target"""
00254     pass
00255 
00256 class InvalidReleaseTargetException(Exception):
00257     pass
00258 
00259 def split_path (path):
00260     """spilt a file name into it's directory name, base name, and extension
00261 
00262     Positional arguments:
00263     path - the file name to split
00264     """
00265     base, has_ext = split(path)
00266     name, ext = splitext(has_ext)
00267     return base, name, ext
00268 
00269 
00270 def get_path_depth (path):
00271     """ Given a path, return the number of directory levels present.
00272         This roughly translates to the number of path separators (os.sep) + 1.
00273         Ex. Given "path/to/dir", this would return 3
00274         Special cases: "." and "/" return 0
00275 
00276     Positional arguments:
00277     path - the path to calculate the depth of
00278     """
00279     normalized_path = normpath(path)
00280     path_depth = 0
00281     head, tail = split(normalized_path)
00282 
00283     while tail and tail != '.':
00284         path_depth += 1
00285         head, tail = split(head)
00286 
00287     return path_depth
00288 
00289 
00290 def args_error (parser, message):
00291     """Abort with an error that was generated by the arguments to a CLI program
00292 
00293     Positional arguments:
00294     parser - the ArgumentParser object that parsed the command line
00295     message - what went wrong
00296     """
00297     parser.error(message)
00298     sys.exit(2)
00299 
00300 
00301 def construct_enum (**enums):
00302     """ Create your own pseudo-enums
00303 
00304     Keyword arguments:
00305     * - a member of the Enum you are creating and it's value
00306     """
00307     return type('Enum', (), enums)
00308 
00309 
00310 def check_required_modules (required_modules, verbose=True):
00311     """ Function checks for Python modules which should be "importable"
00312         before test suite can be used.
00313         @return returns True if all modules are installed already
00314     """
00315     import imp
00316     not_installed_modules = []
00317     for module_name in required_modules:
00318         try:
00319             imp.find_module(module_name)
00320         except ImportError:
00321             # We also test against a rare case: module is an egg file
00322             try:
00323                 __import__(module_name)
00324             except ImportError as exc:
00325                 not_installed_modules.append(module_name)
00326                 if verbose:
00327                     print("Error: %s" % exc)
00328 
00329     if verbose:
00330         if not_installed_modules:
00331             print("Warning: Module(s) %s not installed. Please install "
00332                   "required module(s) before using this script."
00333                   % (', '.join(not_installed_modules)))
00334 
00335     if not_installed_modules:
00336         return False
00337     else:
00338         return True
00339 
00340 def dict_to_ascii (dictionary):
00341     """ Utility function: traverse a dictionary and change all the strings in
00342     the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
00343     because the JSON decoder always returns Unicode string. Based on
00344     http://stackoverflow.com/a/13105359
00345 
00346     Positional arguments:
00347     dictionary - The dict that contains some Unicode that should be ASCII
00348     """
00349     if isinstance(dictionary, dict):
00350         return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
00351                             for key, value in dictionary.items()])
00352     elif isinstance(dictionary, list):
00353         return [dict_to_ascii(element) for element in dictionary]
00354     elif isinstance(dictionary, unicode):
00355         return dictionary.encode('ascii').decode()
00356     else:
00357         return dictionary
00358 
00359 def json_file_to_dict (fname):
00360     """ Read a JSON file and return its Python representation, transforming all
00361     the strings from Unicode to ASCII. The order of keys in the JSON file is
00362     preserved.
00363 
00364     Positional arguments:
00365     fname - the name of the file to parse
00366     """
00367     try:
00368         with open(fname, "r") as file_obj:
00369             return dict_to_ascii(json.load(file_obj,
00370                                            object_pairs_hook=OrderedDict))
00371     except (ValueError, IOError):
00372         sys.stderr.write("Error parsing '%s':\n" % fname)
00373         raise
00374 
00375 # Wowza, double closure
00376 def argparse_type(casedness, prefer_hyphen=False):
00377     def middle(lst, type_name):
00378         def parse_type(string):
00379             """ validate that an argument passed in (as string) is a member of
00380             the list of possible arguments. Offer a suggestion if the case of
00381             the string, or the hyphens/underscores do not match the expected
00382             style of the argument.
00383             """
00384             if not isinstance(string, unicode):
00385                 string = string.decode()
00386             if prefer_hyphen:
00387                 newstring = casedness(string).replace("_", "-")
00388             else:
00389                 newstring = casedness(string).replace("-", "_")
00390             if string in lst:
00391                 return string
00392             elif string not in lst and newstring in lst:
00393                 raise argparse.ArgumentTypeError(
00394                     "{0} is not a supported {1}. Did you mean {2}?".format(
00395                         string, type_name, newstring))
00396             else:
00397                 raise argparse.ArgumentTypeError(
00398                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00399                     format(string, type_name, columnate(lst)))
00400         return parse_type
00401     return middle
00402 
00403 # short cuts for the argparse_type versions
00404 argparse_uppercase_type = argparse_type(unicode.upper, False)
00405 argparse_lowercase_type = argparse_type(unicode.lower, False)
00406 argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True)
00407 argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True)
00408 
00409 def argparse_force_type (case):
00410     """ validate that an argument passed in (as string) is a member of the list
00411     of possible arguments after converting it's case.
00412     """
00413     def middle(lst, type_name):
00414         """ The parser type generator"""
00415         def parse_type(string):
00416             """ The parser type"""
00417             if not isinstance(string, unicode):
00418                 string = string.decode()
00419             for option in lst:
00420                 if case(string) == case(option):
00421                     return option
00422             raise argparse.ArgumentTypeError(
00423                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00424                 format(string, type_name, columnate(lst)))
00425         return parse_type
00426     return middle
00427 
00428 # these two types convert the case of their arguments _before_ validation
00429 argparse_force_uppercase_type = argparse_force_type(unicode.upper)
00430 argparse_force_lowercase_type = argparse_force_type(unicode.lower)
00431 
00432 def argparse_many (func):
00433     """ An argument parser combinator that takes in an argument parser and
00434     creates a new parser that accepts a comma separated list of the same thing.
00435     """
00436     def wrap(string):
00437         """ The actual parser"""
00438         return [func(s) for s in string.split(",")]
00439     return wrap
00440 
00441 def argparse_filestring_type (string):
00442     """ An argument parser that verifies that a string passed in corresponds
00443     to a file"""
00444     if exists(string):
00445         return string
00446     else:
00447         raise argparse.ArgumentTypeError(
00448             "{0}"" does not exist in the filesystem.".format(string))
00449 
00450 def argparse_profile_filestring_type (string):
00451     """ An argument parser that verifies that a string passed in is either
00452     absolute path or a file name (expanded to
00453     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00454     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00455     if exists(string):
00456         return string
00457     elif exists(fpath):
00458         return fpath
00459     else:
00460         raise argparse.ArgumentTypeError(
00461             "{0} does not exist in the filesystem.".format(string))
00462 
00463 def columnate (strings, separator=", ", chars=80):
00464     """ render a list of strings as a in a bunch of columns
00465 
00466     Positional arguments:
00467     strings - the strings to columnate
00468 
00469     Keyword arguments;
00470     separator - the separation between the columns
00471     chars - the maximum with of a row
00472     """
00473     col_width = max(len(s) for s in strings)
00474     total_width = col_width + len(separator)
00475     columns = math.floor(chars / total_width)
00476     output = ""
00477     for i, string in zip(range(len(strings)), strings):
00478         append = string
00479         if i != len(strings) - 1:
00480             append += separator
00481         if i % columns == columns - 1:
00482             append += "\n"
00483         else:
00484             append = append.ljust(total_width)
00485         output += append
00486     return output
00487 
00488 def argparse_dir_not_parent (other):
00489     """fail if argument provided is a parent of the specified directory"""
00490     def parse_type(not_parent):
00491         """The parser type"""
00492         abs_other = abspath(other)
00493         abs_not_parent = abspath(not_parent)
00494         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00495             raise argparse.ArgumentTypeError(
00496                 "{0} may not be a parent directory of {1}".format(
00497                     not_parent, other))
00498         else:
00499             return not_parent
00500     return parse_type
00501 
00502 def argparse_deprecate (replacement_message):
00503     """fail if argument is provided with deprecation warning"""
00504     def parse_type(_):
00505         """The parser type"""
00506         raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
00507     return parse_type
00508 
00509 def print_large_string (large_string):
00510     """ Breaks a string up into smaller pieces before print them
00511 
00512     This is a limitation within Windows, as detailed here:
00513     https://bugs.python.org/issue11395
00514 
00515     Positional arguments:
00516     large_string - the large string to print
00517     """
00518     string_limit = 1000
00519     large_string_len = len(large_string)
00520     num_parts = int(ceil(float(large_string_len) / float(string_limit)))
00521     for string_part in range(num_parts):
00522         start_index = string_part * string_limit
00523         if string_part == num_parts - 1:
00524             sys.stdout.write(large_string[start_index:])
00525         else:
00526             sys.stdout.write(large_string[start_index:
00527                                           start_index + string_limit])
00528     sys.stdout.write("\n")
00529 
00530 def intelhex_offset (filename, offset):
00531     """Load a hex or bin file at a particular offset"""
00532     _, inteltype = splitext(filename)
00533     ih = IntelHex()
00534     if inteltype == ".bin":
00535         ih.loadbin(filename, offset=offset)
00536     elif inteltype == ".hex":
00537         ih.loadhex(filename)
00538     else:
00539         raise ToolException("File %s does not have a known binary file type"
00540                             % filename)
00541     return ih