utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 import sys
00019 import inspect
00020 import os
00021 import argparse
00022 import math
00023 from os import listdir, remove, makedirs
00024 from shutil import copyfile
00025 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00026 from os.path import commonprefix, normpath, dirname
00027 from subprocess import Popen, PIPE, STDOUT, call
00028 from math import ceil
00029 import json
00030 from collections import OrderedDict
00031 import logging
00032 from intelhex import IntelHex
00033 
00034 try:
00035     unicode
00036 except NameError:
00037     unicode = str
00038 
00039 def remove_if_in(lst, thing):
00040     if thing in lst:
00041         lst.remove(thing)
00042 
00043 def compile_worker (job):
00044     """Standard task runner used for compiling
00045 
00046     Positional argumets:
00047     job - a dict containing a list of commands and the remaining arguments
00048           to run_cmd
00049     """
00050     results = []
00051     for command in job['commands']:
00052         try:
00053             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00054                                       chroot=job['chroot'])
00055         except KeyboardInterrupt:
00056             raise ToolException
00057 
00058         results.append({
00059             'code': _rc,
00060             'output': _stderr,
00061             'command': command
00062         })
00063 
00064     return {
00065         'source': job['source'],
00066         'object': job['object'],
00067         'commands': job['commands'],
00068         'results': results
00069     }
00070 
00071 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00072     """A wrapper to run a command as a blocking job"""
00073     text = command if shell else ' '.join(command)
00074     if verbose:
00075         print(text)
00076     return_code = call(command, shell=shell, cwd=cwd)
00077     if check and return_code != 0:
00078         raise Exception('ERROR %d: "%s"' % (return_code, text))
00079 
00080 
00081 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00082     """Run a command in the foreground
00083 
00084     Positional arguments:
00085     command - the command to run
00086 
00087     Keyword arguments:
00088     work_dir - the working directory to run the command in
00089     chroot - the chroot to run the command in
00090     redirect - redirect the stderr to a pipe to be used later
00091     """
00092     if chroot:
00093         # Conventions managed by the web team for the mbed.org build system
00094         chroot_cmd = [
00095             '/usr/sbin/chroot', '--userspec=33:33', chroot
00096         ]
00097         for element in command:
00098             chroot_cmd += [element.replace(chroot, '')]
00099 
00100         logging.debug("Running command %s", ' '.join(chroot_cmd))
00101         command = chroot_cmd
00102         work_dir = None
00103 
00104     try:
00105         process = Popen(command, stdout=PIPE,
00106                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00107         _stdout, _stderr = process.communicate()
00108     except OSError:
00109         print("[OS ERROR] Command: "+(' '.join(command)))
00110         raise
00111 
00112     return _stdout, _stderr, process.returncode
00113 
00114 
00115 def run_cmd_ext (command):
00116     """ A version of run command that checks if the command exists befor running
00117 
00118     Positional arguments:
00119     command - the command line you are trying to invoke
00120     """
00121     assert is_cmd_valid(command[0])
00122     process = Popen(command, stdout=PIPE, stderr=PIPE)
00123     _stdout, _stderr = process.communicate()
00124     return _stdout, _stderr, process.returncode
00125 
00126 
00127 def is_cmd_valid (command):
00128     """ Verify that a command exists and is executable
00129 
00130     Positional arguments:
00131     command - the command to check
00132     """
00133     caller = get_caller_name()
00134     cmd_path = find_cmd_abspath(command)
00135     if not cmd_path:
00136         error("%s: Command '%s' can't be found" % (caller, command))
00137     if not is_exec(cmd_path):
00138         error("%s: Command '%s' resolves to file '%s' which is not executable"
00139               % (caller, command, cmd_path))
00140     return True
00141 
00142 
00143 def is_exec (path):
00144     """A simple check to verify that a path to an executable exists
00145 
00146     Positional arguments:
00147     path - the executable
00148     """
00149     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00150 
00151 
00152 def find_cmd_abspath (command):
00153     """ Returns the absolute path to a command.
00154         None is returned if no absolute path was found.
00155 
00156     Positional arguhments:
00157     command - the command to find the path of
00158     """
00159     if exists(command) or exists(command + '.exe'):
00160         return os.path.abspath(command)
00161     if not 'PATH' in os.environ:
00162         raise Exception("Can't find command path for current platform ('%s')"
00163                         % sys.platform)
00164     path_env = os.environ['PATH']
00165     for path in path_env.split(os.pathsep):
00166         cmd_path = '%s/%s' % (path, command)
00167         if exists(cmd_path) or exists(cmd_path + '.exe'):
00168             return cmd_path
00169 
00170 
00171 def mkdir (path):
00172     """ a wrapped makedirs that only tries to create a directory if it does not
00173     exist already
00174 
00175     Positional arguments:
00176     path - the path to maybe create
00177     """
00178     if not exists(path):
00179         makedirs(path)
00180 
00181 
00182 def copy_file (src, dst):
00183     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00184     permissions (this was causing errors with directories mounted with samba)
00185 
00186     Positional arguments:
00187     src - the source of the copy operation
00188     dst - the destination of the copy operation
00189     """
00190     if isdir(dst):
00191         _, base = split(src)
00192         dst = join(dst, base)
00193     copyfile(src, dst)
00194 
00195 
00196 def delete_dir_files (directory):
00197     """ A function that does rm -rf
00198 
00199     Positional arguments:
00200     directory - the directory to remove
00201     """
00202     if not exists(directory):
00203         return
00204 
00205     for element in listdir(directory):
00206         to_remove = join(directory, element)
00207         if not isdir(to_remove):
00208             remove(to_remove)
00209 
00210 
00211 def get_caller_name (steps=2):
00212     """
00213     When called inside a function, it returns the name
00214     of the caller of that function.
00215 
00216     Keyword arguments:
00217     steps - the number of steps up the stack the calling function is
00218     """
00219     return inspect.stack()[steps][3]
00220 
00221 
00222 def error (msg):
00223     """Fatal error, abort hard
00224 
00225     Positional arguments:
00226     msg - the message to print before crashing
00227     """
00228     print("ERROR: %s" % msg)
00229     sys.exit(1)
00230 
00231 
00232 def rel_path (path, base, dot=False):
00233     """Relative path calculation that optionaly always starts with a dot
00234 
00235     Positional arguments:
00236     path - the path to make relative
00237     base - what to make the path relative to
00238 
00239     Keyword arguments:
00240     dot - if True, the path will always start with a './'
00241     """
00242     final_path = relpath(path, base)
00243     if dot and not final_path.startswith('.'):
00244         final_path = './' + final_path
00245     return final_path
00246 
00247 
00248 class ToolException (Exception):
00249     """A class representing an exception throw by the tools"""
00250     pass
00251 
00252 class NotSupportedException(Exception):
00253     """A class a toolchain not supporting a particular target"""
00254     pass
00255 
00256 class InvalidReleaseTargetException(Exception):
00257     pass
00258 
00259 def split_path (path):
00260     """spilt a file name into it's directory name, base name, and extension
00261 
00262     Positional arguments:
00263     path - the file name to split
00264     """
00265     base, has_ext = split(path)
00266     name, ext = splitext(has_ext)
00267     return base, name, ext
00268 
00269 
00270 def get_path_depth (path):
00271     """ Given a path, return the number of directory levels present.
00272         This roughly translates to the number of path separators (os.sep) + 1.
00273         Ex. Given "path/to/dir", this would return 3
00274         Special cases: "." and "/" return 0
00275 
00276     Positional arguments:
00277     path - the path to calculate the depth of
00278     """
00279     normalized_path = normpath(path)
00280     path_depth = 0
00281     head, tail = split(normalized_path)
00282 
00283     while tail and tail != '.':
00284         path_depth += 1
00285         head, tail = split(head)
00286 
00287     return path_depth
00288 
00289 
00290 def args_error (parser, message):
00291     """Abort with an error that was generated by the arguments to a CLI program
00292 
00293     Positional arguments:
00294     parser - the ArgumentParser object that parsed the command line
00295     message - what went wrong
00296     """
00297     parser.error(message)
00298     sys.exit(2)
00299 
00300 
00301 def construct_enum (**enums):
00302     """ Create your own pseudo-enums
00303 
00304     Keyword arguments:
00305     * - a member of the Enum you are creating and it's value
00306     """
00307     return type('Enum', (), enums)
00308 
00309 
00310 def check_required_modules (required_modules, verbose=True):
00311     """ Function checks for Python modules which should be "importable"
00312         before test suite can be used.
00313         @return returns True if all modules are installed already
00314     """
00315     import imp
00316     not_installed_modules = []
00317     for module_name in required_modules:
00318         try:
00319             imp.find_module(module_name)
00320         except ImportError:
00321             # We also test against a rare case: module is an egg file
00322             try:
00323                 __import__(module_name)
00324             except ImportError as exc:
00325                 not_installed_modules.append(module_name)
00326                 if verbose:
00327                     print("Error: %s" % exc)
00328 
00329     if verbose:
00330         if not_installed_modules:
00331             print("Warning: Module(s) %s not installed. Please install "
00332                   "required module(s) before using this script."
00333                   % (', '.join(not_installed_modules)))
00334 
00335     if not_installed_modules:
00336         return False
00337     else:
00338         return True
00339 
00340 def json_file_to_dict (fname):
00341     """ Read a JSON file and return its Python representation, transforming all
00342     the strings from Unicode to ASCII. The order of keys in the JSON file is
00343     preserved.
00344 
00345     Positional arguments:
00346     fname - the name of the file to parse
00347     """
00348     try:
00349         with open(fname, "r") as file_obj:
00350             return json.loads(file_obj.read().encode('ascii', 'ignore'),
00351                               object_pairs_hook=OrderedDict)
00352     except (ValueError, IOError):
00353         sys.stderr.write("Error parsing '%s':\n" % fname)
00354         raise
00355 
00356 # Wowza, double closure
00357 def argparse_type(casedness, prefer_hyphen=False):
00358     def middle(lst, type_name):
00359         def parse_type(string):
00360             """ validate that an argument passed in (as string) is a member of
00361             the list of possible arguments. Offer a suggestion if the case of
00362             the string, or the hyphens/underscores do not match the expected
00363             style of the argument.
00364             """
00365             if not isinstance(string, unicode):
00366                 string = string.decode()
00367             if prefer_hyphen:
00368                 newstring = casedness(string).replace("_", "-")
00369             else:
00370                 newstring = casedness(string).replace("-", "_")
00371             if string in lst:
00372                 return string
00373             elif string not in lst and newstring in lst:
00374                 raise argparse.ArgumentTypeError(
00375                     "{0} is not a supported {1}. Did you mean {2}?".format(
00376                         string, type_name, newstring))
00377             else:
00378                 raise argparse.ArgumentTypeError(
00379                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00380                     format(string, type_name, columnate(lst)))
00381         return parse_type
00382     return middle
00383 
00384 # short cuts for the argparse_type versions
00385 argparse_uppercase_type = argparse_type(unicode.upper, False)
00386 argparse_lowercase_type = argparse_type(unicode.lower, False)
00387 argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True)
00388 argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True)
00389 
00390 def argparse_force_type (case):
00391     """ validate that an argument passed in (as string) is a member of the list
00392     of possible arguments after converting it's case.
00393     """
00394     def middle(lst, type_name):
00395         """ The parser type generator"""
00396         if not isinstance(lst[0], unicode):
00397             lst = [o.decode() for o in lst]
00398         def parse_type(string):
00399             """ The parser type"""
00400             if not isinstance(string, unicode):
00401                 string = string.decode()
00402             for option in lst:
00403                 if case(string) == case(option):
00404                     return option
00405             raise argparse.ArgumentTypeError(
00406                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00407                 format(string, type_name, columnate(lst)))
00408         return parse_type
00409     return middle
00410 
00411 # these two types convert the case of their arguments _before_ validation
00412 argparse_force_uppercase_type = argparse_force_type(unicode.upper)
00413 argparse_force_lowercase_type = argparse_force_type(unicode.lower)
00414 
00415 def argparse_many (func):
00416     """ An argument parser combinator that takes in an argument parser and
00417     creates a new parser that accepts a comma separated list of the same thing.
00418     """
00419     def wrap(string):
00420         """ The actual parser"""
00421         return [func(s) for s in string.split(",")]
00422     return wrap
00423 
00424 def argparse_filestring_type (string):
00425     """ An argument parser that verifies that a string passed in corresponds
00426     to a file"""
00427     if exists(string):
00428         return string
00429     else:
00430         raise argparse.ArgumentTypeError(
00431             "{0}"" does not exist in the filesystem.".format(string))
00432 
00433 def argparse_profile_filestring_type (string):
00434     """ An argument parser that verifies that a string passed in is either
00435     absolute path or a file name (expanded to
00436     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00437     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00438     if exists(string):
00439         return string
00440     elif exists(fpath):
00441         return fpath
00442     else:
00443         raise argparse.ArgumentTypeError(
00444             "{0} does not exist in the filesystem.".format(string))
00445 
00446 def columnate (strings, separator=", ", chars=80):
00447     """ render a list of strings as a in a bunch of columns
00448 
00449     Positional arguments:
00450     strings - the strings to columnate
00451 
00452     Keyword arguments;
00453     separator - the separation between the columns
00454     chars - the maximum with of a row
00455     """
00456     col_width = max(len(s) for s in strings)
00457     total_width = col_width + len(separator)
00458     columns = math.floor(chars / total_width)
00459     output = ""
00460     for i, string in zip(range(len(strings)), strings):
00461         append = string
00462         if i != len(strings) - 1:
00463             append += separator
00464         if i % columns == columns - 1:
00465             append += "\n"
00466         else:
00467             append = append.ljust(total_width)
00468         output += append
00469     return output
00470 
00471 def argparse_dir_not_parent (other):
00472     """fail if argument provided is a parent of the specified directory"""
00473     def parse_type(not_parent):
00474         """The parser type"""
00475         abs_other = abspath(other)
00476         abs_not_parent = abspath(not_parent)
00477         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00478             raise argparse.ArgumentTypeError(
00479                 "{0} may not be a parent directory of {1}".format(
00480                     not_parent, other))
00481         else:
00482             return not_parent
00483     return parse_type
00484 
00485 def argparse_deprecate (replacement_message):
00486     """fail if argument is provided with deprecation warning"""
00487     def parse_type(_):
00488         """The parser type"""
00489         raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
00490     return parse_type
00491 
00492 def print_large_string (large_string):
00493     """ Breaks a string up into smaller pieces before print them
00494 
00495     This is a limitation within Windows, as detailed here:
00496     https://bugs.python.org/issue11395
00497 
00498     Positional arguments:
00499     large_string - the large string to print
00500     """
00501     string_limit = 1000
00502     large_string_len = len(large_string)
00503     num_parts = int(ceil(float(large_string_len) / float(string_limit)))
00504     for string_part in range(num_parts):
00505         start_index = string_part * string_limit
00506         if string_part == num_parts - 1:
00507             sys.stdout.write(large_string[start_index:])
00508         else:
00509             sys.stdout.write(large_string[start_index:
00510                                           start_index + string_limit])
00511     sys.stdout.write("\n")
00512 
00513 def intelhex_offset (filename, offset):
00514     """Load a hex or bin file at a particular offset"""
00515     _, inteltype = splitext(filename)
00516     ih = IntelHex()
00517     if inteltype == ".bin":
00518         ih.loadbin(filename, offset=offset)
00519     elif inteltype == ".hex":
00520         ih.loadhex(filename)
00521     else:
00522         raise ToolException("File %s does not have a known binary file type"
00523                             % filename)
00524     return ih
00525 
00526 
00527 def integer (maybe_string, base):
00528     """Make an integer of a number or a string"""
00529     if isinstance(maybe_string, int):
00530         return maybe_string
00531     else:
00532         return int(maybe_string, base)