utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 import sys
00018 import inspect
00019 import os
00020 import argparse
00021 import math
00022 from os import listdir, remove, makedirs
00023 from shutil import copyfile
00024 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00025 from os.path import commonprefix, normpath, dirname
00026 from subprocess import Popen, PIPE, STDOUT, call
00027 from math import ceil
00028 import json
00029 from collections import OrderedDict
00030 import logging
00031 from intelhex import IntelHex
00032 
00033 def remove_if_in(lst, thing):
00034     if thing in lst:
00035         lst.remove(thing)
00036 
00037 def compile_worker (job):
00038     """Standard task runner used for compiling
00039 
00040     Positional argumets:
00041     job - a dict containing a list of commands and the remaining arguments
00042           to run_cmd
00043     """
00044     results = []
00045     for command in job['commands']:
00046         try:
00047             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00048                                       chroot=job['chroot'])
00049         except KeyboardInterrupt:
00050             raise ToolException
00051 
00052         results.append({
00053             'code': _rc,
00054             'output': _stderr,
00055             'command': command
00056         })
00057 
00058     return {
00059         'source': job['source'],
00060         'object': job['object'],
00061         'commands': job['commands'],
00062         'results': results
00063     }
00064 
00065 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00066     """A wrapper to run a command as a blocking job"""
00067     text = command if shell else ' '.join(command)
00068     if verbose:
00069         print text
00070     return_code = call(command, shell=shell, cwd=cwd)
00071     if check and return_code != 0:
00072         raise Exception('ERROR %d: "%s"' % (return_code, text))
00073 
00074 
00075 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00076     """Run a command in the forground
00077 
00078     Positional arguments:
00079     command - the command to run
00080 
00081     Keyword arguments:
00082     work_dir - the working directory to run the command in
00083     chroot - the chroot to run the command in
00084     redirect - redirect the stderr to a pipe to be used later
00085     """
00086     if chroot:
00087         # Conventions managed by the web team for the mbed.org build system
00088         chroot_cmd = [
00089             '/usr/sbin/chroot', '--userspec=33:33', chroot
00090         ]
00091         for element in command:
00092             chroot_cmd += [element.replace(chroot, '')]
00093 
00094         logging.debug("Running command %s", ' '.join(chroot_cmd))
00095         command = chroot_cmd
00096         work_dir = None
00097 
00098     try:
00099         process = Popen(command, stdout=PIPE,
00100                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00101         _stdout, _stderr = process.communicate()
00102     except OSError:
00103         print "[OS ERROR] Command: "+(' '.join(command))
00104         raise
00105 
00106     return _stdout, _stderr, process.returncode
00107 
00108 
00109 def run_cmd_ext (command):
00110     """ A version of run command that checks if the command exists befor running
00111 
00112     Positional arguments:
00113     command - the command line you are trying to invoke
00114     """
00115     assert is_cmd_valid(command[0])
00116     process = Popen(command, stdout=PIPE, stderr=PIPE)
00117     _stdout, _stderr = process.communicate()
00118     return _stdout, _stderr, process.returncode
00119 
00120 
00121 def is_cmd_valid (command):
00122     """ Verify that a command exists and is executable
00123 
00124     Positional arguments:
00125     command - the command to check
00126     """
00127     caller = get_caller_name()
00128     cmd_path = find_cmd_abspath(command)
00129     if not cmd_path:
00130         error("%s: Command '%s' can't be found" % (caller, command))
00131     if not is_exec(cmd_path):
00132         error("%s: Command '%s' resolves to file '%s' which is not executable"
00133               % (caller, command, cmd_path))
00134     return True
00135 
00136 
00137 def is_exec (path):
00138     """A simple check to verify that a path to an executable exists
00139 
00140     Positional arguments:
00141     path - the executable
00142     """
00143     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00144 
00145 
00146 def find_cmd_abspath (command):
00147     """ Returns the absolute path to a command.
00148         None is returned if no absolute path was found.
00149 
00150     Positional arguhments:
00151     command - the command to find the path of
00152     """
00153     if exists(command) or exists(command + '.exe'):
00154         return os.path.abspath(command)
00155     if not 'PATH' in os.environ:
00156         raise Exception("Can't find command path for current platform ('%s')"
00157                         % sys.platform)
00158     path_env = os.environ['PATH']
00159     for path in path_env.split(os.pathsep):
00160         cmd_path = '%s/%s' % (path, command)
00161         if exists(cmd_path) or exists(cmd_path + '.exe'):
00162             return cmd_path
00163 
00164 
00165 def mkdir (path):
00166     """ a wrapped makedirs that only tries to create a directory if it does not
00167     exist already
00168 
00169     Positional arguments:
00170     path - the path to maybe create
00171     """
00172     if not exists(path):
00173         makedirs(path)
00174 
00175 
00176 def copy_file (src, dst):
00177     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00178     permissions (this was causing errors with directories mounted with samba)
00179 
00180     Positional arguments:
00181     src - the source of the copy operation
00182     dst - the destination of the copy operation
00183     """
00184     if isdir(dst):
00185         _, base = split(src)
00186         dst = join(dst, base)
00187     copyfile(src, dst)
00188 
00189 
00190 def delete_dir_files (directory):
00191     """ A function that does rm -rf
00192 
00193     Positional arguments:
00194     directory - the directory to remove
00195     """
00196     if not exists(directory):
00197         return
00198 
00199     for element in listdir(directory):
00200         to_remove = join(directory, element)
00201         if not isdir(to_remove):
00202             remove(to_remove)
00203 
00204 
00205 def get_caller_name (steps=2):
00206     """
00207     When called inside a function, it returns the name
00208     of the caller of that function.
00209 
00210     Keyword arguments:
00211     steps - the number of steps up the stack the calling function is
00212     """
00213     return inspect.stack()[steps][3]
00214 
00215 
00216 def error (msg):
00217     """Fatal error, abort hard
00218 
00219     Positional arguments:
00220     msg - the message to print before crashing
00221     """
00222     print("ERROR: %s" % msg)
00223     sys.exit(1)
00224 
00225 
00226 def rel_path (path, base, dot=False):
00227     """Relative path calculation that optionaly always starts with a dot
00228 
00229     Positional arguments:
00230     path - the path to make relative
00231     base - what to make the path relative to
00232 
00233     Keyword arguments:
00234     dot - if True, the path will always start with a './'
00235     """
00236     final_path = relpath(path, base)
00237     if dot and not final_path.startswith('.'):
00238         final_path = './' + final_path
00239     return final_path
00240 
00241 
00242 class ToolException (Exception):
00243     """A class representing an exception throw by the tools"""
00244     pass
00245 
00246 class NotSupportedException(Exception):
00247     """A class a toolchain not supporting a particular target"""
00248     pass
00249 
00250 class InvalidReleaseTargetException(Exception):
00251     pass
00252 
00253 def split_path (path):
00254     """spilt a file name into it's directory name, base name, and extension
00255 
00256     Positional arguments:
00257     path - the file name to split
00258     """
00259     base, has_ext = split(path)
00260     name, ext = splitext(has_ext)
00261     return base, name, ext
00262 
00263 
00264 def get_path_depth (path):
00265     """ Given a path, return the number of directory levels present.
00266         This roughly translates to the number of path separators (os.sep) + 1.
00267         Ex. Given "path/to/dir", this would return 3
00268         Special cases: "." and "/" return 0
00269 
00270     Positional arguments:
00271     path - the path to calculate the depth of
00272     """
00273     normalized_path = normpath(path)
00274     path_depth = 0
00275     head, tail = split(normalized_path)
00276 
00277     while tail and tail != '.':
00278         path_depth += 1
00279         head, tail = split(head)
00280 
00281     return path_depth
00282 
00283 
00284 def args_error (parser, message):
00285     """Abort with an error that was generated by the arguments to a CLI program
00286 
00287     Positional arguments:
00288     parser - the ArgumentParser object that parsed the command line
00289     message - what went wrong
00290     """
00291     parser.error(message)
00292     sys.exit(2)
00293 
00294 
00295 def construct_enum (**enums):
00296     """ Create your own pseudo-enums
00297 
00298     Keyword arguments:
00299     * - a member of the Enum you are creating and it's value
00300     """
00301     return type('Enum', (), enums)
00302 
00303 
00304 def check_required_modules (required_modules, verbose=True):
00305     """ Function checks for Python modules which should be "importable"
00306         before test suite can be used.
00307         @return returns True if all modules are installed already
00308     """
00309     import imp
00310     not_installed_modules = []
00311     for module_name in required_modules:
00312         try:
00313             imp.find_module(module_name)
00314         except ImportError:
00315             # We also test against a rare case: module is an egg file
00316             try:
00317                 __import__(module_name)
00318             except ImportError as exc:
00319                 not_installed_modules.append(module_name)
00320                 if verbose:
00321                     print "Error: %s" % exc
00322 
00323     if verbose:
00324         if not_installed_modules:
00325             print ("Warning: Module(s) %s not installed. Please install " + \
00326                    "required module(s) before using this script.")\
00327                 % (', '.join(not_installed_modules))
00328 
00329     if not_installed_modules:
00330         return False
00331     else:
00332         return True
00333 
00334 def dict_to_ascii (dictionary):
00335     """ Utility function: traverse a dictionary and change all the strings in
00336     the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
00337     because the JSON decoder always returns Unicode string. Based on
00338     http://stackoverflow.com/a/13105359
00339 
00340     Positional arguments:
00341     dictionary - The dict that contains some Unicode that should be ASCII
00342     """
00343     if isinstance(dictionary, dict):
00344         return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
00345                             for key, value in dictionary.iteritems()])
00346     elif isinstance(dictionary, list):
00347         return [dict_to_ascii(element) for element in dictionary]
00348     elif isinstance(dictionary, unicode):
00349         return dictionary.encode('ascii')
00350     else:
00351         return dictionary
00352 
00353 def json_file_to_dict (fname):
00354     """ Read a JSON file and return its Python representation, transforming all
00355     the strings from Unicode to ASCII. The order of keys in the JSON file is
00356     preserved.
00357 
00358     Positional arguments:
00359     fname - the name of the file to parse
00360     """
00361     try:
00362         with open(fname, "r") as file_obj:
00363             return dict_to_ascii(json.load(file_obj,
00364                                            object_pairs_hook=OrderedDict))
00365     except (ValueError, IOError):
00366         sys.stderr.write("Error parsing '%s':\n" % fname)
00367         raise
00368 
00369 # Wowza, double closure
00370 def argparse_type(casedness, prefer_hyphen=False):
00371     def middle(lst, type_name):
00372         def parse_type(string):
00373             """ validate that an argument passed in (as string) is a member of
00374             the list of possible arguments. Offer a suggestion if the case of
00375             the string, or the hyphens/underscores do not match the expected
00376             style of the argument.
00377             """
00378             if prefer_hyphen:
00379                 newstring = casedness(string).replace("_", "-")
00380             else:
00381                 newstring = casedness(string).replace("-", "_")
00382             if string in lst:
00383                 return string
00384             elif string not in lst and newstring in lst:
00385                 raise argparse.ArgumentTypeError(
00386                     "{0} is not a supported {1}. Did you mean {2}?".format(
00387                         string, type_name, newstring))
00388             else:
00389                 raise argparse.ArgumentTypeError(
00390                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00391                     format(string, type_name, columnate(lst)))
00392         return parse_type
00393     return middle
00394 
00395 # short cuts for the argparse_type versions
00396 argparse_uppercase_type = argparse_type(str.upper, False)
00397 argparse_lowercase_type = argparse_type(str.lower, False)
00398 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
00399 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
00400 
00401 def argparse_force_type (case):
00402     """ validate that an argument passed in (as string) is a member of the list
00403     of possible arguments after converting it's case.
00404     """
00405     def middle(lst, type_name):
00406         """ The parser type generator"""
00407         def parse_type(string):
00408             """ The parser type"""
00409             for option in lst:
00410                 if case(string) == case(option):
00411                     return option
00412             raise argparse.ArgumentTypeError(
00413                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00414                 format(string, type_name, columnate(lst)))
00415         return parse_type
00416     return middle
00417 
00418 # these two types convert the case of their arguments _before_ validation
00419 argparse_force_uppercase_type = argparse_force_type(str.upper)
00420 argparse_force_lowercase_type = argparse_force_type(str.lower)
00421 
00422 def argparse_many (func):
00423     """ An argument parser combinator that takes in an argument parser and
00424     creates a new parser that accepts a comma separated list of the same thing.
00425     """
00426     def wrap(string):
00427         """ The actual parser"""
00428         return [func(s) for s in string.split(",")]
00429     return wrap
00430 
00431 def argparse_filestring_type (string):
00432     """ An argument parser that verifies that a string passed in corresponds
00433     to a file"""
00434     if exists(string):
00435         return string
00436     else:
00437         raise argparse.ArgumentTypeError(
00438             "{0}"" does not exist in the filesystem.".format(string))
00439 
00440 def argparse_profile_filestring_type (string):
00441     """ An argument parser that verifies that a string passed in is either
00442     absolute path or a file name (expanded to
00443     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00444     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00445     if exists(string):
00446         return string
00447     elif exists(fpath):
00448         return fpath
00449     else:
00450         raise argparse.ArgumentTypeError(
00451             "{0} does not exist in the filesystem.".format(string))
00452 
00453 def columnate (strings, separator=", ", chars=80):
00454     """ render a list of strings as a in a bunch of columns
00455 
00456     Positional arguments:
00457     strings - the strings to columnate
00458 
00459     Keyword arguments;
00460     separator - the separation between the columns
00461     chars - the maximum with of a row
00462     """
00463     col_width = max(len(s) for s in strings)
00464     total_width = col_width + len(separator)
00465     columns = math.floor(chars / total_width)
00466     output = ""
00467     for i, string in zip(range(len(strings)), strings):
00468         append = string
00469         if i != len(strings) - 1:
00470             append += separator
00471         if i % columns == columns - 1:
00472             append += "\n"
00473         else:
00474             append = append.ljust(total_width)
00475         output += append
00476     return output
00477 
00478 def argparse_dir_not_parent (other):
00479     """fail if argument provided is a parent of the specified directory"""
00480     def parse_type(not_parent):
00481         """The parser type"""
00482         abs_other = abspath(other)
00483         abs_not_parent = abspath(not_parent)
00484         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00485             raise argparse.ArgumentTypeError(
00486                 "{0} may not be a parent directory of {1}".format(
00487                     not_parent, other))
00488         else:
00489             return not_parent
00490     return parse_type
00491 
00492 def argparse_deprecate (replacement_message):
00493     """fail if argument is provided with deprecation warning"""
00494     def parse_type(_):
00495         """The parser type"""
00496         raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
00497     return parse_type
00498 
00499 def print_large_string (large_string):
00500     """ Breaks a string up into smaller pieces before print them
00501 
00502     This is a limitation within Windows, as detailed here:
00503     https://bugs.python.org/issue11395
00504 
00505     Positional arguments:
00506     large_string - the large string to print
00507     """
00508     string_limit = 1000
00509     large_string_len = len(large_string)
00510     num_parts = int(ceil(float(large_string_len) / float(string_limit)))
00511     for string_part in range(num_parts):
00512         start_index = string_part * string_limit
00513         if string_part == num_parts - 1:
00514             sys.stdout.write(large_string[start_index:])
00515         else:
00516             sys.stdout.write(large_string[start_index:
00517                                           start_index + string_limit])
00518     sys.stdout.write("\n")
00519 
00520 def intelhex_offset (filename, offset):
00521     """Load a hex or bin file at a particular offset"""
00522     _, inteltype = splitext(filename)
00523     ih = IntelHex()
00524     if inteltype == ".bin":
00525         ih.loadbin(filename, offset=offset)
00526     elif inteltype == ".hex":
00527         ih.loadhex(filename)
00528     else:
00529         raise ToolException("File %s does not have a known binary file type"
00530                             % filename)
00531     return ih