utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 import sys
00018 import inspect
00019 import os
00020 import argparse
00021 import math
00022 from os import listdir, remove, makedirs
00023 from shutil import copyfile
00024 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00025 from os.path import commonprefix, normpath
00026 from subprocess import Popen, PIPE, STDOUT, call
00027 import json
00028 from collections import OrderedDict
00029 import logging
00030 
00031 def compile_worker (job):
00032     """Standard task runner used for compiling
00033 
00034     Positional argumets:
00035     job - a dict containing a list of commands and the remaining arguments
00036           to run_cmd
00037     """
00038     results = []
00039     for command in job['commands']:
00040         try:
00041             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00042                                       chroot=job['chroot'])
00043         except KeyboardInterrupt:
00044             raise ToolException
00045 
00046         results.append({
00047             'code': _rc,
00048             'output': _stderr,
00049             'command': command
00050         })
00051 
00052     return {
00053         'source': job['source'],
00054         'object': job['object'],
00055         'commands': job['commands'],
00056         'results': results
00057     }
00058 
00059 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00060     """A wrapper to run a command as a blocking job"""
00061     text = command if shell else ' '.join(command)
00062     if verbose:
00063         print text
00064     return_code = call(command, shell=shell, cwd=cwd)
00065     if check and return_code != 0:
00066         raise Exception('ERROR %d: "%s"' % (return_code, text))
00067 
00068 
00069 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00070     """Run a command in the forground
00071 
00072     Positional arguments:
00073     command - the command to run
00074 
00075     Keyword arguments:
00076     work_dir - the working directory to run the command in
00077     chroot - the chroot to run the command in
00078     redirect - redirect the stderr to a pipe to be used later
00079     """
00080     if chroot:
00081         # Conventions managed by the web team for the mbed.org build system
00082         chroot_cmd = [
00083             '/usr/sbin/chroot', '--userspec=33:33', chroot
00084         ]
00085         for element in command:
00086             chroot_cmd += [element.replace(chroot, '')]
00087 
00088         logging.debug("Running command %s", ' '.join(chroot_cmd))
00089         command = chroot_cmd
00090         work_dir = None
00091 
00092     try:
00093         process = Popen(command, stdout=PIPE,
00094                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00095         _stdout, _stderr = process.communicate()
00096     except OSError:
00097         print "[OS ERROR] Command: "+(' '.join(command))
00098         raise
00099 
00100     return _stdout, _stderr, process.returncode
00101 
00102 
00103 def run_cmd_ext (command):
00104     """ A version of run command that checks if the command exists befor running
00105 
00106     Positional arguments:
00107     command - the command line you are trying to invoke
00108     """
00109     assert is_cmd_valid(command[0])
00110     process = Popen(command, stdout=PIPE, stderr=PIPE)
00111     _stdout, _stderr = process.communicate()
00112     return _stdout, _stderr, process.returncode
00113 
00114 
00115 def is_cmd_valid (command):
00116     """ Verify that a command exists and is executable
00117 
00118     Positional arguments:
00119     command - the command to check
00120     """
00121     caller = get_caller_name()
00122     cmd_path = find_cmd_abspath(command)
00123     if not cmd_path:
00124         error("%s: Command '%s' can't be found" % (caller, command))
00125     if not is_exec(cmd_path):
00126         error("%s: Command '%s' resolves to file '%s' which is not executable"
00127               % (caller, command, cmd_path))
00128     return True
00129 
00130 
00131 def is_exec (path):
00132     """A simple check to verify that a path to an executable exists
00133 
00134     Positional arguments:
00135     path - the executable
00136     """
00137     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00138 
00139 
00140 def find_cmd_abspath (command):
00141     """ Returns the absolute path to a command.
00142         None is returned if no absolute path was found.
00143 
00144     Positional arguhments:
00145     command - the command to find the path of
00146     """
00147     if exists(command) or exists(command + '.exe'):
00148         return os.path.abspath(command)
00149     if not 'PATH' in os.environ:
00150         raise Exception("Can't find command path for current platform ('%s')"
00151                         % sys.platform)
00152     path_env = os.environ['PATH']
00153     for path in path_env.split(os.pathsep):
00154         cmd_path = '%s/%s' % (path, command)
00155         if exists(cmd_path) or exists(cmd_path + '.exe'):
00156             return cmd_path
00157 
00158 
00159 def mkdir (path):
00160     """ a wrapped makedirs that only tries to create a directory if it does not
00161     exist already
00162 
00163     Positional arguments:
00164     path - the path to maybe create
00165     """
00166     if not exists(path):
00167         makedirs(path)
00168 
00169 
00170 def copy_file (src, dst):
00171     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00172     permissions (this was causing errors with directories mounted with samba)
00173 
00174     Positional arguments:
00175     src - the source of the copy operation
00176     dst - the destination of the copy operation
00177     """
00178     if isdir(dst):
00179         _, base = split(src)
00180         dst = join(dst, base)
00181     copyfile(src, dst)
00182 
00183 
00184 def delete_dir_files (directory):
00185     """ A function that does rm -rf
00186 
00187     Positional arguments:
00188     directory - the directory to remove
00189     """
00190     if not exists(directory):
00191         return
00192 
00193     for element in listdir(directory):
00194         to_remove = join(directory, element)
00195         if not isdir(to_remove):
00196             remove(file)
00197 
00198 
00199 def get_caller_name (steps=2):
00200     """
00201     When called inside a function, it returns the name
00202     of the caller of that function.
00203 
00204     Keyword arguments:
00205     steps - the number of steps up the stack the calling function is
00206     """
00207     return inspect.stack()[steps][3]
00208 
00209 
00210 def error (msg):
00211     """Fatal error, abort hard
00212 
00213     Positional arguments:
00214     msg - the message to print before crashing
00215     """
00216     print("ERROR: %s" % msg)
00217     sys.exit(1)
00218 
00219 
00220 def rel_path (path, base, dot=False):
00221     """Relative path calculation that optionaly always starts with a dot
00222 
00223     Positional arguments:
00224     path - the path to make relative
00225     base - what to make the path relative to
00226 
00227     Keyword arguments:
00228     dot - if True, the path will always start with a './'
00229     """
00230     final_path = relpath(path, base)
00231     if dot and not final_path.startswith('.'):
00232         final_path = './' + final_path
00233     return final_path
00234 
00235 
00236 class ToolException (Exception):
00237     """A class representing an exception throw by the tools"""
00238     pass
00239 
00240 class NotSupportedException(Exception):
00241     """A class a toolchain not supporting a particular target"""
00242     pass
00243 
00244 class InvalidReleaseTargetException(Exception):
00245     pass
00246 
00247 def split_path (path):
00248     """spilt a file name into it's directory name, base name, and extension
00249 
00250     Positional arguments:
00251     path - the file name to split
00252     """
00253     base, has_ext = split(path)
00254     name, ext = splitext(has_ext)
00255     return base, name, ext
00256 
00257 
00258 def get_path_depth (path):
00259     """ Given a path, return the number of directory levels present.
00260         This roughly translates to the number of path separators (os.sep) + 1.
00261         Ex. Given "path/to/dir", this would return 3
00262         Special cases: "." and "/" return 0
00263 
00264     Positional arguments:
00265     path - the path to calculate the depth of
00266     """
00267     normalized_path = normpath(path)
00268     path_depth = 0
00269     head, tail = split(normalized_path)
00270 
00271     while tail and tail != '.':
00272         path_depth += 1
00273         head, tail = split(head)
00274 
00275     return path_depth
00276 
00277 
00278 def args_error (parser, message):
00279     """Abort with an error that was generated by the arguments to a CLI program
00280 
00281     Positional arguments:
00282     parser - the ArgumentParser object that parsed the command line
00283     message - what went wrong
00284     """
00285     parser.error(message)
00286     sys.exit(2)
00287 
00288 
00289 def construct_enum (**enums):
00290     """ Create your own pseudo-enums
00291 
00292     Keyword arguments:
00293     * - a member of the Enum you are creating and it's value
00294     """
00295     return type('Enum', (), enums)
00296 
00297 
00298 def check_required_modules (required_modules, verbose=True):
00299     """ Function checks for Python modules which should be "importable"
00300         before test suite can be used.
00301         @return returns True if all modules are installed already
00302     """
00303     import imp
00304     not_installed_modules = []
00305     for module_name in required_modules:
00306         try:
00307             imp.find_module(module_name)
00308         except ImportError:
00309             # We also test against a rare case: module is an egg file
00310             try:
00311                 __import__(module_name)
00312             except ImportError as exc:
00313                 not_installed_modules.append(module_name)
00314                 if verbose:
00315                     print "Error: %s" % exc
00316 
00317     if verbose:
00318         if not_installed_modules:
00319             print ("Warning: Module(s) %s not installed. Please install " + \
00320                    "required module(s) before using this script.")\
00321                 % (', '.join(not_installed_modules))
00322 
00323     if not_installed_modules:
00324         return False
00325     else:
00326         return True
00327 
00328 def dict_to_ascii (dictionary):
00329     """ Utility function: traverse a dictionary and change all the strings in
00330     the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
00331     because the JSON decoder always returns Unicode string. Based on
00332     http://stackoverflow.com/a/13105359
00333 
00334     Positional arguments:
00335     dictionary - The dict that contains some Unicode that should be ASCII
00336     """
00337     if isinstance(dictionary, dict):
00338         return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
00339                             for key, value in dictionary.iteritems()])
00340     elif isinstance(dictionary, list):
00341         return [dict_to_ascii(element) for element in dictionary]
00342     elif isinstance(dictionary, unicode):
00343         return dictionary.encode('ascii')
00344     else:
00345         return dictionary
00346 
00347 def json_file_to_dict (fname):
00348     """ Read a JSON file and return its Python representation, transforming all
00349     the strings from Unicode to ASCII. The order of keys in the JSON file is
00350     preserved.
00351 
00352     Positional arguments:
00353     fname - the name of the file to parse
00354     """
00355     try:
00356         with open(fname, "r") as file_obj:
00357             return dict_to_ascii(json.load(file_obj,
00358                                            object_pairs_hook=OrderedDict))
00359     except (ValueError, IOError):
00360         sys.stderr.write("Error parsing '%s':\n" % fname)
00361         raise
00362 
00363 # Wowza, double closure
00364 def argparse_type(casedness, prefer_hyphen=False):
00365     def middle(lst, type_name):
00366         def parse_type(string):
00367             """ validate that an argument passed in (as string) is a member of
00368             the list of possible arguments. Offer a suggestion if the case of
00369             the string, or the hyphens/underscores do not match the expected
00370             style of the argument.
00371             """
00372             if prefer_hyphen:
00373                 newstring = casedness(string).replace("_", "-")
00374             else:
00375                 newstring = casedness(string).replace("-", "_")
00376             if string in lst:
00377                 return string
00378             elif string not in lst and newstring in lst:
00379                 raise argparse.ArgumentTypeError(
00380                     "{0} is not a supported {1}. Did you mean {2}?".format(
00381                         string, type_name, newstring))
00382             else:
00383                 raise argparse.ArgumentTypeError(
00384                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00385                     format(string, type_name, columnate(lst)))
00386         return parse_type
00387     return middle
00388 
00389 # short cuts for the argparse_type versions
00390 argparse_uppercase_type = argparse_type(str.upper, False)
00391 argparse_lowercase_type = argparse_type(str.lower, False)
00392 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
00393 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
00394 
00395 def argparse_force_type (case):
00396     """ validate that an argument passed in (as string) is a member of the list
00397     of possible arguments after converting it's case.
00398     """
00399     def middle(lst, type_name):
00400         """ The parser type generator"""
00401         def parse_type(string):
00402             """ The parser type"""
00403             for option in lst:
00404                 if case(string) == case(option):
00405                     return option
00406             raise argparse.ArgumentTypeError(
00407                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00408                 format(string, type_name, columnate(lst)))
00409         return parse_type
00410     return middle
00411 
00412 # these two types convert the case of their arguments _before_ validation
00413 argparse_force_uppercase_type = argparse_force_type(str.upper)
00414 argparse_force_lowercase_type = argparse_force_type(str.lower)
00415 
00416 def argparse_many (func):
00417     """ An argument parser combinator that takes in an argument parser and
00418     creates a new parser that accepts a comma separated list of the same thing.
00419     """
00420     def wrap(string):
00421         """ The actual parser"""
00422         return [func(s) for s in string.split(",")]
00423     return wrap
00424 
00425 def argparse_filestring_type (string):
00426     """ An argument parser that verifies that a string passed in corresponds
00427     to a file"""
00428     if exists(string):
00429         return string
00430     else:
00431         raise argparse.ArgumentTypeError(
00432             "{0}"" does not exist in the filesystem.".format(string))
00433 
00434 def columnate (strings, separator=", ", chars=80):
00435     """ render a list of strings as a in a bunch of columns
00436 
00437     Positional arguments:
00438     strings - the strings to columnate
00439 
00440     Keyword arguments;
00441     separator - the separation between the columns
00442     chars - the maximum with of a row
00443     """
00444     col_width = max(len(s) for s in strings)
00445     total_width = col_width + len(separator)
00446     columns = math.floor(chars / total_width)
00447     output = ""
00448     for i, string in zip(range(len(strings)), strings):
00449         append = string
00450         if i != len(strings) - 1:
00451             append += separator
00452         if i % columns == columns - 1:
00453             append += "\n"
00454         else:
00455             append = append.ljust(total_width)
00456         output += append
00457     return output
00458 
00459 def argparse_dir_not_parent (other):
00460     """fail if argument provided is a parent of the specified directory"""
00461     def parse_type(not_parent):
00462         """The parser type"""
00463         abs_other = abspath(other)
00464         abs_not_parent = abspath(not_parent)
00465         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00466             raise argparse.ArgumentTypeError(
00467                 "{0} may not be a parent directory of {1}".format(
00468                     not_parent, other))
00469         else:
00470             return not_parent
00471     return parse_type