utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 import sys
00018 import inspect
00019 import os
00020 import argparse
00021 import math
00022 from os import listdir, remove, makedirs
00023 from shutil import copyfile
00024 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00025 from os.path import commonprefix, normpath, dirname
00026 from subprocess import Popen, PIPE, STDOUT, call
00027 import json
00028 from collections import OrderedDict
00029 import logging
00030 
00031 def remove_if_in(lst, thing):
00032     if thing in lst:
00033         lst.remove(thing)
00034 
00035 def compile_worker (job):
00036     """Standard task runner used for compiling
00037 
00038     Positional argumets:
00039     job - a dict containing a list of commands and the remaining arguments
00040           to run_cmd
00041     """
00042     results = []
00043     for command in job['commands']:
00044         try:
00045             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00046                                       chroot=job['chroot'])
00047         except KeyboardInterrupt:
00048             raise ToolException
00049 
00050         results.append({
00051             'code': _rc,
00052             'output': _stderr,
00053             'command': command
00054         })
00055 
00056     return {
00057         'source': job['source'],
00058         'object': job['object'],
00059         'commands': job['commands'],
00060         'results': results
00061     }
00062 
00063 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00064     """A wrapper to run a command as a blocking job"""
00065     text = command if shell else ' '.join(command)
00066     if verbose:
00067         print text
00068     return_code = call(command, shell=shell, cwd=cwd)
00069     if check and return_code != 0:
00070         raise Exception('ERROR %d: "%s"' % (return_code, text))
00071 
00072 
00073 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00074     """Run a command in the forground
00075 
00076     Positional arguments:
00077     command - the command to run
00078 
00079     Keyword arguments:
00080     work_dir - the working directory to run the command in
00081     chroot - the chroot to run the command in
00082     redirect - redirect the stderr to a pipe to be used later
00083     """
00084     if chroot:
00085         # Conventions managed by the web team for the mbed.org build system
00086         chroot_cmd = [
00087             '/usr/sbin/chroot', '--userspec=33:33', chroot
00088         ]
00089         for element in command:
00090             chroot_cmd += [element.replace(chroot, '')]
00091 
00092         logging.debug("Running command %s", ' '.join(chroot_cmd))
00093         command = chroot_cmd
00094         work_dir = None
00095 
00096     try:
00097         process = Popen(command, stdout=PIPE,
00098                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00099         _stdout, _stderr = process.communicate()
00100     except OSError:
00101         print "[OS ERROR] Command: "+(' '.join(command))
00102         raise
00103 
00104     return _stdout, _stderr, process.returncode
00105 
00106 
00107 def run_cmd_ext (command):
00108     """ A version of run command that checks if the command exists befor running
00109 
00110     Positional arguments:
00111     command - the command line you are trying to invoke
00112     """
00113     assert is_cmd_valid(command[0])
00114     process = Popen(command, stdout=PIPE, stderr=PIPE)
00115     _stdout, _stderr = process.communicate()
00116     return _stdout, _stderr, process.returncode
00117 
00118 
00119 def is_cmd_valid (command):
00120     """ Verify that a command exists and is executable
00121 
00122     Positional arguments:
00123     command - the command to check
00124     """
00125     caller = get_caller_name()
00126     cmd_path = find_cmd_abspath(command)
00127     if not cmd_path:
00128         error("%s: Command '%s' can't be found" % (caller, command))
00129     if not is_exec(cmd_path):
00130         error("%s: Command '%s' resolves to file '%s' which is not executable"
00131               % (caller, command, cmd_path))
00132     return True
00133 
00134 
00135 def is_exec (path):
00136     """A simple check to verify that a path to an executable exists
00137 
00138     Positional arguments:
00139     path - the executable
00140     """
00141     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00142 
00143 
00144 def find_cmd_abspath (command):
00145     """ Returns the absolute path to a command.
00146         None is returned if no absolute path was found.
00147 
00148     Positional arguhments:
00149     command - the command to find the path of
00150     """
00151     if exists(command) or exists(command + '.exe'):
00152         return os.path.abspath(command)
00153     if not 'PATH' in os.environ:
00154         raise Exception("Can't find command path for current platform ('%s')"
00155                         % sys.platform)
00156     path_env = os.environ['PATH']
00157     for path in path_env.split(os.pathsep):
00158         cmd_path = '%s/%s' % (path, command)
00159         if exists(cmd_path) or exists(cmd_path + '.exe'):
00160             return cmd_path
00161 
00162 
00163 def mkdir (path):
00164     """ a wrapped makedirs that only tries to create a directory if it does not
00165     exist already
00166 
00167     Positional arguments:
00168     path - the path to maybe create
00169     """
00170     if not exists(path):
00171         makedirs(path)
00172 
00173 
00174 def copy_file (src, dst):
00175     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00176     permissions (this was causing errors with directories mounted with samba)
00177 
00178     Positional arguments:
00179     src - the source of the copy operation
00180     dst - the destination of the copy operation
00181     """
00182     if isdir(dst):
00183         _, base = split(src)
00184         dst = join(dst, base)
00185     copyfile(src, dst)
00186 
00187 
00188 def delete_dir_files (directory):
00189     """ A function that does rm -rf
00190 
00191     Positional arguments:
00192     directory - the directory to remove
00193     """
00194     if not exists(directory):
00195         return
00196 
00197     for element in listdir(directory):
00198         to_remove = join(directory, element)
00199         if not isdir(to_remove):
00200             remove(file)
00201 
00202 
00203 def get_caller_name (steps=2):
00204     """
00205     When called inside a function, it returns the name
00206     of the caller of that function.
00207 
00208     Keyword arguments:
00209     steps - the number of steps up the stack the calling function is
00210     """
00211     return inspect.stack()[steps][3]
00212 
00213 
00214 def error (msg):
00215     """Fatal error, abort hard
00216 
00217     Positional arguments:
00218     msg - the message to print before crashing
00219     """
00220     print("ERROR: %s" % msg)
00221     sys.exit(1)
00222 
00223 
00224 def rel_path (path, base, dot=False):
00225     """Relative path calculation that optionaly always starts with a dot
00226 
00227     Positional arguments:
00228     path - the path to make relative
00229     base - what to make the path relative to
00230 
00231     Keyword arguments:
00232     dot - if True, the path will always start with a './'
00233     """
00234     final_path = relpath(path, base)
00235     if dot and not final_path.startswith('.'):
00236         final_path = './' + final_path
00237     return final_path
00238 
00239 
00240 class ToolException (Exception):
00241     """A class representing an exception throw by the tools"""
00242     pass
00243 
00244 class NotSupportedException(Exception):
00245     """A class a toolchain not supporting a particular target"""
00246     pass
00247 
00248 class InvalidReleaseTargetException(Exception):
00249     pass
00250 
00251 def split_path (path):
00252     """spilt a file name into it's directory name, base name, and extension
00253 
00254     Positional arguments:
00255     path - the file name to split
00256     """
00257     base, has_ext = split(path)
00258     name, ext = splitext(has_ext)
00259     return base, name, ext
00260 
00261 
00262 def get_path_depth (path):
00263     """ Given a path, return the number of directory levels present.
00264         This roughly translates to the number of path separators (os.sep) + 1.
00265         Ex. Given "path/to/dir", this would return 3
00266         Special cases: "." and "/" return 0
00267 
00268     Positional arguments:
00269     path - the path to calculate the depth of
00270     """
00271     normalized_path = normpath(path)
00272     path_depth = 0
00273     head, tail = split(normalized_path)
00274 
00275     while tail and tail != '.':
00276         path_depth += 1
00277         head, tail = split(head)
00278 
00279     return path_depth
00280 
00281 
00282 def args_error (parser, message):
00283     """Abort with an error that was generated by the arguments to a CLI program
00284 
00285     Positional arguments:
00286     parser - the ArgumentParser object that parsed the command line
00287     message - what went wrong
00288     """
00289     parser.error(message)
00290     sys.exit(2)
00291 
00292 
00293 def construct_enum (**enums):
00294     """ Create your own pseudo-enums
00295 
00296     Keyword arguments:
00297     * - a member of the Enum you are creating and it's value
00298     """
00299     return type('Enum', (), enums)
00300 
00301 
00302 def check_required_modules (required_modules, verbose=True):
00303     """ Function checks for Python modules which should be "importable"
00304         before test suite can be used.
00305         @return returns True if all modules are installed already
00306     """
00307     import imp
00308     not_installed_modules = []
00309     for module_name in required_modules:
00310         try:
00311             imp.find_module(module_name)
00312         except ImportError:
00313             # We also test against a rare case: module is an egg file
00314             try:
00315                 __import__(module_name)
00316             except ImportError as exc:
00317                 not_installed_modules.append(module_name)
00318                 if verbose:
00319                     print "Error: %s" % exc
00320 
00321     if verbose:
00322         if not_installed_modules:
00323             print ("Warning: Module(s) %s not installed. Please install " + \
00324                    "required module(s) before using this script.")\
00325                 % (', '.join(not_installed_modules))
00326 
00327     if not_installed_modules:
00328         return False
00329     else:
00330         return True
00331 
00332 def dict_to_ascii (dictionary):
00333     """ Utility function: traverse a dictionary and change all the strings in
00334     the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
00335     because the JSON decoder always returns Unicode string. Based on
00336     http://stackoverflow.com/a/13105359
00337 
00338     Positional arguments:
00339     dictionary - The dict that contains some Unicode that should be ASCII
00340     """
00341     if isinstance(dictionary, dict):
00342         return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
00343                             for key, value in dictionary.iteritems()])
00344     elif isinstance(dictionary, list):
00345         return [dict_to_ascii(element) for element in dictionary]
00346     elif isinstance(dictionary, unicode):
00347         return dictionary.encode('ascii')
00348     else:
00349         return dictionary
00350 
00351 def json_file_to_dict (fname):
00352     """ Read a JSON file and return its Python representation, transforming all
00353     the strings from Unicode to ASCII. The order of keys in the JSON file is
00354     preserved.
00355 
00356     Positional arguments:
00357     fname - the name of the file to parse
00358     """
00359     try:
00360         with open(fname, "r") as file_obj:
00361             return dict_to_ascii(json.load(file_obj,
00362                                            object_pairs_hook=OrderedDict))
00363     except (ValueError, IOError):
00364         sys.stderr.write("Error parsing '%s':\n" % fname)
00365         raise
00366 
00367 # Wowza, double closure
00368 def argparse_type(casedness, prefer_hyphen=False):
00369     def middle(lst, type_name):
00370         def parse_type(string):
00371             """ validate that an argument passed in (as string) is a member of
00372             the list of possible arguments. Offer a suggestion if the case of
00373             the string, or the hyphens/underscores do not match the expected
00374             style of the argument.
00375             """
00376             if prefer_hyphen:
00377                 newstring = casedness(string).replace("_", "-")
00378             else:
00379                 newstring = casedness(string).replace("-", "_")
00380             if string in lst:
00381                 return string
00382             elif string not in lst and newstring in lst:
00383                 raise argparse.ArgumentTypeError(
00384                     "{0} is not a supported {1}. Did you mean {2}?".format(
00385                         string, type_name, newstring))
00386             else:
00387                 raise argparse.ArgumentTypeError(
00388                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00389                     format(string, type_name, columnate(lst)))
00390         return parse_type
00391     return middle
00392 
00393 # short cuts for the argparse_type versions
00394 argparse_uppercase_type = argparse_type(str.upper, False)
00395 argparse_lowercase_type = argparse_type(str.lower, False)
00396 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
00397 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
00398 
00399 def argparse_force_type (case):
00400     """ validate that an argument passed in (as string) is a member of the list
00401     of possible arguments after converting it's case.
00402     """
00403     def middle(lst, type_name):
00404         """ The parser type generator"""
00405         def parse_type(string):
00406             """ The parser type"""
00407             for option in lst:
00408                 if case(string) == case(option):
00409                     return option
00410             raise argparse.ArgumentTypeError(
00411                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00412                 format(string, type_name, columnate(lst)))
00413         return parse_type
00414     return middle
00415 
00416 # these two types convert the case of their arguments _before_ validation
00417 argparse_force_uppercase_type = argparse_force_type(str.upper)
00418 argparse_force_lowercase_type = argparse_force_type(str.lower)
00419 
00420 def argparse_many (func):
00421     """ An argument parser combinator that takes in an argument parser and
00422     creates a new parser that accepts a comma separated list of the same thing.
00423     """
00424     def wrap(string):
00425         """ The actual parser"""
00426         return [func(s) for s in string.split(",")]
00427     return wrap
00428 
00429 def argparse_filestring_type (string):
00430     """ An argument parser that verifies that a string passed in corresponds
00431     to a file"""
00432     if exists(string):
00433         return string
00434     else:
00435         raise argparse.ArgumentTypeError(
00436             "{0}"" does not exist in the filesystem.".format(string))
00437 
00438 def argparse_profile_filestring_type (string):
00439     """ An argument parser that verifies that a string passed in is either
00440     absolute path or a file name (expanded to
00441     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00442     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00443     if exists(string):
00444         return string
00445     elif exists(fpath):
00446         return fpath
00447     else:
00448         raise argparse.ArgumentTypeError(
00449             "{0} does not exist in the filesystem.".format(string))
00450 
00451 def columnate (strings, separator=", ", chars=80):
00452     """ render a list of strings as a in a bunch of columns
00453 
00454     Positional arguments:
00455     strings - the strings to columnate
00456 
00457     Keyword arguments;
00458     separator - the separation between the columns
00459     chars - the maximum with of a row
00460     """
00461     col_width = max(len(s) for s in strings)
00462     total_width = col_width + len(separator)
00463     columns = math.floor(chars / total_width)
00464     output = ""
00465     for i, string in zip(range(len(strings)), strings):
00466         append = string
00467         if i != len(strings) - 1:
00468             append += separator
00469         if i % columns == columns - 1:
00470             append += "\n"
00471         else:
00472             append = append.ljust(total_width)
00473         output += append
00474     return output
00475 
00476 def argparse_dir_not_parent (other):
00477     """fail if argument provided is a parent of the specified directory"""
00478     def parse_type(not_parent):
00479         """The parser type"""
00480         abs_other = abspath(other)
00481         abs_not_parent = abspath(not_parent)
00482         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00483             raise argparse.ArgumentTypeError(
00484                 "{0} may not be a parent directory of {1}".format(
00485                     not_parent, other))
00486         else:
00487             return not_parent
00488     return parse_type