utils.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 import sys
00018 import inspect
00019 import os
00020 import argparse
00021 import math
00022 from os import listdir, remove, makedirs
00023 from shutil import copyfile
00024 from os.path import isdir, join, exists, split, relpath, splitext, abspath
00025 from os.path import commonprefix, normpath, dirname
00026 from subprocess import Popen, PIPE, STDOUT, call
00027 from math import ceil
00028 import json
00029 from collections import OrderedDict
00030 import logging
00031 
00032 def remove_if_in(lst, thing):
00033     if thing in lst:
00034         lst.remove(thing)
00035 
00036 def compile_worker (job):
00037     """Standard task runner used for compiling
00038 
00039     Positional argumets:
00040     job - a dict containing a list of commands and the remaining arguments
00041           to run_cmd
00042     """
00043     results = []
00044     for command in job['commands']:
00045         try:
00046             _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
00047                                       chroot=job['chroot'])
00048         except KeyboardInterrupt:
00049             raise ToolException
00050 
00051         results.append({
00052             'code': _rc,
00053             'output': _stderr,
00054             'command': command
00055         })
00056 
00057     return {
00058         'source': job['source'],
00059         'object': job['object'],
00060         'commands': job['commands'],
00061         'results': results
00062     }
00063 
00064 def cmd (command, check=True, verbose=False, shell=False, cwd=None):
00065     """A wrapper to run a command as a blocking job"""
00066     text = command if shell else ' '.join(command)
00067     if verbose:
00068         print text
00069     return_code = call(command, shell=shell, cwd=cwd)
00070     if check and return_code != 0:
00071         raise Exception('ERROR %d: "%s"' % (return_code, text))
00072 
00073 
00074 def run_cmd (command, work_dir=None, chroot=None, redirect=False):
00075     """Run a command in the forground
00076 
00077     Positional arguments:
00078     command - the command to run
00079 
00080     Keyword arguments:
00081     work_dir - the working directory to run the command in
00082     chroot - the chroot to run the command in
00083     redirect - redirect the stderr to a pipe to be used later
00084     """
00085     if chroot:
00086         # Conventions managed by the web team for the mbed.org build system
00087         chroot_cmd = [
00088             '/usr/sbin/chroot', '--userspec=33:33', chroot
00089         ]
00090         for element in command:
00091             chroot_cmd += [element.replace(chroot, '')]
00092 
00093         logging.debug("Running command %s", ' '.join(chroot_cmd))
00094         command = chroot_cmd
00095         work_dir = None
00096 
00097     try:
00098         process = Popen(command, stdout=PIPE,
00099                         stderr=STDOUT if redirect else PIPE, cwd=work_dir)
00100         _stdout, _stderr = process.communicate()
00101     except OSError:
00102         print "[OS ERROR] Command: "+(' '.join(command))
00103         raise
00104 
00105     return _stdout, _stderr, process.returncode
00106 
00107 
00108 def run_cmd_ext (command):
00109     """ A version of run command that checks if the command exists befor running
00110 
00111     Positional arguments:
00112     command - the command line you are trying to invoke
00113     """
00114     assert is_cmd_valid(command[0])
00115     process = Popen(command, stdout=PIPE, stderr=PIPE)
00116     _stdout, _stderr = process.communicate()
00117     return _stdout, _stderr, process.returncode
00118 
00119 
00120 def is_cmd_valid (command):
00121     """ Verify that a command exists and is executable
00122 
00123     Positional arguments:
00124     command - the command to check
00125     """
00126     caller = get_caller_name()
00127     cmd_path = find_cmd_abspath(command)
00128     if not cmd_path:
00129         error("%s: Command '%s' can't be found" % (caller, command))
00130     if not is_exec(cmd_path):
00131         error("%s: Command '%s' resolves to file '%s' which is not executable"
00132               % (caller, command, cmd_path))
00133     return True
00134 
00135 
00136 def is_exec (path):
00137     """A simple check to verify that a path to an executable exists
00138 
00139     Positional arguments:
00140     path - the executable
00141     """
00142     return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
00143 
00144 
00145 def find_cmd_abspath (command):
00146     """ Returns the absolute path to a command.
00147         None is returned if no absolute path was found.
00148 
00149     Positional arguhments:
00150     command - the command to find the path of
00151     """
00152     if exists(command) or exists(command + '.exe'):
00153         return os.path.abspath(command)
00154     if not 'PATH' in os.environ:
00155         raise Exception("Can't find command path for current platform ('%s')"
00156                         % sys.platform)
00157     path_env = os.environ['PATH']
00158     for path in path_env.split(os.pathsep):
00159         cmd_path = '%s/%s' % (path, command)
00160         if exists(cmd_path) or exists(cmd_path + '.exe'):
00161             return cmd_path
00162 
00163 
00164 def mkdir (path):
00165     """ a wrapped makedirs that only tries to create a directory if it does not
00166     exist already
00167 
00168     Positional arguments:
00169     path - the path to maybe create
00170     """
00171     if not exists(path):
00172         makedirs(path)
00173 
00174 
00175 def copy_file (src, dst):
00176     """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
00177     permissions (this was causing errors with directories mounted with samba)
00178 
00179     Positional arguments:
00180     src - the source of the copy operation
00181     dst - the destination of the copy operation
00182     """
00183     if isdir(dst):
00184         _, base = split(src)
00185         dst = join(dst, base)
00186     copyfile(src, dst)
00187 
00188 
00189 def delete_dir_files (directory):
00190     """ A function that does rm -rf
00191 
00192     Positional arguments:
00193     directory - the directory to remove
00194     """
00195     if not exists(directory):
00196         return
00197 
00198     for element in listdir(directory):
00199         to_remove = join(directory, element)
00200         if not isdir(to_remove):
00201             remove(file)
00202 
00203 
00204 def get_caller_name (steps=2):
00205     """
00206     When called inside a function, it returns the name
00207     of the caller of that function.
00208 
00209     Keyword arguments:
00210     steps - the number of steps up the stack the calling function is
00211     """
00212     return inspect.stack()[steps][3]
00213 
00214 
00215 def error (msg):
00216     """Fatal error, abort hard
00217 
00218     Positional arguments:
00219     msg - the message to print before crashing
00220     """
00221     print("ERROR: %s" % msg)
00222     sys.exit(1)
00223 
00224 
00225 def rel_path (path, base, dot=False):
00226     """Relative path calculation that optionaly always starts with a dot
00227 
00228     Positional arguments:
00229     path - the path to make relative
00230     base - what to make the path relative to
00231 
00232     Keyword arguments:
00233     dot - if True, the path will always start with a './'
00234     """
00235     final_path = relpath(path, base)
00236     if dot and not final_path.startswith('.'):
00237         final_path = './' + final_path
00238     return final_path
00239 
00240 
00241 class ToolException (Exception):
00242     """A class representing an exception throw by the tools"""
00243     pass
00244 
00245 class NotSupportedException(Exception):
00246     """A class a toolchain not supporting a particular target"""
00247     pass
00248 
00249 class InvalidReleaseTargetException(Exception):
00250     pass
00251 
00252 def split_path (path):
00253     """spilt a file name into it's directory name, base name, and extension
00254 
00255     Positional arguments:
00256     path - the file name to split
00257     """
00258     base, has_ext = split(path)
00259     name, ext = splitext(has_ext)
00260     return base, name, ext
00261 
00262 
00263 def get_path_depth (path):
00264     """ Given a path, return the number of directory levels present.
00265         This roughly translates to the number of path separators (os.sep) + 1.
00266         Ex. Given "path/to/dir", this would return 3
00267         Special cases: "." and "/" return 0
00268 
00269     Positional arguments:
00270     path - the path to calculate the depth of
00271     """
00272     normalized_path = normpath(path)
00273     path_depth = 0
00274     head, tail = split(normalized_path)
00275 
00276     while tail and tail != '.':
00277         path_depth += 1
00278         head, tail = split(head)
00279 
00280     return path_depth
00281 
00282 
00283 def args_error (parser, message):
00284     """Abort with an error that was generated by the arguments to a CLI program
00285 
00286     Positional arguments:
00287     parser - the ArgumentParser object that parsed the command line
00288     message - what went wrong
00289     """
00290     parser.error(message)
00291     sys.exit(2)
00292 
00293 
00294 def construct_enum (**enums):
00295     """ Create your own pseudo-enums
00296 
00297     Keyword arguments:
00298     * - a member of the Enum you are creating and it's value
00299     """
00300     return type('Enum', (), enums)
00301 
00302 
00303 def check_required_modules (required_modules, verbose=True):
00304     """ Function checks for Python modules which should be "importable"
00305         before test suite can be used.
00306         @return returns True if all modules are installed already
00307     """
00308     import imp
00309     not_installed_modules = []
00310     for module_name in required_modules:
00311         try:
00312             imp.find_module(module_name)
00313         except ImportError:
00314             # We also test against a rare case: module is an egg file
00315             try:
00316                 __import__(module_name)
00317             except ImportError as exc:
00318                 not_installed_modules.append(module_name)
00319                 if verbose:
00320                     print "Error: %s" % exc
00321 
00322     if verbose:
00323         if not_installed_modules:
00324             print ("Warning: Module(s) %s not installed. Please install " + \
00325                    "required module(s) before using this script.")\
00326                 % (', '.join(not_installed_modules))
00327 
00328     if not_installed_modules:
00329         return False
00330     else:
00331         return True
00332 
00333 def dict_to_ascii (dictionary):
00334     """ Utility function: traverse a dictionary and change all the strings in
00335     the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
00336     because the JSON decoder always returns Unicode string. Based on
00337     http://stackoverflow.com/a/13105359
00338 
00339     Positional arguments:
00340     dictionary - The dict that contains some Unicode that should be ASCII
00341     """
00342     if isinstance(dictionary, dict):
00343         return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
00344                             for key, value in dictionary.iteritems()])
00345     elif isinstance(dictionary, list):
00346         return [dict_to_ascii(element) for element in dictionary]
00347     elif isinstance(dictionary, unicode):
00348         return dictionary.encode('ascii')
00349     else:
00350         return dictionary
00351 
00352 def json_file_to_dict (fname):
00353     """ Read a JSON file and return its Python representation, transforming all
00354     the strings from Unicode to ASCII. The order of keys in the JSON file is
00355     preserved.
00356 
00357     Positional arguments:
00358     fname - the name of the file to parse
00359     """
00360     try:
00361         with open(fname, "r") as file_obj:
00362             return dict_to_ascii(json.load(file_obj,
00363                                            object_pairs_hook=OrderedDict))
00364     except (ValueError, IOError):
00365         sys.stderr.write("Error parsing '%s':\n" % fname)
00366         raise
00367 
00368 # Wowza, double closure
00369 def argparse_type(casedness, prefer_hyphen=False):
00370     def middle(lst, type_name):
00371         def parse_type(string):
00372             """ validate that an argument passed in (as string) is a member of
00373             the list of possible arguments. Offer a suggestion if the case of
00374             the string, or the hyphens/underscores do not match the expected
00375             style of the argument.
00376             """
00377             if prefer_hyphen:
00378                 newstring = casedness(string).replace("_", "-")
00379             else:
00380                 newstring = casedness(string).replace("-", "_")
00381             if string in lst:
00382                 return string
00383             elif string not in lst and newstring in lst:
00384                 raise argparse.ArgumentTypeError(
00385                     "{0} is not a supported {1}. Did you mean {2}?".format(
00386                         string, type_name, newstring))
00387             else:
00388                 raise argparse.ArgumentTypeError(
00389                     "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00390                     format(string, type_name, columnate(lst)))
00391         return parse_type
00392     return middle
00393 
00394 # short cuts for the argparse_type versions
00395 argparse_uppercase_type = argparse_type(str.upper, False)
00396 argparse_lowercase_type = argparse_type(str.lower, False)
00397 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
00398 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
00399 
00400 def argparse_force_type (case):
00401     """ validate that an argument passed in (as string) is a member of the list
00402     of possible arguments after converting it's case.
00403     """
00404     def middle(lst, type_name):
00405         """ The parser type generator"""
00406         def parse_type(string):
00407             """ The parser type"""
00408             for option in lst:
00409                 if case(string) == case(option):
00410                     return option
00411             raise argparse.ArgumentTypeError(
00412                 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
00413                 format(string, type_name, columnate(lst)))
00414         return parse_type
00415     return middle
00416 
00417 # these two types convert the case of their arguments _before_ validation
00418 argparse_force_uppercase_type = argparse_force_type(str.upper)
00419 argparse_force_lowercase_type = argparse_force_type(str.lower)
00420 
00421 def argparse_many (func):
00422     """ An argument parser combinator that takes in an argument parser and
00423     creates a new parser that accepts a comma separated list of the same thing.
00424     """
00425     def wrap(string):
00426         """ The actual parser"""
00427         return [func(s) for s in string.split(",")]
00428     return wrap
00429 
00430 def argparse_filestring_type (string):
00431     """ An argument parser that verifies that a string passed in corresponds
00432     to a file"""
00433     if exists(string):
00434         return string
00435     else:
00436         raise argparse.ArgumentTypeError(
00437             "{0}"" does not exist in the filesystem.".format(string))
00438 
00439 def argparse_profile_filestring_type (string):
00440     """ An argument parser that verifies that a string passed in is either
00441     absolute path or a file name (expanded to
00442     mbed-os/tools/profiles/<fname>.json) of a existing file"""
00443     fpath = join(dirname(__file__), "profiles/{}.json".format(string))
00444     if exists(string):
00445         return string
00446     elif exists(fpath):
00447         return fpath
00448     else:
00449         raise argparse.ArgumentTypeError(
00450             "{0} does not exist in the filesystem.".format(string))
00451 
00452 def columnate (strings, separator=", ", chars=80):
00453     """ render a list of strings as a in a bunch of columns
00454 
00455     Positional arguments:
00456     strings - the strings to columnate
00457 
00458     Keyword arguments;
00459     separator - the separation between the columns
00460     chars - the maximum with of a row
00461     """
00462     col_width = max(len(s) for s in strings)
00463     total_width = col_width + len(separator)
00464     columns = math.floor(chars / total_width)
00465     output = ""
00466     for i, string in zip(range(len(strings)), strings):
00467         append = string
00468         if i != len(strings) - 1:
00469             append += separator
00470         if i % columns == columns - 1:
00471             append += "\n"
00472         else:
00473             append = append.ljust(total_width)
00474         output += append
00475     return output
00476 
00477 def argparse_dir_not_parent (other):
00478     """fail if argument provided is a parent of the specified directory"""
00479     def parse_type(not_parent):
00480         """The parser type"""
00481         abs_other = abspath(other)
00482         abs_not_parent = abspath(not_parent)
00483         if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
00484             raise argparse.ArgumentTypeError(
00485                 "{0} may not be a parent directory of {1}".format(
00486                     not_parent, other))
00487         else:
00488             return not_parent
00489     return parse_type
00490 
00491 def argparse_deprecate (replacement_message):
00492     """fail if argument is provided with deprecation warning"""
00493     def parse_type(_):
00494         """The parser type"""
00495         raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
00496     return parse_type
00497 
00498 def print_large_string (large_string):
00499     """ Breaks a string up into smaller pieces before print them
00500 
00501     This is a limitation within Windows, as detailed here:
00502     https://bugs.python.org/issue11395
00503 
00504     Positional arguments:
00505     large_string - the large string to print
00506     """
00507     string_limit = 1000
00508     large_string_len = len(large_string)
00509     num_parts = int(ceil(float(large_string_len) / float(string_limit)))
00510     for string_part in range(num_parts):
00511         start_index = string_part * string_limit
00512         if string_part == num_parts - 1:
00513             print large_string[start_index:]
00514         else:
00515             end_index = ((string_part + 1) * string_limit) - 1
00516             print large_string[start_index:end_index],