Magnificent7 / Hackathon

Includes library modifications to allow access to AIN_4 (AIN_0 / 5)

mbd_os/tools/utils.py@0:eafc3fd41f75, 2016-09-20 (annotated)

Committer:
bryantaylor
Date:
Tue Sep 20 21:26:12 2016 +0000
Revision:
0:eafc3fd41f75
hackathon

Who changed what in which revision?

UserRevisionLine numberNew contents of line
bryantaylor 0:eafc3fd41f75 1 """
bryantaylor 0:eafc3fd41f75 2 mbed SDK
bryantaylor 0:eafc3fd41f75 3 Copyright (c) 2011-2013 ARM Limited
bryantaylor 0:eafc3fd41f75 4
bryantaylor 0:eafc3fd41f75 5 Licensed under the Apache License, Version 2.0 (the "License");
bryantaylor 0:eafc3fd41f75 6 you may not use this file except in compliance with the License.
bryantaylor 0:eafc3fd41f75 7 You may obtain a copy of the License at
bryantaylor 0:eafc3fd41f75 8
bryantaylor 0:eafc3fd41f75 9 http://www.apache.org/licenses/LICENSE-2.0
bryantaylor 0:eafc3fd41f75 10
bryantaylor 0:eafc3fd41f75 11 Unless required by applicable law or agreed to in writing, software
bryantaylor 0:eafc3fd41f75 12 distributed under the License is distributed on an "AS IS" BASIS,
bryantaylor 0:eafc3fd41f75 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
bryantaylor 0:eafc3fd41f75 14 See the License for the specific language governing permissions and
bryantaylor 0:eafc3fd41f75 15 limitations under the License.
bryantaylor 0:eafc3fd41f75 16 """
bryantaylor 0:eafc3fd41f75 17 import sys
bryantaylor 0:eafc3fd41f75 18 import inspect
bryantaylor 0:eafc3fd41f75 19 import os
bryantaylor 0:eafc3fd41f75 20 import argparse
bryantaylor 0:eafc3fd41f75 21 import math
bryantaylor 0:eafc3fd41f75 22 from os import listdir, remove, makedirs
bryantaylor 0:eafc3fd41f75 23 from shutil import copyfile
bryantaylor 0:eafc3fd41f75 24 from os.path import isdir, join, exists, split, relpath, splitext, abspath
bryantaylor 0:eafc3fd41f75 25 from os.path import commonprefix, normpath
bryantaylor 0:eafc3fd41f75 26 from subprocess import Popen, PIPE, STDOUT, call
bryantaylor 0:eafc3fd41f75 27 import json
bryantaylor 0:eafc3fd41f75 28 from collections import OrderedDict
bryantaylor 0:eafc3fd41f75 29 import logging
bryantaylor 0:eafc3fd41f75 30
bryantaylor 0:eafc3fd41f75 31 def compile_worker(job):
bryantaylor 0:eafc3fd41f75 32 """Standard task runner used for compiling
bryantaylor 0:eafc3fd41f75 33
bryantaylor 0:eafc3fd41f75 34 Positional argumets:
bryantaylor 0:eafc3fd41f75 35 job - a dict containing a list of commands and the remaining arguments
bryantaylor 0:eafc3fd41f75 36 to run_cmd
bryantaylor 0:eafc3fd41f75 37 """
bryantaylor 0:eafc3fd41f75 38 results = []
bryantaylor 0:eafc3fd41f75 39 for command in job['commands']:
bryantaylor 0:eafc3fd41f75 40 try:
bryantaylor 0:eafc3fd41f75 41 _, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
bryantaylor 0:eafc3fd41f75 42 chroot=job['chroot'])
bryantaylor 0:eafc3fd41f75 43 except KeyboardInterrupt:
bryantaylor 0:eafc3fd41f75 44 raise ToolException
bryantaylor 0:eafc3fd41f75 45
bryantaylor 0:eafc3fd41f75 46 results.append({
bryantaylor 0:eafc3fd41f75 47 'code': _rc,
bryantaylor 0:eafc3fd41f75 48 'output': _stderr,
bryantaylor 0:eafc3fd41f75 49 'command': command
bryantaylor 0:eafc3fd41f75 50 })
bryantaylor 0:eafc3fd41f75 51
bryantaylor 0:eafc3fd41f75 52 return {
bryantaylor 0:eafc3fd41f75 53 'source': job['source'],
bryantaylor 0:eafc3fd41f75 54 'object': job['object'],
bryantaylor 0:eafc3fd41f75 55 'commands': job['commands'],
bryantaylor 0:eafc3fd41f75 56 'results': results
bryantaylor 0:eafc3fd41f75 57 }
bryantaylor 0:eafc3fd41f75 58
bryantaylor 0:eafc3fd41f75 59 def cmd(command, check=True, verbose=False, shell=False, cwd=None):
bryantaylor 0:eafc3fd41f75 60 """A wrapper to run a command as a blocking job"""
bryantaylor 0:eafc3fd41f75 61 text = command if shell else ' '.join(command)
bryantaylor 0:eafc3fd41f75 62 if verbose:
bryantaylor 0:eafc3fd41f75 63 print text
bryantaylor 0:eafc3fd41f75 64 return_code = call(command, shell=shell, cwd=cwd)
bryantaylor 0:eafc3fd41f75 65 if check and return_code != 0:
bryantaylor 0:eafc3fd41f75 66 raise Exception('ERROR %d: "%s"' % (return_code, text))
bryantaylor 0:eafc3fd41f75 67
bryantaylor 0:eafc3fd41f75 68
bryantaylor 0:eafc3fd41f75 69 def run_cmd(command, work_dir=None, chroot=None, redirect=False):
bryantaylor 0:eafc3fd41f75 70 """Run a command in the forground
bryantaylor 0:eafc3fd41f75 71
bryantaylor 0:eafc3fd41f75 72 Positional arguments:
bryantaylor 0:eafc3fd41f75 73 command - the command to run
bryantaylor 0:eafc3fd41f75 74
bryantaylor 0:eafc3fd41f75 75 Keyword arguments:
bryantaylor 0:eafc3fd41f75 76 work_dir - the working directory to run the command in
bryantaylor 0:eafc3fd41f75 77 chroot - the chroot to run the command in
bryantaylor 0:eafc3fd41f75 78 redirect - redirect the stderr to a pipe to be used later
bryantaylor 0:eafc3fd41f75 79 """
bryantaylor 0:eafc3fd41f75 80 if chroot:
bryantaylor 0:eafc3fd41f75 81 # Conventions managed by the web team for the mbed.org build system
bryantaylor 0:eafc3fd41f75 82 chroot_cmd = [
bryantaylor 0:eafc3fd41f75 83 '/usr/sbin/chroot', '--userspec=33:33', chroot
bryantaylor 0:eafc3fd41f75 84 ]
bryantaylor 0:eafc3fd41f75 85 for element in command:
bryantaylor 0:eafc3fd41f75 86 chroot_cmd += [element.replace(chroot, '')]
bryantaylor 0:eafc3fd41f75 87
bryantaylor 0:eafc3fd41f75 88 logging.debug("Running command %s", ' '.join(chroot_cmd))
bryantaylor 0:eafc3fd41f75 89 command = chroot_cmd
bryantaylor 0:eafc3fd41f75 90 work_dir = None
bryantaylor 0:eafc3fd41f75 91
bryantaylor 0:eafc3fd41f75 92 try:
bryantaylor 0:eafc3fd41f75 93 process = Popen(command, stdout=PIPE,
bryantaylor 0:eafc3fd41f75 94 stderr=STDOUT if redirect else PIPE, cwd=work_dir)
bryantaylor 0:eafc3fd41f75 95 _stdout, _stderr = process.communicate()
bryantaylor 0:eafc3fd41f75 96 except OSError:
bryantaylor 0:eafc3fd41f75 97 print "[OS ERROR] Command: "+(' '.join(command))
bryantaylor 0:eafc3fd41f75 98 raise
bryantaylor 0:eafc3fd41f75 99
bryantaylor 0:eafc3fd41f75 100 return _stdout, _stderr, process.returncode
bryantaylor 0:eafc3fd41f75 101
bryantaylor 0:eafc3fd41f75 102
bryantaylor 0:eafc3fd41f75 103 def run_cmd_ext(command):
bryantaylor 0:eafc3fd41f75 104 """ A version of run command that checks if the command exists befor running
bryantaylor 0:eafc3fd41f75 105
bryantaylor 0:eafc3fd41f75 106 Positional arguments:
bryantaylor 0:eafc3fd41f75 107 command - the command line you are trying to invoke
bryantaylor 0:eafc3fd41f75 108 """
bryantaylor 0:eafc3fd41f75 109 assert is_cmd_valid(command[0])
bryantaylor 0:eafc3fd41f75 110 process = Popen(command, stdout=PIPE, stderr=PIPE)
bryantaylor 0:eafc3fd41f75 111 _stdout, _stderr = process.communicate()
bryantaylor 0:eafc3fd41f75 112 return _stdout, _stderr, process.returncode
bryantaylor 0:eafc3fd41f75 113
bryantaylor 0:eafc3fd41f75 114
bryantaylor 0:eafc3fd41f75 115 def is_cmd_valid(command):
bryantaylor 0:eafc3fd41f75 116 """ Verify that a command exists and is executable
bryantaylor 0:eafc3fd41f75 117
bryantaylor 0:eafc3fd41f75 118 Positional arguments:
bryantaylor 0:eafc3fd41f75 119 command - the command to check
bryantaylor 0:eafc3fd41f75 120 """
bryantaylor 0:eafc3fd41f75 121 caller = get_caller_name()
bryantaylor 0:eafc3fd41f75 122 cmd_path = find_cmd_abspath(command)
bryantaylor 0:eafc3fd41f75 123 if not cmd_path:
bryantaylor 0:eafc3fd41f75 124 error("%s: Command '%s' can't be found" % (caller, command))
bryantaylor 0:eafc3fd41f75 125 if not is_exec(cmd_path):
bryantaylor 0:eafc3fd41f75 126 error("%s: Command '%s' resolves to file '%s' which is not executable"
bryantaylor 0:eafc3fd41f75 127 % (caller, command, cmd_path))
bryantaylor 0:eafc3fd41f75 128 return True
bryantaylor 0:eafc3fd41f75 129
bryantaylor 0:eafc3fd41f75 130
bryantaylor 0:eafc3fd41f75 131 def is_exec(path):
bryantaylor 0:eafc3fd41f75 132 """A simple check to verify that a path to an executable exists
bryantaylor 0:eafc3fd41f75 133
bryantaylor 0:eafc3fd41f75 134 Positional arguments:
bryantaylor 0:eafc3fd41f75 135 path - the executable
bryantaylor 0:eafc3fd41f75 136 """
bryantaylor 0:eafc3fd41f75 137 return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
bryantaylor 0:eafc3fd41f75 138
bryantaylor 0:eafc3fd41f75 139
bryantaylor 0:eafc3fd41f75 140 def find_cmd_abspath(command):
bryantaylor 0:eafc3fd41f75 141 """ Returns the absolute path to a command.
bryantaylor 0:eafc3fd41f75 142 None is returned if no absolute path was found.
bryantaylor 0:eafc3fd41f75 143
bryantaylor 0:eafc3fd41f75 144 Positional arguhments:
bryantaylor 0:eafc3fd41f75 145 command - the command to find the path of
bryantaylor 0:eafc3fd41f75 146 """
bryantaylor 0:eafc3fd41f75 147 if exists(command) or exists(command + '.exe'):
bryantaylor 0:eafc3fd41f75 148 return os.path.abspath(command)
bryantaylor 0:eafc3fd41f75 149 if not 'PATH' in os.environ:
bryantaylor 0:eafc3fd41f75 150 raise Exception("Can't find command path for current platform ('%s')"
bryantaylor 0:eafc3fd41f75 151 % sys.platform)
bryantaylor 0:eafc3fd41f75 152 path_env = os.environ['PATH']
bryantaylor 0:eafc3fd41f75 153 for path in path_env.split(os.pathsep):
bryantaylor 0:eafc3fd41f75 154 cmd_path = '%s/%s' % (path, command)
bryantaylor 0:eafc3fd41f75 155 if exists(cmd_path) or exists(cmd_path + '.exe'):
bryantaylor 0:eafc3fd41f75 156 return cmd_path
bryantaylor 0:eafc3fd41f75 157
bryantaylor 0:eafc3fd41f75 158
bryantaylor 0:eafc3fd41f75 159 def mkdir(path):
bryantaylor 0:eafc3fd41f75 160 """ a wrapped makedirs that only tries to create a directory if it does not
bryantaylor 0:eafc3fd41f75 161 exist already
bryantaylor 0:eafc3fd41f75 162
bryantaylor 0:eafc3fd41f75 163 Positional arguments:
bryantaylor 0:eafc3fd41f75 164 path - the path to maybe create
bryantaylor 0:eafc3fd41f75 165 """
bryantaylor 0:eafc3fd41f75 166 if not exists(path):
bryantaylor 0:eafc3fd41f75 167 makedirs(path)
bryantaylor 0:eafc3fd41f75 168
bryantaylor 0:eafc3fd41f75 169
bryantaylor 0:eafc3fd41f75 170 def copy_file(src, dst):
bryantaylor 0:eafc3fd41f75 171 """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
bryantaylor 0:eafc3fd41f75 172 permissions (this was causing errors with directories mounted with samba)
bryantaylor 0:eafc3fd41f75 173
bryantaylor 0:eafc3fd41f75 174 Positional arguments:
bryantaylor 0:eafc3fd41f75 175 src - the source of the copy operation
bryantaylor 0:eafc3fd41f75 176 dst - the destination of the copy operation
bryantaylor 0:eafc3fd41f75 177 """
bryantaylor 0:eafc3fd41f75 178 if isdir(dst):
bryantaylor 0:eafc3fd41f75 179 _, base = split(src)
bryantaylor 0:eafc3fd41f75 180 dst = join(dst, base)
bryantaylor 0:eafc3fd41f75 181 copyfile(src, dst)
bryantaylor 0:eafc3fd41f75 182
bryantaylor 0:eafc3fd41f75 183
bryantaylor 0:eafc3fd41f75 184 def delete_dir_files(directory):
bryantaylor 0:eafc3fd41f75 185 """ A function that does rm -rf
bryantaylor 0:eafc3fd41f75 186
bryantaylor 0:eafc3fd41f75 187 Positional arguments:
bryantaylor 0:eafc3fd41f75 188 directory - the directory to remove
bryantaylor 0:eafc3fd41f75 189 """
bryantaylor 0:eafc3fd41f75 190 if not exists(directory):
bryantaylor 0:eafc3fd41f75 191 return
bryantaylor 0:eafc3fd41f75 192
bryantaylor 0:eafc3fd41f75 193 for element in listdir(directory):
bryantaylor 0:eafc3fd41f75 194 to_remove = join(directory, element)
bryantaylor 0:eafc3fd41f75 195 if not isdir(to_remove):
bryantaylor 0:eafc3fd41f75 196 remove(file)
bryantaylor 0:eafc3fd41f75 197
bryantaylor 0:eafc3fd41f75 198
bryantaylor 0:eafc3fd41f75 199 def get_caller_name(steps=2):
bryantaylor 0:eafc3fd41f75 200 """
bryantaylor 0:eafc3fd41f75 201 When called inside a function, it returns the name
bryantaylor 0:eafc3fd41f75 202 of the caller of that function.
bryantaylor 0:eafc3fd41f75 203
bryantaylor 0:eafc3fd41f75 204 Keyword arguments:
bryantaylor 0:eafc3fd41f75 205 steps - the number of steps up the stack the calling function is
bryantaylor 0:eafc3fd41f75 206 """
bryantaylor 0:eafc3fd41f75 207 return inspect.stack()[steps][3]
bryantaylor 0:eafc3fd41f75 208
bryantaylor 0:eafc3fd41f75 209
bryantaylor 0:eafc3fd41f75 210 def error(msg):
bryantaylor 0:eafc3fd41f75 211 """Fatal error, abort hard
bryantaylor 0:eafc3fd41f75 212
bryantaylor 0:eafc3fd41f75 213 Positional arguments:
bryantaylor 0:eafc3fd41f75 214 msg - the message to print before crashing
bryantaylor 0:eafc3fd41f75 215 """
bryantaylor 0:eafc3fd41f75 216 print("ERROR: %s" % msg)
bryantaylor 0:eafc3fd41f75 217 sys.exit(1)
bryantaylor 0:eafc3fd41f75 218
bryantaylor 0:eafc3fd41f75 219
bryantaylor 0:eafc3fd41f75 220 def rel_path(path, base, dot=False):
bryantaylor 0:eafc3fd41f75 221 """Relative path calculation that optionaly always starts with a dot
bryantaylor 0:eafc3fd41f75 222
bryantaylor 0:eafc3fd41f75 223 Positional arguments:
bryantaylor 0:eafc3fd41f75 224 path - the path to make relative
bryantaylor 0:eafc3fd41f75 225 base - what to make the path relative to
bryantaylor 0:eafc3fd41f75 226
bryantaylor 0:eafc3fd41f75 227 Keyword arguments:
bryantaylor 0:eafc3fd41f75 228 dot - if True, the path will always start with a './'
bryantaylor 0:eafc3fd41f75 229 """
bryantaylor 0:eafc3fd41f75 230 final_path = relpath(path, base)
bryantaylor 0:eafc3fd41f75 231 if dot and not final_path.startswith('.'):
bryantaylor 0:eafc3fd41f75 232 final_path = './' + final_path
bryantaylor 0:eafc3fd41f75 233 return final_path
bryantaylor 0:eafc3fd41f75 234
bryantaylor 0:eafc3fd41f75 235
bryantaylor 0:eafc3fd41f75 236 class ToolException(Exception):
bryantaylor 0:eafc3fd41f75 237 """A class representing an exception throw by the tools"""
bryantaylor 0:eafc3fd41f75 238 pass
bryantaylor 0:eafc3fd41f75 239
bryantaylor 0:eafc3fd41f75 240 class NotSupportedException(Exception):
bryantaylor 0:eafc3fd41f75 241 """A class a toolchain not supporting a particular target"""
bryantaylor 0:eafc3fd41f75 242 pass
bryantaylor 0:eafc3fd41f75 243
bryantaylor 0:eafc3fd41f75 244 class InvalidReleaseTargetException(Exception):
bryantaylor 0:eafc3fd41f75 245 pass
bryantaylor 0:eafc3fd41f75 246
bryantaylor 0:eafc3fd41f75 247 def split_path(path):
bryantaylor 0:eafc3fd41f75 248 """spilt a file name into it's directory name, base name, and extension
bryantaylor 0:eafc3fd41f75 249
bryantaylor 0:eafc3fd41f75 250 Positional arguments:
bryantaylor 0:eafc3fd41f75 251 path - the file name to split
bryantaylor 0:eafc3fd41f75 252 """
bryantaylor 0:eafc3fd41f75 253 base, has_ext = split(path)
bryantaylor 0:eafc3fd41f75 254 name, ext = splitext(has_ext)
bryantaylor 0:eafc3fd41f75 255 return base, name, ext
bryantaylor 0:eafc3fd41f75 256
bryantaylor 0:eafc3fd41f75 257
bryantaylor 0:eafc3fd41f75 258 def get_path_depth(path):
bryantaylor 0:eafc3fd41f75 259 """ Given a path, return the number of directory levels present.
bryantaylor 0:eafc3fd41f75 260 This roughly translates to the number of path separators (os.sep) + 1.
bryantaylor 0:eafc3fd41f75 261 Ex. Given "path/to/dir", this would return 3
bryantaylor 0:eafc3fd41f75 262 Special cases: "." and "/" return 0
bryantaylor 0:eafc3fd41f75 263
bryantaylor 0:eafc3fd41f75 264 Positional arguments:
bryantaylor 0:eafc3fd41f75 265 path - the path to calculate the depth of
bryantaylor 0:eafc3fd41f75 266 """
bryantaylor 0:eafc3fd41f75 267 normalized_path = normpath(path)
bryantaylor 0:eafc3fd41f75 268 path_depth = 0
bryantaylor 0:eafc3fd41f75 269 head, tail = split(normalized_path)
bryantaylor 0:eafc3fd41f75 270
bryantaylor 0:eafc3fd41f75 271 while tail and tail != '.':
bryantaylor 0:eafc3fd41f75 272 path_depth += 1
bryantaylor 0:eafc3fd41f75 273 head, tail = split(head)
bryantaylor 0:eafc3fd41f75 274
bryantaylor 0:eafc3fd41f75 275 return path_depth
bryantaylor 0:eafc3fd41f75 276
bryantaylor 0:eafc3fd41f75 277
bryantaylor 0:eafc3fd41f75 278 def args_error(parser, message):
bryantaylor 0:eafc3fd41f75 279 """Abort with an error that was generated by the arguments to a CLI program
bryantaylor 0:eafc3fd41f75 280
bryantaylor 0:eafc3fd41f75 281 Positional arguments:
bryantaylor 0:eafc3fd41f75 282 parser - the ArgumentParser object that parsed the command line
bryantaylor 0:eafc3fd41f75 283 message - what went wrong
bryantaylor 0:eafc3fd41f75 284 """
bryantaylor 0:eafc3fd41f75 285 parser.error(message)
bryantaylor 0:eafc3fd41f75 286 sys.exit(2)
bryantaylor 0:eafc3fd41f75 287
bryantaylor 0:eafc3fd41f75 288
bryantaylor 0:eafc3fd41f75 289 def construct_enum(**enums):
bryantaylor 0:eafc3fd41f75 290 """ Create your own pseudo-enums
bryantaylor 0:eafc3fd41f75 291
bryantaylor 0:eafc3fd41f75 292 Keyword arguments:
bryantaylor 0:eafc3fd41f75 293 * - a member of the Enum you are creating and it's value
bryantaylor 0:eafc3fd41f75 294 """
bryantaylor 0:eafc3fd41f75 295 return type('Enum', (), enums)
bryantaylor 0:eafc3fd41f75 296
bryantaylor 0:eafc3fd41f75 297
bryantaylor 0:eafc3fd41f75 298 def check_required_modules(required_modules, verbose=True):
bryantaylor 0:eafc3fd41f75 299 """ Function checks for Python modules which should be "importable"
bryantaylor 0:eafc3fd41f75 300 before test suite can be used.
bryantaylor 0:eafc3fd41f75 301 @return returns True if all modules are installed already
bryantaylor 0:eafc3fd41f75 302 """
bryantaylor 0:eafc3fd41f75 303 import imp
bryantaylor 0:eafc3fd41f75 304 not_installed_modules = []
bryantaylor 0:eafc3fd41f75 305 for module_name in required_modules:
bryantaylor 0:eafc3fd41f75 306 try:
bryantaylor 0:eafc3fd41f75 307 imp.find_module(module_name)
bryantaylor 0:eafc3fd41f75 308 except ImportError:
bryantaylor 0:eafc3fd41f75 309 # We also test against a rare case: module is an egg file
bryantaylor 0:eafc3fd41f75 310 try:
bryantaylor 0:eafc3fd41f75 311 __import__(module_name)
bryantaylor 0:eafc3fd41f75 312 except ImportError as exc:
bryantaylor 0:eafc3fd41f75 313 not_installed_modules.append(module_name)
bryantaylor 0:eafc3fd41f75 314 if verbose:
bryantaylor 0:eafc3fd41f75 315 print "Error: %s" % exc
bryantaylor 0:eafc3fd41f75 316
bryantaylor 0:eafc3fd41f75 317 if verbose:
bryantaylor 0:eafc3fd41f75 318 if not_installed_modules:
bryantaylor 0:eafc3fd41f75 319 print ("Warning: Module(s) %s not installed. Please install " + \
bryantaylor 0:eafc3fd41f75 320 "required module(s) before using this script.")\
bryantaylor 0:eafc3fd41f75 321 % (', '.join(not_installed_modules))
bryantaylor 0:eafc3fd41f75 322
bryantaylor 0:eafc3fd41f75 323 if not_installed_modules:
bryantaylor 0:eafc3fd41f75 324 return False
bryantaylor 0:eafc3fd41f75 325 else:
bryantaylor 0:eafc3fd41f75 326 return True
bryantaylor 0:eafc3fd41f75 327
bryantaylor 0:eafc3fd41f75 328 def dict_to_ascii(dictionary):
bryantaylor 0:eafc3fd41f75 329 """ Utility function: traverse a dictionary and change all the strings in
bryantaylor 0:eafc3fd41f75 330 the dictionary to ASCII from Unicode. Useful when reading ASCII JSON data,
bryantaylor 0:eafc3fd41f75 331 because the JSON decoder always returns Unicode string. Based on
bryantaylor 0:eafc3fd41f75 332 http://stackoverflow.com/a/13105359
bryantaylor 0:eafc3fd41f75 333
bryantaylor 0:eafc3fd41f75 334 Positional arguments:
bryantaylor 0:eafc3fd41f75 335 dictionary - The dict that contains some Unicode that should be ASCII
bryantaylor 0:eafc3fd41f75 336 """
bryantaylor 0:eafc3fd41f75 337 if isinstance(dictionary, dict):
bryantaylor 0:eafc3fd41f75 338 return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value))
bryantaylor 0:eafc3fd41f75 339 for key, value in dictionary.iteritems()])
bryantaylor 0:eafc3fd41f75 340 elif isinstance(dictionary, list):
bryantaylor 0:eafc3fd41f75 341 return [dict_to_ascii(element) for element in dictionary]
bryantaylor 0:eafc3fd41f75 342 elif isinstance(dictionary, unicode):
bryantaylor 0:eafc3fd41f75 343 return dictionary.encode('ascii')
bryantaylor 0:eafc3fd41f75 344 else:
bryantaylor 0:eafc3fd41f75 345 return dictionary
bryantaylor 0:eafc3fd41f75 346
bryantaylor 0:eafc3fd41f75 347 def json_file_to_dict(fname):
bryantaylor 0:eafc3fd41f75 348 """ Read a JSON file and return its Python representation, transforming all
bryantaylor 0:eafc3fd41f75 349 the strings from Unicode to ASCII. The order of keys in the JSON file is
bryantaylor 0:eafc3fd41f75 350 preserved.
bryantaylor 0:eafc3fd41f75 351
bryantaylor 0:eafc3fd41f75 352 Positional arguments:
bryantaylor 0:eafc3fd41f75 353 fname - the name of the file to parse
bryantaylor 0:eafc3fd41f75 354 """
bryantaylor 0:eafc3fd41f75 355 try:
bryantaylor 0:eafc3fd41f75 356 with open(fname, "r") as file_obj:
bryantaylor 0:eafc3fd41f75 357 return dict_to_ascii(json.load(file_obj,
bryantaylor 0:eafc3fd41f75 358 object_pairs_hook=OrderedDict))
bryantaylor 0:eafc3fd41f75 359 except (ValueError, IOError):
bryantaylor 0:eafc3fd41f75 360 sys.stderr.write("Error parsing '%s':\n" % fname)
bryantaylor 0:eafc3fd41f75 361 raise
bryantaylor 0:eafc3fd41f75 362
bryantaylor 0:eafc3fd41f75 363 # Wowza, double closure
bryantaylor 0:eafc3fd41f75 364 def argparse_type(casedness, prefer_hyphen=False):
bryantaylor 0:eafc3fd41f75 365 def middle(lst, type_name):
bryantaylor 0:eafc3fd41f75 366 def parse_type(string):
bryantaylor 0:eafc3fd41f75 367 """ validate that an argument passed in (as string) is a member of
bryantaylor 0:eafc3fd41f75 368 the list of possible arguments. Offer a suggestion if the case of
bryantaylor 0:eafc3fd41f75 369 the string, or the hyphens/underscores do not match the expected
bryantaylor 0:eafc3fd41f75 370 style of the argument.
bryantaylor 0:eafc3fd41f75 371 """
bryantaylor 0:eafc3fd41f75 372 if prefer_hyphen:
bryantaylor 0:eafc3fd41f75 373 newstring = casedness(string).replace("_", "-")
bryantaylor 0:eafc3fd41f75 374 else:
bryantaylor 0:eafc3fd41f75 375 newstring = casedness(string).replace("-", "_")
bryantaylor 0:eafc3fd41f75 376 if string in lst:
bryantaylor 0:eafc3fd41f75 377 return string
bryantaylor 0:eafc3fd41f75 378 elif string not in lst and newstring in lst:
bryantaylor 0:eafc3fd41f75 379 raise argparse.ArgumentTypeError(
bryantaylor 0:eafc3fd41f75 380 "{0} is not a supported {1}. Did you mean {2}?".format(
bryantaylor 0:eafc3fd41f75 381 string, type_name, newstring))
bryantaylor 0:eafc3fd41f75 382 else:
bryantaylor 0:eafc3fd41f75 383 raise argparse.ArgumentTypeError(
bryantaylor 0:eafc3fd41f75 384 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
bryantaylor 0:eafc3fd41f75 385 format(string, type_name, columnate(lst)))
bryantaylor 0:eafc3fd41f75 386 return parse_type
bryantaylor 0:eafc3fd41f75 387 return middle
bryantaylor 0:eafc3fd41f75 388
bryantaylor 0:eafc3fd41f75 389 # short cuts for the argparse_type versions
bryantaylor 0:eafc3fd41f75 390 argparse_uppercase_type = argparse_type(str.upper, False)
bryantaylor 0:eafc3fd41f75 391 argparse_lowercase_type = argparse_type(str.lower, False)
bryantaylor 0:eafc3fd41f75 392 argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
bryantaylor 0:eafc3fd41f75 393 argparse_lowercase_hyphen_type = argparse_type(str.lower, True)
bryantaylor 0:eafc3fd41f75 394
bryantaylor 0:eafc3fd41f75 395 def argparse_force_type(case):
bryantaylor 0:eafc3fd41f75 396 """ validate that an argument passed in (as string) is a member of the list
bryantaylor 0:eafc3fd41f75 397 of possible arguments after converting it's case.
bryantaylor 0:eafc3fd41f75 398 """
bryantaylor 0:eafc3fd41f75 399 def middle(lst, type_name):
bryantaylor 0:eafc3fd41f75 400 """ The parser type generator"""
bryantaylor 0:eafc3fd41f75 401 def parse_type(string):
bryantaylor 0:eafc3fd41f75 402 """ The parser type"""
bryantaylor 0:eafc3fd41f75 403 for option in lst:
bryantaylor 0:eafc3fd41f75 404 if case(string) == case(option):
bryantaylor 0:eafc3fd41f75 405 return option
bryantaylor 0:eafc3fd41f75 406 raise argparse.ArgumentTypeError(
bryantaylor 0:eafc3fd41f75 407 "{0} is not a supported {1}. Supported {1}s are:\n{2}".
bryantaylor 0:eafc3fd41f75 408 format(string, type_name, columnate(lst)))
bryantaylor 0:eafc3fd41f75 409 return parse_type
bryantaylor 0:eafc3fd41f75 410 return middle
bryantaylor 0:eafc3fd41f75 411
bryantaylor 0:eafc3fd41f75 412 # these two types convert the case of their arguments _before_ validation
bryantaylor 0:eafc3fd41f75 413 argparse_force_uppercase_type = argparse_force_type(str.upper)
bryantaylor 0:eafc3fd41f75 414 argparse_force_lowercase_type = argparse_force_type(str.lower)
bryantaylor 0:eafc3fd41f75 415
bryantaylor 0:eafc3fd41f75 416 def argparse_many(func):
bryantaylor 0:eafc3fd41f75 417 """ An argument parser combinator that takes in an argument parser and
bryantaylor 0:eafc3fd41f75 418 creates a new parser that accepts a comma separated list of the same thing.
bryantaylor 0:eafc3fd41f75 419 """
bryantaylor 0:eafc3fd41f75 420 def wrap(string):
bryantaylor 0:eafc3fd41f75 421 """ The actual parser"""
bryantaylor 0:eafc3fd41f75 422 return [func(s) for s in string.split(",")]
bryantaylor 0:eafc3fd41f75 423 return wrap
bryantaylor 0:eafc3fd41f75 424
bryantaylor 0:eafc3fd41f75 425 def argparse_filestring_type(string):
bryantaylor 0:eafc3fd41f75 426 """ An argument parser that verifies that a string passed in corresponds
bryantaylor 0:eafc3fd41f75 427 to a file"""
bryantaylor 0:eafc3fd41f75 428 if exists(string):
bryantaylor 0:eafc3fd41f75 429 return string
bryantaylor 0:eafc3fd41f75 430 else:
bryantaylor 0:eafc3fd41f75 431 raise argparse.ArgumentTypeError(
bryantaylor 0:eafc3fd41f75 432 "{0}"" does not exist in the filesystem.".format(string))
bryantaylor 0:eafc3fd41f75 433
bryantaylor 0:eafc3fd41f75 434 def columnate(strings, separator=", ", chars=80):
bryantaylor 0:eafc3fd41f75 435 """ render a list of strings as a in a bunch of columns
bryantaylor 0:eafc3fd41f75 436
bryantaylor 0:eafc3fd41f75 437 Positional arguments:
bryantaylor 0:eafc3fd41f75 438 strings - the strings to columnate
bryantaylor 0:eafc3fd41f75 439
bryantaylor 0:eafc3fd41f75 440 Keyword arguments;
bryantaylor 0:eafc3fd41f75 441 separator - the separation between the columns
bryantaylor 0:eafc3fd41f75 442 chars - the maximum with of a row
bryantaylor 0:eafc3fd41f75 443 """
bryantaylor 0:eafc3fd41f75 444 col_width = max(len(s) for s in strings)
bryantaylor 0:eafc3fd41f75 445 total_width = col_width + len(separator)
bryantaylor 0:eafc3fd41f75 446 columns = math.floor(chars / total_width)
bryantaylor 0:eafc3fd41f75 447 output = ""
bryantaylor 0:eafc3fd41f75 448 for i, string in zip(range(len(strings)), strings):
bryantaylor 0:eafc3fd41f75 449 append = string
bryantaylor 0:eafc3fd41f75 450 if i != len(strings) - 1:
bryantaylor 0:eafc3fd41f75 451 append += separator
bryantaylor 0:eafc3fd41f75 452 if i % columns == columns - 1:
bryantaylor 0:eafc3fd41f75 453 append += "\n"
bryantaylor 0:eafc3fd41f75 454 else:
bryantaylor 0:eafc3fd41f75 455 append = append.ljust(total_width)
bryantaylor 0:eafc3fd41f75 456 output += append
bryantaylor 0:eafc3fd41f75 457 return output
bryantaylor 0:eafc3fd41f75 458
bryantaylor 0:eafc3fd41f75 459 def argparse_dir_not_parent(other):
bryantaylor 0:eafc3fd41f75 460 """fail if argument provided is a parent of the specified directory"""
bryantaylor 0:eafc3fd41f75 461 def parse_type(not_parent):
bryantaylor 0:eafc3fd41f75 462 """The parser type"""
bryantaylor 0:eafc3fd41f75 463 abs_other = abspath(other)
bryantaylor 0:eafc3fd41f75 464 abs_not_parent = abspath(not_parent)
bryantaylor 0:eafc3fd41f75 465 if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
bryantaylor 0:eafc3fd41f75 466 raise argparse.ArgumentTypeError(
bryantaylor 0:eafc3fd41f75 467 "{0} may not be a parent directory of {1}".format(
bryantaylor 0:eafc3fd41f75 468 not_parent, other))
bryantaylor 0:eafc3fd41f75 469 else:
bryantaylor 0:eafc3fd41f75 470 return not_parent
bryantaylor 0:eafc3fd41f75 471 return parse_type