__init__.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 
00019 import re
00020 import sys
00021 import json
00022 from os import stat, walk, getcwd, sep, remove
00023 from copy import copy
00024 from time import time, sleep
00025 from shutil import copyfile
00026 from os.path import (join, splitext, exists, relpath, dirname, basename, split,
00027                      abspath, isfile, isdir, normcase)
00028 from inspect import getmro
00029 from copy import deepcopy
00030 from collections import namedtuple
00031 from abc import ABCMeta, abstractmethod
00032 from distutils.spawn import find_executable
00033 from multiprocessing import Pool, cpu_count
00034 from hashlib import md5
00035 
00036 from ..utils import (run_cmd, mkdir, rel_path, ToolException,
00037                     NotSupportedException, split_path, compile_worker)
00038 from ..settings import MBED_ORG_USER, PRINT_COMPILER_OUTPUT_AS_LINK
00039 from .. import hooks
00040 from ..notifier.term import TerminalNotifier
00041 from ..resources import FileType
00042 from ..memap import MemapParser
00043 from ..config import ConfigException
00044 
00045 
00046 #Disables multiprocessing if set to higher number than the host machine CPUs
00047 CPU_COUNT_MIN = 1
00048 CPU_COEF = 1
00049 
00050 class mbedToolchain:
00051     # Verbose logging
00052     VERBOSE = True
00053 
00054     # Compile C files as CPP
00055     COMPILE_C_AS_CPP = False
00056 
00057     # Response files for compiling, includes, linking and archiving.
00058     # Not needed on posix systems where the typical arg limit is 2 megabytes
00059     RESPONSE_FILES = True
00060 
00061     CORTEX_SYMBOLS = {
00062         "Cortex-M0" : ["__CORTEX_M0", "ARM_MATH_CM0", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00063         "Cortex-M0+": ["__CORTEX_M0PLUS", "ARM_MATH_CM0PLUS", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00064         "Cortex-M1" : ["__CORTEX_M3", "ARM_MATH_CM1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00065         "Cortex-M3" : ["__CORTEX_M3", "ARM_MATH_CM3", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00066         "Cortex-M4" : ["__CORTEX_M4", "ARM_MATH_CM4", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00067         "Cortex-M4F" : ["__CORTEX_M4", "ARM_MATH_CM4", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00068         "Cortex-M7" : ["__CORTEX_M7", "ARM_MATH_CM7", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00069         "Cortex-M7F" : ["__CORTEX_M7", "ARM_MATH_CM7", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00070         "Cortex-M7FD" : ["__CORTEX_M7", "ARM_MATH_CM7", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00071         "Cortex-A9" : ["__CORTEX_A9", "ARM_MATH_CA9", "__FPU_PRESENT", "__CMSIS_RTOS", "__EVAL", "__MBED_CMSIS_RTOS_CA9"],
00072         "Cortex-M23-NS": ["__CORTEX_M23", "ARM_MATH_ARMV8MBL", "DOMAIN_NS=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00073         "Cortex-M23": ["__CORTEX_M23", "ARM_MATH_ARMV8MBL", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00074         "Cortex-M33-NS": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "DOMAIN_NS=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00075         "Cortex-M33": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00076         "Cortex-M33F-NS": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "DOMAIN_NS=1", "__FPU_PRESENT", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00077         "Cortex-M33F": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "__FPU_PRESENT", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00078     }
00079 
00080     MBED_CONFIG_FILE_NAME="mbed_config.h"
00081 
00082     PROFILE_FILE_NAME = ".profile"
00083 
00084     __metaclass__ = ABCMeta
00085 
00086     profile_template = {'common':[], 'c':[], 'cxx':[], 'asm':[], 'ld':[]}
00087 
00088     def __init__(self, target, notify=None, macros=None, build_profile=None,
00089                  build_dir=None):
00090         self.target = target
00091         self.name = self.__class__.__name__
00092 
00093         # compile/assemble/link/binary hooks
00094         self.hook = hooks.Hook(target, self)
00095 
00096         # Toolchain flags
00097         self.flags = deepcopy(build_profile or self.profile_template)
00098 
00099         # System libraries provided by the toolchain
00100         self.sys_libs = []
00101 
00102         # User-defined macros
00103         self.macros = macros or []
00104 
00105         # Macros generated from toolchain and target rules/features
00106         self.asm_symbols = None
00107         self.cxx_symbols = None
00108 
00109         # Labels generated from toolchain and target rules/features (used for selective build)
00110         self.labels = None
00111 
00112         # This will hold the initialized config object
00113         self.config = None
00114 
00115         # This will hold the configuration data (as returned by Config.get_config_data())
00116         self.config_data = None
00117 
00118         # This will hold the location of the configuration file or None if there's no configuration available
00119         self.config_file = None
00120 
00121         # Call guard for "get_config_data" (see the comments of get_config_data for details)
00122         self.config_processed = False
00123 
00124         # Non-incremental compile
00125         self.build_all = False
00126 
00127         # Build output dir
00128         self.build_dir = abspath(build_dir) if PRINT_COMPILER_OUTPUT_AS_LINK else build_dir
00129         self.timestamp = time()
00130 
00131         # Number of concurrent build jobs. 0 means auto (based on host system cores)
00132         self.jobs = 0
00133 
00134 
00135         # Output notify function
00136         # This function is passed all events, and expected to handle notification of the
00137         # user, emit the events to a log, etc.
00138         # The API for all notify methods passed into the notify parameter is as follows:
00139         # def notify(Event, Silent)
00140         # Where *Event* is a dict representing the toolchain event that was generated
00141         #            e.g.: a compile succeeded, or a warning was emitted by the compiler
00142         #                  or an application was linked
00143         #       *Silent* is a boolean
00144         if notify:
00145             self.notify = notify
00146         else:
00147             self.notify = TerminalNotifier()
00148 
00149 
00150         # Stats cache is used to reduce the amount of IO requests to stat
00151         # header files during dependency change. See need_update()
00152         self.stat_cache = {}
00153 
00154         # Used by the mbed Online Build System to build in chrooted environment
00155         self.CHROOT = None
00156 
00157         # Call post __init__() hooks before the ARM/GCC_ARM/IAR toolchain __init__() takes over
00158         self.init()
00159 
00160     # Used for post __init__() hooks
00161     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00162     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00163     def init(self):
00164         return True
00165 
00166     def get_output(self):
00167         return self.notifier.get_output()
00168 
00169     def get_symbols(self, for_asm=False):
00170         if for_asm:
00171             if self.asm_symbols is None:
00172                 self.asm_symbols = []
00173 
00174                 # Cortex CPU symbols
00175                 if self.target.core in mbedToolchain.CORTEX_SYMBOLS:
00176                     self.asm_symbols.extend(mbedToolchain.CORTEX_SYMBOLS[self.target.core])
00177 
00178                 # Add target's symbols
00179                 self.asm_symbols += self.target.macros
00180                 # Add extra symbols passed via 'macros' parameter
00181                 self.asm_symbols += self.macros
00182             return list(set(self.asm_symbols))  # Return only unique symbols
00183         else:
00184             if self.cxx_symbols is None:
00185                 # Target and Toolchain symbols
00186                 labels = self.get_labels()
00187                 self.cxx_symbols = ["TARGET_%s" % t for t in labels['TARGET']]
00188                 self.cxx_symbols.extend(["TOOLCHAIN_%s" % t for t in labels['TOOLCHAIN']])
00189 
00190                 # Cortex CPU symbols
00191                 if self.target.core in mbedToolchain.CORTEX_SYMBOLS:
00192                     self.cxx_symbols.extend(mbedToolchain.CORTEX_SYMBOLS[self.target.core])
00193 
00194                 # Symbols defined by the on-line build.system
00195                 self.cxx_symbols.extend(['MBED_BUILD_TIMESTAMP=%s' % self.timestamp, 'TARGET_LIKE_MBED', '__MBED__=1'])
00196                 if MBED_ORG_USER:
00197                     self.cxx_symbols.append('MBED_USERNAME=' + MBED_ORG_USER)
00198 
00199                 # Add target's symbols
00200                 self.cxx_symbols += self.target.macros
00201                 # Add target's hardware
00202                 self.cxx_symbols += ["DEVICE_" + data + "=1" for data in self.target.device_has]
00203                 # Add target's features
00204                 self.cxx_symbols += ["FEATURE_" + data + "=1" for data in self.target.features]
00205                 # Add target's components
00206                 self.cxx_symbols += ["COMPONENT_" + data + "=1" for data in self.target.components]
00207                 # Add extra symbols passed via 'macros' parameter
00208                 self.cxx_symbols += self.macros
00209 
00210                 # Form factor variables
00211                 if hasattr(self.target, 'supported_form_factors'):
00212                     self.cxx_symbols.extend(["TARGET_FF_%s" % t for t in self.target.supported_form_factors])
00213 
00214             return list(set(self.cxx_symbols))  # Return only unique symbols
00215 
00216     # Extend the internal list of macros
00217     def add_macros(self, new_macros):
00218         self.macros.extend(new_macros)
00219 
00220     def get_labels(self):
00221         if self.labels is None:
00222             toolchain_labels = self._get_toolchain_labels()
00223             self.labels = {
00224                 'TARGET': self.target.labels,
00225                 'FEATURE': self.target.features,
00226                 'COMPONENT': self.target.components,
00227                 'TOOLCHAIN': toolchain_labels
00228             }
00229 
00230             # This is a policy decision and it should /really/ be in the config system
00231             # ATM it's here for backward compatibility
00232             if ((("-g" in self.flags['common'] or "-g3" in self.flags['common']) and
00233                  "-O0" in self.flags['common']) or
00234                 ("-r" in self.flags['common'] and
00235                  "-On" in self.flags['common'])):
00236                 self.labels['TARGET'].append("DEBUG")
00237             else:
00238                 self.labels['TARGET'].append("RELEASE")
00239         return self.labels
00240 
00241     def _get_toolchain_labels(self):
00242         toolchain_labels = [c.__name__ for c in getmro(self.__class__)]
00243         toolchain_labels.remove('mbedToolchain')
00244         toolchain_labels.remove('object')
00245         return toolchain_labels
00246 
00247 
00248     # Determine whether a source file needs updating/compiling
00249     def need_update(self, target, dependencies):
00250         if self.build_all:
00251             return True
00252 
00253         if not exists(target):
00254             return True
00255 
00256         target_mod_time = stat(target).st_mtime
00257 
00258         for d in dependencies:
00259             # Some objects are not provided with full path and here we do not have
00260             # information about the library paths. Safe option: assume an update
00261             if not d or not exists(d):
00262                 return True
00263 
00264             if d not in self.stat_cache:
00265                 self.stat_cache[d] = stat(d).st_mtime
00266 
00267             if self.stat_cache[d] >= target_mod_time:
00268                 return True
00269 
00270         return False
00271 
00272 
00273     def scan_repository(self, path):
00274         resources = []
00275 
00276         for root, dirs, files in walk(path):
00277             # Remove ignored directories
00278             for d in copy(dirs):
00279                 if d == '.' or d == '..':
00280                     dirs.remove(d)
00281 
00282             for file in files:
00283                 file_path = join(root, file)
00284                 resources.append(file_path)
00285 
00286         return resources
00287 
00288     def copy_files(self, files_paths, trg_path, resources=None):
00289         # Handle a single file
00290         if not isinstance(files_paths, list):
00291             files_paths = [files_paths]
00292 
00293         for dest, source in files_paths:
00294             target = join(trg_path, dest)
00295             if (target != source) and (self.need_update(target, [source])):
00296                 self.progress("copy", dest)
00297                 mkdir(dirname(target))
00298                 copyfile(source, target)
00299 
00300     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00301     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00302     def relative_object_path(self, build_path, file_ref):
00303         source_dir, name, _ = split_path(file_ref.name)
00304 
00305         obj_dir = relpath(join(build_path, source_dir))
00306         if obj_dir is not self.prev_dir:
00307             self.prev_dir = obj_dir
00308             mkdir(obj_dir)
00309         return join(obj_dir, name + '.o')
00310 
00311     def make_option_file(self, options, naming=".options_{}.txt"):
00312         """ Generate a via file for a pile of defines
00313         ARM, GCC, IAR cross compatible
00314         """
00315         to_write = " ".join(options).encode('utf-8')
00316         new_md5 = md5(to_write).hexdigest()
00317         via_file = join(self.build_dir, naming.format(new_md5))
00318         try:
00319             with open(via_file, "r") as fd:
00320                 old_md5 = md5(fd.read().encode('utf-8')).hexdigest()
00321         except IOError:
00322             old_md5 = None
00323         if old_md5 != new_md5:
00324             with open(via_file, "wb") as fd:
00325                 fd.write(to_write)
00326         return via_file
00327 
00328     def get_inc_file(self, includes):
00329         """Generate a via file for all includes.
00330         ARM, GCC, IAR cross compatible
00331         """
00332         cmd_list = ("-I{}".format(c.replace("\\", "/")) for c in includes if c)
00333         if self.CHROOT:
00334             cmd_list = (c.replace(self.CHROOT, '') for c in cmd_list)
00335         return self.make_option_file(list(cmd_list), naming=".includes_{}.txt")
00336 
00337     def get_link_file(self, cmd):
00338         """Generate a via file for all objects when linking.
00339         ARM, GCC, IAR cross compatible
00340         """
00341         cmd_list = (c.replace("\\", "/") for c in cmd if c)
00342         if self.CHROOT:
00343             cmd_list = (c.replace(self.CHROOT, '') for c in cmd_list)
00344         return self.make_option_file(list(cmd_list), naming=".link_options.txt")
00345 
00346     def get_arch_file(self, objects):
00347         """ Generate a via file for all objects when archiving.
00348         ARM, GCC, IAR cross compatible
00349         """
00350         cmd_list = (c.replace("\\", "/") for c in objects if c)
00351         return self.make_option_file(list(cmd_list), ".archive_files.txt")
00352 
00353     # THIS METHOD IS BEING CALLED BY THE MBED ONLINE BUILD SYSTEM
00354     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00355     def compile_sources(self, resources, inc_dirs=None):
00356         # Web IDE progress bar for project build
00357         files_to_compile = (
00358             resources.get_file_refs(FileType.ASM_SRC) +
00359             resources.get_file_refs(FileType.C_SRC) +
00360             resources.get_file_refs(FileType.CPP_SRC)
00361         )
00362         self.to_be_compiled = len(files_to_compile)
00363         self.compiled = 0
00364 
00365         self.notify.cc_verbose("Macros: "+' '.join(['-D%s' % s for s in self.get_symbols()]))
00366 
00367         inc_paths = resources.get_file_paths(FileType.INC_DIR)
00368         if inc_dirs is not None:
00369             if isinstance(inc_dirs, list):
00370                 inc_paths.extend(inc_dirs)
00371             else:
00372                 inc_paths.append(inc_dirs)
00373         # De-duplicate include paths
00374         inc_paths = set(inc_paths)
00375         # Sort include paths for consistency
00376         inc_paths = sorted(set(inc_paths))
00377         # Unique id of all include paths
00378         self.inc_md5 = md5(' '.join(inc_paths).encode('utf-8')).hexdigest()
00379 
00380         objects = []
00381         queue = []
00382         work_dir = getcwd()
00383         self.prev_dir = None
00384 
00385         # Generate configuration header (this will update self.build_all if needed)
00386         self.get_config_header()
00387         self.dump_build_profile()
00388 
00389         # Sort compile queue for consistency
00390         files_to_compile.sort()
00391         for source in files_to_compile:
00392             object = self.relative_object_path(self.build_dir, source)
00393 
00394             # Queue mode (multiprocessing)
00395             commands = self.compile_command(source.path, object, inc_paths)
00396             if commands is not None:
00397                 queue.append({
00398                     'source': source,
00399                     'object': object,
00400                     'commands': commands,
00401                     'work_dir': work_dir,
00402                     'chroot': self.CHROOT
00403                 })
00404             else:
00405                 self.compiled += 1
00406                 objects.append(object)
00407 
00408         # Use queues/multiprocessing if cpu count is higher than setting
00409         jobs = self.jobs if self.jobs else cpu_count()
00410         if jobs > CPU_COUNT_MIN and len(queue) > jobs:
00411             return self.compile_queue(queue, objects)
00412         else:
00413             return self.compile_seq(queue, objects)
00414 
00415     # Compile source files queue in sequential order
00416     def compile_seq(self, queue, objects):
00417         for item in queue:
00418             result = compile_worker(item)
00419 
00420             self.compiled += 1
00421             self.progress("compile", item['source'].name, build_update=True)
00422             for res in result['results']:
00423                 self.notify.cc_verbose("Compile: %s" % ' '.join(res['command']), result['source'].name)
00424                 self.compile_output([
00425                     res['code'],
00426                     res['output'],
00427                     res['command']
00428                 ])
00429             objects.append(result['object'])
00430         return objects
00431 
00432     # Compile source files queue in parallel by creating pool of worker threads
00433     def compile_queue(self, queue, objects):
00434         jobs_count = int(self.jobs if self.jobs else cpu_count() * CPU_COEF)
00435         p = Pool(processes=jobs_count)
00436 
00437         results = []
00438         for i in range(len(queue)):
00439             results.append(p.apply_async(compile_worker, [queue[i]]))
00440         p.close()
00441 
00442         itr = 0
00443         while len(results):
00444             itr += 1
00445             if itr > 180000:
00446                 p.terminate()
00447                 p.join()
00448                 raise ToolException("Compile did not finish in 5 minutes")
00449 
00450             sleep(0.01)
00451             pending = 0
00452             for r in results:
00453                 if r.ready():
00454                     try:
00455                         result = r.get()
00456                         results.remove(r)
00457 
00458                         self.compiled += 1
00459                         self.progress("compile", result['source'].name, build_update=True)
00460                         for res in result['results']:
00461                             self.notify.cc_verbose("Compile: %s" % ' '.join(res['command']), result['source'].name)
00462                             self.compile_output([
00463                                 res['code'],
00464                                 res['output'],
00465                                 res['command']
00466                             ])
00467                         objects.append(result['object'])
00468                     except ToolException as err:
00469                         if p._taskqueue.queue:
00470                             p._taskqueue.queue.clear()
00471                             sleep(0.5)
00472                         p.terminate()
00473                         p.join()
00474                         raise ToolException(err)
00475                 else:
00476                     pending += 1
00477                     if pending >= jobs_count:
00478                         break
00479 
00480         results = None
00481         p.join()
00482 
00483         return objects
00484 
00485     # Determine the compile command based on type of source file
00486     def compile_command(self, source, object, includes):
00487         # Check dependencies
00488         _, ext = splitext(source)
00489         ext = ext.lower()
00490 
00491         source = abspath(source) if PRINT_COMPILER_OUTPUT_AS_LINK else source
00492 
00493         if ext == '.c' or  ext == '.cpp' or ext == '.cc':
00494             base, _ = splitext(object)
00495             dep_path = base + '.d'
00496             try:
00497                 deps = self.parse_dependencies(dep_path) if (exists(dep_path)) else []
00498             except (IOError, IndexError):
00499                 deps = []
00500             config_file = ([self.config.app_config_location]
00501                            if self.config.app_config_location else [])
00502             deps.extend(config_file)
00503             if ext != '.c' or self.COMPILE_C_AS_CPP:
00504                 deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-cxx"))
00505             else:
00506                 deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-c"))
00507             if len(deps) == 0 or self.need_update(object, deps):
00508                 if ext != '.c' or self.COMPILE_C_AS_CPP:
00509                     return self.compile_cpp(source, object, includes)
00510                 else:
00511                     return self.compile_c(source, object, includes)
00512         elif ext == '.s':
00513             deps = [source]
00514             deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-asm"))
00515             if self.need_update(object, deps):
00516                 return self.assemble(source, object, includes)
00517         else:
00518             return False
00519 
00520         return None
00521 
00522     def parse_dependencies(self, dep_path):
00523         """Parse the dependency information generated by the compiler.
00524 
00525         Positional arguments:
00526         dep_path -- the path to a file generated by a previous run of the compiler
00527 
00528         Return value:
00529         A list of all source files that the dependency file indicated were dependencies
00530 
00531         Side effects:
00532         None
00533 
00534         Note: A default implementation is provided for make-like file formats
00535         """
00536         dependencies = []
00537         buff = open(dep_path).readlines()
00538         if buff:
00539             buff[0] = re.sub('^(.*?)\: ', '', buff[0])
00540             for line in buff:
00541                 filename = line.replace('\\\n', '').strip()
00542                 if filename:
00543                     filename = filename.replace('\\ ', '\a')
00544                     dependencies.extend(((self.CHROOT if self.CHROOT else '') +
00545                                          f.replace('\a', ' '))
00546                                         for f in filename.split(" "))
00547         return list(filter(None, dependencies))
00548 
00549     def is_not_supported_error(self, output):
00550         return "#error directive: [NOT_SUPPORTED]" in output
00551 
00552     @abstractmethod
00553     def parse_output(self, output):
00554         """Take in compiler output and extract sinlge line warnings and errors from it.
00555 
00556         Positional arguments:
00557         output -- a string of all the messages emitted by a run of the compiler
00558 
00559         Return value:
00560         None
00561 
00562         Side effects:
00563         call self.cc_info or self.notify with a description of the event generated by the compiler
00564         """
00565         raise NotImplemented
00566 
00567     def compile_output(self, output=[]):
00568         _rc = output[0]
00569         _stderr = output[1].decode("utf-8")
00570         command = output[2]
00571 
00572         # Parse output for Warnings and Errors
00573         self.parse_output(_stderr)
00574         self.notify.debug("Return: %s"% _rc)
00575         for error_line in _stderr.splitlines():
00576             self.notify.debug("Output: %s"% error_line)
00577 
00578         # Check return code
00579         if _rc != 0:
00580             if self.is_not_supported_error(_stderr):
00581                 raise NotSupportedException(_stderr)
00582             else:
00583                 raise ToolException(_stderr)
00584 
00585     def build_library(self, objects, dir, name):
00586         needed_update = False
00587         lib = self.STD_LIB_NAME % name
00588         fout = join(dir, lib)
00589         if self.need_update(fout, objects):
00590             self.notify.info("Library: %s" % lib)
00591             self.archive(objects, fout)
00592             needed_update = True
00593 
00594         return needed_update
00595 
00596     def link_program(self, r, tmp_path, name):
00597         needed_update = False
00598         ext = 'bin'
00599         if hasattr(self.target, 'OUTPUT_EXT'):
00600             ext = self.target.OUTPUT_EXT
00601 
00602         if hasattr(self.target, 'OUTPUT_NAMING'):
00603             self.notify.var("binary_naming", self.target.OUTPUT_NAMING)
00604             if self.target.OUTPUT_NAMING == "8.3":
00605                 name = name[0:8]
00606                 ext = ext[0:3]
00607 
00608         # Create destination directory
00609         head, tail =  split(name)
00610         new_path = join(tmp_path, head)
00611         mkdir(new_path)
00612 
00613         filename = name+'.'+ext
00614         # Absolute path of the final linked file
00615         full_path = join(tmp_path, filename)
00616         elf = join(tmp_path, name + '.elf')
00617         bin = None if ext == 'elf' else full_path
00618         map = join(tmp_path, name + '.map')
00619 
00620         objects = sorted(set(r.get_file_paths(FileType.OBJECT)))
00621         config_file = ([self.config.app_config_location]
00622                        if self.config.app_config_location else [])
00623         linker_script = [path for _, path in r.get_file_refs(FileType.LD_SCRIPT)
00624                          if path.endswith(self.LINKER_EXT)][-1]
00625         lib_dirs = r.get_file_paths(FileType.LIB_DIR)
00626         libraries = [l for l in r.get_file_paths(FileType.LIB)
00627                      if l.endswith(self.LIBRARY_EXT)]
00628         dependencies = objects + libraries + [linker_script] + config_file
00629         dependencies.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-ld"))
00630         if self.need_update(elf, dependencies):
00631             needed_update = True
00632             self.progress("link", name)
00633             self.link(elf, objects, libraries, lib_dirs, linker_script)
00634 
00635         if bin and self.need_update(bin, [elf]):
00636             needed_update = True
00637             self.progress("elf2bin", name)
00638             self.binary(r, elf, bin)
00639 
00640         # Initialize memap and process map file. This doesn't generate output.
00641         self.mem_stats(map)
00642 
00643         self.notify.var("compile_succeded", True)
00644         self.notify.var("binary", filename)
00645 
00646         return full_path, needed_update
00647 
00648     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00649     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00650     def default_cmd(self, command):
00651         _stdout, _stderr, _rc = run_cmd(command, work_dir=getcwd(), chroot=self.CHROOT)
00652         self.notify.debug("Return: %s"% _rc)
00653 
00654         for output_line in _stdout.splitlines():
00655             self.notify.debug("Output: %s"% output_line)
00656         for error_line in _stderr.splitlines():
00657             self.notify.debug("Errors: %s"% error_line)
00658 
00659         if _rc != 0:
00660             for line in _stderr.splitlines():
00661                 self.notify.tool_error(line)
00662             raise ToolException(_stderr)
00663 
00664     def progress(self, action, file, build_update=False):
00665         if build_update:
00666             percent = 100. * float(self.compiled) / float(self.to_be_compiled)
00667         else:
00668             percent = None
00669         self.notify.progress(action, file, percent)
00670 
00671     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00672     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00673     def mem_stats(self, map):
00674         """! Creates parser object
00675         @param map Path to linker map file to parse and decode
00676         @return None
00677         """
00678         toolchain = self.__class__.__name__
00679 
00680         # Create memap object
00681         memap = MemapParser()
00682 
00683         # Parse and decode a map file
00684         if memap.parse(abspath(map), toolchain) is False:
00685             self.notify.info("Unknown toolchain for memory statistics %s" % toolchain)
00686             return None
00687 
00688         # Store the memap instance for later use
00689         self.memap_instance = memap
00690 
00691         # Note: memory statistics are not returned.
00692         # Need call to generate_output later (depends on depth & output format)
00693 
00694         return None
00695 
00696     def _add_defines_from_region(self, region, suffixes=['_ADDR', '_SIZE']):
00697         for define in [(region.name.upper() + suffixes[0], region.start),
00698                        (region.name.upper() + suffixes[1], region.size)]:
00699             define_string = "-D%s=0x%x" %  define
00700             self.cc.append(define_string)
00701             self.cppc.append(define_string)
00702             self.flags["common"].append(define_string)
00703 
00704     def _add_all_regions(self, region_list, active_region_name):
00705         for region in region_list:
00706             self._add_defines_from_region(region)
00707             if region.active:
00708                 for define in [
00709                         ("%s_START" % active_region_name, "0x%x" % region.start),
00710                         ("%s_SIZE" % active_region_name, "0x%x" % region.size)
00711                 ]:
00712                     define_string = self.make_ld_define(*define)
00713                     self.ld.append(define_string)
00714                     self.flags["ld"].append(define_string)
00715             self.notify.info("  Region %s: size 0x%x, offset 0x%x"
00716                              % (region.name, region.size, region.start))
00717 
00718     def add_regions(self):
00719         """Add regions to the build profile, if there are any.
00720         """
00721         if self.config.has_regions:
00722             regions = list(self.config.regions)
00723             self.notify.info("Using ROM region%s %s in this build." % (
00724                 "s" if len(regions) > 1 else "",
00725                 ", ".join(r.name for r in regions)
00726             ))
00727             self._add_all_regions(regions, "MBED_APP")
00728         if self.config.has_ram_regions:
00729             regions = list(self.config.ram_regions)
00730             self.notify.info("Using RAM region%s %s in this build." % (
00731                 "s" if len(regions) > 1 else "",
00732                 ", ".join(r.name for r in regions)
00733             ))
00734             self._add_all_regions(regions, "MBED_RAM")
00735         try:
00736             rom_start, rom_size = self.config.rom
00737             Region = namedtuple("Region", "name start size")
00738             self._add_defines_from_region(
00739                 Region("MBED_ROM", rom_start, rom_size),
00740                 suffixes=["_START", "_SIZE"]
00741             )
00742         except ConfigException:
00743             pass
00744 
00745     def add_linker_defines(self):
00746         stack_param = "target.boot-stack-size"
00747         params, _ = self.config_data
00748 
00749         if stack_param in params:
00750             define_string = self.make_ld_define("MBED_BOOT_STACK_SIZE", int(params[stack_param].value, 0))
00751             self.ld.append(define_string)
00752             self.flags["ld"].append(define_string)
00753 
00754     # Set the configuration data
00755     def set_config_data(self, config_data):
00756         self.config_data = config_data
00757         # new configuration data can change labels, so clear the cache
00758         self.labels = None
00759         # pass info about softdevice presence to linker (see NRF52)
00760         if "SOFTDEVICE_PRESENT" in config_data[1]:
00761             define_string = self.make_ld_define("SOFTDEVICE_PRESENT", config_data[1]["SOFTDEVICE_PRESENT"].macro_value)
00762             self.ld.append(define_string)
00763             self.flags["ld"].append(define_string)
00764         self.add_regions()
00765         self.add_linker_defines()
00766 
00767     # Creates the configuration header if needed:
00768     # - if there is no configuration data, "mbed_config.h" is not create (or deleted if it exists).
00769     # - if there is configuration data and "mbed_config.h" does not exist, it is created.
00770     # - if there is configuration data similar to the previous configuration data,
00771     #   "mbed_config.h" is left untouched.
00772     # - if there is new configuration data, "mbed_config.h" is overriden.
00773     # The function needs to be called exactly once for the lifetime of this toolchain instance.
00774     # The "config_processed" variable (below) ensures this behaviour.
00775     # The function returns the location of the configuration file, or None if there is no
00776     # configuration data available (and thus no configuration file)
00777     def get_config_header(self):
00778         if self.config_processed: # this function was already called, return its result
00779             return self.config_file
00780         # The config file is located in the build directory
00781         self.config_file = join(self.build_dir, self.MBED_CONFIG_FILE_NAME)
00782         # If the file exists, read its current content in prev_data
00783         if exists(self.config_file):
00784             with open(self.config_file, "r") as f:
00785                 prev_data = f.read()
00786         else:
00787             prev_data = None
00788         # Get the current configuration data
00789         crt_data = self.config.config_to_header(self.config_data) if self.config_data else None
00790         # "changed" indicates if a configuration change was detected
00791         changed = False
00792         if prev_data is not None: # a previous mbed_config.h exists
00793             if crt_data is None: # no configuration data, so "mbed_config.h" needs to be removed
00794                 remove(self.config_file)
00795                 self.config_file = None # this means "config file not present"
00796                 changed = True
00797             elif crt_data != prev_data: # different content of config file
00798                 with open(self.config_file, "w") as f:
00799                     f.write(crt_data)
00800                 changed = True
00801         else: # a previous mbed_config.h does not exist
00802             if crt_data is not None: # there's configuration data available
00803                 with open(self.config_file, "w") as f:
00804                     f.write(crt_data)
00805                 changed = True
00806             else:
00807                 self.config_file = None # this means "config file not present"
00808         # If there was a change in configuration, rebuild everything
00809         self.build_all = changed
00810         # Make sure that this function will only return the location of the configuration
00811         # file for subsequent calls, without trying to manipulate its content in any way.
00812         self.config_processed = True
00813         return self.config_file
00814 
00815     def dump_build_profile(self):
00816         """Dump the current build profile and macros into the `.profile` file
00817         in the build directory"""
00818         for key in ["cxx", "c", "asm", "ld"]:
00819             to_dump = {
00820                 "flags": sorted(self.flags[key]),
00821                 "macros": sorted(self.macros),
00822                 "symbols": sorted(self.get_symbols(for_asm=(key == "asm"))),
00823             }
00824             if key in ["cxx", "c"]:
00825                 to_dump["symbols"].remove('MBED_BUILD_TIMESTAMP=%s' % self.timestamp)
00826                 to_dump["flags"].extend(sorted(self.flags['common']))
00827             where = join(self.build_dir, self.PROFILE_FILE_NAME + "-" + key)
00828             self._overwrite_when_not_equal(where, json.dumps(
00829                 to_dump, sort_keys=True, indent=4))
00830 
00831     @staticmethod
00832     def _overwrite_when_not_equal(filename, content):
00833         if not exists(filename) or content != open(filename).read():
00834             with open(filename, "w") as out:
00835                 out.write(content)
00836 
00837     @staticmethod
00838     def generic_check_executable(tool_key, executable_name, levels_up,
00839                                  nested_dir=None):
00840         """
00841         Positional args:
00842         tool_key: the key to index TOOLCHAIN_PATHS
00843         executable_name: the toolchain's named executable (ex. armcc)
00844         levels_up: each toolchain joins the toolchain_path, some
00845         variable directories (bin, include), and the executable name,
00846         so the TOOLCHAIN_PATH value must be appropriately distanced
00847 
00848         Keyword args:
00849         nested_dir: the directory within TOOLCHAIN_PATHS where the executable
00850           is found (ex: 'bin' for ARM\bin\armcc (necessary to check for path
00851           that will be used by toolchain's compile)
00852 
00853         Returns True if the executable location specified by the user
00854         exists and is valid OR the executable can be found on the PATH.
00855         Returns False otherwise.
00856         """
00857         # Search PATH if user did not specify a path or specified path doesn't
00858         # exist.
00859         if not TOOLCHAIN_PATHS[tool_key] or not exists(TOOLCHAIN_PATHS[tool_key]):
00860             exe = find_executable(executable_name)
00861             if not exe:
00862                 return False
00863             for level in range(levels_up):
00864                 # move up the specified number of directories
00865                 exe = dirname(exe)
00866             TOOLCHAIN_PATHS[tool_key] = exe
00867         if nested_dir:
00868             subdir = join(TOOLCHAIN_PATHS[tool_key], nested_dir,
00869                           executable_name)
00870         else:
00871             subdir = join(TOOLCHAIN_PATHS[tool_key],executable_name)
00872         # User could have specified a path that exists but does not contain exe
00873         return exists(subdir) or exists(subdir +'.exe')
00874 
00875     @abstractmethod
00876     def check_executable(self):
00877         """Returns True if the executable (armcc) location specified by the
00878          user exists OR the executable can be found on the PATH.
00879          Returns False otherwise."""
00880         raise NotImplemented
00881 
00882     @abstractmethod
00883     def get_config_option(self, config_header):
00884         """Generate the compiler option that forces the inclusion of the configuration
00885         header file.
00886 
00887         Positional arguments:
00888         config_header -- The configuration header that will be included within all source files
00889 
00890         Return value:
00891         A list of the command line arguments that will force the inclusion the specified header
00892 
00893         Side effects:
00894         None
00895         """
00896         raise NotImplemented
00897 
00898     @abstractmethod
00899     def get_compile_options(self, defines, includes, for_asm=False):
00900         """Generate the compiler options from the defines and includes
00901 
00902         Positional arguments:
00903         defines -- The preprocessor macros defined on the command line
00904         includes -- The include file search paths
00905 
00906         Keyword arguments:
00907         for_asm -- generate the assembler options instead of the compiler options
00908 
00909         Return value:
00910         A list of the command line arguments that will force the inclusion the specified header
00911 
00912         Side effects:
00913         None
00914         """
00915         raise NotImplemented
00916 
00917     @abstractmethod
00918     def assemble(self, source, object, includes):
00919         """Generate the command line that assembles.
00920 
00921         Positional arguments:
00922         source -- a file path that is the file to assemble
00923         object -- a file path that is the destination object
00924         includes -- a list of all directories where header files may be found
00925 
00926         Return value:
00927         The complete command line, as a list, that would invoke the assembler
00928         on the source file, include all the include paths, and generate
00929         the specified object file.
00930 
00931         Side effects:
00932         None
00933 
00934         Note:
00935         This method should be decorated with @hook_tool.
00936         """
00937         raise NotImplemented
00938 
00939     @abstractmethod
00940     def compile_c(self, source, object, includes):
00941         """Generate the command line that compiles a C source file.
00942 
00943         Positional arguments:
00944         source -- the C source file to compile
00945         object -- the destination object file
00946         includes -- a list of all the directories where header files may be found
00947 
00948         Return value:
00949         The complete command line, as a list, that would invoke the C compiler
00950         on the source file, include all the include paths, and generate the
00951         specified object file.
00952 
00953         Side effects:
00954         None
00955 
00956         Note:
00957         This method should be decorated with @hook_tool.
00958         """
00959         raise NotImplemented
00960 
00961     @abstractmethod
00962     def compile_cpp(self, source, object, includes):
00963         """Generate the command line that compiles a C++ source file.
00964 
00965         Positional arguments:
00966         source -- the C++ source file to compile
00967         object -- the destination object file
00968         includes -- a list of all the directories where header files may be found
00969 
00970         Return value:
00971         The complete command line, as a list, that would invoke the C++ compiler
00972         on the source file, include all the include paths, and generate the
00973         specified object file.
00974 
00975         Side effects:
00976         None
00977 
00978         Note:
00979         This method should be decorated with @hook_tool.
00980         """
00981         raise NotImplemented
00982 
00983     @abstractmethod
00984     def link(self, output, objects, libraries, lib_dirs, mem_map):
00985         """Run the linker to create an executable and memory map.
00986 
00987         Positional arguments:
00988         output -- the file name to place the executable in
00989         objects -- all of the object files to link
00990         libraries -- all of the required libraries
00991         lib_dirs -- where the required libraries are located
00992         mem_map -- the location where the memory map file should be stored
00993 
00994         Return value:
00995         None
00996 
00997         Side effect:
00998         Runs the linker to produce the executable.
00999 
01000         Note:
01001         This method should be decorated with @hook_tool.
01002         """
01003         raise NotImplemented
01004 
01005     @abstractmethod
01006     def archive(self, objects, lib_path):
01007         """Run the command line that creates an archive.
01008 
01009         Positional arguhments:
01010         objects -- a list of all the object files that should be archived
01011         lib_path -- the file name of the resulting library file
01012 
01013         Return value:
01014         None
01015 
01016         Side effect:
01017         Runs the archiving tool to produce the library file.
01018 
01019         Note:
01020         This method should be decorated with @hook_tool.
01021         """
01022         raise NotImplemented
01023 
01024     @abstractmethod
01025     def binary(self, resources, elf, bin):
01026         """Run the command line that will Extract a simplified binary file.
01027 
01028         Positional arguments:
01029         resources -- A resources object (Is not used in any of the toolchains)
01030         elf -- the executable file that is to be converted
01031         bin -- the file name of the to be created simplified binary file
01032 
01033         Return value:
01034         None
01035 
01036         Side effect:
01037         Runs the elf2bin tool to produce the simplified binary file.
01038 
01039         Note:
01040         This method should be decorated with @hook_tool.
01041         """
01042         raise NotImplemented
01043 
01044     @staticmethod
01045     @abstractmethod
01046     def name_mangle(name):
01047         """Mangle a name based on the conventional name mangling of this toolchain
01048 
01049         Positional arguments:
01050         name -- the name to mangle
01051 
01052         Return:
01053         the mangled name as a string
01054         """
01055         raise NotImplemented
01056 
01057     @staticmethod
01058     @abstractmethod
01059     def make_ld_define(name, value):
01060         """Create an argument to the linker that would define a symbol
01061 
01062         Positional arguments:
01063         name -- the symbol to define
01064         value -- the value to give the symbol
01065 
01066         Return:
01067         The linker flag as a string
01068         """
01069         raise NotImplemented
01070 
01071     @staticmethod
01072     @abstractmethod
01073     def redirect_symbol(source, sync, build_dir):
01074         """Redirect a symbol at link time to point at somewhere else
01075 
01076         Positional arguments:
01077         source -- the symbol doing the pointing
01078         sync -- the symbol being pointed to
01079         build_dir -- the directory to put "response files" if needed by the toolchain
01080 
01081         Side Effects:
01082         Possibly create a file in the build directory
01083 
01084         Return:
01085         The linker flag to redirect the symbol, as a string
01086         """
01087         raise NotImplemented
01088 
01089     # Return the list of macros geenrated by the build system
01090     def get_config_macros(self):
01091         return self.config.config_to_macros(self.config_data) if self.config_data else []
01092 
01093     @abstractmethod
01094     def version_check(self):
01095         """Check the version of a compiler being used and raise a
01096         NotSupportedException when it's incorrect.
01097         """
01098         raise NotImplemented
01099 
01100     @property
01101     def report(self):
01102         to_ret = {}
01103         to_ret['c_compiler'] = {'flags': copy(self.flags['c']),
01104                                 'symbols': self.get_symbols()}
01105         to_ret['cxx_compiler'] = {'flags': copy(self.flags['cxx']),
01106                                   'symbols': self.get_symbols()}
01107         to_ret['assembler'] = {'flags': copy(self.flags['asm']),
01108                                'symbols': self.get_symbols(True)}
01109         to_ret['linker'] = {'flags': copy(self.flags['ld'])}
01110         to_ret.update(self.config.report)
01111         return to_ret
01112 
01113 from tools.settings import ARM_PATH, ARMC6_PATH, GCC_ARM_PATH, IAR_PATH
01114 
01115 TOOLCHAIN_PATHS = {
01116     'ARM': ARM_PATH,
01117     'uARM': ARM_PATH,
01118     'ARMC6': ARMC6_PATH,
01119     'GCC_ARM': GCC_ARM_PATH,
01120     'IAR': IAR_PATH
01121 }
01122 
01123 from tools.toolchains.arm import ARM_STD, ARM_MICRO, ARMC6
01124 from tools.toolchains.gcc import GCC_ARM
01125 from tools.toolchains.iar import IAR
01126 
01127 TOOLCHAIN_CLASSES = {
01128     u'ARM': ARM_STD,
01129     u'uARM': ARM_MICRO,
01130     u'ARMC6': ARMC6,
01131     u'GCC_ARM': GCC_ARM,
01132     u'IAR': IAR
01133 }
01134 
01135 TOOLCHAINS = set(TOOLCHAIN_CLASSES.keys())