__init__.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2011-2013 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009     http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 
00019 import re
00020 import sys
00021 import json
00022 from os import stat, walk, getcwd, sep, remove
00023 from copy import copy
00024 from time import time, sleep
00025 from shutil import copyfile
00026 from os.path import (join, splitext, exists, relpath, dirname, basename, split,
00027                      abspath, isfile, isdir, normcase)
00028 from inspect import getmro
00029 from copy import deepcopy
00030 from collections import namedtuple
00031 from abc import ABCMeta, abstractmethod
00032 from distutils.spawn import find_executable
00033 from multiprocessing import Pool, cpu_count
00034 from hashlib import md5
00035 
00036 from ..utils import (run_cmd, mkdir, rel_path, ToolException,
00037                     NotSupportedException, split_path, compile_worker)
00038 from ..settings import MBED_ORG_USER, PRINT_COMPILER_OUTPUT_AS_LINK
00039 from .. import hooks
00040 from ..notifier.term import TerminalNotifier
00041 from ..resources import FileType
00042 from ..memap import MemapParser
00043 from ..config import ConfigException
00044 
00045 
00046 #Disables multiprocessing if set to higher number than the host machine CPUs
00047 CPU_COUNT_MIN = 1
00048 CPU_COEF = 1
00049 
00050 class mbedToolchain:
00051     # Verbose logging
00052     VERBOSE = True
00053 
00054     # Compile C files as CPP
00055     COMPILE_C_AS_CPP = False
00056 
00057     # Response files for compiling, includes, linking and archiving.
00058     # Not needed on posix systems where the typical arg limit is 2 megabytes
00059     RESPONSE_FILES = True
00060 
00061     CORTEX_SYMBOLS = {
00062         "Cortex-M0" : ["__CORTEX_M0", "ARM_MATH_CM0", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00063         "Cortex-M0+": ["__CORTEX_M0PLUS", "ARM_MATH_CM0PLUS", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00064         "Cortex-M1" : ["__CORTEX_M3", "ARM_MATH_CM1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00065         "Cortex-M3" : ["__CORTEX_M3", "ARM_MATH_CM3", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00066         "Cortex-M4" : ["__CORTEX_M4", "ARM_MATH_CM4", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00067         "Cortex-M4F" : ["__CORTEX_M4", "ARM_MATH_CM4", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00068         "Cortex-M7" : ["__CORTEX_M7", "ARM_MATH_CM7", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00069         "Cortex-M7F" : ["__CORTEX_M7", "ARM_MATH_CM7", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00070         "Cortex-M7FD" : ["__CORTEX_M7", "ARM_MATH_CM7", "__FPU_PRESENT=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00071         "Cortex-A9" : ["__CORTEX_A9", "ARM_MATH_CA9", "__FPU_PRESENT", "__CMSIS_RTOS", "__EVAL", "__MBED_CMSIS_RTOS_CA9"],
00072         "Cortex-M23-NS": ["__CORTEX_M23", "ARM_MATH_ARMV8MBL", "DOMAIN_NS=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00073         "Cortex-M23": ["__CORTEX_M23", "ARM_MATH_ARMV8MBL", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00074         "Cortex-M33-NS": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "DOMAIN_NS=1", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00075         "Cortex-M33": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00076         "Cortex-M33F-NS": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "DOMAIN_NS=1", "__FPU_PRESENT", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00077         "Cortex-M33F": ["__CORTEX_M33", "ARM_MATH_ARMV8MML", "__FPU_PRESENT", "__CMSIS_RTOS", "__MBED_CMSIS_RTOS_CM"],
00078     }
00079 
00080     MBED_CONFIG_FILE_NAME="mbed_config.h"
00081 
00082     PROFILE_FILE_NAME = ".profile"
00083 
00084     __metaclass__ = ABCMeta
00085 
00086     profile_template = {'common':[], 'c':[], 'cxx':[], 'asm':[], 'ld':[]}
00087 
00088     def __init__(self, target, notify=None, macros=None, build_profile=None,
00089                  build_dir=None):
00090         self.target = target
00091         self.name = self.__class__.__name__
00092 
00093         # compile/assemble/link/binary hooks
00094         self.hook = hooks.Hook(target, self)
00095 
00096         # Toolchain flags
00097         self.flags = deepcopy(build_profile or self.profile_template)
00098 
00099         # System libraries provided by the toolchain
00100         self.sys_libs = []
00101 
00102         # User-defined macros
00103         self.macros = macros or []
00104 
00105         # Macros generated from toolchain and target rules/features
00106         self.asm_symbols = None
00107         self.cxx_symbols = None
00108 
00109         # Labels generated from toolchain and target rules/features (used for selective build)
00110         self.labels = None
00111 
00112         # This will hold the initialized config object
00113         self.config = None
00114 
00115         # This will hold the configuration data (as returned by Config.get_config_data())
00116         self.config_data = None
00117 
00118         # This will hold the location of the configuration file or None if there's no configuration available
00119         self.config_file = None
00120 
00121         # Call guard for "get_config_data" (see the comments of get_config_data for details)
00122         self.config_processed = False
00123 
00124         # Non-incremental compile
00125         self.build_all = False
00126 
00127         # Build output dir
00128         self.build_dir = abspath(build_dir) if PRINT_COMPILER_OUTPUT_AS_LINK else build_dir
00129         self.timestamp = time()
00130 
00131         # Number of concurrent build jobs. 0 means auto (based on host system cores)
00132         self.jobs = 0
00133 
00134 
00135         # Output notify function
00136         # This function is passed all events, and expected to handle notification of the
00137         # user, emit the events to a log, etc.
00138         # The API for all notify methods passed into the notify parameter is as follows:
00139         # def notify(Event, Silent)
00140         # Where *Event* is a dict representing the toolchain event that was generated
00141         #            e.g.: a compile succeeded, or a warning was emitted by the compiler
00142         #                  or an application was linked
00143         #       *Silent* is a boolean
00144         if notify:
00145             self.notify = notify
00146         else:
00147             self.notify = TerminalNotifier()
00148 
00149 
00150         # Stats cache is used to reduce the amount of IO requests to stat
00151         # header files during dependency change. See need_update()
00152         self.stat_cache = {}
00153 
00154         # Used by the mbed Online Build System to build in chrooted environment
00155         self.CHROOT = None
00156 
00157         # Call post __init__() hooks before the ARM/GCC_ARM/IAR toolchain __init__() takes over
00158         self.init()
00159 
00160     # Used for post __init__() hooks
00161     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00162     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00163     def init(self):
00164         return True
00165 
00166     def get_output(self):
00167         return self.notifier.get_output()
00168 
00169     def get_symbols(self, for_asm=False):
00170         if for_asm:
00171             if self.asm_symbols is None:
00172                 self.asm_symbols = []
00173 
00174                 # Cortex CPU symbols
00175                 if self.target.core in mbedToolchain.CORTEX_SYMBOLS:
00176                     self.asm_symbols.extend(mbedToolchain.CORTEX_SYMBOLS[self.target.core])
00177 
00178                 # Add target's symbols
00179                 self.asm_symbols += self.target.macros
00180                 # Add extra symbols passed via 'macros' parameter
00181                 self.asm_symbols += self.macros
00182             return list(set(self.asm_symbols))  # Return only unique symbols
00183         else:
00184             if self.cxx_symbols is None:
00185                 # Target and Toolchain symbols
00186                 labels = self.get_labels()
00187                 self.cxx_symbols = ["TARGET_%s" % t for t in labels['TARGET']]
00188                 self.cxx_symbols.extend(["TOOLCHAIN_%s" % t for t in labels['TOOLCHAIN']])
00189 
00190                 # Cortex CPU symbols
00191                 if self.target.core in mbedToolchain.CORTEX_SYMBOLS:
00192                     self.cxx_symbols.extend(mbedToolchain.CORTEX_SYMBOLS[self.target.core])
00193 
00194                 # Symbols defined by the on-line build.system
00195                 self.cxx_symbols.extend(['MBED_BUILD_TIMESTAMP=%s' % self.timestamp, 'TARGET_LIKE_MBED', '__MBED__=1'])
00196                 if MBED_ORG_USER:
00197                     self.cxx_symbols.append('MBED_USERNAME=' + MBED_ORG_USER)
00198 
00199                 # Add target's symbols
00200                 self.cxx_symbols += self.target.macros
00201                 # Add target's hardware
00202                 self.cxx_symbols += ["DEVICE_" + data + "=1" for data in self.target.device_has]
00203                 # Add target's features
00204                 self.cxx_symbols += ["FEATURE_" + data + "=1" for data in self.target.features]
00205                 # Add target's components
00206                 self.cxx_symbols += ["COMPONENT_" + data + "=1" for data in self.target.components]
00207                 # Add extra symbols passed via 'macros' parameter
00208                 self.cxx_symbols += self.macros
00209 
00210                 # Form factor variables
00211                 if hasattr(self.target, 'supported_form_factors'):
00212                     self.cxx_symbols.extend(["TARGET_FF_%s" % t for t in self.target.supported_form_factors])
00213 
00214             return list(set(self.cxx_symbols))  # Return only unique symbols
00215 
00216     # Extend the internal list of macros
00217     def add_macros(self, new_macros):
00218         self.macros.extend(new_macros)
00219 
00220     def get_labels(self):
00221         if self.labels is None:
00222             toolchain_labels = self._get_toolchain_labels()
00223             self.labels = {
00224                 'TARGET': self.target.labels,
00225                 'FEATURE': self.target.features,
00226                 'COMPONENT': self.target.components,
00227                 'TOOLCHAIN': toolchain_labels
00228             }
00229 
00230             # This is a policy decision and it should /really/ be in the config system
00231             # ATM it's here for backward compatibility
00232             if ((("-g" in self.flags['common'] or "-g3" in self.flags['common']) and
00233                  "-O0" in self.flags['common']) or
00234                 ("-r" in self.flags['common'] and
00235                  "-On" in self.flags['common'])):
00236                 self.labels['TARGET'].append("DEBUG")
00237             else:
00238                 self.labels['TARGET'].append("RELEASE")
00239         return self.labels
00240 
00241     def _get_toolchain_labels(self):
00242         toolchain_labels = [c.__name__ for c in getmro(self.__class__)]
00243         toolchain_labels.remove('mbedToolchain')
00244         toolchain_labels.remove('object')
00245         return toolchain_labels
00246 
00247 
00248     # Determine whether a source file needs updating/compiling
00249     def need_update(self, target, dependencies):
00250         if self.build_all:
00251             return True
00252 
00253         if not exists(target):
00254             return True
00255 
00256         target_mod_time = stat(target).st_mtime
00257 
00258         for d in dependencies:
00259             # Some objects are not provided with full path and here we do not have
00260             # information about the library paths. Safe option: assume an update
00261             if not d or not exists(d):
00262                 return True
00263 
00264             if d not in self.stat_cache:
00265                 self.stat_cache[d] = stat(d).st_mtime
00266 
00267             if self.stat_cache[d] >= target_mod_time:
00268                 return True
00269 
00270         return False
00271 
00272 
00273     def scan_repository(self, path):
00274         resources = []
00275 
00276         for root, dirs, files in walk(path):
00277             # Remove ignored directories
00278             for d in copy(dirs):
00279                 if d == '.' or d == '..':
00280                     dirs.remove(d)
00281 
00282             for file in files:
00283                 file_path = join(root, file)
00284                 resources.append(file_path)
00285 
00286         return resources
00287 
00288     def copy_files(self, files_paths, trg_path, resources=None):
00289         # Handle a single file
00290         if not isinstance(files_paths, list):
00291             files_paths = [files_paths]
00292 
00293         for dest, source in files_paths:
00294             target = join(trg_path, dest)
00295             if (target != source) and (self.need_update(target, [source])):
00296                 self.progress("copy", dest)
00297                 mkdir(dirname(target))
00298                 copyfile(source, target)
00299 
00300     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00301     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00302     def relative_object_path(self, build_path, file_ref):
00303         source_dir, name, _ = split_path(file_ref.name)
00304 
00305         obj_dir = relpath(join(build_path, source_dir))
00306         if obj_dir is not self.prev_dir:
00307             self.prev_dir = obj_dir
00308             mkdir(obj_dir)
00309         return join(obj_dir, name + '.o')
00310 
00311     def make_option_file(self, options, naming=".options_{}.txt"):
00312         """ Generate a via file for a pile of defines
00313         ARM, GCC, IAR cross compatible
00314         """
00315         to_write = " ".join(options).encode('utf-8')
00316         new_md5 = md5(to_write).hexdigest()
00317         via_file = join(self.build_dir, naming.format(new_md5))
00318         try:
00319             with open(via_file, "r") as fd:
00320                 old_md5 = md5(fd.read().encode('utf-8')).hexdigest()
00321         except IOError:
00322             old_md5 = None
00323         if old_md5 != new_md5:
00324             with open(via_file, "wb") as fd:
00325                 fd.write(to_write)
00326         return via_file
00327 
00328     def get_inc_file(self, includes):
00329         """Generate a via file for all includes.
00330         ARM, GCC, IAR cross compatible
00331         """
00332         cmd_list = ("-I{}".format(c.replace("\\", "/")) for c in includes if c)
00333         if self.CHROOT:
00334             cmd_list = (c.replace(self.CHROOT, '') for c in cmd_list)
00335         return self.make_option_file(list(cmd_list), naming=".includes_{}.txt")
00336 
00337     def get_link_file(self, cmd):
00338         """Generate a via file for all objects when linking.
00339         ARM, GCC, IAR cross compatible
00340         """
00341         cmd_list = (c.replace("\\", "/") for c in cmd if c)
00342         if self.CHROOT:
00343             cmd_list = (c.replace(self.CHROOT, '') for c in cmd_list)
00344         return self.make_option_file(list(cmd_list), naming=".link_options.txt")
00345 
00346     def get_arch_file(self, objects):
00347         """ Generate a via file for all objects when archiving.
00348         ARM, GCC, IAR cross compatible
00349         """
00350         cmd_list = (c.replace("\\", "/") for c in objects if c)
00351         return self.make_option_file(list(cmd_list), ".archive_files.txt")
00352 
00353     # THIS METHOD IS BEING CALLED BY THE MBED ONLINE BUILD SYSTEM
00354     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00355     def compile_sources(self, resources, inc_dirs=None):
00356         # Web IDE progress bar for project build
00357         files_to_compile = (
00358             resources.get_file_refs(FileType.ASM_SRC) +
00359             resources.get_file_refs(FileType.C_SRC) +
00360             resources.get_file_refs(FileType.CPP_SRC)
00361         )
00362         self.to_be_compiled = len(files_to_compile)
00363         self.compiled = 0
00364 
00365         self.notify.cc_verbose("Macros: "+' '.join(['-D%s' % s for s in self.get_symbols()]))
00366 
00367         inc_paths = resources.get_file_paths(FileType.INC_DIR)
00368         if inc_dirs is not None:
00369             if isinstance(inc_dirs, list):
00370                 inc_paths.extend(inc_dirs)
00371             else:
00372                 inc_paths.append(inc_dirs)
00373         # De-duplicate include paths
00374         inc_paths = set(inc_paths)
00375         # Sort include paths for consistency
00376         inc_paths = sorted(set(inc_paths))
00377         # Unique id of all include paths
00378         self.inc_md5 = md5(' '.join(inc_paths).encode('utf-8')).hexdigest()
00379 
00380         objects = []
00381         queue = []
00382         work_dir = getcwd()
00383         self.prev_dir = None
00384 
00385         # Generate configuration header (this will update self.build_all if needed)
00386         self.get_config_header()
00387         self.dump_build_profile()
00388 
00389         # Sort compile queue for consistency
00390         files_to_compile.sort()
00391         for source in files_to_compile:
00392             object = self.relative_object_path(self.build_dir, source)
00393 
00394             # Queue mode (multiprocessing)
00395             commands = self.compile_command(source.path, object, inc_paths)
00396             if commands is not None:
00397                 queue.append({
00398                     'source': source,
00399                     'object': object,
00400                     'commands': commands,
00401                     'work_dir': work_dir,
00402                     'chroot': self.CHROOT
00403                 })
00404             else:
00405                 self.compiled += 1
00406                 objects.append(object)
00407 
00408         # Use queues/multiprocessing if cpu count is higher than setting
00409         jobs = self.jobs if self.jobs else cpu_count()
00410         if jobs > CPU_COUNT_MIN and len(queue) > jobs:
00411             return self.compile_queue(queue, objects)
00412         else:
00413             return self.compile_seq(queue, objects)
00414 
00415     # Compile source files queue in sequential order
00416     def compile_seq(self, queue, objects):
00417         for item in queue:
00418             result = compile_worker(item)
00419 
00420             self.compiled += 1
00421             self.progress("compile", item['source'].name, build_update=True)
00422             for res in result['results']:
00423                 self.notify.cc_verbose("Compile: %s" % ' '.join(res['command']), result['source'])
00424                 self.compile_output([
00425                     res['code'],
00426                     res['output'],
00427                     res['command']
00428                 ])
00429             objects.append(result['object'])
00430         return objects
00431 
00432     # Compile source files queue in parallel by creating pool of worker threads
00433     def compile_queue(self, queue, objects):
00434         jobs_count = int(self.jobs if self.jobs else cpu_count() * CPU_COEF)
00435         p = Pool(processes=jobs_count)
00436 
00437         results = []
00438         for i in range(len(queue)):
00439             results.append(p.apply_async(compile_worker, [queue[i]]))
00440         p.close()
00441 
00442         itr = 0
00443         while len(results):
00444             itr += 1
00445             if itr > 180000:
00446                 p.terminate()
00447                 p.join()
00448                 raise ToolException("Compile did not finish in 5 minutes")
00449 
00450             sleep(0.01)
00451             pending = 0
00452             for r in results:
00453                 if r.ready():
00454                     try:
00455                         result = r.get()
00456                         results.remove(r)
00457 
00458                         self.compiled += 1
00459                         self.progress("compile", result['source'].name, build_update=True)
00460                         for res in result['results']:
00461                             self.notify.cc_verbose("Compile: %s" % ' '.join(res['command']), result['source'])
00462                             self.compile_output([
00463                                 res['code'],
00464                                 res['output'],
00465                                 res['command']
00466                             ])
00467                         objects.append(result['object'])
00468                     except ToolException as err:
00469                         if p._taskqueue.queue:
00470                             p._taskqueue.queue.clear()
00471                             sleep(0.5)
00472                         p.terminate()
00473                         p.join()
00474                         raise ToolException(err)
00475                 else:
00476                     pending += 1
00477                     if pending >= jobs_count:
00478                         break
00479 
00480         results = None
00481         p.join()
00482 
00483         return objects
00484 
00485     # Determine the compile command based on type of source file
00486     def compile_command(self, source, object, includes):
00487         # Check dependencies
00488         _, ext = splitext(source)
00489         ext = ext.lower()
00490 
00491         source = abspath(source) if PRINT_COMPILER_OUTPUT_AS_LINK else source
00492 
00493         if ext == '.c' or  ext == '.cpp' or ext == '.cc':
00494             base, _ = splitext(object)
00495             dep_path = base + '.d'
00496             try:
00497                 deps = self.parse_dependencies(dep_path) if (exists(dep_path)) else []
00498             except (IOError, IndexError):
00499                 deps = []
00500             config_file = ([self.config.app_config_location]
00501                            if self.config.app_config_location else [])
00502             deps.extend(config_file)
00503             if ext != '.c' or self.COMPILE_C_AS_CPP:
00504                 deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-cxx"))
00505             else:
00506                 deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-c"))
00507             if len(deps) == 0 or self.need_update(object, deps):
00508                 if ext != '.c' or self.COMPILE_C_AS_CPP:
00509                     return self.compile_cpp(source, object, includes)
00510                 else:
00511                     return self.compile_c(source, object, includes)
00512         elif ext == '.s':
00513             deps = [source]
00514             deps.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-asm"))
00515             if self.need_update(object, deps):
00516                 return self.assemble(source, object, includes)
00517         else:
00518             return False
00519 
00520         return None
00521 
00522     def parse_dependencies(self, dep_path):
00523         """Parse the dependency information generated by the compiler.
00524 
00525         Positional arguments:
00526         dep_path -- the path to a file generated by a previous run of the compiler
00527 
00528         Return value:
00529         A list of all source files that the dependency file indicated were dependencies
00530 
00531         Side effects:
00532         None
00533 
00534         Note: A default implementation is provided for make-like file formats
00535         """
00536         dependencies = []
00537         buff = open(dep_path).readlines()
00538         if buff:
00539             buff[0] = re.sub('^(.*?)\: ', '', buff[0])
00540             for line in buff:
00541                 filename = line.replace('\\\n', '').strip()
00542                 if filename:
00543                     filename = filename.replace('\\ ', '\a')
00544                     dependencies.extend(((self.CHROOT if self.CHROOT else '') +
00545                                          f.replace('\a', ' '))
00546                                         for f in filename.split(" "))
00547         return list(filter(None, dependencies))
00548 
00549     def is_not_supported_error(self, output):
00550         return "#error directive: [NOT_SUPPORTED]" in output
00551 
00552     @abstractmethod
00553     def parse_output(self, output):
00554         """Take in compiler output and extract sinlge line warnings and errors from it.
00555 
00556         Positional arguments:
00557         output -- a string of all the messages emitted by a run of the compiler
00558 
00559         Return value:
00560         None
00561 
00562         Side effects:
00563         call self.cc_info or self.notify with a description of the event generated by the compiler
00564         """
00565         raise NotImplemented
00566 
00567     def compile_output(self, output=[]):
00568         _rc = output[0]
00569         _stderr = output[1].decode("utf-8")
00570         command = output[2]
00571 
00572         # Parse output for Warnings and Errors
00573         self.parse_output(_stderr)
00574         self.notify.debug("Return: %s"% _rc)
00575         for error_line in _stderr.splitlines():
00576             self.notify.debug("Output: %s"% error_line)
00577 
00578         # Check return code
00579         if _rc != 0:
00580             if self.is_not_supported_error(_stderr):
00581                 raise NotSupportedException(_stderr)
00582             else:
00583                 raise ToolException(_stderr)
00584 
00585     def build_library(self, objects, dir, name):
00586         needed_update = False
00587         lib = self.STD_LIB_NAME % name
00588         fout = join(dir, lib)
00589         if self.need_update(fout, objects):
00590             self.notify.info("Library: %s" % lib)
00591             self.archive(objects, fout)
00592             needed_update = True
00593 
00594         return needed_update
00595 
00596     def link_program(self, r, tmp_path, name):
00597         needed_update = False
00598         ext = 'bin'
00599         if hasattr(self.target, 'OUTPUT_EXT'):
00600             ext = self.target.OUTPUT_EXT
00601 
00602         if hasattr(self.target, 'OUTPUT_NAMING'):
00603             self.notify.var("binary_naming", self.target.OUTPUT_NAMING)
00604             if self.target.OUTPUT_NAMING == "8.3":
00605                 name = name[0:8]
00606                 ext = ext[0:3]
00607 
00608         # Create destination directory
00609         head, tail =  split(name)
00610         new_path = join(tmp_path, head)
00611         mkdir(new_path)
00612 
00613         filename = name+'.'+ext
00614         # Absolute path of the final linked file
00615         full_path = join(tmp_path, filename)
00616         elf = join(tmp_path, name + '.elf')
00617         bin = None if ext == 'elf' else full_path
00618         map = join(tmp_path, name + '.map')
00619 
00620         objects = sorted(set(r.get_file_paths(FileType.OBJECT)))
00621         config_file = ([self.config.app_config_location]
00622                        if self.config.app_config_location else [])
00623         linker_script = [path for _, path in r.get_file_refs(FileType.LD_SCRIPT)
00624                          if path.endswith(self.LINKER_EXT)][-1]
00625         lib_dirs = r.get_file_paths(FileType.LIB_DIR)
00626         libraries = [l for l in r.get_file_paths(FileType.LIB)
00627                      if l.endswith(self.LIBRARY_EXT)]
00628         dependencies = objects + libraries + [linker_script] + config_file
00629         dependencies.append(join(self.build_dir, self.PROFILE_FILE_NAME + "-ld"))
00630         if self.need_update(elf, dependencies):
00631             needed_update = True
00632             self.progress("link", name)
00633             self.link(elf, objects, libraries, lib_dirs, linker_script)
00634 
00635         if bin and self.need_update(bin, [elf]):
00636             needed_update = True
00637             self.progress("elf2bin", name)
00638             self.binary(r, elf, bin)
00639 
00640         # Initialize memap and process map file. This doesn't generate output.
00641         self.mem_stats(map)
00642 
00643         self.notify.var("compile_succeded", True)
00644         self.notify.var("binary", filename)
00645 
00646         return full_path, needed_update
00647 
00648     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00649     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00650     def default_cmd(self, command):
00651         _stdout, _stderr, _rc = run_cmd(command, work_dir=getcwd(), chroot=self.CHROOT)
00652         self.notify.debug("Return: %s"% _rc)
00653 
00654         for output_line in _stdout.splitlines():
00655             self.notify.debug("Output: %s"% output_line)
00656         for error_line in _stderr.splitlines():
00657             self.notify.debug("Errors: %s"% error_line)
00658 
00659         if _rc != 0:
00660             for line in _stderr.splitlines():
00661                 self.notify.tool_error(line)
00662             raise ToolException(_stderr)
00663 
00664     def progress(self, action, file, build_update=False):
00665         if build_update:
00666             percent = 100. * float(self.compiled) / float(self.to_be_compiled)
00667         else:
00668             percent = None
00669         self.notify.progress(action, file, percent)
00670 
00671     # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
00672     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
00673     def mem_stats(self, map):
00674         """! Creates parser object
00675         @param map Path to linker map file to parse and decode
00676         @return None
00677         """
00678         toolchain = self.__class__.__name__
00679 
00680         # Create memap object
00681         memap = MemapParser()
00682 
00683         # Parse and decode a map file
00684         if memap.parse(abspath(map), toolchain) is False:
00685             self.notify.info("Unknown toolchain for memory statistics %s" % toolchain)
00686             return None
00687 
00688         # Store the memap instance for later use
00689         self.memap_instance = memap
00690 
00691         # Note: memory statistics are not returned.
00692         # Need call to generate_output later (depends on depth & output format)
00693 
00694         return None
00695 
00696     def _add_defines_from_region(self, region, suffixes=['_ADDR', '_SIZE']):
00697         for define in [(region.name.upper() + suffixes[0], region.start),
00698                        (region.name.upper() + suffixes[1], region.size)]:
00699             define_string = "-D%s=0x%x" %  define
00700             self.cc.append(define_string)
00701             self.cppc.append(define_string)
00702             self.flags["common"].append(define_string)
00703 
00704     def _add_all_regions(self, region_list, active_region_name):
00705         for region in region_list:
00706             self._add_defines_from_region(region)
00707             if region.active:
00708                 for define in [
00709                         ("%s_START" % active_region_name, "0x%x" % region.start),
00710                         ("%s_SIZE" % active_region_name, "0x%x" % region.size)
00711                 ]:
00712                     define_string = self.make_ld_define(*define)
00713                     self.ld.append(define_string)
00714                     self.flags["ld"].append(define_string)
00715             self.notify.info("  Region %s: size 0x%x, offset 0x%x"
00716                              % (region.name, region.size, region.start))
00717 
00718     def add_regions(self):
00719         """Add regions to the build profile, if there are any.
00720         """
00721         if self.config.has_regions:
00722             regions = list(self.config.regions)
00723             self.notify.info("Using ROM region%s %s in this build." % (
00724                 "s" if len(regions) > 1 else "",
00725                 ", ".join(r.name for r in regions)
00726             ))
00727             self._add_all_regions(regions, "MBED_APP")
00728         if self.config.has_ram_regions:
00729             regions = list(self.config.ram_regions)
00730             self.notify.info("Using RAM region%s %s in this build." % (
00731                 "s" if len(regions) > 1 else "",
00732                 ", ".join(r.name for r in regions)
00733             ))
00734             self._add_all_regions(regions, "MBED_RAM")
00735         try:
00736             rom_start, rom_size = self.config.rom
00737             Region = namedtuple("Region", "name start size")
00738             self._add_defines_from_region(
00739                 Region("MBED_ROM", rom_start, rom_size),
00740                 suffixes=["_START", "_SIZE"]
00741             )
00742         except ConfigException:
00743             pass
00744 
00745     # Set the configuration data
00746     def set_config_data(self, config_data):
00747         self.config_data = config_data
00748         # new configuration data can change labels, so clear the cache
00749         self.labels = None
00750         # pass info about softdevice presence to linker (see NRF52)
00751         if "SOFTDEVICE_PRESENT" in config_data[1]:
00752             define_string = self.make_ld_define("SOFTDEVICE_PRESENT", config_data[1]["SOFTDEVICE_PRESENT"].macro_value)
00753             self.ld.append(define_string)
00754             self.flags["ld"].append(define_string)
00755         self.add_regions()
00756 
00757     # Creates the configuration header if needed:
00758     # - if there is no configuration data, "mbed_config.h" is not create (or deleted if it exists).
00759     # - if there is configuration data and "mbed_config.h" does not exist, it is created.
00760     # - if there is configuration data similar to the previous configuration data,
00761     #   "mbed_config.h" is left untouched.
00762     # - if there is new configuration data, "mbed_config.h" is overriden.
00763     # The function needs to be called exactly once for the lifetime of this toolchain instance.
00764     # The "config_processed" variable (below) ensures this behaviour.
00765     # The function returns the location of the configuration file, or None if there is no
00766     # configuration data available (and thus no configuration file)
00767     def get_config_header(self):
00768         if self.config_processed: # this function was already called, return its result
00769             return self.config_file
00770         # The config file is located in the build directory
00771         self.config_file = join(self.build_dir, self.MBED_CONFIG_FILE_NAME)
00772         # If the file exists, read its current content in prev_data
00773         if exists(self.config_file):
00774             with open(self.config_file, "r") as f:
00775                 prev_data = f.read()
00776         else:
00777             prev_data = None
00778         # Get the current configuration data
00779         crt_data = self.config.config_to_header(self.config_data) if self.config_data else None
00780         # "changed" indicates if a configuration change was detected
00781         changed = False
00782         if prev_data is not None: # a previous mbed_config.h exists
00783             if crt_data is None: # no configuration data, so "mbed_config.h" needs to be removed
00784                 remove(self.config_file)
00785                 self.config_file = None # this means "config file not present"
00786                 changed = True
00787             elif crt_data != prev_data: # different content of config file
00788                 with open(self.config_file, "w") as f:
00789                     f.write(crt_data)
00790                 changed = True
00791         else: # a previous mbed_config.h does not exist
00792             if crt_data is not None: # there's configuration data available
00793                 with open(self.config_file, "w") as f:
00794                     f.write(crt_data)
00795                 changed = True
00796             else:
00797                 self.config_file = None # this means "config file not present"
00798         # If there was a change in configuration, rebuild everything
00799         self.build_all = changed
00800         # Make sure that this function will only return the location of the configuration
00801         # file for subsequent calls, without trying to manipulate its content in any way.
00802         self.config_processed = True
00803         return self.config_file
00804 
00805     def dump_build_profile(self):
00806         """Dump the current build profile and macros into the `.profile` file
00807         in the build directory"""
00808         for key in ["cxx", "c", "asm", "ld"]:
00809             to_dump = {
00810                 "flags": sorted(self.flags[key]),
00811                 "macros": sorted(self.macros),
00812                 "symbols": sorted(self.get_symbols(for_asm=(key == "asm"))),
00813             }
00814             if key in ["cxx", "c"]:
00815                 to_dump["symbols"].remove('MBED_BUILD_TIMESTAMP=%s' % self.timestamp)
00816                 to_dump["flags"].extend(sorted(self.flags['common']))
00817             where = join(self.build_dir, self.PROFILE_FILE_NAME + "-" + key)
00818             self._overwrite_when_not_equal(where, json.dumps(
00819                 to_dump, sort_keys=True, indent=4))
00820 
00821     @staticmethod
00822     def _overwrite_when_not_equal(filename, content):
00823         if not exists(filename) or content != open(filename).read():
00824             with open(filename, "w") as out:
00825                 out.write(content)
00826 
00827     @staticmethod
00828     def generic_check_executable(tool_key, executable_name, levels_up,
00829                                  nested_dir=None):
00830         """
00831         Positional args:
00832         tool_key: the key to index TOOLCHAIN_PATHS
00833         executable_name: the toolchain's named executable (ex. armcc)
00834         levels_up: each toolchain joins the toolchain_path, some
00835         variable directories (bin, include), and the executable name,
00836         so the TOOLCHAIN_PATH value must be appropriately distanced
00837 
00838         Keyword args:
00839         nested_dir: the directory within TOOLCHAIN_PATHS where the executable
00840           is found (ex: 'bin' for ARM\bin\armcc (necessary to check for path
00841           that will be used by toolchain's compile)
00842 
00843         Returns True if the executable location specified by the user
00844         exists and is valid OR the executable can be found on the PATH.
00845         Returns False otherwise.
00846         """
00847         # Search PATH if user did not specify a path or specified path doesn't
00848         # exist.
00849         if not TOOLCHAIN_PATHS[tool_key] or not exists(TOOLCHAIN_PATHS[tool_key]):
00850             exe = find_executable(executable_name)
00851             if not exe:
00852                 return False
00853             for level in range(levels_up):
00854                 # move up the specified number of directories
00855                 exe = dirname(exe)
00856             TOOLCHAIN_PATHS[tool_key] = exe
00857         if nested_dir:
00858             subdir = join(TOOLCHAIN_PATHS[tool_key], nested_dir,
00859                           executable_name)
00860         else:
00861             subdir = join(TOOLCHAIN_PATHS[tool_key],executable_name)
00862         # User could have specified a path that exists but does not contain exe
00863         return exists(subdir) or exists(subdir +'.exe')
00864 
00865     @abstractmethod
00866     def check_executable(self):
00867         """Returns True if the executable (armcc) location specified by the
00868          user exists OR the executable can be found on the PATH.
00869          Returns False otherwise."""
00870         raise NotImplemented
00871 
00872     @abstractmethod
00873     def get_config_option(self, config_header):
00874         """Generate the compiler option that forces the inclusion of the configuration
00875         header file.
00876 
00877         Positional arguments:
00878         config_header -- The configuration header that will be included within all source files
00879 
00880         Return value:
00881         A list of the command line arguments that will force the inclusion the specified header
00882 
00883         Side effects:
00884         None
00885         """
00886         raise NotImplemented
00887 
00888     @abstractmethod
00889     def get_compile_options(self, defines, includes, for_asm=False):
00890         """Generate the compiler options from the defines and includes
00891 
00892         Positional arguments:
00893         defines -- The preprocessor macros defined on the command line
00894         includes -- The include file search paths
00895 
00896         Keyword arguments:
00897         for_asm -- generate the assembler options instead of the compiler options
00898 
00899         Return value:
00900         A list of the command line arguments that will force the inclusion the specified header
00901 
00902         Side effects:
00903         None
00904         """
00905         raise NotImplemented
00906 
00907     @abstractmethod
00908     def assemble(self, source, object, includes):
00909         """Generate the command line that assembles.
00910 
00911         Positional arguments:
00912         source -- a file path that is the file to assemble
00913         object -- a file path that is the destination object
00914         includes -- a list of all directories where header files may be found
00915 
00916         Return value:
00917         The complete command line, as a list, that would invoke the assembler
00918         on the source file, include all the include paths, and generate
00919         the specified object file.
00920 
00921         Side effects:
00922         None
00923 
00924         Note:
00925         This method should be decorated with @hook_tool.
00926         """
00927         raise NotImplemented
00928 
00929     @abstractmethod
00930     def compile_c(self, source, object, includes):
00931         """Generate the command line that compiles a C source file.
00932 
00933         Positional arguments:
00934         source -- the C source file to compile
00935         object -- the destination object file
00936         includes -- a list of all the directories where header files may be found
00937 
00938         Return value:
00939         The complete command line, as a list, that would invoke the C compiler
00940         on the source file, include all the include paths, and generate the
00941         specified object file.
00942 
00943         Side effects:
00944         None
00945 
00946         Note:
00947         This method should be decorated with @hook_tool.
00948         """
00949         raise NotImplemented
00950 
00951     @abstractmethod
00952     def compile_cpp(self, source, object, includes):
00953         """Generate the command line that compiles a C++ source file.
00954 
00955         Positional arguments:
00956         source -- the C++ source file to compile
00957         object -- the destination object file
00958         includes -- a list of all the directories where header files may be found
00959 
00960         Return value:
00961         The complete command line, as a list, that would invoke the C++ compiler
00962         on the source file, include all the include paths, and generate the
00963         specified object file.
00964 
00965         Side effects:
00966         None
00967 
00968         Note:
00969         This method should be decorated with @hook_tool.
00970         """
00971         raise NotImplemented
00972 
00973     @abstractmethod
00974     def link(self, output, objects, libraries, lib_dirs, mem_map):
00975         """Run the linker to create an executable and memory map.
00976 
00977         Positional arguments:
00978         output -- the file name to place the executable in
00979         objects -- all of the object files to link
00980         libraries -- all of the required libraries
00981         lib_dirs -- where the required libraries are located
00982         mem_map -- the location where the memory map file should be stored
00983 
00984         Return value:
00985         None
00986 
00987         Side effect:
00988         Runs the linker to produce the executable.
00989 
00990         Note:
00991         This method should be decorated with @hook_tool.
00992         """
00993         raise NotImplemented
00994 
00995     @abstractmethod
00996     def archive(self, objects, lib_path):
00997         """Run the command line that creates an archive.
00998 
00999         Positional arguhments:
01000         objects -- a list of all the object files that should be archived
01001         lib_path -- the file name of the resulting library file
01002 
01003         Return value:
01004         None
01005 
01006         Side effect:
01007         Runs the archiving tool to produce the library file.
01008 
01009         Note:
01010         This method should be decorated with @hook_tool.
01011         """
01012         raise NotImplemented
01013 
01014     @abstractmethod
01015     def binary(self, resources, elf, bin):
01016         """Run the command line that will Extract a simplified binary file.
01017 
01018         Positional arguments:
01019         resources -- A resources object (Is not used in any of the toolchains)
01020         elf -- the executable file that is to be converted
01021         bin -- the file name of the to be created simplified binary file
01022 
01023         Return value:
01024         None
01025 
01026         Side effect:
01027         Runs the elf2bin tool to produce the simplified binary file.
01028 
01029         Note:
01030         This method should be decorated with @hook_tool.
01031         """
01032         raise NotImplemented
01033 
01034     @staticmethod
01035     @abstractmethod
01036     def name_mangle(name):
01037         """Mangle a name based on the conventional name mangling of this toolchain
01038 
01039         Positional arguments:
01040         name -- the name to mangle
01041 
01042         Return:
01043         the mangled name as a string
01044         """
01045         raise NotImplemented
01046 
01047     @staticmethod
01048     @abstractmethod
01049     def make_ld_define(name, value):
01050         """Create an argument to the linker that would define a symbol
01051 
01052         Positional arguments:
01053         name -- the symbol to define
01054         value -- the value to give the symbol
01055 
01056         Return:
01057         The linker flag as a string
01058         """
01059         raise NotImplemented
01060 
01061     @staticmethod
01062     @abstractmethod
01063     def redirect_symbol(source, sync, build_dir):
01064         """Redirect a symbol at link time to point at somewhere else
01065 
01066         Positional arguments:
01067         source -- the symbol doing the pointing
01068         sync -- the symbol being pointed to
01069         build_dir -- the directory to put "response files" if needed by the toolchain
01070 
01071         Side Effects:
01072         Possibly create a file in the build directory
01073 
01074         Return:
01075         The linker flag to redirect the symbol, as a string
01076         """
01077         raise NotImplemented
01078 
01079     # Return the list of macros geenrated by the build system
01080     def get_config_macros(self):
01081         return self.config.config_to_macros(self.config_data) if self.config_data else []
01082 
01083     @abstractmethod
01084     def version_check(self):
01085         """Check the version of a compiler being used and raise a
01086         NotSupportedException when it's incorrect.
01087         """
01088         raise NotImplemented
01089 
01090     @property
01091     def report(self):
01092         to_ret = {}
01093         to_ret['c_compiler'] = {'flags': copy(self.flags['c']),
01094                                 'symbols': self.get_symbols()}
01095         to_ret['cxx_compiler'] = {'flags': copy(self.flags['cxx']),
01096                                   'symbols': self.get_symbols()}
01097         to_ret['assembler'] = {'flags': copy(self.flags['asm']),
01098                                'symbols': self.get_symbols(True)}
01099         to_ret['linker'] = {'flags': copy(self.flags['ld'])}
01100         to_ret.update(self.config.report)
01101         return to_ret
01102 
01103 from tools.settings import ARM_PATH, ARMC6_PATH, GCC_ARM_PATH, IAR_PATH
01104 
01105 TOOLCHAIN_PATHS = {
01106     'ARM': ARM_PATH,
01107     'uARM': ARM_PATH,
01108     'ARMC6': ARMC6_PATH,
01109     'GCC_ARM': GCC_ARM_PATH,
01110     'IAR': IAR_PATH
01111 }
01112 
01113 from tools.toolchains.arm import ARM_STD, ARM_MICRO, ARMC6
01114 from tools.toolchains.gcc import GCC_ARM
01115 from tools.toolchains.iar import IAR
01116 
01117 TOOLCHAIN_CLASSES = {
01118     u'ARM': ARM_STD,
01119     u'uARM': ARM_MICRO,
01120     u'ARMC6': ARMC6,
01121     u'GCC_ARM': GCC_ARM,
01122     u'IAR': IAR
01123 }
01124 
01125 TOOLCHAINS = set(TOOLCHAIN_CLASSES.keys())