Diff: toolchains/__init__.py

Revision:

24:25bff2709c20

Parent:

22:9e85236d8716

Child:

27:5461402c33f8

diff -r fbae331171fa -r 25bff2709c20 toolchains/__init__.py
--- a/toolchains/__init__.py	Sat Jul 16 22:51:17 2016 +0100
+++ b/toolchains/__init__.py	Mon Aug 01 09:10:17 2016 +0100
@@ -17,7 +17,7 @@
 
 import re
 import sys
-from os import stat, walk, getcwd, sep
+from os import stat, walk, getcwd, sep, remove
 from copy import copy
 from time import time, sleep
 from types import ListType
@@ -26,6 +26,7 @@
 from inspect import getmro
 from copy import deepcopy
 from tools.config import Config
+from abc import ABCMeta, abstractmethod
 
 from multiprocessing import Pool, cpu_count
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path, compile_worker
@@ -188,8 +189,14 @@
 
 
 class mbedToolchain:
+    # Verbose logging
     VERBOSE = True
+
+    # Compile C files as CPP
     COMPILE_C_AS_CPP = False
+
+    # Response files for compiling, includes, linking and archiving.
+    # Not needed on posix systems where the typical arg limit is 2 megabytes
     RESPONSE_FILES = True
 
     CORTEX_SYMBOLS = {
@@ -207,6 +214,8 @@
 
     MBED_CONFIG_FILE_NAME="mbed_config.h"
 
+    __metaclass__ = ABCMeta
+
     def __init__(self, target, options=None, notify=None, macros=None, silent=False, extra_verbose=False):
         self.target = target
         self.name = self.__class__.__name__
@@ -226,9 +235,18 @@
         # Labels generated from toolchain and target rules/features (used for selective build)
         self.labels = None
 
+        # This will hold the initialized config object
+        self.config = None
+
         # This will hold the configuration data (as returned by Config.get_config_data())
         self.config_data = None
 
+        # This will hold the location of the configuration file or None if there's no configuration available
+        self.config_file = None
+
+        # Call guard for "get_config_data" (see the comments of get_config_data for details)
+        self.config_processed = False
+
         # Non-incremental compile
         self.build_all = False
 
@@ -242,8 +260,6 @@
         # Number of concurrent build jobs. 0 means auto (based on host system cores)
         self.jobs = 0
 
-        self.CHROOT = None
-
         # Ignore patterns from .mbedignore files
         self.ignore_patterns = []
 
@@ -285,12 +301,20 @@
         # uVisor spepcific rules
         if 'UVISOR' in self.target.features and 'UVISOR_SUPPORTED' in self.target.extra_labels:
             self.target.core = re.sub(r"F$", '', self.target.core)
-            
+
+        # Stats cache is used to reduce the amount of IO requests to stat
+        # header files during dependency change. See need_update()
         self.stat_cache = {}
 
+        # Used by the mbed Online Build System to build in chrooted environment
+        self.CHROOT = None
+
+        # Call post __init__() hooks before the ARM/GCC_ARM/IAR toolchain __init__() takes over
         self.init()
 
-    # This allows post __init__() hooks. Do not use
+    # Used for post __init__() hooks
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def init(self):
         return True
 
@@ -340,6 +364,8 @@
         elif event['type'] == 'progress':
             self.print_notify(event) # standard handle
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def notify(self, event):
         """ Little closure for notify functions
         """
@@ -392,6 +418,8 @@
             }
         return self.labels
 
+
+    # Determine whether a source file needs updating/compiling
     def need_update(self, target, dependencies):
         if self.build_all:
             return True
@@ -601,6 +629,8 @@
                 mkdir(dirname(target))
                 copyfile(source, target)
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def relative_object_path(self, build_path, base_dir, source):
         source_dir, name, _ = split_path(source)
 
@@ -610,6 +640,8 @@
             mkdir(obj_dir)
         return join(obj_dir, name + '.o')
 
+    # Generate response file for all includes.
+    # ARM, GCC, IAR cross compatible
     def get_inc_file(self, includes):
         include_file = join(self.build_dir, ".includes_%s.txt" % self.inc_md5)
         if not exists(include_file):
@@ -625,6 +657,8 @@
                 f.write(string)
         return include_file
 
+    # Generate response file for all objects when linking.
+    # ARM, GCC, IAR cross compatible
     def get_link_file(self, cmd):
         link_file = join(self.build_dir, ".link_files.txt")
         with open(link_file, "wb") as f:
@@ -639,6 +673,8 @@
             f.write(string)
         return link_file
  
+    # Generate response file for all objects when archiving.
+    # ARM, GCC, IAR cross compatible
     def get_arch_file(self, objects):
         archive_file = join(self.build_dir, ".archive_files.txt")
         with open(archive_file, "wb") as f:
@@ -649,6 +685,8 @@
             f.write(string)
         return archive_file
 
+    # THIS METHOD IS BEING CALLED BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def compile_sources(self, resources, build_path, inc_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
@@ -674,6 +712,9 @@
         work_dir = getcwd()
         self.prev_dir = None
 
+        # Generate configuration header (this will update self.build_all if needed)
+        self.get_config_header()
+
         # Sort compile queue for consistency
         files_to_compile.sort()
         for source in files_to_compile:
@@ -699,6 +740,7 @@
         else:
             return self.compile_seq(queue, objects)
 
+    # Compile source files queue in sequential order
     def compile_seq(self, queue, objects):
         for item in queue:
             result = compile_worker(item)
@@ -715,6 +757,7 @@
             objects.append(result['object'])
         return objects
 
+    # Compile source files queue in parallel by creating pool of worker threads
     def compile_queue(self, queue, objects):
         jobs_count = int(self.jobs if self.jobs else cpu_count() * CPU_COEF)
         p = Pool(processes=jobs_count)
@@ -764,6 +807,7 @@
 
         return objects
 
+    # Determine the compile command based on type of source file
     def compile_command(self, source, object, includes):
         # Check dependencies
         _, ext = splitext(source)
@@ -787,9 +831,39 @@
 
         return None
 
+    @abstractmethod
+    def parse_dependencies(self, dep_path):
+        """Parse the dependency information generated by the compiler.
+
+        Positional arguments:
+        dep_path -- the path to a file generated by a previous run of the compiler
+
+        Return value:
+        A list of all source files that the dependency file indicated were dependencies
+
+        Side effects:
+        None
+        """
+        raise NotImplemented
+
     def is_not_supported_error(self, output):
         return "#error directive: [NOT_SUPPORTED]" in output
 
+    @abstractmethod
+    def parse_output(self, output):
+        """Take in compiler output and extract sinlge line warnings and errors from it.
+
+        Positional arguments:
+        output -- a string of all the messages emitted by a run of the compiler
+
+        Return value:
+        None
+
+        Side effects:
+        call self.cc_info or self.notify with a description of the event generated by the compiler
+        """
+        raise NotImplemented
+
     def compile_output(self, output=[]):
         _rc = output[0]
         _stderr = output[1]
@@ -858,6 +932,8 @@
 
         return bin, needed_update
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def default_cmd(self, command):
         _stdout, _stderr, _rc = run_cmd(command, work_dir=getcwd(), chroot=self.CHROOT)
         self.debug("Return: %s"% _rc)
@@ -876,6 +952,8 @@
     def info(self, message):
         self.notify({'type': 'info', 'message': message})
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def debug(self, message):
         if self.VERBOSE:
             if type(message) is ListType:
@@ -883,11 +961,15 @@
             message = "[DEBUG] " + message
             self.notify({'type': 'debug', 'message': message})
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def cc_info(self, info=None):
         if info is not None:
             info['type'] = 'cc'
             self.notify(info)
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def cc_verbose(self, message, file=""):
         self.debug(message)
 
@@ -903,6 +985,8 @@
     def var(self, key, value):
         self.notify({'type': 'var', 'key': key, 'val': value})
 
+    # THIS METHOD IS BEING OVERRIDDEN BY THE MBED ONLINE BUILD SYSTEM
+    # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
     def mem_stats(self, map):
         """! Creates parser object
         @param map Path to linker map file to parse and decode
@@ -939,25 +1023,201 @@
     def set_config_data(self, config_data):
         self.config_data = config_data
 
-    # Return the location of the config header. This function will create the config
-    # header first if needed. The header will be written in a file called "mbed_conf.h"
-    # located in the project's build directory.
-    # If config headers are not used (self.config_header_content is None), the function
-    # returns None
+    # Creates the configuration header if needed:
+    # - if there is no configuration data, "mbed_config.h" is not create (or deleted if it exists).
+    # - if there is configuration data and "mbed_config.h" does not exist, it is created.
+    # - if there is configuration data similar to the previous configuration data,
+    #   "mbed_config.h" is left untouched.
+    # - if there is new configuration data, "mbed_config.h" is overriden.
+    # The function needs to be called exactly once for the lifetime of this toolchain instance.
+    # The "config_processed" variable (below) ensures this behaviour.
+    # The function returns the location of the configuration file, or None if there is no
+    # configuration data available (and thus no configuration file)
     def get_config_header(self):
-        if self.config_data is None:
-            return None
-        config_file = join(self.build_dir, self.MBED_CONFIG_FILE_NAME)
-        if not exists(config_file):
-            with open(config_file, "wt") as f:
-                f.write(Config.config_to_header(self.config_data))
-        return config_file
+        if self.config_processed: # this function was already called, return its result
+            return self.config_file
+        # The config file is located in the build directory
+        self.config_file = join(self.build_dir, self.MBED_CONFIG_FILE_NAME)
+        # If the file exists, read its current content in prev_data
+        if exists(self.config_file):
+            with open(self.config_file, "rt") as f:
+                prev_data = f.read()
+        else:
+            prev_data = None
+        # Get the current configuration data
+        crt_data = Config.config_to_header(self.config_data) if self.config_data else None
+        # "changed" indicates if a configuration change was detected
+        changed = False
+        if prev_data is not None: # a previous mbed_config.h exists
+            if crt_data is None: # no configuration data, so "mbed_config.h" needs to be removed
+                remove(self.config_file)
+                self.config_file = None # this means "config file not present"
+                changed = True
+            elif crt_data != prev_data: # different content of config file
+                with open(self.config_file, "wt") as f:
+                    f.write(crt_data)
+                changed = True
+        else: # a previous mbed_config.h does not exist
+            if crt_data is not None: # there's configuration data available
+                with open(self.config_file, "wt") as f:
+                    f.write(crt_data)
+                changed = True
+            else:
+                self.config_file = None # this means "config file not present"
+        # If there was a change in configuration, rebuild everything
+        self.build_all = changed
+        # Make sure that this function will only return the location of the configuration
+        # file for subsequent calls, without trying to manipulate its content in any way.
+        self.config_processed = True
+        return self.config_file
+
+    @abstractmethod
+    def get_config_option(self, config_header):
+        """Generate the compiler option that forces the inclusion of the configuration
+        header file.
+
+        Positional arguments:
+        config_header -- The configuration header that will be included within all source files
+
+        Return value:
+        A list of the command line arguments that will force the inclusion the specified header
+
+        Side effects:
+        None
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def assemble(self, source, object, includes):
+        """Generate the command line that assembles.
+
+        Positional arguments:
+        source -- a file path that is the file to assemble
+        object -- a file path that is the destination object
+        includes -- a list of all directories where header files may be found
+
+        Return value:
+        The complete command line, as a list, that would invoke the assembler
+        on the source file, include all the include paths, and generate
+        the specified object file.
+
+        Side effects:
+        None
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def compile_c(self, source, object, includes):
+        """Generate the command line that compiles a C source file.
+
+        Positional arguments:
+        source -- the C source file to compile
+        object -- the destination object file
+        includes -- a list of all the directories where header files may be found
+
+        Return value:
+        The complete command line, as a list, that would invoke the C compiler
+        on the source file, include all the include paths, and generate the
+        specified object file.
+
+        Side effects:
+        None
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def compile_cpp(self, source, object, includes):
+        """Generate the command line that compiles a C++ source file.
+
+        Positional arguments:
+        source -- the C++ source file to compile
+        object -- the destination object file
+        includes -- a list of all the directories where header files may be found
+
+        Return value:
+        The complete command line, as a list, that would invoke the C++ compiler
+        on the source file, include all the include paths, and generate the
+        specified object file.
+
+        Side effects:
+        None
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def link(self, output, objects, libraries, lib_dirs, mem_map):
+        """Run the linker to create an executable and memory map.
+
+        Positional arguments:
+        output -- the file name to place the executable in
+        objects -- all of the object files to link
+        libraries -- all of the required libraries
+        lib_dirs -- where the required libraries are located
+        mem_map -- the location where the memory map file should be stored
+
+        Return value:
+        None
+
+        Side effect:
+        Runs the linker to produce the executable.
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def archive(self, objects, lib_path):
+        """Run the command line that creates an archive.
+
+        Positional arguhments:
+        objects -- a list of all the object files that should be archived
+        lib_path -- the file name of the resulting library file
+
+        Return value:
+        None
+
+        Side effect:
+        Runs the archiving tool to produce the library file.
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
+
+    @abstractmethod
+    def binary(self, resources, elf, bin):
+        """Run the command line that will Extract a simplified binary file.
+
+        Positional arguments:
+        resources -- A resources object (Is not used in any of the toolchains)
+        elf -- the executable file that is to be converted
+        bin -- the file name of the to be created simplified binary file
+
+        Return value:
+        None
+
+        Side effect:
+        Runs the elf2bin tool to produce the simplified binary file.
+
+        Note:
+        This method should be decorated with @hook_tool.
+        """
+        raise NotImplemented
 
     # Return the list of macros geenrated by the build system
     def get_config_macros(self):
         return Config.config_to_macros(self.config_data) if self.config_data else []
 
-
 from tools.settings import ARM_PATH
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
 from tools.settings import IAR_PATH