Diff: build_api.py

Revision:

32:8ea194f6145b

Parent:

30:f12ce67666d0

--- a/build_api.py	Mon Aug 29 11:56:59 2016 +0100
+++ b/build_api.py	Wed Jan 04 11:58:24 2017 -0600
@@ -1,6 +1,6 @@
 """
 mbed SDK
-Copyright (c) 2011-2013 ARM Limited
+Copyright (c) 2011-2016 ARM Limited
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,25 +17,26 @@
 
 import re
 import tempfile
-
 from types import ListType
 from shutil import rmtree
-from os.path import join, exists, basename, abspath, normpath, dirname
-from os import linesep
+from os.path import join, exists, dirname, basename, abspath, normpath
+from os import linesep, remove
 from time import time
 
 from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\
     ToolException, InvalidReleaseTargetException
-from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL,\
-    MBED_COMMON, MBED_CONFIG_FILE
+from tools.paths import MBED_CMSIS_PATH, MBED_TARGETS_PATH, MBED_LIBRARIES,\
+    MBED_HEADER, MBED_DRIVERS, MBED_PLATFORM, MBED_HAL, MBED_CONFIG_FILE,\
+    MBED_LIBRARIES_DRIVERS, MBED_LIBRARIES_PLATFORM, MBED_LIBRARIES_HAL,\
+    BUILD_DIR
 from tools.targets import TARGET_NAMES, TARGET_MAP
 from tools.libraries import Library
 from tools.toolchains import TOOLCHAIN_CLASSES, mbedToolchain
-from tools.targets import set_targets_json_location
-from tools.build_profiles import find_build_profile, get_toolchain_profile, find_targets_json
 from jinja2 import FileSystemLoader
 from jinja2.environment import Environment
 from tools.config import Config
+from tools.build_profiles import find_build_profile, get_toolchain_profile, find_targets_json
+from tools.targets import set_targets_json_location
 
 RELEASE_VERSIONS = ['2', '5']
 
@@ -135,7 +136,7 @@
         toolchain.config.add_config_files(resources.json_files)
 
         # Add features while we find new ones
-        features = toolchain.config.get_features()
+        features = set(toolchain.config.get_features())
         if features == prev_features:
             break
 
@@ -201,12 +202,12 @@
                     ("following toolchains: %s" %
                      ", ".join(supported_toolchains_sorted))
 
-            elif not target.default_build == 'standard':
+            elif not target.default_lib == 'std':
                 result = False
                 reason = ("Target '%s' must set the " % target.name) + \
-                    ("'default_build' to 'standard' to be included in the ") + \
+                    ("'default_lib' to 'std' to be included in the ") + \
                     ("mbed OS 5.0 official release." + linesep) + \
-                    ("Currently it is set to '%s'" % target.default_build)
+                    ("Currently it is set to '%s'" % target.default_lib)
 
         else:
             result = False
@@ -277,9 +278,10 @@
 
 
 def prepare_toolchain(src_paths, target, toolchain_name,
-                      macros=None, options=None, clean=False, jobs=1,
+                      macros=None, clean=False, jobs=1,
                       notify=None, silent=False, verbose=False,
-                      extra_verbose=False, config=None):
+                      extra_verbose=False, config=None,
+                      app_config=None, build_profile=None):
     """ Prepares resource related objects - toolchain, target, config
 
     Positional arguments:
@@ -289,7 +291,6 @@
 
     Keyword arguments:
     macros - additional macros
-    options - general compiler options like debug-symbols or small-build
     clean - Rebuild everything if True
     jobs - how many compilers we can run at once
     notify - Notify function for logs
@@ -297,6 +298,8 @@
     verbose - Write the actual tools command lines used if True
     extra_verbose - even more output!
     config - a Config object to use instead of creating one
+    app_config - location of a chosen mbed_app.json file
+    build_profile - a dict of flags that will be passed to the compiler
     """
 
     # We need to remove all paths which are repeated to avoid
@@ -304,20 +307,14 @@
     src_paths = [src_paths[0]] + list(set(src_paths[1:]))
 
     # If the configuration object was not yet created, create it now
-    config = config or Config(target, src_paths)
-
-    # If the 'target' argument is a string, convert it to a target instance
-    if isinstance(target, basestring):
-        try:
-            target = TARGET_MAP[target]
-        except KeyError:
-            raise KeyError("Target '%s' not found" % target)
+    config = config or Config(target, src_paths, app_config=app_config)
+    target = config.target
 
     # Toolchain instance
     try:
         toolchain = TOOLCHAIN_CLASSES[toolchain_name](
-            target, options, notify, macros, silent,
-            extra_verbose=extra_verbose)
+            target, notify, macros, silent,
+            extra_verbose=extra_verbose, build_profile=build_profile)
     except KeyError:
         raise KeyError("Toolchain %s not supported" % toolchain_name)
 
@@ -329,7 +326,7 @@
     return toolchain
 
 def scan_resources(src_paths, toolchain, dependencies_paths=None,
-                   inc_dirs=None):
+                   inc_dirs=None, base_path=None):
     """ Scan resources using initialized toolcain
 
     Positional arguments
@@ -341,9 +338,9 @@
     """
 
     # Scan src_path
-    resources = toolchain.scan_resources(src_paths[0])
+    resources = toolchain.scan_resources(src_paths[0], base_path=base_path)
     for path in src_paths[1:]:
-        resources.add(toolchain.scan_resources(path))
+        resources.add(toolchain.scan_resources(path, base_path=base_path))
 
     # Scan dependency paths for include dirs
     if dependencies_paths is not None:
@@ -368,11 +365,12 @@
     return resources
 
 def build_project(src_paths, build_path, target, toolchain_name,
-                  libraries_paths=None, options=None, linker_script=None,
+                  libraries_paths=None, linker_script=None,
                   clean=False, notify=None, verbose=False, name=None,
                   macros=None, inc_dirs=None, jobs=1, silent=False,
                   report=None, properties=None, project_id=None,
-                  project_description=None, extra_verbose=False, config=None):
+                  project_description=None, extra_verbose=False, config=None,
+                  app_config=None, build_profile=None):
     """ Build a project. A project may be a test or a user program.
 
     Positional arguments:
@@ -384,7 +382,6 @@
 
     Keyword arguments:
     libraries_paths - The location of libraries to include when linking
-    options - general compiler options like debug-symbols or small-build
     linker_script - the file that drives the linker to do it's job
     clean - Rebuild everything if True
     notify - Notify function for logs
@@ -400,6 +397,8 @@
     project_description - the human-readable version of what this thing does
     extra_verbose - even more output!
     config - a Config object to use instead of creating one
+    app_config - location of a chosen mbed_app.json file
+    build_profile - a dict of flags that will be passed to the compiler
     """
 
     # Convert src_path to a list if needed
@@ -408,16 +407,15 @@
     # Extend src_paths wiht libraries_paths
     if libraries_paths is not None:
         src_paths.extend(libraries_paths)
+        inc_dirs.extend(map(dirname, libraries_paths))
 
     # Build Directory
     if clean and exists(build_path):
         rmtree(build_path)
     mkdir(build_path)
 
-
     ###################################
     # mbed Classic/2.0/libary support #
-    ###################################
 
     # Find build system profile
     profile = None
@@ -452,11 +450,15 @@
 
         mbedToolchain.init = init_hook
 
+    # mbed Classic/2.0/libary support #
+    ###################################
+
     # Pass all params to the unified prepare_toolchain()
     toolchain = prepare_toolchain(
-        src_paths, target, toolchain_name, macros=macros, options=options,
-        clean=clean, jobs=jobs, notify=notify, silent=silent, verbose=verbose,
-        extra_verbose=extra_verbose, config=config)
+        src_paths, target, toolchain_name, macros=macros, clean=clean,
+        jobs=jobs, notify=notify, silent=silent, verbose=verbose,
+        extra_verbose=extra_verbose, config=config, app_config=app_config,
+        build_profile=build_profile)
 
     # The first path will give the name to the library
     if name is None:
@@ -494,10 +496,29 @@
         # Link Program
         res, _ = toolchain.link_program(resources, build_path, name)
 
+        memap_instance = getattr(toolchain, 'memap_instance', None)
+        memap_table = ''
+        if memap_instance:
+            # Write output to stdout in text (pretty table) format
+            memap_table = memap_instance.generate_output('table')
+
+            if not silent:
+                print memap_table
+
+            # Write output to file in JSON format
+            map_out = join(build_path, name + "_map.json")
+            memap_instance.generate_output('json', map_out)
+
+            # Write output to file in CSV format for the CI
+            map_csv = join(build_path, name + "_map.csv")
+            memap_instance.generate_output('csv-ci', map_csv)
+
+        resources.detect_duplicates(toolchain)
+
         if report != None:
             end = time()
             cur_result["elapsed_time"] = end - start
-            cur_result["output"] = toolchain.get_output()
+            cur_result["output"] = toolchain.get_output() + memap_table
             cur_result["result"] = "OK"
             cur_result["memory_usage"] = toolchain.map_outputs
 
@@ -526,10 +547,12 @@
         raise
 
 def build_library(src_paths, build_path, target, toolchain_name,
-                  dependencies_paths=None, options=None, name=None, clean=False,
+                  dependencies_paths=None, name=None, clean=False,
                   archive=True, notify=None, verbose=False, macros=None,
                   inc_dirs=None, jobs=1, silent=False, report=None,
-                  properties=None, extra_verbose=False, project_id=None):
+                  properties=None, extra_verbose=False, project_id=None,
+                  remove_config_header_file=False, app_config=None,
+                  build_profile=None):
     """ Build a library
 
     Positional arguments:
@@ -541,7 +564,6 @@
 
     Keyword arguments:
     dependencies_paths - The location of libraries to include when linking
-    options - general compiler options like debug-symbols or small-build
     name - the name of the library
     clean - Rebuild everything if True
     archive - whether the library will create an archive file
@@ -555,6 +577,9 @@
     properties - UUUUHHHHH beats me
     extra_verbose - even more output!
     project_id - the name that goes in the report
+    remove_config_header_file - delete config header file when done building
+    app_config - location of a chosen mbed_app.json file
+    build_profile - a dict of flags that will be passed to the compiler
     """
 
     # Convert src_path to a list if needed
@@ -576,9 +601,10 @@
 
     # Pass all params to the unified prepare_toolchain()
     toolchain = prepare_toolchain(
-        src_paths, target, toolchain_name, macros=macros, options=options,
-        clean=clean, jobs=jobs, notify=notify, silent=silent, verbose=verbose,
-        extra_verbose=extra_verbose)
+        src_paths, target, toolchain_name, macros=macros, clean=clean,
+        jobs=jobs, notify=notify, silent=silent, verbose=verbose,
+        extra_verbose=extra_verbose, app_config=app_config,
+        build_profile=build_profile)
 
     # The first path will give the name to the library
     if name is None:
@@ -622,6 +648,8 @@
         toolchain.copy_files(resources.objects, build_path, resources=resources)
         toolchain.copy_files(resources.libraries, build_path,
                              resources=resources)
+        toolchain.copy_files(resources.json_files, build_path,
+                             resources=resources)
         if resources.linker_script:
             toolchain.copy_files(resources.linker_script, build_path,
                                  resources=resources)
@@ -638,6 +666,11 @@
         if archive:
             toolchain.build_library(objects, build_path, name)
 
+        if remove_config_header_file:
+            config_header_path = toolchain.get_config_header()
+            if config_header_path:
+                remove(config_header_path)
+
         if report != None:
             end = time()
             cur_result["elapsed_time"] = end - start
@@ -672,9 +705,10 @@
 ### Legacy methods ###
 ######################
 
-def build_lib(lib_id, target, toolchain_name, options=None, verbose=False,
+def build_lib(lib_id, target, toolchain_name, verbose=False,
               clean=False, macros=None, notify=None, jobs=1, silent=False,
-              report=None, properties=None, extra_verbose=False):
+              report=None, properties=None, extra_verbose=False,
+              build_profile=None):
     """ Legacy method for building mbed libraries
 
     Positional arguments:
@@ -683,7 +717,6 @@
     toolchain_name - the name of the build tools
 
     Keyword arguments:
-    options - general compiler options like debug-symbols or small-build
     clean - Rebuild everything if True
     verbose - Write the actual tools command lines used if True
     macros - additional macros
@@ -693,6 +726,7 @@
     report - a dict where a result may be appended
     properties - UUUUHHHHH beats me
     extra_verbose - even more output!
+    build_profile - a dict of flags that will be passed to the compiler
     """
     lib = Library(lib_id)
     if not lib.is_supported(target, toolchain_name):
@@ -748,8 +782,8 @@
     try:
         # Toolchain instance
         toolchain = TOOLCHAIN_CLASSES[toolchain_name](
-            target, options, macros=macros, notify=notify, silent=silent,
-            extra_verbose=extra_verbose)
+            target, macros=macros, notify=notify, silent=silent,
+            extra_verbose=extra_verbose, build_profile=build_profile)
         toolchain.VERBOSE = verbose
         toolchain.jobs = jobs
         toolchain.build_all = clean
@@ -780,6 +814,7 @@
             for path in dependencies_paths:
                 lib_resources = toolchain.scan_resources(path)
                 dependencies_include_dir.extend(lib_resources.inc_dirs)
+                dependencies_include_dir.extend(map(dirname, lib_resources.inc_dirs))
 
         if inc_dirs:
             dependencies_include_dir.extend(inc_dirs)
@@ -837,9 +872,10 @@
 
 # We do have unique legacy conventions about how we build and package the mbed
 # library
-def build_mbed_libs(target, toolchain_name, options=None, verbose=False,
+def build_mbed_libs(target, toolchain_name, verbose=False,
                     clean=False, macros=None, notify=None, jobs=1, silent=False,
-                    report=None, properties=None, extra_verbose=False):
+                    report=None, properties=None, extra_verbose=False,
+                    build_profile=None):
     """ Function returns True is library was built and false if building was
     skipped
 
@@ -848,7 +884,6 @@
     toolchain_name - the name of the build tools
 
     Keyword arguments:
-    options - general compiler options like debug-symbols or small-build
     verbose - Write the actual tools command lines used if True
     clean - Rebuild everything if True
     macros - additional macros
@@ -858,6 +893,7 @@
     report - a dict where a result may be appended
     properties - UUUUHHHHH beats me
     extra_verbose - even more output!
+    build_profile - a dict of flags that will be passed to the compiler
     """
 
     if report != None:
@@ -892,8 +928,8 @@
     try:
         # Toolchain
         toolchain = TOOLCHAIN_CLASSES[toolchain_name](
-            target, options, macros=macros, notify=notify, silent=silent,
-            extra_verbose=extra_verbose)
+            target, macros=macros, notify=notify, silent=silent,
+            extra_verbose=extra_verbose, build_profile=build_profile)
         toolchain.VERBOSE = verbose
         toolchain.jobs = jobs
         toolchain.build_all = clean
@@ -915,7 +951,7 @@
         # CMSIS
         toolchain.info("Building library %s (%s, %s)" %
                        ('CMSIS', target.name, toolchain_name))
-        cmsis_src = join(MBED_TARGETS_PATH, "cmsis")
+        cmsis_src = MBED_CMSIS_PATH
         resources = toolchain.scan_resources(cmsis_src)
 
         toolchain.copy_files(resources.headers, build_target)
@@ -930,29 +966,40 @@
                        ('MBED', target.name, toolchain_name))
 
         # Common Headers
-        toolchain.copy_files(toolchain.scan_resources(MBED_API).headers,
-                             MBED_LIBRARIES)
-        toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers,
-                             MBED_LIBRARIES)
+        toolchain.copy_files([MBED_HEADER], MBED_LIBRARIES)
+        library_incdirs = [dirname(MBED_LIBRARIES), MBED_LIBRARIES]
+
+        for dir, dest in [(MBED_DRIVERS, MBED_LIBRARIES_DRIVERS),
+                          (MBED_PLATFORM, MBED_LIBRARIES_PLATFORM),
+                          (MBED_HAL, MBED_LIBRARIES_HAL)]:
+            resources = toolchain.scan_resources(dir)
+            toolchain.copy_files(resources.headers, dest)
+            library_incdirs.append(dest)
 
         # Target specific sources
-        hal_src = join(MBED_TARGETS_PATH, "hal")
+        hal_src = MBED_TARGETS_PATH
         hal_implementation = toolchain.scan_resources(hal_src)
         toolchain.copy_files(hal_implementation.headers +
                              hal_implementation.hex_files +
-                             hal_implementation.libraries,
+                             hal_implementation.libraries +
+                             [MBED_CONFIG_FILE],
                              build_target, resources=hal_implementation)
+        toolchain.copy_files(hal_implementation.linker_script, build_toolchain)
+        toolchain.copy_files(hal_implementation.bin_files, build_toolchain)
         incdirs = toolchain.scan_resources(build_target).inc_dirs
         objects = toolchain.compile_sources(hal_implementation, tmp_path,
-                                            [MBED_LIBRARIES] + incdirs)
+                                            library_incdirs + incdirs)
+        toolchain.copy_files(objects, build_toolchain)
 
         # Common Sources
-        mbed_resources = toolchain.scan_resources(MBED_COMMON)
-        objects += toolchain.compile_sources(mbed_resources, tmp_path,
-                                             [MBED_LIBRARIES] + incdirs)
+        mbed_resources = None
+        for dir in [MBED_DRIVERS, MBED_PLATFORM, MBED_HAL]:
+            mbed_resources += toolchain.scan_resources(dir)
+
+        objects = toolchain.compile_sources(mbed_resources, tmp_path,
+                                            library_incdirs + incdirs)
 
         # A number of compiled files need to be copied as objects as opposed to
-        # being part of the mbed library, for reasons that have to do with the
         # way the linker search for symbols in archives. These are:
         #   - retarget.o: to make sure that the C standard lib symbols get
         #                 overridden
@@ -1129,9 +1176,9 @@
 
 
 def static_analysis_scan(target, toolchain_name, cppcheck_cmd,
-                         cppcheck_msg_format, options=None, verbose=False,
+                         cppcheck_msg_format, verbose=False,
                          clean=False, macros=None, notify=None, jobs=1,
-                         extra_verbose=False):
+                         extra_verbose=False, build_profile=None):
     """Perform static analysis on a target and toolchain combination
 
     Positional arguments:
@@ -1141,18 +1188,19 @@
     cppcheck_msg_format - the format of the check messages
 
     Keyword arguments:
-    options - things like debug-symbols, or small-build, etc.
     verbose - more printing!
     clean - start from a clean slate
     macros - extra macros to compile with
     notify - the notification event handling function
     jobs - number of commands to run at once
     extra_verbose - even moar printing
+    build_profile - a dict of flags that will be passed to the compiler
     """
     # Toolchain
-    toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options,
-                                                  macros=macros, notify=notify,
-                                                  extra_verbose=extra_verbose)
+    toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, macros=macros,
+                                                  notify=notify,
+                                                  extra_verbose=extra_verbose,
+                                                  build_profile=build_profile)
     toolchain.VERBOSE = verbose
     toolchain.jobs = jobs
     toolchain.build_all = clean
@@ -1168,7 +1216,7 @@
     # CMSIS
     toolchain.info("Static analysis for %s (%s, %s)" %
                    ('CMSIS', target.name, toolchain_name))
-    cmsis_src = join(MBED_TARGETS_PATH, "cmsis")
+    cmsis_src = MBED_CMSIS_PATH
     resources = toolchain.scan_resources(cmsis_src)
 
     # Copy files before analysis
@@ -1211,7 +1259,10 @@
                    ('MBED', target.name, toolchain_name))
 
     # Common Headers
-    toolchain.copy_files(toolchain.scan_resources(MBED_API).headers,
+    toolchain.copy_files([MBED_HEADER], MBED_LIBRARIES)
+    toolchain.copy_files(toolchain.scan_resources(MBED_DRIVERS).headers,
+                         MBED_LIBRARIES)
+    toolchain.copy_files(toolchain.scan_resources(MBED_PLATFORM).headers,
                          MBED_LIBRARIES)
     toolchain.copy_files(toolchain.scan_resources(MBED_HAL).headers,
                          MBED_LIBRARIES)
@@ -1241,8 +1292,8 @@
     # command line
     mbed_includes = ["-I%s" % i for i in mbed_resources.inc_dirs]
     mbed_includes.append("-I%s"% str(build_target))
-    mbed_includes.append("-I%s"% str(MBED_COMMON))
-    mbed_includes.append("-I%s"% str(MBED_API))
+    mbed_includes.append("-I%s"% str(MBED_DRIVERS))
+    mbed_includes.append("-I%s"% str(MBED_PLATFORM))
     mbed_includes.append("-I%s"% str(MBED_HAL))
     mbed_c_sources = " ".join(mbed_resources.c_sources)
     mbed_cpp_sources = " ".join(mbed_resources.cpp_sources)
@@ -1274,9 +1325,9 @@
 
 
 def static_analysis_scan_lib(lib_id, target, toolchain, cppcheck_cmd,
-                             cppcheck_msg_format, options=None, verbose=False,
+                             cppcheck_msg_format, verbose=False,
                              clean=False, macros=None, notify=None, jobs=1,
-                             extra_verbose=False):
+                             extra_verbose=False, build_profile=None):
     """Perform static analysis on a library as if it were to be compiled for a
     particular target and toolchain combination
     """
@@ -1284,9 +1335,9 @@
     if lib.is_supported(target, toolchain):
         static_analysis_scan_library(
             lib.source_dir, lib.build_dir, target, toolchain, cppcheck_cmd,
-            cppcheck_msg_format, lib.dependencies, options, verbose=verbose,
+            cppcheck_msg_format, lib.dependencies, verbose=verbose,
             clean=clean, macros=macros, notify=notify, jobs=jobs,
-            extra_verbose=extra_verbose)
+            extra_verbose=extra_verbose, build_profile=build_profile)
     else:
         print('Library "%s" is not yet supported on target %s with toolchain %s'
               % (lib_id, target.name, toolchain))
@@ -1294,10 +1345,10 @@
 
 def static_analysis_scan_library(src_paths, build_path, target, toolchain_name,
                                  cppcheck_cmd, cppcheck_msg_format,
-                                 dependencies_paths=None, options=None,
+                                 dependencies_paths=None,
                                  name=None, clean=False, notify=None,
                                  verbose=False, macros=None, jobs=1,
-                                 extra_verbose=False):
+                                 extra_verbose=False, build_profile=None):
     """ Function scans library for statically detectable defects
 
     Positional arguments:
@@ -1310,7 +1361,6 @@
 
     Keyword arguments:
     dependencies_paths - the paths to sources that this library depends on
-    options - things like debug-symbols, or small-build, etc.
     name - the name of this library
     clean - start from a clean slate
     notify - the notification event handling function
@@ -1318,6 +1368,7 @@
     macros - extra macros to compile with
     jobs - number of commands to run at once
     extra_verbose - even moar printing
+    build_profile - a dict of flags that will be passed to the compiler
     """
     if type(src_paths) != ListType:
         src_paths = [src_paths]
@@ -1328,9 +1379,10 @@
                             src_path)
 
     # Toolchain instance
-    toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options,
-                                                  macros=macros, notify=notify,
-                                                  extra_verbose=extra_verbose)
+    toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, macros=macros,
+                                                  notify=notify,
+                                                  extra_verbose=extra_verbose,
+                                                  build_profile=build_profile)
     toolchain.VERBOSE = verbose
     toolchain.jobs = jobs