Diff: build_api.py

Revision:

9:2d27d77ada5c

Parent:

8:a8ac6ed29081

Child:

13:ab47a20b66f0

diff -r a8ac6ed29081 -r 2d27d77ada5c build_api.py
--- a/build_api.py	Tue Jun 07 11:35:02 2016 +0100
+++ b/build_api.py	Tue Jun 14 11:07:30 2016 +0100
@@ -19,13 +19,13 @@
 import tempfile
 import colorama
 
-
+from copy import copy
 from types import ListType
 from shutil import rmtree
 from os.path import join, exists, basename, abspath, normpath
 from os import getcwd, walk
 from time import time
-from copy import copy
+import fnmatch
 
 from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException, ToolException
 from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON
@@ -128,7 +128,7 @@
 
     for path in src_paths:
         profile = get_build_profile(path)
-
+ 
     # If the configuration object was not yet created, create it now
     config = config or Config(target, src_paths)
 
@@ -210,9 +210,9 @@
             resources.objects.extend(objects)
 
         # Link Program
-        res, needed_update = toolchain.link_program(resources, build_path, name)
+        res, _ = toolchain.link_program(resources, build_path, name)
 
-        if report != None and needed_update:
+        if report != None:
             end = time()
             cur_result["elapsed_time"] = end - start
             cur_result["output"] = toolchain.get_output()
@@ -237,8 +237,6 @@
             if toolchain_output:
                 cur_result["output"] += toolchain_output
 
-            cur_result["output"] += str(e)
-
             add_result_to_report(report, cur_result)
 
         # Let Exception propagate
@@ -307,26 +305,43 @@
         toolchain.info("Building library %s (%s, %s)" % (name, target.name, toolchain_name))
 
         # Scan Resources
-        resources = []
-        for src_path in src_paths:
-            resources.append(toolchain.scan_resources(src_path))
+        resources = None
+        for path in src_paths:
+            # Scan resources
+            resource = toolchain.scan_resources(path)
+            
+            # Copy headers, objects and static libraries - all files needed for static lib
+            toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
+            toolchain.copy_files(resource.objects, build_path, rel_path=resource.base_path)
+            toolchain.copy_files(resource.libraries, build_path, rel_path=resource.base_path)
+            if resource.linker_script:
+                toolchain.copy_files(resource.linker_script, build_path, rel_path=resource.base_path)
+
+            # Extend resources collection
+            if not resources:
+                resources = resource
+            else:
+                resources.add(resource)
+
+        # We need to add if necessary additional include directories
+        if inc_dirs:
+            if type(inc_dirs) == ListType:
+                resources.inc_dirs.extend(inc_dirs)
+            else:
+                resources.inc_dirs.append(inc_dirs)
 
         # Add extra include directories / files which are required by library
         # This files usually are not in the same directory as source files so
         # previous scan will not include them
         if inc_dirs_ext is not None:
             for inc_ext in inc_dirs_ext:
-                resources.append(toolchain.scan_resources(inc_ext))
+                resources.add(toolchain.scan_resources(inc_ext))
 
         # Dependencies Include Paths
-        dependencies_include_dir = []
         if dependencies_paths is not None:
             for path in dependencies_paths:
                 lib_resources = toolchain.scan_resources(path)
-                dependencies_include_dir.extend(lib_resources.inc_dirs)
-
-        if inc_dirs:
-            dependencies_include_dir.extend(inc_dirs)
+                resources.inc_dirs.extend(lib_resources.inc_dirs)
 
         if archive:
             # Use temp path when building archive
@@ -337,29 +352,21 @@
 
         # Handle configuration
         config = Config(target)
-
-        # Copy headers, objects and static libraries
-        for resource in resources:
-            toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
-            toolchain.copy_files(resource.objects, build_path, rel_path=resource.base_path)
-            toolchain.copy_files(resource.libraries, build_path, rel_path=resource.base_path)
-            if resource.linker_script:
-                toolchain.copy_files(resource.linker_script, build_path, rel_path=resource.base_path)
-            config.add_config_files(resource.json_files)
-
+        # Update the configuration with any .json files found while scanning
+        config.add_config_files(resources.json_files)
+        # And add the configuration macros to the toolchain
         toolchain.add_macros(config.get_config_data_macros())
 
         # Compile Sources
-        objects = []
-        for resource in resources:
-            objects.extend(toolchain.compile_sources(resource, abspath(tmp_path), dependencies_include_dir))
+        for path in src_paths:
+            src = toolchain.scan_resources(path)
+            objects = toolchain.compile_sources(src, abspath(tmp_path), resources.inc_dirs)
+            resources.objects.extend(objects)
 
         if archive:
-            needed_update = toolchain.build_library(objects, build_path, name)
-        else:
-            needed_update = True
+            toolchain.build_library(objects, build_path, name)
 
-        if report != None and needed_update:
+        if report != None:
             end = time()
             cur_result["elapsed_time"] = end - start
             cur_result["output"] = toolchain.get_output()
@@ -382,6 +389,144 @@
             if toolchain_output:
                 cur_result["output"] += toolchain_output
 
+            add_result_to_report(report, cur_result)
+
+        # Let Exception propagate
+        raise e
+
+######################
+### Legacy methods ###
+######################
+
+def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
+    """ Legacy method for building mbed libraries
+        Function builds library in proper directory using all dependencies and macros defined by user.
+    """
+    lib = Library(lib_id)
+    if not lib.is_supported(target, toolchain_name):
+        print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
+        return False
+    
+    # We need to combine macros from parameter list with macros from library definition
+    MACROS = lib.macros if lib.macros else []
+    if macros:
+        macros.extend(MACROS)
+    else:
+        macros = MACROS
+
+    src_paths = lib.source_dir
+    build_path = lib.build_dir
+    dependencies_paths = lib.dependencies
+    inc_dirs = lib.inc_dirs
+    inc_dirs_ext = lib.inc_dirs_ext
+    
+    """ src_path: the path of the source directory
+    build_path: the path of the build directory
+    target: ['LPC1768', 'LPC11U24', 'LPC2368']
+    toolchain: ['ARM', 'uARM', 'GCC_ARM', 'GCC_CR']
+    library_paths: List of paths to additional libraries
+    clean: Rebuild everything if True
+    notify: Notify function for logs
+    verbose: Write the actual tools command lines if True
+    inc_dirs: additional include directories which should be included in build
+    inc_dirs_ext: additional include directories which should be copied to library directory
+    """
+    if type(src_paths) != ListType:
+        src_paths = [src_paths]
+
+    # The first path will give the name to the library
+    name = basename(src_paths[0])
+
+    if report != None:
+        start = time()
+        id_name = name.upper()
+        description = name
+        vendor_label = target.extra_labels[0]
+        cur_result = None
+        prep_report(report, target.name, toolchain_name, id_name)
+        cur_result = create_result(target.name, toolchain_name, id_name, description)
+
+        if properties != None:
+            prep_properties(properties, target.name, toolchain_name, vendor_label)
+
+    for src_path in src_paths:
+        if not exists(src_path):
+            error_msg = "The library source folder does not exist: %s", src_path
+
+            if report != None:
+                cur_result["output"] = error_msg
+                cur_result["result"] = "FAIL"
+                add_result_to_report(report, cur_result)
+
+            raise Exception(error_msg)
+
+    try:
+        # Toolchain instance
+        toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent, extra_verbose=extra_verbose, profile=profile)
+        toolchain.VERBOSE = verbose
+        toolchain.jobs = jobs
+        toolchain.build_all = clean
+
+        toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name))
+
+        # Scan Resources
+        resources = []
+        for src_path in src_paths:
+            resources.append(toolchain.scan_resources(src_path))
+
+        # Add extra include directories / files which are required by library
+        # This files usually are not in the same directory as source files so
+        # previous scan will not include them
+        if inc_dirs_ext is not None:
+            for inc_ext in inc_dirs_ext:
+                resources.append(toolchain.scan_resources(inc_ext))
+
+        # Dependencies Include Paths
+        dependencies_include_dir = []
+        if dependencies_paths is not None:
+            for path in dependencies_paths:
+                lib_resources = toolchain.scan_resources(path)
+                dependencies_include_dir.extend(lib_resources.inc_dirs)
+
+        if inc_dirs:
+            dependencies_include_dir.extend(inc_dirs)
+
+        # Create the desired build directory structure
+        bin_path = join(build_path, toolchain.obj_path)
+        mkdir(bin_path)
+        tmp_path = join(build_path, '.temp', toolchain.obj_path)
+        mkdir(tmp_path)
+
+        # Copy Headers
+        for resource in resources:
+            toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
+        dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
+
+        # Compile Sources
+        objects = []
+        for resource in resources:
+            objects.extend(toolchain.compile_sources(resource, tmp_path, dependencies_include_dir))
+
+        needed_update = toolchain.build_library(objects, bin_path, name)
+
+        if report != None and needed_update:
+            end = time()
+            cur_result["elapsed_time"] = end - start
+            cur_result["output"] = toolchain.get_output()
+            cur_result["result"] = "OK"
+
+            add_result_to_report(report, cur_result)
+
+    except Exception, e:
+        if report != None:
+            end = time()
+            cur_result["result"] = "FAIL"
+            cur_result["elapsed_time"] = end - start
+
+            toolchain_output = toolchain.get_output()
+            if toolchain_output:
+                cur_result["output"] += toolchain_output
+
             cur_result["output"] += str(e)
 
             add_result_to_report(report, cur_result)
@@ -389,34 +534,6 @@
         # Let Exception propagate
         raise e
 
-def build_lib(lib_id, target, toolchain, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
-    """ Wrapper for build_library function.
-        Function builds library in proper directory using all dependencies and macros defined by user.
-    """
-    lib = Library(lib_id)
-    if lib.is_supported(target, toolchain):
-        # We need to combine macros from parameter list with macros from library definition
-        MACROS = lib.macros if lib.macros else []
-        if macros:
-            MACROS.extend(macros)
-
-        return build_library(lib.source_dir, lib.build_dir, target, toolchain, lib.dependencies, options,
-                      verbose=verbose,
-                      silent=silent,
-                      clean=clean,
-                      macros=MACROS,
-                      notify=notify,
-                      inc_dirs=lib.inc_dirs,
-                      inc_dirs_ext=lib.inc_dirs_ext,
-                      jobs=jobs,
-                      report=report,
-                      properties=properties,
-                      extra_verbose=extra_verbose)
-    else:
-        print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
-        return False
-
-
 # We do have unique legacy conventions about how we build and package the mbed library
 def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
     """ Function returns True is library was built and false if building was skipped """
@@ -445,6 +562,9 @@
 
         return False
 
+    for path in src_paths:
+        profile = get_build_profile(path)
+
     try:
         # Toolchain
         toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent, extra_verbose=extra_verbose)
@@ -506,12 +626,12 @@
         for o in separate_objects:
             objects.remove(o)
 
-        needed_update = toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed")
+        toolchain.build_library(objects, BUILD_TOOLCHAIN, "mbed")
 
         for o in separate_objects:
             toolchain.copy_files(o, BUILD_TOOLCHAIN)
 
-        if report != None and needed_update:
+        if report != None:
             end = time()
             cur_result["elapsed_time"] = end - start
             cur_result["output"] = toolchain.get_output()
@@ -538,6 +658,7 @@
         # Let Exception propagate
         raise e
 
+
 def get_unique_supported_toolchains():
     """ Get list of all unique toolchains supported by targets """
     unique_supported_toolchains = []
@@ -554,12 +675,12 @@
     from prettytable import PrettyTable # Only use it in this function so building works without extra modules
 
     # All tests status table print
-    columns = ["Platform"] + unique_supported_toolchains
-    pt = PrettyTable(["Platform"] + unique_supported_toolchains)
+    columns = ["Target"] + unique_supported_toolchains
+    pt = PrettyTable(["Target"] + unique_supported_toolchains)
     # Align table
     for col in columns:
         pt.align[col] = "c"
-    pt.align["Platform"] = "l"
+    pt.align["Target"] = "l"
 
     perm_counter = 0
     target_counter = 0
@@ -571,25 +692,21 @@
         target_counter += 1
 
         row = [target]  # First column is platform name
-        default_toolchain = TARGET_MAP[target].default_toolchain
         for unique_toolchain in unique_supported_toolchains:
-            text = "-"
-            if default_toolchain == unique_toolchain:
-                text = "Default"
-                perm_counter += 1
-            elif unique_toolchain in TARGET_MAP[target].supported_toolchains:
+            if unique_toolchain in TARGET_MAP[target].supported_toolchains:
                 text = "Supported"
                 perm_counter += 1
+            else:
+                text = "-"
+            
             row.append(text)
         pt.add_row(row)
 
     result = pt.get_html_string() if verbose_html else pt.get_string()
     result += "\n"
-    result += "*Default - default on-line compiler\n"
-    result += "*Supported - supported off-line compiler\n"
-    result += "\n"
-    result += "Total platforms: %d\n"% (target_counter)
-    result += "Total permutations: %d"% (perm_counter)
+    result += "Supported targets: %d\n"% (target_counter)
+    if target_counter == 1:
+        result += "Supported toolchains: %d"% (perm_counter)
     return result
 
 
@@ -824,6 +941,66 @@
     with open(filename, 'w+') as f:
         f.write(template.render(failing_builds=build_report_failing, passing_builds=build_report_passing))
 
+
+def scan_for_source_paths(path, exclude_paths=None):
+    ignorepatterns = []
+    paths = []
+    
+    def is_ignored(file_path):
+        for pattern in ignorepatterns:
+            if fnmatch.fnmatch(file_path, pattern):
+                return True
+        return False
+    
+    
+    """ os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
+    When topdown is True, the caller can modify the dirnames list in-place
+    (perhaps using del or slice assignment), and walk() will only recurse into
+    the subdirectories whose names remain in dirnames; this can be used to prune
+    the search, impose a specific order of visiting, or even to inform walk()
+    about directories the caller creates or renames before it resumes walk()
+    again. Modifying dirnames when topdown is False is ineffective, because in
+    bottom-up mode the directories in dirnames are generated before dirpath
+    itself is generated.
+    """
+    for root, dirs, files in walk(path, followlinks=True):
+        # Remove ignored directories
+        # Check if folder contains .mbedignore
+        if ".mbedignore" in files :
+            with open (join(root,".mbedignore"), "r") as f:
+                lines=f.readlines()
+                lines = [l.strip() for l in lines] # Strip whitespaces
+                lines = [l for l in lines if l != ""] # Strip empty lines
+                lines = [l for l in lines if not re.match("^#",l)] # Strip comment lines
+                # Append root path to glob patterns
+                # and append patterns to ignorepatterns
+                ignorepatterns.extend([join(root,line.strip()) for line in lines])
+
+        for d in copy(dirs):
+            dir_path = join(root, d)
+
+            # Always ignore hidden directories
+            if d.startswith('.'):
+                dirs.remove(d)
+
+            # Remove dirs that already match the ignorepatterns
+            # to avoid travelling into them and to prevent them
+            # on appearing in include path.
+            if is_ignored(join(dir_path,"")):
+                dirs.remove(d)
+
+            if exclude_paths:
+                for exclude_path in exclude_paths:
+                    rel_path = relpath(dir_path, exclude_path)
+                    if not (rel_path.startswith('..')):
+                        dirs.remove(d)
+                        break
+
+        # Add root to include paths
+        paths.append(root)
+
+    return paths
+
 def get_build_profile(path):
     profile = None
     builds = get_build_ids(path)