__init__.py

00001 # mbed SDK
00002 # Copyright (c) 2011-2013 ARM Limited
00003 #
00004 # Licensed under the Apache License, Version 2.0 (the "License");
00005 # you may not use this file except in compliance with the License.
00006 # You may obtain a copy of the License at
00007 #
00008 #     http://www.apache.org/licenses/LICENSE-2.0
00009 #
00010 # Unless required by applicable law or agreed to in writing, software
00011 # distributed under the License is distributed on an "AS IS" BASIS,
00012 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013 # See the License for the specific language governing permissions and
00014 # limitations under the License.
00015 
00016 """
00017 # The scanning rules and Resources object.
00018 
00019 A project in Mbed OS contains metadata in the file system as directory names.
00020 These directory names adhere to a set of rules referred to as scanning rules.
00021 The following are the English version of the scanning rules:
00022 
00023 Directory names starting with "TEST_", "TARGET_", "TOOLCHAIN_" and "FEATURE_"
00024 are excluded from a build unless one of the following is true:
00025  * The suffix after "TARGET_" is a target label (see target.labels).
00026  * The suffix after "TOOLCHAIN_" is a toolchain label, defined by the
00027    inheritance hierarchy of the toolchain class.
00028  * The suffix after "FEATURE_" is a member of `target.features`.
00029 
00030 
00031 """
00032 
00033 from __future__ import print_function, division, absolute_import
00034 
00035 import fnmatch
00036 import re
00037 from collections import namedtuple, defaultdict
00038 from copy import copy
00039 from itertools import chain
00040 from os import walk, sep
00041 from os.path import (join, splitext, dirname, relpath, basename, split, normcase,
00042                      abspath, exists)
00043 
00044 from .ignore import MbedIgnoreSet, IGNORE_FILENAME
00045 
00046 # Support legacy build conventions: the original mbed build system did not have
00047 # standard labels for the "TARGET_" and "TOOLCHAIN_" specific directories, but
00048 # had the knowledge of a list of these directories to be ignored.
00049 LEGACY_IGNORE_DIRS = set([
00050     # Legacy Targets
00051     'LPC11U24',
00052     'LPC1768',
00053     'LPC2368',
00054     'LPC4088',
00055     'LPC812',
00056     'KL25Z',
00057 
00058     # Legacy Toolchains
00059     'ARM',
00060     'uARM',
00061     'IAR',
00062     'GCC_ARM',
00063     'GCC_CS',
00064     'GCC_CR',
00065     'GCC_CW',
00066     'GCC_CW_EWL',
00067     'GCC_CW_NEWLIB',
00068     'ARMC6',
00069 
00070     # Tests, here for simplicity
00071     'TESTS',
00072     'TEST_APPS',
00073 ])
00074 LEGACY_TOOLCHAIN_NAMES = {
00075     'ARM_STD':'ARM',
00076     'ARM_MICRO': 'uARM',
00077     'GCC_ARM': 'GCC_ARM',
00078     'GCC_CR': 'GCC_CR',
00079     'IAR': 'IAR',
00080     'ARMC6': 'ARMC6',
00081 }
00082 
00083 
00084 FileRef = namedtuple("FileRef", "name path")
00085 
00086 class FileType(object):
00087     C_SRC = "c"
00088     CPP_SRC = "c++"
00089     ASM_SRC = "s"
00090     HEADER = "header"
00091     INC_DIR = "inc"
00092     LIB_DIR = "libdir"
00093     LIB = "lib"
00094     OBJECT = "o"
00095     HEX = "hex"
00096     BIN = "bin"
00097     JSON = "json"
00098     LD_SCRIPT = "ld"
00099     LIB_REF = "libref"
00100     BLD_REF = "bldref"
00101     REPO_DIR = "repodir"
00102 
00103     def __init__(self):
00104         raise NotImplemented
00105 
00106 class Resources(object):
00107     ALL_FILE_TYPES = [
00108         FileType.C_SRC,
00109         FileType.CPP_SRC,
00110         FileType.ASM_SRC,
00111         FileType.HEADER,
00112         FileType.INC_DIR,
00113         FileType.LIB_DIR,
00114         FileType.LIB,
00115         FileType.OBJECT,
00116         FileType.HEX,
00117         FileType.BIN,
00118         FileType.JSON,
00119         FileType.LD_SCRIPT,
00120         FileType.LIB_REF,
00121         FileType.BLD_REF,
00122         FileType.REPO_DIR,
00123     ]
00124 
00125     def __init__(self, notify, collect_ignores=False):
00126         # publicly accessible things
00127         self.ignored_dirs = []
00128 
00129         # Pre-mbed 2.0 ignore dirs
00130         self._legacy_ignore_dirs = (LEGACY_IGNORE_DIRS)
00131 
00132         # Primate parameters
00133         self._notify = notify
00134         self._collect_ignores = collect_ignores
00135 
00136         # Storage for file references, indexed by file type
00137         self._file_refs = defaultdict(set)
00138 
00139         # Incremental scan related
00140         self._label_paths = []
00141         self._labels = {
00142             "TARGET": [], "TOOLCHAIN": [], "FEATURE": [], "COMPONENT": []
00143         }
00144         self._prefixed_labels = set()
00145 
00146         # Path seperator style (defaults to OS-specific seperator)
00147         self._sep = sep
00148 
00149         self._ignoreset = MbedIgnoreSet()
00150 
00151     def ignore_dir(self, directory):
00152         if self._collect_ignores:
00153             self.ignored_dirs.append(directory)
00154 
00155     def _collect_duplicates(self, dupe_dict, dupe_headers):
00156         for filename in self.s_sources + self.c_sources + self.cpp_sources:
00157             objname, _ = splitext(basename(filename))
00158             dupe_dict.setdefault(objname, set())
00159             dupe_dict[objname] |= set([filename])
00160         for filename in self.headers:
00161             headername = basename(filename)
00162             dupe_headers.setdefault(headername, set())
00163             dupe_headers[headername] |= set([headername])
00164         return dupe_dict, dupe_headers
00165 
00166     def detect_duplicates(self):
00167         """Detect all potential ambiguities in filenames and report them with
00168         a toolchain notification
00169         """
00170         count = 0
00171         dupe_dict, dupe_headers = self._collect_duplicates(dict(), dict())
00172         for objname, filenames in dupe_dict.items():
00173             if len(filenames) > 1:
00174                 count+=1
00175                 self._notify.tool_error(
00176                     "Object file %s.o is not unique! It could be made from: %s"\
00177                     % (objname, " ".join(filenames)))
00178         for headername, locations in dupe_headers.items():
00179             if len(locations) > 1:
00180                 count+=1
00181                 self._notify.tool_error(
00182                     "Header file %s is not unique! It could be: %s" %\
00183                     (headername, " ".join(locations)))
00184         return count
00185 
00186     def win_to_unix(self):
00187         self._sep = "/"
00188         if self._sep != sep:
00189             for file_type in self.ALL_FILE_TYPES:
00190                 v = [f._replace(name=f.name.replace(sep, self._sep)) for
00191                      f in self.get_file_refs(file_type)]
00192                 self._file_refs[file_type] = v
00193 
00194     def __str__(self):
00195         s = []
00196 
00197         for (label, file_type) in (
00198                 ('Include Directories', FileType.INC_DIR),
00199                 ('Headers', FileType.HEADER),
00200 
00201                 ('Assembly sources', FileType.ASM_SRC),
00202                 ('C sources', FileType.C_SRC),
00203                 ('C++ sources', FileType.CPP_SRC),
00204 
00205                 ('Library directories', FileType.LIB_DIR),
00206                 ('Objects', FileType.OBJECT),
00207                 ('Libraries', FileType.LIB),
00208 
00209                 ('Hex files', FileType.HEX),
00210                 ('Bin files', FileType.BIN),
00211                 ('Linker script', FileType.LD_SCRIPT)
00212             ):
00213             resources = self.get_file_refs(file_type)
00214             if resources:
00215                 s.append('%s:\n  ' % label + '\n  '.join(
00216                     "%s -> %s" % (name, path) for name, path in resources))
00217 
00218         return '\n'.join(s)
00219 
00220 
00221     def _add_labels(self, prefix, labels):
00222         self._labels[prefix].extend(labels)
00223         self._prefixed_labels |= set("%s_%s" % (prefix, label) for label in labels)
00224         for path, base_path, into_path in self._label_paths:
00225             if basename(path) in self._prefixed_labels:
00226                 self.add_directory(path, base_path, into_path)
00227         self._label_paths = [(p, b, i) for p, b, i in self._label_paths
00228                              if basename(p) not in self._prefixed_labels]
00229 
00230     def add_target_labels(self, target):
00231         self._add_labels("TARGET", target.labels)
00232         self._add_labels("COMPONENT", target.components)
00233         self.add_features(target.features)
00234 
00235     def add_features(self, features):
00236         self._add_labels("FEATURE", features)
00237 
00238     def add_toolchain_labels(self, toolchain):
00239         for prefix, value in toolchain.get_labels().items():
00240             self._add_labels(prefix, value)
00241         self._legacy_ignore_dirs -= set(
00242             [toolchain.target.name, LEGACY_TOOLCHAIN_NAMES[toolchain.name]])
00243 
00244     def add_ignore_patterns(self, root, base_path, patterns):
00245         real_base = relpath(root, base_path)
00246         self._ignoreset.add_ignore_patterns(real_base, patterns)
00247 
00248     def _not_current_label(self, dirname, label_type):
00249         return (dirname.startswith(label_type + "_") and
00250                 dirname[len(label_type) + 1:] not in self._labels[label_type])
00251 
00252     def add_file_ref(self, file_type, file_name, file_path):
00253         if sep != self._sep:
00254             ref = FileRef(file_name.replace(sep, self._sep), file_path)
00255         else:
00256             ref = FileRef(file_name, file_path)
00257         self._file_refs[file_type].add(ref)
00258 
00259     def get_file_refs(self, file_type):
00260         """Return a list of FileRef for every file of the given type"""
00261         return list(self._file_refs[file_type])
00262 
00263     def _all_parents(self, files):
00264         for name in files:
00265             components = name.split(self._sep)
00266             start_at = 2 if components[0] in set(['', '.']) else 1
00267             for index, directory in reversed(list(enumerate(components))[start_at:]):
00268                 if directory in self._prefixed_labels:
00269                     start_at = index + 1
00270                     break
00271             for n in range(start_at, len(components)):
00272                 parent = self._sep.join(components[:n])
00273                 yield parent
00274 
00275     def _get_from_refs(self, file_type, key):
00276         if file_type is FileType.INC_DIR:
00277             parents = set(self._all_parents(self._get_from_refs(
00278                 FileType.HEADER, key)))
00279             parents.add(".")
00280         else:
00281             parents = set()
00282         return sorted(
00283             list(parents) + [key(f) for f in self.get_file_refs(file_type)]
00284         )
00285 
00286 
00287     def get_file_names(self, file_type):
00288         return self._get_from_refs(file_type, lambda f: f.name)
00289 
00290     def get_file_paths(self, file_type):
00291         return self._get_from_refs(file_type, lambda f: f.path)
00292 
00293     def add_files_to_type(self, file_type, files):
00294         for f in files:
00295             self.add_file_ref(file_type, f, f)
00296 
00297     @property
00298     def inc_dirs(self):
00299         return self.get_file_names(FileType.INC_DIR)
00300 
00301     @property
00302     def headers(self):
00303         return self.get_file_names(FileType.HEADER)
00304 
00305     @property
00306     def s_sources(self):
00307         return self.get_file_names(FileType.ASM_SRC)
00308 
00309     @property
00310     def c_sources(self):
00311         return self.get_file_names(FileType.C_SRC)
00312 
00313     @property
00314     def cpp_sources(self):
00315         return self.get_file_names(FileType.CPP_SRC)
00316 
00317     @property
00318     def lib_dirs(self):
00319         return self.get_file_names(FileType.LIB_DIR)
00320 
00321     @property
00322     def objects(self):
00323         return self.get_file_names(FileType.OBJECT)
00324 
00325     @property
00326     def libraries(self):
00327         return self.get_file_names(FileType.LIB)
00328 
00329     @property
00330     def lib_builds(self):
00331         return self.get_file_names(FileType.BLD_REF)
00332 
00333     @property
00334     def lib_refs(self):
00335         return self.get_file_names(FileType.LIB_REF)
00336 
00337     @property
00338     def linker_script(self):
00339         options = self.get_file_names(FileType.LD_SCRIPT)
00340         if options:
00341             return options[0]
00342         else:
00343             return None
00344 
00345     @property
00346     def hex_files(self):
00347         return self.get_file_names(FileType.HEX)
00348 
00349     @property
00350     def bin_files(self):
00351         return self.get_file_names(FileType.BIN)
00352 
00353     @property
00354     def json_files(self):
00355         return self.get_file_names(FileType.JSON)
00356 
00357     def add_directory(
00358             self,
00359             path,
00360             base_path=None,
00361             into_path=None,
00362             exclude_paths=None,
00363     ):
00364         """ Scan a directory and include its resources in this resources obejct
00365 
00366         Positional arguments:
00367         path - the path to search for resources
00368 
00369         Keyword arguments
00370         base_path - If this is part of an incremental scan, include the origin
00371                     directory root of the scan here
00372         into_path - Pretend that scanned files are within the specified
00373                     directory within a project instead of using their actual path
00374         exclude_paths - A list of paths that are to be excluded from a build
00375         """
00376         self._notify.progress("scan", abspath(path))
00377 
00378         if base_path is None:
00379             base_path = path
00380         if into_path is None:
00381             into_path = path
00382         if self._collect_ignores and path in self.ignored_dirs:
00383             self.ignored_dirs.remove(path)
00384         if exclude_paths:
00385             self.add_ignore_patterns(
00386                 path, base_path, [join(e, "*") for e in exclude_paths])
00387 
00388         for root, dirs, files in walk(path, followlinks=True):
00389             # Check if folder contains .mbedignore
00390             if IGNORE_FILENAME in files:
00391                 real_base = relpath(root, base_path)
00392                 self._ignoreset.add_mbedignore(
00393                     real_base, join(root, IGNORE_FILENAME))
00394 
00395             root_path =join(relpath(root, base_path))
00396             if self._ignoreset.is_ignored(join(root_path,"")):
00397                 self.ignore_dir(root_path)
00398                 dirs[:] = []
00399                 continue
00400 
00401             for d in copy(dirs):
00402                 dir_path = join(root, d)
00403                 if d == '.hg' or d == '.git':
00404                     fake_path = join(into_path, relpath(dir_path, base_path))
00405                     self.add_file_ref(FileType.REPO_DIR, fake_path, dir_path)
00406 
00407                 if (any(self._not_current_label(d, t) for t
00408                         in self._labels.keys())):
00409                     self._label_paths.append((dir_path, base_path, into_path))
00410                     self.ignore_dir(dir_path)
00411                     dirs.remove(d)
00412                 elif (d.startswith('.') or d in self._legacy_ignore_dirs or
00413                       self._ignoreset.is_ignored(join(root_path, d, ""))):
00414                     self.ignore_dir(dir_path)
00415                     dirs.remove(d)
00416 
00417             # Add root to include paths
00418             root = root.rstrip("/")
00419 
00420             for file in files:
00421                 file_path = join(root, file)
00422                 self._add_file(file_path, base_path, into_path)
00423 
00424     _EXT = {
00425         ".c": FileType.C_SRC,
00426         ".cc": FileType.CPP_SRC,
00427         ".cpp": FileType.CPP_SRC,
00428         ".s": FileType.ASM_SRC,
00429         ".h": FileType.HEADER,
00430         ".hh": FileType.HEADER,
00431         ".hpp": FileType.HEADER,
00432         ".o": FileType.OBJECT,
00433         ".hex": FileType.HEX,
00434         ".bin": FileType.BIN,
00435         ".json": FileType.JSON,
00436         ".a": FileType.LIB,
00437         ".ar": FileType.LIB,
00438         ".sct": FileType.LD_SCRIPT,
00439         ".ld": FileType.LD_SCRIPT,
00440         ".icf": FileType.LD_SCRIPT,
00441         ".lib": FileType.LIB_REF,
00442         ".bld": FileType.BLD_REF,
00443     }
00444 
00445     _DIR_EXT = {
00446         ".a": FileType.LIB_DIR,
00447         ".ar": FileType.LIB_DIR,
00448     }
00449 
00450     def _add_file(self, file_path, base_path, into_path):
00451         """ Add a single file into the resources object that was found by
00452         scanning starting as base_path
00453         """
00454 
00455         if  (self._ignoreset.is_ignored(relpath(file_path, base_path)) or
00456              basename(file_path).startswith(".")):
00457             self.ignore_dir(relpath(file_path, base_path))
00458             return
00459 
00460         fake_path = join(into_path, relpath(file_path, base_path))
00461         _, ext = splitext(file_path)
00462         try:
00463             file_type = self._EXT[ext.lower()]
00464             self.add_file_ref(file_type, fake_path, file_path)
00465         except KeyError:
00466             pass
00467         try:
00468             dir_type = self._DIR_EXT[ext.lower()]
00469             self.add_file_ref(dir_type, dirname(fake_path), dirname(file_path))
00470         except KeyError:
00471             pass
00472 
00473 
00474     def scan_with_toolchain(self, src_paths, toolchain, dependencies_paths=None,
00475                             inc_dirs=None, exclude=True):
00476         """ Scan resources using initialized toolcain
00477 
00478         Positional arguments
00479         src_paths - the paths to source directories
00480         toolchain - valid toolchain object
00481 
00482         Keyword arguments
00483         dependencies_paths - dependency paths that we should scan for include dirs
00484         inc_dirs - additional include directories which should be added to
00485                    the scanner resources
00486         exclude - Exclude the toolchain's build directory from the resources
00487         """
00488         self.add_toolchain_labels(toolchain)
00489         for path in src_paths:
00490             if exists(path):
00491                 into_path = relpath(path).strip(".\\/")
00492                 if exclude:
00493                     self.add_directory(
00494                         path,
00495                         into_path=into_path,
00496                         exclude_paths=[toolchain.build_dir]
00497                     )
00498                 else:
00499                     self.add_directory(path, into_path=into_path)
00500 
00501         # Scan dependency paths for include dirs
00502         if dependencies_paths is not None:
00503             toolchain.progress("dep", dependencies_paths)
00504             for dep in dependencies_paths:
00505                 lib_self = self.__class__(self._notify, self._collect_ignores)\
00506                                .scan_with_toolchain([dep], toolchain)
00507                 self.inc_dirs.extend(lib_self.inc_dirs)
00508 
00509         # Add additional include directories if passed
00510         if inc_dirs:
00511             if isinstance(inc_dirs, list):
00512                 self.inc_dirs.extend(inc_dirs)
00513             else:
00514                 self.inc_dirs.append(inc_dirs)
00515 
00516         # Load self into the config system which might expand/modify self
00517         # based on config data
00518         toolchain.config.load_resources(self)
00519 
00520         # Set the toolchain's configuration data
00521         toolchain.set_config_data(toolchain.config.get_config_data())
00522 
00523         return self
00524 
00525     def scan_with_config(self, src_paths, config):
00526         if config.target:
00527             self.add_target_labels(config.target)
00528         for path in src_paths:
00529             if exists(path):
00530                 self.add_directory(path)
00531         config.load_resources(self)
00532         return self
00533