__init__.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2016 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009 http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 from __future__ import print_function, division, absolute_import
00018 
00019 from copy import deepcopy
00020 from six import moves
00021 import json
00022 import six
00023 import os
00024 from os.path import dirname, abspath, exists, join, isabs
00025 import sys
00026 from collections import namedtuple
00027 from os.path import splitext, relpath
00028 from intelhex import IntelHex
00029 from jinja2 import FileSystemLoader, StrictUndefined
00030 from jinja2.environment import Environment
00031 from jsonschema import Draft4Validator, RefResolver
00032 
00033 from ..utils import (json_file_to_dict, intelhex_offset, integer,
00034                      NotSupportedException)
00035 from ..arm_pack_manager import Cache
00036 from ..targets import (CUMULATIVE_ATTRIBUTES, TARGET_MAP, generate_py_target,
00037                        get_resolution_order, Target)
00038 
00039 try:
00040     unicode
00041 except NameError:
00042     unicode = str
00043 PATH_OVERRIDES = set(["target.bootloader_img"])
00044 BOOTLOADER_OVERRIDES = set(["target.bootloader_img", "target.restrict_size",
00045                             "target.header_format", "target.header_offset",
00046                             "target.app_offset",
00047                             "target.mbed_app_start", "target.mbed_app_size"])
00048 
00049 
00050 # Base class for all configuration exceptions
00051 class ConfigException (Exception):
00052     """Config system only exception. Makes it easier to distinguish config
00053     errors"""
00054     pass
00055 
00056 class ConfigParameter (object):
00057     """This class keeps information about a single configuration parameter"""
00058 
00059     def __init__ (self, name, data, unit_name, unit_kind):
00060         """Construct a ConfigParameter
00061 
00062         Positional arguments:
00063         name - the name of the configuration parameter
00064         data - the data associated with the configuration parameter
00065         unit_name - the unit (target/library/application) that defines this
00066                     parameter
00067         unit_ kind - the kind of the unit ("target", "library" or "application")
00068         """
00069         self.name  = self.get_full_name (name, unit_name, unit_kind,
00070                                        allow_prefix=False)
00071         self.defined_by  = self.get_display_name (unit_name, unit_kind)
00072         self.set_value (data.get("value", None), unit_name, unit_kind)
00073         self.help_text  = data.get("help", None)
00074         self.required  = data.get("required", False)
00075         self.macro_name  = data.get("macro_name", "MBED_CONF_%s" %
00076                                    self.sanitize (self.name .upper()))
00077         self.config_errors  = []
00078 
00079     @staticmethod
00080     def get_full_name (name, unit_name, unit_kind, label=None,
00081                       allow_prefix=True):
00082         """Return the full (prefixed) name of a parameter. If the parameter
00083         already has a prefix, check if it is valid
00084 
00085         Positional arguments:
00086         name - the simple (unqualified) name of the parameter
00087         unit_name - the unit (target/library/application) that defines this
00088                     parameter
00089         unit_kind - the kind of the unit ("target", "library" or "application")
00090 
00091         Keyword arguments:
00092         label - the name of the label in the 'target_config_overrides' section
00093         allow_prefix - True to allow the original name to have a prefix, False
00094                        otherwise
00095         """
00096         if name.find('.') == -1: # the name is not prefixed
00097             if unit_kind == "target":
00098                 prefix = "target."
00099             elif unit_kind == "application":
00100                 prefix = "app."
00101             else:
00102                 prefix = unit_name + '.'
00103             return prefix + name
00104         if name in BOOTLOADER_OVERRIDES:
00105             return name
00106         # The name has a prefix, so check if it is valid
00107         if not allow_prefix:
00108             raise ConfigException("Invalid parameter name '%s' in '%s'" %
00109                                   (name, ConfigParameter.get_display_name(
00110                                       unit_name, unit_kind, label)))
00111         temp = name.split(".")
00112         # Check if the parameter syntax is correct (must be
00113         # unit_name.parameter_name)
00114         if len(temp) != 2:
00115             raise ConfigException("Invalid parameter name '%s' in '%s'" %
00116                                   (name, ConfigParameter.get_display_name(
00117                                       unit_name, unit_kind, label)))
00118         prefix = temp[0]
00119         # Check if the given parameter prefix matches the expected prefix
00120         if (unit_kind == "library" and prefix != unit_name) or \
00121            (unit_kind == "target" and prefix != "target"):
00122             raise ConfigException(
00123                 "Invalid prefix '%s' for parameter name '%s' in '%s'" %
00124                 (prefix, name, ConfigParameter.get_display_name(
00125                     unit_name, unit_kind, label)))
00126         return name
00127 
00128     @staticmethod
00129     def get_display_name (unit_name, unit_kind, label=None):
00130         """Return the name displayed for a unit when interrogating the origin
00131         and the last set place of a parameter
00132 
00133         Positional arguments:
00134         unit_name - the unit (target/library/application) that defines this
00135                     parameter
00136         unit_kind - the kind of the unit ("target", "library" or "application")
00137 
00138         Keyword arguments:
00139         label - the name of the label in the 'target_config_overrides' section
00140         """
00141         if unit_kind == "target":
00142             return "target:" + unit_name
00143         elif unit_kind == "application":
00144             return "application%s" % ("[%s]" % label if label else "")
00145         else: # library
00146             return "library:%s%s" % (unit_name, "[%s]" % label if label else "")
00147 
00148     @staticmethod
00149     def sanitize (name):
00150         """ "Sanitize" a name so that it is a valid C macro name. Currently it
00151         simply replaces '.' and '-' with '_'.
00152 
00153         Positional arguments:
00154         name - the name to make into a valid C macro
00155         """
00156         return name.replace('.', '_').replace('-', '_')
00157 
00158     def set_value (self, value, unit_name, unit_kind, label=None):
00159         """ Sets a value for this parameter, remember the place where it was
00160         set.  If the value is a Boolean, it is converted to 1 (for True) or
00161         to 0 (for False).
00162 
00163         Positional arguments:
00164         value - the value of the parameter
00165         unit_name - the unit (target/library/application) that defines this
00166                    parameter
00167         unit_kind - the kind of the unit ("target", "library" or "application")
00168 
00169         Keyword arguments:
00170         label - the name of the label in the 'target_config_overrides' section
00171                (optional)
00172         """
00173         self.value  = int(value) if isinstance(value, bool) else value
00174         self.set_by  = self.get_display_name (unit_name, unit_kind, label)
00175 
00176     def __str__ (self):
00177         """Return the string representation of this configuration parameter
00178 
00179         Arguments: None
00180         """
00181         if self.value  is not None:
00182             return '%s = %s (macro name: "%s")' % \
00183                 (self.name , self.value , self.macro_name )
00184         else:
00185             return '%s has no value' % self.name 
00186 
00187     def get_verbose_description (self):
00188         """Return a verbose description of this configuration parameter as a
00189         string
00190 
00191         Arguments: None
00192         """
00193         desc = "Name: %s%s\n" % \
00194                (self.name , " (required parameter)" if self.required  else "")
00195         if self.help_text :
00196             desc += "    Description: %s\n" % self.help_text 
00197         desc += "    Defined by: %s\n" % self.defined_by 
00198         if not self.value :
00199             return desc + "    No value set"
00200         desc += "    Macro name: %s\n" % self.macro_name 
00201         desc += "    Value: %s (set by %s)" % (self.value , self.set_by )
00202         return desc
00203 
00204 class ConfigMacro (object):
00205     """ A representation of a configuration macro. It handles both macros
00206     without a value (MACRO) and with a value (MACRO=VALUE)
00207     """
00208     def __init__ (self, name, unit_name, unit_kind):
00209         """Construct a ConfigMacro object
00210 
00211         Positional arguments:
00212         name - the macro's name
00213         unit_name - the location where the macro was defined
00214         unit_kind - the type of macro this is
00215         """
00216         self.name  = name
00217         self.defined_by  = ConfigParameter.get_display_name(unit_name, unit_kind)
00218         if name.find("=") != -1:
00219             tmp = name.split("=")
00220             if len(tmp) != 2:
00221                 raise ValueError("Invalid macro definition '%s' in '%s'" %
00222                                  (name, self.defined_by ))
00223             self.macro_name  = tmp[0]
00224             self.macro_value  = tmp[1]
00225         else:
00226             self.macro_name  = name
00227             self.macro_value  = None
00228 
00229 class ConfigCumulativeOverride (object):
00230     """Representation of overrides for cumulative attributes"""
00231     def __init__ (self, name, additions=None, removals=None, strict=False):
00232         """Construct a ConfigCumulativeOverride object
00233 
00234         Positional arguments:
00235         name - the name of the config file this came from ?
00236 
00237         Keyword arguments:
00238         additions - macros to add to the overrides
00239         removals - macros to remove from the overrides
00240         strict - Boolean indicating that attempting to remove from an override
00241                  that does not exist should error
00242         """
00243         self.name  = name
00244         if additions:
00245             self.additions  = set(additions)
00246         else:
00247             self.additions  = set()
00248         if removals:
00249             self.removals  = set(removals)
00250         else:
00251             self.removals  = set()
00252         self.strict  = strict
00253 
00254     def remove_cumulative_overrides (self, overrides):
00255         """Extend the list of override removals.
00256 
00257         Positional arguments:
00258         overrides - a list of names that, when the override is evaluated, will
00259                     be removed
00260         """
00261         for override in overrides:
00262             if override in self.additions :
00263                 raise ConfigException(
00264                     "Configuration conflict. The %s %s both added and removed."
00265                     % (self.name [:-1], override))
00266 
00267         self.removals  |= set(overrides)
00268 
00269     def add_cumulative_overrides (self, overrides):
00270         """Extend the list of override additions.
00271 
00272         Positional arguments:
00273         overrides - a list of a names that, when the override is evaluated, will
00274                     be added to the list
00275         """
00276         for override in overrides:
00277             if override in self.removals  or \
00278                (self.strict  and override not in self.additions ):
00279                 raise ConfigException(
00280                     "Configuration conflict. The %s %s both added and removed."
00281                     % (self.name [:-1], override))
00282 
00283         self.additions  |= set(overrides)
00284 
00285     def strict_cumulative_overrides (self, overrides):
00286         """Remove all overrides that are not the specified ones
00287 
00288         Positional arguments:
00289         overrides - a list of names that will replace the entire attribute when
00290                     this override is evaluated.
00291         """
00292         self.remove_cumulative_overrides (self.additions  - set(overrides))
00293         self.add_cumulative_overrides (overrides)
00294         self.strict  = True
00295 
00296     def update_target (self, target):
00297         """Update the attributes of a target based on this override"""
00298         setattr(target, self.name ,
00299                 list((set(getattr(target, self.name , []))
00300                       | self.additions ) - self.removals ))
00301 
00302 
00303 def _process_config_parameters(data, params, unit_name, unit_kind):
00304     """Process a "config_parameters" section in either a target, a library,
00305     or the application.
00306 
00307     Positional arguments:
00308     data - a dictionary with the configuration parameters
00309     params - storage for the discovered configuration parameters
00310     unit_name - the unit (target/library/application) that defines this
00311                 parameter
00312     unit_kind - the kind of the unit ("target", "library" or "application")
00313     """
00314     for name, val in data.items():
00315         full_name = ConfigParameter.get_full_name(name, unit_name, unit_kind)
00316         # If the parameter was already defined, raise an error
00317         if full_name in params:
00318             raise ConfigException(
00319                 "Parameter name '%s' defined in both '%s' and '%s'" %
00320                 (name, ConfigParameter.get_display_name(unit_name, unit_kind),
00321                  params[full_name].defined_by))
00322         # Otherwise add it to the list of known parameters
00323         # If "val" is not a dictionary, this is a shortcut definition,
00324         # otherwise it is a full definition
00325         params[full_name] = ConfigParameter(name, val if isinstance(val, dict)
00326                                             else {"value": val}, unit_name,
00327                                             unit_kind)
00328     return params
00329 
00330 
00331 def _process_macros(mlist, macros, unit_name, unit_kind):
00332     """Process a macro definition and check for incompatible duplicate
00333     definitions.
00334 
00335     Positional arguments:
00336     mlist - list of macro names to process
00337     macros - dictionary with currently discovered macros
00338     unit_name - the unit (library/application) that defines this macro
00339     unit_kind - the kind of the unit ("library" or "application")
00340     """
00341     for mname in mlist:
00342         macro = ConfigMacro(mname, unit_name, unit_kind)
00343         if (macro.macro_name in macros) and \
00344            (macros[macro.macro_name].name != mname):
00345             # Found an incompatible definition of the macro in another module,
00346             # so raise an error
00347             full_unit_name = ConfigParameter.get_display_name(unit_name,
00348                                                               unit_kind)
00349             raise ConfigException(
00350                 ("Macro '%s' defined in both '%s' and '%s'"
00351                  % (macro.macro_name, macros[macro.macro_name].defined_by,
00352                     full_unit_name)) +
00353                 " with incompatible values")
00354         macros[macro.macro_name] = macro
00355 
00356 
00357 Region = namedtuple("Region", "name start size active filename")
00358 
00359 class Config (object):
00360     """'Config' implements the mbed configuration mechanism"""
00361 
00362     # Libraries and applications have different names for their configuration
00363     # files
00364     __mbed_app_config_name = "mbed_app.json"
00365     __mbed_lib_config_name = "mbed_lib.json"
00366 
00367     __unused_overrides = set(["target.bootloader_img", "target.restrict_size",
00368                               "target.mbed_app_start", "target.mbed_app_size"])
00369 
00370     # Allowed features in configurations
00371     __allowed_features = [
00372         "UVISOR", "BLE", "CLIENT", "IPV4", "LWIP", "COMMON_PAL", "STORAGE", "NANOSTACK",
00373         # Nanostack configurations
00374         "LOWPAN_BORDER_ROUTER", "LOWPAN_HOST", "LOWPAN_ROUTER", "NANOSTACK_FULL", "THREAD_BORDER_ROUTER", "THREAD_END_DEVICE", "THREAD_ROUTER", "ETHERNET_HOST"
00375         ]
00376 
00377     @classmethod
00378     def find_app_config(cls, top_level_dirs):
00379         app_config_location = None
00380         for directory in top_level_dirs:
00381             full_path = os.path.join(directory, cls.__mbed_app_config_name)
00382             if os.path.isfile(full_path):
00383                 if app_config_location is not None:
00384                     raise ConfigException("Duplicate '%s' file in '%s' and '%s'"
00385                                             % (cls.__mbed_app_config_name,
00386                                                cls.app_config_location, full_path))
00387                 else:
00388                     app_config_location = full_path
00389         return app_config_location
00390 
00391     def format_validation_error(self, error, path):
00392         if error.context:
00393             return self.format_validation_error (error.context[0], path)
00394         else:
00395             return "in {} element {}: {}".format(
00396                 path, str(".".join(str(p) for p in error.absolute_path)), error.message)
00397 
00398     def __init__ (self, tgt, top_level_dirs=None, app_config=None):
00399         """Construct a mbed configuration
00400 
00401         Positional arguments:
00402         target - the name of the mbed target used for this configuration
00403                  instance
00404 
00405         Keyword argumets:
00406         top_level_dirs - a list of top level source directories (where
00407                          mbed_app_config.json could be found)
00408         app_config - location of a chosen mbed_app.json file
00409 
00410         NOTE: Construction of a Config object will look for the application
00411         configuration file in top_level_dirs. If found once, it'll parse it.
00412         top_level_dirs may be None (in this case, the constructor will not
00413         search for a configuration file).
00414         """
00415         config_errors = []
00416         self.app_config_location  = app_config
00417         if self.app_config_location  is None and top_level_dirs:
00418             self.app_config_location  = self.find_app_config (top_level_dirs)
00419         try:
00420             self.app_config_data  = json_file_to_dict(self.app_config_location ) \
00421                                    if self.app_config_location  else {}
00422         except ValueError as exc:
00423             self.app_config_data  = {}
00424             config_errors.append(
00425                 ConfigException("Could not parse mbed app configuration from %s"
00426                                 % self.app_config_location ))
00427 
00428 
00429         if self.app_config_location  is not None:
00430             # Validate the format of the JSON file based on schema_app.json
00431             schema_root = os.path.dirname(os.path.abspath(__file__))
00432             schema_path = os.path.join(schema_root, "schema_app.json")
00433             schema      = json_file_to_dict(schema_path)
00434 
00435             url = moves.urllib.request.pathname2url(schema_path)
00436             uri = moves.urllib_parse.urljoin("file://", url)
00437 
00438             resolver = RefResolver(uri, schema)
00439             validator = Draft4Validator(schema, resolver=resolver)
00440 
00441             errors = sorted(validator.iter_errors(self.app_config_data ))
00442 
00443             if errors:
00444                 raise ConfigException("; ".join(
00445                     self.format_validation_error (x, self.app_config_location )
00446                     for x in errors))
00447 
00448         # Update the list of targets with the ones defined in the application
00449         # config, if applicable
00450         self.lib_config_data  = {}
00451         # Make sure that each config is processed only once
00452         self.processed_configs  = {}
00453         if isinstance(tgt, Target):
00454             self.target  = tgt
00455         else:
00456             if tgt in TARGET_MAP:
00457                 self.target  = TARGET_MAP[tgt]
00458             else:
00459                 self.target  = generate_py_target(
00460                     self.app_config_data .get("custom_targets", {}), tgt)
00461         self.target  = deepcopy(self.target )
00462         self.target_labels  = self.target .labels
00463         for override in BOOTLOADER_OVERRIDES:
00464             _, attr = override.split(".")
00465             setattr(self.target , attr, None)
00466 
00467         self.cumulative_overrides  = {key: ConfigCumulativeOverride(key)
00468                                      for key in CUMULATIVE_ATTRIBUTES}
00469 
00470         self._process_config_and_overrides (self.app_config_data , {}, "app",
00471                                            "application")
00472         self.config_errors  = config_errors
00473 
00474     def add_config_files (self, flist):
00475         """Add configuration files
00476 
00477         Positional arguments:
00478         flist - a list of files to add to this configuration
00479         """
00480         for config_file in flist:
00481             if not config_file.endswith(self.__mbed_lib_config_name ):
00482                 continue
00483             full_path = os.path.normpath(os.path.abspath(config_file))
00484             # Check that we didn't already process this file
00485             if full_path in self.processed_configs :
00486                 continue
00487             self.processed_configs [full_path] = True
00488             # Read the library configuration and add a "__full_config_path"
00489             # attribute to it
00490             try:
00491                 cfg = json_file_to_dict(config_file)
00492             except ValueError as exc:
00493                 raise ConfigException(str(exc))
00494 
00495             # Validate the format of the JSON file based on the schema_lib.json
00496             schema_root = os.path.dirname(os.path.abspath(__file__))
00497             schema_path = os.path.join(schema_root, "schema_lib.json")
00498             schema_file = json_file_to_dict(schema_path)
00499 
00500             url = moves.urllib.request.pathname2url(schema_path)
00501             uri = moves.urllib_parse.urljoin("file://", url)
00502 
00503             resolver = RefResolver(uri, schema_file)
00504             validator = Draft4Validator(schema_file, resolver=resolver)
00505 
00506             errors = sorted(validator.iter_errors(cfg))
00507 
00508             if errors:
00509                 raise ConfigException("; ".join(
00510                     self.format_validation_error (x, config_file)
00511                     for x in errors))
00512 
00513             cfg["__config_path"] = full_path
00514 
00515             # If there's already a configuration for a module with the same
00516             # name, exit with error
00517             if cfg["name"] in self.lib_config_data :
00518                 raise ConfigException(
00519                     "Library name '%s' is not unique (defined in '%s' and '%s')"
00520                     % (cfg["name"], full_path,
00521                        self.lib_config_data [cfg["name"]]["__config_path"]))
00522             self.lib_config_data [cfg["name"]] = cfg
00523 
00524     @property
00525     def has_regions (self):
00526         """Does this config have regions defined?"""
00527         for override in BOOTLOADER_OVERRIDES:
00528             _, attr = override.split(".")
00529             if getattr(self.target , attr, None):
00530                 return True
00531         return False
00532 
00533     @property
00534     def sectors (self):
00535         """Return a list of tuples of sector start,size"""
00536         cache = Cache(False, False)
00537         if self.target .device_name not in cache.index:
00538             raise ConfigException("Bootloader not supported on this target: "
00539                                   "targets.json `device_name` not found in "
00540                                   "arm_pack_manager index.")
00541         cmsis_part = cache.index[self.target .device_name]
00542         sectors = cmsis_part['sectors']
00543         if sectors:
00544             return sectors
00545         raise ConfigException("No sector info available")
00546 
00547     @property
00548     def regions (self):
00549         """Generate a list of regions from the config"""
00550         if not self.target .bootloader_supported:
00551             raise ConfigException("Bootloader not supported on this target.")
00552         if not hasattr(self.target , "device_name"):
00553             raise ConfigException("Bootloader not supported on this target: "
00554                                   "targets.json `device_name` not specified.")
00555         cache = Cache(False, False)
00556         if self.target .device_name not in cache.index:
00557             raise ConfigException("Bootloader not supported on this target: "
00558                                   "targets.json `device_name` not found in "
00559                                   "arm_pack_manager index.")
00560         cmsis_part = cache.index[self.target .device_name]
00561         if  ((self.target .bootloader_img or self.target .restrict_size) and
00562              (self.target .mbed_app_start or self.target .mbed_app_size)):
00563             raise ConfigException(
00564                 "target.bootloader_img and target.restirct_size are "
00565                 "incompatible with target.mbed_app_start and "
00566                 "target.mbed_app_size")
00567         try:
00568             rom_size = int(cmsis_part['memory']['IROM1']['size'], 0)
00569             rom_start = int(cmsis_part['memory']['IROM1']['start'], 0)
00570         except KeyError:
00571             try:
00572                 rom_size = int(cmsis_part['memory']['PROGRAM_FLASH']['size'], 0)
00573                 rom_start = int(cmsis_part['memory']['PROGRAM_FLASH']['start'], 0)
00574             except KeyError:
00575                 raise ConfigException("Not enough information in CMSIS packs to "
00576                                       "build a bootloader project")
00577         if self.target .bootloader_img or self.target .restrict_size:
00578             return self._generate_bootloader_build (rom_start, rom_size)
00579         elif self.target .mbed_app_start or self.target .mbed_app_size:
00580             return self._generate_linker_overrides (rom_start, rom_size)
00581         else:
00582             raise ConfigException(
00583                 "Bootloader build requested but no bootlader configuration")
00584 
00585     @staticmethod
00586     def header_member_size(member):
00587         _, _, subtype, _ = member
00588         try:
00589             return int(subtype[:-2]) // 8
00590         except:
00591             if subtype.startswith("CRCITT32"):
00592                 return 32 // 8
00593             elif subtype == "SHA256":
00594                 return 256 // 8
00595             elif subtype == "SHA512":
00596                 return 512 // 8
00597             else:
00598                 raise ValueError("target.header_format: subtype %s is not "
00599                                  "understood" % subtype)
00600 
00601     @staticmethod
00602     def _header_size(format):
00603         return sum(Config.header_member_size(m) for m in format)
00604 
00605     def _make_header_region(self, start, header_format, offset=None):
00606         size = self._header_size (header_format)
00607         region = Region("header", start, size, False, None)
00608         start += size
00609         start = ((start + 7) // 8) * 8
00610         return (start, region)
00611 
00612     @staticmethod
00613     def _assign_new_offset(rom_start, start, new_offset, region_name):
00614         newstart = rom_start + integer(new_offset, 0)
00615         if newstart < start:
00616             raise ConfigException(
00617                 "Can not place % region inside previous region" % region_name)
00618         return newstart
00619 
00620     def _generate_bootloader_build(self, rom_start, rom_size):
00621         start = rom_start
00622         rom_end = rom_start + rom_size
00623         if self.target .bootloader_img:
00624             if isabs(self.target .bootloader_img):
00625                 filename = self.target .bootloader_img
00626             else:
00627                 basedir = abspath(dirname(self.app_config_location ))
00628                 filename = join(basedir, self.target .bootloader_img)
00629             if not exists(filename):
00630                 raise ConfigException("Bootloader %s not found" % filename)
00631             part = intelhex_offset(filename, offset=rom_start)
00632             if part.minaddr() != rom_start:
00633                 raise ConfigException("bootloader executable does not "
00634                                       "start at 0x%x" % rom_start)
00635             part_size = (part.maxaddr() - part.minaddr()) + 1
00636             part_size = Config._align_ceiling(rom_start + part_size, self.sectors ) - rom_start
00637             yield Region("bootloader", rom_start, part_size, False,
00638                          filename)
00639             start = rom_start + part_size
00640             if self.target .header_format:
00641                 if self.target .header_offset:
00642                     start = self._assign_new_offset (
00643                         rom_start, start, self.target .header_offset, "header")
00644                 start, region = self._make_header_region (
00645                     start, self.target .header_format)
00646                 yield region._replace(filename=self.target .header_format)
00647         if self.target .restrict_size is not None:
00648             new_size = int(self.target .restrict_size, 0)
00649             new_size = Config._align_floor(start + new_size, self.sectors ) - start
00650             yield Region("application", start, new_size, True, None)
00651             start += new_size
00652             if self.target .header_format:
00653                 if self.target .header_offset:
00654                     start = self._assign_new_offset (
00655                         rom_start, start, self.target .header_offset, "header")
00656                 start, region = self._make_header_region (
00657                     start, self.target .header_format)
00658                 yield region
00659             if self.target .app_offset:
00660                 start = self._assign_new_offset (
00661                     rom_start, start, self.target .app_offset, "application")
00662             yield Region("post_application", start, rom_end - start,
00663                          False, None)
00664         else:
00665             if self.target .app_offset:
00666                 start = self._assign_new_offset (
00667                     rom_start, start, self.target .app_offset, "application")
00668             yield Region("application", start, rom_end - start,
00669                          True, None)
00670         if start > rom_start + rom_size:
00671             raise ConfigException("Not enough memory on device to fit all "
00672                                   "application regions")
00673     
00674     @staticmethod
00675     def _find_sector(address, sectors):
00676         target_size = -1
00677         target_start = -1
00678         for (start, size) in sectors:
00679             if address < start:
00680                 break
00681             target_start = start
00682             target_size = size
00683         if (target_size < 0):
00684             raise ConfigException("No valid sector found")
00685         return target_start, target_size
00686         
00687     @staticmethod
00688     def _align_floor(address, sectors):
00689         target_start, target_size = Config._find_sector(address, sectors)
00690         sector_num = (address - target_start) // target_size
00691         return target_start + (sector_num * target_size)
00692     
00693     @staticmethod
00694     def _align_ceiling(address, sectors):
00695         target_start, target_size = Config._find_sector(address, sectors)
00696         sector_num = ((address - target_start) + target_size - 1) // target_size
00697         return target_start + (sector_num * target_size)
00698 
00699     @property
00700     def report(self):
00701         return {'app_config': self.app_config_location ,
00702                 'library_configs': map(relpath, self.processed_configs .keys())}
00703 
00704     def _generate_linker_overrides(self, rom_start, rom_size):
00705         if self.target .mbed_app_start is not None:
00706             start = int(self.target .mbed_app_start, 0)
00707         else:
00708             start = rom_start
00709         if self.target .mbed_app_size is not None:
00710             size = int(self.target .mbed_app_size, 0)
00711         else:
00712             size = (rom_size + rom_start) - start
00713         if start < rom_start:
00714             raise ConfigException("Application starts before ROM")
00715         if size + start > rom_size + rom_start:
00716             raise ConfigException("Application ends after ROM")
00717         yield Region("application", start, size, True, None)
00718 
00719     def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
00720         """Process "config_parameters" and "target_config_overrides" into a
00721         given dictionary
00722 
00723         Positional arguments:
00724         data - the configuration data of the library/appliation
00725         params - storage for the discovered configuration parameters
00726         unit_name - the unit (library/application) that defines this parameter
00727         unit_kind - the kind of the unit ("library" or "application")
00728         """
00729         self.config_errors  = []
00730         _process_config_parameters(data.get("config", {}), params, unit_name,
00731                                    unit_kind)
00732         for label, overrides in data.get("target_overrides", {}).items():
00733             # If the label is defined by the target or it has the special value
00734             # "*", process the overrides
00735             if (label == '*') or (label in self.target_labels ):
00736                 # Check for invalid cumulative overrides in libraries
00737                 if (unit_kind == 'library' and
00738                     any(attr.startswith('target.extra_labels') for attr
00739                         in overrides.keys())):
00740                     raise ConfigException(
00741                         "Target override 'target.extra_labels' in " +
00742                         ConfigParameter.get_display_name(unit_name, unit_kind,
00743                                                          label) +
00744                         " is only allowed at the application level")
00745 
00746                 # Parse out cumulative overrides
00747                 for attr, cumulatives in self.cumulative_overrides .items():
00748                     if 'target.'+attr in overrides:
00749                         key = 'target.' + attr
00750                         if not isinstance(overrides[key], list):
00751                             raise ConfigException(
00752                                 "The value of %s.%s is not of type %s" %
00753                                 (unit_name, "target_overrides." + key,
00754                                  "list"))
00755                         cumulatives.strict_cumulative_overrides(overrides[key])
00756                         del overrides[key]
00757 
00758                     if 'target.'+attr+'_add' in overrides:
00759                         key = 'target.' + attr + "_add"
00760                         if not isinstance(overrides[key], list):
00761                             raise ConfigException(
00762                                 "The value of %s.%s is not of type %s" %
00763                                 (unit_name, "target_overrides." + key,
00764                                  "list"))
00765                         cumulatives.add_cumulative_overrides(overrides[key])
00766                         del overrides[key]
00767 
00768                     if 'target.'+attr+'_remove' in overrides:
00769                         key = 'target.' + attr + "_remove"
00770                         if not isinstance(overrides[key], list):
00771                             raise ConfigException(
00772                                 "The value of %s.%s is not of type %s" %
00773                                 (unit_name, "target_overrides." + key,
00774                                  "list"))
00775                         cumulatives.remove_cumulative_overrides(overrides[key])
00776                         del overrides[key]
00777 
00778                 # Consider the others as overrides
00779                 for name, val in overrides.items():
00780                     if (name in PATH_OVERRIDES and "__config_path" in data):
00781                         val = os.path.join(
00782                             os.path.dirname(data["__config_path"]), val)
00783 
00784                     # Get the full name of the parameter
00785                     full_name = ConfigParameter.get_full_name(name, unit_name,
00786                                                               unit_kind, label)
00787                     if full_name in params:
00788                         params[full_name].set_value(val, unit_name, unit_kind,
00789                                                     label)
00790                     elif (name.startswith("target.") and
00791                         (unit_kind is "application" or
00792                          name in BOOTLOADER_OVERRIDES)):
00793                         _, attribute = name.split(".")
00794                         setattr(self.target , attribute, val)
00795                         continue
00796                     else:
00797                         self.config_errors .append(
00798                             ConfigException(
00799                                 "Attempt to override undefined parameter" +
00800                                 (" '%s' in '%s'"
00801                                  % (full_name,
00802                                     ConfigParameter.get_display_name(unit_name,
00803                                                                      unit_kind,
00804                                                                      label)))))
00805 
00806         for cumulatives in self.cumulative_overrides .values():
00807             cumulatives.update_target(self.target )
00808 
00809         return params
00810 
00811     def get_target_config_data (self):
00812         """Read and interpret configuration data defined by targets.
00813 
00814         We consider the resolution order for our target and sort it by level
00815         reversed, so that we first look at the top level target (the parent),
00816         then its direct children, then the children of those children and so on,
00817         until we reach self.target
00818         TODO: this might not work so well in some multiple inheritance scenarios
00819         At each step, look at two keys of the target data:
00820           - config_parameters: used to define new configuration parameters
00821           - config_overrides: used to override already defined configuration
00822                               parameters
00823 
00824         Arguments: None
00825         """
00826         params, json_data = {}, self.target .json_data
00827         resolution_order = [e[0] for e
00828                             in sorted(
00829                                 self.target .resolution_order,
00830                                 key=lambda e: e[1], reverse=True)]
00831         for tname in resolution_order:
00832             # Read the target data directly from its description
00833             target_data = json_data[tname]
00834             # Process definitions first
00835             _process_config_parameters(target_data.get("config", {}), params,
00836                                        tname, "target")
00837             # Then process overrides
00838             for name, val in target_data.get("overrides", {}).items():
00839                 full_name = ConfigParameter.get_full_name(name, tname, "target")
00840                 # If the parameter name is not defined or if there isn't a path
00841                 # from this target to the target where the parameter was defined
00842                 # in the target inheritance tree, raise an error We need to use
00843                 # 'defined_by[7:]' to remove the "target:" prefix from
00844                 # defined_by
00845                 rel_names = [tgt for tgt, _ in
00846                              get_resolution_order(self.target .json_data, tname,
00847                                                   [])]
00848                 if full_name in BOOTLOADER_OVERRIDES:
00849                     continue
00850                 if (full_name not in params) or \
00851                    (params[full_name].defined_by[7:] not in rel_names):
00852                     raise ConfigException(
00853                         "Attempt to override undefined parameter '%s' in '%s'"
00854                         % (name,
00855                            ConfigParameter.get_display_name(tname, "target")))
00856                 # Otherwise update the value of the parameter
00857                 params[full_name].set_value(val, tname, "target")
00858         return params
00859 
00860     def get_lib_config_data (self):
00861         """ Read and interpret configuration data defined by libraries. It is
00862         assumed that "add_config_files" above was already called and the library
00863         configuration data exists in self.lib_config_data
00864 
00865         Arguments: None
00866         """
00867         all_params, macros = {}, {}
00868         for lib_name, lib_data in self.lib_config_data .items():
00869             all_params.update(self._process_config_and_overrides (lib_data, {},
00870                                                                  lib_name,
00871                                                                  "library"))
00872             _process_macros(lib_data.get("macros", []), macros, lib_name,
00873                             "library")
00874         return all_params, macros
00875 
00876     def get_app_config_data (self, params, macros):
00877         """ Read and interpret the configuration data defined by the target. The
00878         target can override any configuration parameter, as well as define its
00879         own configuration data.
00880 
00881         Positional arguments.
00882         params - the dictionary with configuration parameters found so far (in
00883                  the target and in libraries)
00884         macros - the list of macros defined in the configuration
00885         """
00886         app_cfg = self.app_config_data 
00887         # The application can have a "config_parameters" and a
00888         # "target_config_overrides" section just like a library
00889         self._process_config_and_overrides (app_cfg, params, "app",
00890                                            "application")
00891         # The application can also defined macros
00892         _process_macros(app_cfg.get("macros", []), macros, "app",
00893                         "application")
00894 
00895     def get_config_data (self):
00896         """ Return the configuration data in two parts: (params, macros)
00897         params - a dictionary with mapping a name to a ConfigParam
00898         macros - the list of macros defined with "macros" in libraries and in
00899                  the application (as ConfigMacro instances)
00900 
00901         Arguments: None
00902         """
00903         all_params = self.get_target_config_data ()
00904         lib_params, macros = self.get_lib_config_data ()
00905         all_params.update(lib_params)
00906         self.get_app_config_data (all_params, macros)
00907         return all_params, macros
00908 
00909     @staticmethod
00910     def _check_required_parameters(params):
00911         """Check that there are no required parameters without a value
00912 
00913         Positional arguments:
00914         params - the list of parameters to check
00915 
00916         NOTE: This function does not return. Instead, it throws a
00917         ConfigException when any of the required parameters are missing values
00918         """
00919         for param in params.values():
00920             if param.required and (param.value is None):
00921                 raise ConfigException("Required parameter '" + param.name +
00922                                       "' defined by '" + param.defined_by +
00923                                       "' doesn't have a value")
00924 
00925     @staticmethod
00926     def parameters_to_macros (params):
00927         """ Encode the configuration parameters as C macro definitions.
00928 
00929         Positional arguments:
00930         params - a dictionary mapping a name to a ConfigParameter
00931 
00932         Return: a list of strings that encode the configuration parameters as
00933         C pre-processor macros
00934         """
00935         return ['%s=%s' % (m.macro_name, m.value) for m in params.values()
00936                 if m.value is not None]
00937 
00938     @staticmethod
00939     def config_macros_to_macros (macros):
00940         """ Return the macro definitions generated for a dictionary of
00941         ConfigMacros (as returned by get_config_data).
00942 
00943         Positional arguments:
00944         params - a dictionary mapping a name to a ConfigMacro instance
00945 
00946         Return: a list of strings that are the C pre-processor macros
00947         """
00948         return [m.name for m in macros.values()]
00949 
00950     @staticmethod
00951     def config_to_macros (config):
00952         """Convert the configuration data to a list of C macros
00953 
00954         Positional arguments:
00955         config - configuration data as (ConfigParam instances, ConfigMacro
00956                  instances) tuple (as returned by get_config_data())
00957         """
00958         params, macros = config[0], config[1]
00959         Config._check_required_parameters(params)
00960         return Config.config_macros_to_macros(macros) + \
00961             Config.parameters_to_macros(params)
00962 
00963     def get_config_data_macros (self):
00964         """ Convert a Config object to a list of C macros
00965 
00966         Arguments: None
00967         """
00968         return self.config_to_macros (self.get_config_data ())
00969 
00970     def get_features (self):
00971         """ Extract any features from the configuration data
00972 
00973         Arguments: None
00974         """
00975         params, _ = self.get_config_data ()
00976         self._check_required_parameters (params)
00977         self.cumulative_overrides ['features']\
00978             .update_target(self.target )
00979 
00980         for feature in self.target .features:
00981             if feature not in self.__allowed_features :
00982                 raise ConfigException(
00983                     "Feature '%s' is not a supported features" % feature)
00984 
00985         return self.target .features
00986 
00987     def validate_config (self):
00988         """ Validate configuration settings. This either returns True or
00989         raises an exception
00990 
00991         Arguments: None
00992         """
00993         if self.config_errors :
00994             raise self.config_errors [0]
00995         return True
00996 
00997 
00998     @property
00999     def name(self):
01000         if "artifact_name" in self.app_config_data :
01001             return self.app_config_data ["artifact_name"]
01002         else:
01003             return None
01004 
01005     def load_resources (self, resources):
01006         """ Load configuration data from a Resources instance and expand it
01007         based on defined features.
01008 
01009         Positional arguments:
01010         resources - the resources object to load from and expand
01011         """
01012         # Update configuration files until added features creates no changes
01013         prev_features = set()
01014         self.validate_config ()
01015         while True:
01016             # Add/update the configuration with any .json files found while
01017             # scanning
01018             self.add_config_files (resources.json_files)
01019 
01020             # Add features while we find new ones
01021             features = set(self.get_features ())
01022             if features == prev_features:
01023                 break
01024 
01025             for feature in features:
01026                 if feature in resources.features:
01027                     resources.add(resources.features[feature])
01028 
01029             prev_features = features
01030         self.validate_config ()
01031 
01032         if  (hasattr(self.target , "release_versions") and
01033              "5" not in self.target .release_versions and
01034              "rtos" in self.lib_config_data ):
01035             raise NotSupportedException("Target does not support mbed OS 5")
01036 
01037         return resources
01038 
01039     @staticmethod
01040     def config_to_header (config, fname=None):
01041         """ Convert the configuration data to the content of a C header file,
01042         meant to be included to a C/C++ file. The content is returned as a
01043         string.
01044 
01045         Positional arguments:
01046         config - configuration data as (ConfigParam instances, ConfigMacro
01047                  instances) tuple (as returned by get_config_data())
01048 
01049         Keyword arguments:
01050         fname -  also write the content is to the file called "fname".
01051                  WARNING: if 'fname' names an existing file, it will be
01052                  overwritten!
01053         """
01054         params, macros = config[0] or {}, config[1] or {}
01055         Config._check_required_parameters(params)
01056         params_with_values = [p for p in params.values() if p.value is not None]
01057         ctx = {
01058             "cfg_params" : [(p.macro_name, str(p.value), p.set_by)
01059                             for p in params_with_values],
01060             "macros": [(m.macro_name, str(m.macro_value or ""), m.defined_by)
01061                        for m in macros.values()],
01062             "name_len":  max([len(m.macro_name) for m in macros.values()] +
01063                              [len(m.macro_name) for m in params_with_values]
01064                              + [0]),
01065             "val_len" : max([len(str(m.value)) for m in params_with_values] +
01066                             [len(m.macro_value or "") for m in macros.values()]
01067                             + [0]),
01068         }
01069         jinja_loader = FileSystemLoader(dirname(abspath(__file__)))
01070         jinja_environment = Environment(loader=jinja_loader,
01071                                         undefined=StrictUndefined)
01072         header_data = jinja_environment.get_template("header.tmpl").render(ctx)
01073         # If fname is given, write "header_data" to it
01074         if fname:
01075             with open(fname, "w+") as file_desc:
01076                 file_desc.write(header_data)
01077         return header_data
01078 
01079     def get_config_data_header (self, fname=None):
01080         """ Convert a Config instance to the content of a C header file, meant
01081         to be included to a C/C++ file. The content is returned as a string.
01082 
01083         Keyword arguments:
01084         fname - also write the content to the file called "fname".
01085                 WARNING: if 'fname' names an existing file, it will be
01086                 overwritten!
01087         """
01088         return self.config_to_header (self.get_config_data (), fname)