__init__.py

00001 """
00002 mbed SDK
00003 Copyright (c) 2016 ARM Limited
00004 
00005 Licensed under the Apache License, Version 2.0 (the "License");
00006 you may not use this file except in compliance with the License.
00007 You may obtain a copy of the License at
00008 
00009 http://www.apache.org/licenses/LICENSE-2.0
00010 
00011 Unless required by applicable law or agreed to in writing, software
00012 distributed under the License is distributed on an "AS IS" BASIS,
00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00014 See the License for the specific language governing permissions and
00015 limitations under the License.
00016 """
00017 
00018 from copy import deepcopy
00019 import os
00020 from os.path import dirname, abspath, exists, join
00021 import sys
00022 from collections import namedtuple
00023 from os.path import splitext, relpath
00024 from intelhex import IntelHex
00025 from jinja2 import FileSystemLoader, StrictUndefined
00026 from jinja2.environment import Environment
00027 # Implementation of mbed configuration mechanism
00028 from tools.utils import json_file_to_dict, intelhex_offset
00029 from tools.arm_pack_manager import Cache
00030 from tools.targets import CUMULATIVE_ATTRIBUTES, TARGET_MAP, \
00031     generate_py_target, get_resolution_order
00032 
00033 # Base class for all configuration exceptions
00034 class ConfigException (Exception):
00035     """Config system only exception. Makes it easier to distinguish config
00036     errors"""
00037     pass
00038 
00039 class ConfigParameter (object):
00040     """This class keeps information about a single configuration parameter"""
00041 
00042     def __init__ (self, name, data, unit_name, unit_kind):
00043         """Construct a ConfigParameter
00044 
00045         Positional arguments:
00046         name - the name of the configuration parameter
00047         data - the data associated with the configuration parameter
00048         unit_name - the unit (target/library/application) that defines this
00049                     parameter
00050         unit_ kind - the kind of the unit ("target", "library" or "application")
00051         """
00052         self.name  = self.get_full_name (name, unit_name, unit_kind,
00053                                        allow_prefix=False)
00054         self.defined_by  = self.get_display_name (unit_name, unit_kind)
00055         self.set_value (data.get("value", None), unit_name, unit_kind)
00056         self.help_text  = data.get("help", None)
00057         self.required  = data.get("required", False)
00058         self.macro_name  = data.get("macro_name", "MBED_CONF_%s" %
00059                                    self.sanitize (self.name .upper()))
00060         self.config_errors  = []
00061 
00062     @staticmethod
00063     def get_full_name (name, unit_name, unit_kind, label=None,
00064                       allow_prefix=True):
00065         """Return the full (prefixed) name of a parameter. If the parameter
00066         already has a prefix, check if it is valid
00067 
00068         Positional arguments:
00069         name - the simple (unqualified) name of the parameter
00070         unit_name - the unit (target/library/application) that defines this
00071                     parameter
00072         unit_kind - the kind of the unit ("target", "library" or "application")
00073 
00074         Keyword arguments:
00075         label - the name of the label in the 'target_config_overrides' section
00076         allow_prefix - True to allow the original name to have a prefix, False
00077                        otherwise
00078         """
00079         if name.find('.') == -1: # the name is not prefixed
00080             if unit_kind == "target":
00081                 prefix = "target."
00082             elif unit_kind == "application":
00083                 prefix = "app."
00084             else:
00085                 prefix = unit_name + '.'
00086             return prefix + name
00087         # The name has a prefix, so check if it is valid
00088         if not allow_prefix:
00089             raise ConfigException("Invalid parameter name '%s' in '%s'" %
00090                                   (name, ConfigParameter.get_display_name(
00091                                       unit_name, unit_kind, label)))
00092         temp = name.split(".")
00093         # Check if the parameter syntax is correct (must be
00094         # unit_name.parameter_name)
00095         if len(temp) != 2:
00096             raise ConfigException("Invalid parameter name '%s' in '%s'" %
00097                                   (name, ConfigParameter.get_display_name(
00098                                       unit_name, unit_kind, label)))
00099         prefix = temp[0]
00100         # Check if the given parameter prefix matches the expected prefix
00101         if (unit_kind == "library" and prefix != unit_name) or \
00102            (unit_kind == "target" and prefix != "target"):
00103             raise ConfigException(
00104                 "Invalid prefix '%s' for parameter name '%s' in '%s'" %
00105                 (prefix, name, ConfigParameter.get_display_name(
00106                     unit_name, unit_kind, label)))
00107         return name
00108 
00109     @staticmethod
00110     def get_display_name (unit_name, unit_kind, label=None):
00111         """Return the name displayed for a unit when interrogating the origin
00112         and the last set place of a parameter
00113 
00114         Positional arguments:
00115         unit_name - the unit (target/library/application) that defines this
00116                     parameter
00117         unit_kind - the kind of the unit ("target", "library" or "application")
00118 
00119         Keyword arguments:
00120         label - the name of the label in the 'target_config_overrides' section
00121         """
00122         if unit_kind == "target":
00123             return "target:" + unit_name
00124         elif unit_kind == "application":
00125             return "application%s" % ("[%s]" % label if label else "")
00126         else: # library
00127             return "library:%s%s" % (unit_name, "[%s]" % label if label else "")
00128 
00129     @staticmethod
00130     def sanitize (name):
00131         """ "Sanitize" a name so that it is a valid C macro name. Currently it
00132         simply replaces '.' and '-' with '_'.
00133 
00134         Positional arguments:
00135         name - the name to make into a valid C macro
00136         """
00137         return name.replace('.', '_').replace('-', '_')
00138 
00139     def set_value (self, value, unit_name, unit_kind, label=None):
00140         """ Sets a value for this parameter, remember the place where it was
00141         set.  If the value is a Boolean, it is converted to 1 (for True) or
00142         to 0 (for False).
00143 
00144         Positional arguments:
00145         value - the value of the parameter
00146         unit_name - the unit (target/library/application) that defines this
00147                    parameter
00148         unit_kind - the kind of the unit ("target", "library" or "application")
00149 
00150         Keyword arguments:
00151         label - the name of the label in the 'target_config_overrides' section
00152                (optional)
00153         """
00154         self.value  = int(value) if isinstance(value, bool) else value
00155         self.set_by  = self.get_display_name (unit_name, unit_kind, label)
00156 
00157     def __str__ (self):
00158         """Return the string representation of this configuration parameter
00159 
00160         Arguments: None
00161         """
00162         if self.value  is not None:
00163             return '%s = %s (macro name: "%s")' % \
00164                 (self.name , self.value , self.macro_name )
00165         else:
00166             return '%s has no value' % self.name 
00167 
00168     def get_verbose_description (self):
00169         """Return a verbose description of this configuration parameter as a
00170         string
00171 
00172         Arguments: None
00173         """
00174         desc = "Name: %s%s\n" % \
00175                (self.name , " (required parameter)" if self.required  else "")
00176         if self.help_text :
00177             desc += "    Description: %s\n" % self.help_text 
00178         desc += "    Defined by: %s\n" % self.defined_by 
00179         if not self.value :
00180             return desc + "    No value set"
00181         desc += "    Macro name: %s\n" % self.macro_name 
00182         desc += "    Value: %s (set by %s)" % (self.value , self.set_by )
00183         return desc
00184 
00185 class ConfigMacro (object):
00186     """ A representation of a configuration macro. It handles both macros
00187     without a value (MACRO) and with a value (MACRO=VALUE)
00188     """
00189     def __init__ (self, name, unit_name, unit_kind):
00190         """Construct a ConfigMacro object
00191 
00192         Positional arguments:
00193         name - the macro's name
00194         unit_name - the location where the macro was defined
00195         unit_kind - the type of macro this is
00196         """
00197         self.name  = name
00198         self.defined_by  = ConfigParameter.get_display_name(unit_name, unit_kind)
00199         if name.find("=") != -1:
00200             tmp = name.split("=")
00201             if len(tmp) != 2:
00202                 raise ValueError("Invalid macro definition '%s' in '%s'" %
00203                                  (name, self.defined_by ))
00204             self.macro_name  = tmp[0]
00205             self.macro_value  = tmp[1]
00206         else:
00207             self.macro_name  = name
00208             self.macro_value  = None
00209 
00210 class ConfigCumulativeOverride (object):
00211     """Representation of overrides for cumulative attributes"""
00212     def __init__ (self, name, additions=None, removals=None, strict=False):
00213         """Construct a ConfigCumulativeOverride object
00214 
00215         Positional arguments:
00216         name - the name of the config file this came from ?
00217 
00218         Keyword arguments:
00219         additions - macros to add to the overrides
00220         removals - macros to remove from the overrides
00221         strict - Boolean indicating that attempting to remove from an override
00222                  that does not exist should error
00223         """
00224         self.name  = name
00225         if additions:
00226             self.additions  = set(additions)
00227         else:
00228             self.additions  = set()
00229         if removals:
00230             self.removals  = set(removals)
00231         else:
00232             self.removals  = set()
00233         self.strict  = strict
00234 
00235     def remove_cumulative_overrides (self, overrides):
00236         """Extend the list of override removals.
00237 
00238         Positional arguments:
00239         overrides - a list of names that, when the override is evaluated, will
00240                     be removed
00241         """
00242         for override in overrides:
00243             if override in self.additions :
00244                 raise ConfigException(
00245                     "Configuration conflict. The %s %s both added and removed."
00246                     % (self.name [:-1], override))
00247 
00248         self.removals  |= set(overrides)
00249 
00250     def add_cumulative_overrides (self, overrides):
00251         """Extend the list of override additions.
00252 
00253         Positional arguments:
00254         overrides - a list of a names that, when the override is evaluated, will
00255                     be added to the list
00256         """
00257         for override in overrides:
00258             if override in self.removals  or \
00259                (self.strict  and override not in self.additions ):
00260                 raise ConfigException(
00261                     "Configuration conflict. The %s %s both added and removed."
00262                     % (self.name [:-1], override))
00263 
00264         self.additions  |= set(overrides)
00265 
00266     def strict_cumulative_overrides (self, overrides):
00267         """Remove all overrides that are not the specified ones
00268 
00269         Positional arguments:
00270         overrides - a list of names that will replace the entire attribute when
00271                     this override is evaluated.
00272         """
00273         self.remove_cumulative_overrides (self.additions  - set(overrides))
00274         self.add_cumulative_overrides (overrides)
00275         self.strict  = True
00276 
00277     def update_target (self, target):
00278         """Update the attributes of a target based on this override"""
00279         setattr(target, self.name ,
00280                 list((set(getattr(target, self.name , []))
00281                       | self.additions ) - self.removals ))
00282 
00283 
00284 def _process_config_parameters(data, params, unit_name, unit_kind):
00285     """Process a "config_parameters" section in either a target, a library,
00286     or the application.
00287 
00288     Positional arguments:
00289     data - a dictionary with the configuration parameters
00290     params - storage for the discovered configuration parameters
00291     unit_name - the unit (target/library/application) that defines this
00292                 parameter
00293     unit_kind - the kind of the unit ("target", "library" or "application")
00294     """
00295     for name, val in data.items():
00296         full_name = ConfigParameter.get_full_name(name, unit_name, unit_kind)
00297         # If the parameter was already defined, raise an error
00298         if full_name in params:
00299             raise ConfigException(
00300                 "Parameter name '%s' defined in both '%s' and '%s'" %
00301                 (name, ConfigParameter.get_display_name(unit_name, unit_kind),
00302                  params[full_name].defined_by))
00303         # Otherwise add it to the list of known parameters
00304         # If "val" is not a dictionary, this is a shortcut definition,
00305         # otherwise it is a full definition
00306         params[full_name] = ConfigParameter(name, val if isinstance(val, dict)
00307                                             else {"value": val}, unit_name,
00308                                             unit_kind)
00309     return params
00310 
00311 
00312 def _process_macros(mlist, macros, unit_name, unit_kind):
00313     """Process a macro definition and check for incompatible duplicate
00314     definitions.
00315 
00316     Positional arguments:
00317     mlist - list of macro names to process
00318     macros - dictionary with currently discovered macros
00319     unit_name - the unit (library/application) that defines this macro
00320     unit_kind - the kind of the unit ("library" or "application")
00321     """
00322     for mname in mlist:
00323         macro = ConfigMacro(mname, unit_name, unit_kind)
00324         if (macro.macro_name in macros) and \
00325            (macros[macro.macro_name].name != mname):
00326             # Found an incompatible definition of the macro in another module,
00327             # so raise an error
00328             full_unit_name = ConfigParameter.get_display_name(unit_name,
00329                                                               unit_kind)
00330             raise ConfigException(
00331                 ("Macro '%s' defined in both '%s' and '%s'"
00332                  % (macro.macro_name, macros[macro.macro_name].defined_by,
00333                     full_unit_name)) +
00334                 " with incompatible values")
00335         macros[macro.macro_name] = macro
00336 
00337 
00338 def check_dict_types(dict, type_dict, dict_loc):
00339     for key, value in dict.iteritems():
00340         if not isinstance(value, type_dict[key]):
00341             raise ConfigException("The value of %s.%s is not of type %s" %
00342                                   (dict_loc, key, type_dict[key].__name__))
00343 
00344 Region = namedtuple("Region", "name start size active filename")
00345 
00346 class Config (object):
00347     """'Config' implements the mbed configuration mechanism"""
00348 
00349     # Libraries and applications have different names for their configuration
00350     # files
00351     __mbed_app_config_name = "mbed_app.json"
00352     __mbed_lib_config_name = "mbed_lib.json"
00353 
00354     # Allowed keys in configuration dictionaries, and their types
00355     # (targets can have any kind of keys, so this validation is not applicable
00356     # to them)
00357     __allowed_keys = {
00358         "library": {"name": str, "config": dict, "target_overrides": dict,
00359                     "macros": list, "__config_path": str},
00360         "application": {"config": dict, "target_overrides": dict,
00361                         "macros": list, "__config_path": str,
00362                         "artifact_name": str}
00363     }
00364 
00365     __unused_overrides = set(["target.bootloader_img", "target.restrict_size",
00366                               "target.mbed_app_start", "target.mbed_app_size"])
00367 
00368     # Allowed features in configurations
00369     __allowed_features = [
00370         "UVISOR", "BLE", "CLIENT", "IPV4", "LWIP", "COMMON_PAL", "STORAGE", "NANOSTACK",
00371         # Nanostack configurations
00372         "LOWPAN_BORDER_ROUTER", "LOWPAN_HOST", "LOWPAN_ROUTER", "NANOSTACK_FULL", "THREAD_BORDER_ROUTER", "THREAD_END_DEVICE", "THREAD_ROUTER", "ETHERNET_HOST"
00373         ]
00374 
00375     def __init__ (self, tgt, top_level_dirs=None, app_config=None):
00376         """Construct a mbed configuration
00377 
00378         Positional arguments:
00379         target - the name of the mbed target used for this configuration
00380                  instance
00381 
00382         Keyword argumets:
00383         top_level_dirs - a list of top level source directories (where
00384                          mbed_app_config.json could be found)
00385         app_config - location of a chosen mbed_app.json file
00386 
00387         NOTE: Construction of a Config object will look for the application
00388         configuration file in top_level_dirs. If found once, it'll parse it.
00389         top_level_dirs may be None (in this case, the constructor will not
00390         search for a configuration file).
00391         """
00392         config_errors = []
00393         self.app_config_location  = app_config
00394         if self.app_config_location  is None:
00395             for directory in top_level_dirs or []:
00396                 full_path = os.path.join(directory, self.__mbed_app_config_name )
00397                 if os.path.isfile(full_path):
00398                     if self.app_config_location  is not None:
00399                         raise ConfigException("Duplicate '%s' file in '%s' and '%s'"
00400                                               % (self.__mbed_app_config_name ,
00401                                                  self.app_config_location , full_path))
00402                     else:
00403                         self.app_config_location  = full_path
00404         try:
00405             self.app_config_data  = json_file_to_dict(self.app_config_location ) \
00406                                    if self.app_config_location  else {}
00407         except ValueError as exc:
00408             self.app_config_data  = {}
00409             config_errors.append(
00410                 ConfigException("Could not parse mbed app configuration from %s"
00411                                 % self.app_config_location ))
00412 
00413         # Check the keys in the application configuration data
00414         unknown_keys = set(self.app_config_data .keys()) - \
00415                        set(self.__allowed_keys ["application"].keys())
00416         if unknown_keys:
00417             raise ConfigException("Unknown key(s) '%s' in %s" %
00418                                   (",".join(unknown_keys),
00419                                    self.__mbed_app_config_name ))
00420         check_dict_types(self.app_config_data , self.__allowed_keys ["application"],
00421                          "app-config")
00422         # Update the list of targets with the ones defined in the application
00423         # config, if applicable
00424         self.lib_config_data  = {}
00425         # Make sure that each config is processed only once
00426         self.processed_configs  = {}
00427         if isinstance(tgt, basestring):
00428             if tgt in TARGET_MAP:
00429                 self.target  = TARGET_MAP[tgt]
00430             else:
00431                 self.target  = generate_py_target(
00432                     self.app_config_data .get("custom_targets", {}), tgt)
00433 
00434         else:
00435             self.target  = tgt
00436         self.target  = deepcopy(self.target )
00437         self.target_labels  = self.target .labels
00438 
00439         self.cumulative_overrides  = {key: ConfigCumulativeOverride(key)
00440                                      for key in CUMULATIVE_ATTRIBUTES}
00441 
00442         self._process_config_and_overrides (self.app_config_data , {}, "app",
00443                                            "application")
00444         self.config_errors  = config_errors
00445 
00446     def add_config_files (self, flist):
00447         """Add configuration files
00448 
00449         Positional arguments:
00450         flist - a list of files to add to this configuration
00451         """
00452         for config_file in flist:
00453             if not config_file.endswith(self.__mbed_lib_config_name ):
00454                 continue
00455             full_path = os.path.normpath(os.path.abspath(config_file))
00456             # Check that we didn't already process this file
00457             if self.processed_configs .has_key(full_path):
00458                 continue
00459             self.processed_configs [full_path] = True
00460             # Read the library configuration and add a "__full_config_path"
00461             # attribute to it
00462             try:
00463                 cfg = json_file_to_dict(config_file)
00464             except ValueError as exc:
00465                 sys.stderr.write(str(exc) + "\n")
00466                 continue
00467 
00468             cfg["__config_path"] = full_path
00469 
00470             if "name" not in cfg:
00471                 raise ConfigException(
00472                     "Library configured at %s has no name field." % full_path)
00473             # If there's already a configuration for a module with the same
00474             # name, exit with error
00475             if self.lib_config_data .has_key(cfg["name"]):
00476                 raise ConfigException(
00477                     "Library name '%s' is not unique (defined in '%s' and '%s')"
00478                     % (cfg["name"], full_path,
00479                        self.lib_config_data [cfg["name"]]["__config_path"]))
00480             self.lib_config_data [cfg["name"]] = cfg
00481 
00482     @property
00483     def has_regions (self):
00484         """Does this config have regions defined?"""
00485         if 'target_overrides' in self.app_config_data :
00486             target_overrides = self.app_config_data ['target_overrides'].get(
00487                 self.target .name, {})
00488             return ('target.bootloader_img' in target_overrides or
00489                     'target.restrict_size' in target_overrides or
00490                     'target.mbed_app_start' in target_overrides or
00491                     'target.mbed_app_size' in target_overrides)
00492         else:
00493             return False
00494 
00495     @property
00496     def regions (self):
00497         """Generate a list of regions from the config"""
00498         if not self.target .bootloader_supported:
00499             raise ConfigException("Bootloader not supported on this target.")
00500         if not hasattr(self.target , "device_name"):
00501             raise ConfigException("Bootloader not supported on this target: "
00502                                   "targets.json `device_name` not specified.")
00503         cache = Cache(False, False)
00504         if self.target .device_name not in cache.index:
00505             raise ConfigException("Bootloader not supported on this target: "
00506                                   "targets.json `device_name` not found in "
00507                                   "arm_pack_manager index.")
00508         cmsis_part = cache.index[self.target .device_name]
00509         target_overrides = self.app_config_data ['target_overrides'].get(
00510             self.target .name, {})
00511         if  (('target.bootloader_img' in target_overrides or
00512               'target.restrict_size' in target_overrides) and
00513              ('target.mbed_app_start' in target_overrides or
00514               'target.mbed_app_size' in target_overrides)):
00515             raise ConfigException(
00516                 "target.bootloader_img and target.restirct_size are "
00517                 "incompatible with target.mbed_app_start and "
00518                 "target.mbed_app_size")
00519         try:
00520             rom_size = int(cmsis_part['memory']['IROM1']['size'], 0)
00521             rom_start = int(cmsis_part['memory']['IROM1']['start'], 0)
00522         except KeyError:
00523             raise ConfigException("Not enough information in CMSIS packs to "
00524                                   "build a bootloader project")
00525         if  ('target.bootloader_img' in target_overrides or
00526              'target.restrict_size' in target_overrides):
00527             return self._generate_booloader_build (target_overrides,
00528                                                   rom_start, rom_size)
00529         elif ('target.mbed_app_start' in target_overrides or
00530               'target.mbed_app_size' in target_overrides):
00531             return self._generate_linker_overrides (target_overrides,
00532                                                    rom_start, rom_size)
00533         else:
00534             raise ConfigException(
00535                 "Bootloader build requested but no bootlader configuration")
00536 
00537     def _generate_booloader_build(self, target_overrides, rom_start, rom_size):
00538         start = 0
00539         if 'target.bootloader_img' in target_overrides:
00540             basedir = abspath(dirname(self.app_config_location ))
00541             filename = join(basedir, target_overrides['target.bootloader_img'])
00542             if not exists(filename):
00543                 raise ConfigException("Bootloader %s not found" % filename)
00544             part = intelhex_offset(filename, offset=rom_start)
00545             if part.minaddr() != rom_start:
00546                 raise ConfigException("bootloader executable does not "
00547                                       "start at 0x%x" % rom_start)
00548             part_size = (part.maxaddr() - part.minaddr()) + 1
00549             yield Region("bootloader", rom_start + start, part_size, False,
00550                          filename)
00551             start += part_size
00552         if 'target.restrict_size' in target_overrides:
00553             new_size = int(target_overrides['target.restrict_size'], 0)
00554             yield Region("application", rom_start + start, new_size, True, None)
00555             start += new_size
00556             yield Region("post_application", rom_start +start, rom_size - start,
00557                          False, None)
00558         else:
00559             yield Region("application", rom_start + start, rom_size - start,
00560                          True, None)
00561         if start > rom_size:
00562             raise ConfigException("Not enough memory on device to fit all "
00563                                   "application regions")
00564 
00565     @property
00566     def report(self):
00567         return {'app_config': self.app_config_location ,
00568                 'library_configs': map(relpath, self.processed_configs .keys())}
00569 
00570     @staticmethod
00571     def _generate_linker_overrides(target_overrides, rom_start, rom_size):
00572         if 'target.mbed_app_start' in target_overrides:
00573             start = int(target_overrides['target.mbed_app_start'], 0)
00574         else:
00575             start = rom_start
00576         if 'target.mbed_app_size' in target_overrides:
00577             size = int(target_overrides['target.mbed_app_size'], 0)
00578         else:
00579             size = (rom_size + rom_start) - start
00580         if start < rom_start:
00581             raise ConfigException("Application starts before ROM")
00582         if size + start > rom_size + rom_start:
00583             raise ConfigException("Application ends after ROM")
00584         yield Region("application", start, size, True, None)
00585 
00586     def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
00587         """Process "config_parameters" and "target_config_overrides" into a
00588         given dictionary
00589 
00590         Positional arguments:
00591         data - the configuration data of the library/appliation
00592         params - storage for the discovered configuration parameters
00593         unit_name - the unit (library/application) that defines this parameter
00594         unit_kind - the kind of the unit ("library" or "application")
00595         """
00596         self.config_errors  = []
00597         _process_config_parameters(data.get("config", {}), params, unit_name,
00598                                    unit_kind)
00599         for label, overrides in data.get("target_overrides", {}).items():
00600             # If the label is defined by the target or it has the special value
00601             # "*", process the overrides
00602             if (label == '*') or (label in self.target_labels ):
00603                 # Check for invalid cumulative overrides in libraries
00604                 if (unit_kind == 'library' and
00605                     any(attr.startswith('target.extra_labels') for attr
00606                         in overrides.iterkeys())):
00607                     raise ConfigException(
00608                         "Target override 'target.extra_labels' in " +
00609                         ConfigParameter.get_display_name(unit_name, unit_kind,
00610                                                          label) +
00611                         " is only allowed at the application level")
00612 
00613                 # Parse out cumulative overrides
00614                 for attr, cumulatives in self.cumulative_overrides .iteritems():
00615                     if 'target.'+attr in overrides:
00616                         key = 'target.' + attr
00617                         if not isinstance(overrides[key], list):
00618                             raise ConfigException(
00619                                 "The value of %s.%s is not of type %s" %
00620                                 (unit_name, "target_overrides." + key,
00621                                  "list"))
00622                         cumulatives.strict_cumulative_overrides(overrides[key])
00623                         del overrides[key]
00624 
00625                     if 'target.'+attr+'_add' in overrides:
00626                         key = 'target.' + attr + "_add"
00627                         if not isinstance(overrides[key], list):
00628                             raise ConfigException(
00629                                 "The value of %s.%s is not of type %s" %
00630                                 (unit_name, "target_overrides." + key,
00631                                  "list"))
00632                         cumulatives.add_cumulative_overrides(overrides[key])
00633                         del overrides[key]
00634 
00635                     if 'target.'+attr+'_remove' in overrides:
00636                         key = 'target.' + attr + "_remove"
00637                         if not isinstance(overrides[key], list):
00638                             raise ConfigException(
00639                                 "The value of %s.%s is not of type %s" %
00640                                 (unit_name, "target_overrides." + key,
00641                                  "list"))
00642                         cumulatives.remove_cumulative_overrides(overrides[key])
00643                         del overrides[key]
00644 
00645                 # Consider the others as overrides
00646                 for name, val in overrides.items():
00647                     # Get the full name of the parameter
00648                     full_name = ConfigParameter.get_full_name(name, unit_name,
00649                                                               unit_kind, label)
00650                     if full_name in params:
00651                         params[full_name].set_value(val, unit_name, unit_kind,
00652                                                     label)
00653                     elif name in self.__unused_overrides :
00654                         pass
00655                     elif (name.startswith("target.") and
00656                           unit_kind is "application"):
00657                         _, attribute = name.split(".")
00658                         setattr(self.target , attribute, val)
00659                     else:
00660                         self.config_errors .append(
00661                             ConfigException(
00662                                 "Attempt to override undefined parameter" +
00663                                 (" '%s' in '%s'"
00664                                  % (full_name,
00665                                     ConfigParameter.get_display_name(unit_name,
00666                                                                      unit_kind,
00667                                                                      label)))))
00668 
00669         for cumulatives in self.cumulative_overrides .itervalues():
00670             cumulatives.update_target(self.target )
00671 
00672         return params
00673 
00674     def get_target_config_data (self):
00675         """Read and interpret configuration data defined by targets.
00676 
00677         We consider the resolution order for our target and sort it by level
00678         reversed, so that we first look at the top level target (the parent),
00679         then its direct children, then the children of those children and so on,
00680         until we reach self.target
00681         TODO: this might not work so well in some multiple inheritance scenarios
00682         At each step, look at two keys of the target data:
00683           - config_parameters: used to define new configuration parameters
00684           - config_overrides: used to override already defined configuration
00685                               parameters
00686 
00687         Arguments: None
00688         """
00689         params, json_data = {}, self.target .json_data
00690         resolution_order = [e[0] for e
00691                             in sorted(
00692                                 self.target .resolution_order,
00693                                 key=lambda e: e[1], reverse=True)]
00694         for tname in resolution_order:
00695             # Read the target data directly from its description
00696             target_data = json_data[tname]
00697             # Process definitions first
00698             _process_config_parameters(target_data.get("config", {}), params,
00699                                        tname, "target")
00700             # Then process overrides
00701             for name, val in target_data.get("overrides", {}).items():
00702                 full_name = ConfigParameter.get_full_name(name, tname, "target")
00703                 # If the parameter name is not defined or if there isn't a path
00704                 # from this target to the target where the parameter was defined
00705                 # in the target inheritance tree, raise an error We need to use
00706                 # 'defined_by[7:]' to remove the "target:" prefix from
00707                 # defined_by
00708                 rel_names = [tgt for tgt, _ in
00709                              get_resolution_order(self.target .json_data, tname,
00710                                                   [])]
00711                 if full_name in self.__unused_overrides :
00712                     continue
00713                 if (full_name not in params) or \
00714                    (params[full_name].defined_by[7:] not in rel_names):
00715                     raise ConfigException(
00716                         "Attempt to override undefined parameter '%s' in '%s'"
00717                         % (name,
00718                            ConfigParameter.get_display_name(tname, "target")))
00719                 # Otherwise update the value of the parameter
00720                 params[full_name].set_value(val, tname, "target")
00721         return params
00722 
00723     def get_lib_config_data (self):
00724         """ Read and interpret configuration data defined by libraries. It is
00725         assumed that "add_config_files" above was already called and the library
00726         configuration data exists in self.lib_config_data
00727 
00728         Arguments: None
00729         """
00730         all_params, macros = {}, {}
00731         for lib_name, lib_data in self.lib_config_data .items():
00732             unknown_keys = (set(lib_data.keys()) -
00733                             set(self.__allowed_keys ["library"].keys()))
00734             if unknown_keys:
00735                 raise ConfigException("Unknown key(s) '%s' in %s" %
00736                                       (",".join(unknown_keys), lib_name))
00737             check_dict_types(lib_data, self.__allowed_keys ["library"], lib_name)
00738             all_params.update(self._process_config_and_overrides (lib_data, {},
00739                                                                  lib_name,
00740                                                                  "library"))
00741             _process_macros(lib_data.get("macros", []), macros, lib_name,
00742                             "library")
00743         return all_params, macros
00744 
00745     def get_app_config_data (self, params, macros):
00746         """ Read and interpret the configuration data defined by the target. The
00747         target can override any configuration parameter, as well as define its
00748         own configuration data.
00749 
00750         Positional arguments.
00751         params - the dictionary with configuration parameters found so far (in
00752                  the target and in libraries)
00753         macros - the list of macros defined in the configuration
00754         """
00755         app_cfg = self.app_config_data 
00756         # The application can have a "config_parameters" and a
00757         # "target_config_overrides" section just like a library
00758         self._process_config_and_overrides (app_cfg, params, "app",
00759                                            "application")
00760         # The application can also defined macros
00761         _process_macros(app_cfg.get("macros", []), macros, "app",
00762                         "application")
00763 
00764     def get_config_data (self):
00765         """ Return the configuration data in two parts: (params, macros)
00766         params - a dictionary with mapping a name to a ConfigParam
00767         macros - the list of macros defined with "macros" in libraries and in
00768                  the application (as ConfigMacro instances)
00769 
00770         Arguments: None
00771         """
00772         all_params = self.get_target_config_data ()
00773         lib_params, macros = self.get_lib_config_data ()
00774         all_params.update(lib_params)
00775         self.get_app_config_data (all_params, macros)
00776         return all_params, macros
00777 
00778     @staticmethod
00779     def _check_required_parameters(params):
00780         """Check that there are no required parameters without a value
00781 
00782         Positional arguments:
00783         params - the list of parameters to check
00784 
00785         NOTE: This function does not return. Instead, it throws a
00786         ConfigException when any of the required parameters are missing values
00787         """
00788         for param in params.values():
00789             if param.required and (param.value is None):
00790                 raise ConfigException("Required parameter '" + param.name +
00791                                       "' defined by '" + param.defined_by +
00792                                       "' doesn't have a value")
00793 
00794     @staticmethod
00795     def parameters_to_macros (params):
00796         """ Encode the configuration parameters as C macro definitions.
00797 
00798         Positional arguments:
00799         params - a dictionary mapping a name to a ConfigParameter
00800 
00801         Return: a list of strings that encode the configuration parameters as
00802         C pre-processor macros
00803         """
00804         return ['%s=%s' % (m.macro_name, m.value) for m in params.values()
00805                 if m.value is not None]
00806 
00807     @staticmethod
00808     def config_macros_to_macros (macros):
00809         """ Return the macro definitions generated for a dictionary of
00810         ConfigMacros (as returned by get_config_data).
00811 
00812         Positional arguments:
00813         params - a dictionary mapping a name to a ConfigMacro instance
00814 
00815         Return: a list of strings that are the C pre-processor macros
00816         """
00817         return [m.name for m in macros.values()]
00818 
00819     @staticmethod
00820     def config_to_macros (config):
00821         """Convert the configuration data to a list of C macros
00822 
00823         Positional arguments:
00824         config - configuration data as (ConfigParam instances, ConfigMacro
00825                  instances) tuple (as returned by get_config_data())
00826         """
00827         params, macros = config[0], config[1]
00828         Config._check_required_parameters(params)
00829         return Config.config_macros_to_macros(macros) + \
00830             Config.parameters_to_macros(params)
00831 
00832     def get_config_data_macros (self):
00833         """ Convert a Config object to a list of C macros
00834 
00835         Arguments: None
00836         """
00837         return self.config_to_macros (self.get_config_data ())
00838 
00839     def get_features (self):
00840         """ Extract any features from the configuration data
00841 
00842         Arguments: None
00843         """
00844         params, _ = self.get_config_data ()
00845         self._check_required_parameters (params)
00846         self.cumulative_overrides ['features']\
00847             .update_target(self.target )
00848 
00849         for feature in self.target .features:
00850             if feature not in self.__allowed_features :
00851                 raise ConfigException(
00852                     "Feature '%s' is not a supported features" % feature)
00853 
00854         return self.target .features
00855 
00856     def validate_config (self):
00857         """ Validate configuration settings. This either returns True or
00858         raises an exception
00859 
00860         Arguments: None
00861         """
00862         if self.config_errors :
00863             raise self.config_errors [0]
00864         return True
00865 
00866 
00867     @property
00868     def name(self):
00869         if "artifact_name" in self.app_config_data :
00870             return self.app_config_data ["artifact_name"]
00871         else:
00872             return None
00873 
00874     def load_resources (self, resources):
00875         """ Load configuration data from a Resources instance and expand it
00876         based on defined features.
00877 
00878         Positional arguments:
00879         resources - the resources object to load from and expand
00880         """
00881         # Update configuration files until added features creates no changes
00882         prev_features = set()
00883         self.validate_config ()
00884         while True:
00885             # Add/update the configuration with any .json files found while
00886             # scanning
00887             self.add_config_files (resources.json_files)
00888 
00889             # Add features while we find new ones
00890             features = set(self.get_features ())
00891             if features == prev_features:
00892                 break
00893 
00894             for feature in features:
00895                 if feature in resources.features:
00896                     resources.add(resources.features[feature])
00897 
00898             prev_features = features
00899         self.validate_config ()
00900 
00901         return resources
00902 
00903     @staticmethod
00904     def config_to_header (config, fname=None):
00905         """ Convert the configuration data to the content of a C header file,
00906         meant to be included to a C/C++ file. The content is returned as a
00907         string.
00908 
00909         Positional arguments:
00910         config - configuration data as (ConfigParam instances, ConfigMacro
00911                  instances) tuple (as returned by get_config_data())
00912 
00913         Keyword arguments:
00914         fname -  also write the content is to the file called "fname".
00915                  WARNING: if 'fname' names an existing file, it will be
00916                  overwritten!
00917         """
00918         params, macros = config[0] or {}, config[1] or {}
00919         Config._check_required_parameters(params)
00920         params_with_values = [p for p in params.values() if p.value is not None]
00921         ctx = {
00922             "cfg_params" : [(p.macro_name, str(p.value), p.set_by)
00923                             for p in params_with_values],
00924             "macros": [(m.macro_name, str(m.macro_value or ""), m.defined_by)
00925                        for m in macros.values()],
00926             "name_len":  max([len(m.macro_name) for m in macros.values()] +
00927                              [len(m.macro_name) for m in params_with_values]
00928                              + [0]),
00929             "val_len" : max([len(str(m.value)) for m in params_with_values] +
00930                             [len(m.macro_value or "") for m in macros.values()]
00931                             + [0]),
00932         }
00933         jinja_loader = FileSystemLoader(dirname(abspath(__file__)))
00934         jinja_environment = Environment(loader=jinja_loader,
00935                                         undefined=StrictUndefined)
00936         header_data = jinja_environment.get_template("header.tmpl").render(ctx)
00937         # If fname is given, write "header_data" to it
00938         if fname:
00939             with open(fname, "w+") as file_desc:
00940                 file_desc.write(header_data)
00941         return header_data
00942 
00943     def get_config_data_header (self, fname=None):
00944         """ Convert a Config instance to the content of a C header file, meant
00945         to be included to a C/C++ file. The content is returned as a string.
00946 
00947         Keyword arguments:
00948         fname - also write the content to the file called "fname".
00949                 WARNING: if 'fname' names an existing file, it will be
00950                 overwritten!
00951         """
00952         return self.config_to_header (self.get_config_data (), fname)