Anders Blomdell / mbed-sdk-tools

Clone of official tools

export/exporters.py@29:1210849dba19, 2016-08-29 (annotated)

Committer:
screamer
Date:
Mon Aug 29 11:18:36 2016 +0100
Revision:
29:1210849dba19
Parent:
24:25bff2709c20
Child:
31:8ea194f6145b
Port the latest tools patches from https://github.com/ARMmbed/mbed-os

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """Just a template for subclassing"""
screamer 0:66f3b5499f7f 2 import uuid, shutil, os, logging, fnmatch
screamer 0:66f3b5499f7f 3 from os import walk, remove
screamer 0:66f3b5499f7f 4 from os.path import join, dirname, isdir, split
screamer 0:66f3b5499f7f 5 from copy import copy
screamer 0:66f3b5499f7f 6 from jinja2 import Template, FileSystemLoader
screamer 0:66f3b5499f7f 7 from jinja2.environment import Environment
screamer 0:66f3b5499f7f 8 from contextlib import closing
screamer 0:66f3b5499f7f 9 from zipfile import ZipFile, ZIP_DEFLATED
screamer 13:ab47a20b66f0 10 from operator import add
screamer 0:66f3b5499f7f 11
screamer 0:66f3b5499f7f 12 from tools.utils import mkdir
screamer 0:66f3b5499f7f 13 from tools.toolchains import TOOLCHAIN_CLASSES
screamer 0:66f3b5499f7f 14 from tools.targets import TARGET_MAP
screamer 0:66f3b5499f7f 15
screamer 0:66f3b5499f7f 16 from project_generator.generate import Generator
screamer 0:66f3b5499f7f 17 from project_generator.project import Project
screamer 0:66f3b5499f7f 18 from project_generator.settings import ProjectSettings
screamer 0:66f3b5499f7f 19
screamer 13:ab47a20b66f0 20 from tools.config import Config
screamer 13:ab47a20b66f0 21
screamer 0:66f3b5499f7f 22 class OldLibrariesException(Exception): pass
screamer 0:66f3b5499f7f 23
screamer 23:fbae331171fa 24 class FailedBuildException(Exception) : pass
screamer 23:fbae331171fa 25
screamer 24:25bff2709c20 26 # Exporter descriptor for TARGETS
screamer 24:25bff2709c20 27 # TARGETS as class attribute for backward compatibility (allows: if in Exporter.TARGETS)
screamer 24:25bff2709c20 28 class ExporterTargetsProperty(object):
screamer 24:25bff2709c20 29 def __init__(self, func):
screamer 24:25bff2709c20 30 self.func = func
screamer 24:25bff2709c20 31 def __get__(self, inst, cls):
screamer 24:25bff2709c20 32 return self.func(cls)
screamer 24:25bff2709c20 33
screamer 0:66f3b5499f7f 34 class Exporter(object):
screamer 0:66f3b5499f7f 35 TEMPLATE_DIR = dirname(__file__)
screamer 0:66f3b5499f7f 36 DOT_IN_RELATIVE_PATH = False
screamer 0:66f3b5499f7f 37
screamer 13:ab47a20b66f0 38 def __init__(self, target, inputDir, program_name, build_url_resolver, extra_symbols=None, sources_relative=True):
screamer 0:66f3b5499f7f 39 self.inputDir = inputDir
screamer 0:66f3b5499f7f 40 self.target = target
screamer 0:66f3b5499f7f 41 self.program_name = program_name
screamer 0:66f3b5499f7f 42 self.toolchain = TOOLCHAIN_CLASSES[self.get_toolchain()](TARGET_MAP[target])
screamer 0:66f3b5499f7f 43 self.build_url_resolver = build_url_resolver
screamer 0:66f3b5499f7f 44 jinja_loader = FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
screamer 0:66f3b5499f7f 45 self.jinja_environment = Environment(loader=jinja_loader)
screamer 29:1210849dba19 46 self.extra_symbols = extra_symbols if extra_symbols else []
screamer 13:ab47a20b66f0 47 self.config_macros = []
screamer 13:ab47a20b66f0 48 self.sources_relative = sources_relative
screamer 13:ab47a20b66f0 49 self.config_header = None
screamer 0:66f3b5499f7f 50
screamer 0:66f3b5499f7f 51 def get_toolchain(self):
screamer 0:66f3b5499f7f 52 return self.TOOLCHAIN
screamer 0:66f3b5499f7f 53
screamer 13:ab47a20b66f0 54 @property
screamer 13:ab47a20b66f0 55 def flags(self):
screamer 13:ab47a20b66f0 56 return self.toolchain.flags
screamer 13:ab47a20b66f0 57
screamer 13:ab47a20b66f0 58 @property
screamer 13:ab47a20b66f0 59 def progen_flags(self):
screamer 13:ab47a20b66f0 60 if not hasattr(self, "_progen_flag_cache") :
screamer 13:ab47a20b66f0 61 self._progen_flag_cache = dict([(key + "_flags", value) for key,value in self.flags.iteritems()])
screamer 29:1210849dba19 62 asm_defines = ["-D"+symbol for symbol in self.toolchain.get_symbols(True)]
screamer 29:1210849dba19 63 c_defines = ["-D" + symbol for symbol in self.toolchain.get_symbols()]
screamer 29:1210849dba19 64 self._progen_flag_cache['asm_flags'] += asm_defines
screamer 29:1210849dba19 65 self._progen_flag_cache['c_flags'] += c_defines
screamer 29:1210849dba19 66 self._progen_flag_cache['cxx_flags'] += c_defines
screamer 13:ab47a20b66f0 67 if self.config_header:
screamer 13:ab47a20b66f0 68 self._progen_flag_cache['c_flags'] += self.toolchain.get_config_option(self.config_header)
screamer 13:ab47a20b66f0 69 self._progen_flag_cache['cxx_flags'] += self.toolchain.get_config_option(self.config_header)
screamer 13:ab47a20b66f0 70 return self._progen_flag_cache
screamer 13:ab47a20b66f0 71
screamer 0:66f3b5499f7f 72 def __scan_and_copy(self, src_path, trg_path):
screamer 0:66f3b5499f7f 73 resources = self.toolchain.scan_resources(src_path)
screamer 0:66f3b5499f7f 74
screamer 0:66f3b5499f7f 75 for r_type in ['headers', 's_sources', 'c_sources', 'cpp_sources',
screamer 0:66f3b5499f7f 76 'objects', 'libraries', 'linker_script',
screamer 13:ab47a20b66f0 77 'lib_builds', 'lib_refs', 'hex_files', 'bin_files']:
screamer 0:66f3b5499f7f 78 r = getattr(resources, r_type)
screamer 0:66f3b5499f7f 79 if r:
screamer 13:ab47a20b66f0 80 self.toolchain.copy_files(r, trg_path, resources=resources)
screamer 0:66f3b5499f7f 81 return resources
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83 @staticmethod
screamer 0:66f3b5499f7f 84 def _get_dir_grouped_files(files):
screamer 0:66f3b5499f7f 85 """ Get grouped files based on the dirname """
screamer 0:66f3b5499f7f 86 files_grouped = {}
screamer 0:66f3b5499f7f 87 for file in files:
screamer 0:66f3b5499f7f 88 rel_path = os.path.relpath(file, os.getcwd())
screamer 0:66f3b5499f7f 89 dir_path = os.path.dirname(rel_path)
screamer 0:66f3b5499f7f 90 if dir_path == '':
screamer 0:66f3b5499f7f 91 # all files within the current dir go into Source_Files
screamer 0:66f3b5499f7f 92 dir_path = 'Source_Files'
screamer 0:66f3b5499f7f 93 if not dir_path in files_grouped.keys():
screamer 0:66f3b5499f7f 94 files_grouped[dir_path] = []
screamer 0:66f3b5499f7f 95 files_grouped[dir_path].append(file)
screamer 0:66f3b5499f7f 96 return files_grouped
screamer 0:66f3b5499f7f 97
screamer 0:66f3b5499f7f 98 def progen_get_project_data(self):
screamer 0:66f3b5499f7f 99 """ Get ProGen project data """
screamer 0:66f3b5499f7f 100 # provide default data, some tools don't require any additional
screamer 0:66f3b5499f7f 101 # tool specific settings
screamer 0:66f3b5499f7f 102 code_files = []
screamer 0:66f3b5499f7f 103 for r_type in ['c_sources', 'cpp_sources', 's_sources']:
screamer 0:66f3b5499f7f 104 for file in getattr(self.resources, r_type):
screamer 0:66f3b5499f7f 105 code_files.append(file)
screamer 0:66f3b5499f7f 106
screamer 0:66f3b5499f7f 107 sources_files = code_files + self.resources.hex_files + self.resources.objects + \
screamer 0:66f3b5499f7f 108 self.resources.libraries
screamer 0:66f3b5499f7f 109 sources_grouped = Exporter._get_dir_grouped_files(sources_files)
screamer 0:66f3b5499f7f 110 headers_grouped = Exporter._get_dir_grouped_files(self.resources.headers)
screamer 0:66f3b5499f7f 111
screamer 0:66f3b5499f7f 112 project_data = {
screamer 0:66f3b5499f7f 113 'common': {
screamer 0:66f3b5499f7f 114 'sources': sources_grouped,
screamer 0:66f3b5499f7f 115 'includes': headers_grouped,
screamer 0:66f3b5499f7f 116 'build_dir':'.build',
screamer 0:66f3b5499f7f 117 'target': [TARGET_MAP[self.target].progen['target']],
screamer 0:66f3b5499f7f 118 'macros': self.get_symbols(),
screamer 0:66f3b5499f7f 119 'export_dir': [self.inputDir],
screamer 0:66f3b5499f7f 120 'linker_file': [self.resources.linker_script],
screamer 0:66f3b5499f7f 121 }
screamer 0:66f3b5499f7f 122 }
screamer 0:66f3b5499f7f 123 return project_data
screamer 0:66f3b5499f7f 124
screamer 23:fbae331171fa 125 def progen_gen_file(self, tool_name, project_data, progen_build=False):
screamer 13:ab47a20b66f0 126 """ Generate project using ProGen Project API """
screamer 0:66f3b5499f7f 127 settings = ProjectSettings()
screamer 0:66f3b5499f7f 128 project = Project(self.program_name, [project_data], settings)
screamer 0:66f3b5499f7f 129 # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
screamer 0:66f3b5499f7f 130 # thinks it is not dict but a file, and adds them to workspace.
screamer 0:66f3b5499f7f 131 project.project['common']['include_paths'] = self.resources.inc_dirs
screamer 13:ab47a20b66f0 132 project.generate(tool_name, copied=not self.sources_relative)
screamer 23:fbae331171fa 133 if progen_build:
screamer 23:fbae331171fa 134 print("Project exported, building...")
screamer 23:fbae331171fa 135 result = project.build(tool_name)
screamer 23:fbae331171fa 136 if result == -1:
screamer 23:fbae331171fa 137 raise FailedBuildException("Build Failed")
screamer 0:66f3b5499f7f 138
screamer 0:66f3b5499f7f 139 def __scan_all(self, path):
screamer 0:66f3b5499f7f 140 resources = []
screamer 0:66f3b5499f7f 141
screamer 0:66f3b5499f7f 142 for root, dirs, files in walk(path):
screamer 0:66f3b5499f7f 143 for d in copy(dirs):
screamer 0:66f3b5499f7f 144 if d == '.' or d == '..':
screamer 0:66f3b5499f7f 145 dirs.remove(d)
screamer 0:66f3b5499f7f 146
screamer 0:66f3b5499f7f 147 for file in files:
screamer 0:66f3b5499f7f 148 file_path = join(root, file)
screamer 0:66f3b5499f7f 149 resources.append(file_path)
screamer 0:66f3b5499f7f 150
screamer 0:66f3b5499f7f 151 return resources
screamer 0:66f3b5499f7f 152
screamer 13:ab47a20b66f0 153 def scan_and_copy_resources(self, prj_paths, trg_path, relative=False):
screamer 0:66f3b5499f7f 154 # Copy only the file for the required target and toolchain
screamer 0:66f3b5499f7f 155 lib_builds = []
screamer 13:ab47a20b66f0 156 # Create the configuration object
screamer 13:ab47a20b66f0 157 if isinstance(prj_paths, basestring):
screamer 13:ab47a20b66f0 158 prj_paths = [prj_paths]
screamer 13:ab47a20b66f0 159 config = Config(self.target, prj_paths)
screamer 0:66f3b5499f7f 160 for src in ['lib', 'src']:
screamer 13:ab47a20b66f0 161 resources = self.__scan_and_copy(join(prj_paths[0], src), trg_path)
screamer 13:ab47a20b66f0 162 for path in prj_paths[1:]:
screamer 13:ab47a20b66f0 163 resources.add(self.__scan_and_copy(join(path, src), trg_path))
screamer 13:ab47a20b66f0 164
screamer 0:66f3b5499f7f 165 lib_builds.extend(resources.lib_builds)
screamer 0:66f3b5499f7f 166
screamer 0:66f3b5499f7f 167 # The repository files
screamer 13:ab47a20b66f0 168 #for repo_dir in resources.repo_dirs:
screamer 13:ab47a20b66f0 169 # repo_files = self.__scan_all(repo_dir)
screamer 13:ab47a20b66f0 170 # for path in prj_paths:
screamer 13:ab47a20b66f0 171 # self.toolchain.copy_files(repo_files, trg_path, rel_path=join(path, src))
screamer 0:66f3b5499f7f 172
screamer 0:66f3b5499f7f 173 # The libraries builds
screamer 0:66f3b5499f7f 174 for bld in lib_builds:
screamer 0:66f3b5499f7f 175 build_url = open(bld).read().strip()
screamer 0:66f3b5499f7f 176 lib_data = self.build_url_resolver(build_url)
screamer 0:66f3b5499f7f 177 lib_path = lib_data['path'].rstrip('\\/')
screamer 0:66f3b5499f7f 178 self.__scan_and_copy(lib_path, join(trg_path, lib_data['name']))
screamer 0:66f3b5499f7f 179
screamer 0:66f3b5499f7f 180 # Create .hg dir in mbed build dir so it's ignored when versioning
screamer 0:66f3b5499f7f 181 hgdir = join(trg_path, lib_data['name'], '.hg')
screamer 0:66f3b5499f7f 182 mkdir(hgdir)
screamer 0:66f3b5499f7f 183 fhandle = file(join(hgdir, 'keep.me'), 'a')
screamer 0:66f3b5499f7f 184 fhandle.close()
screamer 0:66f3b5499f7f 185
screamer 0:66f3b5499f7f 186 if not relative:
screamer 0:66f3b5499f7f 187 # Final scan of the actual exported resources
screamer 13:ab47a20b66f0 188 resources = self.toolchain.scan_resources(trg_path)
screamer 13:ab47a20b66f0 189 resources.relative_to(trg_path, self.DOT_IN_RELATIVE_PATH)
screamer 0:66f3b5499f7f 190 else:
screamer 0:66f3b5499f7f 191 # use the prj_dir (source, not destination)
screamer 13:ab47a20b66f0 192 resources = self.toolchain.scan_resources(prj_paths[0])
screamer 13:ab47a20b66f0 193 for path in prj_paths[1:]:
screamer 13:ab47a20b66f0 194 resources.add(toolchain.scan_resources(path))
screamer 13:ab47a20b66f0 195
screamer 13:ab47a20b66f0 196 # Loads the resources into the config system which might expand/modify resources based on config data
screamer 13:ab47a20b66f0 197 self.resources = config.load_resources(resources)
screamer 13:ab47a20b66f0 198
screamer 13:ab47a20b66f0 199 if hasattr(self, "MBED_CONFIG_HEADER_SUPPORTED") and self.MBED_CONFIG_HEADER_SUPPORTED :
screamer 13:ab47a20b66f0 200 # Add the configuration file to the target directory
screamer 13:ab47a20b66f0 201 self.config_header = self.toolchain.MBED_CONFIG_FILE_NAME
screamer 13:ab47a20b66f0 202 config.get_config_data_header(join(trg_path, self.config_header))
screamer 13:ab47a20b66f0 203 self.config_macros = []
screamer 29:1210849dba19 204 self.resources.inc_dirs.append(".")
screamer 13:ab47a20b66f0 205 else:
screamer 13:ab47a20b66f0 206 # And add the configuration macros to the toolchain
screamer 13:ab47a20b66f0 207 self.config_macros = config.get_config_data_macros()
screamer 0:66f3b5499f7f 208
screamer 0:66f3b5499f7f 209 def gen_file(self, template_file, data, target_file):
screamer 0:66f3b5499f7f 210 template_path = join(Exporter.TEMPLATE_DIR, template_file)
screamer 0:66f3b5499f7f 211 template = self.jinja_environment.get_template(template_file)
screamer 0:66f3b5499f7f 212 target_text = template.render(data)
screamer 0:66f3b5499f7f 213
screamer 0:66f3b5499f7f 214 target_path = join(self.inputDir, target_file)
screamer 0:66f3b5499f7f 215 logging.debug("Generating: %s" % target_path)
screamer 0:66f3b5499f7f 216 open(target_path, "w").write(target_text)
screamer 0:66f3b5499f7f 217
screamer 0:66f3b5499f7f 218 def get_symbols(self, add_extra_symbols=True):
screamer 0:66f3b5499f7f 219 """ This function returns symbols which must be exported.
screamer 0:66f3b5499f7f 220 Please add / overwrite symbols in each exporter separately
screamer 0:66f3b5499f7f 221 """
screamer 29:1210849dba19 222
screamer 0:66f3b5499f7f 223 # We have extra symbols from e.g. libraries, we want to have them also added to export
screamer 29:1210849dba19 224 extra = self.extra_symbols if add_extra_symbols else []
screamer 29:1210849dba19 225 if hasattr(self, "MBED_CONFIG_HEADER_SUPPORTED") and self.MBED_CONFIG_HEADER_SUPPORTED:
screamer 29:1210849dba19 226 # If the config header is supported, we will preinclude it and do not not
screamer 29:1210849dba19 227 # need the macros as preprocessor flags
screamer 29:1210849dba19 228 return extra
screamer 29:1210849dba19 229
screamer 29:1210849dba19 230 symbols = self.toolchain.get_symbols(True) + self.toolchain.get_symbols() \
screamer 29:1210849dba19 231 + self.config_macros + extra
screamer 0:66f3b5499f7f 232 return symbols
screamer 0:66f3b5499f7f 233
screamer 0:66f3b5499f7f 234 def zip_working_directory_and_clean_up(tempdirectory=None, destination=None, program_name=None, clean=True):
screamer 0:66f3b5499f7f 235 uid = str(uuid.uuid4())
screamer 0:66f3b5499f7f 236 zipfilename = '%s.zip'%uid
screamer 0:66f3b5499f7f 237
screamer 0:66f3b5499f7f 238 logging.debug("Zipping up %s to %s" % (tempdirectory, join(destination, zipfilename)))
screamer 0:66f3b5499f7f 239 # make zip
screamer 0:66f3b5499f7f 240 def zipdir(basedir, archivename):
screamer 0:66f3b5499f7f 241 assert isdir(basedir)
screamer 0:66f3b5499f7f 242 fakeroot = program_name + '/'
screamer 0:66f3b5499f7f 243 with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
screamer 0:66f3b5499f7f 244 for root, _, files in os.walk(basedir):
screamer 0:66f3b5499f7f 245 # NOTE: ignore empty directories
screamer 0:66f3b5499f7f 246 for fn in files:
screamer 0:66f3b5499f7f 247 absfn = join(root, fn)
screamer 0:66f3b5499f7f 248 zfn = fakeroot + '/' + absfn[len(basedir)+len(os.sep):]
screamer 0:66f3b5499f7f 249 z.write(absfn, zfn)
screamer 0:66f3b5499f7f 250
screamer 0:66f3b5499f7f 251 zipdir(tempdirectory, join(destination, zipfilename))
screamer 0:66f3b5499f7f 252
screamer 0:66f3b5499f7f 253 if clean:
screamer 0:66f3b5499f7f 254 shutil.rmtree(tempdirectory)
screamer 0:66f3b5499f7f 255
screamer 0:66f3b5499f7f 256 return join(destination, zipfilename)