the other jimmy / mbed-sdk-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-sdk-tools by mbed official

export/exporters.py@29:1210849dba19, 2016-08-29 (annotated)

Committer:
screamer
Date:
Mon Aug 29 11:18:36 2016 +0100
Revision:
29:1210849dba19
Parent:
24:25bff2709c20
Child:
31:182518299918
Port the latest tools patches from https://github.com/ARMmbed/mbed-os

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """Just a template for subclassing"""
screamer 0:66f3b5499f7f 2 import uuid, shutil, os, logging, fnmatch
screamer 0:66f3b5499f7f 3 from os import walk, remove
screamer 0:66f3b5499f7f 4 from os.path import join, dirname, isdir, split
screamer 0:66f3b5499f7f 5 from copy import copy
screamer 0:66f3b5499f7f 6 from jinja2 import Template, FileSystemLoader
screamer 0:66f3b5499f7f 7 from jinja2.environment import Environment
screamer 0:66f3b5499f7f 8 from contextlib import closing
screamer 0:66f3b5499f7f 9 from zipfile import ZipFile, ZIP_DEFLATED
screamer 13:ab47a20b66f0 10 from operator import add
screamer 0:66f3b5499f7f 11
screamer 0:66f3b5499f7f 12 from tools.utils import mkdir
screamer 0:66f3b5499f7f 13 from tools.toolchains import TOOLCHAIN_CLASSES
screamer 0:66f3b5499f7f 14 from tools.targets import TARGET_MAP
screamer 0:66f3b5499f7f 15
screamer 0:66f3b5499f7f 16 from project_generator.generate import Generator
screamer 0:66f3b5499f7f 17 from project_generator.project import Project
screamer 0:66f3b5499f7f 18 from project_generator.settings import ProjectSettings
screamer 0:66f3b5499f7f 19
screamer 13:ab47a20b66f0 20 from tools.config import Config
screamer 13:ab47a20b66f0 21
screamer 0:66f3b5499f7f 22 class OldLibrariesException(Exception): pass
screamer 0:66f3b5499f7f 23
screamer 23:fbae331171fa 24 class FailedBuildException(Exception) : pass
screamer 23:fbae331171fa 25
screamer 24:25bff2709c20 26 # Exporter descriptor for TARGETS
screamer 24:25bff2709c20 27 # TARGETS as class attribute for backward compatibility (allows: if in Exporter.TARGETS)
screamer 24:25bff2709c20 28 class ExporterTargetsProperty(object):
screamer 24:25bff2709c20 29 def __init__(self, func):
screamer 24:25bff2709c20 30 self.func = func
screamer 24:25bff2709c20 31 def __get__(self, inst, cls):
screamer 24:25bff2709c20 32 return self.func(cls)
screamer 24:25bff2709c20 33
screamer 0:66f3b5499f7f 34 class Exporter(object):
screamer 0:66f3b5499f7f 35 TEMPLATE_DIR = dirname(__file__)
screamer 0:66f3b5499f7f 36 DOT_IN_RELATIVE_PATH = False
screamer 0:66f3b5499f7f 37
screamer 13:ab47a20b66f0 38 def __init__(self, target, inputDir, program_name, build_url_resolver, extra_symbols=None, sources_relative=True):
screamer 0:66f3b5499f7f 39 self.inputDir = inputDir
screamer 0:66f3b5499f7f 40 self.target = target
screamer 0:66f3b5499f7f 41 self.program_name = program_name
screamer 0:66f3b5499f7f 42 self.toolchain = TOOLCHAIN_CLASSES[self.get_toolchain()](TARGET_MAP[target])
screamer 0:66f3b5499f7f 43 self.build_url_resolver = build_url_resolver
screamer 0:66f3b5499f7f 44 jinja_loader = FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
screamer 0:66f3b5499f7f 45 self.jinja_environment = Environment(loader=jinja_loader)
screamer 29:1210849dba19 46 self.extra_symbols = extra_symbols if extra_symbols else []
screamer 13:ab47a20b66f0 47 self.config_macros = []
screamer 13:ab47a20b66f0 48 self.sources_relative = sources_relative
screamer 13:ab47a20b66f0 49 self.config_header = None
screamer 0:66f3b5499f7f 50
screamer 0:66f3b5499f7f 51 def get_toolchain(self):
screamer 0:66f3b5499f7f 52 return self.TOOLCHAIN
screamer 0:66f3b5499f7f 53
screamer 13:ab47a20b66f0 54 @property
screamer 13:ab47a20b66f0 55 def flags(self):
screamer 13:ab47a20b66f0 56 return self.toolchain.flags
screamer 13:ab47a20b66f0 57
screamer 13:ab47a20b66f0 58 @property
screamer 13:ab47a20b66f0 59 def progen_flags(self):
screamer 13:ab47a20b66f0 60 if not hasattr(self, "_progen_flag_cache") :
screamer 13:ab47a20b66f0 61 self._progen_flag_cache = dict([(key + "_flags", value) for key,value in self.flags.iteritems()])
screamer 29:1210849dba19 62 asm_defines = ["-D"+symbol for symbol in self.toolchain.get_symbols(True)]
screamer 29:1210849dba19 63 c_defines = ["-D" + symbol for symbol in self.toolchain.get_symbols()]
screamer 29:1210849dba19 64 self._progen_flag_cache['asm_flags'] += asm_defines
screamer 29:1210849dba19 65 self._progen_flag_cache['c_flags'] += c_defines
screamer 29:1210849dba19 66 self._progen_flag_cache['cxx_flags'] += c_defines
screamer 13:ab47a20b66f0 67 if self.config_header:
screamer 13:ab47a20b66f0 68 self._progen_flag_cache['c_flags'] += self.toolchain.get_config_option(self.config_header)
screamer 13:ab47a20b66f0 69 self._progen_flag_cache['cxx_flags'] += self.toolchain.get_config_option(self.config_header)
screamer 13:ab47a20b66f0 70 return self._progen_flag_cache
screamer 13:ab47a20b66f0 71
screamer 0:66f3b5499f7f 72 def __scan_and_copy(self, src_path, trg_path):
screamer 0:66f3b5499f7f 73 resources = self.toolchain.scan_resources(src_path)
screamer 0:66f3b5499f7f 74
screamer 0:66f3b5499f7f 75 for r_type in ['headers', 's_sources', 'c_sources', 'cpp_sources',
screamer 0:66f3b5499f7f 76 'objects', 'libraries', 'linker_script',
screamer 13:ab47a20b66f0 77 'lib_builds', 'lib_refs', 'hex_files', 'bin_files']:
screamer 0:66f3b5499f7f 78 r = getattr(resources, r_type)
screamer 0:66f3b5499f7f 79 if r:
screamer 13:ab47a20b66f0 80 self.toolchain.copy_files(r, trg_path, resources=resources)
screamer 0:66f3b5499f7f 81 return resources
screamer 0:66f3b5499f7f 82
screamer 0:66f3b5499f7f 83 @staticmethod
screamer 0:66f3b5499f7f 84 def _get_dir_grouped_files(files):
screamer 0:66f3b5499f7f 85 """ Get grouped files based on the dirname """
screamer 0:66f3b5499f7f 86 files_grouped = {}
screamer 0:66f3b5499f7f 87 for file in files:
screamer 0:66f3b5499f7f 88 rel_path = os.path.relpath(file, os.getcwd())
screamer 0:66f3b5499f7f 89 dir_path = os.path.dirname(rel_path)
screamer 0:66f3b5499f7f 90 if dir_path == '':
screamer 0:66f3b5499f7f 91 # all files within the current dir go into Source_Files
screamer 0:66f3b5499f7f 92 dir_path = 'Source_Files'
screamer 0:66f3b5499f7f 93 if not dir_path in files_grouped.keys():
screamer 0:66f3b5499f7f 94 files_grouped[dir_path] = []
screamer 0:66f3b5499f7f 95 files_grouped[dir_path].append(file)
screamer 0:66f3b5499f7f 96 return files_grouped
screamer 0:66f3b5499f7f 97
screamer 0:66f3b5499f7f 98 def progen_get_project_data(self):
screamer 0:66f3b5499f7f 99 """ Get ProGen project data """
screamer 0:66f3b5499f7f 100 # provide default data, some tools don't require any additional
screamer 0:66f3b5499f7f 101 # tool specific settings
screamer 0:66f3b5499f7f 102 code_files = []
screamer 0:66f3b5499f7f 103 for r_type in ['c_sources', 'cpp_sources', 's_sources']:
screamer 0:66f3b5499f7f 104 for file in getattr(self.resources, r_type):
screamer 0:66f3b5499f7f 105 code_files.append(file)
screamer 0:66f3b5499f7f 106
screamer 0:66f3b5499f7f 107 sources_files = code_files + self.resources.hex_files + self.resources.objects + \
screamer 0:66f3b5499f7f 108 self.resources.libraries
screamer 0:66f3b5499f7f 109 sources_grouped = Exporter._get_dir_grouped_files(sources_files)
screamer 0:66f3b5499f7f 110 headers_grouped = Exporter._get_dir_grouped_files(self.resources.headers)
screamer 0:66f3b5499f7f 111
screamer 0:66f3b5499f7f 112 project_data = {
screamer 0:66f3b5499f7f 113 'common': {
screamer 0:66f3b5499f7f 114 'sources': sources_grouped,
screamer 0:66f3b5499f7f 115 'includes': headers_grouped,
screamer 0:66f3b5499f7f 116 'build_dir':'.build',
screamer 0:66f3b5499f7f 117 'target': [TARGET_MAP[self.target].progen['target']],
screamer 0:66f3b5499f7f 118 'macros': self.get_symbols(),
screamer 0:66f3b5499f7f 119 'export_dir': [self.inputDir],
screamer 0:66f3b5499f7f 120 'linker_file': [self.resources.linker_script],
screamer 0:66f3b5499f7f 121 }
screamer 0:66f3b5499f7f 122 }
screamer 0:66f3b5499f7f 123 return project_data
screamer 0:66f3b5499f7f 124
screamer 23:fbae331171fa 125 def progen_gen_file(self, tool_name, project_data, progen_build=False):
screamer 13:ab47a20b66f0 126 """ Generate project using ProGen Project API """
screamer 0:66f3b5499f7f 127 settings = ProjectSettings()
screamer 0:66f3b5499f7f 128 project = Project(self.program_name, [project_data], settings)
screamer 0:66f3b5499f7f 129 # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
screamer 0:66f3b5499f7f 130 # thinks it is not dict but a file, and adds them to workspace.
screamer 0:66f3b5499f7f 131 project.project['common']['include_paths'] = self.resources.inc_dirs
screamer 13:ab47a20b66f0 132 project.generate(tool_name, copied=not self.sources_relative)
screamer 23:fbae331171fa 133 if progen_build:
screamer 23:fbae331171fa 134 print("Project exported, building...")
screamer 23:fbae331171fa 135 result = project.build(tool_name)
screamer 23:fbae331171fa 136 if result == -1:
screamer 23:fbae331171fa 137 raise FailedBuildException("Build Failed")
screamer 0:66f3b5499f7f 138
screamer 0:66f3b5499f7f 139 def __scan_all(self, path):
screamer 0:66f3b5499f7f 140 resources = []
screamer 0:66f3b5499f7f 141
screamer 0:66f3b5499f7f 142 for root, dirs, files in walk(path):
screamer 0:66f3b5499f7f 143 for d in copy(dirs):
screamer 0:66f3b5499f7f 144 if d == '.' or d == '..':
screamer 0:66f3b5499f7f 145 dirs.remove(d)
screamer 0:66f3b5499f7f 146
screamer 0:66f3b5499f7f 147 for file in files:
screamer 0:66f3b5499f7f 148 file_path = join(root, file)
screamer 0:66f3b5499f7f 149 resources.append(file_path)
screamer 0:66f3b5499f7f 150
screamer 0:66f3b5499f7f 151 return resources
screamer 0:66f3b5499f7f 152
screamer 13:ab47a20b66f0 153 def scan_and_copy_resources(self, prj_paths, trg_path, relative=False):
screamer 0:66f3b5499f7f 154 # Copy only the file for the required target and toolchain
screamer 0:66f3b5499f7f 155 lib_builds = []
screamer 13:ab47a20b66f0 156 # Create the configuration object
screamer 13:ab47a20b66f0 157 if isinstance(prj_paths, basestring):
screamer 13:ab47a20b66f0 158 prj_paths = [prj_paths]
screamer 13:ab47a20b66f0 159 config = Config(self.target, prj_paths)
screamer 0:66f3b5499f7f 160 for src in ['lib', 'src']:
screamer 13:ab47a20b66f0 161 resources = self.__scan_and_copy(join(prj_paths[0], src), trg_path)
screamer 13:ab47a20b66f0 162 for path in prj_paths[1:]:
screamer 13:ab47a20b66f0 163 resources.add(self.__scan_and_copy(join(path, src), trg_path))
screamer 13:ab47a20b66f0 164
screamer 0:66f3b5499f7f 165 lib_builds.extend(resources.lib_builds)
screamer 0:66f3b5499f7f 166
screamer 0:66f3b5499f7f 167 # The repository files
screamer 13:ab47a20b66f0 168 #for repo_dir in resources.repo_dirs:
screamer 13:ab47a20b66f0 169 # repo_files = self.__scan_all(repo_dir)
screamer 13:ab47a20b66f0 170 # for path in prj_paths:
screamer 13:ab47a20b66f0 171 # self.toolchain.copy_files(repo_files, trg_path, rel_path=join(path, src))
screamer 0:66f3b5499f7f 172
screamer 0:66f3b5499f7f 173 # The libraries builds
screamer 0:66f3b5499f7f 174 for bld in lib_builds:
screamer 0:66f3b5499f7f 175 build_url = open(bld).read().strip()
screamer 0:66f3b5499f7f 176 lib_data = self.build_url_resolver(build_url)
screamer 0:66f3b5499f7f 177 lib_path = lib_data['path'].rstrip('\\/')
screamer 0:66f3b5499f7f 178 self.__scan_and_copy(lib_path, join(trg_path, lib_data['name']))
screamer 0:66f3b5499f7f 179
screamer 0:66f3b5499f7f 180 # Create .hg dir in mbed build dir so it's ignored when versioning
screamer 0:66f3b5499f7f 181 hgdir = join(trg_path, lib_data['name'], '.hg')
screamer 0:66f3b5499f7f 182 mkdir(hgdir)
screamer 0:66f3b5499f7f 183 fhandle = file(join(hgdir, 'keep.me'), 'a')
screamer 0:66f3b5499f7f 184 fhandle.close()
screamer 0:66f3b5499f7f 185
screamer 0:66f3b5499f7f 186 if not relative:
screamer 0:66f3b5499f7f 187 # Final scan of the actual exported resources
screamer 13:ab47a20b66f0 188 resources = self.toolchain.scan_resources(trg_path)
screamer 13:ab47a20b66f0 189 resources.relative_to(trg_path, self.DOT_IN_RELATIVE_PATH)
screamer 0:66f3b5499f7f 190 else:
screamer 0:66f3b5499f7f 191 # use the prj_dir (source, not destination)
screamer 13:ab47a20b66f0 192 resources = self.toolchain.scan_resources(prj_paths[0])
screamer 13:ab47a20b66f0 193 for path in prj_paths[1:]:
screamer 13:ab47a20b66f0 194 resources.add(toolchain.scan_resources(path))
screamer 13:ab47a20b66f0 195
screamer 13:ab47a20b66f0 196 # Loads the resources into the config system which might expand/modify resources based on config data
screamer 13:ab47a20b66f0 197 self.resources = config.load_resources(resources)
screamer 13:ab47a20b66f0 198
screamer 13:ab47a20b66f0 199 if hasattr(self, "MBED_CONFIG_HEADER_SUPPORTED") and self.MBED_CONFIG_HEADER_SUPPORTED :
screamer 13:ab47a20b66f0 200 # Add the configuration file to the target directory
screamer 13:ab47a20b66f0 201 self.config_header = self.toolchain.MBED_CONFIG_FILE_NAME
screamer 13:ab47a20b66f0 202 config.get_config_data_header(join(trg_path, self.config_header))
screamer 13:ab47a20b66f0 203 self.config_macros = []
screamer 29:1210849dba19 204 self.resources.inc_dirs.append(".")
screamer 13:ab47a20b66f0 205 else:
screamer 13:ab47a20b66f0 206 # And add the configuration macros to the toolchain
screamer 13:ab47a20b66f0 207 self.config_macros = config.get_config_data_macros()
screamer 0:66f3b5499f7f 208
screamer 0:66f3b5499f7f 209 def gen_file(self, template_file, data, target_file):
screamer 0:66f3b5499f7f 210 template_path = join(Exporter.TEMPLATE_DIR, template_file)
screamer 0:66f3b5499f7f 211 template = self.jinja_environment.get_template(template_file)
screamer 0:66f3b5499f7f 212 target_text = template.render(data)
screamer 0:66f3b5499f7f 213
screamer 0:66f3b5499f7f 214 target_path = join(self.inputDir, target_file)
screamer 0:66f3b5499f7f 215 logging.debug("Generating: %s" % target_path)
screamer 0:66f3b5499f7f 216 open(target_path, "w").write(target_text)
screamer 0:66f3b5499f7f 217
screamer 0:66f3b5499f7f 218 def get_symbols(self, add_extra_symbols=True):
screamer 0:66f3b5499f7f 219 """ This function returns symbols which must be exported.
screamer 0:66f3b5499f7f 220 Please add / overwrite symbols in each exporter separately
screamer 0:66f3b5499f7f 221 """
screamer 29:1210849dba19 222
screamer 0:66f3b5499f7f 223 # We have extra symbols from e.g. libraries, we want to have them also added to export
screamer 29:1210849dba19 224 extra = self.extra_symbols if add_extra_symbols else []
screamer 29:1210849dba19 225 if hasattr(self, "MBED_CONFIG_HEADER_SUPPORTED") and self.MBED_CONFIG_HEADER_SUPPORTED:
screamer 29:1210849dba19 226 # If the config header is supported, we will preinclude it and do not not
screamer 29:1210849dba19 227 # need the macros as preprocessor flags
screamer 29:1210849dba19 228 return extra
screamer 29:1210849dba19 229
screamer 29:1210849dba19 230 symbols = self.toolchain.get_symbols(True) + self.toolchain.get_symbols() \
screamer 29:1210849dba19 231 + self.config_macros + extra
screamer 0:66f3b5499f7f 232 return symbols
screamer 0:66f3b5499f7f 233
screamer 0:66f3b5499f7f 234 def zip_working_directory_and_clean_up(tempdirectory=None, destination=None, program_name=None, clean=True):
screamer 0:66f3b5499f7f 235 uid = str(uuid.uuid4())
screamer 0:66f3b5499f7f 236 zipfilename = '%s.zip'%uid
screamer 0:66f3b5499f7f 237
screamer 0:66f3b5499f7f 238 logging.debug("Zipping up %s to %s" % (tempdirectory, join(destination, zipfilename)))
screamer 0:66f3b5499f7f 239 # make zip
screamer 0:66f3b5499f7f 240 def zipdir(basedir, archivename):
screamer 0:66f3b5499f7f 241 assert isdir(basedir)
screamer 0:66f3b5499f7f 242 fakeroot = program_name + '/'
screamer 0:66f3b5499f7f 243 with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
screamer 0:66f3b5499f7f 244 for root, _, files in os.walk(basedir):
screamer 0:66f3b5499f7f 245 # NOTE: ignore empty directories
screamer 0:66f3b5499f7f 246 for fn in files:
screamer 0:66f3b5499f7f 247 absfn = join(root, fn)
screamer 0:66f3b5499f7f 248 zfn = fakeroot + '/' + absfn[len(basedir)+len(os.sep):]
screamer 0:66f3b5499f7f 249 z.write(absfn, zfn)
screamer 0:66f3b5499f7f 250
screamer 0:66f3b5499f7f 251 zipdir(tempdirectory, join(destination, zipfilename))
screamer 0:66f3b5499f7f 252
screamer 0:66f3b5499f7f 253 if clean:
screamer 0:66f3b5499f7f 254 shutil.rmtree(tempdirectory)
screamer 0:66f3b5499f7f 255
screamer 0:66f3b5499f7f 256 return join(destination, zipfilename)