the other jimmy / mbed-sdk-tools

Important changes to repositories hosted on mbed.com

Mbed hosted mercurial repositories are deprecated and are due to be permanently deleted in July 2026.

To keep a copy of this software download the repository Zip archive or clone locally using Mercurial.

It is also possible to export all your personal repositories from the account settings page.

Fork of mbed-sdk-tools by mbed official

export/exporters.py@0:66f3b5499f7f, 2016-05-19 (annotated)

Committer:
screamer
Date:
Thu May 19 19:44:41 2016 +0100
Revision:
0:66f3b5499f7f
Child:
13:ab47a20b66f0
Initial revision

Who changed what in which revision?

UserRevisionLine numberNew contents of line
screamer 0:66f3b5499f7f 1 """Just a template for subclassing"""
screamer 0:66f3b5499f7f 2 import uuid, shutil, os, logging, fnmatch
screamer 0:66f3b5499f7f 3 from os import walk, remove
screamer 0:66f3b5499f7f 4 from os.path import join, dirname, isdir, split
screamer 0:66f3b5499f7f 5 from copy import copy
screamer 0:66f3b5499f7f 6 from jinja2 import Template, FileSystemLoader
screamer 0:66f3b5499f7f 7 from jinja2.environment import Environment
screamer 0:66f3b5499f7f 8 from contextlib import closing
screamer 0:66f3b5499f7f 9 from zipfile import ZipFile, ZIP_DEFLATED
screamer 0:66f3b5499f7f 10
screamer 0:66f3b5499f7f 11 from tools.utils import mkdir
screamer 0:66f3b5499f7f 12 from tools.toolchains import TOOLCHAIN_CLASSES
screamer 0:66f3b5499f7f 13 from tools.targets import TARGET_MAP
screamer 0:66f3b5499f7f 14
screamer 0:66f3b5499f7f 15 from project_generator.generate import Generator
screamer 0:66f3b5499f7f 16 from project_generator.project import Project
screamer 0:66f3b5499f7f 17 from project_generator.settings import ProjectSettings
screamer 0:66f3b5499f7f 18
screamer 0:66f3b5499f7f 19 class OldLibrariesException(Exception): pass
screamer 0:66f3b5499f7f 20
screamer 0:66f3b5499f7f 21 class Exporter(object):
screamer 0:66f3b5499f7f 22 TEMPLATE_DIR = dirname(__file__)
screamer 0:66f3b5499f7f 23 DOT_IN_RELATIVE_PATH = False
screamer 0:66f3b5499f7f 24
screamer 0:66f3b5499f7f 25 def __init__(self, target, inputDir, program_name, build_url_resolver, extra_symbols=None):
screamer 0:66f3b5499f7f 26 self.inputDir = inputDir
screamer 0:66f3b5499f7f 27 self.target = target
screamer 0:66f3b5499f7f 28 self.program_name = program_name
screamer 0:66f3b5499f7f 29 self.toolchain = TOOLCHAIN_CLASSES[self.get_toolchain()](TARGET_MAP[target])
screamer 0:66f3b5499f7f 30 self.build_url_resolver = build_url_resolver
screamer 0:66f3b5499f7f 31 jinja_loader = FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
screamer 0:66f3b5499f7f 32 self.jinja_environment = Environment(loader=jinja_loader)
screamer 0:66f3b5499f7f 33 self.extra_symbols = extra_symbols
screamer 0:66f3b5499f7f 34
screamer 0:66f3b5499f7f 35 def get_toolchain(self):
screamer 0:66f3b5499f7f 36 return self.TOOLCHAIN
screamer 0:66f3b5499f7f 37
screamer 0:66f3b5499f7f 38 def __scan_and_copy(self, src_path, trg_path):
screamer 0:66f3b5499f7f 39 resources = self.toolchain.scan_resources(src_path)
screamer 0:66f3b5499f7f 40
screamer 0:66f3b5499f7f 41 for r_type in ['headers', 's_sources', 'c_sources', 'cpp_sources',
screamer 0:66f3b5499f7f 42 'objects', 'libraries', 'linker_script',
screamer 0:66f3b5499f7f 43 'lib_builds', 'lib_refs', 'repo_files', 'hex_files', 'bin_files']:
screamer 0:66f3b5499f7f 44 r = getattr(resources, r_type)
screamer 0:66f3b5499f7f 45 if r:
screamer 0:66f3b5499f7f 46 self.toolchain.copy_files(r, trg_path, rel_path=src_path)
screamer 0:66f3b5499f7f 47 return resources
screamer 0:66f3b5499f7f 48
screamer 0:66f3b5499f7f 49 @staticmethod
screamer 0:66f3b5499f7f 50 def _get_dir_grouped_files(files):
screamer 0:66f3b5499f7f 51 """ Get grouped files based on the dirname """
screamer 0:66f3b5499f7f 52 files_grouped = {}
screamer 0:66f3b5499f7f 53 for file in files:
screamer 0:66f3b5499f7f 54 rel_path = os.path.relpath(file, os.getcwd())
screamer 0:66f3b5499f7f 55 dir_path = os.path.dirname(rel_path)
screamer 0:66f3b5499f7f 56 if dir_path == '':
screamer 0:66f3b5499f7f 57 # all files within the current dir go into Source_Files
screamer 0:66f3b5499f7f 58 dir_path = 'Source_Files'
screamer 0:66f3b5499f7f 59 if not dir_path in files_grouped.keys():
screamer 0:66f3b5499f7f 60 files_grouped[dir_path] = []
screamer 0:66f3b5499f7f 61 files_grouped[dir_path].append(file)
screamer 0:66f3b5499f7f 62 return files_grouped
screamer 0:66f3b5499f7f 63
screamer 0:66f3b5499f7f 64 def progen_get_project_data(self):
screamer 0:66f3b5499f7f 65 """ Get ProGen project data """
screamer 0:66f3b5499f7f 66 # provide default data, some tools don't require any additional
screamer 0:66f3b5499f7f 67 # tool specific settings
screamer 0:66f3b5499f7f 68 code_files = []
screamer 0:66f3b5499f7f 69 for r_type in ['c_sources', 'cpp_sources', 's_sources']:
screamer 0:66f3b5499f7f 70 for file in getattr(self.resources, r_type):
screamer 0:66f3b5499f7f 71 code_files.append(file)
screamer 0:66f3b5499f7f 72
screamer 0:66f3b5499f7f 73 sources_files = code_files + self.resources.hex_files + self.resources.objects + \
screamer 0:66f3b5499f7f 74 self.resources.libraries
screamer 0:66f3b5499f7f 75 sources_grouped = Exporter._get_dir_grouped_files(sources_files)
screamer 0:66f3b5499f7f 76 headers_grouped = Exporter._get_dir_grouped_files(self.resources.headers)
screamer 0:66f3b5499f7f 77
screamer 0:66f3b5499f7f 78 project_data = {
screamer 0:66f3b5499f7f 79 'common': {
screamer 0:66f3b5499f7f 80 'sources': sources_grouped,
screamer 0:66f3b5499f7f 81 'includes': headers_grouped,
screamer 0:66f3b5499f7f 82 'build_dir':'.build',
screamer 0:66f3b5499f7f 83 'target': [TARGET_MAP[self.target].progen['target']],
screamer 0:66f3b5499f7f 84 'macros': self.get_symbols(),
screamer 0:66f3b5499f7f 85 'export_dir': [self.inputDir],
screamer 0:66f3b5499f7f 86 'linker_file': [self.resources.linker_script],
screamer 0:66f3b5499f7f 87 }
screamer 0:66f3b5499f7f 88 }
screamer 0:66f3b5499f7f 89 return project_data
screamer 0:66f3b5499f7f 90
screamer 0:66f3b5499f7f 91 def progen_gen_file(self, tool_name, project_data):
screamer 0:66f3b5499f7f 92 """" Generate project using ProGen Project API """
screamer 0:66f3b5499f7f 93 settings = ProjectSettings()
screamer 0:66f3b5499f7f 94 project = Project(self.program_name, [project_data], settings)
screamer 0:66f3b5499f7f 95 # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
screamer 0:66f3b5499f7f 96 # thinks it is not dict but a file, and adds them to workspace.
screamer 0:66f3b5499f7f 97 project.project['common']['include_paths'] = self.resources.inc_dirs
screamer 0:66f3b5499f7f 98 project.generate(tool_name, copied=True)
screamer 0:66f3b5499f7f 99
screamer 0:66f3b5499f7f 100 def __scan_all(self, path):
screamer 0:66f3b5499f7f 101 resources = []
screamer 0:66f3b5499f7f 102
screamer 0:66f3b5499f7f 103 for root, dirs, files in walk(path):
screamer 0:66f3b5499f7f 104 for d in copy(dirs):
screamer 0:66f3b5499f7f 105 if d == '.' or d == '..':
screamer 0:66f3b5499f7f 106 dirs.remove(d)
screamer 0:66f3b5499f7f 107
screamer 0:66f3b5499f7f 108 for file in files:
screamer 0:66f3b5499f7f 109 file_path = join(root, file)
screamer 0:66f3b5499f7f 110 resources.append(file_path)
screamer 0:66f3b5499f7f 111
screamer 0:66f3b5499f7f 112 return resources
screamer 0:66f3b5499f7f 113
screamer 0:66f3b5499f7f 114 def scan_and_copy_resources(self, prj_path, trg_path, relative=False):
screamer 0:66f3b5499f7f 115 # Copy only the file for the required target and toolchain
screamer 0:66f3b5499f7f 116 lib_builds = []
screamer 0:66f3b5499f7f 117 for src in ['lib', 'src']:
screamer 0:66f3b5499f7f 118 resources = self.__scan_and_copy(join(prj_path, src), trg_path)
screamer 0:66f3b5499f7f 119 lib_builds.extend(resources.lib_builds)
screamer 0:66f3b5499f7f 120
screamer 0:66f3b5499f7f 121 # The repository files
screamer 0:66f3b5499f7f 122 for repo_dir in resources.repo_dirs:
screamer 0:66f3b5499f7f 123 repo_files = self.__scan_all(repo_dir)
screamer 0:66f3b5499f7f 124 self.toolchain.copy_files(repo_files, trg_path, rel_path=join(prj_path, src))
screamer 0:66f3b5499f7f 125
screamer 0:66f3b5499f7f 126 # The libraries builds
screamer 0:66f3b5499f7f 127 for bld in lib_builds:
screamer 0:66f3b5499f7f 128 build_url = open(bld).read().strip()
screamer 0:66f3b5499f7f 129 lib_data = self.build_url_resolver(build_url)
screamer 0:66f3b5499f7f 130 lib_path = lib_data['path'].rstrip('\\/')
screamer 0:66f3b5499f7f 131 self.__scan_and_copy(lib_path, join(trg_path, lib_data['name']))
screamer 0:66f3b5499f7f 132
screamer 0:66f3b5499f7f 133 # Create .hg dir in mbed build dir so it's ignored when versioning
screamer 0:66f3b5499f7f 134 hgdir = join(trg_path, lib_data['name'], '.hg')
screamer 0:66f3b5499f7f 135 mkdir(hgdir)
screamer 0:66f3b5499f7f 136 fhandle = file(join(hgdir, 'keep.me'), 'a')
screamer 0:66f3b5499f7f 137 fhandle.close()
screamer 0:66f3b5499f7f 138
screamer 0:66f3b5499f7f 139 if not relative:
screamer 0:66f3b5499f7f 140 # Final scan of the actual exported resources
screamer 0:66f3b5499f7f 141 self.resources = self.toolchain.scan_resources(trg_path)
screamer 0:66f3b5499f7f 142 self.resources.relative_to(trg_path, self.DOT_IN_RELATIVE_PATH)
screamer 0:66f3b5499f7f 143 else:
screamer 0:66f3b5499f7f 144 # use the prj_dir (source, not destination)
screamer 0:66f3b5499f7f 145 self.resources = self.toolchain.scan_resources(prj_path)
screamer 0:66f3b5499f7f 146 # Check the existence of a binary build of the mbed library for the desired target
screamer 0:66f3b5499f7f 147 # This prevents exporting the mbed libraries from source
screamer 0:66f3b5499f7f 148 # if not self.toolchain.mbed_libs:
screamer 0:66f3b5499f7f 149 # raise OldLibrariesException()
screamer 0:66f3b5499f7f 150
screamer 0:66f3b5499f7f 151 def gen_file(self, template_file, data, target_file):
screamer 0:66f3b5499f7f 152 template_path = join(Exporter.TEMPLATE_DIR, template_file)
screamer 0:66f3b5499f7f 153 template = self.jinja_environment.get_template(template_file)
screamer 0:66f3b5499f7f 154 target_text = template.render(data)
screamer 0:66f3b5499f7f 155
screamer 0:66f3b5499f7f 156 target_path = join(self.inputDir, target_file)
screamer 0:66f3b5499f7f 157 logging.debug("Generating: %s" % target_path)
screamer 0:66f3b5499f7f 158 open(target_path, "w").write(target_text)
screamer 0:66f3b5499f7f 159
screamer 0:66f3b5499f7f 160 def get_symbols(self, add_extra_symbols=True):
screamer 0:66f3b5499f7f 161 """ This function returns symbols which must be exported.
screamer 0:66f3b5499f7f 162 Please add / overwrite symbols in each exporter separately
screamer 0:66f3b5499f7f 163 """
screamer 0:66f3b5499f7f 164 symbols = self.toolchain.get_symbols()
screamer 0:66f3b5499f7f 165 # We have extra symbols from e.g. libraries, we want to have them also added to export
screamer 0:66f3b5499f7f 166 if add_extra_symbols:
screamer 0:66f3b5499f7f 167 if self.extra_symbols is not None:
screamer 0:66f3b5499f7f 168 symbols.extend(self.extra_symbols)
screamer 0:66f3b5499f7f 169 return symbols
screamer 0:66f3b5499f7f 170
screamer 0:66f3b5499f7f 171 def zip_working_directory_and_clean_up(tempdirectory=None, destination=None, program_name=None, clean=True):
screamer 0:66f3b5499f7f 172 uid = str(uuid.uuid4())
screamer 0:66f3b5499f7f 173 zipfilename = '%s.zip'%uid
screamer 0:66f3b5499f7f 174
screamer 0:66f3b5499f7f 175 logging.debug("Zipping up %s to %s" % (tempdirectory, join(destination, zipfilename)))
screamer 0:66f3b5499f7f 176 # make zip
screamer 0:66f3b5499f7f 177 def zipdir(basedir, archivename):
screamer 0:66f3b5499f7f 178 assert isdir(basedir)
screamer 0:66f3b5499f7f 179 fakeroot = program_name + '/'
screamer 0:66f3b5499f7f 180 with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
screamer 0:66f3b5499f7f 181 for root, _, files in os.walk(basedir):
screamer 0:66f3b5499f7f 182 # NOTE: ignore empty directories
screamer 0:66f3b5499f7f 183 for fn in files:
screamer 0:66f3b5499f7f 184 absfn = join(root, fn)
screamer 0:66f3b5499f7f 185 zfn = fakeroot + '/' + absfn[len(basedir)+len(os.sep):]
screamer 0:66f3b5499f7f 186 z.write(absfn, zfn)
screamer 0:66f3b5499f7f 187
screamer 0:66f3b5499f7f 188 zipdir(tempdirectory, join(destination, zipfilename))
screamer 0:66f3b5499f7f 189
screamer 0:66f3b5499f7f 190 if clean:
screamer 0:66f3b5499f7f 191 shutil.rmtree(tempdirectory)
screamer 0:66f3b5499f7f 192
screamer 0:66f3b5499f7f 193 return join(destination, zipfilename)