Marco Zecchini / Mbed OS Example_RTOS

Rtos API example

mbed-os/tools/importer/importer.py@0:9fca2b23d0ba, 2019-02-23 (annotated)

Committer:
marcozecchini
Date:
Sat Feb 23 12:13:36 2019 +0000
Revision:
0:9fca2b23d0ba
final commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
marcozecchini 0:9fca2b23d0ba 1 import os
marcozecchini 0:9fca2b23d0ba 2 import json
marcozecchini 0:9fca2b23d0ba 3 import sys
marcozecchini 0:9fca2b23d0ba 4 import subprocess
marcozecchini 0:9fca2b23d0ba 5 import logging
marcozecchini 0:9fca2b23d0ba 6 import argparse
marcozecchini 0:9fca2b23d0ba 7 from os.path import dirname, abspath, join
marcozecchini 0:9fca2b23d0ba 8
marcozecchini 0:9fca2b23d0ba 9 # Be sure that the tools directory is in the search path
marcozecchini 0:9fca2b23d0ba 10 ROOT = abspath(join(dirname(__file__), "../.."))
marcozecchini 0:9fca2b23d0ba 11 sys.path.insert(0, ROOT)
marcozecchini 0:9fca2b23d0ba 12
marcozecchini 0:9fca2b23d0ba 13 from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file
marcozecchini 0:9fca2b23d0ba 14
marcozecchini 0:9fca2b23d0ba 15 def del_file(name):
marcozecchini 0:9fca2b23d0ba 16 """ Delete the file in RTOS/CMSIS/features directory of mbed-os
marcozecchini 0:9fca2b23d0ba 17 Args:
marcozecchini 0:9fca2b23d0ba 18 name - name of the file
marcozecchini 0:9fca2b23d0ba 19 """
marcozecchini 0:9fca2b23d0ba 20 result = []
marcozecchini 0:9fca2b23d0ba 21 search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')]
marcozecchini 0:9fca2b23d0ba 22 for path in search_path:
marcozecchini 0:9fca2b23d0ba 23 for root, dirs, files in os.walk(path):
marcozecchini 0:9fca2b23d0ba 24 if name in files:
marcozecchini 0:9fca2b23d0ba 25 result.append(os.path.join(root, name))
marcozecchini 0:9fca2b23d0ba 26 for file in result:
marcozecchini 0:9fca2b23d0ba 27 os.remove(file)
marcozecchini 0:9fca2b23d0ba 28 rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT))
marcozecchini 0:9fca2b23d0ba 29
marcozecchini 0:9fca2b23d0ba 30 def copy_folder(src, dest):
marcozecchini 0:9fca2b23d0ba 31 """ Copy contents of folder in mbed-os listed path
marcozecchini 0:9fca2b23d0ba 32 Args:
marcozecchini 0:9fca2b23d0ba 33 src - src folder path
marcozecchini 0:9fca2b23d0ba 34 dest - destination folder path
marcozecchini 0:9fca2b23d0ba 35 """
marcozecchini 0:9fca2b23d0ba 36 files = os.listdir(src)
marcozecchini 0:9fca2b23d0ba 37 for file in files:
marcozecchini 0:9fca2b23d0ba 38 abs_src_file = os.path.join(src, file)
marcozecchini 0:9fca2b23d0ba 39 if os.path.isfile(abs_src_file):
marcozecchini 0:9fca2b23d0ba 40 abs_dst_file = os.path.join(dest, file)
marcozecchini 0:9fca2b23d0ba 41 mkdir(os.path.dirname(abs_dst_file))
marcozecchini 0:9fca2b23d0ba 42 copy_file(abs_src_file, abs_dst_file)
marcozecchini 0:9fca2b23d0ba 43
marcozecchini 0:9fca2b23d0ba 44 def run_cmd_with_output(command, exit_on_failure=False):
marcozecchini 0:9fca2b23d0ba 45 """ Passes a command to the system and returns a True/False result once the
marcozecchini 0:9fca2b23d0ba 46 command has been executed, indicating success/failure. If the command was
marcozecchini 0:9fca2b23d0ba 47 successful then the output from the command is returned to the caller.
marcozecchini 0:9fca2b23d0ba 48 Commands are passed as a list of tokens.
marcozecchini 0:9fca2b23d0ba 49 E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
marcozecchini 0:9fca2b23d0ba 50
marcozecchini 0:9fca2b23d0ba 51 Args:
marcozecchini 0:9fca2b23d0ba 52 command - system command as a list of tokens
marcozecchini 0:9fca2b23d0ba 53 exit_on_failure - If True exit the program on failure (default = False)
marcozecchini 0:9fca2b23d0ba 54
marcozecchini 0:9fca2b23d0ba 55 Returns:
marcozecchini 0:9fca2b23d0ba 56 result - True/False indicating the success/failure of the command
marcozecchini 0:9fca2b23d0ba 57 output - The output of the command if it was successful, else empty string
marcozecchini 0:9fca2b23d0ba 58 """
marcozecchini 0:9fca2b23d0ba 59 rel_log.debug('[Exec] %s', ' '.join(command))
marcozecchini 0:9fca2b23d0ba 60 returncode = 0
marcozecchini 0:9fca2b23d0ba 61 output = ""
marcozecchini 0:9fca2b23d0ba 62 try:
marcozecchini 0:9fca2b23d0ba 63 output = subprocess.check_output(command, shell=True)
marcozecchini 0:9fca2b23d0ba 64 except subprocess.CalledProcessError as e:
marcozecchini 0:9fca2b23d0ba 65 returncode = e.returncode
marcozecchini 0:9fca2b23d0ba 66
marcozecchini 0:9fca2b23d0ba 67 if exit_on_failure:
marcozecchini 0:9fca2b23d0ba 68 rel_log.error("The command %s failed with return code: %s",
marcozecchini 0:9fca2b23d0ba 69 (' '.join(command)), returncode)
marcozecchini 0:9fca2b23d0ba 70 sys.exit(1)
marcozecchini 0:9fca2b23d0ba 71 return returncode, output
marcozecchini 0:9fca2b23d0ba 72
marcozecchini 0:9fca2b23d0ba 73 def get_curr_sha(repo_path):
marcozecchini 0:9fca2b23d0ba 74 """ Gets the latest SHA for the specified repo
marcozecchini 0:9fca2b23d0ba 75 Args:
marcozecchini 0:9fca2b23d0ba 76 repo_path - path to the repository
marcozecchini 0:9fca2b23d0ba 77
marcozecchini 0:9fca2b23d0ba 78 Returns:
marcozecchini 0:9fca2b23d0ba 79 sha - last commit SHA
marcozecchini 0:9fca2b23d0ba 80 """
marcozecchini 0:9fca2b23d0ba 81 cwd = os.getcwd()
marcozecchini 0:9fca2b23d0ba 82 os.chdir(abspath(repo_path))
marcozecchini 0:9fca2b23d0ba 83
marcozecchini 0:9fca2b23d0ba 84 cmd = "git log --pretty=format:%h -n 1"
marcozecchini 0:9fca2b23d0ba 85 _, sha = run_cmd_with_output(cmd, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 86
marcozecchini 0:9fca2b23d0ba 87 os.chdir(cwd)
marcozecchini 0:9fca2b23d0ba 88 return sha
marcozecchini 0:9fca2b23d0ba 89
marcozecchini 0:9fca2b23d0ba 90 def branch_exists(name):
marcozecchini 0:9fca2b23d0ba 91 """ Check if branch already exists in mbed-os local repository.
marcozecchini 0:9fca2b23d0ba 92 It will not verify if branch is present in remote repository.
marcozecchini 0:9fca2b23d0ba 93 Args:
marcozecchini 0:9fca2b23d0ba 94 name - branch name
marcozecchini 0:9fca2b23d0ba 95 Returns:
marcozecchini 0:9fca2b23d0ba 96 True - If branch is already present
marcozecchini 0:9fca2b23d0ba 97 """
marcozecchini 0:9fca2b23d0ba 98
marcozecchini 0:9fca2b23d0ba 99 cmd = "git branch"
marcozecchini 0:9fca2b23d0ba 100 _, output = run_cmd_with_output(cmd, exit_on_failure=False)
marcozecchini 0:9fca2b23d0ba 101 if name in output:
marcozecchini 0:9fca2b23d0ba 102 return True
marcozecchini 0:9fca2b23d0ba 103 return False
marcozecchini 0:9fca2b23d0ba 104
marcozecchini 0:9fca2b23d0ba 105 def branch_checkout(name):
marcozecchini 0:9fca2b23d0ba 106 """
marcozecchini 0:9fca2b23d0ba 107 Checkout the required branch
marcozecchini 0:9fca2b23d0ba 108 Args:
marcozecchini 0:9fca2b23d0ba 109 name - branch name
marcozecchini 0:9fca2b23d0ba 110 """
marcozecchini 0:9fca2b23d0ba 111 cmd = "git checkout " + name
marcozecchini 0:9fca2b23d0ba 112 run_cmd_with_output(cmd, exit_on_failure=False)
marcozecchini 0:9fca2b23d0ba 113
marcozecchini 0:9fca2b23d0ba 114 def get_last_cherry_pick_sha(branch):
marcozecchini 0:9fca2b23d0ba 115 """
marcozecchini 0:9fca2b23d0ba 116 SHA of last cherry pick commit is returned. SHA should be added to all
marcozecchini 0:9fca2b23d0ba 117 cherry-pick commits with -x option.
marcozecchini 0:9fca2b23d0ba 118
marcozecchini 0:9fca2b23d0ba 119 Args:
marcozecchini 0:9fca2b23d0ba 120 branch - Hash to be verified.
marcozecchini 0:9fca2b23d0ba 121 Returns - SHA if found, else None
marcozecchini 0:9fca2b23d0ba 122 """
marcozecchini 0:9fca2b23d0ba 123 cmd = "git checkout " + branch
marcozecchini 0:9fca2b23d0ba 124 run_cmd_with_output(cmd, exit_on_failure=False)
marcozecchini 0:9fca2b23d0ba 125
marcozecchini 0:9fca2b23d0ba 126 sha = None
marcozecchini 0:9fca2b23d0ba 127 get_commit = "git log -n 1"
marcozecchini 0:9fca2b23d0ba 128 _, output = run_cmd_with_output(get_commit, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 129 lines = output.split('\n')
marcozecchini 0:9fca2b23d0ba 130 for line in lines:
marcozecchini 0:9fca2b23d0ba 131 if 'cherry picked from' in line:
marcozecchini 0:9fca2b23d0ba 132 sha = line.split(' ')[-1]
marcozecchini 0:9fca2b23d0ba 133 return sha[:-1]
marcozecchini 0:9fca2b23d0ba 134 return sha
marcozecchini 0:9fca2b23d0ba 135
marcozecchini 0:9fca2b23d0ba 136 if __name__ == "__main__":
marcozecchini 0:9fca2b23d0ba 137
marcozecchini 0:9fca2b23d0ba 138 parser = argparse.ArgumentParser(description=__doc__,
marcozecchini 0:9fca2b23d0ba 139 formatter_class=argparse.RawDescriptionHelpFormatter)
marcozecchini 0:9fca2b23d0ba 140 parser.add_argument('-l', '--log-level',
marcozecchini 0:9fca2b23d0ba 141 help="Level for providing logging output",
marcozecchini 0:9fca2b23d0ba 142 default='INFO')
marcozecchini 0:9fca2b23d0ba 143 parser.add_argument('-r', '--repo-path',
marcozecchini 0:9fca2b23d0ba 144 help="Git Repository to be imported",
marcozecchini 0:9fca2b23d0ba 145 default=None,
marcozecchini 0:9fca2b23d0ba 146 required=True)
marcozecchini 0:9fca2b23d0ba 147 parser.add_argument('-c', '--config-file',
marcozecchini 0:9fca2b23d0ba 148 help="Configuration file",
marcozecchini 0:9fca2b23d0ba 149 default=None,
marcozecchini 0:9fca2b23d0ba 150 required=True)
marcozecchini 0:9fca2b23d0ba 151 args = parser.parse_args()
marcozecchini 0:9fca2b23d0ba 152 level = getattr(logging, args.log_level.upper())
marcozecchini 0:9fca2b23d0ba 153
marcozecchini 0:9fca2b23d0ba 154 # Set logging level
marcozecchini 0:9fca2b23d0ba 155 logging.basicConfig(level=level)
marcozecchini 0:9fca2b23d0ba 156 rel_log = logging.getLogger("Importer")
marcozecchini 0:9fca2b23d0ba 157
marcozecchini 0:9fca2b23d0ba 158 if (args.repo_path is None) or (args.config_file is None):
marcozecchini 0:9fca2b23d0ba 159 rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.")
marcozecchini 0:9fca2b23d0ba 160 exit(1)
marcozecchini 0:9fca2b23d0ba 161
marcozecchini 0:9fca2b23d0ba 162 json_file = os.path.abspath(args.config_file)
marcozecchini 0:9fca2b23d0ba 163 if not os.path.isfile(json_file):
marcozecchini 0:9fca2b23d0ba 164 rel_log.error("%s not found.", args.config_file)
marcozecchini 0:9fca2b23d0ba 165 exit(1)
marcozecchini 0:9fca2b23d0ba 166
marcozecchini 0:9fca2b23d0ba 167 repo = os.path.abspath(args.repo_path)
marcozecchini 0:9fca2b23d0ba 168 if not os.path.exists(repo):
marcozecchini 0:9fca2b23d0ba 169 rel_log.error("%s not found.", args.repo_path)
marcozecchini 0:9fca2b23d0ba 170 exit(1)
marcozecchini 0:9fca2b23d0ba 171
marcozecchini 0:9fca2b23d0ba 172 sha = get_curr_sha(repo)
marcozecchini 0:9fca2b23d0ba 173 if not sha:
marcozecchini 0:9fca2b23d0ba 174 rel_log.error("Could not obtain latest SHA")
marcozecchini 0:9fca2b23d0ba 175 exit(1)
marcozecchini 0:9fca2b23d0ba 176 rel_log.info("%s SHA = %s", os.path.basename(repo), sha)
marcozecchini 0:9fca2b23d0ba 177
marcozecchini 0:9fca2b23d0ba 178 branch = 'feature_' + os.path.basename(repo) + '_' + sha
marcozecchini 0:9fca2b23d0ba 179 commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha
marcozecchini 0:9fca2b23d0ba 180
marcozecchini 0:9fca2b23d0ba 181 # Read configuration data
marcozecchini 0:9fca2b23d0ba 182 with open(json_file, 'r') as config:
marcozecchini 0:9fca2b23d0ba 183 json_data = json.load(config)
marcozecchini 0:9fca2b23d0ba 184
marcozecchini 0:9fca2b23d0ba 185 '''
marcozecchini 0:9fca2b23d0ba 186 Check if branch exists already, in case branch is present
marcozecchini 0:9fca2b23d0ba 187 we will skip all file transfer and merge operations and will
marcozecchini 0:9fca2b23d0ba 188 jump to cherry-pick
marcozecchini 0:9fca2b23d0ba 189 '''
marcozecchini 0:9fca2b23d0ba 190 if branch_exists(branch):
marcozecchini 0:9fca2b23d0ba 191 rel_log.info("Branch present = %s", branch)
marcozecchini 0:9fca2b23d0ba 192 else:
marcozecchini 0:9fca2b23d0ba 193 data_files = json_data["files"]
marcozecchini 0:9fca2b23d0ba 194 data_folders = json_data["folders"]
marcozecchini 0:9fca2b23d0ba 195
marcozecchini 0:9fca2b23d0ba 196 ## Remove all files listed in .json from mbed-os repo to avoid duplications
marcozecchini 0:9fca2b23d0ba 197 for file in data_files:
marcozecchini 0:9fca2b23d0ba 198 src_file = file['src_file']
marcozecchini 0:9fca2b23d0ba 199 del_file(os.path.basename(src_file))
marcozecchini 0:9fca2b23d0ba 200
marcozecchini 0:9fca2b23d0ba 201 for folder in data_folders:
marcozecchini 0:9fca2b23d0ba 202 dest_folder = folder['dest_folder']
marcozecchini 0:9fca2b23d0ba 203 delete_dir_files(dest_folder)
marcozecchini 0:9fca2b23d0ba 204 rel_log.debug("Deleted = %s", folder)
marcozecchini 0:9fca2b23d0ba 205
marcozecchini 0:9fca2b23d0ba 206 rel_log.info("Removed files/folders listed in json file")
marcozecchini 0:9fca2b23d0ba 207
marcozecchini 0:9fca2b23d0ba 208 ## Copy all the CMSIS files listed in json file to mbed-os
marcozecchini 0:9fca2b23d0ba 209 for file in data_files:
marcozecchini 0:9fca2b23d0ba 210 repo_file = os.path.join(repo, file['src_file'])
marcozecchini 0:9fca2b23d0ba 211 mbed_path = os.path.join(ROOT, file['dest_file'])
marcozecchini 0:9fca2b23d0ba 212 mkdir(os.path.dirname(mbed_path))
marcozecchini 0:9fca2b23d0ba 213 copy_file(repo_file, mbed_path)
marcozecchini 0:9fca2b23d0ba 214 rel_log.debug("Copied = %s", mbed_path)
marcozecchini 0:9fca2b23d0ba 215
marcozecchini 0:9fca2b23d0ba 216 for folder in data_folders:
marcozecchini 0:9fca2b23d0ba 217 repo_folder = os.path.join(repo, folder['src_folder'])
marcozecchini 0:9fca2b23d0ba 218 mbed_path = os.path.join(ROOT, folder['dest_folder'])
marcozecchini 0:9fca2b23d0ba 219 copy_folder(repo_folder, mbed_path)
marcozecchini 0:9fca2b23d0ba 220 rel_log.debug("Copied = %s", mbed_path)
marcozecchini 0:9fca2b23d0ba 221
marcozecchini 0:9fca2b23d0ba 222 ## Create new branch with all changes
marcozecchini 0:9fca2b23d0ba 223 create_branch = "git checkout -b "+ branch
marcozecchini 0:9fca2b23d0ba 224 run_cmd_with_output(create_branch, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 225 rel_log.info("Branch created = %s", branch)
marcozecchini 0:9fca2b23d0ba 226
marcozecchini 0:9fca2b23d0ba 227 add_files = "git add -A"
marcozecchini 0:9fca2b23d0ba 228 run_cmd_with_output(add_files, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 229
marcozecchini 0:9fca2b23d0ba 230 commit_branch = "git commit -m \"" + commit_msg + "\""
marcozecchini 0:9fca2b23d0ba 231 run_cmd_with_output(commit_branch, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 232 rel_log.info("Commit added = %s", mbed_path)
marcozecchini 0:9fca2b23d0ba 233
marcozecchini 0:9fca2b23d0ba 234 ## Checkout the feature branch
marcozecchini 0:9fca2b23d0ba 235 branch_checkout(branch)
marcozecchini 0:9fca2b23d0ba 236 commit_sha = json_data["commit_sha"]
marcozecchini 0:9fca2b23d0ba 237 last_sha = get_last_cherry_pick_sha(branch)
marcozecchini 0:9fca2b23d0ba 238 if not last_sha:
marcozecchini 0:9fca2b23d0ba 239 ## Apply commits specific to mbed-os changes
marcozecchini 0:9fca2b23d0ba 240 for sha in commit_sha:
marcozecchini 0:9fca2b23d0ba 241 cherry_pick_sha = "git cherry-pick -x " + sha
marcozecchini 0:9fca2b23d0ba 242 run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
marcozecchini 0:9fca2b23d0ba 243 rel_log.info("Commit added = %s", cherry_pick_sha)
marcozecchini 0:9fca2b23d0ba 244 ## Few commits are already applied, check the next in sequence
marcozecchini 0:9fca2b23d0ba 245 ## and skip to last applied
marcozecchini 0:9fca2b23d0ba 246 else:
marcozecchini 0:9fca2b23d0ba 247 found = False
marcozecchini 0:9fca2b23d0ba 248 for sha in commit_sha:
marcozecchini 0:9fca2b23d0ba 249 if sha == last_sha:
marcozecchini 0:9fca2b23d0ba 250 found = True
marcozecchini 0:9fca2b23d0ba 251 continue
marcozecchini 0:9fca2b23d0ba 252 if found is True:
marcozecchini 0:9fca2b23d0ba 253 cherry_pick_sha = "git cherry-pick -x " + sha
marcozecchini 0:9fca2b23d0ba 254 run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)