BA / Mbed OS BaBoRo_test2

Backup 1

mbed-os/tools/importer/importer.py@0:02dd72d1d465, 2018-04-24 (annotated)

Committer:
borlanic
Date:
Tue Apr 24 11:45:18 2018 +0000
Revision:
0:02dd72d1d465
BaBoRo_test2 - backup 1

Who changed what in which revision?

UserRevisionLine numberNew contents of line
borlanic 0:02dd72d1d465 1 import os
borlanic 0:02dd72d1d465 2 import json
borlanic 0:02dd72d1d465 3 import sys
borlanic 0:02dd72d1d465 4 import subprocess
borlanic 0:02dd72d1d465 5 import logging
borlanic 0:02dd72d1d465 6 import argparse
borlanic 0:02dd72d1d465 7 from os.path import dirname, abspath, join
borlanic 0:02dd72d1d465 8
borlanic 0:02dd72d1d465 9 # Be sure that the tools directory is in the search path
borlanic 0:02dd72d1d465 10 ROOT = abspath(join(dirname(__file__), "../.."))
borlanic 0:02dd72d1d465 11 sys.path.insert(0, ROOT)
borlanic 0:02dd72d1d465 12
borlanic 0:02dd72d1d465 13 from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file
borlanic 0:02dd72d1d465 14
borlanic 0:02dd72d1d465 15 def del_file(name):
borlanic 0:02dd72d1d465 16 """ Delete the file in RTOS/CMSIS/features directory of mbed-os
borlanic 0:02dd72d1d465 17 Args:
borlanic 0:02dd72d1d465 18 name - name of the file
borlanic 0:02dd72d1d465 19 """
borlanic 0:02dd72d1d465 20 result = []
borlanic 0:02dd72d1d465 21 search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')]
borlanic 0:02dd72d1d465 22 for path in search_path:
borlanic 0:02dd72d1d465 23 for root, dirs, files in os.walk(path):
borlanic 0:02dd72d1d465 24 if name in files:
borlanic 0:02dd72d1d465 25 result.append(os.path.join(root, name))
borlanic 0:02dd72d1d465 26 for file in result:
borlanic 0:02dd72d1d465 27 os.remove(file)
borlanic 0:02dd72d1d465 28 rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT))
borlanic 0:02dd72d1d465 29
borlanic 0:02dd72d1d465 30 def copy_folder(src, dest):
borlanic 0:02dd72d1d465 31 """ Copy contents of folder in mbed-os listed path
borlanic 0:02dd72d1d465 32 Args:
borlanic 0:02dd72d1d465 33 src - src folder path
borlanic 0:02dd72d1d465 34 dest - destination folder path
borlanic 0:02dd72d1d465 35 """
borlanic 0:02dd72d1d465 36 files = os.listdir(src)
borlanic 0:02dd72d1d465 37 for file in files:
borlanic 0:02dd72d1d465 38 abs_src_file = os.path.join(src, file)
borlanic 0:02dd72d1d465 39 if os.path.isfile(abs_src_file):
borlanic 0:02dd72d1d465 40 abs_dst_file = os.path.join(dest, file)
borlanic 0:02dd72d1d465 41 mkdir(os.path.dirname(abs_dst_file))
borlanic 0:02dd72d1d465 42 copy_file(abs_src_file, abs_dst_file)
borlanic 0:02dd72d1d465 43
borlanic 0:02dd72d1d465 44 def run_cmd_with_output(command, exit_on_failure=False):
borlanic 0:02dd72d1d465 45 """ Passes a command to the system and returns a True/False result once the
borlanic 0:02dd72d1d465 46 command has been executed, indicating success/failure. If the command was
borlanic 0:02dd72d1d465 47 successful then the output from the command is returned to the caller.
borlanic 0:02dd72d1d465 48 Commands are passed as a list of tokens.
borlanic 0:02dd72d1d465 49 E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
borlanic 0:02dd72d1d465 50
borlanic 0:02dd72d1d465 51 Args:
borlanic 0:02dd72d1d465 52 command - system command as a list of tokens
borlanic 0:02dd72d1d465 53 exit_on_failure - If True exit the program on failure (default = False)
borlanic 0:02dd72d1d465 54
borlanic 0:02dd72d1d465 55 Returns:
borlanic 0:02dd72d1d465 56 result - True/False indicating the success/failure of the command
borlanic 0:02dd72d1d465 57 output - The output of the command if it was successful, else empty string
borlanic 0:02dd72d1d465 58 """
borlanic 0:02dd72d1d465 59 rel_log.debug('[Exec] %s', ' '.join(command))
borlanic 0:02dd72d1d465 60 returncode = 0
borlanic 0:02dd72d1d465 61 output = ""
borlanic 0:02dd72d1d465 62 try:
borlanic 0:02dd72d1d465 63 output = subprocess.check_output(command, shell=True)
borlanic 0:02dd72d1d465 64 except subprocess.CalledProcessError as e:
borlanic 0:02dd72d1d465 65 returncode = e.returncode
borlanic 0:02dd72d1d465 66
borlanic 0:02dd72d1d465 67 if exit_on_failure:
borlanic 0:02dd72d1d465 68 rel_log.error("The command %s failed with return code: %s",
borlanic 0:02dd72d1d465 69 (' '.join(command)), returncode)
borlanic 0:02dd72d1d465 70 sys.exit(1)
borlanic 0:02dd72d1d465 71 return returncode, output
borlanic 0:02dd72d1d465 72
borlanic 0:02dd72d1d465 73 def get_curr_sha(repo_path):
borlanic 0:02dd72d1d465 74 """ Gets the latest SHA for the specified repo
borlanic 0:02dd72d1d465 75 Args:
borlanic 0:02dd72d1d465 76 repo_path - path to the repository
borlanic 0:02dd72d1d465 77
borlanic 0:02dd72d1d465 78 Returns:
borlanic 0:02dd72d1d465 79 sha - last commit SHA
borlanic 0:02dd72d1d465 80 """
borlanic 0:02dd72d1d465 81 cwd = os.getcwd()
borlanic 0:02dd72d1d465 82 os.chdir(abspath(repo_path))
borlanic 0:02dd72d1d465 83
borlanic 0:02dd72d1d465 84 cmd = "git log --pretty=format:%h -n 1"
borlanic 0:02dd72d1d465 85 _, sha = run_cmd_with_output(cmd, exit_on_failure=True)
borlanic 0:02dd72d1d465 86
borlanic 0:02dd72d1d465 87 os.chdir(cwd)
borlanic 0:02dd72d1d465 88 return sha
borlanic 0:02dd72d1d465 89
borlanic 0:02dd72d1d465 90 def branch_exists(name):
borlanic 0:02dd72d1d465 91 """ Check if branch already exists in mbed-os local repository.
borlanic 0:02dd72d1d465 92 It will not verify if branch is present in remote repository.
borlanic 0:02dd72d1d465 93 Args:
borlanic 0:02dd72d1d465 94 name - branch name
borlanic 0:02dd72d1d465 95 Returns:
borlanic 0:02dd72d1d465 96 True - If branch is already present
borlanic 0:02dd72d1d465 97 """
borlanic 0:02dd72d1d465 98
borlanic 0:02dd72d1d465 99 cmd = "git branch"
borlanic 0:02dd72d1d465 100 _, output = run_cmd_with_output(cmd, exit_on_failure=False)
borlanic 0:02dd72d1d465 101 if name in output:
borlanic 0:02dd72d1d465 102 return True
borlanic 0:02dd72d1d465 103 return False
borlanic 0:02dd72d1d465 104
borlanic 0:02dd72d1d465 105 def branch_checkout(name):
borlanic 0:02dd72d1d465 106 """
borlanic 0:02dd72d1d465 107 Checkout the required branch
borlanic 0:02dd72d1d465 108 Args:
borlanic 0:02dd72d1d465 109 name - branch name
borlanic 0:02dd72d1d465 110 """
borlanic 0:02dd72d1d465 111 cmd = "git checkout " + name
borlanic 0:02dd72d1d465 112 run_cmd_with_output(cmd, exit_on_failure=False)
borlanic 0:02dd72d1d465 113
borlanic 0:02dd72d1d465 114 def get_last_cherry_pick_sha(branch):
borlanic 0:02dd72d1d465 115 """
borlanic 0:02dd72d1d465 116 SHA of last cherry pick commit is returned. SHA should be added to all
borlanic 0:02dd72d1d465 117 cherry-pick commits with -x option.
borlanic 0:02dd72d1d465 118
borlanic 0:02dd72d1d465 119 Args:
borlanic 0:02dd72d1d465 120 branch - Hash to be verified.
borlanic 0:02dd72d1d465 121 Returns - SHA if found, else None
borlanic 0:02dd72d1d465 122 """
borlanic 0:02dd72d1d465 123 cmd = "git checkout " + branch
borlanic 0:02dd72d1d465 124 run_cmd_with_output(cmd, exit_on_failure=False)
borlanic 0:02dd72d1d465 125
borlanic 0:02dd72d1d465 126 sha = None
borlanic 0:02dd72d1d465 127 get_commit = "git log -n 1"
borlanic 0:02dd72d1d465 128 _, output = run_cmd_with_output(get_commit, exit_on_failure=True)
borlanic 0:02dd72d1d465 129 lines = output.split('\n')
borlanic 0:02dd72d1d465 130 for line in lines:
borlanic 0:02dd72d1d465 131 if 'cherry picked from' in line:
borlanic 0:02dd72d1d465 132 sha = line.split(' ')[-1]
borlanic 0:02dd72d1d465 133 return sha[:-1]
borlanic 0:02dd72d1d465 134 return sha
borlanic 0:02dd72d1d465 135
borlanic 0:02dd72d1d465 136 if __name__ == "__main__":
borlanic 0:02dd72d1d465 137
borlanic 0:02dd72d1d465 138 parser = argparse.ArgumentParser(description=__doc__,
borlanic 0:02dd72d1d465 139 formatter_class=argparse.RawDescriptionHelpFormatter)
borlanic 0:02dd72d1d465 140 parser.add_argument('-l', '--log-level',
borlanic 0:02dd72d1d465 141 help="Level for providing logging output",
borlanic 0:02dd72d1d465 142 default='INFO')
borlanic 0:02dd72d1d465 143 parser.add_argument('-r', '--repo-path',
borlanic 0:02dd72d1d465 144 help="Git Repository to be imported",
borlanic 0:02dd72d1d465 145 default=None,
borlanic 0:02dd72d1d465 146 required=True)
borlanic 0:02dd72d1d465 147 parser.add_argument('-c', '--config-file',
borlanic 0:02dd72d1d465 148 help="Configuration file",
borlanic 0:02dd72d1d465 149 default=None,
borlanic 0:02dd72d1d465 150 required=True)
borlanic 0:02dd72d1d465 151 args = parser.parse_args()
borlanic 0:02dd72d1d465 152 level = getattr(logging, args.log_level.upper())
borlanic 0:02dd72d1d465 153
borlanic 0:02dd72d1d465 154 # Set logging level
borlanic 0:02dd72d1d465 155 logging.basicConfig(level=level)
borlanic 0:02dd72d1d465 156 rel_log = logging.getLogger("Importer")
borlanic 0:02dd72d1d465 157
borlanic 0:02dd72d1d465 158 if (args.repo_path is None) or (args.config_file is None):
borlanic 0:02dd72d1d465 159 rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.")
borlanic 0:02dd72d1d465 160 exit(1)
borlanic 0:02dd72d1d465 161
borlanic 0:02dd72d1d465 162 json_file = os.path.abspath(args.config_file)
borlanic 0:02dd72d1d465 163 if not os.path.isfile(json_file):
borlanic 0:02dd72d1d465 164 rel_log.error("%s not found.", args.config_file)
borlanic 0:02dd72d1d465 165 exit(1)
borlanic 0:02dd72d1d465 166
borlanic 0:02dd72d1d465 167 repo = os.path.abspath(args.repo_path)
borlanic 0:02dd72d1d465 168 if not os.path.exists(repo):
borlanic 0:02dd72d1d465 169 rel_log.error("%s not found.", args.repo_path)
borlanic 0:02dd72d1d465 170 exit(1)
borlanic 0:02dd72d1d465 171
borlanic 0:02dd72d1d465 172 sha = get_curr_sha(repo)
borlanic 0:02dd72d1d465 173 if not sha:
borlanic 0:02dd72d1d465 174 rel_log.error("Could not obtain latest SHA")
borlanic 0:02dd72d1d465 175 exit(1)
borlanic 0:02dd72d1d465 176 rel_log.info("%s SHA = %s", os.path.basename(repo), sha)
borlanic 0:02dd72d1d465 177
borlanic 0:02dd72d1d465 178 branch = 'feature_' + os.path.basename(repo) + '_' + sha
borlanic 0:02dd72d1d465 179 commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha
borlanic 0:02dd72d1d465 180
borlanic 0:02dd72d1d465 181 # Read configuration data
borlanic 0:02dd72d1d465 182 with open(json_file, 'r') as config:
borlanic 0:02dd72d1d465 183 json_data = json.load(config)
borlanic 0:02dd72d1d465 184
borlanic 0:02dd72d1d465 185 '''
borlanic 0:02dd72d1d465 186 Check if branch exists already, in case branch is present
borlanic 0:02dd72d1d465 187 we will skip all file transfer and merge operations and will
borlanic 0:02dd72d1d465 188 jump to cherry-pick
borlanic 0:02dd72d1d465 189 '''
borlanic 0:02dd72d1d465 190 if branch_exists(branch):
borlanic 0:02dd72d1d465 191 rel_log.info("Branch present = %s", branch)
borlanic 0:02dd72d1d465 192 else:
borlanic 0:02dd72d1d465 193 data_files = json_data["files"]
borlanic 0:02dd72d1d465 194 data_folders = json_data["folders"]
borlanic 0:02dd72d1d465 195
borlanic 0:02dd72d1d465 196 ## Remove all files listed in .json from mbed-os repo to avoid duplications
borlanic 0:02dd72d1d465 197 for file in data_files:
borlanic 0:02dd72d1d465 198 src_file = file['src_file']
borlanic 0:02dd72d1d465 199 del_file(os.path.basename(src_file))
borlanic 0:02dd72d1d465 200
borlanic 0:02dd72d1d465 201 for folder in data_folders:
borlanic 0:02dd72d1d465 202 dest_folder = folder['dest_folder']
borlanic 0:02dd72d1d465 203 delete_dir_files(dest_folder)
borlanic 0:02dd72d1d465 204 rel_log.debug("Deleted = %s", folder)
borlanic 0:02dd72d1d465 205
borlanic 0:02dd72d1d465 206 rel_log.info("Removed files/folders listed in json file")
borlanic 0:02dd72d1d465 207
borlanic 0:02dd72d1d465 208 ## Copy all the CMSIS files listed in json file to mbed-os
borlanic 0:02dd72d1d465 209 for file in data_files:
borlanic 0:02dd72d1d465 210 repo_file = os.path.join(repo, file['src_file'])
borlanic 0:02dd72d1d465 211 mbed_path = os.path.join(ROOT, file['dest_file'])
borlanic 0:02dd72d1d465 212 mkdir(os.path.dirname(mbed_path))
borlanic 0:02dd72d1d465 213 copy_file(repo_file, mbed_path)
borlanic 0:02dd72d1d465 214 rel_log.debug("Copied = %s", mbed_path)
borlanic 0:02dd72d1d465 215
borlanic 0:02dd72d1d465 216 for folder in data_folders:
borlanic 0:02dd72d1d465 217 repo_folder = os.path.join(repo, folder['src_folder'])
borlanic 0:02dd72d1d465 218 mbed_path = os.path.join(ROOT, folder['dest_folder'])
borlanic 0:02dd72d1d465 219 copy_folder(repo_folder, mbed_path)
borlanic 0:02dd72d1d465 220 rel_log.debug("Copied = %s", mbed_path)
borlanic 0:02dd72d1d465 221
borlanic 0:02dd72d1d465 222 ## Create new branch with all changes
borlanic 0:02dd72d1d465 223 create_branch = "git checkout -b "+ branch
borlanic 0:02dd72d1d465 224 run_cmd_with_output(create_branch, exit_on_failure=True)
borlanic 0:02dd72d1d465 225 rel_log.info("Branch created = %s", branch)
borlanic 0:02dd72d1d465 226
borlanic 0:02dd72d1d465 227 add_files = "git add -A"
borlanic 0:02dd72d1d465 228 run_cmd_with_output(add_files, exit_on_failure=True)
borlanic 0:02dd72d1d465 229
borlanic 0:02dd72d1d465 230 commit_branch = "git commit -m \"" + commit_msg + "\""
borlanic 0:02dd72d1d465 231 run_cmd_with_output(commit_branch, exit_on_failure=True)
borlanic 0:02dd72d1d465 232 rel_log.info("Commit added = %s", mbed_path)
borlanic 0:02dd72d1d465 233
borlanic 0:02dd72d1d465 234 ## Checkout the feature branch
borlanic 0:02dd72d1d465 235 branch_checkout(branch)
borlanic 0:02dd72d1d465 236 commit_sha = json_data["commit_sha"]
borlanic 0:02dd72d1d465 237 last_sha = get_last_cherry_pick_sha(branch)
borlanic 0:02dd72d1d465 238 if not last_sha:
borlanic 0:02dd72d1d465 239 ## Apply commits specific to mbed-os changes
borlanic 0:02dd72d1d465 240 for sha in commit_sha:
borlanic 0:02dd72d1d465 241 cherry_pick_sha = "git cherry-pick -x " + sha
borlanic 0:02dd72d1d465 242 run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
borlanic 0:02dd72d1d465 243 rel_log.info("Commit added = %s", cherry_pick_sha)
borlanic 0:02dd72d1d465 244 ## Few commits are already applied, check the next in sequence
borlanic 0:02dd72d1d465 245 ## and skip to last applied
borlanic 0:02dd72d1d465 246 else:
borlanic 0:02dd72d1d465 247 found = False
borlanic 0:02dd72d1d465 248 for sha in commit_sha:
borlanic 0:02dd72d1d465 249 if sha == last_sha:
borlanic 0:02dd72d1d465 250 found = True
borlanic 0:02dd72d1d465 251 continue
borlanic 0:02dd72d1d465 252 if found is True:
borlanic 0:02dd72d1d465 253 cherry_pick_sha = "git cherry-pick -x " + sha
borlanic 0:02dd72d1d465 254 run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)