importer.py

00001 import os
00002 import json
00003 import sys
00004 import subprocess
00005 import logging
00006 import argparse
00007 from os.path import dirname, abspath, join
00008 
00009 # Be sure that the tools directory is in the search path
00010 ROOT = abspath(join(dirname(__file__), "../.."))
00011 sys.path.insert(0, ROOT)
00012 
00013 from tools.utils import run_cmd, delete_dir_files, mkdir, copy_file
00014 
00015 def del_file(name):
00016     """ Delete the file in RTOS/CMSIS/features directory of mbed-os
00017     Args:
00018     name - name of the file
00019     """
00020     result = []
00021     search_path = [join(ROOT, 'rtos'), join(ROOT, 'cmsis'), join(ROOT, 'features')]
00022     for path in search_path:
00023         for root, dirs, files in os.walk(path):
00024             if name in files:
00025                 result.append(os.path.join(root, name))
00026     for file in result:
00027         os.remove(file)
00028         rel_log.debug("Deleted: %s", os.path.relpath(file, ROOT))
00029 
00030 def copy_folder(src, dest):
00031     """ Copy contents of folder in mbed-os listed path
00032     Args:
00033     src - src folder path
00034     dest - destination folder path
00035     """
00036     files = os.listdir(src)
00037     for file in files:
00038         abs_src_file = os.path.join(src, file)
00039         if os.path.isfile(abs_src_file):
00040             abs_dst_file = os.path.join(dest, file)
00041             mkdir(os.path.dirname(abs_dst_file))
00042             copy_file(abs_src_file, abs_dst_file)
00043 
00044 def run_cmd_with_output(command, exit_on_failure=False):
00045     """ Passes a command to the system and returns a True/False result once the
00046         command has been executed, indicating success/failure. If the command was
00047         successful then the output from the command is returned to the caller.
00048         Commands are passed as a list of tokens.
00049         E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
00050 
00051     Args:
00052     command - system command as a list of tokens
00053     exit_on_failure - If True exit the program on failure (default = False)
00054 
00055     Returns:
00056     result - True/False indicating the success/failure of the command
00057     output - The output of the command if it was successful, else empty string
00058     """
00059     rel_log.debug('[Exec] %s', ' '.join(command))
00060     returncode = 0
00061     output = ""
00062     try:
00063         output = subprocess.check_output(command, shell=True)
00064     except subprocess.CalledProcessError as e:
00065         returncode = e.returncode
00066 
00067         if exit_on_failure:
00068             rel_log.error("The command %s failed with return code: %s",
00069                         (' '.join(command)), returncode)
00070             sys.exit(1)
00071     return returncode, output
00072 
00073 def get_curr_sha(repo_path):
00074     """ Gets the latest SHA for the specified repo
00075     Args:
00076     repo_path - path to the repository
00077 
00078     Returns:
00079     sha - last commit SHA
00080     """
00081     cwd = os.getcwd()
00082     os.chdir(abspath(repo_path))
00083 
00084     cmd = "git log --pretty=format:%h -n 1"
00085     _, sha = run_cmd_with_output(cmd, exit_on_failure=True)
00086 
00087     os.chdir(cwd)
00088     return sha
00089 
00090 def branch_exists(name):
00091     """ Check if branch already exists in mbed-os local repository.
00092     It will not verify if branch is present in remote repository.
00093     Args:
00094     name - branch name
00095     Returns:
00096     True - If branch is already present
00097     """
00098 
00099     cmd = "git branch"
00100     _, output = run_cmd_with_output(cmd, exit_on_failure=False)
00101     if name in output:
00102         return True
00103     return False
00104 
00105 def branch_checkout(name):
00106     """
00107     Checkout the required branch
00108     Args:
00109     name - branch name
00110     """
00111     cmd = "git checkout " + name
00112     run_cmd_with_output(cmd, exit_on_failure=False)
00113 
00114 def get_last_cherry_pick_sha(branch):
00115     """
00116     SHA of last cherry pick commit is returned. SHA should be added to all
00117     cherry-pick commits with -x option.
00118 
00119     Args:
00120     branch - Hash to be verified.
00121     Returns - SHA if found, else None
00122     """
00123     cmd = "git checkout " + branch
00124     run_cmd_with_output(cmd, exit_on_failure=False)
00125 
00126     sha = None
00127     get_commit = "git log -n 1"
00128     _, output = run_cmd_with_output(get_commit, exit_on_failure=True)
00129     lines = output.split('\n')
00130     for line in lines:
00131         if 'cherry picked from' in line:
00132             sha = line.split(' ')[-1]
00133             return sha[:-1]
00134     return sha
00135 
00136 if __name__ == "__main__":
00137 
00138     parser = argparse.ArgumentParser(description=__doc__,
00139                                      formatter_class=argparse.RawDescriptionHelpFormatter)
00140     parser.add_argument('-l', '--log-level',
00141                         help="Level for providing logging output",
00142                         default='INFO')
00143     parser.add_argument('-r', '--repo-path',
00144                         help="Git Repository to be imported",
00145                         default=None,
00146                         required=True)
00147     parser.add_argument('-c', '--config-file',
00148                         help="Configuration file",
00149                         default=None,
00150                         required=True)
00151     args = parser.parse_args()
00152     level = getattr(logging, args.log_level.upper())
00153 
00154     # Set logging level
00155     logging.basicConfig(level=level)
00156     rel_log = logging.getLogger("Importer")
00157 
00158     if (args.repo_path is None) or (args.config_file is None):
00159         rel_log.error("Repository path and config file required as input. Use \"--help\" for more info.")
00160         exit(1)
00161 
00162     json_file = os.path.abspath(args.config_file)
00163     if not os.path.isfile(json_file):
00164         rel_log.error("%s not found.", args.config_file)
00165         exit(1)
00166 
00167     repo = os.path.abspath(args.repo_path)
00168     if not os.path.exists(repo):
00169         rel_log.error("%s not found.", args.repo_path)
00170         exit(1)
00171 
00172     sha = get_curr_sha(repo)
00173     if not sha:
00174         rel_log.error("Could not obtain latest SHA")
00175         exit(1)
00176     rel_log.info("%s SHA = %s", os.path.basename(repo), sha)
00177 
00178     branch = 'feature_' + os.path.basename(repo) + '_' + sha
00179     commit_msg = "[" + os.path.basename(repo) + "]" + ": Updated to " + sha
00180 
00181     # Read configuration data
00182     with open(json_file, 'r') as config:
00183         json_data = json.load(config)
00184 
00185     '''
00186     Check if branch exists already, in case branch is present
00187     we will skip all file transfer and merge operations and will
00188     jump to cherry-pick
00189     '''
00190     if branch_exists(branch):
00191         rel_log.info("Branch present = %s", branch)
00192     else:
00193         data_files = json_data["files"]
00194         data_folders = json_data["folders"]
00195 
00196         ## Remove all files listed in .json from mbed-os repo to avoid duplications
00197         for file in data_files:
00198             src_file = file['src_file']
00199             del_file(os.path.basename(src_file))
00200 
00201         for folder in data_folders:
00202             dest_folder = folder['dest_folder']
00203             delete_dir_files(dest_folder)
00204             rel_log.debug("Deleted = %s", folder)
00205 
00206         rel_log.info("Removed files/folders listed in json file")
00207 
00208         ## Copy all the CMSIS files listed in json file to mbed-os
00209         for file in data_files:
00210             repo_file = os.path.join(repo, file['src_file'])
00211             mbed_path = os.path.join(ROOT, file['dest_file'])
00212             mkdir(os.path.dirname(mbed_path))
00213             copy_file(repo_file, mbed_path)
00214             rel_log.debug("Copied = %s", mbed_path)
00215 
00216         for folder in data_folders:
00217             repo_folder = os.path.join(repo, folder['src_folder'])
00218             mbed_path = os.path.join(ROOT, folder['dest_folder'])
00219             copy_folder(repo_folder, mbed_path)
00220             rel_log.debug("Copied = %s", mbed_path)
00221 
00222         ## Create new branch with all changes
00223         create_branch = "git checkout -b "+ branch
00224         run_cmd_with_output(create_branch, exit_on_failure=True)
00225         rel_log.info("Branch created = %s", branch)
00226 
00227         add_files = "git add -A"
00228         run_cmd_with_output(add_files, exit_on_failure=True)
00229 
00230         commit_branch = "git commit -m \"" + commit_msg + "\""
00231         run_cmd_with_output(commit_branch, exit_on_failure=True)
00232         rel_log.info("Commit added = %s", mbed_path)
00233 
00234     ## Checkout the feature branch
00235     branch_checkout(branch)
00236     commit_sha = json_data["commit_sha"]
00237     last_sha = get_last_cherry_pick_sha(branch)
00238     if not last_sha:
00239         ## Apply commits specific to mbed-os changes
00240         for sha in commit_sha:
00241             cherry_pick_sha = "git cherry-pick -x " + sha
00242             run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)
00243             rel_log.info("Commit added = %s", cherry_pick_sha)
00244     ## Few commits are already applied, check the next in sequence
00245     ## and skip to last applied
00246     else:
00247         found = False
00248         for sha in commit_sha:
00249             if sha == last_sha:
00250                 found = True
00251                 continue
00252             if found is True:
00253                 cherry_pick_sha = "git cherry-pick -x " + sha
00254                 run_cmd_with_output(cherry_pick_sha, exit_on_failure=True)