check_release.py

00001 # Script to check a new mbed 2 release by compiling a set of specified test apps 
00002 # for all currently supported platforms. Each test app must include an mbed library.
00003 # This can either be the pre-compiled version 'mbed' or the source version 'mbed-dev'.
00004 #
00005 # Setup:
00006 # 1. Set up your global .hgrc file
00007 # 
00008 #    If you don't already have a .hgrc file in your $HOME directory, create one there.
00009 #    Then add the following section:
00010 # 
00011 #    [auth]
00012 #    x.prefix = *
00013 #    x.username = <put your mbed org username here>
00014 #    x.password = <put your mbed org password here>
00015 #
00016 #    This has 2 purposes, the first means you don't get prompted for your password 
00017 #    whenever you run hg commands on the commandline. The second is that this script
00018 #    reads these details in order to fully automate the Mercurial commands.
00019 # 
00020 # Edit "check_release.json". This has the following structure:
00021 #{
00022 #  "config" : {
00023 #    "mbed_repo_path" : "C:/Users/annbri01/Work/Mercurial"
00024 #  },
00025 #  "test_list" : [
00026 #    {
00027 #        "name" : "test_compile_mbed_lib",
00028 #        "lib" : "mbed"
00029 #    },
00030 #    {
00031 #        "name" : "test_compile_mbed_dev",
00032 #        "lib" : "mbed-dev"
00033 #    }
00034 #  ],
00035 #  "target_list" : []
00036 #}
00037 #
00038 # The mbed_repo_path field should be changed to point to where your local
00039 # working directory is for Mercurial repositories.
00040 # For each test app you wish to run, add an entry to the test list. The example
00041 # above has 2 test apps  
00042 #     "test_compile_mbed_lib" and "test_compile_mbed_dev"
00043 # The lib field in each says which type of mbed 2 library the app contains.
00044 # These test apps MUST be available as repos in the user's online Mercurial area.
00045 # The target_list allows the user to override the set of targets/platforms used 
00046 # for the compilation.
00047 # E.g to just compile for 2 targets, K64F and K22F :
00048 # "target_list" : ["K64F", "K22F"]
00049 #
00050 # Run the script from the mbed-os directory as follows:
00051 # > python tools/check_release.py 
00052 #
00053 # It will look for local clones of the test app repos. If they don't exist
00054 # it will clone them. It will then read the latest versions of mbed and mbed-dev
00055 # (an assumption is made that both of these are already cloned in your Mercurial area).
00056 # The lib files within the test apps are then updated to the corresponding version in 
00057 # the associated lib itself. The test apps are then committed and pushed back to the users
00058 # fork.
00059 # The test apps will then be compiled for all supported targets and a % result output at 
00060 # the end.
00061 #
00062 # Uses the online compiler API at https://mbed.org/handbook/Compile-API
00063 # Based on the example from https://mbed.org/teams/mbed/code/mbed-API-helper/
00064  
00065  
00066 import os, getpass, sys, json, time, requests, logging
00067 from os.path import dirname, abspath, basename, join
00068 import argparse
00069 import subprocess
00070 import re
00071 import hglib
00072 import argparse
00073 
00074 # Be sure that the tools directory is in the search path
00075 ROOT = abspath(join(dirname(__file__), ".."))
00076 sys.path.insert(0, ROOT)
00077 
00078 from tools.build_api import get_mbed_official_release
00079 
00080 OFFICIAL_MBED_LIBRARY_BUILD = get_mbed_official_release('2')
00081 
00082 def get_compilation_failure(messages):
00083     """ Reads the json formatted 'messages' and checks for compilation errors.
00084         If there is a genuine compilation error then there should be a new 
00085         message containing a severity field = Error and an accompanying message 
00086         with the compile error text. Any other combination is considered an 
00087         internal compile engine failure 
00088     Args:
00089     messages - json formatted text returned by the online compiler API.
00090     
00091     Returns:
00092     Either "Error" or "Internal" to indicate an actual compilation error or an 
00093     internal IDE API fault.
00094 
00095     """
00096     for m in messages:
00097         # Get message text if it exists
00098         try:
00099             message = m['message']
00100             message = message + "\n"
00101         except KeyError:
00102             # Skip this message as it has no 'message' field
00103             continue
00104                  
00105         # Get type of message text
00106         try:
00107             msg_type = m['type']
00108         except KeyError:
00109             # Skip this message as it has no 'type' field
00110             continue
00111         
00112         if msg_type == 'error' or msg_type == 'tool_error':
00113             rel_log.error(message)
00114             return "Error"
00115         else: 
00116             rel_log.debug(message)
00117 
00118     return "Internal"
00119                  
00120 def invoke_api(payload, url, auth, polls, begin="start/"):
00121     """ Sends an API command request to the online IDE. Waits for a task completed 
00122         response before returning the results.
00123 
00124     Args:
00125     payload - Configuration parameters to be passed to the API
00126     url - THe URL for the online compiler API
00127     auth - Tuple containing authentication credentials
00128     polls - Number of times to poll for results
00129     begin - Default value = "start/", start command to be appended to URL
00130     
00131     Returns:
00132     result - True/False indicating the success/failure of the compilation
00133     fail_type - the failure text if the compilation failed, else None
00134     """
00135 
00136     # send task to api
00137     rel_log.debug(url + begin + "| data: " + str(payload))
00138     r = requests.post(url + begin, data=payload, auth=auth)
00139     rel_log.debug(r.request.body)
00140     
00141     if r.status_code != 200:
00142         rel_log.error("HTTP code %d reported.", r.status_code)
00143         return False, "Internal"
00144 
00145     response = r.json()
00146     rel_log.debug(response)
00147     uuid = response['result']['data']['task_id']
00148     rel_log.debug("Task accepted and given ID: %s", uuid)
00149     result = False
00150     fail_type = None
00151     
00152     # It currently seems to take the onlide IDE API ~30s to process the compile
00153     # request and provide a response. Set the poll time to half that in case it 
00154     # does manage to compile quicker.
00155     poll_delay = 15
00156     rel_log.debug("Running with a poll for response delay of: %ss", poll_delay)
00157 
00158     # poll for output
00159     for check in range(polls):
00160         time.sleep(poll_delay)
00161         
00162         try:
00163             r = requests.get(url + "output/%s" % uuid, auth=auth)
00164             
00165         except ConnectionError:
00166             return "Internal"
00167 
00168         response = r.json()
00169 
00170         data = response['result']['data']
00171         if data['task_complete']:
00172             # Task completed. Now determine the result. Should be one of :
00173             # 1) Successful compilation
00174             # 2) Failed compilation with an error message
00175             # 3) Internal failure of the online compiler            
00176             result = bool(data['compilation_success'])
00177             if result:
00178                 rel_log.info("COMPILATION SUCCESSFUL\n")
00179             else:
00180                 # Did this fail due to a genuine compilation error or a failue of 
00181                 # the api itself ?
00182                 rel_log.info("COMPILATION FAILURE\n")
00183                 fail_type = get_compilation_failure(data['new_messages'])
00184             break
00185     else:
00186         rel_log.info("COMPILATION FAILURE\n")
00187         
00188     if not result and fail_type == None:
00189         fail_type = "Internal"
00190         
00191     return result, fail_type
00192 
00193 
00194 def build_repo(target, program, user, pw, polls=25, 
00195                url="https://developer.mbed.org/api/v2/tasks/compiler/"):
00196     """ Wrapper for sending an API command request to the online IDE. Sends a 
00197         build request.
00198 
00199     Args:
00200     target - Target to be built
00201     program - Test program to build
00202     user - mbed username
00203     pw - mbed password
00204     polls - Number of times to poll for results
00205     url - THe URL for the online compiler API
00206     
00207     Returns:
00208     result - True/False indicating the success/failure of the compilation
00209     fail_type - the failure text if the compilation failed, else None
00210     """
00211     payload = {'clean':True, 'target':target, 'program':program}
00212     auth = (user, pw)
00213     return invoke_api(payload, url, auth, polls)
00214 
00215 def run_cmd(command, exit_on_failure=False):
00216     """ Passes a command to the system and returns a True/False result once the 
00217         command has been executed, indicating success/failure. Commands are passed
00218         as a list of tokens. 
00219         E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
00220 
00221     Args:
00222     command - system command as a list of tokens
00223     exit_on_failure - If True exit the program on failure (default = False)
00224     
00225     Returns:
00226     result - True/False indicating the success/failure of the command
00227     """
00228     rel_log.debug('[Exec] %s', ' '.join(command))
00229     return_code = subprocess.call(command, shell=True)
00230     
00231     if return_code:
00232         rel_log.warning("The command '%s' failed with return code: %s",  
00233                         (' '.join(command), return_code))
00234         if exit_on_failure:
00235             sys.exit(1)
00236     
00237     return return_code
00238 
00239 def run_cmd_with_output(command, exit_on_failure=False):
00240     """ Passes a command to the system and returns a True/False result once the 
00241         command has been executed, indicating success/failure. If the command was 
00242         successful then the output from the command is returned to the caller.
00243         Commands are passed as a list of tokens. 
00244         E.g. The command 'git remote -v' would be passed in as ['git', 'remote', '-v']
00245 
00246     Args:
00247     command - system command as a list of tokens
00248     exit_on_failure - If True exit the program on failure (default = False)
00249     
00250     Returns:
00251     result - True/False indicating the success/failure of the command
00252     output - The output of the command if it was successful, else empty string
00253     """
00254     rel_log.debug('[Exec] %s', ' '.join(command))
00255     returncode = 0
00256     output = ""
00257     try:
00258         output = subprocess.check_output(command, shell=True)
00259     except subprocess.CalledProcessError as e:
00260         rel_log.warning("The command '%s' failed with return code: %s", 
00261                         (' '.join(command), e.returncode))
00262         returncode = e.returncode
00263         if exit_on_failure:
00264             sys.exit(1)
00265     return returncode, output
00266 
00267 def upgrade_test_repo(test, user, library, ref, repo_path):
00268     """ Upgrades a local version of a test repo to the latest version of its 
00269         embedded library.
00270         If the test repo is not present in the user area specified in the json 
00271         config file, then it will first be cloned. 
00272     Args:
00273     test - Mercurial test repo name
00274     user - Mercurial user name
00275     library - library name
00276     ref - SHA corresponding to the latest version of the library
00277     repo_path - path to the user's repo area
00278     
00279     Returns:
00280     updated - True if library was updated, False otherwise
00281     """
00282     rel_log.info("Updating test repo: '%s' to SHA: %s", test, ref)
00283     cwd = os.getcwd()
00284 
00285     repo = "https://" + user + '@developer.mbed.org/users/' + user + '/code/' + test 
00286 
00287     # Clone the repo if it doesn't already exist
00288     path = abspath(repo_path + '/' + test)
00289     if not os.path.exists(path):
00290         rel_log.info("Test repo doesn't exist, cloning...")
00291         os.chdir(abspath(repo_path))        
00292         clone_cmd = ['hg', 'clone', repo]
00293         run_cmd(clone_cmd, exit_on_failure=True)
00294     
00295     os.chdir(path)
00296 
00297     client = hglib.open(path)        
00298 
00299     lib_file = library + '.lib'    
00300     if os.path.isfile(lib_file):
00301         # Rename command will fail on some OS's if the target file already exist,
00302         # so ensure if it does, it is deleted first.
00303         bak_file = library + '_bak' 
00304         if os.path.isfile(bak_file):
00305             os.remove(bak_file)
00306         
00307         os.rename(lib_file, bak_file)
00308     else:
00309         rel_log.error("Failure to backup lib file prior to updating.")
00310         return False
00311     
00312     # mbed 2 style lib file contains one line with the following format
00313     # e.g. https://developer.mbed.org/users/<user>/code/mbed-dev/#156823d33999
00314     exp = 'https://developer.mbed.org/users/' + user + '/code/' + library + '/#[A-Za-z0-9]+'
00315     lib_re = re.compile(exp)
00316     updated = False
00317 
00318     # Scan through mbed-os.lib line by line, looking for lib version and update 
00319     # it if found
00320     with open(bak_file, 'r') as ip, open(lib_file, 'w') as op:
00321         for line in ip:
00322 
00323             opline = line
00324             
00325             regexp = lib_re.match(line)
00326             if regexp:
00327                 opline = 'https://developer.mbed.org/users/' + user + '/code/' + library + '/#' + ref
00328                 updated = True
00329     
00330             op.write(opline)
00331 
00332     if updated:
00333 
00334         # Setup the default commit message
00335         commit_message = '"Updating ' + library + ' to ' + ref + '"' 
00336  
00337         # Setup and run the commit command. Need to use the rawcommand in the hglib
00338         # for this in order to pass the string value to the -m option. run_cmd using 
00339         # subprocess does not like this syntax.
00340         try:
00341             client.rawcommand(['commit','-m '+commit_message, lib_file])
00342 
00343             cmd = ['hg', 'push', '-f', repo]
00344             run_cmd(cmd, exit_on_failure=True)
00345             
00346         except:
00347             rel_log.info("Lib file already up to date and thus nothing to commit")
00348                             
00349     os.chdir(cwd)
00350     return updated
00351 
00352 def get_sha(repo_path, library):
00353     """ Gets the latest SHA for the library specified. The library is assumed to be
00354         located at the repo_path. If a SHA cannot be obtained this script will exit.
00355 
00356     Args:
00357     library - library name
00358     repo_path - path to the user's repo area
00359     
00360     Returns:
00361     sha - last commit SHA
00362     """
00363     cwd = os.getcwd()
00364     sha = None
00365     os.chdir(abspath(repo_path + '/' + library))
00366     
00367     cmd = ['hg', 'log', '-l', '1']
00368     ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
00369     
00370     # Output should contain a 4 line string of the form:
00371     # changeset:   135:176b8275d35d
00372     # tag:         tip
00373     # user:        <>
00374     # date:        Thu Feb 02 16:02:30 2017 +0000
00375     # summary:     Release 135 of the mbed library
00376     # All we want is the changeset string after version number
00377     
00378     lines = output.split('\n')
00379     fields = lines[0].split(':')
00380     sha = fields[2]
00381     
00382     os.chdir(cwd)
00383     return sha
00384 
00385 def get_latest_library_versions(repo_path):
00386     """ Returns the latest library versions (SHAs) for 'mbed' and 'mbed-dev'. 
00387     If the SHAs cannot be obtained this script will exit.
00388 
00389     Args:
00390     repo_path - path to the user's repo area
00391     
00392     Returns:
00393     mbed - last commit SHA for mbed library
00394     mbed_dev - last commit SHA for mbed_dev library
00395     
00396     """
00397 
00398     mbed = get_sha(repo_path, 'mbed')
00399     mbed_dev = get_sha(repo_path, 'mbed-dev')
00400 
00401     return mbed, mbed_dev
00402 
00403 def log_results(lst, title):
00404     if len(lst) == 0:
00405         rel_log.info("%s - None", title)
00406     else:        
00407         for entry in lst:
00408             rel_log.info("%s - Test: %s, Target: %s", title, entry[0], entry[1])                
00409 
00410 
00411 if __name__ == '__main__':
00412 
00413     parser = argparse.ArgumentParser(description=__doc__,
00414                                      formatter_class=argparse.RawDescriptionHelpFormatter)
00415     parser.add_argument('-l', '--log-level', 
00416                         help="Level for providing logging output", 
00417                         default='INFO')
00418     args = parser.parse_args()
00419 
00420     default = getattr(logging, 'INFO')
00421     level = getattr(logging, args.log_level.upper(), default)
00422         
00423     # Set logging level
00424     logging.basicConfig(level=level)
00425     rel_log = logging.getLogger("check-release")
00426     
00427     # Read configuration data
00428     with open(os.path.join(os.path.dirname(__file__), "check_release.json")) as config:
00429         json_data = json.load(config)
00430      
00431     supported_targets = []
00432 
00433     if len(json_data["target_list"]) > 0:
00434         # Compile user supplied subset of targets
00435         supported_targets = json_data["target_list"]
00436     else:        
00437         # Get a list of the officially supported mbed-os 2 targets
00438         for tgt in OFFICIAL_MBED_LIBRARY_BUILD:
00439             supported_targets.append(tgt[0])
00440 
00441     ignore_list = []
00442 
00443     if len(json_data["ignore_list"]) > 0:
00444         # List of tuples of (test, target) to be ignored in this test
00445         ignore_list = json_data["ignore_list"]
00446 
00447     config = json_data["config"]
00448     test_list = json_data["test_list"]
00449     repo_path = config["mbed_repo_path"]
00450     tests = []
00451 
00452     # get username
00453     cmd = ['hg', 'config', 'auth.x.username']
00454     ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
00455     output = output.split('\n')
00456     user = output[0]
00457     
00458     # get password
00459     cmd = ['hg', 'config', 'auth.x.password']
00460     ret, output = run_cmd_with_output(cmd, exit_on_failure=True)
00461     output = output.split('\n')
00462     password = output[0]
00463     
00464     mbed, mbed_dev = get_latest_library_versions(repo_path)
00465 
00466     if not mbed or not mbed_dev:
00467         rel_log.error("Could not obtain latest versions of library files!!")
00468         exit(1)
00469         
00470     rel_log.info("Latest mbed lib version = %s", mbed)
00471     rel_log.info("Latest mbed-dev lib version = %s", mbed_dev)    
00472   
00473     # First update test repos to latest versions of their embedded libraries
00474     for test in test_list:
00475         tests.append(test['name'])
00476         upgrade_test_repo(test['name'], user, test['lib'], 
00477                           mbed if test['lib'] == "mbed" else mbed_dev, 
00478                           repo_path)
00479 
00480     total = len(supported_targets) * len(tests)
00481     current = 0
00482     retries = 10
00483     passes = 0
00484     failures = []
00485     skipped = []
00486     
00487     # Compile each test for each supported target
00488     for test in tests:
00489         for target in supported_targets:
00490             
00491             combo = [test, target]
00492             
00493             if combo in ignore_list:
00494                 rel_log.info("SKIPPING TEST: %s, TARGET: %s", test, target)
00495                 total -= 1   
00496                 skipped.append(combo)
00497                 continue
00498                 
00499             current += 1
00500             for retry in range(0, retries):
00501                 rel_log.info("COMPILING (%d/%d): TEST %s, TARGET: %s , attempt %u\n", current, total, test, target,  retry)
00502                 result, mesg = build_repo(target, test, user, password)
00503                 if not result:
00504                     if mesg == 'Internal':
00505                         # Internal compiler error thus retry
00506                         continue
00507                     else:
00508                         # Actual error thus move on to next compilation
00509                         failures.append(combo)
00510                         break
00511                                     
00512                 passes += (int)(result)
00513                 break
00514             else:
00515                 rel_log.error("Compilation failed due to internal errors.")
00516                 rel_log.error("Skipping test/target combination.")
00517                 total -= 1   
00518                 skipped.append(combo)
00519                 
00520     rel_log.info(" SUMMARY OF COMPILATION RESULTS")                
00521     rel_log.info(" ------------------------------")                
00522     rel_log.info(" NUMBER OF TEST APPS: %d, NUMBER OF TARGETS: %d", 
00523                  len(tests), len(supported_targets))   
00524     log_results(failures, " FAILED")
00525     log_results(skipped, " SKIPPED")
00526 
00527     # Output a % pass rate, indicate a failure if not 100% successful
00528     pass_rate = (float(passes) / float(total)) * 100.0
00529     rel_log.info(" PASS RATE %.1f %%\n", pass_rate)
00530     sys.exit(not (pass_rate == 100))