examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and 
00004     the update.py modules.
00005     
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 
00014 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00015 sys.path.insert(0, ROOT)
00016 
00017 from tools.build_api import get_mbed_official_release
00018 from tools.targets import TARGET_MAP
00019 from tools.export import EXPORTERS
00020 
00021 SUPPORTED_TOOLCHAINS = ["ARM", "IAR", "GCC_ARM"]
00022 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() if exp != "cmsis" and exp != "zip"]
00023 
00024 
00025 def print_list (lst):
00026     """Prints to screen the contents of a list
00027 
00028     Args:
00029     lst - a list of any type, to be displayed
00030 
00031     """
00032     if lst:
00033         for thing in lst:
00034             print("# %s" % thing)
00035 
00036 
00037 def print_category(results, index, message):
00038     summary = [example for key, summ in results.iteritems()
00039                for example in summ[index]]
00040     if all(len(s) == 0 for s in summary):
00041         return
00042     print("#")
00043     print("#" * 80)
00044     print("# %s" % message)
00045     print("#" * 80)
00046     split_summ = [s.rsplit(" ", 1) for s in summary]
00047 
00048     print_list(summary)
00049 
00050 
00051 def print_summary (results, export=False):
00052     """Prints to screen the results of compiling/exporting combinations of example programs,
00053        targets and compile toolchains/IDEs.
00054 
00055     Args:
00056     results - results of the compilation stage. See compile_repos() and export_repos()
00057               for details of the format.
00058 
00059     """
00060 
00061     print("#"*80)
00062     print("# Examples compilation summary")
00063     print("#"*80)
00064 
00065     print_category(results, 2, "Passed example combinations")
00066 
00067     second_result = "Failed example combinations" if not export else \
00068         "Failed export example combinations"
00069 
00070     print_category(results, 3, second_result)
00071 
00072     if export:
00073         print_category(results, 4, "Failed build combinations")
00074         print_category(results, 5, "Skipped build combinations")
00075 
00076     print("#")
00077     print("#"*80)
00078 
00079 def valid_choices(allowed_choices, all_choices):
00080     if len(allowed_choices) > 0:
00081         return [t for t in all_choices if t in allowed_choices]
00082     else:
00083         return all_choices
00084 
00085 
00086 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00087     """Generate pairs of target and toolchains
00088 
00089     Args:
00090     allowed_targets - a list of all possible targets
00091     allowed_toolchains - a list of all possible toolchains
00092 
00093     Kwargs:
00094     features - the features that must be in the features array of a
00095                target
00096     """
00097     for target in allowed_targets:
00098         for toolchain in allowed_toolchains:
00099             if all(feature in TARGET_MAP[target].features
00100                     for feature in features):
00101                 yield target, toolchain
00102 
00103 
00104 def target_cross_ide (allowed_targets, allowed_ides, features=[], toolchains=[]):
00105     """Generate pairs of target and ides
00106 
00107     Args:
00108     allowed_targets - a list of all possible targets
00109     allowed_ides - a list of all possible IDEs
00110 
00111     Kwargs:
00112     features - the features that must be in the features array of a
00113                target
00114     """
00115     for target in allowed_targets:
00116         for ide in allowed_ides:
00117             if (target in EXPORTERS[ide].TARGETS and
00118                 (not toolchains or EXPORTERS[ide].TOOLCHAIN in toolchains) and
00119                 all(feature in TARGET_MAP[target].features
00120                     for feature in features)):
00121                 yield target, ide
00122 
00123 
00124 def get_repo_list (example):
00125     """ Returns a list of all the repos associated with the specific example in the json
00126         config file.
00127         If there are repos listed under the mbed section then these will be returned as a 
00128         list. If not then the github single repo with be returned. 
00129         NOTE: This does not currently deal with multiple examples underneath a github 
00130         sourced exampe repo.
00131 
00132     Args:
00133     example - Example for which the repo list is requested
00134     repos - The list of repos and types contained within that example in the json file
00135 
00136     """
00137     repos = []
00138     if len(example['mbed']) > 0:
00139         for repo in example['mbed']:
00140             repos.append({
00141                 'repo': repo,
00142                 'type': 'hg'
00143             })
00144     else:
00145         repos.append({
00146             'repo': example['github'],
00147             'type': 'git'
00148         })
00149     return repos
00150 
00151 
00152 def source_repos (config, examples):
00153     """ Imports each of the repos and its dependencies (.lib files) associated
00154         with the specific examples name from the json config file. Note if
00155         there is already a clone of the repo then it will first be removed to
00156         ensure a clean, up to date cloning.
00157     Args:
00158     config - the json object imported from the file. 
00159 
00160     """
00161     print("\nImporting example repos....\n")
00162     for example in config['examples']:
00163         for repo_info in get_repo_list(example):
00164             name = basename(repo_info['repo'])
00165             if name in examples:
00166                 if os.path.exists(name):
00167                     print("'%s' example directory already exists. Deleting..." % name)
00168                     rmtree(name)
00169 
00170                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00171 
00172 def clone_repos (config, examples):
00173     """ Clones each of the repos associated with the specific examples name from the
00174         json config file. Note if there is already a clone of the repo then it will first
00175         be removed to ensure a clean, up to date cloning.
00176     Args:
00177     config - the json object imported from the file.
00178 
00179     """
00180     print("\nCloning example repos....\n")
00181     for example in config['examples']:
00182         for repo_info in get_repo_list(example):
00183             name = basename(repo_info['repo'])
00184             if name in examples:
00185                 if os.path.exists(name):
00186                     print("'%s' example directory already exists. Deleting..." % name)
00187                     rmtree(name)
00188 
00189                 subprocess.call([repo_info['type'], "clone", repo_info['repo']])
00190 
00191 def deploy_repos (config, examples):
00192     """ If the example directory exists as provided by the json config file,
00193         pull in the examples dependencies by using `mbed-cli deploy`.
00194     Args:
00195     config - the json object imported from the file.
00196 
00197     """
00198     print("\nDeploying example repos....\n")
00199     for example in config['examples']:
00200         for repo_info in get_repo_list(example):
00201             name = basename(repo_info['repo'])
00202             if name in examples:
00203                 if os.path.exists(name):
00204                     os.chdir(name)
00205                     subprocess.call(["mbed-cli", "deploy"])
00206                     os.chdir("..")
00207                 else:
00208                     print("'%s' example directory doesn't exist. Skipping..." % name)
00209 
00210 
00211 def get_num_failures (results, export=False):
00212     """ Returns the number of failed compilations from the results summary
00213     Args:
00214     results - results summary of the compilation stage. See compile_repos() for
00215               details of the format.
00216     num_failures 
00217 
00218     """
00219     num_failures = 0
00220 
00221     for key, val in results.iteritems():
00222         num_failures = num_failures + len(val[3])
00223         if export:
00224             num_failures += len(val[4])
00225 
00226     return num_failures
00227 
00228 def export_repos (config, ides, targets, examples):
00229     """Exports and builds combinations of example programs, targets and IDEs.
00230 
00231         The results are returned in a [key: value] dictionary format:
00232             Where key = The example name from the json config file
00233             value = a list containing: pass_status, successes, export failures, build_failures,
00234             and build_skips
00235 
00236             where pass_status = The overall pass status for the export of the full
00237             set of example programs comprising the example suite.
00238             IE they must build and export) True if all examples pass, false otherwise
00239             successes = list of examples that exported and built (if possible)
00240             If the exporter has no build functionality, then it is a pass
00241             if exported
00242             export_failures = list of examples that failed to export.
00243             build_failures = list of examples that failed to build
00244             build_skips = list of examples that cannot build
00245 
00246             Both successes and failures contain the example name, target and IDE
00247 
00248             Args:
00249             config - the json object imported from the file.
00250             ides - List of IDES to export to
00251     """
00252     results = {}
00253     print("\nExporting example repos....\n")
00254     for example in config['examples']:
00255         if example['name'] not in examples:
00256             continue
00257 
00258         export_failures = []
00259         build_failures = []
00260         build_skips = []
00261         successes = []
00262         exported = True
00263         pass_status = True
00264         if example['export']:
00265             for repo_info in get_repo_list(example):
00266                 example_project_name = basename(repo_info['repo'])
00267                 os.chdir(example_project_name)
00268                 # Check that the target, IDE, and features combinations are valid and return a
00269                 # list of valid combinations to work through
00270                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00271                                                     valid_choices(example['exporters'], ides),
00272                                                     example['features'], example['toolchains']):
00273                     example_name = "{} {} {}".format(example_project_name, target,
00274                                                      ide)
00275                     def status(message):
00276                         print(message + " %s" % example_name)
00277                         sys.stdout.flush()
00278 
00279                     status("Exporting")
00280                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00281                                              "-m", target])
00282                     proc.wait()
00283                     if proc.returncode:
00284                         export_failures.append(example_name)
00285                         status("FAILURE exporting")
00286                     else:
00287                         status("SUCCESS exporting")
00288                         status("Building")
00289                         try:
00290                             if EXPORTERS[ide].build(example_project_name):
00291                                 status("FAILURE building")
00292                                 build_failures.append(example_name)
00293                             else:
00294                                 status("SUCCESS building")
00295                                 successes.append(example_name)
00296                         except TypeError:
00297                             successes.append(example_name)
00298                             build_skips.append(example_name)
00299                 os.chdir("..")
00300 
00301                 if len(build_failures+export_failures) > 0:
00302                     pass_status= False
00303         else:
00304             exported = False
00305 
00306         results[example['name']] = [exported, pass_status, successes,
00307                                     export_failures, build_failures, build_skips]
00308 
00309     return results
00310 
00311 
00312 def compile_repos (config, toolchains, targets, examples):
00313     """Compiles combinations of example programs, targets and compile chains.
00314        
00315        The results are returned in a [key: value] dictionary format:
00316        Where key = The example name from the json config file
00317              value = a list containing: pass_status, successes, and failures
00318              
00319              where pass_status = The overall pass status for the compilation of the full
00320                                  set of example programs comprising the example suite.
00321                                  True if all examples pass, false otherwise
00322                    successes = list of passing examples. 
00323                    failures = list of failing examples.
00324                    
00325                    Both successes and failures contain the example name, target and compile chain
00326 
00327     Args:
00328     config - the json object imported from the file. 
00329     toolchains - List of toolchains to compile for.
00330     results - results of the compilation stage. 
00331 
00332     """
00333     results = {}
00334     print("\nCompiling example repos....\n")
00335     for example in config['examples']:
00336         if example['name'] not in examples:
00337             continue
00338         failures = []
00339         successes = []
00340         compiled = True
00341         pass_status = True
00342         if example['compile']:
00343             for repo_info in get_repo_list(example):
00344                 name = basename(repo_info['repo'])
00345                 os.chdir(name)
00346                 
00347                 # Check that the target, toolchain and features combinations are valid and return a 
00348                 # list of valid combinations to work through
00349                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00350                                                                 valid_choices(example['toolchains'], toolchains),
00351                                                                 example['features']):
00352                     proc = subprocess.Popen(["mbed-cli", "compile", "-t", toolchain,
00353                                              "-m", target, "--silent"])
00354                     proc.wait()
00355                     example_summary = "{} {} {}".format(name, target, toolchain)
00356                     if proc.returncode:
00357                         failures.append(example_summary)
00358                     else:
00359                         successes.append(example_summary)
00360                 os.chdir("..")
00361             
00362             # If there are any compilation failures for the example 'set' then the overall status is fail.
00363             if len(failures) > 0:
00364                 pass_status = False
00365         else:
00366             compiled = False
00367 
00368         results[example['name']] = [compiled, pass_status, successes, failures]
00369 
00370     return results
00371 
00372 
00373 def update_mbedos_version (config, tag, examples):
00374     """ For each example repo identified in the config json object, update the version of 
00375         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00376         example repo has already been cloned.
00377         
00378     Args:
00379     config - the json object imported from the file. 
00380     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00381     
00382     """
00383     print("Updating mbed-os in examples to version %s\n" % tag)
00384     for example in config['examples']:
00385         if example['name'] not in examples:
00386             continue
00387         for repo_info in get_repo_list(example):
00388             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00389             print("\nChanging dir to %s\n" % update_dir)
00390             os.chdir(update_dir)
00391             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00392             os.chdir("../..")
00393