examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and 
00004     the update.py modules.
00005     
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 
00014 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00015 sys.path.insert(0, ROOT)
00016 
00017 from tools.build_api import get_mbed_official_release
00018 from tools.targets import TARGET_MAP
00019 from tools.export import EXPORTERS
00020 
00021 SUPPORTED_TOOLCHAINS = ["ARM", "IAR", "GCC_ARM"]
00022 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() if exp != "cmsis" and exp != "zip"]
00023 
00024 
00025 def print_list (lst):
00026     """Prints to screen the contents of a list
00027 
00028     Args:
00029     lst - a list of any type, to be displayed
00030 
00031     """
00032     if lst:
00033         for thing in lst:
00034             print("# %s" % thing)
00035 
00036 
00037 def print_category(results, index, message):
00038     summary = [example for key, summ in results.iteritems()
00039                for example in summ[index]]
00040     if all(len(s) == 0 for s in summary):
00041         return
00042     print("#")
00043     print("#" * 80)
00044     print("# %s" % message)
00045     print("#" * 80)
00046     split_summ = [s.rsplit(" ", 1) for s in summary]
00047 
00048     print_list(summary)
00049 
00050 
00051 def print_summary (results, export=False):
00052     """Prints to screen the results of compiling/exporting combinations of example programs,
00053        targets and compile toolchains/IDEs.
00054 
00055     Args:
00056     results - results of the compilation stage. See compile_repos() and export_repos()
00057               for details of the format.
00058 
00059     """
00060 
00061     print("#"*80)
00062     print("# Examples compilation summary")
00063     print("#"*80)
00064 
00065     print_category(results, 2, "Passed example combinations")
00066 
00067     second_result = "Failed example combinations" if not export else \
00068         "Failed export example combinations"
00069 
00070     print_category(results, 3, second_result)
00071 
00072     if export:
00073         print_category(results, 4, "Failed build combinations")
00074         print_category(results, 5, "Skipped build combinations")
00075 
00076     print("#")
00077     print("#"*80)
00078 
00079 def valid_choices(allowed_choices, all_choices):
00080     if len(allowed_choices) > 0:
00081         return [t for t in all_choices if t in allowed_choices]
00082     else:
00083         return all_choices
00084 
00085 
00086 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00087     """Generate pairs of target and toolchains
00088 
00089     Args:
00090     allowed_targets - a list of all possible targets
00091     allowed_toolchains - a list of all possible toolchains
00092 
00093     Kwargs:
00094     features - the features that must be in the features array of a
00095                target
00096     """
00097     for target in allowed_targets:
00098         for toolchain in allowed_toolchains:
00099             if all(feature in TARGET_MAP[target].features
00100                     for feature in features):
00101                 yield target, toolchain
00102 
00103 
00104 def target_cross_ide (allowed_targets, allowed_ides, features=[]):
00105     """Generate pairs of target and ides
00106 
00107     Args:
00108     allowed_targets - a list of all possible targets
00109     allowed_ides - a list of all possible IDEs
00110 
00111     Kwargs:
00112     features - the features that must be in the features array of a
00113                target
00114     """
00115     for target in allowed_targets:
00116         for ide in allowed_ides:
00117             if (target in EXPORTERS[ide].TARGETS and
00118                 all(feature in TARGET_MAP[target].features
00119                     for feature in features)):
00120                 yield target, ide
00121 
00122 
00123 def get_repo_list (example):
00124     """ Returns a list of all the repos associated with the specific example in the json
00125         config file.
00126         If there are repos listed under the mbed section then these will be returned as a 
00127         list. If not then the github single repo with be returned. 
00128         NOTE: This does not currently deal with multiple examples underneath a github 
00129         sourced exampe repo.
00130 
00131     Args:
00132     example - Example for which the repo list is requested
00133     repos - The list of repos and types contained within that example in the json file
00134 
00135     """
00136     repos = []
00137     if len(example['mbed']) > 0:
00138         for repo in example['mbed']:
00139             repos.append({
00140                 'repo': repo,
00141                 'type': 'hg'
00142             })
00143     else:
00144         repos.append({
00145             'repo': example['github'],
00146             'type': 'git'
00147         })
00148     return repos
00149 
00150 
00151 def source_repos (config, examples):
00152     """ Imports each of the repos and its dependencies (.lib files) associated
00153         with the specific examples name from the json config file. Note if
00154         there is already a clone of the repo then it will first be removed to
00155         ensure a clean, up to date cloning.
00156     Args:
00157     config - the json object imported from the file. 
00158 
00159     """
00160     print("\nImporting example repos....\n")
00161     for example in config['examples']:
00162         for repo_info in get_repo_list(example):
00163             name = basename(repo_info['repo'])
00164             if name in examples:
00165                 if os.path.exists(name):
00166                     print("'%s' example directory already exists. Deleting..." % name)
00167                     rmtree(name)
00168 
00169                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00170 
00171 def clone_repos (config, examples):
00172     """ Clones each of the repos associated with the specific examples name from the
00173         json config file. Note if there is already a clone of the repo then it will first
00174         be removed to ensure a clean, up to date cloning.
00175     Args:
00176     config - the json object imported from the file.
00177 
00178     """
00179     print("\nCloning example repos....\n")
00180     for example in config['examples']:
00181         for repo_info in get_repo_list(example):
00182             name = basename(repo_info['repo'])
00183             if name in examples:
00184                 if os.path.exists(name):
00185                     print("'%s' example directory already exists. Deleting..." % name)
00186                     rmtree(name)
00187 
00188                 subprocess.call([repo_info['type'], "clone", repo_info['repo']])
00189 
00190 def deploy_repos (config, examples):
00191     """ If the example directory exists as provided by the json config file,
00192         pull in the examples dependencies by using `mbed-cli deploy`.
00193     Args:
00194     config - the json object imported from the file.
00195 
00196     """
00197     print("\nDeploying example repos....\n")
00198     for example in config['examples']:
00199         for repo_info in get_repo_list(example):
00200             name = basename(repo_info['repo'])
00201             if name in examples:
00202                 if os.path.exists(name):
00203                     os.chdir(name)
00204                     subprocess.call(["mbed-cli", "deploy"])
00205                     os.chdir("..")
00206                 else:
00207                     print("'%s' example directory doesn't exist. Skipping..." % name)
00208 
00209 
00210 def get_num_failures (results, export=False):
00211     """ Returns the number of failed compilations from the results summary
00212     Args:
00213     results - results summary of the compilation stage. See compile_repos() for
00214               details of the format.
00215     num_failures 
00216 
00217     """
00218     num_failures = 0
00219 
00220     for key, val in results.iteritems():
00221         num_failures = num_failures + len(val[3])
00222         if export:
00223             num_failures += len(val[4])
00224 
00225     return num_failures
00226 
00227 def export_repos (config, ides, targets, examples):
00228     """Exports and builds combinations of example programs, targets and IDEs.
00229 
00230         The results are returned in a [key: value] dictionary format:
00231             Where key = The example name from the json config file
00232             value = a list containing: pass_status, successes, export failures, build_failures,
00233             and build_skips
00234 
00235             where pass_status = The overall pass status for the export of the full
00236             set of example programs comprising the example suite.
00237             IE they must build and export) True if all examples pass, false otherwise
00238             successes = list of examples that exported and built (if possible)
00239             If the exporter has no build functionality, then it is a pass
00240             if exported
00241             export_failures = list of examples that failed to export.
00242             build_failures = list of examples that failed to build
00243             build_skips = list of examples that cannot build
00244 
00245             Both successes and failures contain the example name, target and IDE
00246 
00247             Args:
00248             config - the json object imported from the file.
00249             ides - List of IDES to export to
00250     """
00251     results = {}
00252     print("\nExporting example repos....\n")
00253     for example in config['examples']:
00254         if example['name'] not in examples:
00255             continue
00256 
00257         export_failures = []
00258         build_failures = []
00259         build_skips = []
00260         successes = []
00261         exported = True
00262         pass_status = True
00263         if example['export']:
00264             for repo_info in get_repo_list(example):
00265                 example_project_name = basename(repo_info['repo'])
00266                 os.chdir(example_project_name)
00267                 # Check that the target, IDE, and features combinations are valid and return a
00268                 # list of valid combinations to work through
00269                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00270                                                     valid_choices(example['exporters'], ides),
00271                                                     example['features']):
00272                     example_name = "{} {} {}".format(example_project_name, target,
00273                                                      ide)
00274                     def status(message):
00275                         print(message + " %s" % example_name)
00276                         sys.stdout.flush()
00277 
00278                     status("Exporting")
00279                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00280                                              "-m", target])
00281                     proc.wait()
00282                     if proc.returncode:
00283                         export_failures.append(example_name)
00284                         status("FAILURE exporting")
00285                     else:
00286                         status("SUCCESS exporting")
00287                         status("Building")
00288                         try:
00289                             if EXPORTERS[ide].build(example_project_name):
00290                                 status("FAILURE building")
00291                                 build_failures.append(example_name)
00292                             else:
00293                                 status("SUCCESS building")
00294                                 successes.append(example_name)
00295                         except TypeError:
00296                             successes.append(example_name)
00297                             build_skips.append(example_name)
00298                 os.chdir("..")
00299 
00300                 if len(build_failures+export_failures) > 0:
00301                     pass_status= False
00302         else:
00303             exported = False
00304 
00305         results[example['name']] = [exported, pass_status, successes,
00306                                     export_failures, build_failures, build_skips]
00307 
00308     return results
00309 
00310 
00311 def compile_repos (config, toolchains, targets, examples):
00312     """Compiles combinations of example programs, targets and compile chains.
00313        
00314        The results are returned in a [key: value] dictionary format:
00315        Where key = The example name from the json config file
00316              value = a list containing: pass_status, successes, and failures
00317              
00318              where pass_status = The overall pass status for the compilation of the full
00319                                  set of example programs comprising the example suite.
00320                                  True if all examples pass, false otherwise
00321                    successes = list of passing examples. 
00322                    failures = list of failing examples.
00323                    
00324                    Both successes and failures contain the example name, target and compile chain
00325 
00326     Args:
00327     config - the json object imported from the file. 
00328     toolchains - List of toolchains to compile for.
00329     results - results of the compilation stage. 
00330 
00331     """
00332     results = {}
00333     print("\nCompiling example repos....\n")
00334     for example in config['examples']:
00335         if example['name'] not in examples:
00336             continue
00337         failures = []
00338         successes = []
00339         compiled = True
00340         pass_status = True
00341         if example['compile']:
00342             for repo_info in get_repo_list(example):
00343                 name = basename(repo_info['repo'])
00344                 os.chdir(name)
00345                 
00346                 # Check that the target, toolchain and features combinations are valid and return a 
00347                 # list of valid combinations to work through
00348                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00349                                                                 valid_choices(example['toolchains'], toolchains),
00350                                                                 example['features']):
00351                     proc = subprocess.Popen(["mbed-cli", "compile", "-t", toolchain,
00352                                              "-m", target, "--silent"])
00353                     proc.wait()
00354                     example_summary = "{} {} {}".format(name, target, toolchain)
00355                     if proc.returncode:
00356                         failures.append(example_summary)
00357                     else:
00358                         successes.append(example_summary)
00359                 os.chdir("..")
00360             
00361             # If there are any compilation failures for the example 'set' then the overall status is fail.
00362             if len(failures) > 0:
00363                 pass_status = False
00364         else:
00365             compiled = False
00366 
00367         results[example['name']] = [compiled, pass_status, successes, failures]
00368 
00369     return results
00370 
00371 
00372 def update_mbedos_version (config, tag, examples):
00373     """ For each example repo identified in the config json object, update the version of 
00374         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00375         example repo has already been cloned.
00376         
00377     Args:
00378     config - the json object imported from the file. 
00379     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00380     
00381     """
00382     print("Updating mbed-os in examples to version %s\n" % tag)
00383     for example in config['examples']:
00384         if example['name'] not in examples:
00385             continue
00386         for repo_info in get_repo_list(example):
00387             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00388             print("\nChanging dir to %s\n" % update_dir)
00389             os.chdir(update_dir)
00390             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00391             os.chdir("../..")
00392