examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and
00004     the update.py modules.
00005 
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 
00014 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00015 sys.path.insert(0, ROOT)
00016 
00017 from tools.build_api import get_mbed_official_release
00018 from tools.targets import TARGET_MAP
00019 from tools.export import EXPORTERS
00020 from tools.project import EXPORTER_ALIASES
00021 from tools.toolchains import TOOLCHAINS
00022 
00023 SUPPORTED_TOOLCHAINS = list(TOOLCHAINS - set(u'uARM'))
00024 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() + EXPORTER_ALIASES.keys()
00025                   if exp != "cmsis" and exp != "zip"]
00026 
00027 
00028 def print_list (lst):
00029     """Prints to screen the contents of a list
00030 
00031     Args:
00032     lst - a list of any type, to be displayed
00033 
00034     """
00035     if lst:
00036         for thing in lst:
00037             print("# %s" % thing)
00038 
00039 
00040 def print_category(results, index, message):
00041     summary = [example for key, summ in results.items()
00042                for example in summ[index]]
00043     if all(len(s) == 0 for s in summary):
00044         return
00045     print("#")
00046     print("#" * 80)
00047     print("# %s" % message)
00048     print("#" * 80)
00049     split_summ = [s.rsplit(" ", 1) for s in summary]
00050 
00051     print_list(summary)
00052 
00053 
00054 def print_summary (results, export=False):
00055     """Prints to screen the results of compiling/exporting combinations of example programs,
00056        targets and compile toolchains/IDEs.
00057 
00058     Args:
00059     results - results of the compilation stage. See compile_repos() and export_repos()
00060               for details of the format.
00061 
00062     """
00063 
00064     print("#"*80)
00065     print("# Examples compilation summary")
00066     print("#"*80)
00067 
00068     print_category(results, 2, "Passed example combinations")
00069 
00070     second_result = "Failed example combinations" if not export else \
00071         "Failed export example combinations"
00072 
00073     print_category(results, 3, second_result)
00074 
00075     if export:
00076         print_category(results, 4, "Failed build combinations")
00077         print_category(results, 5, "Skipped build combinations")
00078 
00079     print("#")
00080     print("#"*80)
00081 
00082 def valid_choices(allowed_choices, all_choices):
00083     if len(allowed_choices) > 0:
00084         return [t for t in all_choices if t in allowed_choices]
00085     else:
00086         return all_choices
00087 
00088 
00089 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00090     """Generate pairs of target and toolchains
00091 
00092     Args:
00093     allowed_targets - a list of all possible targets
00094     allowed_toolchains - a list of all possible toolchains
00095 
00096     Kwargs:
00097     features - the features that must be in the features array of a
00098                target
00099     """
00100     for target in allowed_targets:
00101         for toolchain in allowed_toolchains:
00102             if all(feature in TARGET_MAP[target].features
00103                     for feature in features):
00104                 yield target, toolchain
00105 
00106 
00107 def target_cross_ide (allowed_targets, allowed_ides, features=[], toolchains=[]):
00108     """Generate pairs of target and ides
00109 
00110     Args:
00111     allowed_targets - a list of all possible targets
00112     allowed_ides - a list of all possible IDEs
00113 
00114     Kwargs:
00115     features - the features that must be in the features array of a
00116                target
00117     """
00118     for target in allowed_targets:
00119         for ide in allowed_ides:
00120             if (EXPORTERS[ide].is_target_supported(target) and
00121                 (not toolchains or EXPORTERS[ide].TOOLCHAIN in toolchains) and
00122                 all(feature in TARGET_MAP[target].features
00123                     for feature in features)):
00124                 yield target, ide
00125 
00126 
00127 def get_repo_list (example):
00128     """ Returns a list of all the repos and their types associated with the
00129         specific example in the json config file.
00130         If the key 'test-repo-source' is set to 'mbed', then it will return the
00131         mbed section as a list. Otherwise, it will return the single github repo.
00132         NOTE: This does not currently deal with multiple examples underneath a github
00133         sourced exampe repo.
00134 
00135     Args:
00136     example - Example for which the repo list is requested
00137 
00138     """
00139     repos = []
00140     if example['test-repo-source'] == 'mbed':
00141         for repo in example['mbed']:
00142             repos.append({
00143                 'repo': repo,
00144                 'type': 'hg'
00145             })
00146     else:
00147         repos.append({
00148             'repo': example['github'],
00149             'type': 'git'
00150         })
00151     return repos
00152 
00153 
00154 def source_repos (config, examples):
00155     """ Imports each of the repos and its dependencies (.lib files) associated
00156         with the specific examples name from the json config file. Note if
00157         there is already a clone of the repo then it will first be removed to
00158         ensure a clean, up to date cloning.
00159     Args:
00160     config - the json object imported from the file.
00161 
00162     """
00163     print("\nImporting example repos....\n")
00164     for example in config['examples']:
00165         for repo_info in get_repo_list(example):
00166             name = basename(repo_info['repo'])
00167             if name in examples:
00168                 if os.path.exists(name):
00169                     print("'%s' example directory already exists. Deleting..." % name)
00170                     rmtree(name)
00171 
00172                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00173 
00174 def clone_repos (config, examples , retry = 3):
00175     """ Clones each of the repos associated with the specific examples name from the
00176         json config file. Note if there is already a clone of the repo then it will first
00177         be removed to ensure a clean, up to date cloning.
00178     Args:
00179     config - the json object imported from the file.
00180 
00181     """
00182     print("\nCloning example repos....\n")
00183     for example in config['examples']:
00184         for repo_info in get_repo_list(example):
00185             name = basename(repo_info['repo'])
00186             if name in examples:
00187                 if os.path.exists(name):
00188                     print("'%s' example directory already exists. Deleting..." % name)
00189                     rmtree(name)
00190                 for i in range(0, retry):
00191                     if subprocess.call([repo_info['type'], "clone", repo_info['repo']]) == 0:
00192                         break
00193                 else:
00194                     print("ERROR : unable to clone the repo {}".format(name))
00195 
00196 def deploy_repos (config, examples):
00197     """ If the example directory exists as provided by the json config file,
00198         pull in the examples dependencies by using `mbed-cli deploy`.
00199     Args:
00200     config - the json object imported from the file.
00201 
00202     """
00203     print("\nDeploying example repos....\n")
00204     for example in config['examples']:
00205         for repo_info in get_repo_list(example):
00206             name = basename(repo_info['repo'])
00207             if name in examples:
00208                 if os.path.exists(name):
00209                     os.chdir(name)
00210                     subprocess.call(["mbed-cli", "deploy"])
00211                     os.chdir("..")
00212                 else:
00213                     print("'%s' example directory doesn't exist. Skipping..." % name)
00214 
00215 
00216 def get_num_failures (results, export=False):
00217     """ Returns the number of failed compilations from the results summary
00218     Args:
00219     results - results summary of the compilation stage. See compile_repos() for
00220               details of the format.
00221     num_failures
00222 
00223     """
00224     num_failures = 0
00225 
00226     for key, val in results.items():
00227         num_failures = num_failures + len(val[3])
00228         if export:
00229             num_failures += len(val[4])
00230 
00231     return num_failures
00232 
00233 def export_repos (config, ides, targets, examples):
00234     """Exports and builds combinations of example programs, targets and IDEs.
00235 
00236         The results are returned in a [key: value] dictionary format:
00237             Where key = The example name from the json config file
00238             value = a list containing: pass_status, successes, export failures, build_failures,
00239             and build_skips
00240 
00241             where pass_status = The overall pass status for the export of the full
00242             set of example programs comprising the example suite.
00243             IE they must build and export) True if all examples pass, false otherwise
00244             successes = list of examples that exported and built (if possible)
00245             If the exporter has no build functionality, then it is a pass
00246             if exported
00247             export_failures = list of examples that failed to export.
00248             build_failures = list of examples that failed to build
00249             build_skips = list of examples that cannot build
00250 
00251             Both successes and failures contain the example name, target and IDE
00252 
00253             Args:
00254             config - the json object imported from the file.
00255             ides - List of IDES to export to
00256     """
00257     results = {}
00258     valid_examples = set(examples)
00259     print("\nExporting example repos....\n")
00260     for example in config['examples']:
00261         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00262         common_examples = valid_examples.intersection(set(example_names))
00263         if not common_examples:
00264             continue
00265         export_failures = []
00266         build_failures = []
00267         build_skips = []
00268         successes = []
00269         exported = True
00270         pass_status = True
00271         if example['export']:
00272             for repo_info in get_repo_list(example):
00273                 example_project_name = basename(repo_info['repo'])
00274                 os.chdir(example_project_name)
00275                 # Check that the target, IDE, and features combinations are valid and return a
00276                 # list of valid combinations to work through
00277                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00278                                                     valid_choices(example['exporters'], ides),
00279                                                     example['features'], example['toolchains']):
00280                     example_name = "{} {} {}".format(example_project_name, target,
00281                                                      ide)
00282                     def status(message):
00283                         print(message + " %s" % example_name)
00284                         sys.stdout.flush()
00285 
00286                     status("Exporting")
00287                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00288                                              "-m", target])
00289                     proc.wait()
00290                     if proc.returncode:
00291                         export_failures.append(example_name)
00292                         status("FAILURE exporting")
00293                     else:
00294                         status("SUCCESS exporting")
00295                         status("Building")
00296                         try:
00297                             if EXPORTERS[ide].build(example_project_name, cleanup=False):
00298                                 status("FAILURE building")
00299                                 build_failures.append(example_name)
00300                             else:
00301                                 status("SUCCESS building")
00302                                 successes.append(example_name)
00303                         except TypeError:
00304                             successes.append(example_name)
00305                             build_skips.append(example_name)
00306                 os.chdir("..")
00307 
00308                 if len(build_failures+export_failures) > 0:
00309                     pass_status= False
00310         else:
00311             exported = False
00312 
00313         results[example['name']] = [exported, pass_status, successes,
00314                                     export_failures, build_failures, build_skips]
00315 
00316     return results
00317 
00318 
00319 def compile_repos (config, toolchains, targets, profile, examples):
00320     """Compiles combinations of example programs, targets and compile chains.
00321 
00322        The results are returned in a [key: value] dictionary format:
00323        Where key = The example name from the json config file
00324              value = a list containing: pass_status, successes, and failures
00325 
00326              where pass_status = The overall pass status for the compilation of the full
00327                                  set of example programs comprising the example suite.
00328                                  True if all examples pass, false otherwise
00329                    successes = list of passing examples.
00330                    failures = list of failing examples.
00331 
00332                    Both successes and failures contain the example name, target and compile chain
00333 
00334     Args:
00335     config - the json object imported from the file.
00336     toolchains - List of toolchains to compile for.
00337     results - results of the compilation stage.
00338 
00339     """
00340     results = {}
00341     valid_examples = set(examples)
00342     print("\nCompiling example repos....\n")
00343     for example in config['examples']:
00344         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00345         common_examples = valid_examples.intersection(set(example_names))
00346         if not common_examples:
00347             continue
00348         failures = []
00349         successes = []
00350         compiled = True
00351         pass_status = True
00352         if example['compile']:
00353             for repo_info in get_repo_list(example):
00354                 name = basename(repo_info['repo'])
00355                 os.chdir(name)
00356 
00357                 # Check that the target, toolchain and features combinations are valid and return a
00358                 # list of valid combinations to work through
00359                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00360                                                                 valid_choices(example['toolchains'], toolchains),
00361                                                                 example['features']):
00362                     print("Compiling %s for %s, %s" % (name, target, toolchain))
00363                     build_command = ["mbed-cli", "compile", "-t", toolchain, "-m", target, "-v"]
00364 
00365                     if profile:
00366                         build_command.append("--profile")
00367                         build_command.append(profile)
00368 
00369                     proc = subprocess.Popen(build_command)
00370 
00371                     proc.wait()
00372                     example_summary = "{} {} {}".format(name, target, toolchain)
00373                     if proc.returncode:
00374                         failures.append(example_summary)
00375                     else:
00376                         successes.append(example_summary)
00377                 os.chdir("..")
00378 
00379             # If there are any compilation failures for the example 'set' then the overall status is fail.
00380             if len(failures) > 0:
00381                 pass_status = False
00382         else:
00383             compiled = False
00384 
00385         results[example['name']] = [compiled, pass_status, successes, failures]
00386 
00387     return results
00388 
00389 
00390 def update_mbedos_version (config, tag, examples):
00391     """ For each example repo identified in the config json object, update the version of
00392         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00393         example repo has already been cloned.
00394 
00395     Args:
00396     config - the json object imported from the file.
00397     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00398 
00399     """
00400     print("Updating mbed-os in examples to version %s\n" % tag)
00401     for example in config['examples']:
00402         if example['name'] not in examples:
00403             continue
00404         for repo_info in get_repo_list(example):
00405             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00406             print("\nChanging dir to %s\n" % update_dir)
00407             os.chdir(update_dir)
00408             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00409             os.chdir("../..")