examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and
00004     the update.py modules.
00005 
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 from sets import Set
00014 
00015 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00016 sys.path.insert(0, ROOT)
00017 
00018 from tools.build_api import get_mbed_official_release
00019 from tools.targets import TARGET_MAP
00020 from tools.export import EXPORTERS
00021 from tools.toolchains import TOOLCHAINS
00022 
00023 SUPPORTED_TOOLCHAINS = list(TOOLCHAINS - set(u'uARM'))
00024 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() if exp != "cmsis" and exp != "zip"]
00025 
00026 
00027 def print_list (lst):
00028     """Prints to screen the contents of a list
00029 
00030     Args:
00031     lst - a list of any type, to be displayed
00032 
00033     """
00034     if lst:
00035         for thing in lst:
00036             print("# %s" % thing)
00037 
00038 
00039 def print_category(results, index, message):
00040     summary = [example for key, summ in results.items()
00041                for example in summ[index]]
00042     if all(len(s) == 0 for s in summary):
00043         return
00044     print("#")
00045     print("#" * 80)
00046     print("# %s" % message)
00047     print("#" * 80)
00048     split_summ = [s.rsplit(" ", 1) for s in summary]
00049 
00050     print_list(summary)
00051 
00052 
00053 def print_summary (results, export=False):
00054     """Prints to screen the results of compiling/exporting combinations of example programs,
00055        targets and compile toolchains/IDEs.
00056 
00057     Args:
00058     results - results of the compilation stage. See compile_repos() and export_repos()
00059               for details of the format.
00060 
00061     """
00062 
00063     print("#"*80)
00064     print("# Examples compilation summary")
00065     print("#"*80)
00066 
00067     print_category(results, 2, "Passed example combinations")
00068 
00069     second_result = "Failed example combinations" if not export else \
00070         "Failed export example combinations"
00071 
00072     print_category(results, 3, second_result)
00073 
00074     if export:
00075         print_category(results, 4, "Failed build combinations")
00076         print_category(results, 5, "Skipped build combinations")
00077 
00078     print("#")
00079     print("#"*80)
00080 
00081 def valid_choices(allowed_choices, all_choices):
00082     if len(allowed_choices) > 0:
00083         return [t for t in all_choices if t in allowed_choices]
00084     else:
00085         return all_choices
00086 
00087 
00088 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00089     """Generate pairs of target and toolchains
00090 
00091     Args:
00092     allowed_targets - a list of all possible targets
00093     allowed_toolchains - a list of all possible toolchains
00094 
00095     Kwargs:
00096     features - the features that must be in the features array of a
00097                target
00098     """
00099     for target in allowed_targets:
00100         for toolchain in allowed_toolchains:
00101             if all(feature in TARGET_MAP[target].features
00102                     for feature in features):
00103                 yield target, toolchain
00104 
00105 
00106 def target_cross_ide (allowed_targets, allowed_ides, features=[], toolchains=[]):
00107     """Generate pairs of target and ides
00108 
00109     Args:
00110     allowed_targets - a list of all possible targets
00111     allowed_ides - a list of all possible IDEs
00112 
00113     Kwargs:
00114     features - the features that must be in the features array of a
00115                target
00116     """
00117     for target in allowed_targets:
00118         for ide in allowed_ides:
00119             if (EXPORTERS[ide].is_target_supported(target) and
00120                 (not toolchains or EXPORTERS[ide].TOOLCHAIN in toolchains) and
00121                 all(feature in TARGET_MAP[target].features
00122                     for feature in features)):
00123                 yield target, ide
00124 
00125 
00126 def get_repo_list (example):
00127     """ Returns a list of all the repos and their types associated with the
00128         specific example in the json config file.
00129         If the key 'test-repo-source' is set to 'mbed', then it will return the
00130         mbed section as a list. Otherwise, it will return the single github repo.
00131         NOTE: This does not currently deal with multiple examples underneath a github
00132         sourced exampe repo.
00133 
00134     Args:
00135     example - Example for which the repo list is requested
00136 
00137     """
00138     repos = []
00139     if example['test-repo-source'] == 'mbed':
00140         for repo in example['mbed']:
00141             repos.append({
00142                 'repo': repo,
00143                 'type': 'hg'
00144             })
00145     else:
00146         repos.append({
00147             'repo': example['github'],
00148             'type': 'git'
00149         })
00150     return repos
00151 
00152 
00153 def source_repos (config, examples):
00154     """ Imports each of the repos and its dependencies (.lib files) associated
00155         with the specific examples name from the json config file. Note if
00156         there is already a clone of the repo then it will first be removed to
00157         ensure a clean, up to date cloning.
00158     Args:
00159     config - the json object imported from the file.
00160 
00161     """
00162     print("\nImporting example repos....\n")
00163     for example in config['examples']:
00164         for repo_info in get_repo_list(example):
00165             name = basename(repo_info['repo'])
00166             if name in examples:
00167                 if os.path.exists(name):
00168                     print("'%s' example directory already exists. Deleting..." % name)
00169                     rmtree(name)
00170 
00171                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00172 
00173 def clone_repos (config, examples , retry = 3):
00174     """ Clones each of the repos associated with the specific examples name from the
00175         json config file. Note if there is already a clone of the repo then it will first
00176         be removed to ensure a clean, up to date cloning.
00177     Args:
00178     config - the json object imported from the file.
00179 
00180     """
00181     print("\nCloning example repos....\n")
00182     for example in config['examples']:
00183         for repo_info in get_repo_list(example):
00184             name = basename(repo_info['repo'])
00185             if name in examples:
00186                 if os.path.exists(name):
00187                     print("'%s' example directory already exists. Deleting..." % name)
00188                     rmtree(name)
00189                 for i in range(0, retry):
00190                     if subprocess.call([repo_info['type'], "clone", repo_info['repo']]) == 0:
00191                         break
00192                 else:
00193                     print("ERROR : unable to clone the repo {}".format(name))
00194 
00195 def deploy_repos (config, examples):
00196     """ If the example directory exists as provided by the json config file,
00197         pull in the examples dependencies by using `mbed-cli deploy`.
00198     Args:
00199     config - the json object imported from the file.
00200 
00201     """
00202     print("\nDeploying example repos....\n")
00203     for example in config['examples']:
00204         for repo_info in get_repo_list(example):
00205             name = basename(repo_info['repo'])
00206             if name in examples:
00207                 if os.path.exists(name):
00208                     os.chdir(name)
00209                     subprocess.call(["mbed-cli", "deploy"])
00210                     os.chdir("..")
00211                 else:
00212                     print("'%s' example directory doesn't exist. Skipping..." % name)
00213 
00214 
00215 def get_num_failures (results, export=False):
00216     """ Returns the number of failed compilations from the results summary
00217     Args:
00218     results - results summary of the compilation stage. See compile_repos() for
00219               details of the format.
00220     num_failures
00221 
00222     """
00223     num_failures = 0
00224 
00225     for key, val in results.items():
00226         num_failures = num_failures + len(val[3])
00227         if export:
00228             num_failures += len(val[4])
00229 
00230     return num_failures
00231 
00232 def export_repos (config, ides, targets, examples):
00233     """Exports and builds combinations of example programs, targets and IDEs.
00234 
00235         The results are returned in a [key: value] dictionary format:
00236             Where key = The example name from the json config file
00237             value = a list containing: pass_status, successes, export failures, build_failures,
00238             and build_skips
00239 
00240             where pass_status = The overall pass status for the export of the full
00241             set of example programs comprising the example suite.
00242             IE they must build and export) True if all examples pass, false otherwise
00243             successes = list of examples that exported and built (if possible)
00244             If the exporter has no build functionality, then it is a pass
00245             if exported
00246             export_failures = list of examples that failed to export.
00247             build_failures = list of examples that failed to build
00248             build_skips = list of examples that cannot build
00249 
00250             Both successes and failures contain the example name, target and IDE
00251 
00252             Args:
00253             config - the json object imported from the file.
00254             ides - List of IDES to export to
00255     """
00256     results = {}
00257     valid_examples = Set(examples)
00258     print("\nExporting example repos....\n")
00259     for example in config['examples']:
00260         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00261         common_examples = valid_examples.intersection(Set(example_names))
00262         if not common_examples:
00263             continue
00264         export_failures = []
00265         build_failures = []
00266         build_skips = []
00267         successes = []
00268         exported = True
00269         pass_status = True
00270         if example['export']:
00271             for repo_info in get_repo_list(example):
00272                 example_project_name = basename(repo_info['repo'])
00273                 os.chdir(example_project_name)
00274                 # Check that the target, IDE, and features combinations are valid and return a
00275                 # list of valid combinations to work through
00276                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00277                                                     valid_choices(example['exporters'], ides),
00278                                                     example['features'], example['toolchains']):
00279                     example_name = "{} {} {}".format(example_project_name, target,
00280                                                      ide)
00281                     def status(message):
00282                         print(message + " %s" % example_name)
00283                         sys.stdout.flush()
00284 
00285                     status("Exporting")
00286                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00287                                              "-m", target])
00288                     proc.wait()
00289                     if proc.returncode:
00290                         export_failures.append(example_name)
00291                         status("FAILURE exporting")
00292                     else:
00293                         status("SUCCESS exporting")
00294                         status("Building")
00295                         try:
00296                             if EXPORTERS[ide].build(example_project_name, cleanup=False):
00297                                 status("FAILURE building")
00298                                 build_failures.append(example_name)
00299                             else:
00300                                 status("SUCCESS building")
00301                                 successes.append(example_name)
00302                         except TypeError:
00303                             successes.append(example_name)
00304                             build_skips.append(example_name)
00305                 os.chdir("..")
00306 
00307                 if len(build_failures+export_failures) > 0:
00308                     pass_status= False
00309         else:
00310             exported = False
00311 
00312         results[example['name']] = [exported, pass_status, successes,
00313                                     export_failures, build_failures, build_skips]
00314 
00315     return results
00316 
00317 
00318 def compile_repos (config, toolchains, targets, profile, examples):
00319     """Compiles combinations of example programs, targets and compile chains.
00320 
00321        The results are returned in a [key: value] dictionary format:
00322        Where key = The example name from the json config file
00323              value = a list containing: pass_status, successes, and failures
00324 
00325              where pass_status = The overall pass status for the compilation of the full
00326                                  set of example programs comprising the example suite.
00327                                  True if all examples pass, false otherwise
00328                    successes = list of passing examples.
00329                    failures = list of failing examples.
00330 
00331                    Both successes and failures contain the example name, target and compile chain
00332 
00333     Args:
00334     config - the json object imported from the file.
00335     toolchains - List of toolchains to compile for.
00336     results - results of the compilation stage.
00337 
00338     """
00339     results = {}
00340     valid_examples = Set(examples)
00341     print("\nCompiling example repos....\n")
00342     for example in config['examples']:
00343         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00344         common_examples = valid_examples.intersection(Set(example_names))
00345         if not common_examples:
00346             continue
00347         failures = []
00348         successes = []
00349         compiled = True
00350         pass_status = True
00351         if example['compile']:
00352             for repo_info in get_repo_list(example):
00353                 name = basename(repo_info['repo'])
00354                 os.chdir(name)
00355 
00356                 # Check that the target, toolchain and features combinations are valid and return a
00357                 # list of valid combinations to work through
00358                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00359                                                                 valid_choices(example['toolchains'], toolchains),
00360                                                                 example['features']):
00361                     print("Compiling %s for %s, %s" % (name, target, toolchain))
00362                     build_command = ["mbed-cli", "compile", "-t", toolchain, "-m", target, "-v"]
00363 
00364                     if profile:
00365                         build_command.append("--profile")
00366                         build_command.append(profile)
00367 
00368                     proc = subprocess.Popen(build_command)
00369 
00370                     proc.wait()
00371                     example_summary = "{} {} {}".format(name, target, toolchain)
00372                     if proc.returncode:
00373                         failures.append(example_summary)
00374                     else:
00375                         successes.append(example_summary)
00376                 os.chdir("..")
00377 
00378             # If there are any compilation failures for the example 'set' then the overall status is fail.
00379             if len(failures) > 0:
00380                 pass_status = False
00381         else:
00382             compiled = False
00383 
00384         results[example['name']] = [compiled, pass_status, successes, failures]
00385 
00386     return results
00387 
00388 
00389 def update_mbedos_version (config, tag, examples):
00390     """ For each example repo identified in the config json object, update the version of
00391         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00392         example repo has already been cloned.
00393 
00394     Args:
00395     config - the json object imported from the file.
00396     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00397 
00398     """
00399     print("Updating mbed-os in examples to version %s\n" % tag)
00400     for example in config['examples']:
00401         if example['name'] not in examples:
00402             continue
00403         for repo_info in get_repo_list(example):
00404             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00405             print("\nChanging dir to %s\n" % update_dir)
00406             os.chdir(update_dir)
00407             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00408             os.chdir("../..")