examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and
00004     the update.py modules.
00005 
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 from sets import Set
00014 
00015 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00016 sys.path.insert(0, ROOT)
00017 
00018 from tools.build_api import get_mbed_official_release
00019 from tools.targets import TARGET_MAP
00020 from tools.export import EXPORTERS
00021 
00022 SUPPORTED_TOOLCHAINS = ["ARM", "IAR", "GCC_ARM", "ARMC6"]
00023 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() if exp != "cmsis" and exp != "zip"]
00024 
00025 
00026 def print_list (lst):
00027     """Prints to screen the contents of a list
00028 
00029     Args:
00030     lst - a list of any type, to be displayed
00031 
00032     """
00033     if lst:
00034         for thing in lst:
00035             print("# %s" % thing)
00036 
00037 
00038 def print_category(results, index, message):
00039     summary = [example for key, summ in results.iteritems()
00040                for example in summ[index]]
00041     if all(len(s) == 0 for s in summary):
00042         return
00043     print("#")
00044     print("#" * 80)
00045     print("# %s" % message)
00046     print("#" * 80)
00047     split_summ = [s.rsplit(" ", 1) for s in summary]
00048 
00049     print_list(summary)
00050 
00051 
00052 def print_summary (results, export=False):
00053     """Prints to screen the results of compiling/exporting combinations of example programs,
00054        targets and compile toolchains/IDEs.
00055 
00056     Args:
00057     results - results of the compilation stage. See compile_repos() and export_repos()
00058               for details of the format.
00059 
00060     """
00061 
00062     print("#"*80)
00063     print("# Examples compilation summary")
00064     print("#"*80)
00065 
00066     print_category(results, 2, "Passed example combinations")
00067 
00068     second_result = "Failed example combinations" if not export else \
00069         "Failed export example combinations"
00070 
00071     print_category(results, 3, second_result)
00072 
00073     if export:
00074         print_category(results, 4, "Failed build combinations")
00075         print_category(results, 5, "Skipped build combinations")
00076 
00077     print("#")
00078     print("#"*80)
00079 
00080 def valid_choices(allowed_choices, all_choices):
00081     if len(allowed_choices) > 0:
00082         return [t for t in all_choices if t in allowed_choices]
00083     else:
00084         return all_choices
00085 
00086 
00087 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00088     """Generate pairs of target and toolchains
00089 
00090     Args:
00091     allowed_targets - a list of all possible targets
00092     allowed_toolchains - a list of all possible toolchains
00093 
00094     Kwargs:
00095     features - the features that must be in the features array of a
00096                target
00097     """
00098     for target in allowed_targets:
00099         for toolchain in allowed_toolchains:
00100             if all(feature in TARGET_MAP[target].features
00101                     for feature in features):
00102                 yield target, toolchain
00103 
00104 
00105 def target_cross_ide (allowed_targets, allowed_ides, features=[], toolchains=[]):
00106     """Generate pairs of target and ides
00107 
00108     Args:
00109     allowed_targets - a list of all possible targets
00110     allowed_ides - a list of all possible IDEs
00111 
00112     Kwargs:
00113     features - the features that must be in the features array of a
00114                target
00115     """
00116     for target in allowed_targets:
00117         for ide in allowed_ides:
00118             if (EXPORTERS[ide].is_target_supported(target) and
00119                 (not toolchains or EXPORTERS[ide].TOOLCHAIN in toolchains) and
00120                 all(feature in TARGET_MAP[target].features
00121                     for feature in features)):
00122                 yield target, ide
00123 
00124 
00125 def get_repo_list (example):
00126     """ Returns a list of all the repos and their types associated with the
00127         specific example in the json config file.
00128         If the key 'test-repo-source' is set to 'mbed', then it will return the
00129         mbed section as a list. Otherwise, it will return the single github repo.
00130         NOTE: This does not currently deal with multiple examples underneath a github
00131         sourced exampe repo.
00132 
00133     Args:
00134     example - Example for which the repo list is requested
00135 
00136     """
00137     repos = []
00138     if example['test-repo-source'] == 'mbed':
00139         for repo in example['mbed']:
00140             repos.append({
00141                 'repo': repo,
00142                 'type': 'hg'
00143             })
00144     else:
00145         repos.append({
00146             'repo': example['github'],
00147             'type': 'git'
00148         })
00149     return repos
00150 
00151 
00152 def source_repos (config, examples):
00153     """ Imports each of the repos and its dependencies (.lib files) associated
00154         with the specific examples name from the json config file. Note if
00155         there is already a clone of the repo then it will first be removed to
00156         ensure a clean, up to date cloning.
00157     Args:
00158     config - the json object imported from the file.
00159 
00160     """
00161     print("\nImporting example repos....\n")
00162     for example in config['examples']:
00163         for repo_info in get_repo_list(example):
00164             name = basename(repo_info['repo'])
00165             if name in examples:
00166                 if os.path.exists(name):
00167                     print("'%s' example directory already exists. Deleting..." % name)
00168                     rmtree(name)
00169 
00170                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00171 
00172 def clone_repos (config, examples , retry = 3):
00173     """ Clones each of the repos associated with the specific examples name from the
00174         json config file. Note if there is already a clone of the repo then it will first
00175         be removed to ensure a clean, up to date cloning.
00176     Args:
00177     config - the json object imported from the file.
00178 
00179     """
00180     print("\nCloning example repos....\n")
00181     for example in config['examples']:
00182         for repo_info in get_repo_list(example):
00183             name = basename(repo_info['repo'])
00184             if name in examples:
00185                 if os.path.exists(name):
00186                     print("'%s' example directory already exists. Deleting..." % name)
00187                     rmtree(name)
00188                 for i in range(0, retry):
00189                     if subprocess.call([repo_info['type'], "clone", repo_info['repo']]) == 0:
00190                         break
00191                 else:
00192                     print("ERROR : unable to clone the repo {}".format(name))
00193 
00194 def deploy_repos (config, examples):
00195     """ If the example directory exists as provided by the json config file,
00196         pull in the examples dependencies by using `mbed-cli deploy`.
00197     Args:
00198     config - the json object imported from the file.
00199 
00200     """
00201     print("\nDeploying example repos....\n")
00202     for example in config['examples']:
00203         for repo_info in get_repo_list(example):
00204             name = basename(repo_info['repo'])
00205             if name in examples:
00206                 if os.path.exists(name):
00207                     os.chdir(name)
00208                     subprocess.call(["mbed-cli", "deploy"])
00209                     os.chdir("..")
00210                 else:
00211                     print("'%s' example directory doesn't exist. Skipping..." % name)
00212 
00213 
00214 def get_num_failures (results, export=False):
00215     """ Returns the number of failed compilations from the results summary
00216     Args:
00217     results - results summary of the compilation stage. See compile_repos() for
00218               details of the format.
00219     num_failures
00220 
00221     """
00222     num_failures = 0
00223 
00224     for key, val in results.iteritems():
00225         num_failures = num_failures + len(val[3])
00226         if export:
00227             num_failures += len(val[4])
00228 
00229     return num_failures
00230 
00231 def export_repos (config, ides, targets, examples):
00232     """Exports and builds combinations of example programs, targets and IDEs.
00233 
00234         The results are returned in a [key: value] dictionary format:
00235             Where key = The example name from the json config file
00236             value = a list containing: pass_status, successes, export failures, build_failures,
00237             and build_skips
00238 
00239             where pass_status = The overall pass status for the export of the full
00240             set of example programs comprising the example suite.
00241             IE they must build and export) True if all examples pass, false otherwise
00242             successes = list of examples that exported and built (if possible)
00243             If the exporter has no build functionality, then it is a pass
00244             if exported
00245             export_failures = list of examples that failed to export.
00246             build_failures = list of examples that failed to build
00247             build_skips = list of examples that cannot build
00248 
00249             Both successes and failures contain the example name, target and IDE
00250 
00251             Args:
00252             config - the json object imported from the file.
00253             ides - List of IDES to export to
00254     """
00255     results = {}
00256     valid_examples = Set(examples)
00257     print("\nExporting example repos....\n")
00258     for example in config['examples']:
00259         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00260         common_examples = valid_examples.intersection(Set(example_names))
00261         if not common_examples:
00262             continue
00263         export_failures = []
00264         build_failures = []
00265         build_skips = []
00266         successes = []
00267         exported = True
00268         pass_status = True
00269         if example['export']:
00270             for repo_info in get_repo_list(example):
00271                 example_project_name = basename(repo_info['repo'])
00272                 os.chdir(example_project_name)
00273                 # Check that the target, IDE, and features combinations are valid and return a
00274                 # list of valid combinations to work through
00275                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00276                                                     valid_choices(example['exporters'], ides),
00277                                                     example['features'], example['toolchains']):
00278                     example_name = "{} {} {}".format(example_project_name, target,
00279                                                      ide)
00280                     def status(message):
00281                         print(message + " %s" % example_name)
00282                         sys.stdout.flush()
00283 
00284                     status("Exporting")
00285                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00286                                              "-m", target])
00287                     proc.wait()
00288                     if proc.returncode:
00289                         export_failures.append(example_name)
00290                         status("FAILURE exporting")
00291                     else:
00292                         status("SUCCESS exporting")
00293                         status("Building")
00294                         try:
00295                             if EXPORTERS[ide].build(example_project_name, cleanup=False):
00296                                 status("FAILURE building")
00297                                 build_failures.append(example_name)
00298                             else:
00299                                 status("SUCCESS building")
00300                                 successes.append(example_name)
00301                         except TypeError:
00302                             successes.append(example_name)
00303                             build_skips.append(example_name)
00304                 os.chdir("..")
00305 
00306                 if len(build_failures+export_failures) > 0:
00307                     pass_status= False
00308         else:
00309             exported = False
00310 
00311         results[example['name']] = [exported, pass_status, successes,
00312                                     export_failures, build_failures, build_skips]
00313 
00314     return results
00315 
00316 
00317 def compile_repos (config, toolchains, targets, profile, examples):
00318     """Compiles combinations of example programs, targets and compile chains.
00319 
00320        The results are returned in a [key: value] dictionary format:
00321        Where key = The example name from the json config file
00322              value = a list containing: pass_status, successes, and failures
00323 
00324              where pass_status = The overall pass status for the compilation of the full
00325                                  set of example programs comprising the example suite.
00326                                  True if all examples pass, false otherwise
00327                    successes = list of passing examples.
00328                    failures = list of failing examples.
00329 
00330                    Both successes and failures contain the example name, target and compile chain
00331 
00332     Args:
00333     config - the json object imported from the file.
00334     toolchains - List of toolchains to compile for.
00335     results - results of the compilation stage.
00336 
00337     """
00338     results = {}
00339     valid_examples = Set(examples)
00340     print("\nCompiling example repos....\n")
00341     for example in config['examples']:
00342         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00343         common_examples = valid_examples.intersection(Set(example_names))
00344         if not common_examples:
00345             continue
00346         failures = []
00347         successes = []
00348         compiled = True
00349         pass_status = True
00350         if example['compile']:
00351             for repo_info in get_repo_list(example):
00352                 name = basename(repo_info['repo'])
00353                 os.chdir(name)
00354 
00355                 # Check that the target, toolchain and features combinations are valid and return a
00356                 # list of valid combinations to work through
00357                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00358                                                                 valid_choices(example['toolchains'], toolchains),
00359                                                                 example['features']):
00360                     print("Compiling %s for %s, %s" % (name, target, toolchain))
00361                     build_command = ["mbed-cli", "compile", "-t", toolchain, "-m", target, "-v"]
00362 
00363                     if profile:
00364                         build_command.append("--profile")
00365                         build_command.append(profile)
00366 
00367                     proc = subprocess.Popen(build_command)
00368 
00369                     proc.wait()
00370                     example_summary = "{} {} {}".format(name, target, toolchain)
00371                     if proc.returncode:
00372                         failures.append(example_summary)
00373                     else:
00374                         successes.append(example_summary)
00375                 os.chdir("..")
00376 
00377             # If there are any compilation failures for the example 'set' then the overall status is fail.
00378             if len(failures) > 0:
00379                 pass_status = False
00380         else:
00381             compiled = False
00382 
00383         results[example['name']] = [compiled, pass_status, successes, failures]
00384 
00385     return results
00386 
00387 
00388 def update_mbedos_version (config, tag, examples):
00389     """ For each example repo identified in the config json object, update the version of
00390         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00391         example repo has already been cloned.
00392 
00393     Args:
00394     config - the json object imported from the file.
00395     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00396 
00397     """
00398     print("Updating mbed-os in examples to version %s\n" % tag)
00399     for example in config['examples']:
00400         if example['name'] not in examples:
00401             continue
00402         for repo_info in get_repo_list(example):
00403             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00404             print("\nChanging dir to %s\n" % update_dir)
00405             os.chdir(update_dir)
00406             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00407             os.chdir("../..")