examples_lib.py

00001 """ Import and bulid a bunch of example programs
00002 
00003     This library includes functions that are shared between the examples.py and 
00004     the update.py modules.
00005     
00006  """
00007 import os
00008 from os.path import dirname, abspath, basename
00009 import os.path
00010 import sys
00011 import subprocess
00012 from shutil import rmtree
00013 from sets import Set
00014 
00015 ROOT = abspath(dirname(dirname(dirname(dirname(__file__)))))
00016 sys.path.insert(0, ROOT)
00017 
00018 from tools.build_api import get_mbed_official_release
00019 from tools.targets import TARGET_MAP
00020 from tools.export import EXPORTERS
00021 
00022 SUPPORTED_TOOLCHAINS = ["ARM", "IAR", "GCC_ARM"]
00023 SUPPORTED_IDES = [exp for exp in EXPORTERS.keys() if exp != "cmsis" and exp != "zip"]
00024 
00025 
00026 def print_list (lst):
00027     """Prints to screen the contents of a list
00028 
00029     Args:
00030     lst - a list of any type, to be displayed
00031 
00032     """
00033     if lst:
00034         for thing in lst:
00035             print("# %s" % thing)
00036 
00037 
00038 def print_category(results, index, message):
00039     summary = [example for key, summ in results.iteritems()
00040                for example in summ[index]]
00041     if all(len(s) == 0 for s in summary):
00042         return
00043     print("#")
00044     print("#" * 80)
00045     print("# %s" % message)
00046     print("#" * 80)
00047     split_summ = [s.rsplit(" ", 1) for s in summary]
00048 
00049     print_list(summary)
00050 
00051 
00052 def print_summary (results, export=False):
00053     """Prints to screen the results of compiling/exporting combinations of example programs,
00054        targets and compile toolchains/IDEs.
00055 
00056     Args:
00057     results - results of the compilation stage. See compile_repos() and export_repos()
00058               for details of the format.
00059 
00060     """
00061 
00062     print("#"*80)
00063     print("# Examples compilation summary")
00064     print("#"*80)
00065 
00066     print_category(results, 2, "Passed example combinations")
00067 
00068     second_result = "Failed example combinations" if not export else \
00069         "Failed export example combinations"
00070 
00071     print_category(results, 3, second_result)
00072 
00073     if export:
00074         print_category(results, 4, "Failed build combinations")
00075         print_category(results, 5, "Skipped build combinations")
00076 
00077     print("#")
00078     print("#"*80)
00079 
00080 def valid_choices(allowed_choices, all_choices):
00081     if len(allowed_choices) > 0:
00082         return [t for t in all_choices if t in allowed_choices]
00083     else:
00084         return all_choices
00085 
00086 
00087 def target_cross_toolchain (allowed_targets, allowed_toolchains, features=[]):
00088     """Generate pairs of target and toolchains
00089 
00090     Args:
00091     allowed_targets - a list of all possible targets
00092     allowed_toolchains - a list of all possible toolchains
00093 
00094     Kwargs:
00095     features - the features that must be in the features array of a
00096                target
00097     """
00098     for target in allowed_targets:
00099         for toolchain in allowed_toolchains:
00100             if all(feature in TARGET_MAP[target].features
00101                     for feature in features):
00102                 yield target, toolchain
00103 
00104 
00105 def target_cross_ide (allowed_targets, allowed_ides, features=[], toolchains=[]):
00106     """Generate pairs of target and ides
00107 
00108     Args:
00109     allowed_targets - a list of all possible targets
00110     allowed_ides - a list of all possible IDEs
00111 
00112     Kwargs:
00113     features - the features that must be in the features array of a
00114                target
00115     """
00116     for target in allowed_targets:
00117         for ide in allowed_ides:
00118             if (target in EXPORTERS[ide].TARGETS and
00119                 (not toolchains or EXPORTERS[ide].TOOLCHAIN in toolchains) and
00120                 all(feature in TARGET_MAP[target].features
00121                     for feature in features)):
00122                 yield target, ide
00123 
00124 
00125 def get_repo_list (example):
00126     """ Returns a list of all the repos associated with the specific example in the json
00127         config file.
00128         If there are repos listed under the mbed section then these will be returned as a 
00129         list. If not then the github single repo with be returned. 
00130         NOTE: This does not currently deal with multiple examples underneath a github 
00131         sourced exampe repo.
00132 
00133     Args:
00134     example - Example for which the repo list is requested
00135     repos - The list of repos and types contained within that example in the json file
00136 
00137     """
00138     repos = []
00139     if len(example['mbed']) > 0:
00140         for repo in example['mbed']:
00141             repos.append({
00142                 'repo': repo,
00143                 'type': 'hg'
00144             })
00145     else:
00146         repos.append({
00147             'repo': example['github'],
00148             'type': 'git'
00149         })
00150     return repos
00151 
00152 
00153 def source_repos (config, examples):
00154     """ Imports each of the repos and its dependencies (.lib files) associated
00155         with the specific examples name from the json config file. Note if
00156         there is already a clone of the repo then it will first be removed to
00157         ensure a clean, up to date cloning.
00158     Args:
00159     config - the json object imported from the file. 
00160 
00161     """
00162     print("\nImporting example repos....\n")
00163     for example in config['examples']:
00164         for repo_info in get_repo_list(example):
00165             name = basename(repo_info['repo'])
00166             if name in examples:
00167                 if os.path.exists(name):
00168                     print("'%s' example directory already exists. Deleting..." % name)
00169                     rmtree(name)
00170 
00171                 subprocess.call(["mbed-cli", "import", repo_info['repo']])
00172 
00173 def clone_repos (config, examples):
00174     """ Clones each of the repos associated with the specific examples name from the
00175         json config file. Note if there is already a clone of the repo then it will first
00176         be removed to ensure a clean, up to date cloning.
00177     Args:
00178     config - the json object imported from the file.
00179 
00180     """
00181     print("\nCloning example repos....\n")
00182     for example in config['examples']:
00183         for repo_info in get_repo_list(example):
00184             name = basename(repo_info['repo'])
00185             if name in examples:
00186                 if os.path.exists(name):
00187                     print("'%s' example directory already exists. Deleting..." % name)
00188                     rmtree(name)
00189 
00190                 subprocess.call([repo_info['type'], "clone", repo_info['repo']])
00191 
00192 def deploy_repos (config, examples):
00193     """ If the example directory exists as provided by the json config file,
00194         pull in the examples dependencies by using `mbed-cli deploy`.
00195     Args:
00196     config - the json object imported from the file.
00197 
00198     """
00199     print("\nDeploying example repos....\n")
00200     for example in config['examples']:
00201         for repo_info in get_repo_list(example):
00202             name = basename(repo_info['repo'])
00203             if name in examples:
00204                 if os.path.exists(name):
00205                     os.chdir(name)
00206                     subprocess.call(["mbed-cli", "deploy"])
00207                     os.chdir("..")
00208                 else:
00209                     print("'%s' example directory doesn't exist. Skipping..." % name)
00210 
00211 
00212 def get_num_failures (results, export=False):
00213     """ Returns the number of failed compilations from the results summary
00214     Args:
00215     results - results summary of the compilation stage. See compile_repos() for
00216               details of the format.
00217     num_failures 
00218 
00219     """
00220     num_failures = 0
00221 
00222     for key, val in results.iteritems():
00223         num_failures = num_failures + len(val[3])
00224         if export:
00225             num_failures += len(val[4])
00226 
00227     return num_failures
00228 
00229 def export_repos (config, ides, targets, examples):
00230     """Exports and builds combinations of example programs, targets and IDEs.
00231 
00232         The results are returned in a [key: value] dictionary format:
00233             Where key = The example name from the json config file
00234             value = a list containing: pass_status, successes, export failures, build_failures,
00235             and build_skips
00236 
00237             where pass_status = The overall pass status for the export of the full
00238             set of example programs comprising the example suite.
00239             IE they must build and export) True if all examples pass, false otherwise
00240             successes = list of examples that exported and built (if possible)
00241             If the exporter has no build functionality, then it is a pass
00242             if exported
00243             export_failures = list of examples that failed to export.
00244             build_failures = list of examples that failed to build
00245             build_skips = list of examples that cannot build
00246 
00247             Both successes and failures contain the example name, target and IDE
00248 
00249             Args:
00250             config - the json object imported from the file.
00251             ides - List of IDES to export to
00252     """
00253     results = {}
00254     valid_examples = Set(examples)
00255     print("\nExporting example repos....\n")
00256     for example in config['examples']:
00257         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00258         common_examples = valid_examples.intersection(Set(example_names))
00259         if not common_examples:
00260             continue
00261         export_failures = []
00262         build_failures = []
00263         build_skips = []
00264         successes = []
00265         exported = True
00266         pass_status = True
00267         if example['export']:
00268             for repo_info in get_repo_list(example):
00269                 example_project_name = basename(repo_info['repo'])
00270                 os.chdir(example_project_name)
00271                 # Check that the target, IDE, and features combinations are valid and return a
00272                 # list of valid combinations to work through
00273                 for target, ide in target_cross_ide(valid_choices(example['targets'], targets),
00274                                                     valid_choices(example['exporters'], ides),
00275                                                     example['features'], example['toolchains']):
00276                     example_name = "{} {} {}".format(example_project_name, target,
00277                                                      ide)
00278                     def status(message):
00279                         print(message + " %s" % example_name)
00280                         sys.stdout.flush()
00281 
00282                     status("Exporting")
00283                     proc = subprocess.Popen(["mbed-cli", "export", "-i", ide,
00284                                              "-m", target])
00285                     proc.wait()
00286                     if proc.returncode:
00287                         export_failures.append(example_name)
00288                         status("FAILURE exporting")
00289                     else:
00290                         status("SUCCESS exporting")
00291                         status("Building")
00292                         try:
00293                             if EXPORTERS[ide].build(example_project_name):
00294                                 status("FAILURE building")
00295                                 build_failures.append(example_name)
00296                             else:
00297                                 status("SUCCESS building")
00298                                 successes.append(example_name)
00299                         except TypeError:
00300                             successes.append(example_name)
00301                             build_skips.append(example_name)
00302                 os.chdir("..")
00303 
00304                 if len(build_failures+export_failures) > 0:
00305                     pass_status= False
00306         else:
00307             exported = False
00308 
00309         results[example['name']] = [exported, pass_status, successes,
00310                                     export_failures, build_failures, build_skips]
00311 
00312     return results
00313 
00314 
00315 def compile_repos (config, toolchains, targets, examples):
00316     """Compiles combinations of example programs, targets and compile chains.
00317        
00318        The results are returned in a [key: value] dictionary format:
00319        Where key = The example name from the json config file
00320              value = a list containing: pass_status, successes, and failures
00321              
00322              where pass_status = The overall pass status for the compilation of the full
00323                                  set of example programs comprising the example suite.
00324                                  True if all examples pass, false otherwise
00325                    successes = list of passing examples. 
00326                    failures = list of failing examples.
00327                    
00328                    Both successes and failures contain the example name, target and compile chain
00329 
00330     Args:
00331     config - the json object imported from the file. 
00332     toolchains - List of toolchains to compile for.
00333     results - results of the compilation stage. 
00334 
00335     """
00336     results = {}
00337     valid_examples = Set(examples)
00338     print("\nCompiling example repos....\n")
00339     for example in config['examples']:
00340         example_names = [basename(x['repo']) for x in get_repo_list(example)]
00341         common_examples = valid_examples.intersection(Set(example_names))
00342         if not common_examples:
00343             continue
00344         failures = []
00345         successes = []
00346         compiled = True
00347         pass_status = True
00348         if example['compile']:
00349             for repo_info in get_repo_list(example):
00350                 name = basename(repo_info['repo'])
00351                 os.chdir(name)
00352                 
00353                 # Check that the target, toolchain and features combinations are valid and return a 
00354                 # list of valid combinations to work through
00355                 for target, toolchain in target_cross_toolchain(valid_choices(example['targets'], targets),
00356                                                                 valid_choices(example['toolchains'], toolchains),
00357                                                                 example['features']):
00358                     print("Compiling %s for %s, %s" % (name, target, toolchain))
00359                     proc = subprocess.Popen(["mbed-cli", "compile", "-t", toolchain,
00360                                              "-m", target, "--silent"])
00361                     proc.wait()
00362                     example_summary = "{} {} {}".format(name, target, toolchain)
00363                     if proc.returncode:
00364                         failures.append(example_summary)
00365                     else:
00366                         successes.append(example_summary)
00367                 os.chdir("..")
00368             
00369             # If there are any compilation failures for the example 'set' then the overall status is fail.
00370             if len(failures) > 0:
00371                 pass_status = False
00372         else:
00373             compiled = False
00374 
00375         results[example['name']] = [compiled, pass_status, successes, failures]
00376 
00377     return results
00378 
00379 
00380 def update_mbedos_version (config, tag, examples):
00381     """ For each example repo identified in the config json object, update the version of 
00382         mbed-os to that specified by the supplied GitHub tag. This function assumes that each
00383         example repo has already been cloned.
00384         
00385     Args:
00386     config - the json object imported from the file. 
00387     tag - GitHub tag corresponding to a version of mbed-os to upgrade to.
00388     
00389     """
00390     print("Updating mbed-os in examples to version %s\n" % tag)
00391     for example in config['examples']:
00392         if example['name'] not in examples:
00393             continue
00394         for repo_info in get_repo_list(example):
00395             update_dir =  basename(repo_info['repo']) + "/mbed-os"
00396             print("\nChanging dir to %s\n" % update_dir)
00397             os.chdir(update_dir)
00398             subprocess.call(["mbed-cli", "update", tag, "--clean"])
00399             os.chdir("../..")
00400