run_test.py

00001 #
00002 # DAPLink Interface Firmware
00003 # Copyright (c) 2009-2016, ARM Limited, All Rights Reserved
00004 # SPDX-License-Identifier: Apache-2.0
00005 #
00006 # Licensed under the Apache License, Version 2.0 (the "License"); you may
00007 # not use this file except in compliance with the License.
00008 # You may obtain a copy of the License at
00009 #
00010 # http://www.apache.org/licenses/LICENSE-2.0
00011 #
00012 # Unless required by applicable law or agreed to in writing, software
00013 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
00014 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00015 # See the License for the specific language governing permissions and
00016 # limitations under the License.
00017 #
00018 
00019 """
00020 DAPLink validation and testing tool
00021 
00022 optional arguments:
00023   -h, --help            show this help message and exit
00024   --targetdir TARGETDIR
00025                         Directory with pre-built target test images.
00026   --user USER           MBED username (required for compile-api)
00027   --password PASSWORD   MBED password (required for compile-api)
00028   --firmwaredir FIRMWAREDIR
00029                         Directory with firmware images to test
00030   --firmware {k20dx_k64f_if,lpc11u35_sscity_if,...} (run script with --help to see full list)
00031                         Firmware to test
00032   --project-tool TOOL    choices=['uvision', 'mbedcli'],'Tool used to compile the project',
00033                         default='uvision'
00034   --logdir LOGDIR       Directory to log test results to
00035   --noloadif            Skip load step for interface.
00036   --notestendpt         Dont test the interface USB endpoints.
00037   --loadbl              Load bootloader before test.
00038   --testdl              Run DAPLink specific tests. The DAPLink test tests
00039                         bootloader updates so use with caution
00040   --testfirst           If multiple boards of the same type are found only
00041                         test the first one.
00042   --verbose {Minimal,Normal,Verbose,All}
00043                         Verbose output
00044   --dryrun              Print info on configurations but dont actually run
00045                         tests.
00046   --force               Try to run tests even if there are problems. Delete logs from previous run.
00047 Example usages
00048 ------------------------
00049 
00050 Test all built projects in the repository:
00051 test_all.py --user <username> --password <password>
00052 
00053 Test everything on a single project in the repository:
00054 test_all.py --project <project> --testfirst --user <username>
00055     --password <password>
00056 
00057 Verify that the USB endpoints are working correctly on
00058 an existing board with firmware already loaded:
00059 test_all.py --noloadif --user <username> --password <password>
00060 """
00061 from __future__ import absolute_import
00062 from __future__ import print_function
00063 
00064 import os
00065 import shutil
00066 import argparse
00067 import subprocess
00068 from enum import Enum
00069 from hid_test import test_hid
00070 from serial_test import test_serial
00071 from msd_test import test_mass_storage
00072 from usb_test import test_usb
00073 from daplink_board import get_all_attached_daplink_boards
00074 from project_generator.generate import Generator
00075 from test_info import TestInfo
00076 from daplink_firmware import load_bundle_from_project, load_bundle_from_release
00077 from firmware import Firmware
00078 from target import load_target_bundle, build_target_bundle
00079 from test_daplink import daplink_test
00080 import info
00081 
00082 DEFAULT_TEST_DIR = './test_results'
00083 
00084 VERB_MINIMAL = 'Minimal'    # Just top level errors
00085 VERB_NORMAL = 'Normal'      # Top level errors and warnings
00086 VERB_VERBOSE = 'Verbose'    # All errors and warnings
00087 VERB_ALL = 'All'            # All errors
00088 VERB_LEVELS = [VERB_MINIMAL, VERB_NORMAL, VERB_VERBOSE, VERB_ALL]
00089 
00090 
00091 def test_endpoints(workspace, parent_test):
00092     """Run tests to validate DAPLINK fimrware"""
00093     test_info = parent_test.create_subtest('test_endpoints')
00094     test_hid(workspace, test_info)
00095     test_serial(workspace, test_info)
00096     test_mass_storage(workspace, test_info)
00097     test_usb(workspace, test_info)
00098 
00099 
00100 class TestConfiguration (object):
00101     """Wrap all the resources needed to run a test"""
00102     def __init__(self, name):
00103         self.name  = name
00104         self.board  = None
00105         self.target  = None
00106         self.if_firmware  = None
00107         self.bl_firmware  = None
00108 
00109     def __str__(self):
00110         name_board = '<None>'
00111         name_target = '<None>'
00112         name_if_firmware = '<None>'
00113         name_bl_firmware = '<None>'
00114         if self.board  is not None:
00115             name_board = self.board .name
00116         if self.target  is not None:
00117             name_target = self.target .name
00118         if self.if_firmware  is not None:
00119             name_if_firmware = self.if_firmware .name
00120         if self.bl_firmware  is not None:
00121             name_bl_firmware = self.bl_firmware .name
00122         return "APP=%s BL=%s Board=%s Target=%s" % (name_if_firmware,
00123                                                     name_bl_firmware,
00124                                                     name_board, name_target)
00125 
00126 
00127 class TestManager (object):
00128     """Handle tests configuration running and results"""
00129 
00130     class _STATE(Enum):
00131         INIT = 0
00132         CONFIGURED = 1
00133         COMPLETE = 2
00134 
00135     def __init__(self):
00136         # By default test all configurations and boards
00137         self._target_list  = []
00138         self._board_list  = []
00139         self._firmware_list  = []
00140         self._only_test_first  = False
00141         self._load_if  = True
00142         self._load_bl  = True
00143         self._test_daplink  = True
00144         self._test_ep  = True
00145 
00146         # Internal state
00147         self._state  = self._STATE .INIT
00148         self._test_configuration_list  = None
00149         self._all_tests_pass  = None
00150         self._firmware_filter  = None
00151         self._untested_firmware  = None
00152 
00153     @property
00154     def all_tests_pass(self):
00155         assert self._all_tests_pass  is not None, 'Must call run_tests first'
00156         return self._all_tests_pass 
00157 
00158     def set_test_first_board_only (self, first):
00159         """Only test one board of each type"""
00160         assert isinstance(first, bool)
00161         assert self._state  is self._STATE .INIT
00162         self._only_test_first  = first
00163 
00164     def set_load_if (self, load):
00165         """Load new interface firmware before testing"""
00166         assert isinstance(load, bool)
00167         assert self._state  is self._STATE .INIT
00168         self._load_if  = load
00169 
00170     def set_load_bl (self, load):
00171         """Load new bootloader firmware before testing"""
00172         assert isinstance(load, bool)
00173         assert self._state  is self._STATE .INIT
00174         self._load_bl  = load
00175 
00176     def set_test_daplink (self, run_test):
00177         """Run DAPLink specific tests"""
00178         assert isinstance(run_test, bool)
00179         assert self._state  is self._STATE .INIT
00180         self._test_daplink  = run_test
00181 
00182     def set_test_ep (self, run_test):
00183         """Test each endpoint - MSD, CDC, HID"""
00184         assert isinstance(run_test, bool)
00185         assert self._state  is self._STATE .INIT
00186         self._test_ep  = run_test
00187 
00188     def add_firmware (self, firmware_list):
00189         """Add firmware to be tested"""
00190         assert self._state  is self._STATE .INIT
00191         self._firmware_list .extend(firmware_list)
00192 
00193     def add_boards (self, board_list):
00194         """Add boards to be used for testing"""
00195         assert self._state  is self._STATE .INIT
00196         self._board_list .extend(board_list)
00197 
00198     def add_targets (self, target_list):
00199         """Add targets to be used for testing"""
00200         assert self._state  is self._STATE .INIT
00201         self._target_list .extend(target_list)
00202 
00203     def set_firmware_filter (self, name_list):
00204         """Test only the project names passed given"""
00205         assert self._state  is self._STATE .INIT
00206         assert self._firmware_filter  is None
00207         self._firmware_filter  = set(name_list)
00208 
00209     def run_tests (self):
00210         """Run all configurations"""
00211         # Tests can only be run once per TestManager instance
00212         assert self._state  is self._STATE .CONFIGURED
00213         self._state  = self._STATE .COMPLETE
00214 
00215         all_tests_pass = True
00216         for test_configuration in self._test_configuration_list :
00217             board = test_configuration.board
00218             test_info = TestInfo(test_configuration.name)
00219             test_configuration.test_info = test_info
00220 
00221             test_info.info("Board: %s" % test_configuration.board)
00222             test_info.info("Application: %s" %
00223                            test_configuration.if_firmware)
00224             test_info.info("Bootloader: %s" %
00225                            test_configuration.bl_firmware)
00226             test_info.info("Target: %s" % test_configuration.target)
00227 
00228             
00229             if self._load_if :
00230                 if_path = test_configuration.if_firmware.hex_path
00231                 board.load_interface(if_path, test_info)
00232 
00233             valid_bl = test_configuration.bl_firmware is not None
00234             if self._load_bl  and valid_bl:
00235                 bl_path = test_configuration.bl_firmware.hex_path
00236                 board.load_bootloader(bl_path, test_info)
00237 
00238             board.set_check_fs_on_remount(True)
00239 
00240             if self._test_daplink :
00241                 daplink_test(test_configuration, test_info)
00242 
00243             if self._test_ep :
00244                 test_endpoints(test_configuration, test_info)
00245 
00246             if test_info.get_failed():
00247                 all_tests_pass = False
00248 
00249         self._all_tests_pass  = all_tests_pass
00250 
00251     def print_results(self, info_level):
00252         assert self._state  is self._STATE .COMPLETE
00253         # Print info for boards tested
00254         for test_configuration in self._test_configuration_list :
00255             print('')
00256             test_info = test_configuration.test_info
00257             if info_level == VERB_MINIMAL:
00258                 test_info.print_msg(TestInfo.FAILURE, 0)
00259             elif info_level == VERB_NORMAL:
00260                 test_info.print_msg(TestInfo.WARNING, None)
00261             elif info_level == VERB_VERBOSE:
00262                 test_info.print_msg(TestInfo.WARNING, None)
00263             elif info_level == VERB_ALL:
00264                 test_info.print_msg(TestInfo.INFO, None)
00265             else:
00266                 # This should never happen
00267                 assert False
00268 
00269     def write_test_results(self, directory, git_sha=None, local_changes=None,
00270                            info_level=TestInfo.INFO):
00271         assert self._state  is self._STATE .COMPLETE
00272 
00273         assert not os.path.exists(directory)
00274         os.mkdir(directory)
00275 
00276         # Write out version of tools used for test
00277         tools_file = directory + os.sep + 'requirements.txt'
00278         with open(tools_file, "w") as file_handle:
00279             command = ['pip', 'freeze']
00280             subprocess.check_call(command, stdin=subprocess.PIPE,
00281                                   stdout=file_handle,
00282                                   stderr=subprocess.STDOUT)
00283 
00284         # Write out each test result
00285         for test_configuration in self._test_configuration_list :
00286             test_info = test_configuration.test_info
00287             file_path = directory + os.sep + test_info.name + '.txt'
00288             with open(file_path, 'w') as file_handle:
00289                 file_handle.write("Test configuration: %s\n" %
00290                                   test_configuration)
00291                 file_handle.write("Board: %s\n" % test_configuration.board)
00292                 file_handle.write("Application: %s\n" %
00293                                   test_configuration.if_firmware)
00294                 file_handle.write("Bootloader: %s\n" %
00295                                   test_configuration.bl_firmware)
00296                 file_handle.write("Target: %s\n" % test_configuration.target)
00297                 file_handle.write("\n")
00298                 test_info.print_msg(info_level, None, log_file=file_handle)
00299 
00300         # Write out summary
00301         summary_file = directory + os.sep + 'summary.txt'
00302         with open(summary_file, "w") as file_handle:
00303             # Overall result
00304             if self.all_tests_pass :
00305                 file_handle.write("All tests pass\n\n")
00306             else:
00307                 file_handle.write("One or more tests have failed\n\n")
00308 
00309             if git_sha is not None and local_changes is not None:
00310                 file_handle.write("Git info for test:\n")
00311                 file_handle.write("  Git SHA: %s\n" % git_sha)
00312                 file_handle.write("  Local changes: %s\n" % local_changes)
00313             file_handle.write("\n")
00314 
00315             # Results for each test
00316             file_handle.write("Test settings:\n")
00317             file_handle.write("  Load application before test: %s\n" %
00318                               self._load_if )
00319             file_handle.write("  Load bootloader before test: %s\n" %
00320                               self._load_bl )
00321             file_handle.write("  Run DAPLink specific tests: %s\n" %
00322                               self._test_daplink )
00323             file_handle.write("  Run endpoint tests: %s\n" %
00324                               self._test_ep )
00325             file_handle.write("\n")
00326 
00327             # Results for each test
00328             file_handle.write("Tested configurations:\n")
00329             for test_configuration in self._test_configuration_list :
00330                 test_info = test_configuration.test_info
00331                 test_passed = test_info.get_failed() == 0
00332                 result_str = 'Pass' if test_passed else 'Fail'
00333                 file_handle.write("  %s: %s\n" %
00334                                   (test_configuration, result_str))
00335             file_handle.write("\n")
00336 
00337             # Untested firmware
00338             untested_list = self.get_untested_firmware ()
00339             if len(untested_list) == 0:
00340                 file_handle.write("All firmware in package tested\n")
00341             else:
00342                 file_handle.write("Untested firmware:\n")
00343                 for untested_fw in self.get_untested_firmware ():
00344                     file_handle.write("  %s\n" % untested_fw.name)
00345             file_handle.write("\n")
00346 
00347         # Target test images
00348         target_dir = directory + os.sep + 'target'
00349         os.mkdir(target_dir)
00350         for target in self._target_list :
00351             new_hex = target_dir + os.sep + os.path.basename(target.hex_path)
00352             shutil.copy(target.hex_path, new_hex)
00353             new_bin = target_dir + os.sep + os.path.basename(target.bin_path)
00354             shutil.copy(target.bin_path, new_bin)
00355 
00356     def get_test_configurations(self):
00357         assert self._state  in (self._STATE .CONFIGURED,
00358                                self._STATE .COMPLETE)
00359         return self._test_configuration_list 
00360 
00361     def get_untested_firmware(self):
00362         assert self._state  in (self._STATE .CONFIGURED,
00363                                self._STATE .COMPLETE)
00364         return self._untested_firmware 
00365 
00366     def build_test_configurations(self, parent_test):
00367         assert self._state  is self._STATE .INIT
00368         self._state  = self._STATE .CONFIGURED
00369         test_info = parent_test.create_subtest('Build test configuration')
00370 
00371         # Create table mapping each board id to a list of boards with that ID
00372         board_id_to_board_list = {}
00373         for board in self._board_list :
00374             board_id = board.get_board_id()
00375             if board_id not in board_id_to_board_list:
00376                 board_id_to_board_list[board_id] = []
00377             board_list = board_id_to_board_list[board_id]
00378             if self._only_test_first  and len(board_list) > 1:
00379                 # Ignore this board since we already have one
00380                 test_info.info('Ignoring extra boards of type 0x%x' %
00381                                board_id)
00382                 continue
00383             board_list.append(board)
00384 
00385         # Create a list for bootloader firmware and interface firmware
00386         bootloader_firmware_list = []
00387         filtered_interface_firmware_list = []
00388         for firmware in self._firmware_list :
00389             if firmware.type == Firmware.TYPE.BOOTLOADER:
00390                 bootloader_firmware_list.append(firmware)
00391             elif firmware.type == Firmware.TYPE.INTERFACE:
00392                 name = firmware.name
00393                 if ((self._firmware_filter  is None) or
00394                         (name in self._firmware_filter )):
00395                     filtered_interface_firmware_list.append(firmware)
00396             else:
00397                 assert False, 'Unsupported firmware type "%s"' % firmware.type
00398 
00399         # Create a table mapping name to object with that name
00400         TARGET_NAME_TO_TARGET = {target.name: target for target in
00401                                  self._target_list }
00402         FIRMWARE_NAME_TO_FIRMWARE = {firmware.name: firmware for firmware in
00403                                      filtered_interface_firmware_list}
00404         BL_NAME_TO_BL = {firmware.name: firmware for firmware in
00405                          bootloader_firmware_list}
00406 
00407         # Explicitly specified boards must be present
00408         fw_name_set = set(fw.name for fw in filtered_interface_firmware_list)
00409         if self._firmware_filter  is not None:
00410             assert self._firmware_filter  == fw_name_set
00411 
00412         # Create test configurations for each supported configuration
00413         test_conf_list = []
00414         untested_firmware = set(filtered_interface_firmware_list)
00415         for board_id, family_id, fw_name, bl_fw_name, target_name in info.SUPPORTED_CONFIGURATIONS:
00416             target = None
00417             if_firmware = None
00418             bl_firmware = None
00419             if target_name in TARGET_NAME_TO_TARGET:
00420                 target = TARGET_NAME_TO_TARGET[target_name]
00421             if fw_name in FIRMWARE_NAME_TO_FIRMWARE:
00422                 if_firmware = FIRMWARE_NAME_TO_FIRMWARE[fw_name]
00423             if bl_fw_name in BL_NAME_TO_BL:
00424                 bl_firmware = BL_NAME_TO_BL[bl_fw_name]
00425 
00426             target_required = self._test_ep 
00427             bl_required = self._load_bl  or self._test_daplink 
00428             if if_firmware is None:
00429                 # Skip configuration
00430                 continue
00431             if target_required and target is None:
00432                 # Skip configuration
00433                 test_info.info('No target to test firmware %s' % fw_name)
00434                 continue
00435             if bl_required and bl_firmware is None:
00436                 # Skip configuration
00437                 test_info.info('No bootloader to test firmware %s' % fw_name)
00438                 continue
00439             # Check if there is a board to test this firmware
00440             # and if not skip it
00441             if board_id not in board_id_to_board_list:
00442                 test_info.info('No board to test firmware %s' % fw_name)
00443                 continue
00444 
00445             # Create a test configuration for each board
00446             board_list = board_id_to_board_list[board_id]
00447             for board in board_list:
00448                 test_conf = TestConfiguration(if_firmware.name + ' ' +
00449                                               board.name)
00450                 test_conf.if_firmware = if_firmware
00451                 test_conf.bl_firmware = bl_firmware
00452                 test_conf.board = board
00453                 test_conf.target = target
00454                 test_conf_list.append(test_conf)
00455                 # remove this from the untested list
00456                 if if_firmware in untested_firmware:
00457                     untested_firmware.remove(if_firmware)
00458                 assert bl_firmware not in untested_firmware
00459 
00460         self._untested_firmware  = list(untested_firmware)
00461         self._test_configuration_list  = test_conf_list
00462 
00463 
00464 def get_firmware_names(project_dir):
00465 
00466     # Save current directory
00467     cur_dir = os.getcwd()
00468     os.chdir(project_dir)
00469     try:
00470         all_names = set()
00471         projects = list(Generator('projects.yaml').generate())
00472         for project in projects:
00473             assert project.name not in all_names
00474             all_names.add(project.name)
00475     finally:
00476         # Restore the current directory
00477         os.chdir(cur_dir)
00478     return list(all_names)
00479 
00480 
00481 def get_git_info(project_dir):
00482     cur_dir = os.getcwd()
00483     os.chdir(project_dir)
00484 
00485     # Get the git SHA.
00486     try:
00487         git_sha = subprocess.check_output(["git", "rev-parse",
00488                                            "--verify", "HEAD"])
00489         git_sha = git_sha.strip()
00490     except (subprocess.CalledProcessError, WindowsError):
00491         print("#> ERROR: Failed to get git SHA, do you "
00492               "have git in your PATH environment variable?")
00493         exit(-1)
00494 
00495     # Check are there any local, uncommitted modifications.
00496     try:
00497         subprocess.check_output(["git", "diff", "--no-ext-diff",
00498                                  "--quiet", "--exit-code"])
00499     except subprocess.CalledProcessError:
00500         git_has_changes = True
00501     else:
00502         git_has_changes = False
00503 
00504     os.chdir(cur_dir)
00505 
00506     return git_sha, git_has_changes
00507 
00508 
00509 def main():
00510     self_path = os.path.abspath(__file__)
00511     test_dir = os.path.dirname(self_path)
00512     daplink_dir = os.path.dirname(test_dir)
00513 
00514     # We make assumptions that break if user copies script file outside the test dir
00515     if os.path.basename(test_dir) != "test":
00516         print("Error - this script must reside in the test directory")
00517         exit(-1)
00518 
00519     git_sha, local_changes = get_git_info(daplink_dir)
00520     firmware_list = get_firmware_names(daplink_dir)
00521     firmware_choices = [firmware for firmware in firmware_list if
00522                         firmware.endswith('_if')]
00523 
00524     description = 'DAPLink validation and testing tool'
00525     parser = argparse.ArgumentParser(description=description)
00526     parser.add_argument('--targetdir',
00527                         help='Directory with pre-built target test images.',
00528                         default=None)
00529     parser.add_argument('--user', type=str, default=None,
00530                         help='MBED username (required for compile-api)')
00531     parser.add_argument('--password', type=str, default=None,
00532                         help='MBED password (required for compile-api)')
00533     parser.add_argument('--firmwaredir',
00534                         help='Directory with firmware images to test',
00535                         default=None)
00536     parser.add_argument('--project-tool', choices=['uvision', 'mbedcli'],
00537                         help='Tool used to compile the project',
00538                         default='uvision')
00539     parser.add_argument('--firmware', help='Firmware to test', action='append',
00540                         choices=firmware_choices, default=[], required=False)
00541     parser.add_argument('--logdir', help='Directory to log test results to',
00542                         default=DEFAULT_TEST_DIR)
00543     parser.add_argument('--noloadif', help='Skip load step for interface.',
00544                         default=False, action='store_true')
00545     parser.add_argument('--notestendpt', help='Dont test the interface '
00546                         'USB endpoints.', default=False, action='store_true')
00547     parser.add_argument('--loadbl', help='Load bootloader before test.',
00548                         default=False, action='store_true')
00549     parser.add_argument('--testdl', help='Run DAPLink specific tests. '
00550                         'The DAPLink test tests bootloader updates so use'
00551                         'with caution',
00552                         default=False, action='store_true')
00553     parser.add_argument('--testfirst', help='If multiple boards of the same '
00554                         'type are found only test the first one.',
00555                         default=False, action='store_true')
00556     parser.add_argument('--verbose', help='Verbose output',
00557                         choices=VERB_LEVELS, default=VERB_NORMAL)
00558     parser.add_argument('--dryrun', default=False, action='store_true',
00559                         help='Print info on configurations but dont '
00560                         'actually run tests.')
00561     parser.add_argument('--force', action='store_true', default=False,
00562                         help='Try to run tests even if there are problems. Delete logs from previous run.')
00563     args = parser.parse_args()
00564 
00565     use_prebuilt = args.targetdir is not None
00566     use_compile_api = args.user is not None and args.password is not None
00567 
00568     test_info = TestInfo('DAPLink')
00569 
00570     # Validate args
00571 
00572     # See if user wants to test endpoints. If yes and he didn't provide
00573     # target test binaries, use the Compile API to build them
00574     all_targets = None
00575     if not args.notestendpt:
00576         if not use_prebuilt and not use_compile_api:
00577             print("Endpoint test requires target test images.")
00578             print("  Directory with pre-built target test images")
00579             print("  must be specified with '--targetdir'")
00580             print("OR")
00581             print("  developer.mbed.org login credentials must be ")
00582             print("  specified with '--user' and '--password' so test ")
00583             print("  images can be built with the RESTful Compile API.")
00584             print("NOTE: you can skip the endpoint tests altogether ")
00585             print("with --notestendpt")
00586             
00587             exit(-1)
00588 
00589         if args.targetdir is not None:
00590             target_dir = args.targetdir
00591         else:
00592             target_dir = daplink_dir + os.sep + 'tmp'
00593             build_target_bundle(target_dir, args.user, args.password, test_info)
00594 
00595         target_bundle = load_target_bundle(target_dir)
00596         all_targets = target_bundle.get_target_list()
00597 
00598     if os.path.exists(args.logdir):
00599         if args.force:
00600             shutil.rmtree(args.logdir)
00601         else:
00602             print('Error - test results directory "%s" already exists' %
00603                   args.logdir)
00604             exit(-1)
00605 
00606     # Get all relevant info
00607     if args.firmwaredir is None:
00608         firmware_bundle = load_bundle_from_project(args.project_tool)
00609     else:
00610         firmware_bundle = load_bundle_from_release(args.firmwaredir)
00611 
00612     all_firmware = firmware_bundle.get_firmware_list()
00613     all_boards = get_all_attached_daplink_boards()
00614 
00615     for board in all_boards:
00616         if board.get_mode() == board.MODE_BL:
00617             print('Switching to APP mode on board: %s' % board.unique_id)
00618             try:
00619                 board.set_mode(board.MODE_IF)
00620             except Exception:
00621                 print('Unable to switch mode on board: %s' % board.unique_id)
00622 
00623     # Make sure firmware is present
00624     firmware_explicitly_specified = len(args.firmware) != 0
00625     if firmware_explicitly_specified:
00626         all_firmware_names = set(fw.name for fw in all_firmware)
00627         firmware_missing = False
00628         for firmware_name in args.firmware:
00629             if firmware_name not in all_firmware_names:
00630                 firmware_missing = True
00631                 test_info.failure('Cannot find firmware %s' % firmware_name)
00632         if firmware_missing:
00633             test_info.failure('Firmware missing - aborting test')
00634             exit(-1)
00635 
00636     # Create manager and add resources
00637     tester = TestManager()
00638     tester.add_firmware(all_firmware)
00639     tester.add_boards(all_boards)
00640     if all_targets is not None:
00641         tester.add_targets(all_targets)
00642     if firmware_explicitly_specified:
00643         tester.set_firmware_filter(args.firmware)
00644 
00645     # Configure test manager
00646     tester.set_test_first_board_only(args.testfirst)
00647     tester.set_load_if(not args.noloadif)
00648     tester.set_test_ep(not args.notestendpt)
00649     tester.set_load_bl(args.loadbl)
00650     tester.set_test_daplink(args.testdl)
00651 
00652     # Build test configurations
00653     tester.build_test_configurations(test_info)
00654 
00655     test_config_list = tester.get_test_configurations()
00656     if len(test_config_list) == 0:
00657         test_info.failure("Nothing that can be tested")
00658         exit(-1)
00659     else:
00660         test_info.info('Test configurations to be run:')
00661         index = 0
00662         for test_config in test_config_list:
00663             test_info.info('    %i: %s' % (index, test_config))
00664             index += 1
00665     test_info.info('')
00666 
00667     untested_list = tester.get_untested_firmware()
00668     if len(untested_list) == 0:
00669         test_info.info("All firmware can be tested")
00670     else:
00671         test_info.info('Fimrware that will not be tested:')
00672         for untested_firmware in untested_list:
00673             test_info.info('    %s' % untested_firmware.name)
00674     test_info.info('')
00675 
00676     if firmware_explicitly_specified and len(untested_list) != 0:
00677         test_info.failure("Exiting because not all firmware could be tested")
00678         exit(-1)
00679 
00680     # If this is a dryrun don't run tests, just print info
00681     if args.dryrun:
00682         exit(0)
00683 
00684     # Run tests
00685     tester.run_tests()
00686 
00687     # Print test results
00688     tester.print_results(args.verbose)
00689     tester.write_test_results(args.logdir,
00690                               git_sha=git_sha,
00691                               local_changes=local_changes)
00692 
00693     # Warn about untested boards
00694     print('')
00695     for firmware in tester.get_untested_firmware():
00696         print('Warning - configuration %s is untested' % firmware.name)
00697 
00698     if tester.all_tests_pass:
00699         print("All boards passed")
00700         exit(0)
00701     else:
00702         print("Test Failed")
00703         exit(-1)
00704 
00705 
00706 if __name__ == "__main__":
00707     main()