run_icetea.py

00001 #! /usr/bin/env python2
00002 """
00003 Copyright 2018 ARM Limited
00004 Licensed under the Apache License, Version 2.0 (the "License");
00005 you may not use this file except in compliance with the License.
00006 You may obtain a copy of the License at
00007 
00008     http://www.apache.org/licenses/LICENSE-2.0
00009 
00010 Unless required by applicable law or agreed to in writing, software
00011 distributed under the License is distributed on an "AS IS" BASIS,
00012 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013 See the License for the specific language governing permissions and
00014 limitations under the License.
00015 """
00016 
00017 from __future__ import print_function, division, absolute_import
00018 import sys
00019 import os
00020 import re
00021 from os.path import abspath, join, dirname, relpath, sep
00022 import json
00023 import traceback
00024 from fnmatch import translate
00025 from argparse import ArgumentParser
00026 
00027 ROOT = abspath(join(dirname(__file__), '..'))
00028 sys.path.insert(0, ROOT)
00029 
00030 from tools.config import ConfigException
00031 from tools.utils import cmd, run_cmd
00032 
00033 plugins_path = abspath(join(ROOT, 'TEST_APPS', 'icetea_plugins', 'plugins_to_load.py'))
00034 
00035 
00036 def find_build_from_build_data(build_data, id, target, toolchain):
00037     if 'builds' not in build_data:
00038         raise Exception("build data is in wrong format, does not include builds object")
00039 
00040     for build in build_data['builds']:
00041         if 'id' in build.keys() \
00042                 and id.upper() in build['id'].upper() \
00043                 and 'target_name' in build.keys() \
00044                 and target.upper() == build['target_name'].upper() \
00045                 and 'toolchain_name' in build.keys() \
00046                 and toolchain.upper() == build['toolchain_name'].upper() \
00047                 and 'result' in build.keys() \
00048                 and "OK" == build['result']:
00049             return build
00050     return None
00051 
00052 
00053 def create_test_suite(target, tool, icetea_json_output, build_data, tests_by_name):
00054     """
00055     Create test suite content
00056     :param target:
00057     :param tool:
00058     :param icetea_json_output:
00059     :param build_data:
00060     :return:
00061     """
00062     test_suite = dict()
00063     test_suite['testcases'] = list()
00064 
00065     for test in icetea_json_output:
00066         skip = False
00067 
00068         for dut in test['requirements']['duts'].values():
00069             # Set binary path based on application name
00070             if 'application' in dut.keys() and 'name' in dut['application'].keys():
00071                 build = find_build_from_build_data(
00072                     build_data=build_data,
00073                     id=dut['application']['name'],
00074                     target=target,
00075                     toolchain=tool)
00076                 if build:
00077                     try:
00078                         dut['application']['bin'] = build['bin_fullpath']
00079                     except KeyError:
00080                         raise Exception('Full path is missing from build: {}'.format(build))
00081                 else:
00082                     skip = True
00083 
00084         if not tests_by_name or is_test_in_test_by_name(test['name'], tests_by_name):
00085             test_case = {
00086                 'name': test['name'],
00087                 'config': {
00088                     'requirements': set_allowed_platform(test['requirements'], target)
00089                 }
00090             }
00091 
00092             # Skip test if not binary path
00093             if skip:
00094                 test_case['config']['execution'] = {
00095                     'skip': {
00096                         'value': True,
00097                         'reason': "Test requiring application binary not build"
00098                     }
00099                 }
00100 
00101             test_suite['testcases'].append(test_case)
00102 
00103     return test_suite
00104 
00105 
00106 def set_allowed_platform(requirements, target):
00107     """
00108     Allowed platform restrict icetea to run tests on specific board
00109     This targets tests to the right board in case that user has multiple ones connected same time
00110     """
00111     if '*' not in requirements['duts'].keys():
00112         requirements['duts']['*'] = dict()
00113     requirements['duts']['*']['allowed_platforms'] = [target]
00114     return requirements
00115 
00116 
00117 def get_applications(test):
00118     ret = list()
00119     for dut in test['requirements']['duts'].values():
00120         if 'application' in dut.keys() and 'name' in dut['application'].keys():
00121             ret.append(dut['application']['name'])
00122     return ret
00123 
00124 
00125 def filter_test_by_build_data(icetea_json_output, build_data, target, toolchain):
00126     if not build_data:
00127         return icetea_json_output
00128 
00129     ret = list()
00130     for test in icetea_json_output:
00131         for dut in test['requirements']['duts'].values():
00132             if 'application' in dut.keys() and 'name' in dut['application'].keys():
00133                 id = dut['application']['name']
00134                 if find_build_from_build_data(build_data, id, target, toolchain):
00135                     # Test requiring build found
00136                     ret.append(test)
00137     return ret
00138 
00139 
00140 def filter_test_by_name(icetea_json_output, test_by_name):
00141     if not test_by_name:
00142         return icetea_json_output
00143     ret = list()
00144     for test_temp in icetea_json_output:
00145         if is_test_in_test_by_name(test_temp['name'], test_by_name) and test_temp not in ret:
00146             ret.append(test_temp)
00147     return ret
00148 
00149 
00150 def get_applications_from_test(test):
00151     ret = list()
00152     if u'requirements' in test.keys() and u'duts' in test[u'requirements']:
00153         for name, dut in test[u'requirements'][u'duts'].items():
00154             if u'application' in dut.keys() and u'name' in dut[u'application']:
00155                 ret.append(dut[u'application'][u'name'])
00156     return ret
00157 
00158 
00159 def get_application_list(icetea_json_output, tests_by_name):
00160     """ Return comma separated list of application which are used in tests """
00161     ret = list()
00162     for test in filter_test_by_name(icetea_json_output, tests_by_name):
00163         ret.extend(get_applications_from_test(test))
00164     # Remove duplicates
00165     return list(set(ret))
00166 
00167 
00168 def icetea_tests(target, tcdir, verbose):
00169     command = ['icetea', '--tcdir', tcdir, '--list', '--json', '--platform_filter', target] \
00170               + (['-v'] if verbose else [])
00171 
00172     stdout, stderr, returncode = run_cmd(command)
00173 
00174     if returncode != 0:
00175         raise Exception(
00176             "Error when running icetea. \ncwd:{} \nCommand:'{}' \noutput:{}".format(os.getcwd(), ' '.join(command),
00177                                                                                     stderr.decode()))
00178 
00179     return json.loads(stdout)
00180 
00181 
00182 def is_test_in_test_by_name(test_name, test_by_name):
00183     for tbn_temp in test_by_name:
00184         if re.search(translate(tbn_temp), test_name):
00185             return True
00186     return False
00187 
00188 
00189 def check_tests(icetea_json_output):
00190     """
00191     Check that all tests have all necessary information
00192     :return:
00193     """
00194     for test in icetea_json_output:
00195         if not get_applications_from_test(test):
00196             raise Exception('Test {} does not have application with correct name'.format(test['name']))
00197 
00198 
00199 def load_build_data(build_data_path):
00200     """
00201     :return: build_data.json content as dict and None if build data is not available
00202     """
00203     if not os.path.isfile(build_data_path):
00204         return None
00205     return json.load(open(build_data_path))
00206 
00207 
00208 if __name__ == '__main__':
00209     try:
00210         # Parse Options
00211         parser = ArgumentParser()
00212 
00213         parser.add_argument('-m', '--mcu',
00214                             dest='target',
00215                             default=None,
00216                             help='Test target MCU',
00217                             required=True)
00218 
00219         parser.add_argument('-t', '--toolchain',
00220                             dest='toolchain',
00221                             default=None,
00222                             help='Toolchain',
00223                             required=True)
00224 
00225         parser.add_argument('--build-data',
00226                             dest='build_data',
00227                             default=None,
00228                             help='Detail data from build')
00229 
00230         parser.add_argument('--test-suite',
00231                             dest='test_suite',
00232                             default=None,
00233                             help='Path used for test suite file')
00234 
00235         parser.add_argument('-n', '--tests-by-name',
00236                             dest='tests_by_name',
00237                             default=None,
00238                             help='Limit the tests to a list (ex. test1,test2,test3)')
00239 
00240         parser.add_argument('--tcdir',
00241                             dest='tcdir',
00242                             default='TEST_APPS',
00243                             help='Test case directory',
00244                             required=False)
00245 
00246         parser.add_argument('--compile-list',
00247                             action='store_true',
00248                             dest='compile_list',
00249                             default=False,
00250                             help='List tests, which applications can be compiled')
00251 
00252         parser.add_argument('--run-list',
00253                             action='store_true',
00254                             dest='run_list',
00255                             default=False,
00256                             help='List tests, which applications are compiled and ready for run')
00257 
00258         parser.add_argument('--application-list',
00259                             action='store_true',
00260                             dest='application_list',
00261                             default=False,
00262                             help='List applications that need to be build')
00263 
00264         parser.add_argument('--ignore-checks',
00265                             action='store_true',
00266                             dest='ignore_checks',
00267                             default=False,
00268                             help='Ignore data validation checks')
00269 
00270         parser.add_argument('-v', '--verbose',
00271                             action='store_true',
00272                             dest='verbose',
00273                             default=False,
00274                             help='Verbose diagnostic output')
00275 
00276         options = parser.parse_args()
00277 
00278         icetea_json_output = icetea_tests(options.target, options.tcdir, options.verbose)
00279         tests_by_name = options.tests_by_name.split(',') if options.tests_by_name else None
00280         build_data = load_build_data(options.build_data) if options.build_data else None
00281 
00282         if not options.ignore_checks:
00283             check_tests(icetea_json_output)
00284 
00285         if options.compile_list:
00286             print('Available icetea tests for build \'{}-{}\', location \'{}\''.format(
00287                 options.target, options.toolchain, options.tcdir))
00288             for test in icetea_json_output:
00289                 print(
00290                     'Test Case:\n    Name: {name}\n    Path: .{sep}{filepath}\n    Test applications: .{sep}{apps}'.format(
00291                         name=test['name'],
00292                         sep=sep,
00293                         filepath=relpath(test['filepath'], ROOT),
00294                         apps=''.join(get_applications(test)).replace('-', os.path.sep)))
00295 
00296         elif options.run_list:
00297             print('Available icetea tests for build \'{}-{}\', location \'{}\''.format(
00298                 options.target, options.toolchain, options.tcdir))
00299 
00300             # Filters
00301             tests = filter_test_by_name(icetea_json_output, tests_by_name)
00302             if build_data:
00303                 tests = filter_test_by_build_data(tests, build_data, options.target, options.toolchain)
00304 
00305             for test in tests:
00306                 print('    test \'{name}\''.format(name=test['name']))
00307 
00308         elif options.application_list:
00309             print(','.join(get_application_list(icetea_json_output, tests_by_name)))
00310 
00311         else:
00312             if not build_data:
00313                 raise Exception("Build data file does not exist: {}".format(options.build_data))
00314 
00315             test_suite = create_test_suite(options.target, options.toolchain, icetea_json_output, build_data,
00316                                            tests_by_name)
00317 
00318             if not test_suite['testcases']:
00319                 raise Exception("Test suite is empty. Check that --tcdir and --tests-by-name have correct values")
00320 
00321             if not options.test_suite:
00322                 raise Exception('--test-suite is required when running tests')
00323 
00324             with open(options.test_suite, 'w') as f:
00325                 json.dump(test_suite, f, indent=2)
00326 
00327             # List just for debug
00328             if options.verbose:
00329                 cmd(['icetea', '--tcdir', options.tcdir, '--list'] + (['-v'] if options.verbose else []))
00330 
00331             cmd(['icetea', '--tcdir', options.tcdir, '--suite', options.test_suite, '--clean', '--plugin_path',
00332                  plugins_path] + (['-v'] if options.verbose else []))
00333 
00334     except KeyboardInterrupt as e:
00335         print('\n[CTRL+c] exit')
00336     except ConfigException as e:
00337         # Catching ConfigException here to prevent a traceback
00338         print('[ERROR] {}'.format(e))
00339     except Exception as e:
00340         traceback.print_exc(file=sys.stdout)
00341         print('[ERROR] {}'.format(e))
00342         sys.exit(1)