run_icetea.py

00001 #! /usr/bin/env python2
00002 """
00003 Copyright 2018 ARM Limited
00004 Licensed under the Apache License, Version 2.0 (the "License");
00005 you may not use this file except in compliance with the License.
00006 You may obtain a copy of the License at
00007 
00008     http://www.apache.org/licenses/LICENSE-2.0
00009 
00010 Unless required by applicable law or agreed to in writing, software
00011 distributed under the License is distributed on an "AS IS" BASIS,
00012 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013 See the License for the specific language governing permissions and
00014 limitations under the License.
00015 """
00016 
00017 from __future__ import print_function, division, absolute_import
00018 import sys
00019 import os
00020 import re
00021 from os.path import abspath, join, dirname, relpath, sep
00022 import json
00023 import traceback
00024 from fnmatch import translate
00025 from argparse import ArgumentParser
00026 
00027 ROOT = abspath(join(dirname(__file__), '..'))
00028 sys.path.insert(0, ROOT)
00029 
00030 from tools.config import ConfigException
00031 from tools.utils import cmd, run_cmd
00032 
00033 plugins_path = abspath(join(ROOT, 'TEST_APPS', 'icetea_plugins', 'plugins_to_load.py'))
00034 
00035 
00036 def find_build_from_build_data(build_data, id, target, toolchain):
00037     if 'builds' not in build_data:
00038         raise Exception("build data is in wrong format, does not include builds object")
00039 
00040     for build in build_data['builds']:
00041         if 'id' in build.keys() \
00042                 and id.upper() in build['id'].upper() \
00043                 and 'target_name' in build.keys() \
00044                 and target.upper() == build['target_name'].upper() \
00045                 and 'toolchain_name' in build.keys() \
00046                 and toolchain.upper() == build['toolchain_name'].upper() \
00047                 and 'result' in build.keys() \
00048                 and "OK" == build['result']:
00049             return build
00050     return None
00051 
00052 
00053 def create_test_suite(target, tool, icetea_json_output, build_data, tests_by_name):
00054     """
00055     Create test suite content
00056     :param target:
00057     :param tool:
00058     :param icetea_json_output:
00059     :param build_data:
00060     :return:
00061     """
00062     test_suite = dict()
00063     test_suite['testcases'] = list()
00064 
00065     for test in icetea_json_output:
00066         skip = False
00067 
00068         for dut in test['requirements']['duts'].values():
00069             # Set binary path based on application name
00070             if 'application' in dut.keys() and 'name' in dut['application'].keys():
00071                 build = find_build_from_build_data(
00072                     build_data=build_data,
00073                     id=dut['application']['name'],
00074                     target=target,
00075                     toolchain=tool)
00076                 if build:
00077                     try:
00078                         dut['application']['bin'] = build['bin_fullpath']
00079                     except KeyError:
00080                         raise Exception('Full path is missing from build: {}'.format(build))
00081                 else:
00082                     skip = True
00083 
00084         if not tests_by_name or is_test_in_test_by_name(test['name'], tests_by_name):
00085             test_case = {
00086                 'name': test['name'],
00087                 'config': {
00088                     'requirements': set_allowed_platform(test['requirements'], target)
00089                 }
00090             }
00091 
00092             # Skip test if not binary path
00093             if skip:
00094                 test_case['config']['execution'] = {
00095                     'skip': {
00096                         'value': True,
00097                         'reason': "Test requiring application binary not build"
00098                     }
00099                 }
00100 
00101             test_suite['testcases'].append(test_case)
00102 
00103     return test_suite
00104 
00105 
00106 def set_allowed_platform(requirements, target):
00107     """
00108     Allowed platform restrict icetea to run tests on specific board
00109     This targets tests to the right board in case that user has multiple ones connected same time
00110     """
00111     if '*' not in requirements['duts'].keys():
00112         requirements['duts']['*'] = dict()
00113     requirements['duts']['*']['allowed_platforms'] = [target]
00114     return requirements
00115 
00116 
00117 def get_applications(test):
00118     ret = list()
00119     for dut in test['requirements']['duts'].values():
00120         if 'application' in dut.keys() and 'name' in dut['application'].keys():
00121             ret.append(dut['application']['name'])
00122     return ret
00123 
00124 
00125 def filter_test_by_build_data(icetea_json_output, build_data, target, toolchain):
00126     if not build_data:
00127         return icetea_json_output
00128 
00129     ret = list()
00130     for test in icetea_json_output:
00131         for dut in test['requirements']['duts'].values():
00132             if 'application' in dut.keys() and 'name' in dut['application'].keys():
00133                 id = dut['application']['name']
00134                 if find_build_from_build_data(build_data, id, target, toolchain):
00135                     # Test requiring build found
00136                     ret.append(test)
00137     return ret
00138 
00139 
00140 def filter_test_by_name(icetea_json_output, test_by_name):
00141     if not test_by_name:
00142         return icetea_json_output
00143     ret = list()
00144     for test_temp in icetea_json_output:
00145         if is_test_in_test_by_name(test_temp['name'], test_by_name) and test_temp not in ret:
00146             ret.append(test_temp)
00147     return ret
00148 
00149 
00150 def get_applications_from_test(test):
00151     ret = list()
00152     if u'requirements' in test.keys() and u'duts' in test[u'requirements']:
00153         for name, dut in test[u'requirements'][u'duts'].items():
00154             if u'application' in dut.keys() and u'name' in dut[u'application']:
00155                 ret.append(dut[u'application'][u'name'])
00156     return ret
00157 
00158 
00159 def get_application_list(icetea_json_output, tests_by_name):
00160     """ Return comma separated list of application which are used in tests """
00161     ret = list()
00162     for test in filter_test_by_name(icetea_json_output, tests_by_name):
00163         ret.extend(get_applications_from_test(test))
00164     # Remove duplicates
00165     return list(set(ret))
00166 
00167 
00168 def icetea_tests(target, tcdir, verbose):
00169     if not os.path.exists(tcdir):
00170         raise Exception("Icetea run error: No TEST_APPS folder in {}".format(os.path.curdir))
00171 
00172     command = ['icetea', '--tcdir', tcdir, '--list', '--json', '--platform_filter', target] \
00173               + (['-v'] if verbose else [])
00174 
00175     stdout, stderr, returncode = run_cmd(command)
00176 
00177     if returncode != 0:
00178         additional_information = "\ncwd:{} \nCommand:'{}' \noutput:{}".format(os.getcwd(), ' '.join(command),
00179                                                                               stderr.decode())
00180         raise Exception("Error when running icetea. {}".format(additional_information))
00181 
00182     return json.loads(stdout)
00183 
00184 
00185 def is_test_in_test_by_name(test_name, test_by_name):
00186     for tbn_temp in test_by_name:
00187         if re.search(translate(tbn_temp), test_name):
00188             return True
00189     return False
00190 
00191 
00192 def check_tests(icetea_json_output):
00193     """
00194     Check that all tests have all necessary information
00195     :return:
00196     """
00197     for test in icetea_json_output:
00198         if not get_applications_from_test(test):
00199             raise Exception('Test {} does not have application with correct name'.format(test['name']))
00200 
00201 
00202 def load_build_data(build_data_path):
00203     """
00204     :return: build_data.json content as dict and None if build data is not available
00205     """
00206     if not os.path.isfile(build_data_path):
00207         return None
00208     return json.load(open(build_data_path))
00209 
00210 
00211 if __name__ == '__main__':
00212     try:
00213         # Parse Options
00214         parser = ArgumentParser()
00215 
00216         parser.add_argument('-m', '--mcu',
00217                             dest='target',
00218                             default=None,
00219                             help='Test target MCU',
00220                             required=True)
00221 
00222         parser.add_argument('-t', '--toolchain',
00223                             dest='toolchain',
00224                             default=None,
00225                             help='Toolchain',
00226                             required=True)
00227 
00228         parser.add_argument('--build-data',
00229                             dest='build_data',
00230                             default=None,
00231                             help='Detail data from build')
00232 
00233         parser.add_argument('--test-suite',
00234                             dest='test_suite',
00235                             default=None,
00236                             help='Path used for test suite file')
00237 
00238         parser.add_argument('-n', '--tests-by-name',
00239                             dest='tests_by_name',
00240                             default=None,
00241                             help='Limit the tests to a list (ex. test1,test2,test3)')
00242 
00243         parser.add_argument('--tcdir',
00244                             dest='tcdir',
00245                             default='TEST_APPS',
00246                             help='Test case directory',
00247                             required=False)
00248 
00249         parser.add_argument('--compile-list',
00250                             action='store_true',
00251                             dest='compile_list',
00252                             default=False,
00253                             help='List tests, which applications can be compiled')
00254 
00255         parser.add_argument('--run-list',
00256                             action='store_true',
00257                             dest='run_list',
00258                             default=False,
00259                             help='List tests, which applications are compiled and ready for run')
00260 
00261         parser.add_argument('--application-list',
00262                             action='store_true',
00263                             dest='application_list',
00264                             default=False,
00265                             help='List applications that need to be build')
00266 
00267         parser.add_argument('--ignore-checks',
00268                             action='store_true',
00269                             dest='ignore_checks',
00270                             default=False,
00271                             help='Ignore data validation checks')
00272 
00273         parser.add_argument('-v', '--verbose',
00274                             action='store_true',
00275                             dest='verbose',
00276                             default=False,
00277                             help='Verbose diagnostic output')
00278 
00279         options = parser.parse_args()
00280 
00281         icetea_json_output = icetea_tests(options.target, options.tcdir, options.verbose)
00282         tests_by_name = options.tests_by_name.split(',') if options.tests_by_name else None
00283         build_data = load_build_data(options.build_data) if options.build_data else None
00284 
00285         if not options.ignore_checks:
00286             check_tests(icetea_json_output)
00287 
00288         if options.compile_list:
00289             print('Available icetea tests for build \'{}-{}\', location \'{}\''.format(
00290                 options.target, options.toolchain, options.tcdir))
00291             for test in icetea_json_output:
00292                 print(
00293                     'Test Case:\n    Name: {name}\n    Path: .{sep}{filepath}\n    Test applications: .{sep}{apps}'.format(
00294                         name=test['name'],
00295                         sep=sep,
00296                         filepath=relpath(test['filepath'], ROOT),
00297                         apps=''.join(get_applications(test)).replace('-', os.path.sep)))
00298 
00299         elif options.run_list:
00300             print('Available icetea tests for build \'{}-{}\', location \'{}\''.format(
00301                 options.target, options.toolchain, options.tcdir))
00302 
00303             # Filters
00304             tests = filter_test_by_name(icetea_json_output, tests_by_name)
00305             if build_data:
00306                 tests = filter_test_by_build_data(tests, build_data, options.target, options.toolchain)
00307 
00308             for test in tests:
00309                 print('    test \'{name}\''.format(name=test['name']))
00310 
00311         elif options.application_list:
00312             print(','.join(get_application_list(icetea_json_output, tests_by_name)))
00313 
00314         else:
00315             if not build_data:
00316                 raise Exception("Build data file does not exist: {}".format(options.build_data))
00317 
00318             test_suite = create_test_suite(options.target, options.toolchain, icetea_json_output, build_data,
00319                                            tests_by_name)
00320 
00321             if not test_suite['testcases']:
00322                 raise Exception("Test suite is empty. Check that --tcdir and --tests-by-name have correct values")
00323 
00324             if not options.test_suite:
00325                 raise Exception('--test-suite is required when running tests')
00326 
00327             with open(options.test_suite, 'w') as f:
00328                 json.dump(test_suite, f, indent=2)
00329 
00330             # List just for debug
00331             if options.verbose:
00332                 cmd(['icetea', '--tcdir', options.tcdir, '--list'] + (['-v'] if options.verbose else []))
00333 
00334             cmd(['icetea', '--tcdir', options.tcdir, '--suite', options.test_suite, '--clean', '--plugin_path',
00335                  plugins_path] + (['-v'] if options.verbose else []))
00336 
00337     except KeyboardInterrupt as e:
00338         print('\n[CTRL+c] exit')
00339     except ConfigException as e:
00340         # Catching ConfigException here to prevent a traceback
00341         print('[ERROR] {}'.format(e))
00342     except Exception as e:
00343         traceback.print_exc(file=sys.stdout)
00344         print('[ERROR] {}'.format(e))
00345         sys.exit(1)