singletest.py

00001 #!/usr/bin/env python2
00002 
00003 """
00004 mbed SDK
00005 Copyright (c) 2011-2014 ARM Limited
00006 
00007 Licensed under the Apache License, Version 2.0 (the "License");
00008 you may not use this file except in compliance with the License.
00009 You may obtain a copy of the License at
00010 
00011     http://www.apache.org/licenses/LICENSE-2.0
00012 
00013 Unless required by applicable law or agreed to in writing, software
00014 distributed under the License is distributed on an "AS IS" BASIS,
00015 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00016 See the License for the specific language governing permissions and
00017 limitations under the License.
00018 
00019 Author: Przemyslaw Wirkus <Przemyslaw.Wirkus@arm.com>
00020 """
00021 
00022 """
00023 File format example: test_spec.json:
00024 {
00025     "targets": {
00026         "KL46Z": ["ARM", "GCC_ARM"],
00027         "LPC1768": ["ARM", "GCC_ARM", "GCC_CR", "IAR"],
00028         "LPC11U24": ["uARM"],
00029         "NRF51822": ["ARM"]
00030     }
00031 }
00032 
00033 File format example: muts_all.json:
00034 {
00035     "1" : {"mcu": "LPC1768",
00036            "port":"COM4",
00037            "disk":"J:\\",
00038            "peripherals": ["TMP102", "digital_loop", "port_loop", "analog_loop", "SD"]
00039     },
00040 
00041     "2" : {"mcu": "KL25Z",
00042            "port":"COM7",
00043            "disk":"G:\\",
00044            "peripherals": ["digital_loop", "port_loop", "analog_loop"]
00045     }
00046 }
00047 """
00048 
00049 
00050 # Be sure that the tools directory is in the search path
00051 import sys
00052 from os.path import join, abspath, dirname
00053 
00054 ROOT = abspath(join(dirname(__file__), ".."))
00055 sys.path.insert(0, ROOT)
00056 
00057 
00058 # Check: Extra modules which are required by core test suite
00059 from tools.utils import check_required_modules
00060 check_required_modules(['prettytable', 'serial'])
00061 
00062 # Imports related to mbed build api
00063 from tools.build_api import mcu_toolchain_matrix
00064 
00065 # Imports from TEST API
00066 from tools.test_api import SingleTestRunner
00067 from tools.test_api import singletest_in_cli_mode
00068 from tools.test_api import detect_database_verbose
00069 from tools.test_api import get_json_data_from_file
00070 from tools.test_api import get_avail_tests_summary_table
00071 from tools.test_api import get_default_test_options_parser
00072 from tools.test_api import print_muts_configuration_from_json
00073 from tools.test_api import print_test_configuration_from_json
00074 from tools.test_api import get_autodetected_MUTS_list
00075 from tools.test_api import get_autodetected_TEST_SPEC
00076 from tools.test_api import get_module_avail
00077 from tools.test_exporters import ReportExporter, ResultExporterType
00078 
00079 
00080 # Importing extra modules which can be not installed but if available they can extend test suite functionality
00081 try:
00082     import mbed_lstools
00083     from tools.compliance.ioper_runner import IOperTestRunner
00084     from tools.compliance.ioper_runner import get_available_oper_test_scopes
00085 except:
00086     pass
00087 
00088 def get_version():
00089     """ Returns test script version
00090     """
00091     single_test_version_major = 1
00092     single_test_version_minor = 5
00093     return (single_test_version_major, single_test_version_minor)
00094 
00095 
00096 if __name__ == '__main__':
00097     # Command line options
00098     parser = get_default_test_options_parser()
00099 
00100     parser.description = """This script allows you to run mbed defined test cases for particular MCU(s) and corresponding toolchain(s)."""
00101     parser.epilog = """Example: singletest.py -i test_spec.json -M muts_all.json"""
00102 
00103     opts = parser.parse_args()
00104 
00105     # Print scrip version
00106     if opts.version:
00107         print parser.description
00108         print parser.epilog
00109         print "Version %d.%d"% get_version()
00110         exit(0)
00111 
00112     if opts.db_url and opts.verbose_test_configuration_only:
00113         detect_database_verbose(opts.db_url)
00114         exit(0)
00115 
00116     # Print summary / information about automation test status
00117     if opts.test_automation_report:
00118         print get_avail_tests_summary_table(platform_filter=opts.general_filter_regex)
00119         exit(0)
00120 
00121     # Print summary / information about automation test status
00122     if opts.test_case_report:
00123         test_case_report_cols = ['id',
00124                                  'automated',
00125                                  'description',
00126                                  'peripherals',
00127                                  'host_test',
00128                                  'duration',
00129                                  'source_dir']
00130         print get_avail_tests_summary_table(cols=test_case_report_cols,
00131                                             result_summary=False,
00132                                             join_delim='\n',
00133                                             platform_filter=opts.general_filter_regex)
00134         exit(0)
00135 
00136     # Only prints matrix of supported toolchains
00137     if opts.supported_toolchains:
00138         print mcu_toolchain_matrix(platform_filter=opts.general_filter_regex)
00139         exit(0)
00140 
00141     test_spec = None
00142     MUTs = None
00143 
00144     if hasattr(opts, 'auto_detect') and opts.auto_detect:
00145         # If auto_detect attribute is present, we assume other auto-detection
00146         # parameters like 'toolchains_filter' are also set.
00147         print "MBEDLS: Detecting connected mbed-enabled devices... "
00148 
00149         MUTs = get_autodetected_MUTS_list()
00150 
00151         for mut in MUTs.values():
00152             print "MBEDLS: Detected %s, port: %s, mounted: %s"% (mut['mcu_unique'] if 'mcu_unique' in mut else mut['mcu'],
00153                                     mut['port'],
00154                                     mut['disk'])
00155 
00156         # Set up parameters for test specification filter function (we need to set toolchains per target here)
00157         use_default_toolchain = 'default' in opts.toolchains_filter if opts.toolchains_filter is not None else True
00158         use_supported_toolchains = 'all' in opts.toolchains_filter if opts.toolchains_filter is not None else False
00159         toolchain_filter = opts.toolchains_filter
00160         platform_name_filter = opts.general_filter_regex if opts.general_filter_regex is not None else opts.general_filter_regex
00161         # Test specification with information about each target and associated toolchain
00162         test_spec = get_autodetected_TEST_SPEC(MUTs.values(),
00163                                                use_default_toolchain=use_default_toolchain,
00164                                                use_supported_toolchains=use_supported_toolchains,
00165                                                toolchain_filter=toolchain_filter,
00166                                                platform_name_filter=platform_name_filter)
00167     else:
00168         # Open file with test specification
00169         # test_spec_filename tells script which targets and their toolchain(s)
00170         # should be covered by the test scenario
00171         opts.auto_detect = False
00172         test_spec = get_json_data_from_file(opts.test_spec_filename) if opts.test_spec_filename else None
00173         if test_spec is None:
00174             if not opts.test_spec_filename:
00175                 parser.print_help()
00176             exit(-1)
00177 
00178         # Get extra MUTs if applicable
00179         MUTs = get_json_data_from_file(opts.muts_spec_filename) if opts.muts_spec_filename else None
00180 
00181         if MUTs is None:
00182             if not opts.muts_spec_filename:
00183                 parser.print_help()
00184             exit(-1)
00185 
00186     if opts.verbose_test_configuration_only:
00187         print "MUTs configuration in %s:" % ('auto-detected' if opts.auto_detect else opts.muts_spec_filename)
00188         if MUTs:
00189             print print_muts_configuration_from_json(MUTs, platform_filter=opts.general_filter_regex)
00190         print
00191         print "Test specification in %s:" % ('auto-detected' if opts.auto_detect else opts.test_spec_filename)
00192         if test_spec:
00193             print print_test_configuration_from_json(test_spec)
00194         exit(0)
00195 
00196     if get_module_avail('mbed_lstools'):
00197         if opts.operability_checks:
00198             # Check if test scope is valid and run tests
00199             test_scope = get_available_oper_test_scopes()
00200             if opts.operability_checks in test_scope:
00201                 tests = IOperTestRunner(scope=opts.operability_checks)
00202                 test_results = tests.run()
00203 
00204                 # Export results in form of JUnit XML report to separate file
00205                 if opts.report_junit_file_name:
00206                     report_exporter = ReportExporter(ResultExporterType.JUNIT_OPER)
00207                     report_exporter.report_to_file(test_results, opts.report_junit_file_name)
00208             else:
00209                 print "Unknown interoperability test scope name: '%s'" % (opts.operability_checks)
00210                 print "Available test scopes: %s" % (','.join(["'%s'" % n for n in test_scope]))
00211 
00212             exit(0)
00213 
00214     # Verbose test specification and MUTs configuration
00215     if MUTs and opts.verbose:
00216         print print_muts_configuration_from_json(MUTs)
00217     if test_spec and opts.verbose:
00218         print print_test_configuration_from_json(test_spec)
00219 
00220     if opts.only_build_tests:
00221         # We are skipping testing phase, and suppress summary
00222         opts.suppress_summary = True
00223 
00224     single_test = SingleTestRunner(_global_loops_count=opts.test_global_loops_value,
00225                                    _test_loops_list=opts.test_loops_list,
00226                                    _muts=MUTs,
00227                                    _clean=opts.clean,
00228                                    _parser=parser,
00229                                    _opts=opts,
00230                                    _opts_db_url=opts.db_url,
00231                                    _opts_log_file_name=opts.log_file_name,
00232                                    _opts_report_html_file_name=opts.report_html_file_name,
00233                                    _opts_report_junit_file_name=opts.report_junit_file_name,
00234                                    _opts_report_build_file_name=opts.report_build_file_name,
00235                                    _opts_report_text_file_name=opts.report_text_file_name,
00236                                    _test_spec=test_spec,
00237                                    _opts_goanna_for_mbed_sdk=opts.goanna_for_mbed_sdk,
00238                                    _opts_goanna_for_tests=opts.goanna_for_tests,
00239                                    _opts_shuffle_test_order=opts.shuffle_test_order,
00240                                    _opts_shuffle_test_seed=opts.shuffle_test_seed,
00241                                    _opts_test_by_names=opts.test_by_names,
00242                                    _opts_peripheral_by_names=opts.peripheral_by_names,
00243                                    _opts_test_only_peripheral=opts.test_only_peripheral,
00244                                    _opts_test_only_common=opts.test_only_common,
00245                                    _opts_verbose_skipped_tests=opts.verbose_skipped_tests,
00246                                    _opts_verbose_test_result_only=opts.verbose_test_result_only,
00247                                    _opts_verbose=opts.verbose,
00248                                    _opts_firmware_global_name=opts.firmware_global_name,
00249                                    _opts_only_build_tests=opts.only_build_tests,
00250                                    _opts_parallel_test_exec=opts.parallel_test_exec,
00251                                    _opts_suppress_summary=opts.suppress_summary,
00252                                    _opts_test_x_toolchain_summary=opts.test_x_toolchain_summary,
00253                                    _opts_copy_method=opts.copy_method,
00254                                    _opts_mut_reset_type=opts.mut_reset_type,
00255                                    _opts_jobs=opts.jobs,
00256                                    _opts_waterfall_test=opts.waterfall_test,
00257                                    _opts_consolidate_waterfall_test=opts.consolidate_waterfall_test,
00258                                    _opts_extend_test_timeout=opts.extend_test_timeout,
00259                                    _opts_auto_detect=opts.auto_detect)
00260 
00261     # Runs test suite in CLI mode
00262     if (singletest_in_cli_mode(single_test)):
00263         exit(0)
00264     else:
00265         exit(-1)