Nicolas Borla
/
BBR_1Ebene
BBR 1 Ebene
mbed-os/tools/test_exporters.py@0:fbdae7e6d805, 2018-05-14 (annotated)
- Committer:
- borlanic
- Date:
- Mon May 14 11:29:06 2018 +0000
- Revision:
- 0:fbdae7e6d805
BBR
Who changed what in which revision?
User | Revision | Line number | New contents of line |
---|---|---|---|
borlanic | 0:fbdae7e6d805 | 1 | """ |
borlanic | 0:fbdae7e6d805 | 2 | mbed SDK |
borlanic | 0:fbdae7e6d805 | 3 | Copyright (c) 2011-2014 ARM Limited |
borlanic | 0:fbdae7e6d805 | 4 | |
borlanic | 0:fbdae7e6d805 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); |
borlanic | 0:fbdae7e6d805 | 6 | you may not use this file except in compliance with the License. |
borlanic | 0:fbdae7e6d805 | 7 | You may obtain a copy of the License at |
borlanic | 0:fbdae7e6d805 | 8 | |
borlanic | 0:fbdae7e6d805 | 9 | http://www.apache.org/licenses/LICENSE-2.0 |
borlanic | 0:fbdae7e6d805 | 10 | |
borlanic | 0:fbdae7e6d805 | 11 | Unless required by applicable law or agreed to in writing, software |
borlanic | 0:fbdae7e6d805 | 12 | distributed under the License is distributed on an "AS IS" BASIS, |
borlanic | 0:fbdae7e6d805 | 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
borlanic | 0:fbdae7e6d805 | 14 | See the License for the specific language governing permissions and |
borlanic | 0:fbdae7e6d805 | 15 | limitations under the License. |
borlanic | 0:fbdae7e6d805 | 16 | |
borlanic | 0:fbdae7e6d805 | 17 | Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com> |
borlanic | 0:fbdae7e6d805 | 18 | """ |
borlanic | 0:fbdae7e6d805 | 19 | |
borlanic | 0:fbdae7e6d805 | 20 | from tools.utils import construct_enum, mkdir |
borlanic | 0:fbdae7e6d805 | 21 | from prettytable import PrettyTable |
borlanic | 0:fbdae7e6d805 | 22 | import os |
borlanic | 0:fbdae7e6d805 | 23 | |
borlanic | 0:fbdae7e6d805 | 24 | ResultExporterType = construct_enum(HTML='Html_Exporter', |
borlanic | 0:fbdae7e6d805 | 25 | JUNIT='JUnit_Exporter', |
borlanic | 0:fbdae7e6d805 | 26 | JUNIT_OPER='JUnit_Exporter_Interoperability', |
borlanic | 0:fbdae7e6d805 | 27 | BUILD='Build_Exporter', |
borlanic | 0:fbdae7e6d805 | 28 | TEXT='Text_Exporter', |
borlanic | 0:fbdae7e6d805 | 29 | PRINT='Print_Exporter') |
borlanic | 0:fbdae7e6d805 | 30 | |
borlanic | 0:fbdae7e6d805 | 31 | |
borlanic | 0:fbdae7e6d805 | 32 | class ReportExporter(): |
borlanic | 0:fbdae7e6d805 | 33 | """ Class exports extended test result Python data structure to |
borlanic | 0:fbdae7e6d805 | 34 | different formats like HTML, JUnit XML. |
borlanic | 0:fbdae7e6d805 | 35 | |
borlanic | 0:fbdae7e6d805 | 36 | Parameter 'test_result_ext' format: |
borlanic | 0:fbdae7e6d805 | 37 | |
borlanic | 0:fbdae7e6d805 | 38 | u'uARM': { u'LPC1768': { 'MBED_2': { 0: { 'copy_method': 'shutils.copy()', |
borlanic | 0:fbdae7e6d805 | 39 | 'duration': 20, |
borlanic | 0:fbdae7e6d805 | 40 | 'elapsed_time': 1.7929999828338623, |
borlanic | 0:fbdae7e6d805 | 41 | 'output': 'Host test instrumentation on ...\r\n', |
borlanic | 0:fbdae7e6d805 | 42 | 'result': 'OK', |
borlanic | 0:fbdae7e6d805 | 43 | 'target_name': u'LPC1768', |
borlanic | 0:fbdae7e6d805 | 44 | 'description': 'stdio', |
borlanic | 0:fbdae7e6d805 | 45 | 'id': u'MBED_2', |
borlanic | 0:fbdae7e6d805 | 46 | 'toolchain_name': u'uARM'}}, |
borlanic | 0:fbdae7e6d805 | 47 | """ |
borlanic | 0:fbdae7e6d805 | 48 | CSS_STYLE = """<style> |
borlanic | 0:fbdae7e6d805 | 49 | .name{ |
borlanic | 0:fbdae7e6d805 | 50 | border: 1px solid; |
borlanic | 0:fbdae7e6d805 | 51 | border-radius: 25px; |
borlanic | 0:fbdae7e6d805 | 52 | width: 100px; |
borlanic | 0:fbdae7e6d805 | 53 | } |
borlanic | 0:fbdae7e6d805 | 54 | .tooltip{ |
borlanic | 0:fbdae7e6d805 | 55 | position:absolute; |
borlanic | 0:fbdae7e6d805 | 56 | background-color: #F5DA81; |
borlanic | 0:fbdae7e6d805 | 57 | display:none; |
borlanic | 0:fbdae7e6d805 | 58 | } |
borlanic | 0:fbdae7e6d805 | 59 | </style> |
borlanic | 0:fbdae7e6d805 | 60 | """ |
borlanic | 0:fbdae7e6d805 | 61 | |
borlanic | 0:fbdae7e6d805 | 62 | JAVASCRIPT = """ |
borlanic | 0:fbdae7e6d805 | 63 | <script type="text/javascript"> |
borlanic | 0:fbdae7e6d805 | 64 | function show (elem) { |
borlanic | 0:fbdae7e6d805 | 65 | elem.style.display = "block"; |
borlanic | 0:fbdae7e6d805 | 66 | } |
borlanic | 0:fbdae7e6d805 | 67 | function hide (elem) { |
borlanic | 0:fbdae7e6d805 | 68 | elem.style.display = ""; |
borlanic | 0:fbdae7e6d805 | 69 | } |
borlanic | 0:fbdae7e6d805 | 70 | </script> |
borlanic | 0:fbdae7e6d805 | 71 | """ |
borlanic | 0:fbdae7e6d805 | 72 | |
borlanic | 0:fbdae7e6d805 | 73 | def __init__(self, result_exporter_type, package="test"): |
borlanic | 0:fbdae7e6d805 | 74 | self.result_exporter_type = result_exporter_type |
borlanic | 0:fbdae7e6d805 | 75 | self.package = package |
borlanic | 0:fbdae7e6d805 | 76 | |
borlanic | 0:fbdae7e6d805 | 77 | def report(self, test_summary_ext, test_suite_properties=None, |
borlanic | 0:fbdae7e6d805 | 78 | print_log_for_failures=True): |
borlanic | 0:fbdae7e6d805 | 79 | """ Invokes report depending on exporter_type set in constructor |
borlanic | 0:fbdae7e6d805 | 80 | """ |
borlanic | 0:fbdae7e6d805 | 81 | if self.result_exporter_type == ResultExporterType.HTML: |
borlanic | 0:fbdae7e6d805 | 82 | # HTML exporter |
borlanic | 0:fbdae7e6d805 | 83 | return self.exporter_html(test_summary_ext, test_suite_properties) |
borlanic | 0:fbdae7e6d805 | 84 | elif self.result_exporter_type == ResultExporterType.JUNIT: |
borlanic | 0:fbdae7e6d805 | 85 | # JUNIT exporter for results from test suite |
borlanic | 0:fbdae7e6d805 | 86 | return self.exporter_junit(test_summary_ext, test_suite_properties) |
borlanic | 0:fbdae7e6d805 | 87 | elif self.result_exporter_type == ResultExporterType.JUNIT_OPER: |
borlanic | 0:fbdae7e6d805 | 88 | # JUNIT exporter for interoperability test |
borlanic | 0:fbdae7e6d805 | 89 | return self.exporter_junit_ioper(test_summary_ext, test_suite_properties) |
borlanic | 0:fbdae7e6d805 | 90 | elif self.result_exporter_type == ResultExporterType.PRINT: |
borlanic | 0:fbdae7e6d805 | 91 | # JUNIT exporter for interoperability test |
borlanic | 0:fbdae7e6d805 | 92 | return self.exporter_print(test_summary_ext, print_log_for_failures=print_log_for_failures) |
borlanic | 0:fbdae7e6d805 | 93 | elif self.result_exporter_type == ResultExporterType.TEXT: |
borlanic | 0:fbdae7e6d805 | 94 | return self.exporter_text(test_summary_ext) |
borlanic | 0:fbdae7e6d805 | 95 | return None |
borlanic | 0:fbdae7e6d805 | 96 | |
borlanic | 0:fbdae7e6d805 | 97 | def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None): |
borlanic | 0:fbdae7e6d805 | 98 | """ Stores report to specified file |
borlanic | 0:fbdae7e6d805 | 99 | """ |
borlanic | 0:fbdae7e6d805 | 100 | report = self.report(test_summary_ext, test_suite_properties=test_suite_properties) |
borlanic | 0:fbdae7e6d805 | 101 | self.write_to_file(report, file_name) |
borlanic | 0:fbdae7e6d805 | 102 | |
borlanic | 0:fbdae7e6d805 | 103 | def write_to_file(self, report, file_name): |
borlanic | 0:fbdae7e6d805 | 104 | if report is not None: |
borlanic | 0:fbdae7e6d805 | 105 | dirname = os.path.dirname(file_name) |
borlanic | 0:fbdae7e6d805 | 106 | if dirname: |
borlanic | 0:fbdae7e6d805 | 107 | mkdir(dirname) |
borlanic | 0:fbdae7e6d805 | 108 | with open(file_name, 'w') as f: |
borlanic | 0:fbdae7e6d805 | 109 | f.write(report) |
borlanic | 0:fbdae7e6d805 | 110 | |
borlanic | 0:fbdae7e6d805 | 111 | def get_tooltip_name(self, toolchain, target, test_id, loop_no): |
borlanic | 0:fbdae7e6d805 | 112 | """ Generate simple unique tool-tip name which can be used. |
borlanic | 0:fbdae7e6d805 | 113 | For example as HTML <div> section id attribute. |
borlanic | 0:fbdae7e6d805 | 114 | """ |
borlanic | 0:fbdae7e6d805 | 115 | return "target_test_%s_%s_%s_%s"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no) |
borlanic | 0:fbdae7e6d805 | 116 | |
borlanic | 0:fbdae7e6d805 | 117 | def get_result_div_sections(self, test, test_no): |
borlanic | 0:fbdae7e6d805 | 118 | """ Generates separate <DIV> sections which contains test results output. |
borlanic | 0:fbdae7e6d805 | 119 | """ |
borlanic | 0:fbdae7e6d805 | 120 | |
borlanic | 0:fbdae7e6d805 | 121 | RESULT_COLORS = {'OK': 'LimeGreen', |
borlanic | 0:fbdae7e6d805 | 122 | 'FAIL': 'Orange', |
borlanic | 0:fbdae7e6d805 | 123 | 'ERROR': 'LightCoral', |
borlanic | 0:fbdae7e6d805 | 124 | 'OTHER': 'LightGray', |
borlanic | 0:fbdae7e6d805 | 125 | } |
borlanic | 0:fbdae7e6d805 | 126 | |
borlanic | 0:fbdae7e6d805 | 127 | tooltip_name = self.get_tooltip_name(test['toolchain_name'], test['target_name'], test['id'], test_no) |
borlanic | 0:fbdae7e6d805 | 128 | background_color = RESULT_COLORS[test['result'] if test['result'] in RESULT_COLORS else 'OTHER'] |
borlanic | 0:fbdae7e6d805 | 129 | result_div_style = "background-color: %s"% background_color |
borlanic | 0:fbdae7e6d805 | 130 | |
borlanic | 0:fbdae7e6d805 | 131 | result = """<div class="name" style="%s" onmouseover="show(%s)" onmouseout="hide(%s)"> |
borlanic | 0:fbdae7e6d805 | 132 | <center>%s</center> |
borlanic | 0:fbdae7e6d805 | 133 | <div class = "tooltip" id= "%s"> |
borlanic | 0:fbdae7e6d805 | 134 | <b>%s</b><br /> |
borlanic | 0:fbdae7e6d805 | 135 | <hr /> |
borlanic | 0:fbdae7e6d805 | 136 | <b>%s</b> in <b>%.2f sec</b><br /> |
borlanic | 0:fbdae7e6d805 | 137 | <hr /> |
borlanic | 0:fbdae7e6d805 | 138 | <small> |
borlanic | 0:fbdae7e6d805 | 139 | %s |
borlanic | 0:fbdae7e6d805 | 140 | </small> |
borlanic | 0:fbdae7e6d805 | 141 | </div> |
borlanic | 0:fbdae7e6d805 | 142 | </div> |
borlanic | 0:fbdae7e6d805 | 143 | """% (result_div_style, |
borlanic | 0:fbdae7e6d805 | 144 | tooltip_name, |
borlanic | 0:fbdae7e6d805 | 145 | tooltip_name, |
borlanic | 0:fbdae7e6d805 | 146 | test['result'], |
borlanic | 0:fbdae7e6d805 | 147 | tooltip_name, |
borlanic | 0:fbdae7e6d805 | 148 | test['target_name_unique'], |
borlanic | 0:fbdae7e6d805 | 149 | test['description'], |
borlanic | 0:fbdae7e6d805 | 150 | test['elapsed_time'], |
borlanic | 0:fbdae7e6d805 | 151 | test['output'].replace('\n', '<br />')) |
borlanic | 0:fbdae7e6d805 | 152 | return result |
borlanic | 0:fbdae7e6d805 | 153 | |
borlanic | 0:fbdae7e6d805 | 154 | def get_result_tree(self, test_results): |
borlanic | 0:fbdae7e6d805 | 155 | """ If test was run in a loop (we got few results from the same test) |
borlanic | 0:fbdae7e6d805 | 156 | we will show it in a column to see all results. |
borlanic | 0:fbdae7e6d805 | 157 | This function produces HTML table with corresponding results. |
borlanic | 0:fbdae7e6d805 | 158 | """ |
borlanic | 0:fbdae7e6d805 | 159 | result = '' |
borlanic | 0:fbdae7e6d805 | 160 | for i, test_result in enumerate(test_results): |
borlanic | 0:fbdae7e6d805 | 161 | result += '<table>' |
borlanic | 0:fbdae7e6d805 | 162 | test_ids = sorted(test_result.keys()) |
borlanic | 0:fbdae7e6d805 | 163 | for test_no in test_ids: |
borlanic | 0:fbdae7e6d805 | 164 | test = test_result[test_no] |
borlanic | 0:fbdae7e6d805 | 165 | result += """<tr> |
borlanic | 0:fbdae7e6d805 | 166 | <td valign="top">%s</td> |
borlanic | 0:fbdae7e6d805 | 167 | </tr>"""% self.get_result_div_sections(test, "%d_%d" % (test_no, i)) |
borlanic | 0:fbdae7e6d805 | 168 | result += '</table>' |
borlanic | 0:fbdae7e6d805 | 169 | return result |
borlanic | 0:fbdae7e6d805 | 170 | |
borlanic | 0:fbdae7e6d805 | 171 | def get_all_unique_test_ids(self, test_result_ext): |
borlanic | 0:fbdae7e6d805 | 172 | """ Gets all unique test ids from all ran tests. |
borlanic | 0:fbdae7e6d805 | 173 | We need this to create complete list of all test ran. |
borlanic | 0:fbdae7e6d805 | 174 | """ |
borlanic | 0:fbdae7e6d805 | 175 | result = [] |
borlanic | 0:fbdae7e6d805 | 176 | targets = test_result_ext.keys() |
borlanic | 0:fbdae7e6d805 | 177 | for target in targets: |
borlanic | 0:fbdae7e6d805 | 178 | toolchains = test_result_ext[target].keys() |
borlanic | 0:fbdae7e6d805 | 179 | for toolchain in toolchains: |
borlanic | 0:fbdae7e6d805 | 180 | tests = test_result_ext[target][toolchain].keys() |
borlanic | 0:fbdae7e6d805 | 181 | result.extend(tests) |
borlanic | 0:fbdae7e6d805 | 182 | return sorted(list(set(result))) |
borlanic | 0:fbdae7e6d805 | 183 | |
borlanic | 0:fbdae7e6d805 | 184 | # |
borlanic | 0:fbdae7e6d805 | 185 | # Exporters functions |
borlanic | 0:fbdae7e6d805 | 186 | # |
borlanic | 0:fbdae7e6d805 | 187 | |
borlanic | 0:fbdae7e6d805 | 188 | def exporter_html(self, test_result_ext, test_suite_properties=None): |
borlanic | 0:fbdae7e6d805 | 189 | """ Export test results in proprietary HTML format. |
borlanic | 0:fbdae7e6d805 | 190 | """ |
borlanic | 0:fbdae7e6d805 | 191 | result = """<html> |
borlanic | 0:fbdae7e6d805 | 192 | <head> |
borlanic | 0:fbdae7e6d805 | 193 | <title>mbed SDK test suite test result report</title> |
borlanic | 0:fbdae7e6d805 | 194 | %s |
borlanic | 0:fbdae7e6d805 | 195 | %s |
borlanic | 0:fbdae7e6d805 | 196 | </head> |
borlanic | 0:fbdae7e6d805 | 197 | <body> |
borlanic | 0:fbdae7e6d805 | 198 | """% (self.CSS_STYLE, self.JAVASCRIPT) |
borlanic | 0:fbdae7e6d805 | 199 | |
borlanic | 0:fbdae7e6d805 | 200 | unique_test_ids = self.get_all_unique_test_ids(test_result_ext) |
borlanic | 0:fbdae7e6d805 | 201 | targets = sorted(test_result_ext.keys()) |
borlanic | 0:fbdae7e6d805 | 202 | result += '<table>' |
borlanic | 0:fbdae7e6d805 | 203 | for target in targets: |
borlanic | 0:fbdae7e6d805 | 204 | toolchains = sorted(test_result_ext[target].keys()) |
borlanic | 0:fbdae7e6d805 | 205 | for toolchain in toolchains: |
borlanic | 0:fbdae7e6d805 | 206 | result += '<tr>' |
borlanic | 0:fbdae7e6d805 | 207 | result += '<td></td>' |
borlanic | 0:fbdae7e6d805 | 208 | result += '<td></td>' |
borlanic | 0:fbdae7e6d805 | 209 | |
borlanic | 0:fbdae7e6d805 | 210 | tests = sorted(test_result_ext[target][toolchain].keys()) |
borlanic | 0:fbdae7e6d805 | 211 | for test in unique_test_ids: |
borlanic | 0:fbdae7e6d805 | 212 | result += """<td align="center">%s</td>"""% test |
borlanic | 0:fbdae7e6d805 | 213 | result += """</tr> |
borlanic | 0:fbdae7e6d805 | 214 | <tr> |
borlanic | 0:fbdae7e6d805 | 215 | <td valign="center">%s</td> |
borlanic | 0:fbdae7e6d805 | 216 | <td valign="center"><b>%s</b></td> |
borlanic | 0:fbdae7e6d805 | 217 | """% (toolchain, target) |
borlanic | 0:fbdae7e6d805 | 218 | |
borlanic | 0:fbdae7e6d805 | 219 | for test in unique_test_ids: |
borlanic | 0:fbdae7e6d805 | 220 | test_result = self.get_result_tree(test_result_ext[target][toolchain][test]) if test in tests else '' |
borlanic | 0:fbdae7e6d805 | 221 | result += '<td>%s</td>'% (test_result) |
borlanic | 0:fbdae7e6d805 | 222 | |
borlanic | 0:fbdae7e6d805 | 223 | result += '</tr>' |
borlanic | 0:fbdae7e6d805 | 224 | result += '</table>' |
borlanic | 0:fbdae7e6d805 | 225 | result += '</body></html>' |
borlanic | 0:fbdae7e6d805 | 226 | return result |
borlanic | 0:fbdae7e6d805 | 227 | |
borlanic | 0:fbdae7e6d805 | 228 | def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): |
borlanic | 0:fbdae7e6d805 | 229 | from junit_xml import TestSuite, TestCase |
borlanic | 0:fbdae7e6d805 | 230 | test_suites = [] |
borlanic | 0:fbdae7e6d805 | 231 | test_cases = [] |
borlanic | 0:fbdae7e6d805 | 232 | |
borlanic | 0:fbdae7e6d805 | 233 | for platform in sorted(test_result_ext.keys()): |
borlanic | 0:fbdae7e6d805 | 234 | # {platform : ['Platform', 'Result', 'Scope', 'Description']) |
borlanic | 0:fbdae7e6d805 | 235 | test_cases = [] |
borlanic | 0:fbdae7e6d805 | 236 | for tr_result in test_result_ext[platform]: |
borlanic | 0:fbdae7e6d805 | 237 | result, name, scope, description = tr_result |
borlanic | 0:fbdae7e6d805 | 238 | |
borlanic | 0:fbdae7e6d805 | 239 | classname = 'test.ioper.%s.%s.%s' % (platform, name, scope) |
borlanic | 0:fbdae7e6d805 | 240 | elapsed_sec = 0 |
borlanic | 0:fbdae7e6d805 | 241 | _stdout = description |
borlanic | 0:fbdae7e6d805 | 242 | _stderr = '' |
borlanic | 0:fbdae7e6d805 | 243 | # Test case |
borlanic | 0:fbdae7e6d805 | 244 | tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) |
borlanic | 0:fbdae7e6d805 | 245 | # Test case extra failure / error info |
borlanic | 0:fbdae7e6d805 | 246 | if result == 'FAIL': |
borlanic | 0:fbdae7e6d805 | 247 | tc.add_failure_info(description, _stdout) |
borlanic | 0:fbdae7e6d805 | 248 | elif result == 'ERROR': |
borlanic | 0:fbdae7e6d805 | 249 | tc.add_error_info(description, _stdout) |
borlanic | 0:fbdae7e6d805 | 250 | elif result == 'SKIP' or result == 'NOT_SUPPORTED': |
borlanic | 0:fbdae7e6d805 | 251 | tc.add_skipped_info(description, _stdout) |
borlanic | 0:fbdae7e6d805 | 252 | |
borlanic | 0:fbdae7e6d805 | 253 | test_cases.append(tc) |
borlanic | 0:fbdae7e6d805 | 254 | ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) |
borlanic | 0:fbdae7e6d805 | 255 | test_suites.append(ts) |
borlanic | 0:fbdae7e6d805 | 256 | return TestSuite.to_xml_string(test_suites) |
borlanic | 0:fbdae7e6d805 | 257 | |
borlanic | 0:fbdae7e6d805 | 258 | def exporter_junit(self, test_result_ext, test_suite_properties=None): |
borlanic | 0:fbdae7e6d805 | 259 | """ Export test results in JUnit XML compliant format |
borlanic | 0:fbdae7e6d805 | 260 | """ |
borlanic | 0:fbdae7e6d805 | 261 | from junit_xml import TestSuite, TestCase |
borlanic | 0:fbdae7e6d805 | 262 | test_suites = [] |
borlanic | 0:fbdae7e6d805 | 263 | test_cases = [] |
borlanic | 0:fbdae7e6d805 | 264 | |
borlanic | 0:fbdae7e6d805 | 265 | targets = sorted(test_result_ext.keys()) |
borlanic | 0:fbdae7e6d805 | 266 | for target in targets: |
borlanic | 0:fbdae7e6d805 | 267 | toolchains = sorted(test_result_ext[target].keys()) |
borlanic | 0:fbdae7e6d805 | 268 | for toolchain in toolchains: |
borlanic | 0:fbdae7e6d805 | 269 | test_cases = [] |
borlanic | 0:fbdae7e6d805 | 270 | tests = sorted(test_result_ext[target][toolchain].keys()) |
borlanic | 0:fbdae7e6d805 | 271 | for test in tests: |
borlanic | 0:fbdae7e6d805 | 272 | test_results = test_result_ext[target][toolchain][test] |
borlanic | 0:fbdae7e6d805 | 273 | for test_res in test_results: |
borlanic | 0:fbdae7e6d805 | 274 | test_ids = sorted(test_res.keys()) |
borlanic | 0:fbdae7e6d805 | 275 | for test_no in test_ids: |
borlanic | 0:fbdae7e6d805 | 276 | test_result = test_res[test_no] |
borlanic | 0:fbdae7e6d805 | 277 | name = test_result['description'] |
borlanic | 0:fbdae7e6d805 | 278 | classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id']) |
borlanic | 0:fbdae7e6d805 | 279 | elapsed_sec = test_result['elapsed_time'] |
borlanic | 0:fbdae7e6d805 | 280 | _stdout = test_result['output'] |
borlanic | 0:fbdae7e6d805 | 281 | |
borlanic | 0:fbdae7e6d805 | 282 | if 'target_name_unique' in test_result: |
borlanic | 0:fbdae7e6d805 | 283 | _stderr = test_result['target_name_unique'] |
borlanic | 0:fbdae7e6d805 | 284 | else: |
borlanic | 0:fbdae7e6d805 | 285 | _stderr = test_result['target_name'] |
borlanic | 0:fbdae7e6d805 | 286 | |
borlanic | 0:fbdae7e6d805 | 287 | # Test case |
borlanic | 0:fbdae7e6d805 | 288 | tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) |
borlanic | 0:fbdae7e6d805 | 289 | |
borlanic | 0:fbdae7e6d805 | 290 | # Test case extra failure / error info |
borlanic | 0:fbdae7e6d805 | 291 | message = test_result['result'] |
borlanic | 0:fbdae7e6d805 | 292 | if test_result['result'] == 'FAIL': |
borlanic | 0:fbdae7e6d805 | 293 | tc.add_failure_info(message, _stdout) |
borlanic | 0:fbdae7e6d805 | 294 | elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED': |
borlanic | 0:fbdae7e6d805 | 295 | tc.add_skipped_info(message, _stdout) |
borlanic | 0:fbdae7e6d805 | 296 | elif test_result['result'] != 'OK': |
borlanic | 0:fbdae7e6d805 | 297 | tc.add_error_info(message, _stdout) |
borlanic | 0:fbdae7e6d805 | 298 | |
borlanic | 0:fbdae7e6d805 | 299 | test_cases.append(tc) |
borlanic | 0:fbdae7e6d805 | 300 | |
borlanic | 0:fbdae7e6d805 | 301 | ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) |
borlanic | 0:fbdae7e6d805 | 302 | test_suites.append(ts) |
borlanic | 0:fbdae7e6d805 | 303 | return TestSuite.to_xml_string(test_suites) |
borlanic | 0:fbdae7e6d805 | 304 | |
borlanic | 0:fbdae7e6d805 | 305 | def exporter_print_helper(self, array, print_log=False): |
borlanic | 0:fbdae7e6d805 | 306 | for item in array: |
borlanic | 0:fbdae7e6d805 | 307 | print(" * %s::%s::%s" % (item["target_name"], |
borlanic | 0:fbdae7e6d805 | 308 | item["toolchain_name"], |
borlanic | 0:fbdae7e6d805 | 309 | item["id"])) |
borlanic | 0:fbdae7e6d805 | 310 | if print_log: |
borlanic | 0:fbdae7e6d805 | 311 | log_lines = item["output"].split("\n") |
borlanic | 0:fbdae7e6d805 | 312 | for log_line in log_lines: |
borlanic | 0:fbdae7e6d805 | 313 | print(" %s" % log_line) |
borlanic | 0:fbdae7e6d805 | 314 | |
borlanic | 0:fbdae7e6d805 | 315 | def exporter_print(self, test_result_ext, print_log_for_failures=False): |
borlanic | 0:fbdae7e6d805 | 316 | """ Export test results in print format. |
borlanic | 0:fbdae7e6d805 | 317 | """ |
borlanic | 0:fbdae7e6d805 | 318 | failures = [] |
borlanic | 0:fbdae7e6d805 | 319 | skips = [] |
borlanic | 0:fbdae7e6d805 | 320 | successes = [] |
borlanic | 0:fbdae7e6d805 | 321 | |
borlanic | 0:fbdae7e6d805 | 322 | unique_test_ids = self.get_all_unique_test_ids(test_result_ext) |
borlanic | 0:fbdae7e6d805 | 323 | targets = sorted(test_result_ext.keys()) |
borlanic | 0:fbdae7e6d805 | 324 | |
borlanic | 0:fbdae7e6d805 | 325 | for target in targets: |
borlanic | 0:fbdae7e6d805 | 326 | toolchains = sorted(test_result_ext[target].keys()) |
borlanic | 0:fbdae7e6d805 | 327 | for toolchain in toolchains: |
borlanic | 0:fbdae7e6d805 | 328 | tests = sorted(test_result_ext[target][toolchain].keys()) |
borlanic | 0:fbdae7e6d805 | 329 | for test in tests: |
borlanic | 0:fbdae7e6d805 | 330 | test_runs = test_result_ext[target][toolchain][test] |
borlanic | 0:fbdae7e6d805 | 331 | for test_runner in test_runs: |
borlanic | 0:fbdae7e6d805 | 332 | #test_run = test_result_ext[target][toolchain][test][test_run_number][0] |
borlanic | 0:fbdae7e6d805 | 333 | test_run = test_runner[0] |
borlanic | 0:fbdae7e6d805 | 334 | |
borlanic | 0:fbdae7e6d805 | 335 | if "result" in test_run: |
borlanic | 0:fbdae7e6d805 | 336 | if test_run["result"] == "FAIL": |
borlanic | 0:fbdae7e6d805 | 337 | failures.append(test_run) |
borlanic | 0:fbdae7e6d805 | 338 | elif test_run["result"] == "SKIP" or test_run["result"] == "NOT_SUPPORTED": |
borlanic | 0:fbdae7e6d805 | 339 | skips.append(test_run) |
borlanic | 0:fbdae7e6d805 | 340 | elif test_run["result"] == "OK": |
borlanic | 0:fbdae7e6d805 | 341 | successes.append(test_run) |
borlanic | 0:fbdae7e6d805 | 342 | else: |
borlanic | 0:fbdae7e6d805 | 343 | raise Exception("Unhandled result type: %s" % (test_run["result"])) |
borlanic | 0:fbdae7e6d805 | 344 | else: |
borlanic | 0:fbdae7e6d805 | 345 | raise Exception("'test_run' did not have a 'result' value") |
borlanic | 0:fbdae7e6d805 | 346 | |
borlanic | 0:fbdae7e6d805 | 347 | if successes: |
borlanic | 0:fbdae7e6d805 | 348 | print("\n\nBuild successes:") |
borlanic | 0:fbdae7e6d805 | 349 | self.exporter_print_helper(successes) |
borlanic | 0:fbdae7e6d805 | 350 | |
borlanic | 0:fbdae7e6d805 | 351 | if skips: |
borlanic | 0:fbdae7e6d805 | 352 | print("\n\nBuild skips:") |
borlanic | 0:fbdae7e6d805 | 353 | self.exporter_print_helper(skips) |
borlanic | 0:fbdae7e6d805 | 354 | |
borlanic | 0:fbdae7e6d805 | 355 | if failures: |
borlanic | 0:fbdae7e6d805 | 356 | print("\n\nBuild failures:") |
borlanic | 0:fbdae7e6d805 | 357 | self.exporter_print_helper(failures, print_log=print_log_for_failures) |
borlanic | 0:fbdae7e6d805 | 358 | return False |
borlanic | 0:fbdae7e6d805 | 359 | else: |
borlanic | 0:fbdae7e6d805 | 360 | return True |
borlanic | 0:fbdae7e6d805 | 361 | |
borlanic | 0:fbdae7e6d805 | 362 | def exporter_text(self, test_result_ext): |
borlanic | 0:fbdae7e6d805 | 363 | """ Prints well-formed summary with results (SQL table like) |
borlanic | 0:fbdae7e6d805 | 364 | table shows target x test results matrix across |
borlanic | 0:fbdae7e6d805 | 365 | """ |
borlanic | 0:fbdae7e6d805 | 366 | success_code = 0 # Success code that can be leter returned to |
borlanic | 0:fbdae7e6d805 | 367 | # Pretty table package is used to print results |
borlanic | 0:fbdae7e6d805 | 368 | pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description", |
borlanic | 0:fbdae7e6d805 | 369 | "Elapsed Time", "Timeout"]) |
borlanic | 0:fbdae7e6d805 | 370 | pt.align["Result"] = "l" # Left align |
borlanic | 0:fbdae7e6d805 | 371 | pt.align["Target"] = "l" # Left align |
borlanic | 0:fbdae7e6d805 | 372 | pt.align["Toolchain"] = "l" # Left align |
borlanic | 0:fbdae7e6d805 | 373 | pt.align["Test ID"] = "l" # Left align |
borlanic | 0:fbdae7e6d805 | 374 | pt.align["Test Description"] = "l" # Left align |
borlanic | 0:fbdae7e6d805 | 375 | pt.padding_width = 1 # One space between column edges and contents (default) |
borlanic | 0:fbdae7e6d805 | 376 | |
borlanic | 0:fbdae7e6d805 | 377 | result_dict = {"OK" : 0, |
borlanic | 0:fbdae7e6d805 | 378 | "FAIL" : 0, |
borlanic | 0:fbdae7e6d805 | 379 | "ERROR" : 0, |
borlanic | 0:fbdae7e6d805 | 380 | "UNDEF" : 0, |
borlanic | 0:fbdae7e6d805 | 381 | "IOERR_COPY" : 0, |
borlanic | 0:fbdae7e6d805 | 382 | "IOERR_DISK" : 0, |
borlanic | 0:fbdae7e6d805 | 383 | "IOERR_SERIAL" : 0, |
borlanic | 0:fbdae7e6d805 | 384 | "TIMEOUT" : 0, |
borlanic | 0:fbdae7e6d805 | 385 | "NO_IMAGE" : 0, |
borlanic | 0:fbdae7e6d805 | 386 | "MBED_ASSERT" : 0, |
borlanic | 0:fbdae7e6d805 | 387 | "BUILD_FAILED" : 0, |
borlanic | 0:fbdae7e6d805 | 388 | "NOT_SUPPORTED" : 0 |
borlanic | 0:fbdae7e6d805 | 389 | } |
borlanic | 0:fbdae7e6d805 | 390 | unique_test_ids = self.get_all_unique_test_ids(test_result_ext) |
borlanic | 0:fbdae7e6d805 | 391 | targets = sorted(test_result_ext.keys()) |
borlanic | 0:fbdae7e6d805 | 392 | for target in targets: |
borlanic | 0:fbdae7e6d805 | 393 | toolchains = sorted(test_result_ext[target].keys()) |
borlanic | 0:fbdae7e6d805 | 394 | for toolchain in toolchains: |
borlanic | 0:fbdae7e6d805 | 395 | test_cases = [] |
borlanic | 0:fbdae7e6d805 | 396 | tests = sorted(test_result_ext[target][toolchain].keys()) |
borlanic | 0:fbdae7e6d805 | 397 | for test in tests: |
borlanic | 0:fbdae7e6d805 | 398 | test_results = test_result_ext[target][toolchain][test] |
borlanic | 0:fbdae7e6d805 | 399 | for test_res in test_results: |
borlanic | 0:fbdae7e6d805 | 400 | test_ids = sorted(test_res.keys()) |
borlanic | 0:fbdae7e6d805 | 401 | for test_no in test_ids: |
borlanic | 0:fbdae7e6d805 | 402 | test_result = test_res[test_no] |
borlanic | 0:fbdae7e6d805 | 403 | result_dict[test_result['result']] += 1 |
borlanic | 0:fbdae7e6d805 | 404 | pt.add_row([test_result['result'], |
borlanic | 0:fbdae7e6d805 | 405 | test_result['target_name'], |
borlanic | 0:fbdae7e6d805 | 406 | test_result['toolchain_name'], |
borlanic | 0:fbdae7e6d805 | 407 | test_result['id'], |
borlanic | 0:fbdae7e6d805 | 408 | test_result['description'], |
borlanic | 0:fbdae7e6d805 | 409 | test_result['elapsed_time'], |
borlanic | 0:fbdae7e6d805 | 410 | test_result['duration']]) |
borlanic | 0:fbdae7e6d805 | 411 | result = pt.get_string() |
borlanic | 0:fbdae7e6d805 | 412 | result += "\n" |
borlanic | 0:fbdae7e6d805 | 413 | |
borlanic | 0:fbdae7e6d805 | 414 | # Print result count |
borlanic | 0:fbdae7e6d805 | 415 | result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.items()]) |
borlanic | 0:fbdae7e6d805 | 416 | return result |