Daiki Kato / mbed-os-lychee

mbed-os for GR-LYCHEE

Dependents:   mbed-os-example-blinky-gr-lychee GR-Boads_Camera_sample GR-Boards_Audio_Recoder GR-Boads_Camera_DisplayApp ... more

tools/test_exporters.py@0:f782d9c66c49, 2018-02-02 (annotated)

Committer:
dkato
Date:
Fri Feb 02 05:42:23 2018 +0000
Revision:
0:f782d9c66c49
mbed-os for GR-LYCHEE

Who changed what in which revision?

UserRevisionLine numberNew contents of line
dkato 0:f782d9c66c49 1 """
dkato 0:f782d9c66c49 2 mbed SDK
dkato 0:f782d9c66c49 3 Copyright (c) 2011-2014 ARM Limited
dkato 0:f782d9c66c49 4
dkato 0:f782d9c66c49 5 Licensed under the Apache License, Version 2.0 (the "License");
dkato 0:f782d9c66c49 6 you may not use this file except in compliance with the License.
dkato 0:f782d9c66c49 7 You may obtain a copy of the License at
dkato 0:f782d9c66c49 8
dkato 0:f782d9c66c49 9 http://www.apache.org/licenses/LICENSE-2.0
dkato 0:f782d9c66c49 10
dkato 0:f782d9c66c49 11 Unless required by applicable law or agreed to in writing, software
dkato 0:f782d9c66c49 12 distributed under the License is distributed on an "AS IS" BASIS,
dkato 0:f782d9c66c49 13 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
dkato 0:f782d9c66c49 14 See the License for the specific language governing permissions and
dkato 0:f782d9c66c49 15 limitations under the License.
dkato 0:f782d9c66c49 16
dkato 0:f782d9c66c49 17 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com>
dkato 0:f782d9c66c49 18 """
dkato 0:f782d9c66c49 19
dkato 0:f782d9c66c49 20 from tools.utils import construct_enum, mkdir
dkato 0:f782d9c66c49 21 from prettytable import PrettyTable
dkato 0:f782d9c66c49 22 import os
dkato 0:f782d9c66c49 23
dkato 0:f782d9c66c49 24 ResultExporterType = construct_enum(HTML='Html_Exporter',
dkato 0:f782d9c66c49 25 JUNIT='JUnit_Exporter',
dkato 0:f782d9c66c49 26 JUNIT_OPER='JUnit_Exporter_Interoperability',
dkato 0:f782d9c66c49 27 BUILD='Build_Exporter',
dkato 0:f782d9c66c49 28 TEXT='Text_Exporter',
dkato 0:f782d9c66c49 29 PRINT='Print_Exporter')
dkato 0:f782d9c66c49 30
dkato 0:f782d9c66c49 31
dkato 0:f782d9c66c49 32 class ReportExporter():
dkato 0:f782d9c66c49 33 """ Class exports extended test result Python data structure to
dkato 0:f782d9c66c49 34 different formats like HTML, JUnit XML.
dkato 0:f782d9c66c49 35
dkato 0:f782d9c66c49 36 Parameter 'test_result_ext' format:
dkato 0:f782d9c66c49 37
dkato 0:f782d9c66c49 38 u'uARM': { u'LPC1768': { 'MBED_2': { 0: { 'copy_method': 'shutils.copy()',
dkato 0:f782d9c66c49 39 'duration': 20,
dkato 0:f782d9c66c49 40 'elapsed_time': 1.7929999828338623,
dkato 0:f782d9c66c49 41 'output': 'Host test instrumentation on ...\r\n',
dkato 0:f782d9c66c49 42 'result': 'OK',
dkato 0:f782d9c66c49 43 'target_name': u'LPC1768',
dkato 0:f782d9c66c49 44 'description': 'stdio',
dkato 0:f782d9c66c49 45 'id': u'MBED_2',
dkato 0:f782d9c66c49 46 'toolchain_name': u'uARM'}},
dkato 0:f782d9c66c49 47 """
dkato 0:f782d9c66c49 48 CSS_STYLE = """<style>
dkato 0:f782d9c66c49 49 .name{
dkato 0:f782d9c66c49 50 border: 1px solid;
dkato 0:f782d9c66c49 51 border-radius: 25px;
dkato 0:f782d9c66c49 52 width: 100px;
dkato 0:f782d9c66c49 53 }
dkato 0:f782d9c66c49 54 .tooltip{
dkato 0:f782d9c66c49 55 position:absolute;
dkato 0:f782d9c66c49 56 background-color: #F5DA81;
dkato 0:f782d9c66c49 57 display:none;
dkato 0:f782d9c66c49 58 }
dkato 0:f782d9c66c49 59 </style>
dkato 0:f782d9c66c49 60 """
dkato 0:f782d9c66c49 61
dkato 0:f782d9c66c49 62 JAVASCRIPT = """
dkato 0:f782d9c66c49 63 <script type="text/javascript">
dkato 0:f782d9c66c49 64 function show (elem) {
dkato 0:f782d9c66c49 65 elem.style.display = "block";
dkato 0:f782d9c66c49 66 }
dkato 0:f782d9c66c49 67 function hide (elem) {
dkato 0:f782d9c66c49 68 elem.style.display = "";
dkato 0:f782d9c66c49 69 }
dkato 0:f782d9c66c49 70 </script>
dkato 0:f782d9c66c49 71 """
dkato 0:f782d9c66c49 72
dkato 0:f782d9c66c49 73 def __init__(self, result_exporter_type, package="test"):
dkato 0:f782d9c66c49 74 self.result_exporter_type = result_exporter_type
dkato 0:f782d9c66c49 75 self.package = package
dkato 0:f782d9c66c49 76
dkato 0:f782d9c66c49 77 def report(self, test_summary_ext, test_suite_properties=None,
dkato 0:f782d9c66c49 78 print_log_for_failures=True):
dkato 0:f782d9c66c49 79 """ Invokes report depending on exporter_type set in constructor
dkato 0:f782d9c66c49 80 """
dkato 0:f782d9c66c49 81 if self.result_exporter_type == ResultExporterType.HTML:
dkato 0:f782d9c66c49 82 # HTML exporter
dkato 0:f782d9c66c49 83 return self.exporter_html(test_summary_ext, test_suite_properties)
dkato 0:f782d9c66c49 84 elif self.result_exporter_type == ResultExporterType.JUNIT:
dkato 0:f782d9c66c49 85 # JUNIT exporter for results from test suite
dkato 0:f782d9c66c49 86 return self.exporter_junit(test_summary_ext, test_suite_properties)
dkato 0:f782d9c66c49 87 elif self.result_exporter_type == ResultExporterType.JUNIT_OPER:
dkato 0:f782d9c66c49 88 # JUNIT exporter for interoperability test
dkato 0:f782d9c66c49 89 return self.exporter_junit_ioper(test_summary_ext, test_suite_properties)
dkato 0:f782d9c66c49 90 elif self.result_exporter_type == ResultExporterType.PRINT:
dkato 0:f782d9c66c49 91 # JUNIT exporter for interoperability test
dkato 0:f782d9c66c49 92 return self.exporter_print(test_summary_ext, print_log_for_failures=print_log_for_failures)
dkato 0:f782d9c66c49 93 elif self.result_exporter_type == ResultExporterType.TEXT:
dkato 0:f782d9c66c49 94 return self.exporter_text(test_summary_ext)
dkato 0:f782d9c66c49 95 return None
dkato 0:f782d9c66c49 96
dkato 0:f782d9c66c49 97 def report_to_file(self, test_summary_ext, file_name, test_suite_properties=None):
dkato 0:f782d9c66c49 98 """ Stores report to specified file
dkato 0:f782d9c66c49 99 """
dkato 0:f782d9c66c49 100 report = self.report(test_summary_ext, test_suite_properties=test_suite_properties)
dkato 0:f782d9c66c49 101 self.write_to_file(report, file_name)
dkato 0:f782d9c66c49 102
dkato 0:f782d9c66c49 103 def write_to_file(self, report, file_name):
dkato 0:f782d9c66c49 104 if report is not None:
dkato 0:f782d9c66c49 105 dirname = os.path.dirname(file_name)
dkato 0:f782d9c66c49 106 if dirname:
dkato 0:f782d9c66c49 107 mkdir(dirname)
dkato 0:f782d9c66c49 108 with open(file_name, 'w') as f:
dkato 0:f782d9c66c49 109 f.write(report)
dkato 0:f782d9c66c49 110
dkato 0:f782d9c66c49 111 def get_tooltip_name(self, toolchain, target, test_id, loop_no):
dkato 0:f782d9c66c49 112 """ Generate simple unique tool-tip name which can be used.
dkato 0:f782d9c66c49 113 For example as HTML <div> section id attribute.
dkato 0:f782d9c66c49 114 """
dkato 0:f782d9c66c49 115 return "target_test_%s_%s_%s_%s"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no)
dkato 0:f782d9c66c49 116
dkato 0:f782d9c66c49 117 def get_result_div_sections(self, test, test_no):
dkato 0:f782d9c66c49 118 """ Generates separate <DIV> sections which contains test results output.
dkato 0:f782d9c66c49 119 """
dkato 0:f782d9c66c49 120
dkato 0:f782d9c66c49 121 RESULT_COLORS = {'OK': 'LimeGreen',
dkato 0:f782d9c66c49 122 'FAIL': 'Orange',
dkato 0:f782d9c66c49 123 'ERROR': 'LightCoral',
dkato 0:f782d9c66c49 124 'OTHER': 'LightGray',
dkato 0:f782d9c66c49 125 }
dkato 0:f782d9c66c49 126
dkato 0:f782d9c66c49 127 tooltip_name = self.get_tooltip_name(test['toolchain_name'], test['target_name'], test['id'], test_no)
dkato 0:f782d9c66c49 128 background_color = RESULT_COLORS[test['result'] if test['result'] in RESULT_COLORS else 'OTHER']
dkato 0:f782d9c66c49 129 result_div_style = "background-color: %s"% background_color
dkato 0:f782d9c66c49 130
dkato 0:f782d9c66c49 131 result = """<div class="name" style="%s" onmouseover="show(%s)" onmouseout="hide(%s)">
dkato 0:f782d9c66c49 132 <center>%s</center>
dkato 0:f782d9c66c49 133 <div class = "tooltip" id= "%s">
dkato 0:f782d9c66c49 134 <b>%s</b><br />
dkato 0:f782d9c66c49 135 <hr />
dkato 0:f782d9c66c49 136 <b>%s</b> in <b>%.2f sec</b><br />
dkato 0:f782d9c66c49 137 <hr />
dkato 0:f782d9c66c49 138 <small>
dkato 0:f782d9c66c49 139 %s
dkato 0:f782d9c66c49 140 </small>
dkato 0:f782d9c66c49 141 </div>
dkato 0:f782d9c66c49 142 </div>
dkato 0:f782d9c66c49 143 """% (result_div_style,
dkato 0:f782d9c66c49 144 tooltip_name,
dkato 0:f782d9c66c49 145 tooltip_name,
dkato 0:f782d9c66c49 146 test['result'],
dkato 0:f782d9c66c49 147 tooltip_name,
dkato 0:f782d9c66c49 148 test['target_name_unique'],
dkato 0:f782d9c66c49 149 test['description'],
dkato 0:f782d9c66c49 150 test['elapsed_time'],
dkato 0:f782d9c66c49 151 test['output'].replace('\n', '<br />'))
dkato 0:f782d9c66c49 152 return result
dkato 0:f782d9c66c49 153
dkato 0:f782d9c66c49 154 def get_result_tree(self, test_results):
dkato 0:f782d9c66c49 155 """ If test was run in a loop (we got few results from the same test)
dkato 0:f782d9c66c49 156 we will show it in a column to see all results.
dkato 0:f782d9c66c49 157 This function produces HTML table with corresponding results.
dkato 0:f782d9c66c49 158 """
dkato 0:f782d9c66c49 159 result = ''
dkato 0:f782d9c66c49 160 for i, test_result in enumerate(test_results):
dkato 0:f782d9c66c49 161 result += '<table>'
dkato 0:f782d9c66c49 162 test_ids = sorted(test_result.keys())
dkato 0:f782d9c66c49 163 for test_no in test_ids:
dkato 0:f782d9c66c49 164 test = test_result[test_no]
dkato 0:f782d9c66c49 165 result += """<tr>
dkato 0:f782d9c66c49 166 <td valign="top">%s</td>
dkato 0:f782d9c66c49 167 </tr>"""% self.get_result_div_sections(test, "%d_%d" % (test_no, i))
dkato 0:f782d9c66c49 168 result += '</table>'
dkato 0:f782d9c66c49 169 return result
dkato 0:f782d9c66c49 170
dkato 0:f782d9c66c49 171 def get_all_unique_test_ids(self, test_result_ext):
dkato 0:f782d9c66c49 172 """ Gets all unique test ids from all ran tests.
dkato 0:f782d9c66c49 173 We need this to create complete list of all test ran.
dkato 0:f782d9c66c49 174 """
dkato 0:f782d9c66c49 175 result = []
dkato 0:f782d9c66c49 176 targets = test_result_ext.keys()
dkato 0:f782d9c66c49 177 for target in targets:
dkato 0:f782d9c66c49 178 toolchains = test_result_ext[target].keys()
dkato 0:f782d9c66c49 179 for toolchain in toolchains:
dkato 0:f782d9c66c49 180 tests = test_result_ext[target][toolchain].keys()
dkato 0:f782d9c66c49 181 result.extend(tests)
dkato 0:f782d9c66c49 182 return sorted(list(set(result)))
dkato 0:f782d9c66c49 183
dkato 0:f782d9c66c49 184 #
dkato 0:f782d9c66c49 185 # Exporters functions
dkato 0:f782d9c66c49 186 #
dkato 0:f782d9c66c49 187
dkato 0:f782d9c66c49 188 def exporter_html(self, test_result_ext, test_suite_properties=None):
dkato 0:f782d9c66c49 189 """ Export test results in proprietary HTML format.
dkato 0:f782d9c66c49 190 """
dkato 0:f782d9c66c49 191 result = """<html>
dkato 0:f782d9c66c49 192 <head>
dkato 0:f782d9c66c49 193 <title>mbed SDK test suite test result report</title>
dkato 0:f782d9c66c49 194 %s
dkato 0:f782d9c66c49 195 %s
dkato 0:f782d9c66c49 196 </head>
dkato 0:f782d9c66c49 197 <body>
dkato 0:f782d9c66c49 198 """% (self.CSS_STYLE, self.JAVASCRIPT)
dkato 0:f782d9c66c49 199
dkato 0:f782d9c66c49 200 unique_test_ids = self.get_all_unique_test_ids(test_result_ext)
dkato 0:f782d9c66c49 201 targets = sorted(test_result_ext.keys())
dkato 0:f782d9c66c49 202 result += '<table>'
dkato 0:f782d9c66c49 203 for target in targets:
dkato 0:f782d9c66c49 204 toolchains = sorted(test_result_ext[target].keys())
dkato 0:f782d9c66c49 205 for toolchain in toolchains:
dkato 0:f782d9c66c49 206 result += '<tr>'
dkato 0:f782d9c66c49 207 result += '<td></td>'
dkato 0:f782d9c66c49 208 result += '<td></td>'
dkato 0:f782d9c66c49 209
dkato 0:f782d9c66c49 210 tests = sorted(test_result_ext[target][toolchain].keys())
dkato 0:f782d9c66c49 211 for test in unique_test_ids:
dkato 0:f782d9c66c49 212 result += """<td align="center">%s</td>"""% test
dkato 0:f782d9c66c49 213 result += """</tr>
dkato 0:f782d9c66c49 214 <tr>
dkato 0:f782d9c66c49 215 <td valign="center">%s</td>
dkato 0:f782d9c66c49 216 <td valign="center"><b>%s</b></td>
dkato 0:f782d9c66c49 217 """% (toolchain, target)
dkato 0:f782d9c66c49 218
dkato 0:f782d9c66c49 219 for test in unique_test_ids:
dkato 0:f782d9c66c49 220 test_result = self.get_result_tree(test_result_ext[target][toolchain][test]) if test in tests else ''
dkato 0:f782d9c66c49 221 result += '<td>%s</td>'% (test_result)
dkato 0:f782d9c66c49 222
dkato 0:f782d9c66c49 223 result += '</tr>'
dkato 0:f782d9c66c49 224 result += '</table>'
dkato 0:f782d9c66c49 225 result += '</body></html>'
dkato 0:f782d9c66c49 226 return result
dkato 0:f782d9c66c49 227
dkato 0:f782d9c66c49 228 def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None):
dkato 0:f782d9c66c49 229 from junit_xml import TestSuite, TestCase
dkato 0:f782d9c66c49 230 test_suites = []
dkato 0:f782d9c66c49 231 test_cases = []
dkato 0:f782d9c66c49 232
dkato 0:f782d9c66c49 233 for platform in sorted(test_result_ext.keys()):
dkato 0:f782d9c66c49 234 # {platform : ['Platform', 'Result', 'Scope', 'Description'])
dkato 0:f782d9c66c49 235 test_cases = []
dkato 0:f782d9c66c49 236 for tr_result in test_result_ext[platform]:
dkato 0:f782d9c66c49 237 result, name, scope, description = tr_result
dkato 0:f782d9c66c49 238
dkato 0:f782d9c66c49 239 classname = 'test.ioper.%s.%s.%s' % (platform, name, scope)
dkato 0:f782d9c66c49 240 elapsed_sec = 0
dkato 0:f782d9c66c49 241 _stdout = description
dkato 0:f782d9c66c49 242 _stderr = ''
dkato 0:f782d9c66c49 243 # Test case
dkato 0:f782d9c66c49 244 tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
dkato 0:f782d9c66c49 245 # Test case extra failure / error info
dkato 0:f782d9c66c49 246 if result == 'FAIL':
dkato 0:f782d9c66c49 247 tc.add_failure_info(description, _stdout)
dkato 0:f782d9c66c49 248 elif result == 'ERROR':
dkato 0:f782d9c66c49 249 tc.add_error_info(description, _stdout)
dkato 0:f782d9c66c49 250 elif result == 'SKIP' or result == 'NOT_SUPPORTED':
dkato 0:f782d9c66c49 251 tc.add_skipped_info(description, _stdout)
dkato 0:f782d9c66c49 252
dkato 0:f782d9c66c49 253 test_cases.append(tc)
dkato 0:f782d9c66c49 254 ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases)
dkato 0:f782d9c66c49 255 test_suites.append(ts)
dkato 0:f782d9c66c49 256 return TestSuite.to_xml_string(test_suites)
dkato 0:f782d9c66c49 257
dkato 0:f782d9c66c49 258 def exporter_junit(self, test_result_ext, test_suite_properties=None):
dkato 0:f782d9c66c49 259 """ Export test results in JUnit XML compliant format
dkato 0:f782d9c66c49 260 """
dkato 0:f782d9c66c49 261 from junit_xml import TestSuite, TestCase
dkato 0:f782d9c66c49 262 test_suites = []
dkato 0:f782d9c66c49 263 test_cases = []
dkato 0:f782d9c66c49 264
dkato 0:f782d9c66c49 265 targets = sorted(test_result_ext.keys())
dkato 0:f782d9c66c49 266 for target in targets:
dkato 0:f782d9c66c49 267 toolchains = sorted(test_result_ext[target].keys())
dkato 0:f782d9c66c49 268 for toolchain in toolchains:
dkato 0:f782d9c66c49 269 test_cases = []
dkato 0:f782d9c66c49 270 tests = sorted(test_result_ext[target][toolchain].keys())
dkato 0:f782d9c66c49 271 for test in tests:
dkato 0:f782d9c66c49 272 test_results = test_result_ext[target][toolchain][test]
dkato 0:f782d9c66c49 273 for test_res in test_results:
dkato 0:f782d9c66c49 274 test_ids = sorted(test_res.keys())
dkato 0:f782d9c66c49 275 for test_no in test_ids:
dkato 0:f782d9c66c49 276 test_result = test_res[test_no]
dkato 0:f782d9c66c49 277 name = test_result['description']
dkato 0:f782d9c66c49 278 classname = '%s.%s.%s.%s'% (self.package, target, toolchain, test_result['id'])
dkato 0:f782d9c66c49 279 elapsed_sec = test_result['elapsed_time']
dkato 0:f782d9c66c49 280 _stdout = test_result['output']
dkato 0:f782d9c66c49 281
dkato 0:f782d9c66c49 282 if 'target_name_unique' in test_result:
dkato 0:f782d9c66c49 283 _stderr = test_result['target_name_unique']
dkato 0:f782d9c66c49 284 else:
dkato 0:f782d9c66c49 285 _stderr = test_result['target_name']
dkato 0:f782d9c66c49 286
dkato 0:f782d9c66c49 287 # Test case
dkato 0:f782d9c66c49 288 tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr)
dkato 0:f782d9c66c49 289
dkato 0:f782d9c66c49 290 # Test case extra failure / error info
dkato 0:f782d9c66c49 291 message = test_result['result']
dkato 0:f782d9c66c49 292 if test_result['result'] == 'FAIL':
dkato 0:f782d9c66c49 293 tc.add_failure_info(message, _stdout)
dkato 0:f782d9c66c49 294 elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED':
dkato 0:f782d9c66c49 295 tc.add_skipped_info(message, _stdout)
dkato 0:f782d9c66c49 296 elif test_result['result'] != 'OK':
dkato 0:f782d9c66c49 297 tc.add_error_info(message, _stdout)
dkato 0:f782d9c66c49 298
dkato 0:f782d9c66c49 299 test_cases.append(tc)
dkato 0:f782d9c66c49 300
dkato 0:f782d9c66c49 301 ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain])
dkato 0:f782d9c66c49 302 test_suites.append(ts)
dkato 0:f782d9c66c49 303 return TestSuite.to_xml_string(test_suites)
dkato 0:f782d9c66c49 304
dkato 0:f782d9c66c49 305 def exporter_print_helper(self, array, print_log=False):
dkato 0:f782d9c66c49 306 for item in array:
dkato 0:f782d9c66c49 307 print " * %s::%s::%s" % (item["target_name"], item["toolchain_name"], item["id"])
dkato 0:f782d9c66c49 308 if print_log:
dkato 0:f782d9c66c49 309 log_lines = item["output"].split("\n")
dkato 0:f782d9c66c49 310 for log_line in log_lines:
dkato 0:f782d9c66c49 311 print " %s" % log_line
dkato 0:f782d9c66c49 312
dkato 0:f782d9c66c49 313 def exporter_print(self, test_result_ext, print_log_for_failures=False):
dkato 0:f782d9c66c49 314 """ Export test results in print format.
dkato 0:f782d9c66c49 315 """
dkato 0:f782d9c66c49 316 failures = []
dkato 0:f782d9c66c49 317 skips = []
dkato 0:f782d9c66c49 318 successes = []
dkato 0:f782d9c66c49 319
dkato 0:f782d9c66c49 320 unique_test_ids = self.get_all_unique_test_ids(test_result_ext)
dkato 0:f782d9c66c49 321 targets = sorted(test_result_ext.keys())
dkato 0:f782d9c66c49 322
dkato 0:f782d9c66c49 323 for target in targets:
dkato 0:f782d9c66c49 324 toolchains = sorted(test_result_ext[target].keys())
dkato 0:f782d9c66c49 325 for toolchain in toolchains:
dkato 0:f782d9c66c49 326 tests = sorted(test_result_ext[target][toolchain].keys())
dkato 0:f782d9c66c49 327 for test in tests:
dkato 0:f782d9c66c49 328 test_runs = test_result_ext[target][toolchain][test]
dkato 0:f782d9c66c49 329 for test_runner in test_runs:
dkato 0:f782d9c66c49 330 #test_run = test_result_ext[target][toolchain][test][test_run_number][0]
dkato 0:f782d9c66c49 331 test_run = test_runner[0]
dkato 0:f782d9c66c49 332
dkato 0:f782d9c66c49 333 if "result" in test_run:
dkato 0:f782d9c66c49 334 if test_run["result"] == "FAIL":
dkato 0:f782d9c66c49 335 failures.append(test_run)
dkato 0:f782d9c66c49 336 elif test_run["result"] == "SKIP" or test_run["result"] == "NOT_SUPPORTED":
dkato 0:f782d9c66c49 337 skips.append(test_run)
dkato 0:f782d9c66c49 338 elif test_run["result"] == "OK":
dkato 0:f782d9c66c49 339 successes.append(test_run)
dkato 0:f782d9c66c49 340 else:
dkato 0:f782d9c66c49 341 raise Exception("Unhandled result type: %s" % (test_run["result"]))
dkato 0:f782d9c66c49 342 else:
dkato 0:f782d9c66c49 343 raise Exception("'test_run' did not have a 'result' value")
dkato 0:f782d9c66c49 344
dkato 0:f782d9c66c49 345 if successes:
dkato 0:f782d9c66c49 346 print "\n\nBuild successes:"
dkato 0:f782d9c66c49 347 self.exporter_print_helper(successes)
dkato 0:f782d9c66c49 348
dkato 0:f782d9c66c49 349 if skips:
dkato 0:f782d9c66c49 350 print "\n\nBuild skips:"
dkato 0:f782d9c66c49 351 self.exporter_print_helper(skips)
dkato 0:f782d9c66c49 352
dkato 0:f782d9c66c49 353 if failures:
dkato 0:f782d9c66c49 354 print "\n\nBuild failures:"
dkato 0:f782d9c66c49 355 self.exporter_print_helper(failures, print_log=print_log_for_failures)
dkato 0:f782d9c66c49 356 return False
dkato 0:f782d9c66c49 357 else:
dkato 0:f782d9c66c49 358 return True
dkato 0:f782d9c66c49 359
dkato 0:f782d9c66c49 360 def exporter_text(self, test_result_ext):
dkato 0:f782d9c66c49 361 """ Prints well-formed summary with results (SQL table like)
dkato 0:f782d9c66c49 362 table shows target x test results matrix across
dkato 0:f782d9c66c49 363 """
dkato 0:f782d9c66c49 364 success_code = 0 # Success code that can be leter returned to
dkato 0:f782d9c66c49 365 # Pretty table package is used to print results
dkato 0:f782d9c66c49 366 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description",
dkato 0:f782d9c66c49 367 "Elapsed Time", "Timeout"])
dkato 0:f782d9c66c49 368 pt.align["Result"] = "l" # Left align
dkato 0:f782d9c66c49 369 pt.align["Target"] = "l" # Left align
dkato 0:f782d9c66c49 370 pt.align["Toolchain"] = "l" # Left align
dkato 0:f782d9c66c49 371 pt.align["Test ID"] = "l" # Left align
dkato 0:f782d9c66c49 372 pt.align["Test Description"] = "l" # Left align
dkato 0:f782d9c66c49 373 pt.padding_width = 1 # One space between column edges and contents (default)
dkato 0:f782d9c66c49 374
dkato 0:f782d9c66c49 375 result_dict = {"OK" : 0,
dkato 0:f782d9c66c49 376 "FAIL" : 0,
dkato 0:f782d9c66c49 377 "ERROR" : 0,
dkato 0:f782d9c66c49 378 "UNDEF" : 0,
dkato 0:f782d9c66c49 379 "IOERR_COPY" : 0,
dkato 0:f782d9c66c49 380 "IOERR_DISK" : 0,
dkato 0:f782d9c66c49 381 "IOERR_SERIAL" : 0,
dkato 0:f782d9c66c49 382 "TIMEOUT" : 0,
dkato 0:f782d9c66c49 383 "NO_IMAGE" : 0,
dkato 0:f782d9c66c49 384 "MBED_ASSERT" : 0,
dkato 0:f782d9c66c49 385 "BUILD_FAILED" : 0,
dkato 0:f782d9c66c49 386 "NOT_SUPPORTED" : 0
dkato 0:f782d9c66c49 387 }
dkato 0:f782d9c66c49 388 unique_test_ids = self.get_all_unique_test_ids(test_result_ext)
dkato 0:f782d9c66c49 389 targets = sorted(test_result_ext.keys())
dkato 0:f782d9c66c49 390 for target in targets:
dkato 0:f782d9c66c49 391 toolchains = sorted(test_result_ext[target].keys())
dkato 0:f782d9c66c49 392 for toolchain in toolchains:
dkato 0:f782d9c66c49 393 test_cases = []
dkato 0:f782d9c66c49 394 tests = sorted(test_result_ext[target][toolchain].keys())
dkato 0:f782d9c66c49 395 for test in tests:
dkato 0:f782d9c66c49 396 test_results = test_result_ext[target][toolchain][test]
dkato 0:f782d9c66c49 397 for test_res in test_results:
dkato 0:f782d9c66c49 398 test_ids = sorted(test_res.keys())
dkato 0:f782d9c66c49 399 for test_no in test_ids:
dkato 0:f782d9c66c49 400 test_result = test_res[test_no]
dkato 0:f782d9c66c49 401 result_dict[test_result['result']] += 1
dkato 0:f782d9c66c49 402 pt.add_row([test_result['result'],
dkato 0:f782d9c66c49 403 test_result['target_name'],
dkato 0:f782d9c66c49 404 test_result['toolchain_name'],
dkato 0:f782d9c66c49 405 test_result['id'],
dkato 0:f782d9c66c49 406 test_result['description'],
dkato 0:f782d9c66c49 407 test_result['elapsed_time'],
dkato 0:f782d9c66c49 408 test_result['duration']])
dkato 0:f782d9c66c49 409 result = pt.get_string()
dkato 0:f782d9c66c49 410 result += "\n"
dkato 0:f782d9c66c49 411
dkato 0:f782d9c66c49 412 # Print result count
dkato 0:f782d9c66c49 413 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()])
dkato 0:f782d9c66c49 414 return result