Marco Zecchini
/
Example_RTOS
Rtos API example
Embed:
(wiki syntax)
Show/hide line numbers
test_exporters.py
00001 """ 00002 mbed SDK 00003 Copyright (c) 2011-2014 ARM Limited 00004 00005 Licensed under the Apache License, Version 2.0 (the "License"); 00006 you may not use this file except in compliance with the License. 00007 You may obtain a copy of the License at 00008 00009 http://www.apache.org/licenses/LICENSE-2.0 00010 00011 Unless required by applicable law or agreed to in writing, software 00012 distributed under the License is distributed on an "AS IS" BASIS, 00013 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00014 See the License for the specific language governing permissions and 00015 limitations under the License. 00016 00017 Author: Przemyslaw Wirkus <Przemyslaw.wirkus@arm.com> 00018 """ 00019 00020 from tools.utils import construct_enum, mkdir 00021 from prettytable import PrettyTable 00022 import os 00023 00024 ResultExporterType = construct_enum(HTML='Html_Exporter', 00025 JUNIT='JUnit_Exporter', 00026 JUNIT_OPER='JUnit_Exporter_Interoperability', 00027 BUILD='Build_Exporter', 00028 TEXT='Text_Exporter', 00029 PRINT='Print_Exporter') 00030 00031 00032 class ReportExporter (): 00033 """ Class exports extended test result Python data structure to 00034 different formats like HTML, JUnit XML. 00035 00036 Parameter 'test_result_ext' format: 00037 00038 u'uARM': { u'LPC1768': { 'MBED_2': { 0: { 'copy_method': 'shutils.copy()', 00039 'duration': 20, 00040 'elapsed_time': 1.7929999828338623, 00041 'output': 'Host test instrumentation on ...\r\n', 00042 'result': 'OK', 00043 'target_name': u'LPC1768', 00044 'description': 'stdio', 00045 'id': u'MBED_2', 00046 'toolchain_name': u'uARM'}}, 00047 """ 00048 CSS_STYLE = """<style> 00049 .name{ 00050 border: 1px solid; 00051 border-radius: 25px; 00052 width: 100px; 00053 } 00054 .tooltip{ 00055 position:absolute; 00056 background-color: #F5DA81; 00057 display:none; 00058 } 00059 </style> 00060 """ 00061 00062 JAVASCRIPT = """ 00063 <script type="text/javascript"> 00064 function show (elem) { 00065 elem.style.display = "block"; 00066 } 00067 function hide (elem) { 00068 elem.style.display = ""; 00069 } 00070 </script> 00071 """ 00072 00073 def __init__(self, result_exporter_type, package="test"): 00074 self.result_exporter_type = result_exporter_type 00075 self.package = package 00076 00077 def report (self, test_summary_ext, test_suite_properties=None, 00078 print_log_for_failures=True): 00079 """ Invokes report depending on exporter_type set in constructor 00080 """ 00081 if self.result_exporter_type == ResultExporterType.HTML: 00082 # HTML exporter 00083 return self.exporter_html (test_summary_ext, test_suite_properties) 00084 elif self.result_exporter_type == ResultExporterType.JUNIT: 00085 # JUNIT exporter for results from test suite 00086 return self.exporter_junit (test_summary_ext, test_suite_properties) 00087 elif self.result_exporter_type == ResultExporterType.JUNIT_OPER: 00088 # JUNIT exporter for interoperability test 00089 return self.exporter_junit_ioper (test_summary_ext, test_suite_properties) 00090 elif self.result_exporter_type == ResultExporterType.PRINT: 00091 # JUNIT exporter for interoperability test 00092 return self.exporter_print (test_summary_ext, print_log_for_failures=print_log_for_failures) 00093 elif self.result_exporter_type == ResultExporterType.TEXT: 00094 return self.exporter_text (test_summary_ext) 00095 return None 00096 00097 def report_to_file (self, test_summary_ext, file_name, test_suite_properties=None): 00098 """ Stores report to specified file 00099 """ 00100 report = self.report (test_summary_ext, test_suite_properties=test_suite_properties) 00101 self.write_to_file (report, file_name) 00102 00103 def write_to_file(self, report, file_name): 00104 if report is not None: 00105 dirname = os.path.dirname(file_name) 00106 if dirname: 00107 mkdir(dirname) 00108 with open(file_name, 'w') as f: 00109 f.write(report) 00110 00111 def get_tooltip_name (self, toolchain, target, test_id, loop_no): 00112 """ Generate simple unique tool-tip name which can be used. 00113 For example as HTML <div> section id attribute. 00114 """ 00115 return "target_test_%s_%s_%s_%s"% (toolchain.lower(), target.lower(), test_id.lower(), loop_no) 00116 00117 def get_result_div_sections (self, test, test_no): 00118 """ Generates separate <DIV> sections which contains test results output. 00119 """ 00120 00121 RESULT_COLORS = {'OK': 'LimeGreen', 00122 'FAIL': 'Orange', 00123 'ERROR': 'LightCoral', 00124 'OTHER': 'LightGray', 00125 } 00126 00127 tooltip_name = self.get_tooltip_name (test['toolchain_name'], test['target_name'], test['id'], test_no) 00128 background_color = RESULT_COLORS[test['result'] if test['result'] in RESULT_COLORS else 'OTHER'] 00129 result_div_style = "background-color: %s"% background_color 00130 00131 result = """<div class="name" style="%s" onmouseover="show(%s)" onmouseout="hide(%s)"> 00132 <center>%s</center> 00133 <div class = "tooltip" id= "%s"> 00134 <b>%s</b><br /> 00135 <hr /> 00136 <b>%s</b> in <b>%.2f sec</b><br /> 00137 <hr /> 00138 <small> 00139 %s 00140 </small> 00141 </div> 00142 </div> 00143 """% (result_div_style, 00144 tooltip_name, 00145 tooltip_name, 00146 test['result'], 00147 tooltip_name, 00148 test['target_name_unique'], 00149 test['description'], 00150 test['elapsed_time'], 00151 test['output'].replace('\n', '<br />')) 00152 return result 00153 00154 def get_result_tree (self, test_results): 00155 """ If test was run in a loop (we got few results from the same test) 00156 we will show it in a column to see all results. 00157 This function produces HTML table with corresponding results. 00158 """ 00159 result = '' 00160 for i, test_result in enumerate(test_results): 00161 result += '<table>' 00162 test_ids = sorted(test_result.keys()) 00163 for test_no in test_ids: 00164 test = test_result[test_no] 00165 result += """<tr> 00166 <td valign="top">%s</td> 00167 </tr>"""% self.get_result_div_sections (test, "%d_%d" % (test_no, i)) 00168 result += '</table>' 00169 return result 00170 00171 def get_all_unique_test_ids (self, test_result_ext): 00172 """ Gets all unique test ids from all ran tests. 00173 We need this to create complete list of all test ran. 00174 """ 00175 result = [] 00176 targets = test_result_ext.keys() 00177 for target in targets: 00178 toolchains = test_result_ext[target].keys() 00179 for toolchain in toolchains: 00180 tests = test_result_ext[target][toolchain].keys() 00181 result.extend(tests) 00182 return sorted(list(set(result))) 00183 00184 # 00185 # Exporters functions 00186 # 00187 00188 def exporter_html (self, test_result_ext, test_suite_properties=None): 00189 """ Export test results in proprietary HTML format. 00190 """ 00191 result = """<html> 00192 <head> 00193 <title>mbed SDK test suite test result report</title> 00194 %s 00195 %s 00196 </head> 00197 <body> 00198 """% (self.CSS_STYLE , self.JAVASCRIPT ) 00199 00200 unique_test_ids = self.get_all_unique_test_ids (test_result_ext) 00201 targets = sorted(test_result_ext.keys()) 00202 result += '<table>' 00203 for target in targets: 00204 toolchains = sorted(test_result_ext[target].keys()) 00205 for toolchain in toolchains: 00206 result += '<tr>' 00207 result += '<td></td>' 00208 result += '<td></td>' 00209 00210 tests = sorted(test_result_ext[target][toolchain].keys()) 00211 for test in unique_test_ids: 00212 result += """<td align="center">%s</td>"""% test 00213 result += """</tr> 00214 <tr> 00215 <td valign="center">%s</td> 00216 <td valign="center"><b>%s</b></td> 00217 """% (toolchain, target) 00218 00219 for test in unique_test_ids: 00220 test_result = self.get_result_tree (test_result_ext[target][toolchain][test]) if test in tests else '' 00221 result += '<td>%s</td>'% (test_result) 00222 00223 result += '</tr>' 00224 result += '</table>' 00225 result += '</body></html>' 00226 return result 00227 00228 def exporter_junit_ioper(self, test_result_ext, test_suite_properties=None): 00229 from junit_xml import TestSuite, TestCase 00230 test_suites = [] 00231 test_cases = [] 00232 00233 for platform in sorted(test_result_ext.keys()): 00234 # {platform : ['Platform', 'Result', 'Scope', 'Description']) 00235 test_cases = [] 00236 for tr_result in test_result_ext[platform]: 00237 result, name, scope, description = tr_result 00238 00239 classname = 'test.ioper.%s.%s.%s' % (platform, name, scope) 00240 elapsed_sec = 0 00241 _stdout = description 00242 _stderr = '' 00243 # Test case 00244 tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) 00245 # Test case extra failure / error info 00246 if result == 'FAIL': 00247 tc.add_failure_info(description, _stdout) 00248 elif result == 'ERROR': 00249 tc.add_error_info(description, _stdout) 00250 elif result == 'SKIP' or result == 'NOT_SUPPORTED': 00251 tc.add_skipped_info(description, _stdout) 00252 00253 test_cases.append(tc) 00254 ts = TestSuite("test.suite.ioper.%s" % (platform), test_cases) 00255 test_suites.append(ts) 00256 return TestSuite.to_xml_string(test_suites) 00257 00258 def exporter_junit (self, test_result_ext, test_suite_properties=None): 00259 """ Export test results in JUnit XML compliant format 00260 """ 00261 from junit_xml import TestSuite, TestCase 00262 test_suites = [] 00263 test_cases = [] 00264 00265 targets = sorted(test_result_ext.keys()) 00266 for target in targets: 00267 toolchains = sorted(test_result_ext[target].keys()) 00268 for toolchain in toolchains: 00269 test_cases = [] 00270 tests = sorted(test_result_ext[target][toolchain].keys()) 00271 for test in tests: 00272 test_results = test_result_ext[target][toolchain][test] 00273 for test_res in test_results: 00274 test_ids = sorted(test_res.keys()) 00275 for test_no in test_ids: 00276 test_result = test_res[test_no] 00277 name = test_result['description'] 00278 classname = '%s.%s.%s.%s'% (self.package , target, toolchain, test_result['id']) 00279 elapsed_sec = test_result['elapsed_time'] 00280 _stdout = test_result['output'] 00281 00282 if 'target_name_unique' in test_result: 00283 _stderr = test_result['target_name_unique'] 00284 else: 00285 _stderr = test_result['target_name'] 00286 00287 # Test case 00288 tc = TestCase(name, classname, elapsed_sec, _stdout, _stderr) 00289 00290 # Test case extra failure / error info 00291 message = test_result['result'] 00292 if test_result['result'] == 'FAIL': 00293 tc.add_failure_info(message, _stdout) 00294 elif test_result['result'] == 'SKIP' or test_result["result"] == 'NOT_SUPPORTED': 00295 tc.add_skipped_info(message, _stdout) 00296 elif test_result['result'] != 'OK': 00297 tc.add_error_info(message, _stdout) 00298 00299 test_cases.append(tc) 00300 00301 ts = TestSuite("test.suite.%s.%s"% (target, toolchain), test_cases, properties=test_suite_properties[target][toolchain]) 00302 test_suites.append(ts) 00303 return TestSuite.to_xml_string(test_suites) 00304 00305 def exporter_print_helper(self, array, print_log=False): 00306 for item in array: 00307 print " * %s::%s::%s" % (item["target_name"], item["toolchain_name"], item["id"]) 00308 if print_log: 00309 log_lines = item["output"].split("\n") 00310 for log_line in log_lines: 00311 print " %s" % log_line 00312 00313 def exporter_print (self, test_result_ext, print_log_for_failures=False): 00314 """ Export test results in print format. 00315 """ 00316 failures = [] 00317 skips = [] 00318 successes = [] 00319 00320 unique_test_ids = self.get_all_unique_test_ids (test_result_ext) 00321 targets = sorted(test_result_ext.keys()) 00322 00323 for target in targets: 00324 toolchains = sorted(test_result_ext[target].keys()) 00325 for toolchain in toolchains: 00326 tests = sorted(test_result_ext[target][toolchain].keys()) 00327 for test in tests: 00328 test_runs = test_result_ext[target][toolchain][test] 00329 for test_runner in test_runs: 00330 #test_run = test_result_ext[target][toolchain][test][test_run_number][0] 00331 test_run = test_runner[0] 00332 00333 if "result" in test_run: 00334 if test_run["result"] == "FAIL": 00335 failures.append(test_run) 00336 elif test_run["result"] == "SKIP" or test_run["result"] == "NOT_SUPPORTED": 00337 skips.append(test_run) 00338 elif test_run["result"] == "OK": 00339 successes.append(test_run) 00340 else: 00341 raise Exception("Unhandled result type: %s" % (test_run["result"])) 00342 else: 00343 raise Exception("'test_run' did not have a 'result' value") 00344 00345 if successes: 00346 print "\n\nBuild successes:" 00347 self.exporter_print_helper (successes) 00348 00349 if skips: 00350 print "\n\nBuild skips:" 00351 self.exporter_print_helper (skips) 00352 00353 if failures: 00354 print "\n\nBuild failures:" 00355 self.exporter_print_helper (failures, print_log=print_log_for_failures) 00356 return False 00357 else: 00358 return True 00359 00360 def exporter_text (self, test_result_ext): 00361 """ Prints well-formed summary with results (SQL table like) 00362 table shows target x test results matrix across 00363 """ 00364 success_code = 0 # Success code that can be leter returned to 00365 # Pretty table package is used to print results 00366 pt = PrettyTable(["Result", "Target", "Toolchain", "Test ID", "Test Description", 00367 "Elapsed Time", "Timeout"]) 00368 pt.align["Result"] = "l" # Left align 00369 pt.align["Target"] = "l" # Left align 00370 pt.align["Toolchain"] = "l" # Left align 00371 pt.align["Test ID"] = "l" # Left align 00372 pt.align["Test Description"] = "l" # Left align 00373 pt.padding_width = 1 # One space between column edges and contents (default) 00374 00375 result_dict = {"OK" : 0, 00376 "FAIL" : 0, 00377 "ERROR" : 0, 00378 "UNDEF" : 0, 00379 "IOERR_COPY" : 0, 00380 "IOERR_DISK" : 0, 00381 "IOERR_SERIAL" : 0, 00382 "TIMEOUT" : 0, 00383 "NO_IMAGE" : 0, 00384 "MBED_ASSERT" : 0, 00385 "BUILD_FAILED" : 0, 00386 "NOT_SUPPORTED" : 0 00387 } 00388 unique_test_ids = self.get_all_unique_test_ids (test_result_ext) 00389 targets = sorted(test_result_ext.keys()) 00390 for target in targets: 00391 toolchains = sorted(test_result_ext[target].keys()) 00392 for toolchain in toolchains: 00393 test_cases = [] 00394 tests = sorted(test_result_ext[target][toolchain].keys()) 00395 for test in tests: 00396 test_results = test_result_ext[target][toolchain][test] 00397 for test_res in test_results: 00398 test_ids = sorted(test_res.keys()) 00399 for test_no in test_ids: 00400 test_result = test_res[test_no] 00401 result_dict[test_result['result']] += 1 00402 pt.add_row([test_result['result'], 00403 test_result['target_name'], 00404 test_result['toolchain_name'], 00405 test_result['id'], 00406 test_result['description'], 00407 test_result['elapsed_time'], 00408 test_result['duration']]) 00409 result = pt.get_string() 00410 result += "\n" 00411 00412 # Print result count 00413 result += "Result: " + ' / '.join(['%s %s' % (value, key) for (key, value) in {k: v for k, v in result_dict.items() if v != 0}.iteritems()]) 00414 return result
Generated on Sun Jul 17 2022 08:25:32 by 1.7.2