You are viewing an older revision! See the latest version
Mbed SDK automated test suite reporting
Rationale¶
Test suite and automation tools ought to report test data in at least few formats. Each format should satisfy different need. For example:
- As console output while test suite is running as CLI (command line interface) application.
- HTML report is human readable and can be presented to audience.
- JUnit format (formatted XML) is used by various tools, test result parsers to generate reports, combine test data or just present it in Continuous Integration report page.
- Other like database dump: export test result data directly to database located locally or remotely.
All mentioned above methods are supported by test suite at this time.
- Console output is by default printing compile, build and rest result data. Additionally switches like -v or -V can be used for command line to get extra prints from test suite configuration and diagnostic or MUT (mbed under test) console output respectively.
- HTML reports are used to generate simple HTML page with intuitive and simple test result matrix. HTML report is a single page containing all test results and test console output data.
Below you can see example HTML report with active test result tool-tip:
- http://junit.org/ JUnit test results reports can be used to generate JUnit compliant XML file with test suite test results. Users can export JUnit XML files to e.g. IDEs like Eclipse to see results directly or they can feed JUnit result reports to CI systems for further automated processing.
Below you can see example of JUnit XML report:
<?xml version="1.0" ?>
<testsuites>
<testsuite errors="0" failures="0" name="test.suite.KL25Z.ARM" skipped="0" tests="1" time="0.55">
<properties>
<property name="skipped" value=""/>
<property name="toolchain" value="ARM"/>
<property name="target" value="KL25Z"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="test.path.KL25Z.ARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\KL25Z\ARM\DTCT_1\detect.bin"/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\KL25Z\ARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="test.libs.KL25Z.ARM.DTCT_1" value=""/>
<property name="clean" value="False"/>
<property name="jobs" value="1"/>
<property name="test.loops.KL25Z.ARM.DTCT_1" value="1"/>
<property name="test.duration.KL25Z.ARM.DTCT_1" value="10"/>
</properties>
<testcase classname="test.KL25Z.ARM.DTCT_1" name="Simple detect test" time="0.550000">
<system-out>Host test instrumentation on port: "COM39" and disk: "F:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'KL25Z'
HOST: MUT Target name 'KL25Z', expected 'KL25Z'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID '60bed2a6-60bb-44d9-909d-97acbc58f2ec'
{success}
{end}
</system-out>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" name="test.suite.LPC1768.ARM" skipped="0" tests="1" time="1.37">
<properties>
<property name="test.duration.LPC1768.ARM.DTCT_1" value="10"/>
<property name="skipped" value=""/>
<property name="toolchain" value="ARM"/>
<property name="target" value="LPC1768"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="test.path.LPC1768.ARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\LPC1768\ARM\DTCT_1\detect.bin"/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\LPC1768\ARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="clean" value="False"/>
<property name="jobs" value="1"/>
<property name="test.loops.LPC1768.ARM.DTCT_1" value="1"/>
<property name="test.libs.LPC1768.ARM.DTCT_1" value=""/>
</properties>
<testcase classname="test.LPC1768.ARM.DTCT_1" name="Simple detect test" time="1.370000">
<system-out>Host test instrumentation on port: "COM77" and disk: "E:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'LPC1768'
HOST: MUT Target name 'LPC1768', expected 'LPC1768'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID 'b3c29946-4a0d-4a54-92ba-817fb19a9284'
{success}
{end}
</system-out>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" name="test.suite.NRF51822.ARM" skipped="0" tests="1" time="0.9">
<properties>
<property name="skipped" value=""/>
<property name="toolchain" value="ARM"/>
<property name="target" value="NRF51822"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="test.loops.NRF51822.ARM.DTCT_1" value="1"/>
<property name="test.libs.NRF51822.ARM.DTCT_1" value=""/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NRF51822\ARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="test.duration.NRF51822.ARM.DTCT_1" value="10"/>
<property name="clean" value="False"/>
<property name="jobs" value="1"/>
<property name="test.path.NRF51822.ARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NRF51822\ARM\DTCT_1\detect.hex"/>
</properties>
<testcase classname="test.NRF51822.ARM.DTCT_1" name="Simple detect test" time="0.900000">
<system-out>Host test instrumentation on port: "COM58" and disk: "H:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'NRF51822'
HOST: MUT Target name 'NRF51822', expected 'NRF51822'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID 'faa0309c-35aa-4256-b0c1-80be362af5c1'
{success}
{end}
</system-out>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" name="test.suite.NRF51822.GCC_ARM" skipped="0" tests="1" time="0.91">
<properties>
<property name="test.duration.NRF51822.GCC_ARM.DTCT_1" value="10"/>
<property name="skipped" value=""/>
<property name="toolchain" value="GCC_ARM"/>
<property name="target" value="NRF51822"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="test.path.NRF51822.GCC_ARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NRF51822\GCC_ARM\DTCT_1\detect.hex"/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NRF51822\GCC_ARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="clean" value="False"/>
<property name="test.loops.NRF51822.GCC_ARM.DTCT_1" value="1"/>
<property name="jobs" value="1"/>
<property name="test.libs.NRF51822.GCC_ARM.DTCT_1" value=""/>
</properties>
<testcase classname="test.NRF51822.GCC_ARM.DTCT_1" name="Simple detect test" time="0.910000">
<system-out>Host test instrumentation on port: "COM58" and disk: "H:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'NRF51822'
HOST: MUT Target name 'NRF51822', expected 'NRF51822'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID '73ee32a1-8f2d-434c-bc44-2dc18d5e2d14'
{success}
{end}
</system-out>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" name="test.suite.LPC1768.uARM" skipped="0" tests="1" time="1.3">
<properties>
<property name="test.loops.LPC1768.uARM.DTCT_1" value="1"/>
<property name="skipped" value=""/>
<property name="toolchain" value="uARM"/>
<property name="target" value="LPC1768"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\LPC1768\uARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="test.path.LPC1768.uARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\LPC1768\uARM\DTCT_1\detect.bin"/>
<property name="clean" value="False"/>
<property name="jobs" value="1"/>
<property name="test.libs.LPC1768.uARM.DTCT_1" value=""/>
<property name="test.duration.LPC1768.uARM.DTCT_1" value="10"/>
</properties>
<testcase classname="test.LPC1768.uARM.DTCT_1" name="Simple detect test" time="1.300000">
<system-out>Host test instrumentation on port: "COM77" and disk: "E:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'LPC1768'
HOST: MUT Target name 'LPC1768', expected 'LPC1768'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID 'c8eac564-aa80-4cc1-a09f-b242751a6346'
{success}
{end}
</system-out>
</testcase>
</testsuite>
<testsuite errors="0" failures="0" name="test.suite.NUCLEO_F103RB.uARM" skipped="0" tests="1" time="0.55">
<properties>
<property name="test.duration.NUCLEO_F103RB.uARM.DTCT_1" value="10"/>
<property name="skipped" value=""/>
<property name="toolchain" value="uARM"/>
<property name="target" value="NUCLEO_F103RB"/>
<property name="shuffle_random_seed" value="0.9237068745"/>
<property name="test.libs.NUCLEO_F103RB.uARM.DTCT_1" value=""/>
<property name="build_dir" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NUCLEO_F103RB\uARM"/>
<property name="build_mbed_libs_result" value="True"/>
<property name="test_ids" value=""/>
<property name="test.loops.NUCLEO_F103RB.uARM.DTCT_1" value="1"/>
<property name="clean" value="False"/>
<property name="test.path.NUCLEO_F103RB.uARM.DTCT_1" value="C:\Work\winbb\slave\BuildRelease\build\build\test\NUCLEO_F103RB\uARM\DTCT_1\detect.bin"/>
<property name="jobs" value="1"/>
</properties>
<testcase classname="test.NUCLEO_F103RB.uARM.DTCT_1" name="Simple detect test" time="0.550000">
<system-out>Host test instrumentation on port: "COM64" and disk: "G:\"
{{start}}
{{start}}
HOST: Detecting target name...
MBED: Target 'NUCLEO_F103RB'
HOST: MUT Target name 'NUCLEO_F103RB', expected 'NUCLEO_F103RB'... [OK]
MBED: Test ID 'DTCT_1'
MBED: UUID '234933ae-3849-4269-81d3-2f28744e1595'
{success}
{end}
</system-out>
</testcase>
</testsuite>
</testsuites>