Willie Solano / mbedtls-slave-prot-prod

mbed TLS Build

Dependents:   Slave-prot-prod

tests/scripts/basic-build-test.sh@0:cdf462088d13, 2017-01-05 (annotated)

Committer:
markrad
Date:
Thu Jan 05 00:18:44 2017 +0000
Revision:
0:cdf462088d13
Initial commit

Who changed what in which revision?

UserRevisionLine numberNew contents of line
markrad 0:cdf462088d13 1 #!/bin/sh
markrad 0:cdf462088d13 2
markrad 0:cdf462088d13 3 # basic-build-tests.sh
markrad 0:cdf462088d13 4 #
markrad 0:cdf462088d13 5 # This file is part of mbed TLS (https://tls.mbed.org)
markrad 0:cdf462088d13 6 #
markrad 0:cdf462088d13 7 # Copyright (c) 2016, ARM Limited, All Rights Reserved
markrad 0:cdf462088d13 8 #
markrad 0:cdf462088d13 9 # Purpose
markrad 0:cdf462088d13 10 #
markrad 0:cdf462088d13 11 # Executes the basic test suites, captures the results, and generates a simple
markrad 0:cdf462088d13 12 # test report and code coverage report.
markrad 0:cdf462088d13 13 #
markrad 0:cdf462088d13 14 # The tests include:
markrad 0:cdf462088d13 15 # * Unit tests - executed using tests/scripts/run-test-suite.pl
markrad 0:cdf462088d13 16 # * Self-tests - executed using the test suites above
markrad 0:cdf462088d13 17 # * System tests - executed using tests/ssl-opt.sh
markrad 0:cdf462088d13 18 # * Interoperability tests - executed using tests/compat.sh
markrad 0:cdf462088d13 19 #
markrad 0:cdf462088d13 20 # The tests focus on functionality and do not consider performance.
markrad 0:cdf462088d13 21 #
markrad 0:cdf462088d13 22 # Note the tests self-adapt due to configurations in include/mbedtls/config.h
markrad 0:cdf462088d13 23 # which can lead to some tests being skipped, and can cause the number of
markrad 0:cdf462088d13 24 # available tests to fluctuate.
markrad 0:cdf462088d13 25 #
markrad 0:cdf462088d13 26 # This script has been written to be generic and should work on any shell.
markrad 0:cdf462088d13 27 #
markrad 0:cdf462088d13 28 # Usage: basic-build-tests.sh
markrad 0:cdf462088d13 29 #
markrad 0:cdf462088d13 30
markrad 0:cdf462088d13 31 # Abort on errors (and uninitiliased variables)
markrad 0:cdf462088d13 32 set -eu
markrad 0:cdf462088d13 33
markrad 0:cdf462088d13 34 if [ -d library -a -d include -a -d tests ]; then :; else
markrad 0:cdf462088d13 35 echo "Must be run from mbed TLS root" >&2
markrad 0:cdf462088d13 36 exit 1
markrad 0:cdf462088d13 37 fi
markrad 0:cdf462088d13 38
markrad 0:cdf462088d13 39 : ${OPENSSL:="openssl"}
markrad 0:cdf462088d13 40 : ${OPENSSL_LEGACY:="$OPENSSL"}
markrad 0:cdf462088d13 41 : ${GNUTLS_CLI:="gnutls-cli"}
markrad 0:cdf462088d13 42 : ${GNUTLS_SERV:="gnutls-serv"}
markrad 0:cdf462088d13 43 : ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"}
markrad 0:cdf462088d13 44 : ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"}
markrad 0:cdf462088d13 45
markrad 0:cdf462088d13 46 # To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh
markrad 0:cdf462088d13 47 # we just export the variables they require
markrad 0:cdf462088d13 48 export OPENSSL_CMD="$OPENSSL"
markrad 0:cdf462088d13 49 export GNUTLS_CLI="$GNUTLS_CLI"
markrad 0:cdf462088d13 50 export GNUTLS_SERV="$GNUTLS_SERV"
markrad 0:cdf462088d13 51
markrad 0:cdf462088d13 52 CONFIG_H='include/mbedtls/config.h'
markrad 0:cdf462088d13 53 CONFIG_BAK="$CONFIG_H.bak"
markrad 0:cdf462088d13 54
markrad 0:cdf462088d13 55 # Step 0 - print build environment info
markrad 0:cdf462088d13 56 OPENSSL="$OPENSSL" \
markrad 0:cdf462088d13 57 OPENSSL_LEGACY="$OPENSSL_LEGACY" \
markrad 0:cdf462088d13 58 GNUTLS_CLI="$GNUTLS_CLI" \
markrad 0:cdf462088d13 59 GNUTLS_SERV="$GNUTLS_SERV" \
markrad 0:cdf462088d13 60 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \
markrad 0:cdf462088d13 61 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \
markrad 0:cdf462088d13 62 scripts/output_env.sh
markrad 0:cdf462088d13 63 echo
markrad 0:cdf462088d13 64
markrad 0:cdf462088d13 65 # Step 1 - Make and instrumented build for code coverage
markrad 0:cdf462088d13 66 export CFLAGS=' --coverage -g3 -O0 '
markrad 0:cdf462088d13 67 make clean
markrad 0:cdf462088d13 68 cp "$CONFIG_H" "$CONFIG_BAK"
markrad 0:cdf462088d13 69 scripts/config.pl full
markrad 0:cdf462088d13 70 scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE
markrad 0:cdf462088d13 71 make -j
markrad 0:cdf462088d13 72
markrad 0:cdf462088d13 73
markrad 0:cdf462088d13 74 # Step 2 - Execute the tests
markrad 0:cdf462088d13 75 TEST_OUTPUT=out_${PPID}
markrad 0:cdf462088d13 76 cd tests
markrad 0:cdf462088d13 77
markrad 0:cdf462088d13 78 # Step 2a - Unit Tests
markrad 0:cdf462088d13 79 perl scripts/run-test-suites.pl -v |tee unit-test-$TEST_OUTPUT
markrad 0:cdf462088d13 80 echo
markrad 0:cdf462088d13 81
markrad 0:cdf462088d13 82 # Step 2b - System Tests
markrad 0:cdf462088d13 83 sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT
markrad 0:cdf462088d13 84 echo
markrad 0:cdf462088d13 85
markrad 0:cdf462088d13 86 # Step 2c - Compatibility tests
markrad 0:cdf462088d13 87 sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \
markrad 0:cdf462088d13 88 tee compat-test-$TEST_OUTPUT
markrad 0:cdf462088d13 89 OPENSSL_CMD="$OPENSSL_LEGACY" \
markrad 0:cdf462088d13 90 sh compat.sh -m 'ssl3' |tee -a compat-test-$TEST_OUTPUT
markrad 0:cdf462088d13 91 OPENSSL_CMD="$OPENSSL_LEGACY" \
markrad 0:cdf462088d13 92 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" \
markrad 0:cdf462088d13 93 GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \
markrad 0:cdf462088d13 94 sh compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' | \
markrad 0:cdf462088d13 95 tee -a compat-test-$TEST_OUTPUT
markrad 0:cdf462088d13 96 echo
markrad 0:cdf462088d13 97
markrad 0:cdf462088d13 98 # Step 3 - Process the coverage report
markrad 0:cdf462088d13 99 cd ..
markrad 0:cdf462088d13 100 make lcov |tee tests/cov-$TEST_OUTPUT
markrad 0:cdf462088d13 101
markrad 0:cdf462088d13 102
markrad 0:cdf462088d13 103 # Step 4 - Summarise the test report
markrad 0:cdf462088d13 104 echo
markrad 0:cdf462088d13 105 echo "========================================================================="
markrad 0:cdf462088d13 106 echo "Test Report Summary"
markrad 0:cdf462088d13 107 echo
markrad 0:cdf462088d13 108
markrad 0:cdf462088d13 109 cd tests
markrad 0:cdf462088d13 110
markrad 0:cdf462088d13 111 # Step 4a - Unit tests
markrad 0:cdf462088d13 112 echo "Unit tests - tests/scripts/run-test-suites.pl"
markrad 0:cdf462088d13 113
markrad 0:cdf462088d13 114 PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ')
markrad 0:cdf462088d13 115 SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ')
markrad 0:cdf462088d13 116 TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ')
markrad 0:cdf462088d13 117 FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ')
markrad 0:cdf462088d13 118
markrad 0:cdf462088d13 119 echo "No test suites : $TOTAL_SUITES"
markrad 0:cdf462088d13 120 echo "Passed : $PASSED_TESTS"
markrad 0:cdf462088d13 121 echo "Failed : $FAILED_TESTS"
markrad 0:cdf462088d13 122 echo "Skipped : $SKIPPED_TESTS"
markrad 0:cdf462088d13 123 echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))"
markrad 0:cdf462088d13 124 echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))"
markrad 0:cdf462088d13 125 echo
markrad 0:cdf462088d13 126
markrad 0:cdf462088d13 127 TOTAL_PASS=$PASSED_TESTS
markrad 0:cdf462088d13 128 TOTAL_FAIL=$FAILED_TESTS
markrad 0:cdf462088d13 129 TOTAL_SKIP=$SKIPPED_TESTS
markrad 0:cdf462088d13 130 TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))
markrad 0:cdf462088d13 131 TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS))
markrad 0:cdf462088d13 132
markrad 0:cdf462088d13 133 # Step 4b - TLS Options tests
markrad 0:cdf462088d13 134 echo "TLS Options tests - tests/ssl-opt.sh"
markrad 0:cdf462088d13 135
markrad 0:cdf462088d13 136 PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p')
markrad 0:cdf462088d13 137 SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p')
markrad 0:cdf462088d13 138 TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p')
markrad 0:cdf462088d13 139 FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS))
markrad 0:cdf462088d13 140
markrad 0:cdf462088d13 141 echo "Passed : $PASSED_TESTS"
markrad 0:cdf462088d13 142 echo "Failed : $FAILED_TESTS"
markrad 0:cdf462088d13 143 echo "Skipped : $SKIPPED_TESTS"
markrad 0:cdf462088d13 144 echo "Total exec'd tests : $TOTAL_TESTS"
markrad 0:cdf462088d13 145 echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))"
markrad 0:cdf462088d13 146 echo
markrad 0:cdf462088d13 147
markrad 0:cdf462088d13 148 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS))
markrad 0:cdf462088d13 149 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS))
markrad 0:cdf462088d13 150 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS))
markrad 0:cdf462088d13 151 TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS))
markrad 0:cdf462088d13 152 TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS))
markrad 0:cdf462088d13 153
markrad 0:cdf462088d13 154
markrad 0:cdf462088d13 155 # Step 4c - System Compatibility tests
markrad 0:cdf462088d13 156 echo "System/Compatibility tests - tests/compat.sh"
markrad 0:cdf462088d13 157
markrad 0:cdf462088d13 158 PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
markrad 0:cdf462088d13 159 SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
markrad 0:cdf462088d13 160 EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }')
markrad 0:cdf462088d13 161 FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS))
markrad 0:cdf462088d13 162
markrad 0:cdf462088d13 163 echo "Passed : $PASSED_TESTS"
markrad 0:cdf462088d13 164 echo "Failed : $FAILED_TESTS"
markrad 0:cdf462088d13 165 echo "Skipped : $SKIPPED_TESTS"
markrad 0:cdf462088d13 166 echo "Total exec'd tests : $EXED_TESTS"
markrad 0:cdf462088d13 167 echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))"
markrad 0:cdf462088d13 168 echo
markrad 0:cdf462088d13 169
markrad 0:cdf462088d13 170 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS))
markrad 0:cdf462088d13 171 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS))
markrad 0:cdf462088d13 172 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS))
markrad 0:cdf462088d13 173 TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS))
markrad 0:cdf462088d13 174 TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS))
markrad 0:cdf462088d13 175
markrad 0:cdf462088d13 176
markrad 0:cdf462088d13 177 # Step 4d - Grand totals
markrad 0:cdf462088d13 178 echo "-------------------------------------------------------------------------"
markrad 0:cdf462088d13 179 echo "Total tests"
markrad 0:cdf462088d13 180
markrad 0:cdf462088d13 181 echo "Total Passed : $TOTAL_PASS"
markrad 0:cdf462088d13 182 echo "Total Failed : $TOTAL_FAIL"
markrad 0:cdf462088d13 183 echo "Total Skipped : $TOTAL_SKIP"
markrad 0:cdf462088d13 184 echo "Total exec'd tests : $TOTAL_EXED"
markrad 0:cdf462088d13 185 echo "Total avail tests : $TOTAL_AVAIL"
markrad 0:cdf462088d13 186 echo
markrad 0:cdf462088d13 187
markrad 0:cdf462088d13 188
markrad 0:cdf462088d13 189 # Step 4e - Coverage
markrad 0:cdf462088d13 190 echo "Coverage"
markrad 0:cdf462088d13 191
markrad 0:cdf462088d13 192 LINES_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p')
markrad 0:cdf462088d13 193 LINES_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p')
markrad 0:cdf462088d13 194 FUNCS_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p')
markrad 0:cdf462088d13 195 FUNCS_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) functions)$/\1/p')
markrad 0:cdf462088d13 196
markrad 0:cdf462088d13 197 LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL))
markrad 0:cdf462088d13 198 LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))"
markrad 0:cdf462088d13 199
markrad 0:cdf462088d13 200 FUNCS_PERCENT=$((1000*$FUNCS_TESTED/$FUNCS_TOTAL))
markrad 0:cdf462088d13 201 FUNCS_PERCENT="$(($FUNCS_PERCENT/10)).$(($FUNCS_PERCENT-($FUNCS_PERCENT/10)*10))"
markrad 0:cdf462088d13 202
markrad 0:cdf462088d13 203 echo "Lines Tested : $LINES_TESTED of $LINES_TOTAL $LINES_PERCENT%"
markrad 0:cdf462088d13 204 echo "Functions Tested : $FUNCS_TESTED of $FUNCS_TOTAL $FUNCS_PERCENT%"
markrad 0:cdf462088d13 205 echo
markrad 0:cdf462088d13 206
markrad 0:cdf462088d13 207
markrad 0:cdf462088d13 208 rm unit-test-$TEST_OUTPUT
markrad 0:cdf462088d13 209 rm sys-test-$TEST_OUTPUT
markrad 0:cdf462088d13 210 rm compat-test-$TEST_OUTPUT
markrad 0:cdf462088d13 211 rm cov-$TEST_OUTPUT
markrad 0:cdf462088d13 212
markrad 0:cdf462088d13 213 cd ..
markrad 0:cdf462088d13 214
markrad 0:cdf462088d13 215 make clean
markrad 0:cdf462088d13 216
markrad 0:cdf462088d13 217 if [ -f "$CONFIG_BAK" ]; then
markrad 0:cdf462088d13 218 mv "$CONFIG_BAK" "$CONFIG_H"
markrad 0:cdf462088d13 219 fi