test.sh: help; option --expect-fail to ignore specified failures
diff --git a/CHANGES b/CHANGES
index 9f81cb0..090ae47 100644
--- a/CHANGES
+++ b/CHANGES
@@ -76,6 +76,14 @@
socktype=<val-of-SOCK_DGRAM>
Tests: EXECSOCKETPAIRPACKETS SYSTEMSOCKETPAIRPACKETS
+ Cosmetic corrections of EXEC,SYSTEM tests.
+
+ test.sh: Added option --expect-fail to specify comma separated list of
+ test numbers whose failure shall not cause a failure of the whole
+ script.
+
+ test.sh: Added help text
+
####################### V 1.7.4.4:
Corrections:
@@ -647,7 +655,7 @@
two settings were applied with two different get/set cycles, Thanks to
Alexandre Fenyo for providing an initial patch.
However, the actual fix is part of a conceptual change of the termios
- module that aims for applying all changes in a single tcsetaddr call.
+ module that aims for applying all changes in a single tcsetattr call.
Fixes FreeBSD Bug 198441
Termios options TAB0,TAB1,TAB2,TAB3, and XTABS did not have an effect.
diff --git a/test.sh b/test.sh
index 970649a..bcb7aec 100755
--- a/test.sh
+++ b/test.sh
@@ -11,13 +11,56 @@
# you can eg strace socat with: export TRACE="strace -v -tt -ff -D -x -s 1024 -o /tmp/$USER/socat.strace"
#set -vx
+#TODO: Add options for interface, broadcast-interface
+
+[ -z "$USER" ] && USER="$LOGNAME" # HP-UX
+if [ -z "$TMPDIR" ]; then
+ if [ -z "$TMP" ]; then
+ TMP=/tmp
+ fi
+ TMPDIR="$TMP"
+fi
+#E=-e # Linux
+if [ $(echo "x\c") = "x" ]; then E=""
+elif [ $(echo -e "x\c") = "x" ]; then E="-e"
+else
+ echo "cannot suppress trailing newline on echo" >&2
+ exit 1
+fi
+ECHO="echo $E"
+PRINTF="printf"
+
+usage() {
+ $ECHO "Usage: $0 <options> [<test-spec> ...]"
+ $ECHO "options:"
+ $ECHO "\t-h \t\tShow this help"
+ $ECHO "\t-t <sec> \tBase for timeouts in seconds, default: 0.1"
+ $ECHO "\t-v \t\tBe more verbose, show failed commands"
+ $ECHO "\t-n <num> \tOnly perform test with given number"
+ $ECHO "\t-N <num> \tOnly perform tests starting with given number"
+ $ECHO "\t-C \t\tClear/remove left over certificates from previous runs"
+ $ECHO "\t-x \t\tShow commands executed, even when test succeeded"
+ #$ECHO "\t-d \t\tShow log output of commands, even when they did not fail"
+ $ECHO "\t-foreign \tAllow tests that send packets to Internet"
+ $ECHO "\t-expect-fail N1,N2,... \tIgnore failure of these tests"
+ $ECHO "\ttest-spec \Number of test or name of test"
+ $ECHO "Contents of environment variable OPTS are passed to Socat invokations, e.'g:"
+ $ECHO "OPTS=\"-d -d -d -d -lu\" ./test.sh"
+ $ECHO "TRACE=\"strace -tt -v\" Use trace,valgrind etc.on socat"
+ $ECHO "SOCAT=/path/to/socat \tselect socat executable for test"
+ $ECHO "FILAN=... PROCAN=..."
+ $ECHO "Find the tests' stdout,stderr,diff in $TMPDIR/$USER/\$PID"
+}
+
val_t=0.1
NUMCOND=true
#NUMCOND="test \$N -gt 70"
VERBOSE=
FOREIGN=
+EXPECT_FAIL=
while [ "$1" ]; do
case "X$1" in
+ X-h) usage; exit 0 ;;
X-t?*) val_t="${1#-t}" ;;
X-t) shift; val_t="$1" ;;
X-v) VERBOSE=1 ;; # show commands
@@ -27,6 +70,10 @@
X-N) shift; NUMCOND="test \$N -ge $1" ;;
X-C) rm -f testcert*.conf testcert.dh testcli*.* testsrv*.* ;;
X-foreign) FOREIGN=1 ;; # allow access to 3rd party Internet hosts
+ X-expect-fail|X--expect-fail) shift; EXPECT_FAIL="$1" ;;
+ X-*) echo "Unknown option \"$1\"" >&2
+ usage >&2
+ exit 1 ;;
*) break;
esac
shift
@@ -4207,16 +4254,17 @@
N=$((N+1))
+# I cannot remember the clou of this test, seems rather useless
NAME=CHILDDEFAULT
case "$TESTS" in
*%$N%*|*%functions%*|*%$NAME%*)
if ! eval $NUMCOND; then :
-elif ! a=$(testfeats STDIO EXEC); then
- $PRINTF "test $F_n $TEST... ${YELLOW}Feature $a not available in $SOCAT${NORMAL}\n" $N
+elif ! F=$(testfeats STDIO EXEC); then
+ $PRINTF "test $F_n $TEST... ${YELLOW}Feature $F not available in $SOCAT${NORMAL}\n" $N
numCANT=$((numCANT+1))
listCANT="$listCANT $N"
-elif ! a=$(testaddrs STDIO EXEC); then
- $PRINTF "test $F_n $TEST... ${YELLOW}Address $a not available in $SOCAT${NORMAL}\n" $N
+elif ! A=$(testaddrs STDIO EXEC); then
+ $PRINTF "test $F_n $TEST... ${YELLOW}Address $A not available in $SOCAT${NORMAL}\n" $N
numCANT=$((numCANT+1))
listCANT="$listCANT $N"
else
@@ -4231,8 +4279,9 @@
MYPGID=`expr "\`grep "process group id =" $tf\`" : '[^0-9]*\([0-9]*\).*'`
MYSID=`expr "\`grep "process session id =" $tf\`" : '[^0-9]*\([0-9]*\).*'`
#echo "PID=$MYPID, PPID=$MYPPID, PGID=$MYPGID, SID=$MYSID"
-if [ "$MYPID" = "$MYPPID" -o "$MYPID" = "$MYPGID" -o "$MYPID" = "$MYSID" -o \
- "$MYPPID" = "$MYPGID" -o "$MYPPID" = "$MYSID" -o "$MYPGID" = "$MYSID" ];
+#if [ "$MYPID" = "$MYPPID" -o "$MYPID" = "$MYPGID" -o "$MYPID" = "$MYSID" -o \
+# "$MYPPID" = "$MYPGID" -o "$MYPPID" = "$MYSID" -o "$MYPGID" = "$MYSID" ];
+if [ "$MYPID" = "$MYPPID" ];
then
$PRINTF "$FAILED:\n"
echo "$CMD"
@@ -14569,10 +14618,10 @@
listCANT="$listCANT $N"
else
# Second, set accept-timeout and see if socat exits before kill
-CMD2="$TRACE $SOCAT $opts TCP-LISTEN:$PORT,reuseaddr,accept-timeout=1 PIPE" &
+CMD2="$TRACE $SOCAT $opts TCP-LISTEN:$PORT,reuseaddr,accept-timeout=1 PIPE"
$CMD2 >"$te2" 2>&1 </dev/null &
pid2=$!
-sleep 1
+sleep 2
if kill $pid2 2>"$tk2"; then
$PRINTF "$FAILED\n"
echo "$CMD2" >&2
@@ -15985,14 +16034,42 @@
echo "Used temp directory $TD - you might want to remove it after analysis"
echo "Summary: $((N-1)) tests, $((numOK+numFAIL+numCANT)) selected; $numOK ok, $numFAIL failed, $numCANT could not be performed"
+set -- $listCANT; while [ "$1" ]; do echo "$1"; shift; done >"$td/cannot.lst"
+ln -sf "$td/cannot.lst" .
+set -- $listFAIL; while [ "$1" ]; do echo "$1"; shift; done >"$td/failed.lst"
+ln -sf "$td/failed.lst" .
+sort -n <(cat "$td/cannot.lst" |while read x; do echo "$x CANT"; done) <(cat "$td/failed.lst" |while read x; do echo "$x FAILED"; done) >"$td/result.txt"
+ln -sf "$td/result.txt" .
if [ "$numCANT" -gt 0 ]; then
echo "CANT: $listCANT"
fi
if [ "$numFAIL" -gt 0 ]; then
echo "FAILED: $listFAIL"
- exit 1
fi
-exit 0
+
+if [ -z "$EXPECT_FAIL" ]; then
+ [ "$numFAIL" -eq 0 ]
+ exit # with rc from above statement
+fi
+
+#set -vx
+
+if [ "$EXPECT_FAIL" ]; then
+ diff <(set -- $(echo "$EXPECT_FAIL" |tr ',' ' '); while [ "$1" ]; do echo "$1"; shift; done) "$td/failed.lst" >"$td/failed.diff"
+ ln -sf "$td/failed.diff" .
+ #grep "^"
+ grep "^> " "$td/failed.diff" |awk '{print($2);}' >"$td/failed.unexp"
+ ln -s "$td/failed.unexp" .
+ echo "FAILED unexpected: $(cat "$td/failed.unexp" |xargs echo)"
+ grep "^< " "$td/failed.diff" |awk '{print($2);}' >"$td/ok.unexp"
+ ln -s "$td/ok.unexp" .
+ echo "OK unexpected: $(cat "$td/ok.unexp" |xargs echo)"
+fi
+listFAIL=$(cat "$td/failed.lst" |xargs echo)
+numFAIL="$(wc -l "$td/failed.lst" |awk '{print($1);}')"
+
+! test -s "$td/failed.unexp"
+exit
#==============================================================================
@@ -16032,12 +16109,12 @@
$PRINTF "test $F_n $TEST... ${YELLOW}systemd-socket-activate not available${NORMAL}\n" $N
numCANT=$((numCANT+1))
listCANT="$listCANT $N"
-elif ! a=$(testfeats STDIO IP4 TCP PIPE); then
- $PRINTF "test $F_n $TEST... ${YELLOW}Feature $a not available in $SOCAT${NORMAL}\n" $N
+elif ! F=$(testfeats STDIO IP4 TCP PIPE); then
+ $PRINTF "test $F_n $TEST... ${YELLOW}Feature $F not available in $SOCAT${NORMAL}\n" $N
numCANT=$((numCANT+1))
listCANT="$listCANT $N"
-elif ! a=$(testaddrs - TCP4 TCP4-LISTEN PIPE); then
- $PRINTF "test $F_n $TEST... ${YELLOW}Address $a not available in $SOCAT${NORMAL}\n" $N
+elif ! A=$(testaddrs - TCP4 TCP4-LISTEN PIPE); then
+ $PRINTF "test $F_n $TEST... ${YELLOW}Address $A not available in $SOCAT${NORMAL}\n" $N
numCANT=$((numCANT+1))
listCANT="$listCANT $N"
elif ! o=$(testoptions so-reuseaddr fork ) >/dev/null; then
