run-option-matrix.sh (9464B)
1 #!/bin/sh 2 # This file is in the public domain. 3 # 4 # Re-run already built libmicrohttpd test binaries across the daemon option 5 # matrix defined in src/microhttpd/mhd_opt_matrix.c. 6 # 7 # See --help below for the full documentation. 8 9 set -u 10 11 me=`basename "$0"` 12 13 # --------------------------------------------------------------------------- 14 # Defaults 15 # --------------------------------------------------------------------------- 16 17 # The binaries that honour the matrix (they read the MHD_TEST_* environment 18 # variables), plus a few libcurl based ones that at least get re-run in every 19 # threading/polling mode of the daemon they start themselves. 20 default_tests="src/microhttpd/test_option_matrix 21 src/microhttpd/test_rq_shift_back 22 src/microhttpd/test_chunked_ext 23 src/microhttpd/test_dauth_malformed 24 src/microhttpd/test_raw_requests 25 src/testcurl/test_get 26 src/testcurl/test_post 27 src/testcurl/test_put_chunked 28 src/testcurl/test_process_headers" 29 30 profiles_arg="" 31 jobs=1 32 builddir="" 33 timeout_sec=600 34 verbose=0 35 list_only=0 36 37 usage () 38 { 39 cat <<EOF 40 Usage: $me [OPTION]... [TEST-BINARY]... 41 42 Re-runs libmicrohttpd test binaries across the daemon option matrix. The 43 matrix is defined once, in src/microhttpd/mhd_opt_matrix.c, and is selected 44 per run through the environment, so nothing has to be recompiled: 45 46 MHD_TEST_PROFILE the profile, by name or by index 47 MHD_TEST_MEM_LIMIT override MHD_OPTION_CONNECTION_MEMORY_LIMIT 48 MHD_TEST_DISCIPLINE override MHD_OPTION_CLIENT_DISCIPLINE_LVL 49 MHD_TEST_STRICT_FOR_CLIENT override the same knob via the deprecated 50 MHD_OPTION_STRICT_FOR_CLIENT 51 MHD_TEST_THREADING external | internal | per-connection | pool 52 MHD_TEST_POLL select | poll | epoll 53 54 This script only sets MHD_TEST_PROFILE; the other variables stay available 55 for a manual run. 56 57 Options: 58 --profiles=LIST comma separated profile names or indices to visit 59 (default: every profile of the matrix) 60 --jobs=N run up to N test processes in parallel (default: 1) 61 --builddir=DIR the root of the built tree (default: the directory this 62 script is called from, or the parent of contrib/) 63 --timeout=SEC kill a single run after SEC seconds (default: $timeout_sec) 64 --list-profiles print the profiles of the matrix and exit 65 --verbose echo the output of every run 66 --help print this help and exit 67 68 Arguments: 69 TEST-BINARY... the binaries to run, relative to the build directory or 70 absolute. The default set is: 71 EOF 72 echo "$default_tests" | sed 's/^/ /' 73 cat <<EOF 74 75 Only the src/microhttpd binaries listed above read the 76 MHD_TEST_* variables; the src/testcurl ones ignore them 77 and are simply re-run, which is still useful as a 78 stability check but does not vary the configuration. 79 80 Exit status: 81 0 every run passed (or was skipped) 82 1 at least one run failed 83 2 usage error or nothing could be run 84 85 A run that exits with 77 is reported as SKIP and does not fail the script; 86 99 is reported as ERROR and does fail it. 87 EOF 88 } 89 90 # --------------------------------------------------------------------------- 91 # Argument parsing 92 # --------------------------------------------------------------------------- 93 94 tests="" 95 while [ $# -gt 0 ] ; do 96 case "$1" in 97 --help | -h) 98 usage ; exit 0 ;; 99 --profiles=*) 100 profiles_arg=`expr "x$1" : 'x--profiles=\(.*\)'` ;; 101 --jobs=*) 102 jobs=`expr "x$1" : 'x--jobs=\(.*\)'` ;; 103 -j) 104 shift ; jobs="${1:-1}" ;; 105 --builddir=*) 106 builddir=`expr "x$1" : 'x--builddir=\(.*\)'` ;; 107 --timeout=*) 108 timeout_sec=`expr "x$1" : 'x--timeout=\(.*\)'` ;; 109 --list-profiles) 110 list_only=1 ;; 111 --verbose | -v) 112 verbose=1 ;; 113 -*) 114 echo "$me: unknown option '$1'; try '$me --help'." >&2 ; exit 2 ;; 115 *) 116 tests="$tests 117 $1" ;; 118 esac 119 shift 120 done 121 122 case "$jobs" in 123 '' | *[!0-9]*) echo "$me: --jobs needs a number." >&2 ; exit 2 ;; 124 esac 125 [ "$jobs" -ge 1 ] || jobs=1 126 case "$timeout_sec" in 127 '' | *[!0-9]*) echo "$me: --timeout needs a number." >&2 ; exit 2 ;; 128 esac 129 130 # --------------------------------------------------------------------------- 131 # Locate the built tree 132 # --------------------------------------------------------------------------- 133 134 if [ -z "$builddir" ] ; then 135 if [ -x "src/microhttpd/test_option_matrix" ] ; then 136 builddir="." 137 else 138 d=`dirname "$0"`/.. 139 if [ -x "$d/src/microhttpd/test_option_matrix" ] ; then 140 builddir="$d" 141 else 142 builddir="." 143 fi 144 fi 145 fi 146 if [ ! -d "$builddir" ] ; then 147 echo "$me: '$builddir' is not a directory." >&2 148 exit 2 149 fi 150 151 matrix_bin="$builddir/src/microhttpd/test_option_matrix" 152 153 # --------------------------------------------------------------------------- 154 # The list of profiles 155 # --------------------------------------------------------------------------- 156 157 # The matrix is defined in exactly one place: ask the test binary for it. 158 # The fallback list is only used when the binary has not been built yet, so 159 # that --list-profiles still says something useful. 160 fallback_profiles="default 161 mem-64 162 mem-128 163 mem-256 164 mem-512 165 mem-1024 166 mem-2048 167 mem-3072 168 mem-4096 169 strict-1 170 strict-2 171 strict-3 172 lax-1 173 lax-2 174 lax-3 175 legacy-strict 176 legacy-lax" 177 178 if [ -x "$matrix_bin" ] ; then 179 all_profiles=`"$matrix_bin" --list-profiles 2>/dev/null` || all_profiles="" 180 fi 181 if [ -z "${all_profiles:-}" ] ; then 182 all_profiles="$fallback_profiles" 183 fi 184 185 if [ -n "$profiles_arg" ] ; then 186 profiles=`echo "$profiles_arg" | tr ',' '\n' | sed '/^$/d'` 187 else 188 profiles="$all_profiles" 189 fi 190 191 if [ "$list_only" -eq 1 ] ; then 192 echo "$all_profiles" 193 exit 0 194 fi 195 196 if [ -z "$tests" ] ; then 197 tests="$default_tests" 198 fi 199 200 # --------------------------------------------------------------------------- 201 # Run 202 # --------------------------------------------------------------------------- 203 204 workdir=`mktemp -d "${TMPDIR:-/tmp}/mhd-option-matrix.XXXXXX"` || { 205 echo "$me: cannot create a temporary directory." >&2 ; exit 2 ; } 206 trap 'rm -rf "$workdir"' 0 207 trap 'rm -rf "$workdir" ; exit 130' 1 2 3 15 208 209 timeout_cmd="" 210 if command -v timeout >/dev/null 2>&1 ; then 211 timeout_cmd="timeout -k 5 $timeout_sec" 212 fi 213 214 # Run one (profile, binary) pair and record the outcome. 215 # $1 profile, $2 binary, $3 result file 216 run_one () 217 { 218 _prof="$1" 219 _bin="$2" 220 _out="$3" 221 _path="$_bin" 222 case "$_path" in 223 /*) ;; 224 *) _path="$builddir/$_bin" ;; 225 esac 226 if [ ! -x "$_path" ] ; then 227 echo "MISSING $_prof $_bin" > "$_out" 228 return 0 229 fi 230 MHD_TEST_PROFILE="$_prof" ; export MHD_TEST_PROFILE 231 # shellcheck disable=SC2086 232 $timeout_cmd "$_path" > "$_out.log" 2>&1 233 _rc=$? 234 case "$_rc" in 235 0) echo "PASS $_prof $_bin" > "$_out" ;; 236 77) echo "SKIP $_prof $_bin" > "$_out" ;; 237 99) echo "ERROR $_prof $_bin (exit 99)" > "$_out" ;; 238 124 | 137) 239 echo "TIMEOUT $_prof $_bin (after ${timeout_sec}s)" > "$_out" ;; 240 *) echo "FAIL $_prof $_bin (exit $_rc)" > "$_out" ;; 241 esac 242 return 0 243 } 244 245 n=0 246 running=0 247 for prof in $profiles ; do 248 for t in $tests ; do 249 n=`expr $n + 1` 250 res="$workdir/r$n" 251 echo "$prof" > "$res.prof" 252 echo "$t" > "$res.bin" 253 if [ "$jobs" -gt 1 ] ; then 254 run_one "$prof" "$t" "$res" & 255 running=`expr $running + 1` 256 if [ "$running" -ge "$jobs" ] ; then 257 wait 258 running=0 259 fi 260 else 261 run_one "$prof" "$t" "$res" 262 if [ "$verbose" -eq 1 ] ; then 263 cat "$res" 2>/dev/null 264 fi 265 fi 266 done 267 done 268 wait 269 270 if [ "$n" -eq 0 ] ; then 271 echo "$me: nothing to run." >&2 272 exit 2 273 fi 274 275 # --------------------------------------------------------------------------- 276 # Summary 277 # --------------------------------------------------------------------------- 278 279 total=0 280 n_pass=0 281 n_skip=0 282 n_fail=0 283 n_missing=0 284 285 echo "" 286 echo "==========================================================================" 287 echo " option matrix summary" 288 echo "==========================================================================" 289 290 i=0 291 for prof in $profiles ; do 292 line="" 293 bad=0 294 base=$i 295 for t in $tests ; do 296 i=`expr $i + 1` 297 res="$workdir/r$i" 298 if [ -f "$res" ] ; then 299 status=`cut -d' ' -f1 < "$res"` 300 else 301 status="ERROR" 302 fi 303 total=`expr $total + 1` 304 case "$status" in 305 PASS) n_pass=`expr $n_pass + 1` ;; 306 SKIP) n_skip=`expr $n_skip + 1` ;; 307 MISSING) n_missing=`expr $n_missing + 1` ;; 308 *) n_fail=`expr $n_fail + 1` ; bad=1 ;; 309 esac 310 line="$line $status" 311 done 312 printf '%-16s%s\n' "$prof" "$line" 313 if [ "$bad" -ne 0 ] || [ "$verbose" -eq 1 ] ; then 314 j=$base 315 for t in $tests ; do 316 j=`expr $j + 1` 317 res="$workdir/r$j" 318 [ -f "$res" ] || continue 319 status=`cut -d' ' -f1 < "$res"` 320 case "$status" in 321 PASS | SKIP | MISSING) continue ;; 322 esac 323 echo " --- $t: `cat "$res"`" 324 if [ -f "$res.log" ] ; then 325 sed 's/^/ /' < "$res.log" | tail -n 25 326 fi 327 done 328 fi 329 done 330 331 echo "--------------------------------------------------------------------------" 332 echo "binaries, in the order of the columns above:" 333 for t in $tests ; do 334 echo " $t" 335 done 336 echo "--------------------------------------------------------------------------" 337 echo "total $total, pass $n_pass, skip $n_skip, fail $n_fail, missing $n_missing" 338 if [ "$n_fail" -ne 0 ] ; then 339 echo "RESULT: FAILED" 340 exit 1 341 fi 342 echo "RESULT: OK" 343 exit 0