build.sh (18373B)
1 #!/bin/bash -eu 2 # 3 # OSS-Fuzz build script for GNU libmicrohttpd. 4 # 5 # This file is in the public domain. 6 # 7 # It is executed inside the OSS-Fuzz base-builder image, which exports: 8 # 9 # $SRC parent directory of the checked-out sources 10 # ($SRC/libmicrohttpd, see Dockerfile) 11 # $WORK scratch directory for build artifacts 12 # $OUT where the finished fuzz targets must be installed 13 # $CC $CXX the instrumented compilers 14 # $CFLAGS $CXXFLAGS sanitizer + coverage flags; MUST be honoured and 15 # MUST NOT be replaced 16 # $LIB_FUZZING_ENGINE the fuzzing engine to link against ("-fsanitize=fuzzer", 17 # a path to libFuzzingEngine.a, the AFL driver, ...) 18 # $SANITIZER address | undefined | memory | coverage 19 # $FUZZING_ENGINE libfuzzer | afl | honggfuzz | centipede | none 20 # 21 # The same script can be run outside OSS-Fuzz for a local smoke test; every 22 # variable above has a defensive default below. 23 # 24 # See contrib/oss-fuzz/README for the full story and for the local 25 # infra/helper.py recipe. 26 27 # --------------------------------------------------------------------------- 28 # Defaults, so that the script is also runnable by hand 29 # --------------------------------------------------------------------------- 30 # The shebang already carries -eu, but only when the script is executed 31 # directly; "bash build.sh" silently drops it, and a harness that fails to 32 # link then leaves $OUT without that target and the script still exits 0. 33 set -eu 34 35 SRC="${SRC:-$(cd "$(dirname "$0")/../../.." && pwd)}" 36 WORK="${WORK:-${SRC}/work}" 37 OUT="${OUT:-${SRC}/out}" 38 SANITIZER="${SANITIZER:-address}" 39 FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" 40 ARCHITECTURE="${ARCHITECTURE:-x86_64}" 41 42 # --- engine ------------------------------------------------------------------ 43 # 44 # Under OSS-Fuzz $CC, $CXX and $LIB_FUZZING_ENGINE are exported by the 45 # base-builder image for the engine being built and MUST be used as given, 46 # so everything here is dead code there. It fires only on a local run, 47 # and its job is to make "FUZZING_ENGINE=afl ./build.sh" produce a real 48 # AFL++ target rather than a libFuzzer one that happens to link. 49 # 50 # _mhd_cov_cflags is the coverage instrumentation the engine needs at 51 # compile time. Getting it wrong is the failure mode that matters: the 52 # build succeeds, the target runs, and it finds nothing, because the 53 # engine has no feedback signal. 54 case "${FUZZING_ENGINE}" in 55 libfuzzer) 56 CC="${CC:-clang}" 57 CXX="${CXX:-clang++}" 58 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}" 59 _mhd_cov_cflags="-fsanitize=fuzzer-no-link" 60 _mhd_main="engine" 61 ;; 62 afl) 63 # afl-clang-fast inserts AFL++'s own instrumentation, so libFuzzer's 64 # must not be added on top; libAFLDriver.a supplies a main() that 65 # feeds AFL++ input to LLVMFuzzerTestOneInput(). 66 CC="${CC:-afl-clang-fast}" 67 CXX="${CXX:-afl-clang-fast++}" 68 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-/usr/lib/afl/libAFLDriver.a}" 69 _mhd_cov_cflags="" 70 _mhd_main="engine" 71 ;; 72 honggfuzz) 73 # hfuzz-clang does the same job for honggfuzz. It is not packaged by 74 # Debian; build it from https://github.com/google/honggfuzz and put 75 # its directory on $PATH (see contrib/oss-fuzz/README). 76 CC="${CC:-hfuzz-clang}" 77 CXX="${CXX:-hfuzz-clang++}" 78 # hfuzz-clang links libhfuzz/libhfcommon and supplies main() itself, 79 # so $LIB_FUZZING_ENGINE stays empty. 80 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-}" 81 _mhd_cov_cflags="" 82 _mhd_main="engine" 83 ;; 84 none) 85 # No engine: link the harnesses' own deterministic driver instead. 86 # This is what makes a sanitizer-only smoke test possible without any 87 # fuzzing engine installed at all. 88 CC="${CC:-clang}" 89 CXX="${CXX:-clang++}" 90 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-}" 91 _mhd_cov_cflags="" 92 _mhd_main="builtin" 93 ;; 94 *) 95 echo "ERROR: unknown FUZZING_ENGINE='${FUZZING_ENGINE}'" >&2 96 echo " expected: libfuzzer | afl | honggfuzz | none" >&2 97 exit 1 98 ;; 99 esac 100 101 # --- architecture ------------------------------------------------------------ 102 # 103 # OSS-Fuzz exports $ARCHITECTURE and already has -m32 in $CFLAGS for i386, 104 # so this too only fires locally. 105 # 106 # -no-pie is needed on i386 and only there: a position independent 107 # executable built with -m32 and any sanitizer dies with SEGV at address 108 # 0 before main() on current Linux/clang. It is harmless on x86_64 and 109 # is therefore not applied there, to keep that build byte-comparable with 110 # what OSS-Fuzz produces. 111 case "${ARCHITECTURE}" in 112 x86_64) 113 _mhd_arch_cflags="" 114 ;; 115 i386) 116 _mhd_arch_cflags="-m32 -no-pie" 117 ;; 118 *) 119 echo "ERROR: unknown ARCHITECTURE='${ARCHITECTURE}'" >&2 120 echo " expected: x86_64 | i386" >&2 121 exit 1 122 ;; 123 esac 124 125 # --- $CFLAGS / $CXXFLAGS ----------------------------------------------------- 126 # 127 # Under OSS-Fuzz $CFLAGS and $CXXFLAGS are always exported by the 128 # base-builder image and MUST be used verbatim, so everything below is 129 # dead code there: it fires only when the variable is unset, i.e. only on 130 # a local run. 131 # 132 # The point of the defaults is that a local run must be a *real* fuzzing 133 # run. Two flags are what make it one, and leaving either out produces a 134 # build that looks fine and finds nothing: 135 # 136 # -fsanitize=fuzzer-no-link 137 # installs libFuzzer's coverage instrumentation (SanitizerCoverage 138 # trace-pc-guard + the comparison hooks) in every translation unit 139 # of the library. Without it libFuzzer gets no feedback signal at 140 # all and degenerates into blind random input generation -- 141 # `cov:` stays flat and the corpus never grows. "-no-link" is the 142 # compile-time half; $LIB_FUZZING_ENGINE supplies the driver at 143 # link time. 144 # a sanitizer 145 # libFuzzer by itself only notices a crash the kernel delivers. 146 # ASan/UBSan are what turn a silently-tolerated overflow into a 147 # report. Note that MHD's own oracles -- the 148 # MHD_set_panic_func() tripwire and mhd_assert(), enabled by the 149 # --enable-asserts below -- work without any sanitizer, which is 150 # why SANITIZER=none is still worth something. 151 # 152 # The mapping below mirrors what OSS-Fuzz's own helper passes for each 153 # $SANITIZER value, with one deliberate difference in the "address" case, 154 # noted there. 155 if [ -z "${CFLAGS:-}" ]; then 156 # -O1 OSS-Fuzz's optimisation level for fuzz builds: 157 # fast enough to get exec/s, low enough that 158 # inlining does not destroy the stack traces. 159 # -fno-omit-frame-pointer needed for usable ASan/libFuzzer backtraces. 160 # -gline-tables-only just enough debug info to symbolize; a full -g 161 # would multiply build time and object size. 162 _mhd_base_cflags="-O1 -fno-omit-frame-pointer -gline-tables-only" 163 164 case "${SANITIZER}" in 165 address) 166 # OSS-Fuzz builds "address" and "undefined" as two separate 167 # campaigns, because it has unlimited machine time and wants each 168 # report attributed to one sanitizer. A local run has an afternoon 169 # at most, so the default folds UBSan into the ASan build: two 170 # oracles per CPU-hour instead of one, at a few percent of speed. 171 # Set SANITIZER=undefined explicitly for the split OSS-Fuzz shape. 172 # 173 # -fno-sanitize-recover=undefined is essential: by default UBSan 174 # *prints* and continues, and libFuzzer only records a finding for a 175 # process that dies. Without it UB scrolls past and the run is 176 # reported clean. 177 _mhd_san_cflags="-fsanitize=address,undefined" 178 _mhd_san_cflags="${_mhd_san_cflags} -fsanitize-address-use-after-scope" 179 _mhd_san_cflags="${_mhd_san_cflags} -fno-sanitize-recover=undefined" 180 ;; 181 undefined) 182 _mhd_san_cflags="-fsanitize=undefined -fno-sanitize-recover=undefined" 183 ;; 184 memory) 185 # MSan reports uninitialised reads from *any* uninstrumented code it 186 # links against, so this is only usable when the C library is 187 # instrumented too -- true inside the OSS-Fuzz image, essentially 188 # never true on a distro toolchain. Expect false positives in libc 189 # frames locally; use the OSS-Fuzz container for a real MSan run. 190 # 191 # This is also why build.sh configures --disable-https and 192 # --disable-curl: an uninstrumented GnuTLS would poison every run. 193 # Any harness that needs TLS is skipped in this configuration; see 194 # the $FUZZERS selection below. 195 _mhd_san_cflags="-fsanitize=memory -fsanitize-memory-track-origins" 196 ;; 197 coverage) 198 # A coverage build is not a fuzzing build: it replays an existing 199 # corpus to produce a report, so no sanitizer and no libFuzzer 200 # coverage instrumentation. 201 _mhd_san_cflags="-fprofile-instr-generate -fcoverage-mapping" 202 ;; 203 none | "") 204 _mhd_san_cflags="" 205 ;; 206 *) 207 echo "ERROR: unknown SANITIZER='${SANITIZER}'" >&2 208 echo " expected: address | undefined | memory | coverage | none" >&2 209 exit 1 210 ;; 211 esac 212 213 if [ "${SANITIZER}" = "coverage" ]; then 214 CFLAGS="${_mhd_arch_cflags} ${_mhd_base_cflags} ${_mhd_san_cflags}" 215 else 216 CFLAGS="${_mhd_arch_cflags} ${_mhd_base_cflags} ${_mhd_san_cflags}" 217 CFLAGS="${CFLAGS} ${_mhd_cov_cflags}" 218 fi 219 unset _mhd_base_cflags _mhd_san_cflags 220 fi 221 CXXFLAGS="${CXXFLAGS:-${CFLAGS}}" 222 if [ "${_mhd_main}" = "builtin" ]; then 223 _mhd_no_main="" 224 else 225 _mhd_no_main="-DFUZZ_NO_MAIN" 226 fi 227 unset _mhd_arch_cflags _mhd_cov_cflags _mhd_main 228 229 # Directory holding the libmicrohttpd sources. OSS-Fuzz clones them to 230 # $SRC/libmicrohttpd (see Dockerfile); allow an override for local runs. 231 MHD_SRC="${MHD_SRC:-${SRC}/libmicrohttpd}" 232 233 # Out-of-tree build directory. Keeping the build out of the source tree 234 # means build.sh never modifies the checkout, which matters for the 235 # "run build.sh twice" and "reproduce against a pristine tree" cases. 236 BUILD="${WORK}/mhd-build" 237 238 FUZZERS="fuzz_request fuzz_options fuzz_eventloop fuzz_str fuzz_memorypool fuzz_auth_header fuzz_postprocessor" 239 240 # fuzz_tls is deliberately absent: it needs a TLS backend, and this build 241 # configures --disable-https on purpose (see the rationale below), so the 242 # target would be an empty shell on all three sanitizers. Shipping it 243 # would need a second, HTTPS-enabled build variant, which also gives up 244 # the MemorySanitizer configuration -- an uninstrumented GnuTLS poisons 245 # every MSan run. It is built and tested in tree by "make -C src/fuzz 246 # check" instead. 247 248 mkdir -p "${WORK}" "${OUT}" "${BUILD}" 249 250 echo "=== libmicrohttpd OSS-Fuzz build ===" 251 echo " MHD_SRC = ${MHD_SRC}" 252 echo " BUILD = ${BUILD}" 253 echo " OUT = ${OUT}" 254 echo " SANITIZER = ${SANITIZER}" 255 echo " FUZZING_ENGINE = ${FUZZING_ENGINE}" 256 echo " ARCHITECTURE = ${ARCHITECTURE}" 257 echo " LIB_FUZZING_ENGINE = ${LIB_FUZZING_ENGINE}" 258 echo " CC / CXX = ${CC} / ${CXX}" 259 echo " CFLAGS = ${CFLAGS}" 260 echo " CXXFLAGS = ${CXXFLAGS}" 261 echo " FUZZERS = ${FUZZERS}" 262 263 # --------------------------------------------------------------------------- 264 # 1. Bootstrap (the git checkout ships no 'configure') 265 # --------------------------------------------------------------------------- 266 cd "${MHD_SRC}" 267 if [ ! -x ./configure ]; then 268 echo "--- bootstrapping ---" 269 # ./bootstrap swallows its own failures (it ends in an '|| echo ...' 270 # chain), so its exit status cannot be trusted; check for the product 271 # and fall back to autoreconf. 272 ./bootstrap || true 273 if [ ! -x ./configure ]; then 274 autoreconf -fi 275 fi 276 fi 277 278 # --------------------------------------------------------------------------- 279 # 2. Configure 280 # --------------------------------------------------------------------------- 281 # Rationale for each flag: 282 # 283 # --enable-static --disable-shared 284 # fuzz_str and fuzz_auth_header call MHD-internal symbols 285 # (MHD_hex_to_bin(), MHD_get_rq_dauth_params_(), MHD_pool_create(), 286 # ...) that are compiled with hidden visibility and are therefore 287 # NOT exported from libmicrohttpd.so. Only the static archive can 288 # be linked. Static linking is also what OSS-Fuzz wants: the 289 # target binaries in $OUT must not depend on anything outside $OUT. 290 # --with-pic 291 # keep the static objects position independent so they can be 292 # linked into the (PIE) fuzz targets regardless of compiler default. 293 # --enable-fuzzing 294 # configures src/fuzz/Makefile. Not strictly needed here (the 295 # harnesses are compiled by hand below) but it keeps this build 296 # equivalent to the documented developer build, and it makes 297 # configure fail loudly if src/fuzz/ ever stops being wired up. 298 # --enable-asserts 299 # keeps mhd_assert() alive. Assertions on attacker-reachable paths 300 # are exactly what this campaign is meant to find; without them 301 # findings K1-K7 (src/fuzz/README section 6) are invisible. 302 # --disable-https 303 # deliberate. The harnesses never speak TLS: they hand MHD an 304 # already-connected AF_UNIX socketpair via MHD_add_connection() and 305 # never set MHD_USE_TLS. Enabling HTTPS would (a) add nothing to 306 # coverage, (b) drag GnuTLS - which OSS-Fuzz fuzzes separately - 307 # into the link, and (c) make the MemorySanitizer build impossible 308 # without an MSan-instrumented GnuTLS. With HTTPS off the only 309 # external dependencies are libc and libpthread, so all three 310 # sanitizers are usable. 311 # --disable-curl 312 # the curl-based test suite is not built here and pulling libcurl in 313 # would create the same uninstrumented-dependency problem. 314 # --disable-doc --disable-examples --disable-tools 315 # nothing of that is needed and it only costs build time (and 316 # texinfo/pandoc dependencies). 317 # --disable-dependency-tracking 318 # one-shot build, no need for .deps. 319 # --enable-build-type=neutral 320 # the default, stated explicitly: "neutral" is the one build type 321 # that does NOT inject its own optimisation/debug flags, so the 322 # $CFLAGS handed to us by OSS-Fuzz survive unmodified. 323 # 324 # NOTE: no --enable-sanitizers and no --enable-coverage. OSS-Fuzz supplies 325 # the sanitizer and coverage instrumentation through $CFLAGS; letting 326 # configure add a second, possibly conflicting -fsanitize= set is a classic 327 # way to break an OSS-Fuzz build. 328 cd "${BUILD}" 329 "${MHD_SRC}/configure" \ 330 --enable-static \ 331 --disable-shared \ 332 --with-pic \ 333 --enable-fuzzing \ 334 --enable-asserts \ 335 --disable-https \ 336 --disable-curl \ 337 --disable-doc \ 338 --disable-examples \ 339 --disable-tools \ 340 --disable-dependency-tracking \ 341 --enable-build-type=neutral \ 342 CC="${CC}" \ 343 CFLAGS="${CFLAGS}" \ 344 LDFLAGS="${LDFLAGS:-}" 345 346 # --------------------------------------------------------------------------- 347 # 3. Build the library only 348 # --------------------------------------------------------------------------- 349 # Building just src/microhttpd avoids compiling the (large) test suite and 350 # the src/fuzz check_PROGRAMS, which would be built with the standalone 351 # driver's main() and are useless here. 352 make -j"$(nproc)" -C src/microhttpd libmicrohttpd.la 353 354 MHD_LIB="${BUILD}/src/microhttpd/.libs/libmicrohttpd.a" 355 test -f "${MHD_LIB}" || { 356 echo "ERROR: ${MHD_LIB} was not produced" >&2 357 exit 1 358 } 359 360 # --------------------------------------------------------------------------- 361 # 4. Compile the harnesses as libFuzzer translation units 362 # --------------------------------------------------------------------------- 363 # -DFUZZ_NO_MAIN drops the standalone driver's main() from fuzz_common.h, 364 # because the engine supplies its own. With FUZZING_ENGINE=none there is 365 # no engine, so the harnesses' built-in driver is kept instead and the 366 # result is a self-contained, deterministic, seeded fuzzer that needs no 367 # engine at all. LLVMFuzzerTestOneInput() itself is unconditional in 368 # every harness either way. 369 # 370 # Include path: 371 # -I${BUILD} for the generated MHD_config.h 372 # -I${MHD_SRC} for the in-tree headers next to configure.ac 373 # -I${MHD_SRC}/src/include for microhttpd.h 374 # -I${MHD_SRC}/src/microhttpd for internal.h, mhd_str.h, gen_auth.h, ... 375 # -I${MHD_SRC}/src/fuzz for fuzz_common.h 376 MHD_INCLUDES=( 377 -I"${BUILD}" 378 -I"${MHD_SRC}" 379 -I"${MHD_SRC}/src/include" 380 -I"${MHD_SRC}/src/microhttpd" 381 -I"${MHD_SRC}/src/fuzz" 382 ) 383 384 for fuzzer in ${FUZZERS}; do 385 echo "--- building ${fuzzer} ---" 386 # shellcheck disable=SC2086 387 $CC $CFLAGS \ 388 ${_mhd_no_main} \ 389 "${MHD_INCLUDES[@]}" \ 390 -c "${MHD_SRC}/src/fuzz/${fuzzer}.c" \ 391 -o "${WORK}/${fuzzer}.o" 392 # Link with $CXX: $LIB_FUZZING_ENGINE is a C++ archive for most engines. 393 # shellcheck disable=SC2086 394 $CXX $CXXFLAGS \ 395 "${WORK}/${fuzzer}.o" \ 396 -o "${OUT}/${fuzzer}" \ 397 $LIB_FUZZING_ENGINE \ 398 "${MHD_LIB}" \ 399 -lpthread 400 done 401 402 # --------------------------------------------------------------------------- 403 # 5. Seed corpora, dictionaries and .options files 404 # --------------------------------------------------------------------------- 405 "${MHD_SRC}/contrib/oss-fuzz/make_seed_corpus.sh" "${MHD_SRC}" "${OUT}" 406 407 for fuzzer in ${FUZZERS}; do 408 cp "${MHD_SRC}/contrib/oss-fuzz/dicts/${fuzzer}.dict" "${OUT}/${fuzzer}.dict" 409 cp "${MHD_SRC}/contrib/oss-fuzz/${fuzzer}.options" "${OUT}/${fuzzer}.options" 410 done 411 412 # --------------------------------------------------------------------------- 413 # 6. Verify: every requested target must actually be in $OUT 414 # --------------------------------------------------------------------------- 415 # Belt and braces for the failure that matters most -- a build that 416 # reports success but ships nothing, which on OSS-Fuzz shows up only as a 417 # target that never runs. 418 _mhd_missing="" 419 for fuzzer in ${FUZZERS}; do 420 [ -x "${OUT}/${fuzzer}" ] || _mhd_missing="${_mhd_missing} ${fuzzer}" 421 [ -f "${OUT}/${fuzzer}_seed_corpus.zip" ] || 422 _mhd_missing="${_mhd_missing} ${fuzzer}_seed_corpus.zip" 423 done 424 if [ -n "${_mhd_missing}" ]; then 425 echo "ERROR: build did not produce:${_mhd_missing}" >&2 426 exit 1 427 fi 428 unset _mhd_missing 429 430 echo "=== done; contents of \$OUT ===" 431 ls -la "${OUT}"