build.sh (18086B)
1 #!/bin/bash -eu 2 # 3 # OSS-Fuzz build script for GNU libextractor. 4 # 5 # This file is in the public domain. 6 # 7 # It is executed inside the OSS-Fuzz base-builder image, which exports: 8 # 9 # $SRC parent directory of the checked-out sources 10 # ($SRC/libextractor, see Dockerfile) 11 # $WORK scratch directory for build artifacts 12 # $OUT where the finished fuzz targets must be installed 13 # $CC $CXX the instrumented compilers 14 # $CFLAGS $CXXFLAGS sanitizer + coverage flags; MUST be honoured and 15 # MUST NOT be replaced 16 # $LIB_FUZZING_ENGINE the fuzzing engine to link against 17 # $SANITIZER address | undefined | memory | coverage 18 # $FUZZING_ENGINE libfuzzer | afl | honggfuzz | centipede | none 19 # 20 # The same script can be run outside OSS-Fuzz for a local campaign; every 21 # variable above has a defensive default below. See contrib/oss-fuzz/README. 22 23 # The shebang carries -eu, but "bash build.sh" silently drops it, and a 24 # harness that fails to link would then leave $OUT short one target while 25 # the script still exits 0. 26 set -eu 27 28 SRC="${SRC:-$(cd "$(dirname "$0")/../../.." && pwd)}" 29 WORK="${WORK:-${SRC}/work}" 30 OUT="${OUT:-${SRC}/out}" 31 SANITIZER="${SANITIZER:-address}" 32 FUZZING_ENGINE="${FUZZING_ENGINE:-libfuzzer}" 33 ARCHITECTURE="${ARCHITECTURE:-x86_64}" 34 35 # --- engine ---------------------------------------------------------------- 36 # 37 # Under OSS-Fuzz $CC, $CXX and $LIB_FUZZING_ENGINE come from the 38 # base-builder image and MUST be used as given, so all of this is dead 39 # code there. It fires only on a local run, where its job is to make 40 # "FUZZING_ENGINE=afl ./build.sh" produce a real AFL++ target rather than 41 # a libFuzzer one that happens to link. 42 # 43 # _le_cov_cflags is the coverage instrumentation the engine needs at 44 # compile time. Getting it wrong is the failure mode that matters: the 45 # build succeeds, the target runs, and it finds nothing, because the 46 # engine has no feedback signal at all. 47 case "${FUZZING_ENGINE}" in 48 libfuzzer) 49 CC="${CC:-clang}" 50 CXX="${CXX:-clang++}" 51 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}" 52 _le_cov_cflags="-fsanitize=fuzzer-no-link" 53 _le_main="engine" 54 ;; 55 afl) 56 CC="${CC:-afl-clang-fast}" 57 CXX="${CXX:-afl-clang-fast++}" 58 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-/usr/lib/afl/libAFLDriver.a}" 59 _le_cov_cflags="" 60 _le_main="engine" 61 ;; 62 honggfuzz) 63 CC="${CC:-hfuzz-clang}" 64 CXX="${CXX:-hfuzz-clang++}" 65 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-}" 66 _le_cov_cflags="" 67 _le_main="engine" 68 ;; 69 none) 70 # No engine: link the harnesses' own deterministic driver instead. 71 # This is what makes a sanitizer-only smoke test possible with no 72 # fuzzing engine installed at all. 73 CC="${CC:-clang}" 74 CXX="${CXX:-clang++}" 75 LIB_FUZZING_ENGINE="${LIB_FUZZING_ENGINE:-}" 76 _le_cov_cflags="" 77 _le_main="builtin" 78 ;; 79 *) 80 echo "ERROR: unknown FUZZING_ENGINE='${FUZZING_ENGINE}'" >&2 81 echo " expected: libfuzzer | afl | honggfuzz | none" >&2 82 exit 1 83 ;; 84 esac 85 86 case "${ARCHITECTURE}" in 87 x86_64) _le_arch_cflags="" ;; 88 i386) _le_arch_cflags="-m32 -no-pie" ;; 89 *) 90 echo "ERROR: unknown ARCHITECTURE='${ARCHITECTURE}'" >&2 91 exit 1 92 ;; 93 esac 94 95 # --- $CFLAGS / $CXXFLAGS --------------------------------------------------- 96 # 97 # Dead code under OSS-Fuzz, which always exports these. On a local run 98 # the defaults have to make it a *real* fuzzing build; two flags decide 99 # that, and leaving either out produces a build that looks fine and finds 100 # nothing: 101 # 102 # -fsanitize=fuzzer-no-link installs libFuzzer's coverage 103 # instrumentation in every translation unit. Without it there is 104 # no feedback signal, `cov:` stays flat and the corpus never grows. 105 # a sanitizer libFuzzer only notices a crash the kernel 106 # delivers. ASan is what turns a silently tolerated over-read of 107 # a parser buffer -- the whole bug class this project has -- into 108 # a report. 109 if [ -z "${CFLAGS:-}" ]; then 110 _le_base_cflags="-O1 -fno-omit-frame-pointer -gline-tables-only" 111 112 case "${SANITIZER}" in 113 address) 114 # OSS-Fuzz runs "address" and "undefined" as separate campaigns so 115 # that each report is attributed to one sanitizer. A local run has 116 # an afternoon at most, so the default folds UBSan into the ASan 117 # build: two oracles per CPU-hour instead of one. 118 # 119 # UBSan is deliberately left *recovering* here (no 120 # -fno-sanitize-recover): libextractor's parsers trip a handful of 121 # signed-shift and misaligned-load sites on almost every input, and 122 # halting on those would stop the run before ASan ever gets to the 123 # memory-safety bugs. UBSan reports each source location once per 124 # process, so they still all show up in the log. Set 125 # LE_UBSAN_HALT=1 for the OSS-Fuzz behaviour of dying on the first. 126 _le_san_cflags="-fsanitize=address,undefined" 127 _le_san_cflags="${_le_san_cflags} -fsanitize-address-use-after-scope" 128 if [ "${LE_UBSAN_HALT:-0}" != "0" ]; then 129 _le_san_cflags="${_le_san_cflags} -fno-sanitize-recover=undefined" 130 fi 131 ;; 132 undefined) 133 _le_san_cflags="-fsanitize=undefined -fno-sanitize-recover=undefined" 134 ;; 135 memory) 136 # Only usable when the C library and every dependency is 137 # instrumented too -- true inside the OSS-Fuzz image, essentially 138 # never on a distro toolchain. This is also why the build below 139 # disables every optional third-party parser: an uninstrumented 140 # libjpeg or GnuTLS would poison every run. 141 _le_san_cflags="-fsanitize=memory -fsanitize-memory-track-origins" 142 ;; 143 coverage) 144 _le_san_cflags="-fprofile-instr-generate -fcoverage-mapping" 145 ;; 146 none | "") 147 _le_san_cflags="" 148 ;; 149 *) 150 echo "ERROR: unknown SANITIZER='${SANITIZER}'" >&2 151 exit 1 152 ;; 153 esac 154 155 CFLAGS="${_le_arch_cflags} ${_le_base_cflags} ${_le_san_cflags}" 156 if [ "${SANITIZER}" != "coverage" ]; then 157 CFLAGS="${CFLAGS} ${_le_cov_cflags}" 158 fi 159 unset _le_base_cflags _le_san_cflags 160 fi 161 CXXFLAGS="${CXXFLAGS:-${CFLAGS}}" 162 if [ "${_le_main}" = "builtin" ]; then 163 _le_no_main="" 164 else 165 _le_no_main="-DFUZZ_NO_MAIN" 166 fi 167 unset _le_arch_cflags _le_cov_cflags _le_main 168 169 LE_SRC="${LE_SRC:-${SRC}/libextractor}" 170 171 # Out-of-tree build: build.sh never modifies the checkout, which is what 172 # makes "run it twice" and "reproduce against a pristine tree" work. 173 BUILD="${WORK}/le-build" 174 175 mkdir -p "${WORK}" "${OUT}" "${BUILD}" 176 177 echo "=== libextractor OSS-Fuzz build ===" 178 echo " LE_SRC = ${LE_SRC}" 179 echo " BUILD = ${BUILD}" 180 echo " OUT = ${OUT}" 181 echo " SANITIZER = ${SANITIZER}" 182 echo " FUZZING_ENGINE = ${FUZZING_ENGINE}" 183 echo " LIB_FUZZING_ENGINE = ${LIB_FUZZING_ENGINE}" 184 echo " CC / CXX = ${CC} / ${CXX}" 185 echo " CFLAGS = ${CFLAGS}" 186 187 # --------------------------------------------------------------------------- 188 # 1. Bootstrap (the git checkout ships no 'configure') 189 # --------------------------------------------------------------------------- 190 cd "${LE_SRC}" 191 if [ ! -x ./configure ]; then 192 echo "--- bootstrapping ---" 193 ./bootstrap || true 194 if [ ! -x ./configure ]; then 195 autoreconf -f -I m4 -i 196 fi 197 fi 198 199 # --------------------------------------------------------------------------- 200 # 2. Configure 201 # --------------------------------------------------------------------------- 202 # Rationale for each flag: 203 # 204 # --enable-static 205 # fuzz_datasource, fuzz_unzip and fuzz_ipc call symbols that are 206 # compiled with $(HIDDEN_VISIBILITY_CFLAGS) and are therefore NOT 207 # exported from libextractor.so. Only the static archive can be 208 # linked, and OSS-Fuzz wants static targets anyway: a binary in 209 # $OUT must not depend on anything outside $OUT. 210 # --enable-fuzzing 211 # configures src/fuzz/Makefile. The harnesses are compiled by 212 # hand below, but keeping this on makes configure fail loudly if 213 # src/fuzz/ ever stops being wired up. 214 # --disable-shared 215 # the per-plugin targets link the plugin's own .c files, so no 216 # plugin module is loaded at run time and building them would only 217 # cost time. Note that this is what makes fuzz_extract useless in 218 # this configuration; see the $FUZZERS selection below. 219 # --disable-gsf --disable-glib 220 # step 3 only builds src/common and src/main, so no plugin that 221 # wraps a third-party parser is ever compiled here: those are 222 # fuzzed by their own OSS-Fuzz projects, and an uninstrumented copy 223 # of one would poison the MemorySanitizer build. Turning off the 224 # two that configure would otherwise pull into the *library* link 225 # keeps the dependency set at libc + zlib. 226 # LE_FUZZ_GSF=1 keeps libgsf in, which is what it takes to build the ole2 227 # target (see $GSF_PLUGIN_FUZZER below). Off by default because libgsf 228 # and glib are not instrumented here, so every allocation they make is 229 # invisible to ASan and impossible under MemorySanitizer -- the same 230 # reason the other third-party wrappers are opt-in. Turn it on for a 231 # local campaign: ole2_extractor.c does a good deal of its own parsing 232 # before libgsf is ever called, and that part is ours. 233 _le_gsf_conf="--disable-gsf --disable-glib" 234 if [ "${LE_FUZZ_GSF:-0}" != "0" ]; then 235 _le_gsf_conf="" 236 fi 237 238 cd "${BUILD}" 239 # shellcheck disable=SC2086 240 "${LE_SRC}/configure" \ 241 --enable-static \ 242 --disable-shared \ 243 --with-pic \ 244 --enable-fuzzing \ 245 ${_le_gsf_conf} \ 246 --disable-dependency-tracking \ 247 CC="${CC}" \ 248 CFLAGS="${CFLAGS}" \ 249 LDFLAGS="${LDFLAGS:-}" 250 251 # --------------------------------------------------------------------------- 252 # 3. Build the libraries the harnesses link against 253 # --------------------------------------------------------------------------- 254 make -j"$(nproc)" -C src/common 255 make -j"$(nproc)" -C src/main libextractor.la 256 257 LE_LIB="${BUILD}/src/main/.libs/libextractor.a" 258 LE_COMMON="${BUILD}/src/common/.libs/libextractor_common.a" 259 for f in "${LE_LIB}" "${LE_COMMON}"; do 260 test -f "${f}" || { echo "ERROR: ${f} was not produced" >&2; exit 1; } 261 done 262 263 # The static archives do not record their own dependencies, so read them 264 # off the libtool archives instead of hardcoding a list here. Which ones 265 # are needed depends on what configure found (-lbz2, -lltdl, -ltidy, 266 # -lapparmor, ...), so a hardcoded list would silently rot. 267 le_deps () 268 { 269 sed -n "s/^dependency_libs='\(.*\)'\$/\1/p" "$1" 2>/dev/null 270 } 271 LE_SYSLIBS="$(le_deps "${BUILD}/src/main/libextractor.la") \ 272 $(le_deps "${BUILD}/src/common/libextractor_common.la")" 273 echo " LE_SYSLIBS = ${LE_SYSLIBS}" 274 275 # --------------------------------------------------------------------------- 276 # 4. Compile the harnesses as fuzzing-engine translation units 277 # --------------------------------------------------------------------------- 278 LE_INCLUDES=( 279 -I"${BUILD}" 280 -I"${LE_SRC}" 281 -I"${LE_SRC}/src/include" 282 -I"${LE_SRC}/src/common" 283 -I"${LE_SRC}/src/main" 284 -I"${LE_SRC}/src/plugins" 285 -I"${LE_SRC}/src/fuzz" 286 ) 287 288 # fuzz_extract is deliberately absent: it dlopen()s the installed plugin 289 # modules, which this build does not produce (--disable-shared), and a 290 # target that loads uninstrumented shared objects gives the engine no 291 # feedback from the code it is meant to be fuzzing. It is built and run 292 # in tree by "make -C src/fuzz check" instead. 293 CORE_FUZZERS="fuzz_datasource fuzz_ipc fuzz_convert fuzz_unzip" 294 295 # name:extra sources:extra libs (the plugin's own objects and its 296 # dependencies; keep in sync with src/fuzz/Makefile.am) 297 PLUGIN_FUZZERS=" 298 applefile:applefile_extractor.c pack.c: 299 dvi:dvi_extractor.c: 300 elf:elf_extractor.c pack.c: 301 it:it_extractor.c: 302 man:man_extractor.c: 303 nsf:nsf_extractor.c: 304 nsfe:nsfe_extractor.c: 305 ps:ps_extractor.c: 306 real:real_extractor.c: 307 riff:riff_extractor.c:-lm 308 rtf:rtf_extractor.c: 309 s3m:s3m_extractor.c: 310 sid:sid_extractor.c: 311 wav:wav_extractor.c: 312 xm:xm_extractor.c: 313 deb:deb_extractor.c:-lz 314 msoffice:msoffice_extractor.c:-lz 315 odf:odf_extractor.c:-lz 316 png:png_extractor.c:-lz 317 qt:qt_extractor.c:-lz 318 zip:zip_extractor.c:-lz 319 pecoff:pecoff_extractor.c forensics.c:-lm 320 lnk:lnk_extractor.c forensics.c:-lm 321 sqlite:sqlite_extractor.c forensics.c:-lm 322 tar:tar_extractor.c forensics.c:-lm 323 iso9660:iso9660_extractor.c forensics.c:-lm 324 diskimage:diskimage_extractor.c forensics.c:-lm 325 heif:heif_extractor.c forensics.c:-lm 326 webp:webp_extractor.c forensics.c:-lm 327 plist:plist_extractor.c forensics.c:-lm 328 id3:id3_extractor.c forensics.c:-lm 329 gpx:gpx_extractor.c forensics.c:-lm 330 kml:kml_extractor.c forensics.c:-lm 331 geotiff:geotiff_extractor.c forensics.c:-lm 332 mbox:mbox_extractor.c forensics.c:-lm 333 apk:apk_extractor.c forensics.c:-lm -lz 334 ebook:ebook_extractor.c forensics.c:-lm -lz 335 " 336 337 # Plugins that wrap a third-party parser. NOT part of the OSS-Fuzz build: 338 # giflib, libjpeg, libtiff, FLAC, libvorbis, libarchive and libmagic all 339 # have their own OSS-Fuzz projects, an uninstrumented copy of one would 340 # attribute its bugs to libextractor, and it would make the 341 # MemorySanitizer configuration impossible. Set LE_FUZZ_EXTRA_PLUGINS=1 342 # for a local campaign that wants the *glue* -- the part that turns what 343 # the library returns into a metadata callback, which is ours -- covered 344 # as well. 345 EXTRA_PLUGIN_FUZZERS=" 346 gif:gif_extractor.c:-lgif 347 jpeg:jpeg_extractor.c:-ljpeg 348 tiff:tiff_extractor.c:-ltiff 349 flac:flac_extractor.c:-lFLAC 350 ogg:ogg_extractor.c:-lvorbisfile -lvorbis -logg 351 archive:archive_extractor.c:-larchive 352 mime:mime_extractor.c:-lmagic 353 " 354 355 # ole2 is separate from the list above because it is the one plugin that 356 # needs extra *compiler* flags rather than just a -l, so it can only be 357 # built when configure kept libgsf (LE_FUZZ_GSF=1). It is worth the 358 # trouble: ole2_extractor.c parses the header fields itself before it 359 # hands the stream to libgsf, and a signed-overflow defect sat in exactly 360 # that code until a campaign finally covered this target. 361 GSF_PLUGIN_FUZZER="" 362 if [ "${LE_FUZZ_GSF:-0}" != "0" ]; then 363 if _le_gsf_cflags="$(pkg-config --cflags libgsf-1 2>/dev/null)" && 364 _le_gsf_libs="$(pkg-config --libs libgsf-1 2>/dev/null)"; then 365 GSF_PLUGIN_FUZZER=" 366 ole2:ole2_extractor.c:${_le_gsf_libs} 367 " 368 LE_GSF_CFLAGS="${_le_gsf_cflags}" 369 else 370 echo "WARNING: LE_FUZZ_GSF=1 but pkg-config cannot find libgsf-1;" >&2 371 echo " the ole2 target will be skipped." >&2 372 fi 373 fi 374 375 if [ "${LE_FUZZ_EXTRA_PLUGINS:-0}" != "0" ]; then 376 PLUGIN_FUZZERS="${PLUGIN_FUZZERS}${EXTRA_PLUGIN_FUZZERS}" 377 fi 378 PLUGIN_FUZZERS="${PLUGIN_FUZZERS}${GSF_PLUGIN_FUZZER}" 379 380 # Upper-case the plugin name for -DLE_FUZZ_ID. 381 upper () { echo "$1" | tr '[:lower:]' '[:upper:]'; } 382 383 build_one () 384 { 385 local target="$1"; shift 386 local objs="" 387 local extra_libs="$1"; shift 388 local o 389 390 for src in "$@"; do 391 o="${WORK}/${target}-$(basename "${src}" .c).o" 392 # shellcheck disable=SC2086 393 $CC $CFLAGS ${_le_no_main} ${LE_EXTRA_CPPFLAGS:-} \ 394 "${LE_INCLUDES[@]}" -c "${src}" -o "${o}" 395 objs="${objs} ${o}" 396 done 397 # Link with $CXX: $LIB_FUZZING_ENGINE is a C++ archive for most engines. 398 # shellcheck disable=SC2086 399 $CXX $CXXFLAGS ${objs} -o "${OUT}/${target}" \ 400 $LIB_FUZZING_ENGINE "${LE_LIB}" "${LE_COMMON}" ${extra_libs} \ 401 ${LE_SYSLIBS} -lz -lpthread -ldl 402 } 403 404 FUZZERS="" 405 406 for f in ${CORE_FUZZERS}; do 407 echo "--- building ${f} ---" 408 LE_EXTRA_CPPFLAGS="" \ 409 build_one "${f}" "" "${LE_SRC}/src/fuzz/${f}.c" 410 FUZZERS="${FUZZERS} ${f}" 411 done 412 413 # Iterate line by line, not word by word: the middle field holds a 414 # space-separated source list. A pipe into `while read` would put the 415 # loop in a subshell and lose $FUZZERS, hence the IFS dance. 416 _le_oldifs="${IFS}" 417 IFS=' 418 ' 419 for line in ${PLUGIN_FUZZERS}; do 420 IFS="${_le_oldifs}" 421 [ -n "${line}" ] || { IFS=' 422 '; continue; } 423 name="${line%%:*}" 424 rest="${line#*:}" 425 srcs="${rest%%:*}" 426 libs="${rest#*:}" 427 target="fuzz_${name}" 428 echo "--- building ${target} ---" 429 set -- 430 for s in ${srcs}; do 431 set -- "$@" "${LE_SRC}/src/plugins/${s}" 432 done 433 # ole2_extractor.c includes <gsf/gsf-*.h>, so it is the one plugin that 434 # needs include paths of its own; everything else compiles with the 435 # library's flags alone. 436 _le_plug_cppflags="-DLE_FUZZ_PLUGIN=${name} -DLE_FUZZ_ID=$(upper "${name}")" 437 if [ "${name}" = "ole2" ]; then 438 _le_plug_cppflags="${_le_plug_cppflags} ${LE_GSF_CFLAGS:-}" 439 fi 440 LE_EXTRA_CPPFLAGS="${_le_plug_cppflags}" \ 441 build_one "${target}" "${libs}" "${LE_SRC}/src/fuzz/fuzz_plugin.c" "$@" 442 FUZZERS="${FUZZERS} ${target}" 443 IFS=' 444 ' 445 done 446 IFS="${_le_oldifs}" 447 unset _le_oldifs 448 449 # --------------------------------------------------------------------------- 450 # 5. Seed corpora, dictionaries and .options files 451 # --------------------------------------------------------------------------- 452 FUZZBIN="${OUT}" "${LE_SRC}/contrib/oss-fuzz/make_seed_corpus.sh" \ 453 "${LE_SRC}" "${OUT}" 454 455 for f in ${FUZZERS}; do 456 d="${LE_SRC}/contrib/oss-fuzz/dicts/${f}.dict" 457 [ -f "${d}" ] || d="${LE_SRC}/contrib/oss-fuzz/dicts/fuzz_plugin.dict" 458 cp "${d}" "${OUT}/${f}.dict" 459 o="${LE_SRC}/contrib/oss-fuzz/${f}.options" 460 [ -f "${o}" ] || o="${LE_SRC}/contrib/oss-fuzz/default.options" 461 cp "${o}" "${OUT}/${f}.options" 462 done 463 464 # --------------------------------------------------------------------------- 465 # 6. Verify: every requested target must actually be in $OUT 466 # --------------------------------------------------------------------------- 467 # Belt and braces for the failure that matters most -- a build that 468 # reports success but ships nothing, which on OSS-Fuzz shows up only as a 469 # target that never runs. 470 _le_missing="" 471 for f in ${FUZZERS}; do 472 [ -x "${OUT}/${f}" ] || _le_missing="${_le_missing} ${f}" 473 [ -f "${OUT}/${f}_seed_corpus.zip" ] || 474 _le_missing="${_le_missing} ${f}_seed_corpus.zip" 475 done 476 if [ -n "${_le_missing}" ]; then 477 echo "ERROR: build did not produce:${_le_missing}" >&2 478 exit 1 479 fi 480 unset _le_missing 481 482 echo "=== done; contents of \$OUT ===" 483 ls -la "${OUT}"