run_campaign.sh (13665B)
1 #!/bin/bash 2 # 3 # Run a weighted libextractor fuzzing campaign. 4 # 5 # This file is in the public domain. 6 # 7 # The weights are not guesses. They come from a measured 12-core hour on 8 # the 2026-07-29 tree: every target was given an identical 1200 s slice, 9 # the resulting corpora were replayed through a coverage build, and each 10 # target was credited with the library regions still uncovered in the 11 # source files it covers better than any other target. See 12 # ../../src/fuzz/CAMPAIGN.md for the data and for why a flat allocation 13 # wastes most of its budget. 14 # 15 # Re-measure with: 16 # src/fuzz/CAMPAIGN.md, section "Reproducing the measurement" 17 # 18 set -u 19 20 BIN="${BIN:-}" 21 OUT="${OUT:-./campaign}" 22 CORES="${CORES:-$(nproc)}" 23 HOURS="${HOURS:-}" 24 PROFILE="${PROFILE:-nightly}" 25 WEIGHTS_FILE="" 26 FLOOR="${FLOOR:-300}" 27 SHARD="${SHARD:-1800}" 28 DRYRUN=0 29 30 usage () 31 { 32 cat <<EOF 33 usage: $0 -b BINDIR [-o OUTDIR] [-c CORES] [-t HOURS] [-p PROFILE] [-w FILE] 34 35 -b BINDIR directory holding the built fuzz targets (build.sh's \$OUT) 36 -o OUTDIR where corpora, logs and artifacts go (default ./campaign) 37 -c CORES concurrent targets (default nproc) 38 -t HOURS wall-clock budget; overrides -p 39 -p PROFILE ci | nightly | deep (default nightly) 40 -w FILE weight table "target weight" per line, overrides the built-in 41 -F SECONDS per-target floor (default 300) 42 -S SECONDS shard size: a target allotted more than this is run as 43 several INDEPENDENT jobs of this length, each with its own 44 corpus and PRNG seed, instead of one long one. 0 disables. 45 Default 1800; see CAMPAIGN.md sections 9 and 10. 46 -n print the schedule and exit; run nothing 47 48 Profiles, in wall-clock hours on \$CORES cores: 49 50 ci every target gets the floor and nothing more. On 12 cores 51 this is about 15 minutes and it recovers ~93% of the edge 52 coverage a twenty-minute-per-target run reaches. Cheap 53 enough to run on every push. 54 nightly 2 h. The floor plus a headroom-weighted share. 55 deep 8 h. Past this the fitted curve says each additional 1% of 56 edge coverage costs more than 30 core-hours; spend it on 57 better generators or new targets instead. 58 EOF 59 exit "${1:-1}" 60 } 61 62 while getopts "b:o:c:t:p:w:F:S:nh" o; do 63 case "$o" in 64 b) BIN="$OPTARG" ;; 65 o) OUT="$OPTARG" ;; 66 c) CORES="$OPTARG" ;; 67 t) HOURS="$OPTARG" ;; 68 p) PROFILE="$OPTARG" ;; 69 w) WEIGHTS_FILE="$OPTARG" ;; 70 F) FLOOR="$OPTARG" ;; 71 S) SHARD="$OPTARG" ;; 72 n) DRYRUN=1 ;; 73 h) usage 0 ;; 74 *) usage ;; 75 esac 76 done 77 78 [ -n "${BIN}" ] || usage 79 [ -d "${BIN}" ] || { echo "no such directory: ${BIN}" >&2; exit 1; } 80 81 if [ -z "${HOURS}" ]; then 82 case "${PROFILE}" in 83 ci) HOURS=0 ;; 84 nightly) HOURS=2 ;; 85 deep) HOURS=8 ;; 86 *) echo "unknown profile: ${PROFILE}" >&2; usage ;; 87 esac 88 fi 89 90 # Addressable headroom in library regions, measured 2026-08-07. 91 # 92 # Derived from the corpus the three campaigns of that day accumulated: 93 # the flat 1200 s-per-target measurement run, the tuned four-hour run, 94 # and the two-hour sharded run that started from the first two distilled 95 # together. Each target is credited with the regions still uncovered in 96 # the files it covers better than any other target. Union coverage: 97 # 17595 of 18889 regions, 93.1%. 98 # 99 # "Addressable" excludes two things that no runtime can reach: 100 # plugins/pack.c 89 regions of format codes that neither of its 101 # two callers ever asks for (fixed format strings) 102 # extractor_logging.c 2 regions live only in a debug build 103 # The third-party wrappers (gif jpeg tiff flac ogg archive mime) are 104 # damped to 35%: their uncovered regions are almost all error returns 105 # from giflib/libjpeg/libtiff/FLAC/libvorbis/libarchive/libmagic, which 106 # those projects fuzz themselves, and reaching them from a file input is 107 # disproportionately expensive. 108 # 109 # Treat differences under about 20% as noise. The staged plugins repeat 110 # to nothing like the +-2% the legacy targets manage; fuzz_ebook varied 111 # 16.6% between two runs that differed only in PRNG seed. These numbers 112 # are for apportioning a budget, not for ranking anything. 113 # 114 # NOTE ON THE FLOOR. The 300 s default was derived from the legacy 115 # targets, whose median time to 99% of final edge coverage is 93 s. The 116 # staged plugins take a median of 690 s, and eleven of the sixteen were 117 # still gaining edges at 1200 s. Use -F 900 while they are in the set. 118 # 119 # NOTE ON SHARDING. See -S and CAMPAIGN.md sections 9 and 10. Several 120 # independent short runs beat one long one when the corpus is small 121 # (section 9); once the corpus is mature the shards all converge to 122 # within half a percent and the seed stops mattering (section 10). Keep 123 # -S on regardless: it costs nothing and it is what stops a single long 124 # job from plateauing and burning hours on nothing. Just do not expect 125 # it to buy coverage from an already-good corpus. 126 read -r -d '' BUILTIN_WEIGHTS <<'EOF' 127 fuzz_diskimage 197 128 fuzz_ebook 87 129 fuzz_apk 78 130 fuzz_pecoff 78 131 fuzz_datasource 74 132 fuzz_msoffice 74 133 fuzz_lnk 67 134 fuzz_unzip 56 135 fuzz_rtf 39 136 fuzz_geotiff 33 137 fuzz_id3 30 138 fuzz_kml 30 139 fuzz_ole2 28 140 fuzz_gpx 26 141 fuzz_ogg 26 142 fuzz_mbox 23 143 fuzz_tar 20 144 fuzz_qt 19 145 fuzz_heif 16 146 fuzz_flac 14 147 fuzz_png 12 148 fuzz_mime 10 149 fuzz_plist 10 150 fuzz_dvi 9 151 fuzz_deb 7 152 fuzz_convert 6 153 fuzz_webp 6 154 fuzz_iso9660 5 155 fuzz_elf 4 156 fuzz_nsfe 4 157 fuzz_odf 4 158 fuzz_man 3 159 fuzz_ps 3 160 fuzz_real 2 161 fuzz_sid 2 162 fuzz_gif 1 163 fuzz_it 1 164 fuzz_jpeg 1 165 fuzz_nsf 1 166 fuzz_riff 1 167 fuzz_sqlite 1 168 fuzz_applefile 0 169 fuzz_archive 0 170 fuzz_ipc 0 171 fuzz_s3m 0 172 fuzz_tiff 0 173 fuzz_wav 0 174 fuzz_xm 0 175 fuzz_zip 0 176 EOF 177 178 if [ -n "${WEIGHTS_FILE}" ]; then 179 WEIGHTS="$(cat "${WEIGHTS_FILE}")" 180 else 181 WEIGHTS="${BUILTIN_WEIGHTS}" 182 fi 183 184 mkdir -p "${OUT}/corpus" "${OUT}/logs" "${OUT}/artifacts" 185 export TMPDIR="${OUT}/tmp" 186 mkdir -p "${TMPDIR}" 187 188 # Only schedule targets that were actually built. 189 AVAIL="" 190 while read -r t w; do 191 [ -n "${t}" ] || continue 192 [ -x "${BIN}/${t}" ] || continue 193 AVAIL="${AVAIL}${t} ${w} 194 " 195 done <<EOF 196 ${WEIGHTS} 197 EOF 198 199 NTARGETS=$(printf '%s' "${AVAIL}" | grep -c . || true) 200 [ "${NTARGETS}" -gt 0 ] || { echo "no targets found in ${BIN}" >&2; exit 1; } 201 202 BUDGET=$(awk -v c="${CORES}" -v h="${HOURS}" 'BEGIN{printf "%d", c*h*3600}') 203 # No single job may outlast the campaign: with -t 8 a target given nine 204 # hours would still be running when everything else has finished, and 205 # the run would take nine hours rather than the eight that were asked 206 # for. Hence water-filling -- share out the budget by weight, clamp 207 # anything over the wall clock, share the remainder among the rest, and 208 # repeat until nothing else clamps. 209 CAP=$(awk -v h="${HOURS}" 'BEGIN{printf "%d", (h>0 ? h*3600 : 0)}') 210 211 : > "${OUT}/jobs.txt" 212 printf '%s' "${AVAIL}" | awk -v floor="${FLOOR}" -v budget="${BUDGET}" \ 213 -v cap="${CAP}" ' 214 { name[NR]=$1; w[NR]=$2; n=NR } 215 END { 216 if (cap <= 0 || cap < floor) cap = floor; # ci profile: floor only 217 for (i=1; i<=n; i++) { s[i]=floor; capped[i]=(floor>=cap) } 218 extra = budget - floor*n 219 if (extra < 0) extra = 0 220 for (round=0; round<64 && extra>0; round++) { 221 tw = 0 222 for (i=1; i<=n; i++) if (!capped[i]) tw += w[i] 223 if (tw <= 0) break 224 spill = 0; moved = 0 225 for (i=1; i<=n; i++) { 226 if (capped[i]) continue 227 add = extra * w[i] / tw 228 if (s[i] + add >= cap) { spill += s[i] + add - cap; s[i] = cap; capped[i]=1 } 229 else { s[i] += add } 230 moved = 1 231 } 232 if (!moved) break 233 extra = spill 234 } 235 for (i=1; i<=n; i++) printf "%s:%d\n", name[i], s[i] 236 }' >> "${OUT}/jobs.txt" 237 238 # Split anything longer than the shard size into that many INDEPENDENT 239 # runs -- separate corpus, separate PRNG seed -- rather than one long 240 # one. This is not a scheduling nicety, it is the single biggest effect 241 # the 2026-08-07 measurements found (src/fuzz/CAMPAIGN.md section 9): 242 # 243 # fuzz_ebook 2 x 1200 s -> 678 of 775 regions 244 # 1 x 14089 s -> 595 245 # fuzz_apk 2 x 1200 s -> 824 of 927 regions 246 # 1 x 14407 s -> 792 247 # 248 # Forty core-minutes beating four core-hours. A long libFuzzer run 249 # commits early to one corpus and cannot leave it; the 14089 s ebook job 250 # stopped finding edges after 2570 s and spent the remaining 82% of its 251 # slice on 195 million executions that added nothing. Independent runs 252 # start from different trajectories, and their corpora are unioned 253 # afterwards. 254 if [ "${SHARD}" -gt 0 ]; then 255 awk -F: -v shard="${SHARD}" ' 256 { 257 secs = $2 258 k = int((secs + shard - 1) / shard) # ceil 259 if (k < 1) k = 1 260 each = int(secs / k) 261 for (j = 1; j <= k; j++) printf "%s:%d:%d\n", $1, each, j 262 }' "${OUT}/jobs.txt" > "${OUT}/jobs.tmp" 263 mv "${OUT}/jobs.tmp" "${OUT}/jobs.txt" 264 else 265 awk -F: '{ printf "%s:%d:1\n", $1, $2 }' "${OUT}/jobs.txt" > "${OUT}/jobs.tmp" 266 mv "${OUT}/jobs.tmp" "${OUT}/jobs.txt" 267 fi 268 269 # Longest first: with a fixed number of slots the long jobs must start 270 # early or the tail of the campaign runs on one core. 271 sort -t: -k2 -rn -o "${OUT}/jobs.txt" "${OUT}/jobs.txt" 272 273 echo "=== libextractor campaign ===" 274 echo " targets ${NTARGETS}" 275 echo " cores ${CORES}" 276 echo " profile ${PROFILE} (${HOURS} h wall => $(awk -v b=${BUDGET} 'BEGIN{printf "%.1f", b/3600}') core-hours)" 277 echo " floor ${FLOOR}s" 278 echo " shard $([ "${SHARD}" -gt 0 ] && echo "${SHARD}s" || echo "off")" 279 echo " jobs $(wc -l < "${OUT}/jobs.txt")" 280 echo " out ${OUT}" 281 awk -F: -v c="${CORES}" ' 282 { printf " %-18s %6d s %2d:%02d shard %d\n", $1, $2, $2/3600, ($2%3600)/60, $3; s+=$2 } 283 END { printf " %-18s %6.1f core-hours over %d cores => %.1f h wall (perfect packing)\n", 284 "TOTAL", s/3600, c, s/3600/c } 285 ' "${OUT}/jobs.txt" 286 287 if [ "${DRYRUN}" = "1" ]; then 288 echo "(dry run: nothing executed)" 289 exit 0 290 fi 291 292 # Every shard gets its own copy of the seed corpus: the shards must stay 293 # independent for the whole point of sharding to hold. 294 while IFS=: read -r t secs sh; do 295 [ -n "${t}" ] || continue 296 z="${BIN}/${t}_seed_corpus.zip" 297 mkdir -p "${OUT}/corpus/${t}-${sh}" 298 [ -f "${z}" ] && unzip -qo "${z}" -d "${OUT}/corpus/${t}-${sh}" 2>/dev/null 299 done < "${OUT}/jobs.txt" 300 301 run_one () 302 { 303 local spec="$1" t secs sh dict maxlen leaks rc 304 t="${spec%%:*}" 305 secs="${spec#*:}"; secs="${secs%%:*}" 306 sh="${spec##*:}" 307 dict="" 308 [ -f "${BIN}/${t}.dict" ] && dict="-dict=${BIN}/${t}.dict" 309 maxlen=262144 310 case "${t}" in 311 fuzz_convert) maxlen=4096 ;; 312 fuzz_ipc) maxlen=65536 ;; 313 esac 314 # ole2 drags in glib, whose one-time allocations LeakSanitizer reports 315 # as leaks on every input. They are not ours and they bury everything 316 # else, so leak detection is off for that target only. 317 leaks=1 318 [ "${t}" = "fuzz_ole2" ] && leaks=0 319 mkdir -p "${OUT}/artifacts/${t}" 320 echo "START ${t}#${sh} (${secs}s)" 321 # -seed is what makes two shards of the same target explore 322 # differently; without it libFuzzer picks one from the clock and two 323 # shards started in the same second would be identical runs. 324 ASAN_OPTIONS="allocator_may_return_null=0:detect_stack_use_after_return=1:detect_leaks=${leaks}" \ 325 UBSAN_OPTIONS="print_stacktrace=1:report_error_type=1" \ 326 "${BIN}/${t}" "${OUT}/corpus/${t}-${sh}" \ 327 -max_total_time="${secs}" -fork=1 -seed="${sh}" \ 328 -ignore_crashes=1 -ignore_ooms=1 -ignore_timeouts=1 \ 329 -rss_limit_mb=2560 -timeout=25 -max_len="${maxlen}" \ 330 -print_final_stats=1 \ 331 -artifact_prefix="${OUT}/artifacts/${t}/" \ 332 ${dict} > "${OUT}/logs/${t}-${sh}.log" 2>&1 333 rc=$? 334 echo "DONE ${t}#${sh} rc=${rc} artifacts=$(ls -1 "${OUT}/artifacts/${t}" 2>/dev/null | wc -l)" 335 } 336 export -f run_one 337 export BIN OUT 338 339 date +%s > "${OUT}/started_at" 340 xargs -a "${OUT}/jobs.txt" -P "${CORES}" -I{} \ 341 bash -c 'run_one "$@"' _ {} 2>&1 | tee "${OUT}/campaign.log" 342 date +%s > "${OUT}/finished_at" 343 344 # Union the shards back together. The whole reason for running them 345 # apart is that they explore differently, so the merged corpus is worth 346 # more than any single shard's -- this is what a coverage replay and the 347 # next campaign's seed corpus should use. 348 if [ "${SHARD}" -gt 0 ]; then 349 for d in "${OUT}"/corpus/*-[0-9]*; do 350 [ -d "${d}" ] || continue 351 b=$(basename "${d}"); t="${b%-*}" 352 mkdir -p "${OUT}/corpus-merged/${t}" 353 cp "${d}"/* "${OUT}/corpus-merged/${t}/" 2>/dev/null || true 354 done 355 echo "merged shard corpora into ${OUT}/corpus-merged" 356 fi 357 358 echo 359 echo "=== campaign finished in $(( $(cat "${OUT}/finished_at") - $(cat "${OUT}/started_at") ))s ===" 360 echo "artifacts: $(find "${OUT}/artifacts" -type f | wc -l)" 361 cat <<EOF 362 363 Now run the replay pass. -fork=1 reads only the child's exit status, so 364 a *recovering* UndefinedBehaviorSanitizer report leaves no artifact and 365 no trace in the log -- and seven of the twelve defects fixed in this 366 library were exactly that. The campaign above finds memory-safety bugs; 367 this finds the rest: 368 369 for d in ${OUT}/corpus/*; do 370 t=\$(basename "\$d") 371 UBSAN_OPTIONS=halt_on_error=0 ASAN_OPTIONS=halt_on_error=0 \\ 372 ./src/fuzz/\$t --corpus-dir="\$d" 373 done 2>&1 | grep "runtime error" 374 EOF