make_seed_corpus.sh (9693B)
1 #!/bin/sh 2 # 3 # Build the seed corpora for the libextractor fuzz targets. 4 # 5 # This file is in the public domain. 6 # 7 # Usage: 8 # make_seed_corpus.sh SRCDIR OUTDIR [--plain] 9 # 10 # SRCDIR the libextractor source tree 11 # OUTDIR where to write <target>_seed_corpus.zip (OSS-Fuzz layout), 12 # or, with --plain, where to write the loose corpus files 13 # --plain write loose files into OUTDIR/<target>/ instead of zips; 14 # this is what "make -C src/fuzz refresh-corpus" wants 15 # 16 # Two things go into a corpus: 17 # 18 # 1. the harness' own built-in seeds, if the built binaries are 19 # available (they can write them out with --write-corpus=DIR), and 20 # 21 # 2. the real files under src/plugins/testdata/, which are the whole 22 # point: a fuzzer that has to *invent* a valid OLE2 FAT or a valid 23 # ZIP central directory before it reaches the interesting code will 24 # not get there in any reasonable time. Each is prefixed with the 25 # configuration bytes the harness expects, all zero, which selects 26 # exactly what production does (see fuzz_ec.h). 27 # 28 # The mapping from testdata file to target is by filename prefix, which 29 # is the convention src/plugins/testdata/ already follows. 30 31 set -eu 32 33 SRCDIR="${1:?usage: make_seed_corpus.sh SRCDIR OUTDIR [--plain]}" 34 OUTDIR="${2:?usage: make_seed_corpus.sh SRCDIR OUTDIR [--plain]}" 35 MODE="${3:-zip}" 36 37 TESTDATA="${SRCDIR}/src/plugins/testdata" 38 FUZZBIN="${FUZZBIN:-}" 39 40 mkdir -p "${OUTDIR}" 41 WORK=$(mktemp -d) 42 trap 'rm -rf "${WORK}"' EXIT 43 44 # Number of leading zero bytes each harness wants in front of the file 45 # image. Keep in sync with LE_FUZZ_EC_PREFIX (fuzz_ec.h) and with the 46 # input formats documented at the top of the core harnesses. 47 prefix_len () 48 { 49 case "$1" in 50 fuzz_extract) echo 2 ;; 51 fuzz_datasource) echo 4 ;; 52 fuzz_ipc) echo 0 ;; 53 fuzz_convert) echo 2 ;; 54 *) echo 4 ;; # fuzz_unzip and every fuzz_<plugin> 55 esac 56 } 57 58 # How many configurations the harness offers, i.e. the length of 59 # LE_FUZZ_CONFIGS in src/fuzz/fuzz_plugin_name.h. The fourth prefix byte 60 # selects one of them (fuzz_plugin.c), so a target listed here gets one 61 # seed per configuration instead of just the default. Keep in sync with 62 # that header: a count that is too high only wastes a few duplicate 63 # seeds (the index is taken modulo the real length), one that is too low 64 # silently leaves a configuration unseeded. 65 num_configs () 66 { 67 case "$1" in 68 fuzz_dvi | fuzz_pdf | fuzz_html | fuzz_odf | fuzz_ole2 | fuzz_ebook) 69 echo 2 ;; # NULL, "fulltext" 70 fuzz_thumbnailffmpeg | fuzz_thumbnailgst) 71 echo 5 ;; # NULL, srt, 3x gallery 72 *) echo 1 ;; 73 esac 74 } 75 76 # Which testdata files seed which target. A target with no entry gets 77 # only its built-in seeds. 78 testdata_glob () 79 { 80 case "$1" in 81 fuzz_applefile) echo "applefile_*" ;; 82 fuzz_archive) echo "archive_*" ;; 83 fuzz_deb) echo "deb_*" ;; 84 fuzz_dvi) echo "dvi_*" ;; 85 fuzz_elf) echo "chello-elf" ;; 86 fuzz_flac) echo "flac_*" ;; 87 fuzz_gif) echo "gif_*" ;; 88 fuzz_html) echo "html_*" ;; 89 fuzz_it) echo "it_*" ;; 90 fuzz_jpeg) echo "jpeg_* exiv2_* thumbnail_*" ;; 91 fuzz_man) echo "man_*" ;; 92 fuzz_mime) echo "*" ;; 93 fuzz_mpeg) echo "mpeg_*" ;; 94 fuzz_msoffice) echo "msoffice_*" ;; 95 fuzz_nsf) echo "nsf_*" ;; 96 fuzz_nsfe) echo "nsfe_*" ;; 97 fuzz_odf) echo "odf_*" ;; 98 fuzz_ogg) echo "ogg_*" ;; 99 fuzz_ole2) echo "ole2_* msoffice_biff4.xls" ;; 100 fuzz_pdf) echo "pdf_*" ;; 101 fuzz_png) echo "png_*" ;; 102 fuzz_ps) echo "ps_*" ;; 103 fuzz_qt) echo "gstreamer_sample_sorenson.mov" ;; 104 fuzz_real) echo "ra3.ra audiosig.rm" ;; 105 fuzz_riff) echo "riff_* wav_*" ;; 106 fuzz_rtf) echo "rtf_*" ;; 107 fuzz_s3m) echo "s3m_*" ;; 108 fuzz_sid) echo "sid_*" ;; 109 fuzz_tiff) echo "tiff_*" ;; 110 fuzz_wav) echo "wav_*" ;; 111 fuzz_xm) echo "xm_*" ;; 112 fuzz_zip) echo "zip_* odf_* msoffice_excel.xlsx msoffice_word.docx" ;; 113 fuzz_unzip) echo "zip_* odf_* msoffice_excel.xlsx msoffice_word.docx \ 114 msoffice_powerpoint.pptx" ;; 115 fuzz_datasource) echo "deb_* zip_* png_* ogg_*" ;; 116 fuzz_pecoff) echo "pecoff_*" ;; 117 fuzz_lnk) echo "lnk_*" ;; 118 fuzz_sqlite) echo "sqlite_*" ;; 119 fuzz_tar) echo "tar_* archive_test.tar" ;; 120 fuzz_thumbnailffmpeg) echo "thumbnail_* subtitles.mkv" ;; 121 fuzz_iso9660) echo "iso9660_*" ;; 122 fuzz_diskimage) echo "diskimage_*" ;; 123 fuzz_heif) echo "heif_*" ;; 124 fuzz_webp) echo "webp_*" ;; 125 fuzz_plist) echo "plist_*" ;; 126 fuzz_id3) echo "id3_*" ;; 127 fuzz_gpx) echo "gpx_*" ;; 128 fuzz_kml) echo "kml_*" ;; 129 fuzz_geotiff) echo "geotiff_* tiff_*" ;; 130 fuzz_mbox) echo "mbox_*" ;; 131 fuzz_apk) echo "apk_*" ;; 132 fuzz_ebook) echo "ebook_*" ;; 133 fuzz_extract) echo "*" ;; 134 *) echo "" ;; 135 esac 136 } 137 138 # All targets we know about. Kept explicit rather than derived from the 139 # build so that this script also works before anything is built. 140 TARGETS="fuzz_extract fuzz_datasource fuzz_ipc fuzz_convert fuzz_unzip \ 141 fuzz_applefile fuzz_dvi fuzz_elf fuzz_it fuzz_man fuzz_nsf fuzz_nsfe \ 142 fuzz_ps fuzz_real fuzz_riff fuzz_rtf fuzz_s3m fuzz_sid fuzz_wav fuzz_xm \ 143 fuzz_deb fuzz_msoffice fuzz_odf fuzz_png fuzz_qt fuzz_zip \ 144 fuzz_gif fuzz_jpeg fuzz_tiff fuzz_flac fuzz_ogg fuzz_archive fuzz_mime \ 145 fuzz_ole2 fuzz_mpeg fuzz_html fuzz_pdf \ 146 fuzz_pecoff fuzz_lnk fuzz_sqlite fuzz_tar fuzz_iso9660 fuzz_diskimage \ 147 fuzz_heif fuzz_webp fuzz_plist fuzz_id3 fuzz_gpx fuzz_kml fuzz_geotiff \ 148 fuzz_mbox fuzz_apk fuzz_ebook" 149 150 for t in ${TARGETS}; do 151 d="${WORK}/${t}" 152 mkdir -p "${d}" 153 154 # 1a. the checked-in corpus, which "make -C src/fuzz refresh-corpus" 155 # regenerates from the harnesses' built-in seeds 156 if [ -d "${SRCDIR}/src/fuzz/corpus/${t}" ]; then 157 cp "${SRCDIR}/src/fuzz/corpus/${t}"/* "${d}/" 2>/dev/null || true 158 fi 159 160 # 1b. built-in seeds straight from the binary, if one that understands 161 # --write-corpus is around. A binary linked against libFuzzer does 162 # NOT: our main() is compiled out, libFuzzer's own argument parser 163 # sees an unknown flag and starts fuzzing, so this must be bounded. 164 if [ -n "${FUZZBIN}" ] && [ -x "${FUZZBIN}/${t}" ]; then 165 timeout 10 "${FUZZBIN}/${t}" --write-corpus="${d}" >/dev/null 2>&1 || true 166 fi 167 168 # 2. testdata, with the configuration prefix prepended 169 n=$(prefix_len "${t}") 170 if [ "${n}" -gt 0 ]; then 171 # a file of n zero bytes 172 : > "${WORK}/prefix" 173 i=0 174 while [ "${i}" -lt "${n}" ]; do 175 printf '\000' >> "${WORK}/prefix" 176 i=$((i + 1)) 177 done 178 fi 179 # `set -f` matters: the glob list comes back from a command 180 # substitution and would otherwise be expanded against the *current* 181 # directory before the loop ever starts. 182 set -f 183 for g in $(testdata_glob "${t}"); do 184 set +f 185 for f in "${TESTDATA}"/${g}; do 186 [ -f "${f}" ] || continue 187 case "${f}" in 188 */README) continue ;; 189 esac 190 b=$(basename "${f}") 191 if [ "${n}" -gt 0 ]; then 192 # One copy per plugin configuration. An all-zero prefix selects 193 # configuration 0, which for every plugin is "no options" -- so 194 # without this loop a target whose interesting code is reached 195 # only through an option would be seeded entirely in the dark. 196 # It is not a small effect: on the fulltext plugins the 197 # "fulltext" configuration is worth between +285 (ole2) and 198 # +1938 (pdf) edges over the default, i.e. most of what those 199 # plugins can execute at all. The fuzzer can flip the byte 200 # itself, but it would be starting from a corpus in which none 201 # of that code has ever run, and corpus distillation keeps 202 # pulling it back to the basin it already knows. 203 nc=$(num_configs "${t}") 204 c=0 205 while [ "${c}" -lt "${nc}" ]; do 206 if [ "${c}" -eq 0 ]; then 207 cat "${WORK}/prefix" "${f}" > "${d}/td-${b}" 208 else 209 # prefix with the last byte replaced by the config index 210 dd if="${WORK}/prefix" bs=1 count=$((n - 1)) 2>/dev/null \ 211 > "${WORK}/cprefix" 212 printf "\\$(printf '%03o' "${c}")" >> "${WORK}/cprefix" 213 cat "${WORK}/cprefix" "${f}" > "${d}/td-c${c}-${b}" 214 fi 215 c=$((c + 1)) 216 done 217 else 218 cp "${f}" "${d}/td-${b}" 219 fi 220 done 221 set -f 222 done 223 set +f 224 225 # 3. reproducers of past findings, so that they stay permanent 226 # regressions in the OSS-Fuzz corpus as well 227 kf="${SRCDIR}/src/fuzz/corpus/known-findings" 228 if [ -d "${kf}" ]; then 229 for f in "${kf}/${t}"-*; do 230 [ -f "${f}" ] || continue 231 cp "${f}" "${d}/$(basename "${f}")" 232 done 233 fi 234 235 # A target whose binary was not built and that has no testdata mapping 236 # would otherwise produce an empty archive, which `zip` refuses to 237 # create; give it a one-byte seed so that the corpus always exists. 238 if [ -z "$(ls -A "${d}" 2>/dev/null)" ]; then 239 printf '\000' > "${d}/empty" 240 fi 241 242 if [ "${MODE}" = "--plain" ]; then 243 mkdir -p "${OUTDIR}/${t}" 244 cp "${d}"/* "${OUTDIR}/${t}/" 245 echo "corpus: ${OUTDIR}/${t} ($(ls -1 "${OUTDIR}/${t}" | wc -l) files)" 246 else 247 (cd "${d}" && zip -q -r "${OUTDIR}/${t}_seed_corpus.zip" . ) 248 echo "corpus: ${OUTDIR}/${t}_seed_corpus.zip ($(ls -1 "${d}" | wc -l) files)" 249 fi 250 done