make_seed_corpus.sh (7545B)
1 #!/bin/sh 2 # 3 # Build the seed corpora for the libextractor fuzz targets. 4 # 5 # This file is in the public domain. 6 # 7 # Usage: 8 # make_seed_corpus.sh SRCDIR OUTDIR [--plain] 9 # 10 # SRCDIR the libextractor source tree 11 # OUTDIR where to write <target>_seed_corpus.zip (OSS-Fuzz layout), 12 # or, with --plain, where to write the loose corpus files 13 # --plain write loose files into OUTDIR/<target>/ instead of zips; 14 # this is what "make -C src/fuzz refresh-corpus" wants 15 # 16 # Two things go into a corpus: 17 # 18 # 1. the harness' own built-in seeds, if the built binaries are 19 # available (they can write them out with --write-corpus=DIR), and 20 # 21 # 2. the real files under src/plugins/testdata/, which are the whole 22 # point: a fuzzer that has to *invent* a valid OLE2 FAT or a valid 23 # ZIP central directory before it reaches the interesting code will 24 # not get there in any reasonable time. Each is prefixed with the 25 # configuration bytes the harness expects, all zero, which selects 26 # exactly what production does (see fuzz_ec.h). 27 # 28 # The mapping from testdata file to target is by filename prefix, which 29 # is the convention src/plugins/testdata/ already follows. 30 31 set -eu 32 33 SRCDIR="${1:?usage: make_seed_corpus.sh SRCDIR OUTDIR [--plain]}" 34 OUTDIR="${2:?usage: make_seed_corpus.sh SRCDIR OUTDIR [--plain]}" 35 MODE="${3:-zip}" 36 37 TESTDATA="${SRCDIR}/src/plugins/testdata" 38 FUZZBIN="${FUZZBIN:-}" 39 40 mkdir -p "${OUTDIR}" 41 WORK=$(mktemp -d) 42 trap 'rm -rf "${WORK}"' EXIT 43 44 # Number of leading zero bytes each harness wants in front of the file 45 # image. Keep in sync with LE_FUZZ_EC_PREFIX (fuzz_ec.h) and with the 46 # input formats documented at the top of the core harnesses. 47 prefix_len () 48 { 49 case "$1" in 50 fuzz_extract) echo 2 ;; 51 fuzz_datasource) echo 4 ;; 52 fuzz_ipc) echo 0 ;; 53 fuzz_convert) echo 2 ;; 54 *) echo 4 ;; # fuzz_unzip and every fuzz_<plugin> 55 esac 56 } 57 58 # Which testdata files seed which target. A target with no entry gets 59 # only its built-in seeds. 60 testdata_glob () 61 { 62 case "$1" in 63 fuzz_applefile) echo "applefile_*" ;; 64 fuzz_archive) echo "archive_*" ;; 65 fuzz_deb) echo "deb_*" ;; 66 fuzz_dvi) echo "dvi_*" ;; 67 fuzz_elf) echo "chello-elf" ;; 68 fuzz_flac) echo "flac_*" ;; 69 fuzz_gif) echo "gif_*" ;; 70 fuzz_html) echo "html_*" ;; 71 fuzz_it) echo "it_*" ;; 72 fuzz_jpeg) echo "jpeg_* exiv2_* thumbnail_*" ;; 73 fuzz_man) echo "man_*" ;; 74 fuzz_mime) echo "*" ;; 75 fuzz_mpeg) echo "mpeg_*" ;; 76 fuzz_msoffice) echo "msoffice_*" ;; 77 fuzz_nsf) echo "nsf_*" ;; 78 fuzz_nsfe) echo "nsfe_*" ;; 79 fuzz_odf) echo "odf_*" ;; 80 fuzz_ogg) echo "ogg_*" ;; 81 fuzz_ole2) echo "ole2_* msoffice_biff4.xls" ;; 82 fuzz_png) echo "png_*" ;; 83 fuzz_ps) echo "ps_*" ;; 84 fuzz_qt) echo "gstreamer_sample_sorenson.mov" ;; 85 fuzz_real) echo "ra3.ra audiosig.rm" ;; 86 fuzz_riff) echo "riff_* wav_*" ;; 87 fuzz_rtf) echo "rtf_*" ;; 88 fuzz_s3m) echo "s3m_*" ;; 89 fuzz_sid) echo "sid_*" ;; 90 fuzz_tiff) echo "tiff_*" ;; 91 fuzz_wav) echo "wav_*" ;; 92 fuzz_xm) echo "xm_*" ;; 93 fuzz_zip) echo "zip_* odf_* msoffice_excel.xlsx msoffice_word.docx" ;; 94 fuzz_unzip) echo "zip_* odf_* msoffice_excel.xlsx msoffice_word.docx \ 95 msoffice_powerpoint.pptx" ;; 96 fuzz_datasource) echo "deb_* zip_* png_* ogg_*" ;; 97 fuzz_pecoff) echo "pecoff_*" ;; 98 fuzz_lnk) echo "lnk_*" ;; 99 fuzz_sqlite) echo "sqlite_*" ;; 100 fuzz_tar) echo "tar_* archive_test.tar" ;; 101 fuzz_iso9660) echo "iso9660_*" ;; 102 fuzz_diskimage) echo "diskimage_*" ;; 103 fuzz_heif) echo "heif_*" ;; 104 fuzz_webp) echo "webp_*" ;; 105 fuzz_plist) echo "plist_*" ;; 106 fuzz_id3) echo "id3_*" ;; 107 fuzz_gpx) echo "gpx_*" ;; 108 fuzz_kml) echo "kml_*" ;; 109 fuzz_geotiff) echo "geotiff_* tiff_*" ;; 110 fuzz_mbox) echo "mbox_*" ;; 111 fuzz_apk) echo "apk_*" ;; 112 fuzz_ebook) echo "ebook_*" ;; 113 fuzz_extract) echo "*" ;; 114 *) echo "" ;; 115 esac 116 } 117 118 # All targets we know about. Kept explicit rather than derived from the 119 # build so that this script also works before anything is built. 120 TARGETS="fuzz_extract fuzz_datasource fuzz_ipc fuzz_convert fuzz_unzip \ 121 fuzz_applefile fuzz_dvi fuzz_elf fuzz_it fuzz_man fuzz_nsf fuzz_nsfe \ 122 fuzz_ps fuzz_real fuzz_riff fuzz_rtf fuzz_s3m fuzz_sid fuzz_wav fuzz_xm \ 123 fuzz_deb fuzz_msoffice fuzz_odf fuzz_png fuzz_qt fuzz_zip \ 124 fuzz_gif fuzz_jpeg fuzz_tiff fuzz_flac fuzz_ogg fuzz_archive fuzz_mime \ 125 fuzz_ole2 fuzz_mpeg fuzz_html \ 126 fuzz_pecoff fuzz_lnk fuzz_sqlite fuzz_tar fuzz_iso9660 fuzz_diskimage \ 127 fuzz_heif fuzz_webp fuzz_plist fuzz_id3 fuzz_gpx fuzz_kml fuzz_geotiff \ 128 fuzz_mbox fuzz_apk fuzz_ebook" 129 130 for t in ${TARGETS}; do 131 d="${WORK}/${t}" 132 mkdir -p "${d}" 133 134 # 1a. the checked-in corpus, which "make -C src/fuzz refresh-corpus" 135 # regenerates from the harnesses' built-in seeds 136 if [ -d "${SRCDIR}/src/fuzz/corpus/${t}" ]; then 137 cp "${SRCDIR}/src/fuzz/corpus/${t}"/* "${d}/" 2>/dev/null || true 138 fi 139 140 # 1b. built-in seeds straight from the binary, if one that understands 141 # --write-corpus is around. A binary linked against libFuzzer does 142 # NOT: our main() is compiled out, libFuzzer's own argument parser 143 # sees an unknown flag and starts fuzzing, so this must be bounded. 144 if [ -n "${FUZZBIN}" ] && [ -x "${FUZZBIN}/${t}" ]; then 145 timeout 10 "${FUZZBIN}/${t}" --write-corpus="${d}" >/dev/null 2>&1 || true 146 fi 147 148 # 2. testdata, with the configuration prefix prepended 149 n=$(prefix_len "${t}") 150 if [ "${n}" -gt 0 ]; then 151 # a file of n zero bytes 152 : > "${WORK}/prefix" 153 i=0 154 while [ "${i}" -lt "${n}" ]; do 155 printf '\000' >> "${WORK}/prefix" 156 i=$((i + 1)) 157 done 158 fi 159 # `set -f` matters: the glob list comes back from a command 160 # substitution and would otherwise be expanded against the *current* 161 # directory before the loop ever starts. 162 set -f 163 for g in $(testdata_glob "${t}"); do 164 set +f 165 for f in "${TESTDATA}"/${g}; do 166 [ -f "${f}" ] || continue 167 case "${f}" in 168 */README) continue ;; 169 esac 170 b=$(basename "${f}") 171 if [ "${n}" -gt 0 ]; then 172 cat "${WORK}/prefix" "${f}" > "${d}/td-${b}" 173 else 174 cp "${f}" "${d}/td-${b}" 175 fi 176 done 177 set -f 178 done 179 set +f 180 181 # 3. reproducers of past findings, so that they stay permanent 182 # regressions in the OSS-Fuzz corpus as well 183 kf="${SRCDIR}/src/fuzz/corpus/known-findings" 184 if [ -d "${kf}" ]; then 185 for f in "${kf}/${t}"-*; do 186 [ -f "${f}" ] || continue 187 cp "${f}" "${d}/$(basename "${f}")" 188 done 189 fi 190 191 # A target whose binary was not built and that has no testdata mapping 192 # would otherwise produce an empty archive, which `zip` refuses to 193 # create; give it a one-byte seed so that the corpus always exists. 194 if [ -z "$(ls -A "${d}" 2>/dev/null)" ]; then 195 printf '\000' > "${d}/empty" 196 fi 197 198 if [ "${MODE}" = "--plain" ]; then 199 mkdir -p "${OUTDIR}/${t}" 200 cp "${d}"/* "${OUTDIR}/${t}/" 201 echo "corpus: ${OUTDIR}/${t} ($(ls -1 "${OUTDIR}/${t}" | wc -l) files)" 202 else 203 (cd "${d}" && zip -q -r "${OUTDIR}/${t}_seed_corpus.zip" . ) 204 echo "corpus: ${OUTDIR}/${t}_seed_corpus.zip ($(ls -1 "${d}" | wc -l) files)" 205 fi 206 done