gen_diskimage_testdata.sh (8060B)
1 #!/bin/sh 2 # This file is part of libextractor. 3 # Copyright (C) 2026 Vidyut Samanta and Christian Grothoff 4 # 5 # libextractor is free software; you can redistribute it and/or modify 6 # it under the terms of the GNU General Public License as published 7 # by the Free Software Foundation; either version 3, or (at your 8 # option) any later version. 9 # 10 # Regenerate the test images for the `diskimage' plugin. 11 # 12 # ./contrib/gen_diskimage_testdata.sh [output-directory] 13 # 14 # The default output directory is src/plugins/testdata. Needs the 15 # Debian packages `qemu-utils' (for qemu-img) and `python3'; nothing 16 # else. Everything the test asserts on is either fixed by the format 17 # or patched to a constant here, so running this twice gives files that 18 # produce the same meta data. The images are CC0: they hold no data, 19 # only headers. 20 # 21 # The three files it writes are: 22 # 23 # diskimage_test.qcow2 a QCOW2 v3 differencing image, 1 MiB virtual, 24 # 512 byte clusters, backed by 25 # "diskimage_base.qcow2", with lazy refcounts 26 # and zstd compression turned on so the feature 27 # bit paths are exercised. Complete, untouched 28 # qemu-img output. 29 # diskimage_test.vmdk a monolithicSparse VMDK, 1 MiB virtual, 30 # backed by "diskimage_base.vmdk". The random 31 # CID and parentCID qemu-img picks are 32 # overwritten with fixed values (same length, 33 # so nothing else moves) and the file is then 34 # TRUNCATED to 16 KiB: qemu-img always writes 35 # 64 KiB of mostly zero grain tables and the 36 # test corpus has a size budget. The sparse 37 # header (80 bytes) and the embedded text 38 # descriptor (offset 512, 10 KiB) are what the 39 # plugin reads and both survive intact; the 40 # result is no longer a mountable VMDK. 41 # diskimage_test.vhdx the first 4 KiB of a VHDX, with the creator 42 # string overwritten with a fixed value. 43 # TRUNCATED, and severely: [MS-VHDX] puts the 44 # header at 64 KiB and the region table at a 45 # fixed 192 KiB, and requires the metadata 46 # region to start at a 1 MiB boundary, so the 47 # smallest conformant VHDX is over 2 MiB. That 48 # does not fit in a test corpus, so only the 49 # file identifier block is kept and the test 50 # can only assert the signature-level items. 51 # The plugin's region and metadata walk is 52 # exercised by hand against a full image; see 53 # GEN_FULL below. 54 # 55 # Setting GEN_FULL=1 additionally writes the untruncated images plus a 56 # VHD, under the same names with a ".full" suffix. They are too big to 57 # ship but are what the plugin should really be checked against. 58 59 set -e 60 61 top=$(dirname "$0")/.. 62 out=${1:-$top/src/plugins/testdata} 63 work=$(mktemp -d) 64 trap 'rm -rf "$work"' 0 65 66 mkdir -p "$out" 67 68 # ---------------------------------------------------------------- QCOW2 69 # 70 # The base image only has to exist while the child is created; only the 71 # name it leaves behind in the child matters. 72 qemu-img create -f qcow2 -o cluster_size=512 \ 73 "$work/diskimage_base.qcow2" 1M > /dev/null 74 (cd "$work" && qemu-img create -f qcow2 \ 75 -o cluster_size=512,lazy_refcounts=on,compression_type=zstd \ 76 -b diskimage_base.qcow2 -F qcow2 \ 77 diskimage_test.qcow2 > /dev/null) 78 cp "$work/diskimage_test.qcow2" "$out/diskimage_test.qcow2" 79 80 # ----------------------------------------------------------------- VMDK 81 qemu-img create -f vmdk -o subformat=monolithicSparse \ 82 "$work/diskimage_base.vmdk" 1M > /dev/null 83 (cd "$work" && qemu-img create -f vmdk -o subformat=monolithicSparse \ 84 -b diskimage_base.vmdk -F vmdk \ 85 diskimage_test.vmdk > /dev/null) 86 python3 - "$work/diskimage_test.vmdk" <<'EOF' 87 import re 88 import struct 89 import sys 90 91 path = sys.argv[1] 92 data = bytearray(open(path, 'rb').read()) 93 # The content ID and the parent's content ID are random per run; pin 94 # them. qemu-img prints them with %x, so they are one to eight hex 95 # digits -- rewrite the whole descriptor slot and pad it back to its 96 # original length so no offset in the header has to change. 97 off = struct.unpack_from('<Q', data, 28)[0] * 512 98 size = struct.unpack_from('<Q', data, 36)[0] * 512 99 desc = bytes(data[off:off + size]) 100 desc = re.sub(rb'\nCID=[0-9a-f]{1,8}\n', b'\nCID=1f2e3d4c\n', desc) 101 desc = re.sub(rb'\nparentCID=[0-9a-f]{1,8}\n', b'\nparentCID=5a6b7c8d\n', desc) 102 desc = desc.rstrip(b'\0') 103 if len(desc) > size: 104 raise SystemExit('descriptor no longer fits its slot') 105 data[off:off + size] = desc + b'\0' * (size - len(desc)) 106 open(path, 'wb').write(data[:16384]) 107 EOF 108 cp "$work/diskimage_test.vmdk" "$out/diskimage_test.vmdk" 109 110 # ----------------------------------------------------------------- VHDX 111 qemu-img create -f vhdx -o block_size=1M,log_size=1M \ 112 "$work/diskimage_test.vhdx" 1M > /dev/null 113 python3 - "$work/diskimage_test.vhdx" <<'EOF' 114 import sys 115 116 path = sys.argv[1] 117 data = bytearray(open(path, 'rb').read()) 118 # [MS-VHDX] 2.1: 8 byte signature, then a 512 byte UTF-16LE creator 119 # string. qemu writes its own version number there, which would make 120 # the test depend on the local qemu; overwrite it. 121 creator = 'libextractor testdata 1.0'.encode('utf-16-le') 122 data[8:8 + 512] = creator + b'\0' * (512 - len(creator)) 123 open(path, 'wb').write(data[:4096]) 124 EOF 125 cp "$work/diskimage_test.vhdx" "$out/diskimage_test.vhdx" 126 127 # ------------------------------------------------- optional full images 128 if [ "x$GEN_FULL" = "x1" ]; then 129 qemu-img create -f qcow2 -o cluster_size=512 \ 130 "$work/full_base.qcow2" 1M > /dev/null 131 (cd "$work" && qemu-img create -f qcow2 \ 132 -o cluster_size=512,lazy_refcounts=on,compression_type=zstd \ 133 -b full_base.qcow2 -F qcow2 \ 134 diskimage_test.qcow2.full > /dev/null) 135 cp "$work/diskimage_test.qcow2.full" "$out/diskimage_test.qcow2.full" 136 137 qemu-img create -f vmdk -o subformat=monolithicSparse \ 138 "$work/full_base.vmdk" 1M > /dev/null 139 (cd "$work" && qemu-img create -f vmdk -o subformat=monolithicSparse \ 140 -b full_base.vmdk -F vmdk \ 141 diskimage_test.vmdk.full > /dev/null) 142 cp "$work/diskimage_test.vmdk.full" "$out/diskimage_test.vmdk.full" 143 144 qemu-img create -f vhdx -o block_size=1M,log_size=1M \ 145 "$work/diskimage_test.vhdx.full" 1M > /dev/null 146 cp "$work/diskimage_test.vhdx.full" "$out/diskimage_test.vhdx.full" 147 148 # A VHD (Microsoft "conectix") dynamic disk. There is no committed 149 # sample for this one -- the plugin reads it, but src/plugins has no 150 # diskimage_test.vhd in EXTRA_DIST. The time stamp and the unique ID 151 # qemu writes are per-run, so pin both and redo the footer checksum, 152 # which is the ones' complement of the sum of the 512 footer bytes. 153 qemu-img create -f vpc "$work/diskimage_test.vhd" 1M > /dev/null 154 python3 - "$work/diskimage_test.vhd" <<'EOF' 155 import struct 156 import sys 157 158 path = sys.argv[1] 159 data = bytearray(open(path, 'rb').read()) 160 161 162 def fix(off): 163 """Pin the per-run fields of the 512 byte footer at `off'.""" 164 if bytes(data[off:off + 8]) != b'conectix': 165 return 166 # seconds since 2000-01-01T00:00:00Z; 0x2D24BD00 is 2024-01-01 167 data[off + 24:off + 28] = struct.pack('>I', 0x2D24BD00) 168 data[off + 68:off + 84] = bytes.fromhex('3ffb1d5a09a44e1d9c2d7f6a1b4c8e02') 169 data[off + 64:off + 68] = b'\0\0\0\0' 170 total = sum(data[off:off + 512]) & 0xFFFFFFFF 171 data[off + 64:off + 68] = struct.pack('>I', (~total) & 0xFFFFFFFF) 172 173 174 fix(0) # the mirror copy a dynamic disk keeps up front 175 fix(len(data) - 512) # the real footer 176 open(path, 'wb').write(data) 177 EOF 178 cp "$work/diskimage_test.vhd" "$out/diskimage_test.vhd.full" 179 fi 180 181 echo "wrote:" 182 ls -l "$out"/diskimage_test.*