commit 337e7cb058e39b68521a4175a07cf3138ecbb374 parent c60eb77862edaaa23e6ef95863e90460d70ec873 Author: Christian Grothoff <christian@grothoff.org> Date: Fri, 7 Aug 2026 20:58:00 +0200 more plugins Diffstat:
190 files changed, 32318 insertions(+), 40 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -150,3 +150,35 @@ src/plugins/test_thumbnailffmpeg build-aux/ src/plugins/test_applefile uncrustify.cfg +src/plugins/test_pecoff +src/fuzz/fuzz_pecoff +src/plugins/test_lnk +src/fuzz/fuzz_lnk +src/plugins/test_sqlite +src/fuzz/fuzz_sqlite +src/plugins/test_tar +src/fuzz/fuzz_tar +src/plugins/test_iso9660 +src/fuzz/fuzz_iso9660 +src/plugins/test_diskimage +src/fuzz/fuzz_diskimage +src/plugins/test_heif +src/fuzz/fuzz_heif +src/plugins/test_webp +src/fuzz/fuzz_webp +src/plugins/test_plist +src/fuzz/fuzz_plist +src/plugins/test_id3 +src/fuzz/fuzz_id3 +src/plugins/test_gpx +src/fuzz/fuzz_gpx +src/plugins/test_kml +src/fuzz/fuzz_kml +src/plugins/test_geotiff +src/fuzz/fuzz_geotiff +src/plugins/test_mbox +src/fuzz/fuzz_mbox +src/plugins/test_apk +src/fuzz/fuzz_apk +src/plugins/test_ebook +src/fuzz/fuzz_ebook diff --git a/ChangeLog b/ChangeLog @@ -1,3 +1,25 @@ +Fri Aug 7 09:30:00 AM CEST 2026 + Add sixteen plugins for a fast forensic/file-manager pass over large + volumes, all of which parse headers directly and add no new + dependency: pecoff (Windows PE/COFF executables, including the Rich + header, the PDB path, the imphash and Authenticode common names), + lnk (Windows shell links, including the volume serial, the + originating machine name and the MAC address in the tracker block), + sqlite (the 100-byte header only; the database is never opened, and + WAL and rollback-journal sidecars are identified too), tar, iso9660, + diskimage (QCOW2, VMDK, VHD and VHDX), heif (HEIC/HEIF/AVIF), webp, + plist (binary and XML), id3 (with the Xing/Info and LAME encoder + headers), gpx, kml, geotiff, mbox, apk (Android packages and Java + archives) and ebook (EPUB and MOBI). Only apk and ebook are + conditional, on zlib, as they must inflate zip members. + Add meta types 232-290 for the provenance these formats record: + build, ownership, volume, database, geographic, mail and signing + metadata. Values are appended only, so this is source and binary + compatible, but it raises EXTRACTOR_metatype_get_max(); the shared + library version should be bumped (current++, age++) at the next + release. + Add fuzzing harnesses for all sixteen. -CG + Fri Aug 7 03:40:00 AM CEST 2026 Add video thumbnail and audio preview plugins, in a GStreamer and an FFmpeg (>= 5.1) variant each: thumbnailgst, thumbnailffmpeg, diff --git a/Makefile.am b/Makefile.am @@ -2,7 +2,23 @@ SUBDIRS = m4 po src doc . EXTRA_DIST = config.rpath \ ABOUT-NLS \ - contrib/gen_testmedia.sh + contrib/gen_testmedia.sh \ + contrib/gen_apk_testdata.sh \ + contrib/gen_diskimage_testdata.sh \ + contrib/gen_ebook_testdata.sh \ + contrib/gen_geotiff_testdata.sh \ + contrib/gen_gpx_testdata.sh \ + contrib/gen_heif_testdata.sh \ + contrib/gen_id3_testdata.sh \ + contrib/gen_iso9660_testdata.sh \ + contrib/gen_kml_testdata.sh \ + contrib/gen_lnk_testdata.sh \ + contrib/gen_mbox_testdata.sh \ + contrib/gen_pecoff_testdata.sh \ + contrib/gen_plist_testdata.sh \ + contrib/gen_sqlite_testdata.sh \ + contrib/gen_tar_testdata.sh \ + contrib/gen_webp_testdata.sh pkgconfigdatadir = $(libdir)/pkgconfig pkgconfigdata_DATA = libextractor.pc diff --git a/README b/README @@ -86,6 +86,17 @@ installation. Otherwise the build system may automatically build only a subset of GNU libextractor resulting in mediocre meta data production. +The sixteen plugins meant for a fast pass over a large volume (pecoff, +lnk, sqlite, tar, iso9660, diskimage, heif, webp, plist, id3, gpx, kml, +geotiff, mbox, apk and ebook) parse headers themselves and need none of +the libraries above, so they are always built. The two exceptions are +apk and ebook, which read deflated members of a zip container and so +need zlib. + +Regenerating the files those plugins are tested against is not part of +a normal build; the scripts under contrib/ that do it note the extra +tools they need (mingw-w64, qemu-utils, lame, ffmpeg). + Finally, 'zzuf' is a fuzzing tool that can optionally be detected by the build system and be used for debugging / testing. It is not required at runtime or for normal builds. diff --git a/contrib/gen_apk_testdata.sh b/contrib/gen_apk_testdata.sh @@ -0,0 +1,376 @@ +#!/bin/sh +# Regenerate src/plugins/testdata/apk_test.jar and apk_test.apk. +# +# This file is in the public domain (CC0); so are the files it produces. +# +# Requirements: python3 (standard library only). No Android SDK, no JDK: +# the binary AndroidManifest.xml is written out by hand, and the PKCS#7 +# signature block is a pre-generated, self-contained test certificate +# chain embedded below (originally made with `openssl req`/`openssl cms`, +# see the comment next to the blob). Everything is deterministic: run it +# twice and you get byte-identical files. +# +# Usage: contrib/gen_apk_testdata.sh [output-directory] +# (default output directory: src/plugins/testdata) + +set -e +OUT="${1:-$(dirname "$0")/../src/plugins/testdata}" +mkdir -p "$OUT" +python3 - "$OUT" <<'PYEOF' +import base64, os, struct, sys, zipfile + +OUT = sys.argv[1] +DATE = (2026, 1, 1, 0, 0, 0) # fixed: the archives must be reproducible + +# ---------------------------------------------------------------- signature +# +# A PKCS#7 SignedData over a dummy .SF file, signed by a two-element test +# chain: "libextractor test CA" issues "libextractor test signer" +# (serial 0x1719). Note that the CA certificate comes *first* inside the +# certificates set, so a parser that simply takes the first certificate +# reports the wrong subject; the plugin has to resolve the SignerInfo's +# issuerAndSerialNumber. Regenerate with: +# +# openssl req -x509 -newkey rsa:2048 -keyout ca.key -nodes -out ca.pem \ +# -sha256 -set_serial 0x01 -subj "/CN=libextractor test CA/O=GNU libextractor" \ +# -not_before 20260101000000Z -not_after 20360101000000Z +# openssl req -new -newkey rsa:2048 -keyout leaf.key -nodes -out leaf.csr \ +# -sha256 -subj "/CN=libextractor test signer/O=GNU libextractor/OU=Testing/C=CH" +# openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -set_serial 0x1719 \ +# -sha256 -not_before 20260101000000Z -not_after 20360101000000Z -out leaf.pem +# openssl cms -sign -binary -in CERT.SF -signer leaf.pem -inkey leaf.key \ +# -certfile ca.pem -outform DER -md sha256 -noattr -nosmimecap -out CERT.RSA +# +# The private keys were thrown away; this signature protects nothing. +PKCS7_B64 = """ +MIIIQgYJKoZIhvcNAQcCoIIIMzCCCC8CAQExDTALBglghkgBZQMEAgEwCwYJKoZIhvcNAQcBoIIG +nzCCA0IwggIqoAMCAQICAQEwDQYJKoZIhvcNAQELBQAwOjEdMBsGA1UEAwwUbGliZXh0cmFjdG9y +IHRlc3QgQ0ExGTAXBgNVBAoMEEdOVSBsaWJleHRyYWN0b3IwHhcNMjYwMTAxMDAwMDAwWhcNMzYw +MTAxMDAwMDAwWjA6MR0wGwYDVQQDDBRsaWJleHRyYWN0b3IgdGVzdCBDQTEZMBcGA1UECgwQR05V +IGxpYmV4dHJhY3RvcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALQDKjxbHQvulnUe +Z6DHIMobIBoNn5mDOXeNah3VwCHFMwcO5DJCvzLo5N/D/pIeuov4PaXMXOv4e7/ZqiIu6DV8Nr0A +KhJxCIEVUHoYt3II3gNFmjIQLs1ajAdA+4rjX8mjicgIbfNNgDCf/OGj5ctFdEelOMF+NCDel8jj +61p0B75a8arlaNnk254A785TQfYOBnWXp+4OPkJmF1FRhQXI/19TRbjCdAc9Q3FL8R/pGp9QxH7s +MTR1CfOJ47DtA/upHeYR3S1BMBtRZCadwBHpLDBNT2Yornpek7nm1kZOL8ACT4FXu/O/fYFguW2o +7RtxxSDMYRRTePZ79ylsr8kCAwEAAaNTMFEwHQYDVR0OBBYEFIp+SdLMxJWNBKxJ9y9qwMuQn00j +MB8GA1UdIwQYMBaAFIp+SdLMxJWNBKxJ9y9qwMuQn00jMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBADRcVd4jwMGRM7UKJwc1pQcm3qzXH0/1U+M5+zPAihFKnuZaZaF/OxqOHkA6 +Ht+juuAb57eOaai+qI5IepohdUGLmwepcNEL9FqboRqaL9DkwPmC/lqTM5OrTzMs2Ti8HiT2Degh +6bX6h9n1fI3MNHArvoTb1n70KlEY5D1huCk0NT3ktRGV8uGkSQomo+Kj3lDiPvXIKl5bddQ9KGBG +MuW1tKNQyjm3n5AxFH3ouFcGLIM3s6AAHJiG9rh6RisJS14gS5XAyEn0n6yF7NRiO6gcozGGkxck +bEz7z0/m0FkmfKhUZ+a7LK0liUn42s42Ev2z7TVYF9crkjIpwRymEYMwggNVMIICPaADAgECAgIX +GTANBgkqhkiG9w0BAQsFADA6MR0wGwYDVQQDDBRsaWJleHRyYWN0b3IgdGVzdCBDQTEZMBcGA1UE +CgwQR05VIGxpYmV4dHJhY3RvcjAeFw0yNjAxMDEwMDAwMDBaFw0zNjAxMDEwMDAwMDBaMF0xITAf +BgNVBAMMGGxpYmV4dHJhY3RvciB0ZXN0IHNpZ25lcjEZMBcGA1UECgwQR05VIGxpYmV4dHJhY3Rv +cjEQMA4GA1UECwwHVGVzdGluZzELMAkGA1UEBhMCQ0gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQCplvQUPC2P/KBXnJQVqow9P8wFV4ngj/gN+COIUukWefcRQX4Z+VRf0kJR0lTeW8NY +xLAQzyhPkDwGJBOGPEpN/3iBmXOQa47wO5MiphgxyCK7lkF0sDcAG4tjkixMNmf0HBoouSQpojKi +MT2KTZu5ShJ80eOgfgvIcdwvG9nGXf9d1/3DgE4YwbFRvqmo5FMQmvKct+Up6IDHxMn5Dj4QdQW8 +p1JvQ6O0eyp+98CK8u8VLMgjT5sPVZv6OyMSjXqHQ25kl29tR3CXDTCOQTGOi+T0DzzeqXOHkQhM +ApUUxSCt4LD1p91nLlgLBDXwEhHyepSPDc5aOvOMi9BNyr5pAgMBAAGjQjBAMB0GA1UdDgQWBBS6 ++cBqtk2rLx1XrLysYCsFUvYqJTAfBgNVHSMEGDAWgBSKfknSzMSVjQSsSfcvasDLkJ9NIzANBgkq +hkiG9w0BAQsFAAOCAQEACKKXcPaPxVWHjiXg9P3LBg3RMbSL8Qp6f4Y8OmYhGJFVLZBWR03BsDU3 +W9IiAYD08XznxItKOIjHjABvXPc2aY6yceJkCL+1ohhL3HRcs4+NvgQbevAqyP7mwWmJAb0WxwC7 +YSTjCIFzszudwUmpd1pQya2SQGPTUpWA5aCz+rEJXPCKrWn4Wwb3HkJV1p6p2gYhWt7rD2z+gs2Y +IpDyAheEbMwr2iU1gZLlC02E1PFq3GpYcgB4401U7OgQOdbJovWlVZw061NF6CstJwL1mQIrKdgE +Iu9awPZ+ZL7FQQ/qkWB4QIYtRhZOc6V68jRj4nkwk63p99GWDICFk1yv1zGCAWkwggFlAgEBMEAw +OjEdMBsGA1UEAwwUbGliZXh0cmFjdG9yIHRlc3QgQ0ExGTAXBgNVBAoMEEdOVSBsaWJleHRyYWN0 +b3ICAhcZMAsGCWCGSAFlAwQCATANBgkqhkiG9w0BAQEFAASCAQB4EvSyr8PjR0GHgvi7z81/jKsb +JQ4aqr+x5djxnoG8OZXXVOz2yGVJJLhsTMbSOdnchpfjOfSv0cxpbeowMM7ICJ5VpCVi4O3RXWAy +TJ2siIl0QoDnp1Xvfs8DyFPUTAUFqvukLB16rgQXoDNk4tFOmqfgwnVcIm+QrrwTVqBgGEb63F5K +HOiBKOJttNPLixFVtCc/mOKz5+Fu3N3NMLiS3u++k2/oWh1+5ozp/90AOXmnpL+iGm7/7GwFMPM3 +ZhKf7a+8cldhquc9AeO/lwYJw6XF8+7awEYSiPeEeKNN9SrbnAKOL+npzNN6OY/U5p9rzsV153Ov +YOGnCcC/2rXa +""" +PKCS7 = base64.b64decode("".join(PKCS7_B64.split())) +SF = (b"Signature-Version: 1.0\r\n" + b"Created-By: libextractor testdata generator\r\n" + b"SHA-256-Digest-Manifest: 0000\r\n\r\n") + + +# ------------------------------------------------------------ jar manifest +def fold (line): + """Wrap a manifest header the way the JAR spec does: no line, including + its line break, may exceed 72 bytes, and every continuation line starts + with a single space that is not part of the value.""" + raw = line.encode ("utf-8") + out = raw[:70] + raw = raw[70:] + while raw: + out += b"\r\n " + raw[:69] + raw = raw[69:] + return out + b"\r\n" + + +JAR_HEADERS = [ + "Manifest-Version: 1.0", + "Created-By: 17.0.9+9 (Eclipse Adoptium)", + "Build-Jdk-Spec: 17", + "Build-Jdk: 17.0.9+9", + "Main-Class: org.gnu.libextractor.TestMain", + "Class-Path: lib/support.jar lib/codec.jar", + # deliberately longer than 72 bytes, so that reading it back proves the + # continuation lines are unfolded + "Implementation-Title: GNU libextractor JAR test archive with a very long" + " title that has to be folded", + "Implementation-Version: 1.19.0", + "Implementation-Vendor: GNU libextractor project", + "Specification-Title: libextractor plugin test specification", + "Specification-Version: 1.19", + "Specification-Vendor: GNU", + "Bundle-SymbolicName: org.gnu.libextractor.test", + "Bundle-Name: libextractor test bundle", + "Bundle-Version: 1.19.0.qualifier", + "Bundle-Vendor: GNU libextractor project", + "Bundle-Description: test bundle produced by gen_apk_testdata.sh", + "Bundle-RequiredExecutionEnvironment: JavaSE-17", +] +MANIFEST = b"".join (fold (h) for h in JAR_HEADERS) + b"\r\n" + +# a syntactically valid but empty class file (major version 61 = Java 17) +CLASS = struct.pack (">IHHH", 0xCAFEBABE, 0, 61, 1) + \ + struct.pack (">HHHHH", 0x0021, 0, 0, 0, 0) + + +def zwrite (path, members, compress = zipfile.ZIP_DEFLATED): + buf = [] + with zipfile.ZipFile (path, "w") as z: + for name, data, method in members: + zi = zipfile.ZipInfo (name, DATE) + zi.compress_type = method + zi.create_system = 0 + z.writestr (zi, data) + return buf + + +zwrite (os.path.join (OUT, "apk_test.jar"), [ + ("META-INF/MANIFEST.MF", MANIFEST, zipfile.ZIP_DEFLATED), + ("META-INF/CERT.SF", SF, zipfile.ZIP_DEFLATED), + ("META-INF/CERT.RSA", PKCS7, zipfile.ZIP_STORED), + ("org/gnu/libextractor/TestMain.class", CLASS, zipfile.ZIP_DEFLATED), +]) + + +# ------------------------------------------------------- binary AndroidManifest +# +# Chunk types and value types are from AOSP +# frameworks/base/libs/androidfw/include/androidfw/ResourceTypes.h. +RES_XML = 0x0003 +RES_STRING_POOL = 0x0001 +RES_XML_START_NAMESPACE = 0x0100 +RES_XML_END_NAMESPACE = 0x0101 +RES_XML_START_ELEMENT = 0x0102 +RES_XML_END_ELEMENT = 0x0103 +RES_XML_RESOURCE_MAP = 0x0180 + +TYPE_REFERENCE = 0x01 +TYPE_STRING = 0x03 +TYPE_INT_DEC = 0x10 +TYPE_INT_BOOLEAN = 0x12 + +ANDROID_NS = "http://schemas.android.com/apk/res/android" + +# Attribute resource IDs, read off a real aapt2-built APK. The resource +# map has to cover a prefix of the string pool, so these strings are +# added to the pool first and in this order. +ATTR_IDS = [ + ("label", 0x01010001), + ("icon", 0x01010002), + ("name", 0x01010003), + ("debuggable", 0x0101000f), + ("minSdkVersion", 0x0101020c), + ("versionCode", 0x0101021b), + ("versionName", 0x0101021c), + ("targetSdkVersion", 0x01010270), + ("allowBackup", 0x01010280), + ("usesCleartextTraffic", 0x010104ec), + ("networkSecurityConfig", 0x01010527), + ("compileSdkVersion", 0x01010572), +] + + +class Pool: + def __init__ (self): + self.strings = [] + for name, _ in ATTR_IDS: + self.add (name) + + def add (self, s): + if s in self.strings: + return self.strings.index (s) + self.strings.append (s) + return len (self.strings) - 1 + + def build (self): + """UTF-16LE string pool, which is what aapt emits for + AndroidManifest.xml.""" + data = b"" + offs = [] + for s in self.strings: + offs.append (len (data)) + u = s.encode ("utf-16-le") + assert len (s) < 0x8000 + data += struct.pack ("<H", len (s)) + u + b"\0\0" + while len (data) % 4: + data += b"\0" + hdr_len = 28 + 4 * len (offs) + size = hdr_len + len (data) + return (struct.pack ("<HHIIIIII", RES_STRING_POOL, 28, size, + len (offs), 0, 0, hdr_len, 0) + + struct.pack ("<%dI" % len (offs), *offs) + + data) + + +pool = Pool () +NOSTR = 0xFFFFFFFF + + +def attr (ns, name, value, vtype): + """One ResXMLTree_attribute (20 bytes).""" + ns_i = pool.add (ns) if ns else NOSTR + nm_i = pool.add (name) + if TYPE_STRING == vtype: + raw = pool.add (value) + data = raw + else: + raw = NOSTR + data = value & 0xFFFFFFFF + return struct.pack ("<IIIHBBI", ns_i, nm_i, raw, 8, 0, vtype, data) + + +def start (name, attrs): + nm = pool.add (name) + body = b"".join (attrs) + size = 16 + 20 + len (body) + return (struct.pack ("<HHIII", RES_XML_START_ELEMENT, 16, size, 1, NOSTR) + + struct.pack ("<IIHHHHHH", NOSTR, nm, 20, 20, len (attrs), + 0, 0, 0) + + body) + + +def end (name): + nm = pool.add (name) + return struct.pack ("<HHIIIII", RES_XML_END_ELEMENT, 16, 24, 1, NOSTR, + NOSTR, nm) + + +PERMISSIONS = [ + "android.permission.INTERNET", + "android.permission.ACCESS_FINE_LOCATION", + "android.permission.READ_SMS", + "android.permission.CAMERA", + "android.permission.RECEIVE_BOOT_COMPLETED", +] + +# Strings referenced by attribute values have to exist before the chunks +# are serialised, but `attr' adds them as it goes, so simply build the +# chunk list first and the pool header afterwards. +body = b"" +body += start ("manifest", [ + attr (ANDROID_NS, "versionCode", 1719, TYPE_INT_DEC), + attr (ANDROID_NS, "versionName", "1.19.0", TYPE_STRING), + attr (ANDROID_NS, "compileSdkVersion", 33, TYPE_INT_DEC), + attr (None, "package", "org.gnu.libextractor.apktest", TYPE_STRING), + attr (None, "platformBuildVersionCode", 33, TYPE_INT_DEC), +]) +body += start ("uses-sdk", [ + attr (ANDROID_NS, "minSdkVersion", 21, TYPE_INT_DEC), + attr (ANDROID_NS, "targetSdkVersion", 33, TYPE_INT_DEC), +]) +body += end ("uses-sdk") +for p in PERMISSIONS: + body += start ("uses-permission", + [attr (ANDROID_NS, "name", p, TYPE_STRING)]) + body += end ("uses-permission") +body += start ("application", [ + # a literal label, not the usual @string reference, so that the plugin + # has something real to report + attr (ANDROID_NS, "label", "libextractor APK test", TYPE_STRING), + # a reference, which the plugin must *not* report as a title + attr (ANDROID_NS, "icon", 0x7F080001, TYPE_REFERENCE), + attr (ANDROID_NS, "debuggable", 0xFFFFFFFF, TYPE_INT_BOOLEAN), + attr (ANDROID_NS, "allowBackup", 0, TYPE_INT_BOOLEAN), + attr (ANDROID_NS, "usesCleartextTraffic", 0xFFFFFFFF, TYPE_INT_BOOLEAN), + attr (ANDROID_NS, "networkSecurityConfig", 0x7F110002, TYPE_REFERENCE), +]) +body += end ("application") +body += end ("manifest") + +ns_prefix = pool.add ("android") +ns_uri = pool.add (ANDROID_NS) +ns_start = struct.pack ("<HHIIIII", RES_XML_START_NAMESPACE, 16, 24, 1, + NOSTR, ns_prefix, ns_uri) +ns_end = struct.pack ("<HHIIIII", RES_XML_END_NAMESPACE, 16, 24, 1, + NOSTR, ns_prefix, ns_uri) + +resmap = struct.pack ("<HHI", RES_XML_RESOURCE_MAP, 8, 8 + 4 * len (ATTR_IDS)) +resmap += struct.pack ("<%dI" % len (ATTR_IDS), *[i for _, i in ATTR_IDS]) + +pool_chunk = pool.build () +rest = pool_chunk + resmap + ns_start + body + ns_end +AXML = struct.pack ("<HHI", RES_XML, 8, 8 + len (rest)) + rest + +DEX = b"dex\n035\0" + b"\0" * 104 +SO = b"\x7fELF\x02\x01\x01\x00" + b"\0" * 56 +ARSC = struct.pack ("<HHII", 0x0002, 12, 12, 0) + +APK_MANIFEST = (b"Manifest-Version: 1.0\r\n" + b"Created-By: 1.0 (Android SignApk)\r\n\r\n") + +apk = os.path.join (OUT, "apk_test.apk") +zwrite (apk, [ + # AndroidManifest.xml first, the way aapt lays an APK out + ("AndroidManifest.xml", AXML, zipfile.ZIP_DEFLATED), + ("resources.arsc", ARSC, zipfile.ZIP_STORED), + ("classes.dex", DEX, zipfile.ZIP_DEFLATED), + ("classes2.dex", DEX, zipfile.ZIP_DEFLATED), + ("classes3.dex", DEX, zipfile.ZIP_DEFLATED), + ("lib/arm64-v8a/libtest.so", SO, zipfile.ZIP_DEFLATED), + ("lib/armeabi-v7a/libtest.so", SO, zipfile.ZIP_DEFLATED), + ("lib/x86_64/libtest.so", SO, zipfile.ZIP_DEFLATED), + ("res/xml/network_security_config.xml", b"<network-security-config/>", + zipfile.ZIP_DEFLATED), + ("META-INF/MANIFEST.MF", APK_MANIFEST, zipfile.ZIP_DEFLATED), + ("META-INF/ANDROIDD.SF", SF, zipfile.ZIP_DEFLATED), + ("META-INF/ANDROIDD.RSA", PKCS7, zipfile.ZIP_STORED), +]) + + +# ------------------------------------------------------ APK Signing Block +# +# Splice an APK Signing Block (the v2/v3 signature container) in between +# the last local file entry and the central directory, and move the +# central directory pointer in the end-of-central-directory record along +# with it. Local header offsets are unaffected because everything before +# the central directory stays where it is. +def sig_block (ids): + pairs = b"" + for i in ids: + # the value is a real signer list in a real APK; here it only has to + # be present and correctly framed + value = struct.pack ("<I", i) + b"\0" * 60 + pairs += struct.pack ("<Q", len (value)) + value + # size covers the pairs, the trailing size field and the magic + size = len (pairs) + 8 + 16 + return (struct.pack ("<Q", size) + pairs + struct.pack ("<Q", size) + + b"APK Sig Block 42") + + +APK_SIG_V2 = 0x7109871A +APK_SIG_V3 = 0xF05368C0 + +with open (apk, "rb") as f: + data = f.read () +eocd = data.rindex (b"PK\5\6") +cd_off = struct.unpack_from ("<I", data, eocd + 16)[0] +block = sig_block ([APK_SIG_V2, APK_SIG_V3]) +data = (data[:cd_off] + block + data[cd_off:eocd] + + data[eocd:eocd + 16] + + struct.pack ("<I", cd_off + len (block)) + + data[eocd + 20:]) +with open (apk, "wb") as f: + f.write (data) + +for n in ("apk_test.jar", "apk_test.apk"): + print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n)))) +PYEOF diff --git a/contrib/gen_diskimage_testdata.sh b/contrib/gen_diskimage_testdata.sh @@ -0,0 +1,182 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# libextractor is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; either version 3, or (at your +# option) any later version. +# +# Regenerate the test images for the `diskimage' plugin. +# +# ./contrib/gen_diskimage_testdata.sh [output-directory] +# +# The default output directory is src/plugins/testdata. Needs the +# Debian packages `qemu-utils' (for qemu-img) and `python3'; nothing +# else. Everything the test asserts on is either fixed by the format +# or patched to a constant here, so running this twice gives files that +# produce the same meta data. The images are CC0: they hold no data, +# only headers. +# +# The three files it writes are: +# +# diskimage_test.qcow2 a QCOW2 v3 differencing image, 1 MiB virtual, +# 512 byte clusters, backed by +# "diskimage_base.qcow2", with lazy refcounts +# and zstd compression turned on so the feature +# bit paths are exercised. Complete, untouched +# qemu-img output. +# diskimage_test.vmdk a monolithicSparse VMDK, 1 MiB virtual, +# backed by "diskimage_base.vmdk". The random +# CID and parentCID qemu-img picks are +# overwritten with fixed values (same length, +# so nothing else moves) and the file is then +# TRUNCATED to 16 KiB: qemu-img always writes +# 64 KiB of mostly zero grain tables and the +# test corpus has a size budget. The sparse +# header (80 bytes) and the embedded text +# descriptor (offset 512, 10 KiB) are what the +# plugin reads and both survive intact; the +# result is no longer a mountable VMDK. +# diskimage_test.vhdx the first 4 KiB of a VHDX, with the creator +# string overwritten with a fixed value. +# TRUNCATED, and severely: [MS-VHDX] puts the +# header at 64 KiB and the region table at a +# fixed 192 KiB, and requires the metadata +# region to start at a 1 MiB boundary, so the +# smallest conformant VHDX is over 2 MiB. That +# does not fit in a test corpus, so only the +# file identifier block is kept and the test +# can only assert the signature-level items. +# The plugin's region and metadata walk is +# exercised by hand against a full image; see +# GEN_FULL below. +# +# Setting GEN_FULL=1 additionally writes the untruncated images plus a +# VHD, under the same names with a ".full" suffix. They are too big to +# ship but are what the plugin should really be checked against. + +set -e + +top=$(dirname "$0")/.. +out=${1:-$top/src/plugins/testdata} +work=$(mktemp -d) +trap 'rm -rf "$work"' 0 + +mkdir -p "$out" + +# ---------------------------------------------------------------- QCOW2 +# +# The base image only has to exist while the child is created; only the +# name it leaves behind in the child matters. +qemu-img create -f qcow2 -o cluster_size=512 \ + "$work/diskimage_base.qcow2" 1M > /dev/null +(cd "$work" && qemu-img create -f qcow2 \ + -o cluster_size=512,lazy_refcounts=on,compression_type=zstd \ + -b diskimage_base.qcow2 -F qcow2 \ + diskimage_test.qcow2 > /dev/null) +cp "$work/diskimage_test.qcow2" "$out/diskimage_test.qcow2" + +# ----------------------------------------------------------------- VMDK +qemu-img create -f vmdk -o subformat=monolithicSparse \ + "$work/diskimage_base.vmdk" 1M > /dev/null +(cd "$work" && qemu-img create -f vmdk -o subformat=monolithicSparse \ + -b diskimage_base.vmdk -F vmdk \ + diskimage_test.vmdk > /dev/null) +python3 - "$work/diskimage_test.vmdk" <<'EOF' +import re +import struct +import sys + +path = sys.argv[1] +data = bytearray(open(path, 'rb').read()) +# The content ID and the parent's content ID are random per run; pin +# them. qemu-img prints them with %x, so they are one to eight hex +# digits -- rewrite the whole descriptor slot and pad it back to its +# original length so no offset in the header has to change. +off = struct.unpack_from('<Q', data, 28)[0] * 512 +size = struct.unpack_from('<Q', data, 36)[0] * 512 +desc = bytes(data[off:off + size]) +desc = re.sub(rb'\nCID=[0-9a-f]{1,8}\n', b'\nCID=1f2e3d4c\n', desc) +desc = re.sub(rb'\nparentCID=[0-9a-f]{1,8}\n', b'\nparentCID=5a6b7c8d\n', desc) +desc = desc.rstrip(b'\0') +if len(desc) > size: + raise SystemExit('descriptor no longer fits its slot') +data[off:off + size] = desc + b'\0' * (size - len(desc)) +open(path, 'wb').write(data[:16384]) +EOF +cp "$work/diskimage_test.vmdk" "$out/diskimage_test.vmdk" + +# ----------------------------------------------------------------- VHDX +qemu-img create -f vhdx -o block_size=1M,log_size=1M \ + "$work/diskimage_test.vhdx" 1M > /dev/null +python3 - "$work/diskimage_test.vhdx" <<'EOF' +import sys + +path = sys.argv[1] +data = bytearray(open(path, 'rb').read()) +# [MS-VHDX] 2.1: 8 byte signature, then a 512 byte UTF-16LE creator +# string. qemu writes its own version number there, which would make +# the test depend on the local qemu; overwrite it. +creator = 'libextractor testdata 1.0'.encode('utf-16-le') +data[8:8 + 512] = creator + b'\0' * (512 - len(creator)) +open(path, 'wb').write(data[:4096]) +EOF +cp "$work/diskimage_test.vhdx" "$out/diskimage_test.vhdx" + +# ------------------------------------------------- optional full images +if [ "x$GEN_FULL" = "x1" ]; then + qemu-img create -f qcow2 -o cluster_size=512 \ + "$work/full_base.qcow2" 1M > /dev/null + (cd "$work" && qemu-img create -f qcow2 \ + -o cluster_size=512,lazy_refcounts=on,compression_type=zstd \ + -b full_base.qcow2 -F qcow2 \ + diskimage_test.qcow2.full > /dev/null) + cp "$work/diskimage_test.qcow2.full" "$out/diskimage_test.qcow2.full" + + qemu-img create -f vmdk -o subformat=monolithicSparse \ + "$work/full_base.vmdk" 1M > /dev/null + (cd "$work" && qemu-img create -f vmdk -o subformat=monolithicSparse \ + -b full_base.vmdk -F vmdk \ + diskimage_test.vmdk.full > /dev/null) + cp "$work/diskimage_test.vmdk.full" "$out/diskimage_test.vmdk.full" + + qemu-img create -f vhdx -o block_size=1M,log_size=1M \ + "$work/diskimage_test.vhdx.full" 1M > /dev/null + cp "$work/diskimage_test.vhdx.full" "$out/diskimage_test.vhdx.full" + + # A VHD (Microsoft "conectix") dynamic disk. There is no committed + # sample for this one -- the plugin reads it, but src/plugins has no + # diskimage_test.vhd in EXTRA_DIST. The time stamp and the unique ID + # qemu writes are per-run, so pin both and redo the footer checksum, + # which is the ones' complement of the sum of the 512 footer bytes. + qemu-img create -f vpc "$work/diskimage_test.vhd" 1M > /dev/null + python3 - "$work/diskimage_test.vhd" <<'EOF' +import struct +import sys + +path = sys.argv[1] +data = bytearray(open(path, 'rb').read()) + + +def fix(off): + """Pin the per-run fields of the 512 byte footer at `off'.""" + if bytes(data[off:off + 8]) != b'conectix': + return + # seconds since 2000-01-01T00:00:00Z; 0x2D24BD00 is 2024-01-01 + data[off + 24:off + 28] = struct.pack('>I', 0x2D24BD00) + data[off + 68:off + 84] = bytes.fromhex('3ffb1d5a09a44e1d9c2d7f6a1b4c8e02') + data[off + 64:off + 68] = b'\0\0\0\0' + total = sum(data[off:off + 512]) & 0xFFFFFFFF + data[off + 64:off + 68] = struct.pack('>I', (~total) & 0xFFFFFFFF) + + +fix(0) # the mirror copy a dynamic disk keeps up front +fix(len(data) - 512) # the real footer +open(path, 'wb').write(data) +EOF + cp "$work/diskimage_test.vhd" "$out/diskimage_test.vhd.full" +fi + +echo "wrote:" +ls -l "$out"/diskimage_test.* diff --git a/contrib/gen_ebook_testdata.sh b/contrib/gen_ebook_testdata.sh @@ -0,0 +1,230 @@ +#!/bin/sh +# Regenerate src/plugins/testdata/ebook_test.epub and ebook_test.mobi. +# +# This file is in the public domain (CC0); so are the files it produces. +# +# Requirements: python3 (standard library only). Calibre is deliberately +# not used: the MOBI file is written out by hand from the format +# description on the MobileRead wiki, which keeps it tiny and lets the +# test assert exact EXTH values. Everything is deterministic. +# +# Usage: contrib/gen_ebook_testdata.sh [output-directory] +# (default output directory: src/plugins/testdata) + +set -e +OUT="${1:-$(dirname "$0")/../src/plugins/testdata}" +mkdir -p "$OUT" +python3 - "$OUT" <<'PYEOF' +import os, struct, sys, zipfile + +OUT = sys.argv[1] +DATE = (2026, 1, 1, 0, 0, 0) # fixed: the archive must be reproducible + +# ------------------------------------------------------------------- EPUB +CONTAINER = """<?xml version="1.0" encoding="UTF-8"?> +<container version="1.0" + xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> + <rootfiles> + <rootfile full-path="OEBPS/content.opf" + media-type="application/oebps-package+xml"/> + </rootfiles> +</container> +""" + +# One <item> per manifest entry; the plugin reports the count, so keep +# this list and the number asserted by test_ebook.c in step. +ITEMS = [ + ('ncx', 'toc.ncx', 'application/x-dtbncx+xml'), + ('nav', 'nav.xhtml', 'application/xhtml+xml'), + ('css', 'style.css', 'text/css'), + ('ch1', 'chapter1.xhtml', 'application/xhtml+xml'), + ('ch2', 'chapter2.xhtml', 'application/xhtml+xml'), + ('ch3', 'chapter3.xhtml', 'application/xhtml+xml'), + ('cover', 'cover.png', 'image/png'), +] + +OPF = """<?xml version="1.0" encoding="UTF-8"?> +<package xmlns="http://www.idpf.org/2007/opf" version="3.0" + unique-identifier="bookid"> + <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:opf="http://www.idpf.org/2007/opf"> + <dc:title>The libextractor Test Book</dc:title> + <dc:creator opf:role="aut" opf:file-as="Grothoff, Christian" + >Christian Grothoff</dc:creator> + <dc:contributor opf:role="bkp" + >calibre (7.2.0) [https://calibre-ebook.com]</dc:contributor> + <dc:publisher>GNU Press</dc:publisher> + <dc:language>en-GB</dc:language> + <dc:date>2026-01-01T00:00:00+00:00</dc:date> + <dc:subject>digital forensics</dc:subject> + <dc:subject>metadata extraction</dc:subject> + <dc:description>A minimal EPUB built by gen_ebook_testdata.sh, used + to check that the ebook plugin reads Dublin Core out of the + OPF package document.</dc:description> + <dc:rights>Public domain (CC0) & nothing else</dc:rights> + <dc:identifier id="bookid" opf:scheme="ISBN" + >978-3-16-148410-0</dc:identifier> + <dc:identifier opf:scheme="URI" + >urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719</dc:identifier> + <meta name="calibre:timestamp" content="2026-01-01T00:00:00+00:00"/> + <meta name="generator" content="gen_ebook_testdata.sh"/> + <meta property="dcterms:modified">2026-02-01T00:00:00Z</meta> + </metadata> + <manifest> +%s + </manifest> + <spine toc="ncx"> + <itemref idref="ch1"/> + <itemref idref="ch2"/> + <itemref idref="ch3"/> + </spine> +</package> +""" % "\n".join (' <item id="%s" href="%s" media-type="%s"/>' % i + for i in ITEMS) + +# Font obfuscation, which is what encryption.xml usually means in an +# EPUB; the plugin only reports that it is there. +ENCRYPTION = """<?xml version="1.0" encoding="UTF-8"?> +<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> + <EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"> + <EncryptionMethod + Algorithm="http://www.idpf.org/2008/embedding"/> + <CipherData> + <CipherReference URI="OEBPS/fonts/serif.otf"/> + </CipherData> + </EncryptedData> +</encryption> +""" + +CHAPTER = """<?xml version="1.0" encoding="UTF-8"?> +<html xmlns="http://www.w3.org/1999/xhtml"><head><title>%s</title></head> +<body><h1>%s</h1><p>Nothing to see here.</p></body></html> +""" + +epub = os.path.join (OUT, "ebook_test.epub") +with zipfile.ZipFile (epub, "w") as z: + # the spec requires `mimetype' to be the first member and stored + zi = zipfile.ZipInfo ("mimetype", DATE) + zi.compress_type = zipfile.ZIP_STORED + z.writestr (zi, b"application/epub+zip") + for name, data in (("META-INF/container.xml", CONTAINER), + ("META-INF/encryption.xml", ENCRYPTION), + ("OEBPS/content.opf", OPF), + ("OEBPS/chapter1.xhtml", CHAPTER % ("One", "One")), + ("OEBPS/chapter2.xhtml", CHAPTER % ("Two", "Two")), + ("OEBPS/chapter3.xhtml", CHAPTER % ("Three", "Three")), + ("OEBPS/style.css", "body { margin: 1em; }\n")): + zi = zipfile.ZipInfo (name, DATE) + zi.compress_type = zipfile.ZIP_DEFLATED + z.writestr (zi, data.encode ("utf-8")) + + +# ------------------------------------------------------------------- MOBI +# +# Layout per the MobileRead wiki: a 78-byte PDB header, one 8-byte record +# info entry per record, then the records. Record 0 holds the 16-byte +# PalmDOC header, the MOBI header and the EXTH header. +MOBI_EPOCH = 2082844800 # seconds between 1904-01-01 and 1970-01-01 +CTIME = 1767225600 # 2026-01-01T00:00:00Z +MTIME = 1769904000 # 2026-02-01T00:00:00Z + +DB_NAME = b"The_libextractor_Test_Book" +FULL_NAME = b"The libextractor Test Book" +TEXT = (b"This is the body text of the libextractor MOBI test file. " + b"It exists only so that record 1 is not empty.\n") + +EXTH_RECORDS = [ + (100, b"Christian Grothoff"), # author + (101, b"GNU Press"), # publisher + (103, b"A hand-built minimal MOBI file used by the " + b"libextractor test suite."), # description + (104, b"978-3-16-148410-0"), # ISBN + (105, b"digital forensics"), # subject + (106, b"2026-01-01"), # published + (108, b"calibre (7.2.0) [https://calibre-ebook.com]"), # contributor + (109, b"Public domain (CC0)"), # rights + (113, b"B00LIBEXTR7"), # ASIN + (204, struct.pack (">I", 201)), # creator software + (205, struct.pack (">I", 7)), # creator major + (206, struct.pack (">I", 2)), # creator minor + (208, b"libextractor-test-watermark"), # watermark + (501, b"EBOK"), # cdetype + (502, b"2026-02-01T00:00:00+00:00"), # last update time + (503, FULL_NAME), # updated title + (524, b"en-gb"), # language +] + + +def exth (): + body = b"" + for t, d in EXTH_RECORDS: + body += struct.pack (">II", t, 8 + len (d)) + d + size = 12 + len (body) + pad = (-size) % 4 + return b"EXTH" + struct.pack (">II", size + pad, + len (EXTH_RECORDS)) + body + b"\0" * pad + + +MOBI_HEADER_LEN = 232 + + +def record0 (): + e = exth () + # PalmDOC header, then MOBI header, then EXTH, then the full name + full_name_off = 16 + MOBI_HEADER_LEN + len (e) + mobi = bytearray (MOBI_HEADER_LEN) + mobi[0:4] = b"MOBI" + struct.pack_into (">I", mobi, 4, MOBI_HEADER_LEN) + struct.pack_into (">I", mobi, 8, 2) # Mobipocket book + struct.pack_into (">I", mobi, 12, 65001) # UTF-8 + struct.pack_into (">I", mobi, 16, 0x1719) # unique ID + struct.pack_into (">I", mobi, 20, 6) # file version + struct.pack_into (">I", mobi, 64, 2) # first non-book index + struct.pack_into (">I", mobi, 68, full_name_off) + struct.pack_into (">I", mobi, 72, len (FULL_NAME)) + struct.pack_into (">I", mobi, 76, 9) # locale: English + struct.pack_into (">I", mobi, 88, 6) # min version + struct.pack_into (">I", mobi, 92, 2) # first image index + struct.pack_into (">I", mobi, 112, 0x50) # EXTH flags: 0x40 = present + palmdoc = struct.pack (">HHIHHHH", + 1, # no compression + 0, + len (TEXT), + 1, # record count + 4096, # record size + 1, # encryption: old Mobipocket + 0) + rec = palmdoc + bytes (mobi) + e + FULL_NAME + b"\0" + while len (rec) % 4: + rec += b"\0" + return rec + + +records = [record0 (), TEXT, b"\0"] + +hdr = bytearray (78) +hdr[0:len (DB_NAME)] = DB_NAME +struct.pack_into (">HH", hdr, 32, 0, 0) # attributes, version +struct.pack_into (">III", hdr, 36, + CTIME + MOBI_EPOCH, + MTIME + MOBI_EPOCH, + 0) # create/mod/backup +struct.pack_into (">III", hdr, 48, 0, 0, 0) # modnum, appinfo, sort +hdr[60:68] = b"BOOKMOBI" +struct.pack_into (">II", hdr, 68, 0x1719, 0) +struct.pack_into (">H", hdr, 76, len (records)) + +info_len = 8 * len (records) + 2 # + 2 bytes of padding +off = 78 + info_len +info = b"" +for i, r in enumerate (records): + info += struct.pack (">IBBBB", off, 0, 0, 0, i) + off += len (r) +info += b"\0\0" + +with open (os.path.join (OUT, "ebook_test.mobi"), "wb") as f: + f.write (bytes (hdr) + info + b"".join (records)) + +for n in ("ebook_test.epub", "ebook_test.mobi"): + print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n)))) +PYEOF diff --git a/contrib/gen_fuzz_seeds.sh b/contrib/gen_fuzz_seeds.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# This file is in the public domain. +# +# Extra fuzzing seed inputs, for targets whose uncovered code is +# concentrated in format variants that src/plugins/testdata/ does not +# contain. See src/fuzz/CAMPAIGN.md section 8 for the measurement that +# justifies each one. +# +# ./contrib/gen_fuzz_seeds.sh OUTDIR +# +# writes OUTDIR/<target>/ ready to be merged into a campaign corpus, or +# zipped into <target>_seed_corpus.zip next to the built targets. +# Needs qemu-utils (qemu-img) and GNU tar. +# +# The two targets the 2026-08-07 measurement found to be corpus-starved +# rather than time-starved: +# +# fuzz_diskimage 277 uncovered regions, 73% coverage, because the +# plugin dispatches on four container formats and +# testdata/ holds exactly one small sample of each. +# fuzz_tar 99 uncovered regions, 72% coverage, one plain +# ustar archive in testdata/. +# +# These are campaign seeds only. They deliberately do NOT go into +# src/plugins/testdata/, because the plugin tests assert on the exact +# metadata of the files there. +# +# Every harness input is the file image behind LE_FUZZ_EC_PREFIX bytes +# of configuration; all-zero selects what production does (fuzz_ec.h). +# +set -u +OUT="${1:?usage: gen_extra_seeds.sh OUTDIR}" +PREFIX_LEN=4 + +work=$(mktemp -d) +trap 'rm -rf "${work}"' EXIT + +emit () # emit <target> <file> +{ + local t="$1" f="$2" d="${OUT}/$1" + [ -s "${f}" ] || return 0 + mkdir -p "${d}" + { head -c "${PREFIX_LEN}" /dev/zero; cat "${f}"; } \ + > "${d}/extra-$(basename "${f}")" +} + +# --- diskimage ------------------------------------------------------- +q () { qemu-img create "$@" >/dev/null 2>&1; } + +# QCOW2: v2 and v3, several cluster sizes, backed and standalone, with +# the optional feature bits that the plugin has separate branches for. +for cs in 512 4096 65536; do + q -f qcow2 -o "cluster_size=${cs},compat=1.1" "${work}/q3-${cs}.qcow2" 4M + emit fuzz_diskimage "${work}/q3-${cs}.qcow2" + q -f qcow2 -o "cluster_size=${cs},compat=0.10" "${work}/q2-${cs}.qcow2" 4M + emit fuzz_diskimage "${work}/q2-${cs}.qcow2" +done +q -f qcow2 -o cluster_size=512,lazy_refcounts=on,compat=1.1 \ + "${work}/q-lazy.qcow2" 4M +emit fuzz_diskimage "${work}/q-lazy.qcow2" +q -f qcow2 -o cluster_size=512,compression_type=zstd,compat=1.1 \ + "${work}/q-zstd.qcow2" 4M +emit fuzz_diskimage "${work}/q-zstd.qcow2" +q -f qcow2 -o cluster_size=512,extended_l2=on,compat=1.1 \ + "${work}/q-xl2.qcow2" 4M +emit fuzz_diskimage "${work}/q-xl2.qcow2" +(cd "${work}" && q -f qcow2 -o cluster_size=512,backing_file=q3-512.qcow2,backing_fmt=qcow2 \ + "${work}/q-backed.qcow2" 4M) +emit fuzz_diskimage "${work}/q-backed.qcow2" + +# VMDK: every subformat qemu-img will write. The plugin reads the +# 80-byte sparse header and the embedded text descriptor, and those +# differ per subformat. +for sf in monolithicSparse twoGbMaxExtentSparse streamOptimized \ + monolithicFlat twoGbMaxExtentFlat; do + q -f vmdk -o "subformat=${sf}" "${work}/v-${sf}.vmdk" 4M + emit fuzz_diskimage "${work}/v-${sf}.vmdk" +done + +# VHDX: block size and log size are both header fields with branches. +for bs in 1M 8M; do + for ls in 1M 4M; do + q -f vhdx -o "block_size=${bs},log_size=${ls}" "${work}/x-${bs}-${ls}.vhdx" 4M + emit fuzz_diskimage "${work}/x-${bs}-${ls}.vhdx" + done +done + +# VHD (vpc): dynamic and fixed footers are different code paths. +q -f vpc "${work}/h-dyn.vhd" 4M +emit fuzz_diskimage "${work}/h-dyn.vhd" +q -f vpc -o subformat=fixed "${work}/h-fix.vhd" 2M +emit fuzz_diskimage "${work}/h-fix.vhd" + +# --- tar ------------------------------------------------------------- +td="${work}/t" +mkdir -p "${td}/sub" +echo hello > "${td}/a.txt" +printf 'x%.0s' $(seq 1 3000) > "${td}/sub/big.txt" +ln -s a.txt "${td}/link" 2>/dev/null || true +ln "${td}/a.txt" "${td}/hard" 2>/dev/null || true +# A name long enough to need the GNU/pax long-name extension. +long="${td}/$(printf 'n%.0s' $(seq 1 120))" +echo long > "${long}" + +for fmt in gnu ustar pax v7 oldgnu posix; do + tar --format="${fmt}" -cf "${work}/t-${fmt}.tar" -C "${td}" . 2>/dev/null + emit fuzz_tar "${work}/t-${fmt}.tar" +done +# Sparse members and per-file compression flavours. +tar --format=gnu --sparse -cf "${work}/t-sparse.tar" -C "${td}" . 2>/dev/null +emit fuzz_tar "${work}/t-sparse.tar" +# An archive carrying pax extended headers with unusual keywords. +tar --format=pax --pax-option=exthdr.name=%d/PaxHeaders/%f,comment=libextractor \ + -cf "${work}/t-paxopt.tar" -C "${td}" . 2>/dev/null +emit fuzz_tar "${work}/t-paxopt.tar" + +echo "diskimage extra seeds: $(ls -1 "${OUT}/fuzz_diskimage" 2>/dev/null | wc -l)" +echo "tar extra seeds: $(ls -1 "${OUT}/fuzz_tar" 2>/dev/null | wc -l)" diff --git a/contrib/gen_geotiff_testdata.sh b/contrib/gen_geotiff_testdata.sh @@ -0,0 +1,166 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/geotiff_test.tif. +# +# The fixture is hand-built rather than produced by gdal_translate: that +# keeps it tiny (about 1 KB instead of the several KB GDAL emits), makes +# it byte-for-byte reproducible without pulling in gdal-bin, and -- most +# importantly -- lets us pick round numbers for the tiepoint and the +# pixel scale so that the bounding box the plugin computes from them is +# an exact, checkable constant. +# +# Requires: python3 (no third-party modules). +# +# The file it writes is a classic little-endian TIFF 6.0 baseline image, +# 16x16, 8 bit greyscale, uncompressed, carrying the GeoTIFF 1.1 tags +# for EPSG:32633 (WGS 84 / UTM zone 33N) plus GDAL's two private tags. +# It is dedicated to the public domain under CC0 1.0. +set -e + +out="$(dirname "$0")/../src/plugins/testdata/geotiff_test.tif" + +python3 - "$out" <<'EOF' +import struct +import sys + +# --- TIFF field types ------------------------------------------------- +ASCII = 2 +SHORT = 3 +LONG = 4 +DOUBLE = 12 + +TYPE_SIZE = {ASCII: 1, SHORT: 2, LONG: 4, DOUBLE: 8} + +WIDTH = 16 +HEIGHT = 16 + +# --- the GeoTIFF key directory --------------------------------------- +# GeoAsciiParamsTag holds all ASCII-valued keys concatenated, each one +# terminated by '|' rather than NUL. +GEO_ASCII = "WGS 84 / UTM zone 33N|" "WGS 84|" "WGS 84 / UTM zone 33N|" +GT_CITATION_OFF = 0 +GT_CITATION_LEN = 22 +GEOG_CITATION_OFF = 22 +GEOG_CITATION_LEN = 7 +PCS_CITATION_OFF = 29 +PCS_CITATION_LEN = 22 +assert len(GEO_ASCII) == 51 + +# (KeyID, TIFFTagLocation, Count, Value_Offset), ascending by KeyID as +# the specification requires. +GEO_KEYS = [ + (1024, 0, 1, 1), # GTModelTypeGeoKey = ModelTypeProjected + (1025, 0, 1, 1), # GTRasterTypeGeoKey = RasterPixelIsArea + (1026, 34737, GT_CITATION_LEN, GT_CITATION_OFF), # GTCitationGeoKey + (2048, 0, 1, 4326), # GeographicTypeGeoKey = WGS 84 + (2049, 34737, GEOG_CITATION_LEN, GEOG_CITATION_OFF), # GeogCitation + (3072, 0, 1, 32633), # ProjectedCSTypeGeoKey = UTM zone 33N + (3073, 34737, PCS_CITATION_LEN, PCS_CITATION_OFF), # PCSCitation + (3076, 0, 1, 9001), # ProjLinearUnitsGeoKey = metre +] + +geo_dir = [1, 1, 1, len(GEO_KEYS)] # version 1.1.1, NumberOfKeys +for k in GEO_KEYS: + geo_dir.extend(k) + +GDAL_METADATA = ( + "<GDALMetadata>" + "<Item name=\"ACQUISITIONDATETIME\">2026-03-14T09:15:00Z</Item>" + "<Item name=\"SENSOR_NAME\">libextractor synthetic sensor</Item>" + "<Item name=\"PROCESSING_LEVEL\">L1C</Item>" + "</GDALMetadata>" +) + + +def ascii_field(text): + """NUL-terminated ASCII, as TIFF wants it.""" + return (text + "\0").encode("ascii") + + +# --- the tags, ascending by tag number (TIFF requires sorted IFDs) ---- +# StripOffsets (273) is patched once the layout is known; it is a LONG, +# so its size never changes and the layout is stable. +tags = [ + (256, SHORT, 1, struct.pack("<H", WIDTH)), # ImageWidth + (257, SHORT, 1, struct.pack("<H", HEIGHT)), # ImageLength + (258, SHORT, 1, struct.pack("<H", 8)), # BitsPerSample + (259, SHORT, 1, struct.pack("<H", 1)), # Compression=none + (262, SHORT, 1, struct.pack("<H", 1)), # BlackIsZero + (270, ASCII, 0, ascii_field("libextractor GeoTIFF test fixture")), + (271, ASCII, 0, ascii_field("libextractor")), # Make + (272, ASCII, 0, ascii_field("synthetic raster")), # Model + (273, LONG, 1, struct.pack("<I", 0)), # StripOffsets + (277, SHORT, 1, struct.pack("<H", 1)), # SamplesPerPixel + (278, SHORT, 1, struct.pack("<H", HEIGHT)), # RowsPerStrip + (279, LONG, 1, struct.pack("<I", WIDTH * HEIGHT)), # StripByteCounts + (305, ASCII, 0, ascii_field("libextractor gen_geotiff_testdata 1.0")), + (306, ASCII, 0, ascii_field("2026:03:14 09:15:00")), # DateTime + (315, ASCII, 0, ascii_field("Christian Grothoff")), # Artist + (33432, ASCII, 0, ascii_field("CC0 1.0 Universal (public domain)")), + # ModelPixelScaleTag: 10 m per pixel in x and y, no z scale. + (33550, DOUBLE, 3, struct.pack("<3d", 10.0, 10.0, 0.0)), + # ModelTiepointTag: raster (0,0,0) is model (500000, 5400000, 0), + # i.e. the upper-left corner of the upper-left pixel. + (33922, DOUBLE, 6, + struct.pack("<6d", 0.0, 0.0, 0.0, 500000.0, 5400000.0, 0.0)), + (34735, SHORT, len(geo_dir), + struct.pack("<%dH" % len(geo_dir), *geo_dir)), # GeoKeyDirectory + (34737, ASCII, len(GEO_ASCII), GEO_ASCII.encode("ascii")), + (42112, ASCII, 0, ascii_field(GDAL_METADATA)), # GDAL_METADATA + (42113, ASCII, 0, ascii_field("0")), # GDAL_NODATA +] + +# Fill in the counts we left at 0 (ASCII counts include the NUL). +tags = [(t, ty, (len(v) if 0 == c else c), v) for (t, ty, c, v) in tags] +for (t, ty, c, v) in tags: + assert c * TYPE_SIZE[ty] == len(v), t +assert tags == sorted(tags, key=lambda e: e[0]) + +# --- lay the file out ------------------------------------------------- +header_len = 8 +ifd_len = 2 + 12 * len(tags) + 4 +value_base = header_len + ifd_len + +values = bytearray() +for (t, ty, c, v) in tags: + if len(v) > 4: + if 0 != len(values) % 2: + values.append(0) # TIFF offsets must be word-aligned + values.extend(v) + +image_off = value_base + len(values) + +# Patch StripOffsets now that we know where the image data lands. +tags = [(t, ty, c, struct.pack("<I", image_off) if 273 == t else v) + for (t, ty, c, v) in tags] + +ifd = bytearray(struct.pack("<H", len(tags))) +values = bytearray() +for (t, ty, c, v) in tags: + if len(v) <= 4: + field = v + b"\0" * (4 - len(v)) + else: + if 0 != len(values) % 2: + values.append(0) + field = struct.pack("<I", value_base + len(values)) + values.extend(v) + ifd.extend(struct.pack("<HHI", t, ty, c)) + ifd.extend(field) +ifd.extend(struct.pack("<I", 0)) # no further IFD +assert len(ifd) == ifd_len +assert value_base + len(values) == image_off + +# A fixed, non-random pattern so the file is byte-for-byte reproducible. +image = bytes(((x * 16 + y) & 0xFF) for y in range(HEIGHT) + for x in range(WIDTH)) + +blob = struct.pack("<2sHI", b"II", 42, header_len) + bytes(ifd) \ + + bytes(values) + image +assert len(blob) == image_off + len(image) + +with open(sys.argv[1], "wb") as f: + f.write(blob) +print("wrote %s (%d bytes)" % (sys.argv[1], len(blob))) +EOF diff --git a/contrib/gen_gpx_testdata.sh b/contrib/gen_gpx_testdata.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/gpx_test.gpx. +# +# Requires: nothing beyond a POSIX shell -- the file is plain text and is +# written out literally, so the result is byte-for-byte reproducible. +# +# The file is a GPX 1.1 track that exercises exactly the fields the gpx +# plugin parses: the `creator' attribute (the single most identifying +# item in the format), a full <metadata> block with an explicit <bounds>, +# two waypoints, a two-point route, and one track segment of five points +# carrying timestamps and elevations. The <desc> deliberately contains +# an entity reference so that the unescaping path is covered. +# +# The data is invented (a walk through central Zurich); it is not derived +# from anybody's real location history and is dedicated to the public +# domain (CC0). +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/gpx_test.gpx" + +cat > "$out" <<'EOF' +<?xml version="1.0" encoding="UTF-8"?> +<gpx version="1.1" + creator="Garmin eTrex 30" + xmlns="http://www.topografix.com/GPX/1/1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd"> + <metadata> + <name>Zurich morning walk</name> + <desc>Recorded on the way to the office & back</desc> + <author> + <name>Anna Mueller</name> + <email id="anna.mueller" domain="example.com"/> + <link href="https://example.com/~anna/"> + <text>Anna's tracks</text> + </link> + </author> + <copyright author="Anna Mueller"> + <year>2024</year> + <license>https://creativecommons.org/publicdomain/zero/1.0/</license> + </copyright> + <link href="https://example.com/tracks/2024-03-15.gpx"> + <text>Original upload</text> + </link> + <time>2024-03-15T07:12:00Z</time> + <keywords>walk,commute,zurich</keywords> + <bounds minlat="47.376890" minlon="8.541694" + maxlat="47.385100" maxlon="8.552300"/> + </metadata> + <wpt lat="47.376890" lon="8.541694"> + <ele>408.2</ele> + <name>Home</name> + </wpt> + <wpt lat="47.385100" lon="8.552300"> + <ele>419.3</ele> + <name>Office</name> + </wpt> + <rte> + <name>Planned route</name> + <rtept lat="47.377000" lon="8.542000"/> + <rtept lat="47.385000" lon="8.552000"/> + </rte> + <trk> + <name>Zurich morning walk</name> + <type>walking</type> + <trkseg> + <trkpt lat="47.376890" lon="8.541694"> + <ele>408.2</ele> + <time>2024-03-15T07:12:33Z</time> + </trkpt> + <trkpt lat="47.378120" lon="8.543210"> + <ele>412.5</ele> + <time>2024-03-15T07:14:03Z</time> + </trkpt> + <trkpt lat="47.380450" lon="8.546800"> + <ele>425.0</ele> + <time>2024-03-15T07:17:45Z</time> + </trkpt> + <trkpt lat="47.382990" lon="8.549900"> + <ele>430.7</ele> + <time>2024-03-15T07:21:10Z</time> + </trkpt> + <trkpt lat="47.385100" lon="8.552300"> + <ele>419.3</ele> + <time>2024-03-15T07:24:53Z</time> + </trkpt> + </trkseg> + </trk> +</gpx> +EOF + +echo "wrote $out ($(wc -c < "$out") bytes)" diff --git a/contrib/gen_heif_testdata.sh b/contrib/gen_heif_testdata.sh @@ -0,0 +1,210 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Generate the test data for the "heif" plugin: +# +# src/plugins/testdata/heif_test.heic +# src/plugins/testdata/heif_test.avif +# +# Both files are hand-constructed ISO base media file format containers. +# They are structurally conformant HEIF/AVIF files -- the box tree, the +# item structures and the property associations are exactly what the +# specifications call for -- but the 'mdat' payload is a placeholder +# rather than a real HEVC or AV1 bitstream, so they identify but do not +# decode. That is deliberate: the plugin is a container parser, the +# values it must report are known exactly, and no encoder is needed. +# +# Needs: python3 (no third party modules). Deterministic: no +# timestamps, no random data. +# +# Written to CC0; there is no third party content in these files. +# +# Usage: contrib/gen_heif_testdata.sh [output-directory] + +set -e + +OUT="${1:-src/plugins/testdata}" +mkdir -p "$OUT" + +python3 - "$OUT" <<'EOF' +import struct +import sys + +out = sys.argv[1] + + +def box(btype, payload): + """A plain ISO-BMFF box: 32-bit size, 4CC type, payload.""" + assert len(btype) == 4 + return struct.pack('>I', 8 + len(payload)) + btype.encode('ascii') + payload + + +def fullbox(btype, version, flags, payload): + """A FullBox: a box whose payload starts with version and flags.""" + return box(btype, bytes([version]) + struct.pack('>I', flags)[1:] + payload) + + +def ftyp(major, brands): + return box('ftyp', + major.encode('ascii') + + struct.pack('>I', 0) + + b''.join(b.encode('ascii') for b in brands)) + + +def hdlr(handler): + return fullbox('hdlr', 0, 0, + struct.pack('>I', 0) + + handler.encode('ascii') + + b'\0' * 12 + + b'libextractor test\0') + + +def pitm(item_id): + return fullbox('pitm', 0, 0, struct.pack('>H', item_id)) + + +def infe(item_id, item_type, name): + """ItemInfoEntry version 2: 16-bit item_ID, then the item_type.""" + return fullbox('infe', 2, 0, + struct.pack('>HH', item_id, 0) + + item_type.encode('ascii') + + name.encode('ascii') + b'\0') + + +def iinf(entries): + return fullbox('iinf', 0, 0, + struct.pack('>H', len(entries)) + b''.join(entries)) + + +def iref_thmb(from_id, to_id): + return fullbox('iref', 0, 0, + box('thmb', struct.pack('>HHH', from_id, 1, to_id))) + + +def ispe(width, height): + return fullbox('ispe', 0, 0, struct.pack('>II', width, height)) + + +def pixi(depths): + return fullbox('pixi', 0, 0, bytes([len(depths)]) + bytes(depths)) + + +def irot(quarter_turns): + """ImageRotation: anti-clockwise quarter turns, 0..3.""" + return box('irot', bytes([quarter_turns & 0x03])) + + +def imir(axis): + """ImageMirror: 0 mirrors about a vertical axis, 1 about a horizontal one.""" + return box('imir', bytes([axis & 0x01])) + + +def colr_nclx(primaries, transfer, matrix, full_range): + return box('colr', + b'nclx' + + struct.pack('>HHH', primaries, transfer, matrix) + + bytes([0x80 if full_range else 0x00])) + + +def av1c(high_bitdepth, twelve_bit, monochrome): + """AV1CodecConfigurationBox, the four byte fixed part.""" + flags = (high_bitdepth << 6) | (twelve_bit << 5) | (monochrome << 4) | 0x0C + return box('av1C', bytes([0x81, 0x00, flags, 0x00])) + + +def hvcc(): + """A minimal HEVCDecoderConfigurationRecord shell (no parameter sets).""" + return box('hvcC', + bytes([0x01, 0x01, 0x60, 0x00, 0x00, 0x00, + 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3C, 0xF0, 0x00, 0xFC, 0xFD, 0xF8, + 0xF8, 0x00, 0x00, 0x0F, 0x00])) + + +def ipma(entries): + """ItemPropertyAssociation with 16-bit item IDs and 7-bit indices.""" + payload = struct.pack('>I', len(entries)) + for item_id, props in entries: + payload += struct.pack('>HB', item_id, len(props)) + for essential, index in props: + assert 0 < index < 0x80 + payload += bytes([(0x80 if essential else 0x00) | index]) + return fullbox('ipma', 0, 0, payload) + + +def iprp(props, assoc): + return box('iprp', box('ipco', b''.join(props)) + ipma(assoc)) + + +def meta(children): + return fullbox('meta', 0, 0, b''.join(children)) + + +def mdat(nbytes): + """A placeholder for the coded data; never read by the plugin.""" + return box('mdat', bytes((i * 7 + 3) & 0xFF for i in range(nbytes))) + + +# ---------------------------------------------------------------- HEIC +# +# Major brand 'heic', so the MIME type and the HEVC codec follow from +# the brand alone. Three items: the primary image (id 1), a thumbnail +# (id 2, tied to the primary by an 'iref' of type 'thmb') and an Exif +# metadata item (id 3). +# +# The property container deliberately holds the *thumbnail's* 'ispe' +# first: a parser that simply reports the first 'ispe' it finds says +# 16x12, and only one that follows the 'ipma' association for the +# primary item reports the true 96x64. +heic_props = [ + ispe(16, 12), # 1: the thumbnail's extent + ispe(96, 64), # 2: the primary item's extent + pixi([10, 10, 10]), # 3: 10 bit, i.e. an HDR capture + irot(1), # 4: 90 degrees anti-clockwise + colr_nclx(9, 16, 9, True), # 5: BT.2020 with the PQ curve + imir(0), # 6: mirrored left to right + hvcc(), # 7 +] +heic_assoc = [ + (1, [(False, 2), (False, 3), (True, 4), (False, 5), (True, 6), + (True, 7)]), + (2, [(False, 1)]), +] +heic = (ftyp('heic', ['mif1', 'heic', 'miaf', 'MiHB']) + + meta([hdlr('pict'), + pitm(1), + iinf([infe(1, 'hvc1', 'primary'), + infe(2, 'hvc1', 'thumb'), + infe(3, 'Exif', 'exif')]), + iref_thmb(2, 1), + iprp(heic_props, heic_assoc)]) + + mdat(512)) + +# ---------------------------------------------------------------- AVIF +# +# Major brand 'mif1' with 'avif' among the compatible brands, which is +# what several AVIF writers emit and what forces a reader to look past +# the major brand. One image item, no 'pixi', so the bit depth has to +# come from the 'av1C' configuration record. +avif_props = [ + ispe(128, 72), # 1 + av1c(0, 0, 0), # 2: 8 bit, colour + colr_nclx(1, 13, 6, True), # 3: BT.709 primaries, sRGB curve +] +avif_assoc = [ + (1, [(False, 1), (True, 2), (False, 3)]), +] +avif = (ftyp('mif1', ['mif1', 'miaf', 'MA1B', 'avif']) + + meta([hdlr('pict'), + pitm(1), + iinf([infe(1, 'av01', 'primary')]), + iprp(avif_props, avif_assoc)]) + + mdat(384)) + +for name, data in (('heif_test.heic', heic), ('heif_test.avif', avif)): + path = out + '/' + name + with open(path, 'wb') as f: + f.write(data) + print('%s: %d bytes' % (path, len(data))) +EOF diff --git a/contrib/gen_id3_testdata.sh b/contrib/gen_id3_testdata.sh @@ -0,0 +1,138 @@ +#!/bin/sh +# Regenerate src/plugins/testdata/id3_test.mp3 for test_id3.c. +# +# Debian packages needed: python3, python3-mutagen, lame. +# +# The audio is a one second 440 Hz sine at 44100 Hz, mono, written as a +# WAV by python and encoded by LAME so that the file carries a real +# Xing/LAME tag -- that tag is the point of the exercise, and ffmpeg +# writes its own ("Lavf lame") rather than LAME's, which would not +# exercise the same fields. +# +# Everything is deterministic: no timestamps, no random data. Note +# that the LAME encoder version string ("LAME3.100") and the settings +# summary derived from the LAME tag are asserted on literally in +# test_id3.c; if you regenerate this with a different LAME release, +# those two expectations have to be updated to match. +# +# The tag set is ID3v2.4 on purpose. It exercises all four text +# encodings in one file (TIT2 as UTF-8, TPE1 as UTF-16 with a byte +# order mark, TALB as ISO-8859-1, the rest as written), and the USLT +# and APIC frames are both longer than 127 bytes, which is where the +# synchsafe frame size of 2.4 first differs from the plain integer of +# 2.3. A hand-written ID3v1 tag follows the audio; its comment and +# year fields are the only ones the plugin should use, since the ID3v2 +# tag covers everything else. +# +# This file is part of libextractor; the generated data is CC0. +set -e + +out="$(dirname "$0")/../src/plugins/testdata" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' 0 + +python3 - "$tmp" <<'EOF' +import math +import os +import struct +import sys +import wave + +tmp = sys.argv[1] +w = wave.open(os.path.join(tmp, 'tone.wav'), 'wb') +w.setnchannels(1) +w.setsampwidth(2) +w.setframerate(44100) +w.writeframes(b''.join(struct.pack('<h', int(12000 * math.sin(2 * math.pi + * 440 * i + / 44100))) + for i in range(44100))) +w.close() +EOF + +lame -V 5 --vbr-new -m m -q 2 --quiet "$tmp/tone.wav" "$tmp/id3_test.mp3" + +python3 - "$tmp" <<'EOF' +import os +import struct +import sys +import zlib + +from mutagen.id3 import (ID3, APIC, COMM, PRIV, TALB, TBPM, TCOM, TCON, TCOP, + TDRC, TENC, TFLT, TIT2, TLAN, TMED, TOWN, TPE1, TPE3, + TPOS, TPUB, TRCK, TSSE, TXXX, UFID, USLT, WOAF, WOAR) + +tmp = sys.argv[1] +path = os.path.join(tmp, 'id3_test.mp3') + + +def png(side): + """A deterministic side x side RGB PNG, roughly 300 bytes.""" + raw = b''.join(b'\x00' + bytes(((x * 7) % 256, (y * 11) % 256, + ((x + y) * 3) % 256)[c] + for x in range(side) for c in range(3)) + for y in range(side)) + + def chunk(tag, data): + c = tag + data + return (struct.pack('>I', len(data)) + c + + struct.pack('>I', zlib.crc32(c) & 0xFFFFFFFF)) + + return (b'\x89PNG\r\n\x1a\n' + + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(raw, 9)) + + chunk(b'IEND', b'')) + + +lyrics = ('One two three four five six seven eight nine ten. ' + 'This lyric frame exists only to be longer than one hundred ' + 'and twenty seven bytes, so that the synchsafe frame size of ' + 'ID3v2.4 differs from a plain big endian integer here.') + +tags = ID3() +tags.add(TIT2(encoding=3, text=['Grüße aus Köln'])) +tags.add(TPE1(encoding=1, text=['Bœuf Trio'])) +tags.add(TALB(encoding=0, text=['Café Sessions'])) +tags.add(TCON(encoding=0, text=['(52)'])) +tags.add(TRCK(encoding=0, text=['3/12'])) +tags.add(TPOS(encoding=0, text=['1/2'])) +tags.add(TDRC(encoding=0, text=['2024-03-14'])) +tags.add(TCOM(encoding=0, text=['Ada Lovelace'])) +tags.add(TPE3(encoding=0, text=['Grace Hopper'])) +tags.add(TPUB(encoding=0, text=['GNU Records'])) +tags.add(TCOP(encoding=0, text=['2026 GNU libextractor'])) +tags.add(TLAN(encoding=0, text=['deu'])) +tags.add(TBPM(encoding=0, text=['128'])) +tags.add(TENC(encoding=0, text=['GNU libextractor test rig'])) +tags.add(TSSE(encoding=0, text=['LAME 3.100 -V 5 --vbr-new'])) +tags.add(TOWN(encoding=0, text=['licensee@example.org'])) +tags.add(TFLT(encoding=0, text=['MPG/3'])) +tags.add(TMED(encoding=0, text=['DIG/A/T'])) +tags.add(USLT(encoding=3, lang='eng', desc='', text=lyrics)) +tags.add(UFID(owner='http://musicbrainz.org', + data=b'9f1a3e6c-0000-4000-8000-0123456789ab')) +tags.add(WOAF(url='https://www.gnu.org/software/libextractor/')) +tags.add(WOAR(url='https://www.gnu.org/')) +tags.add(PRIV(owner='com.apple.iTunes', data=b'\x00\x01\x02\x03')) +tags.add(TXXX(encoding=0, desc='apID', text=['buyer@example.org'])) +tags.add(APIC(encoding=0, mime='image/png', type=3, desc='Cover', + data=png(8))) +tags.save(path, v2_version=4, v1=0) + +# An ID3v1 tag written by hand, so that its fields are visibly +# different from the ID3v2 ones: only the comment and the year should +# ever reach the caller, because the ID3v2 tag covers everything else. +def field(s, n): + return s.encode('latin-1')[:n].ljust(n, b'\x00') + + +v1 = (b'TAG' + field('ID3v1 Title', 30) + field('ID3v1 Artist', 30) + + field('ID3v1 Album', 30) + field('1999', 4) + + field('ID3v1 comment field', 30) + bytes([52])) +assert len(v1) == 128 +with open(path, 'ab') as f: + f.write(v1) +EOF + +cp "$tmp/id3_test.mp3" "$out/id3_test.mp3" +ls -l "$out/id3_test.mp3" diff --git a/contrib/gen_iso9660_testdata.sh b/contrib/gen_iso9660_testdata.sh @@ -0,0 +1,184 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/iso9660_test.iso. +# +# Requires: python3 (standard library only). +# +# genisoimage/xorriso are deliberately NOT used. A mastering program +# stamps its own name and the current time into the descriptors, which +# would make the image non-reproducible and would leave us asserting on +# whatever that program happened to write. Hand-building the volume +# descriptor set instead lets every field the plugin reads be an exact, +# known value, and keeps the image down to the 21 sectors it takes to +# hold a system area, four descriptors and one directory extent. +# +# Layout: +# sector 0-15 system area (all zero, mandated by ECMA-119) +# sector 16 primary volume descriptor +# sector 17 boot record, El Torito +# sector 18 supplementary volume descriptor, Joliet (UCS-2 level 3) +# sector 19 volume descriptor set terminator +# sector 20 root directory extent, with SUSP/Rock Ridge entries +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/iso9660_test.iso" + +python3 - "$out" <<'EOF' +import struct +import sys + +SECTOR = 2048 +out = sys.argv[1] + +SYSTEM_ID = "LINUX" +VOLUME_ID = "FORENSIC_TEST_VOL" +VOLUME_SET_ID = "FORENSIC_SET_01" +PUBLISHER = "GNU LIBEXTRACTOR PROJECT" +PREPARER = "LIBEXTRACTOR TEST SUITE" +APPLICATION = "MKISOFS 2.01 (LIBEXTRACTOR TEST IMAGE)" +COPYRIGHT_FILE = "COPYING.TXT" +ABSTRACT_FILE = "ABSTRACT.TXT" +BIBLIO_FILE = "BIBLIO.TXT" + +TOTAL_SECTORS = 21 +ROOT_EXTENT = 20 + + +def astr(text, width): + """a-characters / d-characters: space padded, fixed width.""" + b = text.encode("ascii") + assert len(b) <= width, text + return b + b" " * (width - len(b)) + + +def ucs2(text, width): + """Joliet identifiers are UCS-2BE, padded with U+0020.""" + b = text.encode("utf-16-be")[:width & ~1] + pad = (width - len(b)) // 2 + return b + b"\x00\x20" * pad + b"\x00" * ((width - len(b)) % 2) + + +def both16(v): + return struct.pack("<H", v) + struct.pack(">H", v) + + +def both32(v): + return struct.pack("<I", v) + struct.pack(">I", v) + + +def dtime(text): + """17-byte dec-datetime, 'YYYYMMDDHHMMSSss' plus a GMT offset byte.""" + assert len(text) == 16, text + return text.encode("ascii") + b"\x00" + + +UNSET_DATE = b"0" * 16 + b"\x00" + +# All four dates are pinned; the test asserts on them literally. +CREATED = dtime("2024031512345678") # 2024-03-15T12:34:56Z +MODIFIED = dtime("2025010203040506") # 2025-01-02T03:04:05Z +EXPIRES = dtime("2030060100000000") # 2030-06-01T00:00:00Z +EFFECTIVE = dtime("2024031600000000") # 2024-03-16T00:00:00Z + + +def dir_record(extent, length, flags, ident, system_use=b""): + """A directory record; the caller keeps it even-length.""" + base = bytearray() + base += b"\x00" # length, filled in below + base += b"\x00" # extended attribute length + base += both32(extent) + base += both32(length) + # recording date/time: 1980-01-01 00:00:00 GMT + base += bytes([80, 1, 1, 0, 0, 0, 0]) + base += bytes([flags]) + base += b"\x00" # file unit size + base += b"\x00" # interleave gap size + base += both16(1) # volume sequence number + base += bytes([len(ident)]) + base += ident + if 0 != len(base) % 2: + base += b"\x00" # pad the identifier + base += system_use + assert 0 == len(base) % 2, len(base) + base[0] = len(base) + return bytes(base) + + +ROOT_RECORD = dir_record(ROOT_EXTENT, SECTOR, 0x02, b"\x00") +assert 34 == len(ROOT_RECORD), len(ROOT_RECORD) + + +def volume_descriptor(vtype, ident_enc, escape=b""): + d = bytearray(SECTOR) + d[0] = vtype + d[1:6] = b"CD001" + d[6] = 1 # version + d[7] = 0 # unused / volume flags + d[8:40] = ident_enc(SYSTEM_ID, 32) + d[40:72] = ident_enc(VOLUME_ID, 32) + d[80:88] = both32(TOTAL_SECTORS) + d[88:88 + len(escape)] = escape # unused in a PVD + d[120:124] = both16(2) # volume set size: two volumes + d[124:128] = both16(1) # volume sequence number + d[128:132] = both16(SECTOR) # logical block size + d[132:140] = both32(10) # path table size + d[140:144] = struct.pack("<I", 19) # type-L path table + d[148:152] = struct.pack(">I", 19) # type-M path table + d[156:190] = ROOT_RECORD + d[190:318] = ident_enc(VOLUME_SET_ID, 128) + d[318:446] = ident_enc(PUBLISHER, 128) + d[446:574] = ident_enc(PREPARER, 128) + d[574:702] = ident_enc(APPLICATION, 128) + d[702:739] = ident_enc(COPYRIGHT_FILE, 37) + d[739:776] = ident_enc(ABSTRACT_FILE, 37) + d[776:813] = ident_enc(BIBLIO_FILE, 37) + d[813:830] = CREATED + d[830:847] = MODIFIED + d[847:864] = EXPIRES + d[864:881] = EFFECTIVE + d[881] = 1 # file structure version + return bytes(d) + + +pvd = volume_descriptor(1, astr) + +# Boot record: El Torito names itself in the boot system identifier. +boot = bytearray(SECTOR) +boot[0] = 0 +boot[1:6] = b"CD001" +boot[6] = 1 +bsi = b"EL TORITO SPECIFICATION" +boot[7:7 + len(bsi)] = bsi # NUL padded, per the spec +boot[71:75] = struct.pack("<I", 20) # boot catalog pointer +boot = bytes(boot) + +# Supplementary volume descriptor: Joliet, UCS-2 level 3 ("%/E"). +svd = bytearray(volume_descriptor(2, ucs2, escape=b"%/E")) +svd = bytes(svd) + +term = bytearray(SECTOR) +term[0] = 255 +term[1:6] = b"CD001" +term[6] = 1 +term = bytes(term) + +# Root directory extent. The "." record carries the SUSP `SP' entry and +# a Rock Ridge `RR' entry, which is what the plugin looks for. +susp = (b"SP" + bytes([7, 1, 0xBE, 0xEF, 0]) + + b"RR" + bytes([5, 1, 0x81])) +dot = dir_record(ROOT_EXTENT, SECTOR, 0x02, b"\x00", susp) +dotdot = dir_record(ROOT_EXTENT, SECTOR, 0x02, b"\x01") +rootdir = bytearray(SECTOR) +rootdir[0:len(dot)] = dot +rootdir[len(dot):len(dot) + len(dotdot)] = dotdot +rootdir = bytes(rootdir) + +image = b"\x00" * (16 * SECTOR) + pvd + boot + svd + term + rootdir +assert TOTAL_SECTORS * SECTOR == len(image), len(image) +with open(out, "wb") as f: + f.write(image) +print("wrote %s (%d bytes)" % (out, len(image))) +EOF diff --git a/contrib/gen_kml_testdata.sh b/contrib/gen_kml_testdata.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/kml_test.kml. +# +# Requires: nothing beyond a POSIX shell -- the file is plain text and is +# written out literally, so the result is byte-for-byte reproducible. +# +# The file is an OGC KML 2.2 document that exercises what the kml plugin +# parses: the document name and an HTML-bearing CDATA description, an +# atom:author and atom:link, a <LookAt> (the *viewer's* position, which +# the plugin must not confuse with the data), a <TimeStamp>, three +# Placemarks -- two points and one LineString -- and a <NetworkLink>, +# which is the forensically interesting bit because it means opening the +# file fetches something from a remote host. +# +# The data is invented (the same Zurich walk as gpx_test.gpx, so that the +# two plugins can be compared) and is dedicated to the public domain +# (CC0). Note that KML coordinates are lon,lat[,alt] -- the reverse of +# the GPX attribute order. +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/kml_test.kml" + +cat > "$out" <<'EOF' +<?xml version="1.0" encoding="UTF-8"?> +<kml xmlns="http://www.opengis.net/kml/2.2" + xmlns:atom="http://www.w3.org/2005/Atom" + xmlns:gx="http://www.google.com/kml/ext/2.2"> + <Document> + <name>Zurich morning walk</name> + <description><![CDATA[<p>Exported from the handheld on + <b>2024-03-15</b>. See <a href="https://example.com/~anna/">the + index</a> for the other days.</p>]]></description> + <atom:author> + <atom:name>Anna Mueller</atom:name> + </atom:author> + <atom:link href="https://example.com/~anna/"/> + <TimeStamp> + <when>2024-03-15T07:12:00Z</when> + </TimeStamp> + <LookAt> + <longitude>8.547000</longitude> + <latitude>47.381000</latitude> + <altitude>0</altitude> + <range>1200</range> + <tilt>45</tilt> + <heading>10</heading> + </LookAt> + <Style id="track"> + <LineStyle> + <color>ff0000ff</color> + <width>4</width> + </LineStyle> + </Style> + <Placemark> + <name>Home</name> + <description>Start of the recording</description> + <Point> + <coordinates>8.541694,47.376890,408.2</coordinates> + </Point> + </Placemark> + <Placemark> + <name>Office</name> + <description>End of the recording</description> + <Point> + <coordinates>8.552300,47.385100,419.3</coordinates> + </Point> + </Placemark> + <Placemark> + <name>Walked track</name> + <styleUrl>#track</styleUrl> + <LineString> + <tessellate>1</tessellate> + <coordinates> + 8.541694,47.376890,408.2 + 8.543210,47.378120,412.5 + 8.546800,47.380450,425.0 + 8.549900,47.382990,430.7 + 8.552300,47.385100,419.3 + </coordinates> + </LineString> + </Placemark> + <NetworkLink> + <name>Live position</name> + <Link> + <href>https://tracker.example.net/live/anna.kml</href> + <refreshMode>onInterval</refreshMode> + <refreshInterval>30</refreshInterval> + </Link> + </NetworkLink> + </Document> +</kml> +EOF + +echo "wrote $out ($(wc -c < "$out") bytes)" diff --git a/contrib/gen_lnk_testdata.sh b/contrib/gen_lnk_testdata.sh @@ -0,0 +1,189 @@ +#!/bin/sh +# Regenerate the shell link test file used by src/plugins/test_lnk.c. +# +# Copyright (C) 2026 Christian Grothoff +# The generated file is dedicated to the public domain (CC0 1.0). +# +# The file is assembled from [MS-SHLLINK] rather than produced by +# Windows: that keeps it a few hundred bytes, keeps every timestamp, +# serial number, machine name and MAC address a constant this script +# owns, and lets it carry exactly the structures the plugin parses -- +# an ID list to skip over, a LinkInfo with a volume label and a path +# split across its two halves, unicode string data, and a tracker block +# whose object identifiers are version 1 UUIDs. +# +# Needs nothing but python3. +set -e + +srcdir=$(dirname "$0") +outdir=${1:-"$srcdir/../src/plugins/testdata"} + +python3 - "$outdir/lnk_test.lnk" <<'EOF' +import struct +import sys + +OUT = sys.argv[1] + +# ---------------------------------------------------------- constants +CLSID = bytes([0x01, 0x14, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46]) +CREATION = 1614834367 # 2021-03-04T05:06:07Z +ACCESS = 1657271411 # 2022-07-08T09:10:11Z +WRITE = 1623767400 # 2021-06-15T14:30:00Z +TARGET_SIZE = 123456 +ATTRIBUTES = 0x0021 # READONLY | ARCHIVE +DRIVE_SERIAL = 0x1A2B3C4D # reported as 1A2B-3C4D +DRIVE_TYPE = 3 # fixed disk +VOLUME_LABEL = "EVIDENCE" +BASE_PATH = "C:\\Users\\analyst\\Desktop\\" +PATH_SUFFIX = "evidence.txt" +NAME_STRING = "Evidence file from the lab machine" +RELATIVE_PATH = "..\\..\\Desktop\\evidence.txt" +WORKING_DIR = "C:\\Users\\analyst\\Desktop" +ARGUMENTS = "/quiet /log C:\\temp\\out.log" +ICON_LOCATION = "%SystemRoot%\\system32\\shell32.dll" +MACHINE_ID = b"lab-ws-07" +# 9b3a5e1c-7d42-4f80-a1e6-0c2d4b6f8a31, stored the Microsoft way +DROID_VOLUME = bytes([0x1C, 0x5E, 0x3A, 0x9B, 0x42, 0x7D, 0x80, 0x4F, + 0xA1, 0xE6, 0x0C, 0x2D, 0x4B, 0x6F, 0x8A, 0x31]) +# 5f3e2d1c-a4b6-11d2-8ae7-001b44113ab7: a version 1 UUID whose node +# field is the MAC address 00:1b:44:11:3a:b7. The version nibble lives +# in the high half of byte 7 because the first three fields are stored +# little-endian. +DROID_FILE = bytes([0x1C, 0x2D, 0x3E, 0x5F, 0xB6, 0xA4, 0xD2, 0x11, + 0x8A, 0xE7, 0x00, 0x1B, 0x44, 0x11, 0x3A, 0xB7]) + +LNK_HAS_ID_LIST = 0x01 +LNK_HAS_LINK_INFO = 0x02 +LNK_HAS_NAME = 0x04 +LNK_HAS_RELATIVE_PATH = 0x08 +LNK_HAS_WORKING_DIR = 0x10 +LNK_HAS_ARGUMENTS = 0x20 +LNK_HAS_ICON_LOCATION = 0x40 +LNK_IS_UNICODE = 0x80 + + +def filetime(unix): + """Seconds since the Unix epoch as 100ns units since 1601-01-01.""" + return (unix + 11644473600) * 10000000 + + +def utf16(text): + """UTF-16LE, NUL terminated.""" + return text.encode("utf-16-le") + b"\x00\x00" + + +def ansi(text): + """Single-byte, NUL terminated.""" + return text.encode("ascii") + b"\x00" + + +# ------------------------------------------------------ ShellLinkHeader +flags = (LNK_HAS_ID_LIST | LNK_HAS_LINK_INFO | LNK_HAS_NAME + | LNK_HAS_RELATIVE_PATH | LNK_HAS_WORKING_DIR + | LNK_HAS_ARGUMENTS | LNK_HAS_ICON_LOCATION | LNK_IS_UNICODE) +header = struct.pack("<I16sIIQQQIiIHHII", + 0x0000004C, + CLSID, + flags, + ATTRIBUTES, + filetime(CREATION), + filetime(ACCESS), + filetime(WRITE), + TARGET_SIZE, + 0, # IconIndex + 1, # ShowCommand: SW_SHOWNORMAL + 0, # HotKey + 0, 0, 0) # Reserved1..3 +assert len(header) == 0x4C, hex(len(header)) + +# --------------------------------------------------- LinkTargetIDList +# Two shell items and the terminating empty one. The plugin only has +# to step over this correctly, so the contents are the shortest thing +# that is still shaped like the real structure: the "My Computer" +# folder followed by a drive. +item1 = struct.pack("<BB", 0x1F, 0x50) + bytes( + [0xE0, 0x4F, 0xD0, 0x20, 0xEA, 0x3A, 0x69, 0x10, + 0xA2, 0xD8, 0x08, 0x00, 0x2B, 0x30, 0x30, 0x9D]) +item1 = struct.pack("<H", len(item1) + 2) + item1 +item2 = struct.pack("<B", 0x2F) + b"C:\\" + bytes(10) +item2 = struct.pack("<H", len(item2) + 2) + item2 +idlist = item1 + item2 + struct.pack("<H", 0) + +# ----------------------------------------------------------- LinkInfo +vol_label_w = utf16(VOLUME_LABEL) +volume = struct.pack("<IIIII", + 0, # VolumeIDSize, filled in below + DRIVE_TYPE, + DRIVE_SERIAL, + 0x14, # use the unicode label instead + 0x14) # VolumeLabelOffsetUnicode +volume += vol_label_w +volume = struct.pack("<I", len(volume)) + volume[4:] + +base_a = ansi(BASE_PATH) +suffix_a = ansi(PATH_SUFFIX) +base_w = utf16(BASE_PATH) +suffix_w = utf16(PATH_SUFFIX) + +LI_HEADER = 0x24 +vol_off = LI_HEADER +base_a_off = vol_off + len(volume) +suffix_a_off = base_a_off + len(base_a) +base_w_off = suffix_a_off + len(suffix_a) +base_w_off += base_w_off % 2 # keep UTF-16 aligned +suffix_w_off = base_w_off + len(base_w) +li_size = suffix_w_off + len(suffix_w) + +link_info = struct.pack("<IIIIIIIII", + li_size, + LI_HEADER, + 0x1, # VolumeIDAndLocalBasePath + vol_off, + base_a_off, + 0, # no network link + suffix_a_off, + base_w_off, + suffix_w_off) +link_info += volume + base_a + suffix_a +link_info += bytes(base_w_off - len(link_info)) +link_info += base_w + suffix_w +assert len(link_info) == li_size, (len(link_info), li_size) + +# --------------------------------------------------------- StringData +def string_data(text): + """A counted, *not* NUL terminated UTF-16LE string.""" + raw = text.encode("utf-16-le") + return struct.pack("<H", len(raw) // 2) + raw + + +strings = (string_data(NAME_STRING) + + string_data(RELATIVE_PATH) + + string_data(WORKING_DIR) + + string_data(ARGUMENTS) + + string_data(ICON_LOCATION)) + +# ---------------------------------------------------------- ExtraData +tracker = struct.pack("<IIII", + 0x60, # BlockSize + 0xA0000003, # TrackerDataBlock + 0x58, # Length + 0) # Version +tracker += MACHINE_ID.ljust(16, b"\x00") +tracker += DROID_VOLUME + DROID_FILE # Droid +tracker += DROID_VOLUME + DROID_FILE # DroidBirth +assert len(tracker) == 0x60, hex(len(tracker)) + +extra = tracker + struct.pack("<I", 0) # terminal block + +image = (header + + struct.pack("<H", len(idlist)) + idlist + + link_info + + strings + + extra) + +with open(OUT, "wb") as f: + f.write(image) +EOF + +ls -l "$outdir/lnk_test.lnk" diff --git a/contrib/gen_mbox_testdata.sh b/contrib/gen_mbox_testdata.sh @@ -0,0 +1,135 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/mbox_test.mbox. +# +# Requires: nothing beyond a POSIX shell -- the mailbox is plain ASCII +# (the non-ASCII header values are RFC 2047 encoded words) and is written +# out literally, so the result is byte-for-byte reproducible. +# +# Three messages in Unix mbox (mboxo) form. The first one carries +# everything the plugin is expected to recover: a three-hop Received +# chain with two IPv4 literals and one IPv6 literal, an +# X-Originating-IP, a DKIM-Signature with a known signing domain, an +# Authentication-Results, an X-Mailer, RFC 2047 encoded From and Subject +# (base64/UTF-8 and quoted-printable/ISO-8859-1 in the same field), a +# References/In-Reply-To pair, and a multipart body with two attachments +# -- one with a plain quoted filename and one using the RFC 2231 +# continuation form. +# +# The third message's body contains a `>From ' line: that is the mboxo +# escaping of a body line that would otherwise look like a message +# separator, and it is there so that the message count stays at three. +# +# All names, addresses and hosts are from the RFC 2606 / RFC 5737 / +# RFC 3849 reserved ranges. The file is invented and is dedicated to the +# public domain (CC0). +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/mbox_test.mbox" + +# The two encoded words in the headers below decode to: +# =?UTF-8?B?QW5uYSBNw7xsbGVy?= -> "Anna Muller" with u-umlaut +# =?UTF-8?B?w5xiZXJ3ZWlzdW5nIE3DpHJ6?= -> "Uberweisung Marz" with umlauts +# =?ISO-8859-1?Q?_2024_Beleg?= -> " 2024 Beleg" +# `Date: Tue, 12 Mar 2024 09:41:07 +0100' is Unix time 1710232867, +# i.e. 2024-03-12T08:41:07Z. + +cat > "$out" <<'EOF' +From anna.mueller@example.com Tue Mar 12 09:41:09 2024 +Received: from mx1.example.net (mx1.example.net [198.51.100.24]) + by mail.example.org (Postfix) with ESMTPS id 4B7f2Z1qJz3xYm + for <bob@example.org>; Tue, 12 Mar 2024 09:41:09 +0100 (CET) +Received: from smtp6.example.net (smtp6.example.net [IPv6:2001:db8:1234::a5]) + by mx1.example.net (Postfix) with ESMTPS id 77aa88bb99cc + for <bob@example.org>; Tue, 12 Mar 2024 09:41:08 +0100 (CET) +Received: from wsbeta.corp.example.com (unknown [203.0.113.77]) + by mx1.example.net (Postfix) with ESMTPSA id 9C1a4F0d7b + for <bob@example.org>; Tue, 12 Mar 2024 09:41:07 +0100 (CET) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com; + s=sel2024; t=1710232867; h=from:to:subject:date; + bh=2jmj7l5rSw0yVb-vlWAYkK-YBwk=; + b=Zm9vYmFyYmF6cXV4Y29ycmdlZ3JhdWx0 +Authentication-Results: mail.example.org; + dkim=pass header.d=example.com; + spf=pass smtp.mailfrom=example.com +X-Originating-IP: [192.0.2.153] +Message-ID: <20240312084107.9C1a4F0d7b@wsbeta.corp.example.com> +In-Reply-To: <20240311150122.A1B2C3@mail.example.net> +References: <20240310090000.000001@mail.example.net> + <20240311150122.A1B2C3@mail.example.net> +Date: Tue, 12 Mar 2024 09:41:07 +0100 +From: =?UTF-8?B?QW5uYSBNw7xsbGVy?= <anna.mueller@example.com> +To: Bob Jones <bob@example.org>, carol@example.org +Cc: dave@example.org +Subject: =?UTF-8?B?w5xiZXJ3ZWlzdW5nIE3DpHJ6?= =?ISO-8859-1?Q?_2024_Beleg?= +Organization: Example GmbH +X-Mailer: Microsoft Outlook 16.0 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=_boundary_42" + +--=_boundary_42 +Content-Type: text/plain; charset="ISO-8859-15" +Content-Transfer-Encoding: quoted-printable + +Hallo Bob, + +anbei der Beleg und die Reisekosten. + +Gruss, Anna + +--=_boundary_42 +Content-Type: application/pdf; name="Kontoauszug_2024-03.pdf" +Content-Disposition: attachment; filename="Kontoauszug_2024-03.pdf" +Content-Transfer-Encoding: base64 + +JVBERi0xLjQKJcfsj6IKMSAwIG9iago8PC9UeXBlL0NhdGFsb2c+PgplbmRvYmoK +dHJhaWxlcgo8PC9Sb290IDEgMCBSPj4KJSVFT0YK + +--=_boundary_42 +Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +Content-Disposition: attachment; + filename*0="Reise"; + filename*1="kosten.xlsx" +Content-Transfer-Encoding: base64 + +UEsDBBQAAAAIAA== + +--=_boundary_42-- + +From bob@example.org Tue Mar 12 10:02:11 2024 +Received: from mail.example.org (localhost [127.0.0.1]) + by mail.example.org (Postfix) with ESMTP id 0A1B2C3D4E5F + for <anna.mueller@example.com>; Tue, 12 Mar 2024 10:02:11 +0100 (CET) +Message-ID: <20240312090211.0A1B2C3D4E5F@mail.example.org> +In-Reply-To: <20240312084107.9C1a4F0d7b@wsbeta.corp.example.com> +Date: Tue, 12 Mar 2024 10:02:11 +0100 +From: Bob Jones <bob@example.org> +To: =?UTF-8?B?QW5uYSBNw7xsbGVy?= <anna.mueller@example.com> +Subject: Re: Beleg +User-Agent: Mozilla Thunderbird 115.8.0 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8; format=flowed + +Danke, ist angekommen. + +From daemon@example.org Tue Mar 12 10:15:00 2024 +Received: from relay.example.org (relay.example.org [198.51.100.9]) + by mail.example.org (Postfix) with ESMTP id 112233445566 + for <anna.mueller@example.com>; Tue, 12 Mar 2024 10:15:00 +0100 (CET) +Message-ID: <20240312091500.112233445566@relay.example.org> +Date: Tue, 12 Mar 2024 10:15:00 +0100 +From: Mail Delivery System <daemon@example.org> +To: anna.mueller@example.com +Subject: Delivery Status Notification (Delay) +Content-Type: text/plain; charset=US-ASCII + +>From the transcript of the session: + + 451 4.4.1 reply: read timeout, delivery deferred + +EOF + +echo "wrote $out ($(wc -c < "$out") bytes)" diff --git a/contrib/gen_pecoff_testdata.sh b/contrib/gen_pecoff_testdata.sh @@ -0,0 +1,286 @@ +#!/bin/sh +# Regenerate the PE/COFF test images used by src/plugins/test_pecoff.c. +# +# Copyright (C) 2026 Christian Grothoff +# The generated files are dedicated to the public domain (CC0 1.0). +# +# Two files are produced: +# +# pecoff_test.exe -- a real x86-64 image built with mingw-w64. It is +# linked without the C runtime so that its import table has a +# single entry, which makes the import hash something the test can +# spell out. Needs the Debian package `gcc-mingw-w64-x86-64'; if +# the compiler is missing the existing file is left alone. +# +# pecoff_test.dll -- assembled here with python3 from the format +# specification. mingw cannot produce a Rich header (only +# Microsoft's linker writes one) or a CodeView debug record with a +# fixed GUID, and those are the two artifacts worth testing most. +# +# Both files are byte-for-byte reproducible: every timestamp is a +# constant in this script and the entropy filler is derived from SHA-256 +# of a fixed counter. +set -e + +srcdir=$(dirname "$0") +outdir=${1:-"$srcdir/../src/plugins/testdata"} +work=$(mktemp -d) +trap 'rm -rf "$work"' 0 + +# The build timestamp stamped into pecoff_test.exe: 2023-11-14T22:13:20Z. +EXE_TIMESTAMP=1700000000 + +CC=x86_64-w64-mingw32-gcc +RC=x86_64-w64-mingw32-windres + +if ! command -v "$CC" >/dev/null 2>&1; then + echo "$CC not found (apt-get install gcc-mingw-w64-x86-64);" \ + "keeping the existing pecoff_test.exe" >&2 +else + cat > "$work/pecoff_test.c" <<'EOF' +/* Placed in the public domain (CC0 1.0). */ +__declspec(dllimport) void __stdcall ExitProcess (unsigned int code); + +void mainCRTStartup (void); + +void +mainCRTStartup (void) +{ + ExitProcess (0); +} +EOF + cat > "$work/pecoff_test.rc" <<'EOF' +1 VERSIONINFO +FILEVERSION 1,2,3,4 +PRODUCTVERSION 5,6,7,8 +FILEFLAGSMASK 0x3fL +FILEFLAGS 0x0L +FILEOS 0x40004L +FILETYPE 0x1L +FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "CompanyName", "GNU libextractor" + VALUE "FileDescription", "libextractor PE test binary" + VALUE "FileVersion", "1.2.3.4" + VALUE "InternalName", "pecoff_test" + VALUE "LegalCopyright", "CC0 1.0 Universal" + VALUE "OriginalFilename", "pecoff_test.exe" + VALUE "ProductName", "GNU libextractor test suite" + VALUE "ProductVersion", "5.6.7.8" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END +EOF + "$RC" "$work/pecoff_test.rc" -O coff -o "$work/pecoff_test_res.o" + "$CC" -Os -nostdlib -nostartfiles \ + -Wl,-e,mainCRTStartup -Wl,--no-insert-timestamp \ + -Wl,--dynamicbase -Wl,--nxcompat -Wl,--high-entropy-va \ + -s -o "$work/pecoff_test.exe" \ + "$work/pecoff_test.c" "$work/pecoff_test_res.o" -lkernel32 + # The linker was told to leave TimeDateStamp at zero so that the build + # is reproducible; put a fixed date there instead, since a plausible + # build date is one of the things the plugin reports. + python3 - "$work/pecoff_test.exe" "$EXE_TIMESTAMP" <<'EOF' +import struct +import sys + +path, stamp = sys.argv[1], int(sys.argv[2]) +with open(path, "rb") as f: + image = bytearray(f.read()) +lfanew = struct.unpack_from("<I", image, 0x3C)[0] +assert image[lfanew:lfanew + 4] == b"PE\x00\x00" +struct.pack_into("<I", image, lfanew + 8, stamp) +with open(path, "wb") as f: + f.write(image) +EOF + cp "$work/pecoff_test.exe" "$outdir/pecoff_test.exe" + chmod 644 "$outdir/pecoff_test.exe" +fi + +python3 - "$outdir/pecoff_test.dll" <<'EOF' +"""Assemble a minimal PE32+ DLL carrying the artifacts MSVC leaves +behind: a Rich header, a CodeView record naming the PDB, an export +directory, a certificate table and a high-entropy section.""" +import hashlib +import struct +import sys + +OUT = sys.argv[1] + +TIMESTAMP = 1600000000 # 2020-09-13T12:26:40Z +RICH_KEY = 0x1A2B3C4D +# (product id, build number, use count), as the linker records them +RICH_ENTRIES = [(0x0104, 30729, 34), + (0x00FF, 30729, 12), + (0x0105, 30729, 1), + (0x0001, 0, 5)] +PDB_PATH = (b"C:\\Users\\builder\\source\\repos\\pecoff_test" + b"\\x64\\Release\\pecoff_test.pdb\x00") +CV_GUID = bytes([0x2A, 0x1E, 0x7B, 0x4C, # Data1 = 4C7B1E2A + 0x3D, 0x9F, # Data2 = 9F3D + 0x5C, 0x4B, # Data3 = 4B5C + 0x8E, 0x6F, 0x0A, 0x1B, 0x2C, 0x3D, 0x4E, 0x5F]) +CV_AGE = 7 +SIGNER_CN = b"GNU libextractor test signer" +EXPORT_NAMES = [b"ForensicEntryA", b"ForensicEntryB", b"ForensicEntryC"] + +SEC_ALIGN = 0x1000 +FILE_ALIGN = 0x200 +E_LFANEW = 0x100 + +# ------------------------------------------------------------ DOS stub +dos = bytearray(64) +dos[0:16] = bytes([0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00]) +dos[16:32] = bytes([0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +dos[0x3C:0x40] = struct.pack("<I", E_LFANEW) + +stub = bytes([0x0E, 0x1F, 0xBA, 0x0E, 0x00, 0xB4, 0x09, 0xCD, + 0x21, 0xB8, 0x01, 0x4C, 0xCD, 0x21]) +stub += b"This program cannot be run in DOS mode.\r\r\n$" +stub = stub.ljust(64, b"\x00") + +# --------------------------------------------------------- Rich header +words = [0x536E6144, 0, 0, 0] # 'DanS' plus three pad dwords +for prod, build, uses in RICH_ENTRIES: + words.append((prod << 16) | build) + words.append(uses) +rich = b"".join(struct.pack("<I", w ^ RICH_KEY) for w in words) +rich += b"Rich" + struct.pack("<I", RICH_KEY) +rich = rich.ljust(E_LFANEW - 0x80, b"\x00") +assert len(dos) + len(stub) + len(rich) == E_LFANEW + +# ------------------------------------------------------------ sections +TEXT_RVA, TEXT_OFF, TEXT_SIZE = 0x1000, 0x400, 0x200 +RDATA_RVA, RDATA_OFF, RDATA_SIZE = 0x2000, 0x600, 0x200 +PACK_RVA, PACK_OFF, PACK_SIZE = 0x3000, 0x800, 0x400 +CERT_OFF, CERT_SIZE = 0xC00, 0x200 + +text = b"\x48\x31\xC0\xC3".ljust(TEXT_SIZE, b"\x00") + +# .pack stands in for a packed or encrypted section: bytes with no +# structure, but derived from a counter so the file never changes. +pack = b"".join(hashlib.sha256(b"libextractor-pecoff-%d" % i).digest() + for i in range(PACK_SIZE // 32)) +assert len(pack) == PACK_SIZE + +rd = bytearray(RDATA_SIZE) +EXP_DIR, FUNCS, NAMES, ORDS = 0x000, 0x028, 0x034, 0x040 +DLLNAME, NAME0, DBG_DIR, RSDS = 0x048, 0x058, 0x088, 0x0A8 + +struct.pack_into("<IIHHIIIIIII", rd, EXP_DIR, + 0, # Characteristics + TIMESTAMP, # TimeDateStamp + 0, 0, # Major/MinorVersion + RDATA_RVA + DLLNAME, # Name + 1, # Base + len(EXPORT_NAMES), # NumberOfFunctions + len(EXPORT_NAMES), # NumberOfNames + RDATA_RVA + FUNCS, + RDATA_RVA + NAMES, + RDATA_RVA + ORDS) +for i in range(len(EXPORT_NAMES)): + struct.pack_into("<I", rd, FUNCS + 4 * i, TEXT_RVA + i) + struct.pack_into("<H", rd, ORDS + 2 * i, i) +rd[DLLNAME:DLLNAME + 16] = b"pecoff_test.dll\x00" +at = NAME0 +for i, nm in enumerate(EXPORT_NAMES): + struct.pack_into("<I", rd, NAMES + 4 * i, RDATA_RVA + at) + rd[at:at + len(nm) + 1] = nm + b"\x00" + at += len(nm) + 1 +assert at <= DBG_DIR + +cv = b"RSDS" + CV_GUID + struct.pack("<I", CV_AGE) + PDB_PATH +assert RSDS + len(cv) <= RDATA_SIZE +rd[RSDS:RSDS + len(cv)] = cv + +struct.pack_into("<IIHHIIII", rd, DBG_DIR, + 0, # Characteristics + TIMESTAMP, # TimeDateStamp + 0, 0, # Major/MinorVersion + 2, # IMAGE_DEBUG_TYPE_CODEVIEW + len(cv), # SizeOfData + RDATA_RVA + RSDS, # AddressOfRawData + RDATA_OFF + RSDS) # PointerToRawData + +# --------------------------------------------------------- certificate +# This is not a signature and verifies as nothing: it is the smallest +# blob that gives the commonName scan something to find. 06 03 55 04 03 +# is the DER encoding of the object identifier 2.5.4.3 (commonName). +cert_body = (b"\x30\x82\x01\x00" + b"\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x43\x48" + b"\x31\x35\x30\x33\x06\x03\x55\x04\x03" + + bytes([0x13, len(SIGNER_CN)]) + SIGNER_CN) +cert = (struct.pack("<IHH", CERT_SIZE, 0x0200, 0x0002) + + cert_body).ljust(CERT_SIZE, b"\x00") + +# ---------------------------------------------------------- PE headers +sections = [ + (b".text", 0x10, TEXT_RVA, TEXT_SIZE, TEXT_OFF, 0x60000020), + (b".rdata", 0x200, RDATA_RVA, RDATA_SIZE, RDATA_OFF, 0x40000040), + (b".pack", PACK_SIZE, PACK_RVA, PACK_SIZE, PACK_OFF, 0xC0000040), +] + +OPT_SIZE = 112 + 16 * 8 +coff = struct.pack("<HHIIIHH", + 0x8664, # Machine: AMD64 + len(sections), + TIMESTAMP, + 0, 0, # symbol table + OPT_SIZE, + 0x2022) # EXECUTABLE_IMAGE|LARGE_ADDRESS|DLL + +opt = struct.pack("<HBBIIIII", + 0x020B, # PE32+ + 14, 38, # linker version + TEXT_SIZE, RDATA_SIZE + PACK_SIZE, 0, + TEXT_RVA, # AddressOfEntryPoint + TEXT_RVA) # BaseOfCode +opt += struct.pack("<Q", 0x180000000) # ImageBase +opt += struct.pack("<IIHHHHHHIIIIHH", + SEC_ALIGN, FILE_ALIGN, + 6, 0, # OS version + 0, 0, # image version + 6, 0, # subsystem version + 0, # Win32VersionValue + 0x4000, # SizeOfImage + 0x400, # SizeOfHeaders + 0, # CheckSum + 2, # Subsystem: Windows GUI + 0xC160) # DllCharacteristics +opt += struct.pack("<QQQQ", 0x100000, 0x1000, 0x100000, 0x1000) +opt += struct.pack("<II", 0, 16) # LoaderFlags, NumberOfRvaAndSizes + +dirs = [(0, 0)] * 16 +dirs[0] = (RDATA_RVA + EXP_DIR, 0x88) # export +dirs[4] = (CERT_OFF, CERT_SIZE) # certificate: a file offset! +dirs[6] = (RDATA_RVA + DBG_DIR, 28) # debug +for va, sz in dirs: + opt += struct.pack("<II", va, sz) +assert len(opt) == OPT_SIZE + +sectab = b"" +for name, vsize, rva, rsize, roff, chars in sections: + sectab += struct.pack("<8sIIIIIIHHI", + name, vsize, rva, rsize, roff, + 0, 0, 0, 0, chars) + +headers = bytes(dos) + stub + rich + b"PE\x00\x00" + coff + opt + sectab +assert len(headers) <= 0x400, hex(len(headers)) +image = headers.ljust(0x400, b"\x00") + text + bytes(rd) + pack + cert +assert len(image) == CERT_OFF + CERT_SIZE, hex(len(image)) + +with open(OUT, "wb") as f: + f.write(image) +EOF + +ls -l "$outdir/pecoff_test.exe" "$outdir/pecoff_test.dll" diff --git a/contrib/gen_plist_testdata.sh b/contrib/gen_plist_testdata.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Regenerate the property list test data for src/plugins/test_plist.c. +# +# Needs nothing beyond python3 (Debian package "python3"); plistlib is +# part of the standard library. The output is deterministic: every +# value, including the date, is hard-coded here and asserted on in +# test_plist.c. +# +# The dictionary mixes the keys of an application bundle Info.plist +# (CFBundle*, DT*) with those of an iOS backup Info.plist (DeviceName, +# ProductType, SerialNumber, LastBackupDate) so that one small file +# exercises both halves of the key table. NSHumanReadableCopyright is +# deliberately non-ASCII: in the binary form plistlib then has to store +# it as UTF-16BE, which is the string decoder we most want covered. +# +# This file is part of libextractor; the generated data is CC0. +set -e + +out="$(dirname "$0")/../src/plugins/testdata" + +python3 - "$out" <<'EOF' +import datetime +import os +import plistlib +import sys + +out = sys.argv[1] + +d = { + 'BuildMachineOSBuild': '23A344', + 'CFBundleExecutable': 'ExtractorTest', + 'CFBundleIdentifier': 'org.gnu.libextractor.testbundle', + 'CFBundleName': 'ExtractorTest', + 'CFBundlePackageType': 'APPL', + 'CFBundleShortVersionString': '1.19.0', + 'CFBundleVersion': '4211', + 'DTPlatformName': 'iphoneos', + 'DTXcode': '1520', + 'DeviceName': 'Test iPhone', + 'ExtractorTestInteger': 42, + 'LastBackupDate': datetime.datetime(2024, 3, 14, 15, 9, 26), + 'MinimumOSVersion': '14.0', + 'NSHumanReadableCopyright': 'Copyright © 2026 Grüße', + 'ProductType': 'iPhone14,5', + 'SerialNumber': 'F2LXK0GTQ1GH', +} + +with open(os.path.join(out, 'plist_binary.plist'), 'wb') as f: + plistlib.dump(d, f, fmt=plistlib.FMT_BINARY) +with open(os.path.join(out, 'plist_xml.plist'), 'wb') as f: + plistlib.dump(d, f, fmt=plistlib.FMT_XML) +EOF + +ls -l "$out/plist_binary.plist" "$out/plist_xml.plist" diff --git a/contrib/gen_sqlite_testdata.sh b/contrib/gen_sqlite_testdata.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/sqlite_test.db. +# +# Requires: sqlite3 (Debian package `sqlite3'), python3. +# +# The result is byte-for-byte reproducible: SQLite does not put a +# timestamp in the file, and the two header words that would otherwise +# vary (the SQLite library version at offset 96 and the version-valid-for +# counter at 92) are overwritten afterwards so that the test can assert +# on an exact version string regardless of the sqlite3 that ran here. +# +# The database is deliberately tiny: 1 KiB pages, a handful of rows, and +# a DELETE that is *not* followed by VACUUM so that freelist pages -- the +# place deleted records survive -- are present in the header. +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/sqlite_test.db" + +rm -f "$out" "$out-journal" "$out-wal" "$out-shm" + +sqlite3 "$out" <<'EOF' +PRAGMA page_size = 1024; +PRAGMA encoding = 'UTF-8'; +PRAGMA auto_vacuum = 2; -- incremental: keeps the freelist around +CREATE TABLE messages (id INTEGER PRIMARY KEY, handle TEXT, body TEXT); +CREATE TABLE attachments (id INTEGER PRIMARY KEY, message_id INTEGER, path TEXT); +CREATE INDEX messages_handle ON messages (handle); +INSERT INTO messages (id, handle, body) VALUES + (1, 'alice@example.org', 'the first message'), + (2, 'bob@example.org', 'the second message'), + (3, 'carol@example.org', 'the third message'), + (4, 'dave@example.org', 'the fourth message'), + (5, 'erin@example.org', 'the fifth message'); +INSERT INTO attachments (id, message_id, path) VALUES + (1, 1, '/var/mobile/Media/DCIM/100APPLE/IMG_0001.JPG'), + (2, 3, '/var/mobile/Media/DCIM/100APPLE/IMG_0002.JPG'); +DELETE FROM messages WHERE id IN (2, 4); +DROP INDEX messages_handle; +PRAGMA user_version = 4242; +PRAGMA application_id = 1279613012; +PRAGMA journal_mode = WAL; +EOF + +# 1279613012 == 0x4c455854 == "LEXT"; an arbitrary but printable tag, so +# that the plugin's four-character rendering of the application id is +# exercised as well as the numeric one. WAL mode is set last and leaves +# the write/read format versions at 2 in the closed database, which is +# what the plugin reports as the journal mode. + +python3 - "$out" <<'EOF' +import struct +import sys + +path = sys.argv[1] +with open(path, "r+b") as f: + data = bytearray(f.read()) + if len(data) < 100 or bytes(data[0:16]) != b"SQLite format 3\x00": + raise SystemExit("not a SQLite database: %s" % path) + # Pin the writer version to 3.45.1 (3 * 1000000 + 45 * 1000 + 1) and + # make version-valid-for agree with the change counter, so the file + # does not depend on the sqlite3 that generated it. + struct.pack_into(">I", data, 96, 3045001) + struct.pack_into(">I", data, 92, struct.unpack_from(">I", data, 24)[0]) + f.seek(0) + f.write(data) + f.truncate() +print("wrote %s (%d bytes)" % (path, len(data))) +EOF + +rm -f "$out-journal" "$out-wal" "$out-shm" diff --git a/contrib/gen_tar_testdata.sh b/contrib/gen_tar_testdata.sh @@ -0,0 +1,88 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Regenerate src/plugins/testdata/tar_test.tar. +# +# Requires: python3 (standard library only; `tarfile' is part of it). +# +# Every field the tar plugin reports is set explicitly here, so the test +# can assert on exact values: owner name and group name, numeric uid and +# gid, the permission bits, the modification times, a symlink target and +# a member whose path is long enough to be split across the ustar prefix +# field. Nothing is taken from the machine this runs on, so the output +# is byte-for-byte reproducible. +set -e + +srcdir=$(dirname "$0")/.. +out="$srcdir/src/plugins/testdata/tar_test.tar" + +python3 - "$out" <<'EOF' +import io +import sys +import tarfile + +out = sys.argv[1] + +# 2024-03-15T12:34:56Z; the newest mtime in the archive, which is what +# the plugin reports as the modification date. +NEWEST = 1710506096 +OLDER = 1700000000 + +BODY = b"the quick brown fox jumps over the lazy dog\n" + + +def member(tf, name, kind, mode, size=0, link=""): + ti = tarfile.TarInfo(name) + ti.type = kind + ti.mode = mode + ti.uid = 1000 + ti.gid = 100 + ti.uname = "forensic" + ti.gname = "analysts" + ti.mtime = OLDER + ti.linkname = link + ti.size = size + return ti + + +with tarfile.open(out, "w", format=tarfile.USTAR_FORMAT) as tf: + ti = member(tf, "evidence/", tarfile.DIRTYPE, 0o755) + tf.addfile(ti) + + ti = member(tf, "evidence/notes.txt", tarfile.REGTYPE, 0o644, len(BODY)) + ti.mtime = NEWEST + tf.addfile(ti, io.BytesIO(BODY)) + + ti = member(tf, "evidence/script.sh", tarfile.REGTYPE, 0o755, len(BODY)) + tf.addfile(ti, io.BytesIO(BODY)) + + ti = member(tf, "evidence/latest.txt", tarfile.SYMTYPE, 0o777, + link="notes.txt") + tf.addfile(ti) + + ti = member(tf, "evidence/hardlink.txt", tarfile.LNKTYPE, 0o644, + link="evidence/notes.txt") + tf.addfile(ti) + + # A member owned by root, so that the plugin has two distinct owners + # to report rather than one. + ti = member(tf, "evidence/root-owned.txt", tarfile.REGTYPE, 0o600, + len(BODY)) + ti.uid = 0 + ti.gid = 0 + ti.uname = "root" + ti.gname = "root" + tf.addfile(ti, io.BytesIO(BODY)) + + # 130 characters, so ustar has to split it into the 155-byte prefix + # field and the 100-byte name field. + long_path = ("evidence/" + "/".join("dir%02d" % i for i in range(15)) + + "/deep.txt") + assert len(long_path) > 100, len(long_path) + ti = member(tf, long_path, tarfile.REGTYPE, 0o640, len(BODY)) + tf.addfile(ti, io.BytesIO(BODY)) + +with open(out, "rb") as f: + print("wrote %s (%d bytes)" % (out, len(f.read()))) +EOF diff --git a/contrib/gen_webp_testdata.sh b/contrib/gen_webp_testdata.sh @@ -0,0 +1,155 @@ +#!/bin/sh +# This file is part of libextractor. +# Copyright (C) 2026 Vidyut Samanta and Christian Grothoff +# +# Generate the test data for the "webp" plugin: +# +# src/plugins/testdata/webp_test.webp +# +# An extended (VP8X) animated WebP with an alpha channel, an ICC +# profile, three animation frames and EXIF and XMP chunks -- that is, +# every optional feature the container defines, so that one file +# exercises the whole chunk walk. +# +# The three frames carry a genuine lossless VP8L bitstream, encoded here +# by ffmpeg from a solid colour generated by lavfi (so nothing is read +# from disk and the result does not depend on a sample image). The +# container around it is assembled by hand because no encoder emits all +# of these chunks at once. The ICC, EXIF and XMP payloads are +# placeholders: the plugin only reports that they are present, and +# parsing them is the exiv2 plugin's job. +# +# Needs: ffmpeg built with libwebp (Debian package "ffmpeg") and +# python3. Deterministic: no timestamps, no random data. +# +# Written to CC0; there is no third party content in this file. +# +# Usage: contrib/gen_webp_testdata.sh [output-directory] + +set -e + +OUT="${1:-src/plugins/testdata}" +mkdir -p "$OUT" + +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +# A 40x24 solid colour, encoded losslessly. libwebp emits a bare +# "RIFF....WEBPVP8L" file, whose VP8L chunk we lift out below. +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "color=c=0x1EB45A:s=40x24" \ + -frames:v 1 -c:v libwebp -lossless 1 -pix_fmt bgra \ + "$TMP/frame.webp" + +python3 - "$OUT" "$TMP/frame.webp" <<'EOF' +import struct +import sys + +out = sys.argv[1] +frame_path = sys.argv[2] + +WIDTH = 40 +HEIGHT = 24 + +# VP8X feature flags, in the single flag byte at the start of the +# "VP8X" payload: Rsv(2) ICC(1) Alpha(1) EXIF(1) XMP(1) Anim(1) Rsv(1) +ICC_FLAG = 0x20 +ALPHA_FLAG = 0x10 +EXIF_FLAG = 0x08 +XMP_FLAG = 0x04 +ANIM_FLAG = 0x02 + + +def chunk(fourcc, payload): + """A RIFF chunk: 4CC, 32-bit little-endian size, payload, pad byte.""" + assert len(fourcc) == 4 + data = fourcc.encode('ascii') + struct.pack('<I', len(payload)) + payload + if len(payload) & 1: + data += b'\0' + return data + + +def le24(value): + return struct.pack('<I', value)[:3] + + +def extract_vp8l(path): + """Lift the VP8L chunk (header included) out of a simple WebP file.""" + with open(path, 'rb') as f: + data = f.read() + assert data[0:4] == b'RIFF' and data[8:12] == b'WEBP', 'not a WebP file' + pos = 12 + while pos + 8 <= len(data): + fourcc = data[pos:pos + 4] + size = struct.unpack('<I', data[pos + 4:pos + 8])[0] + if fourcc == b'VP8L': + end = pos + 8 + size + (size & 1) + return data[pos:end] + pos += 8 + size + (size & 1) + raise SystemExit('ffmpeg did not produce a VP8L chunk') + + +vp8l = extract_vp8l(frame_path) + +# A placeholder ICC profile: a 128 byte header saying "RGB display +# profile, no tags". Small, self-describing, and never parsed by us. +icc = (struct.pack('>I', 132) + + b'lcms' + struct.pack('>I', 0x04300000) + + b'mntr' + b'RGB ' + b'XYZ ' + + b'\0' * 12 # creation date, zeroed + + b'acsp' + b'APPL' + b'\0' * 4 + + b'\0' * 12 # device manufacturer/model + + b'\0' * 8 # device attributes + + b'\0' * 4 # rendering intent: perceptual + + struct.pack('>III', 0xF6D6, 0x10000, 0xD32D) # D50 white point + + b'none' + b'\0' * 44 + + struct.pack('>I', 0)) # tag count: zero + +# "Exif\0\0" is not part of the WebP EXIF chunk; the payload is the raw +# TIFF structure. A little-endian TIFF header with an empty IFD. +exif = b'II' + struct.pack('<HI', 42, 8) + struct.pack('<H', 0) \ + + struct.pack('<I', 0) + +xmp = (b'<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>' + b'<x:xmpmeta xmlns:x="adobe:ns:meta/">' + b'<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">' + b'<rdf:Description rdf:about="" xmlns:dc="http://purl.org/dc/elements/1.1/">' + b'<dc:title><rdf:Alt><rdf:li xml:lang="x-default">' + b'libextractor webp test</rdf:li></rdf:Alt></dc:title>' + b'</rdf:Description></rdf:RDF></x:xmpmeta>' + b'<?xpacket end="w"?>') + +vp8x = chunk('VP8X', + bytes([ICC_FLAG | ALPHA_FLAG | EXIF_FLAG | XMP_FLAG | ANIM_FLAG]) + + b'\0\0\0' + + le24(WIDTH - 1) + + le24(HEIGHT - 1)) + +# Background colour is BGRA; a loop count of zero means "forever". +anim = chunk('ANIM', struct.pack('<IH', 0xFF204080, 0)) + +# Frame durations of 100, 150 and 250 ms sum to a round 500 ms. +frames = b'' +for duration in (100, 150, 250): + frames += chunk('ANMF', + le24(0) + le24(0) # frame_x, frame_y + + le24(WIDTH - 1) + le24(HEIGHT - 1) + + le24(duration) + + bytes([0x02]) # dispose, no blend + + vp8l) + +body = (b'WEBP' + + vp8x + + chunk('ICCP', icc) + + anim + + frames + + chunk('EXIF', exif) + + chunk('XMP ', xmp)) + +data = b'RIFF' + struct.pack('<I', len(body)) + body + +path = out + '/webp_test.webp' +with open(path, 'wb') as f: + f.write(data) +print('%s: %d bytes, ICC %d bytes' % (path, len(data), len(icc))) +EOF diff --git a/contrib/oss-fuzz/build.sh b/contrib/oss-fuzz/build.sh @@ -316,6 +316,22 @@ odf:odf_extractor.c:-lz png:png_extractor.c:-lz qt:qt_extractor.c:-lz zip:zip_extractor.c:-lz +pecoff:pecoff_extractor.c forensics.c:-lm +lnk:lnk_extractor.c forensics.c:-lm +sqlite:sqlite_extractor.c forensics.c:-lm +tar:tar_extractor.c forensics.c:-lm +iso9660:iso9660_extractor.c forensics.c:-lm +diskimage:diskimage_extractor.c forensics.c:-lm +heif:heif_extractor.c forensics.c:-lm +webp:webp_extractor.c forensics.c:-lm +plist:plist_extractor.c forensics.c:-lm +id3:id3_extractor.c forensics.c:-lm +gpx:gpx_extractor.c forensics.c:-lm +kml:kml_extractor.c forensics.c:-lm +geotiff:geotiff_extractor.c forensics.c:-lm +mbox:mbox_extractor.c forensics.c:-lm +apk:apk_extractor.c forensics.c:-lm -lz +ebook:ebook_extractor.c forensics.c:-lm -lz " # Plugins that wrap a third-party parser. NOT part of the OSS-Fuzz build: diff --git a/contrib/oss-fuzz/make_seed_corpus.sh b/contrib/oss-fuzz/make_seed_corpus.sh @@ -94,6 +94,22 @@ testdata_glob () fuzz_unzip) echo "zip_* odf_* msoffice_excel.xlsx msoffice_word.docx \ msoffice_powerpoint.pptx" ;; fuzz_datasource) echo "deb_* zip_* png_* ogg_*" ;; + fuzz_pecoff) echo "pecoff_*" ;; + fuzz_lnk) echo "lnk_*" ;; + fuzz_sqlite) echo "sqlite_*" ;; + fuzz_tar) echo "tar_* archive_test.tar" ;; + fuzz_iso9660) echo "iso9660_*" ;; + fuzz_diskimage) echo "diskimage_*" ;; + fuzz_heif) echo "heif_*" ;; + fuzz_webp) echo "webp_*" ;; + fuzz_plist) echo "plist_*" ;; + fuzz_id3) echo "id3_*" ;; + fuzz_gpx) echo "gpx_*" ;; + fuzz_kml) echo "kml_*" ;; + fuzz_geotiff) echo "geotiff_* tiff_*" ;; + fuzz_mbox) echo "mbox_*" ;; + fuzz_apk) echo "apk_*" ;; + fuzz_ebook) echo "ebook_*" ;; fuzz_extract) echo "*" ;; *) echo "" ;; esac @@ -106,7 +122,10 @@ fuzz_applefile fuzz_dvi fuzz_elf fuzz_it fuzz_man fuzz_nsf fuzz_nsfe \ fuzz_ps fuzz_real fuzz_riff fuzz_rtf fuzz_s3m fuzz_sid fuzz_wav fuzz_xm \ fuzz_deb fuzz_msoffice fuzz_odf fuzz_png fuzz_qt fuzz_zip \ fuzz_gif fuzz_jpeg fuzz_tiff fuzz_flac fuzz_ogg fuzz_archive fuzz_mime \ -fuzz_ole2 fuzz_mpeg fuzz_html" +fuzz_ole2 fuzz_mpeg fuzz_html \ +fuzz_pecoff fuzz_lnk fuzz_sqlite fuzz_tar fuzz_iso9660 fuzz_diskimage \ +fuzz_heif fuzz_webp fuzz_plist fuzz_id3 fuzz_gpx fuzz_kml fuzz_geotiff \ +fuzz_mbox fuzz_apk fuzz_ebook" for t in ${TARGETS}; do d="${WORK}/${t}" diff --git a/contrib/oss-fuzz/run_campaign.sh b/contrib/oss-fuzz/run_campaign.sh @@ -24,6 +24,7 @@ HOURS="${HOURS:-}" PROFILE="${PROFILE:-nightly}" WEIGHTS_FILE="" FLOOR="${FLOOR:-300}" +SHARD="${SHARD:-1800}" DRYRUN=0 usage () @@ -38,6 +39,10 @@ usage: $0 -b BINDIR [-o OUTDIR] [-c CORES] [-t HOURS] [-p PROFILE] [-w FILE] -p PROFILE ci | nightly | deep (default nightly) -w FILE weight table "target weight" per line, overrides the built-in -F SECONDS per-target floor (default 300) + -S SECONDS shard size: a target allotted more than this is run as + several INDEPENDENT jobs of this length, each with its own + corpus and PRNG seed, instead of one long one. 0 disables. + Default 1800; see CAMPAIGN.md sections 9 and 10. -n print the schedule and exit; run nothing Profiles, in wall-clock hours on \$CORES cores: @@ -54,7 +59,7 @@ EOF exit "${1:-1}" } -while getopts "b:o:c:t:p:w:F:nh" o; do +while getopts "b:o:c:t:p:w:F:S:nh" o; do case "$o" in b) BIN="$OPTARG" ;; o) OUT="$OPTARG" ;; @@ -63,6 +68,7 @@ while getopts "b:o:c:t:p:w:F:nh" o; do p) PROFILE="$OPTARG" ;; w) WEIGHTS_FILE="$OPTARG" ;; F) FLOOR="$OPTARG" ;; + S) SHARD="$OPTARG" ;; n) DRYRUN=1 ;; h) usage 0 ;; *) usage ;; @@ -81,7 +87,15 @@ if [ -z "${HOURS}" ]; then esac fi -# Addressable headroom in library regions, measured 2026-07-29. +# Addressable headroom in library regions, measured 2026-08-07. +# +# Derived from the corpus the three campaigns of that day accumulated: +# the flat 1200 s-per-target measurement run, the tuned four-hour run, +# and the two-hour sharded run that started from the first two distilled +# together. Each target is credited with the regions still uncovered in +# the files it covers better than any other target. Union coverage: +# 17595 of 18889 regions, 93.1%. +# # "Addressable" excludes two things that no runtime can reach: # plugins/pack.c 89 regions of format codes that neither of its # two callers ever asks for (fixed format strings) @@ -91,37 +105,71 @@ fi # from giflib/libjpeg/libtiff/FLAC/libvorbis/libarchive/libmagic, which # those projects fuzz themselves, and reaching them from a file input is # disproportionately expensive. +# +# Treat differences under about 20% as noise. The staged plugins repeat +# to nothing like the +-2% the legacy targets manage; fuzz_ebook varied +# 16.6% between two runs that differed only in PRNG seed. These numbers +# are for apportioning a budget, not for ranking anything. +# +# NOTE ON THE FLOOR. The 300 s default was derived from the legacy +# targets, whose median time to 99% of final edge coverage is 93 s. The +# staged plugins take a median of 690 s, and eleven of the sixteen were +# still gaining edges at 1200 s. Use -F 900 while they are in the set. +# +# NOTE ON SHARDING. See -S and CAMPAIGN.md sections 9 and 10. Several +# independent short runs beat one long one when the corpus is small +# (section 9); once the corpus is mature the shards all converge to +# within half a percent and the seed stops mattering (section 10). Keep +# -S on regardless: it costs nothing and it is what stops a single long +# job from plateauing and burning hours on nothing. Just do not expect +# it to buy coverage from an already-good corpus. read -r -d '' BUILTIN_WEIGHTS <<'EOF' -fuzz_ole2 119 -fuzz_unzip 119 -fuzz_msoffice 73 +fuzz_diskimage 197 +fuzz_ebook 87 +fuzz_apk 78 +fuzz_pecoff 78 fuzz_datasource 74 -fuzz_qt 56 -fuzz_rtf 56 -fuzz_png 48 +fuzz_msoffice 74 +fuzz_lnk 67 +fuzz_unzip 56 +fuzz_rtf 39 +fuzz_geotiff 33 +fuzz_id3 30 +fuzz_kml 30 +fuzz_ole2 28 +fuzz_gpx 26 fuzz_ogg 26 +fuzz_mbox 23 +fuzz_tar 20 +fuzz_qt 19 +fuzz_heif 16 fuzz_flac 14 +fuzz_png 12 fuzz_mime 10 -fuzz_dvi 10 -fuzz_odf 9 +fuzz_plist 10 +fuzz_dvi 9 fuzz_deb 7 fuzz_convert 6 -fuzz_elf 6 -fuzz_man 4 +fuzz_webp 6 +fuzz_iso9660 5 +fuzz_elf 4 fuzz_nsfe 4 +fuzz_odf 4 +fuzz_man 3 fuzz_ps 3 -fuzz_real 3 +fuzz_real 2 fuzz_sid 2 fuzz_gif 1 +fuzz_it 1 fuzz_jpeg 1 -fuzz_tiff 1 fuzz_nsf 1 fuzz_riff 1 +fuzz_sqlite 1 fuzz_applefile 0 fuzz_archive 0 fuzz_ipc 0 -fuzz_it 0 fuzz_s3m 0 +fuzz_tiff 0 fuzz_wav 0 fuzz_xm 0 fuzz_zip 0 @@ -186,6 +234,38 @@ printf '%s' "${AVAIL}" | awk -v floor="${FLOOR}" -v budget="${BUDGET}" \ } for (i=1; i<=n; i++) printf "%s:%d\n", name[i], s[i] }' >> "${OUT}/jobs.txt" + +# Split anything longer than the shard size into that many INDEPENDENT +# runs -- separate corpus, separate PRNG seed -- rather than one long +# one. This is not a scheduling nicety, it is the single biggest effect +# the 2026-08-07 measurements found (src/fuzz/CAMPAIGN.md section 9): +# +# fuzz_ebook 2 x 1200 s -> 678 of 775 regions +# 1 x 14089 s -> 595 +# fuzz_apk 2 x 1200 s -> 824 of 927 regions +# 1 x 14407 s -> 792 +# +# Forty core-minutes beating four core-hours. A long libFuzzer run +# commits early to one corpus and cannot leave it; the 14089 s ebook job +# stopped finding edges after 2570 s and spent the remaining 82% of its +# slice on 195 million executions that added nothing. Independent runs +# start from different trajectories, and their corpora are unioned +# afterwards. +if [ "${SHARD}" -gt 0 ]; then + awk -F: -v shard="${SHARD}" ' + { + secs = $2 + k = int((secs + shard - 1) / shard) # ceil + if (k < 1) k = 1 + each = int(secs / k) + for (j = 1; j <= k; j++) printf "%s:%d:%d\n", $1, each, j + }' "${OUT}/jobs.txt" > "${OUT}/jobs.tmp" + mv "${OUT}/jobs.tmp" "${OUT}/jobs.txt" +else + awk -F: '{ printf "%s:%d:1\n", $1, $2 }' "${OUT}/jobs.txt" > "${OUT}/jobs.tmp" + mv "${OUT}/jobs.tmp" "${OUT}/jobs.txt" +fi + # Longest first: with a fixed number of slots the long jobs must start # early or the tail of the campaign runs on one core. sort -t: -k2 -rn -o "${OUT}/jobs.txt" "${OUT}/jobs.txt" @@ -195,9 +275,11 @@ echo " targets ${NTARGETS}" echo " cores ${CORES}" echo " profile ${PROFILE} (${HOURS} h wall => $(awk -v b=${BUDGET} 'BEGIN{printf "%.1f", b/3600}') core-hours)" echo " floor ${FLOOR}s" +echo " shard $([ "${SHARD}" -gt 0 ] && echo "${SHARD}s" || echo "off")" +echo " jobs $(wc -l < "${OUT}/jobs.txt")" echo " out ${OUT}" awk -F: -v c="${CORES}" ' - { printf " %-18s %6d s %2d:%02d\n", $1, $2, $2/3600, ($2%3600)/60; s+=$2 } + { printf " %-18s %6d s %2d:%02d shard %d\n", $1, $2, $2/3600, ($2%3600)/60, $3; s+=$2 } END { printf " %-18s %6.1f core-hours over %d cores => %.1f h wall (perfect packing)\n", "TOTAL", s/3600, c, s/3600/c } ' "${OUT}/jobs.txt" @@ -207,19 +289,21 @@ if [ "${DRYRUN}" = "1" ]; then exit 0 fi -for z in "${BIN}"/*_seed_corpus.zip; do - [ -f "${z}" ] || continue - t=$(basename "${z}" _seed_corpus.zip) - [ -x "${BIN}/${t}" ] || continue - mkdir -p "${OUT}/corpus/${t}" - unzip -qo "${z}" -d "${OUT}/corpus/${t}" 2>/dev/null -done +# Every shard gets its own copy of the seed corpus: the shards must stay +# independent for the whole point of sharding to hold. +while IFS=: read -r t secs sh; do + [ -n "${t}" ] || continue + z="${BIN}/${t}_seed_corpus.zip" + mkdir -p "${OUT}/corpus/${t}-${sh}" + [ -f "${z}" ] && unzip -qo "${z}" -d "${OUT}/corpus/${t}-${sh}" 2>/dev/null +done < "${OUT}/jobs.txt" run_one () { - local spec="$1" t secs dict maxlen leaks rc + local spec="$1" t secs sh dict maxlen leaks rc t="${spec%%:*}" - secs="${spec##*:}" + secs="${spec#*:}"; secs="${secs%%:*}" + sh="${spec##*:}" dict="" [ -f "${BIN}/${t}.dict" ] && dict="-dict=${BIN}/${t}.dict" maxlen=262144 @@ -233,18 +317,21 @@ run_one () leaks=1 [ "${t}" = "fuzz_ole2" ] && leaks=0 mkdir -p "${OUT}/artifacts/${t}" - echo "START ${t} (${secs}s)" + echo "START ${t}#${sh} (${secs}s)" + # -seed is what makes two shards of the same target explore + # differently; without it libFuzzer picks one from the clock and two + # shards started in the same second would be identical runs. ASAN_OPTIONS="allocator_may_return_null=0:detect_stack_use_after_return=1:detect_leaks=${leaks}" \ UBSAN_OPTIONS="print_stacktrace=1:report_error_type=1" \ - "${BIN}/${t}" "${OUT}/corpus/${t}" \ - -max_total_time="${secs}" -fork=1 \ + "${BIN}/${t}" "${OUT}/corpus/${t}-${sh}" \ + -max_total_time="${secs}" -fork=1 -seed="${sh}" \ -ignore_crashes=1 -ignore_ooms=1 -ignore_timeouts=1 \ -rss_limit_mb=2560 -timeout=25 -max_len="${maxlen}" \ -print_final_stats=1 \ -artifact_prefix="${OUT}/artifacts/${t}/" \ - ${dict} > "${OUT}/logs/${t}.log" 2>&1 + ${dict} > "${OUT}/logs/${t}-${sh}.log" 2>&1 rc=$? - echo "DONE ${t} rc=${rc} artifacts=$(ls -1 "${OUT}/artifacts/${t}" 2>/dev/null | wc -l)" + echo "DONE ${t}#${sh} rc=${rc} artifacts=$(ls -1 "${OUT}/artifacts/${t}" 2>/dev/null | wc -l)" } export -f run_one export BIN OUT @@ -254,6 +341,20 @@ xargs -a "${OUT}/jobs.txt" -P "${CORES}" -I{} \ bash -c 'run_one "$@"' _ {} 2>&1 | tee "${OUT}/campaign.log" date +%s > "${OUT}/finished_at" +# Union the shards back together. The whole reason for running them +# apart is that they explore differently, so the merged corpus is worth +# more than any single shard's -- this is what a coverage replay and the +# next campaign's seed corpus should use. +if [ "${SHARD}" -gt 0 ]; then + for d in "${OUT}"/corpus/*-[0-9]*; do + [ -d "${d}" ] || continue + b=$(basename "${d}"); t="${b%-*}" + mkdir -p "${OUT}/corpus-merged/${t}" + cp "${d}"/* "${OUT}/corpus-merged/${t}/" 2>/dev/null || true + done + echo "merged shard corpora into ${OUT}/corpus-merged" +fi + echo echo "=== campaign finished in $(( $(cat "${OUT}/finished_at") - $(cat "${OUT}/started_at") ))s ===" echo "artifacts: $(find "${OUT}/artifacts" -type f | wc -l)" diff --git a/doc/libextractor.texi b/doc/libextractor.texi @@ -788,30 +788,52 @@ The @verb{|EXTRACTOR_meta_data_print|} is a simple function which prints the met @itemize @bullet @item +APK (Android packages and Java archives, using zlib) +@item ARCHIVE (using libarchive) @item +DISKIMAGE (QCOW2, VMDK, VHD and VHDX virtual disk images) +@item DVI @item +EBOOK (EPUB and MOBI/AZW) +@item EXIV2 (using libexiv2, 0.23 or later preferred) @item FLAC (using libFLAC) @item +GEOTIFF (georeferencing tags in TIFF images) +@item +GPX (GPS tracks) +@item GIF (using libgif) @item GSTREAMER (using libgstreamer v1.0 or later) @item +HEIF (HEIC, HEIF and AVIF images) +@item +ID3 (MPEG audio with ID3v1/ID3v2 tags, Xing and LAME headers) +@item +ISO9660 (CD/DVD images) +@item HTML (using libtidy) @item IT @item JPEG (using libjpeg v8 or later) @item +KML (Keyhole Markup Language) +@item +LNK (Windows shell links) +@item MAN @item MIDI (using libsmf) @item MIME (using libmagic) @item +MBOX (Unix mailboxes and RFC 5322 messages) +@item MPEG (using libmpeg2) @item MSOFFICE (OOXML: @file{.docx}, @file{.xlsx}, @file{.pptx}; and @@ -819,6 +841,10 @@ Excel 2-4 BIFF streams) @item NSF @item +PECOFF (Windows executables, DLLs and drivers) +@item +PLIST (Apple property lists, binary and XML) +@item NSFE @item ODF @@ -840,6 +866,8 @@ custom document properties) @item S3M @item +SQLITE (SQLite 3 databases, write-ahead logs and journals) +@item SID @item PreviewGst (audio preview, using GStreamer) @@ -855,8 +883,12 @@ ThumbnailGtk (still image thumbnail, using gdk-pixbuf) @item TIFF (with libtiff, tested with v4) @item +TAR (tar archives) +@item WAV @item +WEBP +@item XM @item ZIP @@ -939,6 +971,58 @@ initialized it, an application that runs these plugins @code{EXTRACTOR_OPTION_IN_PROCESS} should not also use out-of-process plugins in the same process afterwards. +@section Forensic and bulk-scan plugins + +Sixteen of the plugins in that list were added for a different job from +the rest: identifying, and saying something useful about, every file on +a large volume in a single fast pass. That purpose shapes how they +behave, in ways worth knowing about before you use them. + +They answer questions about @emph{provenance} rather than about +content --- which machine, user, tool, device or account a file came +from. A Windows executable reports the compiler that built it, the +path its debug symbols were written to (which routinely discloses the +build machine's user name) and the exploit mitigations it opts into; a +shell link reports the volume serial number, the label and the NetBIOS +name of the machine the target lived on, and often that machine's MAC +address; a tar archive reports the user and group names of the system +it was packed on; a mailbox reports the relay chain a message +travelled and the addresses it passed through. + +To keep a bulk scan fast they read headers, not files. Each rejects a +file that is not its format within the first few bytes --- which is the +overwhelmingly common case --- and none of them decodes payload data. +The @code{sqlite} plugin in particular reads only the 100-byte header +and never opens the database: opening an untrusted database would be +both slow and unsafe. For the same reason none of these plugins links +a third-party parsing library, so all but @code{apk} and @code{ebook}, +which must inflate members of a zip container, are built +unconditionally. + +Because they characterise rather than enumerate, plugins that meet +repeating structures --- archive members, mail messages, requested +permissions, map placemarks --- report a bounded number of them +together with an @code{EXTRACTOR_METATYPE_ENTRY_COUNT} giving the true +total. Do not read the absence of a further item as the absence of +further items. + +Several of the meta types these plugins report were introduced with +them, and an application that switches on +@code{enum EXTRACTOR_MetaType} will not know them. Values are only +ever appended, so existing values keep their meaning; use +@code{EXTRACTOR_metatype_get_max()} rather than a hard-coded bound, and +@code{EXTRACTOR_metatype_to_string()} to render a type you do not +recognise. + +Two habits of these formats are worth remembering when reading their +output. Timestamps are reported as ISO 8601 in UTC, but several +formats store local time with no zone at all --- ISO 9660 and the +ISO base media format both do --- so a value may be displaced by the +writer's offset from UTC. And a field being present does not make it +true: every value here was written by whatever produced the file, an +executable's build timestamp is trivially forged, and a disagreement +between two fields is frequently more informative than either. + @node Writing new Plugins @chapter Writing new Plugins diff --git a/src/fuzz/CAMPAIGN.md b/src/fuzz/CAMPAIGN.md @@ -236,3 +236,342 @@ llvm-cov report /tmp/cov/fuzz_unzip -instr-profile=p.profdata The per-job stats lines libFuzzer prints in fork mode (`#N: cov: .. ft: .. corp: .. time: Ns job: M`) are a ready-made time series; that is where sections 2 and 3 come from. + + +## 7. The staged forensic plugins (measured 2026-08-07) + +Sixteen plugins — `apk diskimage ebook geotiff gpx heif id3 iso9660 kml +lnk mbox pecoff plist sqlite tar webp`, about 24 000 lines of +hand-written parser — were added to the tree with in-tree harnesses in +`src/fuzz/Makefile.am`, but they were **not** in +`contrib/oss-fuzz/build.sh` and not in the weight table, so no campaign +had ever run one. They are in both now. + +### The run + + 48 jobs x 1200 s / 16 cores = 3 waves = 3718 s wall, 16 core-hours + +The 1200 s slice is deliberately the same one section 1 used, so the +headroom numbers below are on the same basis as the older ones. Of the +48 jobs, 32 were the sixteen new targets run **twice** with different +PRNG seeds and independent corpora, and 16 were the highest-weight +legacy targets as a regression check. + +| | | +|---|---| +| crashes in the sixteen new targets | **0** | +| crashes elsewhere | 8 artifacts, all `fuzz_ole2`, all one defect (below) | +| recovering UBSan reports | **1 defect**, `geotiff_extractor.c` | +| corpus | 170 MB across 48 corpora | +| union region coverage | 15 607 / 17 380 = **89.8 %** | + +### Two findings + +**`geotiff_extractor.c:1669-1670`, float-cast-overflow.** `emit_image_tags` +guards `width` and `height` with `isfinite()` and `> 0.0` but with no +upper bound, then casts both to `unsigned long long`. A TIFF tag +carrying 7.0e22 makes that cast undefined. The two cases immediately +below it — `bits` and `samples` — already carry the `4096.0 >` bound +that this one is missing, so the fix is to match them. Reproduced in +both replicates, and it is exactly the shape section 1 warns about: a +*recovering* report, invisible to `-fork=1`, found only by the replay +pass. + +**The eight `fuzz_ole2` artifacts are one defect, and it is libgsf's.** +All eight have an identical stack: `iconv` reads past the end of the +buffer `le_input_read` handed it, called from +`gsf_doc_meta_data_read_from_msole` via `g_convert_with_iconv`. This is +the class section 4 already documents for this target. `issue13` no +longer reproduces, so nothing regressed. + +### These plugins do not converge like the old ones + +This is the result that matters for scheduling, and it contradicts +section 3 for the new set: + +| time to reach | legacy median (32 targets) | staged median (16 targets) | +|---|---|---| +| 90 % of final edge coverage | 12 s | 21 s | +| 95 % | 34 s | 115 s | +| 99 % | 93 s | **690 s** | +| still gaining at 1200 s | 4 of 32 | **11 of 16** | + +The 300 s floor was derived from targets that are done in 93 s. It cuts +these off mid-climb. **Run with `-F 900` while they are in the set.** + +### Replicate noise is much larger here, so do not over-read the ranking + +Section 2 established that edge coverage reproduces to about 2 % on the +legacy targets. It does not here. Same target, same slice, different +seed: + +| target | Δ edge coverage | target | Δ edge coverage | +|---|---|---|---| +| fuzz_sqlite | 0.0 % | fuzz_kml | −3.6 % | +| fuzz_id3 | 0.0 % | fuzz_diskimage | −3.7 % | +| fuzz_lnk | −0.1 % | fuzz_apk | −5.7 % | +| fuzz_heif | −0.1 % | fuzz_plist | −6.7 % | +| fuzz_iso9660 | +0.2 % | fuzz_gpx | −7.4 % | +| fuzz_webp | +0.3 % | fuzz_mbox | −9.1 % | +| fuzz_geotiff | +0.7 % | **fuzz_ebook** | **−16.6 %** | +| fuzz_tar | −0.8 % | fuzz_pecoff | +4.4 % | + +The noisy ones are the deep, slow parsers, and they are noisy for the +same reason they are slow: `fuzz_mbox` managed 487 k executions in its +slice where `fuzz_sqlite` managed 60.7 M, a factor of 125, so a single +seed samples far less of the input space. The weights below therefore +come from **llvm-cov region coverage of the merged a+b corpus**, not +from one run's edge count, and a difference of less than about 20 % in +this table should not be treated as a real ordering. + +### Where the remaining code is + +| source file | regions | covered | left | best target | +|---|---|---|---|---| +| `diskimage_extractor.c` | 1030 | 753 | **277** | fuzz_diskimage | +| `apk_extractor.c` | 927 | 824 | 103 | fuzz_apk | +| `pecoff_extractor.c` | 943 | 842 | 101 | fuzz_pecoff | +| `tar_extractor.c` | 356 | 257 | 99 | fuzz_tar | +| `ebook_extractor.c` | 775 | 678 | 97 | fuzz_ebook | +| `forensics.c` | 263 | 201 | 62 | fuzz_lnk | +| `kml_extractor.c` | 807 | 751 | 56 | fuzz_kml | +| `mbox_extractor.c` | 1382 | 1327 | 55 | fuzz_mbox | +| `gpx_extractor.c` | 881 | 846 | 35 | fuzz_gpx | +| `geotiff_extractor.c` | 901 | 868 | 33 | fuzz_geotiff | +| `id3_extractor.c` | 886 | 855 | 31 | fuzz_id3 | +| `heif_extractor.c` | 713 | 695 | 18 | fuzz_heif | +| `plist_extractor.c` | 594 | 580 | 14 | fuzz_plist | +| `iso9660_extractor.c` | 281 | 274 | 7 | fuzz_iso9660 | +| `webp_extractor.c` | 225 | 220 | 5 | fuzz_webp | +| `sqlite_extractor.c` | 549 | 548 | 1 | fuzz_sqlite | + +Three things follow. + +**`fuzz_diskimage` is the new `fuzz_ole2`.** 277 uncovered regions, more +than any other target in the tree, and 73 % coverage where every other +staged plugin is near 90 %. The cause is the corpus, not the runtime: +the plugin dispatches on four container formats — QCOW2, VHD, VHDX, +VMDK — and `src/plugins/testdata/` holds exactly one small sample of +each. Fund it first, and give it better seeds. + +**`forensics.c` is shared by all sixteen and 24 % of it is uncovered.** +It is attributed to `fuzz_lnk` only because that target happens to cover +it best; no single target owns it. 62 regions in code every one of +these plugins calls is worth more attention than its position in the +table suggests. + +**`fuzz_sqlite` and `fuzz_webp` are done.** One and five regions left +after 60.7 M and 39.7 M executions. Give them the floor and nothing +more. + +### What to spend the next campaign on + +1. **Seeds for `fuzz_diskimage`**, and to a lesser extent `fuzz_tar` + (72 % covered). Both are corpus-starved rather than time-starved; + another hour of the same inputs will not find the other 277 regions. +2. **`-F 900`**, per the convergence table above. +3. **A structure-aware generator** is still the honest answer for + `unzip.c` (61 regions left, concentrated in the central-directory + coherency checks). Note that `fuzz_apk` and `fuzz_ebook` link + `unzip.c` but cover it *worse* than `fuzz_unzip` does — 527 and 521 + regions against 607 — so section 4's conclusion stands unchanged: + fund `unzip.c` through `fuzz_unzip` and nowhere else. + + +## 8. The tuned 4-hour run (2026-08-07) + +The campaign of section 7 was re-run with everything it taught applied: +the measured weight table, `-F 900` for the slow-converging staged +plugins, and enriched seeds for the two targets that turned out to be +corpus-starved. + + 49 targets / 16 cores / 4 h = 64 core-hours, water-filled + +### Seeds beat runtime, decisively + +Section 7 attributed `fuzz_diskimage`'s 277 uncovered regions to a +corpus holding exactly one sample of each of its four container formats, +and `fuzz_tar`'s 99 to a single ustar archive. Generating the missing +variants — QCOW2 v2/v3 at three cluster sizes, lazy-refcount, zstd and +backed images, all five VMDK subformats qemu-img will write, VHDX at +four block/log combinations, fixed and dynamic VHD; and GNU/ustar/pax/ +v7/oldgnu/posix tars plus sparse members and pax extended headers — +moved coverage *before any fuzzing at all*: + +| file | seeds only, before | seeds only, after | 1200 s campaign of §7 | +|---|---|---|---| +| `diskimage_extractor.c` | 55.5 % | **66.2 %** | 73.1 % | +| `tar_extractor.c` | 54.2 % | **80.1 %** | 72.2 % | + +The tar figure is the one to remember: eight generated files reach more +of the parser than a twenty-minute fuzzing slice did. When a target's +headroom is concentrated in format variants the corpus does not contain, +generating them is worth more than any amount of additional runtime. + +### Findings + +**`qt_extractor.c`, heap-buffer-overflow — fixed.** An 80-byte input. +`getAtomSize` and `getAtomHeaderSize` both document that +`checkAtomValid` must have run first, because a size field of 1 selects +the 64-bit `struct LongAtom` and is read sixteen bytes wide rather than +eight. `ilstHandler` validates each entry and passes the real end down, +but `iTunesTagHandler` ignored its `size` parameter and tested only the +*declared* size (`asize < hdr + 8`) before handing `pos + hdr` to +`processDataAtom`, which dereferences the child atom immediately. A +child declaring the 64-bit escape therefore read eight bytes past what +had been guaranteed. The fix restores the documented precondition with +the same `checkAtomValid` call the sibling handlers already use. +Reproducer: `corpus/known-findings/fuzz_qt-issue15.bin`. + +**`geotiff_extractor.c`, float-cast-overflow — fixed.** See section 7. +Reproducer: `corpus/known-findings/fuzz_geotiff-issue14.bin`. + +**The ole2 artifacts are a libgsf defect — patch sent to `patches/`.** +Diagnosed rather than merely classified this time: `msole_prop_read` +checks a dictionary entry's length against `end - data + 8` where it +must be `end - data - 8`, and omits `section->char_size`, which is 2 for +the Unicode property sets. A crafted document reads up to `2*len + 16` +bytes past the property-set section. It is invisible with libgsf's own +stdio and memory inputs, whose padded buffers absorb it, and becomes a +hard ASan report only because `le_input_read` in `ole2_extractor.c` +hands libgsf a tightly sized buffer. Rebuilding libgsf 1.14.53 with the +fix takes all twenty artifacts to zero and leaves `test_ole2` and +`test_msoffice` passing. + +Note that `fuzz_qt` ran this campaign against the *unpatched* plugin — +the defect was found an hour in, and rebuilding mid-run would have +invalidated the schedule. Its artifacts are all the one defect. + + +## 9. Independent runs beat long runs + +The four-hour campaign of section 8 gave its six heaviest targets a +single uninterrupted four-hour slice each, because that is what +water-filling a weighted budget does. Replaying the corpora afterwards +showed that this was the wrong thing to do, and by a wide margin. + +Region coverage of each plugin's own source file, same binary, same +seed corpus, only the shape of the schedule differing: + +| target | 2 x 1200 s, corpora unioned | 1 x 14 000 s | best single 1200 s run | +|---|---|---|---| +| `fuzz_ebook` | **678** / 775 | 595 | 669 | +| `fuzz_apk` | **824** / 927 | 792 | 822 | + +Forty core-minutes beat four core-hours, and for both targets even *one* +twenty-minute run beat the four-hour one. + +The mechanism is visible in the logs. `fuzz_ebook`'s four-hour job +reached 1193 edges at t = 2570 s and never found another one — it then +spent 82 % of its slice and 195 million further executions standing +still, while a twenty-minute run from a different PRNG seed reached +1395. libFuzzer commits early to a corpus and mutates within it; when +that corpus is a poor basin there is no mechanism to leave it, and more +time cannot help. A second run from a different seed is not more of the +same work — it is a second sample of a distribution whose spread +(section 7: up to 16.6 % on these plugins) dwarfs what extra runtime +buys. + +This is not universal. Measuring how long each target went without a +new edge before its slice ended: + +| still gaining at the end | plateaued early | +|---|---| +| `fuzz_rtf` 0 %, `fuzz_apk` 1 %, `fuzz_gpx` 2 %, `fuzz_pecoff` 3 %, `fuzz_msoffice` 4 %, `fuzz_diskimage` 5 %, `fuzz_mbox` 5 % | `fuzz_ogg` 87 %, `fuzz_ebook` 82 %, `fuzz_unzip` 61 %, `fuzz_datasource` 48 %, `fuzz_kml` 48 % | + +Note that `fuzz_apk` is in the *left* column and still lost to two short +runs: a target can keep finding new edges at a slow rate and yet be in a +worse basin than a fresh run would have found. Time-to-plateau alone +does not identify which targets to shard. + +### What was changed + +`run_campaign.sh` grew `-S SECONDS` (default 1800). Any target allotted +more than one shard's worth is now run as several independent jobs of +that length, each with its own copy of the seed corpus and its own +`-seed`, and the shard corpora are unioned into `corpus-merged/` at the +end. The core-hour budget and the weight table are untouched; only the +shape changes. On 16 cores for 4 h that is 148 jobs instead of 49. + +The weight table itself was re-derived from the union of every corpus +both campaigns produced, which is the least seed-dependent basis +available: union coverage 17559 / 18889 regions, 93.0 %. + +### The standing advice, revised + +1. **Shard.** Leave `-S` alone. It is worth more than any weight in the + table. +2. **Seeds before time.** Section 8's tar result and section 9's ebook + result point the same way: eight generated files beat a twenty-minute + slice, and a twenty-minute slice beat four hours. When a target + looks starved, ask whether the corpus is missing a format variant + before asking for more core-hours. +3. **Then weights**, for apportioning what is left. + + +## 10. Sharding is a cold-start effect + +Section 9 concluded that several independent short runs beat one long +run, and `-S` was added on that basis. A two-hour, 16-core, 85-job run +tested whether it still holds once the corpus is no longer small. It +does not, and the qualification matters more than the original result. + +The run differed from section 8's in two ways: every target started +from the accumulated corpus of the previous two campaigns rather than +from the shipped seeds, and the oversized allocations were sharded. The +corpus was distilled first (`-merge=1`, 42 707 inputs down to 16 748, +338 MB to 116 MB, coverage-neutral: `diskimage_extractor.c` 831 to 830 +regions, `gpx_extractor.c` 855 to 855). + +Final edge coverage of the shards of one target, against the +cold-start seed-to-seed spread that motivated sharding in the first +place: + +| target | shards (warm) | spread | cold-start Δ (§7) | +|---|---|---|---| +| fuzz_ebook | 1433 1435 1433 1434 | **0.14 %** | −16.6 % | +| fuzz_mbox | 1833 1833 | 0.00 % | −9.1 % | +| fuzz_gpx | 1264 1264 | 0.00 % | −7.4 % | +| fuzz_apk | 1467 1462 1460 1464 | 0.48 % | −5.7 % | +| fuzz_kml | 1078 1081 | 0.28 % | −3.6 % | +| fuzz_diskimage | 824 825 | 0.12 % | −3.7 % | +| fuzz_pecoff | 1284–1288 | 0.31 % | +4.4 % | +| fuzz_lnk, heif, id3, geotiff | identical | 0.00 % | ~0 % | + +From a large corpus the PRNG seed stops mattering: every shard lands in +the same place to within half a percent, where cold-start replicates +differed by up to a sixth. **It is the corpus, not the seed, that +decides where a run ends up.** Sharding buys diversity only while the +corpus is small enough for trajectory to be in play. + +That does not make `-S` wrong — it still prevents the section 9 +pathology, where a single long job plateaus and burns hours on nothing +(`fuzz_ebook` idle for 82 % of a four-hour slice), and it costs nothing. +But it should be understood as insurance against a bad trajectory, not +as a way to buy coverage from a mature corpus. + +The warm start itself was the larger effect. `fuzz_ebook` finished at +1433 edges from every shard, against 1395 for the best cold-start +twenty-minute run and 1163 for the worst — the corpus removed the +variance that section 7 spent its whole budget measuring. + +### Diminishing returns, quantified + +| campaign | cost | union region coverage | +|---|---|---| +| §7 flat measurement | 16 core-hours | 89.8 % | +| §8 tuned, 4 h | 64 core-hours | 93.0 % | +| §10 warm + sharded, 2 h | 32 core-hours | **93.1 %** | + +Thirty-two core-hours bought 36 regions. The library is at the point +this document predicted in section 5: campaigns are now a regression +exercise, and new coverage has to come from seeds, new targets, or +structure-aware generators rather than from runtime. + +### Findings + +None new. All 22 artifacts were `fuzz_ole2`, all carrying the +`g_convert_with_iconv` signature of the libgsf defect in `patches/`, and +all 22 stop reporting against a patched libgsf. The replay pass over +all 85 corpora produced no recovering UndefinedBehaviorSanitizer reports +and no other sanitizer output. diff --git a/src/fuzz/Makefile.am b/src/fuzz/Makefile.am @@ -130,7 +130,21 @@ PLAIN_PLUGIN_FUZZERS = \ fuzz_s3m \ fuzz_sid \ fuzz_wav \ - fuzz_xm + fuzz_xm \ + fuzz_pecoff \ + fuzz_lnk \ + fuzz_sqlite \ + fuzz_tar \ + fuzz_iso9660 \ + fuzz_diskimage \ + fuzz_heif \ + fuzz_webp \ + fuzz_plist \ + fuzz_id3 \ + fuzz_gpx \ + fuzz_kml \ + fuzz_geotiff \ + fuzz_mbox fuzz_applefile_SOURCES = fuzz_plugin.c fuzz_applefile_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ @@ -236,7 +250,124 @@ ZLIB_PLUGIN_FUZZERS = \ fuzz_odf \ fuzz_png \ fuzz_qt \ - fuzz_zip + fuzz_zip \ + fuzz_apk \ + fuzz_ebook + +# The forensic header parsers. They share forensics.c, so it is +# compiled into every one of these harnesses. +fuzz_pecoff_SOURCES = fuzz_plugin.c +fuzz_pecoff_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=pecoff -DLE_FUZZ_ID=PECOFF +fuzz_pecoff_LDADD = $(XLIB) -lm +nodist_fuzz_pecoff_SOURCES = \ + $(PLUGIN_DIR)/pecoff_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_lnk_SOURCES = fuzz_plugin.c +fuzz_lnk_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=lnk -DLE_FUZZ_ID=LNK +fuzz_lnk_LDADD = $(XLIB) -lm +nodist_fuzz_lnk_SOURCES = \ + $(PLUGIN_DIR)/lnk_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_sqlite_SOURCES = fuzz_plugin.c +fuzz_sqlite_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=sqlite -DLE_FUZZ_ID=SQLITE +fuzz_sqlite_LDADD = $(XLIB) -lm +nodist_fuzz_sqlite_SOURCES = \ + $(PLUGIN_DIR)/sqlite_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_tar_SOURCES = fuzz_plugin.c +fuzz_tar_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=tar -DLE_FUZZ_ID=TAR +fuzz_tar_LDADD = $(XLIB) -lm +nodist_fuzz_tar_SOURCES = \ + $(PLUGIN_DIR)/tar_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_iso9660_SOURCES = fuzz_plugin.c +fuzz_iso9660_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=iso9660 -DLE_FUZZ_ID=ISO9660 +fuzz_iso9660_LDADD = $(XLIB) -lm +nodist_fuzz_iso9660_SOURCES = \ + $(PLUGIN_DIR)/iso9660_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_diskimage_SOURCES = fuzz_plugin.c +fuzz_diskimage_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=diskimage -DLE_FUZZ_ID=DISKIMAGE +fuzz_diskimage_LDADD = $(XLIB) -lm +nodist_fuzz_diskimage_SOURCES = \ + $(PLUGIN_DIR)/diskimage_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_heif_SOURCES = fuzz_plugin.c +fuzz_heif_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=heif -DLE_FUZZ_ID=HEIF +fuzz_heif_LDADD = $(XLIB) -lm +nodist_fuzz_heif_SOURCES = \ + $(PLUGIN_DIR)/heif_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_webp_SOURCES = fuzz_plugin.c +fuzz_webp_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=webp -DLE_FUZZ_ID=WEBP +fuzz_webp_LDADD = $(XLIB) -lm +nodist_fuzz_webp_SOURCES = \ + $(PLUGIN_DIR)/webp_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_plist_SOURCES = fuzz_plugin.c +fuzz_plist_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=plist -DLE_FUZZ_ID=PLIST +fuzz_plist_LDADD = $(XLIB) -lm +nodist_fuzz_plist_SOURCES = \ + $(PLUGIN_DIR)/plist_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_id3_SOURCES = fuzz_plugin.c +fuzz_id3_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=id3 -DLE_FUZZ_ID=ID3 +fuzz_id3_LDADD = $(XLIB) -lm +nodist_fuzz_id3_SOURCES = \ + $(PLUGIN_DIR)/id3_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_gpx_SOURCES = fuzz_plugin.c +fuzz_gpx_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=gpx -DLE_FUZZ_ID=GPX +fuzz_gpx_LDADD = $(XLIB) -lm +nodist_fuzz_gpx_SOURCES = \ + $(PLUGIN_DIR)/gpx_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_kml_SOURCES = fuzz_plugin.c +fuzz_kml_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=kml -DLE_FUZZ_ID=KML +fuzz_kml_LDADD = $(XLIB) -lm +nodist_fuzz_kml_SOURCES = \ + $(PLUGIN_DIR)/kml_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_geotiff_SOURCES = fuzz_plugin.c +fuzz_geotiff_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=geotiff -DLE_FUZZ_ID=GEOTIFF +fuzz_geotiff_LDADD = $(XLIB) -lm +nodist_fuzz_geotiff_SOURCES = \ + $(PLUGIN_DIR)/geotiff_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_mbox_SOURCES = fuzz_plugin.c +fuzz_mbox_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=mbox -DLE_FUZZ_ID=MBOX +fuzz_mbox_LDADD = $(XLIB) -lm +nodist_fuzz_mbox_SOURCES = \ + $(PLUGIN_DIR)/mbox_extractor.c \ + $(PLUGIN_DIR)/forensics.c + fuzz_deb_SOURCES = fuzz_plugin.c fuzz_deb_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ @@ -277,6 +408,25 @@ fuzz_zip_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ fuzz_zip_LDFLAGS = -static fuzz_zip_LDADD = $(LE_COMMON) $(XLIB) -lz nodist_fuzz_zip_SOURCES = $(PLUGIN_DIR)/zip_extractor.c + +fuzz_apk_SOURCES = fuzz_plugin.c +fuzz_apk_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=apk -DLE_FUZZ_ID=APK +fuzz_apk_LDFLAGS = -static +fuzz_apk_LDADD = $(LE_COMMON) $(XLIB) -lm -lz +nodist_fuzz_apk_SOURCES = \ + $(PLUGIN_DIR)/apk_extractor.c \ + $(PLUGIN_DIR)/forensics.c + +fuzz_ebook_SOURCES = fuzz_plugin.c +fuzz_ebook_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \ + -DLE_FUZZ_PLUGIN=ebook -DLE_FUZZ_ID=EBOOK +fuzz_ebook_LDFLAGS = -static +fuzz_ebook_LDADD = $(LE_COMMON) $(XLIB) -lm -lz +nodist_fuzz_ebook_SOURCES = \ + $(PLUGIN_DIR)/ebook_extractor.c \ + $(PLUGIN_DIR)/forensics.c + endif diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-000.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-001.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-002.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-003.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-004.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_apk/fuzz_apk-005.bin b/src/fuzz/corpus/fuzz_apk/fuzz_apk-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-000.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-001.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-002.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-003.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-004.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-005.bin b/src/fuzz/corpus/fuzz_diskimage/fuzz_diskimage-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-000.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-001.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-002.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-003.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-004.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-005.bin b/src/fuzz/corpus/fuzz_ebook/fuzz_ebook-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-000.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-001.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-002.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-003.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-004.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-005.bin b/src/fuzz/corpus/fuzz_geotiff/fuzz_geotiff-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-000.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-001.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-002.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-003.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-004.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-005.bin b/src/fuzz/corpus/fuzz_gpx/fuzz_gpx-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-000.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-001.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-002.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-003.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-004.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_heif/fuzz_heif-005.bin b/src/fuzz/corpus/fuzz_heif/fuzz_heif-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-000.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-001.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-002.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-003.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-004.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_id3/fuzz_id3-005.bin b/src/fuzz/corpus/fuzz_id3/fuzz_id3-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-000.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-001.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-002.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-003.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-004.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-005.bin b/src/fuzz/corpus/fuzz_iso9660/fuzz_iso9660-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-000.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-001.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-002.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-003.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-004.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_kml/fuzz_kml-005.bin b/src/fuzz/corpus/fuzz_kml/fuzz_kml-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-000.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-001.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-002.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-003.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-004.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-005.bin b/src/fuzz/corpus/fuzz_lnk/fuzz_lnk-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-000.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-001.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-002.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-003.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-004.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-005.bin b/src/fuzz/corpus/fuzz_mbox/fuzz_mbox-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-000.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-001.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-002.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-003.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-004.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-005.bin b/src/fuzz/corpus/fuzz_pecoff/fuzz_pecoff-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-000.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-001.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-002.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-003.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-004.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_plist/fuzz_plist-005.bin b/src/fuzz/corpus/fuzz_plist/fuzz_plist-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-000.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-001.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-002.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-003.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-004.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-005.bin b/src/fuzz/corpus/fuzz_sqlite/fuzz_sqlite-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-000.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-001.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-002.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-003.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-004.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_tar/fuzz_tar-005.bin b/src/fuzz/corpus/fuzz_tar/fuzz_tar-005.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-000.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-000.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-001.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-001.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-002.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-002.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-003.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-003.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-004.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-004.bin Binary files differ. diff --git a/src/fuzz/corpus/fuzz_webp/fuzz_webp-005.bin b/src/fuzz/corpus/fuzz_webp/fuzz_webp-005.bin Binary files differ. diff --git a/src/fuzz/corpus/known-findings/fuzz_geotiff-issue14.bin b/src/fuzz/corpus/known-findings/fuzz_geotiff-issue14.bin Binary files differ. diff --git a/src/fuzz/corpus/known-findings/fuzz_qt-issue15.bin b/src/fuzz/corpus/known-findings/fuzz_qt-issue15.bin Binary files differ. diff --git a/src/fuzz/fuzz_plugin_name.h b/src/fuzz/fuzz_plugin_name.h @@ -113,6 +113,22 @@ #define LE_ID_OLE2 31 #define LE_ID_THUMBNAILFFMPEG 32 #define LE_ID_PREVIEWOPUS 33 +#define LE_ID_PECOFF 34 +#define LE_ID_LNK 35 +#define LE_ID_SQLITE 36 +#define LE_ID_TAR 37 +#define LE_ID_ISO9660 38 +#define LE_ID_DISKIMAGE 39 +#define LE_ID_HEIF 40 +#define LE_ID_WEBP 41 +#define LE_ID_PLIST 42 +#define LE_ID_ID3 43 +#define LE_ID_GPX 44 +#define LE_ID_KML 45 +#define LE_ID_GEOTIFF 46 +#define LE_ID_MBOX 47 +#define LE_ID_APK 48 +#define LE_ID_EBOOK 49 #define LE_FUZZ_IDVAL LE_FUZZ_CAT (LE_ID_, LE_FUZZ_ID) @@ -174,6 +190,99 @@ #define LE_FUZZ_MAGIC "OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00" #define LE_FUZZ_SHAPE LE_SHAPE_RAW +#elif LE_FUZZ_IDVAL == LE_ID_PECOFF +/* MZ header; the PE header offset lives at 0x3c */ +/* carries a valid e_lfanew at 0x3c pointing at the "PE\0\0" signature; + without it every generated input dies at the second check */ +#define LE_FUZZ_MAGIC \ + "\x4d\x5a" \ + "\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e\x2e" \ + "\x40\x00\x00\x00\x50\x45\x00\x00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_LNK +/* header size 0x4c followed by the shell link CLSID */ +#define LE_FUZZ_MAGIC \ + "\x4c\x00\x00\x00\x01\x14\x02\x00\x00\x00\x00\x00\xc0\x00\x00\x00\x00\x00\x00\x46" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_SQLITE +/* the header is the first 100 bytes */ +#define LE_FUZZ_MAGIC "SQLite format 3\x00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_TAR +/* the ustar magic is at offset 257, so the prefix only seeds the name field */ +#define LE_FUZZ_MAGIC "testfile\x00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_ISO9660 +/* the primary volume descriptor is at offset 32768, out of reach of a prefix */ +#define LE_FUZZ_MAGIC "\x01" "CD001" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_DISKIMAGE +/* qcow2; the VMDK/VHD/VHDX branches are reached from the corpus */ +#define LE_FUZZ_MAGIC "QFI\xfb\x00\x00\x00\x03" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_HEIF +/* ISO base media file format: big-endian length-prefixed boxes */ +#define LE_FUZZ_MAGIC "\x00\x00\x00\x18" "ftypheic\x00\x00\x00\x00mif1heic" +#define LE_FUZZ_SHAPE LE_SHAPE_CHUNK_BE + +#elif LE_FUZZ_IDVAL == LE_ID_WEBP +/* RIFF: little-endian length-prefixed chunks */ +#define LE_FUZZ_MAGIC "RIFF\x00\x00\x00\x00WEBPVP8X" +#define LE_FUZZ_SHAPE LE_SHAPE_CHUNK_LE + +#elif LE_FUZZ_IDVAL == LE_ID_PLIST +/* the trailer is the last 32 bytes */ +#define LE_FUZZ_MAGIC "bplist00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_ID3 +/* ID3v2 header with a synchsafe size */ +#define LE_FUZZ_MAGIC "ID3\x03\x00\x00\x00\x00\x00\x00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_GPX +#define LE_FUZZ_MAGIC \ + "<?xml version=\"1.0\"?>\n<gpx version=\"1.1\" creator=\"x\">" +#define LE_FUZZ_SHAPE LE_SHAPE_TEXT + +#elif LE_FUZZ_IDVAL == LE_ID_KML +#define LE_FUZZ_MAGIC \ + "<?xml version=\"1.0\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">" +#define LE_FUZZ_SHAPE LE_SHAPE_TEXT + +#elif LE_FUZZ_IDVAL == LE_ID_GEOTIFF +/* little-endian classic TIFF; the geo tags live in the IFD */ +#define LE_FUZZ_MAGIC "II\x2a\x00\x08\x00\x00\x00" +#define LE_FUZZ_SHAPE LE_SHAPE_RAW + +#elif LE_FUZZ_IDVAL == LE_ID_MBOX +/* the envelope line alone is rejected: the plugin also requires a + well-formed header block behind it */ +#define LE_FUZZ_MAGIC \ + "From a@b Mon Jan 1 00:00:00 2024\n" \ + "From: a@b\nDate: 1 Jan 2024 00:00:00 +0000\n" +#define LE_FUZZ_SHAPE LE_SHAPE_TEXT + +#elif LE_FUZZ_IDVAL == LE_ID_APK +/* LE_SHAPE_ZIP synthesises the whole container, so a magic prefix + would sit in front of it and invalidate every local header + offset; the zip harness leaves it empty for the same reason */ +#define LE_FUZZ_MAGIC "" +#define LE_FUZZ_SHAPE LE_SHAPE_ZIP + +#elif LE_FUZZ_IDVAL == LE_ID_EBOOK +/* EPUB; the MOBI branch has its "BOOKMOBI" magic at offset 60 and + cannot be expressed as a prefix, so seed the corpus with + testdata/ebook_test.mobi to reach it */ +#define LE_FUZZ_MAGIC "" +#define LE_FUZZ_SHAPE LE_SHAPE_ZIP + #elif LE_FUZZ_IDVAL == LE_ID_PREVIEWOPUS #define LE_FUZZ_MAGIC "OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00" #define LE_FUZZ_SHAPE LE_SHAPE_RAW diff --git a/src/include/extractor.h b/src/include/extractor.h @@ -410,7 +410,325 @@ enum EXTRACTOR_MetaType */ EXTRACTOR_METATYPE_AUTHORING_OS = 231, - EXTRACTOR_METATYPE_LAST = 232 + /* The types below were introduced for formats whose interesting + content is provenance rather than authorship: executables, disk + images, databases, mail and geodata. They are meant to survive a + single fast pass over a file, so each of them is something that can + be read out of a header without decoding the payload. */ + + /** + * Timestamp recorded by the linker or archiver when the file was + * built, as opposed to when the file was created on this filesystem. + */ + EXTRACTOR_METATYPE_BUILD_DATE = 232, + + /** + * Identifier tying a binary to the debug information built with it + * (PE debug directory GUID, Mach-O LC_UUID, ELF build-id). + */ + EXTRACTOR_METATYPE_BUILD_ID = 233, + + /** + * Compiler and linker that produced the file, where the format + * records it separately from the authoring application (the PE + * "Rich" header, for example). + */ + EXTRACTOR_METATYPE_TOOLCHAIN = 234, + + /** + * Path to the debug symbols recorded at build time. Frequently + * discloses the user name and directory layout of the build machine. + */ + EXTRACTOR_METATYPE_DEBUG_PATH = 235, + + /** + * Hash over the imported symbols of an executable ("imphash"). + */ + EXTRACTOR_METATYPE_IMPORT_HASH = 236, + + /** + * Entry point of an executable or archive: a virtual address, or a + * class name for a Java archive. + */ + EXTRACTOR_METATYPE_ENTRY_POINT = 237, + + /** + * Execution environment an executable asks the loader for, such as + * "Windows GUI", "Windows console" or "native". + */ + EXTRACTOR_METATYPE_SUBSYSTEM = 238, + + /** + * Exploit mitigations the binary opts into (ASLR, DEP, control flow + * guard, ...). + */ + EXTRACTOR_METATYPE_SECURITY_MITIGATIONS = 239, + + /** + * Entity that signed the file: a code signing subject, a certificate + * common name, or the signing domain of a mail signature. + */ + EXTRACTOR_METATYPE_SIGNER = 240, + + /** + * Fingerprint of the certificate the file was signed with. + */ + EXTRACTOR_METATYPE_CERTIFICATE_FINGERPRINT = 241, + + /** + * Shannon entropy of the file or of one of its sections; high values + * suggest compressed, encrypted or packed content. + */ + EXTRACTOR_METATYPE_ENTROPY = 242, + + /** + * Name of the product the file belongs to, where that is recorded + * separately from the title. + */ + EXTRACTOR_METATYPE_PRODUCT_NAME = 243, + + /** + * Path this file points at: the target of a shortcut or a symlink. + */ + EXTRACTOR_METATYPE_TARGET_PATH = 244, + + /** + * Directory a shortcut or launcher starts its target in. + */ + EXTRACTOR_METATYPE_WORKING_DIRECTORY = 245, + + /** + * Arguments a shortcut or launcher passes to its target. + */ + EXTRACTOR_METATYPE_COMMAND_LINE = 246, + + /** + * Name of the user owning the file, as recorded inside the format. + */ + EXTRACTOR_METATYPE_OWNER_USER = 247, + + /** + * Name of the group owning the file, as recorded inside the format. + */ + EXTRACTOR_METATYPE_OWNER_GROUP = 248, + + /** + * Numeric user id of the owner, as recorded inside the format. + */ + EXTRACTOR_METATYPE_OWNER_UID = 249, + + /** + * Numeric group id of the owner, as recorded inside the format. + */ + EXTRACTOR_METATYPE_OWNER_GID = 250, + + /** + * Access permissions recorded inside the format, such as a Unix mode. + */ + EXTRACTOR_METATYPE_PERMISSIONS = 251, + + /** + * Time the file was last read, as recorded inside the format. + */ + EXTRACTOR_METATYPE_ACCESS_DATE = 252, + + /** + * Filesystem attribute flags recorded inside the format, such as + * hidden, system, read-only or archive. + */ + EXTRACTOR_METATYPE_ATTRIBUTES = 253, + + /** + * Label of the volume the file describes or was taken from. + */ + EXTRACTOR_METATYPE_VOLUME_NAME = 254, + + /** + * Serial number or UUID identifying a volume or disk image. + */ + EXTRACTOR_METATYPE_VOLUME_SERIAL = 255, + + /** + * Capacity of the volume or virtual disk. + */ + EXTRACTOR_METATYPE_VOLUME_SIZE = 256, + + /** + * Size of the format's allocation unit: a sector, a database page or + * a disk image cluster. + */ + EXTRACTOR_METATYPE_BLOCK_SIZE = 257, + + /** + * File system contained in or described by the file. + */ + EXTRACTOR_METATYPE_FILESYSTEM_TYPE = 258, + + /** + * Image this one is a delta against: a backing file or a + * differencing parent. + */ + EXTRACTOR_METATYPE_PARENT_IMAGE = 259, + + /** + * Unused capacity reported by the format. + */ + EXTRACTOR_METATYPE_FREE_SPACE = 260, + + /** + * System the volume was intended for, as recorded in its descriptor. + */ + EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER = 261, + + /** + * Party that assembled the data on a volume, as distinct from the + * party that published it. + */ + EXTRACTOR_METATYPE_DATA_PREPARER = 262, + + /** + * Date after which the content is no longer considered valid. + */ + EXTRACTOR_METATYPE_EXPIRATION_DATE = 263, + + /** + * Encryption or digital restrictions protecting the content. + */ + EXTRACTOR_METATYPE_ENCRYPTION = 264, + + /** + * Identifier of the application that owns the file, such as a bundle + * identifier or a database application id. + */ + EXTRACTOR_METATYPE_APPLICATION_ID = 265, + + /** + * Version of the schema the content is laid out according to. + */ + EXTRACTOR_METATYPE_SCHEMA_VERSION = 266, + + /** + * Counter the format increments on every modification. + */ + EXTRACTOR_METATYPE_CHANGE_COUNTER = 267, + + /** + * Crash recovery strategy a database is configured for. + */ + EXTRACTOR_METATYPE_JOURNAL_MODE = 268, + + /** + * Number of items the file holds: archive members, database records, + * mail messages, placemarks. + */ + EXTRACTOR_METATYPE_ENTRY_COUNT = 269, + + /** + * Total size of the content once decompressed. + */ + EXTRACTOR_METATYPE_UNCOMPRESSED_SIZE = 270, + + /** + * Coordinate reference system geographic data is expressed in. + */ + EXTRACTOR_METATYPE_COORDINATE_SYSTEM = 271, + + /** + * Geographic extent covered by the file, as west, south, east, north. + */ + EXTRACTOR_METATYPE_BOUNDING_BOX = 272, + + /** + * Length of a recorded path. + */ + EXTRACTOR_METATYPE_DISTANCE = 273, + + /** + * Identifier assigned to a message by the system that created it. + */ + EXTRACTOR_METATYPE_MESSAGE_ID = 274, + + /** + * Identifier of the message this one answers. + */ + EXTRACTOR_METATYPE_IN_REPLY_TO = 275, + + /** + * Addressee of a message. + */ + EXTRACTOR_METATYPE_RECIPIENT = 276, + + /** + * One hop of the path a message travelled, as recorded by a relay. + */ + EXTRACTOR_METATYPE_RECEIVED_FROM = 277, + + /** + * Network address recorded in the file. + */ + EXTRACTOR_METATYPE_IP_ADDRESS = 278, + + /** + * Parameters the encoder was invoked with. + */ + EXTRACTOR_METATYPE_ENCODER_SETTINGS = 279, + + /** + * Whether the bitrate is constant, variable or average. + */ + EXTRACTOR_METATYPE_BITRATE_MODE = 280, + + /** + * Bits per colour component. + */ + EXTRACTOR_METATYPE_COLOR_DEPTH = 281, + + /** + * Colour space or ICC profile the content is expressed in. + */ + EXTRACTOR_METATYPE_COLOR_PROFILE = 282, + + /** + * Specifications a container declares itself compatible with. + */ + EXTRACTOR_METATYPE_COMPATIBLE_BRANDS = 283, + + /** + * Account the content was bought with, as embedded by the store. + */ + EXTRACTOR_METATYPE_PURCHASE_ACCOUNT = 284, + + /** + * Marking identifying the recipient of an individual copy. + */ + EXTRACTOR_METATYPE_WATERMARK = 285, + + /** + * International standard book number. + */ + EXTRACTOR_METATYPE_ISBN = 286, + + /** + * Capability an application asks the platform for. + */ + EXTRACTOR_METATYPE_PERMISSION = 287, + + /** + * Hardware address recorded in the file. + */ + EXTRACTOR_METATYPE_MAC_ADDRESS = 288, + + /** + * Name of the machine the file originated on. + */ + EXTRACTOR_METATYPE_SOURCE_HOST = 289, + + /** + * Oldest platform release the content will run on. + */ + EXTRACTOR_METATYPE_MINIMUM_OS_VERSION = 290, + + EXTRACTOR_METATYPE_LAST = 291 }; /** @} */ /* end of meta data types */ diff --git a/src/main/extractor_metatypes.c b/src/main/extractor_metatypes.c @@ -594,6 +594,143 @@ static const struct MetaTypeDescription meta_type_descriptions[] = { gettext_noop ( "operating system the document was last written on") }, + /* 232 */ + { gettext_noop ("build date"), + gettext_noop ("date the file was compiled or linked") }, + { gettext_noop ("build identifier"), + gettext_noop ( + "identifier tying a binary to the debug information built with it") }, + { gettext_noop ("toolchain"), + gettext_noop ("compiler and linker that produced the file") }, + { gettext_noop ("debug symbol path"), + gettext_noop ("path to the debug symbols recorded at build time") }, + /* 236 */ + { gettext_noop ("import hash"), + gettext_noop ("hash over the imported symbols of an executable") }, + { gettext_noop ("entry point"), + gettext_noop ("address or class where execution begins") }, + { gettext_noop ("subsystem"), + gettext_noop ("execution environment the binary asks the loader for") }, + { gettext_noop ("security mitigations"), + gettext_noop ("exploit mitigations the binary opts into") }, + /* 240 */ + { gettext_noop ("signer"), + gettext_noop ("entity that signed the file") }, + { gettext_noop ("certificate fingerprint"), + gettext_noop ("fingerprint of the signing certificate") }, + { gettext_noop ("entropy"), + gettext_noop ( + "shannon entropy of the file or of one of its sections") }, + { gettext_noop ("product name"), + gettext_noop ("name of the product the file belongs to") }, + /* 244 */ + { gettext_noop ("target path"), + gettext_noop ("path this shortcut or link points at") }, + { gettext_noop ("working directory"), + gettext_noop ("directory the target is started in") }, + { gettext_noop ("command line"), + gettext_noop ("arguments passed to the target") }, + { gettext_noop ("owner user name"), + gettext_noop ("name of the user owning the file") }, + /* 248 */ + { gettext_noop ("owner group name"), + gettext_noop ("name of the group owning the file") }, + { gettext_noop ("owner user id"), + gettext_noop ("numeric user id of the owner") }, + { gettext_noop ("owner group id"), + gettext_noop ("numeric group id of the owner") }, + { gettext_noop ("permissions"), + gettext_noop ("access permissions recorded inside the format") }, + /* 252 */ + { gettext_noop ("last access date"), + gettext_noop ("time the file was last read") }, + { gettext_noop ("file attributes"), + gettext_noop ( + "attribute flags such as hidden, system, read-only or archive") }, + { gettext_noop ("volume name"), + gettext_noop ("label of the volume") }, + { gettext_noop ("volume serial number"), + gettext_noop ("serial number or uuid identifying a volume") }, + /* 256 */ + { gettext_noop ("volume size"), + gettext_noop ("capacity of the volume or virtual disk") }, + { gettext_noop ("block size"), + gettext_noop ("size of the format's allocation unit") }, + { gettext_noop ("file system type"), + gettext_noop ("file system contained in or described by the file") }, + { gettext_noop ("parent image"), + gettext_noop ("image this one is a delta against") }, + /* 260 */ + { gettext_noop ("free space"), + gettext_noop ("unused capacity reported by the format") }, + { gettext_noop ("system identifier"), + gettext_noop ("system the volume was intended for") }, + { gettext_noop ("data preparer"), + gettext_noop ("party that assembled the data on a volume") }, + { gettext_noop ("expiration date"), + gettext_noop ("date after which the content is no longer valid") }, + /* 264 */ + { gettext_noop ("encryption"), + gettext_noop ("encryption or digital restrictions protecting the content")}, + { gettext_noop ("application identifier"), + gettext_noop ("identifier of the application that owns the file") }, + { gettext_noop ("schema version"), + gettext_noop ("version of the schema the content follows") }, + { gettext_noop ("change counter"), + gettext_noop ("counter incremented on every modification") }, + /* 268 */ + { gettext_noop ("journal mode"), + gettext_noop ("crash recovery strategy the database is configured for") }, + { gettext_noop ("entry count"), + gettext_noop ("number of items the file holds") }, + { gettext_noop ("uncompressed size"), + gettext_noop ("total size of the content once decompressed") }, + { gettext_noop ("coordinate reference system"), + gettext_noop ("system geographic data is expressed in") }, + /* 272 */ + { gettext_noop ("bounding box"), + gettext_noop ("geographic extent covered, as west, south, east, north") }, + { gettext_noop ("distance"), + gettext_noop ("length of a recorded path") }, + { gettext_noop ("message identifier"), + gettext_noop ("identifier assigned to a message by its originator") }, + { gettext_noop ("in reply to"), + gettext_noop ("identifier of the message this one answers") }, + /* 276 */ + { gettext_noop ("recipient"), + gettext_noop ("addressee of a message") }, + { gettext_noop ("received from"), + gettext_noop ("one hop of the path a message travelled") }, + { gettext_noop ("ip address"), + gettext_noop ("network address recorded in the file") }, + { gettext_noop ("encoder settings"), + gettext_noop ("parameters the encoder was invoked with") }, + /* 280 */ + { gettext_noop ("bitrate mode"), + gettext_noop ("whether the bitrate is constant, variable or average") }, + { gettext_noop ("color depth"), + gettext_noop ("bits per colour component") }, + { gettext_noop ("color profile"), + gettext_noop ("colour space the content is expressed in") }, + { gettext_noop ("compatible brands"), + gettext_noop ("specifications the container declares compatibility with")}, + /* 284 */ + { gettext_noop ("purchase account"), + gettext_noop ("account the content was bought with") }, + { gettext_noop ("watermark"), + gettext_noop ("marking identifying the recipient of this copy") }, + { gettext_noop ("isbn"), + gettext_noop ("international standard book number") }, + { gettext_noop ("permission"), + gettext_noop ("capability the application asks the platform for") }, + /* 288 */ + { gettext_noop ("mac address"), + gettext_noop ("hardware address recorded in the file") }, + { gettext_noop ("source host"), + gettext_noop ("name of the machine the file originated on") }, + { gettext_noop ("minimum operating system version"), + gettext_noop ("oldest platform release the content will run on") }, + { gettext_noop ("last"), gettext_noop ("last") } }; diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am @@ -90,7 +90,32 @@ EXTRA_DIST = \ testdata/wav_alert.wav \ testdata/xm_diesel.xm \ testdata/zip_test.zip \ + testdata/apk_test.apk \ + testdata/apk_test.jar \ + testdata/diskimage_test.qcow2 \ + testdata/diskimage_test.vhd \ + testdata/diskimage_test.vhdx \ + testdata/diskimage_test.vmdk \ + testdata/ebook_test.epub \ + testdata/ebook_test.mobi \ + testdata/geotiff_test.tif \ + testdata/gpx_test.gpx \ + testdata/heif_test.avif \ + testdata/heif_test.heic \ + testdata/id3_test.mp3 \ + testdata/iso9660_test.iso \ + testdata/kml_test.kml \ + testdata/lnk_test.lnk \ + testdata/mbox_test.mbox \ + testdata/pecoff_test.dll \ + testdata/pecoff_test.exe \ + testdata/plist_binary.plist \ + testdata/plist_xml.plist \ + testdata/sqlite_test.db \ + testdata/tar_test.tar \ + testdata/webp_test.webp \ testdata/README \ + testdata/README.forensics \ testdata/README.media @@ -210,6 +235,12 @@ PLUGIN_ZLIB= \ TEST_ZLIB=test_deb TEST_MSOFFICE=test_msoffice TEST_QT=test_qt +# Android packages and EPUB books keep their metadata in deflated +# members of a zip container, so both need to inflate to say anything. +PLUGIN_APK=libextractor_apk.la +TEST_APK=test_apk +PLUGIN_EBOOK=libextractor_ebook.la +TEST_EBOOK=test_ebook endif if HAVE_LIBVLC @@ -219,21 +250,37 @@ endif plugin_LTLIBRARIES = \ libextractor_applefile.la \ + libextractor_diskimage.la \ libextractor_dvi.la \ libextractor_elf.la \ + libextractor_geotiff.la \ + libextractor_gpx.la \ + libextractor_heif.la \ + libextractor_id3.la \ + libextractor_iso9660.la \ libextractor_it.la \ + libextractor_kml.la \ + libextractor_lnk.la \ libextractor_man.la \ + libextractor_mbox.la \ libextractor_nsf.la \ libextractor_nsfe.la \ + libextractor_pecoff.la \ + libextractor_plist.la \ libextractor_ps.la \ libextractor_real.la \ libextractor_riff.la \ libextractor_rtf.la \ libextractor_s3m.la \ libextractor_sid.la \ + libextractor_sqlite.la \ + libextractor_tar.la \ libextractor_wav.la \ + libextractor_webp.la \ libextractor_xm.la \ + $(PLUGIN_APK) \ $(PLUGIN_ARCHIVE) \ + $(PLUGIN_EBOOK) \ $(PLUGIN_EXIV2) \ $(PLUGIN_FLAC) \ $(PLUGIN_GIF) \ @@ -258,13 +305,24 @@ plugin_LTLIBRARIES = \ check_PROGRAMS = \ test_applefile \ + test_diskimage \ test_dvi \ test_elf \ + test_geotiff \ + test_gpx \ + test_heif \ + test_id3 \ + test_iso9660 \ test_it \ + test_kml \ + test_lnk \ test_man \ + test_mbox \ test_nsf \ test_nsfe \ test_odf \ + test_pecoff \ + test_plist \ test_ps \ test_png \ test_real \ @@ -272,10 +330,15 @@ check_PROGRAMS = \ test_rtf \ test_s3m \ test_sid \ + test_sqlite \ + test_tar \ test_wav \ + test_webp \ test_xm \ test_zip \ + $(TEST_APK) \ $(TEST_ARCHIVE) \ + $(TEST_EBOOK) \ $(TEST_EXIV2) \ $(TEST_FLAC) \ $(TEST_GIF) \ @@ -313,6 +376,202 @@ libtest_la_LIBADD = \ $(top_builddir)/src/main/libextractor.la $(XLIB) + +# Plugins that read provenance out of binary headers. They share +# forensics.c for bounds-checked reads, endian decoding and UTF-8 +# validation; -lm is for the entropy calculation. +libextractor_diskimage_la_SOURCES = \ + diskimage_extractor.c forensics.c forensics.h +libextractor_diskimage_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_diskimage_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_diskimage_SOURCES = \ + test_diskimage.c +test_diskimage_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_geotiff_la_SOURCES = \ + geotiff_extractor.c forensics.c forensics.h +libextractor_geotiff_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_geotiff_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_geotiff_SOURCES = \ + test_geotiff.c +test_geotiff_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_gpx_la_SOURCES = \ + gpx_extractor.c forensics.c forensics.h +libextractor_gpx_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_gpx_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_gpx_SOURCES = \ + test_gpx.c +test_gpx_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_heif_la_SOURCES = \ + heif_extractor.c forensics.c forensics.h +libextractor_heif_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_heif_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_heif_SOURCES = \ + test_heif.c +test_heif_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_id3_la_SOURCES = \ + id3_extractor.c forensics.c forensics.h +libextractor_id3_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_id3_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_id3_SOURCES = \ + test_id3.c +test_id3_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_iso9660_la_SOURCES = \ + iso9660_extractor.c forensics.c forensics.h +libextractor_iso9660_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_iso9660_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_iso9660_SOURCES = \ + test_iso9660.c +test_iso9660_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_kml_la_SOURCES = \ + kml_extractor.c forensics.c forensics.h +libextractor_kml_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_kml_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_kml_SOURCES = \ + test_kml.c +test_kml_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_lnk_la_SOURCES = \ + lnk_extractor.c forensics.c forensics.h +libextractor_lnk_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_lnk_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_lnk_SOURCES = \ + test_lnk.c +test_lnk_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_mbox_la_SOURCES = \ + mbox_extractor.c forensics.c forensics.h +libextractor_mbox_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_mbox_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_mbox_SOURCES = \ + test_mbox.c +test_mbox_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_pecoff_la_SOURCES = \ + pecoff_extractor.c forensics.c forensics.h +libextractor_pecoff_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_pecoff_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_pecoff_SOURCES = \ + test_pecoff.c +test_pecoff_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_plist_la_SOURCES = \ + plist_extractor.c forensics.c forensics.h +libextractor_plist_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_plist_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_plist_SOURCES = \ + test_plist.c +test_plist_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_sqlite_la_SOURCES = \ + sqlite_extractor.c forensics.c forensics.h +libextractor_sqlite_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_sqlite_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_sqlite_SOURCES = \ + test_sqlite.c +test_sqlite_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_tar_la_SOURCES = \ + tar_extractor.c forensics.c forensics.h +libextractor_tar_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_tar_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_tar_SOURCES = \ + test_tar.c +test_tar_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_webp_la_SOURCES = \ + webp_extractor.c forensics.c forensics.h +libextractor_webp_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_webp_la_LIBADD = \ + -lm $(XLIB) $(LE_LIBINTL) + +test_webp_SOURCES = \ + test_webp.c +test_webp_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_apk_la_SOURCES = \ + apk_extractor.c forensics.c forensics.h +libextractor_apk_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_apk_la_LIBADD = \ + $(top_builddir)/src/common/libextractor_common.la -lm $(XLIB) $(LE_LIBINTL) + +test_apk_SOURCES = \ + test_apk.c +test_apk_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + +libextractor_ebook_la_SOURCES = \ + ebook_extractor.c forensics.c forensics.h +libextractor_ebook_la_LDFLAGS = \ + $(PLUGINFLAGS) +libextractor_ebook_la_LIBADD = \ + $(top_builddir)/src/common/libextractor_common.la -lm $(XLIB) $(LE_LIBINTL) + +test_ebook_SOURCES = \ + test_ebook.c +test_ebook_LDADD = \ + $(top_builddir)/src/plugins/libtest.la + libextractor_archive_la_SOURCES = \ archive_extractor.c libextractor_archive_la_LDFLAGS = \ diff --git a/src/plugins/apk_extractor.c b/src/plugins/apk_extractor.c @@ -0,0 +1,1929 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/apk_extractor.c + * @brief plugin to support Android packages and Java archives + * @author Christian Grothoff + * + * An APK and a JAR are both ZIP files, and so are a docx, an ODF + * document and any old archive, so the plugin identifies them by the + * member they must contain -- `AndroidManifest.xml' for an APK, + * `META-INF/MANIFEST.MF' for a JAR -- and returns without a word + * otherwise. An APK contains both, which is why the Android manifest + * is looked for first. + * + * What is worth having here is what the package may do (its declared + * permissions), what built it (the JDK or SDK version), and who signed + * it (the subject of the signing certificate). + * + * References: + * - AOSP frameworks/base/libs/androidfw/include/androidfw/ResourceTypes.h + * for the binary XML ("AXML") chunk format + * - https://source.android.com/docs/security/features/apksigning for the + * APK Signing Block + * - the JAR File Specification for `META-INF/MANIFEST.MF' + * - RFC 5652 (PKCS#7/CMS) and RFC 5280 for the signature block + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" +#include "unzip.h" + + +/** + * Name we report our meta data under. + */ +#define PLUGIN_NAME "apk" + +/** + * Never read more than this from `AndroidManifest.xml'. Real ones are + * a few tens of kilobytes even for very large applications. + */ +#define MAX_AXML (512 * 1024) + +/** + * Never read more than this from `META-INF/MANIFEST.MF'. + */ +#define MAX_MANIFEST (256 * 1024) + +/** + * Never read more than this from a `META-INF/*.RSA' signature block. + */ +#define MAX_PKCS7 (256 * 1024) + +/** + * Never read more than this of the APK Signing Block. + */ +#define MAX_SIG_BLOCK (64 * 1024) + +/** + * Upper bound on the number of ZIP members we walk. The end of central + * directory record counts entries in 16 bits, so this cannot be reached + * by a well-formed archive. + */ +#define MAX_MEMBERS 65535 + +/** + * Upper bound on the number of chunks and elements we walk in the + * binary XML. + */ +#define MAX_CHUNKS 65536 + +/** + * Upper bound on the number of strings in an AXML string pool. + */ +#define MAX_POOL_STRINGS 200000 + +/** + * Upper bound on the nesting depth we follow when walking DER. + */ +#define MAX_DER_DEPTH 16 + +/* AXML chunk types, from ResourceTypes.h */ +#define RES_XML_TYPE 0x0003 +#define RES_STRING_POOL_TYPE 0x0001 +#define RES_XML_START_ELEMENT_TYPE 0x0102 + +/* Res_value data types */ +#define TYPE_STRING 0x03 +#define TYPE_INT_DEC 0x10 +#define TYPE_INT_HEX 0x11 +#define TYPE_INT_BOOLEAN 0x12 + +/* APK Signing Block pair identifiers */ +#define APK_SIG_SCHEME_V2 0x7109871AU +#define APK_SIG_SCHEME_V3 0xF05368C0U +#define APK_SIG_SCHEME_V31 0x1B93AD61U + + +/** + * A parsed AXML string pool. + */ +struct StringPool +{ + /** + * Table of @e count 32 bit offsets into @e data. + */ + const unsigned char *offsets; + + /** + * The string data itself. + */ + const unsigned char *data; + + /** + * Number of bytes in @e data. + */ + size_t data_len; + + /** + * Number of strings in the pool. + */ + uint32_t count; + + /** + * True if the strings are UTF-8, false if they are UTF-16LE. + */ + int utf8; +}; + + +/** + * What we learned from walking the ZIP central directory. + */ +struct ArchiveFacts +{ + /** + * Base name of the `META-INF/<alias>.RSA' (or .DSA, .EC) member, + * which is conventionally the signer's key alias. + */ + char sig_alias[128]; + + /** + * Full member name of the signature block, empty if unsigned. + */ + char sig_member[256]; + + /** + * Distinct ABI directory names seen under `lib/'. + */ + char abis[8][32]; + + /** + * Number of entries used in @e abis. + */ + unsigned int num_abis; + + /** + * Number of `classes*.dex' members. + */ + unsigned int num_dex; + + /** + * Total number of members. + */ + unsigned int num_members; +}; + + +/** + * Find @a needle in @a hay. `memmem()' is a GNU extension and this only + * ever runs over buffers of a few tens of kilobytes. + * + * @param hay buffer to search + * @param hlen number of bytes in @a hay + * @param needle bytes to look for + * @param nlen number of bytes in @a needle + * @return pointer into @a hay, NULL if @a needle does not occur + */ +static const unsigned char * +mem_find (const unsigned char *hay, + size_t hlen, + const char *needle, + size_t nlen) +{ + if ( (0 == nlen) || + (nlen > hlen) ) + return NULL; + for (size_t i = 0; i + nlen <= hlen; i++) + if (0 == memcmp (&hay[i], + needle, + nlen)) + return &hay[i]; + return NULL; +} + + +/** + * Read a whole member out of a ZIP archive. + * + * @param uf the archive + * @param name member to read + * @param cap never read more than this many bytes + * @param[out] len number of bytes read + * @return the member's contents with a NUL appended, NULL on error; + * caller must free + */ +static unsigned char * +read_member (struct EXTRACTOR_UnzipFile *uf, + const char *name, + size_t cap, + size_t *len) +{ + struct EXTRACTOR_UnzipFileInfo fi; + unsigned char *buf; + size_t size; + size_t got = 0; + + *len = 0; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_go_find_local_file (uf, + name, + 2)) + return NULL; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_get_current_file_info (uf, + &fi, + NULL, 0, + NULL, 0, + NULL, 0)) + return NULL; + size = (size_t) fi.uncompressed_size; + if (0 == size) + return NULL; + if (size > cap) + size = cap; + if (NULL == (buf = malloc (size + 1))) + return NULL; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_open_current_file (uf)) + { + free (buf); + return NULL; + } + while (got < size) + { + ssize_t ret; + + ret = EXTRACTOR_common_unzip_read_current_file (uf, + &buf[got], + size - got); + if (0 >= ret) + break; /* error or end of member */ + if (((size_t) ret) > size - got) + break; /* cannot happen, but do not overrun if it does */ + got += (size_t) ret; + } + (void) EXTRACTOR_common_unzip_close_current_file (uf); + if (0 == got) + { + free (buf); + return NULL; + } + buf[got] = '\0'; + *len = got; + return buf; +} + + +/* ------------------------------------------------------------------ */ +/* JAR manifest */ +/* ------------------------------------------------------------------ */ + + +/** + * Report the value of one `META-INF/MANIFEST.MF' header, if it is one + * we care about. + * + * @param ec extraction context + * @param key header name, NUL-terminated + * @param value header value, NUL-terminated and already unfolded + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_manifest_header (struct EXTRACTOR_ExtractContext *ec, + const char *key, + const char *value) +{ + static const struct + { + const char *key; + enum EXTRACTOR_MetaType type; + } map[] = { + { "Main-Class", EXTRACTOR_METATYPE_ENTRY_POINT }, + /* these three name the exact JDK the archive was built with */ + { "Created-By", EXTRACTOR_METATYPE_TOOLCHAIN }, + { "Build-Jdk", EXTRACTOR_METATYPE_TOOLCHAIN }, + { "Build-Jdk-Spec", EXTRACTOR_METATYPE_TOOLCHAIN }, + { "Class-Path", EXTRACTOR_METATYPE_LIBRARY_DEPENDENCY }, + { "Implementation-Title", EXTRACTOR_METATYPE_TITLE }, + { "Implementation-Version", EXTRACTOR_METATYPE_SOFTWARE_VERSION }, + { "Implementation-Vendor", EXTRACTOR_METATYPE_VENDOR }, + { "Implementation-URL", EXTRACTOR_METATYPE_URL }, + { "Specification-Title", EXTRACTOR_METATYPE_SUBJECT }, + { "Specification-Version", EXTRACTOR_METATYPE_FORMAT_VERSION }, + { "Specification-Vendor", EXTRACTOR_METATYPE_ORGANIZATION }, + /* OSGi bundle headers */ + { "Bundle-SymbolicName", EXTRACTOR_METATYPE_APPLICATION_ID }, + { "Bundle-Name", EXTRACTOR_METATYPE_PRODUCT_NAME }, + { "Bundle-Version", EXTRACTOR_METATYPE_PACKAGE_VERSION }, + { "Bundle-Vendor", EXTRACTOR_METATYPE_VENDOR }, + { "Bundle-Description", EXTRACTOR_METATYPE_DESCRIPTION }, + { "Bundle-License", EXTRACTOR_METATYPE_LICENSE }, + { "Bundle-RequiredExecutionEnvironment", + EXTRACTOR_METATYPE_MINIMUM_OS_VERSION }, + { "Import-Package", EXTRACTOR_METATYPE_LIBRARY_DEPENDENCY }, + { NULL, EXTRACTOR_METATYPE_RESERVED } + }; + + for (unsigned int i = 0; NULL != map[i].key; i++) + if (0 == strcasecmp (key, + map[i].key)) + return EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + map[i].type, + value, + strlen (value)); + return 0; +} + + +/** + * Walk the main section of a `META-INF/MANIFEST.MF'. + * + * The format is `Key: Value' with CRLF line breaks, and no line may + * exceed 72 bytes, so long values are broken across continuation lines + * that begin with a single space which is not part of the value. Only + * the main section (everything before the first blank line) describes + * the archive; what follows is one section per signed member. + * + * @param ec extraction context + * @param data the manifest + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +parse_jar_manifest (struct EXTRACTOR_ExtractContext *ec, + const char *data, + size_t len) +{ + char key[128]; + char value[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t pos = 0; + size_t klen = 0; + size_t vlen = 0; + unsigned int headers = 0; + + key[0] = '\0'; + while ( (pos < len) && + (headers < 256) ) + { + const char *nl; + const char *line; + size_t line_len; + const char *colon; + + nl = memchr (&data[pos], + '\n', + len - pos); + line = &data[pos]; + line_len = (NULL == nl) + ? len - pos + : (size_t) (nl - line); + pos += line_len + ((NULL == nl) ? 0 : 1); + if ( (0 < line_len) && + ('\r' == line[line_len - 1]) ) + line_len--; + if (0 == line_len) + break; /* end of the main section */ + if (' ' == line[0]) + { + /* continuation of the previous header */ + size_t add = line_len - 1; + + if (0 == klen) + continue; /* continuation without a header; ignore */ + if (add > sizeof (value) - 1 - vlen) + add = sizeof (value) - 1 - vlen; + memcpy (&value[vlen], + &line[1], + add); + vlen += add; + value[vlen] = '\0'; + continue; + } + /* a new header starts, so the previous one is complete */ + if ( (0 != klen) && + (0 != emit_manifest_header (ec, + key, + value)) ) + return 1; + klen = 0; + vlen = 0; + headers++; + colon = memchr (line, + ':', + line_len); + if (NULL == colon) + continue; + klen = (size_t) (colon - line); + if (klen > sizeof (key) - 1) + klen = sizeof (key) - 1; + memcpy (key, + line, + klen); + key[klen] = '\0'; + vlen = line_len - (size_t) (colon - line) - 1; + if ( (0 < vlen) && + (' ' == colon[1]) ) + { + colon++; + vlen--; + } + if (vlen > sizeof (value) - 1) + vlen = sizeof (value) - 1; + memcpy (value, + &colon[1], + vlen); + value[vlen] = '\0'; + } + if ( (0 != klen) && + (0 != emit_manifest_header (ec, + key, + value)) ) + return 1; + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* PKCS#7 signature block */ +/* ------------------------------------------------------------------ */ + + +/** + * One DER tag-length-value triple. + */ +struct Tlv +{ + /** + * The tag byte. + */ + unsigned int tag; + + /** + * Offset of the value within the enclosing buffer. + */ + size_t start; + + /** + * Number of bytes in the value. + */ + size_t len; + + /** + * Offset just past the value, i.e. where the next TLV begins. + */ + size_t next; +}; + + +/** + * Decode the DER tag-length-value triple at @a pos. + * + * Only the definite-length, low-tag-number form is accepted; a + * certificate that needs anything else is not one we are going to + * understand anyway. + * + * @param buf the buffer + * @param end offset just past the region the TLV must fit in + * @param pos offset of the tag byte + * @param[out] tlv where to store the result + * @return 1 on success, 0 if the encoding is malformed or truncated + */ +static int +der_read (const unsigned char *buf, + size_t end, + size_t pos, + struct Tlv *tlv) +{ + uint64_t len; + size_t p = pos; + + if (p + 2 > end) + return 0; + tlv->tag = buf[p]; + if (0x1F == (tlv->tag & 0x1F)) + return 0; /* high tag number form */ + p++; + if (buf[p] < 0x80) + { + len = buf[p]; + p++; + } + else + { + unsigned int n = buf[p] & 0x7F; + + if ( (0 == n) || + (n > 4) ) + return 0; /* indefinite length, or longer than we will ever want */ + p++; + if (p + n > end) + return 0; + len = 0; + for (unsigned int i = 0; i < n; i++) + len = (len << 8) | buf[p + i]; + p += n; + } + if (len > (uint64_t) (end - p)) + return 0; + tlv->start = p; + tlv->len = (size_t) len; + tlv->next = p + (size_t) len; + return 1; +} + + +/** + * Append the short name for an X.500 attribute type to @a out. + * + * Attribute types we do not have a name for are skipped rather than + * spelled as an OID: a distinguished name that uses one is not one a + * human is going to recognise either way. + * + * @param oid the OID's content bytes + * @param len number of bytes in @a oid + * @return the short name, NULL if unknown + */ +static const char * +x500_name (const unsigned char *oid, + size_t len) +{ + static const struct + { + const char *name; + size_t len; + const unsigned char oid[12]; + } known[] = { + { "CN", 3, { 0x55, 0x04, 0x03 } }, + { "SN", 3, { 0x55, 0x04, 0x04 } }, + { "serialNumber", 3, { 0x55, 0x04, 0x05 } }, + { "C", 3, { 0x55, 0x04, 0x06 } }, + { "L", 3, { 0x55, 0x04, 0x07 } }, + { "ST", 3, { 0x55, 0x04, 0x08 } }, + { "STREET", 3, { 0x55, 0x04, 0x09 } }, + { "O", 3, { 0x55, 0x04, 0x0A } }, + { "OU", 3, { 0x55, 0x04, 0x0B } }, + { "T", 3, { 0x55, 0x04, 0x0C } }, + { "GN", 3, { 0x55, 0x04, 0x2A } }, + { "emailAddress", 9, + { 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x09, 0x01 } }, + { "DC", 10, + { 0x09, 0x92, 0x26, 0x89, 0x93, 0xF2, 0x2C, 0x64, 0x01, 0x19 } }, + { "UID", 10, + { 0x09, 0x92, 0x26, 0x89, 0x93, 0xF2, 0x2C, 0x64, 0x01, 0x01 } }, + { NULL, 0, { 0 } } + }; + + for (unsigned int i = 0; NULL != known[i].name; i++) + if ( (known[i].len == len) && + (0 == memcmp (known[i].oid, + oid, + len)) ) + return known[i].name; + return NULL; +} + + +/** + * Render an X.501 `Name' as an RFC 4514 style distinguished name. + * + * The relative distinguished names are emitted in the order they appear + * in the encoding, which is what `openssl x509 -subject' prints too. + * + * @param buf buffer holding the encoding + * @param name the `Name' TLV (a SEQUENCE OF RelativeDistinguishedName) + * @param[out] out where to write the result, NUL-terminated + * @param outsz number of bytes available in @a out, at least 2 + * @return 1 on success, 0 if nothing could be rendered + */ +static int +der_render_name (const unsigned char *buf, + const struct Tlv *name, + char *out, + size_t outsz) +{ + size_t o = 0; + size_t pos = name->start; + unsigned int rdns = 0; + + out[0] = '\0'; + while ( (pos < name->start + name->len) && + (rdns++ < 32) ) + { + struct Tlv rdn; + size_t apos; + + if (! der_read (buf, + name->start + name->len, + pos, + &rdn)) + break; + pos = rdn.next; + if (0x31 != rdn.tag) + continue; /* not a SET */ + apos = rdn.start; + while (apos < rdn.start + rdn.len) + { + struct Tlv atv; + struct Tlv oid; + struct Tlv value; + const char *label; + + if (! der_read (buf, + rdn.start + rdn.len, + apos, + &atv)) + break; + apos = atv.next; + if (0x30 != atv.tag) + continue; + if (! der_read (buf, + atv.start + atv.len, + atv.start, + &oid)) + continue; + if (0x06 != oid.tag) + continue; + if (! der_read (buf, + atv.start + atv.len, + oid.next, + &value)) + continue; + /* UTF8String, PrintableString, T61String, IA5String; BMPString + and UniversalString are left out because they would need a + conversion we would only ever exercise on broken input */ + if ( (0x0C != value.tag) && + (0x13 != value.tag) && + (0x14 != value.tag) && + (0x16 != value.tag) ) + continue; + label = x500_name (&buf[oid.start], + oid.len); + if (NULL == label) + continue; + if (0 != o) + { + if (o + 2 >= outsz) + break; + out[o++] = ','; + out[o++] = ' '; + } + if (o + strlen (label) + 1 >= outsz) + break; + memcpy (&out[o], + label, + strlen (label)); + o += strlen (label); + out[o++] = '='; + for (size_t i = 0; i < value.len; i++) + { + unsigned char c = buf[value.start + i]; + + if (o + 2 >= outsz) + break; + /* RFC 4514 escaping, so that a CN containing a comma cannot + look like two relative distinguished names */ + if ( (',' == c) || ('+' == c) || ('"' == c) || ('\\' == c) || + ('<' == c) || ('>' == c) || (';' == c) ) + out[o++] = '\\'; + out[o++] = (char) c; + } + } + } + out[o] = '\0'; + return (0 != o); +} + + +/** + * Pull the signer's distinguished name out of a PKCS#7 signature block. + * + * The certificates in a `META-INF/*.RSA' come in no particular order -- + * in a chain signed archive the certificate authority typically comes + * first -- so the signer is identified the way RFC 5652 says: by + * matching the SignerInfo's issuerAndSerialNumber against each + * certificate's issuer and serial number. + * + * @param buf the DER encoding + * @param len number of bytes in @a buf + * @param[out] out where to write the distinguished name, NUL-terminated + * @param outsz number of bytes available in @a out, at least 2 + * @return 1 on success, 0 if the block could not be understood + */ +static int +pkcs7_signer_dn (const unsigned char *buf, + size_t len, + char *out, + size_t outsz) +{ + struct Tlv content_info; + struct Tlv oid; + struct Tlv explicit0; + struct Tlv signed_data; + struct Tlv tlv; + struct Tlv certs; + struct Tlv signer_infos; + struct Tlv signer_issuer; + struct Tlv signer_serial; + size_t pos; + size_t end; + int have_signer = 0; + int have_certs = 0; + unsigned int fields = 0; + + out[0] = '\0'; + if (! der_read (buf, len, 0, &content_info)) + return 0; + if (0x30 != content_info.tag) + return 0; + end = content_info.start + content_info.len; + if (! der_read (buf, end, content_info.start, &oid)) + return 0; + if (0x06 != oid.tag) + return 0; + if (! der_read (buf, end, oid.next, &explicit0)) + return 0; + if (0xA0 != explicit0.tag) + return 0; + if (! der_read (buf, + explicit0.start + explicit0.len, + explicit0.start, + &signed_data)) + return 0; + if (0x30 != signed_data.tag) + return 0; + /* SignedData ::= SEQUENCE { version, digestAlgorithms, contentInfo, + [0] certificates OPTIONAL, + [1] crls OPTIONAL, signerInfos } + The first three fields have to be stepped over positionally: + digestAlgorithms is a SET and so is signerInfos, so a search by tag + would stop at the wrong one. */ + end = signed_data.start + signed_data.len; + pos = signed_data.start; + for (unsigned int i = 0; i < 3; i++) + { + if (! der_read (buf, end, pos, &tlv)) + return 0; + pos = tlv.next; + } + memset (&signer_infos, + 0, + sizeof (signer_infos)); + while ( (pos < end) && + (fields++ < MAX_DER_DEPTH) ) + { + if (! der_read (buf, end, pos, &tlv)) + return 0; + pos = tlv.next; + if (0xA0 == tlv.tag) + { + certs = tlv; + have_certs = 1; + } + else if (0x31 == tlv.tag) + { + signer_infos = tlv; + break; + } + } + if (! have_certs) + return 0; + /* SignerInfo ::= SEQUENCE { version, issuerAndSerialNumber, ... } */ + if (der_read (buf, + signer_infos.start + signer_infos.len, + signer_infos.start, + &tlv) && + (0x30 == tlv.tag)) + { + struct Tlv version; + struct Tlv ias; + + if (der_read (buf, tlv.start + tlv.len, tlv.start, &version) && + der_read (buf, tlv.start + tlv.len, version.next, &ias) && + (0x30 == ias.tag) && + der_read (buf, ias.start + ias.len, ias.start, &signer_issuer) && + der_read (buf, ias.start + ias.len, signer_issuer.next, + &signer_serial) && + (0x30 == signer_issuer.tag) && + (0x02 == signer_serial.tag)) + have_signer = 1; + } + /* walk the certificates and take the one the SignerInfo points at */ + pos = certs.start; + end = certs.start + certs.len; + for (unsigned int i = 0; (pos < end) && (i < 32); i++) + { + struct Tlv cert; + struct Tlv tbs; + struct Tlv field; + struct Tlv serial; + struct Tlv issuer; + struct Tlv validity; + struct Tlv subject; + + if (! der_read (buf, end, pos, &cert)) + break; + pos = cert.next; + if (0x30 != cert.tag) + continue; + if (! der_read (buf, cert.start + cert.len, cert.start, &tbs)) + continue; + if (0x30 != tbs.tag) + continue; + /* TBSCertificate ::= SEQUENCE { [0] version DEFAULT v1, + serialNumber, signature, issuer, validity, subject, ... } */ + if (! der_read (buf, tbs.start + tbs.len, tbs.start, &field)) + continue; + if (0xA0 == field.tag) + { + if (! der_read (buf, tbs.start + tbs.len, field.next, &serial)) + continue; + } + else + { + serial = field; + } + if (0x02 != serial.tag) + continue; + if (! der_read (buf, tbs.start + tbs.len, serial.next, &field)) + continue; /* signature algorithm */ + if (! der_read (buf, tbs.start + tbs.len, field.next, &issuer)) + continue; + if (0x30 != issuer.tag) + continue; + if (! der_read (buf, tbs.start + tbs.len, issuer.next, &validity)) + continue; + if (! der_read (buf, tbs.start + tbs.len, validity.next, &subject)) + continue; + if (0x30 != subject.tag) + continue; + if (have_signer) + { + /* RFC 5652 identifies the signer's certificate by issuer name + and serial number, so match on both */ + if ( (signer_serial.len != serial.len) || + (0 != memcmp (&buf[signer_serial.start], + &buf[serial.start], + serial.len)) ) + continue; + if ( (signer_issuer.len != issuer.len) || + (0 != memcmp (&buf[signer_issuer.start], + &buf[issuer.start], + issuer.len)) ) + continue; + } + return der_render_name (buf, + &subject, + out, + outsz); + } + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* binary XML */ +/* ------------------------------------------------------------------ */ + + +/** + * Fetch string number @a idx out of an AXML string pool, converted to + * UTF-8 and NUL-terminated. + * + * @param sp the pool + * @param idx index of the string + * @param[out] out where to write the string + * @param outsz number of bytes available in @a out, at least 8 + * @return number of bytes written, 0 if the string could not be read + */ +static size_t +pool_string (const struct StringPool *sp, + uint32_t idx, + char *out, + size_t outsz) +{ + const unsigned char *p; + size_t avail; + uint32_t off; + + out[0] = '\0'; + if ( (NULL == sp->offsets) || + (idx >= sp->count) ) + return 0; + off = EXTRACTOR_forensic_le32_ (&sp->offsets[4 * (size_t) idx]); + if (((size_t) off) >= sp->data_len) + return 0; + p = &sp->data[off]; + avail = sp->data_len - off; + if (sp->utf8) + { + size_t k = 0; + size_t n8; + + /* two varints: the length in UTF-16 code units, then the length in + bytes; a leading set high bit means the value takes two bytes */ + if (avail < 1) + return 0; + if (0 != (p[k] & 0x80)) + { + if (avail < 2) + return 0; + k += 2; + } + else + { + k += 1; + } + if (avail < k + 1) + return 0; + if (0 != (p[k] & 0x80)) + { + if (avail < k + 2) + return 0; + n8 = (size_t) (((((size_t) p[k]) & 0x7F) << 8) | p[k + 1]); + k += 2; + } + else + { + n8 = p[k]; + k += 1; + } + if (n8 > avail - k) + return 0; + if (n8 > outsz - 1) + n8 = outsz - 1; + memcpy (out, + &p[k], + n8); + out[n8] = '\0'; + if (! EXTRACTOR_forensic_utf8_valid_ (out, + n8)) + { + out[0] = '\0'; + return 0; + } + return n8; + } + { + size_t k = 2; + size_t n16; + size_t o = 0; + + if (avail < 2) + return 0; + n16 = EXTRACTOR_forensic_le16_ (p); + if (0 != (n16 & 0x8000)) + { + if (avail < 4) + return 0; + n16 = (((n16 & 0x7FFF) << 16) + | EXTRACTOR_forensic_le16_ (&p[2])); + k = 4; + } + if (n16 > (avail - k) / 2) + return 0; + for (size_t i = 0; i < n16; i++) + { + uint32_t cp = EXTRACTOR_forensic_le16_ (&p[k + 2 * i]); + + if ( (0xD800 <= cp) && (0xDBFF >= cp) && (i + 1 < n16) ) + { + uint32_t lo = EXTRACTOR_forensic_le16_ (&p[k + 2 * (i + 1)]); + + if ( (0xDC00 <= lo) && (0xDFFF >= lo) ) + { + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + i++; + } + } + if ( (0xD800 <= cp) && (0xDFFF >= cp) ) + break; /* unpaired surrogate; stop rather than guess */ + if (cp < 0x80) + { + if (o + 2 > outsz) + break; + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + if (o + 3 > outsz) + break; + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + if (o + 4 > outsz) + break; + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + if (o + 5 > outsz) + break; + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + } + out[o] = '\0'; + return o; + } +} + + +/** + * Spell an Android API level as the marketing version people recognise. + * + * @param api the API level + * @return the version string, NULL if the level is not in the table + */ +static const char * +android_release (uint32_t api) +{ + static const char *table[] = { + NULL, "1.0", "1.1", "1.5", "1.6", "2.0", "2.0.1", "2.1", "2.2", + "2.3", "2.3.3", "3.0", "3.1", "3.2", "4.0", "4.0.3", "4.1", "4.2", + "4.3", "4.4", "4.4W", "5.0", "5.1", "6.0", "7.0", "7.1", "8.0", + "8.1", "9", "10", "11", "12", "12L", "13", "14", "15", "16" + }; + + if ( (0 == api) || + (api >= sizeof (table) / sizeof (table[0])) ) + return NULL; + return table[api]; +} + + +/** + * Report an API level, naming the Android release where we know it. + * + * @param ec extraction context + * @param type meta data type + * @param api the API level + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_api_level (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + uint32_t api) +{ + const char *rel = android_release (api); + + if (NULL == rel) + return EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + type, + "Android API %u", + (unsigned int) api); + return EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + type, + "Android API %u (Android %s)", + (unsigned int) api, + rel); +} + + +/** + * Report the attributes of one AXML element, if it is an element we + * care about. + * + * @param ec extraction context + * @param sp the string pool + * @param element the element's name + * @param attrs pointer to the first attribute + * @param count number of attributes + * @param stride number of bytes per attribute + * @param[in,out] permissions number of permissions reported so far + * @param[in,out] have_sdk set once compileSdkVersion has been reported + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_element (struct EXTRACTOR_ExtractContext *ec, + const struct StringPool *sp, + const char *element, + const unsigned char *attrs, + unsigned int count, + size_t stride, + unsigned int *permissions, + int *have_sdk) +{ + int is_manifest = (0 == strcmp (element, "manifest")); + int is_uses_sdk = (0 == strcmp (element, "uses-sdk")); + int is_permission = (0 == strcmp (element, "uses-permission")); + int is_application = (0 == strcmp (element, "application")); + + if (! (is_manifest || is_uses_sdk || is_permission || is_application)) + return 0; + for (unsigned int i = 0; i < count; i++) + { + const unsigned char *a = &attrs[i * stride]; + char name[128]; + char value[EXTRACTOR_FORENSIC_MAX_STRING]; + uint32_t data; + unsigned int dtype; + + if (0 == pool_string (sp, + EXTRACTOR_forensic_le32_ (&a[4]), + name, + sizeof (name))) + continue; /* attribute names are given by resource id only in + files aapt did not produce; we do not guess */ + dtype = a[15]; + data = EXTRACTOR_forensic_le32_ (&a[16]); + value[0] = '\0'; + if (TYPE_STRING == dtype) + (void) pool_string (sp, + data, + value, + sizeof (value)); + if (is_manifest) + { + if ( (0 == strcmp (name, "package")) && + (TYPE_STRING == dtype) ) + { + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_PACKAGE_NAME, + value, + strlen (value))) + return 1; + } + else if (0 == strcmp (name, "versionCode")) + { + if (TYPE_STRING == dtype) + { + if (0 != + EXTRACTOR_forensic_emit_text_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_PACKAGE_VERSION, + value, + strlen (value))) + return 1; + } + else if (0 != + EXTRACTOR_forensic_emit_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_PACKAGE_VERSION, + "%u", + (unsigned int) data)) + return 1; + } + else if ( (0 == strcmp (name, "versionName")) && + (TYPE_STRING == dtype) ) + { + if (0 != + EXTRACTOR_forensic_emit_text_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_SOFTWARE_VERSION, + value, + strlen (value))) + return 1; + } + else if ( (0 == strcmp (name, "compileSdkVersion")) && + (TYPE_STRING != dtype) ) + { + *have_sdk = 1; + if (0 != + emit_api_level (ec, + EXTRACTOR_METATYPE_TOOLCHAIN, + data)) + return 1; + } + else if ( (0 == strcmp (name, "platformBuildVersionCode")) && + (TYPE_STRING != dtype) && + (! *have_sdk) ) + { + /* what older builds of aapt wrote instead of + compileSdkVersion; reporting both would say the same thing + twice */ + *have_sdk = 1; + if (0 != + emit_api_level (ec, + EXTRACTOR_METATYPE_TOOLCHAIN, + data)) + return 1; + } + } + else if (is_uses_sdk) + { + if ( (0 == strcmp (name, "minSdkVersion")) && + (TYPE_STRING != dtype) ) + { + if (0 != + emit_api_level (ec, + EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, + data)) + return 1; + } + else if ( (0 == strcmp (name, "targetSdkVersion")) && + (TYPE_STRING != dtype) ) + { + if (0 != + emit_api_level (ec, + EXTRACTOR_METATYPE_TARGET_OS, + data)) + return 1; + } + } + else if (is_permission) + { + if ( (0 != strcmp (name, "name")) || + (TYPE_STRING != dtype) ) + continue; + /* every declared permission is counted, but only the first few + dozen are reported: what matters on a first pass is the + characterisation, and the total is reported separately */ + (*permissions)++; + if (*permissions > EXTRACTOR_FORENSIC_MAX_ITEMS) + continue; + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_PERMISSION, + value, + strlen (value))) + return 1; + } + else /* is_application */ + { + if (0 == strcmp (name, "label")) + { + /* usually a resource reference, and a bare `@0x7f...' would + tell nobody anything */ + if (TYPE_STRING != dtype) + continue; + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_TITLE, + value, + strlen (value))) + return 1; + } + else if ( (0 == strcmp (name, "debuggable")) && + (TYPE_INT_BOOLEAN == dtype) && + (0 != data) ) + { + /* a debuggable package that was shipped is worth a look */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ATTRIBUTES, + "debuggable")) + return 1; + } + else if ( ( (0 == strcmp (name, "allowBackup")) || + (0 == strcmp (name, "usesCleartextTraffic")) || + (0 == strcmp (name, "extractNativeLibs")) ) && + (TYPE_INT_BOOLEAN == dtype) ) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s: %s", + name, + (0 != data) ? "true" : "false")) + return 1; + } + else if (0 == strcmp (name, "networkSecurityConfig")) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ATTRIBUTES, + "networkSecurityConfig: %s", + ('\0' != value[0]) + ? value + : "present")) + return 1; + } + } + } + return 0; +} + + +/** + * Walk a binary `AndroidManifest.xml'. + * + * The document is a flat list of chunks: a header, a string pool, an + * optional resource map, and then one chunk per XML event. Only the + * start element chunks matter here, and only four element names, so the + * walk stays linear and never recurses. + * + * @param ec extraction context + * @param data the document + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +parse_axml (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *data, + size_t len) +{ + struct StringPool sp; + size_t pos; + size_t total; + uint16_t hdr_size; + unsigned int chunks = 0; + unsigned int permissions = 0; + int have_sdk = 0; + + memset (&sp, + 0, + sizeof (sp)); + if (len < 8) + return 0; + if (RES_XML_TYPE != EXTRACTOR_forensic_le16_ (data)) + return 0; + hdr_size = EXTRACTOR_forensic_le16_ (&data[2]); + total = (size_t) EXTRACTOR_forensic_le32_ (&data[4]); + if ( (hdr_size < 8) || + (((size_t) hdr_size) > len) ) + return 0; + if ( (total > len) || + (total < 8) ) + total = len; /* truncated by our read cap, or by the file */ + pos = hdr_size; + while ( (pos + 8 <= total) && + (chunks++ < MAX_CHUNKS) ) + { + uint16_t type = EXTRACTOR_forensic_le16_ (&data[pos]); + uint16_t chdr = EXTRACTOR_forensic_le16_ (&data[pos + 2]); + uint32_t csize = EXTRACTOR_forensic_le32_ (&data[pos + 4]); + + if ( (csize < 8) || + (chdr < 8) || + (((uint64_t) csize) > total - pos) || + (chdr > csize) ) + break; /* malformed, or no forward progress */ + if (RES_STRING_POOL_TYPE == type) + { + uint32_t count; + uint32_t flags; + uint32_t strings_start; + uint32_t styles_start; + + if (chdr < 28) + { + pos += csize; + continue; + } + count = EXTRACTOR_forensic_le32_ (&data[pos + 8]); + flags = EXTRACTOR_forensic_le32_ (&data[pos + 16]); + strings_start = EXTRACTOR_forensic_le32_ (&data[pos + 20]); + styles_start = EXTRACTOR_forensic_le32_ (&data[pos + 24]); + if ( (count > MAX_POOL_STRINGS) || + (((uint64_t) count) * 4 > csize - 28) || + (strings_start > csize) || + (strings_start < 28) ) + { + pos += csize; + continue; + } + sp.count = count; + sp.offsets = &data[pos + 28]; + sp.data = &data[pos + strings_start]; + sp.data_len = csize - strings_start; + if ( (0 != styles_start) && + (styles_start > strings_start) && + (styles_start <= csize) ) + sp.data_len = styles_start - strings_start; + sp.utf8 = (0 != (flags & 0x100)); + } + else if (RES_XML_START_ELEMENT_TYPE == type) + { + const unsigned char *ext; + char element[128]; + uint16_t attr_start; + uint16_t attr_size; + uint16_t attr_count; + uint64_t need; + + if ( (chdr < 16) || + (csize - chdr < 20) ) + { + pos += csize; + continue; + } + ext = &data[pos + chdr]; + attr_start = EXTRACTOR_forensic_le16_ (&ext[8]); + attr_size = EXTRACTOR_forensic_le16_ (&ext[10]); + attr_count = EXTRACTOR_forensic_le16_ (&ext[12]); + /* every attribute is a ResXMLTree_attribute: four 32 bit words + plus a Res_value, and implementations are allowed to make the + record longer, never shorter */ + if (attr_size < 20) + { + pos += csize; + continue; + } + need = ((uint64_t) attr_start) + + ((uint64_t) attr_count) * attr_size; + if (need > (uint64_t) (csize - chdr)) + { + pos += csize; + continue; + } + if (0 == pool_string (&sp, + EXTRACTOR_forensic_le32_ (&ext[4]), + element, + sizeof (element))) + { + pos += csize; + continue; + } + if (0 != + emit_element (ec, + &sp, + element, + &data[pos + chdr + attr_start], + attr_count, + attr_size, + &permissions, + &have_sdk)) + return 1; + } + pos += csize; + } + if (permissions > EXTRACTOR_FORENSIC_MAX_ITEMS) + return EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_COMMENT, + "%u permissions declared", + permissions); + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* ZIP container */ +/* ------------------------------------------------------------------ */ + + +/** + * Walk the ZIP central directory once and note the things that can be + * read off member names alone. + * + * @param uf the archive + * @param[out] af where to store what we found + */ +static void +scan_members (struct EXTRACTOR_UnzipFile *uf, + struct ArchiveFacts *af) +{ + char name[512]; + unsigned int seen = 0; + + memset (af, + 0, + sizeof (*af)); + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_go_to_first_file (uf)) + return; + do + { + size_t nlen; + + if (seen++ >= MAX_MEMBERS) + break; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_get_current_file_info (uf, + NULL, + name, + sizeof (name), + NULL, 0, + NULL, 0)) + continue; + af->num_members++; + nlen = strlen (name); + if ( (0 == strncmp (name, "classes", strlen ("classes"))) && + (4 < nlen) && + (0 == strcmp (&name[nlen - 4], ".dex")) ) + { + af->num_dex++; + continue; + } + if (0 == strncmp (name, "lib/", strlen ("lib/"))) + { + const char *abi = &name[strlen ("lib/")]; + const char *slash = strchr (abi, '/'); + size_t alen; + + if (NULL == slash) + continue; + alen = (size_t) (slash - abi); + if ( (0 == alen) || + (alen >= sizeof (af->abis[0])) ) + continue; + for (unsigned int i = 0; i < af->num_abis; i++) + if ( (0 == strncmp (af->abis[i], abi, alen)) && + ('\0' == af->abis[i][alen]) ) + { + alen = 0; + break; + } + if (0 == alen) + continue; + if (af->num_abis >= sizeof (af->abis) / sizeof (af->abis[0])) + continue; + memcpy (af->abis[af->num_abis], + abi, + alen); + af->abis[af->num_abis][alen] = '\0'; + af->num_abis++; + continue; + } + if ( ('\0' == af->sig_member[0]) && + (0 == strncmp (name, "META-INF/", strlen ("META-INF/"))) && + (4 < nlen) && + ( (0 == strcasecmp (&name[nlen - 4], ".RSA")) || + (0 == strcasecmp (&name[nlen - 4], ".DSA")) ) ) + { + const char *base = &name[strlen ("META-INF/")]; + + if (NULL == strchr (base, '/')) + { + memcpy (af->sig_member, + name, + (nlen < sizeof (af->sig_member)) + ? nlen + 1 + : sizeof (af->sig_member)); + af->sig_member[sizeof (af->sig_member) - 1] = '\0'; + snprintf (af->sig_alias, + sizeof (af->sig_alias), + "%.*s", + (int) (strlen (base) - 4), + base); + } + } + } + while (EXTRACTOR_UNZIP_OK == + EXTRACTOR_common_unzip_go_to_next_file (uf)); +} + + +/** + * Report which APK signature schemes the file carries. + * + * The v2 and v3 signatures live in an "APK Signing Block", which sits + * between the last local file entry and the central directory and is + * found from the end: the block ends with its own size and the magic + * string `APK Sig Block 42' immediately before the central directory. + * + * @param ec extraction context + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_signing_block (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char eocd[22]; + unsigned char foot[24]; + unsigned char *block; + uint64_t size; + uint64_t cd_off; + uint64_t block_size; + uint64_t start; + size_t read_len; + size_t pos; + int ret = 0; + + size = ec->get_size (ec->cls); + if ( (UINT64_MAX == size) || + (size < 22 + 24) ) + return 0; + /* the usual case is an archive with no comment, so the end of central + directory record is the last 22 bytes */ + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (size - 22), + eocd, + sizeof (eocd))) + return 0; + if (0 != memcmp (eocd, + "PK\x05\x06", + 4)) + { + unsigned char *tail; + size_t tail_len = (size > 65557) ? 65557 : (size_t) size; + const unsigned char *found = NULL; + + if (NULL == (tail = malloc (tail_len))) + return 0; + if (EXTRACTOR_forensic_read_ (ec, + (int64_t) (size - tail_len), + tail, + tail_len)) + for (size_t i = tail_len - 22 + 1; i-- > 0;) + if (0 == memcmp (&tail[i], + "PK\x05\x06", + 4)) + { + found = &tail[i]; + break; + } + if (NULL == found) + { + free (tail); + return 0; + } + memcpy (eocd, + found, + sizeof (eocd)); + free (tail); + } + cd_off = EXTRACTOR_forensic_le32_ (&eocd[16]); + if ( (cd_off < 24) || + (cd_off > size) ) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (cd_off - 24), + foot, + sizeof (foot))) + return 0; + if (0 != memcmp (&foot[8], + "APK Sig Block 42", + 16)) + return 0; + block_size = EXTRACTOR_forensic_le64_ (&foot[0]); + /* written as a subtraction so that a huge size field cannot overflow; + cd_off is at least 24 here */ + if ( (block_size < 24) || + (block_size > cd_off - 8) ) + return 0; + start = cd_off - block_size - 8; + /* the pairs run from just after the leading size field up to the + trailing size field */ + read_len = (size_t) (block_size - 24); + if (read_len > MAX_SIG_BLOCK) + read_len = MAX_SIG_BLOCK; + if (0 == read_len) + return 0; + if (NULL == (block = malloc (read_len))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (start + 8), + block, + read_len)) + { + free (block); + return 0; + } + pos = 0; + for (unsigned int i = 0; (i < 64) && (pos + 12 <= read_len); i++) + { + uint64_t plen = EXTRACTOR_forensic_le64_ (&block[pos]); + uint32_t id = EXTRACTOR_forensic_le32_ (&block[pos + 8]); + const char *scheme = NULL; + + if (plen < 4) + break; /* no forward progress */ + switch (id) + { + case APK_SIG_SCHEME_V2: scheme = "v2"; break; + case APK_SIG_SCHEME_V3: scheme = "v3"; break; + case APK_SIG_SCHEME_V31: scheme = "v3.1"; break; + default: break; + } + if ( (NULL != scheme) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_SIGNER, + "APK Signature Scheme %s", + scheme)) ) + { + ret = 1; + break; + } + if (plen > read_len - pos - 8) + break; + pos += (size_t) plen + 8; + } + free (block); + return ret; +} + + +/** + * Report who signed the archive. + * + * @param ec extraction context + * @param uf the archive + * @param af what the central directory walk found + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_signature (struct EXTRACTOR_ExtractContext *ec, + struct EXTRACTOR_UnzipFile *uf, + const struct ArchiveFacts *af) +{ + unsigned char *der; + size_t der_len; + char dn[EXTRACTOR_FORENSIC_MAX_STRING]; + int ret = 0; + + if ('\0' == af->sig_member[0]) + return 0; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_SIGNER, + "JAR signing (v1), alias %s", + af->sig_alias)) + return 1; + der = read_member (uf, + af->sig_member, + MAX_PKCS7, + &der_len); + if (NULL == der) + return 0; + if (pkcs7_signer_dn (der, + der_len, + dn, + sizeof (dn))) + ret = EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_SIGNER, + dn, + strlen (dn)); + free (der); + return ret; +} + + +/** + * Report the facts that follow from the member names alone. + * + * @param ec extraction context + * @param af what the central directory walk found + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_archive_facts (struct EXTRACTOR_ExtractContext *ec, + const struct ArchiveFacts *af) +{ + for (unsigned int i = 0; i < af->num_abis; i++) + if (0 != + EXTRACTOR_forensic_emit_text_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, + af->abis[i], + strlen (af->abis[i]))) + return 1; + if ( (0 != af->num_dex) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_COMMENT, + "%u dex file%s", + af->num_dex, + (1 == af->num_dex) ? "" : "s")) ) + return 1; + if ( (0 != af->num_members) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + af->num_members)) ) + return 1; + return 0; +} + + +/** + * Main entry method for the apk extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_apk_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_apk_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char magic[4]; + struct EXTRACTOR_UnzipFile *uf; + struct ArchiveFacts af; + unsigned char *member; + size_t member_len; + int is_apk; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + magic, + sizeof (magic))) + return; + if (0 != memcmp (magic, + "PK\x03\x04", + 4)) + return; + if (NULL == (uf = EXTRACTOR_common_unzip_open (ec))) + return; + /* an APK contains a JAR manifest too, so look for the Android + manifest first */ + if (EXTRACTOR_UNZIP_OK == + EXTRACTOR_common_unzip_go_find_local_file (uf, + "AndroidManifest.xml", + 2)) + { + is_apk = 1; + } + else if (EXTRACTOR_UNZIP_OK == + EXTRACTOR_common_unzip_go_find_local_file (uf, + "META-INF/MANIFEST.MF", + 2)) + { + is_apk = 0; + } + else + { + (void) EXTRACTOR_common_unzip_close (uf); + return; /* somebody else's ZIP */ + } + if (0 != + ec->proc (ec->cls, + PLUGIN_NAME, + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + is_apk + ? "application/vnd.android.package-archive" + : "application/java-archive", + is_apk + ? strlen ("application/vnd.android.package-archive") + 1 + : strlen ("application/java-archive") + 1)) + goto CLEANUP; + if (is_apk) + { + member = read_member (uf, + "AndroidManifest.xml", + MAX_AXML, + &member_len); + if (NULL != member) + { + int stop = parse_axml (ec, + member, + member_len); + + free (member); + if (stop) + goto CLEANUP; + } + } + else + { + member = read_member (uf, + "META-INF/MANIFEST.MF", + MAX_MANIFEST, + &member_len); + if (NULL != member) + { + int stop = parse_jar_manifest (ec, + (const char *) member, + member_len); + + free (member); + if (stop) + goto CLEANUP; + } + } + scan_members (uf, + &af); + if (0 != + emit_archive_facts (ec, + &af)) + goto CLEANUP; + if (0 != + emit_signature (ec, + uf, + &af)) + goto CLEANUP; + if (is_apk) + (void) emit_signing_block (ec); +CLEANUP: + (void) EXTRACTOR_common_unzip_close (uf); +} + + +/* end of apk_extractor.c */ diff --git a/src/plugins/diskimage_extractor.c b/src/plugins/diskimage_extractor.c @@ -0,0 +1,1723 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/diskimage_extractor.c + * @brief plugin to support virtual disk images: QCOW/QCOW2, VMDK, + * VHD and VHDX + * @author Christian Grothoff + * + * These four formats answer the same questions, so one plugin answers + * them once: how big does the disk claim to be, which virtual machine + * or tool wrote it, is it a differencing image (and what is its parent, + * because then there are sibling files the investigator also needs), + * and is it encrypted. + * + * Deliberately *not* done here: identifying the guest file system + * inside the image. That means following the allocation tables to the + * first sectors of the guest disk, which is a second-pass job; this + * plugin never reads more than a handful of headers. + * + * References: + * - QCOW2: `docs/interop/qcow2.txt' in the QEMU source tree. + * - VMDK: "Virtual Disk Format 5.0" (VMware technical note). + * - VHD: "Virtual Hard Disk Image Format Specification" 1.0 (Microsoft). + * - VHDX: [MS-VHDX], "Virtual Hard Disk v2 (VHDX) File Format". + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * Name we report our meta data under. + */ +#define DISKIMAGE "diskimage" + +/** + * Evaluate @a call and return 1 from the enclosing function if the + * caller asked us to stop extracting. + */ +#define CHECK(call) do { if (0 != (call)) return 1; } while (0) + +/** + * Largest VMDK text descriptor we will read. Real descriptors are a + * few hundred bytes; the header field giving the size is attacker + * controlled, so it needs a hard cap. + */ +#define VMDK_MAX_DESCRIPTOR (32 * 1024) + +/** + * Largest number of lines we look at in a VMDK descriptor. + */ +#define VMDK_MAX_LINES 1024 + +/** + * Largest number of region table entries [MS-VHDX] allows. + */ +#define VHDX_MAX_REGIONS 2047 + +/** + * Largest number of metadata table entries [MS-VHDX] allows. + */ +#define VHDX_MAX_METADATA 2047 + +/** + * Largest single VHDX metadata item we will read. The largest one we + * care about is the parent locator; everything else is 4 to 16 bytes. + */ +#define VHDX_MAX_ITEM (16 * 1024) + +/** + * Seconds between the VHD epoch (2000-01-01T00:00:00Z) and the Unix + * epoch. + */ +#define VHD_EPOCH 946684800LL + + +/** + * Compare a NUL-terminated key from a file against a literal, ignoring + * ASCII case. Not `strcasecmp()' because that is locale-dependent. + * + * @param key NUL-terminated key from the file + * @param want the literal to compare against + * @return 1 if they match, 0 if not + */ +static int +key_is (const char *key, + const char *want) +{ + size_t i; + + for (i = 0; '\0' != want[i]; i++) + { + char a = key[i]; + char b = want[i]; + + if ( ('A' <= a) && ('Z' >= a) ) + a = (char) (a - 'A' + 'a'); + if ( ('A' <= b) && ('Z' >= b) ) + b = (char) (b - 'A' + 'a'); + if (a != b) + return 0; + } + return ('\0' == key[i]); +} + + +/** + * Copy @a src into @a dst, truncating if it does not fit. + * + * @param dst destination buffer + * @param dst_size number of bytes in @a dst, including the NUL + * @param src NUL-terminated source string + */ +static void +copy_bounded (char *dst, + size_t dst_size, + const char *src) +{ + size_t i; + + for (i = 0; (i + 1 < dst_size) && ('\0' != src[i]); i++) + dst[i] = src[i]; + dst[i] = '\0'; +} + + +/** + * Strip leading and trailing white space from @a s in place. + * + * @param s the string to trim, modified in place + * @return pointer into @a s at the first non-blank character + */ +static char * +trim_inplace (char *s) +{ + size_t len; + + while ( (' ' == *s) || + ('\t' == *s) || + ('\r' == *s) ) + s++; + len = strlen (s); + while ( (0 < len) && + ( (' ' == s[len - 1]) || + ('\t' == s[len - 1]) || + ('\r' == s[len - 1]) ) ) + len--; + s[len] = '\0'; + return s; +} + + +/** + * Remove one layer of double quotes from @a s, in place. + * + * @param s the string, modified in place + * @return pointer to the unquoted string + */ +static char * +unquote_inplace (char *s) +{ + size_t len = strlen (s); + + if ( (2 <= len) && + ('"' == s[0]) && + ('"' == s[len - 1]) ) + { + s[len - 1] = '\0'; + return &s[1]; + } + return s; +} + + +/** + * Parse an unsigned decimal number, rejecting anything that is not + * exactly a number. + * + * @param s the string + * @param[out] value where to store the result + * @return 1 on success, 0 if @a s is not a bounded decimal number + */ +static int +parse_u64 (const char *s, + uint64_t *value) +{ + uint64_t v = 0; + int digits = 0; + + for (size_t i = 0; '\0' != s[i]; i++) + { + if ( ('0' > s[i]) || + ('9' < s[i]) ) + return 0; + if (v > (UINT64_MAX - 9) / 10) + return 0; /* would overflow */ + v = v * 10 + (uint64_t) (s[i] - '0'); + digits++; + if (20 < digits) + return 0; + } + if (0 == digits) + return 0; + *value = v; + return 1; +} + + +/** + * Emit the MIME type of the format we just identified. + * + * @param ec extraction context + * @param mime the MIME type + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_mime (struct EXTRACTOR_ExtractContext *ec, + const char *mime) +{ + return (0 != ec->proc (ec->cls, + DISKIMAGE, + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + mime, + strlen (mime) + 1)) ? 1 : 0; +} + + +/* ------------------------------------------------------------------ */ +/* QCOW / QCOW2 */ +/* ------------------------------------------------------------------ */ + + +/** + * Report the QCOW2 version 3 feature bits that say something about the + * state the image was left in. + * + * @param ec extraction context + * @param incompatible the incompatible feature bit field + * @param compatible the compatible feature bit field + * @param autoclear the autoclear feature bit field + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +qcow_features (struct EXTRACTOR_ExtractContext *ec, + uint64_t incompatible, + uint64_t compatible, + uint64_t autoclear) +{ + /* Bit 0 is left set while the image is open for writing: seeing it + in a file at rest means the writer died without closing it. Bit 1 + is set by QEMU itself when it detects inconsistent metadata. */ + if (0 != (incompatible & 1ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "dirty bit set" + " (image was not closed cleanly)")); + if (0 != (incompatible & 2ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "corrupt bit set" + " (image marked corrupt by QEMU)")); + if (0 != (incompatible & 4ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "data is in an external data file")); + if (0 != (incompatible & 16ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "extended L2 entries")); + if (0 != (compatible & 1ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "lazy refcounts")); + if (0 != (autoclear & 1ULL)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "persistent dirty bitmaps")); + return 0; +} + + +/** + * Extract from a QCOW or QCOW2 image. The magic has already matched. + * + * @param ec extraction context + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_qcow (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char hdr[112]; + size_t have; + uint32_t version; + uint64_t backing_offset; + uint32_t backing_size; + uint64_t size; + + memset (hdr, + 0, + sizeof (hdr)); + /* A version 3 header is 104 or 112 bytes, a version 2 header 72 and + a version 1 header 48; take the largest that the file holds. */ + if (EXTRACTOR_forensic_read_ (ec, 0, hdr, sizeof (hdr))) + have = sizeof (hdr); + else if (EXTRACTOR_forensic_read_ (ec, 0, hdr, 72)) + have = 72; + else if (EXTRACTOR_forensic_read_ (ec, 0, hdr, 48)) + have = 48; + else + return 0; /* magic matched but there is no header behind it */ + version = EXTRACTOR_forensic_be32_ (&hdr[4]); + if ( (0 == version) || + (3 < version) ) + return 0; /* not a QCOW generation we know how to read */ + CHECK (emit_mime (ec, + "application/x-qemu-disk")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + "%s", + (1 == version) ? "QCOW" : "QCOW2")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u", + (unsigned int) version)); + /* The backing file reference and the virtual size sit at the same + place in all three generations. */ + size = EXTRACTOR_forensic_be64_ (&hdr[24]); + if ( (0 != size) && + (size < (1ULL << 62)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SIZE, + size)); + backing_offset = EXTRACTOR_forensic_be64_ (&hdr[8]); + backing_size = EXTRACTOR_forensic_be32_ (&hdr[16]); + if ( (0 != backing_offset) && + (0 != backing_size) && + (EXTRACTOR_FORENSIC_MAX_STRING >= backing_size) && + (backing_offset <= (uint64_t) INT64_MAX - backing_size) ) + { + char name[EXTRACTOR_FORENSIC_MAX_STRING]; + + if (EXTRACTOR_forensic_read_ (ec, + (int64_t) backing_offset, + name, + backing_size)) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + name, + backing_size)); + } + if ( (1 == version) || + (72 > have) ) + return 0; /* everything below moved in, or arrived with, version 2 */ + { + uint32_t cluster_bits = EXTRACTOR_forensic_be32_ (&hdr[20]); + uint32_t crypt = EXTRACTOR_forensic_be32_ (&hdr[32]); + uint32_t snapshots = EXTRACTOR_forensic_be32_ (&hdr[60]); + + /* 9..21 is what QEMU itself accepts; outside that the field is + either garbage or would make 1 << cluster_bits undefined. */ + if ( (9 <= cluster_bits) && + (21 >= cluster_bits) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_BLOCK_SIZE, + 1ULL << cluster_bits)); + if (0 != crypt) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ENCRYPTION, + "%s", + (1 == crypt) + ? "AES" + : ( (2 == crypt) + ? "LUKS" + : "unknown"))); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) snapshots)); + } + if ( (3 == version) && + (104 <= have) ) + { + uint64_t incompatible = EXTRACTOR_forensic_be64_ (&hdr[72]); + uint64_t compatible = EXTRACTOR_forensic_be64_ (&hdr[80]); + uint64_t autoclear = EXTRACTOR_forensic_be64_ (&hdr[88]); + uint32_t header_length = EXTRACTOR_forensic_be32_ (&hdr[100]); + + CHECK (qcow_features (ec, + incompatible, + compatible, + autoclear)); + /* The compression type byte only exists when the matching + incompatible bit is set; otherwise the image is plain zlib. */ + if ( (0 != (incompatible & 8ULL)) && + (104 < header_length) && + (105 <= have) ) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "compression type: %s", + (0 == hdr[104]) + ? "zlib" + : ( (1 == hdr[104]) + ? "zstd" + : "unknown"))); + } + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* VMDK */ +/* ------------------------------------------------------------------ */ + + +/** + * Parse one `RW <sectors> <type> "<file>"' extent line of a VMDK text + * descriptor. + * + * @param ec extraction context + * @param line the line, NUL-terminated, modified in place + * @param[in,out] extents number of extents seen so far + * @param[in,out] sectors total number of sectors seen so far + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vmdk_extent_line (struct EXTRACTOR_ExtractContext *ec, + char *line, + unsigned int *extents, + uint64_t *sectors) +{ + char *p = line; + char *count; + uint64_t n; + + (*extents)++; + while ( ('\0' != *p) && + (' ' != *p) ) + p++; /* skip the access mode */ + while (' ' == *p) + p++; + count = p; + while ( ('0' <= *p) && + ('9' >= *p) ) + p++; + if (' ' == *p) + { + *p = '\0'; + p++; + if ( (parse_u64 (count, + &n)) && + (n < (1ULL << 53)) ) + *sectors += n; + while (' ' == *p) + p++; + while ( ('\0' != *p) && + (' ' != *p) ) + p++; /* skip the extent type */ + while (' ' == *p) + p++; + } + if ('"' == *p) + { + char *end; + + p++; + end = p; + while ( ('\0' != *end) && + ('"' != *end) ) + end++; + if ('"' == *end) + { + *end = '\0'; + /* The extent files are sibling files an investigator has to + collect as well, so name them -- but only the first few. */ + if (EXTRACTOR_FORENSIC_MAX_ITEMS >= *extents) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FILENAME, + p, + strlen (p))); + } + } + return 0; +} + + +/** + * Parse the text descriptor of a VMDK, either the copy embedded in a + * sparse extent header or a stand-alone descriptor file. + * + * @param ec extraction context + * @param text the descriptor, modified in place; need not be + * NUL-terminated, @a len bounds it + * @param len number of bytes in @a text + * @param sparse 1 if this is the copy embedded in a sparse extent + * header, whose binary fields already gave us the version and + * the virtual size; 0 for a stand-alone descriptor file, where + * both have to come from the text + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vmdk_descriptor (struct EXTRACTOR_ExtractContext *ec, + char *text, + size_t len, + int sparse) +{ + char parent_cid[64]; + char parent_hint[EXTRACTOR_FORENSIC_MAX_STRING]; + char adapter[64]; + uint64_t cylinders = 0; + uint64_t heads = 0; + uint64_t track_sectors = 0; + uint64_t total_sectors = 0; + unsigned int extents = 0; + unsigned int lines = 0; + unsigned int keys = 0; + size_t pos = 0; + + parent_cid[0] = '\0'; + parent_hint[0] = '\0'; + adapter[0] = '\0'; + /* A NUL inside the descriptor ends it: everything past it is the + zero padding of the descriptor slot. */ + for (size_t i = 0; i < len; i++) + if ('\0' == text[i]) + { + len = i; + break; + } + while ( (pos < len) && + (VMDK_MAX_LINES > lines) ) + { + char *line; + char *eq; + size_t end = pos; + + lines++; + while ( (end < len) && + ('\n' != text[end]) ) + end++; + line = &text[pos]; + text[end] = '\0'; /* safe: either the '\n' or the end of the slot */ + pos = end + 1; + line = trim_inplace (line); + if ( ('\0' == line[0]) || + ('#' == line[0]) ) + continue; + if ( (0 == strncmp (line, "RW ", 3)) || + (0 == strncmp (line, "RDONLY ", 7)) || + (0 == strncmp (line, "NOACCESS ", 9)) ) + { + CHECK (vmdk_extent_line (ec, + line, + &extents, + &total_sectors)); + continue; + } + eq = strchr (line, + '='); + if (NULL == eq) + continue; + *eq = '\0'; + { + char *key = trim_inplace (line); + char *value = unquote_inplace (trim_inplace (&eq[1])); + + if ('\0' == value[0]) + continue; + /* A descriptor that repeats a key a thousand times must not turn + into a thousand meta data items; a real one has about fifteen + keys, so stopping here loses nothing. */ + if (EXTRACTOR_FORENSIC_MAX_ITEMS <= keys) + continue; + keys++; + if (key_is (key, + "CID")) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SERIAL, + value, + strlen (value))); + else if (key_is (key, + "parentCID")) + copy_bounded (parent_cid, + sizeof (parent_cid), + value); + else if (key_is (key, + "parentFileNameHint")) + copy_bounded (parent_hint, + sizeof (parent_hint), + value); + else if ( (! sparse) && + (key_is (key, + "version")) ) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT_VERSION, + value, + strlen (value))); + else if (key_is (key, + "createType")) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + value, + strlen (value))); + else if (key_is (key, + "ddb.virtualHWVersion")) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_SOFTWARE_VERSION, + "virtual hardware version %.16s", + value)); + else if (key_is (key, + "ddb.toolsVersion")) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_SOFTWARE_VERSION, + "VMware Tools version %.16s", + value)); + else if (key_is (key, + "ddb.adapterType")) + copy_bounded (adapter, + sizeof (adapter), + value); + else if (key_is (key, + "ddb.geometry.cylinders")) + (void) parse_u64 (value, + &cylinders); + else if (key_is (key, + "ddb.geometry.heads")) + (void) parse_u64 (value, + &heads); + else if (key_is (key, + "ddb.geometry.sectors")) + (void) parse_u64 (value, + &track_sectors); + else if ( (0 == strncmp (key, "ddb.uuid", 8)) || + (key_is (key, "ddb.longContentID")) ) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SERIAL, + value, + strlen (value))); + } + } + if ('\0' != adapter[0]) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "adapter type: %.32s", + adapter)); + if ( (0 != cylinders) && + (0 != heads) && + (0 != track_sectors) ) + { + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "geometry: %llu cylinders," + " %llu heads, %llu sectors per track", + (unsigned long long) cylinders, + (unsigned long long) heads, + (unsigned long long) track_sectors)); + } + if (0 != extents) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + extents)); + /* A stand-alone descriptor has no binary header to take the virtual + size from, so add up the extents instead. */ + if ( (! sparse) && + (0 != total_sectors) && + (total_sectors < (1ULL << 53)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SIZE, + total_sectors * 512ULL)); + /* `ffffffff' is the documented "no parent" value. */ + if ( ('\0' != parent_cid[0]) && + (! key_is (parent_cid, + "ffffffff")) ) + { + if ('\0' != parent_hint[0]) + CHECK (EXTRACTOR_forensic_emit_text_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + parent_hint, + strlen (parent_hint))); + else + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + "parent CID %.32s", + parent_cid)); + } + return 0; +} + + +/** + * Extract from a VMDK sparse extent. The `KDMV' magic has already + * matched. + * + * @param ec extraction context + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_vmdk_sparse (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char hdr[80]; + uint64_t capacity; + uint64_t grain; + uint64_t desc_offset; + uint64_t desc_size; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + sizeof (hdr))) + return 0; + CHECK (emit_mime (ec, + "application/x-vmdk")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + "%s", + "VMDK")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u", + (unsigned int) EXTRACTOR_forensic_le32_ ( + &hdr[4]))); + capacity = EXTRACTOR_forensic_le64_ (&hdr[12]); + if ( (0 != capacity) && + (capacity < (1ULL << 53)) ) + { + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SIZE, + capacity * 512ULL)); + } + grain = EXTRACTOR_forensic_le64_ (&hdr[20]); + if ( (0 != grain) && + (grain < (1ULL << 32)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_BLOCK_SIZE, + grain * 512ULL)); + /* An unclean shutdown flag survives in the file, so it says the VM + was not powered down before the image was taken. */ + if (0 != hdr[72]) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "unclean shutdown")); + if (0 != EXTRACTOR_forensic_le16_ (&hdr[77])) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "%s", + "grains are DEFLATE compressed")); + desc_offset = EXTRACTOR_forensic_le64_ (&hdr[28]); + desc_size = EXTRACTOR_forensic_le64_ (&hdr[36]); + if ( (0 == desc_offset) || + (0 == desc_size) ) + return 0; /* descriptor lives in a separate file */ + if ( (desc_offset > (uint64_t) INT64_MAX / 512) || + (desc_size > VMDK_MAX_DESCRIPTOR / 512) ) + return 0; /* nonsense, or more than we are willing to read */ + { + size_t bytes = (size_t) (desc_size * 512); + char *text; + int ret; + + if (NULL == (text = malloc (bytes + 1))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (desc_offset * 512), + text, + bytes)) + { + free (text); + return 0; + } + text[bytes] = '\0'; + ret = vmdk_descriptor (ec, + text, + bytes, + 1); + free (text); + return ret; + } +} + + +/** + * Extract from a stand-alone VMDK text descriptor file. The + * `# Disk DescriptorFile' banner has already matched. + * + * @param ec extraction context + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_vmdk_text (struct EXTRACTOR_ExtractContext *ec) +{ + uint64_t fsize = ec->get_size (ec->cls); + size_t bytes; + char *text; + int ret; + + CHECK (emit_mime (ec, + "application/x-vmdk")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + "%s", + "VMDK")); + if ( (UINT64_MAX == fsize) || + (0 == fsize) ) + return 0; + bytes = (fsize > VMDK_MAX_DESCRIPTOR) ? VMDK_MAX_DESCRIPTOR : (size_t) fsize; + if (NULL == (text = malloc (bytes + 1))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + 0, + text, + bytes)) + { + free (text); + return 0; + } + text[bytes] = '\0'; + ret = vmdk_descriptor (ec, + text, + bytes, + 0); + free (text); + return ret; +} + + +/* ------------------------------------------------------------------ */ +/* VHD (Microsoft Virtual PC / Hyper-V version 1) */ +/* ------------------------------------------------------------------ */ + + +/** + * Read the dynamic disk header of a VHD and report what it says about + * the block size and, for a differencing disk, the parent. + * + * @param ec extraction context + * @param offset file offset of the dynamic disk header + * @param differencing 1 if the footer said this is a differencing disk + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vhd_dynamic_header (struct EXTRACTOR_ExtractContext *ec, + uint64_t offset, + int differencing) +{ + unsigned char dyn[576]; + unsigned char name[512]; + uint32_t block_size; + + if (offset > (uint64_t) INT64_MAX - sizeof (dyn)) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + dyn, + sizeof (dyn))) + return 0; + if (0 != memcmp (dyn, + "cxsparse", + 8)) + return 0; + block_size = EXTRACTOR_forensic_be32_ (&dyn[32]); + if ( (0 != block_size) && + (block_size <= (64 * 1024 * 1024)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_BLOCK_SIZE, + block_size)); + if (! differencing) + return 0; + CHECK (EXTRACTOR_forensic_emit_guid_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + &dyn[40], + 0)); + /* The parent name is UTF-16 *big* endian here, unlike everywhere + else in Windows; swap it so the shared helper can take it. */ + for (size_t i = 0; i + 1 < sizeof (name); i += 2) + { + name[i] = dyn[64 + i + 1]; + name[i + 1] = dyn[64 + i]; + } + CHECK (EXTRACTOR_forensic_emit_utf16le_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + name, + sizeof (name))); + return 0; +} + + +/** + * Extract from a VHD image given its 512 byte footer. + * + * @param ec extraction context + * @param foot the footer + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_vhd (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *foot) +{ + uint32_t format_version; + uint32_t host_os; + uint32_t disk_type; + uint64_t current_size; + uint64_t data_offset; + int64_t created; + + CHECK (emit_mime (ec, + "application/x-vhd")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + "%s", + "VHD")); + format_version = EXTRACTOR_forensic_be32_ (&foot[12]); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u.%u", + (unsigned int) (format_version >> 16), + (unsigned int) (format_version & 0xFFFF))); + /* The creator application is a four character code -- `vpc ' for + Virtual PC, `win ' for Hyper-V, `qemu', `tap ' for Xen, `d2v ' for + Disk2vhd -- followed by a major.minor version. */ + { + uint32_t cver = EXTRACTOR_forensic_be32_ (&foot[32]); + char app[5]; + + memcpy (app, + &foot[28], + 4); + app[4] = '\0'; + for (unsigned int i = 0; i < 4; i++) + if ( (0x20 > (unsigned char) app[i]) || + (0x7E < (unsigned char) app[i]) ) + app[i] = ' '; + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + "%s %u.%u", + app, + (unsigned int) (cver >> 16), + (unsigned int) (cver & 0xFFFF))); + } + host_os = EXTRACTOR_forensic_be32_ (&foot[36]); + if (0x5769326BU == host_os) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_AUTHORING_OS, + "%s", + "Windows")); + else if (0x4D616320U == host_os) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_AUTHORING_OS, + "%s", + "Macintosh")); + /* The time stamp counts seconds from 2000-01-01, not from the Unix + epoch. */ + created = (int64_t) EXTRACTOR_forensic_be32_ (&foot[24]); + CHECK (EXTRACTOR_forensic_emit_unix_time_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_CREATION_DATE, + created + VHD_EPOCH)); + current_size = EXTRACTOR_forensic_be64_ (&foot[48]); + if ( (0 != current_size) && + (current_size < (1ULL << 62)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SIZE, + current_size)); + CHECK (EXTRACTOR_forensic_emit_guid_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SERIAL, + &foot[68], + 0)); + disk_type = EXTRACTOR_forensic_be32_ (&foot[60]); + switch (disk_type) + { + case 2: + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "fixed disk")); + break; + case 3: + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "dynamic disk")); + break; + case 4: + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "differencing disk")); + break; + default: + break; + } + if (0 != (foot[84] & 1)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "saved state present (VM was suspended)")); + data_offset = EXTRACTOR_forensic_be64_ (&foot[16]); + if ( ( (3 == disk_type) || + (4 == disk_type) ) && + (0 != data_offset) && + (UINT64_MAX != data_offset) ) + return vhd_dynamic_header (ec, + data_offset, + 4 == disk_type); + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* VHDX */ +/* ------------------------------------------------------------------ */ + +/** + * Region table GUID of the VHDX metadata region, + * 8B7CA206-4790-4B9A-B8FE-575F050F886E, in file order. + */ +static const unsigned char vhdx_metadata_region[16] = { + 0x06, 0xA2, 0x7C, 0x8B, 0x90, 0x47, 0x9A, 0x4B, + 0xB8, 0xFE, 0x57, 0x5F, 0x05, 0x0F, 0x88, 0x6E +}; + +/** + * Metadata item GUID "File Parameters", + * CAA16737-FA36-4D43-B3B6-33F0AA44E76B, in file order. + */ +static const unsigned char vhdx_file_parameters[16] = { + 0x37, 0x67, 0xA1, 0xCA, 0x36, 0xFA, 0x43, 0x4D, + 0xB3, 0xB6, 0x33, 0xF0, 0xAA, 0x44, 0xE7, 0x6B +}; + +/** + * Metadata item GUID "Virtual Disk Size", + * 2FA54224-CD1B-4876-B211-5DBED83BF4B8, in file order. + */ +static const unsigned char vhdx_virtual_disk_size[16] = { + 0x24, 0x42, 0xA5, 0x2F, 0x1B, 0xCD, 0x76, 0x48, + 0xB2, 0x11, 0x5D, 0xBE, 0xD8, 0x3B, 0xF4, 0xB8 +}; + +/** + * Metadata item GUID "Virtual Disk ID", + * BECA12AB-B2E6-4523-93EF-C309E000C746, in file order. + */ +static const unsigned char vhdx_virtual_disk_id[16] = { + 0xAB, 0x12, 0xCA, 0xBE, 0xE6, 0xB2, 0x23, 0x45, + 0x93, 0xEF, 0xC3, 0x09, 0xE0, 0x00, 0xC7, 0x46 +}; + +/** + * Metadata item GUID "Logical Sector Size", + * 8141BF1D-A96F-4709-BA47-F233A8FAAB5F, in file order. + */ +static const unsigned char vhdx_logical_sector_size[16] = { + 0x1D, 0xBF, 0x41, 0x81, 0x6F, 0xA9, 0x09, 0x47, + 0xBA, 0x47, 0xF2, 0x33, 0xA8, 0xFA, 0xAB, 0x5F +}; + +/** + * Metadata item GUID "Physical Sector Size", + * CDA348C7-445D-4471-9CC9-E9885251C556, in file order. + */ +static const unsigned char vhdx_physical_sector_size[16] = { + 0xC7, 0x48, 0xA3, 0xCD, 0x5D, 0x44, 0x71, 0x44, + 0x9C, 0xC9, 0xE9, 0x88, 0x52, 0x51, 0xC5, 0x56 +}; + +/** + * Metadata item GUID "Parent Locator", + * A8D35F2D-B30B-454D-ABF7-D3D84834AB0C, in file order. + */ +static const unsigned char vhdx_parent_locator[16] = { + 0x2D, 0x5F, 0xD3, 0xA8, 0x0B, 0xB3, 0x4D, 0x45, + 0xAB, 0xF7, 0xD3, 0xD8, 0x48, 0x34, 0xAB, 0x0C +}; + + +/** + * Report what the VHDX parent locator says about the parent image. + * + * The locator is a small table of UTF-16LE key/value pairs; the keys + * that matter are `relative_path', `volume_path' and + * `absolute_win32_path', each of which names the file the differencing + * image depends on. + * + * @param ec extraction context + * @param item the parent locator metadata item + * @param len number of bytes in @a item + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vhdx_parent_paths (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *item, + size_t len) +{ + uint16_t count; + + if (20 > len) + return 0; + count = EXTRACTOR_forensic_le16_ (&item[18]); + if (EXTRACTOR_FORENSIC_MAX_ITEMS < count) + count = EXTRACTOR_FORENSIC_MAX_ITEMS; + for (uint16_t i = 0; i < count; i++) + { + size_t base = 20 + (size_t) i * 12; + uint32_t key_offset; + uint32_t value_offset; + uint16_t key_len; + uint16_t value_len; + char key[64]; + + if (base + 12 > len) + break; + key_offset = EXTRACTOR_forensic_le32_ (&item[base]); + value_offset = EXTRACTOR_forensic_le32_ (&item[base + 4]); + key_len = EXTRACTOR_forensic_le16_ (&item[base + 8]); + value_len = EXTRACTOR_forensic_le16_ (&item[base + 10]); + if ( (key_offset > len) || + (key_len > len - key_offset) || + (value_offset > len) || + (value_len > len - value_offset) ) + continue; /* the entry points outside the item */ + if ( (0 == value_len) || + (0 == key_len) || + (sizeof (key) * 2 <= key_len) ) + continue; + /* The keys are ASCII spelled in UTF-16LE; fold them down by hand + so we can compare them without another conversion buffer. */ + { + size_t out = 0; + + for (size_t k = 0; k + 1 < key_len; k += 2) + { + if (0 != item[key_offset + k + 1]) + { + out = 0; + break; /* not ASCII, so not a key we know */ + } + key[out++] = (char) item[key_offset + k]; + } + if (0 == out) + continue; + key[out] = '\0'; + } + if ( (key_is (key, + "relative_path")) || + (key_is (key, + "volume_path")) || + (key_is (key, + "absolute_win32_path")) ) + CHECK (EXTRACTOR_forensic_emit_utf16le_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_PARENT_IMAGE, + &item[value_offset], + value_len)); + } + return 0; +} + + +/** + * Read one VHDX metadata item and report it. + * + * @param ec extraction context + * @param guid the item GUID from the metadata table + * @param offset absolute file offset of the item + * @param len number of bytes in the item + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vhdx_metadata_item (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *guid, + uint64_t offset, + uint32_t len) +{ + unsigned char item[64]; + + if (0 == memcmp (guid, + vhdx_virtual_disk_size, + sizeof (vhdx_virtual_disk_size))) + { + uint64_t size; + + if ( (8 > len) || + (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + item, + 8)) ) + return 0; + size = EXTRACTOR_forensic_le64_ (item); + if ( (0 != size) && + (size < (1ULL << 62)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SIZE, + size)); + return 0; + } + if (0 == memcmp (guid, + vhdx_virtual_disk_id, + sizeof (vhdx_virtual_disk_id))) + { + if ( (16 > len) || + (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + item, + 16)) ) + return 0; + return EXTRACTOR_forensic_emit_guid_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_VOLUME_SERIAL, + item, + 1); + } + if (0 == memcmp (guid, + vhdx_file_parameters, + sizeof (vhdx_file_parameters))) + { + uint32_t block_size; + uint32_t flags; + + if ( (8 > len) || + (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + item, + 8)) ) + return 0; + block_size = EXTRACTOR_forensic_le32_ (item); + flags = EXTRACTOR_forensic_le32_ (&item[4]); + if ( (0 != block_size) && + (block_size <= (256 * 1024 * 1024)) ) + CHECK (EXTRACTOR_forensic_emit_size_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_BLOCK_SIZE, + block_size)); + if (0 != (flags & 1U)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "fixed disk (blocks left allocated)")); + if (0 != (flags & 2U)) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + "differencing disk")); + return 0; + } + if ( (0 == memcmp (guid, + vhdx_logical_sector_size, + sizeof (vhdx_logical_sector_size))) || + (0 == memcmp (guid, + vhdx_physical_sector_size, + sizeof (vhdx_physical_sector_size))) ) + { + int logical = (0 == memcmp (guid, + vhdx_logical_sector_size, + sizeof (vhdx_logical_sector_size))); + + if ( (4 > len) || + (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + item, + 4)) ) + return 0; + return EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "%s sector size: %u bytes", + logical ? "logical" : "physical", + (unsigned int) EXTRACTOR_forensic_le32_ ( + item)); + } + if (0 == memcmp (guid, + vhdx_parent_locator, + sizeof (vhdx_parent_locator))) + { + unsigned char *locator; + int ret; + + if ( (20 > len) || + (VHDX_MAX_ITEM < len) ) + return 0; + if (NULL == (locator = malloc (len))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + locator, + len)) + { + free (locator); + return 0; + } + ret = vhdx_parent_paths (ec, + locator, + len); + free (locator); + return ret; + } + return 0; +} + + +/** + * Walk the VHDX metadata table and report the items we understand. + * + * @param ec extraction context + * @param region_offset file offset of the metadata region + * @param region_length number of bytes in the metadata region + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vhdx_metadata_table (struct EXTRACTOR_ExtractContext *ec, + uint64_t region_offset, + uint32_t region_length) +{ + unsigned char head[32]; + unsigned char *table; + uint16_t count; + size_t table_bytes; + int ret = 0; + + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) region_offset, + head, + sizeof (head))) + return 0; + if (0 != memcmp (head, + "metadata", + 8)) + return 0; + count = EXTRACTOR_forensic_le16_ (&head[10]); + if (VHDX_MAX_METADATA < count) + return 0; /* more than the format allows: do not trust the table */ + if (0 == count) + return 0; + table_bytes = (size_t) count * 32; + if (region_length < 32 + table_bytes) + return 0; + if (NULL == (table = malloc (table_bytes))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (region_offset + 32), + table, + table_bytes)) + { + free (table); + return 0; + } + for (uint16_t i = 0; i < count; i++) + { + const unsigned char *e = &table[(size_t) i * 32]; + uint32_t item_offset = EXTRACTOR_forensic_le32_ (&e[16]); + uint32_t item_len = EXTRACTOR_forensic_le32_ (&e[20]); + + if ( (0 == item_len) || + (VHDX_MAX_ITEM < item_len) ) + continue; + /* The item offset is relative to the start of the region and both + it and the item have to stay inside it. */ + if ( (item_offset < 32) || + (item_offset > region_length) || + (item_len > region_length - item_offset) ) + continue; + if (0 != vhdx_metadata_item (ec, + e, + region_offset + item_offset, + item_len)) + { + ret = 1; + break; + } + } + free (table); + return ret; +} + + +/** + * Walk a VHDX region table looking for the metadata region. + * + * @param ec extraction context + * @param offset file offset of the region table (192KB or 256KB) + * @param fsize size of the file, UINT64_MAX if unknown + * @param[out] found set to 1 if the metadata region was reported + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +vhdx_region_table (struct EXTRACTOR_ExtractContext *ec, + uint64_t offset, + uint64_t fsize, + int *found) +{ + unsigned char head[16]; + unsigned char *entries; + uint32_t count; + int ret = 0; + + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + head, + sizeof (head))) + return 0; + if (0 != memcmp (head, + "regi", + 4)) + return 0; + count = EXTRACTOR_forensic_le32_ (&head[8]); + if ( (0 == count) || + (VHDX_MAX_REGIONS < count) ) + return 0; + if (NULL == (entries = malloc ((size_t) count * 32))) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (offset + 16), + entries, + (size_t) count * 32)) + { + free (entries); + return 0; + } + for (uint32_t i = 0; i < count; i++) + { + const unsigned char *e = &entries[(size_t) i * 32]; + uint64_t region_offset; + uint32_t region_length; + + if (0 != memcmp (e, + vhdx_metadata_region, + sizeof (vhdx_metadata_region))) + continue; + region_offset = EXTRACTOR_forensic_le64_ (&e[16]); + region_length = EXTRACTOR_forensic_le32_ (&e[24]); + if ( (0 == region_offset) || + (0 == region_length) || + (region_offset > (uint64_t) INT64_MAX - region_length) ) + continue; + if ( (UINT64_MAX != fsize) && + (region_offset + region_length > fsize) ) + continue; /* truncated file: the region is not there */ + *found = 1; + ret = vhdx_metadata_table (ec, + region_offset, + region_length); + break; + } + free (entries); + return ret; +} + + +/** + * Extract from a VHDX image. The `vhdxfile' signature has already + * matched. + * + * @param ec extraction context + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_vhdx (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char creator[512]; + unsigned char head[80]; + uint64_t best_sequence = 0; + int have_head = 0; + uint64_t fsize; + int found = 0; + + memset (head, + 0, + sizeof (head)); + CHECK (emit_mime (ec, + "application/x-vhdx")); + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT, + "%s", + "VHDX")); + if (EXTRACTOR_forensic_read_ (ec, + 8, + creator, + sizeof (creator))) + CHECK (EXTRACTOR_forensic_emit_utf16le_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + creator, + sizeof (creator))); + /* Two copies of the header live at 64KB and 128KB; the one with the + higher sequence number is current. We do not verify the CRC-32C + the format puts on each of them, so a corrupt header is reported + as if it were good. */ + for (unsigned int i = 0; i < 2; i++) + { + unsigned char candidate[80]; + uint64_t sequence; + + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (0x10000 + i * 0x10000), + candidate, + sizeof (candidate))) + continue; + if (0 != memcmp (candidate, + "head", + 4)) + continue; + sequence = EXTRACTOR_forensic_le64_ (&candidate[8]); + if ( (have_head) && + (sequence < best_sequence) ) + continue; + memcpy (head, + candidate, + sizeof (head)); + best_sequence = sequence; + have_head = 1; + } + if (have_head) + { + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u", + (unsigned int) + EXTRACTOR_forensic_le16_ (&head[66]))); + /* A non-zero LogGuid means the log holds entries that were never + replayed into the image: the writer did not shut down. */ + { + int empty = 1; + + for (unsigned int i = 0; i < 16; i++) + if (0 != head[48 + i]) + { + empty = 0; + break; + } + if (! empty) + CHECK (EXTRACTOR_forensic_emit_ (ec, + DISKIMAGE, + EXTRACTOR_METATYPE_COMMENT, + "%s", + "unflushed log present" + " (image was not shut down cleanly)")); + } + } + fsize = ec->get_size (ec->cls); + /* The region table sits at a fixed 192KB, with a backup at 256KB. */ + CHECK (vhdx_region_table (ec, + 0x30000, + fsize, + &found)); + if (! found) + CHECK (vhdx_region_table (ec, + 0x40000, + fsize, + &found)); + return 0; +} + + +/* ------------------------------------------------------------------ */ + + +/** + * Main entry method for the virtual disk image extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_diskimage_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_diskimage_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char magic[16]; + unsigned char foot[512]; + uint64_t fsize; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + magic, + sizeof (magic))) + return; + if (0 == memcmp (magic, + "QFI\xfb", + 4)) + { + (void) extract_qcow (ec); + return; + } + if (0 == memcmp (magic, + "KDMV", + 4)) + { + (void) extract_vmdk_sparse (ec); + return; + } + if (0 == memcmp (magic, + "# Disk Descripto", + 16)) + { + (void) extract_vmdk_text (ec); + return; + } + if (0 == memcmp (magic, + "vhdxfile", + 8)) + { + (void) extract_vhdx (ec); + return; + } + if (0 == memcmp (magic, + "conectix", + 8)) + { + /* A dynamic or differencing VHD mirrors its footer at the front, + so we already have it and need no seek. */ + if (EXTRACTOR_forensic_read_ (ec, + 0, + foot, + sizeof (foot))) + (void) extract_vhd (ec, + foot); + return; + } + /* Only a fixed VHD is left, and its footer is at the very end. That + costs a seek on every file we were handed, so pay it only for the + files that could be one: a VHD is a whole number of 512 byte + sectors plus the footer, hence always 512 byte aligned. (Virtual + PC 2004 and older wrote a 511 byte footer; those are not + recognised.) */ + fsize = ec->get_size (ec->cls); + if ( (UINT64_MAX == fsize) || + (1024 > fsize) || + ((uint64_t) INT64_MAX < fsize) || + (0 != (fsize % 512)) ) + return; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (fsize - 512), + foot, + sizeof (foot))) + return; + if (0 != memcmp (foot, + "conectix", + 8)) + return; + (void) extract_vhd (ec, + foot); +} + + +/* end of diskimage_extractor.c */ diff --git a/src/plugins/ebook_extractor.c b/src/plugins/ebook_extractor.c @@ -0,0 +1,1403 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/ebook_extractor.c + * @brief plugin to support EPUB and MOBI/AZW electronic books + * @author Christian Grothoff + * + * Two unrelated containers share this plugin because they answer the + * same question. An EPUB is a ZIP whose OPF package document carries + * Dublin Core; a MOBI is a Palm database whose EXTH header carries the + * same fields plus, on files that came from a store, a watermark. What + * is worth having here is not the title but the provenance: which tool + * built the file, when, for whom. + * + * References: + * - EPUB 3 OCF and Package Document, https://www.w3.org/TR/epub-33/ + * - MOBI format, https://wiki.mobileread.com/wiki/MOBI + * - PalmDOC / PDB, https://wiki.mobileread.com/wiki/PDB + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" +#include "unzip.h" + + +/** + * Name we report our meta data under. + */ +#define PLUGIN_NAME "ebook" + +/** + * Never read more than this from `META-INF/container.xml'. It holds a + * handful of paths; anything larger is padding. + */ +#define MAX_CONTAINER (64 * 1024) + +/** + * Never read more than this from the OPF package document. Real ones + * are a few kilobytes plus the manifest. + */ +#define MAX_OPF (256 * 1024) + +/** + * Never read more than this from record 0 of a Palm database. The MOBI + * and EXTH headers together are a few kilobytes. + */ +#define MAX_RECORD0 (64 * 1024) + +/** + * Upper bound on the number of EXTH records we will walk. + */ +#define MAX_EXTH 1024 + +/** + * Upper bound on the number of XML elements we will walk in the OPF. + */ +#define MAX_ELEMENTS 8192 + +/** + * Seconds between the Palm epoch (1904-01-01) and the Unix epoch. + */ +#define PALM_EPOCH_OFFSET 2082844800LL + + +/** + * Find @a needle in @a hay. `memmem()' is a GNU extension, and this is + * only ever run over buffers of a few hundred kilobytes. + * + * @param hay buffer to search + * @param hlen number of bytes in @a hay + * @param needle bytes to look for + * @param nlen number of bytes in @a needle + * @return pointer into @a hay, NULL if @a needle does not occur + */ +static const char * +mem_find (const char *hay, + size_t hlen, + const char *needle, + size_t nlen) +{ + if ( (0 == nlen) || + (nlen > hlen) ) + return NULL; + for (size_t i = 0; i + nlen <= hlen; i++) + if (0 == memcmp (&hay[i], + needle, + nlen)) + return &hay[i]; + return NULL; +} + + +/** + * Is @a c a character that may appear in an XML name? + * + * @param c character to test + * @return 1 if @a c may be part of a name + */ +static int +is_name_char (char c) +{ + return ( ( ('a' <= c) && ('z' >= c) ) || + ( ('A' <= c) && ('Z' >= c) ) || + ( ('0' <= c) && ('9' >= c) ) || + ('_' == c) || ('-' == c) || ('.' == c) || (':' == c) ); +} + + +/** + * Is @a c XML white space? + * + * @param c character to test + * @return 1 if @a c is white space + */ +static int +is_space (char c) +{ + return (' ' == c) || ('\t' == c) || ('\r' == c) || ('\n' == c); +} + + +/** + * Append @a cp to @a out as UTF-8. + * + * @param out output buffer + * @param outsz number of bytes available in @a out + * @param pos current write position + * @param cp code point to append + * @return new write position, unchanged if @a cp did not fit + */ +static size_t +utf8_put (char *out, + size_t outsz, + size_t pos, + uint32_t cp) +{ + if ( (cp > 0x10FFFF) || + ( (0xD800 <= cp) && (0xDFFF >= cp) ) ) + return pos; + if (cp < 0x80) + { + if (pos + 1 > outsz) + return pos; + out[pos++] = (char) cp; + } + else if (cp < 0x800) + { + if (pos + 2 > outsz) + return pos; + out[pos++] = (char) (0xC0 | (cp >> 6)); + out[pos++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + if (pos + 3 > outsz) + return pos; + out[pos++] = (char) (0xE0 | (cp >> 12)); + out[pos++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[pos++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + if (pos + 4 > outsz) + return pos; + out[pos++] = (char) (0xF0 | (cp >> 18)); + out[pos++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[pos++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[pos++] = (char) (0x80 | (cp & 0x3F)); + } + return pos; +} + + +/** + * Copy XML character data, resolving the five predefined entities and + * numeric character references and collapsing runs of white space. + * + * The OPF fields we read are laid out across indented lines, so without + * the white space collapsing every value would come back with the + * source file's line breaks embedded in it. + * + * @param data the character data + * @param len number of bytes in @a data + * @param[out] out where to write the result, NUL-terminated + * @param outsz number of bytes available in @a out, at least 2 + * @return number of bytes written, not counting the NUL + */ +static size_t +xml_text (const char *data, + size_t len, + char *out, + size_t outsz) +{ + size_t o = 0; + size_t i = 0; + int pending_space = 0; + + while ( (i < len) && + (o + 1 < outsz) ) + { + if (is_space (data[i])) + { + if (0 != o) + pending_space = 1; + i++; + continue; + } + if (pending_space) + { + out[o++] = ' '; + pending_space = 0; + if (o + 1 >= outsz) + break; + } + if ('&' != data[i]) + { + out[o++] = data[i++]; + continue; + } + { + const char *semi = memchr (&data[i], + ';', + len - i); + size_t elen; + + if ( (NULL == semi) || + ( (size_t) (semi - &data[i]) > 12) ) + { + out[o++] = data[i++]; /* a bare '&', which is not legal XML + but does occur; pass it through */ + continue; + } + elen = (size_t) (semi - &data[i]) - 1; /* between '&' and ';' */ + if ( (3 == elen) && (0 == memcmp (&data[i + 1], "amp", 3)) ) + out[o++] = '&'; + else if ( (2 == elen) && (0 == memcmp (&data[i + 1], "lt", 2)) ) + out[o++] = '<'; + else if ( (2 == elen) && (0 == memcmp (&data[i + 1], "gt", 2)) ) + out[o++] = '>'; + else if ( (4 == elen) && (0 == memcmp (&data[i + 1], "quot", 4)) ) + out[o++] = '"'; + else if ( (4 == elen) && (0 == memcmp (&data[i + 1], "apos", 4)) ) + out[o++] = '\''; + else if ( (2 <= elen) && ('#' == data[i + 1]) ) + { + uint32_t cp = 0; + size_t k = i + 2; + int base = 10; + + if ( ('x' == data[k]) || ('X' == data[k]) ) + { + base = 16; + k++; + } + for (; k < (size_t) (semi - data); k++) + { + unsigned int d; + + if ( ('0' <= data[k]) && ('9' >= data[k]) ) + d = (unsigned int) (data[k] - '0'); + else if ( (16 == base) && ('a' <= data[k]) && ('f' >= data[k]) ) + d = (unsigned int) (data[k] - 'a') + 10; + else if ( (16 == base) && ('A' <= data[k]) && ('F' >= data[k]) ) + d = (unsigned int) (data[k] - 'A') + 10; + else + { + cp = 0; + break; + } + if (cp > 0x10FFFF) + { + cp = 0; + break; + } + cp = cp * (uint32_t) base + d; + } + o = utf8_put (out, + outsz - 1, + o, + cp); + } + else + { + /* an entity we do not know; drop it rather than guess */ + } + i += elen + 2; + } + } + out[o] = '\0'; + return o; +} + + +/** + * Read the value of attribute @a name out of the body of a start tag. + * A namespace prefix on the attribute is ignored, so a request for + * `role' also matches `opf:role'. + * + * @param data the start tag body, after the element name + * @param len number of bytes in @a data + * @param name local name of the attribute + * @param[out] out where to write the value, NUL-terminated + * @param outsz number of bytes available in @a out, at least 2 + * @return 1 if the attribute was found, 0 if not + */ +static int +xml_attr (const char *data, + size_t len, + const char *name, + char *out, + size_t outsz) +{ + size_t nlen = strlen (name); + + out[0] = '\0'; + if (nlen >= len) + return 0; + for (size_t i = 0; i + nlen < len; i++) + { + size_t k; + char quote; + const char *vend; + + if (0 != memcmp (&data[i], + name, + nlen)) + continue; + /* the name must start where an attribute may start: right after + white space, or right after a namespace prefix that does */ + if (0 == i) + continue; + if (is_space (data[i - 1])) + { + /* fine */ + } + else if (':' == data[i - 1]) + { + size_t j = i - 1; + + while ( (0 < j) && + is_name_char (data[j - 1]) && + (':' != data[j - 1]) ) + j--; + if ( (0 == j) || + (! is_space (data[j - 1])) ) + continue; + } + else + { + continue; + } + k = i + nlen; + while ( (k < len) && + is_space (data[k]) ) + k++; + if ( (k >= len) || + ('=' != data[k]) ) + continue; + k++; + while ( (k < len) && + is_space (data[k]) ) + k++; + if ( (k >= len) || + ( ('"' != data[k]) && ('\'' != data[k]) ) ) + continue; + quote = data[k]; + k++; + vend = memchr (&data[k], + quote, + len - k); + if (NULL == vend) + return 0; + (void) xml_text (&data[k], + (size_t) (vend - &data[k]), + out, + outsz); + return 1; + } + return 0; +} + + +/** + * Does @a s, after an optional `urn:isbn:' prefix, look like an ISBN? + * + * @param s NUL-terminated candidate + * @return 1 if @a s is 10 or 13 digits with optional separators + */ +static int +looks_like_isbn (const char *s) +{ + size_t digits = 0; + + if (0 == strncasecmp (s, + "urn:isbn:", + strlen ("urn:isbn:"))) + s += strlen ("urn:isbn:"); + else if (0 == strncasecmp (s, + "isbn:", + strlen ("isbn:"))) + s += strlen ("isbn:"); + for (size_t i = 0; '\0' != s[i]; i++) + { + if ( ('-' == s[i]) || (' ' == s[i]) ) + continue; + if ( ('0' <= s[i]) && ('9' >= s[i]) ) + { + digits++; + continue; + } + if ( ( ('X' == s[i]) || ('x' == s[i]) ) && + (9 == digits) && + ('\0' == s[i + 1]) ) + { + digits++; + continue; + } + return 0; + } + return (10 == digits) || (13 == digits); +} + + +/** + * Read a whole member out of a ZIP archive. + * + * @param uf the archive + * @param name member to read + * @param cap never read more than this many bytes + * @param[out] len number of bytes read + * @return the member's contents with a NUL appended, NULL on error; + * caller must free + */ +static char * +read_member (struct EXTRACTOR_UnzipFile *uf, + const char *name, + size_t cap, + size_t *len) +{ + struct EXTRACTOR_UnzipFileInfo fi; + char *buf; + size_t size; + size_t got = 0; + + *len = 0; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_go_find_local_file (uf, + name, + 2)) + return NULL; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_get_current_file_info (uf, + &fi, + NULL, 0, + NULL, 0, + NULL, 0)) + return NULL; + size = (size_t) fi.uncompressed_size; + if (0 == size) + return NULL; + if (size > cap) + size = cap; + if (NULL == (buf = malloc (size + 1))) + return NULL; + if (EXTRACTOR_UNZIP_OK != + EXTRACTOR_common_unzip_open_current_file (uf)) + { + free (buf); + return NULL; + } + while (got < size) + { + ssize_t ret; + + ret = EXTRACTOR_common_unzip_read_current_file (uf, + &buf[got], + size - got); + if (0 >= ret) + break; /* error or end of member */ + if (((size_t) ret) > size - got) + break; /* cannot happen, but do not overrun if it does */ + got += (size_t) ret; + } + (void) EXTRACTOR_common_unzip_close_current_file (uf); + if (0 == got) + { + free (buf); + return NULL; + } + buf[got] = '\0'; + *len = got; + return buf; +} + + +/** + * Does the archive contain a member with this name? + * + * @param uf the archive + * @param name member to look for + * @return 1 if present + */ +static int +has_member (struct EXTRACTOR_UnzipFile *uf, + const char *name) +{ + return (EXTRACTOR_UNZIP_OK == + EXTRACTOR_common_unzip_go_find_local_file (uf, + name, + 2)); +} + + +/** + * Walk the `<metadata>' element of an OPF package document and report + * what it holds. + * + * This is a bounded text scan, not an XML parser: the OPF is untrusted + * input and everything we want out of it is a flat list of leaf + * elements, so pulling in a parser would buy nothing but attack surface. + * + * @param ec extraction context + * @param opf the package document + * @param len number of bytes in @a opf + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +parse_opf_metadata (struct EXTRACTOR_ExtractContext *ec, + const char *opf, + size_t len) +{ + char value[EXTRACTOR_FORENSIC_MAX_STRING]; + char attr[256]; + const char *mstart; + const char *mend; + const char *p; + unsigned int elements = 0; + /* the repeating elements are capped separately: a file with ten + thousand <dc:subject> entries is characterised well enough by the + first few dozen */ + unsigned int subjects = 0; + unsigned int identifiers = 0; + unsigned int people = 0; + + mstart = mem_find (opf, + len, + "<metadata", + strlen ("<metadata")); + if (NULL == mstart) + return 0; + mend = mem_find (mstart, + len - (size_t) (mstart - opf), + "</metadata", + strlen ("</metadata")); + if (NULL == mend) + mend = opf + len; + p = mstart; + while ( (p < mend) && + (elements++ < MAX_ELEMENTS) ) + { + const char *name; + const char *local; + const char *body; + const char *text; + size_t name_len; + size_t body_len; + size_t text_len; + int self_closing; + char quote = '\0'; + enum EXTRACTOR_MetaType type; + + p = memchr (p, + '<', + (size_t) (mend - p)); + if (NULL == p) + break; + p++; + if (p >= mend) + break; + if ( ('/' == *p) || ('?' == *p) || ('!' == *p) ) + continue; + name = p; + while ( (p < mend) && + is_name_char (*p) ) + p++; + name_len = (size_t) (p - name); + if (0 == name_len) + continue; + body = p; + while (p < mend) + { + if ('\0' != quote) + { + if (*p == quote) + quote = '\0'; + } + else if ( ('"' == *p) || ('\'' == *p) ) + { + quote = *p; + } + else if ('>' == *p) + { + break; + } + p++; + } + if (p >= mend) + break; + body_len = (size_t) (p - body); + self_closing = ( (0 < body_len) && ('/' == p[-1]) ); + if (self_closing) + body_len--; + p++; /* step past '>' */ + text = p; + text_len = 0; + if (! self_closing) + { + const char *e = memchr (p, + '<', + (size_t) (mend - p)); + + text_len = (NULL == e) + ? (size_t) (mend - p) + : (size_t) (e - p); + } + /* strip the namespace prefix; `dc:title' and `title' are the same + element for our purposes because we are already inside + <metadata> */ + local = name; + for (size_t i = 0; i < name_len; i++) + if (':' == name[i]) + local = &name[i + 1]; + name_len -= (size_t) (local - name); + +#define IS(s) ( (strlen (s) == name_len) && \ + (0 == memcmp (local, s, name_len)) ) + + if (IS ("meta")) + { + if ( (xml_attr (body, + body_len, + "name", + attr, + sizeof (attr))) && + (xml_attr (body, + body_len, + "content", + value, + sizeof (value))) ) + { + if (0 == strcasecmp (attr, + "calibre:timestamp")) + type = EXTRACTOR_METATYPE_CREATION_DATE; + else if ( (0 == strcasecmp (attr, + "generator")) || + (0 == strcasecmp (attr, + "calibre:generator")) ) + type = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE; + else + continue; + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + type, + value, + strlen (value))) + return 1; + } + else if (xml_attr (body, + body_len, + "property", + attr, + sizeof (attr))) + { + if (0 != strcasecmp (attr, + "dcterms:modified")) + continue; + (void) xml_text (text, + text_len, + value, + sizeof (value)); + if (0 != + EXTRACTOR_forensic_emit_text_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_MODIFICATION_DATE, + value, + strlen (value))) + return 1; + } + continue; + } + if (0 == text_len) + continue; + (void) xml_text (text, + text_len, + value, + sizeof (value)); + if ('\0' == value[0]) + continue; + if (IS ("title")) + type = EXTRACTOR_METATYPE_TITLE; + else if (IS ("creator")) + { + if (people++ >= EXTRACTOR_FORENSIC_MAX_ITEMS) + continue; + type = EXTRACTOR_METATYPE_AUTHOR_NAME; + } + else if (IS ("publisher")) + type = EXTRACTOR_METATYPE_PUBLISHER; + else if (IS ("language")) + type = EXTRACTOR_METATYPE_LANGUAGE; + else if (IS ("date")) + type = EXTRACTOR_METATYPE_PUBLICATION_DATE; + else if (IS ("subject")) + { + if (subjects++ >= EXTRACTOR_FORENSIC_MAX_ITEMS) + continue; + type = EXTRACTOR_METATYPE_KEYWORDS; + } + else if (IS ("description")) + type = EXTRACTOR_METATYPE_DESCRIPTION; + else if (IS ("rights")) + type = EXTRACTOR_METATYPE_RIGHTS; + else if (IS ("source")) + type = EXTRACTOR_METATYPE_URI; + else if (IS ("contributor")) + { + /* `bkp' is the MARC relator for the agent that made the file -- + which is the tool, and thus the provenance we are after */ + if ( (xml_attr (body, + body_len, + "role", + attr, + sizeof (attr))) && + (0 == strcasecmp (attr, + "bkp")) ) + { + type = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE; + } + else + { + if (people++ >= EXTRACTOR_FORENSIC_MAX_ITEMS) + continue; + type = EXTRACTOR_METATYPE_CONTRIBUTOR_NAME; + } + } + else if (IS ("identifier")) + { + int isbn = 0; + + if (identifiers++ >= EXTRACTOR_FORENSIC_MAX_ITEMS) + continue; + + if (xml_attr (body, + body_len, + "scheme", + attr, + sizeof (attr))) + isbn = (0 == strcasecmp (attr, + "ISBN")); + if (! isbn) + isbn = looks_like_isbn (value); + type = isbn + ? EXTRACTOR_METATYPE_ISBN + : EXTRACTOR_METATYPE_URI; + } + else + { + continue; + } + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + type, + value, + strlen (value))) + return 1; + } +#undef IS + return 0; +} + + +/** + * Handle an EPUB: read `META-INF/container.xml' to find the OPF package + * document, then report what the package document holds. + * + * @param ec extraction context + * @param uf the opened archive + */ +static void +extract_epub (struct EXTRACTOR_ExtractContext *ec, + struct EXTRACTOR_UnzipFile *uf) +{ + char path[512]; + char attr[64]; + char *container; + char *opf = NULL; + size_t container_len; + size_t opf_len = 0; + const char *p; + const char *q; + unsigned int items = 0; + + if (0 != + ec->proc (ec->cls, + PLUGIN_NAME, + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/epub+zip", + strlen ("application/epub+zip") + 1)) + return; + container = read_member (uf, + "META-INF/container.xml", + MAX_CONTAINER, + &container_len); + if (NULL != container) + { + p = mem_find (container, + container_len, + "full-path=", + strlen ("full-path=")); + if (NULL != p) + { + p += strlen ("full-path="); + if ( ('"' == *p) || ('\'' == *p) ) + { + char quote = *p; + + p++; + q = memchr (p, + quote, + container_len - (size_t) (p - container)); + if (NULL != q) + (void) xml_text (p, + (size_t) (q - p), + path, + sizeof (path)); + else + path[0] = '\0'; + } + else + { + path[0] = '\0'; + } + if ('\0' != path[0]) + opf = read_member (uf, + path, + MAX_OPF, + &opf_len); + } + free (container); + } + if (NULL == opf) + { + /* no usable container.xml; the conventional locations are worth a + try before giving up */ + static const char *guesses[] = { + "OEBPS/content.opf", + "content.opf", + "OPS/content.opf", + NULL + }; + + for (unsigned int i = 0; NULL != guesses[i]; i++) + { + opf = read_member (uf, + guesses[i], + MAX_OPF, + &opf_len); + if (NULL != opf) + break; + } + } + if (NULL != opf) + { + /* the <package version="..."> attribute distinguishes EPUB 2 from + EPUB 3, which changes what else we may expect to find */ + p = mem_find (opf, + opf_len, + "<package", + strlen ("<package")); + if (NULL != p) + { + size_t tail = opf_len - (size_t) (p - opf); + const char *gt; + + if (tail > 1024) + tail = 1024; + /* stop at the end of the start tag, so that a `version=' further + down the document cannot be mistaken for this one */ + gt = memchr (p, + '>', + tail); + if (NULL != gt) + tail = (size_t) (gt - p); + if (tail <= strlen ("<package")) + tail = strlen ("<package"); + if (xml_attr (p + strlen ("<package"), + tail - strlen ("<package"), + "version", + attr, + sizeof (attr))) + { + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_FORMAT_VERSION, + attr, + strlen (attr))) + goto CLEANUP; + } + } + if (0 != + parse_opf_metadata (ec, + opf, + opf_len)) + goto CLEANUP; + /* count the manifest entries; `<itemref>' does not match because + the character after `<item' has to be a name delimiter */ + for (size_t i = 0; i + strlen ("<item") < opf_len; i++) + if ( (0 == memcmp (&opf[i], + "<item", + strlen ("<item"))) && + (! is_name_char (opf[i + strlen ("<item")])) ) + items++; + if (0 != items) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + items)) + goto CLEANUP; + } + } + if (has_member (uf, + "META-INF/encryption.xml")) + { + if (0 != + EXTRACTOR_forensic_emit_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENCRYPTION, + "META-INF/encryption.xml present" + " (DRM or font obfuscation)")) + goto CLEANUP; + } + if (has_member (uf, + "META-INF/rights.xml")) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENCRYPTION, + "META-INF/rights.xml present" + " (Adobe Content Server DRM)")) + goto CLEANUP; + } +CLEANUP: + free (opf); +} + + +/** + * Convert a Palm database time stamp. + * + * The format says seconds since 1904, but writers disagree and some use + * the Unix epoch. The two ranges do not overlap for any date between + * 1970 and 2036, so the value itself says which convention was used. + * + * @param raw the 32 bit field + * @return seconds since 1970, 0 if the field was never filled in + */ +static int64_t +palm_time (uint32_t raw) +{ + if (0 == raw) + return 0; + if (((int64_t) raw) >= PALM_EPOCH_OFFSET) + return ((int64_t) raw) - PALM_EPOCH_OFFSET; + return (int64_t) raw; +} + + +/** + * Report one EXTH record. + * + * @param ec extraction context + * @param type EXTH record type + * @param data the record's payload + * @param len number of bytes in @a data + * @param[in,out] have_title set to 1 once a title has been reported + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_exth (struct EXTRACTOR_ExtractContext *ec, + uint32_t type, + const unsigned char *data, + size_t len, + int *have_title) +{ + enum EXTRACTOR_MetaType mt; + + switch (type) + { + case 100: mt = EXTRACTOR_METATYPE_AUTHOR_NAME; break; + case 101: mt = EXTRACTOR_METATYPE_PUBLISHER; break; + case 103: mt = EXTRACTOR_METATYPE_DESCRIPTION; break; + case 104: mt = EXTRACTOR_METATYPE_ISBN; break; + case 105: mt = EXTRACTOR_METATYPE_KEYWORDS; break; + case 106: mt = EXTRACTOR_METATYPE_PUBLICATION_DATE; break; + case 108: + /* the tool that produced the file stamps itself here; calibre and + kindlegen both do */ + mt = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE; + break; + case 109: mt = EXTRACTOR_METATYPE_RIGHTS; break; + case 112: mt = EXTRACTOR_METATYPE_URI; break; + case 113: mt = EXTRACTOR_METATYPE_SERIAL; break; /* ASIN */ + case 208: + /* per-copy marking on files that came from a store */ + mt = EXTRACTOR_METATYPE_WATERMARK; + break; + case 502: mt = EXTRACTOR_METATYPE_MODIFICATION_DATE; break; + case 503: + mt = EXTRACTOR_METATYPE_TITLE; + *have_title = 1; + break; + case 524: mt = EXTRACTOR_METATYPE_LANGUAGE; break; + case 501: + /* `EBOK' for a store book, `PDOC' for a personal document that was + side-loaded or mailed to a device */ + return EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ATTRIBUTES, + "cdetype: %.*s", + (int) ((len > 16) ? 16 : len), + (const char *) data); + case 209: + /* "tamper proof keys": binary, and specific to the copy rather than + to the book, so worth reporting even though we cannot decode it */ + return EXTRACTOR_forensic_emit_hex_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_WATERMARK, + data, + (len > 32) ? 32 : len); + default: + return 0; + } + return EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + mt, + (const char *) data, + len); +} + + +/** + * Walk the EXTH header of a MOBI file. + * + * @param ec extraction context + * @param rec record 0 + * @param rec_len number of bytes in @a rec + * @param off offset of the EXTH header within @a rec + * @param[in,out] have_title set to 1 once a title has been reported + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +parse_exth (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *rec, + size_t rec_len, + size_t off, + int *have_title) +{ + uint32_t count; + uint32_t exth_len; + size_t end; + size_t p; + + if ( (off > rec_len) || + (rec_len - off < 12) ) + return 0; + if (0 != memcmp (&rec[off], + "EXTH", + 4)) + return 0; + exth_len = EXTRACTOR_forensic_be32_ (&rec[off + 4]); + count = EXTRACTOR_forensic_be32_ (&rec[off + 8]); + end = rec_len; + if ( (exth_len >= 12) && + (exth_len <= rec_len - off) ) + end = off + exth_len; + if (count > MAX_EXTH) + count = MAX_EXTH; + p = off + 12; + for (uint32_t i = 0; i < count; i++) + { + uint32_t type; + uint32_t rlen; + + if ( (p + 8 > end) || + (p + 8 < p) ) + break; + type = EXTRACTOR_forensic_be32_ (&rec[p]); + rlen = EXTRACTOR_forensic_be32_ (&rec[p + 4]); + if (rlen < 8) + break; /* no forward progress; a malformed file must not spin */ + if (((size_t) rlen) > end - p) + break; + if (0 != + emit_exth (ec, + type, + &rec[p + 8], + (size_t) rlen - 8, + have_title)) + return 1; + p += rlen; + } + return 0; +} + + +/** + * Handle a Palm database that identifies itself as a MOBI book. + * + * @param ec extraction context + * @param hdr the 78 byte PDB header + */ +static void +extract_mobi (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *hdr) +{ + unsigned char info[16]; + unsigned char *rec = NULL; + uint64_t size; + uint32_t rec0_off; + uint32_t rec1_off; + uint32_t hdr_len; + uint16_t num_records; + size_t rec_len; + int have_title = 0; + + if (0 != + ec->proc (ec->cls, + PLUGIN_NAME, + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-mobipocket-ebook", + strlen ("application/x-mobipocket-ebook") + 1)) + return; + if (0 != + EXTRACTOR_forensic_emit_unix_time_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_CREATION_DATE, + palm_time ( + EXTRACTOR_forensic_be32_ ( + &hdr[36])))) + return; + if (0 != + EXTRACTOR_forensic_emit_unix_time_ ( + ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_MODIFICATION_DATE, + palm_time (EXTRACTOR_forensic_be32_ (&hdr[40])))) + return; + num_records = EXTRACTOR_forensic_be16_ (&hdr[76]); + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) num_records)) + return; + size = ec->get_size (ec->cls); + if ( (2 > num_records) || + (UINT64_MAX == size) ) + goto NAME_ONLY; + /* the record info list follows the header; entry i gives the offset + of record i, so record 0 runs up to the start of record 1 */ + if (! EXTRACTOR_forensic_read_ (ec, + 78, + info, + 16)) + goto NAME_ONLY; + rec0_off = EXTRACTOR_forensic_be32_ (&info[0]); + rec1_off = EXTRACTOR_forensic_be32_ (&info[8]); + if ( (rec1_off <= rec0_off) || + (((uint64_t) rec1_off) > size) ) + goto NAME_ONLY; + rec_len = (size_t) (rec1_off - rec0_off); + if (rec_len > MAX_RECORD0) + rec_len = MAX_RECORD0; + if (rec_len < 16) + goto NAME_ONLY; + if (NULL == (rec = malloc (rec_len))) + goto NAME_ONLY; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) rec0_off, + rec, + rec_len)) + goto NAME_ONLY; + /* PalmDOC header: encryption type at offset 12 */ + switch (EXTRACTOR_forensic_be16_ (&rec[12])) + { + case 0: + break; /* no DRM; saying so would be noise */ + case 1: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENCRYPTION, + "old Mobipocket encryption")) + goto CLEANUP; + break; + case 2: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENCRYPTION, + "Mobipocket DRM")) + goto CLEANUP; + break; + default: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_ENCRYPTION, + "unknown encryption type %u", + (unsigned int) + EXTRACTOR_forensic_be16_ (&rec[12]))) + goto CLEANUP; + break; + } + if ( (rec_len < 32) || + (0 != memcmp (&rec[16], + "MOBI", + 4)) ) + goto NAME_ONLY; /* PalmDOC without a MOBI header */ + hdr_len = EXTRACTOR_forensic_be32_ (&rec[20]); + if ( (hdr_len < 24) || + (((uint64_t) hdr_len) + 16 > rec_len) ) + goto NAME_ONLY; + if (hdr_len >= 28) + { + const char *cs = NULL; + + switch (EXTRACTOR_forensic_be32_ (&rec[16 + 12])) + { + case 1252: cs = "windows-1252"; break; + case 65001: cs = "UTF-8"; break; + default: break; + } + if ( (NULL != cs) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_CHARACTER_SET, + "%s", + cs)) ) + goto CLEANUP; + } + /* EXTH flags live at MOBI header offset 112; bit 0x40 says that an + EXTH header follows the MOBI header */ + if ( (hdr_len >= 116) && + (0 != (EXTRACTOR_forensic_be32_ (&rec[16 + 112]) & 0x40)) ) + { + if (0 != + parse_exth (ec, + rec, + rec_len, + (size_t) hdr_len + 16, + &have_title)) + goto CLEANUP; + } + if ( (! have_title) && + (hdr_len >= 76) ) + { + uint32_t fn_off = EXTRACTOR_forensic_be32_ (&rec[16 + 68]); + uint32_t fn_len = EXTRACTOR_forensic_be32_ (&rec[16 + 72]); + + if ( (fn_len > 0) && + (fn_len <= EXTRACTOR_FORENSIC_MAX_STRING) && + (((uint64_t) fn_off) + fn_len <= rec_len) ) + { + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_TITLE, + (const char *) &rec[fn_off], + fn_len)) + goto CLEANUP; + have_title = 1; + } + } +NAME_ONLY: + if (! have_title) + (void) EXTRACTOR_forensic_emit_text_ (ec, + PLUGIN_NAME, + EXTRACTOR_METATYPE_TITLE, + (const char *) hdr, + 32); +CLEANUP: + free (rec); +} + + +/** + * Main entry method for the ebook extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_ebook_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_ebook_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char hdr[78]; + struct EXTRACTOR_UnzipFile *uf; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + 4)) + return; + if (0 != memcmp (hdr, + "PK\x03\x04", + 4)) + { + /* not a ZIP; the only other thing we handle is a Palm database, + whose type/creator pair sits at offset 60 */ + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + sizeof (hdr))) + return; + if (0 == memcmp (&hdr[60], + "BOOKMOBI", + 8)) + extract_mobi (ec, + hdr); + return; + } + if (NULL == (uf = EXTRACTOR_common_unzip_open (ec))) + return; + /* a plain ZIP, a docx and an ODF file all start `PK\003\004', so + claim the file only if it carries the OCF structure */ + if (has_member (uf, + "META-INF/container.xml")) + { + extract_epub (ec, + uf); + } + else + { + size_t len; + char *mt = read_member (uf, + "mimetype", + 64, + &len); + + if ( (NULL != mt) && + (len >= strlen ("application/epub+zip")) && + (0 == memcmp (mt, + "application/epub+zip", + strlen ("application/epub+zip"))) ) + { + free (mt); + extract_epub (ec, + uf); + } + else + { + free (mt); + } + } + (void) EXTRACTOR_common_unzip_close (uf); +} + + +/* end of ebook_extractor.c */ diff --git a/src/plugins/forensics.c b/src/plugins/forensics.c @@ -0,0 +1,512 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/forensics.c + * @brief helpers shared by the plugins that read provenance out of + * binary headers + * @author Christian Grothoff + */ +#include "platform.h" +#include "forensics.h" + +#include <math.h> +#include <stdarg.h> + + +int +EXTRACTOR_forensic_read_ (struct EXTRACTOR_ExtractContext *ec, + int64_t offset, + void *buf, + size_t len) +{ + unsigned char *out = buf; + size_t got = 0; + + if (0 == len) + return 1; + if ( (0 <= offset) && + (offset != ec->seek (ec->cls, + offset, + SEEK_SET)) ) + return 0; + while (got < len) + { + void *data; + ssize_t ret; + + ret = ec->read (ec->cls, + &data, + len - got); + if (0 >= ret) + return 0; + if (((size_t) ret) > len - got) + return 0; /* the IPC layer is misbehaving; do not overrun */ + memcpy (&out[got], + data, + (size_t) ret); + got += (size_t) ret; + } + return 1; +} + + +int +EXTRACTOR_forensic_utf8_valid_ (const char *data, + size_t len) +{ + const unsigned char *p = (const unsigned char *) data; + size_t i = 0; + + while (i < len) + { + unsigned int extra; + uint32_t cp; + + if (p[i] < 0x80) + { + i++; + continue; + } + if ((0xC2 <= p[i]) && (p[i] <= 0xDF)) + { + extra = 1; + cp = p[i] & 0x1F; + } + else if ((0xE0 <= p[i]) && (p[i] <= 0xEF)) + { + extra = 2; + cp = p[i] & 0x0F; + } + else if ((0xF0 <= p[i]) && (p[i] <= 0xF4)) + { + extra = 3; + cp = p[i] & 0x07; + } + else + { + return 0; /* continuation byte or overlong lead */ + } + if (i + extra >= len) + return 0; /* truncated multi-byte sequence */ + for (unsigned int k = 1; k <= extra; k++) + { + if (0x80 != (p[i + k] & 0xC0)) + return 0; + cp = (cp << 6) | (p[i + k] & 0x3F); + } + /* reject overlong encodings, surrogates and out-of-range values */ + if ( (1 == extra) && (cp < 0x80) ) + return 0; + if ( (2 == extra) && (cp < 0x800) ) + return 0; + if ( (3 == extra) && (cp < 0x10000) ) + return 0; + if ( (0xD800 <= cp) && (cp <= 0xDFFF) ) + return 0; + if (cp > 0x10FFFF) + return 0; + i += extra + 1; + } + return 1; +} + + +size_t +EXTRACTOR_forensic_trim_ (const char *data, + size_t len) +{ + while ( (0 < len) && + ( ('\0' == data[len - 1]) || + (' ' == data[len - 1]) || + ('\t' == data[len - 1]) || + ('\r' == data[len - 1]) || + ('\n' == data[len - 1]) ) ) + len--; + return len; +} + + +int +EXTRACTOR_forensic_parse_octal_ (const char *data, + size_t len, + uint64_t *value) +{ + uint64_t v = 0; + size_t i = 0; + int digits = 0; + + while ( (i < len) && + (' ' == data[i]) ) + i++; + while ( (i < len) && + ('0' <= data[i]) && + ('7' >= data[i]) ) + { + if (v > (UINT64_MAX >> 3)) + return 0; /* would overflow */ + v = (v << 3) | (uint64_t) (data[i] - '0'); + digits++; + i++; + } + if (0 == digits) + return 0; + /* what follows must be padding, not more number */ + while (i < len) + { + if ( ('\0' != data[i]) && + (' ' != data[i]) ) + return 0; + i++; + } + *value = v; + return 1; +} + + +double +EXTRACTOR_forensic_entropy_ (const unsigned char *data, + size_t len) +{ + unsigned long counts[256]; + double entropy = 0.0; + + if (0 == len) + return 0.0; + memset (counts, + 0, + sizeof (counts)); + for (size_t i = 0; i < len; i++) + counts[data[i]]++; + for (unsigned int i = 0; i < 256; i++) + { + double p; + + if (0 == counts[i]) + continue; + p = ((double) counts[i]) / ((double) len); + entropy -= p * log2 (p); + } + return entropy; +} + + +int +EXTRACTOR_forensic_emit_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const char *fmt, + ...) +{ + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + va_list ap; + int len; + + va_start (ap, fmt); + len = vsnprintf (buf, + sizeof (buf), + fmt, + ap); + va_end (ap); + if (0 >= len) + return 0; + if (((size_t) len) >= sizeof (buf)) + len = sizeof (buf) - 1; + return (0 != ec->proc (ec->cls, + plugin, + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + buf, + (size_t) len + 1)) ? 1 : 0; +} + + +int +EXTRACTOR_forensic_emit_text_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const char *data, + size_t len) +{ + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t out = 0; + + /* a NUL inside the field ends the string: these are C strings in + fixed-width slots far more often than they are counted strings */ + for (size_t i = 0; i < len; i++) + if ('\0' == data[i]) + { + len = i; + break; + } + len = EXTRACTOR_forensic_trim_ (data, + len); + if (0 == len) + return 0; + if (len > sizeof (buf) - 1) + len = sizeof (buf) - 1; + for (size_t i = 0; i < len; i++) + { + unsigned char c = (unsigned char) data[i]; + + /* Collapse control characters rather than dropping the value: a + path with a stray tab in it is still the path we want to show, + but it must not be able to forge line structure in the output. */ + if ( (c < 0x20) || (0x7F == c) ) + buf[out++] = ' '; + else + buf[out++] = (char) c; + } + out = EXTRACTOR_forensic_trim_ (buf, + out); + if (0 == out) + return 0; + buf[out] = '\0'; + if (! EXTRACTOR_forensic_utf8_valid_ (buf, + out)) + return 0; /* not text we can hand out; the caller may know better */ + return (0 != ec->proc (ec->cls, + plugin, + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + buf, + out + 1)) ? 1 : 0; +} + + +int +EXTRACTOR_forensic_emit_utf16le_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *data, + size_t bytes) +{ + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t out = 0; + size_t i = 0; + + while (i + 1 < bytes) + { + uint32_t cp = EXTRACTOR_forensic_le16_ (&data[i]); + + i += 2; + if (0 == cp) + break; /* NUL terminator */ + if ( (0xD800 <= cp) && (cp <= 0xDBFF) ) + { + uint32_t lo; + + if (i + 1 >= bytes) + break; /* truncated surrogate pair */ + lo = EXTRACTOR_forensic_le16_ (&data[i]); + if ( (lo < 0xDC00) || (lo > 0xDFFF) ) + break; /* unpaired surrogate; stop rather than guess */ + i += 2; + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + } + else if ( (0xDC00 <= cp) && (cp <= 0xDFFF) ) + { + break; /* stray low surrogate */ + } + if ( (cp < 0x20) || (0x7F == cp) ) + cp = ' '; + if (out + 4 >= sizeof (buf)) + break; + if (cp < 0x80) + { + buf[out++] = (char) cp; + } + else if (cp < 0x800) + { + buf[out++] = (char) (0xC0 | (cp >> 6)); + buf[out++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + buf[out++] = (char) (0xE0 | (cp >> 12)); + buf[out++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + buf[out++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + buf[out++] = (char) (0xF0 | (cp >> 18)); + buf[out++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + buf[out++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + buf[out++] = (char) (0x80 | (cp & 0x3F)); + } + } + out = EXTRACTOR_forensic_trim_ (buf, + out); + if (0 == out) + return 0; + buf[out] = '\0'; + return (0 != ec->proc (ec->cls, + plugin, + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + buf, + out + 1)) ? 1 : 0; +} + + +int +EXTRACTOR_forensic_emit_unix_time_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + int64_t when) +{ + char buf[32]; + struct tm tm; + time_t t = (time_t) when; + + /* Zero and 0xFFFFFFFF are how these fields look when they were never + filled in; anything outside a plausible range is a misparse. */ + if ( (when < 315532800LL) || /* 1980-01-01 */ + (when > 4102444800LL) ) /* 2100-01-01 */ + return 0; + if (NULL == gmtime_r (&t, + &tm)) + return 0; + if (0 == strftime (buf, + sizeof (buf), + "%Y-%m-%dT%H:%M:%SZ", + &tm)) + return 0; + return (0 != ec->proc (ec->cls, + plugin, + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + buf, + strlen (buf) + 1)) ? 1 : 0; +} + + +int +EXTRACTOR_forensic_emit_filetime_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + uint64_t filetime) +{ + /* FILETIME counts 100ns intervals from 1601-01-01; 11644473600 is the + number of seconds from there to the Unix epoch. */ + if (0 == filetime) + return 0; + if (filetime > (uint64_t) 200000000000000000ULL) + return 0; /* well past year 2234; a misparse */ + return EXTRACTOR_forensic_emit_unix_time_ (ec, + plugin, + type, + (int64_t) (filetime / 10000000ULL) + - 11644473600LL); +} + + +int +EXTRACTOR_forensic_emit_size_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + uint64_t bytes) +{ + return EXTRACTOR_forensic_emit_ (ec, + plugin, + type, + "%llu", + (unsigned long long) bytes); +} + + +int +EXTRACTOR_forensic_emit_hex_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *data, + size_t len) +{ + static const char hex[] = "0123456789abcdef"; + char buf[129]; + + if ( (0 == len) || + (len > (sizeof (buf) - 1) / 2) ) + return 0; + for (size_t i = 0; i < len; i++) + { + buf[2 * i] = hex[data[i] >> 4]; + buf[2 * i + 1] = hex[data[i] & 0x0F]; + } + buf[2 * len] = '\0'; + return (0 != ec->proc (ec->cls, + plugin, + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + buf, + 2 * len + 1)) ? 1 : 0; +} + + +int +EXTRACTOR_forensic_emit_guid_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *guid, + int mixed_endian) +{ + uint32_t d1; + uint16_t d2; + uint16_t d3; + int all_zero = 1; + + for (unsigned int i = 0; i < 16; i++) + if (0 != guid[i]) + { + all_zero = 0; + break; + } + if (all_zero) + return 0; + if (mixed_endian) + { + d1 = EXTRACTOR_forensic_le32_ (guid); + d2 = EXTRACTOR_forensic_le16_ (&guid[4]); + d3 = EXTRACTOR_forensic_le16_ (&guid[6]); + } + else + { + d1 = EXTRACTOR_forensic_be32_ (guid); + d2 = EXTRACTOR_forensic_be16_ (&guid[4]); + d3 = EXTRACTOR_forensic_be16_ (&guid[6]); + } + return EXTRACTOR_forensic_emit_ (ec, + plugin, + type, + "%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x", + (unsigned int) d1, + (unsigned int) d2, + (unsigned int) d3, + guid[8], guid[9], + guid[10], guid[11], guid[12], + guid[13], guid[14], guid[15]); +} + + +/* end of forensics.c */ diff --git a/src/plugins/forensics.h b/src/plugins/forensics.h @@ -0,0 +1,331 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/forensics.h + * @brief helpers shared by the plugins that read provenance out of + * binary headers + * @author Christian Grothoff + * + * These plugins all do the same handful of things -- pull a fixed-size + * header out of the stream, pick integers out of it in a documented + * byte order, and hand back strings that came from an untrusted file. + * Doing that once here rather than sixteen times keeps the bounds + * checks and the UTF-8 validation in one place. + */ +#ifndef FORENSICS_H +#define FORENSICS_H + +#include "extractor.h" + + +/** + * Longest string we will ever hand to the caller. Values in these + * formats are identifiers and paths, not documents; anything longer is + * either padding or an attempt to make us allocate. + */ +#define EXTRACTOR_FORENSIC_MAX_STRING 1024 + +/** + * Upper bound on how many items of the same repeating kind (archive + * members, permissions, mail headers) a plugin should report. A first + * pass over a large volume wants a characterisation, not a listing. + */ +#define EXTRACTOR_FORENSIC_MAX_ITEMS 64 + + +/** + * Read exactly @a len bytes at @a offset into @a buf. + * + * `ec->read()' may return short reads and hands back a pointer that is + * only valid until the next call, so every caller would otherwise need + * this loop. + * + * @param ec extraction context + * @param offset absolute position to read from, -1 to read at the + * current position without seeking + * @param buf where to copy the data to + * @param len number of bytes to read + * @return 1 on success, 0 if the file is too short or seeking failed + */ +int +EXTRACTOR_forensic_read_ (struct EXTRACTOR_ExtractContext *ec, + int64_t offset, + void *buf, + size_t len); + + +/** + * Check that @a data is well-formed UTF-8. + * + * `EXTRACTOR_common_convert_to_utf8()' silently passes the input + * through when iconv rejects it, so a plugin that trusts it will emit + * whatever bytes the file contained. Validate before emitting. + * + * @param data bytes to check + * @param len number of bytes in @a data + * @return 1 if @a data is valid UTF-8, 0 if not + */ +int +EXTRACTOR_forensic_utf8_valid_ (const char *data, + size_t len); + + +/** + * Strip trailing spaces and NUL bytes from a fixed-width field. + * + * @param data the field + * @param len width of the field + * @return length of the field with the padding removed + */ +size_t +EXTRACTOR_forensic_trim_ (const char *data, + size_t len); + + +/** + * Parse a NUL- or space-terminated octal number, as used by tar. + * + * @param data the field + * @param len width of the field + * @param[out] value where to store the result + * @return 1 on success, 0 if the field does not hold an octal number + */ +int +EXTRACTOR_forensic_parse_octal_ (const char *data, + size_t len, + uint64_t *value); + + +/** + * Shannon entropy of a buffer, in bits per byte. + * + * @param data the buffer + * @param len number of bytes in @a data + * @return entropy between 0.0 and 8.0, 0.0 for an empty buffer + */ +double +EXTRACTOR_forensic_entropy_ (const unsigned char *data, + size_t len); + + +/** + * Emit a printf-formatted value. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param fmt printf format string + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const char *fmt, + ...) +__attribute__ ((format (printf, 4, 5))); + + +/** + * Emit a string that came out of the file. + * + * Trailing padding is stripped, control characters are replaced, the + * result is capped at #EXTRACTOR_FORENSIC_MAX_STRING and dropped + * entirely if it is empty or not valid UTF-8. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param data the string, need not be NUL-terminated + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_text_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const char *data, + size_t len); + + +/** + * Emit a UTF-16LE string that came out of the file, converting it to + * UTF-8. Used by the Windows formats and by binary property lists. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param data the string + * @param bytes number of bytes (not code units) in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_utf16le_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *data, + size_t bytes); + + +/** + * Emit a time expressed as seconds since the Unix epoch, as ISO 8601 in + * UTC. Implausible values (before 1980, after 2100) are dropped: they + * are almost always a misparse or a zeroed field. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param when seconds since 1970-01-01 + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_unix_time_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + int64_t when); + + +/** + * Emit a Windows FILETIME (100ns units since 1601-01-01) as ISO 8601. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param filetime the FILETIME value + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_filetime_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + uint64_t filetime); + + +/** + * Emit a byte count, as a plain decimal number of bytes. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param bytes the value + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_size_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + uint64_t bytes); + + +/** + * Emit a byte string in lower-case hexadecimal. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param data the bytes + * @param len number of bytes in @a data, at most 64 + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_hex_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *data, + size_t len); + + +/** + * Emit a 16-byte GUID/UUID in the usual 8-4-4-4-12 spelling. + * + * @param ec extraction context + * @param plugin name to report the data under + * @param type meta data type + * @param guid the 16 bytes + * @param mixed_endian 1 if the first three fields are little-endian, as + * in a Microsoft GUID; 0 for a big-endian RFC 4122 UUID + * @return 1 if the caller should stop extracting, 0 to continue + */ +int +EXTRACTOR_forensic_emit_guid_ (struct EXTRACTOR_ExtractContext *ec, + const char *plugin, + enum EXTRACTOR_MetaType type, + const unsigned char *guid, + int mixed_endian); + + +/* Integer accessors. The formats below all document their byte order, + so read it explicitly rather than casting a struct over the buffer: + that also sidesteps every alignment question. */ + +static inline uint16_t +EXTRACTOR_forensic_le16_ (const unsigned char *p) +{ + return (uint16_t) (((uint16_t) p[0]) + | (((uint16_t) p[1]) << 8)); +} + + +static inline uint32_t +EXTRACTOR_forensic_le32_ (const unsigned char *p) +{ + return ((uint32_t) p[0]) + | (((uint32_t) p[1]) << 8) + | (((uint32_t) p[2]) << 16) + | (((uint32_t) p[3]) << 24); +} + + +static inline uint64_t +EXTRACTOR_forensic_le64_ (const unsigned char *p) +{ + return ((uint64_t) EXTRACTOR_forensic_le32_ (p)) + | (((uint64_t) EXTRACTOR_forensic_le32_ (p + 4)) << 32); +} + + +static inline uint16_t +EXTRACTOR_forensic_be16_ (const unsigned char *p) +{ + return (uint16_t) (((uint16_t) p[1]) + | (((uint16_t) p[0]) << 8)); +} + + +static inline uint32_t +EXTRACTOR_forensic_be32_ (const unsigned char *p) +{ + return ((uint32_t) p[3]) + | (((uint32_t) p[2]) << 8) + | (((uint32_t) p[1]) << 16) + | (((uint32_t) p[0]) << 24); +} + + +static inline uint64_t +EXTRACTOR_forensic_be64_ (const unsigned char *p) +{ + return (((uint64_t) EXTRACTOR_forensic_be32_ (p)) << 32) + | ((uint64_t) EXTRACTOR_forensic_be32_ (p + 4)); +} + + +#endif +/* end of forensics.h */ diff --git a/src/plugins/geotiff_extractor.c b/src/plugins/geotiff_extractor.c @@ -0,0 +1,1960 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/geotiff_extractor.c + * @brief plugin to support GeoTIFF files + * @author Christian Grothoff + * + * A GeoTIFF is an ordinary TIFF 6.0 (or BigTIFF) file that additionally + * carries a handful of private tags describing where on the Earth the + * raster sits. We do not link libtiff or libgeotiff: all we need is a + * bounded walk over the image file directory, and the `tiff' plugin -- + * which does link libtiff -- is only built when that library is + * present, so this plugin has to stand on its own anyway. + * + * Deliberately, we say nothing at all about a plain TIFF: without one + * of the four georeferencing tags the file belongs to the `tiff' and + * `exiv2' plugins and we return without emitting anything. + * + * References: OGC GeoTIFF 1.1 (OGC 19-008r4), the original GeoTIFF + * 1.8.2 specification, TIFF 6.0 and Adobe's BigTIFF description. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +#include <math.h> + + +/** + * How many image file directories we are willing to follow. Real + * files have one plus a handful of reduced-resolution overviews. + */ +#define GEOTIFF_MAX_IFD 16 + +/** + * How many entries we look at in a single IFD. GDAL writes a few + * dozen; anything past this is padding or an attack. + */ +#define GEOTIFF_MAX_ENTRIES 512 + +/** + * How many GeoTIFF keys we parse out of the key directory. The + * registry defines well under a hundred. + */ +#define GEOTIFF_MAX_KEYS 512 + +/** + * Cap on the number of bytes of any one tag value we will read (and + * hence allocate). The largest thing we touch is GDAL's XML blob. + */ +#define GEOTIFF_MAX_VALUE 65536 + +/** + * How many `<Item>' elements we pull out of GDAL_METADATA. + */ +#define GEOTIFF_MAX_GDAL_ITEMS 16 + +/** + * Exclusive upper bound on ImageWidth/ImageLength. Both are LONG in + * the TIFF specification, so 2^32 admits every legal file; the point of + * the bound is that the tag is read as a `double' (it may be typed + * DOUBLE in the file) and converting one larger than ULLONG_MAX to + * `unsigned long long' is undefined behaviour. + */ +#define GEOTIFF_MAX_DIM 4294967296.0 + + +/* TIFF field types, from TIFF 6.0 plus the BigTIFF additions. */ +#define TT_BYTE 1 +#define TT_ASCII 2 +#define TT_SHORT 3 +#define TT_LONG 4 +#define TT_RATIONAL 5 +#define TT_SBYTE 6 +#define TT_UNDEFINED 7 +#define TT_SSHORT 8 +#define TT_SLONG 9 +#define TT_SRATIONAL 10 +#define TT_FLOAT 11 +#define TT_DOUBLE 12 +#define TT_LONG8 16 +#define TT_SLONG8 17 +#define TT_IFD8 18 + + +/** + * The tags we care about. Used to index #Context::tags. + */ +enum WantedTag +{ + W_WIDTH = 0, + W_LENGTH, + W_BITS, + W_COMPRESSION, + W_DESCRIPTION, + W_MAKE, + W_MODEL, + W_SAMPLES, + W_SOFTWARE, + W_DATETIME, + W_ARTIST, + W_COPYRIGHT, + W_PIXELSCALE, + W_TIEPOINT, + W_TRANSFORM, + W_GEOKEYS, + W_GEODOUBLE, + W_GEOASCII, + W_GDAL_META, + W_GDAL_NODATA, + W_COUNT +}; + + +/** + * TIFF tag numbers, in the order of `enum WantedTag'. + */ +static const uint16_t wanted_tags[W_COUNT] = { + 256, /* ImageWidth */ + 257, /* ImageLength */ + 258, /* BitsPerSample */ + 259, /* Compression */ + 270, /* ImageDescription */ + 271, /* Make */ + 272, /* Model */ + 277, /* SamplesPerPixel */ + 305, /* Software */ + 306, /* DateTime */ + 315, /* Artist */ + 33432, /* Copyright */ + 33550, /* ModelPixelScaleTag */ + 33922, /* ModelTiepointTag */ + 34264, /* ModelTransformationTag */ + 34735, /* GeoKeyDirectoryTag */ + 34736, /* GeoDoubleParamsTag */ + 34737, /* GeoAsciiParamsTag */ + 42112, /* GDAL_METADATA */ + 42113 /* GDAL_NODATA */ +}; + + +/** + * One directory entry we decided to keep. + */ +struct Entry +{ + /** + * Number of values, from the entry's count field. + */ + uint64_t count; + + /** + * Where the value lives, if it did not fit in the entry. Only + * meaningful when the total size exceeds the inline field. + */ + uint64_t offset; + + /** + * The raw value/offset field, 4 bytes for classic TIFF and 8 for + * BigTIFF. Small values live here directly. + */ + unsigned char val[8]; + + /** + * TIFF field type (one of the TT_* constants). + */ + uint16_t type; + + /** + * True once we have seen this tag. We keep the first occurrence: + * later IFDs are overviews and thumbnails. + */ + int found; +}; + + +/** + * State for one extraction run. + */ +struct Context +{ + /** + * Extraction context we were handed. + */ + struct EXTRACTOR_ExtractContext *ec; + + /** + * Size of the file, used to bounds-check every offset. + */ + uint64_t fsize; + + /** + * True if the file is big-endian (`MM'). + */ + int be; + + /** + * True if the file is BigTIFF (version 43): 8-byte offsets and + * counts, 20-byte directory entries. + */ + int big; + + /** + * The tags we found, indexed by `enum WantedTag'. + */ + struct Entry tags[W_COUNT]; +}; + + +/** + * The GeoTIFF keys we understand, as read out of the key directory. + */ +struct GeoKeys +{ + /** + * GTModelTypeGeoKey (1024): 1 projected, 2 geographic, 3 geocentric. + */ + unsigned int model_type; + + /** + * GTRasterTypeGeoKey (1025): 1 pixel-is-area, 2 pixel-is-point. + */ + unsigned int raster_type; + + /** + * GeographicTypeGeoKey (2048), an EPSG geographic CRS code. + */ + unsigned int geographic; + + /** + * ProjectedCSTypeGeoKey (3072), an EPSG projected CRS code. + */ + unsigned int projected; + + /** + * VerticalCSTypeGeoKey (4096), an EPSG vertical CRS code. + */ + unsigned int vertical; + + /** + * ProjLinearUnitsGeoKey (3076), an EPSG unit-of-measure code. + */ + unsigned int linear_units; + + /** + * Which of the above were actually present, as a bit set of + * 1 << (key index); see the have_* flags below instead. + */ + int have_model_type; + + int have_raster_type; + + int have_geographic; + + int have_projected; + + int have_vertical; + + int have_linear_units; +}; + + +/** + * Read a 16-bit integer in the file's byte order. + * + * @param ctx extraction state, for the byte order + * @param p the two bytes + * @return the value + */ +static uint16_t +geo_u16 (const struct Context *ctx, + const unsigned char *p) +{ + return ctx->be + ? EXTRACTOR_forensic_be16_ (p) + : EXTRACTOR_forensic_le16_ (p); +} + + +/** + * Read a 32-bit integer in the file's byte order. + * + * @param ctx extraction state, for the byte order + * @param p the four bytes + * @return the value + */ +static uint32_t +geo_u32 (const struct Context *ctx, + const unsigned char *p) +{ + return ctx->be + ? EXTRACTOR_forensic_be32_ (p) + : EXTRACTOR_forensic_le32_ (p); +} + + +/** + * Read a 64-bit integer in the file's byte order. + * + * @param ctx extraction state, for the byte order + * @param p the eight bytes + * @return the value + */ +static uint64_t +geo_u64 (const struct Context *ctx, + const unsigned char *p) +{ + return ctx->be + ? EXTRACTOR_forensic_be64_ (p) + : EXTRACTOR_forensic_le64_ (p); +} + + +/** + * Read an IEEE 754 single as stored in the file. + * + * The bit pattern is assembled in host integer order first, so the + * only assumption left is that the host's `float' is IEEE 754 with the + * same byte order as its integers -- true on every platform + * libextractor builds on. + * + * @param ctx extraction state, for the byte order + * @param p the four bytes + * @return the value + */ +static double +geo_f32 (const struct Context *ctx, + const unsigned char *p) +{ + uint32_t bits = geo_u32 (ctx, + p); + float f; + + memcpy (&f, + &bits, + sizeof (f)); + return (double) f; +} + + +/** + * Read an IEEE 754 double as stored in the file. + * + * @param ctx extraction state, for the byte order + * @param p the eight bytes + * @return the value + */ +static double +geo_f64 (const struct Context *ctx, + const unsigned char *p) +{ + uint64_t bits = geo_u64 (ctx, + p); + double d; + + memcpy (&d, + &bits, + sizeof (d)); + return d; +} + + +/** + * Size in bytes of one value of the given TIFF field type. + * + * @param type the TIFF field type + * @return the size, 0 for a type we do not know (which makes the + * caller ignore the tag, as TIFF 6.0 requires) + */ +static size_t +type_size (uint16_t type) +{ + switch (type) + { + case TT_BYTE: + case TT_ASCII: + case TT_SBYTE: + case TT_UNDEFINED: + return 1; + case TT_SHORT: + case TT_SSHORT: + return 2; + case TT_LONG: + case TT_SLONG: + case TT_FLOAT: + return 4; + case TT_RATIONAL: + case TT_SRATIONAL: + case TT_DOUBLE: + case TT_LONG8: + case TT_SLONG8: + case TT_IFD8: + return 8; + default: + return 0; + } +} + + +/** + * Fetch the bytes belonging to @a e, either from the entry itself or + * from wherever in the file it points. + * + * @param ctx extraction state + * @param e the directory entry + * @param cap refuse to read (and allocate) more than this many bytes + * @param[out] len number of bytes returned + * @return the bytes, to be freed by the caller, NULL on any error + */ +static unsigned char * +load_value (const struct Context *ctx, + const struct Entry *e, + size_t cap, + size_t *len) +{ + size_t ts = type_size (e->type); + size_t inline_max = ctx->big ? 8 : 4; + size_t total; + unsigned char *buf; + + if ( (! e->found) || + (0 == ts) || + (0 == e->count) ) + return NULL; + if (cap > GEOTIFF_MAX_VALUE) + cap = GEOTIFF_MAX_VALUE; + /* This is the count-times-size multiplication the file controls; + dividing instead of multiplying keeps it from wrapping. */ + if (e->count > ((uint64_t) cap) / ts) + return NULL; + total = (size_t) (e->count * ts); + buf = malloc (total); + if (NULL == buf) + return NULL; + if (total <= inline_max) + { + memcpy (buf, + e->val, + total); + *len = total; + return buf; + } + if ( (e->offset > ctx->fsize) || + (total > ctx->fsize - e->offset) || + (e->offset > (uint64_t) INT64_MAX) ) + { + free (buf); + return NULL; + } + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) e->offset, + buf, + total)) + { + free (buf); + return NULL; + } + *len = total; + return buf; +} + + +/** + * Interpret element @a idx of an already loaded tag value as a double. + * + * @param ctx extraction state, for the byte order + * @param type the TIFF field type the buffer holds + * @param buf the value bytes + * @param len number of bytes in @a buf + * @param idx which element to read + * @param[out] out where to store the value + * @return 1 on success, 0 if the index or the type does not work out + */ +static int +value_as_double (const struct Context *ctx, + uint16_t type, + const unsigned char *buf, + size_t len, + size_t idx, + double *out) +{ + size_t ts = type_size (type); + const unsigned char *p; + + if ( (0 == ts) || + (idx >= len / ts) ) + return 0; + p = &buf[idx * ts]; + switch (type) + { + case TT_BYTE: + case TT_UNDEFINED: + *out = (double) p[0]; + return 1; + case TT_SBYTE: + *out = (double) (int8_t) p[0]; + return 1; + case TT_SHORT: + *out = (double) geo_u16 (ctx, p); + return 1; + case TT_SSHORT: + *out = (double) (int16_t) geo_u16 (ctx, p); + return 1; + case TT_LONG: + *out = (double) geo_u32 (ctx, p); + return 1; + case TT_SLONG: + *out = (double) (int32_t) geo_u32 (ctx, p); + return 1; + case TT_FLOAT: + *out = geo_f32 (ctx, p); + return 1; + case TT_DOUBLE: + *out = geo_f64 (ctx, p); + return 1; + case TT_RATIONAL: + { + uint32_t num = geo_u32 (ctx, p); + uint32_t den = geo_u32 (ctx, p + 4); + + if (0 == den) + return 0; + *out = ((double) num) / ((double) den); + return 1; + } + case TT_SRATIONAL: + { + int32_t num = (int32_t) geo_u32 (ctx, p); + int32_t den = (int32_t) geo_u32 (ctx, p + 4); + + if (0 == den) + return 0; + *out = ((double) num) / ((double) den); + return 1; + } + case TT_LONG8: + case TT_IFD8: + *out = (double) geo_u64 (ctx, p); + return 1; + case TT_SLONG8: + *out = (double) (int64_t) geo_u64 (ctx, p); + return 1; + default: + return 0; + } +} + + +/** + * Read a single numeric value out of one of our tags. + * + * Only the one element is fetched, never the whole array: + * ModelTiepointTag holds a list of ground control points in files that + * are not a simple grid, and that list can run to many kilobytes we + * have no use for. + * + * @param ctx extraction state + * @param w which tag to read + * @param idx which element of the tag to read + * @param[out] out where to store the value + * @return 1 on success, 0 if the tag is absent or unusable + */ +static int +tag_double (const struct Context *ctx, + enum WantedTag w, + size_t idx, + double *out) +{ + const struct Entry *e = &ctx->tags[w]; + size_t ts = type_size (e->type); + size_t inline_max = ctx->big ? 8 : 4; + unsigned char raw[8]; + uint64_t total; + uint64_t byte_off; + uint64_t room; + + if ( (! e->found) || + (0 == ts) || + (((uint64_t) idx) >= e->count) ) + return 0; + if (e->count > UINT64_MAX / ts) + return 0; /* a count that cannot describe any real array */ + total = e->count * ts; + byte_off = ((uint64_t) idx) * ts; + if (total <= inline_max) + { + memcpy (raw, + &e->val[byte_off], + ts); + } + else + { + if ( (e->offset > ctx->fsize) || + (e->offset > (uint64_t) INT64_MAX) ) + return 0; + room = ctx->fsize - e->offset; + if ( (byte_off > room) || + (ts > room - byte_off) ) + return 0; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) (e->offset + byte_off), + raw, + ts)) + return 0; + } + return value_as_double (ctx, + e->type, + raw, + ts, + 0, + out); +} + + +/** + * Emit a string-valued tag, if it is present. + * + * @param ctx extraction state + * @param w which tag to emit + * @param type meta data type to report it as + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_ascii_tag (const struct Context *ctx, + enum WantedTag w, + enum EXTRACTOR_MetaType type) +{ + unsigned char *buf; + size_t len; + int stop; + + if (TT_ASCII != ctx->tags[w].type) + return 0; + buf = load_value (ctx, + &ctx->tags[w], + EXTRACTOR_FORENSIC_MAX_STRING, + &len); + if (NULL == buf) + return 0; + stop = EXTRACTOR_forensic_emit_text_ (ctx->ec, + "geotiff", + type, + (const char *) buf, + len); + free (buf); + return stop; +} + + +/** + * Walk the chain of image file directories and remember the entries + * for the tags in #wanted_tags. + * + * @param ctx extraction state, updated in place + * @param first offset of the first IFD, from the header + */ +static void +walk_ifds (struct Context *ctx, + uint64_t first) +{ + uint64_t seen[GEOTIFF_MAX_IFD]; + unsigned int nseen = 0; + uint64_t off = first; + size_t esz = ctx->big ? 20 : 12; + size_t cntsz = ctx->big ? 8 : 2; + size_t nextsz = ctx->big ? 8 : 4; + + while ( (0 != off) && + (nseen < GEOTIFF_MAX_IFD) ) + { + unsigned char cnt[8]; + unsigned char *ents; + uint64_t avail; + uint64_t nent; + uint64_t declared; + int truncated = 0; + + /* An IFD that points back at one we already parsed would loop + forever; the offset cap alone does not catch a 2-cycle. */ + for (unsigned int i = 0; i < nseen; i++) + if (seen[i] == off) + return; + seen[nseen++] = off; + if ( (off >= ctx->fsize) || + (off > (uint64_t) INT64_MAX) ) + return; + avail = ctx->fsize - off; + if (avail < cntsz) + return; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) off, + cnt, + cntsz)) + return; + declared = ctx->big + ? geo_u64 (ctx, cnt) + : (uint64_t) geo_u16 (ctx, cnt); + nent = declared; + if (nent > GEOTIFF_MAX_ENTRIES) + { + nent = GEOTIFF_MAX_ENTRIES; + truncated = 1; + } + avail -= cntsz; + if (avail / esz < nent) + { + /* the file is shorter than it claims; take what is there */ + nent = avail / esz; + truncated = 1; + } + if (0 == nent) + return; + ents = malloc ((size_t) (nent * esz)); + if (NULL == ents) + return; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + -1, + ents, + (size_t) (nent * esz))) + { + free (ents); + return; + } + for (uint64_t i = 0; i < nent; i++) + { + const unsigned char *p = &ents[i * esz]; + uint16_t tag = geo_u16 (ctx, p); + + for (unsigned int k = 0; k < W_COUNT; k++) + { + struct Entry *e = &ctx->tags[k]; + + if ( (wanted_tags[k] != tag) || + (e->found) ) + continue; + e->type = geo_u16 (ctx, p + 2); + if (ctx->big) + { + e->count = geo_u64 (ctx, p + 4); + memcpy (e->val, + p + 12, + 8); + e->offset = geo_u64 (ctx, e->val); + } + else + { + e->count = (uint64_t) geo_u32 (ctx, p + 4); + memcpy (e->val, + p + 8, + 4); + e->offset = (uint64_t) geo_u32 (ctx, e->val); + } + e->found = 1; + break; + } + } + free (ents); + if (truncated) + return; /* the next-IFD pointer is not where we think it is */ + /* the next-IFD offset follows the entries */ + if (avail - declared * esz < nextsz) + return; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + -1, + cnt, + nextsz)) + return; + off = ctx->big + ? geo_u64 (ctx, cnt) + : (uint64_t) geo_u32 (ctx, cnt); + } +} + + +/** + * Human-readable name for the EPSG codes a first pass is most likely + * to run into. Anything else is reported as the bare code; resolving + * the full registry would mean shipping it. + * + * @param code the EPSG CRS code + * @param[out] buf where to write the name + * @param buf_size number of bytes in @a buf + * @return 1 if a name was written, 0 if the code is not one we know + */ +static int +epsg_name (unsigned int code, + char *buf, + size_t buf_size) +{ + static const struct + { + unsigned int code; + const char *name; + } known[] = { + { 4326, "WGS 84" }, + { 4269, "NAD83" }, + { 4258, "ETRS89" }, + { 4979, "WGS 84 (3D)" }, + { 3857, "WGS 84 / Pseudo-Mercator" }, + { 3395, "WGS 84 / World Mercator" }, + { 5714, "MSL height" }, + { 5773, "EGM96 height" }, + { 3855, "EGM2008 height" }, + { 0, NULL } + }; + + for (unsigned int i = 0; NULL != known[i].name; i++) + if (known[i].code == code) + { + if (((size_t) snprintf (buf, + buf_size, + "%s", + known[i].name)) >= buf_size) + return 0; + return 1; + } + if ( (32601 <= code) && + (32660 >= code) ) + { + snprintf (buf, + buf_size, + "WGS 84 / UTM zone %uN", + code - 32600); + return 1; + } + if ( (32701 <= code) && + (32760 >= code) ) + { + snprintf (buf, + buf_size, + "WGS 84 / UTM zone %uS", + code - 32700); + return 1; + } + return 0; +} + + +/** + * How many citation strings we remember in order not to report the + * same CRS name twice. + */ +#define GEOTIFF_MAX_CITATIONS 4 + + +/** + * The citation strings emitted so far, so that the very common case of + * GTCitationGeoKey and PCSCitationGeoKey carrying the same text does + * not produce the same value twice. + */ +struct Citations +{ + /** + * The strings, not NUL-terminated. + */ + char text[GEOTIFF_MAX_CITATIONS][128]; + + /** + * Length of each entry in #text. + */ + size_t len[GEOTIFF_MAX_CITATIONS]; + + /** + * How many entries of #text are in use. + */ + unsigned int count; +}; + + +/** + * Emit one GeoTIFF key whose value is a substring of + * GeoAsciiParamsTag. + * + * GeoASCII strings are packed into one blob and terminated by `|' + * rather than by NUL, because a NUL would end the whole TIFF ASCII + * field. + * + * @param ctx extraction state + * @param[in,out] cits citations reported so far + * @param ascii the GeoAsciiParamsTag bytes, NULL if the tag is absent + * @param ascii_len number of bytes in @a ascii + * @param off offset of the substring, from the key's Value_Offset + * @param count length of the substring, from the key's Count + * @param type meta data type to report it as + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_geo_ascii (const struct Context *ctx, + struct Citations *cits, + const unsigned char *ascii, + size_t ascii_len, + size_t off, + size_t count, + enum EXTRACTOR_MetaType type) +{ + size_t n; + + if ( (NULL == ascii) || + (off >= ascii_len) ) + return 0; + n = count; + if (n > ascii_len - off) + n = ascii_len - off; + /* The string runs to its `|' terminator. Trusting Count instead + would run one string into the next whenever Count is wrong, and it + is the file that chooses Count. */ + for (size_t i = 0; i < n; i++) + if ('|' == ascii[off + i]) + { + n = i; + break; + } + if (0 == n) + return 0; + for (unsigned int i = 0; i < cits->count; i++) + { + if ( (cits->len[i] == n) && + (0 == memcmp (cits->text[i], + &ascii[off], + n)) ) + return 0; /* already reported under another key */ + } + if ( (cits->count < GEOTIFF_MAX_CITATIONS) && + (n <= sizeof (cits->text[0])) ) + { + memcpy (cits->text[cits->count], + &ascii[off], + n); + cits->len[cits->count] = n; + cits->count++; + } + return EXTRACTOR_forensic_emit_text_ (ctx->ec, + "geotiff", + type, + (const char *) &ascii[off], + n); +} + + +/** + * Parse GeoKeyDirectoryTag, emitting the human-readable citations as + * we go and collecting the numeric keys into @a gk. + * + * @param ctx extraction state + * @param[out] gk where to store the numeric keys + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +parse_geo_keys (const struct Context *ctx, + struct GeoKeys *gk) +{ + struct Citations cits; + unsigned char *dir; + unsigned char *ascii = NULL; + size_t dir_len; + size_t ascii_len = 0; + size_t nvals; + size_t nkeys; + int stop = 0; + + memset (gk, + 0, + sizeof (*gk)); + memset (&cits, + 0, + sizeof (cits)); + if (TT_SHORT != ctx->tags[W_GEOKEYS].type) + return 0; + dir = load_value (ctx, + &ctx->tags[W_GEOKEYS], + 2 * (4 + 4 * GEOTIFF_MAX_KEYS), + &dir_len); + if (NULL == dir) + return 0; + nvals = dir_len / 2; + if (4 > nvals) + { + free (dir); + return 0; + } + nkeys = geo_u16 (ctx, + &dir[6]); + if (nkeys > GEOTIFF_MAX_KEYS) + nkeys = GEOTIFF_MAX_KEYS; + /* NumberOfKeys is not to be trusted over the tag's own count */ + if (nkeys > (nvals - 4) / 4) + nkeys = (nvals - 4) / 4; + if ( (0 != ctx->tags[W_GEOASCII].found) && + (TT_ASCII == ctx->tags[W_GEOASCII].type) ) + ascii = load_value (ctx, + &ctx->tags[W_GEOASCII], + GEOTIFF_MAX_VALUE, + &ascii_len); + for (size_t i = 0; (i < nkeys) && (0 == stop); i++) + { + const unsigned char *k = &dir[8 + i * 8]; + uint16_t key_id = geo_u16 (ctx, k); + uint16_t location = geo_u16 (ctx, k + 2); + uint16_t count = geo_u16 (ctx, k + 4); + uint16_t value = geo_u16 (ctx, k + 6); + + if (34737 == location) + { + /* a substring of GeoAsciiParamsTag: these are the CRS names */ + switch (key_id) + { + case 1026: /* GTCitationGeoKey */ + case 2049: /* GeogCitationGeoKey */ + case 3073: /* PCSCitationGeoKey */ + case 4097: /* VerticalCitationGeoKey */ + stop = emit_geo_ascii (ctx, + &cits, + ascii, + ascii_len, + value, + count, + EXTRACTOR_METATYPE_COORDINATE_SYSTEM); + break; + default: + break; + } + continue; + } + if (34736 == location) + continue; /* a DOUBLE in GeoDoubleParamsTag; none of the keys + we report is double-valued (they are ellipsoid + parameters and projection constants) */ + if (0 != location) + continue; /* not a location the specification defines */ + if (1 != count) + continue; /* an inline key is a single SHORT by definition */ + switch (key_id) + { + case 1024: /* GTModelTypeGeoKey */ + gk->model_type = value; + gk->have_model_type = 1; + break; + case 1025: /* GTRasterTypeGeoKey */ + gk->raster_type = value; + gk->have_raster_type = 1; + break; + case 2048: /* GeographicTypeGeoKey */ + gk->geographic = value; + gk->have_geographic = 1; + break; + case 3072: /* ProjectedCSTypeGeoKey */ + gk->projected = value; + gk->have_projected = 1; + break; + case 3076: /* ProjLinearUnitsGeoKey */ + gk->linear_units = value; + gk->have_linear_units = 1; + break; + case 4096: /* VerticalCSTypeGeoKey */ + gk->vertical = value; + gk->have_vertical = 1; + break; + default: + break; + } + } + free (ascii); + free (dir); + return stop; +} + + +/** + * Emit the coordinate reference system and the two descriptive keys. + * + * @param ctx extraction state + * @param gk the parsed GeoTIFF keys + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_crs (const struct Context *ctx, + const struct GeoKeys *gk) +{ + static const char *model_names[] = { + NULL, + "projected", + "geographic", + "geocentric" + }; + char name[128]; + unsigned int code = 0; + int have = 0; + + /* A projected CRS is the more specific statement, so it wins; the + geographic key is then only the datum the projection sits on. */ + if ( (gk->have_projected) && + (0 != gk->projected) && + (32767 != gk->projected) ) + { + code = gk->projected; + have = 1; + } + else if ( (gk->have_geographic) && + (0 != gk->geographic) && + (32767 != gk->geographic) ) + { + code = gk->geographic; + have = 1; + } + else if ( (gk->have_vertical) && + (0 != gk->vertical) && + (32767 != gk->vertical) ) + { + code = gk->vertical; + have = 1; + } + if (have) + { + if (epsg_name (code, + name, + sizeof (name))) + { + if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + "EPSG:%u (%s)", + code, + name)) + return 1; + } + else if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + "EPSG:%u", + code)) + { + return 1; + } + } + if ( (gk->have_vertical) && + (have) && + (code != gk->vertical) && + (0 != gk->vertical) && + (32767 != gk->vertical) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + "vertical EPSG:%u", + gk->vertical)) ) + return 1; + if ( (gk->have_model_type) && + (gk->model_type < sizeof (model_names) / sizeof (model_names[0])) && + (NULL != model_names[gk->model_type]) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COMMENT, + "GeoTIFF model type: %s", + model_names[gk->model_type])) ) + return 1; + if ( (gk->have_raster_type) && + ( (1 == gk->raster_type) || + (2 == gk->raster_type) ) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COMMENT, + "GeoTIFF raster type: %s", + (1 == gk->raster_type) + ? "pixel is area" + : "pixel is point")) ) + return 1; + return 0; +} + + +/** + * Name of the unit the model coordinates are expressed in. + * + * @param gk the parsed GeoTIFF keys + * @return a unit name, or NULL if we cannot tell + */ +static const char * +model_units (const struct GeoKeys *gk) +{ + if (gk->have_linear_units) + switch (gk->linear_units) + { + case 9001: + return "m"; + case 9002: + case 9003: + return "ft"; + case 9036: + return "km"; + default: + break; + } + if (gk->have_model_type) + { + if (2 == gk->model_type) + return "degrees"; /* a geographic CRS; angular units by definition */ + if (1 == gk->model_type) + return "m"; /* the overwhelmingly common projected case */ + } + if ( (gk->have_geographic) && + (! gk->have_projected) ) + return "degrees"; + return NULL; +} + + +/** + * Emit the georeferencing derived from the model tags: the pixel + * scale, the extent of the raster, and the tiepoint elevation. + * + * @param ctx extraction state + * @param gk the parsed GeoTIFF keys + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_geometry (const struct Context *ctx, + const struct GeoKeys *gk) +{ + const char *units = model_units (gk); + double width; + double height; + double sx; + double sy; + double sz; + double west = 0.0; + double east = 0.0; + double north = 0.0; + double south = 0.0; + int have_scale; + int have_box = 0; + + if ( (! tag_double (ctx, W_WIDTH, 0, &width)) || + (! tag_double (ctx, W_LENGTH, 0, &height)) || + (! isfinite (width)) || + (! isfinite (height)) || + (1.0 > width) || + (1.0 > height) ) + { + width = 0.0; + height = 0.0; + } + have_scale = (tag_double (ctx, W_PIXELSCALE, 0, &sx) && + tag_double (ctx, W_PIXELSCALE, 1, &sy) && + isfinite (sx) && + isfinite (sy) && + (0.0 != sx) && + (0.0 != sy)); + if ( (! tag_double (ctx, W_PIXELSCALE, 2, &sz)) || + (! isfinite (sz)) ) + sz = 0.0; + if (have_scale) + { + if (NULL != units) + { + if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_IMAGE_RESOLUTION, + "%.10g x %.10g %s/pixel", + sx, + sy, + units)) + return 1; + } + else if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_IMAGE_RESOLUTION, + "%.10g x %.10g per pixel", + sx, + sy)) + { + return 1; + } + } + /* Preferred georeferencing: one tiepoint plus a pixel scale. The + model coordinate of raster point (i,j) is + x = x_tie + (i - i_tie) * sx, y = y_tie - (j - j_tie) * sy + (y grows northwards while j grows downwards, which is why sy is + subtracted). Note that with GTRasterTypeGeoKey = RasterPixelIsArea + the tiepoint names the *corner* of the pixel and the extent below + is exact, whereas with RasterPixelIsPoint it names the pixel + *centre* and the true extent is half a pixel larger on every side. + We do not correct for that: half a pixel is not worth a second + code path, but it is worth knowing about. */ + if (have_scale && + (0.0 < width) && + (0.0 < height)) + { + double ti; + double tj; + double tx; + double ty; + + if (tag_double (ctx, W_TIEPOINT, 0, &ti) && + tag_double (ctx, W_TIEPOINT, 1, &tj) && + tag_double (ctx, W_TIEPOINT, 3, &tx) && + tag_double (ctx, W_TIEPOINT, 4, &ty) && + isfinite (ti) && isfinite (tj) && + isfinite (tx) && isfinite (ty)) + { + double x0 = tx + (0.0 - ti) * sx; + double x1 = tx + (width - ti) * sx; + double y0 = ty - (0.0 - tj) * sy; + double y1 = ty - (height - tj) * sy; + + west = (x0 < x1) ? x0 : x1; + east = (x0 < x1) ? x1 : x0; + south = (y0 < y1) ? y0 : y1; + north = (y0 < y1) ? y1 : y0; + have_box = 1; + } + } + if ( (! have_box) && + (0.0 < width) && + (0.0 < height) ) + { + /* Fall back on ModelTransformationTag, a row-major 4x4 matrix that + maps (i,j,k,1) to (x,y,z,1); only the first two rows matter for + an extent. The mapping may rotate, so take the extremes over + all four raster corners rather than assuming axis alignment. */ + double m[8]; + int ok = 1; + + for (unsigned int i = 0; i < 8; i++) + if ( (! tag_double (ctx, W_TRANSFORM, i, &m[i])) || + (! isfinite (m[i])) ) + { + ok = 0; + break; + } + if (ok) + { + static const double corner[4][2] = { + { 0.0, 0.0 }, { 1.0, 0.0 }, { 0.0, 1.0 }, { 1.0, 1.0 } + }; + + for (unsigned int c = 0; c < 4; c++) + { + double i = corner[c][0] * width; + double j = corner[c][1] * height; + double x = m[0] * i + m[1] * j + m[3]; + double y = m[4] * i + m[5] * j + m[7]; + + if ( (! isfinite (x)) || + (! isfinite (y)) ) + { + have_box = 0; + break; + } + if (0 == c) + { + west = east = x; + south = north = y; + have_box = 1; + continue; + } + if (x < west) + west = x; + if (x > east) + east = x; + if (y < south) + south = y; + if (y > north) + north = y; + } + } + } + /* The box is in whatever units the CRS uses -- degrees for a + geographic CRS, metres for a typical projected one. We emit it + unconverted and let COORDINATE_SYSTEM say which; reprojecting + would mean linking PROJ. */ + if ( (have_box) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_BOUNDING_BOX, + "%.10g,%.10g,%.10g,%.10g", + west, + south, + east, + north)) ) + return 1; + { + double tz; + + /* The tiepoint's z is only meaningful when the file actually + carries a vertical dimension; a plain 2D raster stores 0 there. */ + if ( (tag_double (ctx, W_TIEPOINT, 5, &tz)) && + (isfinite (tz)) && + (0.0 != tz) && + ( (gk->have_vertical) || + (0.0 != sz) ) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_LOCATION_ELEVATION, + "%.10g", + tz)) ) + return 1; + } + return 0; +} + + +/** + * Find @a needle in @a hay. + * + * @param hay where to search + * @param hay_len number of bytes in @a hay + * @param from index to start searching at + * @param needle NUL-terminated string to look for + * @param[out] pos index of the match + * @return 1 if found, 0 if not + */ +static int +find_sub (const char *hay, + size_t hay_len, + size_t from, + const char *needle, + size_t *pos) +{ + size_t nlen = strlen (needle); + + if ( (0 == nlen) || + (hay_len < nlen) ) + return 0; + for (size_t i = from; i + nlen <= hay_len; i++) + if (0 == memcmp (&hay[i], + needle, + nlen)) + { + *pos = i; + return 1; + } + return 0; +} + + +/** + * Pull `<Item name="...">value</Item>' pairs out of GDAL's private + * metadata tag. This is where a GDAL-produced raster keeps its real + * provenance: the acquisition time, the sensor, the processing chain. + * + * We do not decode XML entities or attributes beyond `name'; the point + * is a cheap characterisation, and a value that needed unescaping is + * still recognisable. + * + * @param ctx extraction state + * @param xml the tag contents + * @param len number of bytes in @a xml + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +scan_gdal_metadata (const struct Context *ctx, + const char *xml, + size_t len) +{ + /* GDAL item names that name the thing that made the data rather + than describing the data. */ + static const char *device_items[] = { + "SENSOR", + "SENSOR_NAME", + "SENSORNAME", + "SENSOR_ID", + "INSTRUMENT", + "PLATFORM", + "SATELLITEID", + NULL + }; + unsigned int items = 0; + size_t i = 0; + + while ( (i < len) && + (items < GEOTIFF_MAX_GDAL_ITEMS) ) + { + size_t open; + size_t tag_end; + size_t name_start; + size_t name_end; + size_t close; + size_t name_len; + size_t val_len; + enum EXTRACTOR_MetaType type = EXTRACTOR_METATYPE_COMMENT; + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + int n; + + if (! find_sub (xml, len, i, "<Item", &open)) + break; + if (! find_sub (xml, len, open, ">", &tag_end)) + break; + i = tag_end + 1; + if (! find_sub (xml, tag_end, open, "name=\"", &name_start)) + continue; /* an Item without a name; skip it, keep scanning */ + name_start += strlen ("name=\""); + if (! find_sub (xml, tag_end, name_start, "\"", &name_end)) + continue; + if (! find_sub (xml, len, tag_end, "</Item>", &close)) + break; + name_len = name_end - name_start; + val_len = close - (tag_end + 1); + i = close + strlen ("</Item>"); + items++; + if ( (0 == name_len) || + (64 < name_len) || + (0 == val_len) || + (256 < val_len) ) + continue; + for (unsigned int k = 0; NULL != device_items[k]; k++) + if ( (strlen (device_items[k]) == name_len) && + (0 == memcmp (device_items[k], + &xml[name_start], + name_len)) ) + { + type = EXTRACTOR_METATYPE_SOURCE_DEVICE; + break; + } + if (EXTRACTOR_METATYPE_SOURCE_DEVICE == type) + n = snprintf (buf, + sizeof (buf), + "%.*s", + (int) val_len, + &xml[tag_end + 1]); + else + n = snprintf (buf, + sizeof (buf), + "%.*s=%.*s", + (int) name_len, + &xml[name_start], + (int) val_len, + &xml[tag_end + 1]); + if ( (0 >= n) || + (((size_t) n) >= sizeof (buf)) ) + continue; + if (EXTRACTOR_forensic_emit_text_ (ctx->ec, + "geotiff", + type, + buf, + (size_t) n)) + return 1; + } + return 0; +} + + +/** + * Emit the GDAL private tags, if the file has them. + * + * @param ctx extraction state + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_gdal (const struct Context *ctx) +{ + unsigned char *buf; + size_t len; + int stop; + + if ( (ctx->tags[W_GDAL_NODATA].found) && + (TT_ASCII == ctx->tags[W_GDAL_NODATA].type) ) + { + buf = load_value (ctx, + &ctx->tags[W_GDAL_NODATA], + 64, + &len); + if (NULL != buf) + { + char nod[128]; + int n; + + /* trim to the C string GDAL wrote, then label it: a bare "-9999" + under COMMENT would be unreadable */ + for (size_t i = 0; i < len; i++) + if ('\0' == buf[i]) + { + len = i; + break; + } + n = snprintf (nod, + sizeof (nod), + "GDAL nodata value: %.*s", + (int) len, + (const char *) buf); + free (buf); + if ( (0 < n) && + (((size_t) n) < sizeof (nod)) && + (EXTRACTOR_forensic_emit_text_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COMMENT, + nod, + (size_t) n)) ) + return 1; + } + } + if ( (! ctx->tags[W_GDAL_META].found) || + (TT_ASCII != ctx->tags[W_GDAL_META].type) ) + return 0; + buf = load_value (ctx, + &ctx->tags[W_GDAL_META], + GEOTIFF_MAX_VALUE, + &len); + if (NULL == buf) + return 0; + for (size_t i = 0; i < len; i++) + if ('\0' == buf[i]) + { + len = i; + break; + } + stop = scan_gdal_metadata (ctx, + (const char *) buf, + len); + free (buf); + return stop; +} + + +/** + * Emit the plain TIFF descriptive tags: geometry, codec and the + * provenance strings. + * + * @param ctx extraction state + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_image_tags (const struct Context *ctx) +{ + static const struct + { + unsigned int code; + const char *name; + } codecs[] = { + { 1, "none" }, + { 2, "CCITT modified Huffman RLE" }, + { 3, "CCITT Group 3 fax" }, + { 4, "CCITT Group 4 fax" }, + { 5, "LZW" }, + { 6, "JPEG (old-style)" }, + { 7, "JPEG" }, + { 8, "Adobe Deflate" }, + { 32773, "PackBits" }, + { 32946, "Deflate" }, + { 34712, "JPEG 2000" }, + { 34887, "LERC" }, + { 34925, "LZMA" }, + { 50000, "ZSTD" }, + { 50001, "WEBP" }, + { 0, NULL } + }; + double width; + double height; + double bits; + double samples; + double comp; + + if ( (tag_double (ctx, W_WIDTH, 0, &width)) && + (tag_double (ctx, W_LENGTH, 0, &height)) && + (isfinite (width)) && + (isfinite (height)) && + (0.0 < width) && + (0.0 < height) && + (GEOTIFF_MAX_DIM > width) && + (GEOTIFF_MAX_DIM > height) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "%llux%llu", + (unsigned long long) width, + (unsigned long long) height)) ) + return 1; + if ( (tag_double (ctx, W_BITS, 0, &bits)) && + (isfinite (bits)) && + (0.0 < bits) && + (4096.0 > bits) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_COLOR_DEPTH, + "%llu", + (unsigned long long) bits)) ) + return 1; + if ( (tag_double (ctx, W_SAMPLES, 0, &samples)) && + (isfinite (samples)) && + (0.0 < samples) && + (4096.0 > samples) && + (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_CHANNELS, + "%llu", + (unsigned long long) samples)) ) + return 1; + if ( (tag_double (ctx, W_COMPRESSION, 0, &comp)) && + (isfinite (comp)) && + (0.0 <= comp) && + (65536.0 > comp) ) + { + unsigned int code = (unsigned int) comp; + const char *name = NULL; + + for (unsigned int i = 0; NULL != codecs[i].name; i++) + if (codecs[i].code == code) + { + name = codecs[i].name; + break; + } + if (NULL != name) + { + if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_CODEC, + "%s", + name)) + return 1; + } + else if (EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_CODEC, + "TIFF compression %u", + code)) + { + return 1; + } + } + if (emit_ascii_tag (ctx, + W_SOFTWARE, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE)) + return 1; + if (emit_ascii_tag (ctx, + W_DESCRIPTION, + EXTRACTOR_METATYPE_DESCRIPTION)) + return 1; + if (emit_ascii_tag (ctx, + W_ARTIST, + EXTRACTOR_METATYPE_AUTHOR_NAME)) + return 1; + if (emit_ascii_tag (ctx, + W_COPYRIGHT, + EXTRACTOR_METATYPE_COPYRIGHT)) + return 1; + if (emit_ascii_tag (ctx, + W_MAKE, + EXTRACTOR_METATYPE_DEVICE_MANUFACTURER)) + return 1; + if (emit_ascii_tag (ctx, + W_MODEL, + EXTRACTOR_METATYPE_DEVICE_MODEL)) + return 1; + return 0; +} + + +/** + * Emit TIFF's DateTime tag, which is `YYYY:MM:DD HH:MM:SS' in + * unspecified local time. + * + * We re-spell it as ISO 8601 but deliberately do not append a `Z': + * TIFF 6.0 attaches no time zone to the field, and claiming UTC would + * be inventing information. + * + * @param ctx extraction state + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_datetime (const struct Context *ctx) +{ + unsigned char *buf; + size_t len; + unsigned int v[6]; + int stop; + + if (TT_ASCII != ctx->tags[W_DATETIME].type) + return 0; + buf = load_value (ctx, + &ctx->tags[W_DATETIME], + 64, + &len); + if (NULL == buf) + return 0; + if (19 > len) + { + free (buf); + return 0; + } + /* fixed layout: 4-2-2 date, space, 2-2-2 time, all digits */ + { + static const unsigned int digit_at[] = { + 0, 1, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18 + }; + int ok = ( (':' == buf[4]) && + (':' == buf[7]) && + (' ' == buf[10]) && + (':' == buf[13]) && + (':' == buf[16]) ); + + for (unsigned int i = 0; + (ok) && (i < sizeof (digit_at) / sizeof (digit_at[0])); + i++) + if ( ('0' > buf[digit_at[i]]) || + ('9' < buf[digit_at[i]]) ) + ok = 0; + if (! ok) + { + free (buf); + return 0; + } + } + v[0] = (unsigned int) ((buf[0] - '0') * 1000 + (buf[1] - '0') * 100 + + (buf[2] - '0') * 10 + (buf[3] - '0')); + v[1] = (unsigned int) ((buf[5] - '0') * 10 + (buf[6] - '0')); + v[2] = (unsigned int) ((buf[8] - '0') * 10 + (buf[9] - '0')); + v[3] = (unsigned int) ((buf[11] - '0') * 10 + (buf[12] - '0')); + v[4] = (unsigned int) ((buf[14] - '0') * 10 + (buf[15] - '0')); + v[5] = (unsigned int) ((buf[17] - '0') * 10 + (buf[18] - '0')); + free (buf); + if ( (1900 > v[0]) || + (2200 < v[0]) || + (1 > v[1]) || (12 < v[1]) || + (1 > v[2]) || (31 < v[2]) || + (23 < v[3]) || (59 < v[4]) || (60 < v[5]) ) + return 0; /* a zeroed or garbage field, not a date */ + stop = EXTRACTOR_forensic_emit_ (ctx->ec, + "geotiff", + EXTRACTOR_METATYPE_CREATION_DATE, + "%04u-%02u-%02uT%02u:%02u:%02u", + v[0], v[1], v[2], v[3], v[4], v[5]); + return stop; +} + + +/** + * Main entry method for the GeoTIFF extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_geotiff_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_geotiff_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + struct Context ctx; + struct GeoKeys gk; + unsigned char hdr[16]; + uint64_t first; + uint16_t version; + + memset (&ctx, + 0, + sizeof (ctx)); + ctx.ec = ec; + ctx.fsize = ec->get_size (ec->cls); + if ( (UINT64_MAX == ctx.fsize) || + (16 > ctx.fsize) || + (ctx.fsize > (uint64_t) INT64_MAX) ) + return; + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + sizeof (hdr))) + return; + /* Bail out on the very first bytes: almost nothing we are handed is + a TIFF, and this plugin must cost nothing for the rest. */ + if (0 == memcmp (hdr, "II", 2)) + ctx.be = 0; + else if (0 == memcmp (hdr, "MM", 2)) + ctx.be = 1; + else + return; + version = geo_u16 (&ctx, + &hdr[2]); + if (42 == version) + { + ctx.big = 0; + first = (uint64_t) geo_u32 (&ctx, + &hdr[4]); + } + else if (43 == version) + { + /* BigTIFF: the header states the offset width, which the format + has so far only ever used as 8. */ + ctx.big = 1; + if (8 != geo_u16 (&ctx, &hdr[4])) + return; + if (0 != geo_u16 (&ctx, &hdr[6])) + return; + first = geo_u64 (&ctx, + &hdr[8]); + } + else + { + return; /* a TIFF-like magic we do not know */ + } + walk_ifds (&ctx, + first); + /* This is the whole point of the plugin being separate from `tiff': + unless one of the four georeferencing tags is present the file is + an ordinary TIFF and we say nothing at all about it. */ + if ( (! ctx.tags[W_PIXELSCALE].found) && + (! ctx.tags[W_TIEPOINT].found) && + (! ctx.tags[W_TRANSFORM].found) && + (! ctx.tags[W_GEOKEYS].found) ) + return; + if (0 != + ec->proc (ec->cls, + "geotiff", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "image/tiff", + strlen ("image/tiff") + 1)) + return; + if (0 != + ec->proc (ec->cls, + "geotiff", + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "GeoTIFF", + strlen ("GeoTIFF") + 1)) + return; + if (ctx.big && + (0 != + ec->proc (ec->cls, + "geotiff", + EXTRACTOR_METATYPE_FORMAT_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "BigTIFF", + strlen ("BigTIFF") + 1))) + return; + if (emit_image_tags (&ctx)) + return; + if (emit_datetime (&ctx)) + return; + if (parse_geo_keys (&ctx, + &gk)) + return; + if (emit_crs (&ctx, + &gk)) + return; + if (emit_geometry (&ctx, + &gk)) + return; + if (emit_gdal (&ctx)) + return; +} + + +/* end of geotiff_extractor.c */ diff --git a/src/plugins/gpx_extractor.c b/src/plugins/gpx_extractor.c @@ -0,0 +1,1590 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/gpx_extractor.c + * @brief plugin to support GPX (GPS Exchange Format) track logs + * @author Christian Grothoff + * + * GPX is raw location history: where a device was, when, and -- through + * the `creator' attribute -- which device or application recorded it. + * That makes the header of a GPX file one of the more identifying things + * a volume can contain, which is why this plugin exists. + * + * This is deliberately *not* an XML parser. libextractor has no XML + * dependency and this plugin must stay unconditional, so what follows is + * a bounded, single-pass scanner over at most #GPX_SCAN_CAP bytes. Its + * limits, stated once here rather than repeated at every call site: + * + * - Comments (`<!-- ... -->') and processing instructions are not + * skipped, so an element name that only occurs inside a comment is + * still seen. For the fields below that means a false positive, never + * an out-of-bounds read. + * - CDATA is recognised only when it opens the content of an element + * whose text we ask for; a CDATA section in the middle of mixed + * content is treated as markup. + * - Only the five predefined entities and numeric character references + * are expanded; a document-defined entity is passed through verbatim. + * - Element content is the run of text up to the next `<'; nested + * elements inside a value are not concatenated. + * - Namespace prefixes are skipped, so `<gpx:name>' and `<name>' are the + * same element here. A document that binds a prefix to some *other* + * namespace would be misread; GPX files in the wild do not. + * - Attribute values are read as raw text; the scanner does not care + * whether a document is well-formed. + * + * None of that can make the scanner read outside the buffer or fail to + * make forward progress: every loop below is bounded by the buffer + * length and advances at least one byte per iteration. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +#include <math.h> + + +/** + * How many bytes of the file we are willing to look at. GPX track logs + * routinely run to tens of megabytes; a first pass over a volume must + * not read all of that. Counts derived from a truncated scan are + * reported as lower bounds, with a comment saying so. + */ +#define GPX_SCAN_CAP (1024 * 1024) + +/** + * Longest start tag we will scan for its closing `>'. Keeps a file made + * of one unterminated tag from turning the per-element work into a scan + * of the whole buffer. + */ +#define GPX_MAX_TAG 8192 + +/** + * Longest element text content we will look at. + */ +#define GPX_MAX_TEXT 4096 + +/** + * How far into the file the `<gpx' element has to appear. + */ +#define GPX_MAGIC_WINDOW 2048 + +/** + * Most attributes we parse out of a single start tag. + */ +#define GPX_MAX_ATTRS 64 + +/** + * Mean Earth radius in metres, as used for the haversine sum. + */ +#define GPX_EARTH_RADIUS 6371000.0 + + +/** + * Everything we picked up while walking the track points. + */ +struct GpxScan +{ + /** + * Bounding box of the points seen, west/south/east/north. + */ + double minlon; + double minlat; + double maxlon; + double maxlat; + + /** + * Coordinates of the first point, which is where the recording + * started. + */ + double first_lat; + double first_lon; + + /** + * Elevation range seen inside track points. + */ + double minele; + double maxele; + + /** + * Sum of the great-circle distances between consecutive track points. + */ + double distance; + + /** + * Latitude/longitude of the previous track point, for @e distance. + */ + double prev_lat; + double prev_lon; + + /** + * First and last timestamp seen inside a track point. + */ + int64_t first_time; + int64_t last_time; + + /** + * Number of `<trkpt>', `<wpt>' and `<rtept>' elements. + */ + uint64_t trkpts; + uint64_t wpts; + uint64_t rtepts; + + /** + * Non-zero once at least one point of any kind was seen; before that + * an `<ele>' or `<time>' belongs to the metadata, not to a point. + */ + int seen_point; + + /** + * Non-zero once #prev_lat / #prev_lon hold a track point. + */ + int have_prev; + + /** + * Non-zero once the bounding box fields are meaningful. + */ + int have_bbox; + + /** + * Non-zero once the elevation range fields are meaningful. + */ + int have_ele; +}; + + +/** + * Can @a c appear in an XML name (after the first character)? + * + * @param c character to test + * @return 1 if @a c is a name character, 0 if not + */ +static int +gpx_name_char (char c) +{ + return ( ( ('a' <= c) && ('z' >= c) ) || + ( ('A' <= c) && ('Z' >= c) ) || + ( ('0' <= c) && ('9' >= c) ) || + ('_' == c) || ('-' == c) || ('.' == c) ); +} + + +/** + * Is the element that starts at @a pos named @a tag? + * + * A namespace prefix is skipped, so `<gpx:name' matches "name". + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param tag element name to compare against + * @return 1 on a match, 0 otherwise + */ +static int +gpx_tag_is (const char *buf, + size_t len, + size_t pos, + const char *tag) +{ + size_t i; + size_t start; + size_t tlen = strlen (tag); + + if ( (pos >= len) || + ('<' != buf[pos]) ) + return 0; + i = pos + 1; + start = i; + while ( (i < len) && + (i - start < 64) && + gpx_name_char (buf[i]) ) + i++; + if ( (i < len) && + (':' == buf[i]) ) + { + i++; + start = i; + while ( (i < len) && + (i - start < 64) && + gpx_name_char (buf[i]) ) + i++; + } + if (i - start != tlen) + return 0; + return (0 == memcmp (&buf[start], + tag, + tlen)); +} + + +/** + * Find the `>' that closes the start tag beginning at @a pos. Quoted + * attribute values may contain `>', so quoting is tracked. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @return offset of the `>', or `(size_t) -1' if there is none within + * #GPX_MAX_TAG bytes + */ +static size_t +gpx_tag_end (const char *buf, + size_t len, + size_t pos) +{ + char quote = 0; + + for (size_t i = pos; (i < len) && (i - pos < GPX_MAX_TAG); i++) + { + char c = buf[i]; + + if (0 != quote) + { + if (c == quote) + quote = 0; + continue; + } + if ( ('"' == c) || + ('\'' == c) ) + { + quote = c; + continue; + } + if ('>' == c) + return i; + } + return (size_t) -1; +} + + +/** + * Find the value of attribute @a name in the start tag at @a pos. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param name attribute name, without a namespace prefix + * @param[out] vstart offset of the first byte of the value + * @param[out] vlen number of bytes in the value + * @return 1 if the attribute was found, 0 if not + */ +static int +gpx_attr (const char *buf, + size_t len, + size_t pos, + const char *name, + size_t *vstart, + size_t *vlen) +{ + size_t end = gpx_tag_end (buf, + len, + pos); + size_t i; + size_t nlen = strlen (name); + + if (((size_t) -1) == end) + return 0; + i = pos + 1; + while ( (i < end) && + (gpx_name_char (buf[i]) || + (':' == buf[i]) ) ) + i++; + for (unsigned int n = 0; n < GPX_MAX_ATTRS; n++) + { + size_t astart; + size_t alen; + size_t vs; + size_t ve; + char quote; + + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if (i >= end) + break; + astart = i; + while ( (i < end) && + (gpx_name_char (buf[i]) || + (':' == buf[i]) ) ) + i++; + if (i == astart) + { + i++; /* not a name: skip a byte so we always progress */ + continue; + } + alen = i - astart; + /* a prefixed attribute matches on its local part */ + for (size_t k = 0; k < alen; k++) + if (':' == buf[astart + k]) + { + astart += k + 1; + alen -= k + 1; + break; + } + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if ( (i >= end) || + ('=' != buf[i]) ) + continue; /* valueless attribute; i already advanced past a name */ + i++; + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if (i >= end) + break; + quote = buf[i]; + if ( ('"' == quote) || + ('\'' == quote) ) + { + i++; + vs = i; + while ( (i < end) && + (quote != buf[i]) ) + i++; + ve = i; + if (i < end) + i++; + } + else + { + vs = i; + while ( (i < end) && + (' ' != buf[i]) && ('\t' != buf[i]) && + ('\r' != buf[i]) && ('\n' != buf[i]) && + ('/' != buf[i]) ) + i++; + ve = i; + } + if ( (alen == nlen) && + (0 == memcmp (&buf[astart], + name, + nlen)) ) + { + *vstart = vs; + *vlen = ve - vs; + return 1; + } + } + return 0; +} + + +/** + * Text content of the element whose start tag begins at @a pos. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param[out] tstart offset of the first content byte + * @param[out] tlen number of content bytes + * @return 1 if content was found, 0 if the element is empty or the tag + * is malformed + */ +static int +gpx_text (const char *buf, + size_t len, + size_t pos, + size_t *tstart, + size_t *tlen) +{ + size_t end = gpx_tag_end (buf, + len, + pos); + size_t i; + size_t j; + + if (((size_t) -1) == end) + return 0; + if ( (end > pos) && + ('/' == buf[end - 1]) ) + return 0; /* self-closing element, no content */ + i = end + 1; + if ( (i + 9 <= len) && + (0 == memcmp (&buf[i], + "<![CDATA[", + 9)) ) + { + j = i + 9; + while ( (j + 3 <= len) && + (j - i < GPX_MAX_TEXT) ) + { + if (0 == memcmp (&buf[j], + "]]>", + 3)) + { + *tstart = i + 9; + *tlen = j - (i + 9); + return 1; + } + j++; + } + return 0; + } + j = i; + while ( (j < len) && + ('<' != buf[j]) && + (j - i < GPX_MAX_TEXT) ) + j++; + *tstart = i; + *tlen = j - i; + return (0 != *tlen); +} + + +/** + * Find the next element named @a tag at or after @a from, staying below + * @a limit. + * + * @param buf the buffer + * @param limit offset one past the last byte to search + * @param from where to start searching + * @param tag element name + * @return offset of the `<', or `(size_t) -1' if not found + */ +static size_t +gpx_find (const char *buf, + size_t limit, + size_t from, + const char *tag) +{ + for (size_t i = from; i < limit; i++) + { + if ('<' != buf[i]) + continue; + if (gpx_tag_is (buf, + limit, + i, + tag)) + return i; + } + return (size_t) -1; +} + + +/** + * Expand the five predefined XML entities and numeric character + * references in @a in. Anything else is copied through unchanged. + * + * @param in input text + * @param inlen number of bytes in @a in + * @param[out] out where to write the result + * @param outsize number of bytes available in @a out + * @return number of bytes written to @a out + */ +static size_t +gpx_unescape (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + size_t o = 0; + size_t i = 0; + + while ( (i < inlen) && + (o + 5 < outsize) ) + { + size_t j; + uint32_t cp = 0; + + if ('&' != in[i]) + { + out[o++] = in[i++]; + continue; + } + /* find the terminating `;' within a short window */ + j = i + 1; + while ( (j < inlen) && + (j - i < 12) && + (';' != in[j]) ) + j++; + if ( (j >= inlen) || + (';' != in[j]) ) + { + out[o++] = in[i++]; + continue; + } + /* j is the offset of the `;', so the reference is j + 1 - i bytes */ + if ( (4 == j + 1 - i) && + (0 == memcmp (&in[i], "<", 4)) ) + cp = '<'; + else if ( (4 == j + 1 - i) && + (0 == memcmp (&in[i], ">", 4)) ) + cp = '>'; + else if ( (5 == j + 1 - i) && + (0 == memcmp (&in[i], "&", 5)) ) + cp = '&'; + else if ( (6 == j + 1 - i) && + (0 == memcmp (&in[i], """, 6)) ) + cp = '"'; + else if ( (6 == j + 1 - i) && + (0 == memcmp (&in[i], "'", 6)) ) + cp = '\''; + else if ( (j - i > 2) && + ('#' == in[i + 1]) ) + { + size_t k = i + 2; + int base = 10; + + if ( (k < j) && + ( ('x' == in[k]) || ('X' == in[k]) ) ) + { + base = 16; + k++; + } + if (k == j) + { + out[o++] = in[i++]; + continue; + } + while (k < j) + { + int d; + + if ( ('0' <= in[k]) && ('9' >= in[k]) ) + d = in[k] - '0'; + else if ( (16 == base) && ('a' <= in[k]) && ('f' >= in[k]) ) + d = in[k] - 'a' + 10; + else if ( (16 == base) && ('A' <= in[k]) && ('F' >= in[k]) ) + d = in[k] - 'A' + 10; + else + break; + if (cp > 0x110000 / (uint32_t) base) + { + cp = 0; + break; + } + cp = cp * (uint32_t) base + (uint32_t) d; + k++; + } + if ( (k != j) || + (0 == cp) || + (cp > 0x10FFFF) || + ( (0xD800 <= cp) && (0xDFFF >= cp) ) ) + { + out[o++] = in[i++]; + continue; + } + } + else + { + out[o++] = in[i++]; + continue; /* unknown entity: leave it alone */ + } + if (cp < 0x80) + { + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + i = j + 1; + } + return o; +} + + +/** + * Emit a stretch of XML text after expanding entity references. + * + * @param ec extraction context + * @param type meta data type + * @param data the text + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +gpx_emit_xml (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const char *data, + size_t len) +{ + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t out; + + if (len > sizeof (buf) - 8) + len = sizeof (buf) - 8; + out = gpx_unescape (data, + len, + buf, + sizeof (buf)); + return EXTRACTOR_forensic_emit_text_ (ec, + "gpx", + type, + buf, + out); +} + + +/** + * Parse a decimal number out of a stretch of bytes that is not + * NUL-terminated. + * + * @param data the bytes + * @param len number of bytes in @a data + * @param[out] value where to store the result + * @return 1 on success, 0 if @a data does not start with a number + */ +static int +gpx_parse_double (const char *data, + size_t len, + double *value) +{ + char tmp[64]; + char *endp; + double v; + size_t i = 0; + size_t o = 0; + + while ( (i < len) && + ( (' ' == data[i]) || ('\t' == data[i]) || + ('\r' == data[i]) || ('\n' == data[i]) ) ) + i++; + while ( (i < len) && + (o < sizeof (tmp) - 1) && + ( ( ('0' <= data[i]) && ('9' >= data[i]) ) || + ('+' == data[i]) || ('-' == data[i]) || + ('.' == data[i]) || ('e' == data[i]) || ('E' == data[i]) ) ) + tmp[o++] = data[i++]; + tmp[o] = '\0'; + if (0 == o) + return 0; + v = strtod (tmp, + &endp); + if ( (endp == tmp) || + (! isfinite (v)) ) + return 0; + *value = v; + return 1; +} + + +/** + * Days since 1970-01-01 for a proleptic Gregorian date. (Howard + * Hinnant's `days_from_civil'.) + * + * @param y year + * @param m month, 1-12 + * @param d day of month, 1-31 + * @return day number, negative before the epoch + */ +static int64_t +gpx_days_from_civil (int64_t y, + int64_t m, + int64_t d) +{ + int64_t era; + int64_t yoe; + int64_t doy; + int64_t doe; + + y -= (m <= 2); + era = (y >= 0 ? y : y - 399) / 400; + yoe = y - era * 400; + doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + + +/** + * Parse an ISO 8601 timestamp as GPX writes it: + * `YYYY-MM-DDThh:mm:ss[.fff][Z|(+|-)hh[:mm]]'. A missing zone is taken + * as UTC, which is what the GPX schema requires anyway. + * + * @param s the text + * @param len number of bytes in @a s + * @param[out] when where to store seconds since the Unix epoch + * @return 1 on success, 0 if @a s is not a timestamp + */ +static int +gpx_parse_iso8601 (const char *s, + size_t len, + int64_t *when) +{ + int64_t v[6] = { 0, 0, 0, 0, 0, 0 }; + static const size_t widths[6] = { 4, 2, 2, 2, 2, 2 }; + static const char seps[5] = { '-', '-', 'T', ':', ':' }; + size_t i = 0; + int64_t off = 0; + + while ( (i < len) && + ( (' ' == s[i]) || ('\t' == s[i]) || + ('\r' == s[i]) || ('\n' == s[i]) ) ) + i++; + for (unsigned int f = 0; f < 6; f++) + { + if (i + widths[f] > len) + return 0; + for (size_t k = 0; k < widths[f]; k++) + { + if ( ('0' > s[i + k]) || + ('9' < s[i + k]) ) + return 0; + v[f] = v[f] * 10 + (s[i + k] - '0'); + } + i += widths[f]; + if (f < 5) + { + if (i >= len) + return 0; + if ( (2 == f) && + (' ' == s[i]) ) + i++; /* `YYYY-MM-DD hh:mm:ss' also occurs */ + else if (seps[f] == s[i]) + i++; + else + return 0; + } + } + if ( (v[1] < 1) || (v[1] > 12) || + (v[2] < 1) || (v[2] > 31) || + (v[3] > 23) || (v[4] > 59) || (v[5] > 60) ) + return 0; + if ( (i < len) && + ('.' == s[i]) ) + { + i++; + while ( (i < len) && + ('0' <= s[i]) && ('9' >= s[i]) ) + i++; + } + if ( (i < len) && + ( ('+' == s[i]) || ('-' == s[i]) ) ) + { + int neg = ('-' == s[i]); + int64_t oh = 0; + int64_t om = 0; + + i++; + if (i + 2 > len) + return 0; + for (size_t k = 0; k < 2; k++) + { + if ( ('0' > s[i + k]) || ('9' < s[i + k]) ) + return 0; + oh = oh * 10 + (s[i + k] - '0'); + } + i += 2; + if ( (i < len) && + (':' == s[i]) ) + i++; + if (i + 2 <= len) + { + if ( ('0' <= s[i]) && ('9' >= s[i]) && + ('0' <= s[i + 1]) && ('9' >= s[i + 1]) ) + { + om = (s[i] - '0') * 10 + (s[i + 1] - '0'); + i += 2; + } + } + if ( (oh > 23) || (om > 59) ) + return 0; + off = oh * 3600 + om * 60; + if (neg) + off = -off; + } + *when = gpx_days_from_civil (v[0], + v[1], + v[2]) * 86400 + + v[3] * 3600 + v[4] * 60 + v[5] + - off; + return 1; +} + + +/** + * Great-circle distance between two points, in metres. + * + * @param lat1 latitude of the first point, in degrees + * @param lon1 longitude of the first point, in degrees + * @param lat2 latitude of the second point, in degrees + * @param lon2 longitude of the second point, in degrees + * @return distance in metres + */ +static double +gpx_haversine (double lat1, + double lon1, + double lat2, + double lon2) +{ + double rad = M_PI / 180.0; + double dlat = (lat2 - lat1) * rad; + double dlon = (lon2 - lon1) * rad; + double a; + + a = sin (dlat / 2.0) * sin (dlat / 2.0) + + cos (lat1 * rad) * cos (lat2 * rad) + * sin (dlon / 2.0) * sin (dlon / 2.0); + if (a < 0.0) + a = 0.0; + if (a > 1.0) + a = 1.0; + return 2.0 * GPX_EARTH_RADIUS * asin (sqrt (a)); +} + + +/** + * Record a point of any kind in @a sc. + * + * @param sc scan state + * @param lat latitude in degrees + * @param lon longitude in degrees + * @param track 1 if this is a `<trkpt>' (which also feeds the distance + * sum), 0 for a waypoint or route point + */ +static void +gpx_note_point (struct GpxScan *sc, + double lat, + double lon, + int track) +{ + if ( (lat < -90.0) || (lat > 90.0) || + (lon < -180.0) || (lon > 180.0) ) + return; /* out of range: not a coordinate, do not let it skew the box */ + if (! sc->have_bbox) + { + sc->minlat = sc->maxlat = lat; + sc->minlon = sc->maxlon = lon; + sc->first_lat = lat; + sc->first_lon = lon; + sc->have_bbox = 1; + } + else + { + if (lat < sc->minlat) + sc->minlat = lat; + if (lat > sc->maxlat) + sc->maxlat = lat; + if (lon < sc->minlon) + sc->minlon = lon; + if (lon > sc->maxlon) + sc->maxlon = lon; + } + if (track) + { + if (sc->have_prev) + sc->distance += gpx_haversine (sc->prev_lat, + sc->prev_lon, + lat, + lon); + sc->prev_lat = lat; + sc->prev_lon = lon; + sc->have_prev = 1; + } +} + + +/** + * Walk the buffer once, tallying points, coordinates, elevations and + * timestamps. + * + * The walk is linear and every iteration advances at least one byte, so + * it is bounded by @a len regardless of what the file contains. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param[out] sc scan state to fill in + */ +static void +gpx_walk_points (const char *buf, + size_t len, + struct GpxScan *sc) +{ + for (size_t i = 0; i < len; i++) + { + size_t vs; + size_t vl; + double lat; + double lon; + /* 0 = not a point, 1 = <trkpt>, 2 = <wpt>, 3 = <rtept> */ + unsigned int kind; + + if ('<' != buf[i]) + continue; + if (gpx_tag_is (buf, len, i, "trkpt")) + { + kind = 1; + } + else if (gpx_tag_is (buf, len, i, "wpt")) + { + kind = 2; + } + else if (gpx_tag_is (buf, len, i, "rtept")) + { + kind = 3; + } + else if (sc->seen_point && + gpx_tag_is (buf, len, i, "ele") ) + { + double ele; + + if (gpx_text (buf, len, i, &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &ele) && + (ele > -20000.0) && (ele < 20000.0) ) + { + if (! sc->have_ele) + { + sc->minele = sc->maxele = ele; + sc->have_ele = 1; + } + else + { + if (ele < sc->minele) + sc->minele = ele; + if (ele > sc->maxele) + sc->maxele = ele; + } + } + continue; + } + else if (sc->seen_point && + gpx_tag_is (buf, len, i, "time") ) + { + int64_t when; + + if (gpx_text (buf, len, i, &vs, &vl) && + gpx_parse_iso8601 (&buf[vs], vl, &when) ) + { + if (0 == sc->first_time) + sc->first_time = when; + sc->last_time = when; + } + continue; + } + else + { + continue; + } + if (1 == kind) + sc->trkpts++; + else if (2 == kind) + sc->wpts++; + else + sc->rtepts++; + sc->seen_point = 1; + if (gpx_attr (buf, len, i, "lat", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &lat) && + gpx_attr (buf, len, i, "lon", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &lon) ) + gpx_note_point (sc, + lat, + lon, + (1 == kind)); + } +} + + +/** + * Does @a creator name a piece of hardware rather than an application? + * + * Purely a substring test against the GPS makers whose devices write + * GPX. It only decides whether we *additionally* report the value as + * #EXTRACTOR_METATYPE_DEVICE_MODEL; the value is always reported as + * #EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE. + * + * @param creator the `creator' attribute value + * @param len number of bytes in @a creator + * @return 1 if it looks like a device model, 0 otherwise + */ +static int +gpx_looks_like_device (const char *creator, + size_t len) +{ + static const char *vendors[] = { + "garmin", "etrex", "oregon", "gpsmap", "edge", "fenix", "forerunner", + "montana", "dakota", "colorado", "magellan", "suunto", "polar", + "wahoo", "elemnt", "bryton", "lezyne", "sigma rox", "coros", + "tomtom", "holux", "qstarz", "columbus", "igotu", "trimble", + NULL + }; + char low[128]; + size_t n = len; + + if (n > sizeof (low) - 1) + n = sizeof (low) - 1; + for (size_t i = 0; i < n; i++) + { + char c = creator[i]; + + if ( ('A' <= c) && ('Z' >= c) ) + c = (char) (c - 'A' + 'a'); + low[i] = c; + } + low[n] = '\0'; + for (unsigned int i = 0; NULL != vendors[i]; i++) + if (NULL != strstr (low, + vendors[i])) + return 1; + return 0; +} + + +/** + * Report everything the `<metadata>' block holds. + * + * @param ec extraction context + * @param buf the buffer + * @param len number of bytes in @a buf + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +gpx_do_metadata (struct EXTRACTOR_ExtractContext *ec, + const char *buf, + size_t len) +{ + size_t md; + size_t mdend; + size_t p; + size_t vs; + size_t vl; + + md = gpx_find (buf, + len, + 0, + "metadata"); + if (((size_t) -1) == md) + { + /* GPX 1.0 has no <metadata> element: name, desc, author, email, + url, time, keywords and bounds are direct children of <gpx>. + Treat everything from the root element up to the first <wpt>, + <rte> or <trk> as the metadata block. */ + md = gpx_find (buf, + len, + 0, + "gpx"); + if (((size_t) -1) == md) + return 0; + mdend = len; + for (size_t i = md + 1; i < len; i++) + { + if ('<' != buf[i]) + continue; + if (gpx_tag_is (buf, len, i, "wpt") || + gpx_tag_is (buf, len, i, "rte") || + gpx_tag_is (buf, len, i, "trk") ) + { + mdend = i; + break; + } + } + } + else + { + /* the block ends at `</metadata>'; if that is missing (truncated + file), fall back to the end of the buffer */ + mdend = len; + for (size_t i = md; i + 11 <= len; i++) + if (0 == memcmp (&buf[i], + "</metadata>", + 11)) + { + mdend = i; + break; + } + } + p = gpx_find (buf, mdend, md + 1, "name"); + if ( (((size_t) -1) != p) && + gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_TITLE, &buf[vs], vl) ) + return 1; + p = gpx_find (buf, mdend, md + 1, "desc"); + if ( (((size_t) -1) != p) && + gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_DESCRIPTION, &buf[vs], vl) ) + return 1; + p = gpx_find (buf, mdend, md + 1, "keywords"); + if ( (((size_t) -1) != p) && + gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_KEYWORDS, &buf[vs], vl) ) + return 1; + /* The author is `<author><name>' in GPX 1.1 and a plain + `<author>text</author>' in GPX 1.0. */ + p = gpx_find (buf, mdend, md + 1, "author"); + if (((size_t) -1) != p) + { + size_t a = gpx_find (buf, mdend, p + 1, "name"); + int emitted = 0; + + if ( (((size_t) -1) != a) && + gpx_text (buf, mdend, a, &vs, &vl) ) + { + emitted = 1; + if (gpx_emit_xml (ec, EXTRACTOR_METATYPE_AUTHOR_NAME, &buf[vs], vl)) + return 1; + } + if ( (! emitted) && + gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_AUTHOR_NAME, &buf[vs], vl) ) + return 1; + } + /* `<email id="user" domain="host"/>' in GPX 1.1 -- split so that the + file does not read as a harvestable address -- and a plain + `<email>user@host</email>' in GPX 1.0. */ + p = gpx_find (buf, mdend, md + 1, "email"); + if (((size_t) -1) != p) + { + size_t ds; + size_t dl; + + if (gpx_attr (buf, mdend, p, "id", &vs, &vl) && + gpx_attr (buf, mdend, p, "domain", &ds, &dl) ) + { + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_AUTHOR_EMAIL, + "%.*s@%.*s", + (int) vl, + &buf[vs], + (int) dl, + &buf[ds])) + return 1; + } + else if (gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, + EXTRACTOR_METATYPE_AUTHOR_EMAIL, + &buf[vs], + vl) ) + { + return 1; + } + } + /* `<copyright author="...">' with an optional `<license>' child in + GPX 1.1; a plain text element in GPX 1.0. */ + p = gpx_find (buf, mdend, md + 1, "copyright"); + if (((size_t) -1) != p) + { + if (gpx_attr (buf, mdend, p, "author", &vs, &vl)) + { + if (gpx_emit_xml (ec, EXTRACTOR_METATYPE_COPYRIGHT, &buf[vs], vl)) + return 1; + } + else if (gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_COPYRIGHT, &buf[vs], vl) ) + { + return 1; + } + { + size_t l = gpx_find (buf, mdend, p + 1, "license"); + + if ( (((size_t) -1) != l) && + gpx_text (buf, mdend, l, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_LICENSE, &buf[vs], vl) ) + return 1; + } + } + /* every `<link href="...">' in the block, capped */ + { + size_t l = md; + unsigned int n = 0; + + while (n < EXTRACTOR_FORENSIC_MAX_ITEMS) + { + l = gpx_find (buf, mdend, l + 1, "link"); + if (((size_t) -1) == l) + break; + if (gpx_attr (buf, mdend, l, "href", &vs, &vl)) + { + n++; + if (gpx_emit_xml (ec, EXTRACTOR_METATYPE_URL, &buf[vs], vl)) + return 1; + } + } + } + /* GPX 1.0 spells the same thing `<url>' */ + p = gpx_find (buf, mdend, md + 1, "url"); + if ( (((size_t) -1) != p) && + gpx_text (buf, mdend, p, &vs, &vl) && + gpx_emit_xml (ec, EXTRACTOR_METATYPE_URL, &buf[vs], vl) ) + return 1; + p = gpx_find (buf, mdend, md + 1, "time"); + if (((size_t) -1) != p) + { + int64_t when; + + if (gpx_text (buf, mdend, p, &vs, &vl)) + { + if (gpx_parse_iso8601 (&buf[vs], vl, &when)) + { + if (EXTRACTOR_forensic_emit_unix_time_ (ec, + "gpx", + EXTRACTOR_METATYPE_CREATION_DATE, + when)) + return 1; + } + else if (gpx_emit_xml (ec, + EXTRACTOR_METATYPE_UNKNOWN_DATE, + &buf[vs], + vl)) + { + return 1; + } + } + } + return 0; +} + + +/** + * Main entry method for the GPX extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_gpx_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_gpx_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + char head[GPX_MAGIC_WINDOW]; + char *buf = NULL; + size_t hlen = 0; + size_t len = 0; + size_t cap; + uint64_t fsize; + size_t root; + size_t vs; + size_t vl; + struct GpxScan sc; + int truncated; + + /* Bail out on the first few bytes: almost nothing we are handed is + GPX, and we must not pay for the ones that are not. */ + { + void *data; + ssize_t ret; + + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + return; + while (hlen < sizeof (head)) + { + ret = ec->read (ec->cls, + &data, + sizeof (head) - hlen); + if (0 >= ret) + break; + if (((size_t) ret) > sizeof (head) - hlen) + return; /* the IPC layer is misbehaving */ + memcpy (&head[hlen], + data, + (size_t) ret); + hlen += (size_t) ret; + } + } + if (hlen < 16) + return; + root = gpx_find (head, + hlen, + 0, + "gpx"); + if (((size_t) -1) == root) + return; /* not GPX */ + /* GPX is XML; require the declaration or at least that the document + starts with markup, so that a text file merely containing "<gpx" is + not claimed. */ + { + size_t k = 0; + + if ( (hlen >= 3) && + (0 == memcmp (head, "\xef\xbb\xbf", 3)) ) + k = 3; /* UTF-8 BOM */ + while ( (k < hlen) && + ( (' ' == head[k]) || ('\t' == head[k]) || + ('\r' == head[k]) || ('\n' == head[k]) ) ) + k++; + if ( (k >= hlen) || + ('<' != head[k]) ) + return; + } + + fsize = ec->get_size (ec->cls); + cap = GPX_SCAN_CAP; + if ( (UINT64_MAX != fsize) && + (fsize < (uint64_t) cap) ) + cap = (size_t) fsize; + if (cap < hlen) + cap = hlen; + buf = malloc (cap); + if (NULL == buf) + return; + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + { + free (buf); + return; + } + while (len < cap) + { + void *data; + ssize_t ret; + + ret = ec->read (ec->cls, + &data, + cap - len); + if (0 >= ret) + break; + if (((size_t) ret) > cap - len) + break; /* the IPC layer is misbehaving */ + memcpy (&buf[len], + data, + (size_t) ret); + len += (size_t) ret; + } + if (len < 16) + { + free (buf); + return; + } + truncated = ( (UINT64_MAX == fsize) || + (fsize > (uint64_t) len) ); + + if (0 != ec->proc (ec->cls, + "gpx", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/gpx+xml", + strlen ("application/gpx+xml") + 1)) + goto out; + + root = gpx_find (buf, + (len < GPX_MAGIC_WINDOW) ? len : GPX_MAGIC_WINDOW, + 0, + "gpx"); + if (((size_t) -1) != root) + { + /* The `creator' attribute is the single most identifying field in + the format: it names the device or the application that produced + the track. */ + if (gpx_attr (buf, len, root, "creator", &vs, &vl)) + { + if (gpx_emit_xml (ec, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + &buf[vs], + vl)) + goto out; + if (gpx_looks_like_device (&buf[vs], vl) && + gpx_emit_xml (ec, + EXTRACTOR_METATYPE_DEVICE_MODEL, + &buf[vs], + vl) ) + goto out; + } + if (gpx_attr (buf, len, root, "version", &vs, &vl) && + gpx_emit_xml (ec, + EXTRACTOR_METATYPE_FORMAT_VERSION, + &buf[vs], + vl) ) + goto out; + } + /* GPX coordinates are always WGS 84; the schema does not allow + anything else, so this is a statement about the format. */ + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + "WGS 84")) + goto out; + if (gpx_do_metadata (ec, + buf, + len)) + goto out; + + memset (&sc, + 0, + sizeof (sc)); + gpx_walk_points (buf, + len, + &sc); + + /* An explicit <bounds> beats what we computed: it covers the whole + file, ours only covers what we scanned. */ + { + size_t b = gpx_find (buf, len, 0, "bounds"); + double minlat; + double minlon; + double maxlat; + double maxlon; + int have = 0; + + if (((size_t) -1) != b) + { + if (gpx_attr (buf, len, b, "minlat", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &minlat) && + gpx_attr (buf, len, b, "minlon", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &minlon) && + gpx_attr (buf, len, b, "maxlat", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &maxlat) && + gpx_attr (buf, len, b, "maxlon", &vs, &vl) && + gpx_parse_double (&buf[vs], vl, &maxlon) && + (minlat >= -90.0) && (maxlat <= 90.0) && + (minlon >= -180.0) && (maxlon <= 180.0) ) + have = 1; + } + if ( (! have) && + sc.have_bbox) + { + minlat = sc.minlat; + minlon = sc.minlon; + maxlat = sc.maxlat; + maxlon = sc.maxlon; + have = 1; + } + if (have && + EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_BOUNDING_BOX, + "%.6f,%.6f,%.6f,%.6f", + minlon, + minlat, + maxlon, + maxlat) ) + goto out; + } + if (sc.have_bbox) + { + /* Where the recording started; in a personal track log that is very + often the recorder's home. */ + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_GPS_LATITUDE, + "%.6f", + sc.first_lat)) + goto out; + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_GPS_LONGITUDE, + "%.6f", + sc.first_lon)) + goto out; + } + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%llu", + (unsigned long long) sc.trkpts)) + goto out; + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_COMMENT, + "%llu track points, %llu waypoints," + " %llu route points", + (unsigned long long) sc.trkpts, + (unsigned long long) sc.wpts, + (unsigned long long) sc.rtepts)) + goto out; + if ( (0 != sc.first_time) && + (sc.last_time > sc.first_time) ) + { + int64_t d = sc.last_time - sc.first_time; + + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_DURATION, + "%lld:%02lld:%02lld", + (long long) (d / 3600), + (long long) ((d / 60) % 60), + (long long) (d % 60))) + goto out; + } + if (0 != sc.first_time) + { + /* When the first point was recorded; for a track without a + <metadata><time> this is the only timestamp in the file. */ + if (EXTRACTOR_forensic_emit_unix_time_ (ec, + "gpx", + EXTRACTOR_METATYPE_CREATION_DATE, + sc.first_time)) + goto out; + if (EXTRACTOR_forensic_emit_unix_time_ (ec, + "gpx", + EXTRACTOR_METATYPE_MODIFICATION_DATE, + sc.last_time)) + goto out; + } + if (sc.have_ele) + { + if (sc.maxele - sc.minele < 0.05) + { + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_LOCATION_ELEVATION, + "%.1f m", + sc.minele)) + goto out; + } + else if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_LOCATION_ELEVATION, + "%.1f-%.1f m", + sc.minele, + sc.maxele)) + { + goto out; + } + } + if (sc.distance >= 1.0) + { + /* Haversine sum over the track points we scanned -- free, since we + walked them anyway. Under the scan cap this is a lower bound. */ + if (EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_DISTANCE, + "%.0f m", + sc.distance)) + goto out; + } + /* Counts from a partial scan are lower bounds, and saying so is the + whole point: silently under-reporting how many points a track has + would be worse than not reporting at all. */ + if (truncated && + EXTRACTOR_forensic_emit_ (ec, + "gpx", + EXTRACTOR_METATYPE_COMMENT, + "scan truncated at 1 MiB; the counts," + " bounding box, duration and distance" + " above cover only that prefix of the" + " file") ) + goto out; +out: + free (buf); +} + + +/* end of gpx_extractor.c */ diff --git a/src/plugins/heif_extractor.c b/src/plugins/heif_extractor.c @@ -0,0 +1,1852 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/heif_extractor.c + * @brief plugin to support HEIF/HEIC and AVIF still images + * @author Christian Grothoff + * + * HEIC (the default camera format on recent iPhones) and AVIF (now + * common on the web) are both ISO base media file format containers + * carrying a still image instead of a movie. We parse the container + * only: no HEVC or AV1 bitstream is decoded, and no external decoder + * library is linked, because this is a fast identification pass. + * + * Structure, per ISO/IEC 14496-12 (box structure), ISO/IEC 23008-12 + * (HEIF item structures) and the AV1 Image File Format (AVIF brands): + * + * ftyp major brand + compatible brands + * meta (FullBox) + * hdlr 'pict' for a still image collection + * pitm which item is the displayed image + * iinf / infe the items and their types + * iref thumbnail / auxiliary relationships + * iprp + * ipco the properties, addressed by 1-based index + * ispe pixi irot imir colr av1C hvcC ... + * ipma item -> property index associations + * mdat the coded data (never read here) + * moov only for image sequences (bursts, Live Photos) + * + * Every box walk below is bounded by the extent of its parent box, never + * by the size of the file, and the whole parse shares one box budget and + * one byte budget so that a crafted file cannot make us loop or read the + * whole volume. There is no recursion: the nesting we care about is + * fixed and spelled out as nested calls. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * Total number of boxes we are willing to look at in one file. + */ +#define HEIF_MAX_BOXES 4096 + +/** + * Total number of bytes we are willing to read from one file. Header + * parsing needs a few kilobytes; anything beyond this is a file trying + * to make us do work. + */ +#define HEIF_READ_BUDGET (256 * 1024) + +/** + * Maximum number of compatible brands we report. + */ +#define HEIF_MAX_BRANDS 32 + +/** + * Maximum number of property boxes we index out of 'ipco'. + */ +#define HEIF_MAX_PROPS 64 + +/** + * Maximum number of properties we follow for the primary item. + */ +#define HEIF_MAX_ASSOC 32 + +/** + * Maximum number of 'infe' entries we classify. The reported item + * count still comes from the 'iinf' header, so a truncated walk does + * not falsify #EXTRACTOR_METATYPE_ENTRY_COUNT. + */ +#define HEIF_MAX_INFE 512 + +/** + * Maximum number of item references we follow. + */ +#define HEIF_MAX_IREF 512 + +/** + * Largest 'ipma' box we pull into memory. + */ +#define HEIF_IPMA_MAX 8192 + +/** + * Seconds between 1904-01-01 (the ISO-BMFF epoch) and 1970-01-01. + */ +#define HEIF_EPOCH_OFFSET 2082844800LL + + +/** + * One box, as located by #heif_next_box(). + */ +struct HeifBox +{ + /** + * Four character box type. Not NUL terminated. + */ + unsigned char type[4]; + + /** + * Absolute offset of the first payload byte. + */ + uint64_t payload; + + /** + * Absolute offset one past the last payload byte. + */ + uint64_t end; +}; + + +/** + * One entry of the 'ipco' property container, addressed by its 1-based + * index from 'ipma'. + */ +struct HeifProp +{ + /** + * Four character property box type. + */ + unsigned char type[4]; + + /** + * Absolute offset of the first payload byte. + */ + uint64_t payload; + + /** + * Absolute offset one past the last payload byte. + */ + uint64_t end; +}; + + +/** + * Everything we carry through the parse. + */ +struct HeifState +{ + /** + * Extraction context we were handed. + */ + struct EXTRACTOR_ExtractContext *ec; + + /** + * Properties found in 'ipco', in file order. + */ + struct HeifProp props[HEIF_MAX_PROPS]; + + /** + * Property indices (1-based) associated with the primary item. + */ + unsigned int assoc[HEIF_MAX_ASSOC]; + + /** + * Bytes we may still read. + */ + uint64_t budget; + + /** + * Boxes we may still visit. + */ + unsigned int boxes; + + /** + * Number of entries used in @e props. + */ + unsigned int num_props; + + /** + * Number of entries used in @e assoc. + */ + unsigned int num_assoc; + + /** + * Item ID of the primary item, from 'pitm'. + */ + uint32_t primary_item; + + /** + * Number of items declared by 'iinf'. + */ + uint32_t num_items; + + /** + * Number of coded image items seen. + */ + unsigned int num_image_items; + + /** + * Number of items that are the source of a 'thmb' reference. + */ + unsigned int num_thumbnails; + + /** + * Number of items that are the source of an 'auxl' reference + * (alpha and depth planes). + */ + unsigned int num_auxiliary; + + /** + * Number of 'Exif' items. + */ + unsigned int num_exif; + + /** + * Number of 'mime' (XMP and friends) items. + */ + unsigned int num_mime; + + /** + * Item type of the primary item, if we saw its 'infe'. + */ + unsigned char primary_type[4]; + + /** + * True if @e primary_item is valid. + */ + int have_pitm; + + /** + * True if @e primary_type is valid. + */ + int have_primary_type; + + /** + * True if a 'moov' box was found, which makes this a sequence. + */ + int have_moov; + + /** + * Set once #EXTRACTOR_ExtractContext::proc asked us to stop. + */ + int stop; +}; + + +/** + * Read @a len bytes at @a offset, charged against the read budget. + * + * @param hs parser state + * @param offset absolute offset to read from + * @param buf where to put the data + * @param len number of bytes to read + * @return 1 on success, 0 on a short file or an exhausted budget + */ +static int +heif_read (struct HeifState *hs, + uint64_t offset, + void *buf, + size_t len) +{ + if ( (len > hs->budget) || + (offset > INT64_MAX) ) + return 0; + hs->budget -= len; + return EXTRACTOR_forensic_read_ (hs->ec, + (int64_t) offset, + buf, + len); +} + + +/** + * Test whether @a box has the given four character type. + * + * @param box the box + * @param type four characters to compare against + * @return 1 on a match, 0 otherwise + */ +static int +heif_is (const struct HeifBox *box, + const char *type) +{ + return 0 == memcmp (box->type, + type, + 4); +} + + +/** + * Locate the box that starts at @a pos inside a parent that ends at + * @a limit. + * + * Handles all three ISO-BMFF size encodings: a 32-bit size, size 1 + * meaning a 64-bit largesize follows the type, and size 0 meaning "up + * to the end of the enclosing box". A size that does not cover its own + * header, or that runs past the parent, terminates the walk rather than + * being clamped: a box that lies about its size is not one we want to + * keep reading past. + * + * @param hs parser state + * @param pos absolute offset of the box header + * @param limit absolute offset one past the end of the parent + * @param[out] box the box that was found + * @return 1 on success, 0 if there is no further box here + */ +static int +heif_next_box (struct HeifState *hs, + uint64_t pos, + uint64_t limit, + struct HeifBox *box) +{ + unsigned char hdr[8]; + uint64_t size; + unsigned int hlen = 8; + + if (0 == hs->boxes) + return 0; + hs->boxes--; + if ( (pos >= limit) || + (limit - pos < 8) ) + return 0; + if (! heif_read (hs, + pos, + hdr, + sizeof (hdr))) + return 0; + size = EXTRACTOR_forensic_be32_ (hdr); + memcpy (box->type, + &hdr[4], + 4); + if (1 == size) + { + unsigned char large[8]; + + if (limit - pos < 16) + return 0; + if (! heif_read (hs, + pos + 8, + large, + sizeof (large))) + return 0; + size = EXTRACTOR_forensic_be64_ (large); + hlen = 16; + if (size < 16) + return 0; + } + else if (0 == size) + { + size = limit - pos; /* extends to the end of the parent */ + } + else if (size < 8) + { + return 0; /* smaller than its own header */ + } + if (size > limit - pos) + return 0; + box->payload = pos + hlen; + box->end = pos + size; + if (box->payload > box->end) + return 0; + return 1; +} + + +/** + * Number of payload bytes in @a box. + * + * @param box the box + * @return payload length + */ +static uint64_t +heif_payload_len (const struct HeifBox *box) +{ + return box->end - box->payload; +} + + +/** + * Read the version byte of a FullBox. + * + * @param hs parser state + * @param box the box, which must be a FullBox + * @param[out] version the version byte + * @param[out] flags the 24-bit flags field + * @return 1 on success, 0 if the box is too short + */ +static int +heif_full_header (struct HeifState *hs, + const struct HeifBox *box, + unsigned int *version, + uint32_t *flags) +{ + unsigned char vf[4]; + + if (heif_payload_len (box) < 4) + return 0; + if (! heif_read (hs, + box->payload, + vf, + sizeof (vf))) + return 0; + *version = vf[0]; + *flags = EXTRACTOR_forensic_be32_ (vf) & 0xFFFFFF; + return 1; +} + + +/** + * A brand we recognise, and what it says about the file. + */ +struct HeifBrand +{ + /** + * The four character brand. + */ + const char *brand; + + /** + * MIME type family, without the "-sequence" suffix. + */ + const char *family; + + /** + * Coding format the brand implies, NULL if the brand is generic. + */ + const char *codec; + + /** + * True if the brand denotes an image sequence rather than a still. + */ + int sequence; +}; + + +/** + * Brands that make a file ours. 'mif1'/'msf1' are the generic HEIF + * brands and say nothing about the codec, which is why the compatible + * brand list has to be consulted as well: "mif1" as major brand with + * "avif" among the compatible brands is what several AVIF encoders + * produce. + */ +static const struct HeifBrand heif_brands[] = { + { "heic", "image/heic", "HEVC", 0 }, + { "heix", "image/heic", "HEVC", 0 }, + { "heim", "image/heic", "HEVC", 0 }, + { "heis", "image/heic", "HEVC", 0 }, + { "hevc", "image/heic", "HEVC", 1 }, + { "hevx", "image/heic", "HEVC", 1 }, + { "hevm", "image/heic", "HEVC", 1 }, + { "hevs", "image/heic", "HEVC", 1 }, + { "avif", "image/avif", "AV1", 0 }, + { "avis", "image/avif", "AV1", 1 }, + { "mif1", "image/heif", NULL, 0 }, + { "mif2", "image/heif", NULL, 0 }, + { "msf1", "image/heif", NULL, 1 }, + { NULL, NULL, NULL, 0 } +}; + + +/** + * Look @a brand up in #heif_brands. + * + * @param brand four bytes from the file + * @return the entry, or NULL if we do not know the brand + */ +static const struct HeifBrand * +heif_lookup_brand (const unsigned char *brand) +{ + for (unsigned int i = 0; NULL != heif_brands[i].brand; i++) + if (0 == memcmp (brand, + heif_brands[i].brand, + 4)) + return &heif_brands[i]; + return NULL; +} + + +/** + * Human readable name for an item type as it appears in 'infe'. + * + * @param type four bytes from the file + * @return the name, or NULL if we do not know the type + */ +static const char * +heif_item_type_name (const unsigned char *type) +{ + static const struct + { + const char *type; + const char *name; + } names[] = { + { "hvc1", "HEVC" }, + { "hev1", "HEVC" }, + { "av01", "AV1" }, + { "avc1", "AVC" }, + { "vvc1", "VVC" }, + { "jpeg", "JPEG" }, + { "j2k1", "JPEG 2000" }, + { "grid", "tiled grid" }, + { "iovl", "overlay" }, + { "iden", "identity" }, + { "mask", "mask" }, + { NULL, NULL } + }; + + for (unsigned int i = 0; NULL != names[i].type; i++) + if (0 == memcmp (type, + names[i].type, + 4)) + return names[i].name; + return NULL; +} + + +/** + * True if @a type names a coded or derived image item, as opposed to a + * metadata item such as 'Exif' or 'mime'. + * + * @param type four bytes from the file + * @return 1 if this is an image item + */ +static int +heif_is_image_item (const unsigned char *type) +{ + return NULL != heif_item_type_name (type); +} + + +/** + * Parse the 'pitm' box, which names the item that a viewer displays. + * + * @param hs parser state + * @param box the 'pitm' box + */ +static void +heif_parse_pitm (struct HeifState *hs, + const struct HeifBox *box) +{ + unsigned char buf[4]; + unsigned int version; + uint32_t flags; + + if (! heif_full_header (hs, + box, + &version, + &flags)) + return; + if (0 == version) + { + if (heif_payload_len (box) < 6) + return; + if (! heif_read (hs, + box->payload + 4, + buf, + 2)) + return; + hs->primary_item = EXTRACTOR_forensic_be16_ (buf); + } + else + { + if (heif_payload_len (box) < 8) + return; + if (! heif_read (hs, + box->payload + 4, + buf, + 4)) + return; + hs->primary_item = EXTRACTOR_forensic_be32_ (buf); + } + hs->have_pitm = 1; +} + + +/** + * Parse the 'iinf' box and its 'infe' children: how many items there + * are, what kinds they are, and what the primary item is made of. + * + * @param hs parser state + * @param box the 'iinf' box + */ +static void +heif_parse_iinf (struct HeifState *hs, + const struct HeifBox *box) +{ + unsigned char buf[16]; + unsigned int version; + uint32_t flags; + uint64_t pos; + unsigned int n; + + if (! heif_full_header (hs, + box, + &version, + &flags)) + return; + if (0 == version) + { + if (heif_payload_len (box) < 6) + return; + if (! heif_read (hs, + box->payload + 4, + buf, + 2)) + return; + hs->num_items = EXTRACTOR_forensic_be16_ (buf); + pos = box->payload + 6; + } + else + { + if (heif_payload_len (box) < 8) + return; + if (! heif_read (hs, + box->payload + 4, + buf, + 4)) + return; + hs->num_items = EXTRACTOR_forensic_be32_ (buf); + pos = box->payload + 8; + } + n = 0; + while ( (pos < box->end) && + (n < HEIF_MAX_INFE) ) + { + struct HeifBox infe; + unsigned int iversion; + uint32_t item_id; + const unsigned char *item_type; + size_t need; + + if (! heif_next_box (hs, + pos, + box->end, + &infe)) + break; + if (infe.end <= pos) + break; /* no forward progress */ + pos = infe.end; + n++; + if (! heif_is (&infe, + "infe")) + continue; + /* ItemInfoEntry: version 2 uses a 16-bit item_ID, version 3 and up + a 32-bit one; both are followed by a 16-bit protection index and + then the four character item_type. Versions 0 and 1 predate + item_type and carry a file name instead, which tells us nothing + about the coding format. */ + need = 14; + if (heif_payload_len (&infe) < 4) + continue; + if (heif_payload_len (&infe) < need) + need = (size_t) heif_payload_len (&infe); + if (need > sizeof (buf)) + need = sizeof (buf); + if (! heif_read (hs, + infe.payload, + buf, + need)) + continue; + iversion = buf[0]; + if (2 == iversion) + { + if (need < 12) + continue; + item_id = EXTRACTOR_forensic_be16_ (&buf[4]); + item_type = &buf[8]; + } + else if (iversion >= 3) + { + if (need < 14) + continue; + item_id = EXTRACTOR_forensic_be32_ (&buf[4]); + item_type = &buf[10]; + } + else + { + continue; /* no item_type in this version */ + } + if (heif_is_image_item (item_type)) + hs->num_image_items++; + else if (0 == memcmp (item_type, + "Exif", + 4)) + hs->num_exif++; + else if (0 == memcmp (item_type, + "mime", + 4)) + hs->num_mime++; + if ( (hs->have_pitm) && + (item_id == hs->primary_item) ) + { + memcpy (hs->primary_type, + item_type, + 4); + hs->have_primary_type = 1; + } + } +} + + +/** + * Parse the 'iref' box: which items are thumbnails of, or auxiliary + * planes for, another item. + * + * @param hs parser state + * @param box the 'iref' box + */ +static void +heif_parse_iref (struct HeifState *hs, + const struct HeifBox *box) +{ + unsigned int version; + uint32_t flags; + uint64_t pos; + unsigned int n; + + if (! heif_full_header (hs, + box, + &version, + &flags)) + return; + pos = box->payload + 4; + n = 0; + while ( (pos < box->end) && + (n < HEIF_MAX_IREF) ) + { + struct HeifBox ref; + + if (! heif_next_box (hs, + pos, + box->end, + &ref)) + break; + if (ref.end <= pos) + break; /* no forward progress */ + pos = ref.end; + n++; + /* Each child is one reference: its type is the relationship and its + from_item_ID is the item that plays the subordinate role. */ + if (heif_is (&ref, + "thmb")) + hs->num_thumbnails++; + else if (heif_is (&ref, + "auxl")) + hs->num_auxiliary++; + } +} + + +/** + * Parse the 'ipco' box, indexing its children so that 'ipma' can refer + * to them by their 1-based position. + * + * @param hs parser state + * @param box the 'ipco' box + */ +static void +heif_parse_ipco (struct HeifState *hs, + const struct HeifBox *box) +{ + uint64_t pos = box->payload; + + while ( (pos < box->end) && + (hs->num_props < HEIF_MAX_PROPS) ) + { + struct HeifBox prop; + + if (! heif_next_box (hs, + pos, + box->end, + &prop)) + break; + if (prop.end <= pos) + break; /* no forward progress */ + pos = prop.end; + memcpy (hs->props[hs->num_props].type, + prop.type, + 4); + hs->props[hs->num_props].payload = prop.payload; + hs->props[hs->num_props].end = prop.end; + hs->num_props++; + } +} + + +/** + * Parse the 'ipma' box and collect the property indices associated with + * the primary item. + * + * @param hs parser state + * @param box the 'ipma' box + */ +static void +heif_parse_ipma (struct HeifState *hs, + const struct HeifBox *box) +{ + unsigned char *buf; + size_t len; + size_t off; + unsigned int version; + uint32_t flags; + uint32_t entries; + int wide_index; + + if (! hs->have_pitm) + return; + if (0 != hs->num_assoc) + return; /* a previous 'ipma' already answered the question */ + len = (size_t) heif_payload_len (box); + if (len < 8) + return; + if (len > HEIF_IPMA_MAX) + len = HEIF_IPMA_MAX; + buf = malloc (len); + if (NULL == buf) + return; + if (! heif_read (hs, + box->payload, + buf, + len)) + { + free (buf); + return; + } + version = buf[0]; + flags = EXTRACTOR_forensic_be32_ (buf) & 0xFFFFFF; + wide_index = (0 != (flags & 1)); + entries = EXTRACTOR_forensic_be32_ (&buf[4]); + off = 8; + /* The entry count is a 32-bit field out of the file, so it is not a + loop bound we can trust; the buffer running out is what actually + ends the walk, and `truncated' carries that out of the inner + loop without ever incrementing `e' past its own maximum. */ + for (uint32_t e = 0; e < entries; e++) + { + uint32_t item_id; + unsigned int count; + int mine; + int truncated = 0; + + if (version < 1) + { + if (off + 2 > len) + break; + item_id = EXTRACTOR_forensic_be16_ (&buf[off]); + off += 2; + } + else + { + if (off + 4 > len) + break; + item_id = EXTRACTOR_forensic_be32_ (&buf[off]); + off += 4; + } + if (off >= len) + break; + count = buf[off]; + off++; + mine = (item_id == hs->primary_item); + for (unsigned int a = 0; a < count; a++) + { + unsigned int index; + + if (wide_index) + { + if (off + 2 > len) + { + truncated = 1; + break; + } + index = EXTRACTOR_forensic_be16_ (&buf[off]) & 0x7FFF; + off += 2; + } + else + { + if (off + 1 > len) + { + truncated = 1; + break; + } + index = buf[off] & 0x7F; + off++; + } + if ( (mine) && + (0 != index) && + (hs->num_assoc < HEIF_MAX_ASSOC) ) + hs->assoc[hs->num_assoc++] = index; + } + if (truncated) + break; + } + free (buf); +} + + +/** + * Emit the image dimensions from an 'ispe' property. + * + * @param hs parser state + * @param prop the property + */ +static void +heif_emit_ispe (struct HeifState *hs, + const struct HeifProp *prop) +{ + unsigned char buf[12]; + + if (prop->end - prop->payload < 12) + return; + if (! heif_read (hs, + prop->payload, + buf, + sizeof (buf))) + return; + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "%ux%u", + (unsigned int) EXTRACTOR_forensic_be32_ ( + &buf[4]), + (unsigned int) EXTRACTOR_forensic_be32_ ( + &buf[8]))) + hs->stop = 1; +} + + +/** + * Emit the bit depth from a 'pixi' property. A 10-bit still is the + * usual signal that the file carries HDR content. + * + * @param hs parser state + * @param prop the property + */ +static void +heif_emit_pixi (struct HeifState *hs, + const struct HeifProp *prop) +{ + unsigned char buf[8]; + unsigned int channels; + uint64_t len = prop->end - prop->payload; + + if (len < 5) + return; + if (! heif_read (hs, + prop->payload, + buf, + 5)) + return; + channels = buf[4]; + if ( (0 == channels) || + (channels > 4) || + (len < 5 + channels) ) + return; + if (! heif_read (hs, + prop->payload + 5, + buf, + channels)) + return; + /* All channels normally share a depth; report the first and say how + many planes there are, which distinguishes monochrome from YUV. */ + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COLOR_DEPTH, + "%u", + (unsigned int) buf[0])) + hs->stop = 1; +} + + +/** + * Emit the rotation recorded in an 'irot' property. + * + * @param hs parser state + * @param prop the property + */ +static void +heif_emit_irot (struct HeifState *hs, + const struct HeifProp *prop) +{ + unsigned char buf[1]; + + if (prop->end - prop->payload < 1) + return; + if (! heif_read (hs, + prop->payload, + buf, + 1)) + return; + /* The low two bits count anti-clockwise quarter turns to apply when + displaying the image. */ + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_ORIENTATION, + "rotate %u degrees counter-clockwise", + (unsigned int) (buf[0] & 0x03) * 90)) + hs->stop = 1; +} + + +/** + * Emit the mirroring recorded in an 'imir' property. + * + * @param hs parser state + * @param prop the property + */ +static void +heif_emit_imir (struct HeifState *hs, + const struct HeifProp *prop) +{ + unsigned char buf[1]; + + if (prop->end - prop->payload < 1) + return; + if (! heif_read (hs, + prop->payload, + buf, + 1)) + return; + /* axis == 0 mirrors about a vertical axis, i.e. left and right are + swapped; axis == 1 mirrors about a horizontal axis. */ + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_ORIENTATION, + "mirrored %s", + (0 == (buf[0] & 0x01)) + ? "horizontally" + : "vertically")) + hs->stop = 1; +} + + +/** + * Emit the colour information from a 'colr' property. + * + * @param hs parser state + * @param prop the property + */ +static void +heif_emit_colr (struct HeifState *hs, + const struct HeifProp *prop) +{ + unsigned char buf[12]; + uint64_t len = prop->end - prop->payload; + + if (len < 4) + return; + if (! heif_read (hs, + prop->payload, + buf, + (len < sizeof (buf)) ? (size_t) len : sizeof (buf))) + return; + if (0 == memcmp (buf, + "nclx", + 4)) + { + unsigned int primaries; + unsigned int transfer; + unsigned int matrix; + const char *hdr = NULL; + + if (len < 11) + return; + primaries = EXTRACTOR_forensic_be16_ (&buf[4]); + transfer = EXTRACTOR_forensic_be16_ (&buf[6]); + matrix = EXTRACTOR_forensic_be16_ (&buf[8]); + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COLOR_PROFILE, + "nclx: primaries %u, transfer %u, " + "matrix %u, %s range", + primaries, + transfer, + matrix, + (0 != (buf[10] & 0x80)) + ? "full" + : "limited")) + { + hs->stop = 1; + return; + } + /* Transfer characteristics 16 and 18 are the two HDR curves; they + are worth spelling out because they are what distinguishes an HDR + capture from an ordinary one. */ + if (16 == transfer) + hdr = "HDR: PQ transfer function (SMPTE ST 2084)"; + else if (18 == transfer) + hdr = "HDR: HLG transfer function (ARIB STD-B67)"; + if (NULL != hdr) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COMMENT, + "%s", + hdr)) + hs->stop = 1; + } + return; + } + if ( (0 == memcmp (buf, + "prof", + 4)) || + (0 == memcmp (buf, + "rICC", + 4)) ) + { + /* We deliberately do not parse the ICC payload: exiv2 and the + colour management stack do that far better than we could. */ + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COLOR_PROFILE, + "ICC profile (%.4s), %llu bytes", + (const char *) buf, + (unsigned long long) (len - 4))) + hs->stop = 1; + } +} + + +/** + * Emit what an 'av1C' configuration record says about the bit depth, + * which is the fallback when the file carries no 'pixi'. + * + * @param hs parser state + * @param prop the property + * @param have_pixi true if a 'pixi' property will report the depth + */ +static void +heif_emit_av1c (struct HeifState *hs, + const struct HeifProp *prop, + int have_pixi) +{ + unsigned char buf[4]; + unsigned int depth; + + if (have_pixi) + return; + if (prop->end - prop->payload < 4) + return; + if (! heif_read (hs, + prop->payload, + buf, + sizeof (buf))) + return; + if (0x81 != buf[0]) + return; /* marker bit plus version 1 */ + /* byte 2: seq_tier_0(1) high_bitdepth(1) twelve_bit(1) monochrome(1) + chroma_subsampling_x(1) chroma_subsampling_y(1) + chroma_sample_position(2) */ + if (0 != (buf[2] & 0x40)) + depth = (0 != (buf[2] & 0x20)) ? 12 : 10; + else + depth = 8; + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COLOR_DEPTH, + "%u", + depth)) + hs->stop = 1; +} + + +/** + * Walk the properties selected for the primary item and emit what they + * say. Only the first property of each kind is reported: a file may + * legally associate several, but the extra ones describe alternative + * renderings rather than the image a user would see. + * + * @param hs parser state + */ +static void +heif_emit_properties (struct HeifState *hs) +{ + unsigned int order[HEIF_MAX_PROPS]; + unsigned int n = 0; + int did_ispe = 0; + int did_pixi = 0; + int did_irot = 0; + int did_imir = 0; + int did_colr = 0; + int did_av1c = 0; + int have_pixi = 0; + + if (0 != hs->num_assoc) + { + for (unsigned int i = 0; i < hs->num_assoc; i++) + if (hs->assoc[i] <= hs->num_props) + order[n++] = hs->assoc[i] - 1; + } + else + { + /* No usable association: fall back to file order. The first 'ispe' + in 'ipco' belongs to a thumbnail often enough that this is worth + flagging, but it is still better than reporting nothing. */ + for (unsigned int i = 0; i < hs->num_props; i++) + order[n++] = i; + } + /* 'pixi' wins over 'av1C' for the bit depth, so find out whether one + is coming before deciding to use the configuration record. */ + for (unsigned int i = 0; i < n; i++) + if (0 == memcmp (hs->props[order[i]].type, + "pixi", + 4)) + have_pixi = 1; + for (unsigned int i = 0; i < n; i++) + { + const struct HeifProp *prop = &hs->props[order[i]]; + + if (hs->stop) + return; + if ( (0 == memcmp (prop->type, "ispe", 4)) && + (! did_ispe) ) + { + did_ispe = 1; + heif_emit_ispe (hs, + prop); + } + else if ( (0 == memcmp (prop->type, "pixi", 4)) && + (! did_pixi) ) + { + did_pixi = 1; + heif_emit_pixi (hs, + prop); + } + else if ( (0 == memcmp (prop->type, "irot", 4)) && + (! did_irot) ) + { + did_irot = 1; + heif_emit_irot (hs, + prop); + } + else if ( (0 == memcmp (prop->type, "imir", 4)) && + (! did_imir) ) + { + did_imir = 1; + heif_emit_imir (hs, + prop); + } + else if ( (0 == memcmp (prop->type, "colr", 4)) && + (! did_colr) ) + { + did_colr = 1; + heif_emit_colr (hs, + prop); + } + else if ( (0 == memcmp (prop->type, "av1C", 4)) && + (! did_av1c) ) + { + did_av1c = 1; + heif_emit_av1c (hs, + prop, + have_pixi); + } + } +} + + +/** + * Parse the 'iprp' box: the property container and the associations. + * + * @param hs parser state + * @param box the 'iprp' box + */ +static void +heif_parse_iprp (struct HeifState *hs, + const struct HeifBox *box) +{ + uint64_t pos = box->payload; + + /* 'ipco' always precedes 'ipma' in practice, but do not rely on it: + index the container first in one pass, then read the associations. */ + while (pos < box->end) + { + struct HeifBox child; + + if (! heif_next_box (hs, + pos, + box->end, + &child)) + break; + if (child.end <= pos) + break; /* no forward progress */ + pos = child.end; + if (heif_is (&child, + "ipco")) + heif_parse_ipco (hs, + &child); + } + pos = box->payload; + while (pos < box->end) + { + struct HeifBox child; + + if (! heif_next_box (hs, + pos, + box->end, + &child)) + break; + if (child.end <= pos) + break; + pos = child.end; + if (heif_is (&child, + "ipma")) + heif_parse_ipma (hs, + &child); + } +} + + +/** + * Find where the children of a 'meta' box start. + * + * In ISO/IEC 14496-12 'meta' is a FullBox, so its children begin four + * bytes into the payload. QuickTime writes the same box without the + * version and flags word, and such files do turn up; probe for a + * plausible child box at both offsets rather than guessing. + * + * @param hs parser state + * @param box the 'meta' box + * @param[out] start where the children begin + * @return 1 if a starting offset was found + */ +static int +heif_meta_children (struct HeifState *hs, + const struct HeifBox *box, + uint64_t *start) +{ + struct HeifBox probe; + + if ( (heif_payload_len (box) > 4) && + (heif_next_box (hs, + box->payload + 4, + box->end, + &probe)) ) + { + *start = box->payload + 4; + return 1; + } + if (heif_next_box (hs, + box->payload, + box->end, + &probe)) + { + *start = box->payload; + return 1; + } + return 0; +} + + +/** + * Parse the 'meta' box: the items, their properties and their + * relationships. + * + * @param hs parser state + * @param box the 'meta' box + * @param brand_codec coding format implied by the brand, may be NULL + */ +static void +heif_parse_meta (struct HeifState *hs, + const struct HeifBox *box, + const char *brand_codec) +{ + struct HeifBox pitm; + struct HeifBox iinf; + struct HeifBox iref; + struct HeifBox iprp; + int have_pitm_box = 0; + int have_iinf = 0; + int have_iref = 0; + int have_iprp = 0; + uint64_t pos; + + memset (&pitm, 0, sizeof (pitm)); + memset (&iinf, 0, sizeof (iinf)); + memset (&iref, 0, sizeof (iref)); + memset (&iprp, 0, sizeof (iprp)); + if (! heif_meta_children (hs, + box, + &pos)) + return; + while (pos < box->end) + { + struct HeifBox child; + + if (! heif_next_box (hs, + pos, + box->end, + &child)) + break; + if (child.end <= pos) + break; /* no forward progress */ + pos = child.end; + if ( (heif_is (&child, "pitm")) && + (! have_pitm_box) ) + { + pitm = child; + have_pitm_box = 1; + } + else if ( (heif_is (&child, "iinf")) && + (! have_iinf) ) + { + iinf = child; + have_iinf = 1; + } + else if ( (heif_is (&child, "iref")) && + (! have_iref) ) + { + iref = child; + have_iref = 1; + } + else if ( (heif_is (&child, "iprp")) && + (! have_iprp) ) + { + iprp = child; + have_iprp = 1; + } + } + /* The order below is ours, not the file's: the primary item ID has to + be known before 'iinf' and 'ipma' can be interpreted. */ + if (have_pitm_box) + heif_parse_pitm (hs, + &pitm); + if (have_iinf) + heif_parse_iinf (hs, + &iinf); + if (have_iref) + heif_parse_iref (hs, + &iref); + + /* CODEC: the item type of the primary item is more specific than the + brand, so prefer it and fall back to the brand. */ + if (hs->have_primary_type) + { + const char *name = heif_item_type_name (hs->primary_type); + + if (NULL != name) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_CODEC, + "%s (%.4s)", + name, + (const char *) hs->primary_type)) + { + hs->stop = 1; + return; + } + if ( (0 == memcmp (hs->primary_type, "grid", 4)) || + (0 == memcmp (hs->primary_type, "iovl", 4)) || + (0 == memcmp (hs->primary_type, "iden", 4)) ) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_RESOURCE_TYPE, + "derived image (%.4s)", + (const char *) hs->primary_type)) + { + hs->stop = 1; + return; + } + } + } + } + else if (NULL != brand_codec) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_CODEC, + "%s", + brand_codec)) + { + hs->stop = 1; + return; + } + } + + if (have_iprp) + { + heif_parse_iprp (hs, + &iprp); + heif_emit_properties (hs); + if (hs->stop) + return; + } + + if (0 != hs->num_items) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) hs->num_items)) + { + hs->stop = 1; + return; + } + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COMMENT, + "%u image item%s, %u thumbnail%s, " + "%u auxiliary, %u metadata item%s", + hs->num_image_items, + (1 == hs->num_image_items) ? "" : "s", + hs->num_thumbnails, + (1 == hs->num_thumbnails) ? "" : "s", + hs->num_auxiliary, + hs->num_exif + hs->num_mime, + (1 == hs->num_exif + hs->num_mime) + ? "" : "s")) + { + hs->stop = 1; + return; + } + } + /* We do not touch the payload of these items: parsing EXIF and XMP is + the exiv2 plugin's job. Saying that they are there is what lets a + downstream tool decide to run it. */ + if (0 != hs->num_exif) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COMMENT, + "contains an Exif metadata item")) + { + hs->stop = 1; + return; + } + } + if (0 != hs->num_mime) + { + if (0 != EXTRACTOR_forensic_emit_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_COMMENT, + "contains an XMP or MIME metadata item")) + { + hs->stop = 1; + return; + } + } +} + + +/** + * Parse 'moov' and its 'mvhd', which only a sequence file has. + * + * @param hs parser state + * @param box the 'moov' box + */ +static void +heif_parse_moov (struct HeifState *hs, + const struct HeifBox *box) +{ + uint64_t pos = box->payload; + + while (pos < box->end) + { + struct HeifBox child; + unsigned char buf[28]; + int64_t created; + int64_t modified; + + if (! heif_next_box (hs, + pos, + box->end, + &child)) + break; + if (child.end <= pos) + break; /* no forward progress */ + pos = child.end; + if (! heif_is (&child, + "mvhd")) + continue; + if (heif_payload_len (&child) < 5) + return; + if (! heif_read (hs, + child.payload, + buf, + 5)) + return; + /* The ISO-BMFF epoch is 1904-01-01; these fields are also famously + written as local time labelled as UTC by most muxers, so treat + the result as an approximate wall clock reading, not as UTC. */ + if (0 == buf[0]) + { + if (heif_payload_len (&child) < 20) + return; + if (! heif_read (hs, + child.payload + 4, + buf, + 16)) + return; + created = (int64_t) EXTRACTOR_forensic_be32_ (&buf[0]); + modified = (int64_t) EXTRACTOR_forensic_be32_ (&buf[4]); + } + else + { + if (heif_payload_len (&child) < 32) + return; + if (! heif_read (hs, + child.payload + 4, + buf, + 28)) + return; + created = (int64_t) (EXTRACTOR_forensic_be64_ (&buf[0]) + & INT64_MAX); + modified = (int64_t) (EXTRACTOR_forensic_be64_ (&buf[8]) + & INT64_MAX); + } + if (0 != created) + { + if (0 != EXTRACTOR_forensic_emit_unix_time_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_CREATION_DATE, + created - HEIF_EPOCH_OFFSET)) + { + hs->stop = 1; + return; + } + } + if (0 != modified) + { + if (0 != EXTRACTOR_forensic_emit_unix_time_ (hs->ec, + "heif", + EXTRACTOR_METATYPE_MODIFICATION_DATE, + modified + - HEIF_EPOCH_OFFSET)) + hs->stop = 1; + } + return; + } +} + + +/** + * Main entry method for the HEIF/HEIC/AVIF extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_heif_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_heif_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + struct HeifState hs; + struct HeifBox meta; + struct HeifBox moov; + unsigned char head[16]; + unsigned char brands[HEIF_MAX_BRANDS * 4]; + const struct HeifBrand *major; + const char *family; + const char *codec; + char mime[64]; + char compat[HEIF_MAX_BRANDS * 5 + 1]; + uint64_t fsize; + uint64_t ftyp_size; + unsigned int num_brands; + unsigned int sequence; + uint64_t pos; + int have_meta = 0; + + fsize = ec->get_size (ec->cls); + if ( (UINT64_MAX == fsize) || + (fsize < 16) ) + return; + memset (&hs, + 0, + sizeof (hs)); + memset (&meta, 0, sizeof (meta)); + memset (&moov, 0, sizeof (moov)); + hs.ec = ec; + hs.boxes = HEIF_MAX_BOXES; + hs.budget = HEIF_READ_BUDGET; + + /* Bail out on the first 16 bytes: almost every file we are handed is + not ours, and this is the only read those files should cost. */ + if (! heif_read (&hs, + 0, + head, + sizeof (head))) + return; + if (0 != memcmp (&head[4], + "ftyp", + 4)) + return; + ftyp_size = EXTRACTOR_forensic_be32_ (head); + if ( (ftyp_size < 16) || + (ftyp_size > fsize) ) + return; /* 'ftyp' never uses the 64-bit or to-end-of-file forms */ + major = heif_lookup_brand (&head[8]); + + num_brands = (unsigned int) ((ftyp_size - 16) / 4); + if (num_brands > HEIF_MAX_BRANDS) + num_brands = HEIF_MAX_BRANDS; + if ( (0 != num_brands) && + (! heif_read (&hs, + 16, + brands, + num_brands * 4)) ) + num_brands = 0; + + family = (NULL != major) ? major->family : NULL; + codec = (NULL != major) ? major->codec : NULL; + sequence = (NULL != major) ? (unsigned int) major->sequence : 0; + if (NULL == codec) + { + /* Either the major brand is generic ('mif1'/'msf1') or we do not + know it at all; the compatible brands decide. */ + for (unsigned int i = 0; i < num_brands; i++) + { + const struct HeifBrand *b = heif_lookup_brand (&brands[i * 4]); + + if (NULL == b) + continue; + if (NULL == family) + { + family = b->family; + sequence = (unsigned int) b->sequence; + } + if (NULL != b->codec) + { + codec = b->codec; + family = b->family; + sequence |= (unsigned int) b->sequence; + break; + } + } + } + if (NULL == family) + return; /* neither the major nor any compatible brand is ours */ + + snprintf (mime, + sizeof (mime), + "%s%s", + family, + (0 != sequence) ? "-sequence" : ""); + if (0 != ec->proc (ec->cls, + "heif", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + mime, + strlen (mime) + 1)) + return; + if (0 != EXTRACTOR_forensic_emit_text_ (ec, + "heif", + EXTRACTOR_METATYPE_FORMAT, + (const char *) &head[8], + 4)) + return; + if (0 != num_brands) + { + size_t off = 0; + + for (unsigned int i = 0; i < num_brands; i++) + { + /* Skip the padding brands some encoders write. */ + if (0 == memcmp (&brands[i * 4], + "\0\0\0\0", + 4)) + continue; + if (off + 5 >= sizeof (compat)) + break; + if (0 != off) + compat[off++] = ','; + memcpy (&compat[off], + &brands[i * 4], + 4); + off += 4; + } + if (0 != off) + { + if (0 != EXTRACTOR_forensic_emit_text_ (ec, + "heif", + EXTRACTOR_METATYPE_COMPATIBLE_BRANDS, + compat, + off)) + return; + } + } + + /* Top level walk: we only care about 'meta' and 'moov'. 'mdat' is + stepped over by its size and never read. One pass records where + they are; the emission order below is ours, not the file's. */ + pos = ftyp_size; + while (pos < fsize) + { + struct HeifBox box; + + if (! heif_next_box (&hs, + pos, + fsize, + &box)) + break; + if (box.end <= pos) + break; /* no forward progress */ + pos = box.end; + if ( (heif_is (&box, "meta")) && + (! have_meta) ) + { + meta = box; + have_meta = 1; + } + else if ( (heif_is (&box, "moov")) && + (! hs.have_moov) ) + { + moov = box; + hs.have_moov = 1; + } + } + if (have_meta) + { + heif_parse_meta (&hs, + &meta, + codec); + if (hs.stop) + return; + } + if ( (0 != sequence) || + (hs.have_moov) ) + { + /* A burst, a Live Photo or an animated AVIF: several coded images + that belong together, which is worth calling out because it means + there is more here than the one frame a viewer shows. */ + if (0 != EXTRACTOR_forensic_emit_ (ec, + "heif", + EXTRACTOR_METATYPE_RESOURCE_TYPE, + "image sequence")) + return; + if (0 != EXTRACTOR_forensic_emit_ (ec, + "heif", + EXTRACTOR_METATYPE_COMMENT, + "image sequence (burst, animation or " + "Live Photo)")) + return; + } + if (hs.have_moov) + heif_parse_moov (&hs, + &moov); +} + + +/* end of heif_extractor.c */ diff --git a/src/plugins/id3_extractor.c b/src/plugins/id3_extractor.c @@ -0,0 +1,2014 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/id3_extractor.c + * @brief plugin to support MPEG audio with ID3v2 and ID3v1 tags + * @author Christian Grothoff + * + * There is an unported `plugins/old/mp3_extractor.c' in this tree that + * covers part of the same ground. It got the MPEG frame header tables + * right and they are reused here, but its strategy -- walk every frame + * in the file, up to 31 MB, to decide whether the file is an MP3 -- + * is the opposite of what a first pass over a volume wants, so the + * rest is written fresh. + * + * What we are after is not the song title. It is the provenance: who + * encoded the file (TENC), with what (TSSE and the LAME tag), who + * owned it (TOWN), which player stamped it (PRIV) and which catalogue + * it came from (UFID). The LAME tag in particular is a fingerprint of + * the exact ripping software and its settings. + * + * References: the ID3v2.2/2.3/2.4 specifications from id3.org, and the + * Xing/Info and LAME tag layout as documented at + * gabriel.mp3-tech.org/mp3infotag.html. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +#include <stdarg.h> + + +/** + * Name we report our meta data under. + */ +#define ID3_PLUGIN "id3" + +/** + * MIME type we claim. + */ +#define ID3_MIME "audio/mpeg" + +/** + * Most of an ID3v2 tag we are willing to pull into memory. Parsing + * from a buffer rather than by seeking is what makes the + * unsynchronisation scheme tractable; the price is this read. Real + * tags are a few kilobytes, or a few tens of kilobytes with cover art; + * beyond this bound we parse the frames that fit and stop. + */ +#define ID3_MAX_TAG_READ (256 * 1024) + +/** + * Upper bound on the number of frames we walk. + */ +#define ID3_MAX_FRAMES 128 + +/** + * Longest decoded string we build. + */ +#define ID3_MAX_TEXT 1024 + +/** + * Largest embedded picture we hand on. Anything bigger is skipped + * rather than copied through the IPC channel. + */ +#define ID3_MAX_PICTURE (100 * 1024) + +/** + * Bytes we scan for the first MPEG audio frame. + */ +#define ID3_SYNC_WINDOW 8192 + +/** + * How many TXXX frames we report as unknown user text. + */ +#define ID3_MAX_USER_TEXT 8 + + +/** + * Bitrates in kbit/s. The row is picked from the MPEG version and the + * layer, the column is the four bit field in the frame header. + */ +static const unsigned int bitrate_table[5][16] = { + /* MPEG 1 Layer I */ + { 0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 0 }, + /* MPEG 1 Layer II */ + { 0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 0 }, + /* MPEG 1 Layer III */ + { 0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0 }, + /* MPEG 2 / 2.5 Layer I */ + { 0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256, 0 }, + /* MPEG 2 / 2.5 Layer II and III */ + { 0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0 } +}; + + +/** + * Sampling rates in Hz, indexed by version (0 = MPEG 1, 1 = MPEG 2, + * 2 = MPEG 2.5) and by the two-bit field in the header. + */ +static const unsigned int samplerate_table[3][3] = { + { 44100, 48000, 32000 }, + { 22050, 24000, 16000 }, + { 11025, 12000, 8000 } +}; + + +/** + * Channel modes, in the order the header field numbers them. + */ +static const char *const channel_modes[4] = { + "stereo", + "joint stereo", + "dual channel", + "mono" +}; + + +/** + * The ID3v1 genre numbers, as fixed by the original specification and + * extended by Winamp. ID3v2 `TCON' frames also refer to these by + * number, in the "(52)" spelling. + */ +static const char *const id3_genres[] = { + "Blues", "Classic Rock", "Country", "Dance", "Disco", "Funk", + "Grunge", "Hip-Hop", "Jazz", "Metal", "New Age", "Oldies", + "Other", "Pop", "R&B", "Rap", "Reggae", "Rock", + "Techno", "Industrial", "Alternative", "Ska", "Death Metal", "Pranks", + "Soundtrack", "Euro-Techno", "Ambient", "Trip-Hop", "Vocal", + "Jazz+Funk", "Fusion", "Trance", "Classical", "Instrumental", "Acid", + "House", "Game", "Sound Clip", "Gospel", "Noise", "Alternative Rock", + "Bass", "Soul", "Punk", "Space", "Meditative", "Instrumental Pop", + "Instrumental Rock", "Ethnic", "Gothic", "Darkwave", + "Techno-Industrial", "Electronic", "Pop-Folk", "Eurodance", "Dream", + "Southern Rock", "Comedy", "Cult", "Gangsta", "Top 40", + "Christian Rap", "Pop/Funk", "Jungle", "Native US", "Cabaret", + "New Wave", "Psychedelic", "Rave", "Showtunes", "Trailer", "Lo-Fi", + "Tribal", "Acid Punk", "Acid Jazz", "Polka", "Retro", "Musical", + "Rock & Roll", "Hard Rock", "Folk", "Folk-Rock", "National Folk", + "Swing", "Fast Fusion", "Bebop", "Latin", "Revival", "Celtic", + "Bluegrass", "Avantgarde", "Gothic Rock", "Progressive Rock", + "Psychedelic Rock", "Symphonic Rock", "Slow Rock", "Big Band", + "Chorus", "Easy Listening", "Acoustic", "Humour", "Speech", + "Chanson", "Opera", "Chamber Music", "Sonata", "Symphony", + "Booty Bass", "Primus", "Porn Groove", "Satire", "Slow Jam", "Club", + "Tango", "Samba", "Folklore", "Ballad", "Power Ballad", + "Rhythmic Soul", "Freestyle", "Duet", "Punk Rock", "Drum Solo", + "A Cappella", "Euro-House", "Dance Hall" +}; + + +/** + * An ID3v2 text frame identifier and the meta data type it maps to. + */ +struct FrameMap +{ + /** + * Four character frame identifier. + */ + const char *id; + + /** + * Type to report the frame's text as. + */ + enum EXTRACTOR_MetaType type; +}; + + +/** + * Text frames we understand. The first block is the ordinary + * descriptive tagging; the second is what tells us where the file came + * from. + */ +static const struct FrameMap text_frames[] = { + { "TIT1", EXTRACTOR_METATYPE_GROUP }, + { "TIT2", EXTRACTOR_METATYPE_TITLE }, + { "TIT3", EXTRACTOR_METATYPE_SUBTITLE }, + { "TALB", EXTRACTOR_METATYPE_ALBUM }, + { "TPE1", EXTRACTOR_METATYPE_ARTIST }, + { "TPE2", EXTRACTOR_METATYPE_PERFORMER }, + { "TPE3", EXTRACTOR_METATYPE_CONDUCTOR }, + { "TPE4", EXTRACTOR_METATYPE_INTERPRETATION }, + { "TCOM", EXTRACTOR_METATYPE_COMPOSER }, + { "TEXT", EXTRACTOR_METATYPE_WRITER }, + { "TRCK", EXTRACTOR_METATYPE_TRACK_NUMBER }, + { "TPOS", EXTRACTOR_METATYPE_DISC_NUMBER }, + { "TYER", EXTRACTOR_METATYPE_PUBLICATION_DATE }, + { "TDRL", EXTRACTOR_METATYPE_PUBLICATION_DATE }, + { "TDRC", EXTRACTOR_METATYPE_CREATION_DATE }, + { "TPUB", EXTRACTOR_METATYPE_PUBLISHER }, + { "TCOP", EXTRACTOR_METATYPE_COPYRIGHT }, + { "TLAN", EXTRACTOR_METATYPE_LANGUAGE }, + { "TBPM", EXTRACTOR_METATYPE_BEATS_PER_MINUTE }, + { "TSRC", EXTRACTOR_METATYPE_ISRC }, + { "TMOO", EXTRACTOR_METATYPE_MOOD }, + { "TOAL", EXTRACTOR_METATYPE_ORIGINAL_TITLE }, + { "TOPE", EXTRACTOR_METATYPE_ORIGINAL_PERFORMER }, + { "TOLY", EXTRACTOR_METATYPE_ORIGINAL_WRITER }, + { "TORY", EXTRACTOR_METATYPE_ORIGINAL_RELEASE_YEAR }, + /* provenance */ + { "TENC", EXTRACTOR_METATYPE_ENCODED_BY }, + { "TSSE", EXTRACTOR_METATYPE_ENCODER_SETTINGS }, + { "TOWN", EXTRACTOR_METATYPE_OWNER_USER }, + { "TFLT", EXTRACTOR_METATYPE_SOURCE_DEVICE }, + { "TMED", EXTRACTOR_METATYPE_SOURCE_DEVICE }, + { NULL, EXTRACTOR_METATYPE_RESERVED } +}; + + +/** + * A three character ID3v2.2 frame identifier and the 2.3 identifier + * that means the same thing. + */ +struct FrameAlias +{ + /** + * ID3v2.2 identifier. + */ + const char *old_id; + + /** + * Equivalent ID3v2.3 identifier. + */ + const char *new_id; +}; + + +/** + * ID3v2.2 used three character frame identifiers. Rather than a + * second dispatcher we translate them to their 2.3 equivalents and run + * the same code. `PIC' is deliberately absent: its payload names the + * image format in three characters where `APIC' carries a MIME string, + * so it cannot share the handler, and an attached picture is not worth + * a second one. + */ +static const struct FrameAlias v22_aliases[] = { + { "TT1", "TIT1" }, { "TT2", "TIT2" }, { "TT3", "TIT3" }, + { "TAL", "TALB" }, { "TP1", "TPE1" }, { "TP2", "TPE2" }, + { "TP3", "TPE3" }, { "TP4", "TPE4" }, { "TCM", "TCOM" }, + { "TXT", "TEXT" }, { "TRK", "TRCK" }, { "TPA", "TPOS" }, + { "TYE", "TYER" }, { "TPB", "TPUB" }, { "TCR", "TCOP" }, + { "TLA", "TLAN" }, { "TBP", "TBPM" }, { "TRC", "TSRC" }, + { "TOT", "TOAL" }, { "TOA", "TOPE" }, { "TOL", "TOLY" }, + { "TOR", "TORY" }, { "TEN", "TENC" }, { "TSS", "TSSE" }, + { "TFT", "TFLT" }, { "TMT", "TMED" }, { "TCO", "TCON" }, + { "COM", "COMM" }, { "ULT", "USLT" }, { "UFI", "UFID" }, + { "TXX", "TXXX" }, { "WAF", "WOAF" }, { "WAR", "WOAR" }, + { "WCM", "WCOM" }, { "WPY", "WPAY" }, + { NULL, NULL } +}; + + +/** + * TXXX descriptions that name a store account. The Apple ID of a + * purchase normally lives in an M4A `----:com.apple.iTunes:...' atom + * rather than in ID3, but transcoders copy it across under these + * descriptions, and other stores stamp their own. + */ +static const char *const purchase_descriptions[] = { + "apid", + "apple id", + "purchase account", + "purchased by", + "itunes account", + "itunes_purchase_account", + NULL +}; + + +/** + * What we have already reported, so that the ID3v1 tag at the end of + * the file can fill gaps without contradicting the ID3v2 tag in front + * of it. + */ +struct Id3Context +{ + /** + * Extraction context. + */ + struct EXTRACTOR_ExtractContext *ec; + + /** + * Number of TXXX frames reported so far. + */ + unsigned int user_text; + + /** + * Indexed by meta data type: 1 once we have emitted one. + */ + unsigned char seen[EXTRACTOR_METATYPE_LAST]; +}; + + +/** + * Emit a NUL-terminated string and remember that we did. + * + * @param ctx our state + * @param type meta data type + * @param s the string + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +id3_emit (struct Id3Context *ctx, + enum EXTRACTOR_MetaType type, + const char *s) +{ + /* An empty value is dropped by the emit helper, so it must not + count as having covered this type: the ID3v1 tag may still have + something real to say about it. */ + if ( ('\0' != s[0]) && + (0 < type) && + (type < EXTRACTOR_METATYPE_LAST) ) + ctx->seen[type] = 1; + return EXTRACTOR_forensic_emit_text_ (ctx->ec, + ID3_PLUGIN, + type, + s, + strlen (s)); +} + + +/** + * Convert an ISO-8859-1 string to UTF-8. The bytes cannot simply be + * passed through: every code point from 0x80 up needs two bytes in + * UTF-8, and handing the raw bytes to the caller would produce invalid + * UTF-8 for exactly the accented characters that make a tag + * interesting. + * + * @param in the Latin-1 bytes + * @param len number of bytes in @a in + * @param[out] out where to write the NUL-terminated UTF-8 + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the result does not fit + */ +static int +latin1_to_utf8 (const unsigned char *in, + size_t len, + char *out, + size_t out_size) +{ + size_t o = 0; + + for (size_t i = 0; i < len; i++) + { + if (0 == in[i]) + break; + if (o + 2 >= out_size) + return 0; + if (in[i] < 0x80) + { + out[o++] = (char) in[i]; + } + else + { + out[o++] = (char) (0xC0 | (in[i] >> 6)); + out[o++] = (char) (0x80 | (in[i] & 0x3F)); + } + } + out[o] = '\0'; + return 1; +} + + +/** + * Convert a UTF-16 string to UTF-8. + * + * @param in the UTF-16 bytes + * @param len number of bytes in @a in + * @param big_endian 1 for UTF-16BE, 0 for UTF-16LE + * @param[out] out where to write the NUL-terminated UTF-8 + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the input is malformed or does not fit + */ +static int +utf16_to_utf8 (const unsigned char *in, + size_t len, + int big_endian, + char *out, + size_t out_size) +{ + size_t o = 0; + size_t i = 0; + + while (i + 1 < len) + { + uint32_t cp = big_endian + ? EXTRACTOR_forensic_be16_ (&in[i]) + : EXTRACTOR_forensic_le16_ (&in[i]); + + i += 2; + if (0 == cp) + break; + if ( (0xD800 <= cp) && (cp <= 0xDBFF) ) + { + uint32_t lo; + + if (i + 1 >= len) + return 0; + lo = big_endian + ? EXTRACTOR_forensic_be16_ (&in[i]) + : EXTRACTOR_forensic_le16_ (&in[i]); + if ( (lo < 0xDC00) || (lo > 0xDFFF) ) + return 0; + i += 2; + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + } + else if ( (0xDC00 <= cp) && (cp <= 0xDFFF) ) + { + return 0; + } + if (o + 4 >= out_size) + return 0; + if (cp < 0x80) + { + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + } + out[o] = '\0'; + return 1; +} + + +/** + * Decode a string in the encoding an ID3v2 text frame declares in its + * first byte. + * + * @param enc 0 = ISO-8859-1, 1 = UTF-16 with byte order mark, + * 2 = UTF-16BE, 3 = UTF-8 + * @param in the encoded bytes + * @param len number of bytes in @a in + * @param[out] out where to write the NUL-terminated UTF-8 + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the encoding is unknown or the input is + * malformed + */ +static int +decode_text (unsigned int enc, + const unsigned char *in, + size_t len, + char *out, + size_t out_size) +{ + switch (enc) + { + case 0: + return latin1_to_utf8 (in, + len, + out, + out_size); + case 1: + if (len < 2) + return 0; + if ( (0xFF == in[0]) && (0xFE == in[1]) ) + return utf16_to_utf8 (&in[2], + len - 2, + 0, + out, + out_size); + if ( (0xFE == in[0]) && (0xFF == in[1]) ) + return utf16_to_utf8 (&in[2], + len - 2, + 1, + out, + out_size); + /* no byte order mark although the encoding byte promised one; + little endian is what the taggers that get this wrong write */ + return utf16_to_utf8 (in, + len, + 0, + out, + out_size); + case 2: + return utf16_to_utf8 (in, + len, + 1, + out, + out_size); + case 3: + { + size_t n = len; + + for (size_t i = 0; i < len; i++) + if (0 == in[i]) + { + n = i; + break; + } + if (n >= out_size) + return 0; + if (! EXTRACTOR_forensic_utf8_valid_ ((const char *) in, + n)) + return 0; + memcpy (out, + in, + n); + out[n] = '\0'; + return 1; + } + default: + return 0; + } +} + + +/** + * Find the terminator of the string at the start of @a in. + * + * @param enc text encoding as in #decode_text() + * @param in the bytes + * @param len number of bytes in @a in + * @param[out] next offset of the byte after the terminator + * @return length of the string in bytes, excluding the terminator; + * @a len (and @a next set to @a len) if there is none + */ +static size_t +string_end (unsigned int enc, + const unsigned char *in, + size_t len, + size_t *next) +{ + if ( (1 == enc) || (2 == enc) ) + { + for (size_t i = 0; i + 1 < len; i += 2) + if ( (0 == in[i]) && (0 == in[i + 1]) ) + { + *next = i + 2; + return i; + } + *next = len; + return len; + } + for (size_t i = 0; i < len; i++) + if (0 == in[i]) + { + *next = i + 1; + return i; + } + *next = len; + return len; +} + + +/** + * Report a genre, resolving the numeric references ID3 allows. + * `TCON' may hold "(52)", "52" or "Electronic"; only the last is + * useful to a human. + * + * @param ctx our state + * @param text the frame's text + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_genre (struct Id3Context *ctx, + const char *text) +{ + const char *p = text; + char *endp; + unsigned long n; + + if ('(' == p[0]) + p++; + if ( ('0' <= p[0]) && (p[0] <= '9') ) + { + n = strtoul (p, + &endp, + 10); + if ( ( (')' == *endp) || ('\0' == *endp) ) && + (n < sizeof (id3_genres) / sizeof (id3_genres[0])) ) + return id3_emit (ctx, + EXTRACTOR_METATYPE_GENRE, + id3_genres[n]); + } + return id3_emit (ctx, + EXTRACTOR_METATYPE_GENRE, + text); +} + + +/** + * Handle one ID3v2 frame. + * + * @param ctx our state + * @param id the four character frame identifier, NUL-terminated + * @param data the frame's payload + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +handle_frame (struct Id3Context *ctx, + const char *id, + const unsigned char *data, + size_t len) +{ + char text[ID3_MAX_TEXT]; + + if (0 == len) + return 0; + if ( ('T' == id[0]) && + (0 != strcmp (id, + "TXXX")) ) + { + if (! decode_text (data[0], + &data[1], + len - 1, + text, + sizeof (text))) + return 0; + if (0 == strcmp (id, + "TCON")) + return emit_genre (ctx, + text); + for (unsigned int i = 0; NULL != text_frames[i].id; i++) + if (0 == strcmp (id, + text_frames[i].id)) + return id3_emit (ctx, + text_frames[i].type, + text); + return 0; /* a text frame we do not care about */ + } + if (0 == strcmp (id, + "TXXX")) + { + char value[ID3_MAX_TEXT]; + size_t dlen; + size_t next; + + dlen = string_end (data[0], + &data[1], + len - 1, + &next); + if (! decode_text (data[0], + &data[1], + dlen, + text, + sizeof (text))) + return 0; + if (next >= len - 1) + return 0; + if (! decode_text (data[0], + &data[1 + next], + len - 1 - next, + value, + sizeof (value))) + return 0; + for (unsigned int i = 0; NULL != purchase_descriptions[i]; i++) + { + size_t n = strlen (purchase_descriptions[i]); + size_t k; + + if (strlen (text) != n) + continue; + for (k = 0; k < n; k++) + if (tolower ((unsigned char) text[k]) != purchase_descriptions[i][k]) + break; + if (k == n) + return id3_emit (ctx, + EXTRACTOR_METATYPE_PURCHASE_ACCOUNT, + value); + } + if (ctx->user_text >= ID3_MAX_USER_TEXT) + return 0; + ctx->user_text++; + return (0 != EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_UNKNOWN, + "%s: %s", + text, + value)) ? 1 : 0; + } + if ( ('W' == id[0]) && + (0 != strcmp (id, + "WXXX")) ) + { + /* the plain URL frames carry no encoding byte; they are always + ISO-8859-1 */ + if (! latin1_to_utf8 (data, + len, + text, + sizeof (text))) + return 0; + return id3_emit (ctx, + EXTRACTOR_METATYPE_URL, + text); + } + if (0 == strcmp (id, + "WXXX")) + { + size_t next; + + (void) string_end (data[0], + &data[1], + len - 1, + &next); + if (next >= len - 1) + return 0; + if (! latin1_to_utf8 (&data[1 + next], + len - 1 - next, + text, + sizeof (text))) + return 0; + return id3_emit (ctx, + EXTRACTOR_METATYPE_URL, + text); + } + if ( (0 == strcmp (id, + "COMM")) || + (0 == strcmp (id, + "USLT")) ) + { + size_t next; + + /* encoding byte, three character language code, a short content + descriptor and then the text */ + if (len < 5) + return 0; + (void) string_end (data[0], + &data[4], + len - 4, + &next); + if (next >= len - 4) + return 0; + if (! decode_text (data[0], + &data[4 + next], + len - 4 - next, + text, + sizeof (text))) + return 0; + return id3_emit (ctx, + (0 == strcmp (id, "COMM")) + ? EXTRACTOR_METATYPE_COMMENT + : EXTRACTOR_METATYPE_LYRICS, + text); + } + if (0 == strcmp (id, + "UFID")) + { + char owner[256]; + size_t next; + size_t olen; + int printable = 1; + + olen = string_end (0, + data, + len, + &next); + if ( (0 == olen) || + (next >= len) ) + return 0; + if (! latin1_to_utf8 (data, + olen, + owner, + sizeof (owner))) + return 0; + for (size_t i = next; i < len; i++) + if ( (data[i] < 0x20) || (data[i] > 0x7E) ) + { + printable = 0; + break; + } + if (printable) + { + size_t n = len - next; + + if (n > sizeof (text) / 2) + n = sizeof (text) / 2; + memcpy (text, + &data[next], + n); + text[n] = '\0'; + ctx->seen[EXTRACTOR_METATYPE_SERIAL] = 1; + return (0 != EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_SERIAL, + "%s: %s", + owner, + text)) ? 1 : 0; + } + { + static const char hex[] = "0123456789abcdef"; + size_t n = len - next; + size_t o; + + if (n > 32) + n = 32; + o = strlen (owner); + if (o + 2 + 2 * n + 1 > sizeof (text)) + return 0; + memcpy (text, + owner, + o); + text[o++] = ':'; + text[o++] = ' '; + for (size_t i = 0; i < n; i++) + { + text[o++] = hex[data[next + i] >> 4]; + text[o++] = hex[data[next + i] & 0x0F]; + } + text[o] = '\0'; + } + return id3_emit (ctx, + EXTRACTOR_METATYPE_SERIAL, + text); + } + if (0 == strcmp (id, + "PRIV")) + { + size_t next; + size_t olen; + + /* the owner identifier names the application that stamped the + file -- iTunes, Windows Media Player, a shop's downloader. The + payload behind it is opaque binary and stays where it is. */ + olen = string_end (0, + data, + len, + &next); + if (0 == olen) + return 0; + if (! latin1_to_utf8 (data, + olen, + text, + sizeof (text))) + return 0; + return id3_emit (ctx, + EXTRACTOR_METATYPE_APPLICATION_ID, + text); + } + if (0 == strcmp (id, + "APIC")) + { + char mime[128]; + size_t next; + size_t mlen; + size_t pos; + + mlen = string_end (0, + &data[1], + len - 1, + &next); + if (! latin1_to_utf8 (&data[1], + mlen, + mime, + sizeof (mime))) + return 0; + pos = 1 + next; + if (pos + 1 >= len) + return 0; + pos++; /* picture type byte */ + (void) string_end (data[0], + &data[pos], + len - pos, + &next); + pos += next; + if (pos >= len) + return 0; + if (len - pos > ID3_MAX_PICTURE) + return 0; /* too big to be worth copying through the pipe */ + if ('\0' == mime[0]) + strcpy (mime, + "image/jpeg"); + ctx->seen[EXTRACTOR_METATYPE_COVER_PICTURE] = 1; + return (0 != ctx->ec->proc (ctx->ec->cls, + ID3_PLUGIN, + EXTRACTOR_METATYPE_COVER_PICTURE, + EXTRACTOR_METAFORMAT_BINARY, + mime, + (const char *) &data[pos], + len - pos)) ? 1 : 0; + } + return 0; +} + + +/** + * Undo the ID3v2 unsynchronisation scheme in place: every 0xFF byte + * that was followed by an inserted 0x00 loses that 0x00. + * + * @param[in,out] buf the tag data + * @param len number of bytes in @a buf + * @return number of bytes after the transformation + */ +static size_t +deunsynchronise (unsigned char *buf, + size_t len) +{ + size_t o = 0; + + for (size_t i = 0; i < len; i++) + { + buf[o++] = buf[i]; + if ( (0xFF == buf[i]) && + (i + 1 < len) && + (0x00 == buf[i + 1]) ) + i++; + } + return o; +} + + +/** + * Walk the frames of an ID3v2 tag that has already been read into + * memory. + * + * @param ctx our state + * @param major ID3v2 minor version: 2, 3 or 4 + * @param tag_unsync 1 if the tag header set the unsynchronisation flag + * @param buf the tag body, after the ten byte header + * @param len number of bytes in @a buf + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +walk_frames (struct Id3Context *ctx, + unsigned int major, + int tag_unsync, + unsigned char *buf, + size_t len) +{ + size_t pos = 0; + unsigned int count = 0; + const size_t hdr = (2 == major) ? 6 : 10; + const size_t idlen = (2 == major) ? 3 : 4; + + while ( (pos + hdr <= len) && + (count < ID3_MAX_FRAMES) ) + { + char id[5]; + size_t size; + unsigned char *body; + size_t blen; + unsigned char flags2 = 0; + int frame_unsync = tag_unsync; + + if (0 == buf[pos]) + break; /* padding */ + for (size_t i = 0; i < idlen; i++) + { + unsigned char c = buf[pos + i]; + + if ( ( (c < 'A') || (c > 'Z') ) && + ( (c < '0') || (c > '9') ) ) + return 0; /* not a frame identifier; the tag ends here */ + id[i] = (char) c; + } + id[idlen] = '\0'; + if (2 == major) + { + size = ((size_t) buf[pos + 3] << 16) + | ((size_t) buf[pos + 4] << 8) + | (size_t) buf[pos + 5]; + } + else if (4 == major) + { + /* In 2.4 the frame size is synchsafe -- seven bits per byte -- + while in 2.3 it is a plain big-endian integer. Getting this + backwards is the classic ID3 bug: it only shows up for frames + of 128 bytes or more, so a tagger tested on short titles looks + fine and falls apart on the first embedded picture. Taggers + that write 2.4 headers with 2.3 sizes exist, and they give + themselves away by setting the high bit of a size byte, which + a synchsafe integer never does. */ + if ( (0 != (buf[pos + 4] & 0x80)) || + (0 != (buf[pos + 5] & 0x80)) || + (0 != (buf[pos + 6] & 0x80)) || + (0 != (buf[pos + 7] & 0x80)) ) + size = (size_t) EXTRACTOR_forensic_be32_ (&buf[pos + 4]); + else + size = (((size_t) buf[pos + 4]) << 21) + | (((size_t) buf[pos + 5]) << 14) + | (((size_t) buf[pos + 6]) << 7) + | ((size_t) buf[pos + 7]); + } + else + { + size = (size_t) EXTRACTOR_forensic_be32_ (&buf[pos + 4]); + } + if (0 == size) + break; /* a zero length frame cannot be followed by anything */ + if (size > len - pos - hdr) + break; /* runs past the end of the tag */ + if (2 != major) + flags2 = buf[pos + 9]; + body = &buf[pos + hdr]; + blen = size; + pos += hdr + size; + count++; + if (3 == major) + { + if (0 != (flags2 & 0xC0)) + continue; /* compressed or encrypted */ + if (0 != (flags2 & 0x20)) + { + if (0 == blen) + continue; + body++; + blen--; /* group identifier */ + } + } + else if (4 == major) + { + if (0 != (flags2 & 0x0C)) + continue; /* compressed or encrypted */ + if (0 != (flags2 & 0x40)) + { + if (0 == blen) + continue; + body++; + blen--; /* group identifier */ + } + if (0 != (flags2 & 0x02)) + frame_unsync = 1; + if (0 != (flags2 & 0x01)) + { + if (blen < 4) + continue; + body += 4; + blen -= 4; /* data length indicator */ + } + } + if (frame_unsync) + blen = deunsynchronise (body, + blen); + if (2 == major) + { + unsigned int a; + + for (a = 0; NULL != v22_aliases[a].old_id; a++) + if (0 == strcmp (id, + v22_aliases[a].old_id)) + break; + if (NULL == v22_aliases[a].old_id) + continue; /* a 2.2 frame we do not handle */ + if (handle_frame (ctx, + v22_aliases[a].new_id, + body, + blen)) + return 1; + continue; + } + if (handle_frame (ctx, + id, + body, + blen)) + return 1; + } + return 0; +} + + +/** + * A decoded MPEG audio frame header. + */ +struct MpegHeader +{ + /** + * 1, 2 or 25 for MPEG 1, MPEG 2 and MPEG 2.5. + */ + unsigned int version; + + /** + * 1, 2 or 3. + */ + unsigned int layer; + + /** + * Bitrate in bits per second. + */ + unsigned int bitrate; + + /** + * Sampling rate in Hz. + */ + unsigned int sample_rate; + + /** + * Channel mode, 0 to 3, indexing #channel_modes. + */ + unsigned int mode; + + /** + * Number of samples this frame encodes. + */ + unsigned int samples; + + /** + * Length of the frame in bytes, including the header. + */ + unsigned int frame_len; + + /** + * 1 if a CRC follows the header. + */ + int protected_frame; +}; + + +/** + * Decode and validate a four byte MPEG audio frame header. + * + * Every reserved combination is rejected: without that, one byte in + * every 2048 of arbitrary binary starts something that looks like a + * frame. + * + * @param h the four bytes + * @param[out] f the decoded header + * @return 1 if @a h is a valid header, 0 if not + */ +static int +parse_mpeg_header (const unsigned char *h, + struct MpegHeader *f) +{ + unsigned int vbits; + unsigned int lbits; + unsigned int brindex; + unsigned int srindex; + unsigned int column; + unsigned int vrow; + unsigned int padding; + + if (0xFF != h[0]) + return 0; + if (0xE0 != (h[1] & 0xE0)) + return 0; + vbits = (h[1] >> 3) & 0x03; + lbits = (h[1] >> 1) & 0x03; + if (1 == vbits) + return 0; /* reserved version */ + if (0 == lbits) + return 0; /* reserved layer */ + brindex = (h[2] >> 4) & 0x0F; + srindex = (h[2] >> 2) & 0x03; + if ( (0 == brindex) || + (15 == brindex) ) + return 0; /* "free" and "bad" are both unusable to us */ + if (3 == srindex) + return 0; /* reserved */ + if (3 == (h[3] & 0x03)) + return 0; /* reserved emphasis */ + f->layer = 4 - lbits; + switch (vbits) + { + case 0: + f->version = 25; + vrow = 2; + break; + case 2: + f->version = 2; + vrow = 1; + break; + default: + f->version = 1; + vrow = 0; + break; + } + if (1 == f->version) + column = f->layer - 1; + else + column = (1 == f->layer) ? 3 : 4; + f->bitrate = 1000 * bitrate_table[column][brindex]; + f->sample_rate = samplerate_table[vrow][srindex]; + f->mode = (h[3] >> 6) & 0x03; + f->protected_frame = (0 == (h[1] & 0x01)) ? 1 : 0; + padding = (h[2] >> 1) & 0x01; + if (1 == f->layer) + f->samples = 384; + else if (2 == f->layer) + f->samples = 1152; + else + f->samples = (1 == f->version) ? 1152 : 576; + if (1 == f->layer) + f->frame_len = (12 * f->bitrate / f->sample_rate + padding) * 4; + else + f->frame_len = (f->samples / 8) * f->bitrate / f->sample_rate + padding; + if (f->frame_len < 24) + return 0; + return 1; +} + + +/** + * Report the ReplayGain field at @a p, if it holds one. + * + * @param ctx our state + * @param type meta data type to report it as + * @param p the two bytes + * @param want_name the name code this field must carry (1 = track, + * 2 = album) + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_replay_gain (struct Id3Context *ctx, + enum EXTRACTOR_MetaType type, + const unsigned char *p, + unsigned int want_name) +{ + uint16_t v = EXTRACTOR_forensic_be16_ (p); + unsigned int name = (v >> 13) & 0x07; + unsigned int value = v & 0x01FF; + + if (name != want_name) + return 0; + if (0 == value) + return 0; + ctx->seen[type] = 1; + return (0 != EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + type, + "%s%u.%u dB", + (0 != ((v >> 9) & 0x01)) ? "-" : "+", + value / 10, + value % 10)) ? 1 : 0; +} + + +/** + * Append a comma separated fragment to the encoder settings summary, + * never running past the end of @a buf. + * + * @param buf where to build the summary + * @param size size of @a buf in bytes + * @param[in,out] off current length of the summary + * @param fmt printf format string for the fragment + */ +static void +append_setting (char *buf, + size_t size, + size_t *off, + const char *fmt, + ...) +__attribute__ ((format (printf, 4, 5))); + +static void +append_setting (char *buf, + size_t size, + size_t *off, + const char *fmt, + ...) +{ + va_list ap; + int n; + + if (*off + 3 >= size) + return; + if (0 != *off) + { + buf[(*off)++] = ','; + buf[(*off)++] = ' '; + buf[*off] = '\0'; + } + va_start (ap, fmt); + n = vsnprintf (&buf[*off], + size - *off, + fmt, + ap); + va_end (ap); + if (0 >= n) + return; + if (((size_t) n) >= size - *off) + *off = size - 1; + else + *off += (size_t) n; +} + + +/** + * Read the Xing/Info header and the LAME tag that may sit inside the + * first audio frame. + * + * The LAME tag is the most identifying thing in an MP3: it names the + * encoder build and the settings it ran with, which together pin down + * the ripping software far more precisely than any text frame. + * + * @param ctx our state + * @param f the decoded frame header + * @param frame the frame's bytes + * @param len number of bytes of the frame we have + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_xing (struct Id3Context *ctx, + const struct MpegHeader *f, + const unsigned char *frame, + size_t len) +{ + size_t xo; + size_t p; + uint32_t flags; + uint32_t frames = 0; + uint32_t nbytes = 0; + uint32_t quality = 0; + int have_quality = 0; + int vbr; + char settings[ID3_MAX_TEXT]; + size_t so = 0; + + /* The tag sits just past the side information, whose size depends on + the version and on whether the frame is mono. A CRC, if present, + takes two more bytes right after the header. */ + if (1 == f->version) + xo = (3 == f->mode) ? 21 : 36; + else + xo = (3 == f->mode) ? 13 : 21; + if (f->protected_frame) + xo += 2; + if (xo + 8 > len) + return 0; + if (0 == memcmp (&frame[xo], + "Xing", + 4)) + { + vbr = 1; + } + else if (0 == memcmp (&frame[xo], + "Info", + 4)) + { + vbr = 0; + } + else + { + return 0; + } + flags = EXTRACTOR_forensic_be32_ (&frame[xo + 4]); + p = xo + 8; + if (0 != (flags & 0x01)) + { + if (p + 4 > len) + return 0; + frames = EXTRACTOR_forensic_be32_ (&frame[p]); + p += 4; + } + if (0 != (flags & 0x02)) + { + if (p + 4 > len) + return 0; + nbytes = EXTRACTOR_forensic_be32_ (&frame[p]); + p += 4; + } + if (0 != (flags & 0x04)) + { + if (p + 100 > len) + return 0; + p += 100; /* seek table */ + } + if (0 != (flags & 0x08)) + { + if (p + 4 > len) + return 0; + quality = EXTRACTOR_forensic_be32_ (&frame[p]); + have_quality = 1; + p += 4; + } + if ( (0 != frames) && + (frames < 100000000u) && + (0 != f->sample_rate) ) + { + uint64_t samples = (uint64_t) frames * f->samples; + unsigned int secs = (unsigned int) (samples / f->sample_rate); + + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_DURATION, + "%u:%02u:%02u", + secs / 3600, + (secs / 60) % 60, + secs % 60)) + return 1; + ctx->seen[EXTRACTOR_METATYPE_DURATION] = 1; + if (0 != nbytes) + { + uint64_t avg = ((uint64_t) nbytes) * 8 * f->sample_rate / samples; + + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_BITRATE, + "%llu", + (unsigned long long) avg)) + return 1; + ctx->seen[EXTRACTOR_METATYPE_BITRATE] = 1; + } + } + /* The LAME extension follows the Xing fields. 36 bytes: a nine + character encoder string, then the settings it ran with. */ + if (p + 36 > len) + { + /* no room for the extension; still report what the magic said */ + return id3_emit (ctx, + EXTRACTOR_METATYPE_BITRATE_MODE, + vbr ? "VBR" : "CBR"); + } + { + const unsigned char *l = &frame[p]; + char version[10]; + unsigned int method = l[9] & 0x0F; + unsigned int lowpass = l[10]; + unsigned int abr = l[20]; + unsigned int delay = ((unsigned int) l[21] << 4) | (l[22] >> 4); + unsigned int padding = ((unsigned int) (l[22] & 0x0F) << 8) | l[23]; + unsigned int preset = EXTRACTOR_forensic_be16_ (&l[26]) & 0x07FF; + const char *mode; + int printable = 0; + + /* We do not check the tag's own CRC-16. The field exists, but the + encoders that write this tag do not agree on what it covers -- + LAME and ffmpeg produce different values for the same layout -- + so a mismatch would say more about the writer than about the + data, and rejecting on it would lose real metadata. The nine + character encoder string is self-validating enough: it has to be + printable ASCII with something in it. */ + for (unsigned int i = 0; i < 9; i++) + { + if ( (l[i] < 0x20) || (l[i] > 0x7E) ) + { + printable = 0; + break; + } + if (' ' != l[i]) + printable++; + } + if (printable < 4) + return id3_emit (ctx, + EXTRACTOR_METATYPE_BITRATE_MODE, + vbr ? "VBR" : "CBR"); + memcpy (version, + l, + 9); + version[9] = '\0'; + switch (method) + { + case 1: + mode = "CBR"; + break; + case 2: + mode = "ABR"; + break; + case 8: + mode = "CBR (2 pass)"; + break; + case 9: + mode = "ABR (2 pass)"; + break; + case 0: + mode = vbr ? "VBR" : "CBR"; + break; + default: + mode = "VBR"; + break; + } + if (id3_emit (ctx, + EXTRACTOR_METATYPE_BITRATE_MODE, + mode)) + return 1; + if (id3_emit (ctx, + EXTRACTOR_METATYPE_ENCODER, + version)) + return 1; + for (unsigned int i = 0; i < 9; i++) + if ( ('0' <= version[i]) && (version[i] <= '9') ) + { + if (id3_emit (ctx, + EXTRACTOR_METATYPE_ENCODER_VERSION, + &version[i])) + return 1; + break; + } + settings[0] = '\0'; + if (0 != lowpass) + append_setting (settings, + sizeof (settings), + &so, + "lowpass %u Hz", + 100 * lowpass); + if ( (have_quality) && + (quality <= 100) ) + append_setting (settings, + sizeof (settings), + &so, + "quality %u", + (unsigned int) quality); + append_setting (settings, + sizeof (settings), + &so, + "VBR method %u", + method); + if (0 != abr) + append_setting (settings, + sizeof (settings), + &so, + "min bitrate %u kbit/s", + abr); + if (0 != preset) + append_setting (settings, + sizeof (settings), + &so, + "preset %u", + preset); + append_setting (settings, + sizeof (settings), + &so, + "delay %u, padding %u", + delay, + padding); + if (id3_emit (ctx, + EXTRACTOR_METATYPE_ENCODER_SETTINGS, + settings)) + return 1; + if (emit_replay_gain (ctx, + EXTRACTOR_METATYPE_TRACK_GAIN, + &l[15], + 1)) + return 1; + if (emit_replay_gain (ctx, + EXTRACTOR_METATYPE_ALBUM_GAIN, + &l[17], + 2)) + return 1; + } + return 0; +} + + +/** + * Find and describe the first MPEG audio frame at or after @a start. + * + * @param ctx our state + * @param start where the audio should begin + * @param size size of the file + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_audio (struct Id3Context *ctx, + uint64_t start, + uint64_t size) +{ + unsigned char window[ID3_SYNC_WINDOW]; + struct MpegHeader f; + size_t wlen; + size_t at = 0; + int have = 0; + + if (start >= size) + return 0; + wlen = ((size - start) < sizeof (window)) + ? (size_t) (size - start) + : sizeof (window); + if (wlen < 4) + return 0; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) start, + window, + wlen)) + return 0; + for (size_t i = 0; i + 4 <= wlen; i++) + { + struct MpegHeader next; + + if (0xFF != window[i]) + continue; + if (! parse_mpeg_header (&window[i], + &f)) + continue; + /* Confirm with the frame after this one where we can: a single + header pattern turns up in arbitrary data often enough that it + is not evidence on its own. */ + if (i + f.frame_len + 4 <= wlen) + { + if (! parse_mpeg_header (&window[i + f.frame_len], + &next)) + continue; + if ( (next.version != f.version) || + (next.layer != f.layer) || + (next.sample_rate != f.sample_rate) ) + continue; + } + at = i; + have = 1; + break; + } + if (! have) + return 0; + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_CODEC, + "MPEG %s Layer %s", + (1 == f.version) ? "1" + : ((2 == f.version) ? "2" : "2.5"), + (1 == f.layer) ? "I" + : ((2 == f.layer) ? "II" : "III"))) + return 1; + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_FORMAT, + "MPEG %s Layer %s, %s", + (1 == f.version) ? "1" + : ((2 == f.version) ? "2" : "2.5"), + (1 == f.layer) ? "I" + : ((2 == f.layer) ? "II" : "III"), + channel_modes[f.mode])) + return 1; + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_SAMPLE_RATE, + "%u", + f.sample_rate)) + return 1; + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_CHANNELS, + "%u", + (3 == f.mode) ? 1 : 2)) + return 1; + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_NOMINAL_BITRATE, + "%u", + f.bitrate)) + return 1; + if (extract_xing (ctx, + &f, + &window[at], + wlen - at)) + return 1; + if (! ctx->seen[EXTRACTOR_METATYPE_BITRATE]) + { + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_BITRATE, + "%u", + f.bitrate)) + return 1; + ctx->seen[EXTRACTOR_METATYPE_BITRATE] = 1; + } + return 0; +} + + +/** + * Report a field of the ID3v1 tag, unless the ID3v2 tag already + * supplied one of the same kind. + * + * @param ctx our state + * @param type meta data type + * @param data the fixed width field + * @param len width of the field + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_v1_field (struct Id3Context *ctx, + enum EXTRACTOR_MetaType type, + const unsigned char *data, + size_t len) +{ + char text[128]; + + if (ctx->seen[type]) + return 0; + len = EXTRACTOR_forensic_trim_ ((const char *) data, + len); + if (0 == len) + return 0; + if (! latin1_to_utf8 (data, + len, + text, + sizeof (text))) + return 0; + return id3_emit (ctx, + type, + text); +} + + +/** + * Read the ID3v1 tag in the last 128 bytes of the file. It only ever + * fills gaps: where an ID3v2 tag said anything, it wins, because the + * v1 fields are truncated to 30 bytes and have no declared character + * set. + * + * @param ctx our state + * @param tag the 128 bytes + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_v1 (struct Id3Context *ctx, + const unsigned char *tag) +{ + if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_TITLE, + &tag[3], + 30)) + return 1; + if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_ARTIST, + &tag[33], + 30)) + return 1; + if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_ALBUM, + &tag[63], + 30)) + return 1; + if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_PUBLICATION_DATE, + &tag[93], + 4)) + return 1; + /* ID3v1.1 stole the last two bytes of the comment for a track + number: a NUL in the second to last byte marks it */ + if ( (0 == tag[125]) && + (0 != tag[126]) ) + { + if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_COMMENT, + &tag[97], + 28)) + return 1; + if (! ctx->seen[EXTRACTOR_METATYPE_TRACK_NUMBER]) + { + if (EXTRACTOR_forensic_emit_ (ctx->ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_TRACK_NUMBER, + "%u", + (unsigned int) tag[126])) + return 1; + ctx->seen[EXTRACTOR_METATYPE_TRACK_NUMBER] = 1; + } + } + else if (emit_v1_field (ctx, + EXTRACTOR_METATYPE_COMMENT, + &tag[97], + 30)) + { + return 1; + } + if ( (! ctx->seen[EXTRACTOR_METATYPE_GENRE]) && + (tag[127] < sizeof (id3_genres) / sizeof (id3_genres[0])) ) + return id3_emit (ctx, + EXTRACTOR_METATYPE_GENRE, + id3_genres[tag[127]]); + return 0; +} + + +/** + * Read and parse the ID3v2 tag at the start of the file. + * + * @param ctx our state + * @param head the ten byte tag header + * @param tag_size size of the tag body, from the header + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_v2 (struct Id3Context *ctx, + const unsigned char *head, + uint64_t tag_size) +{ + unsigned char *buf; + size_t len; + size_t skip = 0; + unsigned int major = head[3]; + int tag_unsync = (0 != (head[5] & 0x80)) ? 1 : 0; + int ret; + + len = (tag_size < ID3_MAX_TAG_READ) + ? (size_t) tag_size + : ID3_MAX_TAG_READ; + if (len < 10) + return 0; + buf = malloc (len); + if (NULL == buf) + return 0; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + 10, + buf, + len)) + { + free (buf); + return 0; + } + if ( (tag_unsync) && + (4 > major) ) + { + /* up to 2.3 the whole tag body is unsynchronised as one unit; from + 2.4 it is a per-frame property, handled in walk_frames() */ + len = deunsynchronise (buf, + len); + tag_unsync = 0; + } + if (0 != (head[5] & 0x40)) + { + /* extended header; 2.3 states a size that excludes its own four + byte field, 2.4 a synchsafe size that includes it */ + if (len < 4) + { + free (buf); + return 0; + } + if (4 == major) + skip = (((size_t) buf[0]) << 21) + | (((size_t) buf[1]) << 14) + | (((size_t) buf[2]) << 7) + | ((size_t) buf[3]); + else + skip = (size_t) EXTRACTOR_forensic_be32_ (buf) + 4; + if (skip >= len) + { + free (buf); + return 0; + } + } + ret = walk_frames (ctx, + major, + tag_unsync, + &buf[skip], + len - skip); + free (buf); + return ret; +} + + +/** + * Decide whether the file opens with MPEG audio. + * + * A single frame header is not enough evidence. The pattern is four + * bytes with eleven set bits at the front, and although rejecting + * every reserved field value cuts the false positive rate a long way, + * arbitrary binary still produces one now and then. Requiring the + * next frame to follow exactly where this one says it will, with the + * same version, layer and sampling rate, is what makes it evidence. + * + * @param ec extraction context + * @param head the first four bytes of the file + * @param size size of the file + * @return 1 if the file begins with a confirmed frame, 0 if not + */ +static int +confirm_sync (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *head, + uint64_t size) +{ + struct MpegHeader first; + struct MpegHeader next; + unsigned char raw[4]; + + if (! parse_mpeg_header (head, + &first)) + return 0; + if (((uint64_t) first.frame_len) + 4 > size) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) first.frame_len, + raw, + sizeof (raw))) + return 0; + if (! parse_mpeg_header (raw, + &next)) + return 0; + return ( (next.version == first.version) && + (next.layer == first.layer) && + (next.sample_rate == first.sample_rate) ) ? 1 : 0; +} + + +/** + * Main entry method for the 'audio/mpeg' extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_id3_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_id3_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + struct Id3Context ctx; + unsigned char head[10]; + unsigned char v1[128]; + uint64_t size; + uint64_t tag_size = 0; + uint64_t audio_start = 0; + int have_v2 = 0; + int have_v1 = 0; + + size = ec->get_size (ec->cls); + if ( (UINT64_MAX == size) || + (size < 128) ) + return; + if (! EXTRACTOR_forensic_read_ (ec, + 0, + head, + sizeof (head))) + return; + if ( (0 == memcmp (head, + "ID3", + 3)) && + (head[3] >= 2) && + (head[3] <= 4) && + (0xFF != head[4]) && + (0 == (head[6] & 0x80)) && + (0 == (head[7] & 0x80)) && + (0 == (head[8] & 0x80)) && + (0 == (head[9] & 0x80)) ) + { + /* the tag size is synchsafe: seven bits per byte, so that no run + of bytes in the header can be mistaken for a frame sync */ + tag_size = (((uint64_t) head[6]) << 21) + | (((uint64_t) head[7]) << 14) + | (((uint64_t) head[8]) << 7) + | ((uint64_t) head[9]); + audio_start = 10 + tag_size; + if (0 != (head[5] & 0x10)) + audio_start += 10; /* footer */ + if (audio_start <= size) + have_v2 = 1; + } + if (! have_v2) + { + /* No tag in front. Either the audio starts right here, or this is + one of the files that carry nothing but an ID3v1 tag at the very + end -- the only case in which we look at the tail of a file we + have not yet identified, and the reason for the read below. */ + if (! confirm_sync (ec, + head, + size)) + { + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (size - 128), + v1, + sizeof (v1))) + return; + if (0 != memcmp (v1, + "TAG", + 3)) + return; + have_v1 = 1; + } + } + if (! have_v1) + { + if (EXTRACTOR_forensic_read_ (ec, + (int64_t) (size - 128), + v1, + sizeof (v1))) + have_v1 = (0 == memcmp (v1, + "TAG", + 3)) ? 1 : 0; + } + memset (&ctx, + 0, + sizeof (ctx)); + ctx.ec = ec; + if (EXTRACTOR_forensic_emit_text_ (ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_MIMETYPE, + ID3_MIME, + strlen (ID3_MIME))) + return; + if (have_v2) + { + if (EXTRACTOR_forensic_emit_ (ec, + ID3_PLUGIN, + EXTRACTOR_METATYPE_FORMAT_VERSION, + "ID3v2.%u.%u", + (unsigned int) head[3], + (unsigned int) head[4])) + return; + if (extract_v2 (&ctx, + head, + tag_size)) + return; + } + if (extract_audio (&ctx, + audio_start, + have_v1 ? (size - 128) : size)) + return; + if (have_v1) + (void) extract_v1 (&ctx, + v1); +} + + +/* end of id3_extractor.c */ diff --git a/src/plugins/iso9660_extractor.c b/src/plugins/iso9660_extractor.c @@ -0,0 +1,581 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/iso9660_extractor.c + * @brief plugin to support iso9660 files + * @author Christian Grothoff + * + * Reads the volume descriptor set of an ECMA-119 (ISO 9660) image. The + * descriptors carry the identifiers of the machine, the person and the + * mastering program that produced the disc, which is the provenance a + * first pass is after; the file tree is deliberately not walked. + * + * References: ECMA-119 (ISO 9660), the "El Torito" bootable CD-ROM + * specification, the Joliet specification and IEEE P1282 (Rock Ridge). + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +/** + * Size of a logical sector in the volume descriptor set. ECMA-119 + * fixes this at 2048 regardless of the logical block size the volume + * announces for its files. + */ +#define ISO_SECTOR 2048 + +/** + * Byte offset of the first volume descriptor: sector 16, after the + * 32 KiB system area reserved for boot code. + */ +#define ISO_VD_OFFSET (16 * ISO_SECTOR) + +/** + * How many volume descriptors we are prepared to walk before giving + * up on finding the terminator. + */ +#define ISO_MAX_DESCRIPTORS 32 + +/** + * Bytes of each descriptor we look at while walking: enough for the + * type, the standard identifier, the boot system identifier of a boot + * record and the escape sequences of a supplementary descriptor. + */ +#define ISO_PROBE 96 + + +/** + * Days from 1970-01-01 to @a y - @a m - @a d, proleptic Gregorian. + * + * Written out rather than using timegm(), which is not available + * everywhere libextractor builds. + * + * @param y the year + * @param m the month, 1 to 12 + * @param d the day of month, 1 to 31 + * @return days since the Unix epoch, negative before it + */ +static int64_t +days_from_civil (int64_t y, + unsigned int m, + unsigned int d) +{ + int64_t era; + unsigned int yoe; + unsigned int doy; + unsigned int doe; + + y -= (m <= 2) ? 1 : 0; + era = ((y >= 0) ? y : (y - 399)) / 400; + yoe = (unsigned int) (y - era * 400); + doy = (153 * (m + ((m > 2) ? -3 : 9)) + 2) / 5 + d - 1; + doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + (int64_t) doe - 719468; +} + + +/** + * Parse a 17-byte ECMA-119 `dec-datetime' field. + * + * The field is "YYYYMMDDHHMMSSss" in ASCII digits followed by one + * signed byte holding the offset from GMT in 15-minute units. A field + * that is all ASCII zeroes means "not specified". + * + * @param f the 17 bytes + * @param[out] when where to store the time in seconds since the epoch + * @return 1 if @a f held a usable date + */ +static int +parse_dec_datetime (const unsigned char *f, + int64_t *when) +{ + unsigned int v[7]; + static const unsigned int width[7] = { 4, 2, 2, 2, 2, 2, 2 }; + size_t pos = 0; + int all_zero = 1; + + for (unsigned int i = 0; i < 16; i++) + { + if ( ('0' > f[i]) || + ('9' < f[i]) ) + return 0; /* not a filled-in date field */ + if ('0' != f[i]) + all_zero = 0; + } + if (all_zero) + return 0; /* the documented spelling of "unset" */ + for (unsigned int i = 0; i < 7; i++) + { + v[i] = 0; + for (unsigned int k = 0; k < width[i]; k++) + v[i] = v[i] * 10 + (unsigned int) (f[pos++] - '0'); + } + if ( (v[0] < 1900) || (v[0] > 2200) || + (v[1] < 1) || (v[1] > 12) || + (v[2] < 1) || (v[2] > 31) || + (v[3] > 23) || (v[4] > 59) || (v[5] > 60) ) + return 0; + *when = days_from_civil (v[0], + v[1], + v[2]) * 86400LL + + v[3] * 3600LL + v[4] * 60LL + v[5] + /* the offset is what has to be *subtracted* to reach GMT */ + - ((int64_t) (signed char) f[16]) * 15LL * 60LL; + return 1; +} + + +/** + * Is @a bs a logical block size an ECMA-119 volume may announce? + * + * @param bs the raw field + * @return 1 if @a bs is usable for arithmetic + */ +static int +block_size_ok (uint16_t bs) +{ + return ( (bs >= 512) && + (bs <= 32768) && + (0 == (bs & (bs - 1))) ); +} + + +/** + * Emit a fixed-width text field of the primary volume descriptor. + * + * @param ec extraction context + * @param type meta data type + * @param pvd the primary volume descriptor + * @param off offset of the field within @a pvd + * @param len width of the field + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_field (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const unsigned char *pvd, + size_t off, + size_t len) +{ + return EXTRACTOR_forensic_emit_text_ (ec, + "iso9660", + type, + (const char *) &pvd[off], + len); +} + + +/** + * Parse and emit one of the descriptor's `dec-datetime' fields. + * + * Unset fields (all ASCII zeroes) and implausible ones are dropped + * without a word, which is what a bulk pass wants. + * + * @param ec extraction context + * @param type meta data type + * @param field the 17-byte field + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_date (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const unsigned char *field) +{ + int64_t when; + + if (! parse_dec_datetime (field, + &when)) + return 0; + return EXTRACTOR_forensic_emit_unix_time_ (ec, + "iso9660", + type, + when); +} + + +/** + * Look at the root directory record's system use area for the SUSP and + * Rock Ridge signatures. + * + * The `SP' entry that announces SUSP lives in the "." record of the + * root directory, not in the copy of the root record embedded in the + * descriptor, so this costs one extra sector read. We do it because + * the presence of Rock Ridge changes what a disc actually contains + * (real POSIX names, permissions and symlinks) and one bounded read is + * cheap. + * + * @param ec extraction context + * @param pvd the primary volume descriptor + * @param block_size the volume's logical block size + * @param file_size size of the image, UINT64_MAX if unknown + * @return 1 if Rock Ridge extensions are present + */ +static int +has_rock_ridge (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *pvd, + uint32_t block_size, + uint64_t file_size) +{ + unsigned char sector[ISO_SECTOR]; + uint32_t extent; + size_t want; + uint64_t at; + unsigned int rec_len; + unsigned int fi_len; + unsigned int pos; + + /* the root directory record sits at offset 156 of the descriptor; + its extent is a both-endian 32-bit value at offset 2 */ + extent = EXTRACTOR_forensic_le32_ (&pvd[156 + 2]); + if (0 == extent) + return 0; + at = ((uint64_t) extent) * block_size; + want = (block_size > ISO_SECTOR) ? ISO_SECTOR : (size_t) block_size; + if ( (at > INT64_MAX - want) || + ( (UINT64_MAX != file_size) && + (at + want > file_size) ) ) + return 0; /* the root directory is not inside this file */ + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) at, + sector, + want)) + return 0; + rec_len = sector[0]; + if ( (rec_len < 34) || + (rec_len > want) ) + return 0; + fi_len = sector[32]; + pos = 33 + fi_len; + if (0 != (pos & 1)) + pos++; /* the identifier is padded to an even length */ + while (pos + 4 <= rec_len) + { + unsigned int elen = sector[pos + 2]; + + if (elen < 4) + return 0; /* zero-length entry: refuse to spin */ + if (pos + elen > rec_len) + return 0; + if ( ('R' == sector[pos]) && + ('R' == sector[pos + 1]) ) + return 1; /* the Rock Ridge marker itself */ + if ( ('S' == sector[pos]) && + ('P' == sector[pos + 1]) && + (7 == elen) && + (0xBE == sector[pos + 5]) && + (0xEF == sector[pos + 6]) ) + return 1; /* SUSP, which on a data CD means Rock Ridge */ + pos += elen; + } + return 0; +} + + +/** + * Main entry method for the iso9660 extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_iso9660_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_iso9660_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char probe[ISO_PROBE]; + unsigned char pvd[ISO_SECTOR]; + uint64_t file_size; + uint64_t pvd_at = 0; + uint32_t block_size; + uint32_t space_size; + uint16_t set_size; + int64_t when; + int have_pvd = 0; + int joliet = 0; + int el_torito = 0; + + file_size = ec->get_size (ec->cls); + if ( (UINT64_MAX != file_size) && + (file_size < ISO_VD_OFFSET + ISO_SECTOR) ) + return; /* too short to hold a volume descriptor */ + if (! EXTRACTOR_forensic_read_ (ec, + ISO_VD_OFFSET, + probe, + sizeof (probe))) + return; + if (0 != memcmp (&probe[1], + "CD001", + 5)) + return; /* not an ISO 9660 image */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_MIMETYPE, + "%s", + "application/x-iso9660-image")) + return; + for (unsigned int i = 0; i < ISO_MAX_DESCRIPTORS; i++) + { + uint64_t at = ISO_VD_OFFSET + ((uint64_t) i) * ISO_SECTOR; + + if ( (UINT64_MAX != file_size) && + (at + ISO_SECTOR > file_size) ) + break; + if ( (0 != i) && + (! EXTRACTOR_forensic_read_ (ec, + (int64_t) at, + probe, + sizeof (probe))) ) + break; + if (0 != memcmp (&probe[1], + "CD001", + 5)) + break; /* the descriptor set ended without a terminator */ + switch (probe[0]) + { + case 0: + /* boot record; El Torito puts its name in the boot system + identifier at offset 7 */ + if (0 == memcmp (&probe[7], + "EL TORITO SPECIFICATION", + 23)) + el_torito = 1; + break; + case 1: + if (! have_pvd) + { + have_pvd = 1; + pvd_at = at; + } + break; + case 2: + /* supplementary descriptor; Joliet announces itself with one of + three escape sequences for UCS-2 at offset 88 */ + if ( (0x25 == probe[88]) && + (0x2F == probe[89]) && + ( (0x40 == probe[90]) || + (0x43 == probe[90]) || + (0x45 == probe[90]) ) ) + joliet = 1; + break; + case 255: + i = ISO_MAX_DESCRIPTORS; /* terminator */ + break; + default: + break; + } + } + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "%s", + "ISO 9660")) + return; + if ( (joliet) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "%s", + "Joliet")) ) + return; + if ( (el_torito) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_COMMENT, + "%s", + "bootable (El Torito)")) ) + return; + if (! have_pvd) + return; /* a descriptor set without a primary descriptor */ + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) pvd_at, + pvd, + sizeof (pvd))) + return; + + /* -- the identifier fields, in descriptor order -- */ + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER, + pvd, + 8, + 32)) + return; + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_VOLUME_NAME, + pvd, + 40, + 32)) + return; + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_PUBLISHER, + pvd, + 318, + 128)) + return; + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_DATA_PREPARER, + pvd, + 446, + 128)) + return; + /* the application identifier names the mastering software -- mkisofs, + Nero, ImgBurn -- which is the best provenance the descriptor has */ + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + pvd, + 574, + 128)) + return; + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_COPYRIGHT, + pvd, + 702, + 37)) + return; + if (0 != + emit_field (ec, + EXTRACTOR_METATYPE_ABSTRACT, + pvd, + 739, + 37)) + return; + /* The bibliographic file identifier has no meta type of its own; it + names a file on the disc, so it only means something with a label + attached. */ + { + size_t blen = EXTRACTOR_forensic_trim_ ((const char *) &pvd[776], + 37); + int printable = (0 != blen); + + for (size_t i = 0; i < blen; i++) + if ( (pvd[776 + i] < 0x20) || + (pvd[776 + i] > 0x7e) ) + printable = 0; + if ( (printable) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_COMMENT, + "bibliographic file: %.*s", + (int) blen, + (const char *) &pvd[776])) ) + return; + } + + /* -- geometry -- */ + block_size = EXTRACTOR_forensic_le16_ (&pvd[128]); + if (block_size_ok ((uint16_t) block_size)) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_BLOCK_SIZE, + "%u", + (unsigned int) block_size)) + return; + /* both factors are bounded (2^32 sectors of at most 32 KiB), so + the product cannot overflow 64 bits */ + space_size = EXTRACTOR_forensic_le32_ (&pvd[80]); + if ( (0 != space_size) && + (0 != EXTRACTOR_forensic_emit_size_ (ec, + "iso9660", + EXTRACTOR_METATYPE_VOLUME_SIZE, + ((uint64_t) space_size) + * block_size)) ) + return; + } + else + { + block_size = 0; + } + /* how many volumes the set has; only worth saying when it is a set */ + set_size = EXTRACTOR_forensic_le16_ (&pvd[120]); + if ( (set_size > 1) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) set_size)) ) + return; + + /* -- the four dec-datetime fields -- */ + if (0 != + emit_date (ec, + EXTRACTOR_METATYPE_CREATION_DATE, + &pvd[813])) + return; + if (0 != + emit_date (ec, + EXTRACTOR_METATYPE_MODIFICATION_DATE, + &pvd[830])) + return; + if (0 != + emit_date (ec, + EXTRACTOR_METATYPE_EXPIRATION_DATE, + &pvd[847])) + return; + /* The effective date -- the date before which the volume should not + be used -- has no meta type of its own, so it goes out as a + comment rather than being silently dropped. */ + if (parse_dec_datetime (&pvd[864], + &when)) + { + struct tm tm; + time_t t = (time_t) when; + char buf[32]; + + if ( (when > 315532800LL) && + (when < 4102444800LL) && + (NULL != gmtime_r (&t, + &tm)) && + (0 != strftime (buf, + sizeof (buf), + "%Y-%m-%dT%H:%M:%SZ", + &tm)) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_COMMENT, + "volume effective from %s", + buf)) ) + return; + } + + /* -- Rock Ridge, which needs one more sector -- */ + if ( (0 != block_size) && + (has_rock_ridge (ec, + pvd, + block_size, + file_size)) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "iso9660", + EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "%s", + "Rock Ridge")) ) + return; +} + + +/* end of iso9660_extractor.c */ diff --git a/src/plugins/kml_extractor.c b/src/plugins/kml_extractor.c @@ -0,0 +1,1427 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/kml_extractor.c + * @brief plugin to support KML (Keyhole Markup Language) documents + * @author Christian Grothoff + * + * KML is what Google Earth, Google Maps exports, most GIS tools and a + * fair number of tracking applications write. Like GPX it carries + * coordinates, but it also carries `<NetworkLink>' elements, which make + * opening the document fetch content from a remote host -- worth + * flagging on its own. + * + * KMZ, the zipped form, is a zip archive whose first member is + * `doc.kml'. This plugin does not look inside it: libextractor's zip + * support lives in the core decompressor, not here, and a KMZ therefore + * does not reach this plugin as KML. Nothing below will mistake a zip + * for KML, since the magic check requires markup at the start of the + * file. + * + * As in `gpx_extractor.c', this is a bounded scanner, not an XML parser + * -- libextractor has no XML dependency and this plugin must stay + * unconditional. What that costs, stated once: + * + * - Comments (`<!-- ... -->') and processing instructions are not + * skipped, so an element that only occurs inside a comment is still + * counted. A false positive, never an out-of-bounds read. + * - CDATA is recognised only when it opens the content of an element we + * ask for -- which is the case that matters, because `<description>' + * almost always wraps its HTML in CDATA. + * - Only the five predefined entities and numeric character references + * are expanded. + * - Element content is the run of text up to the next `<'; nested + * elements inside a value are not concatenated. + * - Namespace prefixes are ignored, so `<atom:name>' and `<name>' are + * the same element here. That is what lets `<atom:author><atom:name>' + * be found without a namespace table, and it is why `<Link>' (KML) and + * `<link>' (Atom) have to be told apart by case, which XML guarantees. + * - An `<Update>' or a `<Schema>' can hold elements with the same names + * as the ones we look for; we do not track the containing element, so + * such a document would be reported approximately. + * + * Every loop below is bounded by the buffer length and advances at least + * one byte per iteration. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +#include <math.h> + + +/** + * How many bytes of the file we are willing to look at. KML documents + * exported from a GIS routinely run to many megabytes of coordinates. + */ +#define KML_SCAN_CAP (1024 * 1024) + +/** + * Longest start tag we will scan for its closing `>'. + */ +#define KML_MAX_TAG 8192 + +/** + * Longest element text content we will look at, for everything except + * `<coordinates>' (which gets #KML_MAX_COORD_TEXT). + */ +#define KML_MAX_TEXT 8192 + +/** + * Longest `<coordinates>' body we will walk. + */ +#define KML_MAX_COORD_TEXT (128 * 1024) + +/** + * Most coordinate tuples we will parse out of one `<coordinates>' + * element. + */ +#define KML_MAX_TUPLES 100000 + +/** + * How far into the file the `<kml' element has to appear. + */ +#define KML_MAGIC_WINDOW 2048 + +/** + * Most attributes we parse out of a single start tag. + */ +#define KML_MAX_ATTRS 64 + + +/** + * Everything the single pass over the document collected. + */ +struct KmlScan +{ + /** + * Bounding box over every `<coordinates>' tuple seen. + */ + double minlon; + double minlat; + double maxlon; + double maxlat; + + /** + * The first coordinate in the document. + */ + double first_lat; + double first_lon; + + /** + * Number of `<Placemark>' elements. + */ + uint64_t placemarks; + + /** + * Number of `<NetworkLink>' elements. + */ + uint64_t networklinks; + + /** + * Offset of the first `<Placemark>', or `(size_t) -1'. + */ + size_t first_placemark; + + /** + * Offset of the first `<LookAt>' or `<Camera>', or `(size_t) -1'. + */ + size_t viewpoint; + + /** + * Non-zero once the bounding box fields are meaningful. + */ + int have_bbox; +}; + + +/** + * Can @a c appear in an XML name (after the first character)? + * + * @param c character to test + * @return 1 if @a c is a name character, 0 if not + */ +static int +kml_name_char (char c) +{ + return ( ( ('a' <= c) && ('z' >= c) ) || + ( ('A' <= c) && ('Z' >= c) ) || + ( ('0' <= c) && ('9' >= c) ) || + ('_' == c) || ('-' == c) || ('.' == c) ); +} + + +/** + * Is the element that starts at @a pos named @a tag? A namespace prefix + * is skipped; the comparison is case-sensitive, as XML requires. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param tag element name to compare against + * @return 1 on a match, 0 otherwise + */ +static int +kml_tag_is (const char *buf, + size_t len, + size_t pos, + const char *tag) +{ + size_t i; + size_t start; + size_t tlen = strlen (tag); + + if ( (pos >= len) || + ('<' != buf[pos]) ) + return 0; + i = pos + 1; + start = i; + while ( (i < len) && + (i - start < 64) && + kml_name_char (buf[i]) ) + i++; + if ( (i < len) && + (':' == buf[i]) ) + { + i++; + start = i; + while ( (i < len) && + (i - start < 64) && + kml_name_char (buf[i]) ) + i++; + } + if (i - start != tlen) + return 0; + return (0 == memcmp (&buf[start], + tag, + tlen)); +} + + +/** + * Find the `>' that closes the start tag beginning at @a pos. Quoted + * attribute values may contain `>', so quoting is tracked. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @return offset of the `>', or `(size_t) -1' if there is none within + * #KML_MAX_TAG bytes + */ +static size_t +kml_tag_end (const char *buf, + size_t len, + size_t pos) +{ + char quote = 0; + + for (size_t i = pos; (i < len) && (i - pos < KML_MAX_TAG); i++) + { + char c = buf[i]; + + if (0 != quote) + { + if (c == quote) + quote = 0; + continue; + } + if ( ('"' == c) || + ('\'' == c) ) + { + quote = c; + continue; + } + if ('>' == c) + return i; + } + return (size_t) -1; +} + + +/** + * Find the value of attribute @a name in the start tag at @a pos. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param name attribute name, without a namespace prefix + * @param[out] vstart offset of the first byte of the value + * @param[out] vlen number of bytes in the value + * @return 1 if the attribute was found, 0 if not + */ +static int +kml_attr (const char *buf, + size_t len, + size_t pos, + const char *name, + size_t *vstart, + size_t *vlen) +{ + size_t end = kml_tag_end (buf, + len, + pos); + size_t i; + size_t nlen = strlen (name); + + if (((size_t) -1) == end) + return 0; + i = pos + 1; + while ( (i < end) && + (kml_name_char (buf[i]) || + (':' == buf[i]) ) ) + i++; + for (unsigned int n = 0; n < KML_MAX_ATTRS; n++) + { + size_t astart; + size_t alen; + size_t vs; + size_t ve; + char quote; + + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if (i >= end) + break; + astart = i; + while ( (i < end) && + (kml_name_char (buf[i]) || + (':' == buf[i]) ) ) + i++; + if (i == astart) + { + i++; /* not a name: skip a byte so we always progress */ + continue; + } + alen = i - astart; + for (size_t k = 0; k < alen; k++) + if (':' == buf[astart + k]) + { + astart += k + 1; + alen -= k + 1; + break; + } + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if ( (i >= end) || + ('=' != buf[i]) ) + continue; /* valueless attribute; i already advanced past a name */ + i++; + while ( (i < end) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if (i >= end) + break; + quote = buf[i]; + if ( ('"' == quote) || + ('\'' == quote) ) + { + i++; + vs = i; + while ( (i < end) && + (quote != buf[i]) ) + i++; + ve = i; + if (i < end) + i++; + } + else + { + vs = i; + while ( (i < end) && + (' ' != buf[i]) && ('\t' != buf[i]) && + ('\r' != buf[i]) && ('\n' != buf[i]) && + ('/' != buf[i]) ) + i++; + ve = i; + } + if ( (alen == nlen) && + (0 == memcmp (&buf[astart], + name, + nlen)) ) + { + *vstart = vs; + *vlen = ve - vs; + return 1; + } + } + return 0; +} + + +/** + * Text content of the element whose start tag begins at @a pos. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' + * @param maxtext most content bytes to return + * @param[out] tstart offset of the first content byte + * @param[out] tlen number of content bytes + * @return 1 if content was found, 0 if the element is empty or the tag + * is malformed + */ +static int +kml_text (const char *buf, + size_t len, + size_t pos, + size_t maxtext, + size_t *tstart, + size_t *tlen) +{ + size_t end = kml_tag_end (buf, + len, + pos); + size_t i; + size_t j; + + if (((size_t) -1) == end) + return 0; + if ( (end > pos) && + ('/' == buf[end - 1]) ) + return 0; /* self-closing element, no content */ + i = end + 1; + if ( (i + 9 <= len) && + (0 == memcmp (&buf[i], + "<![CDATA[", + 9)) ) + { + j = i + 9; + while ( (j + 3 <= len) && + (j - i < maxtext) ) + { + if (0 == memcmp (&buf[j], + "]]>", + 3)) + { + *tstart = i + 9; + *tlen = j - (i + 9); + return (0 != *tlen); + } + j++; + } + return 0; + } + j = i; + while ( (j < len) && + ('<' != buf[j]) && + (j - i < maxtext) ) + j++; + *tstart = i; + *tlen = j - i; + return (0 != *tlen); +} + + +/** + * Find the next element named @a tag at or after @a from, staying below + * @a limit. + * + * @param buf the buffer + * @param limit offset one past the last byte to search + * @param from where to start searching + * @param tag element name + * @return offset of the `<', or `(size_t) -1' if not found + */ +static size_t +kml_find (const char *buf, + size_t limit, + size_t from, + const char *tag) +{ + for (size_t i = from; i < limit; i++) + { + if ('<' != buf[i]) + continue; + if (kml_tag_is (buf, + limit, + i, + tag)) + return i; + } + return (size_t) -1; +} + + +/** + * Expand the five predefined XML entities and numeric character + * references in @a in. Anything else is copied through unchanged. + * + * @param in input text + * @param inlen number of bytes in @a in + * @param[out] out where to write the result + * @param outsize number of bytes available in @a out + * @return number of bytes written to @a out + */ +static size_t +kml_unescape (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + size_t o = 0; + size_t i = 0; + + while ( (i < inlen) && + (o + 5 < outsize) ) + { + size_t j; + uint32_t cp = 0; + + if ('&' != in[i]) + { + out[o++] = in[i++]; + continue; + } + j = i + 1; + while ( (j < inlen) && + (j - i < 12) && + (';' != in[j]) ) + j++; + if ( (j >= inlen) || + (';' != in[j]) ) + { + out[o++] = in[i++]; + continue; + } + /* j is the offset of the `;', so the reference is j + 1 - i bytes */ + if ( (4 == j + 1 - i) && + (0 == memcmp (&in[i], "<", 4)) ) + cp = '<'; + else if ( (4 == j + 1 - i) && + (0 == memcmp (&in[i], ">", 4)) ) + cp = '>'; + else if ( (5 == j + 1 - i) && + (0 == memcmp (&in[i], "&", 5)) ) + cp = '&'; + else if ( (6 == j + 1 - i) && + (0 == memcmp (&in[i], """, 6)) ) + cp = '"'; + else if ( (6 == j + 1 - i) && + (0 == memcmp (&in[i], "'", 6)) ) + cp = '\''; + else if ( (j - i > 2) && + ('#' == in[i + 1]) ) + { + size_t k = i + 2; + int base = 10; + + if ( (k < j) && + ( ('x' == in[k]) || ('X' == in[k]) ) ) + { + base = 16; + k++; + } + if (k == j) + { + out[o++] = in[i++]; + continue; + } + while (k < j) + { + int d; + + if ( ('0' <= in[k]) && ('9' >= in[k]) ) + d = in[k] - '0'; + else if ( (16 == base) && ('a' <= in[k]) && ('f' >= in[k]) ) + d = in[k] - 'a' + 10; + else if ( (16 == base) && ('A' <= in[k]) && ('F' >= in[k]) ) + d = in[k] - 'A' + 10; + else + break; + if (cp > 0x110000 / (uint32_t) base) + { + cp = 0; + break; + } + cp = cp * (uint32_t) base + (uint32_t) d; + k++; + } + if ( (k != j) || + (0 == cp) || + (cp > 0x10FFFF) || + ( (0xD800 <= cp) && (0xDFFF >= cp) ) ) + { + out[o++] = in[i++]; + continue; + } + } + else + { + out[o++] = in[i++]; + continue; /* unknown entity: leave it alone */ + } + if (cp < 0x80) + { + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + i = j + 1; + } + return o; +} + + +/** + * Drop HTML markup and collapse runs of white space. KML + * `<description>' values are HTML far more often than they are plain + * text, and the balloon markup is noise for our purposes. + * + * Tags are removed by bracket matching, which is exactly as approximate + * as it sounds: a `<' in the prose that is not markup eats up to the + * next `>'. That is the usual trade-off for markup that was never + * required to be well-formed in the first place. + * + * @param in input text + * @param inlen number of bytes in @a in + * @param[out] out where to write the result + * @param outsize number of bytes available in @a out + * @return number of bytes written to @a out + */ +static size_t +kml_strip_html (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + size_t o = 0; + size_t i = 0; + int space = 0; + + while ( (i < inlen) && + (o + 1 < outsize) ) + { + if ('<' == in[i]) + { + while ( (i < inlen) && + ('>' != in[i]) ) + i++; + if (i < inlen) + i++; + space = 1; + continue; + } + if ( (' ' == in[i]) || ('\t' == in[i]) || + ('\r' == in[i]) || ('\n' == in[i]) ) + { + space = 1; + i++; + continue; + } + if (space && + (0 != o) && + (NULL == strchr (".,;:!?)]}", + in[i])) ) + out[o++] = ' '; + space = 0; + if (o + 1 < outsize) + out[o++] = in[i]; + i++; + } + return o; +} + + +/** + * Emit a stretch of XML text after expanding entity references. + * + * @param ec extraction context + * @param type meta data type + * @param data the text + * @param len number of bytes in @a data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +kml_emit_xml (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const char *data, + size_t len) +{ + char buf[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t out; + + if (len > sizeof (buf) - 8) + len = sizeof (buf) - 8; + out = kml_unescape (data, + len, + buf, + sizeof (buf)); + return EXTRACTOR_forensic_emit_text_ (ec, + "kml", + type, + buf, + out); +} + + +/** + * Parse a decimal number out of a stretch of bytes that is not + * NUL-terminated. + * + * @param data the bytes + * @param len number of bytes in @a data + * @param[out] used number of bytes consumed, may be NULL + * @param[out] value where to store the result + * @return 1 on success, 0 if @a data does not start with a number + */ +static int +kml_parse_double (const char *data, + size_t len, + size_t *used, + double *value) +{ + char tmp[64]; + char *endp; + double v; + size_t i = 0; + size_t o = 0; + + while ( (i < len) && + ( (' ' == data[i]) || ('\t' == data[i]) || + ('\r' == data[i]) || ('\n' == data[i]) ) ) + i++; + while ( (i < len) && + (o < sizeof (tmp) - 1) && + ( ( ('0' <= data[i]) && ('9' >= data[i]) ) || + ('+' == data[i]) || ('-' == data[i]) || + ('.' == data[i]) || ('e' == data[i]) || ('E' == data[i]) ) ) + tmp[o++] = data[i++]; + tmp[o] = '\0'; + if (0 == o) + return 0; + v = strtod (tmp, + &endp); + if ( (endp == tmp) || + (! isfinite (v)) ) + return 0; + if (NULL != used) + *used = i; + *value = v; + return 1; +} + + +/** + * Days since 1970-01-01 for a proleptic Gregorian date. (Howard + * Hinnant's `days_from_civil'.) + * + * @param y year + * @param m month, 1-12 + * @param d day of month, 1-31 + * @return day number, negative before the epoch + */ +static int64_t +kml_days_from_civil (int64_t y, + int64_t m, + int64_t d) +{ + int64_t era; + int64_t yoe; + int64_t doy; + int64_t doe; + + y -= (m <= 2); + era = (y >= 0 ? y : y - 399) / 400; + yoe = y - era * 400; + doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + + +/** + * Parse an ISO 8601 timestamp as KML writes it: + * `YYYY-MM-DDThh:mm:ss[.fff][Z|(+|-)hh[:mm]]'. A missing zone is taken + * as UTC. (KML also allows a bare `YYYY', `YYYY-MM' or `YYYY-MM-DD'; + * those are rejected here rather than guessed at.) + * + * @param s the text + * @param len number of bytes in @a s + * @param[out] when where to store seconds since the Unix epoch + * @return 1 on success, 0 if @a s is not a full timestamp + */ +static int +kml_parse_iso8601 (const char *s, + size_t len, + int64_t *when) +{ + int64_t v[6] = { 0, 0, 0, 0, 0, 0 }; + static const size_t widths[6] = { 4, 2, 2, 2, 2, 2 }; + static const char seps[5] = { '-', '-', 'T', ':', ':' }; + size_t i = 0; + int64_t off = 0; + + while ( (i < len) && + ( (' ' == s[i]) || ('\t' == s[i]) || + ('\r' == s[i]) || ('\n' == s[i]) ) ) + i++; + for (unsigned int f = 0; f < 6; f++) + { + if (i + widths[f] > len) + return 0; + for (size_t k = 0; k < widths[f]; k++) + { + if ( ('0' > s[i + k]) || + ('9' < s[i + k]) ) + return 0; + v[f] = v[f] * 10 + (s[i + k] - '0'); + } + i += widths[f]; + if (f < 5) + { + if (i >= len) + return 0; + if ( (2 == f) && + (' ' == s[i]) ) + i++; + else if (seps[f] == s[i]) + i++; + else + return 0; + } + } + if ( (v[1] < 1) || (v[1] > 12) || + (v[2] < 1) || (v[2] > 31) || + (v[3] > 23) || (v[4] > 59) || (v[5] > 60) ) + return 0; + if ( (i < len) && + ('.' == s[i]) ) + { + i++; + while ( (i < len) && + ('0' <= s[i]) && ('9' >= s[i]) ) + i++; + } + if ( (i < len) && + ( ('+' == s[i]) || ('-' == s[i]) ) ) + { + int neg = ('-' == s[i]); + int64_t oh = 0; + int64_t om = 0; + + i++; + if (i + 2 > len) + return 0; + for (size_t k = 0; k < 2; k++) + { + if ( ('0' > s[i + k]) || ('9' < s[i + k]) ) + return 0; + oh = oh * 10 + (s[i + k] - '0'); + } + i += 2; + if ( (i < len) && + (':' == s[i]) ) + i++; + if ( (i + 2 <= len) && + ('0' <= s[i]) && ('9' >= s[i]) && + ('0' <= s[i + 1]) && ('9' >= s[i + 1]) ) + { + om = (s[i] - '0') * 10 + (s[i + 1] - '0'); + i += 2; + } + if ( (oh > 23) || (om > 59) ) + return 0; + off = oh * 3600 + om * 60; + if (neg) + off = -off; + } + *when = kml_days_from_civil (v[0], + v[1], + v[2]) * 86400 + + v[3] * 3600 + v[4] * 60 + v[5] + - off; + return 1; +} + + +/** + * Fold one coordinate into the bounding box. + * + * @param sc scan state + * @param lon longitude in degrees + * @param lat latitude in degrees + */ +static void +kml_note_coord (struct KmlScan *sc, + double lon, + double lat) +{ + if ( (lat < -90.0) || (lat > 90.0) || + (lon < -180.0) || (lon > 180.0) ) + return; /* out of range: not a coordinate */ + if (! sc->have_bbox) + { + sc->minlat = sc->maxlat = lat; + sc->minlon = sc->maxlon = lon; + sc->first_lat = lat; + sc->first_lon = lon; + sc->have_bbox = 1; + return; + } + if (lat < sc->minlat) + sc->minlat = lat; + if (lat > sc->maxlat) + sc->maxlat = lat; + if (lon < sc->minlon) + sc->minlon = lon; + if (lon > sc->maxlon) + sc->maxlon = lon; +} + + +/** + * Walk the body of one `<coordinates>' element. + * + * KML tuples are `lon,lat[,alt]', white-space separated -- note that + * this is the *reverse* of the GPX attribute order, which is the classic + * way to get a KML bounding box wrong. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param pos offset of the `<' of the `<coordinates>' start tag + * @param sc scan state to update + */ +static void +kml_walk_coordinates (const char *buf, + size_t len, + size_t pos, + struct KmlScan *sc) +{ + size_t ts; + size_t tl; + size_t i = 0; + + if (! kml_text (buf, + len, + pos, + KML_MAX_COORD_TEXT, + &ts, + &tl)) + return; + for (unsigned long n = 0; (n < KML_MAX_TUPLES) && (i < tl); n++) + { + size_t used; + double lon; + double lat; + + while ( (i < tl) && + ( (' ' == buf[ts + i]) || ('\t' == buf[ts + i]) || + ('\r' == buf[ts + i]) || ('\n' == buf[ts + i]) || + (',' == buf[ts + i]) ) ) + i++; + if (i >= tl) + break; + if (! kml_parse_double (&buf[ts + i], + tl - i, + &used, + &lon)) + { + i++; /* garbage: skip a byte so the loop always progresses */ + continue; + } + i += used; + if ( (i >= tl) || + (',' != buf[ts + i]) ) + continue; + i++; + if (! kml_parse_double (&buf[ts + i], + tl - i, + &used, + &lat)) + continue; + i += used; + kml_note_coord (sc, + lon, + lat); + /* an optional altitude follows; skip it if it is there */ + if ( (i < tl) && + (',' == buf[ts + i]) ) + { + i++; + if (kml_parse_double (&buf[ts + i], + tl - i, + &used, + &lon)) + i += used; + } + } +} + + +/** + * Walk the document once, counting placemarks and network links and + * folding every coordinate into the bounding box. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @param[out] sc scan state to fill in + */ +static void +kml_walk (const char *buf, + size_t len, + struct KmlScan *sc) +{ + for (size_t i = 0; i < len; i++) + { + if ('<' != buf[i]) + continue; + if (kml_tag_is (buf, len, i, "Placemark")) + { + sc->placemarks++; + if (((size_t) -1) == sc->first_placemark) + sc->first_placemark = i; + } + else if (kml_tag_is (buf, len, i, "NetworkLink")) + { + sc->networklinks++; + } + else if (kml_tag_is (buf, len, i, "coordinates")) + { + kml_walk_coordinates (buf, + len, + i, + sc); + } + else if ( (((size_t) -1) == sc->viewpoint) && + (kml_tag_is (buf, len, i, "LookAt") || + kml_tag_is (buf, len, i, "Camera") ) ) + { + sc->viewpoint = i; + } + } +} + + +/** + * Report the `<href>' of every `<NetworkLink>' and the `<atom:link + * href=>' of the document, capped. + * + * @param ec extraction context + * @param buf the buffer + * @param len number of bytes in @a buf + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +kml_do_links (struct EXTRACTOR_ExtractContext *ec, + const char *buf, + size_t len) +{ + size_t p; + size_t vs; + size_t vl; + unsigned int n = 0; + + /* <atom:link href="..."/>: the document's own canonical location */ + p = 0; + while (n < EXTRACTOR_FORENSIC_MAX_ITEMS) + { + p = kml_find (buf, + len, + p, + "link"); + if (((size_t) -1) == p) + break; + if (kml_attr (buf, len, p, "href", &vs, &vl)) + { + n++; + if (kml_emit_xml (ec, + EXTRACTOR_METATYPE_URL, + &buf[vs], + vl)) + return 1; + } + p++; + } + /* <href> elements: KML's own <Link>, <Icon> and <NetworkLink> all use + one. These are what make opening the document talk to a network. */ + p = 0; + n = 0; + while (n < EXTRACTOR_FORENSIC_MAX_ITEMS) + { + p = kml_find (buf, + len, + p, + "href"); + if (((size_t) -1) == p) + break; + if (kml_text (buf, len, p, KML_MAX_TEXT, &vs, &vl)) + { + n++; + if (kml_emit_xml (ec, + EXTRACTOR_METATYPE_URL, + &buf[vs], + vl)) + return 1; + } + p++; + } + return 0; +} + + +/** + * Main entry method for the KML extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_kml_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_kml_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + char head[KML_MAGIC_WINDOW]; + char *buf = NULL; + size_t hlen = 0; + size_t len = 0; + size_t cap; + size_t limit; + size_t p; + size_t vs; + size_t vl; + uint64_t fsize; + struct KmlScan sc; + int truncated; + + /* Bail out on the first few bytes: almost nothing we are handed is + KML, and we must not pay for the ones that are not. */ + { + void *data; + ssize_t ret; + + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + return; + while (hlen < sizeof (head)) + { + ret = ec->read (ec->cls, + &data, + sizeof (head) - hlen); + if (0 >= ret) + break; + if (((size_t) ret) > sizeof (head) - hlen) + return; /* the IPC layer is misbehaving */ + memcpy (&head[hlen], + data, + (size_t) ret); + hlen += (size_t) ret; + } + } + if (hlen < 16) + return; + if (((size_t) -1) == kml_find (head, + hlen, + 0, + "kml")) + return; /* not KML */ + { + size_t k = 0; + + if ( (hlen >= 3) && + (0 == memcmp (head, "\xef\xbb\xbf", 3)) ) + k = 3; /* UTF-8 BOM */ + while ( (k < hlen) && + ( (' ' == head[k]) || ('\t' == head[k]) || + ('\r' == head[k]) || ('\n' == head[k]) ) ) + k++; + if ( (k >= hlen) || + ('<' != head[k]) ) + return; /* not markup; in particular, not a KMZ (which is a zip) */ + } + + fsize = ec->get_size (ec->cls); + cap = KML_SCAN_CAP; + if ( (UINT64_MAX != fsize) && + (fsize < (uint64_t) cap) ) + cap = (size_t) fsize; + if (cap < hlen) + cap = hlen; + buf = malloc (cap); + if (NULL == buf) + return; + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + { + free (buf); + return; + } + while (len < cap) + { + void *data; + ssize_t ret; + + ret = ec->read (ec->cls, + &data, + cap - len); + if (0 >= ret) + break; + if (((size_t) ret) > cap - len) + break; /* the IPC layer is misbehaving */ + memcpy (&buf[len], + data, + (size_t) ret); + len += (size_t) ret; + } + if (len < 16) + { + free (buf); + return; + } + truncated = ( (UINT64_MAX == fsize) || + (fsize > (uint64_t) len) ); + + if (0 != ec->proc (ec->cls, + "kml", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/vnd.google-earth.kml+xml", + strlen ("application/vnd.google-earth.kml+xml") + 1)) + goto out; + /* KML is defined on WGS 84 only; the specification does not offer an + alternative, so this says something about the format. */ + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + "WGS 84")) + goto out; + + memset (&sc, + 0, + sizeof (sc)); + sc.first_placemark = (size_t) -1; + sc.viewpoint = (size_t) -1; + kml_walk (buf, + len, + &sc); + + /* The document title is the <name> of the enclosing <Document> or + <Folder>, which in practice is the first <name> before the first + <Placemark>. */ + limit = (((size_t) -1) == sc.first_placemark) ? len : sc.first_placemark; + p = kml_find (buf, limit, 0, "name"); + if ( (((size_t) -1) != p) && + kml_text (buf, limit, p, KML_MAX_TEXT, &vs, &vl) && + kml_emit_xml (ec, EXTRACTOR_METATYPE_TITLE, &buf[vs], vl) ) + goto out; + p = kml_find (buf, limit, 0, "description"); + if ( (((size_t) -1) != p) && + kml_text (buf, limit, p, KML_MAX_TEXT, &vs, &vl) ) + { + char stripped[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t slen; + + /* <description> is HTML far more often than not; the markup is + noise, but the text inside it is exactly what we want. */ + slen = kml_strip_html (&buf[vs], + vl, + stripped, + sizeof (stripped)); + if (kml_emit_xml (ec, + EXTRACTOR_METATYPE_DESCRIPTION, + stripped, + slen)) + goto out; + } + /* <atom:author><atom:name> */ + p = kml_find (buf, len, 0, "author"); + if (((size_t) -1) != p) + { + size_t a = kml_find (buf, len, p + 1, "name"); + + if ( (((size_t) -1) != a) && + kml_text (buf, len, a, KML_MAX_TEXT, &vs, &vl) && + kml_emit_xml (ec, EXTRACTOR_METATYPE_AUTHOR_NAME, &buf[vs], vl) ) + goto out; + } + if (kml_do_links (ec, + buf, + len)) + goto out; + if (0 != sc.networklinks) + { + /* Worth saying out loud: a NetworkLink means the document fetches + content from somewhere else when it is opened. */ + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_COMMENT, + "%llu NetworkLink element%s;" + " opening this document fetches remote" + " content", + (unsigned long long) sc.networklinks, + (1 == sc.networklinks) ? "" : "s")) + goto out; + } + /* <TimeStamp><when> or, failing that, <TimeSpan><begin> */ + { + size_t t = kml_find (buf, len, 0, "TimeStamp"); + size_t w = (size_t) -1; + + if (((size_t) -1) != t) + w = kml_find (buf, len, t + 1, "when"); + if (((size_t) -1) == w) + { + t = kml_find (buf, len, 0, "TimeSpan"); + if (((size_t) -1) != t) + w = kml_find (buf, len, t + 1, "begin"); + } + if ( (((size_t) -1) != w) && + kml_text (buf, len, w, KML_MAX_TEXT, &vs, &vl) ) + { + int64_t when; + + if (kml_parse_iso8601 (&buf[vs], vl, &when)) + { + if (EXTRACTOR_forensic_emit_unix_time_ (ec, + "kml", + EXTRACTOR_METATYPE_CREATION_DATE, + when)) + goto out; + } + else if (kml_emit_xml (ec, + EXTRACTOR_METATYPE_UNKNOWN_DATE, + &buf[vs], + vl)) + { + goto out; + } + } + } + if (((size_t) -1) != sc.first_placemark) + { + /* the first Placemark's <name>, bounded by its </Placemark> */ + size_t pend = len; + size_t nm; + + for (size_t i = sc.first_placemark; i + 12 <= len; i++) + if (0 == memcmp (&buf[i], + "</Placemark>", + 12)) + { + pend = i; + break; + } + nm = kml_find (buf, pend, sc.first_placemark + 1, "name"); + if ( (((size_t) -1) != nm) && + kml_text (buf, pend, nm, KML_MAX_TEXT, &vs, &vl) && + kml_emit_xml (ec, EXTRACTOR_METATYPE_LOCATION_NAME, &buf[vs], vl) ) + goto out; + } + if (sc.have_bbox) + { + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_BOUNDING_BOX, + "%.6f,%.6f,%.6f,%.6f", + sc.minlon, + sc.minlat, + sc.maxlon, + sc.maxlat)) + goto out; + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_GPS_LATITUDE, + "%.6f", + sc.first_lat)) + goto out; + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_GPS_LONGITUDE, + "%.6f", + sc.first_lon)) + goto out; + } + if (((size_t) -1) != sc.viewpoint) + { + /* A <LookAt> or <Camera> is where the *viewer* is put, not where the + data is; reporting it as a GPS position would conflate the two, so + it goes into a comment that says which it is. */ + size_t lonp = kml_find (buf, len, sc.viewpoint + 1, "longitude"); + size_t latp = kml_find (buf, len, sc.viewpoint + 1, "latitude"); + double vlon; + double vlat; + size_t s2; + size_t l2; + + if ( (((size_t) -1) != lonp) && + (((size_t) -1) != latp) && + kml_text (buf, len, lonp, KML_MAX_TEXT, &vs, &vl) && + kml_parse_double (&buf[vs], vl, NULL, &vlon) && + kml_text (buf, len, latp, KML_MAX_TEXT, &s2, &l2) && + kml_parse_double (&buf[s2], l2, NULL, &vlat) && + (vlat >= -90.0) && (vlat <= 90.0) && + (vlon >= -180.0) && (vlon <= 180.0) ) + { + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_COMMENT, + "%s viewpoint at %.6f, %.6f" + " (latitude, longitude)", + kml_tag_is (buf, len, sc.viewpoint, + "Camera") + ? "Camera" : "LookAt", + vlat, + vlon)) + goto out; + } + } + if (EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%llu", + (unsigned long long) sc.placemarks)) + goto out; + if (truncated && + EXTRACTOR_forensic_emit_ (ec, + "kml", + EXTRACTOR_METATYPE_COMMENT, + "scan truncated at 1 MiB; the placemark" + " count and bounding box above cover" + " only that prefix of the file") ) + goto out; +out: + free (buf); +} + + +/* end of kml_extractor.c */ diff --git a/src/plugins/lnk_extractor.c b/src/plugins/lnk_extractor.c @@ -0,0 +1,847 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/lnk_extractor.c + * @brief plugin to support Windows shell links (.lnk shortcuts) + * @author Christian Grothoff + * + * A shell link describes a file that is somewhere else, and in doing so + * it records the machine that file was on: the volume serial number and + * label, the target's own timestamps and size, and -- in the link + * tracker block Windows adds so that a moved target can be found again + * -- the NetBIOS name of the creating machine and a pair of object + * identifiers. Those identifiers are version 1 UUIDs, so their node + * field is the MAC address of the network card that generated them. + * None of that survives in the file the link points at, which is why a + * .lnk is worth more to an investigator than its 2 KB suggest. + * + * Reference: [MS-SHLLINK], "Shell Link (.LNK) Binary File Format", + * https://learn.microsoft.com/openspecs/windows_protocols/ms-shllink/ + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * Size of the ShellLinkHeader, and the value its first field must have. + */ +#define LNK_HEADER_SIZE 0x4C + +/** + * How much of the file we read. Shell links are a couple of kilobytes; + * anything past this is a property store or an icon we do not parse. + */ +#define LNK_MAX_READ (64 * 1024) + +/** + * Most ExtraData blocks we walk. + */ +#define LNK_MAX_BLOCKS 32 + +/** + * Longest path we assemble from a base and a suffix, in bytes. + */ +#define LNK_MAX_PATH 1024 + +/* LinkFlags */ +#define LNK_HAS_ID_LIST 0x00000001 +#define LNK_HAS_LINK_INFO 0x00000002 +#define LNK_HAS_NAME 0x00000004 +#define LNK_HAS_RELATIVE_PATH 0x00000008 +#define LNK_HAS_WORKING_DIR 0x00000010 +#define LNK_HAS_ARGUMENTS 0x00000020 +#define LNK_HAS_ICON_LOCATION 0x00000040 +#define LNK_IS_UNICODE 0x00000080 + + +/** + * The class identifier every shell link starts with, + * 00021401-0000-0000-C000-000000000046, as it is stored: the first + * three fields little-endian, the rest in order. + */ +static const unsigned char lnk_clsid[16] = { + 0x01, 0x14, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x46 +}; + + +/** + * Name of a drive type as recorded in a VolumeID. + * + * @param type the raw field + * @return a static string, NULL if the value is not one we know + */ +static const char * +lnk_drive_type (uint32_t type) +{ + switch (type) + { + case 0: + return "unknown"; + case 1: + return "no root directory"; + case 2: + return "removable"; + case 3: + return "fixed disk"; + case 4: + return "network share"; + case 5: + return "CD-ROM"; + case 6: + return "RAM disk"; + default: + return NULL; + } +} + + +/** + * Append @a what to the comma-separated list in @a buf. + * + * @param buf destination buffer + * @param size number of bytes in @a buf + * @param what text to append + */ +static void +lnk_append (char *buf, + size_t size, + const char *what) +{ + size_t used = strlen (buf); + size_t need = strlen (what); + + if (0 != used) + { + if (used + 2 >= size) + return; + buf[used++] = ','; + buf[used++] = ' '; + buf[used] = '\0'; + } + if (used + need + 1 > size) + return; + memcpy (&buf[used], + what, + need + 1); +} + + +/** + * Emit a path made of a base and a suffix, either of which may be + * empty. + * + * [MS-SHLLINK] splits the target path into a part that identifies the + * volume and a part that is common to the local and the network + * spelling, and both halves are NUL-terminated in the file. Joining + * them here keeps the caller from having to care. + * + * @param ec extraction context + * @param type meta data type to report under + * @param base first half + * @param base_len number of bytes readable at @a base + * @param suffix second half, NULL for none + * @param suffix_len number of bytes readable at @a suffix + * @param unicode 1 if both halves are UTF-16LE, 0 for single bytes + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_emit_path (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const unsigned char *base, + size_t base_len, + const unsigned char *suffix, + size_t suffix_len, + int unicode) +{ + unsigned char buf[LNK_MAX_PATH]; + size_t out = 0; + size_t step = unicode ? 2 : 1; + + if ( (NULL == base) || + (0 == base_len) ) + return 0; + while ( (out + step <= base_len) && + (out + step <= sizeof (buf)) ) + { + if (unicode) + { + if (0 == EXTRACTOR_forensic_le16_ (&base[out])) + break; + } + else if ('\0' == base[out]) + { + break; + } + buf[out] = base[out]; + if (unicode) + buf[out + 1] = base[out + 1]; + out += step; + } + if (NULL != suffix) + { + size_t i = 0; + + while ( (i + step <= suffix_len) && + (out + step <= sizeof (buf)) ) + { + if (unicode) + { + if (0 == EXTRACTOR_forensic_le16_ (&suffix[i])) + break; + } + else if ('\0' == suffix[i]) + { + break; + } + buf[out] = suffix[i]; + if (unicode) + buf[out + 1] = suffix[i + 1]; + out += step; + i += step; + } + } + if (0 == out) + return 0; + if (unicode) + return EXTRACTOR_forensic_emit_utf16le_ (ec, + "lnk", + type, + buf, + out); + return EXTRACTOR_forensic_emit_text_ (ec, + "lnk", + type, + (const char *) buf, + out); +} + + +/** + * Report the volume the target lives on. + * + * The drive serial number is assigned when the volume is formatted and + * is the cheapest way to tell whether two shortcuts came off the same + * disk. + * + * @param ec extraction context + * @param v start of the VolumeID structure + * @param len number of bytes readable at @a v + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_parse_volume_id (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *v, + size_t len) +{ + uint32_t size; + uint32_t drive_type; + uint32_t serial; + uint32_t label; + const char *name; + + if (len < 0x10) + return 0; + size = EXTRACTOR_forensic_le32_ (&v[0]); + if (size < 0x10) + return 0; + if (size < len) + len = size; + drive_type = EXTRACTOR_forensic_le32_ (&v[4]); + serial = EXTRACTOR_forensic_le32_ (&v[8]); + label = EXTRACTOR_forensic_le32_ (&v[12]); + if (0 != serial) + { + /* The spelling Windows itself uses in `dir' and `vol'. */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "lnk", + EXTRACTOR_METATYPE_VOLUME_SERIAL, + "%04X-%04X", + (unsigned int) (serial >> 16), + (unsigned int) (serial & 0xFFFF))) + return 1; + } + if (NULL != (name = lnk_drive_type (drive_type))) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "lnk", + EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "%s", + name)) + return 1; + } + if (0x14 == label) + { + /* The ANSI label is still there, but the unicode one supersedes it. */ + uint32_t wide; + + if (len < 0x14) + return 0; + wide = EXTRACTOR_forensic_le32_ (&v[0x10]); + if ( (wide < len) && + (wide >= 0x14) ) + return EXTRACTOR_forensic_emit_utf16le_ (ec, + "lnk", + EXTRACTOR_METATYPE_VOLUME_NAME, + &v[wide], + len - wide); + return 0; + } + if ( (label < len) && + (label >= 0x10) ) + return EXTRACTOR_forensic_emit_text_ (ec, + "lnk", + EXTRACTOR_METATYPE_VOLUME_NAME, + (const char *) &v[label], + len - label); + return 0; +} + + +/** + * Report what the LinkInfo structure says about the target's location. + * + * @param ec extraction context + * @param li start of the LinkInfo structure + * @param len number of bytes readable at @a li + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_parse_link_info (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *li, + size_t len) +{ + uint32_t header_size; + uint32_t flags; + uint32_t suffix_off; + const unsigned char *suffix = NULL; + size_t suffix_len = 0; + int wide = 0; + + if (len < 0x1C) + return 0; + header_size = EXTRACTOR_forensic_le32_ (&li[4]); + flags = EXTRACTOR_forensic_le32_ (&li[8]); + /* A header of 0x24 or more means the unicode spellings of the two + path halves are present and are the ones to prefer. */ + if ( (header_size >= 0x24) && + (len >= 0x24) ) + wide = 1; + suffix_off = wide + ? EXTRACTOR_forensic_le32_ (&li[0x20]) + : EXTRACTOR_forensic_le32_ (&li[0x18]); + if ( (0 != suffix_off) && + (suffix_off < len) ) + { + suffix = &li[suffix_off]; + suffix_len = len - suffix_off; + } + if (0 != (flags & 0x1)) + { + uint32_t vol_off = EXTRACTOR_forensic_le32_ (&li[0x0C]); + uint32_t base_off = wide + ? EXTRACTOR_forensic_le32_ (&li[0x1C]) + : EXTRACTOR_forensic_le32_ (&li[0x10]); + + if ( (0 != vol_off) && + (vol_off < len) && + (0 != + lnk_parse_volume_id (ec, + &li[vol_off], + len - vol_off)) ) + return 1; + if ( (0 != base_off) && + (base_off < len) && + (0 != + lnk_emit_path (ec, + EXTRACTOR_METATYPE_TARGET_PATH, + &li[base_off], + len - base_off, + suffix, + suffix_len, + wide)) ) + return 1; + } + if (0 != (flags & 0x2)) + { + uint32_t cnrl_off = EXTRACTOR_forensic_le32_ (&li[0x14]); + const unsigned char *cnrl; + size_t cnrl_len; + uint32_t net_off; + int net_wide = 0; + + if ( (0 == cnrl_off) || + (cnrl_off >= len) ) + return 0; + cnrl = &li[cnrl_off]; + cnrl_len = len - cnrl_off; + if (cnrl_len < 0x14) + return 0; + net_off = EXTRACTOR_forensic_le32_ (&cnrl[8]); + if ( (net_off > 0x14) && + (cnrl_len >= 0x18) ) + { + uint32_t wide_off = EXTRACTOR_forensic_le32_ (&cnrl[0x14]); + + if ( (0 != wide_off) && + (wide_off < cnrl_len) ) + { + net_off = wide_off; + net_wide = 1; + } + } + if ( (0 != net_off) && + (net_off < cnrl_len) && + (0 != + lnk_emit_path (ec, + EXTRACTOR_METATYPE_TARGET_PATH, + &cnrl[net_off], + cnrl_len - net_off, + net_wide ? NULL : suffix, + net_wide ? 0 : suffix_len, + net_wide)) ) + return 1; + } + return 0; +} + + +/** + * Report a droid identifier, and the MAC address hiding in it. + * + * The two identifiers in a TrackerDataBlock are the NTFS object + * identifiers of the volume and of the file, generated as version 1 + * UUIDs. A version 1 UUID ends in the node field, which the generator + * fills with the MAC address of a network interface unless it had none + * -- in which case it must set the multicast bit to say so. Checking + * both the version and that bit is what keeps a random identifier from + * being reported as somebody's hardware address. + * + * @param ec extraction context + * @param guid the 16 bytes + * @param[in,out] seen previously emitted identifier, to suppress the + * usual case where the birth and current values are the same + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_emit_droid (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *guid, + unsigned char *seen) +{ + if (0 == memcmp (seen, + guid, + 16)) + return 0; + memcpy (seen, + guid, + 16); + if (0 != + EXTRACTOR_forensic_emit_guid_ (ec, + "lnk", + EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER, + guid, + 1)) + return 1; + if (1 != (guid[7] >> 4)) + return 0; /* not a time-based UUID, so no node field */ + if (0 != (guid[10] & 0x01)) + return 0; /* the generator flagged the node as made up */ + if ( (0 == guid[10]) && (0 == guid[11]) && (0 == guid[12]) && + (0 == guid[13]) && (0 == guid[14]) && (0 == guid[15]) ) + return 0; + return EXTRACTOR_forensic_emit_ (ec, + "lnk", + EXTRACTOR_METATYPE_MAC_ADDRESS, + "%02x:%02x:%02x:%02x:%02x:%02x", + guid[10], guid[11], guid[12], + guid[13], guid[14], guid[15]); +} + + +/** + * Report a TrackerDataBlock. + * + * @param ec extraction context + * @param b start of the block + * @param len number of bytes readable at @a b + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_parse_tracker (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *b, + size_t len) +{ + unsigned char seen[16]; + + if (len < 0x60) + return 0; + if (EXTRACTOR_forensic_le32_ (&b[8]) < 0x58) + return 0; /* the declared payload is too short to hold the fields */ + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + "lnk", + EXTRACTOR_METATYPE_SOURCE_HOST, + (const char *) &b[0x10], + 16)) + return 1; + memset (seen, + 0, + sizeof (seen)); + if (0 != + lnk_emit_droid (ec, + &b[0x20], + seen)) + return 1; /* DroidVolumeId */ + if (0 != + lnk_emit_droid (ec, + &b[0x40], + seen)) + return 1; /* BirthVolumeId */ + memset (seen, + 0, + sizeof (seen)); + if (0 != + lnk_emit_droid (ec, + &b[0x30], + seen)) + return 1; /* DroidFileId */ + return lnk_emit_droid (ec, + &b[0x50], + seen); /* BirthFileId */ +} + + +/** + * Walk the ExtraData block chain at the end of the file. + * + * @param ec extraction context + * @param buf the file contents + * @param len number of bytes in @a buf + * @param pos offset of the first block + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +lnk_parse_extra (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *buf, + size_t len, + size_t pos) +{ + for (unsigned int n = 0; + (pos + 8 <= len) && (n < LNK_MAX_BLOCKS); + n++) + { + uint32_t size = EXTRACTOR_forensic_le32_ (&buf[pos]); + uint32_t sig = EXTRACTOR_forensic_le32_ (&buf[pos + 4]); + size_t have; + + if (size < 8) + break; /* the terminal block, or a size that cannot progress */ + have = len - pos; + if (have > size) + have = size; + switch (sig) + { + case 0xA0000001U: /* EnvironmentVariableDataBlock */ + /* An expandable path, used when the target is behind an + environment variable. The unicode copy starts at 0x110. */ + if (have >= 0x314) + { + if (0 != + lnk_emit_path (ec, + EXTRACTOR_METATYPE_TARGET_PATH, + &buf[pos + 0x110], + 0x204, + NULL, + 0, + 1)) + return 1; + } + else if (have >= 0x110) + { + if (0 != + lnk_emit_path (ec, + EXTRACTOR_METATYPE_TARGET_PATH, + &buf[pos + 0x0C], + 0x104, + NULL, + 0, + 0)) + return 1; + } + break; + case 0xA0000003U: /* TrackerDataBlock */ + if (0 != + lnk_parse_tracker (ec, + &buf[pos], + have)) + return 1; + break; + case 0xA0000006U: /* DarwinDataBlock */ + /* The Windows Installer product code the shortcut belongs to. */ + if (have >= 0x314) + { + if (0 != + lnk_emit_path (ec, + EXTRACTOR_METATYPE_APPLICATION_ID, + &buf[pos + 0x110], + 0x204, + NULL, + 0, + 1)) + return 1; + } + break; + default: + break; + } + pos += size; + } + return 0; +} + + +/** + * Main entry method for the shell link extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_lnk_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_lnk_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char head[LNK_HEADER_SIZE]; + unsigned char *buf; + uint64_t fsize; + size_t len; + size_t pos; + uint32_t flags; + uint32_t attrs; + uint32_t target_size; + uint64_t created; + uint64_t accessed; + uint64_t written; + int wide; + char list[256]; + static const struct + { + uint32_t flag; + enum EXTRACTOR_MetaType type; + } strings[] = { + { LNK_HAS_NAME, EXTRACTOR_METATYPE_DESCRIPTION }, + { LNK_HAS_RELATIVE_PATH, EXTRACTOR_METATYPE_TARGET_PATH }, + { LNK_HAS_WORKING_DIR, EXTRACTOR_METATYPE_WORKING_DIRECTORY }, + { LNK_HAS_ARGUMENTS, EXTRACTOR_METATYPE_COMMAND_LINE }, + { LNK_HAS_ICON_LOCATION, EXTRACTOR_METATYPE_URI } + }; + + /* The magic is a fixed size and a fixed class identifier at offset + zero, so a non-match costs one short read. */ + if (! EXTRACTOR_forensic_read_ (ec, + 0, + head, + sizeof (head))) + return; + if (LNK_HEADER_SIZE != EXTRACTOR_forensic_le32_ (&head[0])) + return; + if (0 != memcmp (&head[4], + lnk_clsid, + sizeof (lnk_clsid))) + return; + if (0 != + ec->proc (ec->cls, + "lnk", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-ms-shortcut", + strlen ("application/x-ms-shortcut") + 1)) + return; + flags = EXTRACTOR_forensic_le32_ (&head[0x14]); + attrs = EXTRACTOR_forensic_le32_ (&head[0x18]); + target_size = EXTRACTOR_forensic_le32_ (&head[0x34]); + wide = (0 != (flags & LNK_IS_UNICODE)); + /* These three are the target's timestamps as they were when the + shortcut was made, not the shortcut's own. */ + created = EXTRACTOR_forensic_le64_ (&head[0x1C]); + accessed = EXTRACTOR_forensic_le64_ (&head[0x24]); + written = EXTRACTOR_forensic_le64_ (&head[0x2C]); + if (0 != + EXTRACTOR_forensic_emit_filetime_ (ec, + "lnk", + EXTRACTOR_METATYPE_CREATION_DATE, + created)) + return; + if (0 != + EXTRACTOR_forensic_emit_filetime_ (ec, + "lnk", + EXTRACTOR_METATYPE_ACCESS_DATE, + accessed)) + return; + if (0 != + EXTRACTOR_forensic_emit_filetime_ (ec, + "lnk", + EXTRACTOR_METATYPE_MODIFICATION_DATE, + written)) + return; + if ( (0 != target_size) && + (0 != + EXTRACTOR_forensic_emit_size_ (ec, + "lnk", + EXTRACTOR_METATYPE_EMBEDDED_FILE_SIZE, + target_size)) ) + return; + list[0] = '\0'; + if (0 != (attrs & 0x0001)) + lnk_append (list, sizeof (list), "READONLY"); + if (0 != (attrs & 0x0002)) + lnk_append (list, sizeof (list), "HIDDEN"); + if (0 != (attrs & 0x0004)) + lnk_append (list, sizeof (list), "SYSTEM"); + if (0 != (attrs & 0x0010)) + lnk_append (list, sizeof (list), "DIRECTORY"); + if (0 != (attrs & 0x0020)) + lnk_append (list, sizeof (list), "ARCHIVE"); + if (0 != (attrs & 0x0080)) + lnk_append (list, sizeof (list), "NORMAL"); + if (0 != (attrs & 0x0100)) + lnk_append (list, sizeof (list), "TEMPORARY"); + if (0 != (attrs & 0x0200)) + lnk_append (list, sizeof (list), "SPARSE"); + if (0 != (attrs & 0x0400)) + lnk_append (list, sizeof (list), "REPARSE_POINT"); + if (0 != (attrs & 0x0800)) + lnk_append (list, sizeof (list), "COMPRESSED"); + if (0 != (attrs & 0x1000)) + lnk_append (list, sizeof (list), "OFFLINE"); + if (0 != (attrs & 0x2000)) + lnk_append (list, sizeof (list), "NOT_CONTENT_INDEXED"); + if (0 != (attrs & 0x4000)) + lnk_append (list, sizeof (list), "ENCRYPTED"); + if ( ('\0' != list[0]) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + "lnk", + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + list)) ) + return; + + fsize = ec->get_size (ec->cls); + if ( (0 == fsize) || + (UINT64_MAX == fsize) ) + return; + len = (fsize > LNK_MAX_READ) ? LNK_MAX_READ : (size_t) fsize; + if (len <= LNK_HEADER_SIZE) + return; + if (NULL == (buf = malloc (len))) + return; + if (! EXTRACTOR_forensic_read_ (ec, + 0, + buf, + len)) + { + free (buf); + return; + } + pos = LNK_HEADER_SIZE; + if (0 != (flags & LNK_HAS_ID_LIST)) + { + size_t id_size; + + if (pos + 2 > len) + goto cleanup; + id_size = EXTRACTOR_forensic_le16_ (&buf[pos]); + pos += 2; + if (id_size > len - pos) + goto cleanup; + pos += id_size; + } + if (0 != (flags & LNK_HAS_LINK_INFO)) + { + uint32_t li_size; + size_t have; + + if (pos + 4 > len) + goto cleanup; + li_size = EXTRACTOR_forensic_le32_ (&buf[pos]); + if (li_size < 0x1C) + goto cleanup; + have = len - pos; + if (have > li_size) + have = li_size; + if (0 != + lnk_parse_link_info (ec, + &buf[pos], + have)) + goto cleanup; + if (li_size > len - pos) + goto cleanup; + pos += li_size; + } + for (unsigned int i = 0; i < sizeof (strings) / sizeof (strings[0]); i++) + { + size_t count; + size_t bytes; + + if (0 == (flags & strings[i].flag)) + continue; + if (pos + 2 > len) + goto cleanup; + count = EXTRACTOR_forensic_le16_ (&buf[pos]); + pos += 2; + bytes = wide ? count * 2 : count; + if (bytes > len - pos) + goto cleanup; + if (0 != bytes) + { + int stop; + + if (wide) + stop = EXTRACTOR_forensic_emit_utf16le_ (ec, + "lnk", + strings[i].type, + &buf[pos], + bytes); + else + stop = EXTRACTOR_forensic_emit_text_ (ec, + "lnk", + strings[i].type, + (const char *) &buf[pos], + bytes); + if (0 != stop) + goto cleanup; + } + pos += bytes; + } + (void) lnk_parse_extra (ec, + buf, + len, + pos); +cleanup: + free (buf); +} + + +/* end of lnk_extractor.c */ diff --git a/src/plugins/mbox_extractor.c b/src/plugins/mbox_extractor.c @@ -0,0 +1,2441 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/mbox_extractor.c + * @brief plugin to support Unix mbox mailboxes and RFC 5322 messages + * @author Christian Grothoff + * + * What is worth having out of a mail file is the routing metadata, not + * the prose: the `Received' chain and the IP literals in it say which + * machines the message actually passed through, `Message-ID' usually + * carries the originating host's name on the right of the `@', + * `X-Mailer' fingerprints the client, and the `d=' tag of a + * `DKIM-Signature' names the domain that vouched for it. Those are the + * fields this plugin goes after. + * + * Only the *first* message is parsed in full; the rest are counted. + * That is the deliberate trade-off of a first pass over a large volume: + * a mailbox can be gigabytes, and a characterisation plus a count beats + * an inventory that never finishes. + * + * Known limits of the scanner, stated once here: + * + * - The message count is the number of lines that begin with `From ' + * (plus the first message). That is the mbox format's well-known + * ambiguity: a *body* line beginning with "From " that the writer + * failed to escape as ">From " is indistinguishable from a separator, + * and inflates the count. There is no fix that does not involve + * parsing every message. + * - MIME parts are found by looking for `Content-Type' and + * `Content-Disposition' lines in the body, not by following the + * boundary; a part header quoted inside a body would be picked up. + * - Only the first message's body is searched for attachment names. + * - Encoded words are decoded for UTF-8, US-ASCII and the ISO-8859-1 + * family; ISO-8859-15 and windows-1252 are decoded *as* ISO-8859-1, + * which is right for every position except the handful those two + * redefine. Any other charset is left as the raw encoded word rather + * than guessed at. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * How many bytes of the mailbox we are willing to read. + */ +#define MBOX_SCAN_CAP (256 * 1024) + +/** + * How much of the start of the file the magic check looks at. + */ +#define MBOX_MAGIC_WINDOW 8192 + +/** + * Largest header block we will walk, in bytes. + */ +#define MBOX_MAX_HDR_BLOCK 65536 + +/** + * Most header fields we will look at in one message. + */ +#define MBOX_MAX_HDR_LINES 1024 + +/** + * Working buffer for one unfolded header value. + */ +#define MBOX_HDR_VALUE 4096 + +/** + * How many bytes of the first message's body we search for MIME part + * headers. + */ +#define MBOX_BODY_SCAN (128 * 1024) + +/** + * Longest text an RFC 2231 parameter may assemble to. + */ +#define MBOX_MAX_PARAM 512 + +/** + * Most RFC 2231 continuation segments we will join. + */ +#define MBOX_MAX_CONT 16 + +/** + * Most distinct attachment names we will remember for de-duplication. + */ +#define MBOX_MAX_FILENAMES 32 + + +/** + * Distinct IP literals seen in the routing headers. + */ +struct MboxIps +{ + /** + * The addresses, in the order they were found. 46 bytes is the + * longest textual IPv6 address plus a NUL. + */ + char ip[EXTRACTOR_FORENSIC_MAX_ITEMS][46]; + + /** + * Number of entries used in @e ip. + */ + unsigned int count; +}; + + +/** + * The single-valued headers of the first message. First occurrence + * wins: a header that appears twice is either a duplicate or an + * injection attempt, and the first one is the one the reader saw. + */ +struct MboxHeaders +{ + char from[1024]; + char to[2048]; + char cc[2048]; + char subject[1024]; + char date[256]; + char msgid[512]; + char inreplyto[512]; + char references[2048]; + char mailer[512]; + char org[512]; + char ctype[1024]; + char dkim[2048]; + char authres[1024]; + char charset[128]; +}; + + +/** + * Everything the plugin allocates in one block, so that every exit path + * is a single `free()'. + */ +struct MboxState +{ + struct MboxHeaders h; + struct MboxIps ips; + char val[MBOX_HDR_VALUE]; + + /** + * Attachment names already reported. A MIME part usually carries the + * same name twice, once as the `name' of its `Content-Type' and once + * as the `filename' of its `Content-Disposition'; reporting it twice + * would suggest two attachments. + */ + char fnames[MBOX_MAX_FILENAMES][MBOX_MAX_PARAM]; + + /** + * Number of entries used in @e fnames. + */ + unsigned int nfnames; +}; + + +/** + * ASCII lower case. Header field names and MIME parameter names are + * ASCII by definition, so no locale is involved. + * + * @param c the character + * @return @a c folded to lower case + */ +static char +mbox_lc (char c) +{ + if ( ('A' <= c) && ('Z' >= c) ) + return (char) (c - 'A' + 'a'); + return c; +} + + +/** + * Value of a hexadecimal digit. + * + * @param c the character + * @return 0-15, or -1 if @a c is not a hexadecimal digit + */ +static int +mbox_hex (char c) +{ + if ( ('0' <= c) && ('9' >= c) ) + return c - '0'; + if ( ('a' <= c) && ('f' >= c) ) + return c - 'a' + 10; + if ( ('A' <= c) && ('F' >= c) ) + return c - 'A' + 10; + return -1; +} + + +/** + * Case-insensitive comparison of a counted string against a + * NUL-terminated one. + * + * @param a the counted string + * @param alen number of bytes in @a a + * @param b the NUL-terminated string + * @return 1 if they are equal ignoring case, 0 otherwise + */ +static int +mbox_ieq (const char *a, + size_t alen, + const char *b) +{ + size_t blen = strlen (b); + + if (alen != blen) + return 0; + for (size_t i = 0; i < alen; i++) + if (mbox_lc (a[i]) != mbox_lc (b[i])) + return 0; + return 1; +} + + +/** + * Store a value in a fixed-width slot, first writer wins. + * + * @param dst the slot + * @param dstsize number of bytes in @a dst + * @param v the value + * @param vlen number of bytes in @a v + */ +static void +mbox_store (char *dst, + size_t dstsize, + const char *v, + size_t vlen) +{ + if ('\0' != dst[0]) + return; + if (vlen >= dstsize) + vlen = dstsize - 1; + memcpy (dst, + v, + vlen); + dst[vlen] = '\0'; +} + + +/** + * Days since 1970-01-01 for a proleptic Gregorian date. (Howard + * Hinnant's `days_from_civil'.) + * + * @param y year + * @param m month, 1-12 + * @param d day of month, 1-31 + * @return day number, negative before the epoch + */ +static int64_t +mbox_days_from_civil (int64_t y, + int64_t m, + int64_t d) +{ + int64_t era; + int64_t yoe; + int64_t doy; + int64_t doe; + + y -= (m <= 2); + era = (y >= 0 ? y : y - 399) / 400; + yoe = y - era * 400; + doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + + +/** + * Parse an RFC 5322 `Date:' value into seconds since the Unix epoch. + * + * Handles the current syntax (`Tue, 12 Mar 2024 09:41:07 +0100') and the + * obsolete forms that are still common in archives: a two- or + * three-digit year, a missing seconds field, and an alphabetic zone + * (`GMT', `UT', `EST' and the rest of the US set). An unrecognised + * alphabetic zone is treated as UTC, which is what RFC 5322 says to do + * with the obsolete military single letters. + * + * @param s the value + * @param len number of bytes in @a s + * @param[out] when where to store the result + * @return 1 on success, 0 if @a s is not a date + */ +static int +mbox_parse_date (const char *s, + size_t len, + int64_t *when) +{ + static const char *months[12] = { + "jan", "feb", "mar", "apr", "may", "jun", + "jul", "aug", "sep", "oct", "nov", "dec" + }; + static const struct + { + const char *name; + int hours; + } zones[] = { + { "ut", 0 }, { "gmt", 0 }, { "z", 0 }, + { "est", -5 }, { "edt", -4 }, + { "cst", -6 }, { "cdt", -5 }, + { "mst", -7 }, { "mdt", -6 }, + { "pst", -8 }, { "pdt", -7 }, + { NULL, 0 } + }; + size_t i = 0; + int64_t day; + int64_t mon = -1; + int64_t year; + int64_t hh; + int64_t mm; + int64_t ss = 0; + int64_t off = 0; + size_t digits; + +#define SKIP_WS() while ( (i < len) && \ + ( (' ' == s[i]) || ('\t' == s[i]) ) ) i++ + + SKIP_WS (); + /* optional `Tue, ' */ + { + size_t j = i; + + while ( (j < len) && + ( ( ('a' <= mbox_lc (s[j])) && ('z' >= mbox_lc (s[j])) ) ) ) + j++; + if ( (j > i) && + (j < len) && + (',' == s[j]) ) + i = j + 1; + } + SKIP_WS (); + digits = 0; + day = 0; + while ( (i < len) && + (digits < 2) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + day = day * 10 + (s[i] - '0'); + i++; + digits++; + } + if ( (0 == digits) || + (day < 1) || (day > 31) ) + return 0; + SKIP_WS (); + if (i + 3 > len) + return 0; + for (unsigned int m = 0; m < 12; m++) + if ( (mbox_lc (s[i]) == months[m][0]) && + (mbox_lc (s[i + 1]) == months[m][1]) && + (mbox_lc (s[i + 2]) == months[m][2]) ) + { + mon = m + 1; + break; + } + if (0 > mon) + return 0; + i += 3; + /* an obsolete long month name may follow; skip the rest of the word */ + while ( (i < len) && + ('a' <= mbox_lc (s[i])) && ('z' >= mbox_lc (s[i])) ) + i++; + SKIP_WS (); + digits = 0; + year = 0; + while ( (i < len) && + (digits < 4) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + year = year * 10 + (s[i] - '0'); + i++; + digits++; + } + if (0 == digits) + return 0; + if (2 == digits) + year += (year < 50) ? 2000 : 1900; /* RFC 5322 4.3 */ + else if (3 == digits) + year += 1900; + else if (4 != digits) + return 0; + SKIP_WS (); + digits = 0; + hh = 0; + while ( (i < len) && + (digits < 2) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + hh = hh * 10 + (s[i] - '0'); + i++; + digits++; + } + if ( (0 == digits) || + (i >= len) || + (':' != s[i]) ) + return 0; + i++; + digits = 0; + mm = 0; + while ( (i < len) && + (digits < 2) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + mm = mm * 10 + (s[i] - '0'); + i++; + digits++; + } + if (0 == digits) + return 0; + if ( (i < len) && + (':' == s[i]) ) + { + i++; + digits = 0; + while ( (i < len) && + (digits < 2) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + ss = ss * 10 + (s[i] - '0'); + i++; + digits++; + } + if (0 == digits) + return 0; + } + if ( (hh > 23) || (mm > 59) || (ss > 60) ) + return 0; + SKIP_WS (); + if ( (i < len) && + ( ('+' == s[i]) || ('-' == s[i]) ) ) + { + int neg = ('-' == s[i]); + int64_t v = 0; + + i++; + digits = 0; + while ( (i < len) && + (digits < 4) && + ('0' <= s[i]) && ('9' >= s[i]) ) + { + v = v * 10 + (s[i] - '0'); + i++; + digits++; + } + if (4 != digits) + return 0; + off = (v / 100) * 3600 + (v % 100) * 60; + if (neg) + off = -off; + } + else if (i < len) + { + size_t j = i; + + while ( (j < len) && + (j - i < 8) && + ('a' <= mbox_lc (s[j])) && ('z' >= mbox_lc (s[j])) ) + j++; + for (unsigned int z = 0; NULL != zones[z].name; z++) + if (mbox_ieq (&s[i], + j - i, + zones[z].name)) + { + off = zones[z].hours * 3600; + break; + } + } +#undef SKIP_WS + *when = mbox_days_from_civil (year, + mon, + day) * 86400 + + hh * 3600 + mm * 60 + ss + - off; + return 1; +} + + +/** + * Decode a base64 body into @a out, ignoring white space. + * + * @param in the encoded text + * @param inlen number of bytes in @a in + * @param[out] out where to write the bytes + * @param outsize number of bytes available in @a out + * @return number of bytes written, or 0 on a malformed input + */ +static size_t +mbox_b64 (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + static const char alpha[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + uint32_t acc = 0; + unsigned int bits = 0; + size_t o = 0; + + for (size_t i = 0; i < inlen; i++) + { + const char *p; + + if ( (' ' == in[i]) || ('\t' == in[i]) || + ('\r' == in[i]) || ('\n' == in[i]) ) + continue; + if ('=' == in[i]) + break; + p = memchr (alpha, + in[i], + 64); + if (NULL == p) + return 0; /* not base64 after all */ + acc = (acc << 6) | (uint32_t) (p - alpha); + bits += 6; + if (bits >= 8) + { + bits -= 8; + if (o >= outsize) + return o; + out[o++] = (char) ((acc >> bits) & 0xFF); + } + } + return o; +} + + +/** + * Decode the quoted-printable variant used inside RFC 2047 encoded + * words, where `_' stands for a space. + * + * @param in the encoded text + * @param inlen number of bytes in @a in + * @param[out] out where to write the bytes + * @param outsize number of bytes available in @a out + * @return number of bytes written + */ +static size_t +mbox_qp (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + size_t o = 0; + + for (size_t i = 0; (i < inlen) && (o < outsize); i++) + { + if ('_' == in[i]) + { + out[o++] = ' '; + continue; + } + if ( ('=' == in[i]) && + (i + 2 < inlen) && + (0 <= mbox_hex (in[i + 1])) && + (0 <= mbox_hex (in[i + 2])) ) + { + out[o++] = (char) ((mbox_hex (in[i + 1]) << 4) + | mbox_hex (in[i + 2])); + i += 2; + continue; + } + out[o++] = in[i]; + } + return o; +} + + +/** + * Which charsets we know how to turn into UTF-8. + */ +enum MboxCharset +{ + /** + * Not a charset we handle; leave the encoded word alone. + */ + MBOX_CS_UNKNOWN = 0, + + /** + * Already UTF-8 (or a subset of it). + */ + MBOX_CS_UTF8 = 1, + + /** + * A single-byte charset whose lower half is ASCII; decoded as + * ISO-8859-1. + */ + MBOX_CS_LATIN1 = 2 +}; + + +/** + * Classify a charset name from an encoded word. + * + * @param name the charset name (a `*' language suffix is tolerated) + * @param len number of bytes in @a name + * @return how to decode the octets + */ +static enum MboxCharset +mbox_charset (const char *name, + size_t len) +{ + /* RFC 2231 allows a `*language' suffix on the charset of an encoded + word; cut it off before comparing. */ + for (size_t i = 0; i < len; i++) + if ('*' == name[i]) + { + len = i; + break; + } + if (mbox_ieq (name, len, "utf-8") || + mbox_ieq (name, len, "utf8") || + mbox_ieq (name, len, "us-ascii") || + mbox_ieq (name, len, "ascii") ) + return MBOX_CS_UTF8; + /* ISO-8859-15 and windows-1252 differ from ISO-8859-1 only in a few + positions; decoding them as Latin-1 gets everything else right and + never produces invalid UTF-8. */ + if (mbox_ieq (name, len, "iso-8859-1") || + mbox_ieq (name, len, "iso8859-1") || + mbox_ieq (name, len, "latin1") || + mbox_ieq (name, len, "iso-8859-15") || + mbox_ieq (name, len, "iso8859-15") || + mbox_ieq (name, len, "windows-1252") || + mbox_ieq (name, len, "cp1252") ) + return MBOX_CS_LATIN1; + return MBOX_CS_UNKNOWN; +} + + +/** + * Append @a len bytes of Latin-1 to @a out as UTF-8. + * + * @param in the bytes + * @param len number of bytes in @a in + * @param[out] out output buffer + * @param outsize number of bytes available in @a out + * @param o current write offset in @a out + * @return the new write offset + */ +static size_t +mbox_latin1 (const char *in, + size_t len, + char *out, + size_t outsize, + size_t o) +{ + for (size_t i = 0; i < len; i++) + { + unsigned char c = (unsigned char) in[i]; + + if (c < 0x80) + { + if (o + 1 >= outsize) + break; + out[o++] = (char) c; + } + else + { + if (o + 2 >= outsize) + break; + out[o++] = (char) (0xC0 | (c >> 6)); + out[o++] = (char) (0x80 | (c & 0x3F)); + } + } + return o; +} + + +/** + * Decode the RFC 2047 encoded words in a header value. + * + * Encoded words that are adjacent (separated only by white space) have + * that white space removed, as RFC 2047 section 6.2 requires -- without + * that, a subject split across two words gains a stray space. + * + * @param in the raw header value + * @param inlen number of bytes in @a in + * @param[out] out where to write the decoded text + * @param outsize number of bytes available in @a out + * @return number of bytes written to @a out + */ +static size_t +mbox_decode_words (const char *in, + size_t inlen, + char *out, + size_t outsize) +{ + size_t i = 0; + size_t o = 0; + int prev_was_word = 0; + + while ( (i < inlen) && + (o + 1 < outsize) ) + { + size_t cs; + size_t cse; + size_t enc; + size_t txt; + size_t end; + enum MboxCharset kind; + + if ( ('=' == in[i]) && + (i + 1 < inlen) && + ('?' == in[i + 1]) ) + { + /* =?charset?E?text?= */ + cs = i + 2; + cse = cs; + while ( (cse < inlen) && + (cse - cs < 64) && + ('?' != in[cse]) ) + cse++; + if ( (cse >= inlen) || + ('?' != in[cse]) || + (cse == cs) ) + { + out[o++] = in[i++]; + prev_was_word = 0; + continue; + } + enc = cse + 1; + if ( (enc + 1 >= inlen) || + ('?' != in[enc + 1]) ) + { + out[o++] = in[i++]; + prev_was_word = 0; + continue; + } + txt = enc + 2; + end = txt; + while ( (end + 1 < inlen) && + (end - txt < 1024) && + ! ( ('?' == in[end]) && ('=' == in[end + 1]) ) ) + end++; + if ( (end + 1 >= inlen) || + ('?' != in[end]) ) + { + out[o++] = in[i++]; + prev_was_word = 0; + continue; + } + kind = mbox_charset (&in[cs], + cse - cs); + if (MBOX_CS_UNKNOWN == kind) + { + out[o++] = in[i++]; /* leave the word verbatim */ + prev_was_word = 0; + continue; + } + { + char raw[1024]; + size_t rlen = 0; + + if ( ('b' == mbox_lc (in[enc])) ) + rlen = mbox_b64 (&in[txt], + end - txt, + raw, + sizeof (raw)); + else if ('q' == mbox_lc (in[enc])) + rlen = mbox_qp (&in[txt], + end - txt, + raw, + sizeof (raw)); + else + rlen = 0; + if (0 == rlen) + { + out[o++] = in[i++]; + prev_was_word = 0; + continue; + } + if (MBOX_CS_UTF8 == kind) + { + if (rlen > outsize - o - 1) + rlen = outsize - o - 1; + memcpy (&out[o], + raw, + rlen); + o += rlen; + } + else + { + o = mbox_latin1 (raw, + rlen, + out, + outsize, + o); + } + } + i = end + 2; + prev_was_word = 1; + continue; + } + if ( (' ' == in[i]) || + ('\t' == in[i]) ) + { + size_t j = i; + + while ( (j < inlen) && + ( (' ' == in[j]) || ('\t' == in[j]) ) ) + j++; + if (prev_was_word && + (j + 1 < inlen) && + ('=' == in[j]) && ('?' == in[j + 1]) ) + { + i = j; /* white space between adjacent encoded words: drop */ + continue; + } + out[o++] = ' '; + i = j; + prev_was_word = 0; + continue; + } + out[o++] = in[i++]; + prev_was_word = 0; + } + return o; +} + + +/** + * Emit a header value, decoding its encoded words first. If decoding + * produced something that is not valid UTF-8 -- a mislabelled charset, + * say -- the raw value is emitted instead, which is at worst unreadable + * but is never wrong. + * + * @param ec extraction context + * @param type meta data type + * @param raw the raw (unfolded) header value + * @param rawlen number of bytes in @a raw + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +mbox_emit_hdr (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const char *raw, + size_t rawlen) +{ + char dec[EXTRACTOR_FORENSIC_MAX_STRING]; + size_t dlen; + + dlen = mbox_decode_words (raw, + rawlen, + dec, + sizeof (dec)); + if (EXTRACTOR_forensic_utf8_valid_ (dec, + dlen)) + return EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + type, + dec, + dlen); + return EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + type, + raw, + rawlen); +} + + +/** + * Is @a s, of length @a len, a dotted-quad IPv4 address? + * + * @param s the candidate + * @param len number of bytes in @a s + * @return 1 if it is, 0 if not + */ +static int +mbox_is_ipv4 (const char *s, + size_t len) +{ + size_t i = 0; + + for (unsigned int part = 0; part < 4; part++) + { + unsigned int v = 0; + unsigned int digits = 0; + + while ( (i < len) && + ('0' <= s[i]) && ('9' >= s[i]) && + (digits < 3) ) + { + v = v * 10 + (unsigned int) (s[i] - '0'); + i++; + digits++; + } + if ( (0 == digits) || + (v > 255) ) + return 0; + if (part < 3) + { + if ( (i >= len) || + ('.' != s[i]) ) + return 0; + i++; + } + } + return (i == len); +} + + +/** + * Is @a s, of length @a len, an IPv6 address? + * + * Deliberately conservative: hexadecimal groups separated by colons, + * at most one `::', at most eight groups, optionally ending in a + * dotted-quad (the IPv4-mapped form that shows up in `Received' lines). + * A zone index (`%eth0') is rejected rather than guessed at. + * + * @param s the candidate + * @param len number of bytes in @a s + * @return 1 if it is, 0 if not + */ +static int +mbox_is_ipv6 (const char *s, + size_t len) +{ + unsigned int groups = 0; + int dbl = 0; + size_t i = 0; + size_t tail; + + if ( (0 == len) || + (len > 45) ) + return 0; + /* a trailing dotted-quad counts as two groups */ + tail = len; + for (size_t k = 0; k < len; k++) + if ('.' == s[k]) + { + size_t start = 0; + + for (size_t j = len; j > 0; j--) + if (':' == s[j - 1]) + { + start = j; + break; + } + if (0 == start) + return 0; + if (! mbox_is_ipv4 (&s[start], + len - start)) + return 0; + tail = start - 1; /* keep the colon that precedes it */ + groups = 2; + break; + } + while (i < tail) + { + unsigned int digits = 0; + + if (':' == s[i]) + { + if ( (i + 1 < tail) && + (':' == s[i + 1]) ) + { + if (dbl) + return 0; /* only one `::' is allowed */ + dbl = 1; + i += 2; + continue; + } + if (0 == i) + return 0; /* a single leading colon is not valid */ + i++; + continue; + } + while ( (i < tail) && + (digits < 4) && + (0 <= mbox_hex (s[i])) ) + { + i++; + digits++; + } + if (0 == digits) + return 0; + groups++; + if (groups > 8) + return 0; + if ( (i < tail) && + (':' != s[i]) ) + return 0; + } + if ( (tail < len) && + (0 == groups) ) + return 0; + if (dbl) + return (groups <= 7); + return (8 == groups); +} + + +/** + * Remember an IP literal, ignoring duplicates. + * + * @param ips the list + * @param s the address text + * @param len number of bytes in @a s + */ +static void +mbox_add_ip (struct MboxIps *ips, + const char *s, + size_t len) +{ + /* `IPv6:' prefixes the literal inside the brackets of a Received + line, per RFC 5321. */ + if ( (len > 5) && + mbox_ieq (s, + 5, + "ipv6:") ) + { + s += 5; + len -= 5; + } + if ( (0 == len) || + (len >= sizeof (ips->ip[0])) ) + return; + if ( (! mbox_is_ipv4 (s, len)) && + (! mbox_is_ipv6 (s, len)) ) + return; + for (unsigned int i = 0; i < ips->count; i++) + if ( (strlen (ips->ip[i]) == len) && + (0 == memcmp (ips->ip[i], s, len)) ) + return; + if (ips->count >= EXTRACTOR_FORENSIC_MAX_ITEMS) + return; + memcpy (ips->ip[ips->count], + s, + len); + ips->ip[ips->count][len] = '\0'; + ips->count++; +} + + +/** + * Harvest the bracketed IP literals out of a `Received' value. + * + * @param v the value + * @param vlen number of bytes in @a v + * @param ips where to collect the addresses + */ +static void +mbox_harvest_bracketed (const char *v, + size_t vlen, + struct MboxIps *ips) +{ + for (size_t i = 0; i < vlen; i++) + { + size_t j; + + if ('[' != v[i]) + continue; + j = i + 1; + while ( (j < vlen) && + (j - i <= 64) && + (']' != v[j]) ) + j++; + if ( (j >= vlen) || + (']' != v[j]) ) + continue; + mbox_add_ip (ips, + &v[i + 1], + j - i - 1); + i = j; + } +} + + +/** + * Harvest IP literals out of a value that is a bare list of them, as in + * `X-Originating-IP' and `X-Forwarded-For'. + * + * @param v the value + * @param vlen number of bytes in @a v + * @param ips where to collect the addresses + */ +static void +mbox_harvest_tokens (const char *v, + size_t vlen, + struct MboxIps *ips) +{ + size_t i = 0; + + while (i < vlen) + { + size_t start; + + while ( (i < vlen) && + (NULL != strchr (" \t,;[]()<>", v[i])) ) + i++; + start = i; + while ( (i < vlen) && + (NULL == strchr (" \t,;[]()<>", v[i])) ) + i++; + if (i > start) + mbox_add_ip (ips, + &v[start], + i - start); + } +} + + +/** + * Split a mailbox into its display name and its addr-spec. + * + * Handles `Name <a@b>', `"Name" <a@b>', `a@b' and the obsolete + * `a@b (Name)'. + * + * @param in the mailbox text + * @param len number of bytes in @a in + * @param[out] nstart offset of the display name in @a in + * @param[out] nlen length of the display name, 0 if there is none + * @param[out] astart offset of the addr-spec in @a in + * @param[out] alen length of the addr-spec, 0 if there is none + */ +static void +mbox_split_addr (const char *in, + size_t len, + size_t *nstart, + size_t *nlen, + size_t *astart, + size_t *alen) +{ + size_t lt = (size_t) -1; + size_t gt = (size_t) -1; + + *nstart = 0; + *nlen = 0; + *astart = 0; + *alen = 0; + for (size_t i = 0; i < len; i++) + if ('<' == in[i]) + { + lt = i; + break; + } + if (((size_t) -1) != lt) + for (size_t i = len; i > lt + 1; i--) + if ('>' == in[i - 1]) + { + gt = i - 1; + break; + } + if ( (((size_t) -1) != lt) && + (((size_t) -1) != gt) ) + { + *astart = lt + 1; + *alen = gt - lt - 1; + *nstart = 0; + *nlen = lt; + } + else + { + size_t par = (size_t) -1; + + for (size_t i = 0; i < len; i++) + if ('(' == in[i]) + { + par = i; + break; + } + if (((size_t) -1) != par) + { + size_t close = len; + + for (size_t i = len; i > par + 1; i--) + if (')' == in[i - 1]) + { + close = i - 1; + break; + } + *astart = 0; + *alen = par; + *nstart = par + 1; + *nlen = (close > par + 1) ? close - par - 1 : 0; + } + else + { + *astart = 0; + *alen = len; + } + } + /* trim both parts and strip the quotes around a quoted display name */ + while ( (*nlen > 0) && + ( (' ' == in[*nstart]) || ('\t' == in[*nstart]) ) ) + { + (*nstart)++; + (*nlen)--; + } + while ( (*nlen > 0) && + ( (' ' == in[*nstart + *nlen - 1]) || + ('\t' == in[*nstart + *nlen - 1]) ) ) + (*nlen)--; + if ( (*nlen >= 2) && + ('"' == in[*nstart]) && + ('"' == in[*nstart + *nlen - 1]) ) + { + (*nstart)++; + *nlen -= 2; + } + while ( (*alen > 0) && + ( (' ' == in[*astart]) || ('\t' == in[*astart]) ) ) + { + (*astart)++; + (*alen)--; + } + while ( (*alen > 0) && + ( (' ' == in[*astart + *alen - 1]) || + ('\t' == in[*astart + *alen - 1]) ) ) + (*alen)--; +} + + +/** + * Emit each address of an address list, capped. + * + * The list is split on commas that are not inside a quoted string, an + * angle-addr or a comment, so that a display name containing a comma + * does not become two recipients. + * + * @param ec extraction context + * @param type meta data type to report each address under + * @param v the header value + * @param vlen number of bytes in @a v + * @param[in,out] emitted how many have been emitted so far + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +mbox_emit_addr_list (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + const char *v, + size_t vlen, + unsigned int *emitted) +{ + size_t start = 0; + int quote = 0; + int angle = 0; + int paren = 0; + + for (size_t i = 0; i <= vlen; i++) + { + int split = (i == vlen); + + if (! split) + { + char c = v[i]; + + if (quote) + { + if ('"' == c) + quote = 0; + continue; + } + switch (c) + { + case '"': + quote = 1; + break; + case '<': + angle++; + break; + case '>': + if (angle > 0) + angle--; + break; + case '(': + paren++; + break; + case ')': + if (paren > 0) + paren--; + break; + case ',': + if ( (0 == angle) && (0 == paren) ) + split = 1; + break; + default: + break; + } + } + if (! split) + continue; + if (i > start) + { + size_t s = start; + size_t l = i - start; + + while ( (l > 0) && + ( (' ' == v[s]) || ('\t' == v[s]) ) ) + { + s++; + l--; + } + while ( (l > 0) && + ( (' ' == v[s + l - 1]) || ('\t' == v[s + l - 1]) ) ) + l--; + if (0 != l) + { + if (*emitted >= EXTRACTOR_FORENSIC_MAX_ITEMS) + return 0; + (*emitted)++; + if (mbox_emit_hdr (ec, + type, + &v[s], + l)) + return 1; + } + } + start = i + 1; + } + return 0; +} + + +/** + * Find the value of a MIME parameter in a structured header value. + * + * Understands `name=token', `name="quoted string"', the RFC 2231 + * extended form `name*=charset'lang'pct-encoded' and the RFC 2231 + * continuation form `name*0=', `name*1=', ... which is joined back + * together in index order. + * + * @param v the header value, after the media type + * @param vlen number of bytes in @a v + * @param name the parameter name, lower case + * @param[out] out where to write the value + * @param outsize number of bytes available in @a out + * @return length of the value, 0 if the parameter is absent + */ +static size_t +mbox_param (const char *v, + size_t vlen, + const char *name, + char *out, + size_t outsize) +{ + size_t nlen = strlen (name); + size_t i = 0; + size_t o = 0; + unsigned int segs = 0; + int simple = 0; + + while ( (i < vlen) && + (segs < MBOX_MAX_CONT) ) + { + size_t ps; + size_t pe; + size_t vs; + size_t ve; + int extended = 0; + int is_cont = 0; + + /* advance to the next `;' separated parameter */ + while ( (i < vlen) && + (';' != v[i]) ) + { + if ('"' == v[i]) + { + i++; + while ( (i < vlen) && + ('"' != v[i]) ) + { + if ( ('\\' == v[i]) && + (i + 1 < vlen) ) + i++; + i++; + } + } + if (i < vlen) + i++; + } + if (i >= vlen) + break; + i++; /* skip the `;' */ + while ( (i < vlen) && + ( (' ' == v[i]) || ('\t' == v[i]) ) ) + i++; + ps = i; + while ( (i < vlen) && + (';' != v[i]) && ('=' != v[i]) && + (' ' != v[i]) && ('\t' != v[i]) ) + i++; + pe = i; + while ( (i < vlen) && + ( (' ' == v[i]) || ('\t' == v[i]) ) ) + i++; + if ( (i >= vlen) || + ('=' != v[i]) ) + continue; + i++; + while ( (i < vlen) && + ( (' ' == v[i]) || ('\t' == v[i]) ) ) + i++; + if ( (i < vlen) && + ('"' == v[i]) ) + { + i++; + vs = i; + while ( (i < vlen) && + ('"' != v[i]) ) + { + if ( ('\\' == v[i]) && + (i + 1 < vlen) ) + i++; + i++; + } + ve = i; + if (i < vlen) + i++; + } + else + { + vs = i; + while ( (i < vlen) && + (';' != v[i]) ) + i++; + ve = i; + while ( (ve > vs) && + ( (' ' == v[ve - 1]) || ('\t' == v[ve - 1]) ) ) + ve--; + } + /* does the parameter name match? */ + if (pe - ps < nlen) + continue; + if (! mbox_ieq (&v[ps], + nlen, + name)) + continue; + if (pe - ps == nlen) + { + simple = 1; + } + else if ('*' == v[ps + nlen]) + { + size_t k = ps + nlen + 1; + + while ( (k < pe) && + ('0' <= v[k]) && ('9' >= v[k]) ) + { + is_cont = 1; + k++; + } + if ( (k < pe) && + ('*' == v[k]) ) + { + extended = 1; + k++; + } + if (k != pe) + continue; /* something else entirely */ + if (! is_cont) + extended = 1; + } + else + { + continue; + } + if (simple && + (0 != segs) ) + continue; /* continuations already won */ + if (extended) + { + /* charset'language'pct-encoded -- drop the first two fields */ + size_t q = vs; + unsigned int seen = 0; + + if (! is_cont) + { + while ( (q < ve) && + (seen < 2) ) + { + if ('\'' == v[q]) + seen++; + q++; + } + if (2 != seen) + q = vs; + } + while ( (q < ve) && + (o + 1 < outsize) ) + { + if ( ('%' == v[q]) && + (q + 2 < ve) && + (0 <= mbox_hex (v[q + 1])) && + (0 <= mbox_hex (v[q + 2])) ) + { + out[o++] = (char) ((mbox_hex (v[q + 1]) << 4) + | mbox_hex (v[q + 2])); + q += 3; + continue; + } + out[o++] = v[q++]; + } + } + else + { + size_t q = vs; + + while ( (q < ve) && + (o + 1 < outsize) ) + { + if ( ('\\' == v[q]) && + (q + 1 < ve) ) + q++; + out[o++] = v[q++]; + } + } + segs++; + if (simple) + break; + } + if (o < outsize) + out[o] = '\0'; + return o; +} + + +/** + * Walk the header block once, handing each field to @a cb. + * + * Continuation lines (those starting with a space or a tab) are folded + * into the previous value with the fold replaced by a single space. + * + * @param buf the buffer + * @param hstart offset of the first header line + * @param hend offset one past the last header byte + * @param[in,out] pos iteration cursor, initialise to @a hstart + * @param[out] nstart offset of the field name + * @param[out] nlen length of the field name + * @param[out] val where to copy the unfolded value + * @param valsize number of bytes available in @a val + * @param[out] vlen number of bytes written to @a val + * @return 1 if a field was returned, 0 at the end of the block + */ +static int +mbox_next_header (const char *buf, + size_t hstart, + size_t hend, + size_t *pos, + size_t *nstart, + size_t *nlen, + char *val, + size_t valsize, + size_t *vlen) +{ + (void) hstart; + while (*pos < hend) + { + size_t ls = *pos; + size_t le; + size_t colon = (size_t) -1; + size_t vs; + size_t o = 0; + + le = ls; + while ( (le < hend) && + ('\n' != buf[le]) ) + le++; + for (size_t i = ls; (i < le) && (i - ls < 128); i++) + { + if (':' == buf[i]) + { + colon = i; + break; + } + /* a field name is printable ASCII other than the colon */ + if ( (buf[i] < 33) || + (buf[i] > 126) ) + break; + } + if ( (((size_t) -1) == colon) || + (colon == ls) ) + { + *pos = (le < hend) ? le + 1 : hend; + continue; /* not a header line; skip it */ + } + *nstart = ls; + *nlen = colon - ls; + vs = colon + 1; + while ( (vs < le) && + ( (' ' == buf[vs]) || ('\t' == buf[vs]) ) ) + vs++; + { + size_t e = le; + + while ( (e > vs) && + ('\r' == buf[e - 1]) ) + e--; + if (e - vs > valsize - 1) + e = vs + valsize - 1; + memcpy (val, + &buf[vs], + e - vs); + o = e - vs; + } + *pos = (le < hend) ? le + 1 : hend; + /* fold in the continuation lines */ + while (*pos < hend) + { + size_t cs = *pos; + size_t ce; + size_t ts; + + if ( (' ' != buf[cs]) && + ('\t' != buf[cs]) ) + break; + ce = cs; + while ( (ce < hend) && + ('\n' != buf[ce]) ) + ce++; + ts = cs; + while ( (ts < ce) && + ( (' ' == buf[ts]) || ('\t' == buf[ts]) ) ) + ts++; + { + size_t e = ce; + + while ( (e > ts) && + ('\r' == buf[e - 1]) ) + e--; + if ( (o + 1 < valsize) && + (0 != o) ) + val[o++] = ' '; + if (e - ts > valsize - 1 - o) + e = ts + (valsize - 1 - o); + if (e > ts) + { + memcpy (&val[o], + &buf[ts], + e - ts); + o += e - ts; + } + } + *pos = (ce < hend) ? ce + 1 : hend; + } + val[o] = '\0'; + *vlen = o; + return 1; + } + return 0; +} + + +/** + * Does the buffer start with something that is plausibly an RFC 5322 + * header block? + * + * Deliberately strict: "a text file with colons in it" is not a mail + * message, and claiming one would be worse than missing a real message. + * We require a well-formed `Name: value' block, with folded + * continuations, terminated by a blank line, and at least one of the + * fields a message cannot really do without. + * + * @param buf the buffer + * @param len number of bytes in @a buf + * @return 1 if this looks like a message, 0 if not + */ +static int +mbox_looks_like_message (const char *buf, + size_t len) +{ + size_t i = 0; + unsigned int fields = 0; + int required = 0; + + if (len > MBOX_MAGIC_WINDOW) + len = MBOX_MAGIC_WINDOW; + for (unsigned int line = 0; line < 200; line++) + { + size_t ls = i; + size_t le; + size_t colon = (size_t) -1; + + if (ls >= len) + return 0; /* ran out before the blank line: too little evidence */ + le = ls; + while ( (le < len) && + ('\n' != buf[le]) ) + le++; + if ( (le == ls) || + ( (le == ls + 1) && ('\r' == buf[ls]) ) ) + return ( (0 != required) && (fields >= 2) ); /* blank line */ + if ( (' ' == buf[ls]) || + ('\t' == buf[ls]) ) + { + if (0 == fields) + return 0; /* a continuation cannot come first */ + i = (le < len) ? le + 1 : len; + continue; + } + for (size_t k = ls; (k < le) && (k - ls < 100); k++) + { + if (':' == buf[k]) + { + colon = k; + break; + } + if ( (buf[k] < 33) || + (buf[k] > 126) ) + return 0; /* not a field name */ + } + if ( (((size_t) -1) == colon) || + (colon == ls) ) + return 0; + fields++; + if (mbox_ieq (&buf[ls], colon - ls, "received") || + mbox_ieq (&buf[ls], colon - ls, "message-id") || + mbox_ieq (&buf[ls], colon - ls, "from") || + mbox_ieq (&buf[ls], colon - ls, "date") || + mbox_ieq (&buf[ls], colon - ls, "subject") ) + required = 1; + if (le >= len) + return 0; + i = le + 1; + } + return 0; +} + + +/** + * Report the attachment names, and the charset of the first MIME part, + * out of the first message's body. + * + * @param ec extraction context + * @param buf the buffer + * @param from offset of the first body byte + * @param to offset one past the last body byte + * @param[in,out] st plugin state, for the charset fallback + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +mbox_do_body (struct EXTRACTOR_ExtractContext *ec, + const char *buf, + size_t from, + size_t to, + struct MboxState *st) +{ + unsigned int emitted = 0; + size_t i = from; + + if (to > from + MBOX_BODY_SCAN) + to = from + MBOX_BODY_SCAN; + while ( (i < to) && + (emitted < EXTRACTOR_FORENSIC_MAX_ITEMS) ) + { + size_t ls = i; + size_t nstart; + size_t nlen; + size_t vlen; + size_t pos = ls; + int is_disp; + int is_type; + + /* advance to the start of the next line first, so that we always + make progress even when the line is not one we want */ + { + size_t le = ls; + + while ( (le < to) && + ('\n' != buf[le]) ) + le++; + i = (le < to) ? le + 1 : to; + } + is_disp = ( (ls + 20 <= to) && + mbox_ieq (&buf[ls], 20, "Content-Disposition:") ); + is_type = ( (ls + 13 <= to) && + mbox_ieq (&buf[ls], 13, "Content-Type:") ); + if ( (! is_disp) && + (! is_type) ) + continue; + if (! mbox_next_header (buf, + ls, + to, + &pos, + &nstart, + &nlen, + st->val, + sizeof (st->val), + &vlen)) + continue; + i = pos; /* skip the folded continuation lines as well */ + { + char param[MBOX_MAX_PARAM]; + size_t plen; + + plen = mbox_param (st->val, + vlen, + "filename", + param, + sizeof (param)); + if (0 == plen) + plen = mbox_param (st->val, + vlen, + "name", + param, + sizeof (param)); + if (0 != plen) + { + int seen = 0; + + for (unsigned int k = 0; k < st->nfnames; k++) + if ( (strlen (st->fnames[k]) == plen) && + (0 == memcmp (st->fnames[k], + param, + plen)) ) + { + seen = 1; + break; + } + if (! seen) + { + if (st->nfnames < MBOX_MAX_FILENAMES) + { + memcpy (st->fnames[st->nfnames], + param, + plen); + st->fnames[st->nfnames][plen] = '\0'; + st->nfnames++; + } + emitted++; + if (mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_FILENAME, + param, + plen)) + return 1; + } + } + if (is_type && + ('\0' == st->h.charset[0]) ) + { + plen = mbox_param (st->val, + vlen, + "charset", + param, + sizeof (param)); + if (0 != plen) + mbox_store (st->h.charset, + sizeof (st->h.charset), + param, + plen); + } + } + } + return 0; +} + + +/** + * Main entry method for the mbox extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_mbox_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_mbox_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + char head[MBOX_MAGIC_WINDOW]; + char *buf = NULL; + struct MboxState *st = NULL; + size_t hlen = 0; + size_t len = 0; + size_t cap; + size_t hstart; + size_t hend; + size_t body; + size_t msg1_end; + size_t pos; + size_t nstart; + size_t nlen; + size_t vlen; + uint64_t fsize; + uint64_t messages = 1; + unsigned int received = 0; + unsigned int recipients = 0; + int is_mbox; + int truncated; + + /* Magic first: for a mailbox the file must literally start with + `From ', and for a bare message the header block has to hold up to + inspection. Nothing else gets past this point. */ + { + void *data; + ssize_t ret; + + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + return; + while (hlen < sizeof (head)) + { + ret = ec->read (ec->cls, + &data, + sizeof (head) - hlen); + if (0 >= ret) + break; + if (((size_t) ret) > sizeof (head) - hlen) + return; /* the IPC layer is misbehaving */ + memcpy (&head[hlen], + data, + (size_t) ret); + hlen += (size_t) ret; + } + } + if (hlen < 32) + return; + is_mbox = (0 == memcmp (head, + "From ", + 5)); + if (! is_mbox) + { + if (! mbox_looks_like_message (head, + hlen)) + return; + } + else + { + /* the envelope line must be followed by a header block */ + size_t nl = 0; + + while ( (nl < hlen) && + ('\n' != head[nl]) ) + nl++; + if ( (nl + 1 >= hlen) || + (! mbox_looks_like_message (&head[nl + 1], + hlen - nl - 1)) ) + return; + } + + fsize = ec->get_size (ec->cls); + cap = MBOX_SCAN_CAP; + if ( (UINT64_MAX != fsize) && + (fsize < (uint64_t) cap) ) + cap = (size_t) fsize; + if (cap < hlen) + cap = hlen; + buf = malloc (cap); + if (NULL == buf) + return; + st = malloc (sizeof (struct MboxState)); + if (NULL == st) + { + free (buf); + return; + } + memset (st, + 0, + sizeof (struct MboxState)); + if (0 != ec->seek (ec->cls, + 0, + SEEK_SET)) + goto out; + while (len < cap) + { + void *data; + ssize_t ret; + + ret = ec->read (ec->cls, + &data, + cap - len); + if (0 >= ret) + break; + if (((size_t) ret) > cap - len) + break; /* the IPC layer is misbehaving */ + memcpy (&buf[len], + data, + (size_t) ret); + len += (size_t) ret; + } + if (len < 32) + goto out; + truncated = ( (UINT64_MAX == fsize) || + (fsize > (uint64_t) len) ); + + if (0 != ec->proc (ec->cls, + "mbox", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + is_mbox ? "application/mbox" : "message/rfc822", + is_mbox + ? strlen ("application/mbox") + 1 + : strlen ("message/rfc822") + 1)) + goto out; + + /* Where the first message's headers start, and where the next message + begins. A line that starts with "From " is a separator; the mboxo + escaping of a body line that would look like one is ">From ", which + is why this counts fewer than it might otherwise. */ + hstart = 0; + msg1_end = len; + if (is_mbox) + { + while ( (hstart < len) && + ('\n' != buf[hstart]) ) + hstart++; + if (hstart < len) + hstart++; + for (size_t i = hstart; i + 6 <= len; i++) + if ( ('\n' == buf[i]) && + (0 == memcmp (&buf[i + 1], + "From ", + 5)) ) + { + if (len == msg1_end) + msg1_end = i + 1; + messages++; + } + } + + /* the header block ends at the first blank line */ + hend = len; + { + size_t i = hstart; + + for (unsigned int line = 0; + (line < MBOX_MAX_HDR_LINES) && (i < len); + line++) + { + size_t le = i; + + while ( (le < len) && + ('\n' != buf[le]) ) + le++; + if ( (le == i) || + ( (le == i + 1) && ('\r' == buf[i]) ) ) + { + hend = i; + break; + } + if (le >= len) + { + hend = len; + break; + } + i = le + 1; + if (i - hstart > MBOX_MAX_HDR_BLOCK) + { + hend = i; + break; + } + } + if (hend > msg1_end) + hend = msg1_end; + body = hend; + while ( (body < msg1_end) && + ( ('\r' == buf[body]) || ('\n' == buf[body]) ) ) + body++; + } + + /* pass over the header fields */ + pos = hstart; + while (mbox_next_header (buf, + hstart, + hend, + &pos, + &nstart, + &nlen, + st->val, + sizeof (st->val), + &vlen)) + { + const char *n = &buf[nstart]; + + if (mbox_ieq (n, nlen, "received")) + { + /* The route the message actually took, most recent hop first. + Everything up to the `;' is the routing part; what follows is + the receiving MTA's timestamp, which we do not need. */ + size_t l = vlen; + + for (size_t k = 0; k < vlen; k++) + if (';' == st->val[k]) + { + l = k; + break; + } + mbox_harvest_bracketed (st->val, + vlen, + &st->ips); + if (received < EXTRACTOR_FORENSIC_MAX_ITEMS) + { + received++; + if (EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_RECEIVED_FROM, + st->val, + l)) + goto out; + } + continue; + } + if (mbox_ieq (n, nlen, "x-originating-ip") || + mbox_ieq (n, nlen, "x-forwarded-for") || + mbox_ieq (n, nlen, "x-sender-ip") ) + { + mbox_harvest_tokens (st->val, + vlen, + &st->ips); + continue; + } + if (mbox_ieq (n, nlen, "from")) + mbox_store (st->h.from, sizeof (st->h.from), st->val, vlen); + else if (mbox_ieq (n, nlen, "to")) + mbox_store (st->h.to, sizeof (st->h.to), st->val, vlen); + else if (mbox_ieq (n, nlen, "cc")) + mbox_store (st->h.cc, sizeof (st->h.cc), st->val, vlen); + else if (mbox_ieq (n, nlen, "subject")) + mbox_store (st->h.subject, sizeof (st->h.subject), st->val, vlen); + else if (mbox_ieq (n, nlen, "date")) + mbox_store (st->h.date, sizeof (st->h.date), st->val, vlen); + else if (mbox_ieq (n, nlen, "message-id")) + mbox_store (st->h.msgid, sizeof (st->h.msgid), st->val, vlen); + else if (mbox_ieq (n, nlen, "in-reply-to")) + mbox_store (st->h.inreplyto, sizeof (st->h.inreplyto), st->val, vlen); + else if (mbox_ieq (n, nlen, "references")) + mbox_store (st->h.references, sizeof (st->h.references), st->val, vlen); + else if (mbox_ieq (n, nlen, "x-mailer") || + mbox_ieq (n, nlen, "user-agent") ) + mbox_store (st->h.mailer, sizeof (st->h.mailer), st->val, vlen); + else if (mbox_ieq (n, nlen, "organization")) + mbox_store (st->h.org, sizeof (st->h.org), st->val, vlen); + else if (mbox_ieq (n, nlen, "content-type")) + mbox_store (st->h.ctype, sizeof (st->h.ctype), st->val, vlen); + else if (mbox_ieq (n, nlen, "dkim-signature")) + mbox_store (st->h.dkim, sizeof (st->h.dkim), st->val, vlen); + else if (mbox_ieq (n, nlen, "authentication-results")) + mbox_store (st->h.authres, sizeof (st->h.authres), st->val, vlen); + } + + /* From: split into the display name and the addr-spec */ + if ('\0' != st->h.from[0]) + { + size_t ns; + size_t nl2; + size_t as; + size_t al; + + mbox_split_addr (st->h.from, + strlen (st->h.from), + &ns, + &nl2, + &as, + &al); + if ( (0 != nl2) && + mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_AUTHOR_NAME, + &st->h.from[ns], + nl2) ) + goto out; + if ( (0 != al) && + EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_AUTHOR_EMAIL, + &st->h.from[as], + al) ) + goto out; + } + if ( ('\0' != st->h.to[0]) && + mbox_emit_addr_list (ec, + EXTRACTOR_METATYPE_RECIPIENT, + st->h.to, + strlen (st->h.to), + &recipients) ) + goto out; + if ( ('\0' != st->h.cc[0]) && + mbox_emit_addr_list (ec, + EXTRACTOR_METATYPE_RECIPIENT, + st->h.cc, + strlen (st->h.cc), + &recipients) ) + goto out; + if ('\0' != st->h.subject[0]) + { + if (mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_SUBJECT, + st->h.subject, + strlen (st->h.subject))) + goto out; + if (mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_TITLE, + st->h.subject, + strlen (st->h.subject))) + goto out; + } + if ('\0' != st->h.date[0]) + { + int64_t when; + + if (mbox_parse_date (st->h.date, + strlen (st->h.date), + &when)) + { + if (EXTRACTOR_forensic_emit_unix_time_ (ec, + "mbox", + EXTRACTOR_METATYPE_CREATION_DATE, + when)) + goto out; + } + else if (EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_UNKNOWN_DATE, + st->h.date, + strlen (st->h.date))) + { + goto out; + } + } + /* Message-ID: the right-hand side of the `@' is, for most mailers, + the name of the machine that composed the message -- often an + internal host name that appears nowhere else in the file. Where it + does not match any domain the sender is known to use, that is a + lead worth following. */ + if ('\0' != st->h.msgid[0]) + { + const char *v = st->h.msgid; + size_t l = strlen (v); + + if ( (l >= 2) && + ('<' == v[0]) && + ('>' == v[l - 1]) ) + { + v++; + l -= 2; + } + if (EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_MESSAGE_ID, + v, + l)) + goto out; + } + { + const char *v = NULL; + size_t l = 0; + + if ('\0' != st->h.inreplyto[0]) + { + v = st->h.inreplyto; + l = strlen (v); + } + else if ('\0' != st->h.references[0]) + { + /* the last entry of References is the message being replied to */ + size_t rl = strlen (st->h.references); + size_t lt = (size_t) -1; + + for (size_t i = rl; i > 0; i--) + if ('<' == st->h.references[i - 1]) + { + lt = i - 1; + break; + } + if (((size_t) -1) != lt) + { + v = &st->h.references[lt]; + l = rl - lt; + } + } + if (NULL != v) + { + if ( (l >= 2) && + ('<' == v[0]) && + ('>' == v[l - 1]) ) + { + v++; + l -= 2; + } + if ( (0 != l) && + EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_IN_REPLY_TO, + v, + l) ) + goto out; + } + } + for (unsigned int i = 0; i < st->ips.count; i++) + if (EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_IP_ADDRESS, + st->ips.ip[i], + strlen (st->ips.ip[i]))) + goto out; + if ( ('\0' != st->h.mailer[0]) && + mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + st->h.mailer, + strlen (st->h.mailer)) ) + goto out; + if ('\0' != st->h.dkim[0]) + { + /* the `d=' tag names the domain that signed the message */ + char dom[256]; + size_t dl; + + dl = mbox_param (st->h.dkim, + strlen (st->h.dkim), + "d", + dom, + sizeof (dom)); + if ( (0 == dl) && + (strlen (st->h.dkim) > 2) && + ('d' == st->h.dkim[0]) && + ('=' == st->h.dkim[1]) ) + { + /* mbox_param() only sees parameters after a `;'; a signature that + opens with the d= tag needs this. */ + size_t k = 2; + + while ( (k < strlen (st->h.dkim)) && + (';' != st->h.dkim[k]) ) + k++; + dl = k - 2; + if (dl >= sizeof (dom)) + dl = sizeof (dom) - 1; + memcpy (dom, + &st->h.dkim[2], + dl); + dom[dl] = '\0'; + } + if ( (0 != dl) && + EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_SIGNER, + dom, + dl) ) + goto out; + } + if ( ('\0' != st->h.authres[0]) && + EXTRACTOR_forensic_emit_ (ec, + "mbox", + EXTRACTOR_METATYPE_COMMENT, + "Authentication-Results: %s", + st->h.authres) ) + goto out; + if ( ('\0' != st->h.org[0]) && + mbox_emit_hdr (ec, + EXTRACTOR_METATYPE_ORGANIZATION, + st->h.org, + strlen (st->h.org)) ) + goto out; + if ('\0' != st->h.ctype[0]) + { + size_t l = strlen (st->h.ctype); + size_t m = 0; + char param[MBOX_MAX_PARAM]; + size_t plen; + + while ( (m < l) && + (';' != st->h.ctype[m]) ) + m++; + if ( (0 != m) && + EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_FORMAT, + st->h.ctype, + m) ) + goto out; + plen = mbox_param (st->h.ctype, + l, + "charset", + param, + sizeof (param)); + if (0 != plen) + mbox_store (st->h.charset, + sizeof (st->h.charset), + param, + plen); + plen = mbox_param (st->h.ctype, + l, + "boundary", + param, + sizeof (param)); + if ( (0 != plen) && + EXTRACTOR_forensic_emit_ (ec, + "mbox", + EXTRACTOR_METATYPE_COMMENT, + "MIME boundary: %.*s", + (int) plen, + param) ) + goto out; + } + /* Attachment names, and -- when the message itself is multipart, so + that its own Content-Type carries no charset -- the charset of the + first part. */ + if ( (body < msg1_end) && + mbox_do_body (ec, + buf, + body, + msg1_end, + st) ) + goto out; + if ( ('\0' != st->h.charset[0]) && + EXTRACTOR_forensic_emit_text_ (ec, + "mbox", + EXTRACTOR_METATYPE_CHARACTER_SET, + st->h.charset, + strlen (st->h.charset)) ) + goto out; + if (EXTRACTOR_forensic_emit_ (ec, + "mbox", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%llu", + (unsigned long long) messages)) + goto out; + if (truncated && + EXTRACTOR_forensic_emit_ (ec, + "mbox", + EXTRACTOR_METATYPE_COMMENT, + "scan truncated at 256 KiB; the message" + " count above covers only that prefix of" + " the file") ) + goto out; +out: + free (st); + free (buf); +} + + +/* end of mbox_extractor.c */ diff --git a/src/plugins/pecoff_extractor.c b/src/plugins/pecoff_extractor.c @@ -0,0 +1,2274 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/pecoff_extractor.c + * @brief plugin to support Windows PE/COFF images (.exe, .dll, .sys) + * @author Christian Grothoff + * + * What makes a PE worth looking at in a forensic pass is not the code + * but the residue the build left behind: the Rich header records every + * tool that touched the object files, the CodeView record names the + * directory the developer built in, the import table fingerprints what + * the binary can do, and the version resource carries the vendor's own + * claim about what it is. All of that lives in a handful of small + * structures, so this plugin reads headers and directories only and + * never walks the code. + * + * References: + * "PE Format", https://learn.microsoft.com/windows/win32/debug/pe-format + * "Windows Authenticode Portable Executable Signature Format" + * Rich header: no vendor documentation exists; the layout used here is + * the one every published analysis agrees on (DanS ... Rich, all + * dwords between the two masked with the key that follows `Rich'). + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * Most sections we will look at. The field is 16 bits wide, but a + * real image has a few dozen at most and the table has to fit in + * SizeOfHeaders. + */ +#define PE_MAX_SECTIONS 96 + +/** + * Largest DOS stub we will search for a Rich header. The stub is + * 64 bytes plus the Rich header itself in every real image. + */ +#define PE_MAX_DOS_STUB 4096 + +/** + * Most Rich header records we will report. + */ +#define PE_MAX_RICH_ENTRIES 32 + +/** + * Most imported modules we will look at. + */ +#define PE_MAX_IMPORT_DLLS 64 + +/** + * Most imported functions we will fold into the import hash. Beyond + * this the hash would not match what other tools compute, so we drop + * it rather than report a value that cannot be compared. + */ +#define PE_MAX_IMPORT_FUNCS 8192 + +/** + * How much of the section holding the import directory we map in one + * read. The descriptors, the lookup tables and the hint/name table + * are laid out next to each other by every linker, so one read + * normally resolves the whole import table. + */ +#define PE_IMPORT_WINDOW (512 * 1024) + +/** + * How much of the resource directory tree we read. The tree sits at + * the front of the section; the leaves point elsewhere and are read + * separately. + */ +#define PE_MAX_RESOURCE_DIR (64 * 1024) + +/** + * Largest VS_VERSIONINFO resource we will parse. + */ +#define PE_MAX_VERSION_RESOURCE (32 * 1024) + +/** + * How many bytes of a section we hash to estimate its entropy. 16 KiB + * is far more than enough to tell packed data from code. + */ +#define PE_ENTROPY_SAMPLE (16 * 1024) + +/** + * How many sections we sample for entropy. + */ +#define PE_MAX_ENTROPY_SECTIONS 8 + +/** + * Smallest section we bother computing entropy for. Below this the + * estimate is dominated by the sample size. + */ +#define PE_MIN_ENTROPY_SECTION 512 + +/** + * How much of the certificate table we scan for subject names. + */ +#define PE_MAX_CERT_SCAN (16 * 1024) + +/** + * Longest single name (module, function, section) we accept. + */ +#define PE_MAX_NAME 256 + + +/** + * One entry of the section table, in the only four fields we need. + */ +struct pe_section +{ + /** + * Address of the section in the loaded image, relative to the image + * base. + */ + uint32_t vaddr; + + /** + * Size of the section in the loaded image. + */ + uint32_t vsize; + + /** + * Offset of the section data in the file. + */ + uint32_t raw_ptr; + + /** + * Number of bytes of section data in the file. + */ + uint32_t raw_size; + + /** + * Section name, NUL-terminated. + */ + char name[9]; +}; + + +/** + * Everything the individual directory parsers need to know about the + * image. + */ +struct pe_context +{ + /** + * Extraction context we were called with. + */ + struct EXTRACTOR_ExtractContext *ec; + + /** + * Size of the file, or 0 if it could not be determined. + */ + uint64_t fsize; + + /** + * Section table. + */ + struct pe_section sections[PE_MAX_SECTIONS]; + + /** + * Number of valid entries in @e sections. + */ + unsigned int num_sections; + + /** + * Number of bytes the headers occupy; an RVA below this maps to the + * same file offset. + */ + uint32_t size_of_headers; + + /** + * True if this is a PE32+ image, in which case the import lookup + * table holds 64-bit entries. + */ + int pe32plus; +}; + + +/* ------------------------------------------------------------------ */ +/* MD5, for the import hash */ +/* ------------------------------------------------------------------ */ + +/* + * The import hash is only useful if it equals what every other tool + * computes for the same file, which pins it to MD5. libextractor does + * not link a crypto library and pulling one in for 60 lines of + * arithmetic would be a poor trade, so RFC 1321 is implemented here. + * It is never used for anything security relevant. + */ + +/** + * Running MD5 state. + */ +struct md5_context +{ + /** + * Chaining value. + */ + uint32_t state[4]; + + /** + * Number of bytes fed in so far. + */ + uint64_t count; + + /** + * Partial block not yet compressed. + */ + unsigned char block[64]; + + /** + * Number of bytes used in @e block. + */ + size_t used; +}; + + +/** + * The per-round additive constants, floor(2^32 * |sin(i + 1)|). + */ +static const uint32_t md5_k[64] = { + 0xd76aa478U, 0xe8c7b756U, 0x242070dbU, 0xc1bdceeeU, + 0xf57c0fafU, 0x4787c62aU, 0xa8304613U, 0xfd469501U, + 0x698098d8U, 0x8b44f7afU, 0xffff5bb1U, 0x895cd7beU, + 0x6b901122U, 0xfd987193U, 0xa679438eU, 0x49b40821U, + 0xf61e2562U, 0xc040b340U, 0x265e5a51U, 0xe9b6c7aaU, + 0xd62f105dU, 0x02441453U, 0xd8a1e681U, 0xe7d3fbc8U, + 0x21e1cde6U, 0xc33707d6U, 0xf4d50d87U, 0x455a14edU, + 0xa9e3e905U, 0xfcefa3f8U, 0x676f02d9U, 0x8d2a4c8aU, + 0xfffa3942U, 0x8771f681U, 0x6d9d6122U, 0xfde5380cU, + 0xa4beea44U, 0x4bdecfa9U, 0xf6bb4b60U, 0xbebfbc70U, + 0x289b7ec6U, 0xeaa127faU, 0xd4ef3085U, 0x04881d05U, + 0xd9d4d039U, 0xe6db99e5U, 0x1fa27cf8U, 0xc4ac5665U, + 0xf4292244U, 0x432aff97U, 0xab9423a7U, 0xfc93a039U, + 0x655b59c3U, 0x8f0ccc92U, 0xffeff47dU, 0x85845dd1U, + 0x6fa87e4fU, 0xfe2ce6e0U, 0xa3014314U, 0x4e0811a1U, + 0xf7537e82U, 0xbd3af235U, 0x2ad7d2bbU, 0xeb86d391U +}; + + +/** + * The per-round rotation amounts. + */ +static const unsigned char md5_r[64] = { + 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, + 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, + 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, + 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21 +}; + + +/** + * Rotate @a v left by @a n bits. + * + * @param v value to rotate + * @param n number of bits, 1 to 31 + * @return the rotated value + */ +static uint32_t +md5_rotl (uint32_t v, + unsigned int n) +{ + return (v << n) | (v >> (32 - n)); +} + + +/** + * Compress one 64-byte block into @a ctx. + * + * @param[in,out] ctx hash state + * @param p the block + */ +static void +md5_compress (struct md5_context *ctx, + const unsigned char *p) +{ + uint32_t m[16]; + uint32_t a = ctx->state[0]; + uint32_t b = ctx->state[1]; + uint32_t c = ctx->state[2]; + uint32_t d = ctx->state[3]; + + for (unsigned int i = 0; i < 16; i++) + m[i] = EXTRACTOR_forensic_le32_ (&p[4 * i]); + for (unsigned int i = 0; i < 64; i++) + { + uint32_t f; + unsigned int g; + uint32_t tmp; + + if (i < 16) + { + f = (b & c) | ((~b) & d); + g = i; + } + else if (i < 32) + { + f = (d & b) | ((~d) & c); + g = (5 * i + 1) % 16; + } + else if (i < 48) + { + f = b ^ c ^ d; + g = (3 * i + 5) % 16; + } + else + { + f = c ^ (b | (~d)); + g = (7 * i) % 16; + } + tmp = d; + d = c; + c = b; + b = b + md5_rotl (a + f + md5_k[i] + m[g], + md5_r[i]); + a = tmp; + } + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; +} + + +/** + * Start a new MD5 computation. + * + * @param[out] ctx hash state to initialise + */ +static void +md5_init (struct md5_context *ctx) +{ + ctx->state[0] = 0x67452301U; + ctx->state[1] = 0xefcdab89U; + ctx->state[2] = 0x98badcfeU; + ctx->state[3] = 0x10325476U; + ctx->count = 0; + ctx->used = 0; +} + + +/** + * Feed @a len bytes into the hash. + * + * @param[in,out] ctx hash state + * @param data the bytes + * @param len number of bytes in @a data + */ +static void +md5_update (struct md5_context *ctx, + const void *data, + size_t len) +{ + const unsigned char *p = data; + + ctx->count += len; + while (0 != len) + { + size_t take = sizeof (ctx->block) - ctx->used; + + if (take > len) + take = len; + memcpy (&ctx->block[ctx->used], + p, + take); + ctx->used += take; + p += take; + len -= take; + if (sizeof (ctx->block) == ctx->used) + { + md5_compress (ctx, + ctx->block); + ctx->used = 0; + } + } +} + + +/** + * Finish the hash and write the 16-byte digest. + * + * @param[in,out] ctx hash state + * @param[out] digest where to store the result + */ +static void +md5_final (struct md5_context *ctx, + unsigned char *digest) +{ + uint64_t bits = ctx->count * 8; + unsigned char tail[8]; + static const unsigned char pad = 0x80; + static const unsigned char zero = 0x00; + + for (unsigned int i = 0; i < 8; i++) + tail[i] = (unsigned char) ((bits >> (8 * i)) & 0xFF); + md5_update (ctx, + &pad, + 1); + while (56 != ctx->used) + md5_update (ctx, + &zero, + 1); + /* md5_update() has bumped ctx->count past the real length, but the + length we are about to append was captured before the padding. */ + md5_update (ctx, + tail, + 8); + for (unsigned int i = 0; i < 4; i++) + for (unsigned int j = 0; j < 4; j++) + digest[4 * i + j] = (unsigned char) ((ctx->state[i] >> (8 * j)) & 0xFF); +} + + +/* ------------------------------------------------------------------ */ +/* address translation */ +/* ------------------------------------------------------------------ */ + + +/** + * Translate a relative virtual address into a file offset. + * + * Every directory in a PE is addressed by RVA, so this is the one + * place where a forged section table turns into an out-of-range read. + * The result is checked against the file size and the amount of data + * the section actually has in the file. + * + * @param ctx image being parsed + * @param rva address relative to the image base + * @param[out] offset where to store the file offset + * @param[out] avail where to store how many bytes are readable there + * @return 1 on success, 0 if @a rva is not backed by file data + */ +static int +pe_rva_to_offset (const struct pe_context *ctx, + uint32_t rva, + uint64_t *offset, + uint64_t *avail) +{ + for (unsigned int i = 0; i < ctx->num_sections; i++) + { + const struct pe_section *s = &ctx->sections[i]; + uint32_t span = s->vsize; + uint32_t delta; + + if (span < s->raw_size) + span = s->raw_size; /* the loader maps whatever is in the file */ + if (0 == span) + continue; + if ( (rva < s->vaddr) || + (rva - s->vaddr >= span) ) + continue; + delta = rva - s->vaddr; + if (delta >= s->raw_size) + return 0; /* in the BSS part of the section; nothing in the file */ + if (((uint64_t) s->raw_ptr) + delta >= ctx->fsize) + return 0; + *offset = ((uint64_t) s->raw_ptr) + delta; + *avail = s->raw_size - delta; + if (*avail > ctx->fsize - *offset) + *avail = ctx->fsize - *offset; + return 1; + } + /* Addresses below SizeOfHeaders are mapped one-to-one; a few linkers + put small directories there. */ + if ( (0 != ctx->size_of_headers) && + (rva < ctx->size_of_headers) && + (rva < ctx->fsize) ) + { + *offset = rva; + *avail = ctx->size_of_headers - rva; + if (*avail > ctx->fsize - *offset) + *avail = ctx->fsize - *offset; + return 1; + } + return 0; +} + + +/** + * Read @a len bytes from the address @a rva. + * + * @param ctx image being parsed + * @param rva address relative to the image base + * @param[out] buf where to store the data + * @param len number of bytes to read + * @return 1 on success, 0 if the address is not backed by that much + * file data + */ +static int +pe_read_rva (struct pe_context *ctx, + uint32_t rva, + void *buf, + size_t len) +{ + uint64_t off; + uint64_t avail; + + if (! pe_rva_to_offset (ctx, + rva, + &off, + &avail)) + return 0; + if (avail < len) + return 0; + return EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) off, + buf, + len); +} + + +/* ------------------------------------------------------------------ */ +/* small formatting helpers */ +/* ------------------------------------------------------------------ */ + + +/** + * Append @a what to the comma-separated list in @a buf. + * + * @param buf destination buffer + * @param size number of bytes in @a buf + * @param what text to append + */ +static void +pe_append (char *buf, + size_t size, + const char *what) +{ + size_t used = strlen (buf); + size_t need = strlen (what); + + if (0 != used) + { + if (used + 2 >= size) + return; + buf[used++] = ','; + buf[used++] = ' '; + buf[used] = '\0'; + } + if (used + need + 1 > size) + return; + memcpy (&buf[used], + what, + need + 1); +} + + +/** + * Name of a COFF machine type. + * + * @param machine the raw field + * @return a static string, NULL if the value is not one we know + */ +static const char * +pe_machine_name (uint16_t machine) +{ + switch (machine) + { + case 0x014c: + return "i386"; + case 0x0162: + return "MIPS R3000"; + case 0x0166: + return "MIPS little-endian"; + case 0x0169: + return "MIPS WCE v2"; + case 0x01a2: + return "Hitachi SH3"; + case 0x01a3: + return "Hitachi SH3 DSP"; + case 0x01a6: + return "Hitachi SH4"; + case 0x01a8: + return "Hitachi SH5"; + case 0x01c0: + return "ARM"; + case 0x01c2: + return "ARM Thumb"; + case 0x01c4: + return "ARM Thumb-2"; + case 0x01d3: + return "Matsushita AM33"; + case 0x01f0: + return "PowerPC"; + case 0x01f1: + return "PowerPC with FPU"; + case 0x0200: + return "Itanium"; + case 0x0266: + return "MIPS16"; + case 0x0284: + return "Alpha 64"; + case 0x0366: + return "MIPS with FPU"; + case 0x0466: + return "MIPS16 with FPU"; + case 0x0ebc: + return "EFI byte code"; + case 0x5032: + return "RISC-V 32"; + case 0x5064: + return "RISC-V 64"; + case 0x5128: + return "RISC-V 128"; + case 0x6232: + return "LoongArch 32"; + case 0x6264: + return "LoongArch 64"; + case 0x8664: + return "x86-64"; + case 0x9041: + return "Mitsubishi M32R"; + case 0xaa64: + return "ARM64"; + case 0xa641: + return "ARM64EC"; + case 0xa64e: + return "ARM64X"; + default: + return NULL; + } +} + + +/** + * Name of a Windows subsystem. + * + * @param subsystem the raw field + * @return a static string, NULL if the value is not one we know + */ +static const char * +pe_subsystem_name (uint16_t subsystem) +{ + switch (subsystem) + { + case 1: + return "native"; + case 2: + return "Windows GUI"; + case 3: + return "Windows console"; + case 5: + return "OS/2 console"; + case 7: + return "POSIX console"; + case 8: + return "native Windows 9x driver"; + case 9: + return "Windows CE GUI"; + case 10: + return "EFI application"; + case 11: + return "EFI boot service driver"; + case 12: + return "EFI runtime driver"; + case 13: + return "EFI ROM"; + case 14: + return "Xbox"; + case 16: + return "Windows boot application"; + default: + return NULL; + } +} + + +/* ------------------------------------------------------------------ */ +/* Rich header */ +/* ------------------------------------------------------------------ */ + + +/** + * Decode the Rich header sitting between the DOS stub and the PE + * signature. + * + * Microsoft's linker records one (product id, build number, use count) + * triple for every tool that contributed to the image, masks the whole + * run with a checksum and terminates it with the literal `Rich' plus + * that mask. Nothing but MSVC writes it, and the build numbers pin + * down the exact toolchain installation, which is why it is the + * standard clustering feature for related binaries. + * + * @param ec extraction context + * @param stub bytes between the end of the DOS header and the PE header + * @param len number of bytes in @a stub + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_rich (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *stub, + size_t len) +{ + size_t rich = 0; + size_t dans = 0; + uint32_t key = 0; + int found = 0; + unsigned int tries = 0; + + if (len < 16) + return 0; + for (size_t i = 0; (i + 8 <= len) && (tries < 8); i++) + { + size_t p; + + if ( ('R' != stub[i]) || + ('i' != stub[i + 1]) || + ('c' != stub[i + 2]) || + ('h' != stub[i + 3]) ) + continue; + tries++; + key = EXTRACTOR_forensic_le32_ (&stub[i + 4]); + p = i; + while (p >= 4) + { + p -= 4; + if (0x536E6144U == (EXTRACTOR_forensic_le32_ (&stub[p]) ^ key)) + { + rich = i; + dans = p; + found = 1; + break; + } + } + if (found) + break; + } + if (! found) + return 0; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_BUILD_ID, + "rich:%08x", + (unsigned int) key)) + return 1; + /* DanS is followed by three masked zero dwords, then the triples. */ + for (size_t q = dans + 16, n = 0; + (q + 8 <= rich) && (n < PE_MAX_RICH_ENTRIES); + q += 8, n++) + { + uint32_t comp = EXTRACTOR_forensic_le32_ (&stub[q]) ^ key; + uint32_t uses = EXTRACTOR_forensic_le32_ (&stub[q + 4]) ^ key; + + if ( (0 == comp) && + (0 == uses) ) + continue; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_TOOLCHAIN, + "Rich: prodID 0x%04x, build %u, count %u", + (unsigned int) (comp >> 16), + (unsigned int) (comp & 0xFFFF), + (unsigned int) uses)) + return 1; + } + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* debug directory */ +/* ------------------------------------------------------------------ */ + + +/** + * Report the CodeView record a debug directory entry points at. + * + * The RSDS record carries the absolute path of the PDB as it existed + * on the build machine, which routinely leaks the developer's user + * name and source tree layout, plus the GUID and age that identify the + * build on a symbol server. + * + * @param ctx image being parsed + * @param rva address of the record + * @param size number of bytes in the record + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_codeview (struct pe_context *ctx, + uint32_t rva, + uint32_t size) +{ + unsigned char buf[PE_MAX_NAME + 32]; + size_t want = size; + + if (size < 24) + return 0; + if (want > sizeof (buf)) + want = sizeof (buf); + if (! pe_read_rva (ctx, + rva, + buf, + want)) + return 0; + if ( ('R' == buf[0]) && + ('S' == buf[1]) && + ('D' == buf[2]) && + ('S' == buf[3]) ) + { + uint32_t age = EXTRACTOR_forensic_le32_ (&buf[20]); + + /* The symbol-server spelling: the GUID with its first three fields + printed as numbers, then the age. That is the string a lookup + against a symbol store is keyed on. */ + if (0 != + EXTRACTOR_forensic_emit_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_BUILD_ID, + "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X%X", + (unsigned int) EXTRACTOR_forensic_le32_ ( + &buf[4]), + (unsigned int) EXTRACTOR_forensic_le16_ ( + &buf[8]), + (unsigned int) EXTRACTOR_forensic_le16_ ( + &buf[10]), + buf[12], buf[13], buf[14], buf[15], + buf[16], buf[17], buf[18], buf[19], + (unsigned int) age)) + return 1; + return EXTRACTOR_forensic_emit_text_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_DEBUG_PATH, + (const char *) &buf[24], + want - 24); + } + if ( ('N' == buf[0]) && + ('B' == buf[1]) && + ('1' == buf[2]) && + ('0' == buf[3]) ) + { + if (0 != + EXTRACTOR_forensic_emit_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_BUILD_ID, + "NB10:%08X%X", + (unsigned int) EXTRACTOR_forensic_le32_ ( + &buf[12]), + (unsigned int) EXTRACTOR_forensic_le32_ ( + &buf[16]))) + return 1; + return EXTRACTOR_forensic_emit_text_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_DEBUG_PATH, + (const char *) &buf[16], + want - 16); + } + return 0; +} + + +/** + * Walk the debug directory. + * + * @param ctx image being parsed + * @param rva address of the directory + * @param size number of bytes in the directory + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_debug (struct pe_context *ctx, + uint32_t rva, + uint32_t size) +{ + unsigned int count = size / 28; + + if (count > 16) + count = 16; + for (unsigned int i = 0; i < count; i++) + { + unsigned char ent[28]; + + if (! pe_read_rva (ctx, + rva + i * 28, + ent, + sizeof (ent))) + return 0; + if (2 != EXTRACTOR_forensic_le32_ (&ent[12])) + continue; /* not IMAGE_DEBUG_TYPE_CODEVIEW */ + if (0 != + pe_parse_codeview (ctx, + EXTRACTOR_forensic_le32_ (&ent[20]), + EXTRACTOR_forensic_le32_ (&ent[16]))) + return 1; + break; /* one CodeView record is all any image has */ + } + return 0; +} + + +/* ------------------------------------------------------------------ */ +/* version resource */ +/* ------------------------------------------------------------------ */ + + +/** + * Round @a v up to the next multiple of four. + * + * Every block in a VS_VERSIONINFO starts on a 32-bit boundary measured + * from the start of the resource. + * + * @param v the value + * @return @a v rounded up, or 0 on overflow + */ +static size_t +pe_align4 (size_t v) +{ + if (v > SIZE_MAX - 3) + return 0; + return (v + 3) & ~((size_t) 3); +} + + +/** + * Number of bytes the UTF-16LE key at @a p occupies, terminator + * included. + * + * @param p start of the key + * @param avail number of bytes readable at @a p + * @return the length, 0 if the key is not terminated within @a avail + */ +static size_t +pe_key_bytes (const unsigned char *p, + size_t avail) +{ + for (size_t i = 0; i + 1 < avail; i += 2) + if (0 == EXTRACTOR_forensic_le16_ (&p[i])) + return i + 2; + return 0; +} + + +/** + * Does the UTF-16LE key at @a p spell @a ascii? + * + * @param p start of the key + * @param avail number of bytes readable at @a p + * @param ascii the name to compare against + * @return 1 if they match, 0 if not + */ +static int +pe_key_is (const unsigned char *p, + size_t avail, + const char *ascii) +{ + size_t i = 0; + + while ('\0' != ascii[i]) + { + if (2 * i + 1 >= avail) + return 0; + if (EXTRACTOR_forensic_le16_ (&p[2 * i]) != (uint16_t) (unsigned char) + ascii[i]) + return 0; + i++; + } + if (2 * i + 1 >= avail) + return 0; + return (0 == EXTRACTOR_forensic_le16_ (&p[2 * i])); +} + + +/** + * Mapping from a VERSIONINFO string name to the meta data type we + * report it as. + */ +struct pe_version_field +{ + /** + * Name as it appears in the StringTable. + */ + const char *key; + + /** + * Type to report the value under. + */ + enum EXTRACTOR_MetaType type; +}; + + +/** + * The StringFileInfo names worth reporting. Everything else in there + * is vendor-specific noise. + */ +static const struct pe_version_field pe_version_fields[] = { + { "CompanyName", EXTRACTOR_METATYPE_COMPANY }, + { "FileDescription", EXTRACTOR_METATYPE_DESCRIPTION }, + { "ProductName", EXTRACTOR_METATYPE_PRODUCT_NAME }, + { "ProductVersion", EXTRACTOR_METATYPE_PRODUCT_VERSION }, + { "FileVersion", EXTRACTOR_METATYPE_SOFTWARE_VERSION }, + { "OriginalFilename", EXTRACTOR_METATYPE_FILENAME }, + { "LegalCopyright", EXTRACTOR_METATYPE_COPYRIGHT }, + { "LegalTrademarks", EXTRACTOR_METATYPE_LICENSE }, + { "InternalName", EXTRACTOR_METATYPE_TITLE }, + { "Comments", EXTRACTOR_METATYPE_COMMENT }, + { NULL, EXTRACTOR_METATYPE_RESERVED } +}; + + +/** + * Report one String entry of a StringTable. + * + * @param ec extraction context + * @param b start of the String structure + * @param len number of bytes readable at @a b + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_version_string (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *b, + size_t len) +{ + size_t keylen; + size_t value; + + if (len < 8) + return 0; + keylen = pe_key_bytes (&b[6], + len - 6); + if (0 == keylen) + return 0; + value = pe_align4 (6 + keylen); + if ( (0 == value) || + (value >= len) ) + return 0; + for (unsigned int i = 0; NULL != pe_version_fields[i].key; i++) + { + if (! pe_key_is (&b[6], + len - 6, + pe_version_fields[i].key)) + continue; + /* The declared value length is in characters for text values and in + bytes for binary ones, and writers disagree about which applies. + The enclosing block length is unambiguous, so hand the rest of + the block to a converter that stops at the NUL. */ + return EXTRACTOR_forensic_emit_utf16le_ (ec, + "pecoff", + pe_version_fields[i].type, + &b[value], + len - value); + } + return 0; +} + + +/** + * Walk one StringTable, which holds the strings for a single language + * and code page. + * + * @param ec extraction context + * @param b start of the StringTable structure + * @param len number of bytes readable at @a b + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_string_table (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *b, + size_t len) +{ + size_t keylen; + size_t pos; + + if (len < 8) + return 0; + keylen = pe_key_bytes (&b[6], + len - 6); + if (0 == keylen) + return 0; + pos = pe_align4 (6 + keylen); + for (unsigned int n = 0; + (0 != pos) && (pos + 6 <= len) && (n < EXTRACTOR_FORENSIC_MAX_ITEMS); + n++) + { + size_t slen = EXTRACTOR_forensic_le16_ (&b[pos]); + size_t have; + + if (slen < 8) + break; /* no forward progress possible */ + have = len - pos; + if (have > slen) + have = slen; + if (0 != + pe_parse_version_string (ec, + &b[pos], + have)) + return 1; + pos = pe_align4 (pos + slen); + } + return 0; +} + + +/** + * Walk a StringFileInfo block and its per-language string tables. + * + * @param ec extraction context + * @param b start of the StringFileInfo structure + * @param len number of bytes readable at @a b + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_string_file_info (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *b, + size_t len) +{ + size_t keylen; + size_t pos; + + if (len < 8) + return 0; + keylen = pe_key_bytes (&b[6], + len - 6); + if (0 == keylen) + return 0; + pos = pe_align4 (6 + keylen); + for (unsigned int n = 0; + (0 != pos) && (pos + 6 <= len) && (n < 8); + n++) + { + size_t tlen = EXTRACTOR_forensic_le16_ (&b[pos]); + size_t have; + + if (tlen < 8) + break; + have = len - pos; + if (have > tlen) + have = tlen; + if (0 != + pe_parse_string_table (ec, + &b[pos], + have)) + return 1; + pos = pe_align4 (pos + tlen); + } + return 0; +} + + +/** + * Parse a VS_VERSIONINFO resource. + * + * @param ec extraction context + * @param b start of the resource + * @param len number of bytes readable at @a b + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_version_info (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *b, + size_t len) +{ + size_t total; + size_t vallen; + size_t keylen; + size_t pos; + + if (len < 8) + return 0; + total = EXTRACTOR_forensic_le16_ (&b[0]); + if ( (total < 8) || + (total > len) ) + total = len; + vallen = EXTRACTOR_forensic_le16_ (&b[2]); + keylen = pe_key_bytes (&b[6], + total - 6); + if (0 == keylen) + return 0; + if (! pe_key_is (&b[6], + total - 6, + "VS_VERSION_INFO")) + return 0; + pos = pe_align4 (6 + keylen); + if ( (0 == pos) || + (pos >= total) ) + return 0; + if ( (52 <= vallen) && + (pos + 52 <= total) && + (0xFEEF04BDU == EXTRACTOR_forensic_le32_ (&b[pos])) ) + { + uint32_t fv_ms = EXTRACTOR_forensic_le32_ (&b[pos + 8]); + uint32_t fv_ls = EXTRACTOR_forensic_le32_ (&b[pos + 12]); + + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u.%u.%u.%u", + (unsigned int) (fv_ms >> 16), + (unsigned int) (fv_ms & 0xFFFF), + (unsigned int) (fv_ls >> 16), + (unsigned int) (fv_ls & 0xFFFF))) + return 1; + } + pos = pe_align4 (pos + vallen); + for (unsigned int n = 0; + (0 != pos) && (pos + 6 <= total) && (n < 8); + n++) + { + size_t clen = EXTRACTOR_forensic_le16_ (&b[pos]); + size_t have; + + if (clen < 8) + break; + have = total - pos; + if (have > clen) + have = clen; + if (pe_key_is (&b[pos + 6], + have - 6, + "StringFileInfo")) + { + if (0 != + pe_parse_string_file_info (ec, + &b[pos], + have)) + return 1; + } + pos = pe_align4 (pos + clen); + } + return 0; +} + + +/** + * Find the RT_VERSION leaf in the resource directory tree and parse + * the resource it points at. + * + * The tree is three levels deep (type, name, language); we want the + * first leaf under type 16 and do not care which language it is. + * + * @param ctx image being parsed + * @param rva address of the resource directory + * @param size number of bytes in the resource directory + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_resources (struct pe_context *ctx, + uint32_t rva, + uint32_t size) +{ + unsigned char *tree; + unsigned char *res; + size_t tlen = size; + uint32_t node = 0; + uint32_t leaf = 0; + uint32_t data_rva; + uint32_t data_size; + int found = 0; + int ret = 0; + + if (size < 16) + return 0; + if (tlen > PE_MAX_RESOURCE_DIR) + tlen = PE_MAX_RESOURCE_DIR; + if (NULL == (tree = malloc (tlen))) + return 0; + if (! pe_read_rva (ctx, + rva, + tree, + tlen)) + { + free (tree); + return 0; + } + /* level 1: resource types */ + { + unsigned int named = EXTRACTOR_forensic_le16_ (&tree[12]); + unsigned int ids = EXTRACTOR_forensic_le16_ (&tree[14]); + size_t base; + + if (named > 1024) + named = 1024; + if (ids > 1024) + ids = 1024; + base = 16 + ((size_t) named) * 8; + for (unsigned int i = 0; i < ids; i++) + { + size_t at = base + ((size_t) i) * 8; + + if (at + 8 > tlen) + break; + if (16 != EXTRACTOR_forensic_le32_ (&tree[at])) + continue; /* not RT_VERSION */ + node = EXTRACTOR_forensic_le32_ (&tree[at + 4]); + if (0 == (node & 0x80000000U)) + { + free (tree); + return 0; /* a type entry must point at a subdirectory */ + } + node &= 0x7FFFFFFFU; + found = 1; + break; + } + } + if (! found) + { + free (tree); + return 0; + } + /* levels 2 and 3: take the first entry each time */ + for (unsigned int level = 0; level < 2; level++) + { + unsigned int named; + unsigned int ids; + + if (((size_t) node) + 16 > tlen) + { + free (tree); + return 0; + } + named = EXTRACTOR_forensic_le16_ (&tree[node + 12]); + ids = EXTRACTOR_forensic_le16_ (&tree[node + 14]); + if (0 == named + ids) + { + free (tree); + return 0; + } + if (((size_t) node) + 24 > tlen) + { + free (tree); + return 0; + } + leaf = EXTRACTOR_forensic_le32_ (&tree[node + 20]); + if (0 == (leaf & 0x80000000U)) + break; /* a data entry: we are at the bottom */ + node = leaf & 0x7FFFFFFFU; + leaf = 0; + } + if ( (0 == leaf) || + (0 != (leaf & 0x80000000U)) || + (((size_t) leaf) + 16 > tlen) ) + { + free (tree); + return 0; + } + data_rva = EXTRACTOR_forensic_le32_ (&tree[leaf]); + data_size = EXTRACTOR_forensic_le32_ (&tree[leaf + 4]); + free (tree); + if (data_size < 8) + return 0; + if (data_size > PE_MAX_VERSION_RESOURCE) + data_size = PE_MAX_VERSION_RESOURCE; + if (NULL == (res = malloc (data_size))) + return 0; + if (pe_read_rva (ctx, + data_rva, + res, + data_size)) + ret = pe_parse_version_info (ctx->ec, + res, + data_size); + free (res); + return ret; +} + + +/* ------------------------------------------------------------------ */ +/* imports */ +/* ------------------------------------------------------------------ */ + + +/** + * A window of file data mapped by relative virtual address. + */ +struct pe_window +{ + /** + * The bytes, or NULL if nothing is mapped. + */ + unsigned char *buf; + + /** + * Address @e buf starts at. + */ + uint32_t base; + + /** + * Number of bytes in @e buf. + */ + size_t len; +}; + + +/** + * Point at @a len bytes at @a rva inside @a w. + * + * @param w the window + * @param rva address wanted + * @param len number of bytes wanted + * @return pointer into the window, NULL if the range is not mapped + */ +static const unsigned char * +pe_window_at (const struct pe_window *w, + uint32_t rva, + size_t len) +{ + uint32_t delta; + + if ( (NULL == w->buf) || + (rva < w->base) ) + return NULL; + delta = rva - w->base; + if ( (delta > w->len) || + (w->len - delta < len) ) + return NULL; + return &w->buf[delta]; +} + + +/** + * Copy a NUL-terminated ASCII name out of @a w. + * + * @param w the window + * @param rva address of the name + * @param[out] out where to store the name + * @param size number of bytes in @a out + * @param lower 1 to fold the name to lower case, as the import hash + * requires, 0 to keep it as the linker wrote it + * @return 1 on success, 0 if the name is not mapped or not terminated + */ +static int +pe_window_name (const struct pe_window *w, + uint32_t rva, + char *out, + size_t size, + int lower) +{ + const unsigned char *p = pe_window_at (w, + rva, + 1); + size_t avail; + size_t i; + + if (NULL == p) + return 0; + avail = w->len - (size_t) (rva - w->base); + if (avail > size - 1) + avail = size - 1; + for (i = 0; i < avail; i++) + { + unsigned char c = p[i]; + + if ('\0' == c) + break; + if ( (c < 0x20) || + (c > 0x7E) ) + return 0; /* module and function names are ASCII */ + if ( (lower) && + ('A' <= c) && + ('Z' >= c) ) + c = (unsigned char) (c - 'A' + 'a'); + out[i] = (char) c; + } + if (i == avail) + return 0; /* not terminated inside the window */ + out[i] = '\0'; + return (0 != i); +} + + +/** + * Compute the import hash over the import table. + * + * The hash is MD5 over `module.function' pairs, lower-cased, joined by + * commas, in the order the linker wrote them; the module's `.dll', + * `.ocx' or `.sys' suffix is dropped. That is the definition every + * other tool uses, so any deviation makes the value useless for + * comparison -- which is why this bails out instead of guessing + * whenever part of the table is out of reach. + * + * @param ctx image being parsed + * @param w window covering the import table + * @param rva address of the import descriptor array + * @param[out] hash where to store the 16-byte digest + * @return 1 on success, 0 if no hash could be computed + */ +static int +pe_import_hash (struct pe_context *ctx, + const struct pe_window *w, + uint32_t rva, + unsigned char *hash) +{ + struct md5_context md5; + size_t thunk_size = ctx->pe32plus ? 8 : 4; + unsigned int total = 0; + int any = 0; + + md5_init (&md5); + for (unsigned int d = 0; d < PE_MAX_IMPORT_DLLS; d++) + { + const unsigned char *desc = pe_window_at (w, + rva + d * 20, + 20); + char lib[PE_MAX_NAME]; + size_t liblen; + uint32_t thunks; + + if (NULL == desc) + return 0; + if ( (0 == EXTRACTOR_forensic_le32_ (&desc[0])) && + (0 == EXTRACTOR_forensic_le32_ (&desc[12])) && + (0 == EXTRACTOR_forensic_le32_ (&desc[16])) ) + break; /* the all-zero terminator */ + if (! pe_window_name (w, + EXTRACTOR_forensic_le32_ (&desc[12]), + lib, + sizeof (lib), + 1)) + return 0; + liblen = strlen (lib); + if ( (liblen > 4) && + ('.' == lib[liblen - 4]) && + ( (0 == strcmp (&lib[liblen - 3], "dll")) || + (0 == strcmp (&lib[liblen - 3], "ocx")) || + (0 == strcmp (&lib[liblen - 3], "sys")) ) ) + { + liblen -= 4; + lib[liblen] = '\0'; + } + thunks = EXTRACTOR_forensic_le32_ (&desc[0]); + if (0 == thunks) + thunks = EXTRACTOR_forensic_le32_ (&desc[16]); + if (0 == thunks) + return 0; + for (unsigned int t = 0;; t++) + { + const unsigned char *te; + uint64_t v; + char func[PE_MAX_NAME]; + + if (total >= PE_MAX_IMPORT_FUNCS) + return 0; /* would no longer match anyone else's value */ + te = pe_window_at (w, + thunks + t * (uint32_t) thunk_size, + thunk_size); + if (NULL == te) + return 0; + v = ctx->pe32plus + ? EXTRACTOR_forensic_le64_ (te) + : (uint64_t) EXTRACTOR_forensic_le32_ (te); + if (0 == v) + break; + if (0 != (v & (ctx->pe32plus + ? 0x8000000000000000ULL + : 0x80000000ULL))) + { + /* Imported by ordinal. Tools that know the export tables of a + few system libraries substitute the real name here; without + those tables the documented fallback spelling is used. */ + snprintf (func, + sizeof (func), + "ord%u", + (unsigned int) (v & 0xFFFFU)); + } + else + { + uint32_t name_rva = (uint32_t) (v & 0x7FFFFFFFU); + + if (name_rva > UINT32_MAX - 2) + return 0; + if (! pe_window_name (w, + name_rva + 2, + func, + sizeof (func), + 1)) + return 0; + } + if (any) + md5_update (&md5, + ",", + 1); + md5_update (&md5, + lib, + liblen); + md5_update (&md5, + ".", + 1); + md5_update (&md5, + func, + strlen (func)); + any = 1; + total++; + } + } + if (! any) + return 0; + md5_final (&md5, + hash); + return 1; +} + + +/** + * Report the imported modules and the import hash. + * + * @param ctx image being parsed + * @param rva address of the import descriptor array + * @param size number of bytes in the import directory + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_imports (struct pe_context *ctx, + uint32_t rva, + uint32_t size) +{ + struct pe_window w = { NULL, 0, 0 }; + unsigned char hash[16]; + unsigned int count = 0; + int ret = 0; + + if (size < 20) + return 0; + /* Map the section the descriptors live in: the lookup tables and the + hint/name table are in there too, so one read covers the lot. */ + for (unsigned int i = 0; i < ctx->num_sections; i++) + { + const struct pe_section *s = &ctx->sections[i]; + size_t len = s->raw_size; + + if ( (0 == s->raw_size) || + (rva < s->vaddr) || + (rva - s->vaddr >= s->raw_size) ) + continue; + if (((uint64_t) s->raw_ptr) >= ctx->fsize) + break; + if (len > ctx->fsize - s->raw_ptr) + len = (size_t) (ctx->fsize - s->raw_ptr); + if (len > PE_IMPORT_WINDOW) + len = PE_IMPORT_WINDOW; + if (NULL == (w.buf = malloc (len))) + break; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) s->raw_ptr, + w.buf, + len)) + { + free (w.buf); + w.buf = NULL; + break; + } + w.base = s->vaddr; + w.len = len; + break; + } + if (NULL == w.buf) + return 0; + for (unsigned int d = 0; + (d < PE_MAX_IMPORT_DLLS) && (count < EXTRACTOR_FORENSIC_MAX_ITEMS); + d++) + { + const unsigned char *desc = pe_window_at (&w, + rva + d * 20, + 20); + char lib[PE_MAX_NAME]; + + if (NULL == desc) + break; + if ( (0 == EXTRACTOR_forensic_le32_ (&desc[0])) && + (0 == EXTRACTOR_forensic_le32_ (&desc[12])) && + (0 == EXTRACTOR_forensic_le32_ (&desc[16])) ) + break; + if (! pe_window_name (&w, + EXTRACTOR_forensic_le32_ (&desc[12]), + lib, + sizeof (lib), + 0)) + break; + count++; + if (0 != + EXTRACTOR_forensic_emit_text_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_LIBRARY_DEPENDENCY, + lib, + strlen (lib))) + { + ret = 1; + goto cleanup; + } + } + if (pe_import_hash (ctx, + &w, + rva, + hash)) + ret = EXTRACTOR_forensic_emit_hex_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_IMPORT_HASH, + hash, + sizeof (hash)); +cleanup: + free (w.buf); + return ret; +} + + +/* ------------------------------------------------------------------ */ +/* exports, certificates, entropy */ +/* ------------------------------------------------------------------ */ + + +/** + * Report the internal name a DLL exports itself under and how many + * entries it exports. + * + * The name in the export directory is the name the module had when it + * was linked, which is often not the name of the file on disk. + * + * @param ctx image being parsed + * @param rva address of the export directory + * @param size number of bytes in the export directory + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_exports (struct pe_context *ctx, + uint32_t rva, + uint32_t size) +{ + unsigned char dir[40]; + char name[PE_MAX_NAME]; + uint64_t off; + uint64_t avail; + uint32_t functions; + + if (size < sizeof (dir)) + return 0; + if (! pe_read_rva (ctx, + rva, + dir, + sizeof (dir))) + return 0; + functions = EXTRACTOR_forensic_le32_ (&dir[20]); + if (functions <= 0xFFFF) + { + if (0 != + EXTRACTOR_forensic_emit_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) functions)) + return 1; + } + if (! pe_rva_to_offset (ctx, + EXTRACTOR_forensic_le32_ (&dir[12]), + &off, + &avail)) + return 0; + if (avail > sizeof (name)) + avail = sizeof (name); + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) off, + name, + (size_t) avail)) + return 0; + return EXTRACTOR_forensic_emit_text_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_FILENAME, + name, + (size_t) avail); +} + + +/** + * Report what the certificate table says. + * + * Parsing PKCS#7 properly means an ASN.1 decoder, which is more than + * this pass can justify. The subject common names are still findable + * without one: the DER encoding of the commonName attribute is the + * fixed five-byte sequence 06 03 55 04 03 followed by a tagged string, + * so a scan finds every CN in the chain. Which one is the signer and + * which the issuer is not decided here, so they are all reported. + * + * @param ctx image being parsed + * @param offset file offset of the certificate table + * @param size number of bytes in the certificate table + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_certificates (struct pe_context *ctx, + uint32_t offset, + uint32_t size) +{ + unsigned char *buf; + size_t len = size; + unsigned int found = 0; + int ret = 0; + + if ( (size < 8) || + (offset >= ctx->fsize) || + (size > ctx->fsize - offset) ) + return 0; + if (len > PE_MAX_CERT_SCAN) + len = PE_MAX_CERT_SCAN; + if (NULL == (buf = malloc (len))) + return 0; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) offset, + buf, + len)) + { + free (buf); + return 0; + } + for (size_t i = 0; + (i + 7 < len) && (found < 8); + i++) + { + size_t vlen; + + if ( (0x06 != buf[i]) || + (0x03 != buf[i + 1]) || + (0x55 != buf[i + 2]) || + (0x04 != buf[i + 3]) || + (0x03 != buf[i + 4]) ) + continue; + /* 0x0C UTF8String, 0x13 PrintableString, 0x16 IA5String */ + if ( (0x0C != buf[i + 5]) && + (0x13 != buf[i + 5]) && + (0x16 != buf[i + 5]) ) + continue; + vlen = buf[i + 6]; + if ( (vlen < 1) || + (vlen > 127) || + (i + 7 + vlen > len) ) + continue; + found++; + if (0 != + EXTRACTOR_forensic_emit_text_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_SIGNER, + (const char *) &buf[i + 7], + vlen)) + { + ret = 1; + break; + } + i += 6 + vlen; + } + free (buf); + if ( (0 == ret) && + (0 == found) ) + ret = EXTRACTOR_forensic_emit_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_SIGNER, + "present"); + return ret; +} + + +/** + * Report the entropy of the most disordered section. + * + * A section above roughly 7.2 bits per byte holds compressed, + * encrypted or packed data rather than code, which is the cheapest + * signal there is that an executable is not what it appears to be. + * + * @param ctx image being parsed + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +pe_parse_entropy (struct pe_context *ctx) +{ + unsigned char *buf; + double best = -1.0; + const char *best_name = NULL; + unsigned int sampled = 0; + + if (NULL == (buf = malloc (PE_ENTROPY_SAMPLE))) + return 0; + for (unsigned int i = 0; + (i < ctx->num_sections) && (sampled < PE_MAX_ENTROPY_SECTIONS); + i++) + { + const struct pe_section *s = &ctx->sections[i]; + size_t len = s->raw_size; + double e; + + if (s->raw_size < PE_MIN_ENTROPY_SECTION) + continue; + if (((uint64_t) s->raw_ptr) >= ctx->fsize) + continue; + if (len > ctx->fsize - s->raw_ptr) + len = (size_t) (ctx->fsize - s->raw_ptr); + if (len > PE_ENTROPY_SAMPLE) + len = PE_ENTROPY_SAMPLE; + if (len < PE_MIN_ENTROPY_SECTION) + continue; + if (! EXTRACTOR_forensic_read_ (ctx->ec, + (int64_t) s->raw_ptr, + buf, + len)) + continue; + sampled++; + e = EXTRACTOR_forensic_entropy_ (buf, + len); + if (e > best) + { + best = e; + best_name = s->name; + } + } + free (buf); + if (NULL == best_name) + return 0; + return EXTRACTOR_forensic_emit_ (ctx->ec, + "pecoff", + EXTRACTOR_METATYPE_ENTROPY, + "%.2f bits/byte (%s)", + best, + best_name); +} + + +/* ------------------------------------------------------------------ */ +/* entry point */ +/* ------------------------------------------------------------------ */ + + +/** + * Main entry method for the PE/COFF extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_pecoff_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_pecoff_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + struct pe_context ctx; + unsigned char dos[64]; + unsigned char coff[24]; + unsigned char opt[256]; + uint32_t e_lfanew; + uint16_t machine; + uint16_t nsec; + uint32_t timestamp; + uint16_t opt_size; + uint16_t characteristics; + uint16_t magic; + uint16_t subsystem; + uint16_t dll_flags; + uint32_t entry_point; + uint32_t num_dirs; + size_t dir_at; + uint32_t dirs[16][2]; + const char *s; + char list[256]; + + memset (&ctx, + 0, + sizeof (ctx)); + ctx.ec = ec; + /* Bail out on the first two bytes: almost nothing we are handed is a + PE, and the magic is at offset zero. */ + if (! EXTRACTOR_forensic_read_ (ec, + 0, + dos, + sizeof (dos))) + return; + if ( ('M' != dos[0]) || + ('Z' != dos[1]) ) + return; + ctx.fsize = ec->get_size (ec->cls); + if ( (0 == ctx.fsize) || + (UINT64_MAX == ctx.fsize) ) + return; + e_lfanew = EXTRACTOR_forensic_le32_ (&dos[0x3C]); + if ( (e_lfanew < sizeof (dos)) || + (((uint64_t) e_lfanew) + 24 > ctx.fsize) ) + return; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) e_lfanew, + coff, + sizeof (coff))) + return; + if ( ('P' != coff[0]) || + ('E' != coff[1]) || + ('\0' != coff[2]) || + ('\0' != coff[3]) ) + return; + machine = EXTRACTOR_forensic_le16_ (&coff[4]); + nsec = EXTRACTOR_forensic_le16_ (&coff[6]); + timestamp = EXTRACTOR_forensic_le32_ (&coff[8]); + opt_size = EXTRACTOR_forensic_le16_ (&coff[20]); + characteristics = EXTRACTOR_forensic_le16_ (&coff[22]); + if (0 != + ec->proc (ec->cls, + "pecoff", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/vnd.microsoft.portable-executable", + strlen ("application/vnd.microsoft.portable-executable") + 1)) + return; + if (opt_size < 96) + return; /* an object file, not an image */ + memset (opt, + 0, + sizeof (opt)); + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) e_lfanew + 24, + opt, + (opt_size > sizeof (opt)) + ? sizeof (opt) + : opt_size)) + return; + magic = EXTRACTOR_forensic_le16_ (&opt[0]); + if (0x20b == magic) + ctx.pe32plus = 1; + else if (0x10b != magic) + return; /* 0x107 is a ROM image, which has no optional header */ + if (ctx.pe32plus && (opt_size < 112)) + return; /* the 64-bit header cannot be that short */ + entry_point = EXTRACTOR_forensic_le32_ (&opt[16]); + subsystem = EXTRACTOR_forensic_le16_ (&opt[68]); + dll_flags = EXTRACTOR_forensic_le16_ (&opt[70]); + ctx.size_of_headers = EXTRACTOR_forensic_le32_ (&opt[60]); + dir_at = ctx.pe32plus ? 112 : 96; + num_dirs = EXTRACTOR_forensic_le32_ (&opt[dir_at - 4]); + if (num_dirs > 16) + num_dirs = 16; + memset (dirs, + 0, + sizeof (dirs)); + for (uint32_t i = 0; i < num_dirs; i++) + { + if (dir_at + ((size_t) i) * 8 + 8 > opt_size) + break; + if (dir_at + ((size_t) i) * 8 + 8 > sizeof (opt)) + break; + dirs[i][0] = EXTRACTOR_forensic_le32_ (&opt[dir_at + ((size_t) i) * 8]); + dirs[i][1] = EXTRACTOR_forensic_le32_ (&opt[dir_at + ((size_t) i) * 8 + 4]); + } + + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_FORMAT, + "%s", + ctx.pe32plus ? "PE32+" : "PE32")) + return; + if (0 != (characteristics & 0x2000)) + s = "DLL"; + else if (0 != (characteristics & 0x1000)) + s = "system file"; + else if (0 != (characteristics & 0x0002)) + s = "executable"; + else + s = NULL; + if ( (NULL != s) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_RESOURCE_TYPE, + "%s", + s)) ) + return; + if (NULL != (s = pe_machine_name (machine))) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, + "%s", + s)) + return; + } + else if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, + "0x%04x", + (unsigned int) machine)) + { + return; + } + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_TARGET_OS, + "Windows")) + return; + if (NULL != (s = pe_subsystem_name (subsystem))) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_SUBSYSTEM, + "%s", + s)) + return; + } + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, + "%u.%u", + (unsigned int) EXTRACTOR_forensic_le16_ ( + &opt[40]), + (unsigned int) EXTRACTOR_forensic_le16_ ( + &opt[42]))) + return; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_TOOLCHAIN, + "linker %u.%u", + (unsigned int) opt[2], + (unsigned int) opt[3])) + return; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_ENTRY_POINT, + "0x%08x", + (unsigned int) entry_point)) + return; + /* A reproducible build puts a hash here rather than a time; the + helper drops anything outside a plausible range, which is exactly + the behaviour we want. */ + if (0 != + EXTRACTOR_forensic_emit_unix_time_ (ec, + "pecoff", + EXTRACTOR_METATYPE_BUILD_DATE, + (int64_t) timestamp)) + return; + list[0] = '\0'; + if (0 != (dll_flags & 0x0020)) + pe_append (list, sizeof (list), "HIGH_ENTROPY_VA"); + if (0 != (dll_flags & 0x0040)) + pe_append (list, sizeof (list), "DYNAMIC_BASE"); + if (0 != (dll_flags & 0x0080)) + pe_append (list, sizeof (list), "FORCE_INTEGRITY"); + if (0 != (dll_flags & 0x0100)) + pe_append (list, sizeof (list), "NX_COMPAT"); + if (0 != (dll_flags & 0x0200)) + pe_append (list, sizeof (list), "NO_ISOLATION"); + if (0 != (dll_flags & 0x0400)) + pe_append (list, sizeof (list), "NO_SEH"); + if (0 != (dll_flags & 0x0800)) + pe_append (list, sizeof (list), "NO_BIND"); + if (0 != (dll_flags & 0x1000)) + pe_append (list, sizeof (list), "APPCONTAINER"); + if (0 != (dll_flags & 0x2000)) + pe_append (list, sizeof (list), "WDM_DRIVER"); + if (0 != (dll_flags & 0x4000)) + pe_append (list, sizeof (list), "GUARD_CF"); + if (0 != (dll_flags & 0x8000)) + pe_append (list, sizeof (list), "TERMINAL_SERVER_AWARE"); + if ( ('\0' != list[0]) && + (0 != + EXTRACTOR_forensic_emit_ (ec, + "pecoff", + EXTRACTOR_METATYPE_SECURITY_MITIGATIONS, + "%s", + list)) ) + return; + + /* section table */ + if (nsec > PE_MAX_SECTIONS) + nsec = PE_MAX_SECTIONS; + for (uint16_t i = 0; i < nsec; i++) + { + struct pe_section *sec = &ctx.sections[ctx.num_sections]; + unsigned char sh[40]; + uint64_t at = ((uint64_t) e_lfanew) + 24 + opt_size + ((uint64_t) i) * 40; + + if (at + sizeof (sh) > ctx.fsize) + break; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) at, + sh, + sizeof (sh))) + break; + memcpy (sec->name, + sh, + 8); + sec->name[8] = '\0'; + for (unsigned int k = 0; k < 8; k++) + { + unsigned char c = (unsigned char) sec->name[k]; + + if ( (c < 0x20) || + (c > 0x7E) ) + sec->name[k] = '\0'; + } + sec->vsize = EXTRACTOR_forensic_le32_ (&sh[8]); + sec->vaddr = EXTRACTOR_forensic_le32_ (&sh[12]); + sec->raw_size = EXTRACTOR_forensic_le32_ (&sh[16]); + sec->raw_ptr = EXTRACTOR_forensic_le32_ (&sh[20]); + /* Clamp what the section claims to have in the file to what the + file actually holds, once, so that everything downstream can + trust raw_ptr + raw_size. */ + if (((uint64_t) sec->raw_ptr) >= ctx.fsize) + sec->raw_size = 0; + else if (sec->raw_size > ctx.fsize - sec->raw_ptr) + sec->raw_size = (uint32_t) (ctx.fsize - sec->raw_ptr); + ctx.num_sections++; + } + + /* Rich header, in the gap between the DOS header and the PE header */ + if (e_lfanew > sizeof (dos)) + { + size_t stub_len = e_lfanew - sizeof (dos); + unsigned char *stub; + + if (stub_len > PE_MAX_DOS_STUB) + stub_len = PE_MAX_DOS_STUB; + if (NULL != (stub = malloc (stub_len))) + { + int stop = 0; + + if (EXTRACTOR_forensic_read_ (ec, + (int64_t) sizeof (dos), + stub, + stub_len)) + stop = pe_parse_rich (ec, + stub, + stub_len); + free (stub); + if (stop) + return; + } + } + + if ( (0 != dirs[6][0]) && + (0 != pe_parse_debug (&ctx, + dirs[6][0], + dirs[6][1])) ) + return; + if ( (0 != dirs[2][0]) && + (0 != pe_parse_resources (&ctx, + dirs[2][0], + dirs[2][1])) ) + return; + if ( (0 != dirs[1][0]) && + (0 != pe_parse_imports (&ctx, + dirs[1][0], + dirs[1][1])) ) + return; + if ( (0 != dirs[0][0]) && + (0 != pe_parse_exports (&ctx, + dirs[0][0], + dirs[0][1])) ) + return; + /* Data directory 4 is the one field in the whole format that holds a + file offset rather than an address. */ + if ( (0 != dirs[4][0]) && + (0 != pe_parse_certificates (&ctx, + dirs[4][0], + dirs[4][1])) ) + return; + (void) pe_parse_entropy (&ctx); +} + + +/* end of pecoff_extractor.c */ diff --git a/src/plugins/plist_extractor.c b/src/plugins/plist_extractor.c @@ -0,0 +1,1353 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/plist_extractor.c + * @brief plugin to support Apple property lists, binary and XML + * @author Christian Grothoff + * + * Property lists are the configuration and manifest format of macOS + * and iOS; the forensically interesting ones are the `Info.plist' of + * an application bundle (which names the vendor, the SDK and the tool + * chain that built it) and the `Info.plist' / `Manifest.plist' of an + * iOS backup (which names the device, its serial number and when it + * was last backed up). + * + * Two on-disk representations exist. The binary one is documented by + * Apple's CoreFoundation sources (CFBinaryPlist.c): a `bplist00' + * signature, a body of tagged objects, an offset table and a 32-byte + * trailer at the very end of the file. The XML one is a DTD-bound + * document of alternating <key> and value elements. + * + * This is a first-pass identifier, not a plist library. We decode the + * top-level dictionary one level deep and stop; we never build an + * object graph, never follow arrays or nested dictionaries, and never + * read more than a bounded slice of the file. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +#include <math.h> + + +/** + * Upper bound on the object count we accept from the trailer. A + * plist with more objects than this is either not a plist or is not + * something a first pass should be walking. + */ +#define PLIST_MAX_OBJECTS 100000 + +/** + * How many key/value pairs of the top-level dictionary we decode. + */ +#define PLIST_MAX_PAIRS 64 + +/** + * How many pairs whose key we do not recognise we report as + * #EXTRACTOR_METATYPE_UNKNOWN. The point is a characterisation, not + * a dump of the file. + */ +#define PLIST_MAX_UNKNOWN 12 + +/** + * Longest unrecognised value we are willing to report in the + * "key: value" form. + */ +#define PLIST_MAX_UNKNOWN_VALUE 80 + +/** + * Bytes we look at to decide whether an XML file is a plist. + */ +#define PLIST_SNIFF_SIZE 1024 + +/** + * Bytes of an XML plist we scan for key/value pairs. + */ +#define PLIST_XML_SCAN_SIZE (64 * 1024) + +/** + * Size of the buffers holding one decoded key or value, in bytes. + * Keys and values in these files are identifiers, versions and dates. + */ +#define PLIST_MAX_TEXT 256 + +/** + * Seconds between the Unix epoch and the Core Foundation epoch of + * 2001-01-01. Binary plist dates count from the latter. + */ +#define PLIST_EPOCH_DELTA 978307200LL + +/** + * Name we report our meta data under. + */ +#define PLIST_PLUGIN "plist" + +/** + * MIME type of both representations. `application/xml' would also be + * true of the XML form, but it is useless for identification. + */ +#define PLIST_MIME "application/x-plist" + + +/** + * A well-known property list key and the meta data type it maps to. + */ +struct PlistKey +{ + /** + * Key as it appears in the file. + */ + const char *key; + + /** + * Type to report the associated value as. + */ + enum EXTRACTOR_MetaType type; +}; + + +/** + * Keys worth reporting as something better than "unknown". The + * CFBundle* and DT* families come from application bundles, the rest + * from iOS backup manifests. + */ +static const struct PlistKey plist_keys[] = { + { "CFBundleIdentifier", EXTRACTOR_METATYPE_APPLICATION_ID }, + { "CFBundleName", EXTRACTOR_METATYPE_PACKAGE_NAME }, + { "CFBundleDisplayName", EXTRACTOR_METATYPE_PACKAGE_NAME }, + { "CFBundleShortVersionString", EXTRACTOR_METATYPE_SOFTWARE_VERSION }, + { "CFBundleVersion", EXTRACTOR_METATYPE_PACKAGE_VERSION }, + { "CFBundleExecutable", EXTRACTOR_METATYPE_FILENAME }, + { "CFBundlePackageType", EXTRACTOR_METATYPE_RESOURCE_TYPE }, + { "MinimumOSVersion", EXTRACTOR_METATYPE_MINIMUM_OS_VERSION }, + { "LSMinimumSystemVersion", EXTRACTOR_METATYPE_MINIMUM_OS_VERSION }, + { "NSHumanReadableCopyright", EXTRACTOR_METATYPE_COPYRIGHT }, + { "DTPlatformName", EXTRACTOR_METATYPE_TARGET_PLATFORM }, + { "DTSDKName", EXTRACTOR_METATYPE_TARGET_PLATFORM }, + { "UIDeviceFamily", EXTRACTOR_METATYPE_TARGET_PLATFORM }, + { "DTXcode", EXTRACTOR_METATYPE_TOOLCHAIN }, + { "DTCompiler", EXTRACTOR_METATYPE_TOOLCHAIN }, + { "BuildMachineOSBuild", EXTRACTOR_METATYPE_AUTHORING_OS }, + { "DeviceName", EXTRACTOR_METATYPE_DEVICE_MODEL }, + { "ProductType", EXTRACTOR_METATYPE_DEVICE_MODEL }, + { "ProductVersion", EXTRACTOR_METATYPE_DEVICE_MODEL }, + { "SerialNumber", EXTRACTOR_METATYPE_SERIAL }, + { "UniqueDeviceID", EXTRACTOR_METATYPE_SERIAL }, + { "LastBackupDate", EXTRACTOR_METATYPE_MODIFICATION_DATE }, + { NULL, EXTRACTOR_METATYPE_RESERVED } +}; + + +/** + * State of the binary plist we are decoding: everything the trailer + * told us, already validated against the file size. + */ +struct BPlist +{ + /** + * Number of objects, from the trailer. + */ + uint64_t num_objects; + + /** + * Absolute offset of the offset table. + */ + uint64_t offset_table; + + /** + * Index of the root object. + */ + uint64_t top_object; + + /** + * Width in bytes of an entry in the offset table, 1 to 8. + */ + unsigned int offset_int_size; + + /** + * Width in bytes of an object reference, 1 to 8. + */ + unsigned int object_ref_size; +}; + + +/** + * Look up @a key in #plist_keys. + * + * @param key NUL-terminated key from the file + * @param[out] type meta data type to use for the value + * @return 1 if the key is well-known, 0 if not + */ +static int +lookup_key (const char *key, + enum EXTRACTOR_MetaType *type) +{ + for (unsigned int i = 0; NULL != plist_keys[i].key; i++) + if (0 == strcmp (key, + plist_keys[i].key)) + { + *type = plist_keys[i].type; + return 1; + } + return 0; +} + + +/** + * Read a big-endian unsigned integer of @a len bytes. + * + * @param p the bytes + * @param len number of bytes, 1 to 8 + * @return the value + */ +static uint64_t +be_n (const unsigned char *p, + unsigned int len) +{ + uint64_t v = 0; + + for (unsigned int i = 0; i < len; i++) + v = (v << 8) | (uint64_t) p[i]; + return v; +} + + +/** + * Convert a UTF-16BE string to UTF-8. Binary plists store their + * non-ASCII strings this way; #EXTRACTOR_forensic_emit_utf16le_() + * cannot be used because it reads the other byte order. + * + * @param in the UTF-16BE bytes + * @param bytes number of bytes (not code units) in @a in + * @param[out] out where to write the NUL-terminated UTF-8 + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the input is malformed or does not fit + */ +static int +utf16be_to_utf8 (const unsigned char *in, + size_t bytes, + char *out, + size_t out_size) +{ + size_t o = 0; + size_t i = 0; + + while (i + 1 < bytes) + { + uint32_t cp = EXTRACTOR_forensic_be16_ (&in[i]); + + i += 2; + if (0 == cp) + break; /* NUL terminator */ + if ( (0xD800 <= cp) && (cp <= 0xDBFF) ) + { + uint32_t lo; + + if (i + 1 >= bytes) + return 0; /* truncated surrogate pair */ + lo = EXTRACTOR_forensic_be16_ (&in[i]); + if ( (lo < 0xDC00) || (lo > 0xDFFF) ) + return 0; /* unpaired high surrogate */ + i += 2; + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + } + else if ( (0xDC00 <= cp) && (cp <= 0xDFFF) ) + { + return 0; /* stray low surrogate */ + } + if (o + 4 >= out_size) + return 0; /* does not fit */ + if (cp < 0x80) + { + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + } + out[o] = '\0'; + return 1; +} + + +/** + * Format @a unix_time as ISO 8601 in UTC, the spelling + * #EXTRACTOR_forensic_emit_unix_time_() uses. + * + * @param unix_time seconds since 1970-01-01 + * @param[out] out where to write the NUL-terminated result + * @param out_size size of @a out in bytes, at least 21 + * @return 1 on success, 0 if the value is implausible + */ +static int +format_time (int64_t unix_time, + char *out, + size_t out_size) +{ + struct tm tm; + time_t t = (time_t) unix_time; + + if ( (unix_time < 315532800LL) || /* 1980-01-01 */ + (unix_time > 4102444800LL) ) /* 2100-01-01 */ + return 0; + if (NULL == gmtime_r (&t, + &tm)) + return 0; + return (0 != strftime (out, + out_size, + "%Y-%m-%dT%H:%M:%SZ", + &tm)) ? 1 : 0; +} + + +/** + * Resolve an object reference to its absolute offset in the file. + * + * The offset table is read one entry at a time rather than in one + * block: at #PLIST_MAX_PAIRS pairs we need at most 129 entries, and + * eight-byte offsets times #PLIST_MAX_OBJECTS would otherwise be an + * 800 KB read on a file we may well reject a moment later. + * + * @param ec extraction context + * @param st validated trailer data + * @param ref object index + * @param[out] off absolute offset of the object + * @return 1 on success, 0 if @a ref is out of range or the offset it + * names does not point into the object area + */ +static int +object_offset (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st, + uint64_t ref, + uint64_t *off) +{ + unsigned char raw[8]; + uint64_t v; + + if (ref >= st->num_objects) + return 0; + /* no overflow: num_objects <= PLIST_MAX_OBJECTS, offset_int_size <= 8 + and offset_table + num_objects * offset_int_size was checked to be + within the file when the trailer was parsed */ + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (st->offset_table + + ref * st->offset_int_size), + raw, + st->offset_int_size)) + return 0; + v = be_n (raw, + st->offset_int_size); + /* objects live between the 8-byte signature and the offset table */ + if ( (v < 8) || + (v >= st->offset_table) ) + return 0; + *off = v; + return 1; +} + + +/** + * Read the object reference stored at @a pos. + * + * @param ec extraction context + * @param st validated trailer data + * @param pos absolute offset of the reference + * @param[out] ref where to store the object index + * @return 1 on success, 0 on error + */ +static int +read_ref (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st, + uint64_t pos, + uint64_t *ref) +{ + unsigned char raw[8]; + + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) pos, + raw, + st->object_ref_size)) + return 0; + *ref = be_n (raw, + st->object_ref_size); + return 1; +} + + +/** + * Decode the integer object at @a pos. + * + * @param ec extraction context + * @param st validated trailer data + * @param pos absolute offset of the object + * @param[out] value the value; eight-byte integers are signed, the + * shorter widths unsigned, as CFBinaryPlist writes them + * @param[out] next offset just past the object, may be NULL + * @return 1 on success, 0 if there is no integer object at @a pos + */ +static int +read_integer (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st, + uint64_t pos, + int64_t *value, + uint64_t *next) +{ + unsigned char marker; + unsigned char raw[8]; + unsigned int nbytes; + + if (pos >= st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) pos, + &marker, + 1)) + return 0; + if (0x10 != (marker & 0xF0)) + return 0; + if (3 < (marker & 0x0F)) + return 0; /* 128-bit integers exist; we do not need them */ + nbytes = 1u << (marker & 0x0F); + if ( (pos + 1 + nbytes) > st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (pos + 1), + raw, + nbytes)) + return 0; + /* one, two and four byte integers are unsigned and always fit; the + eight byte form is signed, which is exactly what the cast does */ + *value = (int64_t) be_n (raw, + nbytes); + if (NULL != next) + *next = pos + 1 + nbytes; + return 1; +} + + +/** + * Decode the marker byte at @a pos and the collection/string length + * that goes with it. A low nibble of 0x0F means the length follows as + * an integer object rather than being the nibble itself. + * + * @param ec extraction context + * @param st validated trailer data + * @param pos absolute offset of the object + * @param[out] marker the marker byte + * @param[out] len the decoded length + * @param[out] body offset of the first byte after marker and length + * @return 1 on success, 0 on error + */ +static int +read_marker (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st, + uint64_t pos, + unsigned char *marker, + uint64_t *len, + uint64_t *body) +{ + if (pos >= st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) pos, + marker, + 1)) + return 0; + if (0x0F != (*marker & 0x0F)) + { + *len = *marker & 0x0F; + *body = pos + 1; + return 1; + } + { + int64_t v; + + if (! read_integer (ec, + st, + pos + 1, + &v, + body)) + return 0; + if ( (v < 0) || + ((uint64_t) v > st->offset_table) ) + return 0; + *len = (uint64_t) v; + } + return 1; +} + + +/** + * Render the object @a ref as a string, if it is one of the scalar + * kinds we care about. Anything else -- arrays, sets, dictionaries, + * UIDs, nulls -- is deliberately not decoded. + * + * @param ec extraction context + * @param st validated trailer data + * @param ref object index + * @param[out] out where to write the NUL-terminated UTF-8 rendering + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the object is of another kind or is + * malformed + */ +static int +render_value (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st, + uint64_t ref, + char *out, + size_t out_size) +{ + uint64_t pos; + uint64_t len; + uint64_t body; + unsigned char marker; + + if (! object_offset (ec, + st, + ref, + &pos)) + return 0; + if (! read_marker (ec, + st, + pos, + &marker, + &len, + &body)) + return 0; + switch (marker & 0xF0) + { + case 0x00: + if (0x08 == marker) + { + strcpy (out, + "false"); + return 1; + } + if (0x09 == marker) + { + strcpy (out, + "true"); + return 1; + } + return 0; + case 0x10: + { + int64_t v; + + if (! read_integer (ec, + st, + pos, + &v, + NULL)) + return 0; + snprintf (out, + out_size, + "%lld", + (long long) v); + return 1; + } + case 0x20: + { + unsigned char raw[8]; + unsigned int nbytes = 1u << (marker & 0x0F); + double d; + + if ( (4 != nbytes) && + (8 != nbytes) ) + return 0; + if ( (pos + 1 + nbytes) > st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (pos + 1), + raw, + nbytes)) + return 0; + if (4 == nbytes) + { + uint32_t bits = EXTRACTOR_forensic_be32_ (raw); + float f; + + memcpy (&f, + &bits, + sizeof (f)); + d = (double) f; + } + else + { + uint64_t bits = EXTRACTOR_forensic_be64_ (raw); + + memcpy (&d, + &bits, + sizeof (d)); + } + if (! isfinite (d)) + return 0; + snprintf (out, + out_size, + "%.10g", + d); + return 1; + } + case 0x30: + { + unsigned char raw[8]; + uint64_t bits; + double d; + + if (0x33 != marker) + return 0; + if ( (pos + 9) > st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (pos + 1), + raw, + 8)) + return 0; + bits = EXTRACTOR_forensic_be64_ (raw); + memcpy (&d, + &bits, + sizeof (d)); + /* seconds since 2001-01-01; reject anything that cannot be a + date before converting, so the cast below is defined */ + if ( (! isfinite (d)) || + (d < -1.0e12) || + (d > 1.0e12) ) + return 0; + return format_time ((int64_t) d + PLIST_EPOCH_DELTA, + out, + out_size); + } + case 0x40: + /* the payload of a data object is not text; its size is still + worth knowing */ + if (body + len < body) + return 0; /* overflow */ + if (body + len > st->offset_table) + return 0; + snprintf (out, + out_size, + "%llu bytes of data", + (unsigned long long) len); + return 1; + case 0x50: + { + unsigned char raw[PLIST_MAX_TEXT]; + + if (0 == len) + return 0; + if (len > sizeof (raw)) + len = sizeof (raw); + if (body + len > st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) body, + raw, + (size_t) len)) + return 0; + /* an "ASCII" string with the high bit set is not one; refuse it + rather than guess at a code page */ + for (uint64_t i = 0; i < len; i++) + if (0 != (raw[i] & 0x80)) + return 0; + if (len >= out_size) + len = out_size - 1; + memcpy (out, + raw, + (size_t) len); + out[len] = '\0'; + return 1; + } + case 0x60: + { + unsigned char raw[2 * PLIST_MAX_TEXT]; + uint64_t bytes; + + if (0 == len) + return 0; + if (len > sizeof (raw) / 2) + len = sizeof (raw) / 2; + bytes = 2 * len; + if (body + bytes > st->offset_table) + return 0; + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) body, + raw, + (size_t) bytes)) + return 0; + return utf16be_to_utf8 (raw, + (size_t) bytes, + out, + out_size); + } + default: + return 0; + } +} + + +/** + * Decode the top-level dictionary of a binary property list and report + * the keys we recognise. + * + * @param ec extraction context + * @param st validated trailer data + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +extract_binary_dict (struct EXTRACTOR_ExtractContext *ec, + const struct BPlist *st) +{ + uint64_t pos; + uint64_t count; + uint64_t body; + uint64_t pairs; + unsigned char marker; + unsigned int unknown = 0; + + if (! object_offset (ec, + st, + st->top_object, + &pos)) + return 0; + if (! read_marker (ec, + st, + pos, + &marker, + &count, + &body)) + return 0; + if (0xD0 != (marker & 0xF0)) + return 0; /* the root is not a dictionary; nothing for us here */ + if ( (0 == count) || + (count > PLIST_MAX_OBJECTS) ) + return 0; + /* the reference block holds count keys followed by count values */ + if (count > (UINT64_MAX / (2 * (uint64_t) st->object_ref_size))) + return 0; + if (body + 2 * count * st->object_ref_size > st->offset_table) + return 0; + pairs = (count < PLIST_MAX_PAIRS) ? count : PLIST_MAX_PAIRS; + for (uint64_t i = 0; i < pairs; i++) + { + char key[PLIST_MAX_TEXT]; + char value[PLIST_MAX_TEXT]; + uint64_t kref; + uint64_t vref; + enum EXTRACTOR_MetaType type; + + if (! read_ref (ec, + st, + body + i * st->object_ref_size, + &kref)) + return 0; + if (! read_ref (ec, + st, + body + (count + i) * st->object_ref_size, + &vref)) + return 0; + if (! render_value (ec, + st, + kref, + key, + sizeof (key))) + continue; /* key is not a string; skip the pair */ + if (! render_value (ec, + st, + vref, + value, + sizeof (value))) + continue; /* value is a collection or something we do not decode */ + if (lookup_key (key, + &type)) + { + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + type, + value, + strlen (value))) + return 1; + continue; + } + if (unknown >= PLIST_MAX_UNKNOWN) + continue; + if (strlen (value) > PLIST_MAX_UNKNOWN_VALUE) + continue; + unknown++; + if (EXTRACTOR_forensic_emit_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_UNKNOWN, + "%s: %s", + key, + value)) + return 1; + } + return 0; +} + + +/** + * Handle a file that starts with the `bplist' signature. + * + * @param ec extraction context + * @param size size of the file + * @param header the first eight bytes of the file + */ +static void +extract_binary (struct EXTRACTOR_ExtractContext *ec, + uint64_t size, + const unsigned char *header) +{ + unsigned char trailer[32]; + struct BPlist st; + + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) (size - 32), + trailer, + sizeof (trailer))) + return; + /* CFBinaryPlistTrailer: five unused bytes, sortVersion, + offsetIntSize, objectRefSize, then three big-endian 64-bit + fields. (Note that the leading run of unused bytes is five, not + six: the sixth byte is _sortVersion.) */ + st.offset_int_size = trailer[6]; + st.object_ref_size = trailer[7]; + st.num_objects = EXTRACTOR_forensic_be64_ (&trailer[8]); + st.top_object = EXTRACTOR_forensic_be64_ (&trailer[16]); + st.offset_table = EXTRACTOR_forensic_be64_ (&trailer[24]); + if ( (st.offset_int_size < 1) || + (st.offset_int_size > 8) || + (st.object_ref_size < 1) || + (st.object_ref_size > 8) ) + return; + if ( (0 == st.num_objects) || + (st.num_objects > PLIST_MAX_OBJECTS) ) + return; + if (st.top_object >= st.num_objects) + return; + if ( (st.offset_table < 8) || + (st.offset_table > size - 32) ) + return; + /* no overflow: num_objects <= 100000 and offset_int_size <= 8 */ + if (st.num_objects * st.offset_int_size > size - 32 - st.offset_table) + return; + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_MIMETYPE, + PLIST_MIME, + strlen (PLIST_MIME))) + return; + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_FORMAT, + "Binary property list", + strlen ("Binary property list"))) + return; + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_FORMAT_VERSION, + (const char *) &header[6], + 2)) + return; + if (EXTRACTOR_forensic_emit_size_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_ENTRY_COUNT, + st.num_objects)) + return; + (void) extract_binary_dict (ec, + &st); +} + + +/** + * Find @a needle in @a haystack. Written out rather than using + * memmem(), which is a GNU extension. + * + * @param haystack where to search + * @param hlen number of bytes in @a haystack + * @param needle what to search for + * @param nlen number of bytes in @a needle + * @return pointer to the first occurrence, NULL if there is none + */ +static const char * +find_bytes (const char *haystack, + size_t hlen, + const char *needle, + size_t nlen) +{ + if ( (0 == nlen) || + (hlen < nlen) ) + return NULL; + for (size_t i = 0; i + nlen <= hlen; i++) + if (0 == memcmp (&haystack[i], + needle, + nlen)) + return &haystack[i]; + return NULL; +} + + +/** + * Copy XML character data, resolving the five predefined entities and + * numeric character references. + * + * @param in the text between the tags + * @param len number of bytes in @a in + * @param[out] out where to write the NUL-terminated result + * @param out_size size of @a out in bytes + * @return 1 on success, 0 if the text does not fit + */ +static int +xml_unescape (const char *in, + size_t len, + char *out, + size_t out_size) +{ + size_t o = 0; + size_t i = 0; + + while (i < len) + { + const char *semi; + size_t elen; + + if ('&' != in[i]) + { + if (o + 1 >= out_size) + return 0; + out[o++] = in[i++]; + continue; + } + semi = find_bytes (&in[i], + len - i, + ";", + 1); + if ( (NULL == semi) || + ((size_t) (semi - &in[i]) > 10) ) + { + if (o + 1 >= out_size) + return 0; + out[o++] = in[i++]; /* a bare ampersand; keep it */ + continue; + } + elen = (size_t) (semi - &in[i]) + 1; + if (o + 4 >= out_size) + return 0; + if ( (5 == elen) && + (0 == memcmp (&in[i], "&", 5)) ) + out[o++] = '&'; + else if ( (4 == elen) && + (0 == memcmp (&in[i], "<", 4)) ) + out[o++] = '<'; + else if ( (4 == elen) && + (0 == memcmp (&in[i], ">", 4)) ) + out[o++] = '>'; + else if ( (6 == elen) && + (0 == memcmp (&in[i], """, 6)) ) + out[o++] = '"'; + else if ( (6 == elen) && + (0 == memcmp (&in[i], "'", 6)) ) + out[o++] = '\''; + else if ( (2 < elen) && + ('#' == in[i + 1]) ) + { + unsigned long cp; + char num[12]; + size_t nlen = elen - 3; + + if (nlen >= sizeof (num)) + return 0; + memcpy (num, + &in[i + 2], + nlen); + num[nlen] = '\0'; + cp = strtoul (('x' == num[0]) || ('X' == num[0]) + ? &num[1] : num, + NULL, + (('x' == num[0]) || ('X' == num[0])) ? 16 : 10); + if ( (0 == cp) || + (cp > 0x10FFFF) || + ( (0xD800 <= cp) && (cp <= 0xDFFF) ) ) + return 0; + if (cp < 0x80) + { + out[o++] = (char) cp; + } + else if (cp < 0x800) + { + out[o++] = (char) (0xC0 | (cp >> 6)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else if (cp < 0x10000) + { + out[o++] = (char) (0xE0 | (cp >> 12)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + else + { + out[o++] = (char) (0xF0 | (cp >> 18)); + out[o++] = (char) (0x80 | ((cp >> 12) & 0x3F)); + out[o++] = (char) (0x80 | ((cp >> 6) & 0x3F)); + out[o++] = (char) (0x80 | (cp & 0x3F)); + } + } + else + { + out[o++] = '&'; /* entity we do not know; keep the text */ + i++; + continue; + } + i += elen; + } + out[o] = '\0'; + return 1; +} + + +/** + * One value element we are prepared to read out of an XML plist. + */ +struct XmlValueTag +{ + /** + * Opening tag, including the angle brackets. + */ + const char *open; + + /** + * Matching closing tag, NULL for an empty element such as <true/>. + */ + const char *close; + + /** + * Text to report for an empty element, NULL otherwise. + */ + const char *literal; +}; + + +/** + * Value elements of the plist DTD that carry a scalar. Collections + * (<dict>, <array>) are absent on purpose: we do not descend. + */ +static const struct XmlValueTag xml_value_tags[] = { + { "<string>", "</string>", NULL }, + { "<integer>", "</integer>", NULL }, + { "<real>", "</real>", NULL }, + { "<date>", "</date>", NULL }, + { "<true/>", NULL, "true" }, + { "<false/>", NULL, "false" }, + { NULL, NULL, NULL } +}; + + +/** + * Scan an XML property list for <key>/value pairs. + * + * This is a bounded byte scan and not an XML parser: it will be + * confused by a <key> inside a comment or a CDATA section, and it does + * not track nesting, so a key of the same name inside a nested + * dictionary is treated like a top-level one. For a first pass whose + * job is to say "this bundle is org.example.app, built with that SDK" + * that is an acceptable trade; adding a real XML parser would mean a + * dependency and a great deal more attack surface. What it must never + * do -- and does not -- is read outside @a buf. + * + * @param ec extraction context + * @param buf the first bytes of the file + * @param len number of bytes in @a buf + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +scan_xml (struct EXTRACTOR_ExtractContext *ec, + const char *buf, + size_t len) +{ + const char *p = buf; + const char *end = buf + len; + unsigned int pairs = 0; + unsigned int unknown = 0; + + while ( (p < end) && + (pairs < PLIST_MAX_PAIRS) ) + { + const char *k; + const char *ke; + const char *q; + char key[PLIST_MAX_TEXT]; + char value[PLIST_MAX_TEXT]; + enum EXTRACTOR_MetaType type; + unsigned int i; + + k = find_bytes (p, + (size_t) (end - p), + "<key>", + 5); + if (NULL == k) + break; + k += 5; + ke = find_bytes (k, + (size_t) (end - k), + "</key>", + 6); + if (NULL == ke) + break; + p = ke + 6; + pairs++; + if (! xml_unescape (k, + (size_t) (ke - k), + key, + sizeof (key))) + continue; + q = p; + while ( (q < end) && + ( (' ' == *q) || ('\t' == *q) || + ('\r' == *q) || ('\n' == *q) ) ) + q++; + if ( (q >= end) || + ('<' != *q) ) + continue; + for (i = 0; NULL != xml_value_tags[i].open; i++) + { + size_t olen = strlen (xml_value_tags[i].open); + + if ((size_t) (end - q) < olen) + continue; + if (0 != memcmp (q, + xml_value_tags[i].open, + olen)) + continue; + if (NULL == xml_value_tags[i].close) + { + strcpy (value, + xml_value_tags[i].literal); + p = q + olen; + } + else + { + const char *vs = q + olen; + const char *ve; + size_t clen = strlen (xml_value_tags[i].close); + + ve = find_bytes (vs, + (size_t) (end - vs), + xml_value_tags[i].close, + clen); + if (NULL == ve) + return 0; /* truncated; nothing more to be had */ + p = ve + clen; + if (! xml_unescape (vs, + (size_t) (ve - vs), + value, + sizeof (value))) + value[0] = '\0'; + } + break; + } + if (NULL == xml_value_tags[i].open) + continue; /* <dict>, <array>, <data> or an unknown element */ + if ('\0' == value[0]) + continue; + if (lookup_key (key, + &type)) + { + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + type, + value, + strlen (value))) + return 1; + continue; + } + if (unknown >= PLIST_MAX_UNKNOWN) + continue; + if (strlen (value) > PLIST_MAX_UNKNOWN_VALUE) + continue; + unknown++; + if (EXTRACTOR_forensic_emit_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_UNKNOWN, + "%s: %s", + key, + value)) + return 1; + } + return 0; +} + + +/** + * Handle a file that looks like an XML property list. + * + * @param ec extraction context + * @param size size of the file + */ +static void +extract_xml (struct EXTRACTOR_ExtractContext *ec, + uint64_t size) +{ + char *buf; + size_t len; + + len = (size < PLIST_XML_SCAN_SIZE) + ? (size_t) size + : PLIST_XML_SCAN_SIZE; + buf = malloc (len); + if (NULL == buf) + return; + if (! EXTRACTOR_forensic_read_ (ec, + 0, + buf, + len)) + { + free (buf); + return; + } + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_MIMETYPE, + PLIST_MIME, + strlen (PLIST_MIME))) + { + free (buf); + return; + } + if (EXTRACTOR_forensic_emit_text_ (ec, + PLIST_PLUGIN, + EXTRACTOR_METATYPE_FORMAT, + "XML property list", + strlen ("XML property list"))) + { + free (buf); + return; + } + (void) scan_xml (ec, + buf, + len); + free (buf); +} + + +/** + * Decide whether @a buf is the beginning of an XML property list. + * + * Both conditions matter. The `<plist' is what keeps us from + * claiming every XML document on the volume; the leading `<' is what + * keeps us from claiming a file that merely mentions a plist in its + * first kilobyte. + * + * @param buf the first bytes of the file + * @param len number of bytes in @a buf + * @return 1 if this is an XML plist, 0 if not + */ +static int +looks_like_xml_plist (const char *buf, + size_t len) +{ + size_t i = 0; + + if ( (len >= 3) && + (0xEF == (unsigned char) buf[0]) && + (0xBB == (unsigned char) buf[1]) && + (0xBF == (unsigned char) buf[2]) ) + i = 3; /* UTF-8 byte order mark */ + while ( (i < len) && + ( (' ' == buf[i]) || ('\t' == buf[i]) || + ('\r' == buf[i]) || ('\n' == buf[i]) ) ) + i++; + if ( (i >= len) || + ('<' != buf[i]) ) + return 0; + if (NULL != find_bytes (buf, + len, + "<plist", + 6)) + return 1; + if (NULL != find_bytes (buf, + len, + "<!DOCTYPE plist", + 15)) + return 1; + return 0; +} + + +/** + * Main entry method for the plist extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_plist_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_plist_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char head[PLIST_SNIFF_SIZE]; + uint64_t size; + size_t hlen; + + size = ec->get_size (ec->cls); + if ( (UINT64_MAX == size) || + (size < 16) ) + return; + hlen = (size < sizeof (head)) ? (size_t) size : sizeof (head); + if (! EXTRACTOR_forensic_read_ (ec, + 0, + head, + hlen)) + return; + if ( (hlen >= 8) && + (0 == memcmp (head, + "bplist", + 6)) && + (isdigit ((unsigned char) head[6])) && + (isdigit ((unsigned char) head[7])) ) + { + if (size < 40) + return; /* eight-byte signature plus a 32-byte trailer at least */ + extract_binary (ec, + size, + head); + return; + } + if (looks_like_xml_plist ((const char *) head, + hlen)) + extract_xml (ec, + size); +} + + +/* end of plist_extractor.c */ diff --git a/src/plugins/qt_extractor.c b/src/plugins/qt_extractor.c @@ -1350,6 +1350,16 @@ iTunesTagHandler (const char *input, if (asize < hdr + 8) /* header + at least one atom */ return -1; + /* #processDataAtom immediately reads the child atom's own header, and + both #getAtomSize and #getAtomHeaderSize require #checkAtomValid to + have run first. The test above only says that the *declared* size + of this entry leaves room for eight more bytes; a child whose size + field is the 64-bit escape value 1 is a `struct LongAtom' and is + read sixteen bytes wide, which is eight past that guarantee. */ + if (! checkAtomValid (input, + size, + pos + hdr)) + return -1; for (unsigned int i = 0; NULL != it_to_extr_table[i].atom_type; diff --git a/src/plugins/sqlite_extractor.c b/src/plugins/sqlite_extractor.c @@ -0,0 +1,549 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/sqlite_extractor.c + * @brief plugin to support sqlite files + * @author Christian Grothoff + * + * This plugin reads the 100-byte SQLite database header and nothing + * else. It deliberately does not link libsqlite3 and never opens the + * database: handing an untrusted file to a full SQL engine is a large + * attack surface, and a bulk pass over millions of files cannot afford + * the page-cache and journal recovery work that opening implies. + * Everything reported below lives in the first 100 bytes. + * + * Reference: "The SQLite Database File Format", https://sqlite.org/fileformat.html + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +/** + * Number of bytes in the SQLite database header. + */ +#define SQLITE_HEADER_SIZE 100 + +/** + * Number of bytes we need to tell the three members of the family + * apart: the database magic is 16 bytes, the write-ahead log header is + * 32 and the rollback journal magic is 8. + */ +#define SQLITE_PROBE_SIZE 32 + +/** + * The 16-byte magic every SQLite 3 database file starts with, + * including its terminating NUL. + */ +static const char sqlite_magic[16] = "SQLite format 3"; + +/** + * Magic of a rollback journal file (the `-journal' sidecar). + */ +static const unsigned char journal_magic[8] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7 +}; + + +/** + * Is @a ps a page size SQLite could actually have written? + * + * Legal values are the powers of two from 512 to 32768, plus the + * escape value 1 which stands for 65536. Rejecting anything else + * keeps a corrupt or forged header from turning into an absurd + * "database size" further down. + * + * @param ps the raw 16-bit field + * @return the page size in bytes, 0 if @a ps is not legal + */ +static uint32_t +decode_page_size (uint16_t ps) +{ + if (1 == ps) + return 65536; /* the documented escape for a 64 KiB page */ + if ( (ps < 512) || + (0 != (ps & (ps - 1))) ) + return 0; + return ps; +} + + +/** + * Report the SQLite library version that wrote a file. + * + * The version number is stored as X*1000000 + Y*1000 + Z. + * + * We report this as #EXTRACTOR_METATYPE_MODIFIED_BY_SOFTWARE rather + * than CREATED_BY_SOFTWARE because that is what the field actually + * means: the header documents it as the version that *most recently + * modified* the file, and it is rewritten on every commit. A database + * created by 3.8 and later written by 3.45 carries 3.45 here, so + * calling it the creator would be wrong in exactly the cases where the + * value is interesting. + * + * @param ec extraction context + * @param type meta data type to report under + * @param version the raw version number + * @return 1 if the caller should stop extracting, 0 to continue + */ +static int +emit_version (struct EXTRACTOR_ExtractContext *ec, + enum EXTRACTOR_MetaType type, + uint32_t version) +{ + unsigned int major; + unsigned int minor; + unsigned int patch; + + if (0 == version) + return 0; + major = version / 1000000; + minor = (version / 1000) % 1000; + patch = version % 1000; + if ( (major < 3) || + (major > 9) ) + return 0; /* not a version number we recognise; do not guess */ + return EXTRACTOR_forensic_emit_ (ec, + "sqlite", + type, + "SQLite %u.%u.%u", + major, + minor, + patch); +} + + +/** + * Handle a write-ahead log file (the `-wal' sidecar). + * + * A bulk pass meets these constantly next to the databases they belong + * to, and they hold committed-but-not-yet-checkpointed pages, so + * naming them as such is worth the twenty lines. + * + * @param ec extraction context + * @param hdr the first #SQLITE_PROBE_SIZE bytes of the file + */ +static void +handle_wal (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *hdr) +{ + uint32_t page_size; + + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_MIMETYPE, + "%s", + "application/vnd.sqlite3-wal")) + return; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_FORMAT, + "%s", + "SQLite write-ahead log")) + return; + /* file format version, offset 4, big-endian */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_FORMAT_VERSION, + "%u", + (unsigned int) + EXTRACTOR_forensic_be32_ (&hdr[4]))) + return; + /* the WAL page size is a plain 32-bit value, not the escaped 16-bit + field the database header uses */ + page_size = EXTRACTOR_forensic_be32_ (&hdr[8]); + if ( (page_size >= 512) && + (page_size <= 65536) && + (0 == (page_size & (page_size - 1))) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_BLOCK_SIZE, + "%u", + (unsigned int) page_size)) ) + return; + /* checkpoint sequence number, offset 12: how many times this log has + been reset, i.e. roughly how much history the pair has seen */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_CHANGE_COUNTER, + "%u", + (unsigned int) + EXTRACTOR_forensic_be32_ (&hdr[12]))) + return; + /* salt-1/salt-2 at 16/20 identify the WAL incarnation and let an + analyst match a log against the database it belongs to */ + (void) EXTRACTOR_forensic_emit_hex_ (ec, + "sqlite", + EXTRACTOR_METATYPE_VOLUME_SERIAL, + &hdr[16], + 8); +} + + +/** + * Handle a rollback journal file (the `-journal' sidecar). + * + * @param ec extraction context + * @param hdr the first #SQLITE_PROBE_SIZE bytes of the file + */ +static void +handle_journal (struct EXTRACTOR_ExtractContext *ec, + const unsigned char *hdr) +{ + uint32_t page_size; + uint32_t db_pages; + + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_MIMETYPE, + "%s", + "application/vnd.sqlite3-journal")) + return; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_FORMAT, + "%s", + "SQLite rollback journal")) + return; + /* page size at offset 24, sector size at 20, initial database size in + pages at 16 -- all big-endian */ + page_size = EXTRACTOR_forensic_be32_ (&hdr[24]); + if ( (page_size >= 512) && + (page_size <= 65536) && + (0 == (page_size & (page_size - 1))) ) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_BLOCK_SIZE, + "%u", + (unsigned int) page_size)) + return; + db_pages = EXTRACTOR_forensic_be32_ (&hdr[16]); + /* the size the database had before the transaction started: a + journal left behind says the transaction never completed */ + if ( (0 != db_pages) && + (db_pages <= UINT32_MAX / page_size) && + (0 != EXTRACTOR_forensic_emit_size_ (ec, + "sqlite", + EXTRACTOR_METATYPE_VOLUME_SIZE, + ((uint64_t) db_pages) + * page_size)) ) + return; + } + (void) EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_COMMENT, + "%s", + "hot journal: a transaction was" + " interrupted or the database is open"); +} + + +/** + * Main entry method for the sqlite extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_sqlite_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_sqlite_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char hdr[SQLITE_HEADER_SIZE]; + uint64_t file_size; + uint32_t page_size; + uint32_t page_count; + uint32_t change_counter; + uint32_t version_valid_for; + uint32_t freelist_pages; + uint32_t app_id; + uint16_t raw_page_size; + unsigned char write_version; + unsigned char read_version; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + SQLITE_PROBE_SIZE)) + return; /* too short to be anything of ours */ + /* The two sidecar formats share the family but not the magic; check + them first, they are cheap and would otherwise fall through. */ + if ( (0x37 == hdr[0]) && + (0x7f == hdr[1]) && + (0x06 == hdr[2]) && + ( (0x82 == hdr[3]) || + (0x83 == hdr[3]) ) ) + { + handle_wal (ec, + hdr); + return; + } + if (0 == memcmp (hdr, + journal_magic, + sizeof (journal_magic))) + { + handle_journal (ec, + hdr); + return; + } + if (0 != memcmp (hdr, + sqlite_magic, + sizeof (sqlite_magic))) + return; /* not a SQLite database */ + if (! EXTRACTOR_forensic_read_ (ec, + 0, + hdr, + SQLITE_HEADER_SIZE)) + return; /* magic matched but the header is truncated */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_MIMETYPE, + "%s", + "application/vnd.sqlite3")) + return; + file_size = ec->get_size (ec->cls); + + /* -- page size, offset 16, big-endian, 1 meaning 65536 -- */ + raw_page_size = EXTRACTOR_forensic_be16_ (&hdr[16]); + page_size = decode_page_size (raw_page_size); + if ( (0 != page_size) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_BLOCK_SIZE, + "%u", + (unsigned int) page_size)) ) + return; + + /* -- journal mode, from the write/read format versions at 18 and 19 -- */ + write_version = hdr[18]; + read_version = hdr[19]; + if ( (1 == write_version) || + (2 == write_version) ) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_JOURNAL_MODE, + "%s", + (2 == write_version) + ? "wal" + : "rollback journal")) + return; + } + if ( (read_version > 2) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_COMMENT, + "read format version %u:" + " newer than this library understands", + (unsigned int) read_version)) ) + return; + + /* -- change counter (24) against version-valid-for (92) -- */ + change_counter = EXTRACTOR_forensic_be32_ (&hdr[24]); + version_valid_for = EXTRACTOR_forensic_be32_ (&hdr[92]); + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_CHANGE_COUNTER, + "%u", + (unsigned int) change_counter)) + return; + + /* -- page count, offset 28 -- */ + page_count = EXTRACTOR_forensic_be32_ (&hdr[28]); + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + (unsigned int) page_count)) + return; + /* The page count is only authoritative when version-valid-for equals + the change counter; otherwise the file was last written by a + SQLite older than 3.7.0, which did not maintain the field. */ + if ( (change_counter != version_valid_for) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_COMMENT, + "%s", + "stale page count: last written by" + " a SQLite older than 3.7.0")) ) + return; + + /* -- free space: freelist page count at 36, in pages -- */ + freelist_pages = EXTRACTOR_forensic_be32_ (&hdr[36]); + if (0 != page_size) + { + /* a 32-bit page count times a page size of at most 64 KiB stays + well inside 64 bits */ + /* Freelist pages are where deleted rows survive, so a large value + against a small database is a strong sign of recoverable data. */ + if (0 != + EXTRACTOR_forensic_emit_size_ (ec, + "sqlite", + EXTRACTOR_METATYPE_FREE_SPACE, + ((uint64_t) freelist_pages) + * page_size)) + return; + } + + /* -- schema cookie, offset 40 -- */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_SCHEMA_VERSION, + "schema cookie %u", + (unsigned int) + EXTRACTOR_forensic_be32_ (&hdr[40]))) + return; + /* -- user_version, offset 60; also a schema version, but the one the + application chose rather than the one SQLite maintains -- */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_SCHEMA_VERSION, + "user_version %u", + (unsigned int) + EXTRACTOR_forensic_be32_ (&hdr[60]))) + return; + + /* -- vacuum modes: largest root b-tree page (52) and incremental + vacuum flag (64) -- */ + if (0 != EXTRACTOR_forensic_be32_ (&hdr[52])) + { + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + (0 != EXTRACTOR_forensic_be32_ (&hdr[64])) + ? "incremental vacuum" + : "auto-vacuum")) + return; + } + + /* -- text encoding, offset 56 -- */ + switch (EXTRACTOR_forensic_be32_ (&hdr[56])) + { + case 1: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_CHARACTER_SET, + "%s", + "UTF-8")) + return; + break; + case 2: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_CHARACTER_SET, + "%s", + "UTF-16LE")) + return; + break; + case 3: + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_CHARACTER_SET, + "%s", + "UTF-16BE")) + return; + break; + default: + break; /* 0 means "not yet decided"; anything else is corrupt */ + } + + /* -- application id, offset 68 -- */ + app_id = EXTRACTOR_forensic_be32_ (&hdr[68]); + if (0 != app_id) + { + /* This is the single best answer to "which program wrote this + file". Values are registered upstream in SQLite's `magic.txt'; + many of them are chosen to spell a four-character ASCII tag, so + render that too when every byte is printable. */ + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_APPLICATION_ID, + "%u", + (unsigned int) app_id)) + return; + if ( (0x20 <= hdr[68]) && (hdr[68] < 0x7f) && + (0x20 <= hdr[69]) && (hdr[69] < 0x7f) && + (0x20 <= hdr[70]) && (hdr[70] < 0x7f) && + (0x20 <= hdr[71]) && (hdr[71] < 0x7f) && + (0 != EXTRACTOR_forensic_emit_text_ (ec, + "sqlite", + EXTRACTOR_METATYPE_APPLICATION_ID, + (const char *) &hdr[68], + 4)) ) + return; + } + + /* -- the SQLite version that last wrote the file, offset 96 -- */ + if (0 != + emit_version (ec, + EXTRACTOR_METATYPE_MODIFIED_BY_SOFTWARE, + EXTRACTOR_forensic_be32_ (&hdr[96]))) + return; + + /* -- logical size, and whether the file lives up to it -- */ + if ( (0 != page_size) && + (0 != page_count) ) + { + uint64_t logical = ((uint64_t) page_count) * page_size; + + if (0 != + EXTRACTOR_forensic_emit_size_ (ec, + "sqlite", + EXTRACTOR_METATYPE_VOLUME_SIZE, + logical)) + return; + /* A file materially shorter than its own page count claims was + truncated -- by a carver, a partial copy or a wiper. Allow one + page of slack so that a merely unflushed tail does not trip it. */ + if ( (UINT64_MAX != file_size) && + (logical > page_size) && + (file_size < logical - page_size) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "sqlite", + EXTRACTOR_METATYPE_COMMENT, + "truncated database: header claims" + " %llu bytes, file holds %llu", + (unsigned long long) logical, + (unsigned long long) file_size)) ) + return; + } +} + + +/* end of sqlite_extractor.c */ diff --git a/src/plugins/tar_extractor.c b/src/plugins/tar_extractor.c @@ -0,0 +1,774 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/tar_extractor.c + * @brief plugin to support tar files + * @author Christian Grothoff + * + * There is already an `archive' plugin wrapping libarchive, but it is + * optional and reports what is *in* the archive rather than who made + * it. This one is unconditional, reads nothing but the 512-byte member + * headers, and concentrates on the ownership fields, which leak the + * usernames, group names and numeric ids of the machine the archive was + * created on -- exactly the provenance a first pass wants. + * + * References: POSIX.1-1988 ustar, GNU tar's extensions and POSIX.1-2001 + * pax extended headers. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + +/** + * Size of a tar header and of the blocks the archive is padded to. + */ +#define TAR_BLOCK 512 + +/** + * Largest number of member headers we will walk. Each one costs a + * seek and a 512-byte read, so this bounds us at 128 KiB of reads even + * for an archive with a million members; we report a characterisation, + * not an inventory. + */ +#define TAR_MAX_MEMBERS 256 + +/** + * How many *distinct* values of a repeating attribute (owner, group, + * mode) we remember and report. Archives are made by one or two users; + * more than a handful means the value is not identifying anyway. + */ +#define TAR_MAX_DISTINCT 16 + +/** + * Longest GNU long name (typeflag 'L'/'K') payload we will read. The + * field it replaces is 100 bytes, so anything past this is a path no + * filesystem would accept. + */ +#define TAR_MAX_LONGNAME 1024 + +/** + * Largest pax extended header payload we will read and parse. + */ +#define TAR_MAX_PAX 4096 + + +/** + * What the ownership fields of the archive looked like, accumulated + * over the members we walked. + */ +struct tar_state +{ + /** + * Distinct owner user names, NUL-terminated. + */ + char unames[TAR_MAX_DISTINCT][33]; + + /** + * Distinct owner group names, NUL-terminated. + */ + char gnames[TAR_MAX_DISTINCT][33]; + + /** + * Distinct numeric user ids. + */ + uint64_t uids[TAR_MAX_DISTINCT]; + + /** + * Distinct numeric group ids. + */ + uint64_t gids[TAR_MAX_DISTINCT]; + + /** + * Distinct permission bits. + */ + uint64_t modes[TAR_MAX_DISTINCT]; + + /** + * Number of entries used in @e unames. + */ + unsigned int n_unames; + + /** + * Number of entries used in @e gnames. + */ + unsigned int n_gnames; + + /** + * Number of entries used in @e uids. + */ + unsigned int n_uids; + + /** + * Number of entries used in @e gids. + */ + unsigned int n_gids; + + /** + * Number of entries used in @e modes. + */ + unsigned int n_modes; +}; + + +/** + * Is @a block entirely zero? Two such blocks end an archive; we stop + * at the first, which is what every reader does in practice. + * + * @param block the 512 bytes to check + * @return 1 if all bytes are zero + */ +static int +is_zero_block (const unsigned char *block) +{ + for (unsigned int i = 0; i < TAR_BLOCK; i++) + if (0 != block[i]) + return 0; + return 1; +} + + +/** + * Verify the header checksum of @a block. + * + * The checksum is the sum of all 512 header bytes with the checksum + * field itself read as spaces. Historical tars disagreed on whether + * the bytes are signed, so accept either reading. + * + * @param block the header block + * @return 1 if the stored checksum matches + */ +static int +checksum_ok (const unsigned char *block) +{ + uint64_t stored; + uint32_t usum = 0; + int32_t ssum = 0; + + if (! EXTRACTOR_forensic_parse_octal_ ((const char *) &block[148], + 8, + &stored)) + return 0; + for (unsigned int i = 0; i < TAR_BLOCK; i++) + { + if ( (i >= 148) && + (i < 156) ) + { + usum += (uint32_t) ' '; + ssum += (int32_t) ' '; + continue; + } + usum += (uint32_t) block[i]; + ssum += (int32_t) (signed char) block[i]; + } + return ( (stored == (uint64_t) usum) || + ( (ssum > 0) && + (stored == (uint64_t) ssum) ) ); +} + + +/** + * Read a numeric header field. + * + * Fields are normally NUL- or space-terminated octal. GNU tar escapes + * values that do not fit by setting the high bit of the first byte and + * storing the rest base-256, big-endian; we accept the positive form + * and refuse the rest rather than misparse it. + * + * @param field the field + * @param len width of the field + * @param[out] value where to store the result + * @return 1 on success, 0 if the field cannot be read + */ +static int +tar_number (const unsigned char *field, + size_t len, + uint64_t *value) +{ + if (0 == len) + return 0; + if (0 != (field[0] & 0x80)) + { + uint64_t v = 0; + size_t i = 1; + + if (0x80 != field[0]) + return 0; /* 0xff is the negative form; nothing else is defined */ + while ( (i < len) && + (0 == field[i]) ) + i++; + if (len - i > 8) + return 0; /* does not fit in 64 bits */ + for (; i < len; i++) + v = (v << 8) | field[i]; + *value = v; + return 1; + } + if (0 == EXTRACTOR_forensic_trim_ ((const char *) field, + len)) + { + /* an all-NUL field is how several tars spell "zero" */ + *value = 0; + return 1; + } + return EXTRACTOR_forensic_parse_octal_ ((const char *) field, + len, + value); +} + + +/** + * Could @a block be a header of a pre-POSIX (v7) tar? + * + * v7 has no magic at all, so the only defence against claiming every + * file in the world is to insist that the fields we do understand make + * sense. The caller has already verified the checksum. + * + * @param block the header block + * @return 1 if this looks like a v7 header + */ +static int +plausible_v7 (const unsigned char *block) +{ + uint64_t v; + + if ('\0' == block[0]) + return 0; /* a member with no name */ + for (unsigned int i = 0; i < 100; i++) + { + if ('\0' == block[i]) + break; + if ( (block[i] < 0x20) || + (0x7f == block[i]) ) + return 0; /* control characters in a file name */ + } + if (! EXTRACTOR_forensic_parse_octal_ ((const char *) &block[100], + 8, + &v)) + return 0; /* mode */ + if (! EXTRACTOR_forensic_parse_octal_ ((const char *) &block[136], + 12, + &v)) + return 0; /* mtime */ + switch (block[156]) + { + case '\0': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + return 1; + default: + return 0; + } +} + + +/** + * Remember @a value if it is new. + * + * @param seen array of values seen so far + * @param[in,out] count number of used slots in @a seen + * @param value the value to record + * @return 1 if @a value had not been seen before and was recorded + */ +static int +add_distinct_num (uint64_t *seen, + unsigned int *count, + uint64_t value) +{ + for (unsigned int i = 0; i < *count; i++) + if (seen[i] == value) + return 0; + if (*count >= TAR_MAX_DISTINCT) + return 0; + seen[(*count)++] = value; + return 1; +} + + +/** + * Remember the fixed-width string @a value if it is new and not empty. + * + * @param seen array of 33-byte slots seen so far + * @param[in,out] count number of used slots in @a seen + * @param value the field, not necessarily NUL-terminated + * @param len width of the field, at most 32 + * @return 1 if @a value had not been seen before and was recorded + */ +static int +add_distinct_str (char seen[][33], + unsigned int *count, + const char *value, + size_t len) +{ + char tmp[33]; + + if (len > sizeof (tmp) - 1) + len = sizeof (tmp) - 1; + memcpy (tmp, + value, + len); + tmp[len] = '\0'; + len = strlen (tmp); + len = EXTRACTOR_forensic_trim_ (tmp, + len); + tmp[len] = '\0'; + if (0 == len) + return 0; + for (unsigned int i = 0; i < *count; i++) + if (0 == strcmp (seen[i], + tmp)) + return 0; + if (*count >= TAR_MAX_DISTINCT) + return 0; + memcpy (seen[(*count)++], + tmp, + len + 1); + return 1; +} + + +/** + * Pull the overriding path and link target out of a pax extended + * header payload. + * + * The payload is a sequence of "<len> <key>=<value>\n" records where + * @a len counts the whole record including its own digits and the + * newline. We only care about the two keys that override a field we + * would otherwise report truncated; a zero or nonsensical length ends + * the parse so that a crafted payload cannot spin here. + * + * @param data the payload + * @param len number of bytes in @a data + * @param[out] path buffer of #TAR_MAX_LONGNAME + 1 bytes for "path" + * @param[out] linkpath buffer of #TAR_MAX_LONGNAME + 1 bytes for + * "linkpath" + */ +static void +parse_pax (const char *data, + size_t len, + char *path, + char *linkpath) +{ + size_t pos = 0; + + while (pos < len) + { + size_t reclen = 0; + size_t digits = 0; + size_t key; + size_t eq; + char *dst = NULL; + + while ( (pos + digits < len) && + ('0' <= data[pos + digits]) && + ('9' >= data[pos + digits]) ) + { + if (reclen > TAR_MAX_PAX) + return; /* absurd record length */ + reclen = reclen * 10 + (size_t) (data[pos + digits] - '0'); + digits++; + } + if ( (0 == digits) || + (reclen <= digits + 1) || + (reclen > len - pos) ) + return; /* no forward progress possible */ + key = pos + digits + 1; /* skip the space after the length */ + for (eq = key; eq < pos + reclen; eq++) + if ('=' == data[eq]) + break; + if (eq < pos + reclen) + { + size_t klen = eq - key; + + if ( (4 == klen) && + (0 == memcmp (&data[key], + "path", + 4)) ) + dst = path; + if ( (8 == klen) && + (0 == memcmp (&data[key], + "linkpath", + 8)) ) + dst = linkpath; + if (NULL != dst) + { + size_t vlen = pos + reclen - eq - 1; + + if ( (vlen > 0) && + ('\n' == data[eq + vlen]) ) + vlen--; /* drop the record terminator */ + if (vlen > TAR_MAX_LONGNAME) + vlen = TAR_MAX_LONGNAME; + memcpy (dst, + &data[eq + 1], + vlen); + dst[vlen] = '\0'; + } + } + pos += reclen; + } +} + + +/** + * Main entry method for the tar extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_tar_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_tar_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + unsigned char block[TAR_BLOCK]; + struct tar_state st; + char longname[TAR_MAX_LONGNAME + 1]; + char longlink[TAR_MAX_LONGNAME + 1]; + char name[TAR_BLOCK]; + const char *format; + uint64_t file_size; + uint64_t offset = 0; + uint64_t total_size = 0; + int64_t newest = 0; + unsigned int members = 0; + unsigned int names_emitted = 0; + unsigned int links_emitted = 0; + int have_ustar; + int truncated_walk = 0; + + if (! EXTRACTOR_forensic_read_ (ec, + 0, + block, + TAR_BLOCK)) + return; /* too short for even one header */ + have_ustar = (0 == memcmp (&block[257], + "ustar", + 5)) && + ( ('\0' == block[262]) || + (' ' == block[262]) ); + if (have_ustar) + { + /* POSIX writes "ustar\0" followed by "00"; GNU writes "ustar \0" */ + format = (' ' == block[262]) ? "gnu" : "ustar"; + } + else + { + if (! checksum_ok (block)) + return; /* not a tar */ + if (! plausible_v7 (block)) + return; /* not a tar */ + format = "v7"; + } + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_MIMETYPE, + "%s", + "application/x-tar")) + return; + memset (&st, + 0, + sizeof (st)); + longname[0] = '\0'; + longlink[0] = '\0'; + file_size = ec->get_size (ec->cls); + while (members < TAR_MAX_MEMBERS) + { + uint64_t size; + uint64_t mode; + uint64_t num; + unsigned char typeflag; + + if ( (UINT64_MAX != file_size) && + (offset + TAR_BLOCK > file_size) ) + break; + if (0 != offset) + { + if (! EXTRACTOR_forensic_read_ (ec, + (int64_t) offset, + block, + TAR_BLOCK)) + break; + } + if (is_zero_block (block)) + break; /* end-of-archive marker */ + if (! checksum_ok (block)) + break; /* corrupt or not a member header after all */ + if (! tar_number (&block[124], + 12, + &size)) + break; /* unreadable size: we can no longer find the next header */ + if ( (size > UINT64_MAX - 2 * TAR_BLOCK) || + ( (UINT64_MAX != file_size) && + (size > file_size) ) ) + break; /* would overflow the walk or run past the file */ + typeflag = block[156]; + switch (typeflag) + { + case 'x': + case 'g': + { + char pax[TAR_MAX_PAX]; + size_t want = (size > TAR_MAX_PAX) ? TAR_MAX_PAX : (size_t) size; + + /* pax extended header; the payload is metadata for the member + whose header comes next, not a member of its own */ + format = "pax"; + if ( (0 != want) && + (EXTRACTOR_forensic_read_ (ec, + (int64_t) (offset + TAR_BLOCK), + pax, + want)) ) + parse_pax (pax, + want, + longname, + longlink); + break; + } + case 'L': + case 'K': + { + size_t want = (size > TAR_MAX_LONGNAME) ? TAR_MAX_LONGNAME + : (size_t) size; + + /* GNU long name/link: the payload is the real name of the + member whose header comes next */ + format = "gnu"; + if ( (0 != want) && + (EXTRACTOR_forensic_read_ (ec, + (int64_t) (offset + TAR_BLOCK), + ('L' == typeflag) ? longname + : longlink, + want)) ) + { + if ('L' == typeflag) + longname[want] = '\0'; + else + longlink[want] = '\0'; + } + break; + } + default: + { + const char *member_name; + size_t member_len; + + members++; + /* regular files carry content; everything else has size 0 or a + size that does not describe stored bytes */ + if ( ('0' == typeflag) || + ('\0' == typeflag) || + ('7' == typeflag) ) + total_size += size; + if ('\0' != longname[0]) + { + member_name = longname; + member_len = strlen (longname); + } + else if ( (have_ustar) && + ('\0' != block[345]) ) + { + size_t plen; + size_t nlen; + + /* ustar splits long paths as prefix + '/' + name */ + plen = strnlen ((const char *) &block[345], + 155); + nlen = strnlen ((const char *) &block[0], + 100); + if (plen > sizeof (name) - 2) + plen = sizeof (name) - 2; + if (nlen > sizeof (name) - 2 - plen) + nlen = sizeof (name) - 2 - plen; + memcpy (name, + &block[345], + plen); + name[plen] = '/'; + memcpy (&name[plen + 1], + block, + nlen); + name[plen + 1 + nlen] = '\0'; + member_name = name; + member_len = plen + 1 + nlen; + } + else + { + member_name = (const char *) block; + member_len = strnlen ((const char *) block, + 100); + } + if (names_emitted < EXTRACTOR_FORENSIC_MAX_ITEMS) + { + names_emitted++; + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + "tar", + EXTRACTOR_METATYPE_FILENAME, + member_name, + member_len)) + return; + } + if ( ( ('1' == typeflag) || + ('2' == typeflag) ) && + (links_emitted < EXTRACTOR_FORENSIC_MAX_ITEMS) ) + { + const char *target; + size_t target_len; + + if ('\0' != longlink[0]) + { + target = longlink; + target_len = strlen (longlink); + } + else + { + target = (const char *) &block[157]; + target_len = strnlen (target, + 100); + } + links_emitted++; + if (0 != + EXTRACTOR_forensic_emit_text_ (ec, + "tar", + EXTRACTOR_METATYPE_TARGET_PATH, + target, + target_len)) + return; + } + /* -- ownership: the reason this plugin exists -- */ + if ( (have_ustar) && + (add_distinct_str (st.unames, + &st.n_unames, + (const char *) &block[265], + 32)) && + (0 != EXTRACTOR_forensic_emit_text_ (ec, + "tar", + EXTRACTOR_METATYPE_OWNER_USER, + (const char *) &block[265], + 32)) ) + return; + if ( (have_ustar) && + (add_distinct_str (st.gnames, + &st.n_gnames, + (const char *) &block[297], + 32)) && + (0 != EXTRACTOR_forensic_emit_text_ (ec, + "tar", + EXTRACTOR_METATYPE_OWNER_GROUP, + (const char *) &block[297], + 32)) ) + return; + if ( (tar_number (&block[108], + 8, + &num)) && + (add_distinct_num (st.uids, + &st.n_uids, + num)) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_OWNER_UID, + "%llu", + (unsigned long long) num)) ) + return; + if ( (tar_number (&block[116], + 8, + &num)) && + (add_distinct_num (st.gids, + &st.n_gids, + num)) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_OWNER_GID, + "%llu", + (unsigned long long) num)) ) + return; + if ( (tar_number (&block[100], + 8, + &mode)) && + (add_distinct_num (st.modes, + &st.n_modes, + mode & 07777)) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_PERMISSIONS, + "0%llo", + (unsigned long long) (mode + & 07777))) ) + return; + if ( (tar_number (&block[136], + 12, + &num)) && + (num < (uint64_t) INT64_MAX) && + (((int64_t) num) > newest) ) + newest = (int64_t) num; + longname[0] = '\0'; + longlink[0] = '\0'; + break; + } + } + offset += TAR_BLOCK + ((size + TAR_BLOCK - 1) / TAR_BLOCK) * TAR_BLOCK; + } + if (members >= TAR_MAX_MEMBERS) + truncated_walk = 1; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_FORMAT, + "%s", + format)) + return; + if (0 != + EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + members)) + return; + if (0 != + EXTRACTOR_forensic_emit_size_ (ec, + "tar", + EXTRACTOR_METATYPE_UNCOMPRESSED_SIZE, + total_size)) + return; + /* the newest member: when the archive was assembled, near enough */ + if (0 != + EXTRACTOR_forensic_emit_unix_time_ (ec, + "tar", + EXTRACTOR_METATYPE_MODIFICATION_DATE, + newest)) + return; + if ( (truncated_walk) && + (0 != EXTRACTOR_forensic_emit_ (ec, + "tar", + EXTRACTOR_METATYPE_COMMENT, + "member walk stopped at the cap of %u;" + " counts and sizes are lower bounds", + (unsigned int) TAR_MAX_MEMBERS)) ) + return; +} + + +/* end of tar_extractor.c */ diff --git a/src/plugins/test_apk.c b/src/plugins/test_apk.c @@ -0,0 +1,136 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_apk.c + * @brief testcase for the apk plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a UTF-8 text item. + */ +#define TXT(type, s) { type, EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + s, strlen (s) + 1, 0 } + + +/** + * Main function for the apk testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData apk_sol[] = { + TXT (EXTRACTOR_METATYPE_MIMETYPE, + "application/vnd.android.package-archive"), + /* everything down to the target architectures comes out of the + binary AndroidManifest.xml */ + TXT (EXTRACTOR_METATYPE_PACKAGE_NAME, "org.gnu.libextractor.apktest"), + TXT (EXTRACTOR_METATYPE_PACKAGE_VERSION, "1719"), + TXT (EXTRACTOR_METATYPE_SOFTWARE_VERSION, "1.19.0"), + /* android:compileSdkVersion="33" */ + TXT (EXTRACTOR_METATYPE_TOOLCHAIN, "Android API 33 (Android 13)"), + TXT (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, + "Android API 21 (Android 5.0)"), + TXT (EXTRACTOR_METATYPE_TARGET_OS, "Android API 33 (Android 13)"), + TXT (EXTRACTOR_METATYPE_PERMISSION, "android.permission.INTERNET"), + TXT (EXTRACTOR_METATYPE_PERMISSION, + "android.permission.ACCESS_FINE_LOCATION"), + TXT (EXTRACTOR_METATYPE_PERMISSION, "android.permission.READ_SMS"), + TXT (EXTRACTOR_METATYPE_PERMISSION, "android.permission.CAMERA"), + TXT (EXTRACTOR_METATYPE_PERMISSION, + "android.permission.RECEIVE_BOOT_COMPLETED"), + /* android:label is a literal string here; android:icon is a + resource reference and must not be reported */ + TXT (EXTRACTOR_METATYPE_TITLE, "libextractor APK test"), + TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "debuggable"), + TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "allowBackup: false"), + TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "usesCleartextTraffic: true"), + TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "networkSecurityConfig: present"), + /* read off the member names in the central directory */ + TXT (EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, "arm64-v8a"), + TXT (EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, "armeabi-v7a"), + TXT (EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, "x86_64"), + TXT (EXTRACTOR_METATYPE_COMMENT, "3 dex files"), + TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "12"), + TXT (EXTRACTOR_METATYPE_SIGNER, "JAR signing (v1), alias ANDROIDD"), + /* the certificate authority comes first in the PKCS#7 block, so + this also proves the SignerInfo is resolved rather than the first + certificate taken */ + TXT (EXTRACTOR_METATYPE_SIGNER, + "CN=libextractor test signer, O=GNU libextractor," + " OU=Testing, C=CH"), + /* from the APK Signing Block spliced in before the central + directory */ + TXT (EXTRACTOR_METATYPE_SIGNER, "APK Signature Scheme v2"), + TXT (EXTRACTOR_METATYPE_SIGNER, "APK Signature Scheme v3"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct SolutionData jar_sol[] = { + TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/java-archive"), + TXT (EXTRACTOR_METATYPE_TOOLCHAIN, "17.0.9+9 (Eclipse Adoptium)"), + TXT (EXTRACTOR_METATYPE_TOOLCHAIN, "17"), + TXT (EXTRACTOR_METATYPE_TOOLCHAIN, "17.0.9+9"), + TXT (EXTRACTOR_METATYPE_ENTRY_POINT, "org.gnu.libextractor.TestMain"), + TXT (EXTRACTOR_METATYPE_LIBRARY_DEPENDENCY, + "lib/support.jar lib/codec.jar"), + /* this header is 98 bytes long in the manifest and therefore + folded across two lines; getting it back in one piece proves the + continuation lines are unfolded */ + TXT (EXTRACTOR_METATYPE_TITLE, + "GNU libextractor JAR test archive with a very long title" + " that has to be folded"), + TXT (EXTRACTOR_METATYPE_SOFTWARE_VERSION, "1.19.0"), + TXT (EXTRACTOR_METATYPE_VENDOR, "GNU libextractor project"), + TXT (EXTRACTOR_METATYPE_SUBJECT, + "libextractor plugin test specification"), + TXT (EXTRACTOR_METATYPE_FORMAT_VERSION, "1.19"), + TXT (EXTRACTOR_METATYPE_ORGANIZATION, "GNU"), + TXT (EXTRACTOR_METATYPE_APPLICATION_ID, "org.gnu.libextractor.test"), + TXT (EXTRACTOR_METATYPE_PRODUCT_NAME, "libextractor test bundle"), + TXT (EXTRACTOR_METATYPE_PACKAGE_VERSION, "1.19.0.qualifier"), + TXT (EXTRACTOR_METATYPE_VENDOR, "GNU libextractor project"), + TXT (EXTRACTOR_METATYPE_DESCRIPTION, + "test bundle produced by gen_apk_testdata.sh"), + TXT (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, "JavaSE-17"), + TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "4"), + TXT (EXTRACTOR_METATYPE_SIGNER, "JAR signing (v1), alias CERT"), + TXT (EXTRACTOR_METATYPE_SIGNER, + "CN=libextractor test signer, O=GNU libextractor," + " OU=Testing, C=CH"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/apk_test.apk", apk_sol }, + { "testdata/apk_test.jar", jar_sol }, + { NULL, NULL } + }; + + return ET_main ("apk", ps); +} + + +/* end of test_apk.c */ diff --git a/src/plugins/test_diskimage.c b/src/plugins/test_diskimage.c @@ -0,0 +1,378 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_diskimage.c + * @brief testcase for the diskimage plugin + * @author Christian Grothoff + * + * The three inputs come from contrib/gen_diskimage_testdata.sh; read + * its header for what each of them is and where it was truncated. + * The VHDX is only the file identifier block, because [MS-VHDX] puts + * the region table at a fixed 192 KiB offset and there is no way to + * make a conformant VHDX small enough for a test corpus -- so the + * items the plugin reads out of the metadata region cannot be asserted + * here. + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Main function for the diskimage testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData qcow2_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-qemu-disk", + strlen ("application/x-qemu-disk") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "QCOW2", + strlen ("QCOW2") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "3", + strlen ("3") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_VOLUME_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1048576", + strlen ("1048576") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_BLOCK_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "512", + strlen ("512") + 1, + 0 + }, + /* the point of the whole file: a differencing image names the + image it depends on, so the investigator knows to go find it */ + { + EXTRACTOR_METATYPE_PARENT_IMAGE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "diskimage_base.qcow2", + strlen ("diskimage_base.qcow2") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ENTRY_COUNT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "0", + strlen ("0") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ATTRIBUTES, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "lazy refcounts", + strlen ("lazy refcounts") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "compression type: zstd", + strlen ("compression type: zstd") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct SolutionData vmdk_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-vmdk", + strlen ("application/x-vmdk") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "VMDK", + strlen ("VMDK") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1", + strlen ("1") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_VOLUME_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1048576", + strlen ("1048576") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_BLOCK_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "65536", + strlen ("65536") + 1, + 0 + }, + /* from the text descriptor embedded in the sparse extent header */ + { + EXTRACTOR_METATYPE_VOLUME_SERIAL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1f2e3d4c", + strlen ("1f2e3d4c") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "monolithicSparse", + strlen ("monolithicSparse") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_PARENT_IMAGE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "diskimage_base.vmdk", + strlen ("diskimage_base.vmdk") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "diskimage_test.vmdk", + strlen ("diskimage_test.vmdk") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ENTRY_COUNT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1", + strlen ("1") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_SOFTWARE_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "virtual hardware version 4", + strlen ("virtual hardware version 4") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_SOFTWARE_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "VMware Tools version 2147483647", + strlen ("VMware Tools version 2147483647") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "adapter type: ide", + strlen ("adapter type: ide") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "geometry: 2 cylinders, 16 heads, 63 sectors per track", + strlen ("geometry: 2 cylinders, 16 heads, 63 sectors per track") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct SolutionData vhdx_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-vhdx", + strlen ("application/x-vhdx") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "VHDX", + strlen ("VHDX") + 1, + 0 + }, + /* proves the 512 byte UTF-16LE creator string was decoded */ + { + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "libextractor testdata 1.0", + strlen ("libextractor testdata 1.0") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct SolutionData vhd_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/x-vhd", + strlen ("application/x-vhd") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "VHD", + strlen ("VHD") + 1, + 0 + }, + /* the footer is at the *end* of the file, so getting this at all + proves the tail seek fired */ + { + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "qemu 5.3", + strlen ("qemu 5.3") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_AUTHORING_OS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Windows", + strlen ("Windows") + 1, + 0 + }, + /* VHD counts seconds from 2000-01-01, not from the Unix epoch; + this value only comes out right if that offset was applied */ + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-01-01T00:00:00Z", + strlen ("2024-01-01T00:00:00Z") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_VOLUME_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1079296", + strlen ("1079296") + 1, + 0 + }, + /* a big-endian RFC 4122 UUID, unlike the mixed-endian GUIDs the + Windows formats use -- this asserts the other branch of + EXTRACTOR_forensic_emit_guid_() */ + { + EXTRACTOR_METATYPE_VOLUME_SERIAL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "3ffb1d5a-09a4-4e1d-9c2d-7f6a1b4c8e02", + strlen ("3ffb1d5a-09a4-4e1d-9c2d-7f6a1b4c8e02") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ATTRIBUTES, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "dynamic disk", + strlen ("dynamic disk") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1.0", + strlen ("1.0") + 1, + 0 + }, + /* the block size comes from the dynamic disk header, which is + reached through an offset stored in the footer */ + { + EXTRACTOR_METATYPE_BLOCK_SIZE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2097152", + strlen ("2097152") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/diskimage_test.qcow2", + qcow2_sol }, + { "testdata/diskimage_test.vmdk", + vmdk_sol }, + { "testdata/diskimage_test.vhdx", + vhdx_sol }, + { "testdata/diskimage_test.vhd", + vhd_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("diskimage", ps); +} + + +/* end of test_diskimage.c */ diff --git a/src/plugins/test_ebook.c b/src/plugins/test_ebook.c @@ -0,0 +1,126 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_ebook.c + * @brief testcase for the ebook plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a UTF-8 text item; @a s is compared as a prefix of what + * the plugin produced. + */ +#define TXT(type, s) { type, EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + s, strlen (s) + 1, 0 } + + +/** + * Main function for the ebook testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData epub_sol[] = { + TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/epub+zip"), + /* <package version="3.0"> */ + TXT (EXTRACTOR_METATYPE_FORMAT_VERSION, "3.0"), + TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"), + TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"), + /* dc:contributor with opf:role="bkp" is the tool, not a person */ + TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + "calibre (7.2.0) [https://calibre-ebook.com]"), + TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"), + TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-GB"), + TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01T00:00:00+00:00"), + TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"), + TXT (EXTRACTOR_METATYPE_KEYWORDS, "metadata extraction"), + /* the description is indented across three lines in the OPF, so + this also proves the white space collapsing */ + TXT (EXTRACTOR_METATYPE_DESCRIPTION, + "A minimal EPUB built by gen_ebook_testdata.sh, used to check" + " that the ebook plugin reads Dublin Core out of the OPF" + " package document."), + /* "&" has to come back as a bare ampersand */ + TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0) & nothing else"), + /* dc:identifier with opf:scheme="ISBN" */ + TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"), + /* ... and the one that is not an ISBN */ + TXT (EXTRACTOR_METATYPE_URI, + "urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719"), + TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00+00:00"), + TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, "gen_ebook_testdata.sh"), + TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"), + /* seven <item> entries in the manifest, and not the three + <itemref> entries in the spine */ + TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "7"), + TXT (EXTRACTOR_METATYPE_ENCRYPTION, + "META-INF/encryption.xml present (DRM or font obfuscation)"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct SolutionData mobi_sol[] = { + TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/x-mobipocket-ebook"), + /* PDB header offsets 36/40, seconds since 1904 */ + TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00Z"), + TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"), + /* ... and the same moment again, from EXTH 502 */ + TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00+00:00"), + TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "3"), + /* PalmDOC header offset 12 */ + TXT (EXTRACTOR_METATYPE_ENCRYPTION, "old Mobipocket encryption"), + /* MOBI header offset 28 */ + TXT (EXTRACTOR_METATYPE_CHARACTER_SET, "UTF-8"), + TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"), + TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"), + TXT (EXTRACTOR_METATYPE_DESCRIPTION, + "A hand-built minimal MOBI file used by the libextractor" + " test suite."), + TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"), + TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"), + TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01"), + TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + "calibre (7.2.0) [https://calibre-ebook.com]"), + TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0)"), + TXT (EXTRACTOR_METATYPE_SERIAL, "B00LIBEXTR7"), + TXT (EXTRACTOR_METATYPE_WATERMARK, "libextractor-test-watermark"), + TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "cdetype: EBOK"), + /* EXTH 503, which is why the PDB database name + "The_libextractor_Test_Book" is not reported as the title */ + TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"), + TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-gb"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/ebook_test.epub", epub_sol }, + { "testdata/ebook_test.mobi", mobi_sol }, + { NULL, NULL } + }; + + return ET_main ("ebook", ps); +} + + +/* end of test_ebook.c */ diff --git a/src/plugins/test_geotiff.c b/src/plugins/test_geotiff.c @@ -0,0 +1,263 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_geotiff.c + * @brief testcase for the geotiff plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Main function for the geotiff testcase. + * + * The fixture is built by contrib/gen_geotiff_testdata.sh: a 16x16 + * raster at EPSG:32633 whose tiepoint puts raster (0,0) at model + * (500000, 5400000) with a 10 m pixel. The bounding box below is the + * value that proves the plugin combined the tiepoint, the pixel scale + * and the image dimensions rather than reading any one of them alone. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData geotiff_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "image/tiff", + strlen ("image/tiff") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "GeoTIFF", + strlen ("GeoTIFF") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "16x16", + strlen ("16x16") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COLOR_DEPTH, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "8", + strlen ("8") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CHANNELS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1", + strlen ("1") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CODEC, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "none", + strlen ("none") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "libextractor gen_geotiff_testdata 1.0", + strlen ("libextractor gen_geotiff_testdata 1.0") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_DESCRIPTION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "libextractor GeoTIFF test fixture", + strlen ("libextractor GeoTIFF test fixture") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Christian Grothoff", + strlen ("Christian Grothoff") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COPYRIGHT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "CC0 1.0 Universal (public domain)", + strlen ("CC0 1.0 Universal (public domain)") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_DEVICE_MANUFACTURER, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "libextractor", + strlen ("libextractor") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_DEVICE_MODEL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "synthetic raster", + strlen ("synthetic raster") + 1, + 0 + }, + /* TIFF DateTime is `2026:03:14 09:15:00' in the file */ + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2026-03-14T09:15:00", + strlen ("2026-03-14T09:15:00") + 1, + 0 + }, + /* ProjectedCSTypeGeoKey 32633, resolved through the built-in + table of UTM zones */ + { + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "EPSG:32633 (WGS 84 / UTM zone 33N)", + strlen ("EPSG:32633 (WGS 84 / UTM zone 33N)") + 1, + 0 + }, + /* GTCitationGeoKey / PCSCitationGeoKey, `|'-terminated inside + GeoAsciiParamsTag */ + { + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "WGS 84 / UTM zone 33N", + strlen ("WGS 84 / UTM zone 33N") + 1, + 0 + }, + /* GeogCitationGeoKey */ + { + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "WGS 84", + strlen ("WGS 84") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "GeoTIFF model type: projected", + strlen ("GeoTIFF model type: projected") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "GeoTIFF raster type: pixel is area", + strlen ("GeoTIFF raster type: pixel is area") + 1, + 0 + }, + /* ModelPixelScaleTag, in the linear unit named by + ProjLinearUnitsGeoKey (9001, metre) */ + { + EXTRACTOR_METATYPE_IMAGE_RESOLUTION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "10 x 10 m/pixel", + strlen ("10 x 10 m/pixel") + 1, + 0 + }, + /* tiepoint (0,0) -> (500000, 5400000), 10 m pixel, 16x16 raster: + west,south,east,north in EPSG:32633 metres */ + { + EXTRACTOR_METATYPE_BOUNDING_BOX, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "500000,5399840,500160,5400000", + strlen ("500000,5399840,500160,5400000") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "GDAL nodata value: 0", + strlen ("GDAL nodata value: 0") + 1, + 0 + }, + /* out of GDAL_METADATA's <Item name="..."> elements */ + { + EXTRACTOR_METATYPE_SOURCE_DEVICE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "libextractor synthetic sensor", + strlen ("libextractor synthetic sensor") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "ACQUISITIONDATETIME=2026-03-14T09:15:00Z", + strlen ("ACQUISITIONDATETIME=2026-03-14T09:15:00Z") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "PROCESSING_LEVEL=L1C", + strlen ("PROCESSING_LEVEL=L1C") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/geotiff_test.tif", + geotiff_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("geotiff", ps); +} + + +/* end of test_geotiff.c */ diff --git a/src/plugins/test_gpx.c b/src/plugins/test_gpx.c @@ -0,0 +1,271 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_gpx.c + * @brief testcase for the gpx plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Main function for the gpx testcase. + * + * The expected values below all come from + * `contrib/gen_gpx_testdata.sh'; see that script for how the file is + * built. The interesting assertions are the ones that prove the + * scanner actually walked the track: the bounding box, the first + * coordinate, the point counts, the duration between the first and the + * last `<time>' and the haversine sum over the five points. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData gpx_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/gpx+xml", + strlen ("application/gpx+xml") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Garmin eTrex 30", + strlen ("Garmin eTrex 30") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_DEVICE_MODEL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Garmin eTrex 30", + strlen ("Garmin eTrex 30") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT_VERSION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1.1", + strlen ("1.1") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "WGS 84", + strlen ("WGS 84") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Zurich morning walk", + strlen ("Zurich morning walk") + 1, + 0 + }, + /* the `&' in the source must have come back out as `&' */ + { + EXTRACTOR_METATYPE_DESCRIPTION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Recorded on the way to the office & back", + strlen ("Recorded on the way to the office & back") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_KEYWORDS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "walk,commute,zurich", + strlen ("walk,commute,zurich") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Anna Mueller", + strlen ("Anna Mueller") + 1, + 0 + }, + /* <email id="anna.mueller" domain="example.com"/>, rejoined */ + { + EXTRACTOR_METATYPE_AUTHOR_EMAIL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "anna.mueller@example.com", + strlen ("anna.mueller@example.com") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COPYRIGHT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Anna Mueller", + strlen ("Anna Mueller") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_LICENSE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "https://creativecommons.org/publicdomain/zero/1.0/", + strlen ("https://creativecommons.org/publicdomain/zero/1.0/") + 1, + 0 + }, + /* <metadata><author><link href=> */ + { + EXTRACTOR_METATYPE_URL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "https://example.com/~anna/", + strlen ("https://example.com/~anna/") + 1, + 0 + }, + /* <metadata><link href=> */ + { + EXTRACTOR_METATYPE_URL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "https://example.com/tracks/2024-03-15.gpx", + strlen ("https://example.com/tracks/2024-03-15.gpx") + 1, + 0 + }, + /* <metadata><time> */ + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-03-15T07:12:00Z", + strlen ("2024-03-15T07:12:00Z") + 1, + 0 + }, + /* time of the first track point */ + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-03-15T07:12:33Z", + strlen ("2024-03-15T07:12:33Z") + 1, + 0 + }, + /* time of the last track point */ + { + EXTRACTOR_METATYPE_MODIFICATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-03-15T07:24:53Z", + strlen ("2024-03-15T07:24:53Z") + 1, + 0 + }, + /* west,south,east,north -- from the explicit <bounds> */ + { + EXTRACTOR_METATYPE_BOUNDING_BOX, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "8.541694,47.376890,8.552300,47.385100", + strlen ("8.541694,47.376890,8.552300,47.385100") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_GPS_LATITUDE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "47.376890", + strlen ("47.376890") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_GPS_LONGITUDE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "8.541694", + strlen ("8.541694") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ENTRY_COUNT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "5", + strlen ("5") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "5 track points, 2 waypoints, 2 route points", + strlen ("5 track points, 2 waypoints, 2 route points") + 1, + 0 + }, + /* 07:12:33 to 07:24:53 */ + { + EXTRACTOR_METATYPE_DURATION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "0:12:20", + strlen ("0:12:20") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_LOCATION_ELEVATION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "408.2-430.7 m", + strlen ("408.2-430.7 m") + 1, + 0 + }, + /* haversine sum over the five track points */ + { + EXTRACTOR_METATYPE_DISTANCE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1215 m", + strlen ("1215 m") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/gpx_test.gpx", + gpx_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("gpx", ps); +} + + +/* end of test_gpx.c */ diff --git a/src/plugins/test_heif.c b/src/plugins/test_heif.c @@ -0,0 +1,119 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_heif.c + * @brief testcase for the heif plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a UTF-8 solution entry. + */ +#define UTF8(t, s) { (t), EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + (s), strlen (s) + 1, 0 } + + +/** + * Main function for the heif testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + /* testdata/heif_test.heic carries two 'ispe' properties: the + thumbnail's 16x12 comes first in 'ipco', the primary item's 96x64 + second. Asserting on 96x64 is what proves the 'pitm' -> 'ipma' -> + 'ipco' association walk works rather than "report the first + 'ispe' you trip over". */ + struct SolutionData heif_heic_sol[] = { + UTF8 (EXTRACTOR_METATYPE_MIMETYPE, + "image/heic"), + UTF8 (EXTRACTOR_METATYPE_FORMAT, + "heic"), + UTF8 (EXTRACTOR_METATYPE_COMPATIBLE_BRANDS, + "mif1,heic,miaf,MiHB"), + UTF8 (EXTRACTOR_METATYPE_CODEC, + "HEVC (hvc1)"), + UTF8 (EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "96x64"), + UTF8 (EXTRACTOR_METATYPE_COLOR_DEPTH, + "10"), + UTF8 (EXTRACTOR_METATYPE_ORIENTATION, + "rotate 90 degrees counter-clockwise"), + UTF8 (EXTRACTOR_METATYPE_ORIENTATION, + "mirrored horizontally"), + UTF8 (EXTRACTOR_METATYPE_COLOR_PROFILE, + "nclx: primaries 9, transfer 16, matrix 9, full range"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "HDR: PQ transfer function (SMPTE ST 2084)"), + UTF8 (EXTRACTOR_METATYPE_ENTRY_COUNT, + "3"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "2 image items, 1 thumbnail, 0 auxiliary, 1 metadata item"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "contains an Exif metadata item"), + { 0, 0, NULL, NULL, 0, -1 } + }; + /* testdata/heif_test.avif has the generic major brand 'mif1' and only + names 'avif' among its compatible brands, so both the MIME type and + the codec have to come from that list. It also has no 'pixi', so + the bit depth has to come out of the 'av1C' configuration record. */ + struct SolutionData heif_avif_sol[] = { + UTF8 (EXTRACTOR_METATYPE_MIMETYPE, + "image/avif"), + UTF8 (EXTRACTOR_METATYPE_FORMAT, + "mif1"), + UTF8 (EXTRACTOR_METATYPE_COMPATIBLE_BRANDS, + "mif1,miaf,MA1B,avif"), + UTF8 (EXTRACTOR_METATYPE_CODEC, + "AV1 (av01)"), + UTF8 (EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "128x72"), + UTF8 (EXTRACTOR_METATYPE_COLOR_DEPTH, + "8"), + UTF8 (EXTRACTOR_METATYPE_COLOR_PROFILE, + "nclx: primaries 1, transfer 13, matrix 6, full range"), + UTF8 (EXTRACTOR_METATYPE_ENTRY_COUNT, + "1"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "1 image item, 0 thumbnails, 0 auxiliary, 0 metadata items"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/heif_test.heic", + heif_heic_sol }, + { "testdata/heif_test.avif", + heif_avif_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("heif", ps); +} + + +/* end of test_heif.c */ diff --git a/src/plugins/test_id3.c b/src/plugins/test_id3.c @@ -0,0 +1,143 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_id3.c + * @brief testcase for the id3 plugin + * @author Christian Grothoff + * + * The file under test is described in contrib/gen_id3_testdata.sh. + * The expectations worth understanding: + * + * - The title, artist and album prove the four ID3v2 text encodings: + * they are stored as UTF-8, as UTF-16 with a byte order mark and as + * ISO-8859-1 respectively, and all three carry characters outside + * ASCII, so a plugin that passed the bytes through unchanged would + * fail here rather than silently emit mojibake. + * - The genre is stored as "(52)" and has to come back resolved. + * - The lyrics and the cover picture are both longer than 127 bytes, + * which is where the synchsafe frame size of ID3v2.4 first differs + * from the plain integer of 2.3. Everything after them in the tag + * only parses if that was handled correctly. + * - The publication date and the comment are the only two items that + * come from the ID3v1 tag at the end of the file; every other field + * of that tag is deliberately different from its ID3v2 counterpart + * and must lose to it. + * - The encoder, encoder version, encoder settings, track gain, + * bitrate, bitrate mode and duration all come out of the Xing and + * LAME tags inside the first audio frame. + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Expected string item. + */ +#define STR(t, s) { t, EXTRACTOR_METAFORMAT_UTF8, "text/plain", s, \ + strlen (s) + 1, 0, 0, NULL } + + +/** + * Main function for the ID3 testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData id3_test_sol[] = { + STR (EXTRACTOR_METATYPE_MIMETYPE, "audio/mpeg"), + STR (EXTRACTOR_METATYPE_FORMAT_VERSION, "ID3v2.4.0"), + /* text frames, one per encoding */ + STR (EXTRACTOR_METATYPE_TITLE, "Grüße aus Köln"), + STR (EXTRACTOR_METATYPE_ARTIST, "Bœuf Trio"), + STR (EXTRACTOR_METATYPE_ALBUM, "Café Sessions"), + STR (EXTRACTOR_METATYPE_GENRE, "Electronic"), + STR (EXTRACTOR_METATYPE_TRACK_NUMBER, "3/12"), + STR (EXTRACTOR_METATYPE_DISC_NUMBER, "1/2"), + STR (EXTRACTOR_METATYPE_CREATION_DATE, "2024-03-14"), + STR (EXTRACTOR_METATYPE_COMPOSER, "Ada Lovelace"), + STR (EXTRACTOR_METATYPE_CONDUCTOR, "Grace Hopper"), + STR (EXTRACTOR_METATYPE_PUBLISHER, "GNU Records"), + STR (EXTRACTOR_METATYPE_COPYRIGHT, "2026 GNU libextractor"), + STR (EXTRACTOR_METATYPE_LANGUAGE, "deu"), + STR (EXTRACTOR_METATYPE_BEATS_PER_MINUTE, "128"), + STR (EXTRACTOR_METATYPE_LYRICS, + "One two three four five six seven eight nine ten. " + "This lyric frame exists only to be longer than one hundred " + "and twenty seven bytes, so that the synchsafe frame size of " + "ID3v2.4 differs from a plain big endian integer here."), + /* provenance */ + STR (EXTRACTOR_METATYPE_ENCODED_BY, "GNU libextractor test rig"), + STR (EXTRACTOR_METATYPE_ENCODER_SETTINGS, + "LAME 3.100 -V 5 --vbr-new"), + STR (EXTRACTOR_METATYPE_OWNER_USER, "licensee@example.org"), + STR (EXTRACTOR_METATYPE_SOURCE_DEVICE, "MPG/3"), + STR (EXTRACTOR_METATYPE_SOURCE_DEVICE, "DIG/A/T"), + STR (EXTRACTOR_METATYPE_SERIAL, + "http://musicbrainz.org: 9f1a3e6c-0000-4000-8000-0123456789ab"), + STR (EXTRACTOR_METATYPE_APPLICATION_ID, "com.apple.iTunes"), + STR (EXTRACTOR_METATYPE_PURCHASE_ACCOUNT, "buyer@example.org"), + STR (EXTRACTOR_METATYPE_URL, + "https://www.gnu.org/software/libextractor/"), + STR (EXTRACTOR_METATYPE_URL, "https://www.gnu.org/"), + { + EXTRACTOR_METATYPE_COVER_PICTURE, + EXTRACTOR_METAFORMAT_BINARY, + "image/png", + "\x89PNG\r\n\x1a\n", + 8, + 0, 0, NULL + }, + /* the MPEG frame header, the Xing tag and the LAME tag */ + STR (EXTRACTOR_METATYPE_CODEC, "MPEG 1 Layer III"), + STR (EXTRACTOR_METATYPE_FORMAT, "MPEG 1 Layer III, mono"), + STR (EXTRACTOR_METATYPE_SAMPLE_RATE, "44100"), + STR (EXTRACTOR_METATYPE_CHANNELS, "1"), + STR (EXTRACTOR_METATYPE_NOMINAL_BITRATE, "128000"), + STR (EXTRACTOR_METATYPE_BITRATE, "40233"), + STR (EXTRACTOR_METATYPE_BITRATE_MODE, "VBR"), + STR (EXTRACTOR_METATYPE_DURATION, "0:00:01"), + STR (EXTRACTOR_METATYPE_ENCODER, "LAME3.100"), + STR (EXTRACTOR_METATYPE_ENCODER_VERSION, "3.100"), + STR (EXTRACTOR_METATYPE_ENCODER_SETTINGS, + "lowpass 17000 Hz, quality 50, VBR method 5, " + "min bitrate 32 kbit/s, preset 450, delay 576, padding 1404"), + STR (EXTRACTOR_METATYPE_TRACK_GAIN, "-6.1 dB"), + /* the only two items the ID3v1 tag is allowed to contribute */ + STR (EXTRACTOR_METATYPE_PUBLICATION_DATE, "1999"), + STR (EXTRACTOR_METATYPE_COMMENT, "ID3v1 comment field"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/id3_test.mp3", + id3_test_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("id3", ps); +} + + +/* end of test_id3.c */ diff --git a/src/plugins/test_iso9660.c b/src/plugins/test_iso9660.c @@ -0,0 +1,110 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_iso9660.c + * @brief testcase for the iso9660 plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a plain UTF-8 text solution. + */ +#define TEXT_SOLUTION(type, value) \ + { (type), EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + (value), strlen (value) + 1, 0, 0, NULL } + + +/** + * Main function for the iso9660 testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData iso_test_sol[] = { + TEXT_SOLUTION (EXTRACTOR_METATYPE_MIMETYPE, + "application/x-iso9660-image"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "ISO 9660"), + /* supplementary volume descriptor with the "%/E" escape */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "Joliet"), + /* SUSP `SP' and `RR' entries in the root directory's "." record */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "Rock Ridge"), + /* boot record naming the El Torito specification */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_COMMENT, + "bootable (El Torito)"), + /* -- the identifier fields of the primary volume descriptor -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER, + "LINUX"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_VOLUME_NAME, + "FORENSIC_TEST_VOL"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PUBLISHER, + "GNU LIBEXTRACTOR PROJECT"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_DATA_PREPARER, + "LIBEXTRACTOR TEST SUITE"), + /* the application identifier: the mastering software */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + "MKISOFS 2.01 (LIBEXTRACTOR TEST IMAGE)"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_COPYRIGHT, + "COPYING.TXT"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_ABSTRACT, + "ABSTRACT.TXT"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_COMMENT, + "bibliographic file: BIBLIO.TXT"), + /* -- geometry: 21 sectors of 2048 bytes -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_BLOCK_SIZE, + "2048"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_VOLUME_SIZE, + "43008"), + /* volume set size is 2 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_ENTRY_COUNT, + "2"), + /* -- the four dec-datetime fields, each with a GMT offset of 0 -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_CREATION_DATE, + "2024-03-15T12:34:56Z"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_MODIFICATION_DATE, + "2025-01-02T03:04:05Z"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_EXPIRATION_DATE, + "2030-06-01T00:00:00Z"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_COMMENT, + "volume effective from 2024-03-16T00:00:00Z"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/iso9660_test.iso", + iso_test_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("iso9660", ps); +} + + +/* end of test_iso9660.c */ diff --git a/src/plugins/test_kml.c b/src/plugins/test_kml.c @@ -0,0 +1,193 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_kml.c + * @brief testcase for the kml plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Main function for the kml testcase. + * + * The expected values come from `contrib/gen_kml_testdata.sh'. The + * assertions that prove the scanner did its job rather than just + * recognising the magic are the bounding box (which is only correct if + * the lon,lat tuple order was honoured), the first coordinate, the + * placemark count, the description with its HTML stripped out of the + * CDATA section, and the LookAt comment -- the viewer's position, which + * must *not* have been reported as the data's position. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData kml_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/vnd.google-earth.kml+xml", + strlen ("application/vnd.google-earth.kml+xml") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COORDINATE_SYSTEM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "WGS 84", + strlen ("WGS 84") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Zurich morning walk", + strlen ("Zurich morning walk") + 1, + 0 + }, + /* out of a CDATA section, with the HTML markup removed */ + { + EXTRACTOR_METATYPE_DESCRIPTION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Exported from the handheld on 2024-03-15. See the index for the" + " other days.", + strlen ("Exported from the handheld on 2024-03-15. See the index" + " for the other days.") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Anna Mueller", + strlen ("Anna Mueller") + 1, + 0 + }, + /* <atom:link href=> */ + { + EXTRACTOR_METATYPE_URL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "https://example.com/~anna/", + strlen ("https://example.com/~anna/") + 1, + 0 + }, + /* <NetworkLink><Link><href> */ + { + EXTRACTOR_METATYPE_URL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "https://tracker.example.net/live/anna.kml", + strlen ("https://tracker.example.net/live/anna.kml") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "1 NetworkLink element; opening this document fetches remote" + " content", + strlen ("1 NetworkLink element; opening this document fetches" + " remote content") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-03-15T07:12:00Z", + strlen ("2024-03-15T07:12:00Z") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_LOCATION_NAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Home", + strlen ("Home") + 1, + 0 + }, + /* west,south,east,north over all three Placemarks */ + { + EXTRACTOR_METATYPE_BOUNDING_BOX, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "8.541694,47.376890,8.552300,47.385100", + strlen ("8.541694,47.376890,8.552300,47.385100") + 1, + 0 + }, + /* KML tuples are lon,lat -- getting this right is the whole test */ + { + EXTRACTOR_METATYPE_GPS_LATITUDE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "47.376890", + strlen ("47.376890") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_GPS_LONGITUDE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "8.541694", + strlen ("8.541694") + 1, + 0 + }, + /* the viewer's position, kept apart from the data's */ + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "LookAt viewpoint at 47.381000, 8.547000 (latitude, longitude)", + strlen ("LookAt viewpoint at 47.381000, 8.547000" + " (latitude, longitude)") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ENTRY_COUNT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "3", + strlen ("3") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/kml_test.kml", + kml_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("kml", ps); +} + + +/* end of test_kml.c */ diff --git a/src/plugins/test_lnk.c b/src/plugins/test_lnk.c @@ -0,0 +1,107 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_lnk.c + * @brief testcase for the lnk plugin + * @author Christian Grothoff + * + * The input is generated by contrib/gen_lnk_testdata.sh, which owns + * every constant asserted here. + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a solution whose value is spelled out in full. + */ +#define TEXT(type, value) \ + { type, EXTRACTOR_METAFORMAT_UTF8, "text/plain", value, \ + strlen (value) + 1, 0, 0, NULL } + + +/** + * Main function for the shell link testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData lnk_sol[] = { + TEXT (EXTRACTOR_METATYPE_MIMETYPE, + "application/x-ms-shortcut"), + /* the target's timestamps, taken from the shell link header */ + TEXT (EXTRACTOR_METATYPE_CREATION_DATE, + "2021-03-04T05:06:07Z"), + TEXT (EXTRACTOR_METATYPE_ACCESS_DATE, + "2022-07-08T09:10:11Z"), + TEXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, + "2021-06-15T14:30:00Z"), + TEXT (EXTRACTOR_METATYPE_EMBEDDED_FILE_SIZE, + "123456"), + TEXT (EXTRACTOR_METATYPE_ATTRIBUTES, + "READONLY, ARCHIVE"), + /* the VolumeID inside the LinkInfo structure */ + TEXT (EXTRACTOR_METATYPE_VOLUME_SERIAL, + "1A2B-3C4D"), + TEXT (EXTRACTOR_METATYPE_FILESYSTEM_TYPE, + "fixed disk"), + TEXT (EXTRACTOR_METATYPE_VOLUME_NAME, + "EVIDENCE"), + /* LocalBasePathUnicode joined with CommonPathSuffixUnicode */ + TEXT (EXTRACTOR_METATYPE_TARGET_PATH, + "C:\\Users\\analyst\\Desktop\\evidence.txt"), + /* the UTF-16 string data section */ + TEXT (EXTRACTOR_METATYPE_DESCRIPTION, + "Evidence file from the lab machine"), + TEXT (EXTRACTOR_METATYPE_TARGET_PATH, + "..\\..\\Desktop\\evidence.txt"), + TEXT (EXTRACTOR_METATYPE_WORKING_DIRECTORY, + "C:\\Users\\analyst\\Desktop"), + TEXT (EXTRACTOR_METATYPE_COMMAND_LINE, + "/quiet /log C:\\temp\\out.log"), + TEXT (EXTRACTOR_METATYPE_URI, + "%SystemRoot%\\system32\\shell32.dll"), + /* the TrackerDataBlock: the machine that made the shortcut */ + TEXT (EXTRACTOR_METATYPE_SOURCE_HOST, + "lab-ws-07"), + TEXT (EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER, + "9b3a5e1c-7d42-4f80-a1e6-0c2d4b6f8a31"), + TEXT (EXTRACTOR_METATYPE_SYSTEM_IDENTIFIER, + "5f3e2d1c-a4b6-11d2-8ae7-001b44113ab7"), + /* the node field of that version 1 UUID */ + TEXT (EXTRACTOR_METATYPE_MAC_ADDRESS, + "00:1b:44:11:3a:b7"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/lnk_test.lnk", + lnk_sol }, + { NULL, NULL } + }; + + return ET_main ("lnk", ps); +} + + +/* end of test_lnk.c */ diff --git a/src/plugins/test_mbox.c b/src/plugins/test_mbox.c @@ -0,0 +1,318 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_mbox.c + * @brief testcase for the mbox plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Main function for the mbox testcase. + * + * The mailbox comes from `contrib/gen_mbox_testdata.sh'. The + * assertions that matter are the ones a lazier parser would get wrong: + * the RFC 2047 subject (base64 UTF-8 and quoted-printable ISO-8859-1 in + * one field, with the white space between the two encoded words + * removed), the RFC 5322 date converted from +0100 to UTC, the three + * IP literals out of the Received chain plus the one from + * X-Originating-IP, the DKIM signing domain, and the attachment name + * that only exists as two RFC 2231 continuation segments. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData mbox_sol[] = { + { + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "application/mbox", + strlen ("application/mbox") + 1, + 0 + }, + /* the route, most recent hop first, cut at the receiving MTA's + timestamp but otherwise untouched */ + { + EXTRACTOR_METATYPE_RECEIVED_FROM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "from mx1.example.net (mx1.example.net [198.51.100.24])" + " by mail.example.org (Postfix) with ESMTPS id 4B7f2Z1qJz3xYm" + " for <bob@example.org>", + strlen ("from mx1.example.net (mx1.example.net [198.51.100.24])" + " by mail.example.org (Postfix) with ESMTPS id" + " 4B7f2Z1qJz3xYm for <bob@example.org>") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_RECEIVED_FROM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "from smtp6.example.net (smtp6.example.net" + " [IPv6:2001:db8:1234::a5]) by mx1.example.net (Postfix) with" + " ESMTPS id 77aa88bb99cc for <bob@example.org>", + strlen ("from smtp6.example.net (smtp6.example.net" + " [IPv6:2001:db8:1234::a5]) by mx1.example.net (Postfix)" + " with ESMTPS id 77aa88bb99cc for <bob@example.org>") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_RECEIVED_FROM, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "from wsbeta.corp.example.com (unknown [203.0.113.77])" + " by mx1.example.net (Postfix) with ESMTPSA id 9C1a4F0d7b" + " for <bob@example.org>", + strlen ("from wsbeta.corp.example.com (unknown [203.0.113.77])" + " by mx1.example.net (Postfix) with ESMTPSA id 9C1a4F0d7b" + " for <bob@example.org>") + 1, + 0 + }, + /* =?UTF-8?B?QW5uYSBNw7xsbGVy?= */ + { + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Anna M\xc3\xbcller", + strlen ("Anna M\xc3\xbcller") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_AUTHOR_EMAIL, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "anna.mueller@example.com", + strlen ("anna.mueller@example.com") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_RECIPIENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Bob Jones <bob@example.org>", + strlen ("Bob Jones <bob@example.org>") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_RECIPIENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "carol@example.org", + strlen ("carol@example.org") + 1, + 0 + }, + /* from Cc: */ + { + EXTRACTOR_METATYPE_RECIPIENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "dave@example.org", + strlen ("dave@example.org") + 1, + 0 + }, + /* base64/UTF-8 word, then a quoted-printable ISO-8859-1 word; the + white space between the two must have been dropped */ + { + EXTRACTOR_METATYPE_SUBJECT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "\xc3\x9c" "berweisung M\xc3\xa4rz 2024 Beleg", + strlen ("\xc3\x9c" "berweisung M\xc3\xa4rz 2024 Beleg") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "\xc3\x9c" "berweisung M\xc3\xa4rz 2024 Beleg", + strlen ("\xc3\x9c" "berweisung M\xc3\xa4rz 2024 Beleg") + 1, + 0 + }, + /* Date: Tue, 12 Mar 2024 09:41:07 +0100 */ + { + EXTRACTOR_METATYPE_CREATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2024-03-12T08:41:07Z", + strlen ("2024-03-12T08:41:07Z") + 1, + 0 + }, + /* the right-hand side is the composing workstation's name */ + { + EXTRACTOR_METATYPE_MESSAGE_ID, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "20240312084107.9C1a4F0d7b@wsbeta.corp.example.com", + strlen ("20240312084107.9C1a4F0d7b@wsbeta.corp.example.com") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_IN_REPLY_TO, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "20240311150122.A1B2C3@mail.example.net", + strlen ("20240311150122.A1B2C3@mail.example.net") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_IP_ADDRESS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "198.51.100.24", + strlen ("198.51.100.24") + 1, + 0 + }, + /* out of an `[IPv6:...]' literal */ + { + EXTRACTOR_METATYPE_IP_ADDRESS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "2001:db8:1234::a5", + strlen ("2001:db8:1234::a5") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_IP_ADDRESS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "203.0.113.77", + strlen ("203.0.113.77") + 1, + 0 + }, + /* out of X-Originating-IP, which is not bracketed the same way */ + { + EXTRACTOR_METATYPE_IP_ADDRESS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "192.0.2.153", + strlen ("192.0.2.153") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Microsoft Outlook 16.0", + strlen ("Microsoft Outlook 16.0") + 1, + 0 + }, + /* the `d=' tag of the DKIM-Signature */ + { + EXTRACTOR_METATYPE_SIGNER, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "example.com", + strlen ("example.com") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Authentication-Results: mail.example.org; dkim=pass" + " header.d=example.com; spf=pass smtp.mailfrom=example.com", + strlen ("Authentication-Results: mail.example.org; dkim=pass" + " header.d=example.com; spf=pass" + " smtp.mailfrom=example.com") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_ORGANIZATION, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Example GmbH", + strlen ("Example GmbH") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FORMAT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "multipart/mixed", + strlen ("multipart/mixed") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "MIME boundary: =_boundary_42", + strlen ("MIME boundary: =_boundary_42") + 1, + 0 + }, + { + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Kontoauszug_2024-03.pdf", + strlen ("Kontoauszug_2024-03.pdf") + 1, + 0 + }, + /* only present as `filename*0=' + `filename*1=' */ + { + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "Reisekosten.xlsx", + strlen ("Reisekosten.xlsx") + 1, + 0 + }, + /* the message is multipart, so the charset comes from its first + part rather than from its own Content-Type */ + { + EXTRACTOR_METATYPE_CHARACTER_SET, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "ISO-8859-15", + strlen ("ISO-8859-15") + 1, + 0 + }, + /* three messages: the `>From ' line in the third message's body is + the mboxo escaping and must not have been counted */ + { + EXTRACTOR_METATYPE_ENTRY_COUNT, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "3", + strlen ("3") + 1, + 0 + }, + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/mbox_test.mbox", + mbox_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("mbox", ps); +} + + +/* end of test_mbox.c */ diff --git a/src/plugins/test_pecoff.c b/src/plugins/test_pecoff.c @@ -0,0 +1,177 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_pecoff.c + * @brief testcase for the pecoff plugin + * @author Christian Grothoff + * + * Both inputs are generated by contrib/gen_pecoff_testdata.sh. The + * .exe is a real mingw-w64 image, which is what exercises the resource + * and import parsing; the .dll is assembled from the specification and + * carries the MSVC-only artifacts (Rich header, CodeView record) that + * mingw cannot produce. + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a solution whose value is spelled out in full. + */ +#define TEXT(type, value) \ + { type, EXTRACTOR_METAFORMAT_UTF8, "text/plain", value, \ + strlen (value) + 1, 0, 0, NULL } + +/** + * Shorthand for a solution matched by a regular expression, used where + * the exact value depends on the toolchain that built the input or on + * the last digit of a floating point conversion. + */ +#define MATCH(type, re) \ + { type, EXTRACTOR_METAFORMAT_UTF8, "text/plain", re, \ + strlen (re), 0, 1, NULL } + + +/** + * Main function for the PE/COFF testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData pecoff_exe_sol[] = { + TEXT (EXTRACTOR_METATYPE_MIMETYPE, + "application/vnd.microsoft.portable-executable"), + TEXT (EXTRACTOR_METATYPE_FORMAT, + "PE32+"), + TEXT (EXTRACTOR_METATYPE_RESOURCE_TYPE, + "executable"), + TEXT (EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, + "x86-64"), + TEXT (EXTRACTOR_METATYPE_TARGET_OS, + "Windows"), + TEXT (EXTRACTOR_METATYPE_SUBSYSTEM, + "Windows console"), + TEXT (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, + "4.0"), + /* the GNU linker writes its own version here */ + MATCH (EXTRACTOR_METATYPE_TOOLCHAIN, + "^linker 2\\.[0-9]+$"), + TEXT (EXTRACTOR_METATYPE_ENTRY_POINT, + "0x00001000"), + TEXT (EXTRACTOR_METATYPE_BUILD_DATE, + "2023-11-14T22:13:20Z"), + TEXT (EXTRACTOR_METATYPE_SECURITY_MITIGATIONS, + "HIGH_ENTROPY_VA, DYNAMIC_BASE, NX_COMPAT"), + TEXT (EXTRACTOR_METATYPE_LIBRARY_DEPENDENCY, + "KERNEL32.dll"), + /* MD5 of "kernel32.exitprocess"; matches what pefile computes */ + TEXT (EXTRACTOR_METATYPE_IMPORT_HASH, + "f9ade0aa18f660a34a4fa23392e21838"), + /* the VS_VERSIONINFO resource */ + TEXT (EXTRACTOR_METATYPE_FORMAT_VERSION, + "1.2.3.4"), + TEXT (EXTRACTOR_METATYPE_COMPANY, + "GNU libextractor"), + TEXT (EXTRACTOR_METATYPE_DESCRIPTION, + "libextractor PE test binary"), + TEXT (EXTRACTOR_METATYPE_SOFTWARE_VERSION, + "1.2.3.4"), + TEXT (EXTRACTOR_METATYPE_TITLE, + "pecoff_test"), + TEXT (EXTRACTOR_METATYPE_COPYRIGHT, + "CC0 1.0 Universal"), + TEXT (EXTRACTOR_METATYPE_FILENAME, + "pecoff_test.exe"), + TEXT (EXTRACTOR_METATYPE_PRODUCT_NAME, + "GNU libextractor test suite"), + TEXT (EXTRACTOR_METATYPE_PRODUCT_VERSION, + "5.6.7.8"), + MATCH (EXTRACTOR_METATYPE_ENTROPY, + "^2\\.[89][0-9] bits/byte \\(\\.rsrc\\)$"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct SolutionData pecoff_dll_sol[] = { + TEXT (EXTRACTOR_METATYPE_MIMETYPE, + "application/vnd.microsoft.portable-executable"), + TEXT (EXTRACTOR_METATYPE_FORMAT, + "PE32+"), + TEXT (EXTRACTOR_METATYPE_RESOURCE_TYPE, + "DLL"), + TEXT (EXTRACTOR_METATYPE_TARGET_ARCHITECTURE, + "x86-64"), + TEXT (EXTRACTOR_METATYPE_TARGET_OS, + "Windows"), + TEXT (EXTRACTOR_METATYPE_SUBSYSTEM, + "Windows GUI"), + TEXT (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, + "6.0"), + TEXT (EXTRACTOR_METATYPE_TOOLCHAIN, + "linker 14.38"), + TEXT (EXTRACTOR_METATYPE_ENTRY_POINT, + "0x00001000"), + TEXT (EXTRACTOR_METATYPE_BUILD_DATE, + "2020-09-13T12:26:40Z"), + TEXT (EXTRACTOR_METATYPE_SECURITY_MITIGATIONS, + "HIGH_ENTROPY_VA, DYNAMIC_BASE, NX_COMPAT, GUARD_CF," + " TERMINAL_SERVER_AWARE"), + /* the Rich header: the mask, then the tool records under it */ + TEXT (EXTRACTOR_METATYPE_BUILD_ID, + "rich:1a2b3c4d"), + TEXT (EXTRACTOR_METATYPE_TOOLCHAIN, + "Rich: prodID 0x0104, build 30729, count 34"), + TEXT (EXTRACTOR_METATYPE_TOOLCHAIN, + "Rich: prodID 0x00ff, build 30729, count 12"), + TEXT (EXTRACTOR_METATYPE_TOOLCHAIN, + "Rich: prodID 0x0105, build 30729, count 1"), + TEXT (EXTRACTOR_METATYPE_TOOLCHAIN, + "Rich: prodID 0x0001, build 0, count 5"), + /* the CodeView record, in symbol-server spelling */ + TEXT (EXTRACTOR_METATYPE_BUILD_ID, + "4C7B1E2A9F3D4B5C8E6F0A1B2C3D4E5F7"), + TEXT (EXTRACTOR_METATYPE_DEBUG_PATH, + "C:\\Users\\builder\\source\\repos\\pecoff_test" + "\\x64\\Release\\pecoff_test.pdb"), + TEXT (EXTRACTOR_METATYPE_ENTRY_COUNT, + "3"), + TEXT (EXTRACTOR_METATYPE_FILENAME, + "pecoff_test.dll"), + TEXT (EXTRACTOR_METATYPE_SIGNER, + "GNU libextractor test signer"), + MATCH (EXTRACTOR_METATYPE_ENTROPY, + "^7\\.[78][0-9] bits/byte \\(\\.pack\\)$"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/pecoff_test.exe", + pecoff_exe_sol }, + { "testdata/pecoff_test.dll", + pecoff_dll_sol }, + { NULL, NULL } + }; + + return ET_main ("pecoff", ps); +} + + +/* end of test_pecoff.c */ diff --git a/src/plugins/test_plist.c b/src/plugins/test_plist.c @@ -0,0 +1,116 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_plist.c + * @brief testcase for the plist plugin + * @author Christian Grothoff + * + * Both test files hold the same dictionary, written once in each of + * the two representations, so the two solution sets differ only where + * the representations genuinely do: the binary form has a version and + * an object count, the XML form does not. + * + * The values worth watching are the copyright (non-ASCII, and thus + * UTF-16BE in the binary file) and the backup date, which only comes + * out right if the 2001 epoch and the big-endian IEEE double were both + * handled. + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Expected string item. + */ +#define STR(t, s) { t, EXTRACTOR_METAFORMAT_UTF8, "text/plain", s, \ + strlen (s) + 1, 0, 0, NULL } + + +/** + * Main function for the plist testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData plist_binary_sol[] = { + STR (EXTRACTOR_METATYPE_MIMETYPE, "application/x-plist"), + STR (EXTRACTOR_METATYPE_FORMAT, "Binary property list"), + STR (EXTRACTOR_METATYPE_FORMAT_VERSION, "00"), + STR (EXTRACTOR_METATYPE_ENTRY_COUNT, "32"), + STR (EXTRACTOR_METATYPE_AUTHORING_OS, "23A344"), + STR (EXTRACTOR_METATYPE_FILENAME, "ExtractorTest"), + STR (EXTRACTOR_METATYPE_APPLICATION_ID, + "org.gnu.libextractor.testbundle"), + STR (EXTRACTOR_METATYPE_PACKAGE_NAME, "ExtractorTest"), + STR (EXTRACTOR_METATYPE_RESOURCE_TYPE, "APPL"), + STR (EXTRACTOR_METATYPE_SOFTWARE_VERSION, "1.19.0"), + STR (EXTRACTOR_METATYPE_PACKAGE_VERSION, "4211"), + STR (EXTRACTOR_METATYPE_TARGET_PLATFORM, "iphoneos"), + STR (EXTRACTOR_METATYPE_TOOLCHAIN, "1520"), + STR (EXTRACTOR_METATYPE_DEVICE_MODEL, "Test iPhone"), + STR (EXTRACTOR_METATYPE_DEVICE_MODEL, "iPhone14,5"), + STR (EXTRACTOR_METATYPE_UNKNOWN, "ExtractorTestInteger: 42"), + STR (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2024-03-14T15:09:26Z"), + STR (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, "14.0"), + STR (EXTRACTOR_METATYPE_COPYRIGHT, "Copyright © 2026 Grüße"), + STR (EXTRACTOR_METATYPE_SERIAL, "F2LXK0GTQ1GH"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct SolutionData plist_xml_sol[] = { + STR (EXTRACTOR_METATYPE_MIMETYPE, "application/x-plist"), + STR (EXTRACTOR_METATYPE_FORMAT, "XML property list"), + STR (EXTRACTOR_METATYPE_AUTHORING_OS, "23A344"), + STR (EXTRACTOR_METATYPE_FILENAME, "ExtractorTest"), + STR (EXTRACTOR_METATYPE_APPLICATION_ID, + "org.gnu.libextractor.testbundle"), + STR (EXTRACTOR_METATYPE_PACKAGE_NAME, "ExtractorTest"), + STR (EXTRACTOR_METATYPE_RESOURCE_TYPE, "APPL"), + STR (EXTRACTOR_METATYPE_SOFTWARE_VERSION, "1.19.0"), + STR (EXTRACTOR_METATYPE_PACKAGE_VERSION, "4211"), + STR (EXTRACTOR_METATYPE_TARGET_PLATFORM, "iphoneos"), + STR (EXTRACTOR_METATYPE_TOOLCHAIN, "1520"), + STR (EXTRACTOR_METATYPE_DEVICE_MODEL, "Test iPhone"), + STR (EXTRACTOR_METATYPE_DEVICE_MODEL, "iPhone14,5"), + STR (EXTRACTOR_METATYPE_UNKNOWN, "ExtractorTestInteger: 42"), + STR (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2024-03-14T15:09:26Z"), + STR (EXTRACTOR_METATYPE_MINIMUM_OS_VERSION, "14.0"), + STR (EXTRACTOR_METATYPE_COPYRIGHT, "Copyright © 2026 Grüße"), + STR (EXTRACTOR_METATYPE_SERIAL, "F2LXK0GTQ1GH"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/plist_binary.plist", + plist_binary_sol }, + { "testdata/plist_xml.plist", + plist_xml_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("plist", ps); +} + + +/* end of test_plist.c */ diff --git a/src/plugins/test_sqlite.c b/src/plugins/test_sqlite.c @@ -0,0 +1,100 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_sqlite.c + * @brief testcase for the sqlite plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a plain UTF-8 text solution. + */ +#define TEXT_SOLUTION(type, value) \ + { (type), EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + (value), strlen (value) + 1, 0, 0, NULL } + + +/** + * Main function for the sqlite testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData sqlite_db_sol[] = { + TEXT_SOLUTION (EXTRACTOR_METATYPE_MIMETYPE, + "application/vnd.sqlite3"), + /* PRAGMA page_size = 1024 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_BLOCK_SIZE, + "1024"), + /* PRAGMA journal_mode = WAL leaves write format version 2 behind */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_JOURNAL_MODE, + "wal"), + /* file change counter, in step with version-valid-for */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_CHANGE_COUNTER, + "11"), + /* five 1 KiB pages */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_ENTRY_COUNT, + "5"), + /* one freelist page left behind by the DELETE */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_FREE_SPACE, + "1024"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_SCHEMA_VERSION, + "schema cookie 4"), + /* PRAGMA user_version = 4242 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_SCHEMA_VERSION, + "user_version 4242"), + /* PRAGMA auto_vacuum = 2 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_ATTRIBUTES, + "incremental vacuum"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_CHARACTER_SET, + "UTF-8"), + /* PRAGMA application_id = 1279613012 == 0x4c455854 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_APPLICATION_ID, + "1279613012"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_APPLICATION_ID, + "LEXT"), + /* pinned to 3.45.1 by the generator */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_MODIFIED_BY_SOFTWARE, + "SQLite 3.45.1"), + /* 5 pages of 1024 bytes */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_VOLUME_SIZE, + "5120"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/sqlite_test.db", + sqlite_db_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("sqlite", ps); +} + + +/* end of test_sqlite.c */ diff --git a/src/plugins/test_tar.c b/src/plugins/test_tar.c @@ -0,0 +1,158 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_tar.c + * @brief testcase for the tar plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a plain UTF-8 text solution. + */ +#define TEXT_SOLUTION(type, value) \ + { (type), EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + (value), strlen (value) + 1, 0, 0, NULL } + + +/** + * Main function for the tar testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + struct SolutionData tar_test_sol[] = { + TEXT_SOLUTION (EXTRACTOR_METATYPE_MIMETYPE, + "application/x-tar"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FORMAT, + "ustar"), + /* seven members; the directory and the links count too */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_ENTRY_COUNT, + "7"), + /* four regular files of 44 bytes each */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_UNCOMPRESSED_SIZE, + "176"), + /* the newest member mtime, 1710506096 */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_MODIFICATION_DATE, + "2024-03-15T12:34:56Z"), + /* -- ownership, the point of the plugin -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_USER, + "forensic"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_USER, + "root"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GROUP, + "analysts"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GROUP, + "root"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_UID, + "1000"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_UID, + "0"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GID, + "100"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GID, + "0"), + /* -- the distinct permission bits -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0755"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0644"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0777"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0600"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0640"), + /* -- member names, including one reassembled from the ustar + prefix field -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/notes.txt"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/script.sh"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/latest.txt"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/hardlink.txt"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/root-owned.txt"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "evidence/dir00/dir01/dir02/dir03/dir04/dir05/dir06/dir07" + "/dir08/dir09/dir10/dir11/dir12/dir13/dir14/deep.txt"), + /* -- link targets -- */ + TEXT_SOLUTION (EXTRACTOR_METATYPE_TARGET_PATH, + "notes.txt"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_TARGET_PATH, + "evidence/notes.txt"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + /* The archive plugin's sample doubles as a second, independently + produced input: it is a GNU-format tar made in 2002, so it checks + the magic discrimination and the ownership fields against a file + this plugin's generator never touched. */ + struct SolutionData archive_test_sol[] = { + TEXT_SOLUTION (EXTRACTOR_METATYPE_MIMETYPE, + "application/x-tar"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FORMAT, + "gnu"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_ENTRY_COUNT, + "2"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_USER, + "grothoff"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GROUP, + "grothoff"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_UID, + "1000"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_OWNER_GID, + "1000"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_PERMISSIONS, + "0644"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_UNCOMPRESSED_SIZE, + "1980"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_MODIFICATION_DATE, + "2002-06-05T22:29:14Z"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "test.html"), + TEXT_SOLUTION (EXTRACTOR_METATYPE_FILENAME, + "test.jpg"), + { 0, 0, NULL, NULL, 0, -1, 0, NULL } + }; + struct ProblemSet ps[] = { + { "testdata/tar_test.tar", + tar_test_sol }, + { "testdata/archive_test.tar", + archive_test_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("tar", ps); +} + + +/* end of test_tar.c */ diff --git a/src/plugins/test_webp.c b/src/plugins/test_webp.c @@ -0,0 +1,93 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file plugins/test_webp.c + * @brief testcase for the webp plugin + * @author Christian Grothoff + */ +#include "platform.h" +#include "test_lib.h" + + +/** + * Shorthand for a UTF-8 solution entry. + */ +#define UTF8(t, s) { (t), EXTRACTOR_METAFORMAT_UTF8, "text/plain", \ + (s), strlen (s) + 1, 0 } + + +/** + * Main function for the webp testcase. + * + * @param argc number of arguments (ignored) + * @param argv arguments (ignored) + * @return 0 on success + */ +int +main (int argc, char *argv[]) +{ + /* testdata/webp_test.webp is an extended (VP8X) animated WebP with + every optional chunk the container defines. The canvas size comes + from the VP8X header, the codec from the sub-chunk inside the first + animation frame, and the duration from summing the three frame + durations (100 + 150 + 250 ms), so the assertions below only hold + if the whole chunk walk works. */ + struct SolutionData webp_sol[] = { + UTF8 (EXTRACTOR_METATYPE_MIMETYPE, + "image/webp"), + UTF8 (EXTRACTOR_METATYPE_FORMAT, + "WebP (extended)"), + UTF8 (EXTRACTOR_METATYPE_CODEC, + "VP8L (lossless)"), + UTF8 (EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "40x24"), + UTF8 (EXTRACTOR_METATYPE_ATTRIBUTES, + "alpha, animation, ICC profile, EXIF, XMP"), + /* a bare number, as the heif plugin reports it: whether there is an + alpha channel is asserted through ATTRIBUTES above */ + UTF8 (EXTRACTOR_METATYPE_COLOR_DEPTH, + "8"), + UTF8 (EXTRACTOR_METATYPE_COLOR_PROFILE, + "ICC profile, 136 bytes"), + UTF8 (EXTRACTOR_METATYPE_ENTRY_COUNT, + "3"), + UTF8 (EXTRACTOR_METATYPE_DURATION, + "500 ms"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "animation: 3 frames, looping forever, background #ff204080 BGRA"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "contains an EXIF chunk"), + UTF8 (EXTRACTOR_METATYPE_COMMENT, + "contains an XMP chunk"), + { 0, 0, NULL, NULL, 0, -1 } + }; + struct ProblemSet ps[] = { + { "testdata/webp_test.webp", + webp_sol }, + { NULL, NULL } + }; + + (void) argc; + (void) argv; + return ET_main ("webp", ps); +} + + +/* end of test_webp.c */ diff --git a/src/plugins/testdata/README.forensics b/src/plugins/testdata/README.forensics @@ -0,0 +1,418 @@ +heif -- heif_test.heic, heif_test.avif + Generated by contrib/gen_heif_testdata.sh (python3 only, deterministic). + Hand-constructed ISO base media file format containers: the box tree, + the item structures and the property associations are exactly what + ISO/IEC 14496-12, ISO/IEC 23008-12 and the AV1 Image File Format + specify, but the 'mdat' payload is a placeholder rather than a real + HEVC or AV1 bitstream, so the files identify but do not decode. That + is on purpose: the plugin is a container parser and the values it must + report are then known exactly. + + heif_test.heic (916 bytes): major brand 'heic'. Three items -- the + primary image (id 1), a thumbnail (id 2, tied to the primary by an + 'iref' of type 'thmb') and an 'Exif' metadata item (id 3). The 'ipco' + property container holds the thumbnail's 16x12 'ispe' *first* and the + primary item's 96x64 'ispe' second, so a parser that reports the first + 'ispe' it finds gets the wrong answer and only a correct + pitm -> ipma -> ipco walk reports 96x64. Also carries pixi (10 bit), + irot (90 degrees), imir, colr nclx with transfer 16 (PQ, i.e. HDR) and + an hvcC shell. + + heif_test.avif (631 bytes): major brand 'mif1' with 'avif' only among + the compatible brands, which is what several AVIF writers emit; both + the MIME type and the codec therefore have to come from that list. + One image item, 128x72, no 'pixi', so the bit depth has to be taken + from the 'av1C' configuration record. + + Licence: CC0. No third party content. + +webp -- webp_test.webp + Generated by contrib/gen_webp_testdata.sh (needs ffmpeg with libwebp, + Debian package "ffmpeg", plus python3; deterministic). + + webp_test.webp (752 bytes): an extended (VP8X) animated WebP carrying + every optional chunk the container defines -- ICCP, ANIM, three ANMF + frames, EXIF and XMP -- with the alpha, animation, ICC, EXIF and XMP + feature flags set. Canvas 40x24. The three frames hold a genuine + lossless VP8L bitstream encoded by libwebp from a solid colour + generated by lavfi (nothing is read from disk); libwebp reads the + result back as a valid 3 frame RGBA animation. Frame durations are + 100, 150 and 250 ms, summing to 500 ms. The ICC (136 bytes), EXIF and + XMP payloads are placeholders: the plugin only reports that they are + present and parsing them is the exiv2 plugin's job. + + Licence: CC0. No third party content. + +sqlite -- sqlite_test.db + SQLite 3 database, 5120 bytes. Generated by contrib/gen_sqlite_testdata.sh + with the sqlite3 CLI: 1 KiB pages, UTF-8, incremental auto-vacuum, WAL + journal mode, application_id 1279613012 (= "LEXT"), user_version 4242, two + tables and an index, with rows deleted and an index dropped so that a + freelist page survives. The generator then rewrites the SQLite version at + header offset 96 to 3045001 ("SQLite 3.45.1") and makes version-valid-for + agree with the change counter, so the file does not depend on the sqlite3 + that produced it. test_sqlite asserts on the page size, journal mode, + change counter, page count, free space, schema cookie, user_version, + vacuum mode, encoding, application id (numeric and as its four-character + tag), writer version and logical size. Licence: CC0 / public domain. + +tar -- tar_test.tar + POSIX ustar archive, 10240 bytes. Generated by contrib/gen_tar_testdata.sh + using Python's tarfile with every ownership field set explicitly: + uname "forensic", gname "analysts", uid 1000, gid 100, plus one member + owned by root/root. Contains a directory, three regular files, a symlink, + a hard link and one member whose 107-character path has to be split across + the ustar prefix field. Modification times are hard-coded (1700000000 and + 1710506096); nothing is read from the machine that runs the generator. + test_tar asserts on the format, member count, summed member size, newest + mtime, both owners and groups, both uid/gid pairs, the five distinct + permission masks, all seven member names and both link targets. + Licence: CC0 / public domain. + +iso9660 -- iso9660_test.iso + ISO 9660 (ECMA-119) image, 43008 bytes = 21 sectors. Generated by + contrib/gen_iso9660_testdata.sh, which builds the volume descriptor set + by hand with Python struct rather than calling genisoimage or xorriso: a + mastering program stamps its own name and the current time into the + descriptors, which would be neither reproducible nor assertable. Layout is + 16 zero sectors of system area, then the primary volume descriptor, an + El Torito boot record, a Joliet supplementary descriptor (escape "%/E"), + the set terminator and one root directory extent whose "." record carries + SUSP "SP" and Rock Ridge "RR" entries. This is over the 32 KB guideline + and cannot be smaller: ECMA-119 puts the first descriptor at byte 32768. + test_iso9660 asserts on the system/volume/publisher/preparer/application + identifiers, the copyright, abstract and bibliographic file identifiers, + the block size, volume size, volume set size, all four dec-datetime fields + and the Joliet, Rock Ridge and El Torito detections. + Licence: CC0 / public domain. + + +geotiff -- geotiff_test.tif + Generated by contrib/gen_geotiff_testdata.sh (python3 only, + deterministic; no gdal-bin needed). Hand-built with struct rather + than produced by gdal_translate: GDAL's own output is several KB and + its numbers are whatever the projection happens to give, whereas a + hand-built file can be 1 KB and can use round numbers for the + tiepoint and the pixel scale so that the bounding box the plugin + derives from them is an exact constant to assert on. + + geotiff_test.tif (1099 bytes): a classic little-endian TIFF 6.0 + baseline image, 16x16, 8 bit greyscale, uncompressed, with the + GeoTIFF 1.1 tags for EPSG:32633 (WGS 84 / UTM zone 33N). + ModelPixelScaleTag is 10 m, ModelTiepointTag maps raster (0,0) to + model (500000, 5400000), so the extent is exactly + 500000,5399840,500160,5400000 -- a value that can only come out right + if the tiepoint, the scale and the image dimensions were all read and + combined. GeoKeyDirectoryTag carries eight keys covering all three + TIFFTagLocation cases the specification allows (inline SHORT and + '|'-terminated substrings of GeoAsciiParamsTag), and GDAL's two + private tags (GDAL_METADATA with three <Item> elements, GDAL_NODATA) + are present so the provenance scan is exercised too. Byte order, + BigTIFF and the ModelTransformationTag path are covered by variants + built during development but not committed, since asserting on them + would only duplicate this file's expectations. + + Licence: CC0. No third party content. + +diskimage (QCOW2 / VMDK / VHD / VHDX) +------------------------------- + + Generated by contrib/gen_diskimage_testdata.sh (needs qemu-utils and + python3). qemu-img is the producer for all three; python3 only + overwrites the fields qemu-img randomises per run, so re-running the + script gives byte-identical files. + + diskimage_test.qcow2 (1792 bytes): a QCOW2 version 3 differencing + image, 1 MiB virtual, 512 byte clusters, backed by + "diskimage_base.qcow2" (which the script creates in a temporary + directory and throws away -- the plugin never opens it, only reports + its name). Created with lazy_refcounts=on and compression_type=zstd + so that the compatible-feature and incompatible-feature paths and the + version 3 compression type byte are all exercised. Untouched + qemu-img output; the QCOW2 header has no per-run fields. + + diskimage_test.vmdk (16384 bytes): a monolithicSparse VMDK, 1 MiB + virtual, 64 KiB grains, backed by "diskimage_base.vmdk". Covers both + the binary sparse extent header and the text descriptor embedded in + it at offset 512. The CID and parentCID qemu-img picks at random are + rewritten to 1f2e3d4c and 5a6b7c8d and the descriptor slot is padded + back to its original length, so no offset in the header changes. + TRUNCATED from the 64 KiB qemu-img writes to 16 KiB: everything past + the descriptor is zeroed grain tables. The file is therefore no + longer a mountable VMDK, but the header (80 bytes) and the descriptor + (offset 512, 10 KiB) are intact and are all the plugin reads. + + diskimage_test.vhdx (4096 bytes): the file identifier block of a + VHDX, with the UTF-16LE creator string overwritten with a fixed value + so the test does not depend on the local qemu version. TRUNCATED, + and unavoidably so: [MS-VHDX] fixes the header at 64 KiB and the + region table at 192 KiB, and requires the metadata region to start at + a 1 MiB boundary, so the smallest conformant VHDX is over 2 MiB. The + test can only assert the signature-level items; the plugin's region + table, metadata table and parent locator walk was checked by hand + against full qemu-img output and against a hand-built conformant + image with a parent locator and a non-zero LogGuid. + + There is no diskimage_test.vhd: the plugin reads the Microsoft VHD + ("conectix") footer as well, but src/plugins/Makefile.am has no + EXTRA_DIST entry for one. Run the generator with GEN_FULL=1 to get + a deterministic sample plus the untruncated VMDK and VHDX. + + Licence: CC0. The images contain no data at all, only headers. + +pecoff -- pecoff_test.exe, pecoff_test.dll + Generated by contrib/gen_pecoff_testdata.sh. The .exe needs the + Debian package "gcc-mingw-w64-x86-64"; the .dll needs only python3. + Both are deterministic: the linker is told not to insert a timestamp + and the script writes a fixed one afterwards. + + pecoff_test.exe (4608 bytes): a real x86-64 PE32+ image built by + mingw-w64 from a three line program, linked without the C runtime + (-nostdlib) so that its whole import table is KERNEL32.dll!ExitProcess + and the import hash is therefore a value the test can spell out -- + MD5("kernel32.exitprocess"), which is what pefile computes for the + same file. Carries a VERSIONINFO resource written by windres, which + is what exercises the resource directory walk and the nested, + 32-bit-aligned, UTF-16 StringFileInfo parser. TimeDateStamp is set to + 1700000000 (2023-11-14T22:13:20Z) after linking. + + pecoff_test.dll (3584 bytes): assembled from the PE specification with + python3, because mingw cannot produce the two artifacts that matter + most here. A Rich header (Microsoft's linker writes those and nothing + else does) masked with the key 0x1A2B3C4D and holding four + (product id, build, use count) triples; a CodeView RSDS record naming + C:\Users\builder\source\repos\pecoff_test\x64\Release\pecoff_test.pdb + with a fixed GUID and age; an export directory with three named + exports; a certificate table; and a .pack section filled with SHA-256 + output so that its entropy lands where a packed section's would. The + certificate table is not a signature and verifies as nothing -- it is + the shortest DER that gives the commonName scan something to find, and + the name it carries is "GNU libextractor test signer". Verified + against pefile: machine, subsystem, Rich values, CodeView record and + export names all read back as intended. + + Licence: CC0. No third party content. + +lnk -- lnk_test.lnk + Generated by contrib/gen_lnk_testdata.sh (python3 only, deterministic). + + lnk_test.lnk (706 bytes): a Windows shell link assembled from + [MS-SHLLINK] rather than produced by Windows, so that every value the + test asserts is a constant the generator owns. It carries a + LinkTargetIDList (two shell items) that the parser has to step over + without looking at, a LinkInfo whose VolumeID has drive type "fixed", + serial 1A2B-3C4D and the unicode volume label EVIDENCE, and whose + target path is split across LocalBasePathUnicode + ("C:\Users\analyst\Desktop\") and CommonPathSuffixUnicode + ("evidence.txt") so that the join is tested; UTF-16 string data for + all five optional strings; and a TrackerDataBlock naming the machine + "lab-ws-07" whose droid file identifier 5f3e2d1c-a4b6-11d2-8ae7- + 001b44113ab7 is a version 1 UUID carrying the MAC address + 00:1b:44:11:3a:b7. The three FILETIMEs in the header are the + target's, and all differ: 2021-03-04T05:06:07Z (creation), + 2021-06-15T14:30:00Z (write) and 2022-07-08T09:10:11Z (access). + + Cross-checked with liblnk's lnkinfo (20240423), which reads back the + same timestamps, serial, label, paths, strings and tracker GUIDs. + + Licence: CC0. No third party content. The machine name, MAC address + and user name are invented. + +plist -- plist_binary.plist, plist_xml.plist + Generated by contrib/gen_plist_testdata.sh (python3 only, using the + standard library's plistlib; deterministic). Both files hold the same + dictionary written once in each representation, so the two solution + sets in test_plist.c differ only where the formats genuinely do. + + The dictionary mixes the keys of an application bundle Info.plist + (CFBundleIdentifier, CFBundleShortVersionString, DTPlatformName, + DTXcode, BuildMachineOSBuild, ...) with those of an iOS backup + Info.plist (DeviceName, ProductType, SerialNumber, LastBackupDate), + which is the forensic case the plugin exists for, plus one integer + under an unrecognised key so the "key: value" fallback is exercised. + + plist_binary.plist (613 bytes): bplist00, 32 objects, two byte offsets + and one byte object references. NSHumanReadableCopyright is + "Copyright (C) 2026 Grüße" with a copyright sign and two umlauts, so + plistlib has to store it as a UTF-16BE string object (marker 0x6n) -- + the string decoder most likely to be got wrong, and the one the ASCII + path (0x5n) cannot fake. LastBackupDate is a date object (0x33), an + eight byte big-endian IEEE double counting seconds from 2001-01-01, + and only comes back as 2024-03-14T15:09:26Z if both the epoch shift + and the byte order were handled. + + plist_xml.plist (1126 bytes): the same dictionary as a DTD-bound XML + document, which is what the bounded <key>/value scan reads. + + The parser was also checked against the XML property lists shipped by + Debian's cmake and qt6 packages, and against a binary plist produced + by libplist's plistutil (an implementation independent of plistlib, + and one that interleaves the key and value object indices rather than + writing them in two runs). Those files are not committed. + + Licence: CC0. Every identifier, serial number and device name in the + file is invented. + +id3 -- id3_test.mp3 + Generated by contrib/gen_id3_testdata.sh. Debian packages needed: + python3, python3-mutagen and lame. Deterministic. + + id3_test.mp3 (7546 bytes): one second of a 440 Hz sine at 44100 Hz, + mono, written as a WAV by python and encoded by LAME 3.100 at -V 5 + --vbr-new. LAME is the encoder rather than ffmpeg on purpose: ffmpeg + writes its own Xing/LAME tag stamped "Lavf lame", which does not carry + the lowpass, ReplayGain, preset or VBR method fields that make the + LAME tag the fingerprint it is. Note that the encoder version string + ("LAME3.100") and the settings summary derived from that tag are + asserted on literally, so regenerating with a different LAME release + means updating those two expectations. + + The tag is ID3v2.4 and exercises all four text encodings in one file: + TIT2 is UTF-8, TPE1 is UTF-16 with a byte order mark, TALB is + ISO-8859-1, and all three carry characters outside ASCII, so a plugin + that passed the bytes through would produce invalid UTF-8 rather than + quietly wrong text. TCON is stored as "(52)" and has to come back + resolved to "Electronic". The USLT and APIC frames are both longer + than 127 bytes, which is where the synchsafe frame size of ID3v2.4 + first differs from the plain big-endian integer of 2.3 -- everything + after them only parses if that was handled correctly. The provenance + frames are TENC, TSSE, TOWN, TFLT, TMED, a UFID with a MusicBrainz + style owner, a PRIV owned by com.apple.iTunes, two URL frames and a + TXXX with the description "apID". The attached picture is a 208 byte + PNG built by the generator with zlib. + + A hand-written ID3v1 tag follows the audio. Every one of its fields + is deliberately different from its ID3v2 counterpart, so only the two + it alone supplies -- the year 1999 and the comment -- may reach the + caller; anything else showing up means the gap filling is broken. + + The parser was also checked during development against files written + by three other producers, none of them committed: mutagen at ID3v2.3 + (plain frame sizes, UTF-16 text), ffmpeg's own ID3v2.3 writer and its + "Lavf"/"Lavc" Xing tag, a hand-built ID3v2.2 tag with three character + frame identifiers, a hand-built unsynchronised ID3v2.3 tag, an + ID3v1-only file, a stereo file (which puts the Xing tag at offset 36 + rather than 21) and a CBR file (an "Info" tag rather than "Xing"). + + Licence: CC0. The audio is a generated sine tone; the licensee + address, purchase account, MusicBrainz identifier and every name in + the tag are invented. + +apk -- apk_test.apk, apk_test.jar + Generated by contrib/gen_apk_testdata.sh (python3 only, deterministic). + + apk_test.apk (4723 bytes): a hand-built Android package. The + AndroidManifest.xml is real binary XML written out from the chunk + layout in AOSP's ResourceTypes.h -- a UTF-16LE string pool (which is + what aapt emits for the manifest), a resource map with the attribute + resource ids read off a real aapt2 build, and START/END_ELEMENT + chunks. It declares package org.gnu.libextractor.apktest, + versionCode 1719, versionName 1.19.0, compileSdkVersion 33, + minSdkVersion 21, targetSdkVersion 33 and five uses-permission + elements. The <application> element mixes literal and reference + valued attributes on purpose: android:label is a literal string and + must be reported, android:icon and android:networkSecurityConfig are + resource references and a bare @0x7f... must not be reported as a + title. The archive also holds three classes*.dex stubs, three + lib/<abi>/ stubs (arm64-v8a, armeabi-v7a, x86_64), a META-INF JAR + manifest (so that the plugin has to prefer the Android manifest), a + v1 signature under the alias ANDROIDD, and an APK Signing Block with + v2 and v3 identifiers spliced in between the last entry and the + central directory. + + apk_test.jar (3108 bytes): a JAR whose META-INF/MANIFEST.MF carries + the JDK identification (Created-By, Build-Jdk, Build-Jdk-Spec), the + Implementation-*/Specification-* triples, the OSGi Bundle-* headers + and a Class-Path. Implementation-Title is 98 bytes long and is + therefore folded across two lines, so reading it back proves the + continuation lines are unfolded. + + Both archives are signed by the same PKCS#7 block, a throwaway + two-element chain ("libextractor test CA" issuing "libextractor test + signer", serial 0x1719) generated once with openssl and embedded in + the generator as base64; the private keys were discarded, so the + signature protects nothing. The certificate authority comes first + inside the certificates set, which is what makes the test prove that + the signer is resolved through the SignerInfo's issuerAndSerialNumber + rather than simply taken from the first certificate. + + Licence: CC0. No third party content; every name, package + identifier and certificate subject is invented. + +ebook -- ebook_test.epub, ebook_test.mobi + Generated by contrib/gen_ebook_testdata.sh (python3 only, + deterministic). Calibre is deliberately not used. + + ebook_test.epub (2645 bytes): an OCF container with the uncompressed + `mimetype' member first as the specification requires, a + META-INF/container.xml pointing at OEBPS/content.opf, and an EPUB 3 + package document with rich Dublin Core: title, creator, publisher, + language, date, two subjects, a description indented across three + lines (so that reading it back proves the white space collapsing), a + rights field containing an & entity, one dc:identifier with + opf:scheme="ISBN" and one that is a urn:uuid, a dc:contributor with + opf:role="bkp" naming the tool, <meta name="calibre:timestamp">, + <meta name="generator"> and <meta property="dcterms:modified">. The + manifest holds seven <item> entries and the spine three <itemref> + entries, so the entry count also proves the two are told apart. A + META-INF/encryption.xml (font obfuscation) is present. + + ebook_test.mobi (946 bytes): a hand-built Palm database, type/creator + BOOKMOBI, three records. The PDB creation and modification times are + written in the 1904 epoch (2026-01-01 and 2026-02-01). Record 0 + holds a PalmDOC header with encryption type 1, a 232 byte MOBI header + declaring UTF-8 text and the EXTH flag, and an EXTH header with + seventeen records covering author, publisher, description, ISBN, + subject, published date, contributor (the tool), rights, ASIN, + creator software triple, watermark, cdetype, last update time, + updated title and language, followed by the full name. + + Licence: CC0. No third party content; the book, its author, its + ISBN and its watermark are invented. + +gpx / kml / mbox +---------------- + + Generated by contrib/gen_gpx_testdata.sh, contrib/gen_kml_testdata.sh + and contrib/gen_mbox_testdata.sh. All three are plain text written + out literally by a POSIX shell heredoc -- no tools, no timestamps + taken from the clock, byte-for-byte reproducible. + + gpx_test.gpx (2146 bytes): a GPX 1.1 track with creator="Garmin eTrex + 30" (the field that names the recording device, and the one the + plugin also reports as a device model), a full <metadata> block with + name, desc carrying an & entity, author name and a split + <email id= domain=>, copyright with a <license>, two <link href=>, + a <time> of 2024-03-15T07:12:00Z, keywords, and an explicit <bounds>. + Two <wpt>, a <rte> of two <rtept> and one <trkseg> of five <trkpt> + with elevations 408.2-430.7 m and times from 07:12:33 to 07:24:53, + which is where the 0:12:20 duration and the 1215 m haversine sum in + the test come from. The coordinates are a walk through central + Zurich, invented, not anybody's real track. + + kml_test.kml (2027 bytes): an OGC KML 2.2 Document over the same + coordinates, so that the two plugins can be compared -- note that the + tuples are lon,lat,alt, the reverse of the GPX attribute order, which + is what the bounding box assertion actually tests. It has a + <description> whose HTML sits in a CDATA section, an atom:author and + atom:link, a <TimeStamp>, a <LookAt> (the viewer's position, which + the plugin reports as a comment rather than as the data's position), + three Placemarks (two Points and a LineString) and a <NetworkLink> + whose <href> points at a remote host. + + mbox_test.mbox (3504 bytes): three messages in mbox (mboxo) form. + The first carries a three-hop Received chain with two IPv4 literals + and one [IPv6:...] literal, an X-Originating-IP, a DKIM-Signature + with d=example.com, an Authentication-Results, X-Mailer, an RFC 2047 + From and Subject (base64/UTF-8 and quoted-printable/ISO-8859-1 in one + field), In-Reply-To and References, and a multipart/mixed body with + two attachments -- one with a quoted filename and one that only + exists as RFC 2231 filename*0/filename*1 continuation segments. + Date: Tue, 12 Mar 2024 09:41:07 +0100 is Unix time 1710232867. The + third message's body holds a `>From ' line, the mboxo escaping, so + the message count staying at three proves the separator scan is not + fooled by it. + + Licence: CC0. All names, addresses and hosts are from the RFC 2606 / + RFC 5737 / RFC 3849 reserved ranges; the coordinates and the mail are + invented. diff --git a/src/plugins/testdata/apk_test.apk b/src/plugins/testdata/apk_test.apk Binary files differ. diff --git a/src/plugins/testdata/apk_test.jar b/src/plugins/testdata/apk_test.jar Binary files differ. diff --git a/src/plugins/testdata/diskimage_test.qcow2 b/src/plugins/testdata/diskimage_test.qcow2 Binary files differ. diff --git a/src/plugins/testdata/diskimage_test.vhd b/src/plugins/testdata/diskimage_test.vhd Binary files differ. diff --git a/src/plugins/testdata/diskimage_test.vhdx b/src/plugins/testdata/diskimage_test.vhdx Binary files differ. diff --git a/src/plugins/testdata/diskimage_test.vmdk b/src/plugins/testdata/diskimage_test.vmdk Binary files differ. diff --git a/src/plugins/testdata/ebook_test.epub b/src/plugins/testdata/ebook_test.epub Binary files differ. diff --git a/src/plugins/testdata/ebook_test.mobi b/src/plugins/testdata/ebook_test.mobi Binary files differ. diff --git a/src/plugins/testdata/geotiff_test.tif b/src/plugins/testdata/geotiff_test.tif Binary files differ. diff --git a/src/plugins/testdata/gpx_test.gpx b/src/plugins/testdata/gpx_test.gpx @@ -0,0 +1,68 @@ +<?xml version="1.0" encoding="UTF-8"?> +<gpx version="1.1" + creator="Garmin eTrex 30" + xmlns="http://www.topografix.com/GPX/1/1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd"> + <metadata> + <name>Zurich morning walk</name> + <desc>Recorded on the way to the office & back</desc> + <author> + <name>Anna Mueller</name> + <email id="anna.mueller" domain="example.com"/> + <link href="https://example.com/~anna/"> + <text>Anna's tracks</text> + </link> + </author> + <copyright author="Anna Mueller"> + <year>2024</year> + <license>https://creativecommons.org/publicdomain/zero/1.0/</license> + </copyright> + <link href="https://example.com/tracks/2024-03-15.gpx"> + <text>Original upload</text> + </link> + <time>2024-03-15T07:12:00Z</time> + <keywords>walk,commute,zurich</keywords> + <bounds minlat="47.376890" minlon="8.541694" + maxlat="47.385100" maxlon="8.552300"/> + </metadata> + <wpt lat="47.376890" lon="8.541694"> + <ele>408.2</ele> + <name>Home</name> + </wpt> + <wpt lat="47.385100" lon="8.552300"> + <ele>419.3</ele> + <name>Office</name> + </wpt> + <rte> + <name>Planned route</name> + <rtept lat="47.377000" lon="8.542000"/> + <rtept lat="47.385000" lon="8.552000"/> + </rte> + <trk> + <name>Zurich morning walk</name> + <type>walking</type> + <trkseg> + <trkpt lat="47.376890" lon="8.541694"> + <ele>408.2</ele> + <time>2024-03-15T07:12:33Z</time> + </trkpt> + <trkpt lat="47.378120" lon="8.543210"> + <ele>412.5</ele> + <time>2024-03-15T07:14:03Z</time> + </trkpt> + <trkpt lat="47.380450" lon="8.546800"> + <ele>425.0</ele> + <time>2024-03-15T07:17:45Z</time> + </trkpt> + <trkpt lat="47.382990" lon="8.549900"> + <ele>430.7</ele> + <time>2024-03-15T07:21:10Z</time> + </trkpt> + <trkpt lat="47.385100" lon="8.552300"> + <ele>419.3</ele> + <time>2024-03-15T07:24:53Z</time> + </trkpt> + </trkseg> + </trk> +</gpx> diff --git a/src/plugins/testdata/heif_test.avif b/src/plugins/testdata/heif_test.avif Binary files differ. diff --git a/src/plugins/testdata/heif_test.heic b/src/plugins/testdata/heif_test.heic Binary files differ. diff --git a/src/plugins/testdata/id3_test.mp3 b/src/plugins/testdata/id3_test.mp3 Binary files differ. diff --git a/src/plugins/testdata/iso9660_test.iso b/src/plugins/testdata/iso9660_test.iso Binary files differ. diff --git a/src/plugins/testdata/kml_test.kml b/src/plugins/testdata/kml_test.kml @@ -0,0 +1,68 @@ +<?xml version="1.0" encoding="UTF-8"?> +<kml xmlns="http://www.opengis.net/kml/2.2" + xmlns:atom="http://www.w3.org/2005/Atom" + xmlns:gx="http://www.google.com/kml/ext/2.2"> + <Document> + <name>Zurich morning walk</name> + <description><![CDATA[<p>Exported from the handheld on + <b>2024-03-15</b>. See <a href="https://example.com/~anna/">the + index</a> for the other days.</p>]]></description> + <atom:author> + <atom:name>Anna Mueller</atom:name> + </atom:author> + <atom:link href="https://example.com/~anna/"/> + <TimeStamp> + <when>2024-03-15T07:12:00Z</when> + </TimeStamp> + <LookAt> + <longitude>8.547000</longitude> + <latitude>47.381000</latitude> + <altitude>0</altitude> + <range>1200</range> + <tilt>45</tilt> + <heading>10</heading> + </LookAt> + <Style id="track"> + <LineStyle> + <color>ff0000ff</color> + <width>4</width> + </LineStyle> + </Style> + <Placemark> + <name>Home</name> + <description>Start of the recording</description> + <Point> + <coordinates>8.541694,47.376890,408.2</coordinates> + </Point> + </Placemark> + <Placemark> + <name>Office</name> + <description>End of the recording</description> + <Point> + <coordinates>8.552300,47.385100,419.3</coordinates> + </Point> + </Placemark> + <Placemark> + <name>Walked track</name> + <styleUrl>#track</styleUrl> + <LineString> + <tessellate>1</tessellate> + <coordinates> + 8.541694,47.376890,408.2 + 8.543210,47.378120,412.5 + 8.546800,47.380450,425.0 + 8.549900,47.382990,430.7 + 8.552300,47.385100,419.3 + </coordinates> + </LineString> + </Placemark> + <NetworkLink> + <name>Live position</name> + <Link> + <href>https://tracker.example.net/live/anna.kml</href> + <refreshMode>onInterval</refreshMode> + <refreshInterval>30</refreshInterval> + </Link> + </NetworkLink> + </Document> +</kml> diff --git a/src/plugins/testdata/lnk_test.lnk b/src/plugins/testdata/lnk_test.lnk Binary files differ. diff --git a/src/plugins/testdata/mbox_test.mbox b/src/plugins/testdata/mbox_test.mbox @@ -0,0 +1,92 @@ +From anna.mueller@example.com Tue Mar 12 09:41:09 2024 +Received: from mx1.example.net (mx1.example.net [198.51.100.24]) + by mail.example.org (Postfix) with ESMTPS id 4B7f2Z1qJz3xYm + for <bob@example.org>; Tue, 12 Mar 2024 09:41:09 +0100 (CET) +Received: from smtp6.example.net (smtp6.example.net [IPv6:2001:db8:1234::a5]) + by mx1.example.net (Postfix) with ESMTPS id 77aa88bb99cc + for <bob@example.org>; Tue, 12 Mar 2024 09:41:08 +0100 (CET) +Received: from wsbeta.corp.example.com (unknown [203.0.113.77]) + by mx1.example.net (Postfix) with ESMTPSA id 9C1a4F0d7b + for <bob@example.org>; Tue, 12 Mar 2024 09:41:07 +0100 (CET) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=example.com; + s=sel2024; t=1710232867; h=from:to:subject:date; + bh=2jmj7l5rSw0yVb-vlWAYkK-YBwk=; + b=Zm9vYmFyYmF6cXV4Y29ycmdlZ3JhdWx0 +Authentication-Results: mail.example.org; + dkim=pass header.d=example.com; + spf=pass smtp.mailfrom=example.com +X-Originating-IP: [192.0.2.153] +Message-ID: <20240312084107.9C1a4F0d7b@wsbeta.corp.example.com> +In-Reply-To: <20240311150122.A1B2C3@mail.example.net> +References: <20240310090000.000001@mail.example.net> + <20240311150122.A1B2C3@mail.example.net> +Date: Tue, 12 Mar 2024 09:41:07 +0100 +From: =?UTF-8?B?QW5uYSBNw7xsbGVy?= <anna.mueller@example.com> +To: Bob Jones <bob@example.org>, carol@example.org +Cc: dave@example.org +Subject: =?UTF-8?B?w5xiZXJ3ZWlzdW5nIE3DpHJ6?= =?ISO-8859-1?Q?_2024_Beleg?= +Organization: Example GmbH +X-Mailer: Microsoft Outlook 16.0 +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=_boundary_42" + +--=_boundary_42 +Content-Type: text/plain; charset="ISO-8859-15" +Content-Transfer-Encoding: quoted-printable + +Hallo Bob, + +anbei der Beleg und die Reisekosten. + +Gruss, Anna + +--=_boundary_42 +Content-Type: application/pdf; name="Kontoauszug_2024-03.pdf" +Content-Disposition: attachment; filename="Kontoauszug_2024-03.pdf" +Content-Transfer-Encoding: base64 + +JVBERi0xLjQKJcfsj6IKMSAwIG9iago8PC9UeXBlL0NhdGFsb2c+PgplbmRvYmoK +dHJhaWxlcgo8PC9Sb290IDEgMCBSPj4KJSVFT0YK + +--=_boundary_42 +Content-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +Content-Disposition: attachment; + filename*0="Reise"; + filename*1="kosten.xlsx" +Content-Transfer-Encoding: base64 + +UEsDBBQAAAAIAA== + +--=_boundary_42-- + +From bob@example.org Tue Mar 12 10:02:11 2024 +Received: from mail.example.org (localhost [127.0.0.1]) + by mail.example.org (Postfix) with ESMTP id 0A1B2C3D4E5F + for <anna.mueller@example.com>; Tue, 12 Mar 2024 10:02:11 +0100 (CET) +Message-ID: <20240312090211.0A1B2C3D4E5F@mail.example.org> +In-Reply-To: <20240312084107.9C1a4F0d7b@wsbeta.corp.example.com> +Date: Tue, 12 Mar 2024 10:02:11 +0100 +From: Bob Jones <bob@example.org> +To: =?UTF-8?B?QW5uYSBNw7xsbGVy?= <anna.mueller@example.com> +Subject: Re: Beleg +User-Agent: Mozilla Thunderbird 115.8.0 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8; format=flowed + +Danke, ist angekommen. + +From daemon@example.org Tue Mar 12 10:15:00 2024 +Received: from relay.example.org (relay.example.org [198.51.100.9]) + by mail.example.org (Postfix) with ESMTP id 112233445566 + for <anna.mueller@example.com>; Tue, 12 Mar 2024 10:15:00 +0100 (CET) +Message-ID: <20240312091500.112233445566@relay.example.org> +Date: Tue, 12 Mar 2024 10:15:00 +0100 +From: Mail Delivery System <daemon@example.org> +To: anna.mueller@example.com +Subject: Delivery Status Notification (Delay) +Content-Type: text/plain; charset=US-ASCII + +>From the transcript of the session: + + 451 4.4.1 reply: read timeout, delivery deferred + diff --git a/src/plugins/testdata/pecoff_test.dll b/src/plugins/testdata/pecoff_test.dll Binary files differ. diff --git a/src/plugins/testdata/pecoff_test.exe b/src/plugins/testdata/pecoff_test.exe Binary files differ. diff --git a/src/plugins/testdata/plist_binary.plist b/src/plugins/testdata/plist_binary.plist Binary files differ. diff --git a/src/plugins/testdata/plist_xml.plist b/src/plugins/testdata/plist_xml.plist @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>BuildMachineOSBuild</key> + <string>23A344</string> + <key>CFBundleExecutable</key> + <string>ExtractorTest</string> + <key>CFBundleIdentifier</key> + <string>org.gnu.libextractor.testbundle</string> + <key>CFBundleName</key> + <string>ExtractorTest</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleShortVersionString</key> + <string>1.19.0</string> + <key>CFBundleVersion</key> + <string>4211</string> + <key>DTPlatformName</key> + <string>iphoneos</string> + <key>DTXcode</key> + <string>1520</string> + <key>DeviceName</key> + <string>Test iPhone</string> + <key>ExtractorTestInteger</key> + <integer>42</integer> + <key>LastBackupDate</key> + <date>2024-03-14T15:09:26Z</date> + <key>MinimumOSVersion</key> + <string>14.0</string> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2026 Grüße</string> + <key>ProductType</key> + <string>iPhone14,5</string> + <key>SerialNumber</key> + <string>F2LXK0GTQ1GH</string> +</dict> +</plist> diff --git a/src/plugins/testdata/sqlite_test.db b/src/plugins/testdata/sqlite_test.db Binary files differ. diff --git a/src/plugins/testdata/tar_test.tar b/src/plugins/testdata/tar_test.tar Binary files differ. diff --git a/src/plugins/testdata/webp_test.webp b/src/plugins/testdata/webp_test.webp Binary files differ. diff --git a/src/plugins/thumbnailgtk_extractor.c b/src/plugins/thumbnailgtk_extractor.c @@ -44,6 +44,13 @@ */ #define MAX_IMAGE_SIZE (32 * 1024 * 1024) +/** + * Largest decoded image we are willing to let gdk-pixbuf allocate, + * in pixels. At four bytes per pixel this caps the decode at 128 MB. + * Anything past it gets its dimensions reported but no thumbnail. + */ +#define MAX_IMAGE_PIXELS (32 * 1024 * 1024) + /** * Global handle to MAGIC data. @@ -52,6 +59,72 @@ static magic_t magic; /** + * What #handle_size_prepared saw in the image header, so that the + * dimensions we report are the ones the file claims and not the ones + * we asked the loader to decode at. + */ +struct SizeInfo +{ + /** + * Width from the image header, in pixels. + */ + gint width; + + /** + * Height from the image header, in pixels. + */ + gint height; + + /** + * Set once the header announces more than #MAX_IMAGE_PIXELS pixels. + * The caller stops feeding the loader when it sees this, which is + * what actually prevents the allocation. + */ + int too_big; +}; + + +/** + * Called by the loader once it has parsed the image header and knows + * how big the image claims to be, but before it decodes any of it. + * + * #MAX_IMAGE_SIZE bounds the *compressed* file, which says nothing at + * all about the decoded size: GIF stores its dimensions in 16 bits, so + * a four-kilobyte file can legally announce 65535x65535 and cost + * 65535 * 65535 * 4 bytes -- about 17 GB -- to decode. + * + * Note that #gdk_pixbuf_loader_set_size does NOT help here. It is + * documented as scaling the image while loading, but the GIF loader + * decodes at full size and calls #gdk_pixbuf_scale_simple itself + * afterwards, so the peak allocation is unchanged -- measured at 16 GB + * for the reproducer either way. The only thing that keeps the memory + * from being allocated is to stop handing the loader data, which is + * why this merely raises a flag and the feeding loop acts on it. + * + * @param loader the loader parsing the image + * @param width width the image header claims + * @param height height the image header claims + * @param cls our `struct SizeInfo *' + */ +static void +handle_size_prepared (GdkPixbufLoader *loader, + gint width, + gint height, + gpointer cls) +{ + struct SizeInfo *si = cls; + + (void) loader; + si->width = width; + si->height = height; + if ( (0 >= width) || + (0 >= height) || + (((gint64) width) * ((gint64) height) > MAX_IMAGE_PIXELS) ) + si->too_big = 1; +} + + +/** * Main method for the gtk-thumbnailer plugin. * * @param ec extraction context @@ -63,6 +136,7 @@ void EXTRACTOR_thumbnailgtk_extract_method (struct EXTRACTOR_ExtractContext *ec) { GdkPixbufLoader *loader; + struct SizeInfo si; GdkPixbuf *in; GdkPixbuf *out; size_t length; @@ -114,12 +188,58 @@ EXTRACTOR_thumbnailgtk_extract_method (struct EXTRACTOR_ExtractContext *ec) } loader = gdk_pixbuf_loader_new (); - gdk_pixbuf_loader_write (loader, - buf, - size, NULL); + si.width = 0; + si.height = 0; + si.too_big = 0; + g_signal_connect (loader, + "size-prepared", + G_CALLBACK (&handle_size_prepared), + &si); + /* Feed the loader in chunks rather than in one call, so that + #handle_size_prepared -- which fires as soon as the header has been + parsed -- can stop us before the image body is decoded. A single + write of the whole buffer would decode everything before returning + and the flag would come too late to be of any use. */ + off = 0; + while (off < size) + { + size_t chunk = size - off; + + if (chunk > 4096) + chunk = 4096; + if (! gdk_pixbuf_loader_write (loader, + ((const guchar *) buf) + off, + chunk, + NULL)) + break; + if (si.too_big) + break; + off += chunk; + } free (buf); gdk_pixbuf_loader_close (loader, NULL); + if (si.too_big) + { + /* Report what the header claimed, but do not decode it. */ + if (0 < si.width) + { + snprintf (format, + sizeof (format), + "%ux%u", + (unsigned int) si.width, + (unsigned int) si.height); + (void) ec->proc (ec->cls, + "thumbnailgtk", + EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + format, + strlen (format) + 1); + } + g_object_unref (loader); + return; + } in = gdk_pixbuf_loader_get_pixbuf (loader); if (NULL == in) { @@ -128,8 +248,12 @@ EXTRACTOR_thumbnailgtk_extract_method (struct EXTRACTOR_ExtractContext *ec) } g_object_ref (in); g_object_unref (loader); - height = gdk_pixbuf_get_height (in); - width = gdk_pixbuf_get_width (in); + /* The pixbuf may have been decoded smaller than the file asked for + (see #handle_size_prepared), so report what the header said. */ + height = (0 < si.height) ? (unsigned long) si.height + : (unsigned long) gdk_pixbuf_get_height (in); + width = (0 < si.width) ? (unsigned long) si.width + : (unsigned long) gdk_pixbuf_get_width (in); snprintf (format, sizeof (format), "%ux%u", diff --git a/src/plugins/webp_extractor.c b/src/plugins/webp_extractor.c @@ -0,0 +1,700 @@ +/* + This file is part of libextractor. + Copyright (C) 2026 Vidyut Samanta and Christian Grothoff + + libextractor is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + libextractor is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with libextractor; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. + */ +/** + * @file plugins/webp_extractor.c + * @brief plugin to support WebP images + * @author Christian Grothoff + * + * WebP is a RIFF container. We read the chunk headers and the few + * bytes of each bitstream header that carry the canvas size; no VP8 or + * VP8L data is decoded and libwebp is not linked, because this is a + * fast identification pass. + * + * Layout, per the WebP container specification: + * + * "RIFF" <uint32le size> "WEBP" + * then chunks of <FourCC> <uint32le size> <payload> [pad byte] + * + * A simple file has exactly one chunk, "VP8 " (lossy) or "VP8L" + * (lossless). An extended file starts with "VP8X", whose feature flags + * say what else to expect: "ICCP", "ALPH", "ANIM"/"ANMF", "EXIF" and + * "XMP ". + * + * The AVI plugin ("riff") also opens RIFF files, so this one insists on + * the "WEBP" form type; the two never claim the same file. + */ +#include "platform.h" +#include "extractor.h" +#include "forensics.h" + + +/** + * Maximum number of top level chunks we look at. + */ +#define WEBP_MAX_CHUNKS 1024 + +/** + * Maximum number of animation frames we walk. The frame count we + * report is the number we actually saw, so this caps the reported + * value too, which is what a characterisation pass wants. + */ +#define WEBP_MAX_FRAMES 512 + +/** + * Total number of bytes we are willing to read from one file. + */ +#define WEBP_READ_BUDGET (128 * 1024) + +/* VP8X feature flags, in the single flag byte at the start of the + "VP8X" payload. Bit 0 (0x80) and bit 1 (0x40) are reserved. */ +#define WEBP_FLAG_ICC 0x20 +#define WEBP_FLAG_ALPHA 0x10 +#define WEBP_FLAG_EXIF 0x08 +#define WEBP_FLAG_XMP 0x04 +#define WEBP_FLAG_ANIMATION 0x02 + + +/** + * Everything we learn while walking the chunks. + */ +struct WebpState +{ + /** + * Extraction context we were handed. + */ + struct EXTRACTOR_ExtractContext *ec; + + /** + * Bytes we may still read. + */ + uint64_t budget; + + /** + * Canvas width in pixels, 0 if unknown. + */ + uint32_t width; + + /** + * Canvas height in pixels, 0 if unknown. + */ + uint32_t height; + + /** + * Size of the ICC profile in bytes, 0 if there is none. + */ + uint64_t icc_size; + + /** + * Sum of the frame durations in milliseconds. + */ + uint64_t duration; + + /** + * Number of "ANMF" frames seen. + */ + unsigned int frames; + + /** + * Loop count from "ANIM", 0 meaning "forever". + */ + unsigned int loop_count; + + /** + * Background colour from "ANIM", as BGRA. + */ + uint32_t background; + + /** + * Feature flags from "VP8X". + */ + unsigned char flags; + + /** + * True if a "VP8X" chunk was seen. + */ + int extended; + + /** + * True if an "ANIM" chunk was seen. + */ + int animated; + + /** + * True if lossy VP8 data was seen. + */ + int lossy; + + /** + * True if lossless VP8L data was seen. + */ + int lossless; + + /** + * True if an "ALPH" chunk or a VP8L alpha flag was seen. + */ + int alpha; + + /** + * True if an "EXIF" chunk was seen. + */ + int exif; + + /** + * True if an "XMP " chunk was seen. + */ + int xmp; +}; + + +/** + * Read @a len bytes at @a offset, charged against the read budget. + * + * @param ws parser state + * @param offset absolute offset to read from + * @param buf where to put the data + * @param len number of bytes to read + * @return 1 on success, 0 on a short file or an exhausted budget + */ +static int +webp_read (struct WebpState *ws, + uint64_t offset, + void *buf, + size_t len) +{ + if ( (len > ws->budget) || + (offset > INT64_MAX) ) + return 0; + ws->budget -= len; + return EXTRACTOR_forensic_read_ (ws->ec, + (int64_t) offset, + buf, + len); +} + + +/** + * Read a 24-bit little-endian value. + * + * @param p three bytes + * @return the value + */ +static uint32_t +webp_le24 (const unsigned char *p) +{ + return ((uint32_t) p[0]) + | (((uint32_t) p[1]) << 8) + | (((uint32_t) p[2]) << 16); +} + + +/** + * Pull the canvas size out of a lossy "VP8 " key frame header. + * + * The payload starts with a three byte frame tag, then the three byte + * start code 0x9d 0x01 0x2a, then two 16-bit little-endian values whose + * low 14 bits are the width and the height; the top two bits of each + * are an upscaling hint that does not change the coded size. + * + * @param ws parser state + * @param payload absolute offset of the chunk payload + * @param len length of the chunk payload + */ +static void +webp_parse_vp8 (struct WebpState *ws, + uint64_t payload, + uint64_t len) +{ + unsigned char buf[10]; + + ws->lossy = 1; + if (len < 10) + return; + if (! webp_read (ws, + payload, + buf, + sizeof (buf))) + return; + if (0 != (buf[0] & 0x01)) + return; /* not a key frame; no size here */ + if ( (0x9d != buf[3]) || + (0x01 != buf[4]) || + (0x2a != buf[5]) ) + return; /* start code missing */ + ws->width = EXTRACTOR_forensic_le16_ (&buf[6]) & 0x3FFF; + ws->height = EXTRACTOR_forensic_le16_ (&buf[8]) & 0x3FFF; +} + + +/** + * Pull the canvas size out of a lossless "VP8L" header. + * + * The payload starts with the signature byte 0x2f followed by a 32-bit + * little-endian word packing (width - 1) in bits 0..13, (height - 1) in + * bits 14..27, an alpha hint in bit 28 and the version in bits 29..31. + * + * @param ws parser state + * @param payload absolute offset of the chunk payload + * @param len length of the chunk payload + */ +static void +webp_parse_vp8l (struct WebpState *ws, + uint64_t payload, + uint64_t len) +{ + unsigned char buf[5]; + uint32_t bits; + + ws->lossless = 1; + if (len < 5) + return; + if (! webp_read (ws, + payload, + buf, + sizeof (buf))) + return; + if (0x2f != buf[0]) + return; /* signature missing */ + bits = EXTRACTOR_forensic_le32_ (&buf[1]); + ws->width = (bits & 0x3FFF) + 1; + ws->height = ((bits >> 14) & 0x3FFF) + 1; + if (0 != ((bits >> 28) & 0x01)) + ws->alpha = 1; +} + + +/** + * Parse the "VP8X" extended header: the feature flags and the canvas + * size, which are stored as 24-bit values one less than the real size. + * + * @param ws parser state + * @param payload absolute offset of the chunk payload + * @param len length of the chunk payload + */ +static void +webp_parse_vp8x (struct WebpState *ws, + uint64_t payload, + uint64_t len) +{ + unsigned char buf[10]; + + ws->extended = 1; + if (len < 10) + return; + if (! webp_read (ws, + payload, + buf, + sizeof (buf))) + return; + ws->flags = buf[0]; + ws->width = webp_le24 (&buf[4]) + 1; + ws->height = webp_le24 (&buf[7]) + 1; + if (0 != (ws->flags & WEBP_FLAG_ALPHA)) + ws->alpha = 1; +} + + +/** + * Parse an "ANIM" global animation header. + * + * @param ws parser state + * @param payload absolute offset of the chunk payload + * @param len length of the chunk payload + */ +static void +webp_parse_anim (struct WebpState *ws, + uint64_t payload, + uint64_t len) +{ + unsigned char buf[6]; + + ws->animated = 1; + if (len < 6) + return; + if (! webp_read (ws, + payload, + buf, + sizeof (buf))) + return; + ws->background = EXTRACTOR_forensic_le32_ (buf); + ws->loop_count = EXTRACTOR_forensic_le16_ (&buf[4]); +} + + +/** + * Parse an "ANMF" frame header, and look inside the frame for the + * sub-chunk that says how it is coded. The frame duration is a 24-bit + * value in milliseconds. + * + * @param ws parser state + * @param payload absolute offset of the chunk payload + * @param len length of the chunk payload + */ +static void +webp_parse_anmf (struct WebpState *ws, + uint64_t payload, + uint64_t len) +{ + unsigned char buf[16]; + + if (ws->frames >= WEBP_MAX_FRAMES) + return; + ws->frames++; + if (len < 16) + return; + if (! webp_read (ws, + payload, + buf, + sizeof (buf))) + return; + ws->duration += webp_le24 (&buf[12]); + /* The frame payload holds an optional "ALPH" chunk and then one + "VP8 " or "VP8L" chunk; peek at the first one only, so that an + extended file still reports a codec. */ + if ( (0 != ws->lossy) || + (0 != ws->lossless) || + (len < 24) ) + return; + if (! webp_read (ws, + payload + 16, + buf, + 8)) + return; + if (0 == memcmp (buf, + "ALPH", + 4)) + { + ws->alpha = 1; + return; + } + if (0 == memcmp (buf, + "VP8 ", + 4)) + ws->lossy = 1; + else if (0 == memcmp (buf, + "VP8L", + 4)) + ws->lossless = 1; +} + + +/** + * Append @a name to the comma separated list in @a buf, if @a present. + * + * @param buf the list being built, always NUL terminated + * @param size number of bytes in @a buf + * @param[in,out] off current length of the list + * @param present whether the feature is there at all + * @param name what to append + */ +static void +webp_append (char *buf, + size_t size, + size_t *off, + int present, + const char *name) +{ + size_t need; + + if (! present) + return; + need = strlen (name) + ((0 != *off) ? 2 : 0); + if (*off + need + 1 > size) + return; + if (0 != *off) + { + buf[(*off)++] = ','; + buf[(*off)++] = ' '; + } + memcpy (&buf[*off], + name, + strlen (name)); + *off += strlen (name); + buf[*off] = '\0'; +} + + +/** + * Main entry method for the 'image/webp' extraction plugin. + * + * @param ec extraction context provided to the plugin + */ +void +EXTRACTOR_webp_extract_method (struct EXTRACTOR_ExtractContext *ec); + +void +EXTRACTOR_webp_extract_method (struct EXTRACTOR_ExtractContext *ec) +{ + struct WebpState ws; + unsigned char head[12]; + uint64_t fsize; + uint64_t riff_end; + uint64_t pos; + uint32_t riff_size; + char attrs[256]; + size_t aoff = 0; + + fsize = ec->get_size (ec->cls); + if ( (UINT64_MAX == fsize) || + (fsize < 12) ) + return; + memset (&ws, + 0, + sizeof (ws)); + ws.ec = ec; + ws.budget = WEBP_READ_BUDGET; + + /* Twelve bytes decide it; a file that is not ours costs nothing more + than this one read. */ + if (! webp_read (&ws, + 0, + head, + sizeof (head))) + return; + if ( (0 != memcmp (head, + "RIFF", + 4)) || + (0 != memcmp (&head[8], + "WEBP", + 4)) ) + return; + riff_size = EXTRACTOR_forensic_le32_ (&head[4]); + /* The RIFF size counts everything after the eight byte RIFF header; + trust the file only as far as its actual length. */ + riff_end = (uint64_t) riff_size + 8; + if (riff_end > fsize) + riff_end = fsize; + + pos = 12; + for (unsigned int n = 0; + (n < WEBP_MAX_CHUNKS) && (pos + 8 <= riff_end); + n++) + { + unsigned char chdr[8]; + uint64_t csize; + uint64_t payload; + uint64_t advance; + + if (! webp_read (&ws, + pos, + chdr, + sizeof (chdr))) + break; + csize = EXTRACTOR_forensic_le32_ (&chdr[4]); + payload = pos + 8; + if (csize > riff_end - payload) + csize = riff_end - payload; /* truncated file; use what is there */ + if (0 == memcmp (chdr, + "VP8X", + 4)) + webp_parse_vp8x (&ws, + payload, + csize); + else if (0 == memcmp (chdr, + "VP8 ", + 4)) + webp_parse_vp8 (&ws, + payload, + csize); + else if (0 == memcmp (chdr, + "VP8L", + 4)) + webp_parse_vp8l (&ws, + payload, + csize); + else if (0 == memcmp (chdr, + "ANIM", + 4)) + webp_parse_anim (&ws, + payload, + csize); + else if (0 == memcmp (chdr, + "ANMF", + 4)) + webp_parse_anmf (&ws, + payload, + csize); + else if (0 == memcmp (chdr, + "ALPH", + 4)) + ws.alpha = 1; + else if (0 == memcmp (chdr, + "ICCP", + 4)) + ws.icc_size = csize; + else if (0 == memcmp (chdr, + "EXIF", + 4)) + ws.exif = 1; + else if (0 == memcmp (chdr, + "XMP ", + 4)) + ws.xmp = 1; + /* Chunks are padded to an even length. The eight byte header + guarantees forward progress even for a zero length chunk. */ + advance = 8 + csize + (csize & 1); + if (advance < 8) + break; + pos += advance; + } + + if (0 != ec->proc (ec->cls, + "webp", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + "image/webp", + strlen ("image/webp") + 1)) + return; + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_FORMAT, + "WebP (%s)", + ws.extended + ? "extended" + : (ws.lossless ? "lossless" : "lossy"))) + return; + if ( (ws.lossy) || + (ws.lossless) ) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_CODEC, + "%s", + ws.lossless + ? "VP8L (lossless)" + : "VP8 (lossy)")) + return; + } + if ( (0 != ws.width) && + (0 != ws.height) ) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_IMAGE_DIMENSIONS, + "%ux%u", + (unsigned int) ws.width, + (unsigned int) ws.height)) + return; + } + /* One ATTRIBUTES line naming the optional features, so that a caller + can filter on "has an alpha channel" or "is animated" without + having to reason about the flag byte. */ + attrs[0] = '\0'; + webp_append (attrs, + sizeof (attrs), + &aoff, + ws.alpha, + "alpha"); + webp_append (attrs, + sizeof (attrs), + &aoff, + ws.animated, + "animation"); + webp_append (attrs, + sizeof (attrs), + &aoff, + 0 != ws.icc_size, + "ICC profile"); + webp_append (attrs, + sizeof (attrs), + &aoff, + ws.exif, + "EXIF"); + webp_append (attrs, + sizeof (attrs), + &aoff, + ws.xmp, + "XMP"); + if (0 != aoff) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_ATTRIBUTES, + "%s", + attrs)) + return; + } + /* WebP has no depth field of its own: both bitstreams are 8 bits per + channel by definition. Report the bare number so that a caller can + compare it with what the other image plugins report; whether there + is an alpha channel is an attribute, not a depth. */ + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_COLOR_DEPTH, + "8")) + return; + if (0 != ws.icc_size) + { + /* Not parsed here on purpose: the ICC payload is the exiv2 and + colour management plugins' business. */ + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_COLOR_PROFILE, + "ICC profile, %llu bytes", + (unsigned long long) ws.icc_size)) + return; + } + if (ws.animated) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_ENTRY_COUNT, + "%u", + ws.frames)) + return; + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_DURATION, + "%llu ms", + (unsigned long long) ws.duration)) + return; + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_COMMENT, + "animation: %u frame%s, %s, " + "background #%08x BGRA", + ws.frames, + (1 == ws.frames) ? "" : "s", + (0 == ws.loop_count) + ? "looping forever" + : "finite loop count", + (unsigned int) ws.background)) + return; + } + if (ws.exif) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_COMMENT, + "contains an EXIF chunk")) + return; + } + if (ws.xmp) + { + if (0 != EXTRACTOR_forensic_emit_ (ec, + "webp", + EXTRACTOR_METATYPE_COMMENT, + "contains an XMP chunk")) + return; + } +} + + +/* end of webp_extractor.c */
