gen_apk_testdata.sh - libextractor

gen_apk_testdata.sh (15275B)
      1 #!/bin/sh
      2 # Regenerate src/plugins/testdata/apk_test.jar and apk_test.apk.
      3 #
      4 # This file is in the public domain (CC0); so are the files it produces.
      5 #
      6 # Requirements: python3 (standard library only).  No Android SDK, no JDK:
      7 # the binary AndroidManifest.xml is written out by hand, and the PKCS#7
      8 # signature block is a pre-generated, self-contained test certificate
      9 # chain embedded below (originally made with `openssl req`/`openssl cms`,
     10 # see the comment next to the blob).  Everything is deterministic: run it
     11 # twice and you get byte-identical files.
     12 #
     13 # Usage: contrib/gen_apk_testdata.sh [output-directory]
     14 #        (default output directory: src/plugins/testdata)
     15 
     16 set -e
     17 OUT="${1:-$(dirname "$0")/../src/plugins/testdata}"
     18 mkdir -p "$OUT"
     19 python3 - "$OUT" <<'PYEOF'
     20 import base64, os, struct, sys, zipfile
     21 
     22 OUT = sys.argv[1]
     23 DATE = (2026, 1, 1, 0, 0, 0)      # fixed: the archives must be reproducible
     24 
     25 # ---------------------------------------------------------------- signature
     26 #
     27 # A PKCS#7 SignedData over a dummy .SF file, signed by a two-element test
     28 # chain: "libextractor test CA" issues "libextractor test signer"
     29 # (serial 0x1719).  Note that the CA certificate comes *first* inside the
     30 # certificates set, so a parser that simply takes the first certificate
     31 # reports the wrong subject; the plugin has to resolve the SignerInfo's
     32 # issuerAndSerialNumber.  Regenerate with:
     33 #
     34 #   openssl req -x509 -newkey rsa:2048 -keyout ca.key -nodes -out ca.pem \
     35 #     -sha256 -set_serial 0x01 -subj "/CN=libextractor test CA/O=GNU libextractor" \
     36 #     -not_before 20260101000000Z -not_after 20360101000000Z
     37 #   openssl req -new -newkey rsa:2048 -keyout leaf.key -nodes -out leaf.csr \
     38 #     -sha256 -subj "/CN=libextractor test signer/O=GNU libextractor/OU=Testing/C=CH"
     39 #   openssl x509 -req -in leaf.csr -CA ca.pem -CAkey ca.key -set_serial 0x1719 \
     40 #     -sha256 -not_before 20260101000000Z -not_after 20360101000000Z -out leaf.pem
     41 #   openssl cms -sign -binary -in CERT.SF -signer leaf.pem -inkey leaf.key \
     42 #     -certfile ca.pem -outform DER -md sha256 -noattr -nosmimecap -out CERT.RSA
     43 #
     44 # The private keys were thrown away; this signature protects nothing.
     45 PKCS7_B64 = """
     46 MIIIQgYJKoZIhvcNAQcCoIIIMzCCCC8CAQExDTALBglghkgBZQMEAgEwCwYJKoZIhvcNAQcBoIIG
     47 nzCCA0IwggIqoAMCAQICAQEwDQYJKoZIhvcNAQELBQAwOjEdMBsGA1UEAwwUbGliZXh0cmFjdG9y
     48 IHRlc3QgQ0ExGTAXBgNVBAoMEEdOVSBsaWJleHRyYWN0b3IwHhcNMjYwMTAxMDAwMDAwWhcNMzYw
     49 MTAxMDAwMDAwWjA6MR0wGwYDVQQDDBRsaWJleHRyYWN0b3IgdGVzdCBDQTEZMBcGA1UECgwQR05V
     50 IGxpYmV4dHJhY3RvcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALQDKjxbHQvulnUe
     51 Z6DHIMobIBoNn5mDOXeNah3VwCHFMwcO5DJCvzLo5N/D/pIeuov4PaXMXOv4e7/ZqiIu6DV8Nr0A
     52 KhJxCIEVUHoYt3II3gNFmjIQLs1ajAdA+4rjX8mjicgIbfNNgDCf/OGj5ctFdEelOMF+NCDel8jj
     53 61p0B75a8arlaNnk254A785TQfYOBnWXp+4OPkJmF1FRhQXI/19TRbjCdAc9Q3FL8R/pGp9QxH7s
     54 MTR1CfOJ47DtA/upHeYR3S1BMBtRZCadwBHpLDBNT2Yornpek7nm1kZOL8ACT4FXu/O/fYFguW2o
     55 7RtxxSDMYRRTePZ79ylsr8kCAwEAAaNTMFEwHQYDVR0OBBYEFIp+SdLMxJWNBKxJ9y9qwMuQn00j
     56 MB8GA1UdIwQYMBaAFIp+SdLMxJWNBKxJ9y9qwMuQn00jMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
     57 hvcNAQELBQADggEBADRcVd4jwMGRM7UKJwc1pQcm3qzXH0/1U+M5+zPAihFKnuZaZaF/OxqOHkA6
     58 Ht+juuAb57eOaai+qI5IepohdUGLmwepcNEL9FqboRqaL9DkwPmC/lqTM5OrTzMs2Ti8HiT2Degh
     59 6bX6h9n1fI3MNHArvoTb1n70KlEY5D1huCk0NT3ktRGV8uGkSQomo+Kj3lDiPvXIKl5bddQ9KGBG
     60 MuW1tKNQyjm3n5AxFH3ouFcGLIM3s6AAHJiG9rh6RisJS14gS5XAyEn0n6yF7NRiO6gcozGGkxck
     61 bEz7z0/m0FkmfKhUZ+a7LK0liUn42s42Ev2z7TVYF9crkjIpwRymEYMwggNVMIICPaADAgECAgIX
     62 GTANBgkqhkiG9w0BAQsFADA6MR0wGwYDVQQDDBRsaWJleHRyYWN0b3IgdGVzdCBDQTEZMBcGA1UE
     63 CgwQR05VIGxpYmV4dHJhY3RvcjAeFw0yNjAxMDEwMDAwMDBaFw0zNjAxMDEwMDAwMDBaMF0xITAf
     64 BgNVBAMMGGxpYmV4dHJhY3RvciB0ZXN0IHNpZ25lcjEZMBcGA1UECgwQR05VIGxpYmV4dHJhY3Rv
     65 cjEQMA4GA1UECwwHVGVzdGluZzELMAkGA1UEBhMCQ0gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
     66 ggEKAoIBAQCplvQUPC2P/KBXnJQVqow9P8wFV4ngj/gN+COIUukWefcRQX4Z+VRf0kJR0lTeW8NY
     67 xLAQzyhPkDwGJBOGPEpN/3iBmXOQa47wO5MiphgxyCK7lkF0sDcAG4tjkixMNmf0HBoouSQpojKi
     68 MT2KTZu5ShJ80eOgfgvIcdwvG9nGXf9d1/3DgE4YwbFRvqmo5FMQmvKct+Up6IDHxMn5Dj4QdQW8
     69 p1JvQ6O0eyp+98CK8u8VLMgjT5sPVZv6OyMSjXqHQ25kl29tR3CXDTCOQTGOi+T0DzzeqXOHkQhM
     70 ApUUxSCt4LD1p91nLlgLBDXwEhHyepSPDc5aOvOMi9BNyr5pAgMBAAGjQjBAMB0GA1UdDgQWBBS6
     71 +cBqtk2rLx1XrLysYCsFUvYqJTAfBgNVHSMEGDAWgBSKfknSzMSVjQSsSfcvasDLkJ9NIzANBgkq
     72 hkiG9w0BAQsFAAOCAQEACKKXcPaPxVWHjiXg9P3LBg3RMbSL8Qp6f4Y8OmYhGJFVLZBWR03BsDU3
     73 W9IiAYD08XznxItKOIjHjABvXPc2aY6yceJkCL+1ohhL3HRcs4+NvgQbevAqyP7mwWmJAb0WxwC7
     74 YSTjCIFzszudwUmpd1pQya2SQGPTUpWA5aCz+rEJXPCKrWn4Wwb3HkJV1p6p2gYhWt7rD2z+gs2Y
     75 IpDyAheEbMwr2iU1gZLlC02E1PFq3GpYcgB4401U7OgQOdbJovWlVZw061NF6CstJwL1mQIrKdgE
     76 Iu9awPZ+ZL7FQQ/qkWB4QIYtRhZOc6V68jRj4nkwk63p99GWDICFk1yv1zGCAWkwggFlAgEBMEAw
     77 OjEdMBsGA1UEAwwUbGliZXh0cmFjdG9yIHRlc3QgQ0ExGTAXBgNVBAoMEEdOVSBsaWJleHRyYWN0
     78 b3ICAhcZMAsGCWCGSAFlAwQCATANBgkqhkiG9w0BAQEFAASCAQB4EvSyr8PjR0GHgvi7z81/jKsb
     79 JQ4aqr+x5djxnoG8OZXXVOz2yGVJJLhsTMbSOdnchpfjOfSv0cxpbeowMM7ICJ5VpCVi4O3RXWAy
     80 TJ2siIl0QoDnp1Xvfs8DyFPUTAUFqvukLB16rgQXoDNk4tFOmqfgwnVcIm+QrrwTVqBgGEb63F5K
     81 HOiBKOJttNPLixFVtCc/mOKz5+Fu3N3NMLiS3u++k2/oWh1+5ozp/90AOXmnpL+iGm7/7GwFMPM3
     82 ZhKf7a+8cldhquc9AeO/lwYJw6XF8+7awEYSiPeEeKNN9SrbnAKOL+npzNN6OY/U5p9rzsV153Ov
     83 YOGnCcC/2rXa
     84 """
     85 PKCS7 = base64.b64decode("".join(PKCS7_B64.split()))
     86 SF = (b"Signature-Version: 1.0\r\n"
     87       b"Created-By: libextractor testdata generator\r\n"
     88       b"SHA-256-Digest-Manifest: 0000\r\n\r\n")
     89 
     90 
     91 # ------------------------------------------------------------ jar manifest
     92 def fold (line):
     93   """Wrap a manifest header the way the JAR spec does: no line, including
     94   its line break, may exceed 72 bytes, and every continuation line starts
     95   with a single space that is not part of the value."""
     96   raw = line.encode ("utf-8")
     97   out = raw[:70]
     98   raw = raw[70:]
     99   while raw:
    100     out += b"\r\n " + raw[:69]
    101     raw = raw[69:]
    102   return out + b"\r\n"
    103 
    104 
    105 JAR_HEADERS = [
    106   "Manifest-Version: 1.0",
    107   "Created-By: 17.0.9+9 (Eclipse Adoptium)",
    108   "Build-Jdk-Spec: 17",
    109   "Build-Jdk: 17.0.9+9",
    110   "Main-Class: org.gnu.libextractor.TestMain",
    111   "Class-Path: lib/support.jar lib/codec.jar",
    112   # deliberately longer than 72 bytes, so that reading it back proves the
    113   # continuation lines are unfolded
    114   "Implementation-Title: GNU libextractor JAR test archive with a very long"
    115   " title that has to be folded",
    116   "Implementation-Version: 1.19.0",
    117   "Implementation-Vendor: GNU libextractor project",
    118   "Specification-Title: libextractor plugin test specification",
    119   "Specification-Version: 1.19",
    120   "Specification-Vendor: GNU",
    121   "Bundle-SymbolicName: org.gnu.libextractor.test",
    122   "Bundle-Name: libextractor test bundle",
    123   "Bundle-Version: 1.19.0.qualifier",
    124   "Bundle-Vendor: GNU libextractor project",
    125   "Bundle-Description: test bundle produced by gen_apk_testdata.sh",
    126   "Bundle-RequiredExecutionEnvironment: JavaSE-17",
    127 ]
    128 MANIFEST = b"".join (fold (h) for h in JAR_HEADERS) + b"\r\n"
    129 
    130 # a syntactically valid but empty class file (major version 61 = Java 17)
    131 CLASS = struct.pack (">IHHH", 0xCAFEBABE, 0, 61, 1) + \
    132         struct.pack (">HHHHH", 0x0021, 0, 0, 0, 0)
    133 
    134 
    135 def zwrite (path, members, compress = zipfile.ZIP_DEFLATED):
    136   buf = []
    137   with zipfile.ZipFile (path, "w") as z:
    138     for name, data, method in members:
    139       zi = zipfile.ZipInfo (name, DATE)
    140       zi.compress_type = method
    141       zi.create_system = 0
    142       z.writestr (zi, data)
    143   return buf
    144 
    145 
    146 zwrite (os.path.join (OUT, "apk_test.jar"), [
    147   ("META-INF/MANIFEST.MF", MANIFEST, zipfile.ZIP_DEFLATED),
    148   ("META-INF/CERT.SF", SF, zipfile.ZIP_DEFLATED),
    149   ("META-INF/CERT.RSA", PKCS7, zipfile.ZIP_STORED),
    150   ("org/gnu/libextractor/TestMain.class", CLASS, zipfile.ZIP_DEFLATED),
    151 ])
    152 
    153 
    154 # ------------------------------------------------------- binary AndroidManifest
    155 #
    156 # Chunk types and value types are from AOSP
    157 # frameworks/base/libs/androidfw/include/androidfw/ResourceTypes.h.
    158 RES_XML                 = 0x0003
    159 RES_STRING_POOL         = 0x0001
    160 RES_XML_START_NAMESPACE = 0x0100
    161 RES_XML_END_NAMESPACE   = 0x0101
    162 RES_XML_START_ELEMENT   = 0x0102
    163 RES_XML_END_ELEMENT     = 0x0103
    164 RES_XML_RESOURCE_MAP    = 0x0180
    165 
    166 TYPE_REFERENCE   = 0x01
    167 TYPE_STRING      = 0x03
    168 TYPE_INT_DEC     = 0x10
    169 TYPE_INT_BOOLEAN = 0x12
    170 
    171 ANDROID_NS = "http://schemas.android.com/apk/res/android"
    172 
    173 # Attribute resource IDs, read off a real aapt2-built APK.  The resource
    174 # map has to cover a prefix of the string pool, so these strings are
    175 # added to the pool first and in this order.
    176 ATTR_IDS = [
    177   ("label",                0x01010001),
    178   ("icon",                 0x01010002),
    179   ("name",                 0x01010003),
    180   ("debuggable",           0x0101000f),
    181   ("minSdkVersion",        0x0101020c),
    182   ("versionCode",          0x0101021b),
    183   ("versionName",          0x0101021c),
    184   ("targetSdkVersion",     0x01010270),
    185   ("allowBackup",          0x01010280),
    186   ("usesCleartextTraffic", 0x010104ec),
    187   ("networkSecurityConfig", 0x01010527),
    188   ("compileSdkVersion",    0x01010572),
    189 ]
    190 
    191 
    192 class Pool:
    193   def __init__ (self):
    194     self.strings = []
    195     for name, _ in ATTR_IDS:
    196       self.add (name)
    197 
    198   def add (self, s):
    199     if s in self.strings:
    200       return self.strings.index (s)
    201     self.strings.append (s)
    202     return len (self.strings) - 1
    203 
    204   def build (self):
    205     """UTF-16LE string pool, which is what aapt emits for
    206     AndroidManifest.xml."""
    207     data = b""
    208     offs = []
    209     for s in self.strings:
    210       offs.append (len (data))
    211       u = s.encode ("utf-16-le")
    212       assert len (s) < 0x8000
    213       data += struct.pack ("<H", len (s)) + u + b"\0\0"
    214     while len (data) % 4:
    215       data += b"\0"
    216     hdr_len = 28 + 4 * len (offs)
    217     size = hdr_len + len (data)
    218     return (struct.pack ("<HHIIIIII", RES_STRING_POOL, 28, size,
    219                          len (offs), 0, 0, hdr_len, 0)
    220             + struct.pack ("<%dI" % len (offs), *offs)
    221             + data)
    222 
    223 
    224 pool = Pool ()
    225 NOSTR = 0xFFFFFFFF
    226 
    227 
    228 def attr (ns, name, value, vtype):
    229   """One ResXMLTree_attribute (20 bytes)."""
    230   ns_i = pool.add (ns) if ns else NOSTR
    231   nm_i = pool.add (name)
    232   if TYPE_STRING == vtype:
    233     raw = pool.add (value)
    234     data = raw
    235   else:
    236     raw = NOSTR
    237     data = value & 0xFFFFFFFF
    238   return struct.pack ("<IIIHBBI", ns_i, nm_i, raw, 8, 0, vtype, data)
    239 
    240 
    241 def start (name, attrs):
    242   nm = pool.add (name)
    243   body = b"".join (attrs)
    244   size = 16 + 20 + len (body)
    245   return (struct.pack ("<HHIII", RES_XML_START_ELEMENT, 16, size, 1, NOSTR)
    246           + struct.pack ("<IIHHHHHH", NOSTR, nm, 20, 20, len (attrs),
    247                          0, 0, 0)
    248           + body)
    249 
    250 
    251 def end (name):
    252   nm = pool.add (name)
    253   return struct.pack ("<HHIIIII", RES_XML_END_ELEMENT, 16, 24, 1, NOSTR,
    254                       NOSTR, nm)
    255 
    256 
    257 PERMISSIONS = [
    258   "android.permission.INTERNET",
    259   "android.permission.ACCESS_FINE_LOCATION",
    260   "android.permission.READ_SMS",
    261   "android.permission.CAMERA",
    262   "android.permission.RECEIVE_BOOT_COMPLETED",
    263 ]
    264 
    265 # Strings referenced by attribute values have to exist before the chunks
    266 # are serialised, but `attr' adds them as it goes, so simply build the
    267 # chunk list first and the pool header afterwards.
    268 body = b""
    269 body += start ("manifest", [
    270   attr (ANDROID_NS, "versionCode", 1719, TYPE_INT_DEC),
    271   attr (ANDROID_NS, "versionName", "1.19.0", TYPE_STRING),
    272   attr (ANDROID_NS, "compileSdkVersion", 33, TYPE_INT_DEC),
    273   attr (None, "package", "org.gnu.libextractor.apktest", TYPE_STRING),
    274   attr (None, "platformBuildVersionCode", 33, TYPE_INT_DEC),
    275 ])
    276 body += start ("uses-sdk", [
    277   attr (ANDROID_NS, "minSdkVersion", 21, TYPE_INT_DEC),
    278   attr (ANDROID_NS, "targetSdkVersion", 33, TYPE_INT_DEC),
    279 ])
    280 body += end ("uses-sdk")
    281 for p in PERMISSIONS:
    282   body += start ("uses-permission",
    283                  [attr (ANDROID_NS, "name", p, TYPE_STRING)])
    284   body += end ("uses-permission")
    285 body += start ("application", [
    286   # a literal label, not the usual @string reference, so that the plugin
    287   # has something real to report
    288   attr (ANDROID_NS, "label", "libextractor APK test", TYPE_STRING),
    289   # a reference, which the plugin must *not* report as a title
    290   attr (ANDROID_NS, "icon", 0x7F080001, TYPE_REFERENCE),
    291   attr (ANDROID_NS, "debuggable", 0xFFFFFFFF, TYPE_INT_BOOLEAN),
    292   attr (ANDROID_NS, "allowBackup", 0, TYPE_INT_BOOLEAN),
    293   attr (ANDROID_NS, "usesCleartextTraffic", 0xFFFFFFFF, TYPE_INT_BOOLEAN),
    294   attr (ANDROID_NS, "networkSecurityConfig", 0x7F110002, TYPE_REFERENCE),
    295 ])
    296 body += end ("application")
    297 body += end ("manifest")
    298 
    299 ns_prefix = pool.add ("android")
    300 ns_uri = pool.add (ANDROID_NS)
    301 ns_start = struct.pack ("<HHIIIII", RES_XML_START_NAMESPACE, 16, 24, 1,
    302                         NOSTR, ns_prefix, ns_uri)
    303 ns_end = struct.pack ("<HHIIIII", RES_XML_END_NAMESPACE, 16, 24, 1,
    304                       NOSTR, ns_prefix, ns_uri)
    305 
    306 resmap = struct.pack ("<HHI", RES_XML_RESOURCE_MAP, 8, 8 + 4 * len (ATTR_IDS))
    307 resmap += struct.pack ("<%dI" % len (ATTR_IDS), *[i for _, i in ATTR_IDS])
    308 
    309 pool_chunk = pool.build ()
    310 rest = pool_chunk + resmap + ns_start + body + ns_end
    311 AXML = struct.pack ("<HHI", RES_XML, 8, 8 + len (rest)) + rest
    312 
    313 DEX = b"dex\n035\0" + b"\0" * 104
    314 SO = b"\x7fELF\x02\x01\x01\x00" + b"\0" * 56
    315 ARSC = struct.pack ("<HHII", 0x0002, 12, 12, 0)
    316 
    317 APK_MANIFEST = (b"Manifest-Version: 1.0\r\n"
    318                 b"Created-By: 1.0 (Android SignApk)\r\n\r\n")
    319 
    320 apk = os.path.join (OUT, "apk_test.apk")
    321 zwrite (apk, [
    322   # AndroidManifest.xml first, the way aapt lays an APK out
    323   ("AndroidManifest.xml", AXML, zipfile.ZIP_DEFLATED),
    324   ("resources.arsc", ARSC, zipfile.ZIP_STORED),
    325   ("classes.dex", DEX, zipfile.ZIP_DEFLATED),
    326   ("classes2.dex", DEX, zipfile.ZIP_DEFLATED),
    327   ("classes3.dex", DEX, zipfile.ZIP_DEFLATED),
    328   ("lib/arm64-v8a/libtest.so", SO, zipfile.ZIP_DEFLATED),
    329   ("lib/armeabi-v7a/libtest.so", SO, zipfile.ZIP_DEFLATED),
    330   ("lib/x86_64/libtest.so", SO, zipfile.ZIP_DEFLATED),
    331   ("res/xml/network_security_config.xml", b"<network-security-config/>",
    332    zipfile.ZIP_DEFLATED),
    333   ("META-INF/MANIFEST.MF", APK_MANIFEST, zipfile.ZIP_DEFLATED),
    334   ("META-INF/ANDROIDD.SF", SF, zipfile.ZIP_DEFLATED),
    335   ("META-INF/ANDROIDD.RSA", PKCS7, zipfile.ZIP_STORED),
    336 ])
    337 
    338 
    339 # ------------------------------------------------------ APK Signing Block
    340 #
    341 # Splice an APK Signing Block (the v2/v3 signature container) in between
    342 # the last local file entry and the central directory, and move the
    343 # central directory pointer in the end-of-central-directory record along
    344 # with it.  Local header offsets are unaffected because everything before
    345 # the central directory stays where it is.
    346 def sig_block (ids):
    347   pairs = b""
    348   for i in ids:
    349     # the value is a real signer list in a real APK; here it only has to
    350     # be present and correctly framed
    351     value = struct.pack ("<I", i) + b"\0" * 60
    352     pairs += struct.pack ("<Q", len (value)) + value
    353   # size covers the pairs, the trailing size field and the magic
    354   size = len (pairs) + 8 + 16
    355   return (struct.pack ("<Q", size) + pairs + struct.pack ("<Q", size)
    356           + b"APK Sig Block 42")
    357 
    358 
    359 APK_SIG_V2 = 0x7109871A
    360 APK_SIG_V3 = 0xF05368C0
    361 
    362 with open (apk, "rb") as f:
    363   data = f.read ()
    364 eocd = data.rindex (b"PK\5\6")
    365 cd_off = struct.unpack_from ("<I", data, eocd + 16)[0]
    366 block = sig_block ([APK_SIG_V2, APK_SIG_V3])
    367 data = (data[:cd_off] + block + data[cd_off:eocd]
    368         + data[eocd:eocd + 16]
    369         + struct.pack ("<I", cd_off + len (block))
    370         + data[eocd + 20:])
    371 with open (apk, "wb") as f:
    372   f.write (data)
    373 
    374 for n in ("apk_test.jar", "apk_test.apk"):
    375   print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n))))
    376 PYEOF
	libextractor GNU libextractor
	Log \| Files \| Refs \| Submodules \| README \| LICENSE