gen_ebook_testdata.sh (9520B)
1 #!/bin/sh 2 # Regenerate src/plugins/testdata/ebook_test.epub and ebook_test.mobi. 3 # 4 # This file is in the public domain (CC0); so are the files it produces. 5 # 6 # Requirements: python3 (standard library only). Calibre is deliberately 7 # not used: the MOBI file is written out by hand from the format 8 # description on the MobileRead wiki, which keeps it tiny and lets the 9 # test assert exact EXTH values. Everything is deterministic. 10 # 11 # Usage: contrib/gen_ebook_testdata.sh [output-directory] 12 # (default output directory: src/plugins/testdata) 13 14 set -e 15 OUT="${1:-$(dirname "$0")/../src/plugins/testdata}" 16 mkdir -p "$OUT" 17 python3 - "$OUT" <<'PYEOF' 18 import os, struct, sys, zipfile 19 20 OUT = sys.argv[1] 21 DATE = (2026, 1, 1, 0, 0, 0) # fixed: the archive must be reproducible 22 23 # ------------------------------------------------------------------- EPUB 24 CONTAINER = """<?xml version="1.0" encoding="UTF-8"?> 25 <container version="1.0" 26 xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> 27 <rootfiles> 28 <rootfile full-path="OEBPS/content.opf" 29 media-type="application/oebps-package+xml"/> 30 </rootfiles> 31 </container> 32 """ 33 34 # One <item> per manifest entry; the plugin reports the count, so keep 35 # this list and the number asserted by test_ebook.c in step. 36 ITEMS = [ 37 ('ncx', 'toc.ncx', 'application/x-dtbncx+xml'), 38 ('nav', 'nav.xhtml', 'application/xhtml+xml'), 39 ('css', 'style.css', 'text/css'), 40 ('ch1', 'chapter1.xhtml', 'application/xhtml+xml'), 41 ('ch2', 'chapter2.xhtml', 'application/xhtml+xml'), 42 ('ch3', 'chapter3.xhtml', 'application/xhtml+xml'), 43 ('cover', 'cover.png', 'image/png'), 44 ] 45 46 OPF = """<?xml version="1.0" encoding="UTF-8"?> 47 <package xmlns="http://www.idpf.org/2007/opf" version="3.0" 48 unique-identifier="bookid"> 49 <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" 50 xmlns:opf="http://www.idpf.org/2007/opf"> 51 <dc:title>The libextractor Test Book</dc:title> 52 <dc:creator opf:role="aut" opf:file-as="Grothoff, Christian" 53 >Christian Grothoff</dc:creator> 54 <dc:contributor opf:role="bkp" 55 >calibre (7.2.0) [https://calibre-ebook.com]</dc:contributor> 56 <dc:publisher>GNU Press</dc:publisher> 57 <dc:language>en-GB</dc:language> 58 <dc:date>2026-01-01T00:00:00+00:00</dc:date> 59 <dc:subject>digital forensics</dc:subject> 60 <dc:subject>metadata extraction</dc:subject> 61 <dc:description>A minimal EPUB built by gen_ebook_testdata.sh, used 62 to check that the ebook plugin reads Dublin Core out of the 63 OPF package document.</dc:description> 64 <dc:rights>Public domain (CC0) & nothing else</dc:rights> 65 <dc:identifier id="bookid" opf:scheme="ISBN" 66 >978-3-16-148410-0</dc:identifier> 67 <dc:identifier opf:scheme="URI" 68 >urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719</dc:identifier> 69 <meta name="calibre:timestamp" content="2026-01-01T00:00:00+00:00"/> 70 <meta name="generator" content="gen_ebook_testdata.sh"/> 71 <meta property="dcterms:modified">2026-02-01T00:00:00Z</meta> 72 </metadata> 73 <manifest> 74 %s 75 </manifest> 76 <spine toc="ncx"> 77 <itemref idref="ch1"/> 78 <itemref idref="ch2"/> 79 <itemref idref="ch3"/> 80 </spine> 81 </package> 82 """ % "\n".join (' <item id="%s" href="%s" media-type="%s"/>' % i 83 for i in ITEMS) 84 85 # Font obfuscation, which is what encryption.xml usually means in an 86 # EPUB; the plugin only reports that it is there. 87 ENCRYPTION = """<?xml version="1.0" encoding="UTF-8"?> 88 <encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> 89 <EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"> 90 <EncryptionMethod 91 Algorithm="http://www.idpf.org/2008/embedding"/> 92 <CipherData> 93 <CipherReference URI="OEBPS/fonts/serif.otf"/> 94 </CipherData> 95 </EncryptedData> 96 </encryption> 97 """ 98 99 CHAPTER = """<?xml version="1.0" encoding="UTF-8"?> 100 <html xmlns="http://www.w3.org/1999/xhtml"><head><title>%s</title></head> 101 <body><h1>%s</h1><p>Nothing to see here.</p></body></html> 102 """ 103 104 epub = os.path.join (OUT, "ebook_test.epub") 105 with zipfile.ZipFile (epub, "w") as z: 106 # the spec requires `mimetype' to be the first member and stored 107 zi = zipfile.ZipInfo ("mimetype", DATE) 108 zi.compress_type = zipfile.ZIP_STORED 109 z.writestr (zi, b"application/epub+zip") 110 for name, data in (("META-INF/container.xml", CONTAINER), 111 ("META-INF/encryption.xml", ENCRYPTION), 112 ("OEBPS/content.opf", OPF), 113 ("OEBPS/chapter1.xhtml", CHAPTER % ("One", "One")), 114 ("OEBPS/chapter2.xhtml", CHAPTER % ("Two", "Two")), 115 ("OEBPS/chapter3.xhtml", CHAPTER % ("Three", "Three")), 116 ("OEBPS/style.css", "body { margin: 1em; }\n")): 117 zi = zipfile.ZipInfo (name, DATE) 118 zi.compress_type = zipfile.ZIP_DEFLATED 119 z.writestr (zi, data.encode ("utf-8")) 120 121 122 # ------------------------------------------------------------------- MOBI 123 # 124 # Layout per the MobileRead wiki: a 78-byte PDB header, one 8-byte record 125 # info entry per record, then the records. Record 0 holds the 16-byte 126 # PalmDOC header, the MOBI header and the EXTH header. 127 MOBI_EPOCH = 2082844800 # seconds between 1904-01-01 and 1970-01-01 128 CTIME = 1767225600 # 2026-01-01T00:00:00Z 129 MTIME = 1769904000 # 2026-02-01T00:00:00Z 130 131 DB_NAME = b"The_libextractor_Test_Book" 132 FULL_NAME = b"The libextractor Test Book" 133 TEXT = (b"This is the body text of the libextractor MOBI test file. " 134 b"It exists only so that record 1 is not empty.\n") 135 136 EXTH_RECORDS = [ 137 (100, b"Christian Grothoff"), # author 138 (101, b"GNU Press"), # publisher 139 (103, b"A hand-built minimal MOBI file used by the " 140 b"libextractor test suite."), # description 141 (104, b"978-3-16-148410-0"), # ISBN 142 (105, b"digital forensics"), # subject 143 (106, b"2026-01-01"), # published 144 (108, b"calibre (7.2.0) [https://calibre-ebook.com]"), # contributor 145 (109, b"Public domain (CC0)"), # rights 146 (113, b"B00LIBEXTR7"), # ASIN 147 (204, struct.pack (">I", 201)), # creator software 148 (205, struct.pack (">I", 7)), # creator major 149 (206, struct.pack (">I", 2)), # creator minor 150 (208, b"libextractor-test-watermark"), # watermark 151 (501, b"EBOK"), # cdetype 152 (502, b"2026-02-01T00:00:00+00:00"), # last update time 153 (503, FULL_NAME), # updated title 154 (524, b"en-gb"), # language 155 ] 156 157 158 def exth (): 159 body = b"" 160 for t, d in EXTH_RECORDS: 161 body += struct.pack (">II", t, 8 + len (d)) + d 162 size = 12 + len (body) 163 pad = (-size) % 4 164 return b"EXTH" + struct.pack (">II", size + pad, 165 len (EXTH_RECORDS)) + body + b"\0" * pad 166 167 168 MOBI_HEADER_LEN = 232 169 170 171 def record0 (): 172 e = exth () 173 # PalmDOC header, then MOBI header, then EXTH, then the full name 174 full_name_off = 16 + MOBI_HEADER_LEN + len (e) 175 mobi = bytearray (MOBI_HEADER_LEN) 176 mobi[0:4] = b"MOBI" 177 struct.pack_into (">I", mobi, 4, MOBI_HEADER_LEN) 178 struct.pack_into (">I", mobi, 8, 2) # Mobipocket book 179 struct.pack_into (">I", mobi, 12, 65001) # UTF-8 180 struct.pack_into (">I", mobi, 16, 0x1719) # unique ID 181 struct.pack_into (">I", mobi, 20, 6) # file version 182 struct.pack_into (">I", mobi, 64, 2) # first non-book index 183 struct.pack_into (">I", mobi, 68, full_name_off) 184 struct.pack_into (">I", mobi, 72, len (FULL_NAME)) 185 struct.pack_into (">I", mobi, 76, 9) # locale: English 186 struct.pack_into (">I", mobi, 88, 6) # min version 187 struct.pack_into (">I", mobi, 92, 2) # first image index 188 struct.pack_into (">I", mobi, 112, 0x50) # EXTH flags: 0x40 = present 189 palmdoc = struct.pack (">HHIHHHH", 190 1, # no compression 191 0, 192 len (TEXT), 193 1, # record count 194 4096, # record size 195 1, # encryption: old Mobipocket 196 0) 197 rec = palmdoc + bytes (mobi) + e + FULL_NAME + b"\0" 198 while len (rec) % 4: 199 rec += b"\0" 200 return rec 201 202 203 records = [record0 (), TEXT, b"\0"] 204 205 hdr = bytearray (78) 206 hdr[0:len (DB_NAME)] = DB_NAME 207 struct.pack_into (">HH", hdr, 32, 0, 0) # attributes, version 208 struct.pack_into (">III", hdr, 36, 209 CTIME + MOBI_EPOCH, 210 MTIME + MOBI_EPOCH, 211 0) # create/mod/backup 212 struct.pack_into (">III", hdr, 48, 0, 0, 0) # modnum, appinfo, sort 213 hdr[60:68] = b"BOOKMOBI" 214 struct.pack_into (">II", hdr, 68, 0x1719, 0) 215 struct.pack_into (">H", hdr, 76, len (records)) 216 217 info_len = 8 * len (records) + 2 # + 2 bytes of padding 218 off = 78 + info_len 219 info = b"" 220 for i, r in enumerate (records): 221 info += struct.pack (">IBBBB", off, 0, 0, 0, i) 222 off += len (r) 223 info += b"\0\0" 224 225 with open (os.path.join (OUT, "ebook_test.mobi"), "wb") as f: 226 f.write (bytes (hdr) + info + b"".join (records)) 227 228 for n in ("ebook_test.epub", "ebook_test.mobi"): 229 print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n)))) 230 PYEOF