gen_ebook_testdata.sh (17866B)
1 #!/bin/sh 2 # Regenerate src/plugins/testdata/ebook_test.epub and ebook_test.mobi, 3 # and the full text fixtures ebook_fulltext.epub and ebook_fulltext.mobi. 4 # 5 # This file is in the public domain (CC0); so are the files it produces. 6 # 7 # Requirements: python3 (standard library only). Calibre is deliberately 8 # not used: the MOBI file is written out by hand from the format 9 # description on the MobileRead wiki, which keeps it tiny and lets the 10 # test assert exact EXTH values. Everything is deterministic. 11 # 12 # Usage: contrib/gen_ebook_testdata.sh [output-directory] 13 # (default output directory: src/plugins/testdata) 14 15 set -e 16 OUT="${1:-$(dirname "$0")/../src/plugins/testdata}" 17 mkdir -p "$OUT" 18 python3 - "$OUT" <<'PYEOF' 19 import os, struct, sys, zipfile 20 21 OUT = sys.argv[1] 22 DATE = (2026, 1, 1, 0, 0, 0) # fixed: the archive must be reproducible 23 24 # ------------------------------------------------------------------- EPUB 25 CONTAINER = """<?xml version="1.0" encoding="UTF-8"?> 26 <container version="1.0" 27 xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> 28 <rootfiles> 29 <rootfile full-path="OEBPS/content.opf" 30 media-type="application/oebps-package+xml"/> 31 </rootfiles> 32 </container> 33 """ 34 35 # One <item> per manifest entry; the plugin reports the count, so keep 36 # this list and the number asserted by test_ebook.c in step. 37 ITEMS = [ 38 ('ncx', 'toc.ncx', 'application/x-dtbncx+xml'), 39 ('nav', 'nav.xhtml', 'application/xhtml+xml'), 40 ('css', 'style.css', 'text/css'), 41 ('ch1', 'chapter1.xhtml', 'application/xhtml+xml'), 42 ('ch2', 'chapter2.xhtml', 'application/xhtml+xml'), 43 ('ch3', 'chapter3.xhtml', 'application/xhtml+xml'), 44 ('cover', 'cover.png', 'image/png'), 45 ] 46 47 OPF = """<?xml version="1.0" encoding="UTF-8"?> 48 <package xmlns="http://www.idpf.org/2007/opf" version="3.0" 49 unique-identifier="bookid"> 50 <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" 51 xmlns:opf="http://www.idpf.org/2007/opf"> 52 <dc:title>The libextractor Test Book</dc:title> 53 <dc:creator opf:role="aut" opf:file-as="Grothoff, Christian" 54 >Christian Grothoff</dc:creator> 55 <dc:contributor opf:role="bkp" 56 >calibre (7.2.0) [https://calibre-ebook.com]</dc:contributor> 57 <dc:publisher>GNU Press</dc:publisher> 58 <dc:language>en-GB</dc:language> 59 <dc:date>2026-01-01T00:00:00+00:00</dc:date> 60 <dc:subject>digital forensics</dc:subject> 61 <dc:subject>metadata extraction</dc:subject> 62 <dc:description>A minimal EPUB built by gen_ebook_testdata.sh, used 63 to check that the ebook plugin reads Dublin Core out of the 64 OPF package document.</dc:description> 65 <dc:rights>Public domain (CC0) & nothing else</dc:rights> 66 <dc:identifier id="bookid" opf:scheme="ISBN" 67 >978-3-16-148410-0</dc:identifier> 68 <dc:identifier opf:scheme="URI" 69 >urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719</dc:identifier> 70 <meta name="calibre:timestamp" content="2026-01-01T00:00:00+00:00"/> 71 <meta name="generator" content="gen_ebook_testdata.sh"/> 72 <meta property="dcterms:modified">2026-02-01T00:00:00Z</meta> 73 </metadata> 74 <manifest> 75 %s 76 </manifest> 77 <spine toc="ncx"> 78 <itemref idref="ch1"/> 79 <itemref idref="ch2"/> 80 <itemref idref="ch3"/> 81 </spine> 82 </package> 83 """ % "\n".join (' <item id="%s" href="%s" media-type="%s"/>' % i 84 for i in ITEMS) 85 86 # Font obfuscation, which is what encryption.xml usually means in an 87 # EPUB; the plugin only reports that it is there. 88 ENCRYPTION = """<?xml version="1.0" encoding="UTF-8"?> 89 <encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> 90 <EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"> 91 <EncryptionMethod 92 Algorithm="http://www.idpf.org/2008/embedding"/> 93 <CipherData> 94 <CipherReference URI="OEBPS/fonts/serif.otf"/> 95 </CipherData> 96 </EncryptedData> 97 </encryption> 98 """ 99 100 CHAPTER = """<?xml version="1.0" encoding="UTF-8"?> 101 <html xmlns="http://www.w3.org/1999/xhtml"><head><title>%s</title></head> 102 <body><h1>%s</h1><p>Nothing to see here.</p></body></html> 103 """ 104 105 epub = os.path.join (OUT, "ebook_test.epub") 106 with zipfile.ZipFile (epub, "w") as z: 107 # the spec requires `mimetype' to be the first member and stored 108 zi = zipfile.ZipInfo ("mimetype", DATE) 109 zi.compress_type = zipfile.ZIP_STORED 110 z.writestr (zi, b"application/epub+zip") 111 for name, data in (("META-INF/container.xml", CONTAINER), 112 ("META-INF/encryption.xml", ENCRYPTION), 113 ("OEBPS/content.opf", OPF), 114 ("OEBPS/chapter1.xhtml", CHAPTER % ("One", "One")), 115 ("OEBPS/chapter2.xhtml", CHAPTER % ("Two", "Two")), 116 ("OEBPS/chapter3.xhtml", CHAPTER % ("Three", "Three")), 117 ("OEBPS/style.css", "body { margin: 1em; }\n")): 118 zi = zipfile.ZipInfo (name, DATE) 119 zi.compress_type = zipfile.ZIP_DEFLATED 120 z.writestr (zi, data.encode ("utf-8")) 121 122 123 # ------------------------------------------------------------------- MOBI 124 # 125 # Layout per the MobileRead wiki: a 78-byte PDB header, one 8-byte record 126 # info entry per record, then the records. Record 0 holds the 16-byte 127 # PalmDOC header, the MOBI header and the EXTH header. 128 MOBI_EPOCH = 2082844800 # seconds between 1904-01-01 and 1970-01-01 129 CTIME = 1767225600 # 2026-01-01T00:00:00Z 130 MTIME = 1769904000 # 2026-02-01T00:00:00Z 131 132 DB_NAME = b"The_libextractor_Test_Book" 133 FULL_NAME = b"The libextractor Test Book" 134 TEXT = (b"This is the body text of the libextractor MOBI test file. " 135 b"It exists only so that record 1 is not empty.\n") 136 137 EXTH_RECORDS = [ 138 (100, b"Christian Grothoff"), # author 139 (101, b"GNU Press"), # publisher 140 (103, b"A hand-built minimal MOBI file used by the " 141 b"libextractor test suite."), # description 142 (104, b"978-3-16-148410-0"), # ISBN 143 (105, b"digital forensics"), # subject 144 (106, b"2026-01-01"), # published 145 (108, b"calibre (7.2.0) [https://calibre-ebook.com]"), # contributor 146 (109, b"Public domain (CC0)"), # rights 147 (113, b"B00LIBEXTR7"), # ASIN 148 (204, struct.pack (">I", 201)), # creator software 149 (205, struct.pack (">I", 7)), # creator major 150 (206, struct.pack (">I", 2)), # creator minor 151 (208, b"libextractor-test-watermark"), # watermark 152 (501, b"EBOK"), # cdetype 153 (502, b"2026-02-01T00:00:00+00:00"), # last update time 154 (503, FULL_NAME), # updated title 155 (524, b"en-gb"), # language 156 ] 157 158 159 def exth (): 160 body = b"" 161 for t, d in EXTH_RECORDS: 162 body += struct.pack (">II", t, 8 + len (d)) + d 163 size = 12 + len (body) 164 pad = (-size) % 4 165 return b"EXTH" + struct.pack (">II", size + pad, 166 len (EXTH_RECORDS)) + body + b"\0" * pad 167 168 169 MOBI_HEADER_LEN = 232 170 171 172 def record0 (): 173 e = exth () 174 # PalmDOC header, then MOBI header, then EXTH, then the full name 175 full_name_off = 16 + MOBI_HEADER_LEN + len (e) 176 mobi = bytearray (MOBI_HEADER_LEN) 177 mobi[0:4] = b"MOBI" 178 struct.pack_into (">I", mobi, 4, MOBI_HEADER_LEN) 179 struct.pack_into (">I", mobi, 8, 2) # Mobipocket book 180 struct.pack_into (">I", mobi, 12, 65001) # UTF-8 181 struct.pack_into (">I", mobi, 16, 0x1719) # unique ID 182 struct.pack_into (">I", mobi, 20, 6) # file version 183 struct.pack_into (">I", mobi, 64, 2) # first non-book index 184 struct.pack_into (">I", mobi, 68, full_name_off) 185 struct.pack_into (">I", mobi, 72, len (FULL_NAME)) 186 struct.pack_into (">I", mobi, 76, 9) # locale: English 187 struct.pack_into (">I", mobi, 88, 6) # min version 188 struct.pack_into (">I", mobi, 92, 2) # first image index 189 struct.pack_into (">I", mobi, 112, 0x50) # EXTH flags: 0x40 = present 190 palmdoc = struct.pack (">HHIHHHH", 191 1, # no compression 192 0, 193 len (TEXT), 194 1, # record count 195 4096, # record size 196 1, # encryption: old Mobipocket 197 0) 198 rec = palmdoc + bytes (mobi) + e + FULL_NAME + b"\0" 199 while len (rec) % 4: 200 rec += b"\0" 201 return rec 202 203 204 records = [record0 (), TEXT, b"\0"] 205 206 hdr = bytearray (78) 207 hdr[0:len (DB_NAME)] = DB_NAME 208 struct.pack_into (">HH", hdr, 32, 0, 0) # attributes, version 209 struct.pack_into (">III", hdr, 36, 210 CTIME + MOBI_EPOCH, 211 MTIME + MOBI_EPOCH, 212 0) # create/mod/backup 213 struct.pack_into (">III", hdr, 48, 0, 0, 0) # modnum, appinfo, sort 214 hdr[60:68] = b"BOOKMOBI" 215 struct.pack_into (">II", hdr, 68, 0x1719, 0) 216 struct.pack_into (">H", hdr, 76, len (records)) 217 218 info_len = 8 * len (records) + 2 # + 2 bytes of padding 219 off = 78 + info_len 220 info = b"" 221 for i, r in enumerate (records): 222 info += struct.pack (">IBBBB", off, 0, 0, 0, i) 223 off += len (r) 224 info += b"\0\0" 225 226 with open (os.path.join (OUT, "ebook_test.mobi"), "wb") as f: 227 f.write (bytes (hdr) + info + b"".join (records)) 228 229 230 # --------------------------------------------------- full text: EPUB 231 # 232 # The spine order (nav, c3, c1, c2, notes) differs from the manifest 233 # order; the package document is not in a conventional place; one 234 # chapter lives in a sibling directory ("../"), one has a 235 # percent-encoded name, one is declared ISO-8859-1, one is listed in 236 # encryption.xml (and holds garbage), and one is linear="no". The 237 # expected text is spelled out in test_ebook.c. 238 FT_CONTAINER = CONTAINER.replace ("OEBPS/content.opf", "OPS/package.opf") 239 240 FT_ITEMS = [ 241 ('nav', 'nav.xhtml', 'application/xhtml+xml', ' properties="nav"'), 242 ('c1', 'text/c1.xhtml', 'application/xhtml+xml', ''), 243 ('c2', 'text/chapter%20two.xhtml', 'application/xhtml+xml', ''), 244 ('c3', '../extra/c3.xhtml', 'application/xhtml+xml', ''), 245 ('secret', 'text/secret.xhtml', 'application/xhtml+xml', ''), 246 ('notes', 'text/notes.xhtml', 'application/xhtml+xml', ''), 247 ('css', 'style.css', 'text/css', ''), 248 ] 249 250 FT_OPF = """<?xml version="1.0" encoding="UTF-8"?> 251 <package xmlns="http://www.idpf.org/2007/opf" version="3.0" 252 unique-identifier="bookid"> 253 <metadata xmlns:dc="http://purl.org/dc/elements/1.1/"> 254 <dc:title>Full Text Test</dc:title> 255 <dc:identifier id="bookid">urn:uuid:0f9d5a1e-1719-4c0b-9a3e-00000000f7f7</dc:identifier> 256 <dc:language>en</dc:language> 257 </metadata> 258 <manifest> 259 %s 260 </manifest> 261 <spine> 262 <itemref idref="nav"/> 263 <itemref idref="c3"/> 264 <itemref idref="c1"/> 265 <itemref idref="secret"/> 266 <itemref idref="c2"/> 267 <itemref idref="notes" linear="no"/> 268 </spine> 269 </package> 270 """ % "\n".join (' <item id="%s" href="%s" media-type="%s"%s/>' % i 271 for i in FT_ITEMS) 272 273 FT_ENCRYPTION = ENCRYPTION.replace ("OEBPS/fonts/serif.otf", 274 "OPS/text/secret.xhtml") 275 276 XHTML = """<?xml version="1.0" encoding="%s"?> 277 <!DOCTYPE html> 278 <html xmlns="http://www.w3.org/1999/xhtml" 279 xmlns:epub="http://www.idpf.org/2007/ops"> 280 <head><title>%s</title> 281 <style>p { margin: 0; }</style> 282 <script>var no = "<p>script text</p>";</script> 283 </head> 284 <body> 285 %s 286 </body> 287 </html> 288 """ 289 290 FT_NAV = XHTML % ("UTF-8", "Contents", """<nav epub:type="toc" id="toc"> 291 <h1>Contents</h1> 292 <ol><li><a href="../extra/c3.xhtml">Chapter Alpha</a></li> 293 <li><a href="text/c1.xhtml">Chapter Beta</a></li></ol> 294 </nav>""") 295 296 FT_C3 = XHTML % ("UTF-8", "Alpha", """<h1>Chapter Alpha</h1> 297 <p>Alpha text comes 298 first in the reading order.</p> 299 <span epub:type="pagebreak" id="page2" title="2">2</span> 300 <p>Second paragraph of alpha—with a dash & an ampersand.</p>""") 301 302 FT_C1 = XHTML % ("ISO-8859-1", "Beta", """<header>Running head</header> 303 <section> 304 <header><h2>Chapter Beta</h2></header> 305 <p>Beta: café crème.<a epub:type="noteref" href="notes.xhtml#n1" 306 >*</a></p> 307 <aside epub:type="footnote" id="n1"><p>Footnote text</p></aside> 308 </section> 309 <footer>Page footer</footer>""") 310 311 FT_C2 = XHTML % ("UTF-8", "Gamma", """<h1>Chapter Gamma</h1> 312 <p>Gamma line one<br/>line two<br/><br/>Gamma new paragraph here.</p> 313 <div hidden="hidden">Hidden text</div> 314 <p aria-hidden="true">Decoration</p> 315 <p>Last… <em>really</em> last.</p>""") 316 317 FT_NOTES = XHTML % ("UTF-8", "Notes", """<p id="n1">Note text</p>""") 318 319 ftepub = os.path.join (OUT, "ebook_fulltext.epub") 320 with zipfile.ZipFile (ftepub, "w") as z: 321 zi = zipfile.ZipInfo ("mimetype", DATE) 322 zi.compress_type = zipfile.ZIP_STORED 323 z.writestr (zi, b"application/epub+zip") 324 for name, data in ( 325 ("META-INF/container.xml", FT_CONTAINER.encode ("utf-8")), 326 ("META-INF/encryption.xml", FT_ENCRYPTION.encode ("utf-8")), 327 ("OPS/package.opf", FT_OPF.encode ("utf-8")), 328 ("OPS/nav.xhtml", FT_NAV.encode ("utf-8")), 329 ("OPS/text/c1.xhtml", FT_C1.encode ("iso-8859-1")), 330 ("OPS/text/chapter two.xhtml", FT_C2.encode ("utf-8")), 331 ("extra/c3.xhtml", FT_C3.encode ("utf-8")), 332 ("OPS/text/secret.xhtml", bytes (range (256)) * 4), 333 ("OPS/text/notes.xhtml", FT_NOTES.encode ("utf-8")), 334 ("OPS/style.css", b"p { margin: 0; }\n")): 335 zi = zipfile.ZipInfo (name, DATE) 336 zi.compress_type = zipfile.ZIP_DEFLATED 337 z.writestr (zi, data) 338 339 340 # --------------------------------------------------- full text: MOBI 341 # 342 # PalmDOC compressed, UTF-8, text split into small records so that a 343 # multibyte character straddles a record boundary, and every record 344 # carries both kinds of trailing entries (extra data flags 0x3). 345 FT_TEXT = ("<html><head><guide><reference type=\"toc\" title=\"Contents\" " 346 "filepos=0000000100 /></guide></head><body>" 347 "<h1>The Test Book</h1>" 348 "<p>First paragraph with an ümlaut, repeated text, " 349 "repeated text, repeated text.</p>" 350 "<mbp:pagebreak/>" 351 "<p height=\"1em\" width=\"0pt\">Second chapter starts " 352 "here — à la carte.</p>" 353 "<p>Line<br>break and & entity.</p>" 354 "</body></html>").encode ("utf-8") 355 FT_RECORD = 64 356 357 358 def palmdoc_compress (data): 359 """Greedy PalmDOC LZ77; uses all four kinds of codes.""" 360 out = bytearray () 361 i = 0 362 n = len (data) 363 while i < n: 364 # back reference: distance 1..2047, length 3..10 365 best_len = 0 366 best_dist = 0 367 for dist in range (1, min (i, 2047) + 1): 368 l = 0 369 while (l < 10) and (i + l < n) and (data[i + l - dist] == data[i + l]): 370 l += 1 371 if l > best_len: 372 best_len = l 373 best_dist = dist 374 if best_len >= 3: 375 code = 0x8000 | (best_dist << 3) | (best_len - 3) 376 out += struct.pack (">H", code) 377 i += best_len 378 continue 379 c = data[i] 380 if (c == 0x20) and (i + 1 < n) and (0x40 <= data[i + 1] <= 0x7f): 381 out.append (data[i + 1] ^ 0x80) 382 i += 2 383 continue 384 if (c == 0) or (0x09 <= c <= 0x7f): 385 out.append (c) 386 i += 1 387 continue 388 # literal run of up to 8 bytes 389 j = i 390 while (j < n) and (j - i < 8) and not ((data[j] == 0) or 391 (0x09 <= data[j] <= 0x7f)): 392 j += 1 393 out.append (j - i) 394 out += data[i:j] 395 i = j 396 return bytes (out) 397 398 399 def text_records (): 400 recs = [] 401 pos = 0 402 n = len (FT_TEXT) 403 while pos < n: 404 end = min (pos + FT_RECORD, n) 405 # continuation bytes of a character that straddles the boundary 406 k = 0 407 while (end + k < n) and (FT_TEXT[end + k] & 0xC0) == 0x80: 408 k += 1 409 rec = palmdoc_compress (FT_TEXT[pos:end]) 410 rec += FT_TEXT[end:end + k] + bytes ([k]) # multibyte overlap 411 rec += b"\x00\x00\x83" # 3-byte TBS entry 412 recs.append (rec) 413 pos = end 414 return recs 415 416 417 def ft_record0 (n_text): 418 mobi = bytearray (MOBI_HEADER_LEN) 419 mobi[0:4] = b"MOBI" 420 struct.pack_into (">I", mobi, 4, MOBI_HEADER_LEN) 421 struct.pack_into (">I", mobi, 8, 2) # Mobipocket book 422 struct.pack_into (">I", mobi, 12, 65001) # UTF-8 423 struct.pack_into (">I", mobi, 20, 6) # file version 424 struct.pack_into (">I", mobi, 68, 16 + MOBI_HEADER_LEN) 425 struct.pack_into (">I", mobi, 72, len (b"Full Text Test")) 426 struct.pack_into (">H", mobi, 0xE2, 3) # extra data flags 427 palmdoc = struct.pack (">HHIHHHH", 428 2, # PalmDOC compression 429 0, 430 len (FT_TEXT), 431 n_text, 432 FT_RECORD, 433 0, # not encrypted 434 0) 435 rec = palmdoc + bytes (mobi) + b"Full Text Test\0" 436 while len (rec) % 4: 437 rec += b"\0" 438 return rec 439 440 441 def write_pdb (path, name, records): 442 hdr = bytearray (78) 443 hdr[0:len (name)] = name 444 struct.pack_into (">III", hdr, 36, 445 CTIME + MOBI_EPOCH, MTIME + MOBI_EPOCH, 0) 446 hdr[60:68] = b"BOOKMOBI" 447 struct.pack_into (">H", hdr, 76, len (records)) 448 off = 78 + 8 * len (records) + 2 449 info = b"" 450 for i, r in enumerate (records): 451 info += struct.pack (">IBBBB", off, 0, 0, 0, i) 452 off += len (r) 453 info += b"\0\0" 454 with open (path, "wb") as f: 455 f.write (bytes (hdr) + info + b"".join (records)) 456 457 458 trecs = text_records () 459 write_pdb (os.path.join (OUT, "ebook_fulltext.mobi"), 460 b"Full_Text_Test", 461 [ft_record0 (len (trecs))] + trecs + [b"\xe9\x8e\x0d\x0a"]) 462 463 for n in ("ebook_test.epub", "ebook_test.mobi", 464 "ebook_fulltext.epub", "ebook_fulltext.mobi"): 465 print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n)))) 466 PYEOF