commit 8975fde974153e97261709829be1d2262a0204fc
parent 45f0a3c4caf3f06e5801289b524f9e615b94ee87
Author: Christian Grothoff <christian@grothoff.org>
Date: Sun, 1 Nov 2026 19:46:07 +0100
Add full text, subtitle and gallery extraction options
New opt-in plugin options that make libextractor report the contents of
a document rather than only its metadata. Every option is off by
default: without it, each plugin behaves exactly as before.
Public API: two new meta types, EXTRACTOR_METATYPE_FULL_TEXT and
EXTRACTOR_METATYPE_SUBTITLE_TEXT.
"fulltext" (pdf, dvi, odf, ole2, html, ebook) reports the running text
of a document as one FULL_TEXT item.
pdf: poppler's text boxes are grouped into lines by vertical overlap,
margin columns of numbers are dropped, and a two-column layout is
detected from a gutter that (nearly) no line crosses so that the left
column is read before the right one. Ligatures are expanded and
accents composed. At most 5000 pages and 60 s.
dvi: the opcode stream of every page is interpreted under bounded
opcode, stack, page, line and byte budgets; glyphs are mapped to
Unicode by the encoding guessed from the font name (OT1 with
ligatures/quotes/dashes/accents, OT1 typewriter, T1/Cork including
cm-super and lmodern, 8r, math italic and symbols; math extension and
dingbat fonts are dropped). Word breaks come from horizontal moves
over 0.17 em, new lines from vertical moves over 0.7 em.
odf: content.xml (at most 64 MiB decompressed) is scanned
iteratively with a bounded element stack. text:p/text:h are
paragraphs, text:s/text:tab/text:line-break are spaces, entities and
numeric character references are decoded. Comments,
footnotes/endnotes, tracked-change deletions, page number/count
fields, frame alt text and speaker notes are skipped; namespace
prefixes are resolved from the root element.
ole2: for Word 97-2003 the FIB is parsed from the WordDocument
stream, the piece table (CLX) is read from 0Table/1Table and only the
main text [0, ccpText) is converted, so headers/footers, notes,
comments and text boxes stay out. 8-bit pieces use the cp1252
mapping of [MS-DOC] 2.9.73. Field instructions are dropped and
results kept, except PAGE/NUMPAGES/SECTIONPAGES. Excel and
PowerPoint files get no full text.
html: the file (up to 16 MiB) is converted to UTF-8 per BOM / XML
declaration / META charset, sniffing UTF-8 vs. windows-1252
otherwise, parsed with libtidy from memory and walked from <body>
with bounded depth. Script, style, noscript, template, nav, aside,
page-level header/footer, margin landmark roles and hidden elements
are dropped.
ebook: EPUB text follows the linear OPF spine (hrefs resolved
relative to the OPF and percent-decoded, DRM-listed documents
skipped), walking the archive once so a hostile spine cannot make
lookups quadratic. MOBI text records are read, trailing entries
stripped and PalmDOC decompressed; encrypted and HUFF/CDIC books are
skipped.
Shared helpers: fulltext.[ch] (buffering, margin/furniture filtering,
page joining) and pagelayout.[ch], which decides paragraph ends from
line geometry - vertical gap vs. typical line spacing, first-line
indentation, font size change, short last line of a justified block,
continuation across page breaks - and composes base letter plus
combining accent into precomposed characters.
"srt:LANG" and "gallery:N" (thumbnailffmpeg, thumbnailgst) dump the
first text subtitle track of a language as a SubRip file, and replace
the single thumbnail by a contact sheet of N frames, re-sampling flat
frames from within their slice. Both plugins also report the language
of every subtitle stream, ffmpeg's also for files without a usable
video stream. The gst side demuxes through appsrc ! parsebin, accepts
text/x-raw and application/x-ass/x-ssa, strips Pango markup, composes
the sheet in memory via gst_video_convert_sample(), and caps and halves
an oversized sheet like the ffmpeg plugin does. mediautil.[ch] and
mediagst.h hold what the two share.
IPC: a no-op seek (cls, 0, SEEK_CUR) now sends a rate-limited
one-byte MESSAGE_ALIVE keep-alive. Out of process, plugins silent for
500 ms were killed, which made full text of large documents and big
galleries fail; hung plugins are still killed. Used by the fulltext
buffer, pdf, dvi, odf and both video plugins.
extract: UTF-8 values between 1 and 32 MiB are no longer dropped.
Tests: synthetic fixtures generated by contrib/gen_html_testdata.sh,
contrib/gen_odf_testdata.sh, contrib/gen_fulltext_pdf_dvi_testdata.sh,
extended contrib/gen_ebook_testdata.sh and contrib/gen_testmedia.sh
(subtitled video). test_pdf/test_dvi share test_paged_fulltext.h;
test_odf, test_ole2, test_html, test_ebook, test_gstreamer,
test_thumbnailffmpeg and test_thumbnailgst cover the new options, and
each checks that nothing extra appears without them.
Fuzzing: new fuzz_html, fuzz_pdf (linked as C++) and fuzz_thumbnailgst
targets; fuzz_dvi, fuzz_odf, fuzz_ole2, fuzz_ebook and the ffmpeg
target also fuzz the new configurations. Leak suppressions use the
slow unwinder where they match on library frames, GStreamer's global
leaks are suppressed and at-exit reports no longer leave a stale
reproducer.
Documentation: doc/extract.1 and doc/libextractor.texi describe the
fulltext, srt and gallery options.
Assisted by LLM
Diffstat:
64 files changed, 17137 insertions(+), 543 deletions(-)
diff --git a/Makefile.am b/Makefile.am
@@ -7,6 +7,7 @@ EXTRA_DIST = config.rpath \
contrib/gen_diskimage_testdata.sh \
contrib/gen_ebook_testdata.sh \
contrib/gen_geotiff_testdata.sh \
+ contrib/gen_fulltext_pdf_dvi_testdata.sh \
contrib/gen_gpx_testdata.sh \
contrib/gen_heif_testdata.sh \
contrib/gen_id3_testdata.sh \
@@ -14,6 +15,7 @@ EXTRA_DIST = config.rpath \
contrib/gen_kml_testdata.sh \
contrib/gen_lnk_testdata.sh \
contrib/gen_mbox_testdata.sh \
+ contrib/gen_odf_testdata.sh \
contrib/gen_pecoff_testdata.sh \
contrib/gen_plist_testdata.sh \
contrib/gen_sqlite_testdata.sh \
diff --git a/configure.ac b/configure.ac
@@ -24,9 +24,9 @@ AC_CANONICAL_TARGET
AC_CANONICAL_HOST
AC_CANONICAL_TARGET
-LIB_VERSION_CURRENT=4
-LIB_VERSION_REVISION=10
-LIB_VERSION_AGE=1
+LIB_VERSION_CURRENT=5
+LIB_VERSION_REVISION=0
+LIB_VERSION_AGE=2
AC_SUBST(LIB_VERSION_CURRENT)
AC_SUBST(LIB_VERSION_REVISION)
AC_SUBST(LIB_VERSION_AGE)
diff --git a/contrib/gen_ebook_testdata.sh b/contrib/gen_ebook_testdata.sh
@@ -1,5 +1,6 @@
#!/bin/sh
-# Regenerate src/plugins/testdata/ebook_test.epub and ebook_test.mobi.
+# Regenerate src/plugins/testdata/ebook_test.epub and ebook_test.mobi,
+# and the full text fixtures ebook_fulltext.epub and ebook_fulltext.mobi.
#
# This file is in the public domain (CC0); so are the files it produces.
#
@@ -225,6 +226,241 @@ info += b"\0\0"
with open (os.path.join (OUT, "ebook_test.mobi"), "wb") as f:
f.write (bytes (hdr) + info + b"".join (records))
-for n in ("ebook_test.epub", "ebook_test.mobi"):
+
+# --------------------------------------------------- full text: EPUB
+#
+# The spine order (nav, c3, c1, c2, notes) differs from the manifest
+# order; the package document is not in a conventional place; one
+# chapter lives in a sibling directory ("../"), one has a
+# percent-encoded name, one is declared ISO-8859-1, one is listed in
+# encryption.xml (and holds garbage), and one is linear="no". The
+# expected text is spelled out in test_ebook.c.
+FT_CONTAINER = CONTAINER.replace ("OEBPS/content.opf", "OPS/package.opf")
+
+FT_ITEMS = [
+ ('nav', 'nav.xhtml', 'application/xhtml+xml', ' properties="nav"'),
+ ('c1', 'text/c1.xhtml', 'application/xhtml+xml', ''),
+ ('c2', 'text/chapter%20two.xhtml', 'application/xhtml+xml', ''),
+ ('c3', '../extra/c3.xhtml', 'application/xhtml+xml', ''),
+ ('secret', 'text/secret.xhtml', 'application/xhtml+xml', ''),
+ ('notes', 'text/notes.xhtml', 'application/xhtml+xml', ''),
+ ('css', 'style.css', 'text/css', ''),
+]
+
+FT_OPF = """<?xml version="1.0" encoding="UTF-8"?>
+<package xmlns="http://www.idpf.org/2007/opf" version="3.0"
+ unique-identifier="bookid">
+ <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
+ <dc:title>Full Text Test</dc:title>
+ <dc:identifier id="bookid">urn:uuid:0f9d5a1e-1719-4c0b-9a3e-00000000f7f7</dc:identifier>
+ <dc:language>en</dc:language>
+ </metadata>
+ <manifest>
+%s
+ </manifest>
+ <spine>
+ <itemref idref="nav"/>
+ <itemref idref="c3"/>
+ <itemref idref="c1"/>
+ <itemref idref="secret"/>
+ <itemref idref="c2"/>
+ <itemref idref="notes" linear="no"/>
+ </spine>
+</package>
+""" % "\n".join (' <item id="%s" href="%s" media-type="%s"%s/>' % i
+ for i in FT_ITEMS)
+
+FT_ENCRYPTION = ENCRYPTION.replace ("OEBPS/fonts/serif.otf",
+ "OPS/text/secret.xhtml")
+
+XHTML = """<?xml version="1.0" encoding="%s"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml"
+ xmlns:epub="http://www.idpf.org/2007/ops">
+<head><title>%s</title>
+<style>p { margin: 0; }</style>
+<script>var no = "<p>script text</p>";</script>
+</head>
+<body>
+%s
+</body>
+</html>
+"""
+
+FT_NAV = XHTML % ("UTF-8", "Contents", """<nav epub:type="toc" id="toc">
+<h1>Contents</h1>
+<ol><li><a href="../extra/c3.xhtml">Chapter Alpha</a></li>
+<li><a href="text/c1.xhtml">Chapter Beta</a></li></ol>
+</nav>""")
+
+FT_C3 = XHTML % ("UTF-8", "Alpha", """<h1>Chapter Alpha</h1>
+<p>Alpha text comes
+ first in the reading order.</p>
+<span epub:type="pagebreak" id="page2" title="2">2</span>
+<p>Second paragraph of alpha—with a dash & an ampersand.</p>""")
+
+FT_C1 = XHTML % ("ISO-8859-1", "Beta", """<header>Running head</header>
+<section>
+<header><h2>Chapter Beta</h2></header>
+<p>Beta: café crème.<a epub:type="noteref" href="notes.xhtml#n1"
+>*</a></p>
+<aside epub:type="footnote" id="n1"><p>Footnote text</p></aside>
+</section>
+<footer>Page footer</footer>""")
+
+FT_C2 = XHTML % ("UTF-8", "Gamma", """<h1>Chapter Gamma</h1>
+<p>Gamma line one<br/>line two<br/><br/>Gamma new paragraph here.</p>
+<div hidden="hidden">Hidden text</div>
+<p aria-hidden="true">Decoration</p>
+<p>Last… <em>really</em> last.</p>""")
+
+FT_NOTES = XHTML % ("UTF-8", "Notes", """<p id="n1">Note text</p>""")
+
+ftepub = os.path.join (OUT, "ebook_fulltext.epub")
+with zipfile.ZipFile (ftepub, "w") as z:
+ zi = zipfile.ZipInfo ("mimetype", DATE)
+ zi.compress_type = zipfile.ZIP_STORED
+ z.writestr (zi, b"application/epub+zip")
+ for name, data in (
+ ("META-INF/container.xml", FT_CONTAINER.encode ("utf-8")),
+ ("META-INF/encryption.xml", FT_ENCRYPTION.encode ("utf-8")),
+ ("OPS/package.opf", FT_OPF.encode ("utf-8")),
+ ("OPS/nav.xhtml", FT_NAV.encode ("utf-8")),
+ ("OPS/text/c1.xhtml", FT_C1.encode ("iso-8859-1")),
+ ("OPS/text/chapter two.xhtml", FT_C2.encode ("utf-8")),
+ ("extra/c3.xhtml", FT_C3.encode ("utf-8")),
+ ("OPS/text/secret.xhtml", bytes (range (256)) * 4),
+ ("OPS/text/notes.xhtml", FT_NOTES.encode ("utf-8")),
+ ("OPS/style.css", b"p { margin: 0; }\n")):
+ zi = zipfile.ZipInfo (name, DATE)
+ zi.compress_type = zipfile.ZIP_DEFLATED
+ z.writestr (zi, data)
+
+
+# --------------------------------------------------- full text: MOBI
+#
+# PalmDOC compressed, UTF-8, text split into small records so that a
+# multibyte character straddles a record boundary, and every record
+# carries both kinds of trailing entries (extra data flags 0x3).
+FT_TEXT = ("<html><head><guide><reference type=\"toc\" title=\"Contents\" "
+ "filepos=0000000100 /></guide></head><body>"
+ "<h1>The Test Book</h1>"
+ "<p>First paragraph with an ümlaut, repeated text, "
+ "repeated text, repeated text.</p>"
+ "<mbp:pagebreak/>"
+ "<p height=\"1em\" width=\"0pt\">Second chapter starts "
+ "here — à la carte.</p>"
+ "<p>Line<br>break and & entity.</p>"
+ "</body></html>").encode ("utf-8")
+FT_RECORD = 64
+
+
+def palmdoc_compress (data):
+ """Greedy PalmDOC LZ77; uses all four kinds of codes."""
+ out = bytearray ()
+ i = 0
+ n = len (data)
+ while i < n:
+ # back reference: distance 1..2047, length 3..10
+ best_len = 0
+ best_dist = 0
+ for dist in range (1, min (i, 2047) + 1):
+ l = 0
+ while (l < 10) and (i + l < n) and (data[i + l - dist] == data[i + l]):
+ l += 1
+ if l > best_len:
+ best_len = l
+ best_dist = dist
+ if best_len >= 3:
+ code = 0x8000 | (best_dist << 3) | (best_len - 3)
+ out += struct.pack (">H", code)
+ i += best_len
+ continue
+ c = data[i]
+ if (c == 0x20) and (i + 1 < n) and (0x40 <= data[i + 1] <= 0x7f):
+ out.append (data[i + 1] ^ 0x80)
+ i += 2
+ continue
+ if (c == 0) or (0x09 <= c <= 0x7f):
+ out.append (c)
+ i += 1
+ continue
+ # literal run of up to 8 bytes
+ j = i
+ while (j < n) and (j - i < 8) and not ((data[j] == 0) or
+ (0x09 <= data[j] <= 0x7f)):
+ j += 1
+ out.append (j - i)
+ out += data[i:j]
+ i = j
+ return bytes (out)
+
+
+def text_records ():
+ recs = []
+ pos = 0
+ n = len (FT_TEXT)
+ while pos < n:
+ end = min (pos + FT_RECORD, n)
+ # continuation bytes of a character that straddles the boundary
+ k = 0
+ while (end + k < n) and (FT_TEXT[end + k] & 0xC0) == 0x80:
+ k += 1
+ rec = palmdoc_compress (FT_TEXT[pos:end])
+ rec += FT_TEXT[end:end + k] + bytes ([k]) # multibyte overlap
+ rec += b"\x00\x00\x83" # 3-byte TBS entry
+ recs.append (rec)
+ pos = end
+ return recs
+
+
+def ft_record0 (n_text):
+ mobi = bytearray (MOBI_HEADER_LEN)
+ mobi[0:4] = b"MOBI"
+ struct.pack_into (">I", mobi, 4, MOBI_HEADER_LEN)
+ struct.pack_into (">I", mobi, 8, 2) # Mobipocket book
+ struct.pack_into (">I", mobi, 12, 65001) # UTF-8
+ struct.pack_into (">I", mobi, 20, 6) # file version
+ struct.pack_into (">I", mobi, 68, 16 + MOBI_HEADER_LEN)
+ struct.pack_into (">I", mobi, 72, len (b"Full Text Test"))
+ struct.pack_into (">H", mobi, 0xE2, 3) # extra data flags
+ palmdoc = struct.pack (">HHIHHHH",
+ 2, # PalmDOC compression
+ 0,
+ len (FT_TEXT),
+ n_text,
+ FT_RECORD,
+ 0, # not encrypted
+ 0)
+ rec = palmdoc + bytes (mobi) + b"Full Text Test\0"
+ while len (rec) % 4:
+ rec += b"\0"
+ return rec
+
+
+def write_pdb (path, name, records):
+ hdr = bytearray (78)
+ hdr[0:len (name)] = name
+ struct.pack_into (">III", hdr, 36,
+ CTIME + MOBI_EPOCH, MTIME + MOBI_EPOCH, 0)
+ hdr[60:68] = b"BOOKMOBI"
+ struct.pack_into (">H", hdr, 76, len (records))
+ off = 78 + 8 * len (records) + 2
+ info = b""
+ for i, r in enumerate (records):
+ info += struct.pack (">IBBBB", off, 0, 0, 0, i)
+ off += len (r)
+ info += b"\0\0"
+ with open (path, "wb") as f:
+ f.write (bytes (hdr) + info + b"".join (records))
+
+
+trecs = text_records ()
+write_pdb (os.path.join (OUT, "ebook_fulltext.mobi"),
+ b"Full_Text_Test",
+ [ft_record0 (len (trecs))] + trecs + [b"\xe9\x8e\x0d\x0a"])
+
+for n in ("ebook_test.epub", "ebook_test.mobi",
+ "ebook_fulltext.epub", "ebook_fulltext.mobi"):
print ("%s: %d bytes" % (n, os.path.getsize (os.path.join (OUT, n))))
PYEOF
diff --git a/contrib/gen_fulltext_pdf_dvi_testdata.sh b/contrib/gen_fulltext_pdf_dvi_testdata.sh
@@ -0,0 +1,135 @@
+#!/bin/sh
+# This file is part of libextractor.
+# Copyright (C) 2026 Christian Grothoff
+#
+# Regenerate src/plugins/testdata/pdf_fulltext.pdf and
+# src/plugins/testdata/dvi_fulltext.dvi, the test documents for the
+# "fulltext" option of the pdf and dvi plugins.
+#
+# Requires: pdflatex and latex (TeX Live) with the fancyhdr, geometry
+# and lineno packages and the EC fonts' TFM files. SOURCE_DATE_EPOCH
+# makes the output reproducible for a given TeX Live version.
+#
+# Both documents come from the same LaTeX source (on A5 paper, so that
+# there are few words per page):
+# - a running header ("Notes on Paged Text" / "Working Draft") and a
+# page number in the footer of every page, which must not end up in
+# the full text;
+# - indented paragraphs, a one-line paragraph, a paragraph that runs
+# across a page break, several words hyphenated at line ends, the
+# fi/ff/ffl ligatures, TeX quotes and dashes and accented letters;
+# - PDF only: a two-column page with line numbers in the outer
+# margins (lineno), which the extractor must drop, reading the left
+# column before the right one;
+# - DVI only: a paragraph set in the T1 (Cork) encoding.
+#
+# The text is invented and is dedicated to the public domain (CC0).
+set -e
+
+srcdir=$(cd "$(dirname "$0")/.." && pwd)
+out="$srcdir/src/plugins/testdata"
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+SOURCE_DATE_EPOCH=1700000000
+FORCE_SOURCE_DATE=1
+export SOURCE_DATE_EPOCH FORCE_SOURCE_DATE
+
+cat > "$tmp/ft.tex" <<'TEX'
+\documentclass[12pt]{article}
+\usepackage[a5paper,margin=20mm,headheight=14pt]{geometry}
+\ifdefined\dvimode\else\usepackage[switch]{lineno}\fi
+\usepackage{fancyhdr}
+\pagestyle{fancy}
+\fancyhf{}
+\fancyhead[L]{Notes on Paged Text}
+\fancyhead[R]{Working Draft}
+\fancyfoot[C]{\thepage}
+\renewcommand{\headrulewidth}{0pt}
+\hyphenation{re-con-struc-tion}
+\emergencystretch=2em
+\begin{document}
+\section*{Introduction}
+Paged documents scatter their words across many small boxes, and a
+careful extractor has to put them back in reading order. This first
+paragraph is deliberately long so that it spans several lines of the
+page and so that the justification of the lines becomes visible to
+anybody who looks at the result with a critical eye. A few more
+sentences follow here only to fill up the first page, so that a later
+paragraph is forced to continue on the next page. The quick brown dog
+sleeps under the old oak tree while the children play in the garden
+and the bees buzz around the flowers.
+
+The second paragraph is indented like every other paragraph that
+follows another one, and the extractor should notice that indentation
+and start a new paragraph here. Its reconstruction depends on the
+geometry alone, since the file itself does not record where one
+paragraph ends and the next one begins.
+
+Short paragraph number three.
+
+The fourth paragraph mentions a few typographic details: the office
+has fluffy waffles, an efficient staff and a ``quoted phrase'' with
+pages 10--20 and a pause---like this one. The caf\'e serves na\"ive
+guests.
+
+The fifth paragraph runs across the page break, right past the
+running header and the page number, neither of which must appear in
+the extracted text. We continue writing some more text here so that
+this paragraph covers enough lines to overflow from the first page
+onto the second page, where it simply goes on as if nothing had
+happened, and the extractor must join both halves into one single
+paragraph without inserting any break at all. To make sure that this
+really happens, the paragraph goes on for a while longer with a few
+more sentences of plain filler text that nobody will ever read, apart
+from the test suite, which checks every single word of it.
+
+The sixth paragraph is the last one of the single column part and it
+ends the page with a sentence about zebras jumping over lazy
+foxes.
+
+\ifdefined\dvimode
+\newpage
+{\fontencoding{T1}\selectfont
+The seventh paragraph is set in the T1 encoding: Stra\ss e,
+\"Uberma\ss, \'etude and \c{c}a va.\par}
+\else
+\setlength{\columnsep}{18pt}
+\twocolumn
+\small\sloppy
+\begin{linenumbers}
+The seventh paragraph opens the two column part of the document. The
+left column is read completely before the right column, and the line
+numbers printed in the margins next to each column must be dropped by
+the extractor because they are not part of the text at all.
+
+The eighth paragraph continues in the left column with another few
+lines of harmless filler text about apples, pears and plums that grow
+in an orchard near the river.
+
+The ninth paragraph keeps going with filler about the weather, which
+was mild and pleasant in the morning but turned rather cold later on.
+The ninth paragraph keeps going with filler about the weather, which
+was mild and pleasant in the morning but turned rather cold later on.
+
+\newpage
+The tenth paragraph is the first one of the right column and it talks
+about lighthouses that guide ships safely through the night along
+rocky coasts.
+
+The eleventh paragraph concludes the document with a final remark.
+\end{linenumbers}
+\fi
+\end{document}
+TEX
+
+cd "$tmp"
+# twice each, so that lineno's references settle
+pdflatex -interaction=batchmode '\pdftrailerid{}\input{ft.tex}' >/dev/null
+pdflatex -interaction=batchmode '\pdftrailerid{}\input{ft.tex}' >/dev/null
+latex -interaction=batchmode -jobname=ftdvi '\def\dvimode{}\input{ft.tex}' \
+ >/dev/null
+latex -interaction=batchmode -jobname=ftdvi '\def\dvimode{}\input{ft.tex}' \
+ >/dev/null
+cp ft.pdf "$out/pdf_fulltext.pdf"
+cp ftdvi.dvi "$out/dvi_fulltext.dvi"
diff --git a/contrib/gen_html_testdata.sh b/contrib/gen_html_testdata.sh
@@ -0,0 +1,68 @@
+#!/bin/sh
+# Regenerate src/plugins/testdata/html_fulltext.html, the fixture for the
+# html plugin's "fulltext" option.
+#
+# This file is in the public domain (CC0); so is the file it produces.
+#
+# Requirements: python3 (standard library only). The page is written in
+# ISO-8859-1 (declared via http-equiv) so that the test also covers the
+# charset conversion; it carries all the page furniture the plugin must
+# drop (head, script, style, noscript, template, nav, page-level
+# header/footer, aside, landmark roles, hidden elements). The expected
+# text is spelled out in src/plugins/test_html.c.
+#
+# Usage: contrib/gen_html_testdata.sh [output-directory]
+# (default output directory: src/plugins/testdata)
+
+set -e
+OUT="${1:-$(dirname "$0")/../src/plugins/testdata}"
+mkdir -p "$OUT"
+python3 - "$OUT" <<'PYEOF'
+import os, sys
+
+OUT = sys.argv[1]
+
+PAGE = """<!DOCTYPE html>
+<html lang="de">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<title>Käse und Brot</title>
+<meta name="author" content="Christian Grothoff">
+<style>p { margin: 0; } nav { display: block; }</style>
+<script>document.write("<p>script text</p>");</script>
+</head>
+<body>
+<header><a href="/">Site banner</a></header>
+<nav><ul><li><a href="/">Home</a></li><li><a href="/a">About</a></li></ul></nav>
+<div role="navigation">Breadcrumbs > here</div>
+<main>
+<article>
+<header><h1>Der Käse & das Brot</h1></header>
+<p>Hello <b>world</b>, again here:
+ été — done.</p>
+<p>Line one<br>line two<br><br>New paragraph after two breaks.</p>
+<div hidden>Hidden by attribute</div>
+<span aria-hidden="true">Decorative icon</span>
+<div style="display: none">Hidden by style</div>
+<ul><li>Item A</li><li>Item <i>B</i></li></ul>
+<table><tr><th>Cell 1</th><td>Cell 2</td></tr></table>
+<noscript>Please enable JavaScript</noscript>
+<template><p>Template text</p></template>
+<blockquote>Quoted text</blockquote>
+<p>Soft­hyphen and <span>in</span>line elements.</p>
+<footer>Article byline</footer>
+</article>
+<aside>Sidebar text</aside>
+</main>
+<div role="contentinfo">Legal notice</div>
+<footer>Copyright 2026</footer>
+<script>var x = "</div>more script";</script>
+</body>
+</html>
+"""
+
+with open (os.path.join (OUT, "html_fulltext.html"), "wb") as f:
+ f.write (PAGE.encode ("iso-8859-1"))
+print ("html_fulltext.html: %d bytes"
+ % os.path.getsize (os.path.join (OUT, "html_fulltext.html")))
+PYEOF
diff --git a/contrib/gen_odf_testdata.sh b/contrib/gen_odf_testdata.sh
@@ -0,0 +1,261 @@
+#!/bin/sh
+# Regenerate the synthetic ODF / Word test files used by the "fulltext"
+# tests in src/plugins/test_odf.c and src/plugins/test_ole2.c.
+#
+# Copyright (C) 2026 Christian Grothoff
+# The generated files are dedicated to the public domain (CC0 1.0).
+#
+# The ODF packages (odf_fulltext.odt, .odp, .ods) are zips of
+# hand-written XML, so every byte is owned by this script. The text
+# document deliberately carries all the material the full-text
+# extraction must leave out -- a running header and footer (in
+# styles.xml), a footnote, a comment, page-number and page-count
+# fields, tracked-change deletions, the alternative text of a frame --
+# next to the material it must keep: headings, a list, a table, a
+# bookmark reference, a text frame, spaces/tabs/line breaks and
+# character references. The presentation has speaker notes, the
+# spreadsheet a cell comment.
+#
+# ole2_fulltext.doc is the text document (minus the tracked changes,
+# which in Word format would need character-property parsing to
+# drop, and minus the text frame, which Word stores outside of the
+# main text) converted to Word 97 by LibreOffice, if it is installed:
+# soffice --headless --convert-to doc
+# Its header/footer, footnote, comment and PAGE/NUMPAGES fields then
+# live where Word puts them, which is what test_ole2 checks.
+#
+# Needs python3; LibreOffice only for the .doc.
+set -e
+
+srcdir=$(dirname "$0")
+outdir=${1:-"$srcdir/../src/plugins/testdata"}
+
+python3 - "$outdir" <<'EOF'
+import os
+import sys
+import zipfile
+
+OUT = sys.argv[1]
+DATE = (2026, 1, 2, 3, 4, 6)
+
+NS = ('xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" '
+ 'xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" '
+ 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" '
+ 'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" '
+ 'xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" '
+ 'xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:'
+ 'xsl-fo-compatible:1.0" '
+ 'xmlns:xlink="http://www.w3.org/1999/xlink" '
+ 'xmlns:dc="http://purl.org/dc/elements/1.1/" '
+ 'xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0" '
+ 'xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:'
+ 'svg-compatible:1.0" '
+ 'xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:'
+ 'presentation:1.0" '
+ 'office:version="1.2"')
+
+XML = '<?xml version="1.0" encoding="UTF-8"?>\n'
+
+
+def meta(title):
+ return (XML + '<office:document-meta ' + NS + '><office:meta>'
+ '<meta:generator>gen_odf_testdata.sh</meta:generator>'
+ '<dc:title>' + title + '</dc:title>'
+ '</office:meta></office:document-meta>')
+
+
+def manifest(mime):
+ return (XML + '<manifest:manifest xmlns:manifest='
+ '"urn:oasis:names:tc:opendocument:xmlns:manifest:1.0" '
+ 'manifest:version="1.2">'
+ '<manifest:file-entry manifest:full-path="/" '
+ 'manifest:media-type="' + mime + '"/>'
+ '<manifest:file-entry manifest:full-path="content.xml" '
+ 'manifest:media-type="text/xml"/>'
+ '<manifest:file-entry manifest:full-path="styles.xml" '
+ 'manifest:media-type="text/xml"/>'
+ '<manifest:file-entry manifest:full-path="meta.xml" '
+ 'manifest:media-type="text/xml"/>'
+ '</manifest:manifest>')
+
+
+def write(name, mime, title, content, styles):
+ with zipfile.ZipFile(os.path.join(OUT, name), 'w') as z:
+ def add(fn, data, method):
+ zi = zipfile.ZipInfo(fn, DATE)
+ zi.compress_type = method
+ zi.external_attr = 0o644 << 16
+ z.writestr(zi, data)
+ add('mimetype', mime, zipfile.ZIP_STORED)
+ add('content.xml', XML + content, zipfile.ZIP_DEFLATED)
+ add('styles.xml', XML + styles, zipfile.ZIP_DEFLATED)
+ add('meta.xml', meta(title), zipfile.ZIP_DEFLATED)
+ add('META-INF/manifest.xml', manifest(mime), zipfile.ZIP_DEFLATED)
+
+
+def text_styles():
+ return ('<office:document-styles ' + NS + '>'
+ '<office:automatic-styles>'
+ '<style:page-layout style:name="pm1"/>'
+ '</office:automatic-styles>'
+ '<office:master-styles>'
+ '<style:master-page style:name="Standard" '
+ 'style:page-layout-name="pm1">'
+ '<style:header><text:p>RUNNING HEADER TEXT</text:p>'
+ '</style:header>'
+ '<style:footer><text:p>Footer page '
+ '<text:page-number text:select-page="current">1'
+ '</text:page-number></text:p></style:footer>'
+ '</style:master-page>'
+ '</office:master-styles></office:document-styles>')
+
+
+def text_content(tracked, frame):
+ body = []
+ if tracked:
+ body.append(
+ '<text:tracked-changes>'
+ '<text:changed-region text:id="ct1"><text:deletion>'
+ '<office:change-info><dc:creator>Editor</dc:creator>'
+ '<dc:date>2026-01-02T03:04:05</dc:date></office:change-info>'
+ '<text:p>DELETED TEXT</text:p>'
+ '</text:deletion></text:changed-region>'
+ '</text:tracked-changes>')
+ body.append('<text:h text:outline-level="1">Synthetic Heading</text:h>')
+ body.append(
+ '<text:p>First paragraph with spaces<text:s text:c="3"/>and'
+ '<text:tab/>a tab<text:line-break/>and a break; entities '
+ '& <tag> ☺ €'
+ '<text:note text:id="ftn1" text:note-class="footnote">'
+ '<text:note-citation>1</text:note-citation><text:note-body>'
+ '<text:p>FOOTNOTE BODY TEXT</text:p></text:note-body></text:note>'
+ ' and the sentence '
+ + ('<text:change text:change-id="ct1"/>' if tracked else '')
+ + 'continues'
+ '<office:annotation><dc:creator>Reviewer</dc:creator>'
+ '<dc:date>2026-01-02T03:04:05</dc:date>'
+ '<text:p>ANNOTATION TEXT</text:p></office:annotation>.</text:p>')
+ body.append(
+ '<text:p>This is page <text:page-number text:select-page="current">'
+ '7</text:page-number> of <text:page-count>9</text:page-count>'
+ ' pages.</text:p>')
+ body.append(
+ '<text:list><text:list-item><text:p>List item one</text:p>'
+ '</text:list-item><text:list-item><text:p>List item two</text:p>'
+ '</text:list-item></text:list>')
+ body.append(
+ '<table:table table:name="T1">'
+ '<table:table-column table:number-columns-repeated="2"/>'
+ '<table:table-row>'
+ '<table:table-cell office:value-type="string">'
+ '<text:p>Cell A1</text:p></table:table-cell>'
+ '<table:table-cell office:value-type="string">'
+ '<text:p>Cell B1</text:p></table:table-cell></table:table-row>'
+ '<table:table-row>'
+ '<table:table-cell office:value-type="string">'
+ '<text:p>Cell A2</text:p></table:table-cell>'
+ '<table:table-cell office:value-type="string">'
+ '<text:p>Cell B2</text:p></table:table-cell></table:table-row>'
+ '</table:table>')
+ body.append(
+ '<text:p><text:bookmark text:name="b1"/>See '
+ '<text:bookmark-ref text:reference-format="text" text:ref-name="b1">'
+ 'Chapter One</text:bookmark-ref> for details.</text:p>')
+ if frame:
+ body.append(
+ '<text:p>Before the frame<draw:frame draw:name="F1" '
+ 'text:anchor-type="as-char" svg:width="4cm" svg:height="1cm">'
+ '<draw:text-box><text:p>Frame caption text</text:p>'
+ '</draw:text-box><svg:title>ALT TITLE</svg:title>'
+ '<svg:desc>ALT TEXT</svg:desc></draw:frame>after the frame.'
+ '</text:p>')
+ body.append('<text:p>Last paragraph.</text:p>')
+ return ('<office:document-content ' + NS + '>'
+ '<office:automatic-styles/><office:body><office:text>'
+ + ''.join(body)
+ + '</office:text></office:body></office:document-content>')
+
+
+write('odf_fulltext.odt',
+ 'application/vnd.oasis.opendocument.text',
+ 'Synthetic full text test',
+ text_content(True, True),
+ text_styles())
+
+# input for the Word conversion below; not part of the test data
+write('ole2_fulltext.tmp.odt',
+ 'application/vnd.oasis.opendocument.text',
+ 'Synthetic full text test',
+ text_content(False, False),
+ text_styles())
+
+
+def slide(name, title, lines, notes):
+ return ('<draw:page draw:name="' + name + '" '
+ 'draw:master-page-name="Default">'
+ '<draw:frame presentation:class="title" svg:x="1cm" svg:y="1cm" '
+ 'svg:width="20cm" svg:height="3cm"><draw:text-box>'
+ '<text:p>' + title + '</text:p></draw:text-box></draw:frame>'
+ '<draw:frame presentation:class="outline" svg:x="1cm" '
+ 'svg:y="5cm" svg:width="20cm" svg:height="10cm"><draw:text-box>'
+ '<text:list>'
+ + ''.join('<text:list-item><text:p>' + ln
+ + '</text:p></text:list-item>' for ln in lines)
+ + '</text:list></draw:text-box></draw:frame>'
+ '<presentation:notes><draw:frame presentation:class="notes" '
+ 'svg:x="1cm" svg:y="1cm" svg:width="10cm" svg:height="5cm">'
+ '<draw:text-box><text:p>' + notes + '</text:p></draw:text-box>'
+ '</draw:frame></presentation:notes></draw:page>')
+
+
+write('odf_fulltext.odp',
+ 'application/vnd.oasis.opendocument.presentation',
+ 'Synthetic slides',
+ '<office:document-content ' + NS + '><office:body><office:presentation>'
+ + slide('p1', 'First slide title', ['Bullet one', 'Bullet two'],
+ 'SPEAKER NOTES ONE')
+ + slide('p2', 'Second slide title', ['Bullet three'],
+ 'SPEAKER NOTES TWO')
+ + '</office:presentation></office:body></office:document-content>',
+ '<office:document-styles ' + NS + '><office:master-styles>'
+ '<style:master-page style:name="Default"/>'
+ '</office:master-styles></office:document-styles>')
+
+write('odf_fulltext.ods',
+ 'application/vnd.oasis.opendocument.spreadsheet',
+ 'Synthetic sheet',
+ '<office:document-content ' + NS + '><office:body><office:spreadsheet>'
+ '<table:table table:name="Sheet1">'
+ '<table:table-column table:number-columns-repeated="2"/>'
+ '<table:table-row>'
+ '<table:table-cell office:value-type="string"><text:p>Name</text:p>'
+ '</table:table-cell>'
+ '<table:table-cell office:value-type="string"><text:p>Amount</text:p>'
+ '</table:table-cell></table:table-row>'
+ '<table:table-row>'
+ '<table:table-cell office:value-type="string">'
+ '<office:annotation><dc:creator>Reviewer</dc:creator>'
+ '<text:p>CELL COMMENT</text:p></office:annotation>'
+ '<text:p>Apples</text:p></table:table-cell>'
+ '<table:table-cell office:value-type="float" office:value="42">'
+ '<text:p>42</text:p></table:table-cell></table:table-row>'
+ '<table:table-row table:number-rows-repeated="3">'
+ '<table:table-cell table:number-columns-repeated="2"/>'
+ '</table:table-row>'
+ '</table:table></office:spreadsheet></office:body>'
+ '</office:document-content>',
+ '<office:document-styles ' + NS + '/>')
+EOF
+
+if command -v soffice > /dev/null 2>&1
+then
+ tmp=$(mktemp -d)
+ soffice -env:UserInstallation="file://$tmp/profile" --headless \
+ --convert-to doc:"MS Word 97" --outdir "$tmp" \
+ "$outdir/ole2_fulltext.tmp.odt" > /dev/null
+ mv "$tmp/ole2_fulltext.tmp.doc" "$outdir/ole2_fulltext.doc"
+ rm -rf "$tmp"
+else
+ echo "soffice not found, not regenerating ole2_fulltext.doc" >&2
+fi
+rm -f "$outdir/ole2_fulltext.tmp.odt"
diff --git a/contrib/gen_testmedia.sh b/contrib/gen_testmedia.sh
@@ -55,4 +55,67 @@ $FFMPEG -hide_banner -loglevel error -y \
-map "[v]" -map "[a]" -c:v libtheora -q:v 3 -c:a libvorbis -q:a -2 -ac 1 \
"$OUT/preview_tone.ogv"
-ls -l "$OUT/thumbnail_pattern.ogv" "$OUT/preview_tone.ogg" "$OUT/preview_tone.ogv"
+# A 16:9 video with three text subtitle tracks (English and German as
+# SubRip, French as ASS) for the "srt:LANG" option, and flat stretches
+# for the "gallery:N" option: with gallery:4 the tiles are sampled at
+# 5s (black -> must be shifted), 15s (red), 25s (white -> must be
+# shifted) and 35s (blue).
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/en.srt" << 'EOSRT'
+1
+00:00:01,000 --> 00:00:04,000
+Hello, world!
+
+2
+00:00:12,500 --> 00:00:15,250
+This is the second line
+spanning two rows.
+
+3
+00:00:33,000 --> 00:00:38,000
+Goodbye.
+EOSRT
+cat > "$TMP/de.srt" << 'EOSRT'
+1
+00:00:01,000 --> 00:00:04,000
+Grüß Gott, Welt!
+
+2
+00:00:12,500 --> 00:00:15,250
+Das ist die zweite Zeile
+über zwei Reihen.
+
+3
+00:00:33,000 --> 00:00:38,000
+Auf Wiedersehen.
+EOSRT
+cat > "$TMP/fr.ass" << 'EOASS'
+[Script Info]
+ScriptType: v4.00+
+PlayResX: 160
+PlayResY: 90
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+Style: Default,Arial,12,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,0,2,10,10,10,1
+
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+Dialogue: 0,0:00:01.00,0:00:04.00,Default,,0,0,0,,Bonjour, {\i1}le monde{\i0} !
+Dialogue: 0,0:00:12.50,0:00:15.25,Default,,0,0,0,,Deuxième ligne\Nsur deux rangs.
+EOASS
+$FFMPEG -hide_banner -loglevel error -y \
+ -f lavfi -i "color=c=black:s=160x90:r=5:d=6" \
+ -f lavfi -i "color=c=red:s=160x90:r=5:d=14,drawbox=x=10:y=10:w=24:h=24:color=white:t=fill" \
+ -f lavfi -i "color=c=white:s=160x90:r=5:d=6" \
+ -f lavfi -i "color=c=blue:s=160x90:r=5:d=14,drawbox=x=116:y=56:w=24:h=24:color=yellow:t=fill" \
+ -i "$TMP/en.srt" -i "$TMP/de.srt" -i "$TMP/fr.ass" \
+ -filter_complex "[0:v][1:v][2:v][3:v]concat=n=4:v=1:a=0[v]" \
+ -map "[v]" -map 4 -map 5 -map 6 \
+ -c:v libtheora -q:v 3 -g 10 -c:s:0 srt -c:s:1 srt -c:s:2 ass \
+ -metadata:s:s:0 language=eng -metadata:s:s:1 language=ger \
+ -metadata:s:s:2 language=fre \
+ "$OUT/subtitles.mkv"
+
+ls -l "$OUT/subtitles.mkv" "$OUT/thumbnail_pattern.ogv" "$OUT/preview_tone.ogg" "$OUT/preview_tone.ogv"
diff --git a/contrib/oss-fuzz/build.sh b/contrib/oss-fuzz/build.sh
@@ -296,7 +296,7 @@ CORE_FUZZERS="fuzz_datasource fuzz_ipc fuzz_convert fuzz_unzip"
# dependencies; keep in sync with src/fuzz/Makefile.am)
PLUGIN_FUZZERS="
applefile:applefile_extractor.c pack.c:
-dvi:dvi_extractor.c:
+dvi:dvi_extractor.c fulltext.c pagelayout.c:-lm
elf:elf_extractor.c pack.c:
it:it_extractor.c:
man:man_extractor.c:
@@ -331,7 +331,7 @@ kml:kml_extractor.c forensics.c:-lm
geotiff:geotiff_extractor.c forensics.c:-lm
mbox:mbox_extractor.c forensics.c:-lm
apk:apk_extractor.c forensics.c:-lm -lz
-ebook:ebook_extractor.c forensics.c:-lm -lz
+ebook:ebook_extractor.c forensics.c fulltext.c:-lm -lz
"
# Plugins that wrap a third-party parser. NOT part of the OSS-Fuzz build:
@@ -350,6 +350,7 @@ flac:flac_extractor.c:-lFLAC
ogg:ogg_extractor.c:-lvorbisfile -lvorbis -logg
archive:archive_extractor.c:-larchive
mime:mime_extractor.c:-lmagic
+html:html_extractor.c fulltext.c:-ltidy -lmagic
"
# ole2 is separate from the list above because it is the one plugin that
diff --git a/contrib/oss-fuzz/make_seed_corpus.sh b/contrib/oss-fuzz/make_seed_corpus.sh
@@ -79,6 +79,7 @@ testdata_glob ()
fuzz_odf) echo "odf_*" ;;
fuzz_ogg) echo "ogg_*" ;;
fuzz_ole2) echo "ole2_* msoffice_biff4.xls" ;;
+ fuzz_pdf) echo "pdf_*" ;;
fuzz_png) echo "png_*" ;;
fuzz_ps) echo "ps_*" ;;
fuzz_qt) echo "gstreamer_sample_sorenson.mov" ;;
@@ -98,6 +99,7 @@ testdata_glob ()
fuzz_lnk) echo "lnk_*" ;;
fuzz_sqlite) echo "sqlite_*" ;;
fuzz_tar) echo "tar_* archive_test.tar" ;;
+ fuzz_thumbnailffmpeg) echo "thumbnail_* subtitles.mkv" ;;
fuzz_iso9660) echo "iso9660_*" ;;
fuzz_diskimage) echo "diskimage_*" ;;
fuzz_heif) echo "heif_*" ;;
@@ -122,7 +124,7 @@ fuzz_applefile fuzz_dvi fuzz_elf fuzz_it fuzz_man fuzz_nsf fuzz_nsfe \
fuzz_ps fuzz_real fuzz_riff fuzz_rtf fuzz_s3m fuzz_sid fuzz_wav fuzz_xm \
fuzz_deb fuzz_msoffice fuzz_odf fuzz_png fuzz_qt fuzz_zip \
fuzz_gif fuzz_jpeg fuzz_tiff fuzz_flac fuzz_ogg fuzz_archive fuzz_mime \
-fuzz_ole2 fuzz_mpeg fuzz_html \
+fuzz_ole2 fuzz_mpeg fuzz_html fuzz_pdf \
fuzz_pecoff fuzz_lnk fuzz_sqlite fuzz_tar fuzz_iso9660 fuzz_diskimage \
fuzz_heif fuzz_webp fuzz_plist fuzz_id3 fuzz_gpx fuzz_kml fuzz_geotiff \
fuzz_mbox fuzz_apk fuzz_ebook"
diff --git a/doc/extract.1 b/doc/extract.1
@@ -1,4 +1,4 @@
-.TH EXTRACT 1 "Aug 7, 2012" "libextractor 1.0.0"
+.TH EXTRACT 1 "Nov 1, 2026" "libextractor 1.19"
.\" $Id
.SH NAME
extract
@@ -24,7 +24,7 @@ extract
\&...
.br
.SH DESCRIPTION
-This manual page documents version 1.0.0 of the
+This manual page documents version 1.19 of the
.B extract
command.
.PP
@@ -42,7 +42,9 @@ option.
Display the output in BiBTeX format.
.TP 8
.B \-g
-Use grep\-friendly output (all keywords on a single line for each file). Use the verbose option to print the filename first, followed by the keywords. Use the verbose option twice to also display the keyword types. This option will not print keyword types or non\-textual metadata.
+Use grep\-friendly output (all keywords on a single line for each file). Use the verbose option to print the filename first, followed by the keywords. Use the verbose option twice to also display the keyword types. This option will not print keyword types or non\-textual metadata. Note that it cannot keep the one\-line promise for keywords that contain newlines themselves, such as the full text (see
+.B "PLUGIN OPTIONS"
+below).
.TP 8
.B \-h
Print a brief summary of the options.
@@ -51,7 +53,50 @@ Print a brief summary of the options.
Run plugins in\-process (for debugging). By default, each plugin is run in its own process.
.TP 8
.BI \-l " libraries"
-Use the specified libraries to extract keywords. The general format of libraries is .I [[\-]LIBRARYNAME[:[\-]LIBRARYNAME]*] where LIBRARYNAME is a libextractor compatible library and typically of the form .Ijpeg\. The minus before the libraryname indicates that this library should be removed from the existing list. To run only a few selected plugins, use \-l in combination with \-n.
+Use the specified plugins to extract keywords.
+.I libraries
+is a colon\-separated list of entries of the form
+.RI [ \- ] name [ ( options ) ]
+where
+.I name
+is the short name of a libextractor plugin, for example
+.BR pdf ,
+.B html
+or
+.B odf
+for the plugins installed as
+.IR libextractor_pdf.so ,
+.I libextractor_html.so
+and
+.IR libextractor_odf.so .
+Neither the
+.B libextractor_
+prefix nor the
+.B .so
+suffix is part of the name.
+A leading minus sign removes a plugin from the list built so far instead of adding it.
+.I options
+is a comma\-separated plugin configuration string; see
+.B "PLUGIN OPTIONS"
+below.
+.IP
+Only the
+.I last
+.B \-l
+option on the command line is used, so several plugins have to be
+named in a single colon\-separated list. Adding a plugin that is
+already in the list has no effect; in particular it does
+.I not
+change the options of the plugin that is already there. To pass
+options to a plugin that the default set already provides, either drop
+the default set with
+.B \-n
+or remove the plugin before adding it again, as in
+.BR \-l\ \(dq\-pdf:pdf(fulltext)\(dq .
+To run only a few selected plugins, use
+.B \-l
+in combination with
+.BR \-n .
.TP 8
.B \-L
Print a list of all known keyword types.
@@ -60,9 +105,11 @@ Print a list of all known keyword types.
Load the file into memory and perform extraction from memory (for debugging).
.TP 8
.B \-n
-Do not use the default set of extractors (typically all standard extractors, currently mp3, ogg, jpg, gif, png, tiff, real, html, pdf and mime\-types), use only the extractors specified with the .B \-l option.
-.TP
-.B \-p " type"
+Do not use the default set of extractors (typically all standard extractors, currently mp3, ogg, jpg, gif, png, tiff, real, html, pdf and mime\-types), use only the extractors specified with the
+.B \-l
+option.
+.TP 8
+.BI \-p " type"
Print only the keywords matching the specified type. By default, all keywords that are found and not removed as duplicates are printed.
.TP 8
.B \-v
@@ -71,29 +118,171 @@ Print the version number and exit.
.B \-V
Be verbose. This option can be specified multiple times to increase verbosity further.
.TP 8
-.I \-x " type"
+.BI \-x " type"
Exclude keywords of the specified type from the output. By default, all keywords that are found and not removed as duplicates are printed.
+
+.SH "PLUGIN OPTIONS"
+A plugin can be given a configuration string by appending it in
+parentheses to the plugin name in the argument of
+.BR \-l ,
+as in
+.BR \-l\ \(dqpdf(fulltext)\(dq .
+The string is a comma\-separated list of tokens; which tokens are
+understood is up to the individual plugin, and a plugin silently
+ignores an option it does not know. The parentheses are special to
+the shell, so the argument of
+.B \-l
+normally has to be quoted.
+.PP
+The options of the thumbnail plugins
+.RB ( size ,
+.BR format ,
+.BR srt
+and
+.BR gallery )
+are described in the Texinfo manual. The option understood by the
+largest number of plugins is
+.BR fulltext .
+.SS "Full text"
+The
+.BR html ,
+.BR pdf ,
+.BR ebook ,
+.BR odf ,
+.B ole2
+and
+.B dvi
+plugins can report the entire text of a document as a single keyword
+of type
+.B "full text"
+.RB ( EXTRACTOR_METATYPE_FULL_TEXT ,
+UTF\-8, MIME type
+.BR text/plain ).
+As that text can be large, it is only reported if the plugin is given
+the option
+.BR fulltext .
+.PP
+Within a paragraph, runs of whitespace \(em including the line breaks
+of the input \(em are reported as a single space; paragraphs are
+separated by exactly two newlines. Where the format makes it
+possible, text that is not part of the running text is left out: page
+numbers, running headers and footers, navigation bars and the like.
+The text is truncated at 16 MiB.
+.PP
+To extract the text of a document, drop the default plugins with
+.B \-n
+and name just the plugin for that format:
+.PP
+.RS
+.nf
+$ cat article.html
+<html>
+<head><title>On Cheese</title>
+<meta name="author" content="Christian Grothoff"></head>
+<body>
+<nav><a href="/">Home</a> | <a href="/up">Up</a></nav>
+<h1>On Cheese</h1>
+<p>The first paragraph is written
+over three
+source lines.</p>
+<p>The second paragraph follows.</p>
+</body>
+</html>
+
+$ extract \-n \-l "html(fulltext)" article.html
+Keywords for file article.html:
+title \- On Cheese
+author name \- Christian Grothoff
+full text \- On Cheese
+
+The first paragraph is written over three source lines.
+
+The second paragraph follows.
+.fi
+.RE
+.PP
+The navigation bar was dropped, the line breaks inside the first
+paragraph became single spaces, and the break between the two
+paragraphs was kept.
+.PP
+Use
+.B \-p
+to print nothing but the text, which is what a full\-text indexer
+wants:
+.PP
+.RS
+.nf
+$ extract \-n \-l "pdf(fulltext)" \-p "full text" book.pdf
+Keywords for file book.pdf:
+full text \- Chapter One
+
+It was a dark and stormy night [...]
+.fi
+.RE
+.PP
+Conversely,
+.B \-x
+keeps the ordinary meta data and suppresses the text:
+.PP
+.RS
+.nf
+$ extract \-n \-l "pdf(fulltext)" \-x "full text" book.pdf
+Keywords for file book.pdf:
+mimetype \- application/pdf
+title \- Paul Clifford
+author name \- Edward Bulwer\-Lytton
+page count \- 384
+.fi
+.RE
+.PP
+Several plugins can be configured in one list, so that a directory of
+mixed documents can be handled in a single run:
+.PP
+.RS
+.nf
+$ extract \-n \-l "pdf(fulltext):odf(fulltext):html(fulltext)" \e
+ \-p "full text" *
+.fi
+.RE
+.PP
+If the remaining meta data of the default plugin set is wanted as
+well, remove the plugin from that set before adding it back with the
+option; adding it a second time would have no effect:
+.PP
+.RS
+.nf
+$ extract \-l "\-pdf:pdf(fulltext)" book.pdf
+.fi
+.RE
.SH "SEE ALSO"
.BR libextractor (3)
\- description of the libextractor library
.br
.SH EXAMPLES
+The sample files used below are those of the test suite, in
+.IR src/plugins/testdata/ .
+.PP
.nf
-$ extract test/test.jpg
-comment \- (C) 2001 by Christian Grothoff, using gimp 1.2 1
+$ extract \-n \-l jpeg jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
mimetype \- image/jpeg
+image dimensions \- 3x3
+comment \- (C) 2001 by Christian Grothoff, using gimp 1.2 1
-$ extract \-V \-x comment test/test.jpg
-Keywords for file test/test.jpg:
+$ extract \-n \-l jpeg \-x comment jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
mimetype \- image/jpeg
+image dimensions \- 3x3
-$ extract \-p comment test/test.jpg
+$ extract \-n \-l jpeg \-p comment jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
comment \- (C) 2001 by Christian Grothoff, using gimp 1.2 1
-$ extract \-nV \-l png.so \-p comment test/test.jpg test/test.png
-Keywords for file test/test.jpg:
-Keywords for file test/test.png:
+$ extract \-n \-l gif \-p comment jpeg_image.jpg gif_image.gif
+Keywords for file jpeg_image.jpg:
+Keywords for file gif_image.gif:
comment \- Testing keyword extraction
+.fi
.SH "LEGAL NOTICE"
libextractor and the extract tool are released under the GPL. libextractor is a GNU package.
diff --git a/doc/libextractor.texi b/doc/libextractor.texi
@@ -426,6 +426,15 @@ default plugins. Use ``-l NAME'' to specifically load a certain
plugin. For example, specify ``-n -l mime'' to only use the MIME
plugin.
+The argument of ``-l'' is a colon-separated list of plugin names, each
+optionally preceded by a minus sign to remove the plugin instead of
+adding it and optionally followed by a plugin configuration in
+parentheses, as in ``-l "-pdf:pdf(fulltext)"''. Only the last ``-l''
+option on the command line is used, so several plugins have to be
+named in a single list. The options understood by the individual
+plugins are described in the chapter @emph{Existing Plugins}, under
+@emph{Video thumbnails and audio previews} and @emph{Full text}.
+
Using the ``-p'' option the output of @command{extract} can be limited
to only certain keyword types. Similarly, using the ``-x'' option,
certain keyword types can be excluded. A list of all known keyword
@@ -438,21 +447,30 @@ options.
@section Common usage examples for ``extract''
+The sample files used here are those of the test suite, in
+@file{src/plugins/testdata/}. Note that the argument of @code{-l} is
+the short name of the plugin (@code{jpeg}, not @file{jpeg.so} or
+@file{libextractor_jpeg.so}).
+
@example
-$ extract test/test.jpg
-comment - (C) 2001 by Christian Grothoff, using gimp 1.2 1
+$ extract -n -l jpeg jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
mimetype - image/jpeg
+image dimensions - 3x3
+comment - (C) 2001 by Christian Grothoff, using gimp 1.2 1
-$ extract -V -x comment test/test.jpg
-Keywords for file test/test.jpg:
+$ extract -n -l jpeg -x comment jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
mimetype - image/jpeg
+image dimensions - 3x3
-$ extract -p comment test/test.jpg
+$ extract -n -l jpeg -p comment jpeg_image.jpg
+Keywords for file jpeg_image.jpg:
comment - (C) 2001 by Christian Grothoff, using gimp 1.2 1
-$ extract -nV -l png.so -p comment test/test.jpg test/test.png
-Keywords for file test/test.jpg:
-Keywords for file test/test.png:
+$ extract -n -l gif -p comment jpeg_image.jpg gif_image.gif
+Keywords for file jpeg_image.jpg:
+Keywords for file gif_image.gif:
comment - Testing keyword extraction
@end example
@@ -956,11 +974,34 @@ Opus bitrate of the audio preview (default: 24000).
@item deadline=@var{milliseconds}
Wall clock budget for the whole extraction; when it runs out, the
plugin reports nothing rather than keeping the application waiting
-(default: 10000).
+(default: 10000, or 60000 if @code{srt} or @code{gallery} is given).
+@end table
+
+The two thumbnail plugins also understand:
+
+@table @code
+@item srt:@var{lang}
+Additionally dump the complete subtitle track in language @var{lang}
+as an SRT file (@code{EXTRACTOR_METATYPE_SUBTITLE_TEXT}, UTF-8, MIME
+type @code{application/x-subrip}). @var{lang} may be given as ISO
+639-1 (@code{de}), ISO 639-2/B (@code{ger}) or ISO 639-2/T
+(@code{deu}) code; only the first matching track is dumped, and only
+text subtitles (SubRip, ASS/SSA, WebVTT, timed text) can be converted.
+Without this option the plugins merely list the language of every
+subtitle track as @code{EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE}.
+@item gallery:@var{n}
+Instead of a single thumbnail, emit one image showing @var{n} (at most
+100) frames taken at roughly even intervals across the whole video,
+arranged in a grid chosen so that the image is as close to 16:9 as
+possible. @code{size} then applies to each tile. A frame that shows
+basically nothing (99% of its pixels within a narrow colour range, such
+as a black or white screen) is replaced by one taken a little earlier
+or later.
@end table
For example, @code{"thumbnailgst(size=256,format=png)"} asks for a
-256x256 PNG.
+256x256 PNG, and @code{"thumbnailffmpeg(gallery:12,srt:en)"} for a
+contact sheet of twelve frames plus the English subtitles.
Two caveats apply to the GStreamer variants. Their decoders and
encoders are themselves GStreamer plugins, resolved at run time, so a
@@ -971,6 +1012,38 @@ initialized it, an application that runs these plugins
@code{EXTRACTOR_OPTION_IN_PROCESS} should not also use out-of-process
plugins in the same process afterwards.
+@section Full text
+
+The @code{html}, @code{pdf}, @code{ebook}, @code{odf}, @code{ole2}
+and @code{dvi} plugins can report the entire text of a document as a
+single @code{EXTRACTOR_METATYPE_FULL_TEXT} item in
+@code{EXTRACTOR_METAFORMAT_UTF8} with MIME type @code{text/plain}. As
+the text can be large, this is only done if the plugin is given the
+option @code{fulltext}, as in
+
+@example
+pl = EXTRACTOR_plugin_add_config (NULL,
+ "pdf(fulltext):odf(fulltext)",
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+@end example
+
+@noindent
+or, on the command line, @code{extract -n -l "pdf(fulltext)" file.pdf}.
+Note the @code{-n}: @code{EXTRACTOR_plugin_add} leaves the list
+unchanged if a plugin of that name is already in it, so adding
+@code{pdf(fulltext)} on top of the default plugin set --- which
+already contains @code{pdf}, without the option --- has no effect at
+all. Either drop the default set with @code{-n}, or remove the plugin
+before adding it back, as in
+@code{extract -l "-pdf:pdf(fulltext)" file.pdf}.
+
+Within a paragraph, runs of whitespace (including line breaks) are
+reported as a single space; paragraphs are separated by exactly two
+newlines (@code{"\n\n"}). Where the format makes it possible, text
+that is not part of the running text is left out: page numbers,
+running headers and footers, navigation bars (HTML) and the like. The
+text is truncated at 16 MiB.
+
@section Forensic and bulk-scan plugins
Sixteen of the plugins in that list were added for a different job from
diff --git a/src/fuzz/Makefile.am b/src/fuzz/Makefile.am
@@ -156,8 +156,9 @@ nodist_fuzz_applefile_SOURCES = \
fuzz_dvi_SOURCES = fuzz_plugin.c
fuzz_dvi_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \
-DLE_FUZZ_PLUGIN=dvi -DLE_FUZZ_ID=DVI
-fuzz_dvi_LDADD = $(XLIB) $(SOCKET_LIBS)
-nodist_fuzz_dvi_SOURCES = $(PLUGIN_DIR)/dvi_extractor.c
+fuzz_dvi_LDADD = $(XLIB) $(SOCKET_LIBS) -lm
+nodist_fuzz_dvi_SOURCES = $(PLUGIN_DIR)/dvi_extractor.c \
+ $(PLUGIN_DIR)/fulltext.c $(PLUGIN_DIR)/pagelayout.c
fuzz_elf_SOURCES = fuzz_plugin.c
fuzz_elf_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \
@@ -387,7 +388,8 @@ fuzz_odf_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \
-DLE_FUZZ_PLUGIN=odf -DLE_FUZZ_ID=ODF
fuzz_odf_LDFLAGS = -static
fuzz_odf_LDADD = $(LE_COMMON) $(XLIB) -lz
-nodist_fuzz_odf_SOURCES = $(PLUGIN_DIR)/odf_extractor.c
+nodist_fuzz_odf_SOURCES = $(PLUGIN_DIR)/odf_extractor.c \
+ $(PLUGIN_DIR)/fulltext.c
fuzz_png_SOURCES = fuzz_plugin.c
fuzz_png_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \
@@ -425,7 +427,8 @@ fuzz_ebook_LDFLAGS = -static
fuzz_ebook_LDADD = $(LE_COMMON) $(XLIB) -lm -lz
nodist_fuzz_ebook_SOURCES = \
$(PLUGIN_DIR)/ebook_extractor.c \
- $(PLUGIN_DIR)/forensics.c
+ $(PLUGIN_DIR)/forensics.c \
+ $(PLUGIN_DIR)/fulltext.c
endif
@@ -496,14 +499,28 @@ fuzz_mime_LDADD = $(XLIB) -lmagic
nodist_fuzz_mime_SOURCES = $(PLUGIN_DIR)/mime_extractor.c
endif
+# The html plugin: libtidy builds the tree, but the walk over it (and,
+# with the "fulltext" option, the charset sniffing and conversion) is
+# ours.
+if HAVE_MAGIC
+if HAVE_TIDY
+HTML_FUZZER = fuzz_html
+fuzz_html_SOURCES = fuzz_plugin.c
+fuzz_html_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) $(TIDY_CFLAGS) \
+ -DLE_FUZZ_PLUGIN=html -DLE_FUZZ_ID=HTML
+fuzz_html_LDADD = $(XLIB) -ltidy -lmagic $(LTLIBICONV)
+nodist_fuzz_html_SOURCES = \
+ $(PLUGIN_DIR)/html_extractor.c \
+ $(PLUGIN_DIR)/fulltext.c
+endif
+endif
+
# The FFmpeg-based media plugins. What gets instrumented here is our own
# glue -- the AVIO callbacks against the fault-injecting extraction
# context of fuzz_ec.h, the option parser and the size arithmetic -- and
# not FFmpeg's demuxers, which upstream fuzzes far more thoroughly than
-# we could. Their GStreamer counterparts are deliberately absent: they
-# dlopen() their decoders and start threads, which does not fit an
-# in-process deterministic harness (the gstreamer plugin has no target
-# here either).
+# we could. previewgst and the gstreamer plugin have no target here;
+# thumbnailgst has one (below) for its srt/gallery glue.
if HAVE_FFMPEG
THUMBNAILFFMPEG_FUZZER = fuzz_thumbnailffmpeg
fuzz_thumbnailffmpeg_SOURCES = fuzz_plugin.c
@@ -517,6 +534,28 @@ nodist_fuzz_thumbnailffmpeg_SOURCES = \
$(PLUGIN_DIR)/mediautil.c
endif
+# thumbnailgst, dynamically linked (GStreamer loads its elements as
+# modules at run time anyway). GStreamer's own threads are fine for the
+# in-process driver: every pipeline is shut down (joining its threads)
+# before the plugin returns. A hostile file can stall GStreamer's
+# preroll until the plugin's deadline (10s by default) runs out, and
+# the harness may run the plugin twice per input, hence the longer
+# watchdog.
+if HAVE_GSTREAMER_MEDIA
+THUMBNAILGST_FUZZER = fuzz_thumbnailgst
+fuzz_thumbnailgst_SOURCES = fuzz_plugin.c
+fuzz_thumbnailgst_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) \
+ $(GSTREAMER_CFLAGS) $(GSTREAMER_APP_CFLAGS) $(GSTREAMER_VIDEO_CFLAGS) \
+ -DLE_FUZZ_PLUGIN=thumbnailgst -DLE_FUZZ_ID=THUMBNAILGST \
+ -DFUZZ_DEFAULT_TIMEOUT=45
+fuzz_thumbnailgst_LDADD = $(XLIB) $(GSTREAMER_LIBS) $(GSTREAMER_APP_LIBS) \
+ $(GSTREAMER_VIDEO_LIBS) -lmagic -lpthread
+nodist_fuzz_thumbnailgst_SOURCES = \
+ $(PLUGIN_DIR)/thumbnailgst_extractor.c \
+ $(PLUGIN_DIR)/mediagst.c \
+ $(PLUGIN_DIR)/mediautil.c
+endif
+
if HAVE_FFMPEG_AUDIO
PREVIEWOPUS_FUZZER = fuzz_previewopus
fuzz_previewopus_SOURCES = fuzz_plugin.c
@@ -537,7 +576,21 @@ fuzz_ole2_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) $(GSF_CFLAGS) \
-DLE_FUZZ_PLUGIN=ole2 -DLE_FUZZ_ID=OLE2
fuzz_ole2_LDFLAGS = -static
fuzz_ole2_LDADD = $(LE_COMMON) $(XLIB) $(GSF_LIBS)
-nodist_fuzz_ole2_SOURCES = $(PLUGIN_DIR)/ole2_extractor.c
+nodist_fuzz_ole2_SOURCES = $(PLUGIN_DIR)/ole2_extractor.c \
+ $(PLUGIN_DIR)/fulltext.c
+endif
+
+# pdf_extractor.cc is C++, which makes automake link this one with $(CXX).
+if HAVE_POPPLER
+PDF_FUZZER = fuzz_pdf
+fuzz_pdf_SOURCES = fuzz_plugin.c
+fuzz_pdf_CPPFLAGS = $(AM_CPPFLAGS) -I$(PLUGIN_DIR) $(POPPLER_CFLAGS) \
+ -DLE_FUZZ_PLUGIN=pdf -DLE_FUZZ_ID=PDF
+fuzz_pdf_LDADD = $(XLIB) $(POPPLER_LIBS) -lm
+nodist_fuzz_pdf_SOURCES = \
+ $(PLUGIN_DIR)/pdf_extractor.cc \
+ $(PLUGIN_DIR)/fulltext.c \
+ $(PLUGIN_DIR)/pagelayout.c
endif
@@ -553,8 +606,11 @@ check_PROGRAMS = \
$(OGG_FUZZER) \
$(ARCHIVE_FUZZER) \
$(MIME_FUZZER) \
+ $(HTML_FUZZER) \
$(OLE2_FUZZER) \
+ $(PDF_FUZZER) \
$(THUMBNAILFFMPEG_FUZZER) \
+ $(THUMBNAILGST_FUZZER) \
$(PREVIEWOPUS_FUZZER)
TESTS = $(check_PROGRAMS)
diff --git a/src/fuzz/fuzz_common.h b/src/fuzz/fuzz_common.h
@@ -795,6 +795,7 @@ fuzz_run_file (const char *path)
alarm (FUZZ_DEFAULT_TIMEOUT);
(void) LLVMFuzzerTestOneInput (buf, n);
alarm (0);
+ fuzz_cur_input = NULL; /* nothing to dump for an at-exit report */
free (buf);
return 0;
}
@@ -1072,6 +1073,7 @@ main (int argc, char *const *argv)
if (0 != timeout)
alarm (0);
}
+ fuzz_cur_input = NULL; /* nothing to dump for an at-exit report */
free (buf);
printf ("%s: %llu iterations completed, no findings\n",
FUZZ_HARNESS_NAME,
diff --git a/src/fuzz/fuzz_ec.h b/src/fuzz/fuzz_ec.h
@@ -82,7 +82,7 @@
* Upper bound the plugin side enforces on a single meta data item; see
* #MAX_META_DATA in src/main/extractor_ipc.h.
*/
-#define LE_FUZZ_MAX_META_DATA (32 * 1024)
+#define LE_FUZZ_MAX_META_DATA (32 * 1024 * 1024)
/**
* Number of configuration bytes that precede the file image in every
diff --git a/src/fuzz/fuzz_plugin.c b/src/fuzz/fuzz_plugin.c
@@ -66,6 +66,63 @@ LE_FUZZ_EXTRACT_METHOD (struct EXTRACTOR_ExtractContext *ec);
extern const char *
LE_FUZZ_OPTIONS_METHOD (void) __attribute__ ((weak));
+#ifdef LE_FUZZ_LSAN_SUPPRESSIONS
+/**
+ * Leak suppressions for plugins whose library keeps global state that
+ * is never released (see fuzz_plugin_name.h).
+ *
+ * @return suppression list in LSan syntax
+ */
+const char *
+__lsan_default_suppressions (void);
+
+const char *
+__lsan_default_suppressions (void)
+{
+ return LE_FUZZ_LSAN_SUPPRESSIONS;
+}
+
+
+/**
+ * The suppressions match on library frames; the default frame-pointer
+ * unwinder loses those in libraries built without frame pointers, so
+ * use the slow unwinder for allocations.
+ *
+ * @return default options for LeakSanitizer
+ */
+const char *
+__lsan_default_options (void);
+
+const char *
+__lsan_default_options (void)
+{
+ return "fast_unwind_on_malloc=0";
+}
+
+
+#endif
+
+#ifndef LE_FUZZ_CONFIGS
+/**
+ * Plugin configurations to fuzz. fuzz_plugin_name.h may define
+ * LE_FUZZ_CONFIGS as a comma-separated list of configuration strings
+ * for plugins that have options; entry 0 is always NULL, i.e. the
+ * configuration used when the user gives no options.
+ */
+#define LE_FUZZ_CONFIGS NULL
+#endif
+
+/**
+ * Configurations selected by the fourth prefix byte.
+ */
+static const char *const le_fuzz_configs[] = { LE_FUZZ_CONFIGS };
+
+/**
+ * Number of entries in #le_fuzz_configs.
+ */
+#define LE_FUZZ_NUM_CONFIGS (sizeof (le_fuzz_configs) \
+ / sizeof (le_fuzz_configs[0]))
+
/* ------------------------------------------------------------------ */
/* The fuzz target */
@@ -81,8 +138,7 @@ LLVMFuzzerTestOneInput (const uint8_t *data,
fuzz_ignore_sigpipe ();
if (! le_fuzz_ec_setup (&ec, &st, data, size))
return 0;
- if (NULL != LE_FUZZ_OPTIONS_METHOD)
- ec.config = LE_FUZZ_OPTIONS_METHOD ();
+ ec.config = le_fuzz_configs[data[3] % LE_FUZZ_NUM_CONFIGS];
LE_FUZZ_EXTRACT_METHOD (&ec);
if (0 != (st.faults & LE_FUZZ_RUN_TWICE))
{
@@ -437,7 +493,8 @@ le_fuzz_build_seeds (void)
b[0] = (uint8_t) (i + 1); /* window */
b[1] = (uint8_t) ((4 == i) ? LE_FUZZ_RUN_TWICE : 0); /* faults */
b[2] = (uint8_t) ((5 == i) ? 1 : 0);
- b[3] = 0;
+ /* cycle through the plugin configurations */
+ b[3] = (uint8_t) (i % LE_FUZZ_NUM_CONFIGS);
}
le_fuzz_seed_len[i] = len;
}
diff --git a/src/fuzz/fuzz_plugin_name.h b/src/fuzz/fuzz_plugin_name.h
@@ -129,6 +129,8 @@
#define LE_ID_MBOX 47
#define LE_ID_APK 48
#define LE_ID_EBOOK 49
+#define LE_ID_PDF 60
+#define LE_ID_THUMBNAILGST 70
#define LE_FUZZ_IDVAL LE_FUZZ_CAT (LE_ID_, LE_FUZZ_ID)
@@ -149,6 +151,14 @@
/* DVI preamble: 0xF7 0x02 */
#define LE_FUZZ_MAGIC "\xf7\x02"
#define LE_FUZZ_SHAPE LE_SHAPE_RAW
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
+
+#elif LE_FUZZ_IDVAL == LE_ID_PDF
+/* the interesting inputs are real documents: seed the corpus with
+ testdata/pdf_*.pdf (make_seed_corpus.sh does) */
+#define LE_FUZZ_MAGIC "%PDF-1.4\n"
+#define LE_FUZZ_SHAPE LE_SHAPE_TEXT
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
#elif LE_FUZZ_IDVAL == LE_ID_ELF
#define LE_FUZZ_MAGIC "\x7f" "ELF\x01\x01\x01\x00"
@@ -165,6 +175,7 @@
#elif LE_FUZZ_IDVAL == LE_ID_HTML
#define LE_FUZZ_MAGIC "<!DOCTYPE html>\n<html>"
#define LE_FUZZ_SHAPE LE_SHAPE_TEXT
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
#elif LE_FUZZ_IDVAL == LE_ID_IT
/* Impulse Tracker module */
@@ -189,6 +200,32 @@
to recognize it as video/* or the plugin declines to look at all */
#define LE_FUZZ_MAGIC "OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00"
#define LE_FUZZ_SHAPE LE_SHAPE_RAW
+#define LE_FUZZ_CONFIGS NULL, "srt:en", "gallery:4", \
+ "gallery:9,format=png,srt:de", "size=32,gallery:100"
+
+#elif LE_FUZZ_IDVAL == LE_ID_THUMBNAILGST
+/* Matroska (EBML header with DocType "matroska"), what the srt and
+ gallery options are tested against; libmagic has to call it video/*
+ or the plugin declines to look at all */
+#define LE_FUZZ_MAGIC \
+ "\x1a\x45\xdf\xa3\xa3\x42\x86\x81\x01\x42\xf7\x81\x01\x42\xf2" \
+ "\x81\x04\x42\xf3\x81\x08\x42\x82\x88matroska\x42\x87\x81\x04" \
+ "\x42\x85\x81\x02"
+#define LE_FUZZ_SHAPE LE_SHAPE_RAW
+/* srt and gallery raise the default deadline to 60s; a hostile file can
+ stall GStreamer's preroll for all of it, which the harness watchdog
+ (20s) would report as a hang, so the deadline is lowered explicitly */
+#define LE_FUZZ_CONFIGS NULL, "srt:en,deadline=5000", \
+ "gallery:4,deadline=5000", \
+ "gallery:9,format=png,srt:de,deadline=5000", \
+ "size=32,gallery:100,deadline=5000"
+/* GStreamer never frees its registry, type system and thread pools;
+ only suppress leaks with a GStreamer frame on the stack, so that
+ leaks in the plugin's own glue are still reported */
+#define LE_FUZZ_LSAN_SUPPRESSIONS \
+ "leak:libgstreamer-1.0.so\n" \
+ "leak:libgstbase-1.0.so\n" \
+ "leak:/gstreamer-1.0/\n"
#elif LE_FUZZ_IDVAL == LE_ID_PECOFF
/* MZ header; the PE header offset lives at 0x3c */
@@ -282,6 +319,7 @@
testdata/ebook_test.mobi to reach it */
#define LE_FUZZ_MAGIC ""
#define LE_FUZZ_SHAPE LE_SHAPE_ZIP
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
#elif LE_FUZZ_IDVAL == LE_ID_PREVIEWOPUS
#define LE_FUZZ_MAGIC "OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00"
@@ -307,6 +345,7 @@
#elif LE_FUZZ_IDVAL == LE_ID_ODF
#define LE_FUZZ_MAGIC ""
#define LE_FUZZ_SHAPE LE_SHAPE_ZIP
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
#elif LE_FUZZ_IDVAL == LE_ID_OGG
#define LE_FUZZ_MAGIC "OggS\x00\x02"
@@ -315,6 +354,7 @@
#elif LE_FUZZ_IDVAL == LE_ID_OLE2
#define LE_FUZZ_MAGIC "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
#define LE_FUZZ_SHAPE LE_SHAPE_RAW
+#define LE_FUZZ_CONFIGS NULL, "fulltext"
#elif LE_FUZZ_IDVAL == LE_ID_PNG
#define LE_FUZZ_MAGIC "\x89PNG\r\n\x1a\n"
diff --git a/src/include/extractor.h b/src/include/extractor.h
@@ -728,7 +728,19 @@ enum EXTRACTOR_MetaType
*/
EXTRACTOR_METATYPE_MINIMUM_OS_VERSION = 290,
- EXTRACTOR_METATYPE_LAST = 291
+ /**
+ * Full text of the document as UTF-8, paragraphs separated by
+ * "\n\n". Only produced on request (plugin option "fulltext").
+ */
+ EXTRACTOR_METATYPE_FULL_TEXT = 291,
+
+ /**
+ * Complete subtitle track as a UTF-8 SRT file. Only produced on
+ * request (plugin option "srt:LANG").
+ */
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT = 292,
+
+ EXTRACTOR_METATYPE_LAST = 293
};
/** @} */ /* end of meta data types */
@@ -830,6 +842,13 @@ struct EXTRACTOR_ExtractContext
* Seek in the file. Use `SEEK_CUR` for @a whence and @a pos of 0 to
* obtain the current position in the file.
*
+ * Out of process, a plugin that neither reads, seeks nor reports meta
+ * data for 500 ms is presumed hung and killed. A plugin that
+ * legitimately computes for longer without touching the file (full
+ * text of a large document, a video gallery, ...) must call
+ * `seek (cls, 0, SEEK_CUR)` regularly (at least every 250 ms) to show
+ * it is still making progress; the call is cheap and rate-limited.
+ *
* @param cls the @e cls member of this struct
* @param pos position to seek (see 'man lseek')
* @param whence how to see (absolute to start, relative, absolute to end)
diff --git a/src/main/extractor_ipc.c b/src/main/extractor_ipc.c
@@ -67,6 +67,11 @@ EXTRACTOR_IPC_process_reply_ (struct EXTRACTOR_PluginList *plugin,
size--;
data++;
continue;
+ case MESSAGE_ALIVE: /* still busy; receiving it was the point */
+ ret++;
+ size--;
+ data++;
+ continue;
case MESSAGE_SEEK: /* Seek */
if (size < sizeof (struct SeekRequestMessage))
{
diff --git a/src/main/extractor_ipc.h b/src/main/extractor_ipc.h
@@ -313,6 +313,15 @@ struct MetaMessage
*/
#define MESSAGE_CONTINUE_EXTRACTING 0x07
+/**
+ * Sent from plugin to LE to show that the plugin is still making
+ * progress although it has not read, sought or reported anything for a
+ * while (LE closes channels that stay silent for too long). Only one
+ * byte, no reply. The plugin side sends it (rate-limited) whenever the
+ * plugin calls `seek (cls, 0, SEEK_CUR)`.
+ */
+#define MESSAGE_ALIVE 0x08
+
/**
* Definition of an IPC communication channel with
diff --git a/src/main/extractor_metatypes.c b/src/main/extractor_metatypes.c
@@ -730,6 +730,11 @@ static const struct MetaTypeDescription meta_type_descriptions[] = {
gettext_noop ("name of the machine the file originated on") },
{ gettext_noop ("minimum operating system version"),
gettext_noop ("oldest platform release the content will run on") },
+ { gettext_noop ("full text"),
+ gettext_noop ("entire text content of the document") },
+ /* 292 */
+ { gettext_noop ("subtitle text"),
+ gettext_noop ("complete subtitle track in SRT format") },
{ gettext_noop ("last"),
gettext_noop ("last") }
diff --git a/src/main/extractor_plugin_main.c b/src/main/extractor_plugin_main.c
@@ -100,10 +100,64 @@ struct ProcessingContext
* Output stream.
*/
int out;
+
+ /**
+ * When we last sent a #MESSAGE_ALIVE, in milliseconds.
+ */
+ uint64_t last_alive;
};
/**
+ * Minimum time between two #MESSAGE_ALIVE, in milliseconds; well below
+ * the silence LE tolerates (500 ms).
+ */
+#define ALIVE_INTERVAL_MS 100
+
+
+/**
+ * Get a monotonic time stamp.
+ *
+ * @return current time in milliseconds since some arbitrary epoch
+ */
+static uint64_t
+now_ms (void)
+{
+#if WINDOWS
+ return (uint64_t) GetTickCount64 ();
+#else
+ struct timespec ts;
+
+ if (0 != clock_gettime (CLOCK_MONOTONIC,
+ &ts))
+ return 0;
+ return ((uint64_t) ts.tv_sec) * 1000 + ts.tv_nsec / 1000000;
+#endif
+}
+
+
+/**
+ * Tell LE that we are still busy, unless we did so very recently.
+ *
+ * @param pc processing context
+ */
+static void
+send_alive (struct ProcessingContext *pc)
+{
+ unsigned char alive = MESSAGE_ALIVE;
+ uint64_t now = now_ms ();
+
+ if (now - pc->last_alive < ALIVE_INTERVAL_MS)
+ return;
+ pc->last_alive = now;
+ if (-1 == EXTRACTOR_write_all_ (pc->out,
+ &alive,
+ sizeof (alive)))
+ LOG ("Failed to send MESSAGE_ALIVE\n");
+}
+
+
+/**
* Moves current absolute buffer position to 'pos' in 'whence' mode.
* Will move logical position without shifting the buffer, if possible.
* Will not move beyond the end of file.
@@ -127,6 +181,13 @@ plugin_env_seek (void *cls,
unsigned char reply;
uint16_t wval;
+ if ( (SEEK_CUR == whence) &&
+ (0 == pos) )
+ {
+ /* a no-op seek is how a busy plugin keeps its channel open */
+ send_alive (pc);
+ return (int64_t) pc->read_position;
+ }
switch (whence)
{
case SEEK_CUR:
diff --git a/src/main/iconv.c b/src/main/iconv.c
@@ -45,8 +45,8 @@ iconv_helper (iconv_t cd,
size_t outSize;
size_t outLeft;
- if (inSize > 1024 * 1024)
- return NULL; /* too big to be meta data */
+ if (inSize > 32 * 1024 * 1024)
+ return NULL; /* larger than MAX_META_DATA, cannot be meta data */
i = in;
/* reset iconv */
iconv (cd, NULL, NULL, NULL, NULL);
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
@@ -36,6 +36,7 @@ EXTRA_DIST = \
testdata/chello-elf \
testdata/deb_bzip2.deb \
testdata/dvi_ora.dvi \
+ testdata/dvi_fulltext.dvi \
testdata/exiv2_iptc.jpg \
testdata/flac_kraftwerk.flac \
testdata/gif_image.gif \
@@ -43,6 +44,7 @@ EXTRA_DIST = \
testdata/gstreamer_barsandtone.flv \
testdata/gstreamer_sample_sorenson.mov \
testdata/html_grothoff.html \
+ testdata/html_fulltext.html \
testdata/it_dawn.it \
testdata/jpeg_image.jpg \
testdata/man_extract.1 \
@@ -59,13 +61,18 @@ EXTRA_DIST = \
testdata/nsf_arkanoid.nsf \
testdata/nsfe_classics.nsfe \
testdata/odf_cg.odt \
+ testdata/odf_fulltext.odp \
+ testdata/odf_fulltext.ods \
+ testdata/odf_fulltext.odt \
testdata/ole2_blair.doc \
testdata/ole2_excel.xls \
+ testdata/ole2_fulltext.doc \
testdata/ole2_msword.doc \
testdata/ole2_powerpoint.ppt \
testdata/ole2_starwriter40.sdw \
testdata/ogg_courseclear.ogg \
testdata/pdf_extract.pdf \
+ testdata/pdf_fulltext.pdf \
testdata/png_image.png \
testdata/preview_tone.ogg \
testdata/preview_tone.ogv \
@@ -83,6 +90,7 @@ EXTRA_DIST = \
testdata/s3m_2nd_pm.s3m \
testdata/applefile_test.applesingle \
testdata/sid_wizball.sid \
+ testdata/subtitles.mkv \
testdata/thumbnail_pattern.ogv \
testdata/thumbnail_torsten.jpg \
testdata/tiff_haute.tiff \
@@ -98,6 +106,8 @@ EXTRA_DIST = \
testdata/diskimage_test.vmdk \
testdata/ebook_test.epub \
testdata/ebook_test.mobi \
+ testdata/ebook_fulltext.epub \
+ testdata/ebook_fulltext.mobi \
testdata/geotiff_test.tif \
testdata/gpx_test.gpx \
testdata/heif_test.avif \
@@ -561,11 +571,12 @@ test_apk_LDADD = \
$(top_builddir)/src/plugins/libtest.la
libextractor_ebook_la_SOURCES = \
- ebook_extractor.c forensics.c forensics.h
+ ebook_extractor.c forensics.c forensics.h fulltext.c fulltext.h
libextractor_ebook_la_LDFLAGS = \
$(PLUGINFLAGS)
libextractor_ebook_la_LIBADD = \
- $(top_builddir)/src/common/libextractor_common.la -lm $(XLIB) $(LE_LIBINTL)
+ $(top_builddir)/src/common/libextractor_common.la -lm $(LTLIBICONV) \
+ $(XLIB) $(LE_LIBINTL)
test_ebook_SOURCES = \
test_ebook.c
@@ -608,14 +619,16 @@ libextractor_applefile_la_LIBADD = \
libextractor_dvi_la_SOURCES = \
- dvi_extractor.c
+ dvi_extractor.c \
+ fulltext.c fulltext.h \
+ pagelayout.c pagelayout.h
libextractor_dvi_la_LDFLAGS = \
$(PLUGINFLAGS)
libextractor_dvi_la_LIBADD = \
- $(XLIB) $(SOCKET_LIBS)
+ $(XLIB) $(SOCKET_LIBS) -lm
test_dvi_SOURCES = \
- test_dvi.c
+ test_dvi.c test_paged_fulltext.h
test_dvi_LDADD = \
$(top_builddir)/src/plugins/libtest.la
@@ -707,9 +720,10 @@ test_thumbnailgst_SOURCES = \
test_media_lib.c test_media_lib.h
test_thumbnailgst_LDADD = \
$(top_builddir)/src/plugins/libtest.la \
- $(TEST_MEDIA_LIBS)
+ $(top_builddir)/src/main/libextractor.la \
+ $(TEST_MEDIA_LIBS) $(GSTREAMER_LIBS)
test_thumbnailgst_CFLAGS = \
- $(AM_CFLAGS) $(TEST_MEDIA_CFLAGS)
+ $(AM_CFLAGS) $(TEST_MEDIA_CFLAGS) $(GSTREAMER_CFLAGS)
libextractor_previewgst_la_SOURCES = \
@@ -777,13 +791,13 @@ test_previewopus_CFLAGS = \
libextractor_html_la_SOURCES = \
- html_extractor.c
+ html_extractor.c fulltext.c fulltext.h
libextractor_html_la_CFLAGS = \
$(TIDY_CFLAGS)
libextractor_html_la_LDFLAGS = \
$(PLUGINFLAGS)
libextractor_html_la_LIBADD = \
- -ltidy -lmagic $(XLIB)
+ -ltidy -lmagic $(LTLIBICONV) $(XLIB)
test_html_SOURCES = \
test_html.c
@@ -912,7 +926,8 @@ test_msoffice_LDADD = \
libextractor_odf_la_SOURCES = \
- odf_extractor.c
+ odf_extractor.c \
+ fulltext.c fulltext.h
libextractor_odf_la_LDFLAGS = \
$(PLUGINFLAGS)
libextractor_odf_la_LIBADD = \
@@ -925,7 +940,8 @@ test_odf_LDADD = \
libextractor_ole2_la_SOURCES = \
- ole2_extractor.c
+ ole2_extractor.c \
+ fulltext.c fulltext.h
libextractor_ole2_la_CFLAGS = \
$(GSF_CFLAGS)
libextractor_ole2_la_CPPFLAGS = \
@@ -956,16 +972,18 @@ test_ogg_LDADD = \
libextractor_pdf_la_SOURCES = \
- pdf_extractor.cc
+ pdf_extractor.cc \
+ fulltext.c fulltext.h \
+ pagelayout.c pagelayout.h
libextractor_pdf_la_LDFLAGS = \
$(PLUGINFLAGS)
libextractor_pdf_la_CPPFLAGS = \
$(AM_CPPFLAGS) $(POPPLER_CFLAGS)
libextractor_pdf_la_LIBADD = \
- $(POPPLER_LIBS) $(XLIB)
+ $(POPPLER_LIBS) $(XLIB) -lm
test_pdf_SOURCES = \
- test_pdf.c
+ test_pdf.c test_paged_fulltext.h
test_pdf_LDADD = \
$(top_builddir)/src/plugins/libtest.la
diff --git a/src/plugins/dvi_extractor.c b/src/plugins/dvi_extractor.c
@@ -24,6 +24,9 @@
*/
#include "platform.h"
#include "extractor.h"
+#include "fulltext.h"
+#include "pagelayout.h"
+#include <math.h>
/**
@@ -152,6 +155,1470 @@ getShortAt (const void *data)
}
+/* ****************** full text extraction ********************** */
+
+/**
+ * Maximum number of fonts we track.
+ */
+#define DVI_MAX_FONTS 1024
+
+/**
+ * Size of the font hash table (a power of two, larger than
+ * #DVI_MAX_FONTS).
+ */
+#define DVI_FONT_HASH 2048
+
+/**
+ * Maximum depth of the push/pop stack.
+ */
+#define DVI_MAX_STACK 1024
+
+/**
+ * Maximum number of pages we extract text from.
+ */
+#define DVI_MAX_PAGES 50000
+
+/**
+ * Maximum number of lines we collect over the whole document.
+ */
+#define DVI_MAX_LINES 500000
+
+/**
+ * Maximum number of bytes in one line.
+ */
+#define DVI_MAX_LINE_BYTES (64 * 1024)
+
+/**
+ * Font size assumed for undefined fonts (10pt in TeX's scaled points).
+ */
+#define DVI_DEFAULT_SIZE 655360
+
+/**
+ * Markers in the encoding tables for glyphs that expand to several
+ * characters. Real code points are never below 0x20.
+ */
+#define LIG_FF 1
+#define LIG_FI 2
+#define LIG_FL 3
+#define LIG_FFI 4
+#define LIG_FFL 5
+#define LIG_SS 6
+
+
+/**
+ * Font encodings we know how to map to Unicode.
+ */
+enum DviEncoding
+{
+ /**
+ * TeX text (OT1), e.g. cmr10.
+ */
+ ENC_OT1,
+
+ /**
+ * TeX typewriter text (OT1 variant), e.g. cmtt10.
+ */
+ ENC_OT1TT,
+
+ /**
+ * Cork encoding (T1), e.g. ecrm1000, sfrm1000, ec-lmr10.
+ */
+ ENC_T1,
+
+ /**
+ * ASCII plus ISO-8859-1 upper half (roughly TeXBase1, 8r).
+ */
+ ENC_LATIN1,
+
+ /**
+ * TeX math italic, e.g. cmmi10.
+ */
+ ENC_MATHIT,
+
+ /**
+ * TeX math symbols, e.g. cmsy10.
+ */
+ ENC_SYMBOL,
+
+ /**
+ * Font whose glyphs we do not map (math extension, dingbats, ...).
+ */
+ ENC_NONE
+};
+
+
+/**
+ * A font defined in the DVI file.
+ */
+struct DviFont
+{
+ /**
+ * Font number used in the DVI file.
+ */
+ uint32_t k;
+
+ /**
+ * Scaled size in DVI units.
+ */
+ double size;
+
+ /**
+ * Encoding guessed from the font name.
+ */
+ enum DviEncoding enc;
+};
+
+
+/**
+ * Registers of the DVI machine.
+ */
+struct DviRegs
+{
+ int64_t h;
+ int64_t v;
+ int64_t w;
+ int64_t x;
+ int64_t y;
+ int64_t z;
+};
+
+
+/**
+ * State of the full-text extraction.
+ */
+struct DviText
+{
+ /**
+ * Extraction context, for keep-alives.
+ */
+ struct EXTRACTOR_ExtractContext *ec;
+
+ /**
+ * The DVI file.
+ */
+ const unsigned char *data;
+
+ /**
+ * Number of bytes in @e data.
+ */
+ size_t size;
+
+ /**
+ * Defined fonts.
+ */
+ struct DviFont fonts[DVI_MAX_FONTS];
+
+ /**
+ * Number of entries in @e fonts.
+ */
+ unsigned int num_fonts;
+
+ /**
+ * Hash table (open addressing) mapping font numbers to 1 + their
+ * index in @e fonts; 0 marks an empty slot.
+ */
+ uint16_t font_hash[DVI_FONT_HASH];
+
+ /**
+ * Pages collected so far.
+ */
+ struct EXTRACTOR_FullTextPage *pages;
+
+ /**
+ * Geometry of the lines of each page.
+ */
+ struct EXTRACTOR_PageLineGeom **geoms;
+
+ /**
+ * Number of pages in @e pages / @e geoms.
+ */
+ unsigned int num_pages;
+
+ /**
+ * Allocated length of the geometry array of the current page.
+ */
+ unsigned int geom_cap;
+
+ /**
+ * Text of the current line.
+ */
+ char *lbuf;
+
+ /**
+ * Bytes in @e lbuf.
+ */
+ size_t llen;
+
+ /**
+ * Allocated size of @e lbuf.
+ */
+ size_t lcap;
+
+ /**
+ * 1 if a line is being collected.
+ */
+ int in_line;
+
+ /**
+ * 1 if a space must precede the next character of the line.
+ */
+ int pending_space;
+
+ /**
+ * Accent (combining code point) waiting for its base character, 0
+ * for none.
+ */
+ uint32_t pending_accent;
+
+ /**
+ * 1 if @e pending_accent was placed over the preceding base character
+ * (@e last_cp), so that it may belong to it rather than to the next.
+ */
+ int accent_post_ok;
+
+ /**
+ * Last base character appended to the line (for accents placed
+ * after their base), 0 if none.
+ */
+ uint32_t last_cp;
+
+ /**
+ * Offset of @e last_cp in @e lbuf.
+ */
+ size_t last_cp_off;
+
+ /**
+ * Baseline of the current line.
+ */
+ int64_t line_v;
+
+ /**
+ * Left end of the current line.
+ */
+ double line_x0;
+
+ /**
+ * (Estimated) right end of the current line.
+ */
+ double line_x1;
+
+ /**
+ * Font size at the start of the current line.
+ */
+ double line_size;
+
+ /**
+ * (Estimated) horizontal position after the last glyph.
+ */
+ double last_end;
+
+ /**
+ * Total number of lines collected.
+ */
+ unsigned int total_lines;
+
+ /**
+ * Total number of text bytes collected.
+ */
+ size_t total_bytes;
+
+ /**
+ * Smallest and largest baseline seen.
+ */
+ int64_t vmin;
+ int64_t vmax;
+
+ /**
+ * 1 if @e vmin and @e vmax are set.
+ */
+ int have_v;
+
+ /**
+ * Remaining number of opcodes we are willing to interpret.
+ */
+ uint64_t budget;
+
+ /**
+ * 1 if we ran out of memory or into a limit: stop collecting.
+ */
+ int full;
+};
+
+
+/**
+ * Approximate widths (in 1/1000 em) of the glyphs of cmr10, used to
+ * estimate where a glyph ends (DVI does not record glyph widths).
+ */
+static const uint16_t cmr_widths[128] = {
+ /* 0x00 Greek capitals */
+ 625, 833, 778, 694, 667, 750, 722, 778, 722, 778, 722,
+ /* 0x0B ff fi fl ffi ffl */
+ 583, 556, 556, 833, 833,
+ /* 0x10 dotless i/j, accents */
+ 278, 306, 500, 500, 500, 500, 500, 750, 444,
+ /* 0x19 ss ae oe o/ AE OE O/ */
+ 500, 722, 778, 500, 903, 1014, 778,
+ /* 0x20 */
+ 278, 278, 500, 833, 500, 833, 778, 278, 389, 389, 500, 778, 278, 333,
+ 278, 500,
+ /* 0x30 digits */
+ 500, 500, 500, 500, 500, 500, 500, 500, 500, 500,
+ /* 0x3A */
+ 278, 278, 278, 778, 472, 472,
+ /* 0x40 @ A-O */
+ 778, 750, 708, 722, 764, 681, 653, 785, 750, 361, 514, 778, 625, 917,
+ 750, 778,
+ /* 0x50 P-Z [ " ] ^ . */
+ 681, 778, 736, 556, 722, 750, 750, 1028, 750, 750, 611, 278, 500, 278,
+ 500, 278,
+ /* 0x60 ` a-o */
+ 278, 500, 556, 444, 556, 444, 306, 500, 556, 278, 306, 528, 278, 833,
+ 556, 500,
+ /* 0x70 p-z -- --- accents */
+ 556, 528, 392, 394, 389, 556, 528, 722, 528, 528, 444, 500, 1000, 500,
+ 500, 500
+};
+
+
+/**
+ * Greek capitals at 0x00-0x0A of OT1 and math italic fonts.
+ */
+static const uint16_t greek_caps[11] = {
+ 0x0393, 0x0394, 0x0398, 0x039B, 0x039E, 0x03A0, 0x03A3, 0x03A5,
+ 0x03A6, 0x03A8, 0x03A9
+};
+
+/**
+ * Glyphs 0x10-0x1F of OT1 fonts.
+ */
+static const uint16_t ot1_10[16] = {
+ 0x0131, 0x0237, 0x0300, 0x0301, 0x030C, 0x0306, 0x0304, 0x030A,
+ 0x0327, 0x00DF, 0x00E6, 0x0153, 0x00F8, 0x00C6, 0x0152, 0x00D8
+};
+
+/**
+ * Glyphs 0x00-0x1F of T1 fonts.
+ */
+static const uint16_t t1_00[32] = {
+ 0x0300, 0x0301, 0x0302, 0x0303, 0x0308, 0x030B, 0x030A, 0x030C,
+ 0x0306, 0x0304, 0x0307, 0x0327, 0x0328, 0x201A, 0x2039, 0x203A,
+ 0x201C, 0x201D, 0x201E, 0x00AB, 0x00BB, 0x2013, 0x2014, 0,
+ 0, 0x0131, 0x0237, LIG_FF, LIG_FI, LIG_FL, LIG_FFI, LIG_FFL
+};
+
+/**
+ * Glyphs 0x80-0xBF of T1 fonts.
+ */
+static const uint16_t t1_80[64] = {
+ 0x0102, 0x0104, 0x0106, 0x010C, 0x010E, 0x011A, 0x0118, 0x011E,
+ 0x0139, 0x013D, 0x0141, 0x0143, 0x0147, 0x014A, 0x0150, 0x0154,
+ 0x0158, 0x015A, 0x0160, 0x015E, 0x0164, 0x0162, 0x0170, 0x016E,
+ 0x0178, 0x0179, 0x017D, 0x017B, 0x0132, 0x0130, 0x0111, 0x00A7,
+ 0x0103, 0x0105, 0x0107, 0x010D, 0x010F, 0x011B, 0x0119, 0x011F,
+ 0x013A, 0x013E, 0x0142, 0x0144, 0x0148, 0x014B, 0x0151, 0x0155,
+ 0x0159, 0x015B, 0x0161, 0x015F, 0x0165, 0x0163, 0x0171, 0x016F,
+ 0x00FF, 0x017A, 0x017E, 0x017C, 0x0133, 0x00A1, 0x00BF, 0x00A3
+};
+
+/**
+ * Glyphs 0x0B-0x27 of math italic fonts (lower-case Greek).
+ */
+static const uint16_t mathit_0b[29] = {
+ 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03F5, 0x03B6, 0x03B7, 0x03B8,
+ 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03C0, 0x03C1,
+ 0x03C3, 0x03C4, 0x03C5, 0x03D5, 0x03C7, 0x03C8, 0x03C9, 0x03B5,
+ 0x03D1, 0x03D6, 0x03F1, 0x03C2, 0x03C6
+};
+
+/**
+ * Glyphs 0x00-0x7F of math symbol fonts (0: not mapped).
+ */
+static const uint16_t symbol_map[128] = {
+ 0x2212, 0x22C5, 0x00D7, 0x2217, 0x00F7, 0x22C4, 0x00B1, 0x2213,
+ 0x2295, 0x2296, 0x2297, 0x2298, 0x2299, 0x25EF, 0x2218, 0x2022,
+ 0x224D, 0x2261, 0x2286, 0x2287, 0x2264, 0x2265, 0x2AAF, 0x2AB0,
+ 0x223C, 0x2248, 0x2282, 0x2283, 0x226A, 0x226B, 0x227A, 0x227B,
+ 0x2190, 0x2192, 0x2191, 0x2193, 0x2194, 0x2197, 0x2198, 0x2243,
+ 0x21D0, 0x21D2, 0x21D1, 0x21D3, 0x21D4, 0x2196, 0x2199, 0x221D,
+ 0x2032, 0x221E, 0x2208, 0x220B, 0x25B3, 0x25BD, 0, 0,
+ 0x2200, 0x2203, 0x00AC, 0x2205, 0x211C, 0x2111, 0x22A4, 0x22A5,
+ 0x2135, 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+ 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
+ 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W',
+ 'X', 'Y', 'Z', 0x222A, 0x2229, 0x228E, 0x2227, 0x2228,
+ 0x22A2, 0x22A3, 0x230A, 0x230B, 0x2308, 0x2309, '{', '}',
+ 0x27E8, 0x27E9, '|', 0x2016, 0x2195, 0x21D5, '\\', 0x2240,
+ 0x221A, 0x2A3F, 0x2207, 0x222B, 0x2294, 0x2293, 0x2291, 0x2292,
+ 0x00A7, 0x2020, 0x2021, 0x00B6, 0x2663, 0x2662, 0x2661, 0x2660
+};
+
+
+/**
+ * Map glyph @a c of a font with encoding @a enc to Unicode.
+ *
+ * @param enc font encoding
+ * @param c glyph code
+ * @return code point, a LIG_* marker, or 0 if the glyph has no text
+ */
+static uint32_t
+dvi_map_glyph (enum DviEncoding enc,
+ uint32_t c)
+{
+ if (c > 255)
+ return 0;
+ switch (enc)
+ {
+ case ENC_OT1:
+ case ENC_OT1TT:
+ if (c >= 128)
+ return 0;
+ if (c < 0x0B)
+ return greek_caps[c];
+ if (ENC_OT1TT == enc)
+ {
+ switch (c)
+ {
+ case 0x0B: return 0x2191;
+ case 0x0C: return 0x2193;
+ case 0x0D: return '\'';
+ case 0x0E: return 0x00A1;
+ case 0x0F: return 0x00BF;
+ case 0x7F: return 0x0308;
+ default:
+ break;
+ }
+ if (c >= 0x20)
+ return c;
+ }
+ if (c < 0x10)
+ return LIG_FF + (c - 0x0B);
+ if (c < 0x20)
+ return ot1_10[c - 0x10];
+ switch (c)
+ {
+ case 0x20: return 0x0337; /* stroke of \l and \L */
+ case 0x22: return 0x201D;
+ case 0x27: return 0x2019;
+ case 0x3C: return 0x00A1;
+ case 0x3E: return 0x00BF;
+ case 0x5C: return 0x201C;
+ case 0x5E: return 0x0302;
+ case 0x5F: return 0x0307;
+ case 0x60: return 0x2018;
+ case 0x7B: return 0x2013;
+ case 0x7C: return 0x2014;
+ case 0x7D: return 0x030B;
+ case 0x7E: return 0x0303;
+ case 0x7F: return 0x0308;
+ default:
+ return c;
+ }
+ case ENC_T1:
+ if (c < 0x20)
+ return t1_00[c];
+ if (c == 0x7F)
+ return '-';
+ if (c < 0x80)
+ return c;
+ if (c < 0xC0)
+ return t1_80[c - 0x80];
+ switch (c)
+ {
+ case 0xD7: return 0x0152;
+ case 0xDF: return LIG_SS;
+ case 0xF7: return 0x0153;
+ case 0xFF: return 0x00DF;
+ default:
+ return c;
+ }
+ case ENC_LATIN1:
+ if ( ( (c >= 0x20) && (c < 0x7F) ) ||
+ (c >= 0xA0) )
+ return c;
+ return 0;
+ case ENC_MATHIT:
+ if (c < 0x0B)
+ return greek_caps[c];
+ if (c < 0x28)
+ return mathit_0b[c - 0x0B];
+ if ( (c >= 0x30) && (c <= 0x3E) && (c != 0x3D) )
+ return c;
+ if (0x3D == c)
+ return '/';
+ if (0x40 == c)
+ return 0x2202;
+ if ( ( (c >= 'A') && (c <= 'Z') ) ||
+ ( (c >= 'a') && (c <= 'z') ) )
+ return c;
+ if (0x60 == c)
+ return 0x2113;
+ if (0x7B == c)
+ return 0x0131;
+ if (0x7C == c)
+ return 0x0237;
+ return 0;
+ case ENC_SYMBOL:
+ if (c >= 128)
+ return 0;
+ return symbol_map[c];
+ case ENC_NONE:
+ default:
+ return 0;
+ }
+}
+
+
+/**
+ * Estimate the width of glyph @a c, in 1/1000 em.
+ *
+ * @param enc font encoding
+ * @param c glyph code
+ * @return estimated width
+ */
+static unsigned int
+dvi_glyph_width (enum DviEncoding enc,
+ uint32_t c)
+{
+ switch (enc)
+ {
+ case ENC_OT1:
+ return (c < 128) ? cmr_widths[c] : 500;
+ case ENC_OT1TT:
+ return 525;
+ case ENC_T1:
+ case ENC_LATIN1:
+ if ( (c >= 0x21) && (c < 0x7F) &&
+ (c != '"') && (c != '<') && (c != '>') && (c != '\\') &&
+ (c != '_') && (c < '{') )
+ return cmr_widths[c];
+ if (c >= 0x80)
+ return ( (c & 0x3F) < 0x20) ? 720 : 500;
+ return 500;
+ default:
+ return 600;
+ }
+}
+
+
+/**
+ * Guess the encoding of a font from its name.
+ *
+ * @param name font name (not 0-terminated)
+ * @param len length of @a name
+ * @return guessed encoding
+ */
+static enum DviEncoding
+dvi_font_encoding (const char *name,
+ size_t len)
+{
+ static const char *const none[] = {
+ "cmex", "lmex", "msam", "msbm", "lasy", "wasy", "line", "lcircle",
+ "tc", "ts1-", "eufm", "eusm", "eurm", "rsfs", "stmary", "bbm", "dsrom",
+ "pzd", "psy", "wncy", "manfnt", "logo", "esint", "cmbsy10x",
+ NULL
+ };
+ static const char *const tt[] = {
+ "cmtt", "cmsltt", "cmitt", "cmtcsc", "lmtt", "rm-lmtt", NULL
+ };
+ static const char *const sf[] = {
+ "rm", "ti", "bx", "sl", "ss", "tt", "cc", "bi", "bl", "si", "sx",
+ "so", "sq", "xc", "tl", "st", "oc", "ci", "ui", "vi", "bo",
+ NULL
+ };
+ char lc[64];
+ size_t i;
+ size_t end;
+
+ if (len >= sizeof (lc))
+ len = sizeof (lc) - 1;
+ for (i = 0; i < len; i++)
+ lc[i] = (char) tolower ((unsigned char) name[i]);
+ lc[len] = '\0';
+ for (i = 0; NULL != none[i]; i++)
+ if (0 == strncmp (lc, none[i], strlen (none[i])))
+ return ENC_NONE;
+ if ( (0 == strncmp (lc, "cmmi", 4)) ||
+ (0 == strncmp (lc, "lmmi", 4)) ||
+ (0 == strncmp (lc, "rm-lmmi", 7)) )
+ return ENC_MATHIT;
+ if ( (0 == strncmp (lc, "cmsy", 4)) ||
+ (0 == strncmp (lc, "cmbsy", 5)) ||
+ (0 == strncmp (lc, "lmsy", 4)) ||
+ (0 == strncmp (lc, "lmbsy", 5)) )
+ return ENC_SYMBOL;
+ for (i = 0; NULL != tt[i]; i++)
+ if (0 == strncmp (lc, tt[i], strlen (tt[i])))
+ return ENC_OT1TT;
+ if ( (0 == strncmp (lc, "ec-", 3)) ||
+ (0 == strncmp (lc, "t1-", 3)) ||
+ (0 == strncmp (lc, "cs-", 3)) ||
+ (0 == strncmp (lc, "qx-", 3)) ||
+ (0 == strncmp (lc, "l7x-", 4)) ||
+ (0 == strncmp (lc, "t5-", 3)) ||
+ (0 == strncmp (lc, "texnansi-", 9)) )
+ return ENC_T1;
+ if ( (len > 2) &&
+ (0 == strncmp (lc, "ec", 2)) &&
+ isalpha ((unsigned char) lc[2]) )
+ return ENC_T1;
+ if ( (len > 4) &&
+ (0 == strncmp (lc, "sf", 2)) )
+ for (i = 0; NULL != sf[i]; i++)
+ if (0 == strncmp (&lc[2], sf[i], 2))
+ return ENC_T1;
+ /* Karl Berry's font names: <supplier><family><variant><encoding> */
+ end = len;
+ while ( (end > 0) &&
+ isdigit ((unsigned char) lc[end - 1]) )
+ end--;
+ for (i = (end > 4) ? end - 4 : 0; i + 1 < end; i++)
+ {
+ if ('8' == lc[i])
+ {
+ if ('t' == lc[i + 1])
+ return ENC_T1;
+ if ('r' == lc[i + 1])
+ return ENC_LATIN1;
+ if ( ('c' == lc[i + 1]) ||
+ ('y' == lc[i + 1]) )
+ return ENC_NONE;
+ }
+ }
+ return ENC_OT1;
+}
+
+
+/**
+ * Check whether @a cp is a combining accent.
+ *
+ * @param cp code point
+ * @return 1 if combining
+ */
+static int
+is_accent (uint32_t cp)
+{
+ return ( (cp >= 0x0300) && (cp < 0x0370) ) ? 1 : 0;
+}
+
+
+/**
+ * Append raw bytes to the current line.
+ *
+ * @param t state
+ * @param s bytes to append
+ * @param len number of bytes
+ */
+static void
+line_bytes (struct DviText *t,
+ const char *s,
+ size_t len)
+{
+ if (t->llen + len > DVI_MAX_LINE_BYTES)
+ return;
+ if (t->llen + len > t->lcap)
+ {
+ size_t ncap = (0 == t->lcap) ? 256 : t->lcap;
+ char *n;
+
+ while (ncap < t->llen + len)
+ ncap *= 2;
+ n = realloc (t->lbuf,
+ ncap);
+ if (NULL == n)
+ {
+ t->full = 1;
+ return;
+ }
+ t->lbuf = n;
+ t->lcap = ncap;
+ }
+ memcpy (&t->lbuf[t->llen],
+ s,
+ len);
+ t->llen += len;
+}
+
+
+/**
+ * Append a code point to the current line, preceded by a space if one
+ * is pending.
+ *
+ * @param t state
+ * @param cp code point (not a LIG_* marker)
+ */
+static void
+line_cp (struct DviText *t,
+ uint32_t cp)
+{
+ char enc[4];
+ size_t n;
+
+ if ( (t->pending_space) &&
+ (t->llen > 0) )
+ line_bytes (t, " ", 1);
+ t->pending_space = 0;
+ if (cp < 0x80)
+ {
+ enc[0] = (char) cp;
+ n = 1;
+ }
+ else if (cp < 0x800)
+ {
+ enc[0] = (char) (0xC0 | (cp >> 6));
+ enc[1] = (char) (0x80 | (cp & 0x3F));
+ n = 2;
+ }
+ else
+ {
+ enc[0] = (char) (0xE0 | (cp >> 12));
+ enc[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ enc[2] = (char) (0x80 | (cp & 0x3F));
+ n = 3;
+ }
+ line_bytes (t, enc, n);
+}
+
+
+static void
+resolve_accent (struct DviText *t);
+
+
+/**
+ * Finish the current line (if any) and add it to the current page.
+ *
+ * @param t state
+ */
+static void
+flush_line (struct DviText *t)
+{
+ struct EXTRACTOR_FullTextPage *page;
+ struct EXTRACTOR_PageLineGeom *g;
+
+ if (! t->in_line)
+ return;
+ resolve_accent (t);
+ t->in_line = 0;
+ t->pending_space = 0;
+ t->last_cp = 0;
+ if ( (0 == t->llen) ||
+ (t->full) ||
+ (0 == t->num_pages) )
+ {
+ t->llen = 0;
+ return;
+ }
+ if ( (t->total_lines >= DVI_MAX_LINES) ||
+ (t->total_bytes + t->llen >= EXTRACTOR_FULLTEXT_MAX) )
+ {
+ t->full = 1;
+ t->llen = 0;
+ return;
+ }
+ page = &t->pages[t->num_pages - 1];
+ if (page->num_lines >= t->geom_cap)
+ {
+ unsigned int ncap = (0 == t->geom_cap) ? 32 : 2 * t->geom_cap;
+
+ g = realloc (t->geoms[t->num_pages - 1],
+ ncap * sizeof (*g));
+ if (NULL == g)
+ {
+ t->full = 1;
+ t->llen = 0;
+ return;
+ }
+ t->geoms[t->num_pages - 1] = g;
+ t->geom_cap = ncap;
+ }
+ g = &t->geoms[t->num_pages - 1][page->num_lines];
+ if (0 != EXTRACTOR_fulltext_page_add_ (page,
+ t->lbuf,
+ t->llen,
+ -1.0,
+ 0))
+ {
+ t->full = 1;
+ t->llen = 0;
+ return;
+ }
+ g->x0 = t->line_x0;
+ g->x1 = t->line_x1;
+ g->y = (double) t->line_v;
+ g->size = t->line_size;
+ if ( (! t->have_v) ||
+ (t->line_v < t->vmin) )
+ t->vmin = t->line_v;
+ if ( (! t->have_v) ||
+ (t->line_v > t->vmax) )
+ t->vmax = t->line_v;
+ t->have_v = 1;
+ t->total_lines++;
+ t->total_bytes += t->llen;
+ t->llen = 0;
+}
+
+
+/**
+ * Attach a pending accent to the preceding base character if it was
+ * placed over it; otherwise drop it.
+ *
+ * @param t state
+ */
+static void
+resolve_accent (struct DviText *t)
+{
+ uint32_t comp;
+ int pending_space;
+
+ if (0 == t->pending_accent)
+ return;
+ if ( (t->accent_post_ok) &&
+ (0 != t->last_cp) &&
+ (t->last_cp_off <= t->llen) )
+ {
+ comp = EXTRACTOR_pagelayout_compose_ (t->last_cp,
+ t->pending_accent);
+ pending_space = t->pending_space;
+ t->pending_space = 0;
+ if (0 != comp)
+ {
+ t->llen = t->last_cp_off;
+ line_cp (t, comp);
+ }
+ else
+ {
+ line_cp (t, t->pending_accent);
+ }
+ t->pending_space = pending_space;
+ t->last_cp = 0;
+ }
+ t->pending_accent = 0;
+ t->accent_post_ok = 0;
+}
+
+
+/**
+ * Typeset glyph @a c of @a font at (@a h, @a v).
+ *
+ * @param t state
+ * @param font the font, NULL if undefined
+ * @param c glyph code
+ * @param h horizontal position
+ * @param v vertical position
+ * @return estimated width of the glyph
+ */
+static double
+put_glyph (struct DviText *t,
+ const struct DviFont *font,
+ uint32_t c,
+ int64_t h,
+ int64_t v)
+{
+ enum DviEncoding enc = (NULL == font) ? ENC_OT1 : font->enc;
+ double size = (NULL == font) ? DVI_DEFAULT_SIZE : font->size;
+ double width = size * dvi_glyph_width (enc, c) / 1000.0;
+ double hd = (double) h;
+ uint32_t cp;
+ double gap;
+
+ cp = dvi_map_glyph (enc, c);
+ if (0 == cp)
+ return width;
+ if (t->in_line)
+ {
+ double big = (size > t->line_size) ? size : t->line_size;
+ double dv = (double) v - (double) t->line_v;
+
+ if (fabs (dv) > 0.7 * big)
+ flush_line (t);
+ }
+ if (! t->in_line)
+ {
+ t->in_line = 1;
+ t->line_v = v;
+ t->line_x0 = hd;
+ t->line_x1 = hd;
+ t->line_size = size;
+ t->last_end = hd;
+ t->pending_space = 0;
+ t->pending_accent = 0;
+ t->accent_post_ok = 0;
+ t->last_cp = 0;
+ }
+ gap = hd - t->last_end;
+ if (gap > 0.17 * size)
+ t->pending_space = 1;
+ if (hd + width > t->line_x1)
+ t->line_x1 = hd + width;
+ t->last_end = hd + width;
+ if (is_accent (cp))
+ {
+ resolve_accent (t);
+ t->pending_accent = cp;
+ /* the accent overlaps the preceding base character: it belongs to
+ that one unless the next glyph is set under it, too */
+ t->accent_post_ok = ( (0 != t->last_cp) &&
+ (gap < -0.1 * size) &&
+ (! t->pending_space) ) ? 1 : 0;
+ return width;
+ }
+ if ( (0 != t->pending_accent) &&
+ (gap >= 0) )
+ resolve_accent (t);
+ if (cp < 0x20)
+ {
+ static const char *const ligs[] = {
+ "", "ff", "fi", "fl", "ffi", "ffl", "SS"
+ };
+ const char *l = ligs[cp];
+
+ t->pending_accent = 0;
+ t->accent_post_ok = 0;
+ while ('\0' != *l)
+ {
+ line_cp (t, (unsigned char) *l);
+ l++;
+ }
+ t->last_cp = 0;
+ return width;
+ }
+ if (0 != t->pending_accent)
+ {
+ uint32_t comp = EXTRACTOR_pagelayout_compose_ (cp,
+ t->pending_accent);
+
+ if (0 != comp)
+ {
+ line_cp (t, comp);
+ }
+ else
+ {
+ line_cp (t, cp);
+ line_cp (t, t->pending_accent);
+ }
+ t->pending_accent = 0;
+ t->accent_post_ok = 0;
+ t->last_cp = 0;
+ return width;
+ }
+ {
+ size_t before = t->llen;
+
+ line_cp (t, cp);
+ if (t->llen > before)
+ {
+ t->last_cp_off = t->llen - ( (cp < 0x80) ? 1 : (cp < 0x800) ? 2 : 3);
+ t->last_cp = cp;
+ }
+ else
+ {
+ t->last_cp = 0;
+ }
+ }
+ return width;
+}
+
+
+/**
+ * Read an unsigned big-endian integer of @a n bytes.
+ *
+ * @param t state
+ * @param pos position to read at, updated
+ * @param n number of bytes (1-4)
+ * @param[out] ret value read
+ * @return 0 on success, -1 if out of bounds
+ */
+static int
+read_u (const struct DviText *t,
+ size_t *pos,
+ unsigned int n,
+ uint32_t *ret)
+{
+ uint32_t r = 0;
+ unsigned int i;
+
+ if ( (*pos > t->size) ||
+ (t->size - *pos < n) )
+ return -1;
+ for (i = 0; i < n; i++)
+ r = (r << 8) | t->data[*pos + i];
+ *pos += n;
+ *ret = r;
+ return 0;
+}
+
+
+/**
+ * Read a signed big-endian integer of @a n bytes.
+ *
+ * @param t state
+ * @param pos position to read at, updated
+ * @param n number of bytes (1-4)
+ * @param[out] ret value read
+ * @return 0 on success, -1 if out of bounds
+ */
+static int
+read_s (const struct DviText *t,
+ size_t *pos,
+ unsigned int n,
+ int64_t *ret)
+{
+ uint32_t u;
+
+ if (0 != read_u (t, pos, n, &u))
+ return -1;
+ if ( (n < 4) &&
+ (u & (1U << (8 * n - 1))) )
+ *ret = (int64_t) u - ((int64_t) 1 << (8 * n));
+ else if (4 == n)
+ *ret = (int64_t) (int32_t) u;
+ else
+ *ret = (int64_t) u;
+ return 0;
+}
+
+
+/**
+ * Hash a font number.
+ *
+ * @param k font number
+ * @return initial slot in the font hash table
+ */
+static unsigned int
+font_slot (uint32_t k)
+{
+ return (unsigned int) ((k * 2654435761U) >> 21) & (DVI_FONT_HASH - 1);
+}
+
+
+/**
+ * Look up font @a k.
+ *
+ * @param t state
+ * @param k font number
+ * @return the font, NULL if undefined
+ */
+static const struct DviFont *
+find_font (const struct DviText *t,
+ uint32_t k)
+{
+ unsigned int h = font_slot (k);
+
+ while (0 != t->font_hash[h])
+ {
+ const struct DviFont *f = &t->fonts[t->font_hash[h] - 1];
+
+ if (f->k == k)
+ return f;
+ h = (h + 1) & (DVI_FONT_HASH - 1);
+ }
+ return NULL;
+}
+
+
+/**
+ * Parse a fnt_def command (opcode already consumed) and register the
+ * font.
+ *
+ * @param t state
+ * @param pos position after the opcode, updated
+ * @param n number of bytes of the font number (1-4)
+ * @return 0 on success, -1 on malformed input
+ */
+static int
+parse_fnt_def (struct DviText *t,
+ size_t *pos,
+ unsigned int n)
+{
+ uint32_t k;
+ uint32_t csum;
+ int64_t s;
+ uint32_t d;
+ uint32_t a;
+ uint32_t l;
+ unsigned int i;
+
+ if ( (0 != read_u (t, pos, n, &k)) ||
+ (0 != read_u (t, pos, 4, &csum)) ||
+ (0 != read_s (t, pos, 4, &s)) ||
+ (0 != read_u (t, pos, 4, &d)) ||
+ (0 != read_u (t, pos, 1, &a)) ||
+ (0 != read_u (t, pos, 1, &l)) )
+ return -1;
+ if (t->size - *pos < a + l)
+ return -1;
+ if ( (NULL == find_font (t, k)) &&
+ (t->num_fonts < DVI_MAX_FONTS) )
+ {
+ unsigned int h = font_slot (k);
+
+ while (0 != t->font_hash[h])
+ h = (h + 1) & (DVI_FONT_HASH - 1);
+ t->font_hash[h] = (uint16_t) (t->num_fonts + 1);
+ struct DviFont *f = &t->fonts[t->num_fonts++];
+
+ f->k = k;
+ if (s < 0)
+ s = -s;
+ if ( (0 == s) ||
+ (s > (1 << 30)) )
+ s = DVI_DEFAULT_SIZE;
+ f->size = (double) s;
+ f->enc = dvi_font_encoding ((const char *) &t->data[*pos + a],
+ l);
+ }
+ *pos += a + l;
+ (void) csum;
+ (void) d;
+ return 0;
+}
+
+
+/**
+ * Interpret the page starting with the bop at @a bop and collect its
+ * text into a new page.
+ *
+ * @param t state
+ * @param bop offset of the page's bop command
+ */
+static void
+interpret_page (struct DviText *t,
+ size_t bop)
+{
+ struct DviRegs r;
+ struct DviRegs *stack;
+ unsigned int depth;
+ const struct DviFont *font;
+ size_t pos;
+ struct EXTRACTOR_FullTextPage *np;
+ struct EXTRACTOR_PageLineGeom **ng;
+
+ np = realloc (t->pages,
+ (t->num_pages + 1) * sizeof (*np));
+ if (NULL == np)
+ {
+ t->full = 1;
+ return;
+ }
+ t->pages = np;
+ ng = realloc (t->geoms,
+ (t->num_pages + 1) * sizeof (*ng));
+ if (NULL == ng)
+ {
+ t->full = 1;
+ return;
+ }
+ t->geoms = ng;
+ memset (&t->pages[t->num_pages], 0, sizeof (*np));
+ t->geoms[t->num_pages] = NULL;
+ t->num_pages++;
+ t->geom_cap = 0;
+ stack = malloc (DVI_MAX_STACK * sizeof (*stack));
+ if (NULL == stack)
+ {
+ t->full = 1;
+ return;
+ }
+ memset (&r, 0, sizeof (r));
+ depth = 0;
+ font = NULL;
+ pos = bop + 45;
+ while ( (pos < t->size) &&
+ (! t->full) )
+ {
+ unsigned char op = t->data[pos++];
+ uint32_t u;
+ int64_t s;
+ int64_t s2;
+
+ if (0 == t->budget)
+ {
+ t->full = 1;
+ break;
+ }
+ t->budget--;
+ if (0 == (t->budget & 0xFFFF))
+ (void) t->ec->seek (t->ec->cls,
+ 0,
+ SEEK_CUR); /* keep the host from giving up */
+ if (op < 128)
+ {
+ r.h += (int64_t) put_glyph (t, font, op, r.h, r.v);
+ continue;
+ }
+ if ( (op >= 171) &&
+ (op <= 234) )
+ {
+ font = find_font (t, op - 171);
+ continue;
+ }
+ switch (op)
+ {
+ case 128: case 129: case 130: case 131: /* set1..4 */
+ if (0 != read_u (t, &pos, op - 127, &u))
+ goto DONE;
+ r.h += (int64_t) put_glyph (t, font, u, r.h, r.v);
+ break;
+ case 133: case 134: case 135: case 136: /* put1..4 */
+ if (0 != read_u (t, &pos, op - 132, &u))
+ goto DONE;
+ (void) put_glyph (t, font, u, r.h, r.v);
+ break;
+ case 132: /* set_rule */
+ if ( (0 != read_s (t, &pos, 4, &s)) ||
+ (0 != read_s (t, &pos, 4, &s2)) )
+ goto DONE;
+ r.h += s2;
+ break;
+ case 137: /* put_rule */
+ if ( (0 != read_s (t, &pos, 4, &s)) ||
+ (0 != read_s (t, &pos, 4, &s2)) )
+ goto DONE;
+ break;
+ case 138: /* nop */
+ break;
+ case 141: /* push */
+ if (depth >= DVI_MAX_STACK)
+ goto DONE;
+ stack[depth++] = r;
+ break;
+ case 142: /* pop */
+ if (0 == depth)
+ goto DONE;
+ r = stack[--depth];
+ break;
+ case 143: case 144: case 145: case 146: /* right1..4 */
+ if (0 != read_s (t, &pos, op - 142, &s))
+ goto DONE;
+ r.h += s;
+ break;
+ case 147: /* w0 */
+ r.h += r.w;
+ break;
+ case 148: case 149: case 150: case 151: /* w1..4 */
+ if (0 != read_s (t, &pos, op - 147, &s))
+ goto DONE;
+ r.w = s;
+ r.h += s;
+ break;
+ case 152: /* x0 */
+ r.h += r.x;
+ break;
+ case 153: case 154: case 155: case 156: /* x1..4 */
+ if (0 != read_s (t, &pos, op - 152, &s))
+ goto DONE;
+ r.x = s;
+ r.h += s;
+ break;
+ case 157: case 158: case 159: case 160: /* down1..4 */
+ if (0 != read_s (t, &pos, op - 156, &s))
+ goto DONE;
+ r.v += s;
+ break;
+ case 161: /* y0 */
+ r.v += r.y;
+ break;
+ case 162: case 163: case 164: case 165: /* y1..4 */
+ if (0 != read_s (t, &pos, op - 161, &s))
+ goto DONE;
+ r.y = s;
+ r.v += s;
+ break;
+ case 166: /* z0 */
+ r.v += r.z;
+ break;
+ case 167: case 168: case 169: case 170: /* z1..4 */
+ if (0 != read_s (t, &pos, op - 166, &s))
+ goto DONE;
+ r.z = s;
+ r.v += s;
+ break;
+ case 235: case 236: case 237: case 238: /* fnt1..4 */
+ if (0 != read_u (t, &pos, op - 234, &u))
+ goto DONE;
+ font = find_font (t, u);
+ break;
+ case 239: case 240: case 241: case 242: /* xxx1..4 */
+ if (0 != read_u (t, &pos, op - 238, &u))
+ goto DONE;
+ if (t->size - pos < u)
+ goto DONE;
+ pos += u;
+ break;
+ case 243: case 244: case 245: case 246: /* fnt_def1..4 */
+ if (0 != parse_fnt_def (t, &pos, op - 242))
+ goto DONE;
+ break;
+ case 140: /* eop */
+ default: /* bop, pre, post, post_post, undefined */
+ goto DONE;
+ }
+ /* keep the registers far away from overflowing */
+ if ( (r.h > INT64_C (1) << 50) || (r.h < -(INT64_C (1) << 50)) ||
+ (r.v > INT64_C (1) << 50) || (r.v < -(INT64_C (1) << 50)) )
+ goto DONE;
+ }
+DONE:
+ flush_line (t);
+ free (stack);
+}
+
+
+/**
+ * Extract the full text of the DVI file and emit it.
+ *
+ * @param ec extraction context
+ * @param data the DVI file
+ * @param size number of bytes in @a data
+ * @param post offset of the postamble
+ * @param page_count number of pages
+ * @return 0 to continue, 1 if the callback asked us to stop
+ */
+static int
+dvi_fulltext (struct EXTRACTOR_ExtractContext *ec,
+ const unsigned char *data,
+ size_t size,
+ size_t post,
+ unsigned int page_count)
+{
+ struct DviText *t;
+ size_t *bops;
+ size_t pos;
+ unsigned int n;
+ unsigned int i;
+ int64_t l = 0;
+ double top;
+ double bottom;
+ int ret = 0;
+
+ if ( (0 == page_count) ||
+ (post > size) ||
+ (size - post < 29) )
+ return 0;
+ t = calloc (1, sizeof (*t));
+ if (NULL == t)
+ return 0;
+ t->ec = ec;
+ t->data = data;
+ t->size = size;
+ t->budget = 4 * (uint64_t) size + 1000000;
+ /* fonts from the postamble */
+ pos = post + 29;
+ while (pos < size)
+ {
+ unsigned char op = data[pos++];
+
+ if (138 == op)
+ continue;
+ if ( (op < 243) ||
+ (op > 246) ||
+ (0 != parse_fnt_def (t, &pos, op - 242)) )
+ break;
+ }
+ pos = post + 17;
+ (void) read_s (t, &pos, 4, &l);
+ /* list of pages, in order */
+ if (page_count > DVI_MAX_PAGES)
+ page_count = DVI_MAX_PAGES;
+ bops = malloc (page_count * sizeof (size_t));
+ if (NULL == bops)
+ {
+ free (t);
+ return 0;
+ }
+ {
+ uint32_t p;
+ uint32_t prev;
+ unsigned int total;
+
+ total = 0;
+ p = getIntAt (&data[post + 1]);
+ prev = UINT32_MAX;
+ while ( (UINT32_MAX != p) &&
+ ((size_t) p + 45 <= size) &&
+ (p < prev) &&
+ (139 == data[p]) )
+ {
+ total++;
+ prev = p;
+ p = getIntAt (&data[p + 41]);
+ }
+ n = (total < page_count) ? total : page_count;
+ p = getIntAt (&data[post + 1]);
+ prev = UINT32_MAX;
+ while ( (total > 0) &&
+ (UINT32_MAX != p) &&
+ ((size_t) p + 45 <= size) &&
+ (p < prev) &&
+ (139 == data[p]) )
+ {
+ total--;
+ if (total < n)
+ bops[total] = p;
+ prev = p;
+ p = getIntAt (&data[p + 41]);
+ }
+ }
+ for (i = 0; (i < n) && (! t->full); i++)
+ interpret_page (t, bops[i]);
+ free (bops);
+ /* relative vertical positions */
+ top = 0;
+ bottom = (l > 0) ? (double) l : 0;
+ if (t->have_v)
+ {
+ if ((double) t->vmin < top)
+ top = (double) t->vmin;
+ if ((double) t->vmax > bottom)
+ bottom = (double) t->vmax;
+ }
+ for (i = 0; i < t->num_pages; i++)
+ {
+ unsigned int j;
+
+ for (j = 0; j < t->pages[i].num_lines; j++)
+ {
+ double y = -1;
+
+ if (bottom > top)
+ y = (t->geoms[i][j].y - top) / (bottom - top);
+ if (y > 1)
+ y = 1;
+ t->pages[i].lines[j].y = y;
+ }
+ }
+ if (t->num_pages > 0)
+ {
+ struct EXTRACTOR_FullText ft;
+
+ EXTRACTOR_fulltext_filter_margins_ (t->pages,
+ t->num_pages);
+ EXTRACTOR_pagelayout_paragraphs_ (t->pages,
+ t->geoms,
+ t->num_pages,
+ 0.06);
+ EXTRACTOR_fulltext_init_ (&ft);
+ EXTRACTOR_fulltext_append_pages_ (&ft,
+ t->pages,
+ t->num_pages);
+ ret = EXTRACTOR_fulltext_emit_ (&ft,
+ ec,
+ "dvi");
+ EXTRACTOR_fulltext_clear_ (&ft);
+ }
+ for (i = 0; i < t->num_pages; i++)
+ {
+ EXTRACTOR_fulltext_page_clear_ (&t->pages[i]);
+ free (t->geoms[i]);
+ }
+ free (t->pages);
+ free (t->geoms);
+ free (t->lbuf);
+ free (t);
+ return (0 != ret) ? 1 : 0;
+}
+
+
/**
* Main entry method for the 'application/x-dvi' extraction plugin.
*
@@ -174,6 +1641,7 @@ EXTRACTOR_dvi_extract_method (struct EXTRACTOR_ExtractContext *ec)
uint64_t size;
uint64_t off;
ssize_t iret;
+ uint32_t post;
if (40 >= (iret = ec->read (ec->cls, &buf, 1024)))
return;
@@ -221,6 +1689,7 @@ EXTRACTOR_dvi_extract_method (struct EXTRACTOR_ExtractContext *ec)
/* assert pos at 'post' command */
if (data[pos] != 248)
goto CLEANUP;
+ post = pos;
pageCount = 0;
opos = pos;
pos = getIntAt (&data[opos + 1]);
@@ -320,9 +1789,16 @@ EXTRACTOR_dvi_extract_method (struct EXTRACTOR_ExtractContext *ec)
pos += len + 5;
break;
default: /* unsupported opcode, abort scan */
- goto CLEANUP;
+ goto FULLTEXT;
}
}
+FULLTEXT:
+ if (EXTRACTOR_fulltext_requested_ (ec->config))
+ (void) dvi_fulltext (ec,
+ data,
+ (size_t) size,
+ post,
+ pageCount);
CLEANUP:
free (data);
}
diff --git a/src/plugins/ebook_extractor.c b/src/plugins/ebook_extractor.c
@@ -29,6 +29,23 @@
* is worth having here is not the title but the provenance: which tool
* built the file, when, for whom.
*
+ * With the "fulltext" option the book's text is reported as one
+ * #EXTRACTOR_METATYPE_FULL_TEXT item:
+ * - EPUB: the linear entries of the OPF spine, in spine order (not
+ * manifest or archive order); each (X)HTML content document is
+ * converted by a tolerant markup scanner that drops head, script,
+ * style, nav, aside, page-level header/footer, hidden elements, ARIA
+ * landmark roles for the page margins and page break markers
+ * (epub:type="pagebreak"); block elements, <br><br> and chapter
+ * boundaries end paragraphs. Documents listed in encryption.xml
+ * (DRM) are skipped.
+ * - MOBI/AZW: the text records, uncompressed or PalmDOC compressed,
+ * with the trailing entries stripped, run through the same scanner;
+ * <mbp:pagebreak/> ends a paragraph. Encrypted books and HUFF/CDIC
+ * compression (used by some Amazon-built files) are not supported.
+ * For KF8 (AZW3) the raw markup is scanned in storage order, which
+ * is the reading order of the text.
+ *
* References:
* - EPUB 3 OCF and Package Document, https://www.w3.org/TR/epub-33/
* - MOBI format, https://wiki.mobileread.com/wiki/MOBI
@@ -37,6 +54,7 @@
#include "platform.h"
#include "extractor.h"
#include "forensics.h"
+#include "fulltext.h"
#include "unzip.h"
@@ -439,20 +457,18 @@ looks_like_isbn (const char *s)
/**
- * Read a whole member out of a ZIP archive.
+ * Read the current member of a ZIP archive.
*
- * @param uf the archive
- * @param name member to read
+ * @param uf the archive, positioned on the member
* @param cap never read more than this many bytes
* @param[out] len number of bytes read
* @return the member's contents with a NUL appended, NULL on error;
* caller must free
*/
static char *
-read_member (struct EXTRACTOR_UnzipFile *uf,
- const char *name,
- size_t cap,
- size_t *len)
+read_current (struct EXTRACTOR_UnzipFile *uf,
+ size_t cap,
+ size_t *len)
{
struct EXTRACTOR_UnzipFileInfo fi;
char *buf;
@@ -461,11 +477,6 @@ read_member (struct EXTRACTOR_UnzipFile *uf,
*len = 0;
if (EXTRACTOR_UNZIP_OK !=
- EXTRACTOR_common_unzip_go_find_local_file (uf,
- name,
- 2))
- return NULL;
- if (EXTRACTOR_UNZIP_OK !=
EXTRACTOR_common_unzip_get_current_file_info (uf,
&fi,
NULL, 0,
@@ -511,6 +522,34 @@ read_member (struct EXTRACTOR_UnzipFile *uf,
/**
+ * Read a whole member out of a ZIP archive.
+ *
+ * @param uf the archive
+ * @param name member to read
+ * @param cap never read more than this many bytes
+ * @param[out] len number of bytes read
+ * @return the member's contents with a NUL appended, NULL on error;
+ * caller must free
+ */
+static char *
+read_member (struct EXTRACTOR_UnzipFile *uf,
+ const char *name,
+ size_t cap,
+ size_t *len)
+{
+ *len = 0;
+ if (EXTRACTOR_UNZIP_OK !=
+ EXTRACTOR_common_unzip_go_find_local_file (uf,
+ name,
+ 2))
+ return NULL;
+ return read_current (uf,
+ cap,
+ len);
+}
+
+
+/**
* Does the archive contain a member with this name?
*
* @param uf the archive
@@ -799,201 +838,2059 @@ parse_opf_metadata (struct EXTRACTOR_ExtractContext *ec,
}
+/* ------------------------------------------------------------------ */
+/* Full text: character sets */
+/* ------------------------------------------------------------------ */
+
/**
- * Handle an EPUB: read `META-INF/container.xml' to find the OPF package
- * document, then report what the package document holds.
+ * Never inflate more than this from one EPUB content document.
+ */
+#define MAX_CHAPTER (8 * 1024 * 1024)
+
+/**
+ * Never inflate more than this from all EPUB content documents
+ * together, nor decompress more than this from the text records of a
+ * MOBI file. The full text itself is capped at #EXTRACTOR_FULLTEXT_MAX,
+ * and markup rarely more than triples the size of the text.
+ */
+#define MAX_BOOK_MARKUP (64 * 1024 * 1024)
+
+/**
+ * Upper bound on the number of manifest items we keep.
+ */
+#define MAX_MANIFEST 4096
+
+/**
+ * Upper bound on the number of spine entries we follow.
+ */
+#define MAX_SPINE 4096
+
+/**
+ * Upper bound on the size of one MOBI text record (normally 4096 bytes
+ * of text plus a few trailing bytes).
+ */
+#define MAX_TEXT_RECORD (64 * 1024)
+
+/**
+ * Upper bound on what one MOBI text record may decompress to.
+ */
+#define MAX_TEXT_RECORD_OUT (64 * 1024)
+
+/**
+ * How many attributes of one element we look at.
+ */
+#define MAX_ATTRS 64
+
+
+/**
+ * Code points of windows-1252 bytes 0x80 to 0x9F.
+ */
+static const uint16_t cp1252[32] = {
+ 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
+ 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
+ 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
+ 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178
+};
+
+
+/**
+ * Convert windows-1252 (a superset of ISO-8859-1 in the printable
+ * range) to UTF-8.
*
- * @param ec extraction context
- * @param uf the opened archive
+ * @param data input
+ * @param len number of bytes in @a data
+ * @param[out] olen number of bytes in the result
+ * @return converted text, NULL on allocation failure; caller frees
*/
-static void
-extract_epub (struct EXTRACTOR_ExtractContext *ec,
- struct EXTRACTOR_UnzipFile *uf)
+static char *
+cp1252_to_utf8 (const char *data,
+ size_t len,
+ size_t *olen)
{
- char path[512];
- char attr[64];
- char *container;
- char *opf = NULL;
- size_t container_len;
- size_t opf_len = 0;
- const char *p;
- const char *q;
- unsigned int items = 0;
+ char *out;
+ size_t o = 0;
- if (0 !=
- ec->proc (ec->cls,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_MIMETYPE,
- EXTRACTOR_METAFORMAT_UTF8,
- "text/plain",
- "application/epub+zip",
- strlen ("application/epub+zip") + 1))
- return;
- container = read_member (uf,
- "META-INF/container.xml",
- MAX_CONTAINER,
- &container_len);
- if (NULL != container)
+ if (NULL == (out = malloc (3 * len + 1)))
+ return NULL;
+ for (size_t i = 0; i < len; i++)
{
- p = mem_find (container,
- container_len,
- "full-path=",
- strlen ("full-path="));
- if (NULL != p)
- {
- p += strlen ("full-path=");
- if ( ('"' == *p) || ('\'' == *p) )
- {
- char quote = *p;
+ unsigned char c = (unsigned char) data[i];
- p++;
- q = memchr (p,
- quote,
- container_len - (size_t) (p - container));
- if (NULL != q)
- (void) xml_text (p,
- (size_t) (q - p),
- path,
- sizeof (path));
- else
- path[0] = '\0';
- }
- else
- {
- path[0] = '\0';
- }
- if ('\0' != path[0])
- opf = read_member (uf,
- path,
- MAX_OPF,
- &opf_len);
- }
- free (container);
+ if (c < 0x80)
+ out[o++] = (char) c;
+ else if (c < 0xA0)
+ o = utf8_put (out, 3 * len, o, cp1252[c - 0x80]);
+ else
+ o = utf8_put (out, 3 * len, o, c);
}
- if (NULL == opf)
- {
- /* no usable container.xml; the conventional locations are worth a
- try before giving up */
- static const char *guesses[] = {
- "OEBPS/content.opf",
- "content.opf",
- "OPS/content.opf",
- NULL
- };
+ *olen = o;
+ return out;
+}
- for (unsigned int i = 0; NULL != guesses[i]; i++)
- {
- opf = read_member (uf,
- guesses[i],
- MAX_OPF,
- &opf_len);
- if (NULL != opf)
- break;
- }
- }
- if (NULL != opf)
+
+/**
+ * Convert UTF-16 to UTF-8. Unpaired surrogates become U+FFFD.
+ *
+ * @param data input
+ * @param len number of bytes in @a data
+ * @param big_endian 1 for UTF-16BE, 0 for UTF-16LE
+ * @param[out] olen number of bytes in the result
+ * @return converted text, NULL on allocation failure; caller frees
+ */
+static char *
+utf16_to_utf8 (const char *data,
+ size_t len,
+ int big_endian,
+ size_t *olen)
+{
+ const unsigned char *u = (const unsigned char *) data;
+ char *out;
+ size_t o = 0;
+ size_t cap = 2 * len + 4; /* 2 bytes in never make more than 3 out */
+
+ if (NULL == (out = malloc (cap)))
+ return NULL;
+ for (size_t i = 0; i + 1 < len; i += 2)
{
- /* the <package version="..."> attribute distinguishes EPUB 2 from
- EPUB 3, which changes what else we may expect to find */
- p = mem_find (opf,
- opf_len,
- "<package",
- strlen ("<package"));
- if (NULL != p)
+ uint32_t cp = big_endian
+ ? (uint32_t) ((u[i] << 8) | u[i + 1])
+ : (uint32_t) ((u[i + 1] << 8) | u[i]);
+
+ if ( (cp >= 0xD800) && (cp <= 0xDBFF) && (i + 3 < len) )
{
- size_t tail = opf_len - (size_t) (p - opf);
- const char *gt;
+ uint32_t lo = big_endian
+ ? (uint32_t) ((u[i + 2] << 8) | u[i + 3])
+ : (uint32_t) ((u[i + 3] << 8) | u[i + 2]);
- if (tail > 1024)
- tail = 1024;
- /* stop at the end of the start tag, so that a `version=' further
- down the document cannot be mistaken for this one */
- gt = memchr (p,
- '>',
- tail);
- if (NULL != gt)
- tail = (size_t) (gt - p);
- if (tail <= strlen ("<package"))
- tail = strlen ("<package");
- if (xml_attr (p + strlen ("<package"),
- tail - strlen ("<package"),
- "version",
- attr,
- sizeof (attr)))
+ if ( (lo >= 0xDC00) && (lo <= 0xDFFF) )
{
- if (0 !=
- EXTRACTOR_forensic_emit_text_ (ec,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_FORMAT_VERSION,
- attr,
- strlen (attr)))
- goto CLEANUP;
+ cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00);
+ i += 2;
}
}
- if (0 !=
- parse_opf_metadata (ec,
- opf,
- opf_len))
- goto CLEANUP;
- /* count the manifest entries; `<itemref>' does not match because
- the character after `<item' has to be a name delimiter */
- for (size_t i = 0; i + strlen ("<item") < opf_len; i++)
- if ( (0 == memcmp (&opf[i],
- "<item",
- strlen ("<item"))) &&
- (! is_name_char (opf[i + strlen ("<item")])) )
- items++;
- if (0 != items)
- {
- if (0 !=
- EXTRACTOR_forensic_emit_ (ec,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_ENTRY_COUNT,
- "%u",
- items))
- goto CLEANUP;
- }
- }
- if (has_member (uf,
- "META-INF/encryption.xml"))
- {
- if (0 !=
- EXTRACTOR_forensic_emit_ (
- ec,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_ENCRYPTION,
- "META-INF/encryption.xml present"
- " (DRM or font obfuscation)"))
- goto CLEANUP;
- }
- if (has_member (uf,
- "META-INF/rights.xml"))
- {
- if (0 !=
- EXTRACTOR_forensic_emit_ (ec,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_ENCRYPTION,
- "META-INF/rights.xml present"
- " (Adobe Content Server DRM)"))
- goto CLEANUP;
+ if ( (cp >= 0xD800) && (cp <= 0xDFFF) )
+ cp = 0xFFFD;
+ o = utf8_put (out, cap, o, cp);
}
-CLEANUP:
- free (opf);
+ *olen = o;
+ return out;
}
/**
- * Convert a Palm database time stamp.
- *
- * The format says seconds since 1904, but writers disagree and some use
- * the Unix epoch. The two ranges do not overlap for any date between
- * 1970 and 2036, so the value itself says which convention was used.
+ * Convert @a data from the named character set to UTF-8. UTF-8 input
+ * is copied as is (invalid sequences are dealt with when the text is
+ * appended to the full text); UTF-16 and windows-1252 are converted
+ * here, anything else by iconv if we have it.
*
- * @param raw the 32 bit field
- * @return seconds since 1970, 0 if the field was never filled in
+ * @param data input
+ * @param len number of bytes in @a data
+ * @param cs lower-case charset name
+ * @param[out] olen number of bytes in the result
+ * @return converted text, NULL on allocation failure; caller frees
*/
-static int64_t
-palm_time (uint32_t raw)
+static char *
+convert_to_utf8 (const char *data,
+ size_t len,
+ const char *cs,
+ size_t *olen)
+{
+ char *out;
+
+ *olen = 0;
+ if ( (0 == strcmp (cs, "iso-8859-1")) ||
+ (0 == strcmp (cs, "iso8859-1")) ||
+ (0 == strcmp (cs, "latin1")) ||
+ (0 == strcmp (cs, "us-ascii")) ||
+ (0 == strcmp (cs, "ascii")) ||
+ (0 == strcmp (cs, "cp1252")) ||
+ (0 == strcmp (cs, "windows-1252")) )
+ return cp1252_to_utf8 (data, len, olen);
+ if ( (0 == strcmp (cs, "utf-16be")) ||
+ (0 == strcmp (cs, "utf-16")) )
+ return utf16_to_utf8 (data, len, 1, olen);
+ if (0 == strcmp (cs, "utf-16le"))
+ return utf16_to_utf8 (data, len, 0, olen);
+#if HAVE_ICONV
+ if ( ('\0' != cs[0]) &&
+ (0 != strcmp (cs, "utf-8")) &&
+ (0 != strcmp (cs, "utf8")) )
+ {
+ iconv_t cd = iconv_open ("UTF-8", cs);
+
+ if ((iconv_t) -1 != cd)
+ {
+ char *ib = (char *) data;
+ size_t il = len;
+ char *ob;
+ size_t ol;
+
+ /* no charset produces more than 4 bytes of UTF-8 per byte */
+ if (NULL == (out = malloc (4 * len + 4)))
+ {
+ iconv_close (cd);
+ return NULL;
+ }
+ ob = out;
+ ol = 4 * len;
+ while (il > 0)
+ {
+ if ((size_t) -1 !=
+ iconv (cd, &ib, &il, &ob, &ol))
+ break;
+ if ( (E2BIG == errno) ||
+ (ol < 3) )
+ break;
+ /* skip one undecodable byte */
+ *ob++ = (char) 0xEF;
+ *ob++ = (char) 0xBF;
+ *ob++ = (char) 0xBD;
+ ol -= 3;
+ ib++;
+ il--;
+ }
+ iconv_close (cd);
+ *olen = (size_t) (ob - out);
+ return out;
+ }
+ }
+#endif
+ if (NULL == (out = malloc (len + 1)))
+ return NULL;
+ memcpy (out, data, len);
+ *olen = len;
+ return out;
+}
+
+
+/**
+ * Read a charset name after `encoding' or `charset' at @a p: optional
+ * white space, '=', optional white space and quotes, then the name.
+ *
+ * @param p where to start
+ * @param end end of the buffer
+ * @param[out] cs where to write the lower-case name
+ * @param cs_size number of bytes available in @a cs
+ * @return 1 if a name was found
+ */
+static int
+charset_value (const char *p,
+ const char *end,
+ char *cs,
+ size_t cs_size)
+{
+ size_t o = 0;
+
+ while ( (p < end) && is_space (*p))
+ p++;
+ if ( (p >= end) || ('=' != *p) )
+ return 0;
+ p++;
+ while ( (p < end) &&
+ (is_space (*p) || ('"' == *p) || ('\'' == *p) ) )
+ p++;
+ while ( (p < end) &&
+ (o + 1 < cs_size) &&
+ (isalnum ((unsigned char) *p) ||
+ ('-' == *p) || ('_' == *p) || ('.' == *p) || (':' == *p) ) )
+ cs[o++] = (char) tolower ((unsigned char) *p++);
+ cs[o] = '\0';
+ return (0 != o);
+}
+
+
+/**
+ * Convert an EPUB content document to UTF-8: honour a byte order mark,
+ * UTF-16 without one, and the encoding in the XML declaration or a
+ * META charset declaration; default to UTF-8.
+ *
+ * @param data the document
+ * @param len number of bytes in @a data
+ * @param[out] olen number of bytes in the result
+ * @return the document in UTF-8, NULL on allocation failure
+ */
+static char *
+xhtml_to_utf8 (const char *data,
+ size_t len,
+ size_t *olen)
+{
+ const unsigned char *u = (const unsigned char *) data;
+ size_t scan = (len > 1024) ? 1024 : len;
+ char cs[64];
+
+ if ( (len >= 3) && (0xEF == u[0]) && (0xBB == u[1]) && (0xBF == u[2]) )
+ return convert_to_utf8 (&data[3], len - 3, "utf-8", olen);
+ if ( (len >= 2) && (0xFE == u[0]) && (0xFF == u[1]) )
+ return convert_to_utf8 (&data[2], len - 2, "utf-16be", olen);
+ if ( (len >= 2) && (0xFF == u[0]) && (0xFE == u[1]) )
+ return convert_to_utf8 (&data[2], len - 2, "utf-16le", olen);
+ if ( (len >= 4) && (0 == u[0]) && ('<' == u[1]) && (0 == u[2]) )
+ return convert_to_utf8 (data, len, "utf-16be", olen);
+ if ( (len >= 4) && ('<' == u[0]) && (0 == u[1]) && (0 == u[3]) )
+ return convert_to_utf8 (data, len, "utf-16le", olen);
+ cs[0] = '\0';
+ if ( (scan > 5) && (0 == memcmp (data, "<?xml", 5)) )
+ {
+ const char *gt = memchr (data, '>', scan);
+ size_t dlen = (NULL == gt) ? scan : (size_t) (gt - data);
+ const char *e = mem_find (data, dlen, "encoding", strlen ("encoding"));
+
+ if (NULL != e)
+ (void) charset_value (e + strlen ("encoding"),
+ data + dlen,
+ cs,
+ sizeof (cs));
+ }
+ if ('\0' == cs[0])
+ {
+ const char *m = mem_find (data, scan, "charset", strlen ("charset"));
+
+ if (NULL != m)
+ (void) charset_value (m + strlen ("charset"),
+ data + scan,
+ cs,
+ sizeof (cs));
+ }
+ /* a document in UTF-8 that claims UTF-16 cannot be UTF-16: we would
+ have seen the zero bytes */
+ if (0 == strncmp (cs, "utf-16", 6))
+ cs[0] = '\0';
+ return convert_to_utf8 (data, len, cs, olen);
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Full text: (X)HTML markup */
+/* ------------------------------------------------------------------ */
+
+/**
+ * HTML named character references for U+00A0 to U+00FF, in order.
+ */
+static const char *const latin1_entities[96] = {
+ "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect",
+ "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr",
+ "deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot",
+ "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest",
+ "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil",
+ "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml",
+ "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times",
+ "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig",
+ "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil",
+ "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml",
+ "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide",
+ "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml"
+};
+
+
+/**
+ * Other HTML named character references common in books.
+ */
+static const struct
+{
+ /**
+ * Entity name.
+ */
+ const char *name;
+
+ /**
+ * Code point.
+ */
+ uint32_t cp;
+} other_entities[] = {
+ { "amp", '&' }, { "lt", '<' }, { "gt", '>' }, { "quot", '"' },
+ { "apos", '\'' }, { "OElig", 0x152 }, { "oelig", 0x153 },
+ { "Scaron", 0x160 }, { "scaron", 0x161 }, { "Yuml", 0x178 },
+ { "fnof", 0x192 }, { "circ", 0x2C6 }, { "tilde", 0x2DC },
+ { "ensp", 0x2002 }, { "emsp", 0x2003 }, { "thinsp", 0x2009 },
+ { "zwnj", 0x200C }, { "zwj", 0x200D }, { "lrm", 0x200E },
+ { "rlm", 0x200F }, { "ndash", 0x2013 }, { "mdash", 0x2014 },
+ { "lsquo", 0x2018 }, { "rsquo", 0x2019 }, { "sbquo", 0x201A },
+ { "ldquo", 0x201C }, { "rdquo", 0x201D }, { "bdquo", 0x201E },
+ { "dagger", 0x2020 }, { "Dagger", 0x2021 }, { "bull", 0x2022 },
+ { "hellip", 0x2026 }, { "permil", 0x2030 }, { "prime", 0x2032 },
+ { "Prime", 0x2033 }, { "lsaquo", 0x2039 }, { "rsaquo", 0x203A },
+ { "oline", 0x203E }, { "euro", 0x20AC }, { "trade", 0x2122 },
+ { "larr", 0x2190 }, { "rarr", 0x2192 }, { "minus", 0x2212 },
+ { NULL, 0 }
+};
+
+
+/**
+ * Decode the character reference at @a p (just after the '&').
+ *
+ * @param p text after the '&'
+ * @param len number of bytes available at @a p
+ * @param[out] cp the code point
+ * @return number of bytes consumed after the '&' (including the ';'),
+ * 0 if this is not a character reference we know
+ */
+static size_t
+decode_entity (const char *p,
+ size_t len,
+ uint32_t *cp)
+{
+ size_t n = 0;
+
+ if ( (len >= 2) && ('#' == p[0]) )
+ {
+ uint32_t v = 0;
+ int base = 10;
+ size_t k = 1;
+ size_t digits = 0;
+
+ if ( ('x' == p[1]) || ('X' == p[1]) )
+ {
+ base = 16;
+ k = 2;
+ }
+ for (; (k < len) && (digits < 8); k++, digits++)
+ {
+ unsigned int d;
+
+ if ( ('0' <= p[k]) && ('9' >= p[k]) )
+ d = (unsigned int) (p[k] - '0');
+ else if ( (16 == base) && ('a' <= p[k]) && ('f' >= p[k]) )
+ d = (unsigned int) (p[k] - 'a') + 10;
+ else if ( (16 == base) && ('A' <= p[k]) && ('F' >= p[k]) )
+ d = (unsigned int) (p[k] - 'A') + 10;
+ else
+ break;
+ v = v * (uint32_t) base + d;
+ }
+ if ( (0 == digits) ||
+ (k >= len) ||
+ (';' != p[k]) )
+ return 0;
+ *cp = (v > 0x10FFFF) ? 0xFFFD : v;
+ return k + 1;
+ }
+ while ( (n < len) &&
+ (n < 10) &&
+ isalnum ((unsigned char) p[n]) )
+ n++;
+ if ( (0 == n) ||
+ (n >= len) ||
+ (';' != p[n]) )
+ return 0;
+ for (unsigned int i = 0; i < 96; i++)
+ if ( (strlen (latin1_entities[i]) == n) &&
+ (0 == memcmp (latin1_entities[i], p, n)) )
+ {
+ *cp = 0xA0 + i;
+ return n + 1;
+ }
+ for (unsigned int i = 0; NULL != other_entities[i].name; i++)
+ if ( (strlen (other_entities[i].name) == n) &&
+ (0 == memcmp (other_entities[i].name, p, n)) )
+ {
+ *cp = other_entities[i].cp;
+ return n + 1;
+ }
+ return 0;
+}
+
+
+/**
+ * State of the conversion of one (X)HTML document to text.
+ */
+struct MarkupState
+{
+ /**
+ * Where the text goes.
+ */
+ struct EXTRACTOR_FullText *ft;
+
+ /**
+ * Local name of the element whose content we are skipping, "" if
+ * none.
+ */
+ char skip_name[32];
+
+ /**
+ * Nesting level of @e skip_name elements inside the skipped one.
+ */
+ unsigned int skip_nest;
+
+ /**
+ * Number of open sectioning elements (article, section, main, ...);
+ * a header or footer inside one is not page furniture.
+ */
+ unsigned int sections;
+
+ /**
+ * 1 if the last thing we saw was a <br>, so another one ends the
+ * paragraph.
+ */
+ int after_br;
+};
+
+
+/**
+ * Is @a name (lower case) in the 0-terminated list @a list?
+ *
+ * @param name element name
+ * @param list list of names
+ * @return 1 if found
+ */
+static int
+name_in (const char *name,
+ const char *const *list)
+{
+ for (unsigned int i = 0; NULL != list[i]; i++)
+ if (0 == strcmp (name, list[i]))
+ return 1;
+ return 0;
+}
+
+
+/**
+ * Elements that end a paragraph.
+ */
+static const char *const block_elements[] = {
+ "address", "article", "aside", "blockquote", "body", "caption",
+ "center", "dd", "details", "dialog", "dir", "div", "dl", "dt",
+ "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2",
+ "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "html", "legend",
+ "li", "listing", "main", "menu", "nav", "ol", "p", "pre", "section",
+ "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul",
+ "xmp", "title", "head", "noscript",
+ /* DTBook and FB2-style structure some EPUBs carry */
+ "level", "level1", "level2", "level3", "level4", "level5", "level6",
+ "hd", "h", "note", "annotation", "epigraph", "poem", "stanza", "v",
+ NULL
+};
+
+/**
+ * Elements whose content is never document text.
+ */
+static const char *const skip_elements[] = {
+ "head", "title", "script", "style", "noscript", "template", "nav",
+ "aside", "select", "datalist", "iframe", "object", "applet", "rp", "rt",
+ "guide", "svg", "math", NULL
+};
+
+/**
+ * Elements whose content is not markup (the end tag is searched for
+ * literally).
+ */
+static const char *const raw_text_elements[] = {
+ "script", "style", "xmp", NULL
+};
+
+/**
+ * HTML elements without an end tag; they must never open a skipped
+ * region.
+ */
+static const char *const void_elements[] = {
+ "area", "base", "br", "col", "embed", "hr", "img", "input", "link",
+ "meta", "param", "source", "track", "wbr", NULL
+};
+
+/**
+ * Elements that start a sectioning context.
+ */
+static const char *const sectioning_elements[] = {
+ "article", "section", "main", "aside", "nav", NULL
+};
+
+
+/**
+ * Does a start tag's attribute list mark the element as not part of
+ * the text: hidden, aria-hidden="true", a hiding inline style, an ARIA
+ * landmark role for the page margins, or a page break marker
+ * (epub:type="pagebreak", role="doc-pagebreak")?
+ *
+ * @param a attribute text of the start tag (after the name)
+ * @param len number of bytes in @a a
+ * @return 1 to skip the element
+ */
+static int
+attrs_skip (const char *a,
+ size_t len)
+{
+ static const char *const roles[] = {
+ "navigation", "banner", "contentinfo", "complementary", "search",
+ "doc-pagebreak", NULL
+ };
+ size_t i = 0;
+ unsigned int n = 0;
+
+ while ( (i < len) && (n++ < MAX_ATTRS) )
+ {
+ const char *name;
+ size_t nlen;
+ const char *val = NULL;
+ size_t vlen = 0;
+ char lname[32];
+ char lval[256];
+
+ while ( (i < len) && (is_space (a[i]) || ('/' == a[i]) ) )
+ i++;
+ name = &a[i];
+ while ( (i < len) && (! is_space (a[i])) &&
+ ('=' != a[i]) && ('/' != a[i]) )
+ i++;
+ nlen = (size_t) (&a[i] - name);
+ if (0 == nlen)
+ break;
+ while ( (i < len) && is_space (a[i]))
+ i++;
+ if ( (i < len) && ('=' == a[i]) )
+ {
+ i++;
+ while ( (i < len) && is_space (a[i]))
+ i++;
+ if ( (i < len) && ( ('"' == a[i]) || ('\'' == a[i]) ) )
+ {
+ const char *q = memchr (&a[i + 1], a[i], len - i - 1);
+
+ val = &a[i + 1];
+ vlen = (NULL == q) ? len - i - 1 : (size_t) (q - val);
+ i += vlen + 2;
+ }
+ else
+ {
+ val = &a[i];
+ while ( (i < len) && (! is_space (a[i])) )
+ i++;
+ vlen = (size_t) (&a[i] - val);
+ }
+ }
+ if (nlen >= sizeof (lname))
+ continue;
+ for (size_t k = 0; k < nlen; k++)
+ lname[k] = (char) tolower ((unsigned char) name[k]);
+ lname[nlen] = '\0';
+ {
+ size_t o = 0;
+
+ /* lower-case, and drop white space so that `display: none'
+ matches */
+ for (size_t k = 0; (k < vlen) && (o + 1 < sizeof (lval)); k++)
+ if (! is_space (val[k]))
+ lval[o++] = (char) tolower ((unsigned char) val[k]);
+ else if ( (0 != o) && (' ' != lval[o - 1]) )
+ lval[o++] = ' ';
+ lval[o] = '\0';
+ }
+ if (0 == strcmp (lname, "hidden"))
+ {
+ /* hidden="until-found" is collapsed content that find-in-page
+ reveals, i.e. part of the text */
+ if (0 != strcmp (lval, "until-found"))
+ return 1;
+ continue;
+ }
+ if (0 == strcmp (lname, "aria-hidden"))
+ {
+ if (0 == strcmp (lval, "true"))
+ return 1;
+ continue;
+ }
+ if (0 == strcmp (lname, "style"))
+ {
+ char css[256];
+ size_t o = 0;
+
+ for (size_t k = 0; '\0' != lval[k]; k++)
+ if (' ' != lval[k])
+ css[o++] = lval[k];
+ css[o] = '\0';
+ if ( (NULL != strstr (css, "display:none")) ||
+ (NULL != strstr (css, "visibility:hidden")) )
+ return 1;
+ continue;
+ }
+ if ( (0 == strcmp (lname, "role")) ||
+ (0 == strcmp (lname, "epub:type")) )
+ {
+ const char *p = lval;
+
+ while ('\0' != *p)
+ {
+ size_t tl = strcspn (p, " ");
+
+ if ( (tl == strlen ("pagebreak")) &&
+ (0 == memcmp (p, "pagebreak", tl)) )
+ return 1;
+ for (unsigned int r = 0; NULL != roles[r]; r++)
+ if ( (tl == strlen (roles[r])) &&
+ (0 == memcmp (p, roles[r], tl)) )
+ return 1;
+ p += tl;
+ while (' ' == *p)
+ p++;
+ }
+ }
+ }
+ return 0;
+}
+
+
+/**
+ * Handle a <br>: one is a space, two in a row end the paragraph.
+ *
+ * @param ms conversion state
+ */
+static void
+markup_br (struct MarkupState *ms)
+{
+ if (ms->after_br)
+ {
+ EXTRACTOR_fulltext_paragraph_ (ms->ft);
+ ms->after_br = 0;
+ }
+ else
+ {
+ EXTRACTOR_fulltext_space_ (ms->ft);
+ ms->after_br = 1;
+ }
+}
+
+
+/**
+ * End the current paragraph.
+ *
+ * @param ms conversion state
+ */
+static void
+markup_par (struct MarkupState *ms)
+{
+ EXTRACTOR_fulltext_paragraph_ (ms->ft);
+ ms->after_br = 0;
+}
+
+
+/**
+ * Append character data (between tags) to the text.
+ *
+ * @param ms conversion state
+ * @param p the text, may contain character references
+ * @param len number of bytes at @a p
+ */
+static void
+markup_chars (struct MarkupState *ms,
+ const char *p,
+ size_t len)
+{
+ size_t i = 0;
+
+ while (i < len)
+ {
+ size_t run = i;
+ uint32_t cp;
+ size_t used;
+
+ while ( (run < len) && ('&' != p[run]) )
+ {
+ if (ms->after_br && (! is_space (p[run])) )
+ ms->after_br = 0;
+ run++;
+ }
+ EXTRACTOR_fulltext_append_ (ms->ft, &p[i], run - i);
+ if (run >= len)
+ return;
+ used = decode_entity (&p[run + 1], len - run - 1, &cp);
+ if (0 == used)
+ {
+ EXTRACTOR_fulltext_append_ (ms->ft, "&", 1);
+ i = run + 1;
+ ms->after_br = 0;
+ continue;
+ }
+ EXTRACTOR_fulltext_append_cp_ (ms->ft, cp);
+ if ( (0xA0 != cp) && (' ' != cp) )
+ ms->after_br = 0;
+ i = run + 1 + used;
+ }
+}
+
+
+/**
+ * Find the end of the tag starting at @a p, honouring quoted
+ * attribute values.
+ *
+ * @param p start of the tag body (after the name)
+ * @param end end of the buffer
+ * @return pointer to the closing '>', or @a end
+ */
+static const char *
+tag_end (const char *p,
+ const char *end)
+{
+ char quote = '\0';
+
+ for (; p < end; p++)
+ {
+ if ('\0' != quote)
+ {
+ if (*p == quote)
+ quote = '\0';
+ }
+ else if ( ('"' == *p) || ('\'' == *p) )
+ quote = *p;
+ else if ('>' == *p)
+ return p;
+ }
+ return end;
+}
+
+
+/**
+ * Find the case-insensitive end tag `</name' at or after @a p.
+ *
+ * @param p where to search from
+ * @param end end of the buffer
+ * @param name lower-case element name
+ * @return pointer to the '<' of the end tag, or @a end
+ */
+static const char *
+find_end_tag (const char *p,
+ const char *end,
+ const char *name)
+{
+ size_t nlen = strlen (name);
+
+ for (; p + 2 + nlen <= end; p++)
+ {
+ if ( ('<' != p[0]) || ('/' != p[1]) )
+ continue;
+ if (0 == strncasecmp (&p[2], name, nlen))
+ return p;
+ }
+ return end;
+}
+
+
+/**
+ * Convert one (X)HTML document to text. This is a tolerant scanner,
+ * not a parser: EPUB content documents are XHTML, but MOBI text is
+ * HTML 3.2 with unclosed elements, so all we track is which element's
+ * content we are skipping and whether we are inside a sectioning
+ * element. Block-level elements end paragraphs, as do MOBI page
+ * breaks (<mbp:pagebreak/>); one <br> is a space and two end a
+ * paragraph. Memory and time are linear in @a len; there is no
+ * recursion.
+ *
+ * @param ft where the text goes
+ * @param data the document in UTF-8
+ * @param len number of bytes in @a data
+ */
+static void
+markup_to_text (struct EXTRACTOR_FullText *ft,
+ const char *data,
+ size_t len)
+{
+ struct MarkupState ms;
+ const char *p = data;
+ const char *end = data + len;
+
+ memset (&ms, 0, sizeof (ms));
+ ms.ft = ft;
+ while ( (p < end) && (! ft->full) )
+ {
+ const char *lt;
+ const char *name;
+ const char *gt;
+ char local[32];
+ size_t nlen;
+ int closing;
+ int self_closing;
+
+ lt = memchr (p, '<', (size_t) (end - p));
+ if (NULL == lt)
+ lt = end;
+ if ('\0' == ms.skip_name[0])
+ markup_chars (&ms, p, (size_t) (lt - p));
+ p = lt;
+ if (p >= end)
+ break;
+ if ( (end - p >= 4) && (0 == memcmp (p, "<!--", 4)) )
+ {
+ const char *c = mem_find (p + 4, (size_t) (end - p - 4), "-->", 3);
+
+ p = (NULL == c) ? end : c + 3;
+ continue;
+ }
+ if ( (end - p >= 9) && (0 == memcmp (p, "<![CDATA[", 9)) )
+ {
+ const char *c = mem_find (p + 9, (size_t) (end - p - 9), "]]>", 3);
+ const char *ce = (NULL == c) ? end : c;
+
+ if ('\0' == ms.skip_name[0])
+ {
+ EXTRACTOR_fulltext_append_ (ft, p + 9, (size_t) (ce - p - 9));
+ ms.after_br = 0;
+ }
+ p = (NULL == c) ? end : c + 3;
+ continue;
+ }
+ if ( (end - p >= 2) && ( ('!' == p[1]) || ('?' == p[1]) ) )
+ {
+ gt = memchr (p, '>', (size_t) (end - p));
+ p = (NULL == gt) ? end : gt + 1;
+ continue;
+ }
+ closing = ( (end - p >= 2) && ('/' == p[1]) );
+ name = p + (closing ? 2 : 1);
+ nlen = 0;
+ while ( (name + nlen < end) &&
+ is_name_char (name[nlen]) )
+ nlen++;
+ if ( (0 == nlen) ||
+ (! isalpha ((unsigned char) name[0])) )
+ {
+ /* a stray '<' in the text */
+ if ('\0' == ms.skip_name[0])
+ markup_chars (&ms, "<", 1);
+ p++;
+ continue;
+ }
+ gt = tag_end (name + nlen, end);
+ self_closing = ( (! closing) && (gt > name + nlen) && ('/' == gt[-1]) );
+ {
+ /* the local name, lower case, without namespace prefix */
+ const char *l = name;
+ size_t ll;
+
+ for (size_t k = 0; k < nlen; k++)
+ if (':' == name[k])
+ l = &name[k + 1];
+ ll = (size_t) (name + nlen - l);
+ if (ll >= sizeof (local))
+ ll = sizeof (local) - 1;
+ for (size_t k = 0; k < ll; k++)
+ local[k] = (char) tolower ((unsigned char) l[k]);
+ local[ll] = '\0';
+ }
+ p = (gt < end) ? gt + 1 : end;
+
+ if ( (! closing) && (! self_closing) &&
+ name_in (local, raw_text_elements) )
+ {
+ /* no markup inside; the content is never text, except for the
+ obsolete <xmp>, which is rare enough to skip too */
+ const char *et = find_end_tag (p, end, local);
+
+ if (et < end)
+ et = tag_end (et, end);
+ p = (et < end) ? et + 1 : end;
+ if ('\0' == ms.skip_name[0])
+ markup_par (&ms);
+ continue;
+ }
+ if ('\0' != ms.skip_name[0])
+ {
+ if (0 != strcmp (local, ms.skip_name))
+ continue;
+ if (closing)
+ {
+ if (0 == --ms.skip_nest)
+ {
+ ms.skip_name[0] = '\0';
+ if (name_in (local, block_elements))
+ markup_par (&ms);
+ }
+ }
+ else if (! self_closing)
+ {
+ ms.skip_nest++;
+ }
+ continue;
+ }
+ if (closing)
+ {
+ if (name_in (local, sectioning_elements) && (ms.sections > 0))
+ ms.sections--;
+ if (name_in (local, block_elements))
+ markup_par (&ms);
+ continue;
+ }
+ if (name_in (local, skip_elements) ||
+ ( ( (0 == strcmp (local, "header")) ||
+ (0 == strcmp (local, "footer")) ) &&
+ (0 == ms.sections) ) ||
+ attrs_skip (name + nlen, (size_t) (gt - name - nlen)) )
+ {
+ if (name_in (local, block_elements))
+ markup_par (&ms);
+ if ( (! self_closing) &&
+ (! name_in (local, void_elements)) )
+ {
+ strcpy (ms.skip_name, local);
+ ms.skip_nest = 1;
+ }
+ continue;
+ }
+ if (0 == strcmp (local, "br"))
+ {
+ markup_br (&ms);
+ continue;
+ }
+ if ( (0 == strcmp (local, "pagebreak")) ||
+ name_in (local, block_elements) )
+ markup_par (&ms);
+ if ( (! self_closing) &&
+ name_in (local, sectioning_elements) )
+ ms.sections++;
+ }
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Full text: EPUB */
+/* ------------------------------------------------------------------ */
+
+/**
+ * One entry of the OPF manifest.
+ */
+struct ManifestItem
+{
+ /**
+ * The item's id.
+ */
+ char *id;
+
+ /**
+ * The item's href, relative to the OPF.
+ */
+ char *href;
+};
+
+
+/**
+ * Find the next start tag named @a name (not a longer name that begins
+ * with it) in @a data from offset @a *pos.
+ *
+ * @param data buffer to search
+ * @param len number of bytes in @a data
+ * @param[in,out] pos where to start; set to just past the tag
+ * @param name element name including the '<', e.g. "<item"
+ * @param[out] body start of the attributes
+ * @param[out] body_len number of bytes of attributes
+ * @return 1 if found
+ */
+static int
+next_tag (const char *data,
+ size_t len,
+ size_t *pos,
+ const char *name,
+ const char **body,
+ size_t *body_len)
+{
+ size_t nlen = strlen (name);
+
+ while (*pos < len)
+ {
+ const char *t = mem_find (&data[*pos], len - *pos, name, nlen);
+ const char *gt;
+ size_t off;
+
+ if (NULL == t)
+ {
+ *pos = len;
+ return 0;
+ }
+ off = (size_t) (t - data) + nlen;
+ if ( (off < len) &&
+ is_name_char (data[off]) )
+ {
+ *pos = off;
+ continue;
+ }
+ gt = tag_end (&data[off], data + len);
+ *body = &data[off];
+ *body_len = (size_t) (gt - &data[off]);
+ *pos = (size_t) (gt - data) + ( (gt < data + len) ? 1 : 0);
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Value of a hex digit.
+ *
+ * @param c character
+ * @return value, or -1 if @a c is not a hex digit
+ */
+static int
+hex_val (char c)
+{
+ if ( ('0' <= c) && ('9' >= c) )
+ return c - '0';
+ if ( ('a' <= c) && ('f' >= c) )
+ return c - 'a' + 10;
+ if ( ('A' <= c) && ('F' >= c) )
+ return c - 'A' + 10;
+ return -1;
+}
+
+
+/**
+ * Resolve @a href relative to the directory of @a base (a path inside
+ * the container): drop the fragment, undo percent-encoding and
+ * normalise `.' and `..' segments.
+ *
+ * @param base path of the document @a href appears in
+ * @param href the reference
+ * @param[out] out where to write the member name
+ * @param outsz number of bytes available in @a out
+ * @return 1 on success, 0 if @a href is not a local path we can use
+ */
+static int
+resolve_href (const char *base,
+ const char *href,
+ char *out,
+ size_t outsz)
+{
+ char tmp[1024];
+ size_t t = 0;
+ size_t o = 0;
+ const char *slash;
+ const char *seg;
+
+ if ('\0' == href[0])
+ return 0;
+ /* a scheme (http:, data:, ...) means the resource is not in the
+ container */
+ for (size_t i = 0; '\0' != href[i]; i++)
+ {
+ if (':' == href[i])
+ return 0;
+ if ( ('/' == href[i]) || ('#' == href[i]) || ('?' == href[i]) )
+ break;
+ }
+ if ('/' != href[0])
+ {
+ slash = strrchr (base, '/');
+ if (NULL != slash)
+ {
+ size_t bl = (size_t) (slash - base) + 1;
+
+ if (bl >= sizeof (tmp))
+ return 0;
+ memcpy (tmp, base, bl);
+ t = bl;
+ }
+ }
+ for (size_t i = 0; ('\0' != href[i]) && ('#' != href[i]) &&
+ ('?' != href[i]); i++)
+ {
+ char c = href[i];
+
+ if ( ('%' == c) &&
+ (hex_val (href[i + 1]) >= 0) &&
+ (hex_val (href[i + 2]) >= 0) )
+ {
+ c = (char) (hex_val (href[i + 1]) * 16 + hex_val (href[i + 2]));
+ i += 2;
+ }
+ if ('\0' == c)
+ return 0;
+ if (t + 1 >= sizeof (tmp))
+ return 0;
+ tmp[t++] = c;
+ }
+ tmp[t] = '\0';
+ /* normalise the segments into out */
+ seg = tmp;
+ while ('\0' != *seg)
+ {
+ size_t sl = strcspn (seg, "/");
+
+ if ( (0 == sl) ||
+ ( (1 == sl) && ('.' == seg[0]) ) )
+ {
+ /* empty or `.' */
+ }
+ else if ( (2 == sl) && ('.' == seg[0]) && ('.' == seg[1]) )
+ {
+ /* `..': drop the last segment of out */
+ while ( (o > 0) && ('/' != out[o - 1]) )
+ o--;
+ if (o > 0)
+ o--;
+ while ( (o > 0) && ('/' != out[o - 1]) )
+ o--;
+ }
+ else
+ {
+ if (o + sl + 2 > outsz)
+ return 0;
+ memcpy (&out[o], seg, sl);
+ o += sl;
+ out[o++] = '/';
+ }
+ seg += sl;
+ if ('/' == *seg)
+ seg++;
+ }
+ if (0 == o)
+ return 0;
+ out[o - 1] = '\0'; /* replace the trailing '/' */
+ return 1;
+}
+
+
+/**
+ * Release a manifest.
+ *
+ * @param items the manifest
+ * @param n number of entries in @a items
+ */
+static void
+free_manifest (struct ManifestItem *items,
+ unsigned int n)
+{
+ for (unsigned int i = 0; i < n; i++)
+ {
+ free (items[i].id);
+ free (items[i].href);
+ }
+ free (items);
+}
+
+
+/**
+ * One content document in the spine.
+ */
+struct SpineDoc
+{
+ /**
+ * Member name in the archive.
+ */
+ char *member;
+
+ /**
+ * The document's text (normalized as by #EXTRACTOR_FullText), NULL
+ * if not (yet) read. Shared between entries for the same member.
+ */
+ char *text;
+
+ /**
+ * Number of bytes in @e text.
+ */
+ size_t text_len;
+
+ /**
+ * 1 if this entry owns @e text.
+ */
+ int owner;
+};
+
+
+/**
+ * Index entry for looking up spine documents by member name.
+ */
+struct SpineKey
+{
+ /**
+ * Member name (owned by the `struct SpineDoc`).
+ */
+ const char *member;
+
+ /**
+ * Position in the spine.
+ */
+ unsigned int idx;
+};
+
+
+/**
+ * Order spine keys by member name (ignoring case, as the archive
+ * lookup does), then by spine position.
+ *
+ * @param a a `struct SpineKey`
+ * @param b a `struct SpineKey`
+ * @return comparison result
+ */
+static int
+cmp_spine_key (const void *a,
+ const void *b)
+{
+ const struct SpineKey *ka = a;
+ const struct SpineKey *kb = b;
+ int r = strcasecmp (ka->member,
+ kb->member);
+
+ if (0 != r)
+ return r;
+ return (ka->idx < kb->idx) ? -1 : (ka->idx > kb->idx);
+}
+
+
+/**
+ * Parse the manifest and spine of the OPF: the content documents in
+ * reading order.
+ *
+ * @param opf the package document
+ * @param opf_len number of bytes in @a opf
+ * @param opf_path member name of the package document
+ * @param enc contents of META-INF/encryption.xml, or NULL
+ * @param enc_len number of bytes in @a enc
+ * @param[out] docs the spine, caller frees (also the member names)
+ * @return number of entries in @a docs
+ */
+static unsigned int
+parse_spine (const char *opf,
+ size_t opf_len,
+ const char *opf_path,
+ const char *enc,
+ size_t enc_len,
+ struct SpineDoc **docs)
+{
+ struct ManifestItem *items;
+ struct SpineDoc *d;
+ unsigned int n_items = 0;
+ unsigned int n_docs = 0;
+ unsigned int n_refs = 0;
+ size_t pos;
+ const char *body;
+ size_t body_len;
+ const char *spine;
+ const char *spine_end;
+
+ *docs = NULL;
+ spine = mem_find (opf, opf_len, "<spine", strlen ("<spine"));
+ if (NULL == spine)
+ return 0;
+ spine_end = mem_find (spine,
+ opf_len - (size_t) (spine - opf),
+ "</spine",
+ strlen ("</spine"));
+ if (NULL == spine_end)
+ spine_end = opf + opf_len;
+ items = calloc (MAX_MANIFEST, sizeof (struct ManifestItem));
+ d = calloc (MAX_SPINE, sizeof (struct SpineDoc));
+ if ( (NULL == items) ||
+ (NULL == d) )
+ {
+ free (items);
+ free (d);
+ return 0;
+ }
+ pos = 0;
+ while ( (n_items < MAX_MANIFEST) &&
+ next_tag (opf, opf_len, &pos, "<item", &body, &body_len) )
+ {
+ char id[256];
+ char href[1024];
+ char mt[128];
+
+ if ( (! xml_attr (body, body_len, "id", id, sizeof (id))) ||
+ (! xml_attr (body, body_len, "href", href, sizeof (href))) )
+ continue;
+ if (! xml_attr (body, body_len, "media-type", mt, sizeof (mt)))
+ mt[0] = '\0';
+ if ( (NULL == strstr (mt, "html")) &&
+ (NULL == strstr (mt, "oeb1-document")) &&
+ (NULL == strstr (mt, "dtbook")) )
+ continue; /* images, style sheets, fonts, the NCX */
+ items[n_items].id = strdup (id);
+ items[n_items].href = strdup (href);
+ if ( (NULL == items[n_items].id) ||
+ (NULL == items[n_items].href) )
+ {
+ free (items[n_items].id);
+ free (items[n_items].href);
+ break;
+ }
+ n_items++;
+ }
+ pos = 0;
+ while ( (n_refs++ < MAX_SPINE) &&
+ next_tag (spine,
+ (size_t) (spine_end - spine),
+ &pos,
+ "<itemref",
+ &body,
+ &body_len) )
+ {
+ char idref[256];
+ char linear[16];
+ char member[1024];
+ unsigned int i;
+
+ if (! xml_attr (body, body_len, "idref", idref, sizeof (idref)))
+ continue;
+ /* linear="no" marks auxiliary content (pop-up notes, answer keys)
+ that is outside the reading order */
+ if ( (xml_attr (body, body_len, "linear", linear, sizeof (linear))) &&
+ (0 == strcmp (linear, "no")) )
+ continue;
+ for (i = 0; i < n_items; i++)
+ if (0 == strcmp (items[i].id, idref))
+ break;
+ if (i == n_items)
+ continue;
+ if (! resolve_href (opf_path,
+ items[i].href,
+ member,
+ sizeof (member)))
+ continue;
+ /* content documents that are listed in encryption.xml are DRM
+ protected; font obfuscation only lists fonts */
+ if ( (NULL != enc) &&
+ (NULL != mem_find (enc, enc_len, member, strlen (member))) )
+ continue;
+ if (NULL == (d[n_docs].member = strdup (member)))
+ break;
+ n_docs++;
+ }
+ free_manifest (items, n_items);
+ *docs = d;
+ return n_docs;
+}
+
+
+/**
+ * Append @a text, which is normalized as by #EXTRACTOR_FullText
+ * (paragraphs separated by "\n\n"), to @a ft.
+ *
+ * @param ft where to append
+ * @param text the text
+ * @param len number of bytes in @a text
+ */
+static void
+append_normalized (struct EXTRACTOR_FullText *ft,
+ const char *text,
+ size_t len)
+{
+ size_t i = 0;
+
+ while ( (i < len) &&
+ (! ft->full) )
+ {
+ const char *nl = memchr (&text[i], '\n', len - i);
+ size_t pl = (NULL == nl) ? len - i : (size_t) (nl - &text[i]);
+
+ EXTRACTOR_fulltext_append_ (ft, &text[i], pl);
+ EXTRACTOR_fulltext_paragraph_ (ft);
+ i += pl;
+ while ( (i < len) && ('\n' == text[i]) )
+ i++;
+ }
+}
+
+
+/**
+ * Collect the text of an EPUB in reading order (the linear entries of
+ * the OPF spine) and emit it.
+ *
+ * Looking a member up by name walks the central directory, so doing
+ * that once per spine entry would be quadratic in a hostile archive.
+ * Instead, the archive is walked once, each content document found in
+ * the spine is converted to text as it comes along, and the texts are
+ * put in spine order at the end.
+ *
+ * @param ec extraction context
+ * @param uf the archive
+ * @param opf the package document
+ * @param opf_len number of bytes in @a opf
+ * @param opf_path member name of the package document
+ * @return 1 if the caller should stop extracting, 0 to continue
+ */
+static int
+epub_fulltext (struct EXTRACTOR_ExtractContext *ec,
+ struct EXTRACTOR_UnzipFile *uf,
+ const char *opf,
+ size_t opf_len,
+ const char *opf_path)
+{
+ struct EXTRACTOR_FullText ft;
+ struct SpineDoc *docs;
+ struct SpineKey *order;
+ unsigned int n_docs;
+ size_t budget = MAX_BOOK_MARKUP;
+ size_t text_total = 0;
+ size_t enc_len = 0;
+ char *enc;
+ int ret;
+
+ enc = read_member (uf,
+ "META-INF/encryption.xml",
+ MAX_OPF,
+ &enc_len);
+ n_docs = parse_spine (opf,
+ opf_len,
+ opf_path,
+ enc,
+ enc_len,
+ &docs);
+ free (enc);
+ if (0 == n_docs)
+ {
+ free (docs);
+ return 0;
+ }
+ if (NULL == (order = malloc (n_docs * sizeof (struct SpineKey))))
+ {
+ for (unsigned int i = 0; i < n_docs; i++)
+ free (docs[i].member);
+ free (docs);
+ return 0;
+ }
+ for (unsigned int i = 0; i < n_docs; i++)
+ {
+ order[i].member = docs[i].member;
+ order[i].idx = i;
+ }
+ qsort (order, n_docs, sizeof (struct SpineKey), &cmp_spine_key);
+
+ if (EXTRACTOR_UNZIP_OK ==
+ EXTRACTOR_common_unzip_go_to_first_file (uf))
+ {
+ do
+ {
+ struct EXTRACTOR_UnzipFileInfo fi;
+ struct EXTRACTOR_FullText cft;
+ char name[1024];
+ unsigned int lo = 0;
+ unsigned int hi = n_docs;
+ unsigned int first;
+ char *doc;
+ char *utf8;
+ size_t doc_len;
+ size_t utf8_len;
+
+ if (EXTRACTOR_UNZIP_OK !=
+ EXTRACTOR_common_unzip_get_current_file_info (uf,
+ &fi,
+ name,
+ sizeof (name),
+ NULL, 0,
+ NULL, 0))
+ break;
+ name[sizeof (name) - 1] = '\0';
+ /* first spine entry for this member (binary search) */
+ while (lo < hi)
+ {
+ unsigned int mid = lo + (hi - lo) / 2;
+
+ if (strcasecmp (order[mid].member, name) < 0)
+ lo = mid + 1;
+ else
+ hi = mid;
+ }
+ if ( (lo >= n_docs) ||
+ (0 != strcasecmp (order[lo].member, name)) )
+ continue;
+ first = order[lo].idx;
+ if (NULL != docs[first].text)
+ continue; /* the same name twice in the archive */
+ doc = read_current (uf,
+ (budget < MAX_CHAPTER) ? budget : MAX_CHAPTER,
+ &doc_len);
+ if (NULL == doc)
+ continue;
+ budget -= doc_len;
+ utf8 = xhtml_to_utf8 (doc, doc_len, &utf8_len);
+ free (doc);
+ if (NULL == utf8)
+ continue;
+ EXTRACTOR_fulltext_init_ (&cft);
+ EXTRACTOR_fulltext_set_context_ (&cft, ec);
+ markup_to_text (&cft, utf8, utf8_len);
+ free (utf8);
+ if (0 == cft.len)
+ {
+ EXTRACTOR_fulltext_clear_ (&cft);
+ continue;
+ }
+ text_total += cft.len;
+ /* every spine entry for this member shares the text */
+ for (unsigned int k = lo;
+ (k < n_docs) &&
+ (0 == strcasecmp (order[k].member, name));
+ k++)
+ {
+ docs[order[k].idx].text = cft.buf;
+ docs[order[k].idx].text_len = cft.len;
+ }
+ docs[first].owner = 1;
+ }
+ while ( (budget > 0) &&
+ (text_total < EXTRACTOR_FULLTEXT_MAX) &&
+ (EXTRACTOR_UNZIP_OK ==
+ EXTRACTOR_common_unzip_go_to_next_file (uf)) );
+ }
+ free (order);
+
+ EXTRACTOR_fulltext_init_ (&ft);
+ EXTRACTOR_fulltext_set_context_ (&ft, ec);
+ for (unsigned int i = 0; (i < n_docs) && (! ft.full); i++)
+ {
+ /* each spine entry is a chapter, which always ends a paragraph */
+ if (NULL != docs[i].text)
+ append_normalized (&ft, docs[i].text, docs[i].text_len);
+ EXTRACTOR_fulltext_paragraph_ (&ft);
+ }
+ for (unsigned int i = 0; i < n_docs; i++)
+ {
+ if (docs[i].owner)
+ free (docs[i].text);
+ free (docs[i].member);
+ }
+ free (docs);
+ ret = EXTRACTOR_fulltext_emit_ (&ft, ec, PLUGIN_NAME);
+ EXTRACTOR_fulltext_clear_ (&ft);
+ return (0 != ret) ? 1 : 0;
+}
+
+
+/**
+ * Handle an EPUB: read `META-INF/container.xml' to find the OPF package
+ * document, then report what the package document holds.
+ *
+ * @param ec extraction context
+ * @param uf the opened archive
+ */
+static void
+extract_epub (struct EXTRACTOR_ExtractContext *ec,
+ struct EXTRACTOR_UnzipFile *uf)
+{
+ char path[512];
+ char opf_path[512];
+ char attr[64];
+ char *container;
+ char *opf = NULL;
+ size_t container_len;
+ size_t opf_len = 0;
+ const char *p;
+ const char *q;
+ unsigned int items = 0;
+
+ if (0 !=
+ ec->proc (ec->cls,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/epub+zip",
+ strlen ("application/epub+zip") + 1))
+ return;
+ container = read_member (uf,
+ "META-INF/container.xml",
+ MAX_CONTAINER,
+ &container_len);
+ if (NULL != container)
+ {
+ p = mem_find (container,
+ container_len,
+ "full-path=",
+ strlen ("full-path="));
+ if (NULL != p)
+ {
+ p += strlen ("full-path=");
+ if ( ('"' == *p) || ('\'' == *p) )
+ {
+ char quote = *p;
+
+ p++;
+ q = memchr (p,
+ quote,
+ container_len - (size_t) (p - container));
+ if (NULL != q)
+ (void) xml_text (p,
+ (size_t) (q - p),
+ path,
+ sizeof (path));
+ else
+ path[0] = '\0';
+ }
+ else
+ {
+ path[0] = '\0';
+ }
+ if ('\0' != path[0])
+ {
+ opf = read_member (uf,
+ path,
+ MAX_OPF,
+ &opf_len);
+ if (NULL != opf)
+ memcpy (opf_path, path, sizeof (opf_path));
+ }
+ }
+ free (container);
+ }
+ if (NULL == opf)
+ {
+ /* no usable container.xml; the conventional locations are worth a
+ try before giving up */
+ static const char *guesses[] = {
+ "OEBPS/content.opf",
+ "content.opf",
+ "OPS/content.opf",
+ NULL
+ };
+
+ for (unsigned int i = 0; NULL != guesses[i]; i++)
+ {
+ opf = read_member (uf,
+ guesses[i],
+ MAX_OPF,
+ &opf_len);
+ if (NULL != opf)
+ {
+ snprintf (opf_path, sizeof (opf_path), "%s", guesses[i]);
+ break;
+ }
+ }
+ }
+ if (NULL != opf)
+ {
+ /* the <package version="..."> attribute distinguishes EPUB 2 from
+ EPUB 3, which changes what else we may expect to find */
+ p = mem_find (opf,
+ opf_len,
+ "<package",
+ strlen ("<package"));
+ if (NULL != p)
+ {
+ size_t tail = opf_len - (size_t) (p - opf);
+ const char *gt;
+
+ if (tail > 1024)
+ tail = 1024;
+ /* stop at the end of the start tag, so that a `version=' further
+ down the document cannot be mistaken for this one */
+ gt = memchr (p,
+ '>',
+ tail);
+ if (NULL != gt)
+ tail = (size_t) (gt - p);
+ if (tail <= strlen ("<package"))
+ tail = strlen ("<package");
+ if (xml_attr (p + strlen ("<package"),
+ tail - strlen ("<package"),
+ "version",
+ attr,
+ sizeof (attr)))
+ {
+ if (0 !=
+ EXTRACTOR_forensic_emit_text_ (ec,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_FORMAT_VERSION,
+ attr,
+ strlen (attr)))
+ goto CLEANUP;
+ }
+ }
+ if (0 !=
+ parse_opf_metadata (ec,
+ opf,
+ opf_len))
+ goto CLEANUP;
+ /* count the manifest entries; `<itemref>' does not match because
+ the character after `<item' has to be a name delimiter */
+ for (size_t i = 0; i + strlen ("<item") < opf_len; i++)
+ if ( (0 == memcmp (&opf[i],
+ "<item",
+ strlen ("<item"))) &&
+ (! is_name_char (opf[i + strlen ("<item")])) )
+ items++;
+ if (0 != items)
+ {
+ if (0 !=
+ EXTRACTOR_forensic_emit_ (ec,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_ENTRY_COUNT,
+ "%u",
+ items))
+ goto CLEANUP;
+ }
+ }
+ if (has_member (uf,
+ "META-INF/encryption.xml"))
+ {
+ if (0 !=
+ EXTRACTOR_forensic_emit_ (
+ ec,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_ENCRYPTION,
+ "META-INF/encryption.xml present"
+ " (DRM or font obfuscation)"))
+ goto CLEANUP;
+ }
+ if (has_member (uf,
+ "META-INF/rights.xml"))
+ {
+ if (0 !=
+ EXTRACTOR_forensic_emit_ (ec,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_ENCRYPTION,
+ "META-INF/rights.xml present"
+ " (Adobe Content Server DRM)"))
+ goto CLEANUP;
+ }
+ if ( (NULL != opf) &&
+ (EXTRACTOR_fulltext_requested_ (ec->config)) )
+ (void) epub_fulltext (ec,
+ uf,
+ opf,
+ opf_len,
+ opf_path);
+CLEANUP:
+ free (opf);
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Full text: MOBI */
+/* ------------------------------------------------------------------ */
+
+/**
+ * Size of the trailing entries at the end of a MOBI text record, as
+ * announced by the "extra data flags" of the MOBI header: bit 0 is the
+ * multibyte-character overlap (its size is in the low two bits of the
+ * last byte before the other entries), every other bit is an entry
+ * whose size is a backward-encoded variable-length integer at its end.
+ *
+ * @param d the record
+ * @param size number of bytes in @a d
+ * @param flags extra data flags
+ * @return number of bytes to strip from the end of the record
+ */
+static size_t
+trailing_size (const unsigned char *d,
+ size_t size,
+ uint16_t flags)
+{
+ size_t num = 0;
+
+ for (unsigned int bit = 1; bit < 16; bit++)
+ {
+ size_t end;
+ uint32_t v = 0;
+
+ if (0 == (flags & (1u << bit)))
+ continue;
+ end = size - num;
+ if (0 == end)
+ return size;
+ for (size_t k = (end >= 4) ? end - 4 : 0; k < end; k++)
+ {
+ if (0 != (d[k] & 0x80))
+ v = 0;
+ v = (v << 7) | (d[k] & 0x7F);
+ }
+ if (v > end)
+ return size;
+ num += v;
+ }
+ if (0 != (flags & 1))
+ {
+ if (num >= size)
+ return size;
+ num += (d[size - num - 1] & 0x3) + 1;
+ }
+ return (num > size) ? size : num;
+}
+
+
+/**
+ * Decompress one PalmDOC (LZ77 variant) compressed record.
+ *
+ * @param in compressed data
+ * @param len number of bytes in @a in
+ * @param[out] out where to write the text
+ * @param outsz number of bytes available in @a out
+ * @return number of bytes written
+ */
+static size_t
+palmdoc_decompress (const unsigned char *in,
+ size_t len,
+ unsigned char *out,
+ size_t outsz)
+{
+ size_t i = 0;
+ size_t o = 0;
+
+ while ( (i < len) && (o < outsz) )
+ {
+ unsigned int c = in[i++];
+
+ if ( (0 == c) ||
+ ( (c >= 0x09) && (c <= 0x7F) ) )
+ {
+ out[o++] = (unsigned char) c;
+ }
+ else if (c <= 0x08)
+ {
+ /* c literal bytes follow */
+ for (unsigned int k = 0; (k < c) && (i < len) && (o < outsz); k++)
+ out[o++] = in[i++];
+ }
+ else if (c <= 0xBF)
+ {
+ unsigned int pair;
+ size_t dist;
+ unsigned int n;
+
+ if (i >= len)
+ break;
+ pair = ((c << 8) | in[i++]) & 0x3FFF;
+ dist = pair >> 3;
+ n = (pair & 7) + 3;
+ if ( (0 == dist) ||
+ (dist > o) )
+ break; /* corrupt: points before the start */
+ for (unsigned int k = 0; (k < n) && (o < outsz); k++, o++)
+ out[o] = out[o - dist];
+ }
+ else
+ {
+ /* a space followed by the character c ^ 0x80 */
+ out[o++] = ' ';
+ if (o < outsz)
+ out[o++] = (unsigned char) (c ^ 0x80);
+ }
+ }
+ return o;
+}
+
+
+/**
+ * Collect the text of a MOBI book and emit it. Handles uncompressed
+ * and PalmDOC-compressed books; HUFF/CDIC compression and encrypted
+ * books are skipped.
+ *
+ * @param ec extraction context
+ * @param rec record 0
+ * @param rec_len number of bytes in @a rec (at least 16)
+ * @param hdr_len length of the MOBI header, 0 if there is none
+ * @param num_records number of records in the database
+ * @param size size of the file
+ */
+static void
+mobi_fulltext (struct EXTRACTOR_ExtractContext *ec,
+ const unsigned char *rec,
+ size_t rec_len,
+ uint32_t hdr_len,
+ uint16_t num_records,
+ uint64_t size)
+{
+ struct EXTRACTOR_FullText ft;
+ uint16_t compression = EXTRACTOR_forensic_be16_ (&rec[0]);
+ uint16_t text_records = EXTRACTOR_forensic_be16_ (&rec[8]);
+ uint16_t extra_flags = 0;
+ const char *cs = "windows-1252";
+ unsigned char *info = NULL;
+ unsigned char *in = NULL;
+ unsigned char *text = NULL;
+ size_t text_len = 0;
+ size_t text_cap = 0;
+ unsigned int n_info;
+ char *utf8;
+ size_t utf8_len;
+
+ if (0 != EXTRACTOR_forensic_be16_ (&rec[12]))
+ return; /* encrypted */
+ if ( (1 != compression) &&
+ (2 != compression) )
+ return; /* HUFF/CDIC (17480) or unknown */
+ if (hdr_len >= 28)
+ {
+ if (65001 == EXTRACTOR_forensic_be32_ (&rec[16 + 12]))
+ cs = "utf-8";
+ }
+ /* the extra data flags exist from MOBI header length 0xE4 on */
+ if ( (hdr_len >= 0xE4) &&
+ (rec_len >= 16 + 0xE4) )
+ extra_flags = EXTRACTOR_forensic_be16_ (&rec[16 + 0xE2]);
+ if (text_records >= num_records)
+ text_records = num_records - 1;
+ if (0 == text_records)
+ return;
+ /* offsets of records 0 .. text_records + 1 */
+ n_info = (unsigned int) text_records + 2;
+ if (n_info > num_records)
+ n_info = num_records;
+ if (NULL == (info = malloc (8 * (size_t) n_info)))
+ return;
+ if (! EXTRACTOR_forensic_read_ (ec, 78, info, 8 * (size_t) n_info))
+ goto CLEANUP;
+ if ( (NULL == (in = malloc (MAX_TEXT_RECORD))) )
+ goto CLEANUP;
+ for (unsigned int r = 1; r <= text_records; r++)
+ {
+ uint64_t start = EXTRACTOR_forensic_be32_ (&info[8 * r]);
+ uint64_t stop = (r + 1 < n_info)
+ ? EXTRACTOR_forensic_be32_ (&info[8 * (r + 1)])
+ : size;
+ size_t len;
+ size_t strip;
+
+ if ( (stop <= start) ||
+ (stop > size) ||
+ (stop - start > MAX_TEXT_RECORD) )
+ continue;
+ len = (size_t) (stop - start);
+ if (! EXTRACTOR_forensic_read_ (ec, (int64_t) start, in, len))
+ continue;
+ strip = trailing_size (in, len, extra_flags);
+ len -= strip;
+ if (text_len + MAX_TEXT_RECORD_OUT > text_cap)
+ {
+ size_t ncap = (0 == text_cap) ? 256 * 1024 : 2 * text_cap;
+ unsigned char *nt;
+
+ if (ncap > MAX_BOOK_MARKUP)
+ break;
+ if (NULL == (nt = realloc (text, ncap)))
+ break;
+ text = nt;
+ text_cap = ncap;
+ }
+ if (1 == compression)
+ {
+ if (len > MAX_TEXT_RECORD_OUT)
+ len = MAX_TEXT_RECORD_OUT;
+ memcpy (&text[text_len], in, len);
+ text_len += len;
+ }
+ else
+ {
+ text_len += palmdoc_decompress (in,
+ len,
+ &text[text_len],
+ MAX_TEXT_RECORD_OUT);
+ }
+ }
+ if (0 == text_len)
+ goto CLEANUP;
+ utf8 = convert_to_utf8 ((const char *) text, text_len, cs, &utf8_len);
+ if (NULL == utf8)
+ goto CLEANUP;
+ free (text);
+ text = NULL;
+ EXTRACTOR_fulltext_init_ (&ft);
+ EXTRACTOR_fulltext_set_context_ (&ft, ec);
+ markup_to_text (&ft, utf8, utf8_len);
+ free (utf8);
+ (void) EXTRACTOR_fulltext_emit_ (&ft, ec, PLUGIN_NAME);
+ EXTRACTOR_fulltext_clear_ (&ft);
+CLEANUP:
+ free (text);
+ free (in);
+ free (info);
+}
+
+
+/**
+ * Convert a Palm database time stamp.
+ *
+ * The format says seconds since 1904, but writers disagree and some use
+ * the Unix epoch. The two ranges do not overlap for any date between
+ * 1970 and 2036, so the value itself says which convention was used.
+ *
+ * @param raw the 32 bit field
+ * @return seconds since 1970, 0 if the field was never filled in
+ */
+static int64_t
+palm_time (uint32_t raw)
{
if (0 == raw)
return 0;
@@ -1156,10 +3053,11 @@ extract_mobi (struct EXTRACTOR_ExtractContext *ec,
uint64_t size;
uint32_t rec0_off;
uint32_t rec1_off;
- uint32_t hdr_len;
+ uint32_t hdr_len = 0;
uint16_t num_records;
size_t rec_len;
int have_title = 0;
+ int rec_ok = 0;
if (0 !=
ec->proc (ec->cls,
@@ -1221,6 +3119,7 @@ extract_mobi (struct EXTRACTOR_ExtractContext *ec,
rec,
rec_len))
goto NAME_ONLY;
+ rec_ok = 1;
/* PalmDOC header: encryption type at offset 12 */
switch (EXTRACTOR_forensic_be16_ (&rec[12]))
{
@@ -1261,7 +3160,10 @@ extract_mobi (struct EXTRACTOR_ExtractContext *ec,
hdr_len = EXTRACTOR_forensic_be32_ (&rec[20]);
if ( (hdr_len < 24) ||
(((uint64_t) hdr_len) + 16 > rec_len) )
+ {
+ hdr_len = 0;
goto NAME_ONLY;
+ }
if (hdr_len >= 28)
{
const char *cs = NULL;
@@ -1315,12 +3217,21 @@ extract_mobi (struct EXTRACTOR_ExtractContext *ec,
}
}
NAME_ONLY:
- if (! have_title)
- (void) EXTRACTOR_forensic_emit_text_ (ec,
- PLUGIN_NAME,
- EXTRACTOR_METATYPE_TITLE,
- (const char *) hdr,
- 32);
+ if ( (! have_title) &&
+ (0 != EXTRACTOR_forensic_emit_text_ (ec,
+ PLUGIN_NAME,
+ EXTRACTOR_METATYPE_TITLE,
+ (const char *) hdr,
+ 32)) )
+ goto CLEANUP;
+ if (rec_ok &&
+ (EXTRACTOR_fulltext_requested_ (ec->config)) )
+ mobi_fulltext (ec,
+ rec,
+ rec_len,
+ hdr_len,
+ num_records,
+ size);
CLEANUP:
free (rec);
}
diff --git a/src/plugins/fulltext.c b/src/plugins/fulltext.c
@@ -0,0 +1,983 @@
+/*
+ This file is part of libextractor.
+ Copyright (C) 2026 Christian Grothoff
+
+ libextractor is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ libextractor is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with libextractor; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+/**
+ * @file plugins/fulltext.c
+ * @brief helpers shared by the plugins implementing the "fulltext" option
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "fulltext.h"
+#include <strings.h>
+
+
+const char *
+EXTRACTOR_config_find_ (const char *config,
+ const char *key)
+{
+ const char *pos;
+ size_t klen;
+
+ if ( (NULL == config) ||
+ (NULL == key) )
+ return NULL;
+ klen = strlen (key);
+ pos = config;
+ while (1)
+ {
+ if ( (0 == strncmp (pos,
+ key,
+ klen)) &&
+ ( ('\0' == pos[klen]) ||
+ (',' == pos[klen]) ||
+ ('=' == pos[klen]) ||
+ (':' == pos[klen]) ) )
+ return &pos[klen];
+ pos = strchr (pos, ',');
+ if (NULL == pos)
+ return NULL;
+ pos++;
+ }
+}
+
+
+int
+EXTRACTOR_fulltext_requested_ (const char *config)
+{
+ return (NULL != EXTRACTOR_config_find_ (config,
+ "fulltext")) ? 1 : 0;
+}
+
+
+void
+EXTRACTOR_fulltext_init_ (struct EXTRACTOR_FullText *ft)
+{
+ memset (ft, 0, sizeof (*ft));
+}
+
+
+void
+EXTRACTOR_fulltext_set_context_ (struct EXTRACTOR_FullText *ft,
+ struct EXTRACTOR_ExtractContext *ec)
+{
+ ft->ec = ec;
+}
+
+
+/**
+ * Number of units of work between two keep-alives; the plugin side
+ * of the IPC rate-limits them by time anyway, so this only bounds
+ * the cost of asking.
+ */
+#define TICKS_PER_ALIVE 4096
+
+
+void
+EXTRACTOR_fulltext_tick_ (struct EXTRACTOR_FullText *ft)
+{
+ if (NULL == ft->ec)
+ return;
+ if (++ft->ticks < TICKS_PER_ALIVE)
+ return;
+ ft->ticks = 0;
+ (void) ft->ec->seek (ft->ec->cls,
+ 0,
+ SEEK_CUR);
+}
+
+
+/**
+ * Check whether @a cp is whitespace for our purposes.
+ *
+ * @param cp code point to check
+ * @return 1 if @a cp is whitespace
+ */
+static int
+is_space_cp (uint32_t cp)
+{
+ switch (cp)
+ {
+ case ' ':
+ case '\t':
+ case '\n':
+ case '\r':
+ case '\f':
+ case '\v':
+ case 0x85:
+ case 0xA0:
+ case 0x1680:
+ case 0x2028:
+ case 0x2029:
+ case 0x202F:
+ case 0x205F:
+ case 0x3000:
+ return 1;
+ default:
+ return ( (cp >= 0x2000) && (cp <= 0x200A) ) ? 1 : 0;
+ }
+}
+
+
+/**
+ * Check whether @a cp must be dropped silently: control characters,
+ * soft hyphens, zero-width characters and byte order marks.
+ *
+ * @param cp code point to check
+ * @return 1 if @a cp is to be dropped
+ */
+static int
+is_dropped_cp (uint32_t cp)
+{
+ if ( (cp < 0x20) ||
+ ( (cp >= 0x7F) && (cp < 0xA0) ) )
+ return 1;
+ switch (cp)
+ {
+ case 0xAD:
+ case 0x200B:
+ case 0x200C:
+ case 0x200D:
+ case 0x2060:
+ case 0xFEFF:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+
+/**
+ * Make room for @a extra more bytes plus the terminating 0.
+ *
+ * @param ft buffer to grow
+ * @param extra number of bytes we want to append
+ * @return 0 on success, -1 if the buffer is full
+ */
+static int
+reserve (struct EXTRACTOR_FullText *ft,
+ size_t extra)
+{
+ size_t need;
+ size_t ncap;
+ char *nbuf;
+
+ if (ft->full)
+ return -1;
+ need = ft->len + extra + 1;
+ if (need > EXTRACTOR_FULLTEXT_MAX)
+ {
+ ft->full = 1;
+ return -1;
+ }
+ if (need <= ft->cap)
+ return 0;
+ ncap = (0 == ft->cap) ? 4096 : ft->cap;
+ while (ncap < need)
+ ncap *= 2;
+ if (ncap > EXTRACTOR_FULLTEXT_MAX)
+ ncap = EXTRACTOR_FULLTEXT_MAX;
+ nbuf = realloc (ft->buf,
+ ncap);
+ if (NULL == nbuf)
+ {
+ ft->full = 1;
+ return -1;
+ }
+ ft->buf = nbuf;
+ ft->cap = ncap;
+ return 0;
+}
+
+
+void
+EXTRACTOR_fulltext_append_cp_ (struct EXTRACTOR_FullText *ft,
+ uint32_t cp)
+{
+ char enc[4];
+ size_t elen;
+ const char *sep;
+ size_t slen;
+
+ if (ft->full)
+ return;
+ EXTRACTOR_fulltext_tick_ (ft);
+ if (is_space_cp (cp))
+ {
+ EXTRACTOR_fulltext_space_ (ft);
+ return;
+ }
+ if (is_dropped_cp (cp))
+ return;
+ if ( (cp > 0x10FFFF) ||
+ ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
+ cp = 0xFFFD;
+ if (cp < 0x80)
+ {
+ enc[0] = (char) cp;
+ elen = 1;
+ }
+ else if (cp < 0x800)
+ {
+ enc[0] = (char) (0xC0 | (cp >> 6));
+ enc[1] = (char) (0x80 | (cp & 0x3F));
+ elen = 2;
+ }
+ else if (cp < 0x10000)
+ {
+ enc[0] = (char) (0xE0 | (cp >> 12));
+ enc[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ enc[2] = (char) (0x80 | (cp & 0x3F));
+ elen = 3;
+ }
+ else
+ {
+ enc[0] = (char) (0xF0 | (cp >> 18));
+ enc[1] = (char) (0x80 | ((cp >> 12) & 0x3F));
+ enc[2] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ enc[3] = (char) (0x80 | (cp & 0x3F));
+ elen = 4;
+ }
+ sep = "";
+ slen = 0;
+ if (ft->len > 0)
+ {
+ if (ft->pending_par)
+ {
+ sep = "\n\n";
+ slen = 2;
+ }
+ else if (ft->pending_space)
+ {
+ sep = " ";
+ slen = 1;
+ }
+ }
+ if (0 != reserve (ft,
+ slen + elen))
+ return;
+ memcpy (&ft->buf[ft->len],
+ sep,
+ slen);
+ ft->len += slen;
+ memcpy (&ft->buf[ft->len],
+ enc,
+ elen);
+ ft->len += elen;
+ ft->buf[ft->len] = '\0';
+ ft->pending_par = 0;
+ ft->pending_space = 0;
+}
+
+
+/**
+ * Decode one UTF-8 sequence.
+ *
+ * @param s input
+ * @param len number of bytes available at @a s, at least 1
+ * @param[out] cp decoded code point, U+FFFD if invalid
+ * @return number of bytes consumed (at least 1)
+ */
+static size_t
+decode_utf8 (const unsigned char *s,
+ size_t len,
+ uint32_t *cp)
+{
+ uint32_t c;
+ size_t n;
+ size_t i;
+ uint32_t min;
+
+ c = s[0];
+ if (c < 0x80)
+ {
+ *cp = c;
+ return 1;
+ }
+ if ( (c & 0xE0) == 0xC0)
+ {
+ n = 2;
+ c &= 0x1F;
+ min = 0x80;
+ }
+ else if ( (c & 0xF0) == 0xE0)
+ {
+ n = 3;
+ c &= 0x0F;
+ min = 0x800;
+ }
+ else if ( (c & 0xF8) == 0xF0)
+ {
+ n = 4;
+ c &= 0x07;
+ min = 0x10000;
+ }
+ else
+ {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ if (n > len)
+ {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ for (i = 1; i < n; i++)
+ {
+ if ( (s[i] & 0xC0) != 0x80)
+ {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ c = (c << 6) | (s[i] & 0x3F);
+ }
+ if ( (c < min) ||
+ (c > 0x10FFFF) ||
+ ( (c >= 0xD800) && (c <= 0xDFFF) ) )
+ {
+ *cp = 0xFFFD;
+ return 1;
+ }
+ *cp = c;
+ return n;
+}
+
+
+void
+EXTRACTOR_fulltext_append_ (struct EXTRACTOR_FullText *ft,
+ const char *text,
+ size_t len)
+{
+ const unsigned char *s = (const unsigned char *) text;
+ size_t off;
+ uint32_t cp;
+
+ off = 0;
+ while ( (off < len) &&
+ (! ft->full) )
+ {
+ off += decode_utf8 (&s[off],
+ len - off,
+ &cp);
+ EXTRACTOR_fulltext_append_cp_ (ft,
+ cp);
+ }
+}
+
+
+void
+EXTRACTOR_fulltext_append_str_ (struct EXTRACTOR_FullText *ft,
+ const char *text)
+{
+ if (NULL == text)
+ return;
+ EXTRACTOR_fulltext_append_ (ft,
+ text,
+ strlen (text));
+}
+
+
+void
+EXTRACTOR_fulltext_space_ (struct EXTRACTOR_FullText *ft)
+{
+ if (ft->len > 0)
+ ft->pending_space = 1;
+}
+
+
+void
+EXTRACTOR_fulltext_paragraph_ (struct EXTRACTOR_FullText *ft)
+{
+ if (ft->len > 0)
+ ft->pending_par = 1;
+}
+
+
+int
+EXTRACTOR_fulltext_emit_ (const struct EXTRACTOR_FullText *ft,
+ struct EXTRACTOR_ExtractContext *ec,
+ const char *plugin_name)
+{
+ if (0 == ft->len)
+ return 0;
+ return ec->proc (ec->cls,
+ plugin_name,
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ ft->buf,
+ ft->len + 1);
+}
+
+
+void
+EXTRACTOR_fulltext_clear_ (struct EXTRACTOR_FullText *ft)
+{
+ free (ft->buf);
+ memset (ft, 0, sizeof (*ft));
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Page furniture */
+/* ------------------------------------------------------------------ */
+
+/**
+ * Words that may introduce a page number ("Page 3", "Seite 3", ...).
+ */
+static const char *const page_words[] = {
+ "page", "p", "pg", "pp", "seite", "s", "pagina", "página", "pag",
+ "страница", "стр", "sida", "side", "sivu", "strona", "str", NULL
+};
+
+/**
+ * Words that may join page number and page count ("3 of 17").
+ */
+static const char *const of_words[] = {
+ "of", "von", "de", "sur", "di", "van", "av", "af", "из", "z", "/", NULL
+};
+
+
+/**
+ * Check whether @a tok (of @a len bytes) is in @a list, ignoring ASCII
+ * case and a trailing '.'.
+ *
+ * @param tok token to check
+ * @param len length of @a tok
+ * @param list NULL-terminated word list
+ * @return 1 if found
+ */
+static int
+in_list (const char *tok,
+ size_t len,
+ const char *const *list)
+{
+ unsigned int i;
+
+ if ( (len > 1) &&
+ ('.' == tok[len - 1]) )
+ len--;
+ for (i = 0; NULL != list[i]; i++)
+ if ( (strlen (list[i]) == len) &&
+ (0 == strncasecmp (tok,
+ list[i],
+ len)) )
+ return 1;
+ return 0;
+}
+
+
+/**
+ * Check whether @a tok is a number: 1-5 decimal digits, or a roman
+ * numeral of at most 8 characters.
+ *
+ * @param tok token to check
+ * @param len length of @a tok
+ * @return 1 if @a tok is a number
+ */
+static int
+is_number (const char *tok,
+ size_t len)
+{
+ size_t i;
+ int digits = 1;
+ int roman = 1;
+
+ if ( (0 == len) ||
+ (len > 8) )
+ return 0;
+ for (i = 0; i < len; i++)
+ {
+ char c = tok[i];
+
+ if (! isdigit ((unsigned char) c))
+ digits = 0;
+ if (NULL == strchr ("ivxlcdmIVXLCDM", c))
+ roman = 0;
+ }
+ if (digits && (len <= 5))
+ return 1;
+ return roman;
+}
+
+
+/**
+ * Check whether @a c is a decoration character around page numbers.
+ *
+ * @param c character to check
+ * @return 1 if @a c is decoration
+ */
+static int
+is_decoration (char c)
+{
+ return (NULL != strchr (" \t\r\n-.,:;[]()|*~<>_=#", c)) ? 1 : 0;
+}
+
+
+int
+EXTRACTOR_fulltext_is_page_number_ (const char *line,
+ size_t len)
+{
+ const char *tok[4];
+ size_t tlen[4];
+ unsigned int ntok;
+ size_t pos;
+ size_t start;
+ size_t end;
+
+ /* strip ASCII decorations and UTF-8 en/em dashes at both ends */
+ start = 0;
+ end = len;
+ while (start < end)
+ {
+ if (is_decoration (line[start]))
+ start++;
+ else if ( (start + 3 <= end) &&
+ (0 == memcmp (&line[start], "\xE2\x80", 2)) &&
+ ( ((unsigned char) line[start + 2] >= 0x93) &&
+ ((unsigned char) line[start + 2] <= 0x94) ) )
+ start += 3;
+ else
+ break;
+ }
+ while (end > start)
+ {
+ if (is_decoration (line[end - 1]))
+ end--;
+ else if ( (end >= start + 3) &&
+ (0 == memcmp (&line[end - 3], "\xE2\x80", 2)) &&
+ ( ((unsigned char) line[end - 1] >= 0x93) &&
+ ((unsigned char) line[end - 1] <= 0x94) ) )
+ end -= 3;
+ else
+ break;
+ }
+ if (start == end)
+ return 0;
+ /* tokenize on spaces, and split '/' into its own token */
+ ntok = 0;
+ pos = start;
+ while (pos < end)
+ {
+ size_t ts;
+
+ while ( (pos < end) &&
+ ( (' ' == line[pos]) || ('\t' == line[pos]) ) )
+ pos++;
+ if (pos == end)
+ break;
+ if (ntok == 4)
+ return 0;
+ ts = pos;
+ if ('/' == line[pos])
+ pos++;
+ else
+ while ( (pos < end) &&
+ (' ' != line[pos]) &&
+ ('\t' != line[pos]) &&
+ ('/' != line[pos]) )
+ pos++;
+ tok[ntok] = &line[ts];
+ tlen[ntok] = pos - ts;
+ ntok++;
+ }
+ /* "N" | "WORD N" | "N OF M" | "WORD N OF M" */
+ pos = 0;
+ if ( (ntok > 1) &&
+ in_list (tok[0], tlen[0], page_words) )
+ pos = 1;
+ if ( (pos >= ntok) ||
+ (! is_number (tok[pos], tlen[pos])) )
+ return 0;
+ pos++;
+ if (pos == ntok)
+ return 1;
+ if ( (pos + 2 == ntok) &&
+ in_list (tok[pos], tlen[pos], of_words) &&
+ is_number (tok[pos + 1], tlen[pos + 1]) )
+ return 1;
+ return 0;
+}
+
+
+int
+EXTRACTOR_fulltext_page_add_ (struct EXTRACTOR_FullTextPage *page,
+ const char *text,
+ size_t len,
+ double y,
+ int par_end)
+{
+ struct EXTRACTOR_FullTextLine *l;
+
+ if (page->num_lines == page->cap_lines)
+ {
+ unsigned int ncap;
+ struct EXTRACTOR_FullTextLine *nl;
+
+ ncap = (0 == page->cap_lines) ? 32 : 2 * page->cap_lines;
+ if (ncap > 1024 * 1024)
+ return -1;
+ nl = realloc (page->lines,
+ ncap * sizeof (*nl));
+ if (NULL == nl)
+ return -1;
+ page->lines = nl;
+ page->cap_lines = ncap;
+ }
+ l = &page->lines[page->num_lines];
+ l->text = malloc (len + 1);
+ if (NULL == l->text)
+ return -1;
+ memcpy (l->text,
+ text,
+ len);
+ l->text[len] = '\0';
+ l->y = y;
+ l->par_end = par_end;
+ l->skip = 0;
+ page->num_lines++;
+ return 0;
+}
+
+
+void
+EXTRACTOR_fulltext_page_clear_ (struct EXTRACTOR_FullTextPage *page)
+{
+ unsigned int i;
+
+ for (i = 0; i < page->num_lines; i++)
+ free (page->lines[i].text);
+ free (page->lines);
+ memset (page, 0, sizeof (*page));
+}
+
+
+/**
+ * Maximum number of lines at the top and at the bottom of a page we
+ * consider as page furniture.
+ */
+#define MARGIN_LINES 3
+
+/**
+ * Relative height of the top and bottom margin bands.
+ */
+#define MARGIN_BAND 0.12
+
+/**
+ * Lines longer than this are never page furniture.
+ */
+#define MARGIN_MAX_LEN 200
+
+
+/**
+ * A margin line candidate for running header/footer detection.
+ */
+struct Candidate
+{
+ /**
+ * Normalized text: lower-case ASCII, digits and whitespace removed.
+ */
+ char *key;
+
+ /**
+ * The line this candidate refers to.
+ */
+ struct EXTRACTOR_FullTextLine *line;
+
+ /**
+ * Page number of the line.
+ */
+ unsigned int page;
+
+ /**
+ * 0 for the top band, 1 for the bottom band.
+ */
+ int band;
+};
+
+
+/**
+ * Order candidates by band, then key, then page.
+ *
+ * @param a first candidate
+ * @param b second candidate
+ * @return comparison result
+ */
+static int
+cmp_candidate (const void *a,
+ const void *b)
+{
+ const struct Candidate *ca = a;
+ const struct Candidate *cb = b;
+ int r;
+
+ if (ca->band != cb->band)
+ return ca->band - cb->band;
+ r = strcmp (ca->key,
+ cb->key);
+ if (0 != r)
+ return r;
+ if (ca->page != cb->page)
+ return (ca->page < cb->page) ? -1 : 1;
+ return 0;
+}
+
+
+/**
+ * Normalize @a text for running header comparison.
+ *
+ * @param text line text
+ * @return normalized copy, NULL on allocation failure
+ */
+static char *
+normalize_key (const char *text)
+{
+ size_t len = strlen (text);
+ char *key;
+ size_t i;
+ size_t o;
+
+ key = malloc (len + 1);
+ if (NULL == key)
+ return NULL;
+ o = 0;
+ for (i = 0; i < len; i++)
+ {
+ unsigned char c = (unsigned char) text[i];
+
+ if (isdigit (c) || isspace (c))
+ continue;
+ key[o++] = (char) tolower (c);
+ }
+ key[o] = '\0';
+ return key;
+}
+
+
+/**
+ * Check whether line @a i of @a page is in the margin band @a band.
+ *
+ * @param page page to check
+ * @param i line index
+ * @param band 0 for top, 1 for bottom
+ * @return 1 if in the band
+ */
+static int
+in_band (const struct EXTRACTOR_FullTextPage *page,
+ unsigned int i,
+ int band)
+{
+ const struct EXTRACTOR_FullTextLine *l = &page->lines[i];
+
+ if (strlen (l->text) > MARGIN_MAX_LEN)
+ return 0;
+ if (l->y < 0)
+ {
+ /* position unknown: only the first/last lines qualify */
+ return (0 == band) ? (i < 2) : (i + 2 >= page->num_lines);
+ }
+ return (0 == band) ? (l->y < MARGIN_BAND) : (l->y > 1.0 - MARGIN_BAND);
+}
+
+
+void
+EXTRACTOR_fulltext_filter_margins_ (struct EXTRACTOR_FullTextPage *pages,
+ unsigned int num_pages)
+{
+ struct Candidate *cands;
+ size_t ncands;
+ size_t i;
+ unsigned int p;
+
+ if (0 == num_pages)
+ return;
+ /* pass 1: page numbers */
+ for (p = 0; p < num_pages; p++)
+ {
+ struct EXTRACTOR_FullTextPage *page = &pages[p];
+ unsigned int j;
+
+ /* do not strip the only line of a page */
+ if (page->num_lines < 2)
+ continue;
+ for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
+ if (in_band (page, j, 0) &&
+ EXTRACTOR_fulltext_is_page_number_ (page->lines[j].text,
+ strlen (page->lines[j].text)))
+ page->lines[j].skip = 1;
+ for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
+ {
+ unsigned int k = page->num_lines - 1 - j;
+
+ if (in_band (page, k, 1) &&
+ EXTRACTOR_fulltext_is_page_number_ (page->lines[k].text,
+ strlen (page->lines[k].text)))
+ page->lines[k].skip = 1;
+ }
+ }
+ if (num_pages < 3)
+ return;
+ /* pass 2: running headers and footers */
+ if (num_pages > 1000000)
+ return;
+ cands = calloc ((size_t) num_pages * 2 * MARGIN_LINES,
+ sizeof (*cands));
+ if (NULL == cands)
+ return;
+ ncands = 0;
+ for (p = 0; p < num_pages; p++)
+ {
+ struct EXTRACTOR_FullTextPage *page = &pages[p];
+ unsigned int j;
+ int band;
+
+ if (page->num_lines < 2)
+ continue;
+ for (band = 0; band < 2; band++)
+ for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
+ {
+ unsigned int k = (0 == band) ? j : page->num_lines - 1 - j;
+ char *key;
+
+ if ( (page->lines[k].skip) ||
+ (! in_band (page, k, band)) )
+ continue;
+ key = normalize_key (page->lines[k].text);
+ if (NULL == key)
+ continue;
+ if ('\0' == key[0])
+ {
+ free (key);
+ continue;
+ }
+ cands[ncands].key = key;
+ cands[ncands].line = &page->lines[k];
+ cands[ncands].page = p;
+ cands[ncands].band = band;
+ ncands++;
+ }
+ }
+ qsort (cands,
+ ncands,
+ sizeof (*cands),
+ &cmp_candidate);
+ i = 0;
+ while (i < ncands)
+ {
+ size_t j;
+ unsigned int distinct_pages;
+
+ distinct_pages = 1;
+ for (j = i + 1; j < ncands; j++)
+ {
+ if ( (cands[j].band != cands[i].band) ||
+ (0 != strcmp (cands[j].key, cands[i].key)) )
+ break;
+ if (cands[j].page != cands[j - 1].page)
+ distinct_pages++;
+ }
+ if ( (distinct_pages >= 2) &&
+ (2 * distinct_pages >= num_pages) )
+ {
+ size_t k;
+
+ for (k = i; k < j; k++)
+ cands[k].line->skip = 1;
+ }
+ i = j;
+ }
+ for (i = 0; i < ncands; i++)
+ free (cands[i].key);
+ free (cands);
+}
+
+
+/**
+ * Check whether the hyphen at the end of @a ft is part of a compound
+ * word: the word it ends already contains a hyphen, or the word
+ * @a next starts with does.
+ *
+ * @param ft text so far, ending in '-'
+ * @param next text of the next line
+ * @return 1 if the hyphen is to be kept
+ */
+static int
+is_compound (const struct EXTRACTOR_FullText *ft,
+ const char *next)
+{
+ size_t i;
+
+ for (i = ft->len - 1; i > 0; i--)
+ {
+ char c = ft->buf[i - 1];
+
+ if (' ' == c)
+ break;
+ if ('-' == c)
+ return 1;
+ }
+ for (i = 0; ('\0' != next[i]) && (' ' != next[i]); i++)
+ if ('-' == next[i])
+ return 1;
+ return 0;
+}
+
+
+void
+EXTRACTOR_fulltext_append_pages_ (struct EXTRACTOR_FullText *ft,
+ const struct EXTRACTOR_FullTextPage *pages,
+ unsigned int num_pages)
+{
+ unsigned int p;
+ unsigned int j;
+
+ for (p = 0; p < num_pages; p++)
+ for (j = 0; j < pages[p].num_lines; j++)
+ {
+ const struct EXTRACTOR_FullTextLine *l = &pages[p].lines[j];
+
+ if (l->skip)
+ continue;
+ /* re-join "hyphen-\nated" words; a compound broken at one of its
+ own hyphens ("register-\nto-register") keeps the hyphen */
+ if ( (ft->pending_space) &&
+ (! ft->pending_par) &&
+ (ft->len >= 2) &&
+ ('-' == ft->buf[ft->len - 1]) &&
+ isalpha ((unsigned char) ft->buf[ft->len - 2]) &&
+ islower ((unsigned char) l->text[0]) )
+ {
+ if (! is_compound (ft,
+ l->text))
+ {
+ ft->len--;
+ ft->buf[ft->len] = '\0';
+ }
+ ft->pending_space = 0;
+ }
+ EXTRACTOR_fulltext_append_str_ (ft,
+ l->text);
+ if (l->par_end)
+ EXTRACTOR_fulltext_paragraph_ (ft);
+ else
+ EXTRACTOR_fulltext_space_ (ft);
+ }
+}
+
+
+/* end of fulltext.c */
diff --git a/src/plugins/fulltext.h b/src/plugins/fulltext.h
@@ -0,0 +1,366 @@
+/*
+ This file is part of libextractor.
+ Copyright (C) 2026 Christian Grothoff
+
+ libextractor is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ libextractor is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with libextractor; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+/**
+ * @file plugins/fulltext.h
+ * @brief helpers shared by the plugins implementing the "fulltext" option
+ * @author Christian Grothoff
+ *
+ * The full text is reported as a single #EXTRACTOR_METATYPE_FULL_TEXT
+ * item in UTF-8 with MIME type "text/plain". Within a paragraph, all
+ * whitespace is collapsed into a single space; paragraphs are separated
+ * by exactly "\n\n"; there is no leading or trailing whitespace.
+ */
+#ifndef FULLTEXT_H
+#define FULLTEXT_H
+
+#include "extractor.h"
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Largest full text we are willing to emit, in bytes (including the
+ * terminating 0). Must stay below MAX_META_DATA in
+ * src/main/extractor_ipc.h; longer text is truncated at a character
+ * boundary.
+ */
+#define EXTRACTOR_FULLTEXT_MAX (16 * 1024 * 1024)
+
+
+/**
+ * Check whether the comma-separated plugin configuration @a config
+ * contains the token @a key (exactly, or followed by '=' or ':').
+ *
+ * @param config plugin configuration, may be NULL
+ * @param key token to look for, e.g. "fulltext"
+ * @return NULL if @a key is absent; otherwise a pointer to the
+ * character after @a key in @a config (one of '\0', ',', '='
+ * or ':')
+ */
+const char *
+EXTRACTOR_config_find_ (const char *config,
+ const char *key);
+
+
+/**
+ * Check whether the "fulltext" option is set in @a config.
+ *
+ * @param config plugin configuration, may be NULL
+ * @return 1 if full-text extraction was requested, 0 if not
+ */
+int
+EXTRACTOR_fulltext_requested_ (const char *config);
+
+
+/**
+ * A growing UTF-8 text buffer that normalizes whitespace and paragraph
+ * breaks as text is appended.
+ */
+struct EXTRACTOR_FullText
+{
+ /**
+ * The text so far, 0-terminated if @e len > 0.
+ */
+ char *buf;
+
+ /**
+ * Number of bytes in @e buf, excluding the terminating 0.
+ */
+ size_t len;
+
+ /**
+ * Allocated size of @e buf.
+ */
+ size_t cap;
+
+ /**
+ * 1 if a space must precede the next non-space text.
+ */
+ int pending_space;
+
+ /**
+ * 1 if a paragraph break must precede the next non-space text.
+ */
+ int pending_par;
+
+ /**
+ * 1 if we ran into #EXTRACTOR_FULLTEXT_MAX or out of memory;
+ * all further text is discarded.
+ */
+ int full;
+
+ /**
+ * Extraction context to keep alive while we work, may be NULL; set
+ * with #EXTRACTOR_fulltext_set_context_().
+ */
+ struct EXTRACTOR_ExtractContext *ec;
+
+ /**
+ * Units of work done since the last keep-alive.
+ */
+ unsigned int ticks;
+};
+
+
+/**
+ * Let @a ft keep the host informed that the plugin is still working
+ * (out of process, a plugin that is silent for too long is presumed
+ * hung): every few thousand characters appended, or calls to
+ * #EXTRACTOR_fulltext_tick_(), `seek (cls, 0, SEEK_CUR)` is called
+ * on @a ec.
+ *
+ * @param ft buffer to modify
+ * @param ec extraction context of the plugin
+ */
+void
+EXTRACTOR_fulltext_set_context_ (struct EXTRACTOR_FullText *ft,
+ struct EXTRACTOR_ExtractContext *ec);
+
+
+/**
+ * Account for one unit of work that did not append text (e.g. a tag
+ * that was skipped), for the keep-alive.
+ *
+ * @param ft buffer being filled
+ */
+void
+EXTRACTOR_fulltext_tick_ (struct EXTRACTOR_FullText *ft);
+
+
+/**
+ * Initialize @a ft to the empty text.
+ *
+ * @param[out] ft buffer to initialize
+ */
+void
+EXTRACTOR_fulltext_init_ (struct EXTRACTOR_FullText *ft);
+
+
+/**
+ * Append @a len bytes of @a text. Invalid UTF-8 is replaced by U+FFFD,
+ * control characters are dropped, and any run of whitespace (including
+ * line breaks and U+00A0) becomes a single space.
+ *
+ * @param ft buffer to append to
+ * @param text text to append, need not be 0-terminated
+ * @param len number of bytes in @a text
+ */
+void
+EXTRACTOR_fulltext_append_ (struct EXTRACTOR_FullText *ft,
+ const char *text,
+ size_t len);
+
+
+/**
+ * Append a 0-terminated string.
+ *
+ * @param ft buffer to append to
+ * @param text 0-terminated text to append
+ */
+void
+EXTRACTOR_fulltext_append_str_ (struct EXTRACTOR_FullText *ft,
+ const char *text);
+
+
+/**
+ * Append a single Unicode code point, subject to the same
+ * normalization as #EXTRACTOR_fulltext_append_().
+ *
+ * @param ft buffer to append to
+ * @param cp code point to append
+ */
+void
+EXTRACTOR_fulltext_append_cp_ (struct EXTRACTOR_FullText *ft,
+ uint32_t cp);
+
+
+/**
+ * Request a word break: the next text is separated from what came
+ * before by a single space (unless a paragraph break is pending).
+ *
+ * @param ft buffer to modify
+ */
+void
+EXTRACTOR_fulltext_space_ (struct EXTRACTOR_FullText *ft);
+
+
+/**
+ * End the current paragraph: the next text is preceded by "\n\n".
+ * Multiple calls without text in between have the same effect as one.
+ *
+ * @param ft buffer to modify
+ */
+void
+EXTRACTOR_fulltext_paragraph_ (struct EXTRACTOR_FullText *ft);
+
+
+/**
+ * Emit the text (if non-empty) as #EXTRACTOR_METATYPE_FULL_TEXT.
+ *
+ * @param ft buffer to emit
+ * @param ec extraction context
+ * @param plugin_name short name of the plugin
+ * @return return value of the `proc` callback (0 if nothing was emitted)
+ */
+int
+EXTRACTOR_fulltext_emit_ (const struct EXTRACTOR_FullText *ft,
+ struct EXTRACTOR_ExtractContext *ec,
+ const char *plugin_name);
+
+
+/**
+ * Release the memory held by @a ft and reset it to the empty text.
+ *
+ * @param ft buffer to clean up
+ */
+void
+EXTRACTOR_fulltext_clear_ (struct EXTRACTOR_FullText *ft);
+
+
+/**
+ * Check whether @a line (already whitespace-trimmed or not) looks like
+ * a page number or page label, e.g. "12", "- 12 -", "xiv", "Page 3",
+ * "3 of 17", "[4]", "Seite 7".
+ *
+ * @param line text to check
+ * @param len number of bytes in @a line
+ * @return 1 if @a line looks like a page number, 0 if not
+ */
+int
+EXTRACTOR_fulltext_is_page_number_ (const char *line,
+ size_t len);
+
+
+/**
+ * One line of text on a page, as collected by a plugin for paged
+ * formats (PDF, DVI) before margin filtering.
+ */
+struct EXTRACTOR_FullTextLine
+{
+ /**
+ * UTF-8 text of the line, 0-terminated, owned by the page.
+ */
+ char *text;
+
+ /**
+ * Vertical position of the line's baseline, relative to the page
+ * height, in [0,1] with 0 at the top. Negative if unknown.
+ */
+ double y;
+
+ /**
+ * 1 if this line ends a paragraph (the plugin decided that from the
+ * layout, e.g. vertical gap, indentation, short last line).
+ */
+ int par_end;
+
+ /**
+ * Set by #EXTRACTOR_fulltext_filter_margins_() if the line is page
+ * furniture (page number, running header/footer) and must be skipped.
+ */
+ int skip;
+};
+
+
+/**
+ * All lines of one page, in reading order.
+ */
+struct EXTRACTOR_FullTextPage
+{
+ /**
+ * Lines of the page.
+ */
+ struct EXTRACTOR_FullTextLine *lines;
+
+ /**
+ * Number of entries in @e lines.
+ */
+ unsigned int num_lines;
+
+ /**
+ * Allocated size of @e lines.
+ */
+ unsigned int cap_lines;
+};
+
+
+/**
+ * Append a line to @a page. @a text is copied.
+ *
+ * @param page page to append to
+ * @param text UTF-8 text of the line
+ * @param len number of bytes in @a text
+ * @param y relative vertical position in [0,1], or negative if unknown
+ * @param par_end 1 if the line ends a paragraph
+ * @return 0 on success, -1 on allocation failure
+ */
+int
+EXTRACTOR_fulltext_page_add_ (struct EXTRACTOR_FullTextPage *page,
+ const char *text,
+ size_t len,
+ double y,
+ int par_end);
+
+
+/**
+ * Mark page furniture in @a pages as `skip`: the first/last lines of
+ * a page within the top/bottom margin band that look like page numbers,
+ * or whose text (with digits ignored) recurs in the same band on at
+ * least half of the pages (when there are at least 3 pages).
+ *
+ * @param pages array of pages
+ * @param num_pages number of entries in @a pages
+ */
+void
+EXTRACTOR_fulltext_filter_margins_ (struct EXTRACTOR_FullTextPage *pages,
+ unsigned int num_pages);
+
+
+/**
+ * Append all non-skipped lines of @a pages to @a ft. Lines are joined
+ * with a space (words hyphenated across a line break are re-joined);
+ * `par_end` lines end a paragraph. A paragraph continues across page
+ * boundaries unless the last line of the page has `par_end`.
+ *
+ * @param ft buffer to append to
+ * @param pages array of pages
+ * @param num_pages number of entries in @a pages
+ */
+void
+EXTRACTOR_fulltext_append_pages_ (struct EXTRACTOR_FullText *ft,
+ const struct EXTRACTOR_FullTextPage *pages,
+ unsigned int num_pages);
+
+
+/**
+ * Release the lines of @a page.
+ *
+ * @param page page to clean up
+ */
+void
+EXTRACTOR_fulltext_page_clear_ (struct EXTRACTOR_FullTextPage *page);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/plugins/html_extractor.c b/src/plugins/html_extractor.c
@@ -22,9 +22,22 @@
* @file plugins/html_extractor.c
* @brief plugin to support HTML files
* @author Christian Grothoff
+ *
+ * With the "fulltext" option the text of the <body> is reported as one
+ * #EXTRACTOR_METATYPE_FULL_TEXT item. The file is then read as a whole
+ * (up to #MAX_FULLTEXT_INPUT bytes), converted to UTF-8 according to
+ * its byte order mark, XML declaration or META charset (sniffing UTF-8
+ * versus windows-1252 without one) and parsed by libtidy from memory.
+ * The walk over the tree drops script, style, noscript, template, the
+ * page margins (nav, aside, header and footer outside of an article or
+ * section, elements with an ARIA role of navigation, banner,
+ * contentinfo, complementary or search) and hidden elements (hidden,
+ * aria-hidden="true", display:none); block-level elements end
+ * paragraphs, as do two consecutive <br>.
*/
#include "platform.h"
#include "extractor.h"
+#include "fulltext.h"
#include <magic.h>
#if HAVE_TIDY_H
#include <tidy.h>
@@ -174,92 +187,30 @@ eof_cb (void *sourceData)
/**
- * Main entry method for the 'text/html' extraction plugin.
+ * Report the TITLE and the META tags found in the document head.
*
- * @param ec extraction context provided to the plugin
+ * @param ec extraction context
+ * @param doc parsed document
+ * @return 1 if the caller should stop extracting, 0 to continue
*/
-void
-EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec);
-
-void
-EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
+static int
+report_head (struct EXTRACTOR_ExtractContext *ec,
+ TidyDoc doc)
{
- TidyDoc doc;
TidyNode head;
TidyNode child;
TidyNode title;
- TidyInputSource src;
const char *name;
TidyBuffer tbuf;
TidyAttr attr;
enum EXTRACTOR_MetaType type;
- ssize_t iret;
- void *data;
- const char *mime;
- if (-1 == (iret = ec->read (ec->cls,
- &data,
- 16 * 1024)))
- return;
- if (NULL == (mime = magic_buffer (magic, data, iret)))
- return;
- if (0 != strncmp (mime,
- "text/html",
- strlen ("text/html")))
- return; /* not HTML */
-
- if (0 != ec->seek (ec->cls, 0, SEEK_SET))
- return; /* seek failed !? */
-
- tidyInitSource (&src, ec,
- &get_byte_cb,
- &unget_byte_cb,
- &eof_cb);
- if (NULL == (doc = tidyCreate ()))
- return;
- tidySetReportFilter (doc, &report_cb);
- tidySetAppData (doc, ec);
- if (0 > tidyParseSource (doc, &src))
- {
- tidyRelease (doc);
- return;
- }
- if (1 != tidyStatus (doc))
- {
- tidyRelease (doc);
- return;
- }
if (NULL == (head = tidyGetHead (doc)))
- {
- tidyRelease (doc);
- return;
- }
+ return 0;
for (child = tidyGetChild (head); NULL != child; child = tidyGetNext (child))
{
switch (tidyNodeGetType (child))
{
- case TidyNode_Root:
- break;
- case TidyNode_DocType:
- break;
- case TidyNode_Comment:
- break;
- case TidyNode_ProcIns:
- break;
- case TidyNode_Text:
- break;
- case TidyNode_CDATA:
- break;
- case TidyNode_Section:
- break;
- case TidyNode_Asp:
- break;
- case TidyNode_Jste:
- break;
- case TidyNode_Php:
- break;
- case TidyNode_XmlDecl:
- break;
case TidyNode_Start:
case TidyNode_StartEnd:
name = tidyNodeGetName (child);
@@ -282,7 +233,7 @@ EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
tbuf.size))
{
tidyBufFree (&tbuf);
- goto CLEANUP;
+ return 1;
}
tidyBufFree (&tbuf);
break;
@@ -312,17 +263,935 @@ EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
"text/plain",
name,
strlen (name) + 1))
- goto CLEANUP;
+ return 1;
break;
}
break;
- case TidyNode_End:
+ default:
+ break;
+ }
+ }
+ return 0;
+}
+
+
+/* ------------------------------------------------------------------ */
+/* Full text */
+/* ------------------------------------------------------------------ */
+
+/**
+ * Largest HTML file we read for the "fulltext" option. libtidy keeps
+ * the whole tree in memory (several times the size of the input), and
+ * the text we may emit is capped at #EXTRACTOR_FULLTEXT_MAX anyway;
+ * longer files are truncated.
+ */
+#define MAX_FULLTEXT_INPUT (16 * 1024 * 1024)
+
+/**
+ * How deep into the element tree we descend; anything nested deeper
+ * is ignored. Real documents stay far below this, and it bounds the
+ * stack use of #walk_children().
+ */
+#define MAX_DEPTH 256
+
+/**
+ * How many attributes of one element we look at.
+ */
+#define MAX_ATTRS 64
+
+/**
+ * How far into the file we look for a charset declaration.
+ */
+#define CHARSET_SCAN 8192
+
+
+/**
+ * State of the full text walk over the tidy tree.
+ */
+struct TextWalk
+{
+ /**
+ * Text collected so far.
+ */
+ struct EXTRACTOR_FullText ft;
+
+ /**
+ * The document.
+ */
+ TidyDoc doc;
+
+ /**
+ * Scratch buffer for text node values.
+ */
+ TidyBuffer tb;
+
+ /**
+ * 1 if the last thing we saw was a <br> (with nothing but white
+ * space after it), so another one ends the paragraph.
+ */
+ int after_br;
+};
+
+
+/**
+ * Find @a needle in @a hay, ignoring ASCII case.
+ *
+ * @param hay buffer to search
+ * @param hlen number of bytes in @a hay
+ * @param needle 0-terminated lower-case string to look for
+ * @return offset of the match, or @a hlen if there is none
+ */
+static size_t
+find_ci (const char *hay,
+ size_t hlen,
+ const char *needle)
+{
+ size_t nlen = strlen (needle);
+
+ if (nlen > hlen)
+ return hlen;
+ for (size_t i = 0; i + nlen <= hlen; i++)
+ {
+ size_t k;
+
+ for (k = 0; k < nlen; k++)
+ if (tolower ((unsigned char) hay[i + k]) != needle[k])
+ break;
+ if (k == nlen)
+ return i;
+ }
+ return hlen;
+}
+
+
+/**
+ * Parse a charset name following "charset" / "encoding" at @a p:
+ * optional white space, '=', optional white space and quotes, then the
+ * name.
+ *
+ * @param p where to start
+ * @param end end of the buffer
+ * @param[out] cs where to write the name (0-terminated)
+ * @param cs_size number of bytes available in @a cs
+ * @return 1 if a name was found
+ */
+static int
+parse_charset_value (const char *p,
+ const char *end,
+ char *cs,
+ size_t cs_size)
+{
+ size_t o = 0;
+
+ while ( (p < end) && isspace ((unsigned char) *p))
+ p++;
+ if ( (p >= end) || ('=' != *p) )
+ return 0;
+ p++;
+ while ( (p < end) &&
+ (isspace ((unsigned char) *p) || ('"' == *p) || ('\'' == *p) ) )
+ p++;
+ while ( (p < end) &&
+ (o + 1 < cs_size) &&
+ (isalnum ((unsigned char) *p) ||
+ ('-' == *p) || ('_' == *p) || ('.' == *p) || (':' == *p) ) )
+ cs[o++] = (char) tolower ((unsigned char) *p++);
+ cs[o] = '\0';
+ return (0 != o);
+}
+
+
+/**
+ * Is @a data valid UTF-8?
+ *
+ * @param data bytes to check
+ * @param len number of bytes in @a data
+ * @return 1 if valid
+ */
+static int
+is_utf8 (const unsigned char *data,
+ size_t len)
+{
+ size_t i = 0;
+
+ while (i < len)
+ {
+ unsigned int n;
+ uint32_t cp;
+
+ if (data[i] < 0x80)
+ {
+ i++;
+ continue;
+ }
+ if ( (data[i] >= 0xC2) && (data[i] <= 0xDF) )
+ n = 1;
+ else if ( (data[i] >= 0xE0) && (data[i] <= 0xEF) )
+ n = 2;
+ else if ( (data[i] >= 0xF0) && (data[i] <= 0xF4) )
+ n = 3;
+ else
+ return 0;
+ if (n > len - i - 1)
+ return 0;
+ cp = data[i] & (0x3F >> n);
+ for (unsigned int k = 1; k <= n; k++)
+ {
+ if (0x80 != (data[i + k] & 0xC0))
+ return 0;
+ cp = (cp << 6) | (data[i + k] & 0x3F);
+ }
+ if ( ( (2 == n) && (cp < 0x800) ) ||
+ ( (3 == n) && ( (cp < 0x10000) || (cp > 0x10FFFF) ) ) ||
+ ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
+ return 0;
+ i += n + 1;
+ }
+ return 1;
+}
+
+
+/**
+ * Determine the character set of an HTML document: a byte order mark,
+ * then an XML declaration, then a META charset / http-equiv
+ * declaration; without any of those, UTF-8 if the bytes are valid
+ * UTF-8 and windows-1252 otherwise. Labels are mapped as browsers do
+ * (ISO-8859-1 and ASCII mean windows-1252; a META tag claiming UTF-16
+ * means UTF-8).
+ *
+ * @param data start of the document
+ * @param len number of bytes in @a data
+ * @param[out] cs where to write the iconv name of the charset
+ * @param cs_size number of bytes available in @a cs
+ * @return number of BOM bytes to skip
+ */
+static size_t
+sniff_charset (const char *data,
+ size_t len,
+ char *cs,
+ size_t cs_size)
+{
+ const unsigned char *u = (const unsigned char *) data;
+ size_t scan = (len > CHARSET_SCAN) ? CHARSET_SCAN : len;
+ char found[64];
+ int have = 0;
+
+ if ( (len >= 3) && (0xEF == u[0]) && (0xBB == u[1]) && (0xBF == u[2]) )
+ {
+ snprintf (cs, cs_size, "UTF-8");
+ return 3;
+ }
+ if ( (len >= 2) && (0xFE == u[0]) && (0xFF == u[1]) )
+ {
+ snprintf (cs, cs_size, "UTF-16BE");
+ return 2;
+ }
+ if ( (len >= 2) && (0xFF == u[0]) && (0xFE == u[1]) )
+ {
+ snprintf (cs, cs_size, "UTF-16LE");
+ return 2;
+ }
+ if ( (scan >= 5) && (0 == memcmp (data, "<?xml", 5)) )
+ {
+ const char *gt = memchr (data, '>', scan);
+ size_t dlen = (NULL == gt) ? scan : (size_t) (gt - data);
+ size_t off = find_ci (data, dlen, "encoding");
+
+ if (off < dlen)
+ have = parse_charset_value (&data[off + strlen ("encoding")],
+ &data[dlen],
+ found,
+ sizeof (found));
+ }
+ for (size_t pos = 0; (! have) && (pos < scan); )
+ {
+ size_t off = find_ci (&data[pos], scan - pos, "<meta");
+ const char *tag;
+ const char *gt;
+ size_t tlen;
+ size_t c;
+
+ if (off >= scan - pos)
+ break;
+ tag = &data[pos + off];
+ gt = memchr (tag, '>', scan - pos - off);
+ tlen = (NULL == gt) ? scan - pos - off : (size_t) (gt - tag);
+ c = find_ci (tag, tlen, "charset");
+ if (c < tlen)
+ have = parse_charset_value (&tag[c + strlen ("charset")],
+ &tag[tlen],
+ found,
+ sizeof (found));
+ pos += off + 5;
+ }
+ if (have)
+ {
+ if ( (0 == strcmp (found, "iso-8859-1")) ||
+ (0 == strcmp (found, "iso8859-1")) ||
+ (0 == strcmp (found, "latin1")) ||
+ (0 == strcmp (found, "l1")) ||
+ (0 == strcmp (found, "us-ascii")) ||
+ (0 == strcmp (found, "ascii")) ||
+ (0 == strcmp (found, "cp1252")) ||
+ (0 == strcmp (found, "windows-1252")) )
+ snprintf (cs, cs_size, "WINDOWS-1252");
+ else if ( (0 == strncmp (found, "utf-16", 6)) ||
+ (0 == strncmp (found, "utf16", 5)) ||
+ (0 == strcmp (found, "utf8")) ||
+ (0 == strcmp (found, "utf-8")) )
+ snprintf (cs, cs_size, "UTF-8");
+ else
+ snprintf (cs, cs_size, "%s", found);
+ return 0;
+ }
+ if (is_utf8 (u, len))
+ snprintf (cs, cs_size, "UTF-8");
+ else
+ snprintf (cs, cs_size, "WINDOWS-1252");
+ return 0;
+}
+
+
+/**
+ * Code points of windows-1252 bytes 0x80 to 0x9F.
+ */
+static const uint16_t cp1252[32] = {
+ 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
+ 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
+ 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
+ 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178
+};
+
+
+/**
+ * Append the UTF-8 encoding of @a cp (which must be below 0x10000)
+ * to @a out.
+ *
+ * @param out output buffer, at least 3 bytes must be available
+ * @param cp code point
+ * @return number of bytes written
+ */
+static size_t
+put_utf8 (char *out,
+ uint32_t cp)
+{
+ if (cp < 0x80)
+ {
+ out[0] = (char) cp;
+ return 1;
+ }
+ if (cp < 0x800)
+ {
+ out[0] = (char) (0xC0 | (cp >> 6));
+ out[1] = (char) (0x80 | (cp & 0x3F));
+ return 2;
+ }
+ out[0] = (char) (0xE0 | (cp >> 12));
+ out[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ out[2] = (char) (0x80 | (cp & 0x3F));
+ return 3;
+}
+
+
+/**
+ * Convert @a data from charset @a cs to UTF-8. Undecodable bytes
+ * become U+FFFD. If the charset is unknown, the bytes are taken as
+ * UTF-8 if they are valid UTF-8, and as windows-1252 otherwise.
+ *
+ * @param data input
+ * @param len number of bytes in @a data
+ * @param cs charset name as understood by iconv
+ * @param[out] olen number of bytes in the result
+ * @return converted text, NULL on allocation failure; caller frees
+ */
+static char *
+to_utf8 (const char *data,
+ size_t len,
+ const char *cs,
+ size_t *olen)
+{
+ char *out;
+ size_t o = 0;
+
+ *olen = 0;
+ if (0 == strcmp (cs, "UTF-8"))
+ {
+ if (NULL == (out = malloc (len + 1)))
+ return NULL;
+ memcpy (out, data, len);
+ *olen = len;
+ return out;
+ }
+ /* no conversion produces more than 3 bytes of UTF-8 per input byte
+ (a UTF-16 surrogate pair is 4 bytes in and 4 bytes out) */
+ if (NULL == (out = malloc (3 * len + 4)))
+ return NULL;
+#if HAVE_ICONV
+ if (0 != strcmp (cs, "WINDOWS-1252"))
+ {
+ iconv_t cd = iconv_open ("UTF-8", cs);
+
+ if ((iconv_t) -1 != cd)
+ {
+ char *ib = (char *) data;
+ size_t il = len;
+ char *ob = out;
+ size_t ol = 3 * len;
+
+ while (il > 0)
+ {
+ if ((size_t) -1 !=
+ iconv (cd, &ib, &il, &ob, &ol))
+ break;
+ if ( (E2BIG == errno) ||
+ (ol < 3) )
+ break;
+ /* skip one undecodable byte */
+ ob += put_utf8 (ob, 0xFFFD);
+ ol -= 3;
+ ib++;
+ il--;
+ }
+ iconv_close (cd);
+ *olen = (size_t) (ob - out);
+ return out;
+ }
+ if (is_utf8 ((const unsigned char *) data, len))
+ {
+ memcpy (out, data, len);
+ *olen = len;
+ return out;
+ }
+ }
+#else
+ if ( (0 != strcmp (cs, "WINDOWS-1252")) &&
+ (is_utf8 ((const unsigned char *) data, len)) )
+ {
+ memcpy (out, data, len);
+ *olen = len;
+ return out;
+ }
+#endif
+ for (size_t i = 0; i < len; i++)
+ {
+ unsigned char c = (unsigned char) data[i];
+
+ if (c < 0x80)
+ out[o++] = (char) c;
+ else if (c < 0xA0)
+ o += put_utf8 (&out[o], cp1252[c - 0x80]);
+ else
+ o += put_utf8 (&out[o], c);
+ }
+ *olen = o;
+ return out;
+}
+
+
+/**
+ * Read the whole input (up to #MAX_FULLTEXT_INPUT bytes).
+ *
+ * @param ec extraction context, positioned at the start
+ * @param[out] len number of bytes read
+ * @return the data, NULL on error; caller frees
+ */
+static char *
+read_all (struct EXTRACTOR_ExtractContext *ec,
+ size_t *len)
+{
+ char *buf = NULL;
+ size_t size = 0;
+ size_t cap = 0;
+
+ *len = 0;
+ while (size < MAX_FULLTEXT_INPUT)
+ {
+ void *data;
+ ssize_t got;
+ size_t want = MAX_FULLTEXT_INPUT - size;
+
+ if (want > 64 * 1024)
+ want = 64 * 1024;
+ got = ec->read (ec->cls, &data, want);
+ if (got <= 0)
+ break;
+ if ((size_t) got > want)
+ got = (ssize_t) want;
+ if (size + (size_t) got > cap)
+ {
+ size_t ncap = (0 == cap) ? 64 * 1024 : cap;
+ char *nbuf;
+
+ while (ncap < size + (size_t) got)
+ ncap *= 2;
+ if (ncap > MAX_FULLTEXT_INPUT)
+ ncap = MAX_FULLTEXT_INPUT;
+ if (NULL == (nbuf = realloc (buf, ncap)))
+ break;
+ buf = nbuf;
+ cap = ncap;
+ }
+ memcpy (&buf[size], data, (size_t) got);
+ size += (size_t) got;
+ }
+ *len = size;
+ return buf;
+}
+
+
+/**
+ * Does the space-separated token list @a list contain @a tok (ASCII
+ * case-insensitively)?
+ *
+ * @param list token list
+ * @param tok token to look for
+ * @return 1 if found
+ */
+static int
+has_token (const char *list,
+ const char *tok)
+{
+ size_t tlen = strlen (tok);
+ const char *p = list;
+
+ while ('\0' != *p)
+ {
+ size_t n;
+
+ while (isspace ((unsigned char) *p))
+ p++;
+ n = 0;
+ while ( ('\0' != p[n]) && (! isspace ((unsigned char) p[n])) )
+ n++;
+ if ( (n == tlen) &&
+ (0 == strncasecmp (p, tok, n)) )
+ return 1;
+ p += n;
+ }
+ return 0;
+}
+
+
+/**
+ * Does @a style (a CSS declaration list) hide the element, i.e. set
+ * display:none or visibility:hidden?
+ *
+ * @param style value of the style attribute
+ * @return 1 if the element is hidden
+ */
+static int
+style_hides (const char *style)
+{
+ char norm[256];
+ size_t o = 0;
+
+ for (size_t i = 0; ('\0' != style[i]) && (o + 1 < sizeof (norm)); i++)
+ if (! isspace ((unsigned char) style[i]))
+ norm[o++] = (char) tolower ((unsigned char) style[i]);
+ norm[o] = '\0';
+ return ( (NULL != strstr (norm, "display:none")) ||
+ (NULL != strstr (norm, "visibility:hidden")) );
+}
+
+
+/**
+ * Should the element @a node (and everything in it) be left out of the
+ * full text because of its attributes: hidden, aria-hidden="true", an
+ * inline style that hides it, or an ARIA landmark role for the page
+ * margins (navigation, banner, contentinfo, complementary, search) or a
+ * page break marker?
+ *
+ * @param node element to check
+ * @return 1 to skip the element
+ */
+static int
+attrs_skip (TidyNode node)
+{
+ static const char *roles[] = {
+ "navigation", "banner", "contentinfo", "complementary", "search",
+ "doc-pagebreak", NULL
+ };
+ unsigned int n = 0;
+
+ for (TidyAttr a = tidyAttrFirst (node);
+ (NULL != a) && (n < MAX_ATTRS);
+ a = tidyAttrNext (a), n++)
+ {
+ const char *name = tidyAttrName (a);
+ const char *val = tidyAttrValue (a);
+
+ if (NULL == name)
+ continue;
+ if (0 == strcasecmp (name, "hidden"))
+ {
+ /* hidden="until-found" is collapsed content that find-in-page
+ reveals, i.e. part of the document text */
+ if ( (NULL == val) ||
+ (0 != strcasecmp (val, "until-found")) )
+ return 1;
+ continue;
+ }
+ if (NULL == val)
+ continue;
+ if ( (0 == strcasecmp (name, "aria-hidden")) &&
+ (0 == strcasecmp (val, "true")) )
+ return 1;
+ if ( (0 == strcasecmp (name, "style")) &&
+ (style_hides (val)) )
+ return 1;
+ if (0 == strcasecmp (name, "role"))
+ for (unsigned int i = 0; NULL != roles[i]; i++)
+ if (has_token (val, roles[i]))
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Is @a id an element whose content is never document text?
+ *
+ * @param id element to check
+ * @return 1 to skip the element
+ */
+static int
+tag_skip (TidyTagId id)
+{
+ switch (id)
+ {
+ case TidyTag_HEAD:
+ case TidyTag_TITLE:
+ case TidyTag_SCRIPT:
+ case TidyTag_STYLE:
+ case TidyTag_NOSCRIPT:
+ case TidyTag_TEMPLATE:
+ case TidyTag_NAV:
+ case TidyTag_ASIDE:
+ case TidyTag_SELECT:
+ case TidyTag_DATALIST:
+ case TidyTag_IFRAME:
+ case TidyTag_OBJECT:
+ case TidyTag_APPLET:
+ case TidyTag_RP:
+ case TidyTag_RT:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+
+/**
+ * Does @a id end a paragraph (i.e. is it a block-level element)?
+ *
+ * @param id element to check
+ * @return 1 for block-level elements
+ */
+static int
+tag_is_block (TidyTagId id)
+{
+ switch (id)
+ {
+ case TidyTag_ADDRESS:
+ case TidyTag_ARTICLE:
+ case TidyTag_BLOCKQUOTE:
+ case TidyTag_BODY:
+ case TidyTag_CAPTION:
+ case TidyTag_CENTER:
+ case TidyTag_DD:
+ case TidyTag_DETAILS:
+ case TidyTag_DIALOG:
+ case TidyTag_DIR:
+ case TidyTag_DIV:
+ case TidyTag_DL:
+ case TidyTag_DT:
+ case TidyTag_FIELDSET:
+ case TidyTag_FIGCAPTION:
+ case TidyTag_FIGURE:
+ case TidyTag_FOOTER:
+ case TidyTag_FORM:
+ case TidyTag_FRAMESET:
+ case TidyTag_H1:
+ case TidyTag_H2:
+ case TidyTag_H3:
+ case TidyTag_H4:
+ case TidyTag_H5:
+ case TidyTag_H6:
+ case TidyTag_HEADER:
+ case TidyTag_HGROUP:
+ case TidyTag_HR:
+ case TidyTag_HTML:
+ case TidyTag_LEGEND:
+ case TidyTag_LI:
+ case TidyTag_LISTING:
+ case TidyTag_MAIN:
+ case TidyTag_MENU:
+ case TidyTag_NOFRAMES:
+ case TidyTag_OL:
+ case TidyTag_P:
+ case TidyTag_PLAINTEXT:
+ case TidyTag_PRE:
+ case TidyTag_SECTION:
+ case TidyTag_SUMMARY:
+ case TidyTag_TABLE:
+ case TidyTag_TBODY:
+ case TidyTag_TD:
+ case TidyTag_TFOOT:
+ case TidyTag_TH:
+ case TidyTag_THEAD:
+ case TidyTag_TR:
+ case TidyTag_UL:
+ case TidyTag_XMP:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+
+/**
+ * Does @a id start a sectioning context, inside which <header> and
+ * <footer> belong to that section rather than to the page?
+ *
+ * @param id element to check
+ * @return 1 for article, aside, main, nav and section
+ */
+static int
+tag_is_sectioning (TidyTagId id)
+{
+ return (TidyTag_ARTICLE == id) ||
+ (TidyTag_ASIDE == id) ||
+ (TidyTag_MAIN == id) ||
+ (TidyTag_NAV == id) ||
+ (TidyTag_SECTION == id);
+}
+
+
+/**
+ * Append the text of a text node.
+ *
+ * @param w walk state
+ * @param node the text node
+ */
+static void
+add_text (struct TextWalk *w,
+ TidyNode node)
+{
+ tidyBufClear (&w->tb);
+ if (! tidyNodeGetValue (w->doc, node, &w->tb))
+ return;
+ if (0 == w->tb.size)
+ return;
+ for (uint i = 0; i < w->tb.size; i++)
+ if (! isspace (w->tb.bp[i]))
+ {
+ w->after_br = 0;
+ break;
+ }
+ EXTRACTOR_fulltext_append_ (&w->ft,
+ (const char *) w->tb.bp,
+ w->tb.size);
+}
+
+
+/**
+ * Append the text of the children of @a node.
+ *
+ * @param w walk state
+ * @param node parent node
+ * @param depth nesting depth of @a node
+ * @param in_section 1 if @a node is inside a sectioning element
+ */
+static void
+walk_children (struct TextWalk *w,
+ TidyNode node,
+ unsigned int depth,
+ int in_section)
+{
+ if (depth >= MAX_DEPTH)
+ return;
+ for (TidyNode c = tidyGetChild (node);
+ (NULL != c) && (! w->ft.full);
+ c = tidyGetNext (c))
+ {
+ TidyTagId id;
+ int block;
+
+ switch (tidyNodeGetType (c))
+ {
+ case TidyNode_Text:
+ case TidyNode_CDATA:
+ add_text (w, c);
+ break;
+ case TidyNode_Start:
+ case TidyNode_StartEnd:
+ id = tidyNodeGetId (c);
+ if (tag_skip (id))
+ break;
+ /* a page-level header or footer is page furniture; one inside
+ an article or section is that section's heading or byline */
+ if ( ( (TidyTag_HEADER == id) || (TidyTag_FOOTER == id) ) &&
+ (! in_section) )
+ break;
+ if (attrs_skip (c))
+ break;
+ if (TidyTag_BR == id)
+ {
+ if (w->after_br)
+ {
+ EXTRACTOR_fulltext_paragraph_ (&w->ft);
+ w->after_br = 0;
+ }
+ else
+ {
+ EXTRACTOR_fulltext_space_ (&w->ft);
+ w->after_br = 1;
+ }
+ break;
+ }
+ block = tag_is_block (id);
+ if (block)
+ {
+ EXTRACTOR_fulltext_paragraph_ (&w->ft);
+ w->after_br = 0;
+ }
+ walk_children (w,
+ c,
+ depth + 1,
+ in_section || tag_is_sectioning (id));
+ if (block)
+ {
+ EXTRACTOR_fulltext_paragraph_ (&w->ft);
+ w->after_br = 0;
+ }
break;
default:
break;
}
}
+}
+
+
+/**
+ * Extract meta data and the full text. Unlike the default mode, this
+ * reads the whole file, converts it to UTF-8 according to its declared
+ * (or sniffed) charset and hands tidy a UTF-8 buffer.
+ *
+ * @param ec extraction context, positioned at the start of the file
+ */
+static void
+extract_fulltext (struct EXTRACTOR_ExtractContext *ec)
+{
+ struct TextWalk w;
+ TidyBuffer in;
+ TidyNode body;
+ char cs[64];
+ char *raw;
+ char *utf8;
+ size_t raw_len;
+ size_t utf8_len;
+ size_t skip;
+
+ if (NULL == (raw = read_all (ec, &raw_len)))
+ return;
+ skip = sniff_charset (raw, raw_len, cs, sizeof (cs));
+ utf8 = to_utf8 (&raw[skip], raw_len - skip, cs, &utf8_len);
+ free (raw);
+ if (NULL == utf8)
+ return;
+ if (0 == utf8_len)
+ {
+ free (utf8);
+ return;
+ }
+ memset (&w, 0, sizeof (w));
+ EXTRACTOR_fulltext_init_ (&w.ft);
+ EXTRACTOR_fulltext_set_context_ (&w.ft, ec);
+ if (NULL == (w.doc = tidyCreate ()))
+ {
+ free (utf8);
+ return;
+ }
+ tidySetReportFilter (w.doc, &report_cb);
+ tidySetInCharEncoding (w.doc, "utf8");
+ tidyBufInit (&in);
+ tidyBufAttach (&in, (byte *) utf8, (uint) utf8_len);
+ tidyBufInit (&w.tb);
+ if (0 > tidyParseBuffer (w.doc, &in))
+ goto CLEANUP;
+ /* the default mode insists on status 1 (warnings), which a document
+ that no longer has an encoding problem may not produce; errors (2)
+ still suppress the head items as they do there */
+ if ( (2 > tidyStatus (w.doc)) &&
+ (0 != report_head (ec, w.doc)) )
+ goto CLEANUP;
+ if (NULL != (body = tidyGetBody (w.doc)))
+ walk_children (&w, body, 0, 0);
+ (void) EXTRACTOR_fulltext_emit_ (&w.ft, ec, "html");
CLEANUP:
+ EXTRACTOR_fulltext_clear_ (&w.ft);
+ tidyBufFree (&w.tb);
+ tidyBufDetach (&in);
+ tidyRelease (w.doc);
+ free (utf8);
+}
+
+
+/**
+ * Main entry method for the 'text/html' extraction plugin.
+ *
+ * @param ec extraction context provided to the plugin
+ */
+void
+EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec);
+
+void
+EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
+{
+ TidyDoc doc;
+ TidyInputSource src;
+ ssize_t iret;
+ void *data;
+ const char *mime;
+
+ if (-1 == (iret = ec->read (ec->cls,
+ &data,
+ 16 * 1024)))
+ return;
+ if (NULL == (mime = magic_buffer (magic, data, iret)))
+ return;
+ if (0 != strncmp (mime,
+ "text/html",
+ strlen ("text/html")))
+ return; /* not HTML */
+
+ if (0 != ec->seek (ec->cls, 0, SEEK_SET))
+ return; /* seek failed !? */
+
+ if (EXTRACTOR_fulltext_requested_ (ec->config))
+ {
+ extract_fulltext (ec);
+ return;
+ }
+ tidyInitSource (&src, ec,
+ &get_byte_cb,
+ &unget_byte_cb,
+ &eof_cb);
+ if (NULL == (doc = tidyCreate ()))
+ return;
+ tidySetReportFilter (doc, &report_cb);
+ tidySetAppData (doc, ec);
+ if (0 > tidyParseSource (doc, &src))
+ {
+ tidyRelease (doc);
+ return;
+ }
+ if (1 != tidyStatus (doc))
+ {
+ tidyRelease (doc);
+ return;
+ }
+ (void) report_head (ec, doc);
tidyRelease (doc);
}
diff --git a/src/plugins/mediaffmpeg.c b/src/plugins/mediaffmpeg.c
@@ -201,6 +201,18 @@ EXTRACTOR_ffmpeg_close_ (struct EXTRACTOR_FFmpegIO *io)
int
+EXTRACTOR_ffmpeg_reopen_ (struct EXTRACTOR_FFmpegIO *io)
+{
+ EXTRACTOR_ffmpeg_close_ (io);
+ if (0 != io->ec->seek (io->ec->cls,
+ 0,
+ SEEK_SET))
+ return -1;
+ return EXTRACTOR_ffmpeg_open_ (io);
+}
+
+
+int
EXTRACTOR_ffmpeg_open_stream_ (struct EXTRACTOR_FFmpegIO *io,
enum AVMediaType type,
AVCodecContext **ctx)
diff --git a/src/plugins/mediaffmpeg.h b/src/plugins/mediaffmpeg.h
@@ -102,6 +102,18 @@ EXTRACTOR_ffmpeg_close_ (struct EXTRACTOR_FFmpegIO *io);
/**
+ * Close @a io, rewind the extraction context and open it again, so that
+ * the next av_read_frame() starts at the beginning of the file with
+ * fresh demuxer state (and no stream discarded).
+ *
+ * @param[in,out] io I/O state to reopen
+ * @return 0 on success; on failure @a io is closed
+ */
+int
+EXTRACTOR_ffmpeg_reopen_ (struct EXTRACTOR_FFmpegIO *io);
+
+
+/**
* Find the best stream of @a type and open a decoder for it.
*
* @param io open I/O state
diff --git a/src/plugins/mediagst.c b/src/plugins/mediagst.c
@@ -48,9 +48,11 @@ feed_data (GstElement *appsrc,
ssize_t data_len;
uint8_t *le_data;
+ g_mutex_lock (&io->lock);
if ( (0 != io->length) &&
(io->offset >= io->length) )
{
+ g_mutex_unlock (&io->lock);
gst_app_src_end_of_stream (GST_APP_SRC (appsrc));
return;
}
@@ -59,12 +61,14 @@ feed_data (GstElement *appsrc,
size = io->length - io->offset;
if (0 == size)
{
+ g_mutex_unlock (&io->lock);
gst_app_src_end_of_stream (GST_APP_SRC (appsrc));
return;
}
mem = gst_allocator_alloc (NULL, size, NULL);
if (! gst_memory_map (mem, &mi, GST_MAP_WRITE))
{
+ g_mutex_unlock (&io->lock);
gst_memory_unref (mem);
gst_app_src_end_of_stream (GST_APP_SRC (appsrc));
return;
@@ -88,6 +92,7 @@ feed_data (GstElement *appsrc,
gst_memory_unmap (mem, &mi);
if (0 == accumulated)
{
+ g_mutex_unlock (&io->lock);
gst_memory_unref (mem);
gst_app_src_end_of_stream (GST_APP_SRC (appsrc));
return;
@@ -99,6 +104,7 @@ feed_data (GstElement *appsrc,
GST_BUFFER_OFFSET (buffer) = io->offset;
GST_BUFFER_OFFSET_END (buffer) = io->offset + accumulated;
io->offset += accumulated;
+ g_mutex_unlock (&io->lock);
gst_app_src_push_buffer (GST_APP_SRC (appsrc), buffer);
}
@@ -118,31 +124,35 @@ seek_data (GstElement *appsrc,
{
int64_t ret;
+ gboolean ok;
+
(void) appsrc;
+ g_mutex_lock (&io->lock);
ret = io->ec->seek (io->ec->cls,
position,
SEEK_SET);
if (0 > ret)
+ {
+ g_mutex_unlock (&io->lock);
return FALSE;
+ }
io->offset = (uint64_t) ret;
- return io->offset == position;
+ ok = (io->offset == position);
+ g_mutex_unlock (&io->lock);
+ return ok;
}
/**
- * Implementation of playbin's "source-setup" signal: configure the
- * appsrc it created for our "appsrc://" URI.
+ * Configure @a source to read from the extraction context of @a io.
*
- * @param playbin the pipeline
* @param source the appsrc to configure
* @param io our execution context
*/
static void
-source_setup (GstElement *playbin,
- GstElement *source,
- struct EXTRACTOR_GstIO *io)
+configure_appsrc (GstElement *source,
+ struct EXTRACTOR_GstIO *io)
{
- (void) playbin;
if (0 != io->length)
g_object_set (source,
"size", (gint64) io->length,
@@ -167,6 +177,71 @@ source_setup (GstElement *playbin,
}
+/**
+ * Implementation of playbin's "source-setup" signal: configure the
+ * appsrc it created for our "appsrc://" URI.
+ *
+ * @param playbin the pipeline
+ * @param source the appsrc to configure
+ * @param io our execution context
+ */
+static void
+source_setup (GstElement *playbin,
+ GstElement *source,
+ struct EXTRACTOR_GstIO *io)
+{
+ (void) playbin;
+ configure_appsrc (source,
+ io);
+}
+
+
+/**
+ * Initialize @a io for reading @a ec from the beginning.
+ *
+ * @param[out] io I/O state to initialize
+ * @param ec extraction context to read from
+ */
+static void
+io_init (struct EXTRACTOR_GstIO *io,
+ struct EXTRACTOR_ExtractContext *ec)
+{
+ uint64_t size;
+
+ io->ec = ec;
+ io->offset = 0;
+ /* a zero-filled GMutex is a valid, unlocked mutex that needs no
+ g_mutex_clear(), which saves every caller a cleanup call */
+ memset (&io->lock, 0, sizeof (io->lock));
+ io->last_ping = g_get_monotonic_time ();
+ size = ec->get_size (ec->cls);
+ io->length = (UINT64_MAX == size) ? 0 : size;
+ /* an earlier pipeline may have left the context anywhere */
+ (void) ec->seek (ec->cls,
+ 0,
+ SEEK_SET);
+}
+
+
+GstElement *
+EXTRACTOR_gst_appsrc_ (struct EXTRACTOR_GstIO *io,
+ struct EXTRACTOR_ExtractContext *ec)
+{
+ GstElement *source;
+
+ if (! gst_is_initialized ())
+ gst_init (NULL, NULL);
+ io_init (io,
+ ec);
+ if (NULL == (source = gst_element_factory_make ("appsrc",
+ NULL)))
+ return NULL;
+ configure_appsrc (source,
+ io);
+ return source;
+}
+
+
GstElement *
EXTRACTOR_gst_playbin_ (struct EXTRACTOR_GstIO *io,
struct EXTRACTOR_ExtractContext *ec,
@@ -175,14 +250,11 @@ EXTRACTOR_gst_playbin_ (struct EXTRACTOR_GstIO *io,
GstElement *audio_sink)
{
GstElement *pipeline;
- uint64_t size;
if (! gst_is_initialized ())
gst_init (NULL, NULL);
- io->ec = ec;
- io->offset = 0;
- size = ec->get_size (ec->cls);
- io->length = (UINT64_MAX == size) ? 0 : size;
+ io_init (io,
+ ec);
pipeline = gst_element_factory_make ("playbin", NULL);
if ( (NULL == pipeline) ||
(NULL == video_sink) ||
@@ -211,6 +283,79 @@ EXTRACTOR_gst_playbin_ (struct EXTRACTOR_GstIO *io,
}
+void
+EXTRACTOR_gst_keepalive_ (struct EXTRACTOR_GstIO *io)
+{
+ gint64 now = g_get_monotonic_time ();
+
+ if (now - io->last_ping < 100 * G_TIME_SPAN_MILLISECOND)
+ return;
+ io->last_ping = now;
+ g_mutex_lock (&io->lock);
+ /* a no-op seek makes the plugin side of the IPC tell the host that
+ we are still busy (MESSAGE_ALIVE) */
+ (void) io->ec->seek (io->ec->cls,
+ 0,
+ SEEK_CUR);
+ g_mutex_unlock (&io->lock);
+}
+
+
+int
+EXTRACTOR_gst_wait_async_alive_ (GstElement *pipeline,
+ GstClockTime timeout,
+ struct EXTRACTOR_GstIO *io)
+{
+ GstBus *bus;
+ GstMessage *msg;
+ GstClockTime end;
+ int ret = 0;
+
+ if (0 == timeout)
+ return 0;
+ if (NULL == (bus = gst_element_get_bus (pipeline)))
+ return 0;
+ end = (GstClockTime) g_get_monotonic_time () * GST_USECOND + timeout;
+ while (1)
+ {
+ GstClockTime now = (GstClockTime) g_get_monotonic_time () * GST_USECOND;
+ GstClockTime slice;
+
+ if (now >= end)
+ break; /* timed out */
+ slice = end - now;
+ if (slice > 100 * GST_MSECOND)
+ slice = 100 * GST_MSECOND;
+ msg = gst_bus_timed_pop_filtered (bus,
+ slice,
+ GST_MESSAGE_ASYNC_DONE
+ | GST_MESSAGE_ERROR
+ | GST_MESSAGE_EOS);
+ if (NULL == msg)
+ {
+ EXTRACTOR_gst_keepalive_ (io);
+ continue;
+ }
+ if (GST_MESSAGE_ASYNC_DONE != GST_MESSAGE_TYPE (msg))
+ {
+ /* an error, or the file ended before we ever saw a frame */
+ gst_message_unref (msg);
+ break;
+ }
+ if (GST_OBJECT (pipeline) == GST_MESSAGE_SRC (msg))
+ {
+ gst_message_unref (msg);
+ ret = 1;
+ break;
+ }
+ /* ASYNC_DONE from some element inside the pipeline; keep waiting */
+ gst_message_unref (msg);
+ }
+ gst_object_unref (bus);
+ return ret;
+}
+
+
int
EXTRACTOR_gst_wait_async_ (GstElement *pipeline,
GstClockTime timeout)
diff --git a/src/plugins/mediagst.h b/src/plugins/mediagst.h
@@ -67,6 +67,19 @@ struct EXTRACTOR_GstIO
* Size of the file, 0 if unknown.
*/
uint64_t length;
+
+ /**
+ * Serializes access to @e ec and @e offset: appsrc reads from
+ * GStreamer's streaming threads, EXTRACTOR_gst_keepalive_() from the
+ * plugin's thread.
+ */
+ GMutex lock;
+
+ /**
+ * When EXTRACTOR_gst_keepalive_() last talked to the host, in
+ * microseconds of g_get_monotonic_time().
+ */
+ gint64 last_ping;
};
@@ -90,6 +103,49 @@ EXTRACTOR_gst_playbin_ (struct EXTRACTOR_GstIO *io,
/**
+ * Create an appsrc that reads from @a ec (from the beginning), for use
+ * in a hand-built pipeline.
+ *
+ * @param[out] io I/O state to initialize; must outlive the element
+ * @param ec extraction context to read from
+ * @return the source, NULL if GStreamer could not provide one
+ */
+GstElement *
+EXTRACTOR_gst_appsrc_ (struct EXTRACTOR_GstIO *io,
+ struct EXTRACTOR_ExtractContext *ec);
+
+
+/**
+ * Show the host that we are still alive. Out of process, libextractor
+ * closes the channel of a plugin that has been silent (no meta data, no
+ * seek request) for 500 ms; a plugin that works for longer without
+ * touching the file -- decoding frames it already has, say -- must
+ * therefore make itself heard, which a no-op seek (`SEEK_CUR` by 0)
+ * does. Serialized with the streaming threads under @e io's lock and
+ * rate limited, so it can be called often.
+ *
+ * @param io I/O state of the pipeline
+ */
+void
+EXTRACTOR_gst_keepalive_ (struct EXTRACTOR_GstIO *io);
+
+
+/**
+ * Like EXTRACTOR_gst_wait_async_(), but calls EXTRACTOR_gst_keepalive_()
+ * while waiting.
+ *
+ * @param pipeline pipeline to watch
+ * @param timeout how long to wait, in nanoseconds
+ * @param io I/O state of the pipeline
+ * @return 1 if the pipeline is ready, 0 on error, EOS or timeout
+ */
+int
+EXTRACTOR_gst_wait_async_alive_ (GstElement *pipeline,
+ GstClockTime timeout,
+ struct EXTRACTOR_GstIO *io);
+
+
+/**
* Wait for @a pipeline to finish its current asynchronous state change.
*
* @param pipeline pipeline to watch
diff --git a/src/plugins/mediautil.c b/src/plugins/mediautil.c
@@ -37,6 +37,7 @@ EXTRACTOR_media_options_parse_ (const char *config,
struct EXTRACTOR_MediaOptions *opt)
{
const char *pos;
+ int explicit_deadline = 0;
opt->size = 128;
opt->png = 0;
@@ -44,6 +45,8 @@ EXTRACTOR_media_options_parse_ (const char *config,
opt->length = 15;
opt->bitrate = 24000;
opt->deadline = 10000;
+ opt->srt_lang[0] = '\0';
+ opt->gallery = 0;
if (NULL == config)
return;
pos = config;
@@ -113,7 +116,45 @@ EXTRACTOR_media_options_parse_ (const char *config,
if ( (endp != pos) &&
(v >= 100) &&
(v <= 120000) )
+ {
opt->deadline = (unsigned int) v;
+ explicit_deadline = 1;
+ }
+ }
+ else if ( (0 == strncmp (pos,
+ "srt:",
+ strlen ("srt:"))) ||
+ (0 == strncmp (pos,
+ "srt=",
+ strlen ("srt="))) )
+ {
+ size_t n;
+
+ pos += strlen ("srt:");
+ n = strcspn (pos, ",");
+ if ( (n > 0) &&
+ (n < sizeof (opt->srt_lang)) )
+ {
+ memcpy (opt->srt_lang,
+ pos,
+ n);
+ opt->srt_lang[n] = '\0';
+ }
+ }
+ else if ( (0 == strncmp (pos,
+ "gallery:",
+ strlen ("gallery:"))) ||
+ (0 == strncmp (pos,
+ "gallery=",
+ strlen ("gallery="))) )
+ {
+ pos += strlen ("gallery:");
+ v = strtoull (pos, &endp, 10);
+ if ( (endp != pos) &&
+ (v >= 1) )
+ opt->gallery = (v > EXTRACTOR_MEDIA_MAX_GALLERY)
+ ? EXTRACTOR_MEDIA_MAX_GALLERY
+ : (unsigned int) v;
}
/* advance to the next assignment */
pos = strchr (pos, ',');
@@ -121,6 +162,12 @@ EXTRACTOR_media_options_parse_ (const char *config,
break;
pos++;
}
+ /* a gallery or a full subtitle track means decoding across the
+ entire file, which the single-thumbnail budget does not cover */
+ if ( (! explicit_deadline) &&
+ ( (0 != opt->gallery) ||
+ ('\0' != opt->srt_lang[0]) ) )
+ opt->deadline = 60000;
}
@@ -281,4 +328,507 @@ EXTRACTOR_media_fade_s16_ (int16_t *samples,
}
+/**
+ * ISO 639 codes of a language: 639-1, 639-2/T, 639-2/B.
+ */
+struct LanguageCodes
+{
+ const char *iso1;
+ const char *iso2t;
+ const char *iso2b;
+};
+
+/**
+ * Languages for which the three codes differ in ways a plain prefix
+ * comparison would not catch. Languages not listed here still match
+ * if the user gives the same code the stream uses.
+ */
+static const struct LanguageCodes languages[] = {
+ { "ar", "ara", "ara" }, { "bg", "bul", "bul" }, { "bn", "ben", "ben" },
+ { "bo", "bod", "tib" }, { "ca", "cat", "cat" }, { "cs", "ces", "cze" },
+ { "cy", "cym", "wel" }, { "da", "dan", "dan" }, { "de", "deu", "ger" },
+ { "el", "ell", "gre" }, { "en", "eng", "eng" }, { "eo", "epo", "epo" },
+ { "es", "spa", "spa" }, { "et", "est", "est" }, { "eu", "eus", "baq" },
+ { "fa", "fas", "per" }, { "fi", "fin", "fin" }, { "fr", "fra", "fre" },
+ { "ga", "gle", "gle" }, { "gl", "glg", "glg" }, { "he", "heb", "heb" },
+ { "hi", "hin", "hin" }, { "hr", "hrv", "hrv" }, { "hu", "hun", "hun" },
+ { "hy", "hye", "arm" }, { "id", "ind", "ind" }, { "is", "isl", "ice" },
+ { "it", "ita", "ita" }, { "ja", "jpn", "jpn" }, { "ka", "kat", "geo" },
+ { "ko", "kor", "kor" }, { "lt", "lit", "lit" }, { "lv", "lav", "lav" },
+ { "mk", "mkd", "mac" }, { "ms", "msa", "may" }, { "mt", "mlt", "mlt" },
+ { "my", "mya", "bur" }, { "nb", "nob", "nob" }, { "nl", "nld", "dut" },
+ { "nn", "nno", "nno" }, { "no", "nor", "nor" }, { "pl", "pol", "pol" },
+ { "pt", "por", "por" }, { "ro", "ron", "rum" }, { "ru", "rus", "rus" },
+ { "sk", "slk", "slo" }, { "sl", "slv", "slv" }, { "sq", "sqi", "alb" },
+ { "sr", "srp", "srp" }, { "sv", "swe", "swe" }, { "sw", "swa", "swa" },
+ { "ta", "tam", "tam" }, { "th", "tha", "tha" }, { "tr", "tur", "tur" },
+ { "uk", "ukr", "ukr" }, { "ur", "urd", "urd" }, { "vi", "vie", "vie" },
+ { "zh", "zho", "chi" },
+ { NULL, NULL, NULL }
+};
+
+
+/**
+ * Reduce a language tag to its primary subtag, lower-cased.
+ *
+ * @param tag language tag, e.g. "de-AT"
+ * @param[out] out buffer for the primary subtag
+ * @param out_size size of @a out
+ */
+static void
+primary_subtag (const char *tag,
+ char *out,
+ size_t out_size)
+{
+ size_t i;
+
+ for (i = 0; (i + 1 < out_size) && ('\0' != tag[i]); i++)
+ {
+ if ( ('-' == tag[i]) ||
+ ('_' == tag[i]) )
+ break;
+ out[i] = (char) tolower ((unsigned char) tag[i]);
+ }
+ out[i] = '\0';
+}
+
+
+/**
+ * Map a primary language subtag to its row in #languages.
+ *
+ * @param code lower-case 2- or 3-letter code
+ * @return index into #languages, -1 if unknown
+ */
+static int
+language_index (const char *code)
+{
+ unsigned int i;
+
+ for (i = 0; NULL != languages[i].iso1; i++)
+ if ( (0 == strcmp (code, languages[i].iso1)) ||
+ (0 == strcmp (code, languages[i].iso2t)) ||
+ (0 == strcmp (code, languages[i].iso2b)) )
+ return (int) i;
+ return -1;
+}
+
+
+int
+EXTRACTOR_media_lang_matches_ (const char *want,
+ const char *have)
+{
+ char w[16];
+ char h[16];
+ int wi;
+
+ if ( (NULL == want) ||
+ (NULL == have) )
+ return 0;
+ primary_subtag (want, w, sizeof (w));
+ primary_subtag (have, h, sizeof (h));
+ if ( ('\0' == w[0]) ||
+ ('\0' == h[0]) )
+ return 0;
+ if (0 == strcmp (w, h))
+ return 1;
+ wi = language_index (w);
+ return ( (-1 != wi) &&
+ (wi == language_index (h)) ) ? 1 : 0;
+}
+
+
+/**
+ * Largest spread of a channel (in levels out of 256) that still
+ * counts as "narrow".
+ */
+#define FLAT_RANGE 32
+
+/**
+ * Upper bound on the number of pixels we look at per frame.
+ */
+#define FLAT_SAMPLES (256 * 1024)
+
+
+int
+EXTRACTOR_media_frame_is_flat_ (const uint8_t *rgb,
+ unsigned int width,
+ unsigned int height,
+ size_t stride)
+{
+ uint32_t hist[3][256];
+ uint64_t total;
+ uint64_t need;
+ unsigned int step;
+ unsigned int x;
+ unsigned int y;
+ unsigned int c;
+
+ if ( (0 == width) ||
+ (0 == height) )
+ return 1;
+ memset (hist, 0, sizeof (hist));
+ step = 1;
+ while ( ((uint64_t) width / step) * ((uint64_t) height / step)
+ > FLAT_SAMPLES)
+ step++;
+ total = 0;
+ for (y = 0; y < height; y += step)
+ {
+ const uint8_t *row = rgb + (size_t) y * stride;
+
+ for (x = 0; x < width; x += step)
+ {
+ hist[0][row[3 * x]]++;
+ hist[1][row[3 * x + 1]]++;
+ hist[2][row[3 * x + 2]]++;
+ total++;
+ }
+ }
+ need = (total * 99 + 99) / 100;
+ for (c = 0; c < 3; c++)
+ {
+ uint64_t win = 0;
+ uint64_t best = 0;
+ unsigned int i;
+
+ /* largest population of any FLAT_RANGE-wide window */
+ for (i = 0; i < 256; i++)
+ {
+ win += hist[c][i];
+ if (i >= FLAT_RANGE)
+ win -= hist[c][i - FLAT_RANGE];
+ if (win > best)
+ best = win;
+ }
+ if (best < need)
+ return 0;
+ }
+ return 1;
+}
+
+
+void
+EXTRACTOR_media_gallery_layout_ (unsigned int n,
+ unsigned int tile_w,
+ unsigned int tile_h,
+ unsigned int *cols,
+ unsigned int *rows)
+{
+ const double target = 16.0 / 9.0;
+ double best_score = -1;
+ unsigned int best_waste = 0;
+ unsigned int c;
+
+ if (0 == n)
+ n = 1;
+ if (0 == tile_w)
+ tile_w = 1;
+ if (0 == tile_h)
+ tile_h = 1;
+ *cols = n;
+ *rows = 1;
+ for (c = 1; c <= n; c++)
+ {
+ unsigned int r = (n + c - 1) / c;
+ double aspect = ((double) c * tile_w) / ((double) r * tile_h);
+ double score = (aspect > target) ? aspect / target : target / aspect;
+ unsigned int waste = c * r - n;
+
+ if ( (best_score < 0) ||
+ (score < best_score - 1e-9) ||
+ ( (score < best_score + 1e-9) &&
+ (waste < best_waste) ) )
+ {
+ best_score = score;
+ best_waste = waste;
+ *cols = c;
+ *rows = r;
+ }
+ }
+}
+
+
+uint64_t
+EXTRACTOR_media_gallery_time_ (unsigned int i,
+ unsigned int n,
+ uint64_t duration_ms)
+{
+ if (0 == n)
+ return 0;
+ return (duration_ms / n) * i + duration_ms / (2 * (uint64_t) n);
+}
+
+
+/**
+ * Append @a len bytes to @a srt.
+ *
+ * @param srt file to append to
+ * @param data bytes to append
+ * @param len number of bytes
+ * @return 0 on success, -1 if full
+ */
+static int
+srt_put (struct EXTRACTOR_MediaSrt *srt,
+ const char *data,
+ size_t len)
+{
+ if (srt->full)
+ return -1;
+ if (srt->len + len + 1 > EXTRACTOR_MEDIA_MAX_SRT)
+ {
+ srt->full = 1;
+ return -1;
+ }
+ if (srt->len + len + 1 > srt->cap)
+ {
+ size_t ncap = (0 == srt->cap) ? 4096 : srt->cap;
+ char *nbuf;
+
+ while (ncap < srt->len + len + 1)
+ ncap *= 2;
+ if (ncap > EXTRACTOR_MEDIA_MAX_SRT)
+ ncap = EXTRACTOR_MEDIA_MAX_SRT;
+ nbuf = realloc (srt->buf, ncap);
+ if (NULL == nbuf)
+ {
+ srt->full = 1;
+ return -1;
+ }
+ srt->buf = nbuf;
+ srt->cap = ncap;
+ }
+ memcpy (&srt->buf[srt->len], data, len);
+ srt->len += len;
+ srt->buf[srt->len] = '\0';
+ return 0;
+}
+
+
+/**
+ * Length of the valid UTF-8 sequence at @a s.
+ *
+ * @param s input
+ * @param len bytes available, at least 1
+ * @return length of the sequence, 0 if invalid
+ */
+static size_t
+utf8_seq_len (const unsigned char *s,
+ size_t len)
+{
+ size_t n;
+ size_t i;
+ uint32_t cp;
+
+ if (s[0] < 0x80)
+ return 1;
+ if ((s[0] & 0xE0) == 0xC0)
+ n = 2, cp = s[0] & 0x1F;
+ else if ((s[0] & 0xF0) == 0xE0)
+ n = 3, cp = s[0] & 0x0F;
+ else if ((s[0] & 0xF8) == 0xF0)
+ n = 4, cp = s[0] & 0x07;
+ else
+ return 0;
+ if (n > len)
+ return 0;
+ for (i = 1; i < n; i++)
+ {
+ if ((s[i] & 0xC0) != 0x80)
+ return 0;
+ cp = (cp << 6) | (s[i] & 0x3F);
+ }
+ if ( ( (2 == n) && (cp < 0x80) ) ||
+ ( (3 == n) && (cp < 0x800) ) ||
+ ( (4 == n) && (cp < 0x10000) ) ||
+ (cp > 0x10FFFF) ||
+ ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
+ return 0;
+ return n;
+}
+
+
+/**
+ * Format @a ms as "HH:MM:SS,mmm".
+ *
+ * @param ms time in milliseconds
+ * @param[out] out buffer of at least 32 bytes
+ */
+static void
+srt_time (uint64_t ms,
+ char out[32])
+{
+ snprintf (out,
+ 32,
+ "%02llu:%02llu:%02llu,%03llu",
+ (unsigned long long) (ms / 3600000),
+ (unsigned long long) ((ms / 60000) % 60),
+ (unsigned long long) ((ms / 1000) % 60),
+ (unsigned long long) (ms % 1000));
+}
+
+
+void
+EXTRACTOR_media_srt_add_ (struct EXTRACTOR_MediaSrt *srt,
+ uint64_t start_ms,
+ uint64_t end_ms,
+ const char *text,
+ size_t text_len,
+ int is_ass)
+{
+ const unsigned char *in = (const unsigned char *) text;
+ char *clean;
+ size_t i;
+ size_t o;
+ size_t line_start;
+ char hdr[96];
+ char t1[32];
+ char t2[32];
+
+ if ( (srt->full) ||
+ (NULL == text) )
+ return;
+ i = 0;
+ if (is_ass)
+ {
+ unsigned int commas = 8;
+
+ if ( (text_len >= 9) &&
+ (0 == strncmp (text, "Dialogue:", 9)) )
+ commas = 9;
+ while ( (i < text_len) &&
+ (commas > 0) )
+ if (',' == in[i++])
+ commas--;
+ }
+ clean = malloc (text_len - i + 1);
+ if (NULL == clean)
+ return;
+ o = 0;
+ line_start = 0;
+ while ( (i < text_len) &&
+ ('\0' != in[i]) )
+ {
+ size_t n;
+
+ if (is_ass &&
+ ('{' == in[i]) )
+ {
+ /* ASS override block */
+ while ( (i < text_len) && ('}' != in[i]) )
+ i++;
+ i++;
+ continue;
+ }
+ if (is_ass &&
+ ('\\' == in[i]) &&
+ (i + 1 < text_len) &&
+ ( ('N' == in[i + 1]) || ('n' == in[i + 1]) ||
+ ('h' == in[i + 1]) ) )
+ {
+ clean[o++] = ('h' == in[i + 1]) ? ' ' : '\n';
+ i += 2;
+ }
+ else if ('\r' == in[i])
+ {
+ i++;
+ continue;
+ }
+ else
+ {
+ n = utf8_seq_len (&in[i], text_len - i);
+ if (0 == n)
+ {
+ clean[o++] = '?';
+ i++;
+ }
+ else
+ {
+ memcpy (&clean[o], &in[i], n);
+ o += n;
+ i += n;
+ }
+ }
+ if ('\n' == clean[o - 1])
+ {
+ /* drop trailing blanks, and the line itself if it is empty */
+ size_t e = o - 1;
+
+ while ( (e > line_start) &&
+ ( (' ' == clean[e - 1]) || ('\t' == clean[e - 1]) ) )
+ e--;
+ if (e == line_start)
+ o = line_start;
+ else
+ {
+ clean[e] = '\n';
+ o = e + 1;
+ line_start = o;
+ }
+ }
+ }
+ while ( (o > 0) &&
+ ( ('\n' == clean[o - 1]) || (' ' == clean[o - 1]) ||
+ ('\t' == clean[o - 1]) ) )
+ o--;
+ if (0 == o)
+ {
+ free (clean);
+ return;
+ }
+ if (end_ms < start_ms)
+ end_ms = start_ms;
+ srt_time (start_ms, t1);
+ srt_time (end_ms, t2);
+ snprintf (hdr,
+ sizeof (hdr),
+ "%u\n%s --> %s\n",
+ srt->count + 1,
+ t1,
+ t2);
+ {
+ size_t old_len = srt->len;
+
+ if ( (0 == srt_put (srt, hdr, strlen (hdr))) &&
+ (0 == srt_put (srt, clean, o)) &&
+ (0 == srt_put (srt, "\n\n", 2)) )
+ {
+ srt->count++;
+ }
+ else if (NULL != srt->buf)
+ {
+ /* never emit a partial cue */
+ srt->len = old_len;
+ srt->buf[old_len] = '\0';
+ }
+ }
+ free (clean);
+}
+
+
+int
+EXTRACTOR_media_srt_emit_ (const struct EXTRACTOR_MediaSrt *srt,
+ struct EXTRACTOR_ExtractContext *ec,
+ const char *plugin_name)
+{
+ if ( (0 == srt->count) ||
+ (0 == srt->len) )
+ return 0;
+ return ec->proc (ec->cls,
+ plugin_name,
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "application/x-subrip",
+ srt->buf,
+ srt->len + 1);
+}
+
+
+void
+EXTRACTOR_media_srt_clear_ (struct EXTRACTOR_MediaSrt *srt)
+{
+ free (srt->buf);
+ memset (srt, 0, sizeof (*srt));
+}
+
+
/* end of mediautil.c */
diff --git a/src/plugins/mediautil.h b/src/plugins/mediautil.h
@@ -92,10 +92,192 @@ struct EXTRACTOR_MediaOptions
* Wall clock budget for the entire extraction, in milliseconds.
*/
unsigned int deadline;
+
+ /**
+ * Language of the subtitle track to dump as SRT (option "srt:LANG"),
+ * empty string if no subtitle dump was requested.
+ */
+ char srt_lang[16];
+
+ /**
+ * Number of tiles in the thumbnail gallery (option "gallery:N"),
+ * 0 for a plain single thumbnail.
+ */
+ unsigned int gallery;
};
/**
+ * Largest number of tiles in a gallery.
+ */
+#define EXTRACTOR_MEDIA_MAX_GALLERY 100
+
+/**
+ * Largest gallery sheet in pixels. Far below #EXTRACTOR_MEDIA_MAX_PIXELS:
+ * the sheet is encoded in one uninterruptible call, and a larger one
+ * cannot be squeezed into #EXTRACTOR_MEDIA_MAX_GALLERY_BYTES anyway.
+ * Tiles are scaled down to fit.
+ */
+#define EXTRACTOR_MEDIA_MAX_GALLERY_PIXELS (4 * 1024 * 1024)
+
+/**
+ * Maximum number of bytes we are willing to emit for a gallery.
+ */
+#define EXTRACTOR_MEDIA_MAX_GALLERY_BYTES (4 * 1024 * 1024)
+
+/**
+ * Maximum number of bytes we are willing to emit for an SRT dump.
+ */
+#define EXTRACTOR_MEDIA_MAX_SRT (8 * 1024 * 1024)
+
+
+/**
+ * Check whether the language tag @a have of a stream matches the
+ * language @a want requested by the user. Both may be ISO 639-1
+ * ("de"), ISO 639-2/B ("ger") or ISO 639-2/T ("deu"), optionally with
+ * a region ("de-AT", "de_AT"); comparison ignores case.
+ *
+ * @param want requested language
+ * @param have language tag of the stream, may be NULL
+ * @return 1 if they denote the same language, 0 if not
+ */
+int
+EXTRACTOR_media_lang_matches_ (const char *want,
+ const char *have);
+
+
+/**
+ * Check whether a frame shows basically nothing: at least 99% of its
+ * pixels fall into a narrow color range in every channel (e.g. almost
+ * all black, almost all white, a uniform color, a fade).
+ *
+ * @param rgb packed RGB24 pixels
+ * @param width width in pixels
+ * @param height height in pixels
+ * @param stride bytes per row
+ * @return 1 if the frame is (nearly) flat, 0 if it has content
+ */
+int
+EXTRACTOR_media_frame_is_flat_ (const uint8_t *rgb,
+ unsigned int width,
+ unsigned int height,
+ size_t stride);
+
+
+/**
+ * Choose a grid for @a n tiles of @a tile_w x @a tile_h pixels such
+ * that the entire sheet is as close to 16:9 as possible (ties are
+ * broken towards fewer empty cells).
+ *
+ * @param n number of tiles, at least 1
+ * @param tile_w width of one tile
+ * @param tile_h height of one tile
+ * @param[out] cols number of columns
+ * @param[out] rows number of rows
+ */
+void
+EXTRACTOR_media_gallery_layout_ (unsigned int n,
+ unsigned int tile_w,
+ unsigned int tile_h,
+ unsigned int *cols,
+ unsigned int *rows);
+
+
+/**
+ * Time stamp in the gallery for tile @a i of @a n: the midpoints of
+ * @a n equal slices of @a duration_ms.
+ *
+ * @param i tile index, in [0,n)
+ * @param n number of tiles
+ * @param duration_ms length of the media
+ * @return position of tile @a i in milliseconds
+ */
+uint64_t
+EXTRACTOR_media_gallery_time_ (unsigned int i,
+ unsigned int n,
+ uint64_t duration_ms);
+
+
+/**
+ * An SRT file under construction.
+ */
+struct EXTRACTOR_MediaSrt
+{
+ /**
+ * The text so far.
+ */
+ char *buf;
+
+ /**
+ * Number of bytes in @e buf.
+ */
+ size_t len;
+
+ /**
+ * Allocated size of @e buf.
+ */
+ size_t cap;
+
+ /**
+ * Number of cues written so far.
+ */
+ unsigned int count;
+
+ /**
+ * 1 if we hit #EXTRACTOR_MEDIA_MAX_SRT or ran out of memory.
+ */
+ int full;
+};
+
+
+/**
+ * Append one cue to @a srt. @a text may contain ASS dialogue lines
+ * ("0,0,Default,,0,0,0,,Hello\\Nworld", as FFmpeg produces them) or
+ * plain text; ASS override tags are removed, "\\N" becomes a line
+ * break, invalid UTF-8 is replaced by '?', and empty lines inside the
+ * cue are dropped. Cues without text are ignored.
+ *
+ * @param srt file to append to
+ * @param start_ms start of the cue in milliseconds
+ * @param end_ms end of the cue in milliseconds
+ * @param text text of the cue
+ * @param text_len number of bytes in @a text
+ * @param is_ass 1 if @a text is an ASS dialogue event (fields before
+ * the text are skipped), 0 if it is plain text
+ */
+void
+EXTRACTOR_media_srt_add_ (struct EXTRACTOR_MediaSrt *srt,
+ uint64_t start_ms,
+ uint64_t end_ms,
+ const char *text,
+ size_t text_len,
+ int is_ass);
+
+
+/**
+ * Emit @a srt as #EXTRACTOR_METATYPE_SUBTITLE_TEXT (if non-empty).
+ *
+ * @param srt file to emit
+ * @param ec extraction context
+ * @param plugin_name short name of the plugin
+ * @return return value of `proc` (0 if nothing was emitted)
+ */
+int
+EXTRACTOR_media_srt_emit_ (const struct EXTRACTOR_MediaSrt *srt,
+ struct EXTRACTOR_ExtractContext *ec,
+ const char *plugin_name);
+
+
+/**
+ * Release @a srt.
+ *
+ * @param srt file to release
+ */
+void
+EXTRACTOR_media_srt_clear_ (struct EXTRACTOR_MediaSrt *srt);
+
+
+/**
* A wall clock deadline.
*/
struct EXTRACTOR_MediaDeadline
diff --git a/src/plugins/odf_extractor.c b/src/plugins/odf_extractor.c
@@ -26,6 +26,7 @@
#include <ctype.h>
#include "extractor.h"
#include "unzip.h"
+#include "fulltext.h"
/**
* Maximum length of a filename allowed inside the ZIP archive.
@@ -37,6 +38,31 @@
*/
#define METAFILE "meta.xml"
+/**
+ * Name of the file with the document body in OO documents.
+ */
+#define CONTENTFILE "content.xml"
+
+/**
+ * Largest decompressed size of content.xml we are willing to look at
+ * for the "fulltext" option; anything beyond is ignored (this also
+ * bounds the damage a zip bomb can do).
+ */
+#define MAX_CONTENT_SIZE (64 * 1024 * 1024)
+
+/**
+ * Maximum XML element nesting depth we keep track of. Deeper elements
+ * are still balanced correctly, but treated as having no special
+ * meaning.
+ */
+#define MAX_XML_DEPTH 256
+
+/**
+ * Maximum number of namespace prefixes declared on the root element
+ * that we remember.
+ */
+#define MAX_NS_PREFIXES 64
+
/**
* Mapping from ODF meta data strings to LE types.
@@ -147,6 +173,916 @@ libextractor_oo_getmimetype (struct EXTRACTOR_UnzipFile *uf)
}
+/* ******************** full text of content.xml ******************** */
+
+/**
+ * Namespaces the full-text scanner cares about.
+ */
+enum OdfNamespace
+{
+ NS_OTHER = 0,
+ NS_OFFICE,
+ NS_TEXT,
+ NS_TABLE,
+ NS_DRAW,
+ NS_PRESENTATION,
+ NS_STYLE,
+ NS_SVG
+};
+
+
+/**
+ * Map from namespace URI to #OdfNamespace. Both ODF (OASIS) and the
+ * legacy OpenOffice.org 1.x namespaces are supported.
+ */
+static const struct
+{
+ const char *uri;
+ enum OdfNamespace ns;
+} ns_uris[] = {
+ { "urn:oasis:names:tc:opendocument:xmlns:office:1.0", NS_OFFICE },
+ { "urn:oasis:names:tc:opendocument:xmlns:text:1.0", NS_TEXT },
+ { "urn:oasis:names:tc:opendocument:xmlns:table:1.0", NS_TABLE },
+ { "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0", NS_DRAW },
+ { "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0",
+ NS_PRESENTATION },
+ { "urn:oasis:names:tc:opendocument:xmlns:style:1.0", NS_STYLE },
+ { "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0", NS_SVG },
+ { "http://openoffice.org/2000/office", NS_OFFICE },
+ { "http://openoffice.org/2000/text", NS_TEXT },
+ { "http://openoffice.org/2000/table", NS_TABLE },
+ { "http://openoffice.org/2000/drawing", NS_DRAW },
+ { "http://openoffice.org/2000/presentation", NS_PRESENTATION },
+ { "http://openoffice.org/2000/style", NS_STYLE },
+ { "http://www.w3.org/2000/svg", NS_SVG },
+ { NULL, NS_OTHER }
+};
+
+
+/**
+ * Conventional prefixes, used for elements whose prefix was not
+ * declared on the root element.
+ */
+static const struct
+{
+ const char *prefix;
+ enum OdfNamespace ns;
+} ns_prefixes[] = {
+ { "office", NS_OFFICE },
+ { "text", NS_TEXT },
+ { "table", NS_TABLE },
+ { "draw", NS_DRAW },
+ { "presentation", NS_PRESENTATION },
+ { "style", NS_STYLE },
+ { "svg", NS_SVG },
+ { NULL, NS_OTHER }
+};
+
+
+/**
+ * What an element means for the full text.
+ */
+enum OdfElementKind
+{
+ /**
+ * Nothing special, its content (if any) is used.
+ */
+ EK_OTHER = 0,
+
+ /**
+ * A paragraph (text:p, text:h): text is only taken from inside
+ * paragraphs, and each one ends with a paragraph break.
+ */
+ EK_PARA,
+
+ /**
+ * Material that is not part of the running text (annotations,
+ * notes, deleted text, page-number fields, speaker notes, ...):
+ * the whole subtree is skipped.
+ */
+ EK_SKIP,
+
+ /**
+ * An element standing for white space (text:s, text:tab,
+ * text:line-break).
+ */
+ EK_SPACE,
+
+ /**
+ * The office:body element; only text inside it is used.
+ */
+ EK_BODY
+};
+
+
+/**
+ * Classification of the elements that are not #EK_OTHER.
+ */
+static const struct
+{
+ enum OdfNamespace ns;
+ const char *local;
+ enum OdfElementKind kind;
+} element_kinds[] = {
+ { NS_OFFICE, "body", EK_BODY },
+ { NS_TEXT, "p", EK_PARA },
+ { NS_TEXT, "h", EK_PARA },
+ { NS_TEXT, "s", EK_SPACE },
+ { NS_TEXT, "tab", EK_SPACE },
+ { NS_TEXT, "tab-stop", EK_SPACE }, /* OOo 1.x name of text:tab */
+ { NS_TEXT, "line-break", EK_SPACE },
+ /* comments */
+ { NS_OFFICE, "annotation", EK_SKIP },
+ /* footnotes and endnotes (ODF, and OOo 1.x) */
+ { NS_TEXT, "note", EK_SKIP },
+ { NS_TEXT, "footnote", EK_SKIP },
+ { NS_TEXT, "endnote", EK_SKIP },
+ /* change tracking: holds the text of deletions */
+ { NS_TEXT, "tracked-changes", EK_SKIP },
+ /* page furniture fields */
+ { NS_TEXT, "page-number", EK_SKIP },
+ { NS_TEXT, "page-count", EK_SKIP },
+ { NS_TEXT, "page-continuation", EK_SKIP },
+ { NS_TEXT, "page-continuation-string", EK_SKIP },
+ { NS_TEXT, "page-variable-get", EK_SKIP },
+ /* speaker notes of presentations */
+ { NS_PRESENTATION, "notes", EK_SKIP },
+ /* alternative text / titles of drawing objects */
+ { NS_SVG, "title", EK_SKIP },
+ { NS_SVG, "desc", EK_SKIP },
+ /* headers, footers and master pages normally live in styles.xml
+ (which we do not read); skip them should they show up here */
+ { NS_OFFICE, "master-styles", EK_SKIP },
+ { NS_STYLE, "master-page", EK_SKIP },
+ { NS_STYLE, "header", EK_SKIP },
+ { NS_STYLE, "header-left", EK_SKIP },
+ { NS_STYLE, "header-first", EK_SKIP },
+ { NS_STYLE, "footer", EK_SKIP },
+ { NS_STYLE, "footer-left", EK_SKIP },
+ { NS_STYLE, "footer-first", EK_SKIP },
+ { NS_OTHER, NULL, EK_OTHER }
+};
+
+
+/**
+ * A namespace prefix declared on the root element.
+ */
+struct OdfPrefix
+{
+ /**
+ * The prefix (not 0-terminated, points into the document).
+ */
+ const char *prefix;
+
+ /**
+ * Number of bytes in @e prefix.
+ */
+ size_t prefix_len;
+
+ /**
+ * Namespace the prefix is bound to.
+ */
+ enum OdfNamespace ns;
+};
+
+
+/**
+ * State of the full-text scanner.
+ */
+struct OdfScanner
+{
+ /**
+ * Where the text goes.
+ */
+ struct EXTRACTOR_FullText ft;
+
+ /**
+ * Prefixes declared on the root element.
+ */
+ struct OdfPrefix prefixes[MAX_NS_PREFIXES];
+
+ /**
+ * Number of valid entries in @e prefixes.
+ */
+ unsigned int num_prefixes;
+
+ /**
+ * Kinds of the currently open elements.
+ */
+ unsigned char stack[MAX_XML_DEPTH];
+
+ /**
+ * Number of valid entries in @e stack.
+ */
+ unsigned int depth;
+
+ /**
+ * Number of open elements nested deeper than #MAX_XML_DEPTH.
+ */
+ unsigned int overflow;
+
+ /**
+ * Number of open #EK_SKIP elements.
+ */
+ unsigned int skip_depth;
+
+ /**
+ * Number of open #EK_PARA elements.
+ */
+ unsigned int para_depth;
+
+ /**
+ * Number of open #EK_BODY elements.
+ */
+ unsigned int body_depth;
+
+ /**
+ * 1 once we have seen the root element.
+ */
+ int have_root;
+};
+
+
+/**
+ * Is character data at the current position part of the full text?
+ *
+ * @param sc scanner state
+ * @return 1 if yes
+ */
+static int
+odf_text_active (const struct OdfScanner *sc)
+{
+ return (0 < sc->body_depth) &&
+ (0 < sc->para_depth) &&
+ (0 == sc->skip_depth);
+}
+
+
+/**
+ * Is @a c XML white space?
+ *
+ * @param c character to test
+ * @return 1 if yes
+ */
+static int
+odf_is_ws (char c)
+{
+ return (' ' == c) || ('\t' == c) || ('\n' == c) || ('\r' == c);
+}
+
+
+/**
+ * Find the namespace of the qualified name @a name.
+ *
+ * @param sc scanner state (for the declared prefixes)
+ * @param name qualified name
+ * @param name_len number of bytes in @a name
+ * @param[out] local set to the local part of @a name
+ * @param[out] local_len set to the length of @a local
+ * @return namespace of @a name
+ */
+static enum OdfNamespace
+odf_resolve (const struct OdfScanner *sc,
+ const char *name,
+ size_t name_len,
+ const char **local,
+ size_t *local_len)
+{
+ const char *colon;
+ size_t plen;
+ unsigned int i;
+
+ colon = memchr (name, ':', name_len);
+ if (NULL == colon)
+ {
+ *local = name;
+ *local_len = name_len;
+ return NS_OTHER;
+ }
+ plen = colon - name;
+ *local = colon + 1;
+ *local_len = name_len - plen - 1;
+ for (i = 0; i < sc->num_prefixes; i++)
+ if ( (sc->prefixes[i].prefix_len == plen) &&
+ (0 == memcmp (sc->prefixes[i].prefix, name, plen)) )
+ return sc->prefixes[i].ns;
+ for (i = 0; NULL != ns_prefixes[i].prefix; i++)
+ if ( (strlen (ns_prefixes[i].prefix) == plen) &&
+ (0 == memcmp (ns_prefixes[i].prefix, name, plen)) )
+ return ns_prefixes[i].ns;
+ return NS_OTHER;
+}
+
+
+/**
+ * Classify the element @a name.
+ *
+ * @param sc scanner state
+ * @param name qualified element name
+ * @param name_len number of bytes in @a name
+ * @return what the element means for the full text
+ */
+static enum OdfElementKind
+odf_classify (const struct OdfScanner *sc,
+ const char *name,
+ size_t name_len)
+{
+ enum OdfNamespace ns;
+ const char *local;
+ size_t local_len;
+ unsigned int i;
+
+ ns = odf_resolve (sc, name, name_len, &local, &local_len);
+ if (NS_OTHER == ns)
+ return EK_OTHER;
+ for (i = 0; NULL != element_kinds[i].local; i++)
+ if ( (element_kinds[i].ns == ns) &&
+ (strlen (element_kinds[i].local) == local_len) &&
+ (0 == memcmp (element_kinds[i].local, local, local_len)) )
+ return element_kinds[i].kind;
+ return EK_OTHER;
+}
+
+
+/**
+ * Record the "xmlns:PREFIX" declarations found in the attributes of
+ * the root element.
+ *
+ * @param sc scanner state to update
+ * @param attrs start of the attributes
+ * @param end end of the attributes
+ */
+static void
+odf_parse_ns_decls (struct OdfScanner *sc,
+ const char *attrs,
+ const char *end)
+{
+ const char *p = attrs;
+
+ while (p < end)
+ {
+ const char *an;
+ size_t an_len;
+ const char *val;
+ size_t val_len;
+ char quote;
+ unsigned int i;
+
+ while ( (p < end) && odf_is_ws (*p))
+ p++;
+ an = p;
+ while ( (p < end) && ('=' != *p) && (! odf_is_ws (*p)) )
+ p++;
+ an_len = p - an;
+ while ( (p < end) && odf_is_ws (*p))
+ p++;
+ if ( (p >= end) || ('=' != *p) )
+ return;
+ p++;
+ while ( (p < end) && odf_is_ws (*p))
+ p++;
+ if ( (p >= end) || ( ('"' != *p) && ('\'' != *p) ) )
+ return;
+ quote = *p++;
+ val = p;
+ while ( (p < end) && (quote != *p) )
+ p++;
+ if (p >= end)
+ return;
+ val_len = p - val;
+ p++;
+ if ( (an_len <= 6) ||
+ (0 != memcmp (an, "xmlns:", 6)) ||
+ (sc->num_prefixes >= MAX_NS_PREFIXES) )
+ continue;
+ for (i = 0; NULL != ns_uris[i].uri; i++)
+ {
+ if ( (strlen (ns_uris[i].uri) != val_len) ||
+ (0 != memcmp (ns_uris[i].uri, val, val_len)) )
+ continue;
+ sc->prefixes[sc->num_prefixes].prefix = an + 6;
+ sc->prefixes[sc->num_prefixes].prefix_len = an_len - 6;
+ sc->prefixes[sc->num_prefixes].ns = ns_uris[i].ns;
+ sc->num_prefixes++;
+ break;
+ }
+ }
+}
+
+
+/**
+ * Append XML character data to the full text, decoding the predefined
+ * entities and numeric character references. Other entity references
+ * are kept literally (we never expand entities declared in a DTD).
+ *
+ * @param ft where to append the text
+ * @param text character data
+ * @param len number of bytes in @a text
+ */
+static void
+odf_append_chardata (struct EXTRACTOR_FullText *ft,
+ const char *text,
+ size_t len)
+{
+ static const struct
+ {
+ const char *name;
+ uint32_t cp;
+ } entities[] = {
+ { "lt", '<' },
+ { "gt", '>' },
+ { "amp", '&' },
+ { "quot", '"' },
+ { "apos", '\'' },
+ { NULL, 0 }
+ };
+ size_t start = 0;
+ size_t i = 0;
+
+ while (i < len)
+ {
+ const char *semi;
+ size_t elen;
+ uint32_t cp;
+ int ok;
+
+ if ('&' != text[i])
+ {
+ i++;
+ continue;
+ }
+ /* longest valid reference is "" / "" */
+ elen = len - i - 1;
+ if (elen > 12)
+ elen = 12;
+ semi = memchr (&text[i + 1], ';', elen);
+ if (NULL == semi)
+ {
+ i++;
+ continue;
+ }
+ elen = semi - &text[i + 1];
+ ok = 0;
+ cp = 0;
+ if ( (elen >= 2) &&
+ ('#' == text[i + 1]) )
+ {
+ size_t j = i + 2;
+ int hex = 0;
+
+ if ( ('x' == text[j]) || ('X' == text[j]) )
+ {
+ hex = 1;
+ j++;
+ }
+ ok = (j < i + 1 + elen);
+ for (; j < i + 1 + elen; j++)
+ {
+ unsigned char c = (unsigned char) text[j];
+ unsigned int d;
+
+ if ( (c >= '0') && (c <= '9') )
+ d = c - '0';
+ else if (hex && (c >= 'a') && (c <= 'f'))
+ d = c - 'a' + 10;
+ else if (hex && (c >= 'A') && (c <= 'F'))
+ d = c - 'A' + 10;
+ else
+ {
+ ok = 0;
+ break;
+ }
+ cp = cp * (hex ? 16 : 10) + d;
+ if (cp > 0x10FFFF)
+ {
+ ok = 0;
+ break;
+ }
+ }
+ if ( (0 == cp) ||
+ ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
+ ok = 0;
+ }
+ else
+ {
+ unsigned int k;
+
+ for (k = 0; NULL != entities[k].name; k++)
+ if ( (strlen (entities[k].name) == elen) &&
+ (0 == memcmp (entities[k].name, &text[i + 1], elen)) )
+ {
+ cp = entities[k].cp;
+ ok = 1;
+ break;
+ }
+ }
+ if (! ok)
+ {
+ i++;
+ continue;
+ }
+ EXTRACTOR_fulltext_append_ (ft, &text[start], i - start);
+ EXTRACTOR_fulltext_append_cp_ (ft, cp);
+ i += elen + 2;
+ start = i;
+ }
+ EXTRACTOR_fulltext_append_ (ft, &text[start], len - start);
+}
+
+
+/**
+ * Find @a needle in the @a len bytes at @a hay.
+ *
+ * @param hay where to search
+ * @param len number of bytes at @a hay
+ * @param needle 0-terminated string to find
+ * @return NULL if not found
+ */
+static const char *
+odf_find (const char *hay,
+ size_t len,
+ const char *needle)
+{
+ size_t nlen = strlen (needle);
+ const char *end = hay + len;
+ const char *p = hay;
+
+ while ( (size_t) (end - p) >= nlen)
+ {
+ p = memchr (p, needle[0], (end - p) - nlen + 1);
+ if (NULL == p)
+ return NULL;
+ if (0 == memcmp (p, needle, nlen))
+ return p;
+ p++;
+ }
+ return NULL;
+}
+
+
+/**
+ * Handle a start tag (or empty-element tag).
+ *
+ * @param sc scanner state
+ * @param name qualified element name
+ * @param name_len number of bytes in @a name
+ * @param attrs start of the attributes
+ * @param attrs_end end of the attributes
+ * @param empty 1 if this is an empty-element tag ("<x/>")
+ * @return 0 to continue, -1 if this is not an ODF document
+ */
+static int
+odf_start_element (struct OdfScanner *sc,
+ const char *name,
+ size_t name_len,
+ const char *attrs,
+ const char *attrs_end,
+ int empty)
+{
+ enum OdfElementKind kind;
+
+ if (! sc->have_root)
+ {
+ const char *local;
+ size_t local_len;
+
+ sc->have_root = 1;
+ odf_parse_ns_decls (sc, attrs, attrs_end);
+ if ( (NS_OFFICE != odf_resolve (sc, name, name_len, &local, &local_len)) ||
+ (strlen ("document-content") != local_len) ||
+ (0 != memcmp (local, "document-content", local_len)) )
+ return -1;
+ }
+ kind = odf_classify (sc, name, name_len);
+ if (EK_SPACE == kind)
+ {
+ if (odf_text_active (sc))
+ EXTRACTOR_fulltext_space_ (&sc->ft);
+ }
+ else if ( (EK_PARA == kind) &&
+ (0 == sc->skip_depth) )
+ {
+ /* a paragraph nested in another one (e.g. in a text frame) is a
+ paragraph of its own */
+ EXTRACTOR_fulltext_paragraph_ (&sc->ft);
+ }
+ if (empty)
+ return 0;
+ if ( (0 < sc->overflow) ||
+ (MAX_XML_DEPTH == sc->depth) )
+ {
+ sc->overflow++;
+ return 0;
+ }
+ sc->stack[sc->depth++] = (unsigned char) kind;
+ switch (kind)
+ {
+ case EK_PARA:
+ sc->para_depth++;
+ break;
+ case EK_SKIP:
+ sc->skip_depth++;
+ break;
+ case EK_BODY:
+ sc->body_depth++;
+ break;
+ default:
+ break;
+ }
+ return 0;
+}
+
+
+/**
+ * Handle an end tag. We do not check that it matches the start tag;
+ * a malformed document merely yields odd text.
+ *
+ * @param sc scanner state
+ */
+static void
+odf_end_element (struct OdfScanner *sc)
+{
+ if (0 < sc->overflow)
+ {
+ sc->overflow--;
+ return;
+ }
+ if (0 == sc->depth)
+ return;
+ switch ((enum OdfElementKind) sc->stack[--sc->depth])
+ {
+ case EK_PARA:
+ sc->para_depth--;
+ if (0 == sc->skip_depth)
+ EXTRACTOR_fulltext_paragraph_ (&sc->ft);
+ break;
+ case EK_SKIP:
+ sc->skip_depth--;
+ break;
+ case EK_BODY:
+ sc->body_depth--;
+ break;
+ default:
+ break;
+ }
+}
+
+
+/**
+ * Scan the XML of content.xml and collect the running text. The
+ * scanner is iterative, runs in time linear in @a len and only keeps
+ * a bounded element stack.
+ *
+ * @param sc scanner state, initialized
+ * @param xml the document
+ * @param len number of bytes in @a xml
+ */
+static void
+odf_scan (struct OdfScanner *sc,
+ const char *xml,
+ size_t len)
+{
+ const char *p = xml;
+ const char *end = xml + len;
+
+ while ( (p < end) &&
+ (! sc->ft.full) )
+ {
+ const char *lt;
+ const char *q;
+
+ EXTRACTOR_fulltext_tick_ (&sc->ft);
+ if ('<' != *p)
+ {
+ lt = memchr (p, '<', end - p);
+ if (NULL == lt)
+ lt = end;
+ if (odf_text_active (sc))
+ odf_append_chardata (&sc->ft, p, lt - p);
+ p = lt;
+ continue;
+ }
+ if ( (end - p >= 4) &&
+ (0 == memcmp (p, "<!--", 4)) )
+ {
+ q = odf_find (p + 4, end - p - 4, "-->");
+ if (NULL == q)
+ return;
+ p = q + 3;
+ continue;
+ }
+ if ( (end - p >= 9) &&
+ (0 == memcmp (p, "<![CDATA[", 9)) )
+ {
+ q = odf_find (p + 9, end - p - 9, "]]>");
+ if (NULL == q)
+ return;
+ if (odf_text_active (sc))
+ EXTRACTOR_fulltext_append_ (&sc->ft, p + 9, q - (p + 9));
+ p = q + 3;
+ continue;
+ }
+ if ( (end - p >= 2) &&
+ ('?' == p[1]) )
+ {
+ q = odf_find (p + 2, end - p - 2, "?>");
+ if (NULL == q)
+ return;
+ p = q + 2;
+ continue;
+ }
+ if ( (end - p >= 2) &&
+ ('!' == p[1]) )
+ {
+ /* DOCTYPE; we do not support internal subsets (and thus never
+ expand user-defined entities) */
+ q = memchr (p, '>', end - p);
+ if (NULL == q)
+ return;
+ p = q + 1;
+ continue;
+ }
+ if ( (end - p >= 2) &&
+ ('/' == p[1]) )
+ {
+ q = memchr (p, '>', end - p);
+ if (NULL == q)
+ return;
+ odf_end_element (sc);
+ p = q + 1;
+ continue;
+ }
+ /* start tag */
+ {
+ const char *name = p + 1;
+ const char *attrs;
+ char quote = '\0';
+ int empty;
+
+ q = name;
+ while ( (q < end) &&
+ (! odf_is_ws (*q)) &&
+ ('/' != *q) &&
+ ('>' != *q) )
+ q++;
+ attrs = q;
+ /* find the closing '>', honouring quoted attribute values */
+ while (q < end)
+ {
+ if ('\0' != quote)
+ {
+ if (quote == *q)
+ quote = '\0';
+ }
+ else if ( ('"' == *q) || ('\'' == *q) )
+ quote = *q;
+ else if ('>' == *q)
+ break;
+ q++;
+ }
+ if (q >= end)
+ return;
+ empty = (q > attrs) && ('/' == q[-1]);
+ if (0 != odf_start_element (sc,
+ name,
+ attrs - name,
+ attrs,
+ empty ? q - 1 : q,
+ empty))
+ return;
+ p = q + 1;
+ }
+ }
+}
+
+
+/**
+ * Read the file @a name from the archive, at most @a max_size bytes of
+ * it (the rest is silently ignored).
+ *
+ * @param uf archive to read from
+ * @param name name of the file to read
+ * @param max_size maximum number of bytes to return
+ * @param[out] size set to the number of bytes read
+ * @return NULL on error, otherwise the data (0-terminated), to be freed
+ */
+static char *
+odf_read_member (struct EXTRACTOR_UnzipFile *uf,
+ const char *name,
+ size_t max_size,
+ size_t *size)
+{
+ char filename_inzip[MAXFILENAME];
+ struct EXTRACTOR_UnzipFileInfo file_info;
+ char *buf;
+ size_t cap;
+ size_t off;
+
+ if (EXTRACTOR_UNZIP_OK !=
+ EXTRACTOR_common_unzip_go_find_local_file (uf,
+ name,
+ 2))
+ return NULL;
+ if (EXTRACTOR_UNZIP_OK !=
+ EXTRACTOR_common_unzip_get_current_file_info (uf,
+ &file_info,
+ filename_inzip,
+ sizeof (filename_inzip),
+ NULL, 0, NULL, 0))
+ return NULL;
+ if (EXTRACTOR_UNZIP_OK !=
+ EXTRACTOR_common_unzip_open_current_file (uf))
+ return NULL;
+ /* the size in the header is only a hint, do not trust it */
+ cap = 64 * 1024;
+ if ( (file_info.uncompressed_size > cap) &&
+ (file_info.uncompressed_size < 4 * 1024 * 1024) )
+ cap = file_info.uncompressed_size;
+ if (cap > max_size)
+ cap = max_size;
+ if (NULL == (buf = malloc (cap + 1)))
+ {
+ EXTRACTOR_common_unzip_close_current_file (uf);
+ return NULL;
+ }
+ off = 0;
+ while (off < max_size)
+ {
+ ssize_t got;
+
+ if (off == cap)
+ {
+ size_t ncap = (cap > max_size / 2) ? max_size : 2 * cap;
+ char *nbuf;
+
+ if (NULL == (nbuf = realloc (buf, ncap + 1)))
+ break;
+ buf = nbuf;
+ cap = ncap;
+ }
+ got = EXTRACTOR_common_unzip_read_current_file (uf,
+ &buf[off],
+ cap - off);
+ if (got <= 0)
+ break;
+ off += (size_t) got;
+ }
+ EXTRACTOR_common_unzip_close_current_file (uf);
+ buf[off] = '\0';
+ *size = off;
+ return buf;
+}
+
+
+/**
+ * Extract the running text of the document body (content.xml) and
+ * report it as #EXTRACTOR_METATYPE_FULL_TEXT.
+ *
+ * Only content.xml is read, so headers and footers (which live in the
+ * master pages in styles.xml) never show up. Within the body we skip
+ * comments (office:annotation), footnotes and endnotes (text:note:
+ * they interrupt the sentence they are anchored in, and are not part
+ * of the running text), deleted text of tracked changes, page number
+ * and page count fields, alternative text of drawing objects and the
+ * speaker notes of presentations. Paragraphs and headings (text:p,
+ * text:h) become paragraphs; this also covers list items, table cells
+ * (spreadsheets), text frames and the text boxes on presentation
+ * slides and drawings.
+ *
+ * @param uf archive to read from
+ * @param ec extraction context
+ * @return 0 to continue extracting, 1 to stop
+ */
+static int
+odf_extract_fulltext (struct EXTRACTOR_UnzipFile *uf,
+ struct EXTRACTOR_ExtractContext *ec)
+{
+ struct OdfScanner *sc;
+ char *xml;
+ size_t xml_size;
+ int ret;
+
+ if (NULL == (xml = odf_read_member (uf,
+ CONTENTFILE,
+ MAX_CONTENT_SIZE,
+ &xml_size)))
+ return 0;
+ if (NULL == (sc = calloc (1, sizeof (*sc))))
+ {
+ free (xml);
+ return 0;
+ }
+ EXTRACTOR_fulltext_init_ (&sc->ft);
+ EXTRACTOR_fulltext_set_context_ (&sc->ft, ec);
+ odf_scan (sc, xml, xml_size);
+ free (xml);
+ ret = EXTRACTOR_fulltext_emit_ (&sc->ft, ec, "odf");
+ EXTRACTOR_fulltext_clear_ (&sc->ft);
+ free (sc);
+ return (0 != ret) ? 1 : 0;
+}
+
+
/**
* Main entry method for the ODF extraction plugin.
*
@@ -188,8 +1124,7 @@ EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec)
2))
{
/* metafile not found */
- EXTRACTOR_common_unzip_close (uf);
- return;
+ goto FULLTEXT;
}
if (EXTRACTOR_UNZIP_OK !=
EXTRACTOR_common_unzip_get_current_file_info (uf,
@@ -199,15 +1134,13 @@ EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec)
NULL, 0, NULL, 0))
{
/* problems accessing metafile */
- EXTRACTOR_common_unzip_close (uf);
- return;
+ goto FULLTEXT;
}
if (EXTRACTOR_UNZIP_OK !=
EXTRACTOR_common_unzip_open_current_file (uf))
{
/* problems with unzip */
- EXTRACTOR_common_unzip_close (uf);
- return;
+ goto FULLTEXT;
}
buf_size = file_info.uncompressed_size;
@@ -215,20 +1148,19 @@ EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec)
{
/* too big to be meta-data! */
EXTRACTOR_common_unzip_close_current_file (uf);
- EXTRACTOR_common_unzip_close (uf);
- return;
+ goto FULLTEXT;
}
if (NULL == (buf = malloc (buf_size + 1)))
{
/* out of memory */
EXTRACTOR_common_unzip_close_current_file (uf);
- EXTRACTOR_common_unzip_close (uf);
- return;
+ goto FULLTEXT;
}
if (buf_size != EXTRACTOR_common_unzip_read_current_file (uf, buf, buf_size))
{
EXTRACTOR_common_unzip_close_current_file (uf);
- goto CLEANUP;
+ free (buf);
+ goto FULLTEXT;
}
EXTRACTOR_common_unzip_close_current_file (uf);
/* we don't do "proper" parsing of the meta-data but rather use some heuristics
@@ -306,7 +1238,10 @@ EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec)
"text/plain",
key,
epos - spos + 1))
+ {
+ free (buf);
goto CLEANUP;
+ }
pbuf = epos;
}
else
@@ -314,8 +1249,12 @@ EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec)
}
}
}
-CLEANUP:
free (buf);
+FULLTEXT:
+ if (EXTRACTOR_fulltext_requested_ (ec->config))
+ (void) odf_extract_fulltext (uf,
+ ec);
+CLEANUP:
EXTRACTOR_common_unzip_close (uf);
}
diff --git a/src/plugins/ole2_extractor.c b/src/plugins/ole2_extractor.c
@@ -32,6 +32,7 @@
#include "extractor.h"
#include "convert.h"
#include "msoffice_biff.h"
+#include "fulltext.h"
#include <glib-object.h>
#include <string.h>
#include <stdio.h>
@@ -820,6 +821,604 @@ history_extract (GsfInput *stream,
}
+/* ********************** Word 97-2003 full text ********************** */
+
+/**
+ * Largest CLX (piece table and property modifiers) we are willing to
+ * load. Real documents have a few kilobytes at most.
+ */
+#define WORD_MAX_CLX (16 * 1024 * 1024)
+
+/**
+ * Maximum nesting depth of fields we keep track of; fields nested more
+ * deeply are balanced correctly, but all their text is dropped.
+ */
+#define WORD_MAX_FIELD_DEPTH 32
+
+/**
+ * Number of characters of a field instruction we remember to decide
+ * which kind of field it is.
+ */
+#define WORD_FIELD_INSTR_LEN 32
+
+/**
+ * Maximum number of characters of the main text we convert. Pieces
+ * may all point to the same bytes, so without this limit a small file
+ * could keep us busy for a long time (e.g. with text that is hidden
+ * inside a field instruction and thus never fills the output buffer).
+ */
+#define WORD_MAX_CHARS (32 * 1024 * 1024)
+
+/**
+ * Size of the chunks in which we read the text of a piece.
+ */
+#define WORD_READ_CHUNK 4096
+
+
+/**
+ * A field that is currently open in the text stream.
+ */
+struct WordField
+{
+ /**
+ * The start of the field instruction (ASCII only).
+ */
+ char instr[WORD_FIELD_INSTR_LEN];
+
+ /**
+ * Number of valid bytes in @e instr.
+ */
+ unsigned int instr_len;
+
+ /**
+ * 0 while we are in the field instruction, 1 once we got to the
+ * field result.
+ */
+ int in_result;
+
+ /**
+ * 1 if the result of this field must be dropped (page numbers).
+ */
+ int hide_result;
+};
+
+
+/**
+ * State while converting the main text of a Word document.
+ */
+struct WordText
+{
+ /**
+ * Where the text goes.
+ */
+ struct EXTRACTOR_FullText ft;
+
+ /**
+ * Stack of open fields.
+ */
+ struct WordField fields[WORD_MAX_FIELD_DEPTH];
+
+ /**
+ * Number of entries in @e fields.
+ */
+ unsigned int depth;
+
+ /**
+ * Number of open fields beyond #WORD_MAX_FIELD_DEPTH.
+ */
+ unsigned int overflow;
+
+ /**
+ * 1 if the text at the current position is visible (not part of a
+ * field instruction or a dropped field result).
+ */
+ int visible;
+
+ /**
+ * Pending UTF-16 high surrogate, 0 for none.
+ */
+ uint16_t high_surrogate;
+};
+
+
+/**
+ * Mapping of the bytes 0x80..0x9F in "compressed" (8-bit) text pieces
+ * to Unicode ([MS-DOC] 2.9.73, FcCompressed); 0 means that the byte
+ * maps to itself (and thus is a C1 control that we drop).
+ */
+static const uint16_t word_c1_map[32] = {
+ 0, 0, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
+ 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0, 0, 0,
+ 0, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
+ 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0, 0, 0x0178
+};
+
+
+/**
+ * Read a little-endian 16-bit value.
+ *
+ * @param p where to read
+ * @return the value
+ */
+static uint16_t
+word_le16 (const guint8 *p)
+{
+ return (uint16_t) (p[0] | (p[1] << 8));
+}
+
+
+/**
+ * Read a little-endian 32-bit value.
+ *
+ * @param p where to read
+ * @return the value
+ */
+static uint32_t
+word_le32 (const guint8 *p)
+{
+ return ((uint32_t) p[0])
+ | (((uint32_t) p[1]) << 8)
+ | (((uint32_t) p[2]) << 16)
+ | (((uint32_t) p[3]) << 24);
+}
+
+
+/**
+ * Recompute whether text at the current position is visible.
+ *
+ * @param wt conversion state
+ */
+static void
+word_update_visible (struct WordText *wt)
+{
+ unsigned int i;
+
+ wt->visible = (0 == wt->overflow);
+ for (i = 0; i < wt->depth; i++)
+ if ( (! wt->fields[i].in_result) ||
+ (wt->fields[i].hide_result) )
+ wt->visible = 0;
+}
+
+
+/**
+ * Check whether the field instruction @a instr is one of the fields
+ * showing page numbers (PAGE, NUMPAGES, SECTIONPAGES), whose result we
+ * drop.
+ *
+ * @param f the field
+ * @return 1 if the field result must be dropped
+ */
+static int
+word_is_page_field (const struct WordField *f)
+{
+ static const char *const names[] = {
+ "PAGE", "NUMPAGES", "SECTIONPAGES", NULL
+ };
+ unsigned int s = 0;
+ unsigned int e;
+ unsigned int i;
+
+ while ( (s < f->instr_len) &&
+ (' ' == f->instr[s]) )
+ s++;
+ e = s;
+ while ( (e < f->instr_len) &&
+ (' ' != f->instr[e]) &&
+ ('\\' != f->instr[e]) )
+ e++;
+ for (i = 0; NULL != names[i]; i++)
+ if ( (strlen (names[i]) == e - s) &&
+ (0 == strncasecmp (names[i], &f->instr[s], e - s)) )
+ return 1;
+ return 0;
+}
+
+
+/**
+ * Process one character of the main document text.
+ *
+ * @param wt conversion state
+ * @param cp the character (Unicode code point or Word control code)
+ */
+static void
+word_char (struct WordText *wt,
+ uint32_t cp)
+{
+ struct WordField *top;
+
+ switch (cp)
+ {
+ case 0x13: /* field begin */
+ if ( (0 < wt->overflow) ||
+ (WORD_MAX_FIELD_DEPTH == wt->depth) )
+ {
+ wt->overflow++;
+ }
+ else
+ {
+ top = &wt->fields[wt->depth++];
+ top->instr_len = 0;
+ top->in_result = 0;
+ top->hide_result = 0;
+ }
+ word_update_visible (wt);
+ return;
+ case 0x14: /* field separator */
+ if ( (0 == wt->overflow) &&
+ (0 < wt->depth) )
+ {
+ top = &wt->fields[wt->depth - 1];
+ if (! top->in_result)
+ {
+ top->in_result = 1;
+ top->hide_result = word_is_page_field (top);
+ word_update_visible (wt);
+ }
+ }
+ return;
+ case 0x15: /* field end */
+ if (0 < wt->overflow)
+ wt->overflow--;
+ else if (0 < wt->depth)
+ wt->depth--;
+ word_update_visible (wt);
+ return;
+ default:
+ break;
+ }
+ if ( (0 == wt->overflow) &&
+ (0 < wt->depth) )
+ {
+ top = &wt->fields[wt->depth - 1];
+ if ( (! top->in_result) &&
+ (top->instr_len < WORD_FIELD_INSTR_LEN) )
+ top->instr[top->instr_len++] = (cp >= 0x20) && (cp < 0x7F)
+ ? (char) cp
+ : ' ';
+ }
+ if (! wt->visible)
+ return;
+ switch (cp)
+ {
+ case 0x07: /* table cell / row end */
+ case 0x0C: /* page or section break */
+ case 0x0D: /* paragraph end */
+ EXTRACTOR_fulltext_paragraph_ (&wt->ft);
+ return;
+ case 0x09: /* tab */
+ case 0x0B: /* line break */
+ EXTRACTOR_fulltext_space_ (&wt->ft);
+ return;
+ case 0x1E: /* non-breaking hyphen */
+ EXTRACTOR_fulltext_append_cp_ (&wt->ft, '-');
+ return;
+ case 0x1F: /* optional hyphen */
+ return;
+ default:
+ break;
+ }
+ if (cp < 0x20)
+ return; /* anchors of pictures, notes, comments, ... */
+ EXTRACTOR_fulltext_append_cp_ (&wt->ft, cp);
+}
+
+
+/**
+ * Process one UTF-16 code unit of the main document text.
+ *
+ * @param wt conversion state
+ * @param cu the code unit
+ */
+static void
+word_utf16 (struct WordText *wt,
+ uint16_t cu)
+{
+ if ( (cu >= 0xDC00) && (cu <= 0xDFFF) &&
+ (0 != wt->high_surrogate) )
+ {
+ uint32_t cp;
+
+ cp = 0x10000
+ + (((uint32_t) (wt->high_surrogate - 0xD800)) << 10)
+ + (cu - 0xDC00);
+ wt->high_surrogate = 0;
+ word_char (wt, cp);
+ return;
+ }
+ if (0 != wt->high_surrogate)
+ {
+ wt->high_surrogate = 0;
+ word_char (wt, 0xFFFD);
+ }
+ if ( (cu >= 0xD800) && (cu <= 0xDBFF) )
+ {
+ wt->high_surrogate = cu;
+ return;
+ }
+ if ( (cu >= 0xDC00) && (cu <= 0xDFFF) )
+ {
+ word_char (wt, 0xFFFD);
+ return;
+ }
+ word_char (wt, cu);
+}
+
+
+/**
+ * Convert the text of one piece.
+ *
+ * @param wt conversion state
+ * @param word the WordDocument stream
+ * @param off offset of the text in @a word
+ * @param nchars number of characters in the piece
+ * @param compressed 1 for 8-bit text, 0 for UTF-16LE
+ * @return 0 on success, -1 if the piece is outside of @a word
+ */
+static int
+word_piece (struct WordText *wt,
+ GsfInput *word,
+ uint64_t off,
+ uint64_t nchars,
+ int compressed)
+{
+ guint8 buf[WORD_READ_CHUNK];
+ uint64_t nbytes;
+ uint64_t size;
+
+ size = (uint64_t) gsf_input_size (word);
+ nbytes = compressed ? nchars : 2 * nchars;
+ if ( (off > size) ||
+ (nbytes > size - off) )
+ return -1;
+ if (gsf_input_seek (word, (gsf_off_t) off, G_SEEK_SET))
+ return -1;
+ while ( (0 < nbytes) &&
+ (! wt->ft.full) )
+ {
+ size_t want = (nbytes > sizeof (buf)) ? sizeof (buf) : (size_t) nbytes;
+ size_t i;
+
+ if (NULL == gsf_input_read (word, want, buf))
+ return -1;
+ nbytes -= want;
+ if (compressed)
+ {
+ for (i = 0; i < want; i++)
+ {
+ uint32_t cp = buf[i];
+
+ if ( (cp >= 0x80) &&
+ (cp <= 0x9F) )
+ {
+ if (0 == word_c1_map[cp - 0x80])
+ continue;
+ cp = word_c1_map[cp - 0x80];
+ }
+ word_char (wt, cp);
+ }
+ }
+ else
+ {
+ for (i = 0; i + 1 < want; i += 2)
+ word_utf16 (wt, word_le16 (&buf[i]));
+ }
+ }
+ return 0;
+}
+
+
+/**
+ * Read @a len bytes at @a off from @a in into a fresh buffer.
+ *
+ * @param in stream to read from
+ * @param off where to start
+ * @param len number of bytes to read
+ * @return NULL on error
+ */
+static guint8 *
+word_read_range (GsfInput *in,
+ uint32_t off,
+ uint32_t len)
+{
+ guint8 *buf;
+ gsf_off_t size;
+
+ size = gsf_input_size (in);
+ if ( (0 == len) ||
+ ((gsf_off_t) off > size) ||
+ ((gsf_off_t) len > size - (gsf_off_t) off) )
+ return NULL;
+ if (NULL == (buf = malloc (len)))
+ return NULL;
+ if ( (gsf_input_seek (in, (gsf_off_t) off, G_SEEK_SET)) ||
+ (NULL == gsf_input_read (in, len, buf)) )
+ {
+ free (buf);
+ return NULL;
+ }
+ return buf;
+}
+
+
+/**
+ * Extract the main text of a Word 97-2003 document and report it as
+ * #EXTRACTOR_METATYPE_FULL_TEXT.
+ *
+ * We parse the File Information Block at the start of the
+ * WordDocument stream, load the piece table (CLX) from the table
+ * stream it points to and convert the character positions
+ * [0, ccpText). That range is the main document only: headers and
+ * footers, footnotes and endnotes, comments and text boxes all come
+ * after it and are thus left out. Paragraph marks, table cell/row
+ * marks and page/section breaks end a paragraph. Of fields, only the
+ * result is kept (not the instruction), and the results of page
+ * number fields (PAGE, NUMPAGES, SECTIONPAGES) are dropped.
+ *
+ * Limitations: deleted text of tracked changes and hidden text are
+ * still included (dropping them requires parsing character
+ * properties); documents older than Word 97 are ignored.
+ *
+ * @param infile the OLE2 container
+ * @param ec extraction context
+ * @return 0 to continue extracting, 1 to stop
+ */
+static int
+word_extract_fulltext (GsfInfile *infile,
+ struct EXTRACTOR_ExtractContext *ec)
+{
+ GsfInput *word;
+ GsfInput *table = NULL;
+ guint8 fib[34];
+ guint8 *hdr = NULL;
+ guint8 *clx = NULL;
+ struct WordText *wt = NULL;
+ uint16_t flags;
+ uint16_t csw;
+ uint16_t cslw;
+ uint16_t cb_fclcb;
+ uint32_t fib_len;
+ uint32_t ccp_text;
+ uint32_t fc_clx;
+ uint32_t lcb_clx;
+ uint32_t pos;
+ uint32_t lcb_plc;
+ uint32_t npieces;
+ uint32_t next_cp;
+ const guint8 *plc;
+ uint32_t i;
+ int ret = 0;
+
+ if (NULL == (word = gsf_infile_child_by_name (infile, "WordDocument")))
+ return 0;
+ if ( (gsf_input_seek (word, 0, G_SEEK_SET)) ||
+ (NULL == gsf_input_read (word, sizeof (fib), fib)) )
+ goto CLEANUP;
+ if (0xA5EC != word_le16 (&fib[0]))
+ goto CLEANUP;
+ flags = word_le16 (&fib[0x0A]);
+ if (0 != (flags & 0x0100))
+ goto CLEANUP; /* fEncrypted */
+ csw = word_le16 (&fib[32]);
+ if (0x000E != csw)
+ goto CLEANUP; /* not a Word 97 or later FIB */
+ /* FibBase, csw, fibRgW, cslw, fibRgLw, cbRgFcLcb, fibRgFcLcb up to
+ and including fcClx/lcbClx (pair #33) */
+ fib_len = 34 + 2 * 14 + 2 + 4 * 22 + 2 + 8 * 34;
+ if (NULL == (hdr = word_read_range (word, 0, fib_len)))
+ goto CLEANUP;
+ cslw = word_le16 (&hdr[34 + 2 * 14]);
+ if (0x0016 != cslw)
+ goto CLEANUP;
+ ccp_text = word_le32 (&hdr[34 + 2 * 14 + 2 + 4 * 3]);
+ cb_fclcb = word_le16 (&hdr[34 + 2 * 14 + 2 + 4 * 22]);
+ if (cb_fclcb < 34)
+ goto CLEANUP;
+ fc_clx = word_le32 (&hdr[34 + 2 * 14 + 2 + 4 * 22 + 2 + 8 * 33]);
+ lcb_clx = word_le32 (&hdr[34 + 2 * 14 + 2 + 4 * 22 + 2 + 8 * 33 + 4]);
+ if (ccp_text > WORD_MAX_CHARS)
+ ccp_text = WORD_MAX_CHARS;
+ if ( (0 == ccp_text) ||
+ (lcb_clx > WORD_MAX_CLX) )
+ goto CLEANUP;
+ table = gsf_infile_child_by_name (infile,
+ (0 != (flags & 0x0200))
+ ? "1Table"
+ : "0Table");
+ if (NULL == table)
+ goto CLEANUP;
+ if (NULL == (clx = word_read_range (table, fc_clx, lcb_clx)))
+ goto CLEANUP;
+ /* skip the Prc entries (property modifiers), find the Pcdt */
+ pos = 0;
+ while ( (pos < lcb_clx) &&
+ (0x01 == clx[pos]) )
+ {
+ int16_t cb_grpprl;
+
+ if (lcb_clx - pos < 3)
+ goto CLEANUP;
+ cb_grpprl = (int16_t) word_le16 (&clx[pos + 1]);
+ if (cb_grpprl < 0)
+ goto CLEANUP;
+ pos += 3;
+ if ((uint32_t) cb_grpprl > lcb_clx - pos)
+ goto CLEANUP;
+ pos += (uint32_t) cb_grpprl;
+ }
+ if ( (lcb_clx - pos < 5) ||
+ (0x02 != clx[pos]) )
+ goto CLEANUP;
+ lcb_plc = word_le32 (&clx[pos + 1]);
+ pos += 5;
+ if ( (lcb_plc > lcb_clx - pos) ||
+ (lcb_plc < 4 + 12) ||
+ (0 != (lcb_plc - 4) % 12) )
+ goto CLEANUP;
+ plc = &clx[pos];
+ npieces = (lcb_plc - 4) / 12;
+ if (NULL == (wt = calloc (1, sizeof (*wt))))
+ goto CLEANUP;
+ EXTRACTOR_fulltext_init_ (&wt->ft);
+ EXTRACTOR_fulltext_set_context_ (&wt->ft, ec);
+ wt->visible = 1;
+ next_cp = 0;
+ for (i = 0; i < npieces; i++)
+ {
+ uint32_t cp_start = word_le32 (&plc[4 * i]);
+ uint32_t cp_end = word_le32 (&plc[4 * (i + 1)]);
+ const guint8 *pcd = &plc[4 * (npieces + 1) + 8 * i];
+ uint32_t fc = word_le32 (&pcd[2]);
+ int compressed = (0 != (fc & 0x40000000));
+ uint64_t off;
+
+ off = compressed
+ ? (fc & 0x3FFFFFFF) / 2
+ : (fc & 0x3FFFFFFF);
+
+ if (wt->ft.full)
+ break;
+ if (next_cp >= ccp_text)
+ break;
+ if (cp_end > ccp_text)
+ cp_end = ccp_text;
+ /* never convert a character position twice, pieces must be in
+ order and must not overlap */
+ if (cp_start < next_cp)
+ {
+ if (cp_end <= next_cp)
+ continue;
+ off += (uint64_t) (next_cp - cp_start) * (compressed ? 1 : 2);
+ cp_start = next_cp;
+ }
+ if (cp_end <= cp_start)
+ continue;
+ if (0 != word_piece (wt,
+ word,
+ off,
+ cp_end - cp_start,
+ compressed))
+ continue;
+ next_cp = cp_end;
+ }
+ if (0 != wt->high_surrogate)
+ word_char (wt, 0xFFFD);
+ ret = (0 != EXTRACTOR_fulltext_emit_ (&wt->ft, ec, "ole2")) ? 1 : 0;
+CLEANUP:
+ if (NULL != wt)
+ {
+ EXTRACTOR_fulltext_clear_ (&wt->ft);
+ free (wt);
+ }
+ free (clx);
+ free (hdr);
+ if (NULL != table)
+ g_object_unref (G_OBJECT (table));
+ g_object_unref (G_OBJECT (word));
+ return ret;
+}
+
+
/* *************************** custom GSF input method ***************** */
#define LE_TYPE_INPUT (le_input_get_type ())
@@ -1218,9 +1817,7 @@ EXTRACTOR_ole2_extract_method (struct EXTRACTOR_ExtractContext *ec)
if (0 != ret)
goto CLEANUP;
- if (lcb < 6)
- goto CLEANUP;
- for (i = 0; i<gsf_infile_num_children (infile); i++)
+ for (i = 0; (lcb >= 6) && (i<gsf_infile_num_children (infile)); i++)
{
if (ret != 0)
break;
@@ -1237,6 +1834,10 @@ EXTRACTOR_ole2_extract_method (struct EXTRACTOR_ExtractContext *ec)
g_object_unref (G_OBJECT (src));
}
}
+ if ( (0 == ret) &&
+ (EXTRACTOR_fulltext_requested_ (ec->config)) )
+ ret = word_extract_fulltext (infile,
+ ec);
CLEANUP:
g_object_unref (G_OBJECT (infile));
g_object_unref (G_OBJECT (input));
diff --git a/src/plugins/pagelayout.c b/src/plugins/pagelayout.c
@@ -0,0 +1,653 @@
+/*
+ This file is part of libextractor.
+ Copyright (C) 2026 Christian Grothoff
+
+ libextractor is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ libextractor is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with libextractor; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+/**
+ * @file plugins/pagelayout.c
+ * @brief paragraph detection from line geometry for paged formats
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "pagelayout.h"
+#include <math.h>
+
+/**
+ * Maximum number of left margins (text columns, indentation levels
+ * of lists, ...) we track per page.
+ */
+#define MAX_MARGINS 8
+
+/**
+ * Lines whose left end is within this many font sizes of a margin
+ * start "at" that margin.
+ */
+#define AT_MARGIN 0.3
+
+/**
+ * Minimum indentation of a paragraph's first line, in font sizes.
+ */
+#define MIN_INDENT 0.5
+
+/**
+ * Maximum indentation of a paragraph's first line, in font sizes.
+ */
+#define MAX_INDENT 4.0
+
+
+/**
+ * Per-margin statistics.
+ */
+struct Margin
+{
+ /**
+ * Horizontal position of the margin.
+ */
+ double x;
+
+ /**
+ * Typical right end of the lines starting at this margin.
+ */
+ double right;
+
+ /**
+ * 1 if most lines at this margin end at @e right (justified text).
+ */
+ int justified;
+};
+
+
+/**
+ * Layout statistics of one page, computed over the non-skipped lines.
+ */
+struct PageStats
+{
+ /**
+ * Left margins.
+ */
+ struct Margin margins[MAX_MARGINS];
+
+ /**
+ * Number of entries in @e margins.
+ */
+ unsigned int num_margins;
+
+ /**
+ * Median font size.
+ */
+ double size;
+
+ /**
+ * Typical baseline distance of consecutive lines.
+ */
+ double spacing;
+
+ /**
+ * Relative uncertainty of the right ends.
+ */
+ double tol;
+};
+
+
+/**
+ * Compare two doubles for qsort().
+ *
+ * @param a first value
+ * @param b second value
+ * @return comparison result
+ */
+static int
+cmp_double (const void *a,
+ const void *b)
+{
+ double x = *(const double *) a;
+ double y = *(const double *) b;
+
+ if (x < y)
+ return -1;
+ if (x > y)
+ return 1;
+ return 0;
+}
+
+
+/**
+ * Return the value at quantile @a q of the @a n values in @a v (which
+ * are sorted in place).
+ *
+ * @param v values, @a n > 0
+ * @param n number of values
+ * @param q quantile in [0,1]
+ * @return the quantile
+ */
+static double
+quantile (double *v,
+ size_t n,
+ double q)
+{
+ size_t i;
+
+ qsort (v,
+ n,
+ sizeof (double),
+ &cmp_double);
+ i = (size_t) (q * (double) (n - 1) + 0.5);
+ if (i >= n)
+ i = n - 1;
+ return v[i];
+}
+
+
+/**
+ * Check whether a geometry value is usable.
+ *
+ * @param d value to check
+ * @return 1 if finite
+ */
+static int
+ok (double d)
+{
+ return isfinite (d) ? 1 : 0;
+}
+
+
+/**
+ * Find the margin a line starting at @a x belongs to: the closest
+ * margin to the left within #MAX_INDENT font sizes (or just right of
+ * it within #AT_MARGIN).
+ *
+ * @param st page statistics
+ * @param x left end of the line
+ * @return margin index, -1 if none
+ */
+static int
+find_margin (const struct PageStats *st,
+ double x)
+{
+ int best = -1;
+ double bestd = 0;
+ unsigned int i;
+
+ for (i = 0; i < st->num_margins; i++)
+ {
+ double d = x - st->margins[i].x;
+
+ if ( (d < -AT_MARGIN * st->size) ||
+ (d > MAX_INDENT * st->size) )
+ continue;
+ if ( (-1 == best) ||
+ (fabs (d) < bestd) )
+ {
+ best = (int) i;
+ bestd = fabs (d);
+ }
+ }
+ return best;
+}
+
+
+/**
+ * Check whether a line starting at @a x starts at a left margin.
+ *
+ * @param st page statistics
+ * @param x left end of the line
+ * @return 1 if at a margin
+ */
+static int
+at_margin (const struct PageStats *st,
+ double x)
+{
+ unsigned int i;
+
+ for (i = 0; i < st->num_margins; i++)
+ if (fabs (x - st->margins[i].x) <= AT_MARGIN * st->size)
+ return 1;
+ return 0;
+}
+
+
+/**
+ * Check whether line @a b (with page statistics @a st) is the indented
+ * first line of a paragraph, given that it follows line @a a.
+ *
+ * @param st statistics of the page of @a b
+ * @param a previous line
+ * @param b line to check
+ * @return 1 if @a b is indented
+ */
+static int
+is_indented (const struct PageStats *st,
+ const struct EXTRACTOR_PageLineGeom *a,
+ const struct EXTRACTOR_PageLineGeom *b)
+{
+ unsigned int i;
+
+ if (at_margin (st,
+ b->x0))
+ return 0;
+ /* a block of lines that are all indented alike (a quote, a list) */
+ if (fabs (a->x0 - b->x0) < AT_MARGIN * st->size)
+ return 0;
+ for (i = 0; i < st->num_margins; i++)
+ {
+ double d = b->x0 - st->margins[i].x;
+
+ if ( (d >= MIN_INDENT * st->size) &&
+ (d <= MAX_INDENT * st->size) )
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Check whether line @a a ends well before the right margin of a
+ * justified block.
+ *
+ * @param st statistics of the page of @a a
+ * @param a line to check
+ * @return 1 if @a a is short
+ */
+static int
+is_short (const struct PageStats *st,
+ const struct EXTRACTOR_PageLineGeom *a)
+{
+ int m;
+ const struct Margin *mg;
+ double width;
+ double slack;
+
+ m = find_margin (st,
+ a->x0);
+ if (-1 == m)
+ return 0;
+ mg = &st->margins[m];
+ if (! mg->justified)
+ return 0;
+ width = mg->right - mg->x;
+ if (width <= 0)
+ return 0;
+ slack = (0.05 + 2 * st->tol) * width;
+ if (slack < 2 * st->size)
+ slack = 2 * st->size;
+ return (a->x1 < mg->right - slack) ? 1 : 0;
+}
+
+
+/**
+ * Check whether the font size changes noticeably from @a a to @a b.
+ *
+ * @param a first line
+ * @param b second line
+ * @return 1 on a size change
+ */
+static int
+size_change (const struct EXTRACTOR_PageLineGeom *a,
+ const struct EXTRACTOR_PageLineGeom *b)
+{
+ double mx = (a->size > b->size) ? a->size : b->size;
+
+ return (fabs (a->size - b->size) > 0.15 * mx) ? 1 : 0;
+}
+
+
+/**
+ * Compute the layout statistics of page @a page.
+ *
+ * @param page the page
+ * @param g geometry of its lines
+ * @param tol relative uncertainty of right ends
+ * @param[out] st statistics to fill
+ * @return 0 on success, -1 on allocation failure or if the page has
+ * no usable lines
+ */
+static int
+page_stats (const struct EXTRACTOR_FullTextPage *page,
+ const struct EXTRACTOR_PageLineGeom *g,
+ double tol,
+ struct PageStats *st)
+{
+ double *v;
+ double *xs;
+ size_t n;
+ size_t k;
+ unsigned int i;
+ unsigned int prev;
+ int have_prev;
+
+ memset (st, 0, sizeof (*st));
+ st->tol = tol;
+ if (0 == page->num_lines)
+ return -1;
+ v = malloc (page->num_lines * sizeof (double));
+ xs = malloc (page->num_lines * sizeof (double));
+ if ( (NULL == v) ||
+ (NULL == xs) )
+ {
+ free (v);
+ free (xs);
+ return -1;
+ }
+ /* median font size */
+ n = 0;
+ for (i = 0; i < page->num_lines; i++)
+ if ( (! page->lines[i].skip) &&
+ (g[i].size > 0) )
+ v[n++] = g[i].size;
+ if (0 == n)
+ {
+ free (v);
+ free (xs);
+ return -1;
+ }
+ st->size = quantile (v, n, 0.5);
+ /* typical line spacing */
+ n = 0;
+ have_prev = 0;
+ prev = 0;
+ for (i = 0; i < page->num_lines; i++)
+ {
+ double dy;
+
+ if (page->lines[i].skip)
+ continue;
+ if (have_prev)
+ {
+ dy = g[i].y - g[prev].y;
+ if ( (dy > 0.3 * st->size) &&
+ (dy < 3 * st->size) )
+ v[n++] = dy;
+ }
+ prev = i;
+ have_prev = 1;
+ }
+ st->spacing = (n > 0) ? quantile (v, n, 0.5) : 1.2 * st->size;
+ /* left margins: clusters of line starts */
+ n = 0;
+ for (i = 0; i < page->num_lines; i++)
+ if (! page->lines[i].skip)
+ xs[n++] = g[i].x0;
+ qsort (xs,
+ n,
+ sizeof (double),
+ &cmp_double);
+ k = 0;
+ while (k < n)
+ {
+ size_t e = k + 1;
+ size_t cnt;
+
+ while ( (e < n) &&
+ (xs[e] - xs[e - 1] <= AT_MARGIN * st->size) )
+ e++;
+ cnt = e - k;
+ if ( ( (cnt >= 2) &&
+ (cnt * 100 >= n * 15) ) ||
+ ( (n < 4) &&
+ (0 == k) ) )
+ {
+ if (st->num_margins < MAX_MARGINS)
+ {
+ st->margins[st->num_margins].x = xs[k];
+ st->num_margins++;
+ }
+ else
+ {
+ /* replace the smallest cluster? keep it simple: stop */
+ break;
+ }
+ }
+ k = e;
+ }
+ /* right ends per margin */
+ for (i = 0; i < st->num_margins; i++)
+ {
+ struct Margin *mg = &st->margins[i];
+ unsigned int j;
+ size_t at;
+ size_t near;
+ double slack;
+
+ n = 0;
+ for (j = 0; j < page->num_lines; j++)
+ if ( (! page->lines[j].skip) &&
+ ((int) i == find_margin (st, g[j].x0)) )
+ v[n++] = g[j].x1;
+ if (n < 3)
+ continue;
+ mg->right = quantile (v, n, 0.75);
+ slack = tol * (mg->right - mg->x);
+ if (slack < 0.5 * st->size)
+ slack = 0.5 * st->size;
+ near = 0;
+ for (at = 0; at < n; at++)
+ if (fabs (v[at] - mg->right) <= slack)
+ near++;
+ mg->justified = (2 * near >= n) ? 1 : 0;
+ }
+ free (v);
+ free (xs);
+ return 0;
+}
+
+
+/**
+ * Decide whether line @a a ends a paragraph, given that line @a b (on
+ * the same page) follows it.
+ *
+ * @param st page statistics
+ * @param a the line
+ * @param b the next line
+ * @return 1 if @a a ends a paragraph
+ */
+static int
+ends_paragraph (const struct PageStats *st,
+ const struct EXTRACTOR_PageLineGeom *a,
+ const struct EXTRACTOR_PageLineGeom *b)
+{
+ double dy = b->y - a->y;
+ double big;
+ double lim;
+
+ big = (a->size > b->size) ? a->size : b->size;
+ lim = 1.15 * big;
+ if (lim < st->spacing)
+ lim = st->spacing;
+ if ( (dy > 0) &&
+ (dy > 1.3 * lim) )
+ return 1;
+ if (size_change (a, b))
+ return 1;
+ if (is_indented (st, a, b))
+ return 1;
+ if (is_short (st, a))
+ return 1;
+ return 0;
+}
+
+
+void
+EXTRACTOR_pagelayout_paragraphs_ (struct EXTRACTOR_FullTextPage *pages,
+ struct EXTRACTOR_PageLineGeom *const *geoms,
+ unsigned int num_pages,
+ double width_tolerance)
+{
+ struct PageStats *stats;
+ unsigned int p;
+ int have_last;
+ unsigned int last_p = 0;
+ unsigned int last_i = 0;
+
+ if (0 == num_pages)
+ return;
+ stats = calloc (num_pages,
+ sizeof (*stats));
+ if (NULL == stats)
+ return;
+ have_last = 0;
+ for (p = 0; p < num_pages; p++)
+ {
+ struct EXTRACTOR_FullTextPage *page = &pages[p];
+ const struct EXTRACTOR_PageLineGeom *g = geoms[p];
+ unsigned int i;
+ int have_prev;
+ unsigned int prev = 0;
+
+ for (i = 0; i < page->num_lines; i++)
+ {
+ page->lines[i].par_end = 0;
+ if ( (! ok (g[i].x0)) ||
+ (! ok (g[i].x1)) ||
+ (! ok (g[i].y)) ||
+ (! ok (g[i].size)) )
+ page->lines[i].skip = 1;
+ }
+ if (0 != page_stats (page,
+ g,
+ width_tolerance,
+ &stats[p]))
+ continue;
+ have_prev = 0;
+ for (i = 0; i < page->num_lines; i++)
+ {
+ if (page->lines[i].skip)
+ continue;
+ if (have_prev)
+ {
+ page->lines[prev].par_end = ends_paragraph (&stats[p],
+ &g[prev],
+ &g[i]);
+ }
+ else if (have_last)
+ {
+ /* first line of this page, previous page's last line */
+ const struct EXTRACTOR_PageLineGeom *a = &geoms[last_p][last_i];
+
+ if ( (is_short (&stats[last_p], a)) ||
+ (size_change (a, &g[i])) ||
+ (is_indented (&stats[p], a, &g[i])) )
+ pages[last_p].lines[last_i].par_end = 1;
+ }
+ prev = i;
+ have_prev = 1;
+ }
+ if (have_prev)
+ {
+ have_last = 1;
+ last_p = p;
+ last_i = prev;
+ }
+ }
+ if (have_last)
+ pages[last_p].lines[last_i].par_end = 1;
+ free (stats);
+}
+
+
+/**
+ * Precomposed characters for base letter + combining accent.
+ */
+struct Composition
+{
+ /**
+ * Combining accent.
+ */
+ uint16_t accent;
+
+ /**
+ * Base letters (ASCII).
+ */
+ const char *bases;
+
+ /**
+ * Precomposed characters, same order as @e bases.
+ */
+ uint16_t composed[26];
+};
+
+
+/**
+ * Table of precomposed characters.
+ */
+static const struct Composition compositions[] = {
+ { 0x0300, "AEIOUaeiou",
+ { 0xC0, 0xC8, 0xCC, 0xD2, 0xD9, 0xE0, 0xE8, 0xEC, 0xF2, 0xF9 } },
+ { 0x0301, "AEIOUYaeiouyCcNnSsZzLlRr",
+ { 0xC1, 0xC9, 0xCD, 0xD3, 0xDA, 0xDD, 0xE1, 0xE9, 0xED, 0xF3, 0xFA,
+ 0xFD, 0x106, 0x107, 0x143, 0x144, 0x15A, 0x15B, 0x179, 0x17A,
+ 0x139, 0x13A, 0x154, 0x155 } },
+ { 0x0302, "AEIOUaeiou",
+ { 0xC2, 0xCA, 0xCE, 0xD4, 0xDB, 0xE2, 0xEA, 0xEE, 0xF4, 0xFB } },
+ { 0x0303, "ANOano",
+ { 0xC3, 0xD1, 0xD5, 0xE3, 0xF1, 0xF5 } },
+ { 0x0304, "AaEeIiOoUu",
+ { 0x100, 0x101, 0x112, 0x113, 0x12A, 0x12B, 0x14C, 0x14D, 0x16A,
+ 0x16B } },
+ { 0x0306, "AaGgUu",
+ { 0x102, 0x103, 0x11E, 0x11F, 0x16C, 0x16D } },
+ { 0x0307, "ZzIEeCcGg",
+ { 0x17B, 0x17C, 0x130, 0x116, 0x117, 0x10A, 0x10B, 0x120, 0x121 } },
+ { 0x0308, "AEIOUaeiouyY",
+ { 0xC4, 0xCB, 0xCF, 0xD6, 0xDC, 0xE4, 0xEB, 0xEF, 0xF6, 0xFC, 0xFF,
+ 0x178 } },
+ { 0x030A, "AaUu",
+ { 0xC5, 0xE5, 0x16E, 0x16F } },
+ { 0x030B, "OoUu",
+ { 0x150, 0x151, 0x170, 0x171 } },
+ { 0x030C, "CcDdEeNnRrSsTtZz",
+ { 0x10C, 0x10D, 0x10E, 0x10F, 0x11A, 0x11B, 0x147, 0x148, 0x158,
+ 0x159, 0x160, 0x161, 0x164, 0x165, 0x17D, 0x17E } },
+ { 0x0327, "CcSsTtKkLlNnRr",
+ { 0xC7, 0xE7, 0x15E, 0x15F, 0x162, 0x163, 0x136, 0x137, 0x13B, 0x13C,
+ 0x145, 0x146, 0x156, 0x157 } },
+ { 0x0328, "AaEeIiUu",
+ { 0x104, 0x105, 0x118, 0x119, 0x12E, 0x12F, 0x172, 0x173 } },
+ { 0x0337, "Ll",
+ { 0x141, 0x142 } },
+ { 0, NULL, { 0 } }
+};
+
+
+uint32_t
+EXTRACTOR_pagelayout_compose_ (uint32_t base,
+ uint32_t accent)
+{
+ unsigned int i;
+ const char *pos;
+
+ if (0x0131 == base)
+ base = 'i';
+ if (0x0237 == base)
+ base = 'j';
+ if ( (base >= 0x80) ||
+ (! isalpha ((int) base)) )
+ return 0;
+ for (i = 0; NULL != compositions[i].bases; i++)
+ {
+ if (compositions[i].accent != accent)
+ continue;
+ pos = strchr (compositions[i].bases,
+ (int) base);
+ if (NULL == pos)
+ return 0;
+ return compositions[i].composed[pos - compositions[i].bases];
+ }
+ return 0;
+}
+
+
+/* end of pagelayout.c */
diff --git a/src/plugins/pagelayout.h b/src/plugins/pagelayout.h
@@ -0,0 +1,110 @@
+/*
+ This file is part of libextractor.
+ Copyright (C) 2026 Christian Grothoff
+
+ libextractor is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ libextractor is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with libextractor; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+/**
+ * @file plugins/pagelayout.h
+ * @brief paragraph detection from line geometry for paged formats
+ * (PDF, DVI) implementing the "fulltext" option
+ * @author Christian Grothoff
+ */
+#ifndef PAGELAYOUT_H
+#define PAGELAYOUT_H
+
+#include "extractor.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "fulltext.h"
+
+/**
+ * Geometry of one line of text on a page. All values are in the
+ * same (arbitrary) unit, e.g. PDF points or DVI units.
+ */
+struct EXTRACTOR_PageLineGeom
+{
+ /**
+ * Horizontal position of the left end of the line.
+ */
+ double x0;
+
+ /**
+ * Horizontal position of the right end of the line (may be an
+ * estimate).
+ */
+ double x1;
+
+ /**
+ * Vertical position of the baseline, growing downwards.
+ */
+ double y;
+
+ /**
+ * Font size (or line height) of the line.
+ */
+ double size;
+};
+
+
+/**
+ * Decide from the line geometry which lines end a paragraph, and set
+ * the `par_end` flags in @a pages accordingly. Lines marked `skip`
+ * (by #EXTRACTOR_fulltext_filter_margins_(), which should thus be
+ * called first) are ignored.
+ *
+ * A line ends a paragraph if the vertical gap to the next line is
+ * noticeably larger than the page's typical line spacing, if the next
+ * line is indented relative to a left margin of the page, if the font
+ * size changes noticeably, or if the line ends well before the right
+ * margin of a justified text block. The last line of a page ends a paragraph if it is short or
+ * if the first line of the next page is indented.
+ *
+ * @param pages array of pages
+ * @param geoms geometry of the lines, `geoms[p][i]` belongs to
+ * `pages[p].lines[i]`
+ * @param num_pages number of entries in @a pages and @a geoms
+ * @param width_tolerance relative uncertainty of the `x1` values
+ * (e.g. 0.02 if exact, larger if the right ends are estimated)
+ */
+void
+EXTRACTOR_pagelayout_paragraphs_ (struct EXTRACTOR_FullTextPage *pages,
+ struct EXTRACTOR_PageLineGeom *const *geoms,
+ unsigned int num_pages,
+ double width_tolerance);
+
+/**
+ * Compose a Latin base letter and a combining accent into the
+ * corresponding precomposed character, for the common accents
+ * (U+0300-U+030C, U+0327, U+0328, and U+0337 for the Polish L with
+ * stroke). Dotless i and j count as i and j.
+ *
+ * @param base base letter
+ * @param accent combining accent
+ * @return precomposed character, 0 if there is none
+ */
+uint32_t
+EXTRACTOR_pagelayout_compose_ (uint32_t base,
+ uint32_t accent);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/plugins/pdf_extractor.cc b/src/plugins/pdf_extractor.cc
@@ -25,9 +25,15 @@
*/
#include "platform.h"
#include "extractor.h"
+#include "pagelayout.h"
#include <poppler/cpp/poppler-document.h>
#include <poppler/cpp/poppler-global.h>
+#include <poppler/cpp/poppler-page.h>
+#include <algorithm>
+#include <cmath>
+#include <ctime>
#include <string>
+#include <utility>
#include <vector>
@@ -229,6 +235,860 @@ read_all (struct EXTRACTOR_ExtractContext *ec,
}
+/* ****************** full text extraction ********************** */
+
+/**
+ * Maximum number of pages we extract text from.
+ */
+#define PDF_FT_MAX_PAGES 5000
+
+/**
+ * Maximum number of text boxes (words) we look at per page.
+ */
+#define PDF_FT_MAX_WORDS 100000
+
+/**
+ * Maximum number of lines we collect over the whole document.
+ */
+#define PDF_FT_MAX_LINES 500000
+
+/**
+ * Wall-clock budget for the full text extraction, in seconds.
+ */
+#define PDF_FT_DEADLINE 60
+
+/**
+ * Relative height of the top and bottom margin bands in which lines
+ * separated from the body are treated as page furniture that never
+ * splits into columns.
+ */
+#define PDF_FT_BAND 0.12
+
+
+/**
+ * A word (poppler text box) on a page.
+ */
+struct Word
+{
+ /**
+ * Bounding box, in points, origin top left.
+ */
+ double x0;
+ double y0;
+ double x1;
+ double y1;
+
+ /**
+ * UTF-8 text.
+ */
+ std::string text;
+
+ /**
+ * 1 if the word is followed by a space.
+ */
+ bool space_after;
+
+ /**
+ * 1 if the word consists of digits only.
+ */
+ bool numeric;
+};
+
+
+/**
+ * A line of words (in reading order once built).
+ */
+struct Line
+{
+ /**
+ * Indices into the word array, sorted left to right.
+ */
+ std::vector<size_t> words;
+
+ /**
+ * Vertical extent of the line's first word (for grouping).
+ */
+ double core_y0;
+ double core_y1;
+
+ /**
+ * Baseline (median bottom of the words).
+ */
+ double base;
+
+ /**
+ * Horizontal extent.
+ */
+ double x0;
+ double x1;
+
+ /**
+ * Median word height.
+ */
+ double size;
+};
+
+
+/**
+ * Median of a vector of doubles.
+ *
+ * @param v values (reordered), not empty
+ * @return the median
+ */
+static double
+median (std::vector<double> &v)
+{
+ size_t mid = v.size () / 2;
+
+ std::nth_element (v.begin (),
+ v.begin () + mid,
+ v.end ());
+ return v[mid];
+}
+
+
+/**
+ * Current monotonic time in seconds.
+ *
+ * @return seconds
+ */
+static double
+now_s ()
+{
+ struct timespec ts;
+
+ if (0 != clock_gettime (CLOCK_MONOTONIC, &ts))
+ return 0;
+ return (double) ts.tv_sec + ts.tv_nsec / 1e9;
+}
+
+
+/**
+ * Append the text of @a w to @a out, expanding the Latin ligature
+ * characters U+FB00..U+FB06.
+ *
+ * @param out string to append to
+ * @param w text to append
+ */
+static void
+append_expanded (std::string &out,
+ const std::string &w)
+{
+ static const char *const ligs[] = {
+ "ff", "fi", "fl", "ffi", "ffl", "st", "st"
+ };
+ size_t i = 0;
+
+ while (i < w.size ())
+ {
+ if ( (i + 2 < w.size ()) &&
+ ((unsigned char) w[i] == 0xEF) &&
+ ((unsigned char) w[i + 1] == 0xAC) &&
+ ((unsigned char) w[i + 2] <= 0x86) &&
+ ((unsigned char) w[i + 2] >= 0x80) )
+ {
+ out += ligs[(unsigned char) w[i + 2] - 0x80];
+ i += 3;
+ continue;
+ }
+ out += w[i];
+ i++;
+ }
+}
+
+
+/**
+ * Replace a Latin letter followed by a combining accent by the
+ * precomposed character (poppler reports TeX's accents that way).
+ *
+ * @param s UTF-8 text, modified in place
+ */
+static void
+compose_accents (std::string &s)
+{
+ std::string out;
+ size_t i = 0;
+
+ out.reserve (s.size ());
+ while (i < s.size ())
+ {
+ unsigned char c = (unsigned char) s[i];
+ uint32_t base = 0;
+ size_t blen = 0;
+
+ if ( (c < 0x80) &&
+ isalpha (c) )
+ {
+ base = c;
+ blen = 1;
+ }
+ else if ( (0xC4 == c) &&
+ (i + 1 < s.size ()) &&
+ (0xB1 == (unsigned char) s[i + 1]) )
+ {
+ base = 0x131;
+ blen = 2;
+ }
+ if ( (0 != base) &&
+ (i + blen + 1 < s.size ()) )
+ {
+ unsigned char m0 = (unsigned char) s[i + blen];
+ unsigned char m1 = (unsigned char) s[i + blen + 1];
+
+ if ( ( (0xCC == m0) || (0xCD == m0) ) &&
+ ( (m1 & 0xC0) == 0x80) )
+ {
+ uint32_t mark = ((uint32_t) (m0 & 0x1F) << 6) | (m1 & 0x3F);
+ uint32_t comp = EXTRACTOR_pagelayout_compose_ (base,
+ mark);
+
+ if (0 != comp)
+ {
+ /* all precomposed results are below U+0800 */
+ out += (char) (0xC0 | (comp >> 6));
+ out += (char) (0x80 | (comp & 0x3F));
+ i += blen + 2;
+ continue;
+ }
+ }
+ }
+ out += s[i];
+ i++;
+ }
+ s.swap (out);
+}
+
+
+/**
+ * Drop columns of line numbers (or similar pure numbers) in the left
+ * and right page margins, i.e. numbers that are outside of all other
+ * text on the page and aligned with each other.
+ *
+ * @param words words of the page, filtered in place
+ */
+static void
+drop_marginal_numbers (std::vector<Word> &words)
+{
+ double tmin = HUGE_VAL;
+ double tmax = -HUGE_VAL;
+ std::vector<double> hs;
+ double h;
+
+ for (const Word &w : words)
+ if (! w.numeric)
+ {
+ tmin = std::min (tmin, w.x0);
+ tmax = std::max (tmax, w.x1);
+ hs.push_back (w.y1 - w.y0);
+ }
+ if (hs.empty ())
+ return;
+ h = median (hs);
+ for (int side = 0; side < 2; side++)
+ {
+ std::vector<size_t> cand;
+ double lo = HUGE_VAL;
+ double hi = -HUGE_VAL;
+
+ for (size_t i = 0; i < words.size (); i++)
+ {
+ const Word &w = words[i];
+
+ if (! w.numeric)
+ continue;
+ if ( (0 == side) &&
+ (w.x1 < tmin - 0.3 * h) )
+ {
+ cand.push_back (i);
+ lo = std::min (lo, w.x1);
+ hi = std::max (hi, w.x1);
+ }
+ if ( (1 == side) &&
+ (w.x0 > tmax + 0.3 * h) )
+ {
+ cand.push_back (i);
+ lo = std::min (lo, w.x0);
+ hi = std::max (hi, w.x0);
+ }
+ }
+ if ( (cand.size () < 5) ||
+ (hi - lo > 4 * h) )
+ continue;
+ for (size_t i : cand)
+ words[i].text.clear ();
+ }
+ words.erase (std::remove_if (words.begin (),
+ words.end (),
+ [] (const Word &w) {
+ return w.text.empty ();
+ }),
+ words.end ());
+}
+
+
+/**
+ * Group the words of a page into lines by their vertical position.
+ *
+ * @param words words of the page
+ * @param[out] lines resulting lines, sorted top to bottom, each with
+ * its words sorted left to right
+ */
+static void
+group_lines (const std::vector<Word> &words,
+ std::vector<Line> &lines)
+{
+ std::vector<size_t> order (words.size ());
+
+ for (size_t i = 0; i < words.size (); i++)
+ order[i] = i;
+ std::sort (order.begin (),
+ order.end (),
+ [&words] (size_t a, size_t b) {
+ double ca = words[a].y0 + words[a].y1;
+ double cb = words[b].y0 + words[b].y1;
+
+ if (ca != cb)
+ return ca < cb;
+ return words[a].x0 < words[b].x0;
+ });
+ for (size_t i : order)
+ {
+ const Word &w = words[i];
+ double wh = w.y1 - w.y0;
+ bool placed = false;
+
+ /* check the last two lines (a superscript may have opened one) */
+ for (size_t k = 0; (k < 2) && (k < lines.size ()); k++)
+ {
+ Line &l = lines[lines.size () - 1 - k];
+ double lh = l.core_y1 - l.core_y0;
+ double ov = std::min (w.y1, l.core_y1) - std::max (w.y0, l.core_y0);
+
+ if (ov >= 0.5 * std::min (wh, lh))
+ {
+ l.words.push_back (i);
+ placed = true;
+ break;
+ }
+ }
+ if (placed)
+ continue;
+ lines.emplace_back ();
+ lines.back ().words.push_back (i);
+ lines.back ().core_y0 = w.y0;
+ lines.back ().core_y1 = w.y1;
+ }
+ for (Line &l : lines)
+ {
+ std::vector<double> bases;
+ std::vector<double> hs;
+
+ std::sort (l.words.begin (),
+ l.words.end (),
+ [&words] (size_t a, size_t b) {
+ return words[a].x0 < words[b].x0;
+ });
+ l.x0 = HUGE_VAL;
+ l.x1 = -HUGE_VAL;
+ for (size_t i : l.words)
+ {
+ bases.push_back (words[i].y1);
+ hs.push_back (words[i].y1 - words[i].y0);
+ l.x0 = std::min (l.x0, words[i].x0);
+ l.x1 = std::max (l.x1, words[i].x1);
+ }
+ l.base = median (bases);
+ l.size = median (hs);
+ }
+ std::stable_sort (lines.begin (),
+ lines.end (),
+ [] (const Line &a, const Line &b) {
+ return a.base < b.base;
+ });
+}
+
+
+/**
+ * Compute the horizontal coverage of a line: its words, with gaps of
+ * less than half the line height closed.
+ *
+ * @param words words of the page
+ * @param l the line
+ * @param[out] iv resulting intervals, left to right
+ */
+static void
+line_coverage (const std::vector<Word> &words,
+ const Line &l,
+ std::vector<std::pair<double, double> > &iv)
+{
+ iv.clear ();
+ for (size_t i : l.words)
+ {
+ const Word &w = words[i];
+
+ if ( (! iv.empty ()) &&
+ (w.x0 - iv.back ().second < 0.5 * l.size) )
+ iv.back ().second = std::max (iv.back ().second, w.x1);
+ else
+ iv.push_back (std::make_pair (w.x0, w.x1));
+ }
+}
+
+
+/**
+ * Look for a vertical gutter separating two text columns.
+ *
+ * @param words words of the page
+ * @param lines lines of the page
+ * @param furniture which lines are page furniture (ignored)
+ * @param[out] g0 left end of the gutter
+ * @param[out] g1 right end of the gutter
+ * @return true if a gutter was found
+ */
+static bool
+find_gutter (const std::vector<Word> &words,
+ const std::vector<Line> &lines,
+ const std::vector<bool> &furniture,
+ double &g0,
+ double &g1)
+{
+ double left = HUGE_VAL;
+ double right = -HUGE_VAL;
+ std::vector<double> sizes;
+ std::vector<std::pair<double, double> > iv;
+ size_t nbins;
+ size_t body = 0;
+ double bw;
+ double sz;
+
+ for (size_t i = 0; i < lines.size (); i++)
+ {
+ if (furniture[i])
+ continue;
+ left = std::min (left, lines[i].x0);
+ right = std::max (right, lines[i].x1);
+ sizes.push_back (lines[i].size);
+ body++;
+ }
+ if (body < 6)
+ return false;
+ sz = median (sizes);
+ if ( (! (right - left > 10 * sz)) ||
+ (! (sz > 0)) )
+ return false;
+ nbins = 1000;
+ bw = (right - left) / nbins;
+ std::vector<long> cnt (nbins + 1, 0);
+
+ /* difference array: +1 where an interval starts, -1 after its end */
+ for (size_t i = 0; i < lines.size (); i++)
+ {
+ if (furniture[i])
+ continue;
+ line_coverage (words, lines[i], iv);
+ for (const auto &p : iv)
+ {
+ double f0 = std::max (0.0, (p.first - left) / bw);
+ double f1 = std::max (0.0, (p.second - left) / bw);
+ size_t b0 = (size_t) std::min (f0, (double) nbins);
+ size_t b1 = (size_t) std::min (f1 + 1, (double) nbins);
+
+ cnt[b0]++;
+ cnt[b1]--;
+ }
+ }
+ for (size_t b = 1; b < nbins; b++)
+ cnt[b] += cnt[b - 1];
+ /* widest run of rarely covered bins in the middle half */
+ long thr = (long) (body / 5);
+ size_t best_s = 0;
+ size_t best_len = 0;
+ size_t run_s = 0;
+ size_t run_len = 0;
+
+ for (size_t b = nbins / 4; b < 3 * nbins / 4; b++)
+ {
+ if (cnt[b] <= thr)
+ {
+ if (0 == run_len)
+ run_s = b;
+ run_len++;
+ if (run_len > best_len)
+ {
+ best_len = run_len;
+ best_s = run_s;
+ }
+ }
+ else
+ {
+ run_len = 0;
+ }
+ }
+ if ( (0 == best_len) ||
+ (best_len * bw < std::max (4.0, 0.3 * sz)) )
+ return false;
+ g0 = left + best_s * bw;
+ g1 = left + (best_s + best_len) * bw;
+ /* both sides must hold a good share of the text */
+ size_t nl = 0;
+ size_t nr = 0;
+
+ for (size_t i = 0; i < lines.size (); i++)
+ {
+ bool l = false;
+ bool r = false;
+
+ if (furniture[i])
+ continue;
+ for (size_t w : lines[i].words)
+ {
+ if (words[w].x1 <= g0 + bw)
+ l = true;
+ if (words[w].x0 >= g1 - bw)
+ r = true;
+ }
+ if (l)
+ nl++;
+ if (r)
+ nr++;
+ }
+ return (10 * nl >= 3 * body) && (10 * nr >= 3 * body);
+}
+
+
+/**
+ * Build the text of a line from its words and add it to @a page.
+ *
+ * @param words words of the page
+ * @param widx indices of the words of the line, left to right
+ * @param page_top top of the page
+ * @param page_h height of the page
+ * @param page page to add to
+ * @param geom geometry array of @a page to append to
+ * @param total_bytes running total of text bytes, updated
+ * @return 0 on success, -1 to stop (limit reached or no memory)
+ */
+static int
+emit_line (const std::vector<Word> &words,
+ const std::vector<size_t> &widx,
+ double page_top,
+ double page_h,
+ struct EXTRACTOR_FullTextPage *page,
+ std::vector<struct EXTRACTOR_PageLineGeom> &geom,
+ size_t &total_bytes)
+{
+ std::string text;
+ std::vector<double> bases;
+ std::vector<double> hs;
+ struct EXTRACTOR_PageLineGeom g;
+ double y;
+
+ if (widx.empty ())
+ return 0;
+ g.x0 = HUGE_VAL;
+ g.x1 = -HUGE_VAL;
+ for (size_t k = 0; k < widx.size (); k++)
+ {
+ const Word &w = words[widx[k]];
+
+ bases.push_back (w.y1);
+ hs.push_back (w.y1 - w.y0);
+ g.x0 = std::min (g.x0, w.x0);
+ g.x1 = std::max (g.x1, w.x1);
+ }
+ g.y = median (bases);
+ g.size = median (hs);
+ for (size_t k = 0; k < widx.size (); k++)
+ {
+ const Word &w = words[widx[k]];
+
+ if (k > 0)
+ {
+ const Word &p = words[widx[k - 1]];
+
+ if ( (p.space_after) ||
+ (w.x0 - p.x1 > 0.15 * g.size) )
+ text += ' ';
+ }
+ append_expanded (text, w.text);
+ }
+ compose_accents (text);
+ if (text.empty ())
+ return 0;
+ if (total_bytes + text.size () >= EXTRACTOR_FULLTEXT_MAX)
+ return -1;
+ y = (g.y - page_top) / page_h;
+ if (! (y >= 0))
+ y = 0;
+ if (y > 1)
+ y = 1;
+ if (0 != EXTRACTOR_fulltext_page_add_ (page,
+ text.data (),
+ text.size (),
+ y,
+ 0))
+ return -1;
+ geom.push_back (g);
+ total_bytes += text.size ();
+ return 0;
+}
+
+
+/**
+ * Collect the lines of page @a pg in reading order.
+ *
+ * @param pg the page
+ * @param page page to fill
+ * @param geom geometry of the lines to fill
+ * @param total_bytes running total of text bytes, updated
+ * @return 0 on success, -1 to stop
+ */
+static int
+collect_page (poppler::page *pg,
+ struct EXTRACTOR_FullTextPage *page,
+ std::vector<struct EXTRACTOR_PageLineGeom> &geom,
+ size_t &total_bytes)
+{
+ poppler::rectf rect = pg->page_rect ();
+ double top = rect.top ();
+ double ph = rect.height ();
+ std::vector<poppler::text_box> boxes;
+ std::vector<Word> words;
+ std::vector<Line> lines;
+
+ if ( (! (ph > 0)) ||
+ (! std::isfinite (ph)) ||
+ (! std::isfinite (top)) )
+ return 0;
+ boxes = pg->text_list ();
+ for (const poppler::text_box &b : boxes)
+ {
+ poppler::rectf r = b.bbox ();
+ std::vector<char> u = b.text ().to_utf8 ();
+ Word w;
+
+ if (words.size () >= PDF_FT_MAX_WORDS)
+ break;
+ if ( (0 != b.rotation ()) ||
+ (u.empty ()) )
+ continue;
+ w.x0 = r.left ();
+ w.x1 = r.right ();
+ w.y0 = r.top ();
+ w.y1 = r.bottom ();
+ if ( (! std::isfinite (w.x0)) ||
+ (! std::isfinite (w.x1)) ||
+ (! std::isfinite (w.y0)) ||
+ (! std::isfinite (w.y1)) ||
+ (! (w.y1 > w.y0)) ||
+ (w.x1 < w.x0) ||
+ (w.y1 - w.y0 > ph) )
+ continue;
+ w.text.assign (u.data (), u.size ());
+ w.space_after = b.has_space_after ();
+ w.numeric = (w.text.size () <= 4);
+ for (char c : w.text)
+ if (! isdigit ((unsigned char) c))
+ w.numeric = false;
+ words.push_back (std::move (w));
+ }
+ boxes.clear ();
+ if (words.empty ())
+ return 0;
+ drop_marginal_numbers (words);
+ group_lines (words, lines);
+ if (lines.empty ())
+ return 0;
+
+ /* page furniture: lines in the top/bottom band set apart from
+ the rest by a clear vertical gap */
+ std::vector<bool> furniture (lines.size (), false);
+ {
+ std::vector<double> gaps;
+ double spacing;
+
+ for (size_t i = 1; i < lines.size (); i++)
+ {
+ double d = lines[i].base - lines[i - 1].base;
+
+ if ( (d > 0.3 * lines[i].size) &&
+ (d < 3 * lines[i].size) )
+ gaps.push_back (d);
+ }
+ spacing = gaps.empty () ? 0 : median (gaps);
+ for (size_t i = 0; i + 1 < lines.size (); i++)
+ {
+ if ( (lines[i].base - top) / ph > PDF_FT_BAND)
+ break;
+ if (lines[i + 1].base - lines[i].base <= 1.3 * spacing)
+ break;
+ furniture[i] = true;
+ }
+ for (size_t i = lines.size () - 1; i > 0; i--)
+ {
+ if ( (lines[i].base - top) / ph < 1.0 - PDF_FT_BAND)
+ break;
+ if (lines[i].base - lines[i - 1].base <= 1.3 * spacing)
+ break;
+ furniture[i] = true;
+ }
+ }
+
+ double g0 = 0;
+ double g1 = 0;
+ bool cols = find_gutter (words, lines, furniture, g0, g1);
+ std::vector<std::vector<size_t> > left_col;
+ std::vector<std::vector<size_t> > right_col;
+ std::vector<std::pair<double, double> > iv;
+
+ for (size_t i = 0; i <= lines.size (); i++)
+ {
+ bool spanning = true;
+
+ if ( (i < lines.size ()) &&
+ cols &&
+ (! furniture[i]) )
+ {
+ line_coverage (words, lines[i], iv);
+ spanning = false;
+ for (const auto &p : iv)
+ if ( (p.first < g1) &&
+ (p.second > g0) )
+ spanning = true;
+ }
+ if (spanning)
+ {
+ /* flush the column region above */
+ for (const auto &l : left_col)
+ if (0 != emit_line (words, l, top, ph, page, geom, total_bytes))
+ return -1;
+ for (const auto &l : right_col)
+ if (0 != emit_line (words, l, top, ph, page, geom, total_bytes))
+ return -1;
+ left_col.clear ();
+ right_col.clear ();
+ if (i == lines.size ())
+ break;
+ if (0 != emit_line (words, lines[i].words, top, ph, page, geom,
+ total_bytes))
+ return -1;
+ continue;
+ }
+ std::vector<size_t> l;
+ std::vector<size_t> r;
+
+ for (size_t w : lines[i].words)
+ {
+ if (words[w].x1 <= g1)
+ l.push_back (w);
+ else
+ r.push_back (w);
+ }
+ if (! l.empty ())
+ left_col.push_back (std::move (l));
+ if (! r.empty ())
+ right_col.push_back (std::move (r));
+ }
+ return 0;
+}
+
+
+/**
+ * Extract the full text of @a doc and emit it.
+ *
+ * @param ec extraction context
+ * @param doc the document
+ * @return 0 to continue, 1 if the callback asked us to stop
+ */
+static int
+pdf_fulltext (struct EXTRACTOR_ExtractContext *ec,
+ poppler::document *doc)
+{
+ std::vector<struct EXTRACTOR_FullTextPage> pages;
+ std::vector<std::vector<struct EXTRACTOR_PageLineGeom> > geoms;
+ std::vector<struct EXTRACTOR_PageLineGeom *> gptrs;
+ size_t total_bytes = 0;
+ size_t total_lines = 0;
+ double deadline = now_s () + PDF_FT_DEADLINE;
+ int npages = doc->pages ();
+ int ret = 0;
+
+ if (npages > PDF_FT_MAX_PAGES)
+ npages = PDF_FT_MAX_PAGES;
+ try
+ {
+ pages.reserve (npages);
+ geoms.reserve (npages);
+ for (int i = 0; i < npages; i++)
+ {
+ poppler::page *pg;
+ struct EXTRACTOR_FullTextPage fp;
+ int r;
+
+ if (now_s () > deadline)
+ break;
+ /* keep the host from presuming us hung */
+ ec->seek (ec->cls, 0, SEEK_CUR);
+ pg = doc->create_page (i);
+ if (NULL == pg)
+ continue;
+ memset (&fp, 0, sizeof (fp));
+ pages.push_back (fp);
+ geoms.emplace_back ();
+ r = collect_page (pg,
+ &pages.back (),
+ geoms.back (),
+ total_bytes);
+ delete pg;
+ total_lines += pages.back ().num_lines;
+ if ( (0 != r) ||
+ (total_lines > PDF_FT_MAX_LINES) )
+ break;
+ }
+ }
+ catch (...)
+ {
+ /* out of memory: go with what we have */
+ }
+ /* we may have failed half-way; keep geometry and lines in sync */
+ while (pages.size () > geoms.size ())
+ {
+ EXTRACTOR_fulltext_page_clear_ (&pages.back ());
+ pages.pop_back ();
+ }
+ for (size_t i = 0; i < pages.size (); i++)
+ while (pages[i].num_lines > geoms[i].size ())
+ {
+ pages[i].num_lines--;
+ free (pages[i].lines[pages[i].num_lines].text);
+ }
+ if (! pages.empty ())
+ {
+ struct EXTRACTOR_FullText ft;
+
+ for (size_t i = 0; i < geoms.size (); i++)
+ gptrs.push_back (geoms[i].data ());
+ EXTRACTOR_fulltext_filter_margins_ (pages.data (),
+ (unsigned int) pages.size ());
+ EXTRACTOR_pagelayout_paragraphs_ (pages.data (),
+ gptrs.data (),
+ (unsigned int) pages.size (),
+ 0.02);
+ EXTRACTOR_fulltext_init_ (&ft);
+ EXTRACTOR_fulltext_append_pages_ (&ft,
+ pages.data (),
+ (unsigned int) pages.size ());
+ ret = EXTRACTOR_fulltext_emit_ (&ft,
+ ec,
+ "pdf");
+ EXTRACTOR_fulltext_clear_ (&ft);
+ }
+ for (size_t i = 0; i < pages.size (); i++)
+ EXTRACTOR_fulltext_page_clear_ (&pages[i]);
+ return (0 != ret) ? 1 : 0;
+}
+
+
/**
* Main entry method for the PDF extraction plugin.
*
@@ -288,6 +1148,8 @@ EXTRACTOR_pdf_extract_method (struct EXTRACTOR_ExtractContext *ec)
EXTRACTOR_METATYPE_MODIFICATION_DATE,
doc->get_modification_date_t ()))
goto CLEANUP;
+ if (EXTRACTOR_fulltext_requested_ (ec->config))
+ (void) pdf_fulltext (ec, doc);
CLEANUP:
delete doc;
}
diff --git a/src/plugins/test_dvi.c b/src/plugins/test_dvi.c
@@ -24,6 +24,88 @@
*/
#include "platform.h"
#include "test_lib.h"
+#include "test_paged_fulltext.h"
+
+
+/**
+ * Check the full text of testdata/dvi_fulltext.dvi.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+validate_dvi_fulltext (const void *data,
+ size_t data_len)
+{
+ static const char *const pars[] = {
+ "Introduction",
+ "Paged documents scatter their words",
+ "The second paragraph is indented",
+ "Short paragraph number three.",
+ "The fourth paragraph",
+ "The fifth paragraph runs across the page break",
+ "The sixth paragraph",
+ "The seventh paragraph is set in the T1 encoding",
+ NULL
+ };
+ static const char *const needles[] = {
+ /* OT1 accents: \'e and \"\i */
+ "The caf\xc3\xa9 serves na\xc3\xafve guests.",
+ /* T1 glyphs */
+ "The seventh paragraph is set in the T1 encoding: Stra\xc3\x9f" "e, "
+ "\xc3\x9c" "berma\xc3\x9f, \xc3\xa9tude and \xc3\xa7" "a va.",
+ NULL
+ };
+ static const char *const absent[] = {
+ NULL
+ };
+ const char *s = data;
+
+ if (0 == check_form (s, data_len))
+ {
+ fprintf (stderr, "Malformed full text\n");
+ return 0;
+ }
+ return check_paragraphs (s, pars) &&
+ check_contains (s, common_needles, common_absent) &&
+ check_contains (s, needles, absent) &&
+ check_no_numbers (s);
+}
+
+
+/**
+ * Check the full text of testdata/dvi_ora.dvi.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+validate_ora_fulltext (const void *data,
+ size_t data_len)
+{
+ static const char *const needles[] = {
+ "Optimal Bitwise Register Allocation using Integer Linear Programming"
+ "\n\n",
+ "\n\nAbstract\n\nThis paper addresses the problem of optimal global "
+ "register allocation.",
+ "\n\nThe traditional approach towards optimal register allocation is "
+ "to color the interference graph [7].",
+ NULL
+ };
+ static const char *const absent[] = {
+ NULL
+ };
+ const char *s = data;
+
+ if (0 == check_form (s, data_len))
+ {
+ fprintf (stderr, "Malformed full text\n");
+ return 0;
+ }
+ return check_contains (s, needles, absent);
+}
/**
@@ -117,12 +199,75 @@ main (int argc, char *argv[])
},
{ 0, 0, NULL, NULL, 0, -1 }
};
+ struct SolutionData dvi_fulltext_sol[] = {
+ {
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/x-dvi",
+ strlen ("application/x-dvi") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_PAGE_COUNT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "3",
+ strlen ("3") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^Introduction\n\nPaged documents scatter",
+ 0,
+ 0,
+ 1,
+ &validate_dvi_fulltext
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct SolutionData dvi_ora_ft_sol[] = {
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^Optimal Bitwise Register Allocation",
+ 0,
+ 0,
+ 1,
+ &validate_ora_fulltext
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
struct ProblemSet ps[] = {
{ "testdata/dvi_ora.dvi",
dvi_ora_sol },
{ NULL, NULL }
};
- return ET_main ("dvi", ps);
+ struct ProblemSet ps_ft[] = {
+ { "testdata/dvi_fulltext.dvi",
+ dvi_fulltext_sol },
+ { "testdata/dvi_ora.dvi",
+ dvi_ora_ft_sol },
+ { NULL, NULL }
+ };
+ int ret = 0;
+
+ ret |= ET_main ("dvi", ps);
+ /* without the option, there must be no full text */
+ ret |= check_fulltext_count ("dvi",
+ "testdata/dvi_fulltext.dvi",
+ 0);
+ ret |= check_fulltext_count ("dvi",
+ "testdata/dvi_ora.dvi",
+ 0);
+ ret |= ET_main ("dvi(fulltext)", ps_ft);
+ ret |= check_fulltext_count ("dvi(fulltext)",
+ "testdata/dvi_fulltext.dvi",
+ 1);
+ return ret;
}
diff --git a/src/plugins/test_ebook.c b/src/plugins/test_ebook.c
@@ -35,6 +35,196 @@
/**
+ * Check the shape of a full text: 0-terminated, no leading or trailing
+ * white space, paragraphs separated by exactly "\n\n", no other line
+ * breaks, no runs of spaces, no tabs.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data, including the 0
+ * @return 1 if the shape is right
+ */
+static int
+fulltext_shape_ok (const char *data,
+ size_t data_len)
+{
+ size_t len;
+
+ if ( (data_len < 2) ||
+ ('\0' != data[data_len - 1]) ||
+ (strlen (data) != data_len - 1) )
+ return 0;
+ len = data_len - 1;
+ if ( (' ' == data[0]) || ('\n' == data[0]) ||
+ (' ' == data[len - 1]) || ('\n' == data[len - 1]) )
+ return 0;
+ for (size_t i = 0; i < len; i++)
+ {
+ if ('\t' == data[i])
+ return 0;
+ if ( (' ' == data[i]) &&
+ ( (' ' == data[i + 1]) || ('\n' == data[i + 1]) ) )
+ return 0;
+ if ('\n' == data[i])
+ {
+ if ( ('\n' != data[i + 1]) ||
+ ('\n' == data[i + 2]) ||
+ (' ' == data[i + 2]) )
+ return 0;
+ i++;
+ }
+ }
+ return 1;
+}
+
+
+/**
+ * Closure for #count_cb().
+ */
+struct Count
+{
+ /**
+ * Number of full text items seen.
+ */
+ unsigned int fulltext;
+
+ /**
+ * Number of full text items that are not normalized as promised.
+ */
+ unsigned int bad;
+};
+
+
+/**
+ * Count full text items.
+ *
+ * @param cls a `struct Count`
+ * @param plugin_name ignored
+ * @param type meta data type
+ * @param format ignored
+ * @param data_mime_type ignored
+ * @param data meta data
+ * @param data_len number of bytes in @a data
+ * @return 0 (continue)
+ */
+static int
+count_cb (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct Count *c = cls;
+
+ if (EXTRACTOR_METATYPE_FULL_TEXT != type)
+ return 0;
+ c->fulltext++;
+ if (! fulltext_shape_ok (data, data_len))
+ c->bad++;
+ return 0;
+}
+
+
+/**
+ * Count the full text items the plugin reports for @a file.
+ *
+ * @param config plugin name with options, e.g. "ebook(fulltext)"
+ * @param file file to extract from
+ * @return number of full text items, UINT_MAX if one of them is
+ * malformed
+ */
+static unsigned int
+count_fulltext (const char *config,
+ const char *file)
+{
+ struct EXTRACTOR_PluginList *pl;
+ struct Count c = { 0, 0 };
+
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ config,
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ EXTRACTOR_extract (pl, file, NULL, 0, &count_cb, &c);
+ EXTRACTOR_plugin_remove_all (pl);
+ if (0 != c.bad)
+ {
+ fprintf (stderr,
+ "Full text of %s is not normalized\n",
+ file);
+ return UINT_MAX;
+ }
+ return c.fulltext;
+}
+
+
+#define EPUB_ITEMS \
+ TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/epub+zip"), \
+ /* <package version="3.0"> */ \
+ TXT (EXTRACTOR_METATYPE_FORMAT_VERSION, "3.0"), \
+ TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"), \
+ TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"), \
+ /* dc:contributor with opf:role="bkp" is the tool, not a person */ \
+ TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, \
+ "calibre (7.2.0) [https://calibre-ebook.com]"), \
+ TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"), \
+ TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-GB"), \
+ TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01T00:00:00+00:00"), \
+ TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"), \
+ TXT (EXTRACTOR_METATYPE_KEYWORDS, "metadata extraction"), \
+ /* the description is indented across three lines in the OPF, so \
+ this also proves the white space collapsing */\
+ TXT (EXTRACTOR_METATYPE_DESCRIPTION, \
+ "A minimal EPUB built by gen_ebook_testdata.sh, used to check" \
+ " that the ebook plugin reads Dublin Core out of the OPF" \
+ " package document."), \
+ /* "&" has to come back as a bare ampersand */ \
+ TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0) & nothing else"), \
+ /* dc:identifier with opf:scheme="ISBN" */ \
+ TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"), \
+ /* ... and the one that is not an ISBN */ \
+ TXT (EXTRACTOR_METATYPE_URI, \
+ "urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719"), \
+ TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00+00:00"), \
+ TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, "gen_ebook_testdata.sh"), \
+ TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"), \
+ /* seven <item> entries in the manifest, and not the three \
+ <itemref> entries in the spine */\
+ TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "7"), \
+ TXT (EXTRACTOR_METATYPE_ENCRYPTION, \
+ "META-INF/encryption.xml present (DRM or font obfuscation)")
+
+#define MOBI_ITEMS \
+ TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/x-mobipocket-ebook"), \
+ /* PDB header offsets 36/40, seconds since 1904 */ \
+ TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00Z"), \
+ TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"), \
+ /* ... and the same moment again, from EXTH 502 */ \
+ TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00+00:00"), \
+ TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "3"), \
+ /* PalmDOC header offset 12 */ \
+ TXT (EXTRACTOR_METATYPE_ENCRYPTION, "old Mobipocket encryption"), \
+ /* MOBI header offset 28 */ \
+ TXT (EXTRACTOR_METATYPE_CHARACTER_SET, "UTF-8"), \
+ TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"), \
+ TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"), \
+ TXT (EXTRACTOR_METATYPE_DESCRIPTION, \
+ "A hand-built minimal MOBI file used by the libextractor" \
+ " test suite."), \
+ TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"), \
+ TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"), \
+ TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01"), \
+ TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, \
+ "calibre (7.2.0) [https://calibre-ebook.com]"), \
+ TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0)"), \
+ TXT (EXTRACTOR_METATYPE_SERIAL, "B00LIBEXTR7"), \
+ TXT (EXTRACTOR_METATYPE_WATERMARK, "libextractor-test-watermark"), \
+ TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "cdetype: EBOK"), \
+ /* EXTH 503, which is why the PDB database name \
+ "The_libextractor_Test_Book" is not reported as the title */\
+ TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"), \
+ TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-gb")
+
+/**
* Main function for the ebook testcase.
*
* @param argc number of arguments (ignored)
@@ -45,81 +235,128 @@ int
main (int argc, char *argv[])
{
struct SolutionData epub_sol[] = {
- TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/epub+zip"),
- /* <package version="3.0"> */
- TXT (EXTRACTOR_METATYPE_FORMAT_VERSION, "3.0"),
- TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"),
- TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"),
- /* dc:contributor with opf:role="bkp" is the tool, not a person */
- TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
- "calibre (7.2.0) [https://calibre-ebook.com]"),
- TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"),
- TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-GB"),
- TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01T00:00:00+00:00"),
- TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"),
- TXT (EXTRACTOR_METATYPE_KEYWORDS, "metadata extraction"),
- /* the description is indented across three lines in the OPF, so
- this also proves the white space collapsing */
- TXT (EXTRACTOR_METATYPE_DESCRIPTION,
- "A minimal EPUB built by gen_ebook_testdata.sh, used to check"
- " that the ebook plugin reads Dublin Core out of the OPF"
- " package document."),
- /* "&" has to come back as a bare ampersand */
- TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0) & nothing else"),
- /* dc:identifier with opf:scheme="ISBN" */
- TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"),
- /* ... and the one that is not an ISBN */
- TXT (EXTRACTOR_METATYPE_URI,
- "urn:uuid:0f9d5a1e-1719-4c0b-9a3e-000000001719"),
- TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00+00:00"),
- TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE, "gen_ebook_testdata.sh"),
- TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"),
- /* seven <item> entries in the manifest, and not the three
- <itemref> entries in the spine */
- TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "7"),
- TXT (EXTRACTOR_METATYPE_ENCRYPTION,
- "META-INF/encryption.xml present (DRM or font obfuscation)"),
+ EPUB_ITEMS,
{ 0, 0, NULL, NULL, 0, -1 }
};
struct SolutionData mobi_sol[] = {
- TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/x-mobipocket-ebook"),
- /* PDB header offsets 36/40, seconds since 1904 */
- TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00Z"),
- TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"),
- /* ... and the same moment again, from EXTH 502 */
- TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00+00:00"),
- TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "3"),
- /* PalmDOC header offset 12 */
- TXT (EXTRACTOR_METATYPE_ENCRYPTION, "old Mobipocket encryption"),
- /* MOBI header offset 28 */
- TXT (EXTRACTOR_METATYPE_CHARACTER_SET, "UTF-8"),
- TXT (EXTRACTOR_METATYPE_AUTHOR_NAME, "Christian Grothoff"),
- TXT (EXTRACTOR_METATYPE_PUBLISHER, "GNU Press"),
- TXT (EXTRACTOR_METATYPE_DESCRIPTION,
- "A hand-built minimal MOBI file used by the libextractor"
- " test suite."),
- TXT (EXTRACTOR_METATYPE_ISBN, "978-3-16-148410-0"),
- TXT (EXTRACTOR_METATYPE_KEYWORDS, "digital forensics"),
- TXT (EXTRACTOR_METATYPE_PUBLICATION_DATE, "2026-01-01"),
- TXT (EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
- "calibre (7.2.0) [https://calibre-ebook.com]"),
- TXT (EXTRACTOR_METATYPE_RIGHTS, "Public domain (CC0)"),
- TXT (EXTRACTOR_METATYPE_SERIAL, "B00LIBEXTR7"),
- TXT (EXTRACTOR_METATYPE_WATERMARK, "libextractor-test-watermark"),
- TXT (EXTRACTOR_METATYPE_ATTRIBUTES, "cdetype: EBOK"),
- /* EXTH 503, which is why the PDB database name
- "The_libextractor_Test_Book" is not reported as the title */
- TXT (EXTRACTOR_METATYPE_TITLE, "The libextractor Test Book"),
- TXT (EXTRACTOR_METATYPE_LANGUAGE, "en-gb"),
+ MOBI_ITEMS,
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+#define EPUB_FT_ITEMS \
+ TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/epub+zip"), \
+ TXT (EXTRACTOR_METATYPE_FORMAT_VERSION, "3.0"), \
+ TXT (EXTRACTOR_METATYPE_TITLE, "Full Text Test"), \
+ TXT (EXTRACTOR_METATYPE_URI, \
+ "urn:uuid:0f9d5a1e-1719-4c0b-9a3e-00000000f7f7"), \
+ TXT (EXTRACTOR_METATYPE_LANGUAGE, "en"), \
+ TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "7"), \
+ TXT (EXTRACTOR_METATYPE_ENCRYPTION, \
+ "META-INF/encryption.xml present (DRM or font obfuscation)")
+#define MOBI_FT_ITEMS \
+ TXT (EXTRACTOR_METATYPE_MIMETYPE, "application/x-mobipocket-ebook"), \
+ TXT (EXTRACTOR_METATYPE_CREATION_DATE, "2026-01-01T00:00:00Z"), \
+ TXT (EXTRACTOR_METATYPE_MODIFICATION_DATE, "2026-02-01T00:00:00Z"), \
+ TXT (EXTRACTOR_METATYPE_ENTRY_COUNT, "8"), \
+ TXT (EXTRACTOR_METATYPE_CHARACTER_SET, "UTF-8"), \
+ TXT (EXTRACTOR_METATYPE_TITLE, "Full Text Test")
+ struct SolutionData epub_ft_sol[] = {
+ EPUB_FT_ITEMS,
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct SolutionData mobi_ft_sol[] = {
+ MOBI_FT_ITEMS,
{ 0, 0, NULL, NULL, 0, -1 }
};
struct ProblemSet ps[] = {
{ "testdata/ebook_test.epub", epub_sol },
{ "testdata/ebook_test.mobi", mobi_sol },
+ { "testdata/ebook_fulltext.epub", epub_ft_sol },
+ { "testdata/ebook_fulltext.mobi", mobi_ft_sol },
{ NULL, NULL }
};
+ /* The spine order is nav, c3, c1, (secret,) c2, (notes): the
+ navigation document contributes nothing, the DRM-protected
+ `secret' and the linear="no" `notes' are skipped; c1 is
+ ISO-8859-1; page-level header/footer, the footnote aside, the
+ page break marker, hidden elements, head, style and script are
+ dropped. */
+#define EPUB_TEXT \
+ "Chapter Alpha\n\n" \
+ "Alpha text comes first in the reading order.\n\n" \
+ "Second paragraph of alpha\xe2\x80\x94with a dash & an ampersand.\n\n" \
+ "Chapter Beta\n\n" \
+ "Beta: caf\xc3\xa9 cr\xc3\xa8me.*\n\n" \
+ "Chapter Gamma\n\n" \
+ "Gamma line one line two\n\n" \
+ "Gamma new paragraph here.\n\n" \
+ "Last\xe2\x80\xa6 really last."
+ /* PalmDOC compressed, with trailing entries and a character that
+ straddles two records; <mbp:pagebreak/> ends a paragraph */
+#define MOBI_TEXT \
+ "The Test Book\n\n" \
+ "First paragraph with an \xc3\xbcmlaut, repeated text, repeated text," \
+ " repeated text.\n\n" \
+ "Second chapter starts here \xe2\x80\x94 \xc3\xa0 la carte.\n\n" \
+ "Line break and & entity."
+ /* the chapters of ebook_test.epub in spine order */
+#define EPUB_SIMPLE_TEXT \
+ "One\n\nNothing to see here.\n\n" \
+ "Two\n\nNothing to see here.\n\n" \
+ "Three\n\nNothing to see here."
+ /* with the option: the same items, plus the full text */
+ struct SolutionData epub_sol_ft[] = {
+ EPUB_ITEMS,
+ TXT (EXTRACTOR_METATYPE_FULL_TEXT, EPUB_SIMPLE_TEXT),
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct SolutionData epub_ft_sol_ft[] = {
+ EPUB_FT_ITEMS,
+ TXT (EXTRACTOR_METATYPE_FULL_TEXT, EPUB_TEXT),
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct SolutionData mobi_ft_sol_ft[] = {
+ MOBI_FT_ITEMS,
+ TXT (EXTRACTOR_METATYPE_FULL_TEXT, MOBI_TEXT),
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct ProblemSet ps_ft[] = {
+ { "testdata/ebook_test.epub", epub_sol_ft },
+ /* ebook_test.mobi is flagged as encrypted: no full text */
+ { "testdata/ebook_test.mobi", mobi_sol },
+ { "testdata/ebook_fulltext.epub", epub_ft_sol_ft },
+ { "testdata/ebook_fulltext.mobi", mobi_ft_sol_ft },
+ { NULL, NULL }
+ };
+ static const char *const files[] = {
+ "testdata/ebook_test.epub",
+ "testdata/ebook_test.mobi",
+ "testdata/ebook_fulltext.epub",
+ "testdata/ebook_fulltext.mobi",
+ NULL
+ };
+ int ret;
+
+ if (0 != (ret = ET_main ("ebook", ps)))
+ return ret;
+ if (0 != (ret = ET_main ("ebook(fulltext)", ps_ft)))
+ return ret;
+ /* ET_main only warns about extra items: make sure that the full text
+ appears exactly when it was asked for (and not for the encrypted
+ MOBI) */
+ for (unsigned int i = 0; NULL != files[i]; i++)
+ {
+ unsigned int want = (1 == i) ? 0 : 1;
- return ET_main ("ebook", ps);
+ if ( (0 != count_fulltext ("ebook", files[i])) ||
+ (want != count_fulltext ("ebook(fulltext)", files[i])) )
+ {
+ fprintf (stderr,
+ "Wrong number of full text items for %s\n",
+ files[i]);
+ return 1;
+ }
+ }
+ return 0;
}
diff --git a/src/plugins/test_gstreamer.c b/src/plugins/test_gstreamer.c
@@ -27,6 +27,7 @@
#include <glib.h>
#include <gst/gst.h>
#include <gst/pbutils/pbutils.h>
+#include <sys/wait.h>
/**
@@ -110,6 +111,79 @@ discoverer_main (GstDiscoverer *dc, const char *filename)
/**
+ * Check that the language of every subtitle track of
+ * testdata/subtitles.mkv is reported.
+ *
+ * Must run in a process in which GStreamer has not been initialized:
+ * ET_main() forks the extraction process, and a GStreamer initialized
+ * before a fork() does not work in the child.
+ *
+ * @return 0 on success (or if GStreamer lacks the plugins to read the
+ * file), 1 on failure
+ */
+static int
+check_subtitle_languages (void)
+{
+ /* GStreamer normalizes the ISO 639-2 codes of the Matroska tracks
+ ("eng", "ger", "fre") to ISO 639-1; accept either spelling */
+ struct SolutionData subtitles_sol[] = {
+ {
+ EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^(en|eng)$",
+ 0,
+ 0,
+ 1
+ },
+ {
+ EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^(de|ger|deu)$",
+ 0,
+ 0,
+ 1
+ },
+ {
+ EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^(fr|fre|fra)$",
+ 0,
+ 0,
+ 1
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct ProblemSet ps[] = {
+ { "testdata/subtitles.mkv", subtitles_sol },
+ { NULL, NULL }
+ };
+ int ret;
+ GstDiscoverer *dc;
+
+ g_print ("Running subtitle language test on GStreamer:\n");
+ ret = ET_main ("gstreamer", ps);
+ if (0 == ret)
+ return 0;
+ /* only now may we initialize GStreamer ourselves: find out whether
+ the failure is just a missing GStreamer plugin */
+ gst_init (NULL, NULL);
+ dc = gst_discoverer_new (10 * GST_SECOND, NULL);
+ if ( (NULL != dc) &&
+ (GST_DISCOVERER_MISSING_PLUGINS ==
+ discoverer_main (dc, "testdata/subtitles.mkv")) )
+ ret = 0;
+ if (NULL != dc)
+ g_object_unref (dc);
+ if (0 != ret)
+ g_print ("Subtitle language test FAILED\n");
+ return ret;
+}
+
+
+/**
* Main function for the GStreamer testcase.
*
* @param argc number of arguments (ignored)
@@ -122,8 +196,22 @@ main (int argc, char *argv[])
GError *err = NULL;
GstDiscoverer *dc;
int result = 0;
+ int subtitles_failed = 0;
GstDiscovererResult pre_test;
+ {
+ pid_t pid;
+ int status;
+
+ pid = fork ();
+ if (0 == pid)
+ _exit (check_subtitle_languages ());
+ if ( (-1 == pid) ||
+ (pid != waitpid (pid, &status, 0)) ||
+ (! WIFEXITED (status)) ||
+ (0 != WEXITSTATUS (status)) )
+ subtitles_failed = 1;
+ }
gst_init (&argc, &argv);
dc = gst_discoverer_new (10 * GST_SECOND, &err);
if (NULL == dc)
@@ -1659,7 +1747,7 @@ main (int argc, char *argv[])
/* do not fail hard, as we know many users have outdated gstreamer packages */
result = 0;
}
- return result;
+ return subtitles_failed;
}
diff --git a/src/plugins/test_html.c b/src/plugins/test_html.c
@@ -27,6 +27,156 @@
/**
+ * Check the shape of a full text: 0-terminated, no leading or trailing
+ * white space, paragraphs separated by exactly "\n\n", no other line
+ * breaks, no runs of spaces, no tabs.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data, including the 0
+ * @return 1 if the shape is right
+ */
+static int
+fulltext_shape_ok (const char *data,
+ size_t data_len)
+{
+ size_t len;
+
+ if ( (data_len < 2) ||
+ ('\0' != data[data_len - 1]) ||
+ (strlen (data) != data_len - 1) )
+ return 0;
+ len = data_len - 1;
+ if ( (' ' == data[0]) || ('\n' == data[0]) ||
+ (' ' == data[len - 1]) || ('\n' == data[len - 1]) )
+ return 0;
+ for (size_t i = 0; i < len; i++)
+ {
+ if ('\t' == data[i])
+ return 0;
+ if ( (' ' == data[i]) &&
+ ( (' ' == data[i + 1]) || ('\n' == data[i + 1]) ) )
+ return 0;
+ if ('\n' == data[i])
+ {
+ if ( ('\n' != data[i + 1]) ||
+ ('\n' == data[i + 2]) ||
+ (' ' == data[i + 2]) )
+ return 0;
+ i++;
+ }
+ }
+ return 1;
+}
+
+
+/**
+ * Validate the full text of html_grothoff.html.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+check_grothoff_text (const void *data,
+ size_t data_len)
+{
+ const char *s = data;
+
+ if (! fulltext_shape_ok (s, data_len))
+ return 0;
+ /* the heading, then the list items, each its own paragraph */
+ if (NULL == strstr (s, "Welcome to Christian Grothoff\n\nWelcome\n\n"
+ "Computer Science\n\nLinux\n\n"))
+ return 0;
+ /* nothing from the head or its script */
+ if ( (NULL != strstr (s, "top.location")) ||
+ (NULL != strstr (s, "Homepage of")) )
+ return 0;
+ return 1;
+}
+
+
+/**
+ * Closure for #count_cb().
+ */
+struct Count
+{
+ /**
+ * Number of full text items seen.
+ */
+ unsigned int fulltext;
+
+ /**
+ * Number of full text items that are not normalized as promised.
+ */
+ unsigned int bad;
+};
+
+
+/**
+ * Count full text items.
+ *
+ * @param cls a `struct Count`
+ * @param plugin_name ignored
+ * @param type meta data type
+ * @param format ignored
+ * @param data_mime_type ignored
+ * @param data meta data
+ * @param data_len number of bytes in @a data
+ * @return 0 (continue)
+ */
+static int
+count_cb (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct Count *c = cls;
+
+ if (EXTRACTOR_METATYPE_FULL_TEXT != type)
+ return 0;
+ c->fulltext++;
+ if (! fulltext_shape_ok (data, data_len))
+ c->bad++;
+ return 0;
+}
+
+
+/**
+ * Count the full text items the plugin reports for @a file.
+ *
+ * @param config plugin name with options, e.g. "html(fulltext)"
+ * @param file file to extract from
+ * @return number of full text items, UINT_MAX if one of them is
+ * malformed
+ */
+static unsigned int
+count_fulltext (const char *config,
+ const char *file)
+{
+ struct EXTRACTOR_PluginList *pl;
+ struct Count c = { 0, 0 };
+
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ config,
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ EXTRACTOR_extract (pl, file, NULL, 0, &count_cb, &c);
+ EXTRACTOR_plugin_remove_all (pl);
+ if (0 != c.bad)
+ {
+ fprintf (stderr,
+ "Full text of %s is not normalized\n",
+ file);
+ return UINT_MAX;
+ }
+ return c.fulltext;
+}
+
+
+/**
* Main function for the HTML testcase.
*
* @param argc number of arguments (ignored)
@@ -111,12 +261,102 @@ main (int argc, char *argv[])
},
{ 0, 0, NULL, NULL, 0, -1 }
};
+ struct SolutionData html_fulltext_sol[] = {
+ {
+ EXTRACTOR_METATYPE_AUTHOR_NAME,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "Christian Grothoff",
+ strlen ("Christian Grothoff") + 1,
+ 0
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
struct ProblemSet ps[] = {
{ "testdata/html_grothoff.html",
html_grothoff_sol },
+ { "testdata/html_fulltext.html",
+ html_fulltext_sol },
+ { NULL, NULL }
+ };
+ /* with the option: the same items, plus the full text */
+#define FT_TEXT \
+ "Der K\xc3\xa4se & das Brot\n\n" \
+ "Hello world, again here: \xc3\xa9t\xc3\xa9 \xe2\x80\x94 done.\n\n" \
+ "Line one line two\n\n" \
+ "New paragraph after two breaks.\n\n" \
+ "Item A\n\nItem B\n\n" \
+ "Cell 1\n\nCell 2\n\n" \
+ "Quoted text\n\n" \
+ "Softhyphen and inline elements.\n\n" \
+ "Article byline"
+ struct SolutionData html_grothoff_ft_sol[] = {
+ { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "Christian Grothoff", strlen ("Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_DESCRIPTION, EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain", "Homepage of Christian Grothoff",
+ strlen ("Homepage of Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_AUTHOR_NAME, EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain", "Christian Grothoff",
+ strlen ("Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_KEYWORDS, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "Christian,Grothoff", strlen ("Christian,Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "Welcome to Christian Grothoff",
+ strlen ("Welcome to Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_LANGUAGE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "en", strlen ("en") + 1, 0 },
+ { EXTRACTOR_METATYPE_PUBLISHER, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "Christian Grothoff", strlen ("Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_UNKNOWN_DATE, EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain", "2000-08-20", strlen ("2000-08-20") + 1, 0 },
+ { EXTRACTOR_METATYPE_RIGHTS, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "(C) 2000 by Christian Grothoff",
+ strlen ("(C) 2000 by Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_FULL_TEXT, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "^Welcome to Christian Grothoff\n", 0, 0, 1,
+ &check_grothoff_text },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct SolutionData html_fulltext_ft_sol[] = {
+ /* in the option's code path the declared charset is honoured, so
+ the title comes out right */
+ { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ "K\xc3\xa4se und Brot", strlen ("K\xc3\xa4se und Brot") + 1, 0 },
+ { EXTRACTOR_METATYPE_AUTHOR_NAME, EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain", "Christian Grothoff",
+ strlen ("Christian Grothoff") + 1, 0 },
+ { EXTRACTOR_METATYPE_FULL_TEXT, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
+ FT_TEXT, strlen (FT_TEXT) + 1, 0 },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ struct ProblemSet ps_ft[] = {
+ { "testdata/html_grothoff.html",
+ html_grothoff_ft_sol },
+ { "testdata/html_fulltext.html",
+ html_fulltext_ft_sol },
{ NULL, NULL }
};
- return ET_main ("html", ps);
+ int ret;
+
+ if (0 != (ret = ET_main ("html", ps)))
+ return ret;
+ if (0 != (ret = ET_main ("html(fulltext)", ps_ft)))
+ return ret;
+ /* ET_main only warns about extra items: make sure that the full text
+ appears exactly when it was asked for */
+ if ( (0 != count_fulltext ("html", "testdata/html_fulltext.html")) ||
+ (0 != count_fulltext ("html", "testdata/html_grothoff.html")) ||
+ (1 != count_fulltext ("html(fulltext)",
+ "testdata/html_fulltext.html")) ||
+ (1 != count_fulltext ("html(fulltext)",
+ "testdata/html_grothoff.html")) )
+ {
+ fprintf (stderr,
+ "Full text reported without the option, or not with it\n");
+ return 1;
+ }
+ return 0;
}
diff --git a/src/plugins/test_media_lib.c b/src/plugins/test_media_lib.c
@@ -140,34 +140,37 @@ unfilter (uint8_t *raw,
#endif
-int
-TEST_media_png_center (const void *data,
+uint8_t *
+TEST_media_png_decode (const void *data,
size_t data_len,
- uint8_t rgb[3])
+ uint32_t *width,
+ uint32_t *height)
{
#if ! TEST_MEDIA_HAVE_PNG
(void) data;
(void) data_len;
- (void) rgb;
- return 0;
+ (void) width;
+ (void) height;
+ return NULL;
#else
static const uint8_t magic[] = { 137, 'P', 'N', 'G', '\r', '\n', 26, '\n' };
const uint8_t *d = data;
uint8_t *idat = NULL;
uint8_t *raw = NULL;
+ uint8_t *rgb = NULL;
size_t idat_len = 0;
size_t pos = sizeof (magic);
uLongf raw_len;
- uint32_t width = 0;
- uint32_t height = 0;
+ uint32_t w = 0;
+ uint32_t h = 0;
uint32_t bpp = 0;
uint32_t stride;
- const uint8_t *px;
- int ret = 0;
+ uint32_t x;
+ uint32_t y;
if ( (data_len < sizeof (magic) + 12) ||
(0 != memcmp (d, magic, sizeof (magic))) )
- return 0;
+ return NULL;
while (pos + 12 <= data_len)
{
uint32_t clen = get_u32 (&d[pos]);
@@ -181,8 +184,8 @@ TEST_media_png_center (const void *data,
{
if (clen < 13)
goto cleanup;
- width = get_u32 (&cdata[0]);
- height = get_u32 (&cdata[4]);
+ w = get_u32 (&cdata[0]);
+ h = get_u32 (&cdata[4]);
if ( (8 != cdata[8]) ||
(0 != cdata[12]) )
goto cleanup; /* not 8 bits per channel, or interlaced */
@@ -192,10 +195,10 @@ TEST_media_png_center (const void *data,
bpp = 4;
else
goto cleanup; /* palette or greyscale; our encoders emit neither */
- if ( (0 == width) ||
- (0 == height) ||
- (width > 4096) ||
- (height > 4096) )
+ if ( (0 == w) ||
+ (0 == h) ||
+ (w > 16384) ||
+ (h > 16384) )
goto cleanup;
}
else if (0 == memcmp (ctype, "IDAT", 4))
@@ -219,8 +222,8 @@ TEST_media_png_center (const void *data,
if ( (0 == bpp) ||
(NULL == idat) )
goto cleanup;
- stride = width * bpp;
- raw_len = (uLongf) height * (stride + 1);
+ stride = w * bpp;
+ raw_len = (uLongf) h * (stride + 1);
if (NULL == (raw = malloc (raw_len)))
goto cleanup;
if (Z_OK != uncompress (raw,
@@ -228,24 +231,50 @@ TEST_media_png_center (const void *data,
idat,
(uLong) idat_len))
goto cleanup;
- if (raw_len != (uLongf) height * (stride + 1))
+ if (raw_len != (uLongf) h * (stride + 1))
goto cleanup;
- if (! unfilter (raw, width, height, bpp))
+ if (! unfilter (raw, w, h, bpp))
goto cleanup;
- px = &raw[(height / 2) * (stride + 1) + 1 + (width / 2) * bpp];
- rgb[0] = px[0];
- rgb[1] = px[1];
- rgb[2] = px[2];
- ret = 1;
+ if (NULL == (rgb = malloc ((size_t) w * h * 3)))
+ goto cleanup;
+ for (y = 0; y < h; y++)
+ for (x = 0; x < w; x++)
+ memcpy (&rgb[((size_t) y * w + x) * 3],
+ &raw[(size_t) y * (stride + 1) + 1 + (size_t) x * bpp],
+ 3);
+ *width = w;
+ *height = h;
cleanup:
free (raw);
free (idat);
- return ret;
+ return rgb;
#endif
}
int
+TEST_media_png_center (const void *data,
+ size_t data_len,
+ uint8_t rgb[3])
+{
+ uint8_t *img;
+ uint32_t width;
+ uint32_t height;
+
+ if (NULL == (img = TEST_media_png_decode (data,
+ data_len,
+ &width,
+ &height)))
+ return 0;
+ memcpy (rgb,
+ &img[((size_t) (height / 2) * width + width / 2) * 3],
+ 3);
+ free (img);
+ return 1;
+}
+
+
+int
TEST_media_png_center_is (const void *data,
size_t data_len,
uint8_t r,
diff --git a/src/plugins/test_media_lib.h b/src/plugins/test_media_lib.h
@@ -34,6 +34,26 @@
#include <stdlib.h>
/**
+ * Decode the PNG in @a data into packed 8-bit RGB (an alpha channel is
+ * dropped). Only the subset of PNG our own encoders produce is
+ * understood: 8 bits per channel, RGB or RGBA, not interlaced.
+ * Requires zlib; without it this always fails (see
+ * #TEST_MEDIA_HAVE_PNG).
+ *
+ * @param data the PNG image
+ * @param data_len number of bytes in @a data
+ * @param[out] width set to the width of the image
+ * @param[out] height set to the height of the image
+ * @return the pixels (to be freed with free()), NULL on error
+ */
+uint8_t *
+TEST_media_png_decode (const void *data,
+ size_t data_len,
+ uint32_t *width,
+ uint32_t *height);
+
+
+/**
* Decode the PNG in @a data and return its centre pixel.
*
* Only the subset of PNG our own encoders produce is understood: 8 bits
diff --git a/src/plugins/test_odf.c b/src/plugins/test_odf.c
@@ -27,6 +27,184 @@
/**
+ * Check the general shape of a full text: valid UTF-8, 0-terminated,
+ * no leading or trailing white space, paragraphs separated by exactly
+ * "\n\n", no other line breaks, no double spaces.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data, including the 0
+ * @return 1 if the shape is right
+ */
+static int
+fulltext_shape_ok (const char *data,
+ size_t data_len)
+{
+ const unsigned char *t = (const unsigned char *) data;
+ size_t len;
+ size_t i;
+
+ if ( (data_len < 2) ||
+ ('\0' != data[data_len - 1]) ||
+ (strlen (data) != data_len - 1) )
+ return 0;
+ len = data_len - 1;
+ if (isspace (t[0]) || isspace (t[len - 1]))
+ return 0;
+ for (i = 0; i < len; i++)
+ {
+ if ('\n' == t[i])
+ {
+ if ( (i + 2 >= len) ||
+ ('\n' != t[i + 1]) ||
+ isspace (t[i + 2]) ||
+ isspace (t[i - 1]) )
+ return 0;
+ i++;
+ continue;
+ }
+ if ( (' ' == t[i]) && (' ' == t[i + 1]) )
+ return 0;
+ if ( (t[i] < 0x20) )
+ return 0;
+ if (t[i] >= 0x80)
+ {
+ unsigned int n;
+ unsigned int k;
+
+ if ( (t[i] & 0xE0) == 0xC0)
+ n = 1;
+ else if ( (t[i] & 0xF0) == 0xE0)
+ n = 2;
+ else if ( (t[i] & 0xF8) == 0xF0)
+ n = 3;
+ else
+ return 0;
+ if (i + n >= len)
+ return 0;
+ for (k = 1; k <= n; k++)
+ if ( (t[i + k] & 0xC0) != 0x80)
+ return 0;
+ i += n;
+ }
+ }
+ return 1;
+}
+
+
+/**
+ * Expected end of the full text of odf_cg.odt.
+ */
+#define CG_TAIL \
+ "\n\nFocus Areas of Research and Teaching\n\nProgramming languages, " \
+ "Computer security, Networking, Software tools"
+
+
+/**
+ * Validate the full text of odf_cg.odt (verified against
+ * "soffice --headless --convert-to txt").
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+validate_cg (const void *data,
+ size_t data_len)
+{
+ const char *t = data;
+
+ if (! fulltext_shape_ok (t, data_len))
+ return 0;
+ return (0 == strncmp (t,
+ "Christian Grothoff, PhD (UCLA)\n\n"
+ "TU M\xc3\xbc" "nchen, Department of Informatics, ",
+ strlen ("Christian Grothoff, PhD (UCLA)\n\n"
+ "TU M\xc3\xbc" "nchen, Department of "
+ "Informatics, "))) &&
+ (NULL != strstr (t,
+ "\n\n1996\xe2\x80\x93" "2000 Diplom II in "
+ "Mathematik, BUGH Wuppertal\n\n")) &&
+ (NULL != strstr (t,
+ "\n\nProfessional Activities\n\nDr. Grothoff has "
+ "a history of developing useful tools")) &&
+ (NULL != strstr (t,
+ "He is also the maintainer of GNU libextractor, "
+ "a GNU project for metadata extraction.")) &&
+ (data_len > strlen (CG_TAIL)) &&
+ (0 == strcmp (t + data_len - 1 - strlen (CG_TAIL),
+ CG_TAIL));
+}
+
+
+/**
+ * Called with each item from the default (no option) run; counts
+ * #EXTRACTOR_METATYPE_FULL_TEXT items.
+ *
+ * @param cls pointer to an `unsigned int` counter
+ * @param plugin_name unused
+ * @param type type of the item
+ * @param format unused
+ * @param data_mime_type unused
+ * @param data unused
+ * @param data_len unused
+ * @return 0 (continue)
+ */
+static int
+count_fulltext (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ unsigned int *cnt = cls;
+
+ (void) plugin_name;
+ (void) format;
+ (void) data_mime_type;
+ (void) data;
+ (void) data_len;
+ if (EXTRACTOR_METATYPE_FULL_TEXT == type)
+ (*cnt)++;
+ return 0;
+}
+
+
+/**
+ * Check that without the "fulltext" option no full text is reported.
+ *
+ * @param files NULL-terminated list of files to check
+ * @return 0 on success
+ */
+static int
+check_no_fulltext (const char *const *files)
+{
+ struct EXTRACTOR_PluginList *pl;
+ unsigned int cnt = 0;
+ unsigned int i;
+
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ "odf",
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ for (i = 0; NULL != files[i]; i++)
+ EXTRACTOR_extract (pl,
+ files[i],
+ NULL, 0,
+ &count_fulltext,
+ &cnt);
+ EXTRACTOR_plugin_remove_all (pl);
+ if (0 != cnt)
+ {
+ fprintf (stderr,
+ "Got full text without the fulltext option\n");
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
* Main function for the ODF testcase.
*
* @param argc number of arguments (ignored)
@@ -86,14 +264,172 @@ main (int argc, char *argv[])
strlen ("Anhang 1: Profile der beteiligten Wissenschaftler") + 1,
0
},
+ /* only expected with the "fulltext" option; its 'solved' is -1
+ (end of list) for the default run */
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^Christian Grothoff",
+ 0,
+ -1,
+ 1,
+ &validate_cg
+ },
{ 0, 0, NULL, NULL, 0, -1 }
};
+#define ODT_FT \
+ "Synthetic Heading\n\n" \
+ "First paragraph with spaces and a tab and a break; entities & <tag> " \
+ "\xe2\x98\xba \xe2\x82\xac and the sentence continues.\n\n" \
+ "This is page of pages.\n\n" \
+ "List item one\n\n" \
+ "List item two\n\n" \
+ "Cell A1\n\nCell B1\n\nCell A2\n\nCell B2\n\n" \
+ "See Chapter One for details.\n\n" \
+ "Before the frame\n\nFrame caption text\n\nafter the frame.\n\n" \
+ "Last paragraph."
+ struct SolutionData odt_ft_sol[] = {
+ {
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/vnd.oasis.opendocument.text",
+ strlen ("application/vnd.oasis.opendocument.text") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "gen_odf_testdata.sh",
+ strlen ("gen_odf_testdata.sh") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_TITLE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "Synthetic full text test",
+ strlen ("Synthetic full text test") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ ODT_FT,
+ strlen (ODT_FT) + 1,
+ 0
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+#define ODP_FT \
+ "First slide title\n\nBullet one\n\nBullet two\n\n" \
+ "Second slide title\n\nBullet three"
+ struct SolutionData odp_ft_sol[] = {
+ {
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/vnd.oasis.opendocument.presentation",
+ strlen ("application/vnd.oasis.opendocument.presentation") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "gen_odf_testdata.sh",
+ strlen ("gen_odf_testdata.sh") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_TITLE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "Synthetic slides",
+ strlen ("Synthetic slides") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ ODP_FT,
+ strlen (ODP_FT) + 1,
+ 0
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+#define ODS_FT "Name\n\nAmount\n\nApples\n\n42"
+ struct SolutionData ods_ft_sol[] = {
+ {
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/vnd.oasis.opendocument.spreadsheet",
+ strlen ("application/vnd.oasis.opendocument.spreadsheet") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "gen_odf_testdata.sh",
+ strlen ("gen_odf_testdata.sh") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_TITLE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "Synthetic sheet",
+ strlen ("Synthetic sheet") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ ODS_FT,
+ strlen (ODS_FT) + 1,
+ 0
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
+ const char *const ft_files[] = {
+ "testdata/odf_cg.odt",
+ "testdata/odf_fulltext.odt",
+ "testdata/odf_fulltext.odp",
+ "testdata/odf_fulltext.ods",
+ NULL
+ };
struct ProblemSet ps[] = {
{ "testdata/odf_cg.odt",
odf_cg_sol },
{ NULL, NULL }
};
- return ET_main ("odf", ps);
+ struct ProblemSet ps_ft[] = {
+ { "testdata/odf_cg.odt",
+ odf_cg_sol },
+ { "testdata/odf_fulltext.odt",
+ odt_ft_sol },
+ { "testdata/odf_fulltext.odp",
+ odp_ft_sol },
+ { "testdata/odf_fulltext.ods",
+ ods_ft_sol },
+ { NULL, NULL }
+ };
+ size_t ft_idx = sizeof (odf_cg_sol) / sizeof (odf_cg_sol[0]) - 2;
+ int ret;
+
+ if (0 != (ret = ET_main ("odf", ps)))
+ return ret;
+ if (0 != (ret = check_no_fulltext (ft_files)))
+ return ret;
+ odf_cg_sol[ft_idx].solved = 0;
+ return ET_main ("odf(fulltext)", ps_ft);
}
diff --git a/src/plugins/test_ole2.c b/src/plugins/test_ole2.c
@@ -27,6 +27,271 @@
/**
+ * Check the general shape of a full text: valid UTF-8, 0-terminated,
+ * no leading or trailing white space, paragraphs separated by exactly
+ * "\n\n", no other line breaks, no double spaces.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data, including the 0
+ * @return 1 if the shape is right
+ */
+static int
+fulltext_shape_ok (const char *data,
+ size_t data_len)
+{
+ const unsigned char *t = (const unsigned char *) data;
+ size_t len;
+ size_t i;
+
+ if ( (data_len < 2) ||
+ ('\0' != data[data_len - 1]) ||
+ (strlen (data) != data_len - 1) )
+ return 0;
+ len = data_len - 1;
+ if (isspace (t[0]) || isspace (t[len - 1]))
+ return 0;
+ for (i = 0; i < len; i++)
+ {
+ if ('\n' == t[i])
+ {
+ if ( (i + 2 >= len) ||
+ ('\n' != t[i + 1]) ||
+ isspace (t[i + 2]) ||
+ isspace (t[i - 1]) )
+ return 0;
+ i++;
+ continue;
+ }
+ if ( (' ' == t[i]) && (' ' == t[i + 1]) )
+ return 0;
+ if ( (t[i] < 0x20) )
+ return 0;
+ if (t[i] >= 0x80)
+ {
+ unsigned int n;
+ unsigned int k;
+
+ if ( (t[i] & 0xE0) == 0xC0)
+ n = 1;
+ else if ( (t[i] & 0xF0) == 0xE0)
+ n = 2;
+ else if ( (t[i] & 0xF8) == 0xF0)
+ n = 3;
+ else
+ return 0;
+ if (i + n >= len)
+ return 0;
+ for (k = 1; k <= n; k++)
+ if ( (t[i + k] & 0xC0) != 0x80)
+ return 0;
+ i += n;
+ }
+ }
+ return 1;
+}
+
+
+/**
+ * Expected start of the full text of ole2_blair.doc (verified against
+ * "soffice --headless --convert-to txt").
+ */
+#define BLAIR_HEAD \
+ "IRAQ \xe2\x80\x93 ITS INFRASTRUCTURE OF CONCEALMENT, DECEPTION AND " \
+ "INTIMIDATION\n\nThis report draws upon a number of sources, "
+
+/**
+ * Expected end of the full text of ole2_blair.doc.
+ */
+#define BLAIR_TAIL \
+ "\n\nThe overall effect of the systems of control and intimidation is " \
+ "that every Iraqi is suspicious of all except closest family."
+
+
+/**
+ * Validate the full text of ole2_blair.doc. Its footer holds a PAGE
+ * field ("- 19 -"), and its organisation chart ("SADDAM AND HIS SONS") is
+ * in text boxes; both
+ * must not show up.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+validate_blair (const void *data,
+ size_t data_len)
+{
+ const char *t = data;
+
+ if (! fulltext_shape_ok (t, data_len))
+ return 0;
+ return (0 == strncmp (t, BLAIR_HEAD, strlen (BLAIR_HEAD))) &&
+ (data_len > strlen (BLAIR_TAIL)) &&
+ (0 == strcmp (t + data_len - 1 - strlen (BLAIR_TAIL),
+ BLAIR_TAIL)) &&
+ (NULL != strstr (t,
+ "\n\nBut Iraq has singularly failed to do this."
+ "\n\n")) &&
+ (NULL != strstr (t,
+ "Inspection is not a game of \xe2\x80\x9c" "catch "
+ "as catch can\xe2\x80\x9d.\xe2\x80\x9d\n\n")) &&
+ (NULL == strstr (t, "- 19 -")) &&
+ (NULL == strstr (t, "PAGE")) &&
+ (NULL == strstr (t, "SADDAM"));
+}
+
+
+/**
+ * Called with each item from the default (no option) run; counts
+ * #EXTRACTOR_METATYPE_FULL_TEXT items.
+ *
+ * @param cls pointer to an `unsigned int` counter
+ * @param plugin_name unused
+ * @param type type of the item
+ * @param format unused
+ * @param data_mime_type unused
+ * @param data unused
+ * @param data_len unused
+ * @return 0 (continue)
+ */
+static int
+count_fulltext (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ unsigned int *cnt = cls;
+
+ (void) plugin_name;
+ (void) format;
+ (void) data_mime_type;
+ (void) data;
+ (void) data_len;
+ if (EXTRACTOR_METATYPE_FULL_TEXT == type)
+ (*cnt)++;
+ return 0;
+}
+
+
+/**
+ * Closure for #capture_fulltext.
+ */
+struct Capture
+{
+ /**
+ * Number of full text items seen.
+ */
+ unsigned int cnt;
+
+ /**
+ * Copy of the last full text seen, or NULL.
+ */
+ char *text;
+};
+
+
+/**
+ * Called with each item; remembers the full text.
+ *
+ * @param cls a `struct Capture`
+ * @param plugin_name unused
+ * @param type type of the item
+ * @param format unused
+ * @param data_mime_type unused
+ * @param data the item
+ * @param data_len number of bytes in @a data
+ * @return 0 (continue)
+ */
+static int
+capture_fulltext (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct Capture *cap = cls;
+
+ (void) plugin_name;
+ (void) format;
+ (void) data_mime_type;
+ if (EXTRACTOR_METATYPE_FULL_TEXT != type)
+ return 0;
+ cap->cnt++;
+ free (cap->text);
+ cap->text = NULL;
+ if ( (0 < data_len) &&
+ ('\0' == data[data_len - 1]) )
+ cap->text = strdup (data);
+ return 0;
+}
+
+
+/**
+ * Run the plugin with configuration @a config on each of @a files and
+ * compare the full text with @a expected.
+ *
+ * @param config plugin name and options
+ * @param files NULL-terminated list of files
+ * @param expected expected full text for each file, NULL for none
+ * @return 0 on success
+ */
+static int
+check_fulltext (const char *config,
+ const char *const *files,
+ const char *const *expected)
+{
+ struct EXTRACTOR_PluginList *pl;
+ unsigned int i;
+ int ret = 0;
+
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ config,
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ for (i = 0; NULL != files[i]; i++)
+ {
+ struct Capture cap = { 0, NULL };
+
+ EXTRACTOR_extract (pl,
+ files[i],
+ NULL, 0,
+ &capture_fulltext,
+ &cap);
+ if (NULL == expected[i])
+ {
+ if (0 != cap.cnt)
+ {
+ fprintf (stderr,
+ "%s: unexpected full text from %s\n",
+ config,
+ files[i]);
+ ret = 1;
+ }
+ }
+ else if ( (1 != cap.cnt) ||
+ (NULL == cap.text) ||
+ (! fulltext_shape_ok (cap.text, strlen (cap.text) + 1)) ||
+ (0 != strcmp (cap.text, expected[i])) )
+ {
+ fprintf (stderr,
+ "%s: wrong full text from %s: got `%s'\n",
+ config,
+ files[i],
+ (NULL == cap.text) ? "(none)" : cap.text);
+ ret = 1;
+ }
+ free (cap.text);
+ }
+ EXTRACTOR_plugin_remove_all (pl);
+ return ret;
+}
+
+
+/**
* Main function for the OLE2 testcase.
*
* @param argc number of arguments (ignored)
@@ -119,6 +384,16 @@ main (int argc, char *argv[])
strlen ("Windows 1.0") + 1,
0
},
+ /* only expected with the "fulltext" option; its 'solved' is -1
+ (end of list) for the default run */
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "This is a test document for libextractor.",
+ strlen ("This is a test document for libextractor.") + 1,
+ -1
+ },
{ 0, 0, NULL, NULL, 0, -1 }
};
@@ -468,6 +743,17 @@ main (int argc, char *argv[])
strlen ("Windows 4.0") + 1,
0
},
+ /* only expected with the "fulltext" option */
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^IRAQ",
+ 0,
+ -1,
+ 1,
+ &validate_blair
+ },
{ 0, 0, NULL, NULL, 0, -1 }
};
#endif
@@ -663,7 +949,59 @@ main (int argc, char *argv[])
ole2_powerpoint_sol },
{ NULL, NULL }
};
- return ET_main ("ole2", ps);
+ /* ole2_fulltext.doc is odf_fulltext.odt converted by LibreOffice (see
+ contrib/gen_odf_testdata.sh); its header/footer, footnote, comment
+ and PAGE/NUMPAGES fields must not show up. The bookmark reference
+ got lost in the conversion. */
+ const char *const ft_files[] = {
+ "testdata/ole2_fulltext.doc",
+ "testdata/ole2_starwriter40.sdw",
+ "testdata/ole2_excel.xls",
+ "testdata/ole2_powerpoint.ppt",
+ NULL
+ };
+ const char *const ft_expected[] = {
+ "Synthetic Heading\n\n"
+ "First paragraph with spaces and a tab and a break; entities & <tag> "
+ "\xe2\x98\xba \xe2\x82\xac and the sentence continues.\n\n"
+ "This is page of pages.\n\n"
+ "List item one\n\n"
+ "List item two\n\n"
+ "Cell A1\n\nCell B1\n\nCell A2\n\nCell B2\n\n"
+ "See for details.\n\n"
+ "Last paragraph.",
+ NULL,
+ NULL,
+ NULL
+ };
+ const char *const no_ft_expected[] = {
+ NULL, NULL, NULL, NULL
+ };
+ const char *const default_files[] = {
+ "testdata/ole2_msword.doc",
+ "testdata/ole2_blair.doc",
+ NULL
+ };
+ int ret;
+ unsigned int i;
+
+ if (0 != (ret = ET_main ("ole2", ps)))
+ return ret;
+ if (0 != check_fulltext ("ole2", ft_files, no_ft_expected))
+ return 1;
+ if (0 != check_fulltext ("ole2", default_files, no_ft_expected))
+ return 1;
+ for (i = 0; -1 != ole2_msword_sol[i].solved; i++)
+ ;
+ ole2_msword_sol[i].solved = 0;
+#if HAVE_ICONV
+ for (i = 0; -1 != ole2_blair_sol[i].solved; i++)
+ ;
+ ole2_blair_sol[i].solved = 0;
+#endif
+ if (0 != (ret = ET_main ("ole2(fulltext)", ps)))
+ return ret;
+ return check_fulltext ("ole2(fulltext)", ft_files, ft_expected);
}
diff --git a/src/plugins/test_paged_fulltext.h b/src/plugins/test_paged_fulltext.h
@@ -0,0 +1,325 @@
+/*
+ This file is part of libextractor.
+ Copyright (C) 2026 Christian Grothoff
+
+ libextractor is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ libextractor is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with libextractor; see the file COPYING. If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+*/
+/**
+ * @file plugins/test_paged_fulltext.h
+ * @brief checks on the "fulltext" output of the paged-format plugins
+ * (pdf, dvi) for the documents made by
+ * contrib/gen_fulltext_pdf_dvi_testdata.sh
+ * @author Christian Grothoff
+ */
+#ifndef TEST_PAGED_FULLTEXT_H
+#define TEST_PAGED_FULLTEXT_H
+
+#include "platform.h"
+#include "test_lib.h"
+#include <ctype.h>
+#include <unistd.h>
+
+/**
+ * Number of FULL_TEXT items seen by #count_fulltext().
+ */
+static unsigned int fulltext_items;
+
+
+/**
+ * Count FULL_TEXT items.
+ *
+ * @param cls unused
+ * @param plugin_name name of the plugin
+ * @param type type of the meta data
+ * @param format format of the meta data
+ * @param data_mime_type mime type of the meta data
+ * @param data the meta data
+ * @param data_len number of bytes in @a data
+ * @return 0 (continue)
+ */
+static int
+count_fulltext (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ (void) cls;
+ (void) plugin_name;
+ (void) format;
+ (void) data_mime_type;
+ (void) data;
+ (void) data_len;
+ if (EXTRACTOR_METATYPE_FULL_TEXT == type)
+ fulltext_items++;
+ return 0;
+}
+
+
+/**
+ * Check that running @a plugin on @a filename yields exactly
+ * @a expected FULL_TEXT items.
+ *
+ * @param plugin plugin name and options
+ * @param filename file to extract from
+ * @param expected number of FULL_TEXT items expected
+ * @return 0 on success, 1 on failure
+ */
+static int
+check_fulltext_count (const char *plugin,
+ const char *filename,
+ unsigned int expected)
+{
+ struct EXTRACTOR_PluginList *pl;
+
+ if (0 != access (filename, R_OK))
+ return 0;
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ plugin,
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ fulltext_items = 0;
+ EXTRACTOR_extract (pl,
+ filename,
+ NULL, 0,
+ &count_fulltext,
+ NULL);
+ EXTRACTOR_plugin_remove_all (pl);
+ if (expected == fulltext_items)
+ return 0;
+ fprintf (stderr,
+ "%s on %s: expected %u FULL_TEXT items, got %u\n",
+ plugin,
+ filename,
+ expected,
+ fulltext_items);
+ return 1;
+}
+
+
+/**
+ * Check the general form of a full text: valid UTF-8, no leading or
+ * trailing whitespace, no control characters except the "\n\n"
+ * paragraph separators, no double spaces.
+ *
+ * @param data the text, 0-terminated
+ * @param data_len number of bytes in @a data including the 0
+ * @return number of paragraphs, 0 if malformed
+ */
+static unsigned int
+check_form (const char *data,
+ size_t data_len)
+{
+ const unsigned char *s = (const unsigned char *) data;
+ size_t len;
+ size_t i;
+ unsigned int pars = 1;
+
+ if ( (0 == data_len) ||
+ ('\0' != data[data_len - 1]) )
+ return 0;
+ len = data_len - 1;
+ if ( (0 == len) ||
+ (strlen (data) != len) ||
+ (isspace (s[0])) ||
+ (isspace (s[len - 1])) )
+ return 0;
+ for (i = 0; i < len; i++)
+ {
+ unsigned char c = s[i];
+ unsigned int n;
+ unsigned int k;
+
+ if ('\n' == c)
+ {
+ if ( (i + 2 >= len) ||
+ ('\n' != s[i + 1]) ||
+ (isspace (s[i + 2])) ||
+ (' ' == s[i - 1]) )
+ return 0;
+ pars++;
+ i++;
+ continue;
+ }
+ if (' ' == c)
+ {
+ if (' ' == s[i + 1])
+ return 0;
+ continue;
+ }
+ if (c < 0x20)
+ return 0;
+ if (c < 0x80)
+ continue;
+ if ( (c & 0xE0) == 0xC0)
+ n = 1;
+ else if ( (c & 0xF0) == 0xE0)
+ n = 2;
+ else if ( (c & 0xF8) == 0xF0)
+ n = 3;
+ else
+ return 0;
+ if (i + n >= len)
+ return 0;
+ for (k = 1; k <= n; k++)
+ if ( (s[i + k] & 0xC0) != 0x80)
+ return 0;
+ i += n;
+ }
+ return pars;
+}
+
+
+/**
+ * Check that the paragraphs of @a data start with @a starts, in order.
+ *
+ * @param data the text
+ * @param starts NULL-terminated list of paragraph prefixes
+ * @return 1 if all match
+ */
+static int
+check_paragraphs (const char *data,
+ const char *const *starts)
+{
+ const char *p = data;
+ unsigned int i;
+
+ for (i = 0; NULL != starts[i]; i++)
+ {
+ if (NULL == p)
+ {
+ fprintf (stderr, "Missing paragraph `%s'\n", starts[i]);
+ return 0;
+ }
+ if (0 != strncmp (p, starts[i], strlen (starts[i])))
+ {
+ fprintf (stderr,
+ "Paragraph %u starts with `%.40s', expected `%s'\n",
+ i, p, starts[i]);
+ return 0;
+ }
+ p = strstr (p, "\n\n");
+ if (NULL != p)
+ p += 2;
+ }
+ if (NULL != p)
+ {
+ fprintf (stderr, "Unexpected paragraph `%.40s'\n", p);
+ return 0;
+ }
+ return 1;
+}
+
+
+/**
+ * Check that all of @a needles occur in @a data, and none of
+ * @a absent.
+ *
+ * @param data the text
+ * @param needles NULL-terminated list of required substrings
+ * @param absent NULL-terminated list of forbidden substrings
+ * @return 1 if so
+ */
+static int
+check_contains (const char *data,
+ const char *const *needles,
+ const char *const *absent)
+{
+ unsigned int i;
+
+ for (i = 0; NULL != needles[i]; i++)
+ if (NULL == strstr (data, needles[i]))
+ {
+ fprintf (stderr, "Full text lacks `%s'\n", needles[i]);
+ return 0;
+ }
+ for (i = 0; NULL != absent[i]; i++)
+ if (NULL != strstr (data, absent[i]))
+ {
+ fprintf (stderr, "Full text contains `%s'\n", absent[i]);
+ return 0;
+ }
+ return 1;
+}
+
+
+/**
+ * Check that no word of @a data consists of digits only (page and line
+ * numbers must be gone; the test documents have no other numbers).
+ *
+ * @param data the text
+ * @return 1 if there is no such word
+ */
+static int
+check_no_numbers (const char *data)
+{
+ const char *p = data;
+
+ while ('\0' != *p)
+ {
+ const char *e = p;
+ int digits = 1;
+
+ while ( ('\0' != *e) && (' ' != *e) && ('\n' != *e) )
+ {
+ if (! isdigit ((unsigned char) *e))
+ digits = 0;
+ e++;
+ }
+ if ( (e > p) && digits)
+ {
+ fprintf (stderr, "Full text contains number `%.*s'\n",
+ (int) (e - p), p);
+ return 0;
+ }
+ p = e;
+ while ( (' ' == *p) || ('\n' == *p) )
+ p++;
+ }
+ return 1;
+}
+
+
+/**
+ * Sentences both test documents must yield: re-joined hyphenations,
+ * ligatures, quotes and dashes, and a paragraph across a page break.
+ */
+static const char *const common_needles[] = {
+ "put them back in reading order. This first paragraph",
+ "the extractor should notice that indentation and start",
+ "the office has fluffy waffles, an efficient staff and a "
+ "\xe2\x80\x9cquoted phrase\xe2\x80\x9d with pages 10\xe2\x80\x93" "20 "
+ "and a pause\xe2\x80\x94like this one.",
+ "nobody will ever read, apart from the test suite, which checks every "
+ "single word of it.",
+ "and the extractor must join both halves",
+ "about zebras jumping over lazy foxes.",
+ NULL
+};
+
+/**
+ * Page furniture that must be gone.
+ */
+static const char *const common_absent[] = {
+ "Notes on Paged Text",
+ "Working Draft",
+ "extrac- tor",
+ "hap- pens",
+ NULL
+};
+
+#endif
diff --git a/src/plugins/test_pdf.c b/src/plugins/test_pdf.c
@@ -24,6 +24,58 @@
*/
#include "platform.h"
#include "test_lib.h"
+#include "test_paged_fulltext.h"
+
+
+/**
+ * Check the full text of testdata/pdf_fulltext.pdf.
+ *
+ * @param data the full text
+ * @param data_len number of bytes in @a data
+ * @return 1 if acceptable
+ */
+static int
+validate_pdf_fulltext (const void *data,
+ size_t data_len)
+{
+ static const char *const pars[] = {
+ "Introduction",
+ "Paged documents scatter their words",
+ "The second paragraph is indented",
+ "Short paragraph number three.",
+ "The fourth paragraph",
+ "The fifth paragraph runs across the page break",
+ "The sixth paragraph",
+ "The seventh paragraph opens the two column part",
+ "The eighth paragraph continues in the left column",
+ "The ninth paragraph",
+ "The tenth paragraph is the first one of the right column",
+ "The eleventh paragraph concludes the document with a final remark.",
+ NULL
+ };
+ static const char *const needles[] = {
+ "The caf\xc3\xa9 serves na\xc3\xafve guests.",
+ "the line numbers printed in the margins next to each column must be "
+ "dropped by the extractor because they are not part of the text at all.",
+ "apples, pears and plums that grow in an orchard near the river.",
+ "that guide ships safely through the night along rocky coasts.",
+ NULL
+ };
+ static const char *const absent[] = {
+ NULL
+ };
+ const char *s = data;
+
+ if (0 == check_form (s, data_len))
+ {
+ fprintf (stderr, "Malformed full text\n");
+ return 0;
+ }
+ return check_paragraphs (s, pars) &&
+ check_contains (s, common_needles, common_absent) &&
+ check_contains (s, needles, absent) &&
+ check_no_numbers (s);
+}
/**
@@ -127,12 +179,63 @@ main (int argc, char *argv[])
},
{ 0, 0, NULL, NULL, 0, -1 }
};
+ struct SolutionData pdf_fulltext_sol[] = {
+ {
+ EXTRACTOR_METATYPE_MIMETYPE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "application/pdf",
+ strlen ("application/pdf") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_PAGE_COUNT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "3",
+ strlen ("3") + 1,
+ 0
+ },
+ {
+ EXTRACTOR_METATYPE_FULL_TEXT,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ "^Introduction\n\nPaged documents scatter",
+ 0,
+ 0,
+ 1,
+ &validate_pdf_fulltext
+ },
+ { 0, 0, NULL, NULL, 0, -1 }
+ };
struct ProblemSet ps[] = {
{ "testdata/pdf_extract.pdf",
pdf_extract_sol },
{ NULL, NULL }
};
- return ET_main ("pdf", ps);
+ struct ProblemSet ps_ft[] = {
+ { "testdata/pdf_extract.pdf",
+ pdf_extract_sol },
+ { "testdata/pdf_fulltext.pdf",
+ pdf_fulltext_sol },
+ { NULL, NULL }
+ };
+ int ret = 0;
+
+ ret |= ET_main ("pdf", ps);
+ /* without the option, there must be no full text */
+ ret |= check_fulltext_count ("pdf",
+ "testdata/pdf_fulltext.pdf",
+ 0);
+ ret |= ET_main ("pdf(fulltext)", ps_ft);
+ /* a document without text has no full text either */
+ ret |= check_fulltext_count ("pdf(fulltext)",
+ "testdata/pdf_extract.pdf",
+ 0);
+ ret |= check_fulltext_count ("pdf(fulltext)",
+ "testdata/pdf_fulltext.pdf",
+ 1);
+ return ret;
}
diff --git a/src/plugins/test_thumbnailffmpeg.c b/src/plugins/test_thumbnailffmpeg.c
@@ -27,6 +27,11 @@
* hand, is decoded: testdata/thumbnail_pattern.ogv is black for the
* first 20 seconds and red afterwards, so the colour of the thumbnail
* says whether the plugin really sampled where it claims to.
+ *
+ * testdata/subtitles.mkv (see contrib/gen_testmedia.sh) exercises the
+ * subtitle languages, the "srt:LANG" dump and the "gallery:N" contact
+ * sheet. For those, every item the plugin emits is collected, so that
+ * the tests can also check what must NOT be there.
*/
#include "platform.h"
#include "test_lib.h"
@@ -53,6 +58,754 @@ validate_red (const void *data,
/**
+ * The subtitled test video.
+ */
+#define SUBTITLES_MKV "testdata/subtitles.mkv"
+
+/**
+ * Upper bound for a gallery, see EXTRACTOR_MEDIA_MAX_GALLERY_BYTES.
+ */
+#define MAX_GALLERY_BYTES (4 * 1024 * 1024)
+
+/**
+ * Upper bound for the pixels of a gallery sheet (the plugin's
+ * GALLERY_MAX_SHEET_PIXELS).
+ */
+#define MAX_GALLERY_PIXELS (4 * 1024 * 1024)
+
+/**
+ * Most items we collect from one run.
+ */
+#define MAX_ITEMS 32
+
+/**
+ * Expected SRT dump of the English track.
+ */
+static const char srt_en[] =
+ "1\n"
+ "00:00:01,000 --> 00:00:04,000\n"
+ "Hello, world!\n"
+ "\n"
+ "2\n"
+ "00:00:12,500 --> 00:00:15,250\n"
+ "This is the second line\n"
+ "spanning two rows.\n"
+ "\n"
+ "3\n"
+ "00:00:33,000 --> 00:00:38,000\n"
+ "Goodbye.\n"
+ "\n";
+
+/**
+ * Expected SRT dump of the German track.
+ */
+static const char srt_de[] =
+ "1\n"
+ "00:00:01,000 --> 00:00:04,000\n"
+ "Gr\xc3\xbc\xc3\x9f Gott, Welt!\n"
+ "\n"
+ "2\n"
+ "00:00:12,500 --> 00:00:15,250\n"
+ "Das ist die zweite Zeile\n"
+ "\xc3\xbc" "ber zwei Reihen.\n"
+ "\n"
+ "3\n"
+ "00:00:33,000 --> 00:00:38,000\n"
+ "Auf Wiedersehen.\n"
+ "\n";
+
+/**
+ * Expected SRT dump of the French (ASS) track: override tags and the
+ * ASS fields are gone, "\N" became a line break.
+ */
+static const char srt_fr[] =
+ "1\n"
+ "00:00:01,000 --> 00:00:04,000\n"
+ "Bonjour, le monde !\n"
+ "\n"
+ "2\n"
+ "00:00:12,500 --> 00:00:15,250\n"
+ "Deuxi\xc3\xa8me ligne\n"
+ "sur deux rangs.\n"
+ "\n";
+
+
+/**
+ * One item the plugin emitted.
+ */
+struct Item
+{
+ /**
+ * Type of the item.
+ */
+ enum EXTRACTOR_MetaType type;
+
+ /**
+ * Format of the item.
+ */
+ enum EXTRACTOR_MetaFormat format;
+
+ /**
+ * Mime type, "" if none.
+ */
+ char mime[64];
+
+ /**
+ * Copy of the data.
+ */
+ char *data;
+
+ /**
+ * Number of bytes in @e data.
+ */
+ size_t data_len;
+};
+
+
+/**
+ * Everything the plugin emitted for one file.
+ */
+struct Items
+{
+ /**
+ * The items, in the order they were emitted.
+ */
+ struct Item item[MAX_ITEMS];
+
+ /**
+ * Number of valid entries in @e item.
+ */
+ unsigned int count;
+
+ /**
+ * Set if there were more than #MAX_ITEMS.
+ */
+ int overflow;
+};
+
+
+/**
+ * Collect an item. Signature of #EXTRACTOR_MetaDataProcessor.
+ *
+ * @param cls our `struct Items`
+ * @param plugin_name name of the plugin
+ * @param type type of the item
+ * @param format format of the item
+ * @param data_mime_type mime type of the item
+ * @param data the item
+ * @param data_len number of bytes in @a data
+ * @return 0 (always continue)
+ */
+static int
+collect (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct Items *items = cls;
+ struct Item *it;
+
+ (void) plugin_name;
+ if (items->count == MAX_ITEMS)
+ {
+ items->overflow = 1;
+ return 0;
+ }
+ it = &items->item[items->count];
+ it->type = type;
+ it->format = format;
+ snprintf (it->mime,
+ sizeof (it->mime),
+ "%s",
+ (NULL == data_mime_type) ? "" : data_mime_type);
+ if (NULL == (it->data = malloc (data_len + 1)))
+ abort ();
+ memcpy (it->data,
+ data,
+ data_len);
+ it->data[data_len] = '\0';
+ it->data_len = data_len;
+ items->count++;
+ return 0;
+}
+
+
+/**
+ * Release what collect() allocated.
+ *
+ * @param[in,out] items items to release
+ */
+static void
+items_clear (struct Items *items)
+{
+ unsigned int i;
+
+ for (i = 0; i < items->count; i++)
+ free (items->item[i].data);
+ memset (items, 0, sizeof (*items));
+}
+
+
+/**
+ * Run the plugin with configuration @a config on @a filename and
+ * collect everything it emits.
+ *
+ * @param config the plugin with its options, e.g. "thumbnailffmpeg(srt:de)"
+ * @param in_process 1 to run the plugin in-process
+ * @param filename file to extract from
+ * @param[out] items set to what the plugin emitted
+ */
+static void
+run_collect (const char *config,
+ int in_process,
+ const char *filename,
+ struct Items *items)
+{
+ struct EXTRACTOR_PluginList *pl;
+
+ memset (items, 0, sizeof (*items));
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ config,
+ in_process
+ ? EXTRACTOR_OPTION_IN_PROCESS
+ : EXTRACTOR_OPTION_DEFAULT_POLICY);
+ if (NULL == pl)
+ {
+ fprintf (stderr,
+ "Failed to load %s\n",
+ config);
+ abort ();
+ }
+ EXTRACTOR_extract (pl,
+ filename,
+ NULL,
+ 0,
+ &collect,
+ items);
+ EXTRACTOR_plugin_remove_all (pl);
+}
+
+
+/**
+ * Count the items of type @a type.
+ *
+ * @param items items to search
+ * @param type type to look for
+ * @param[out] last set to the last item of that type, may be NULL
+ * @return number of items of type @a type
+ */
+static unsigned int
+count_type (const struct Items *items,
+ enum EXTRACTOR_MetaType type,
+ const struct Item **last)
+{
+ unsigned int n = 0;
+ unsigned int i;
+
+ for (i = 0; i < items->count; i++)
+ if (items->item[i].type == type)
+ {
+ n++;
+ if (NULL != last)
+ *last = &items->item[i];
+ }
+ return n;
+}
+
+
+/**
+ * Check that the subtitle languages were reported exactly once each,
+ * in stream order, and that there is exactly one thumbnail.
+ *
+ * @param items what the plugin emitted
+ * @param what name of the run, for diagnostics
+ * @return 0 if everything is as expected
+ */
+static int
+check_languages (const struct Items *items,
+ const char *what)
+{
+ static const char *const expected[] = { "eng", "ger", "fre" };
+ unsigned int n = 0;
+ unsigned int i;
+
+ for (i = 0; i < items->count; i++)
+ {
+ const struct Item *it = &items->item[i];
+
+ if (EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE != it->type)
+ continue;
+ if ( (n >= 3) ||
+ (EXTRACTOR_METAFORMAT_UTF8 != it->format) ||
+ (it->data_len != strlen (expected[n]) + 1) ||
+ (0 != strcmp (it->data,
+ expected[n])) )
+ {
+ fprintf (stderr,
+ "%s: unexpected subtitle language #%u `%s'\n",
+ what,
+ n,
+ it->data);
+ return 1;
+ }
+ n++;
+ }
+ if (3 != n)
+ {
+ fprintf (stderr,
+ "%s: got %u subtitle languages, expected 3\n",
+ what,
+ n);
+ return 1;
+ }
+ if (1 != count_type (items,
+ EXTRACTOR_METATYPE_THUMBNAIL,
+ NULL))
+ {
+ fprintf (stderr,
+ "%s: expected exactly one thumbnail\n",
+ what);
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Check a run with "srt:LANG".
+ *
+ * @param lang language to request
+ * @param in_process 1 to run the plugin in-process
+ * @param expected expected SRT file, NULL if none must be emitted
+ * @return 0 on success
+ */
+static int
+test_srt (const char *lang,
+ int in_process,
+ const char *expected)
+{
+ struct Items items;
+ const struct Item *it = NULL;
+ char config[64];
+ unsigned int n;
+ int ret = 0;
+
+ snprintf (config,
+ sizeof (config),
+ "thumbnailffmpeg(srt:%s)",
+ lang);
+ run_collect (config,
+ in_process,
+ SUBTITLES_MKV,
+ &items);
+ ret |= check_languages (&items,
+ config);
+ n = count_type (&items,
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT,
+ &it);
+ if (NULL == expected)
+ {
+ if (0 != n)
+ {
+ fprintf (stderr,
+ "%s: got subtitles for a language that is not there\n",
+ config);
+ ret = 1;
+ }
+ }
+ else if (1 != n)
+ {
+ fprintf (stderr,
+ "%s: got %u subtitle dumps, expected 1\n",
+ config,
+ n);
+ ret = 1;
+ }
+ else if ( (EXTRACTOR_METAFORMAT_UTF8 != it->format) ||
+ (0 != strcmp (it->mime,
+ "application/x-subrip")) ||
+ (it->data_len != strlen (expected) + 1) ||
+ (0 != memcmp (it->data,
+ expected,
+ it->data_len)) )
+ {
+ fprintf (stderr,
+ "%s: got\n%s\nexpected\n%s\n",
+ config,
+ it->data,
+ expected);
+ ret = 1;
+ }
+ items_clear (&items);
+ return ret;
+}
+
+
+/**
+ * Read a big endian 32 bit number.
+ *
+ * @param p bytes to read
+ * @return the number
+ */
+static uint32_t
+be32 (const uint8_t *p)
+{
+ return (((uint32_t) p[0]) << 24)
+ | (((uint32_t) p[1]) << 16)
+ | (((uint32_t) p[2]) << 8)
+ | ((uint32_t) p[3]);
+}
+
+
+/**
+ * Get the dimensions of a PNG from its IHDR chunk.
+ *
+ * @param it the item holding the PNG
+ * @param[out] width set to the width
+ * @param[out] height set to the height
+ * @return 1 if @a it is a PNG
+ */
+static int
+png_size (const struct Item *it,
+ uint32_t *width,
+ uint32_t *height)
+{
+ static const uint8_t magic[] = { 137, 'P', 'N', 'G', '\r', '\n', 26, '\n' };
+ const uint8_t *d = (const uint8_t *) it->data;
+
+ if ( (it->data_len < 8 + 8 + 13) ||
+ (0 != memcmp (d, magic, sizeof (magic))) ||
+ (0 != memcmp (&d[12], "IHDR", 4)) )
+ return 0;
+ *width = be32 (&d[16]);
+ *height = be32 (&d[20]);
+ return 1;
+}
+
+
+/**
+ * Rough colour of a tile.
+ */
+enum TileColor
+{
+ /**
+ * Anything goes, as long as the tile is not basically black or white.
+ */
+ TC_ANY,
+
+ /**
+ * Mostly red.
+ */
+ TC_RED,
+
+ /**
+ * Mostly blue.
+ */
+ TC_BLUE
+};
+
+
+/**
+ * Check one tile of a decoded gallery.
+ *
+ * @param rgb the decoded sheet
+ * @param width width of the sheet
+ * @param x0 left edge of the tile
+ * @param y0 top edge of the tile
+ * @param tw width of the tile
+ * @param th height of the tile
+ * @param color colour the tile should have
+ * @param what name of the run, for diagnostics
+ * @param idx index of the tile, for diagnostics
+ * @return 0 if the tile looks right
+ */
+static int
+check_tile (const uint8_t *rgb,
+ uint32_t width,
+ uint32_t x0,
+ uint32_t y0,
+ uint32_t tw,
+ uint32_t th,
+ enum TileColor color,
+ const char *what,
+ unsigned int idx)
+{
+ uint64_t sum[3] = { 0, 0, 0 };
+ uint64_t dark = 0;
+ uint64_t light = 0;
+ uint64_t n = (uint64_t) tw * th;
+ uint32_t x;
+ uint32_t y;
+ unsigned int c;
+
+ for (y = y0; y < y0 + th; y++)
+ for (x = x0; x < x0 + tw; x++)
+ {
+ const uint8_t *p = &rgb[((size_t) y * width + x) * 3];
+
+ for (c = 0; c < 3; c++)
+ sum[c] += p[c];
+ if ( (p[0] < 48) && (p[1] < 48) && (p[2] < 48) )
+ dark++;
+ if ( (p[0] > 208) && (p[1] > 208) && (p[2] > 208) )
+ light++;
+ }
+ for (c = 0; c < 3; c++)
+ sum[c] /= n;
+ if ( (dark * 2 > n) ||
+ (light * 2 > n) )
+ {
+ fprintf (stderr,
+ "%s: tile %u is mostly %s, the flat frame was not replaced\n",
+ what,
+ idx,
+ (dark * 2 > n) ? "black" : "white");
+ return 1;
+ }
+ if ( ( (TC_RED == color) &&
+ ( (sum[0] < sum[1] + 96) || (sum[0] < sum[2] + 96) ) ) ||
+ ( (TC_BLUE == color) &&
+ ( (sum[2] < sum[0] + 96) || (sum[2] < sum[1] + 96) ) ) )
+ {
+ fprintf (stderr,
+ "%s: tile %u has average rgb(%u,%u,%u), expected %s\n",
+ what,
+ idx,
+ (unsigned int) sum[0],
+ (unsigned int) sum[1],
+ (unsigned int) sum[2],
+ (TC_RED == color) ? "red" : "blue");
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Run the plugin with a gallery configuration and check that it emits
+ * exactly one image of an acceptable size and no other thumbnail.
+ *
+ * @param config the plugin with its options
+ * @param png 1 if we asked for a PNG
+ * @param[out] items set to what the plugin emitted
+ * @param[out] gallery set to the gallery item
+ * @return 0 on success
+ */
+static int
+run_gallery (const char *config,
+ int png,
+ struct Items *items,
+ const struct Item **gallery)
+{
+ static const uint8_t jpeg_magic[] = { 0xFF, 0xD8, 0xFF };
+ uint32_t w;
+ uint32_t h;
+
+ run_collect (config,
+ 0,
+ SUBTITLES_MKV,
+ items);
+ if (0 != check_languages (items,
+ config))
+ return 1;
+ if (0 != count_type (items,
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT,
+ NULL))
+ {
+ fprintf (stderr,
+ "%s: got subtitles without asking for them\n",
+ config);
+ return 1;
+ }
+ count_type (items,
+ EXTRACTOR_METATYPE_THUMBNAIL,
+ gallery);
+ if ( (EXTRACTOR_METAFORMAT_BINARY != (*gallery)->format) ||
+ (0 != strcmp ((*gallery)->mime,
+ png ? "image/png" : "image/jpeg")) ||
+ ((*gallery)->data_len > MAX_GALLERY_BYTES) )
+ {
+ fprintf (stderr,
+ "%s: gallery has the wrong type or size (%s, %u bytes)\n",
+ config,
+ (*gallery)->mime,
+ (unsigned int) (*gallery)->data_len);
+ return 1;
+ }
+ if (! png)
+ {
+ if ( ((*gallery)->data_len < sizeof (jpeg_magic)) ||
+ (0 != memcmp ((*gallery)->data,
+ jpeg_magic,
+ sizeof (jpeg_magic))) )
+ {
+ fprintf (stderr,
+ "%s: gallery is not a JPEG\n",
+ config);
+ return 1;
+ }
+ return 0;
+ }
+ if ( (! png_size (*gallery, &w, &h)) ||
+ (0 == w) ||
+ (0 == h) ||
+ ((uint64_t) w * h > MAX_GALLERY_PIXELS) )
+ {
+ fprintf (stderr,
+ "%s: gallery is not a PNG of acceptable size\n",
+ config);
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Check "gallery:N" on the subtitled video.
+ *
+ * @return 0 on success
+ */
+static int
+test_gallery (void)
+{
+ /* with gallery:4 the tiles are sampled at 5s (black, must be shifted
+ into the red part), 15s (red), 25s (white, must be shifted into the
+ blue part) and 35s (blue) */
+ static const enum TileColor colors4[] = {
+ TC_RED, TC_RED, TC_BLUE, TC_BLUE
+ };
+ struct Items items;
+ const struct Item *g = NULL;
+ const char *config;
+ uint8_t *rgb;
+ uint32_t w;
+ uint32_t h;
+ unsigned int i;
+ int ret = 0;
+
+ /* 160x90 tiles; the sheet closest to 16:9 for four of them is 2x2 */
+ config = "thumbnailffmpeg(gallery:4,format=png,size=160)";
+ if (0 != run_gallery (config, 1, &items, &g))
+ ret = 1;
+ else if ( (! png_size (g, &w, &h)) ||
+ (320 != w) ||
+ (180 != h) )
+ {
+ fprintf (stderr,
+ "%s: gallery is %ux%u, expected 320x180\n",
+ config,
+ (unsigned int) w,
+ (unsigned int) h);
+ ret = 1;
+ }
+ else if (NULL != (rgb = TEST_media_png_decode (g->data,
+ g->data_len,
+ &w,
+ &h)))
+ {
+ for (i = 0; i < 4; i++)
+ ret |= check_tile (rgb,
+ w,
+ (i % 2) * 160,
+ (i / 2) * 90,
+ 160,
+ 90,
+ colors4[i],
+ config,
+ i);
+ free (rgb);
+ }
+#if TEST_MEDIA_HAVE_PNG
+ else
+ {
+ fprintf (stderr,
+ "%s: failed to decode the gallery\n",
+ config);
+ ret = 1;
+ }
+#endif
+ items_clear (&items);
+
+ /* a single tile, sampled at 20s: white, so it must be shifted */
+ config = "thumbnailffmpeg(gallery:1,format=png,size=160)";
+ if (0 != run_gallery (config, 1, &items, &g))
+ ret = 1;
+ else if ( (! png_size (g, &w, &h)) ||
+ (160 != w) ||
+ (90 != h) )
+ {
+ fprintf (stderr,
+ "%s: gallery is %ux%u, expected 160x90\n",
+ config,
+ (unsigned int) w,
+ (unsigned int) h);
+ ret = 1;
+ }
+ else if (NULL != (rgb = TEST_media_png_decode (g->data,
+ g->data_len,
+ &w,
+ &h)))
+ {
+ ret |= check_tile (rgb, w, 0, 0, 160, 90, TC_ANY, config, 0);
+ free (rgb);
+ }
+ items_clear (&items);
+
+ config = "thumbnailffmpeg(gallery:1)";
+ ret |= run_gallery (config, 0, &items, &g);
+ items_clear (&items);
+ config = "thumbnailffmpeg(gallery:100)";
+ ret |= run_gallery (config, 0, &items, &g);
+ items_clear (&items);
+ config = "thumbnailffmpeg(gallery:100,format=png,size=1024)";
+ ret |= run_gallery (config, 1, &items, &g);
+ items_clear (&items);
+ return ret;
+}
+
+
+/**
+ * Check the subtitle languages and the absence of optional output in
+ * a run without options.
+ *
+ * @return 0 on success
+ */
+static int
+test_default (void)
+{
+ struct Items items;
+ int ret = 0;
+
+ run_collect ("thumbnailffmpeg",
+ 0,
+ SUBTITLES_MKV,
+ &items);
+ ret |= check_languages (&items,
+ "default");
+ if (0 != count_type (&items,
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT,
+ NULL))
+ {
+ fprintf (stderr,
+ "default: got subtitles without asking for them\n");
+ ret = 1;
+ }
+ if (items.count != 4)
+ {
+ fprintf (stderr,
+ "default: got %u items, expected 4\n",
+ items.count);
+ ret = 1;
+ }
+ items_clear (&items);
+ return ret;
+}
+
+
+/**
* Main function for the thumbnailffmpeg testcase.
*
* @param argc number of arguments (ignored)
@@ -104,8 +857,27 @@ main (int argc, char *argv[])
ps_jpeg);
if (0 != ret)
return ret;
- return ET_main ("thumbnailffmpeg(format=png,size=32)",
- ps_png);
+ ret = ET_main ("thumbnailffmpeg(format=png,size=32)",
+ ps_png);
+ if (0 != ret)
+ return ret;
+ if (0 != access (SUBTITLES_MKV,
+ R_OK))
+ {
+ fprintf (stderr,
+ "Failed to access %s, skipping subtitle tests\n",
+ SUBTITLES_MKV);
+ return 0;
+ }
+ ret |= test_default ();
+ ret |= test_srt ("en", 0, srt_en);
+ ret |= test_srt ("eng", 0, srt_en);
+ ret |= test_srt ("de", 0, srt_de);
+ ret |= test_srt ("de", 1, srt_de);
+ ret |= test_srt ("fr", 0, srt_fr);
+ ret |= test_srt ("xx", 0, NULL);
+ ret |= test_gallery ();
+ return ret;
}
diff --git a/src/plugins/test_thumbnailgst.c b/src/plugins/test_thumbnailgst.c
@@ -31,12 +31,48 @@
#include "platform.h"
#include "test_lib.h"
#include "test_media_lib.h"
+#include <gst/gst.h>
#ifndef WINDOWS
#include <sys/wait.h>
#endif
/**
+ * The English subtitle track of testdata/subtitles.mkv, as SRT (see
+ * contrib/gen_testmedia.sh).
+ */
+#define SRT_EN \
+ "1\n00:00:01,000 --> 00:00:04,000\nHello, world!\n\n" \
+ "2\n00:00:12,500 --> 00:00:15,250\nThis is the second line\n" \
+ "spanning two rows.\n\n" \
+ "3\n00:00:33,000 --> 00:00:38,000\nGoodbye.\n\n"
+
+/**
+ * The German subtitle track of testdata/subtitles.mkv.
+ */
+#define SRT_DE \
+ "1\n00:00:01,000 --> 00:00:04,000\nGr\xC3\xBC\xC3\x9F Gott, Welt!\n\n" \
+ "2\n00:00:12,500 --> 00:00:15,250\nDas ist die zweite Zeile\n" \
+ "\xC3\xBC" "ber zwei Reihen.\n\n" \
+ "3\n00:00:33,000 --> 00:00:38,000\nAuf Wiedersehen.\n\n"
+
+/**
+ * The French (ASS) subtitle track of testdata/subtitles.mkv; the
+ * override tags ("{\i1}") are gone and "\N" became a line break.
+ */
+#define SRT_FR \
+ "1\n00:00:01,000 --> 00:00:04,000\nBonjour, le monde !\n\n" \
+ "2\n00:00:12,500 --> 00:00:15,250\nDeuxi\xC3\xA8me ligne\n" \
+ "sur deux rangs.\n\n"
+
+/**
+ * Tile size for the default "size=128" and the 160x90 test video.
+ */
+#define TILE_W 128
+#define TILE_H 72
+
+
+/**
* Check that the thumbnail was taken from the red part of the video.
*
* @param data the PNG the plugin produced
@@ -56,6 +92,365 @@ validate_red (const void *data,
/**
+ * Run ET_main() in a child process. Once GStreamer has been
+ * initialized in a process (which ET_main's in-process leg does), the
+ * fork() of a later out-of-process extraction inherits a GStreamer that
+ * no longer works in the child, so every round gets a fresh process.
+ *
+ * @param plugin plugin name and options
+ * @param ps problems to solve
+ * @return 0 on success
+ */
+static int
+run_round (const char *plugin,
+ struct ProblemSet *ps)
+{
+#ifndef WINDOWS
+ pid_t pid;
+ int status;
+
+ fprintf (stderr,
+ "Round `%s'\n",
+ plugin);
+ pid = fork ();
+ if (0 == pid)
+ _exit (ET_main (plugin,
+ ps));
+ if (-1 == pid)
+ return 1;
+ if (pid != waitpid (pid,
+ &status,
+ 0))
+ return 1;
+ if ( (! WIFEXITED (status)) ||
+ (0 != WEXITSTATUS (status)) )
+ {
+ fprintf (stderr,
+ "Round `%s' FAILED\n",
+ plugin);
+ return 1;
+ }
+ return 0;
+#else
+ return ET_main (plugin,
+ ps);
+#endif
+}
+
+
+/**
+ * Check (in a child process, see run_round()) whether GStreamer has
+ * all of the given elements.
+ *
+ * @param names NULL-terminated list of element factory names
+ * @return 1 if all are available
+ */
+static int
+have_elements (const char *const *names)
+{
+#ifndef WINDOWS
+ pid_t pid;
+ int status;
+
+ pid = fork ();
+ if (0 == pid)
+ {
+ unsigned int i;
+
+ gst_init (NULL, NULL);
+ for (i = 0; NULL != names[i]; i++)
+ {
+ GstElementFactory *f = gst_element_factory_find (names[i]);
+
+ if (NULL == f)
+ {
+ fprintf (stderr,
+ "GStreamer element `%s' missing, skipping\n",
+ names[i]);
+ _exit (1);
+ }
+ gst_object_unref (f);
+ }
+ _exit (0);
+ }
+ if (-1 == pid)
+ return 0;
+ if (pid != waitpid (pid,
+ &status,
+ 0))
+ return 0;
+ return (WIFEXITED (status) &&
+ (0 == WEXITSTATUS (status)));
+#else
+ (void) names;
+ return 1;
+#endif
+}
+
+
+/**
+ * What a plugin run produced.
+ */
+struct Result
+{
+ /**
+ * Number of thumbnails.
+ */
+ unsigned int thumbnails;
+
+ /**
+ * Number of SRT dumps.
+ */
+ unsigned int srts;
+
+ /**
+ * Last thumbnail.
+ */
+ char *thumbnail;
+
+ /**
+ * Size of @e thumbnail.
+ */
+ size_t thumbnail_len;
+
+ /**
+ * MIME type of @e thumbnail.
+ */
+ char mime[32];
+};
+
+
+/**
+ * Record the meta data of interest.
+ *
+ * @param cls the `struct Result`
+ * @param plugin_name name of the plugin
+ * @param type type of the meta data
+ * @param format format of the meta data
+ * @param data_mime_type MIME type of the meta data
+ * @param data the meta data
+ * @param data_len number of bytes in @a data
+ * @return 0 (continue)
+ */
+static int
+collect_cb (void *cls,
+ const char *plugin_name,
+ enum EXTRACTOR_MetaType type,
+ enum EXTRACTOR_MetaFormat format,
+ const char *data_mime_type,
+ const char *data,
+ size_t data_len)
+{
+ struct Result *r = cls;
+
+ (void) plugin_name;
+ (void) format;
+ if (EXTRACTOR_METATYPE_SUBTITLE_TEXT == type)
+ r->srts++;
+ if (EXTRACTOR_METATYPE_THUMBNAIL == type)
+ {
+ r->thumbnails++;
+ free (r->thumbnail);
+ r->thumbnail = malloc (data_len);
+ if (NULL != r->thumbnail)
+ memcpy (r->thumbnail, data, data_len);
+ r->thumbnail_len = data_len;
+ snprintf (r->mime,
+ sizeof (r->mime),
+ "%s",
+ (NULL != data_mime_type) ? data_mime_type : "");
+ }
+ return 0;
+}
+
+
+/**
+ * Run the plugin (out of process only, so that GStreamer is never
+ * initialized in this process) and record what it produced.
+ *
+ * @param config plugin name and options
+ * @param[out] r where to store the result
+ */
+static void
+collect (const char *config,
+ struct Result *r)
+{
+ struct EXTRACTOR_PluginList *pl;
+
+ memset (r, 0, sizeof (*r));
+ pl = EXTRACTOR_plugin_add_config (NULL,
+ config,
+ EXTRACTOR_OPTION_DEFAULT_POLICY);
+ if (NULL == pl)
+ return;
+ EXTRACTOR_extract (pl,
+ "testdata/subtitles.mkv",
+ NULL, 0,
+ &collect_cb,
+ r);
+ EXTRACTOR_plugin_remove_all (pl);
+}
+
+
+/**
+ * Check a gallery run: exactly one PNG of @a w x @a h pixels (0 to
+ * accept any size), no SRT, within the size limit, and none of the
+ * first @a check_tiles tiles predominantly black or white.
+ *
+ * @param config plugin name and options
+ * @param w expected width, 0 for any
+ * @param h expected height, 0 for any
+ * @param check_tiles number of tiles to inspect
+ * @param colors expected dominant channel of each inspected tile
+ * (0 red, 2 blue), NULL for no check
+ * @return 0 on success
+ */
+static int
+check_gallery (const char *config,
+ uint32_t w,
+ uint32_t h,
+ unsigned int check_tiles,
+ const unsigned int *colors)
+{
+ struct Result r;
+ uint32_t pw;
+ uint32_t ph;
+ uint8_t *img;
+ unsigned int t;
+ int ret = 0;
+
+ collect (config,
+ &r);
+ if ( (1 != r.thumbnails) ||
+ (0 != r.srts) ||
+ (0 != strcmp (r.mime, "image/png")) ||
+ (r.thumbnail_len > 4 * 1024 * 1024) )
+ {
+ fprintf (stderr,
+ "%s: got %u thumbnails (%s, %u bytes), %u SRTs\n",
+ config,
+ r.thumbnails,
+ r.mime,
+ (unsigned int) r.thumbnail_len,
+ r.srts);
+ free (r.thumbnail);
+ return 1;
+ }
+ img = TEST_media_png_decode (r.thumbnail,
+ r.thumbnail_len,
+ &pw,
+ &ph);
+ if (NULL == img)
+ {
+ fprintf (stderr,
+ "%s: cannot decode PNG\n",
+ config);
+ free (r.thumbnail);
+#if TEST_MEDIA_HAVE_PNG
+ return 1;
+#else
+ return 0;
+#endif
+ }
+ free (r.thumbnail);
+ if ( ( (0 != w) && (pw != w) ) ||
+ ( (0 != h) && (ph != h) ) )
+ {
+ fprintf (stderr,
+ "%s: gallery is %ux%u, expected %ux%u\n",
+ config,
+ (unsigned int) pw,
+ (unsigned int) ph,
+ (unsigned int) w,
+ (unsigned int) h);
+ free (img);
+ return 1;
+ }
+ for (t = 0; t < check_tiles; t++)
+ {
+ uint32_t cols = pw / TILE_W;
+ uint32_t x0 = (t % cols) * TILE_W;
+ uint32_t y0 = (t / cols) * TILE_H;
+ uint64_t sum[3] = { 0, 0, 0 };
+ uint64_t black = 0;
+ uint64_t white = 0;
+ uint64_t total = (uint64_t) TILE_W * TILE_H;
+ uint32_t x;
+ uint32_t y;
+
+ if (y0 + TILE_H > ph)
+ break;
+ for (y = y0; y < y0 + TILE_H; y++)
+ for (x = x0; x < x0 + TILE_W; x++)
+ {
+ const uint8_t *px = &img[((size_t) y * pw + x) * 3];
+
+ if ( (px[0] < 48) && (px[1] < 48) && (px[2] < 48) )
+ black++;
+ if ( (px[0] > 208) && (px[1] > 208) && (px[2] > 208) )
+ white++;
+ sum[0] += px[0];
+ sum[1] += px[1];
+ sum[2] += px[2];
+ }
+ if ( (2 * black > total) ||
+ (2 * white > total) )
+ {
+ fprintf (stderr,
+ "%s: tile %u is predominantly %s\n",
+ config,
+ t,
+ (2 * black > total) ? "black" : "white");
+ ret = 1;
+ }
+ if ( (NULL != colors) &&
+ ( (sum[colors[t]] / total < 160) ||
+ (sum[(colors[t] + 1) % 3] / total > 96) ) )
+ {
+ fprintf (stderr,
+ "%s: tile %u has mean rgb(%u,%u,%u), wrong colour\n",
+ config,
+ t,
+ (unsigned int) (sum[0] / total),
+ (unsigned int) (sum[1] / total),
+ (unsigned int) (sum[2] / total));
+ ret = 1;
+ }
+ }
+ free (img);
+ return ret;
+}
+
+
+/**
+ * Check that a run produces a thumbnail but no SRT.
+ *
+ * @param config plugin name and options
+ * @return 0 on success
+ */
+static int
+check_no_srt (const char *config)
+{
+ struct Result r;
+
+ collect (config,
+ &r);
+ free (r.thumbnail);
+ if ( (0 != r.srts) ||
+ (1 != r.thumbnails) )
+ {
+ fprintf (stderr,
+ "%s: got %u SRTs and %u thumbnails, expected 0 and 1\n",
+ config,
+ r.srts,
+ r.thumbnails);
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
* Main function for the thumbnailgst testcase.
*
* @param argc number of arguments (ignored)
@@ -65,6 +460,15 @@ validate_red (const void *data,
int
main (int argc, char *argv[])
{
+ static const char *const srt_elements[] = {
+ "appsrc", "parsebin", "matroskademux", "fakesink",
+ "playbin", "theoradec", "jpegenc", NULL
+ };
+ static const char *const gallery_elements[] = {
+ "appsrc", "appsink", "playbin", "matroskademux", "theoradec",
+ "videoconvert", "videoscale", "pngenc", "fakesink", NULL
+ };
+ static const unsigned int colors4[] = { 0, 0, 2, 2 };
uint8_t jpeg_magic[] = { 0xFF, 0xD8, 0xFF };
uint8_t png_magic[] = { 137, 'P', 'N', 'G', '\r', '\n', 26, '\n' };
struct SolutionData thumbnail_jpeg_sol[] = {
@@ -91,6 +495,33 @@ main (int argc, char *argv[])
},
{ 0, 0, NULL, NULL, 0, -1 }
};
+#define SRT_SOL(name, text) \
+ struct SolutionData name[] = { \
+ { \
+ EXTRACTOR_METATYPE_THUMBNAIL, \
+ EXTRACTOR_METAFORMAT_BINARY, \
+ "image/jpeg", \
+ (const char *) jpeg_magic, \
+ sizeof (jpeg_magic), \
+ 0 \
+ }, \
+ { \
+ EXTRACTOR_METATYPE_SUBTITLE_TEXT, \
+ EXTRACTOR_METAFORMAT_UTF8, \
+ "application/x-subrip", \
+ text, \
+ sizeof (text), \
+ 0 \
+ }, \
+ { 0, 0, NULL, NULL, 0, -1 } \
+ }
+
+
+ SRT_SOL (srt_en_sol, SRT_EN);
+ SRT_SOL (srt_eng_sol, SRT_EN);
+ SRT_SOL (srt_de_sol, SRT_DE);
+ SRT_SOL (srt_fr_sol, SRT_FR);
+#undef SRT_SOL
struct ProblemSet ps_jpeg[] = {
{ "testdata/thumbnail_pattern.ogv",
thumbnail_jpeg_sol },
@@ -101,40 +532,77 @@ main (int argc, char *argv[])
thumbnail_png_sol },
{ NULL, NULL }
};
-#ifndef WINDOWS
- pid_t pid;
- int status;
+ struct ProblemSet ps_srt_en[] = {
+ { "testdata/subtitles.mkv", srt_en_sol },
+ { NULL, NULL }
+ };
+ struct ProblemSet ps_srt_eng[] = {
+ { "testdata/subtitles.mkv", srt_eng_sol },
+ { NULL, NULL }
+ };
+ struct ProblemSet ps_srt_de[] = {
+ { "testdata/subtitles.mkv", srt_de_sol },
+ { NULL, NULL }
+ };
+ struct ProblemSet ps_srt_fr[] = {
+ { "testdata/subtitles.mkv", srt_fr_sol },
+ { NULL, NULL }
+ };
+ int ret = 0;
- /* The two rounds must not share a process: ET_main() runs each round
- out-of-process and then in-process, and once GStreamer has been
- initialized in this process (by the in-process leg of the first
- round), the fork() that the out-of-process leg of the second round
- performs inherits a GStreamer that no longer works in the child.
- So run the first round in a child of our own, before any of that
- has happened. */
- pid = fork ();
- if (0 == pid)
- _exit (ET_main ("thumbnailgst",
- ps_jpeg));
- if (-1 == pid)
- return 77; /* cannot fork, skip */
- if (pid != waitpid (pid,
- &status,
- 0))
+ (void) argc;
+ (void) argv;
+ /* collect() does not go through ET_main(), which would set this */
+ if (0 != putenv ("LIBEXTRACTOR_PREFIX=./.libs/"))
return 1;
- if ( (! WIFEXITED (status)) ||
- (0 != WEXITSTATUS (status)) )
- return 1;
-#else
- int ret;
-
- ret = ET_main ("thumbnailgst",
- ps_jpeg);
- if (0 != ret)
- return ret;
-#endif
- return ET_main ("thumbnailgst(format=png,size=32)",
- ps_png);
+ /* None of the rounds may initialize GStreamer in this process; see
+ run_round(). */
+ ret |= run_round ("thumbnailgst",
+ ps_jpeg);
+ ret |= run_round ("thumbnailgst(format=png,size=32)",
+ ps_png);
+ if (have_elements (srt_elements))
+ {
+ ret |= check_no_srt ("thumbnailgst");
+ ret |= run_round ("thumbnailgst(srt:en)",
+ ps_srt_en);
+ ret |= run_round ("thumbnailgst(srt:eng)",
+ ps_srt_eng);
+ ret |= run_round ("thumbnailgst(srt:de)",
+ ps_srt_de);
+ ret |= run_round ("thumbnailgst(srt:fr)",
+ ps_srt_fr);
+ ret |= check_no_srt ("thumbnailgst(srt:xx)");
+ }
+ if (have_elements (gallery_elements))
+ {
+ /* 4 tiles of 128x72 are closest to 16:9 as 2x2; tiles 0 and 2 are
+ sampled at 5s (black) and 25s (white) and must have been moved
+ into the red and blue stretches */
+ ret |= check_gallery ("thumbnailgst(gallery:4,format=png)",
+ 2 * TILE_W,
+ 2 * TILE_H,
+ 4,
+ colors4);
+ ret |= check_gallery ("thumbnailgst(gallery:1,format=png)",
+ TILE_W,
+ TILE_H,
+ 1,
+ NULL);
+ /* 0.4s slices cannot escape the black and white stretches, so
+ only the limits are checked */
+ ret |= check_gallery ("thumbnailgst(gallery:100,format=png)",
+ 10 * TILE_W,
+ 10 * TILE_H,
+ 0,
+ NULL);
+ ret |= check_gallery ("thumbnailgst(size=32,gallery:100,format=png)",
+ 0,
+ 0,
+ 0,
+ NULL);
+ }
+ return ret;
}
diff --git a/src/plugins/testdata/README.media b/src/plugins/testdata/README.media
@@ -20,6 +20,9 @@ Theora and Vorbis in an Ogg container were chosen because both plugin
backends can decode them without any additional codec package: FFmpeg has
both built in, and for GStreamer they live in gst-plugins-base.
+ subtitles.mkv 40s, Theora, 160x90, 5fps, subtitles eng/ger
+ (SubRip) and fre (ASS), flat black/white
+ stretches for the gallery (10K)
thumbnail_pattern.ogv 90s, Theora, 160x120, 5fps, no audio (14K)
Solid black for the first 20 seconds, then solid red. The
diff --git a/src/plugins/testdata/dvi_fulltext.dvi b/src/plugins/testdata/dvi_fulltext.dvi
Binary files differ.
diff --git a/src/plugins/testdata/ebook_fulltext.epub b/src/plugins/testdata/ebook_fulltext.epub
Binary files differ.
diff --git a/src/plugins/testdata/ebook_fulltext.mobi b/src/plugins/testdata/ebook_fulltext.mobi
Binary files differ.
diff --git a/src/plugins/testdata/html_fulltext.html b/src/plugins/testdata/html_fulltext.html
@@ -0,0 +1,37 @@
+<!DOCTYPE html>
+<html lang="de">
+<head>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<title>Käse und Brot</title>
+<meta name="author" content="Christian Grothoff">
+<style>p { margin: 0; } nav { display: block; }</style>
+<script>document.write("<p>script text</p>");</script>
+</head>
+<body>
+<header><a href="/">Site banner</a></header>
+<nav><ul><li><a href="/">Home</a></li><li><a href="/a">About</a></li></ul></nav>
+<div role="navigation">Breadcrumbs > here</div>
+<main>
+<article>
+<header><h1>Der Käse & das Brot</h1></header>
+<p>Hello <b>world</b>, again here:
+ été — done.</p>
+<p>Line one<br>line two<br><br>New paragraph after two breaks.</p>
+<div hidden>Hidden by attribute</div>
+<span aria-hidden="true">Decorative icon</span>
+<div style="display: none">Hidden by style</div>
+<ul><li>Item A</li><li>Item <i>B</i></li></ul>
+<table><tr><th>Cell 1</th><td>Cell 2</td></tr></table>
+<noscript>Please enable JavaScript</noscript>
+<template><p>Template text</p></template>
+<blockquote>Quoted text</blockquote>
+<p>Soft­hyphen and <span>in</span>line elements.</p>
+<footer>Article byline</footer>
+</article>
+<aside>Sidebar text</aside>
+</main>
+<div role="contentinfo">Legal notice</div>
+<footer>Copyright 2026</footer>
+<script>var x = "</div>more script";</script>
+</body>
+</html>
diff --git a/src/plugins/testdata/odf_fulltext.odp b/src/plugins/testdata/odf_fulltext.odp
Binary files differ.
diff --git a/src/plugins/testdata/odf_fulltext.ods b/src/plugins/testdata/odf_fulltext.ods
Binary files differ.
diff --git a/src/plugins/testdata/odf_fulltext.odt b/src/plugins/testdata/odf_fulltext.odt
Binary files differ.
diff --git a/src/plugins/testdata/ole2_fulltext.doc b/src/plugins/testdata/ole2_fulltext.doc
Binary files differ.
diff --git a/src/plugins/testdata/pdf_fulltext.pdf b/src/plugins/testdata/pdf_fulltext.pdf
Binary files differ.
diff --git a/src/plugins/testdata/subtitles.mkv b/src/plugins/testdata/subtitles.mkv
Binary files differ.
diff --git a/src/plugins/thumbnailffmpeg_extractor.c b/src/plugins/thumbnailffmpeg_extractor.c
@@ -31,6 +31,23 @@
* every libav and FFmpeg API since 2008; this one deliberately requires
* FFmpeg 5.1 or newer and uses only the send/receive APIs, so that there
* is nothing to keep in sync with upstream churn.
+ *
+ * Besides the thumbnail, the plugin always reports the language of every
+ * subtitle stream. Two options add more expensive output:
+ *
+ * - "srt:LANG" dumps the first text subtitle stream in language LANG as
+ * a complete SubRip file. This demuxes the entire file (without
+ * decoding the video), and is done first, on a fresh demuxer.
+ * - "gallery:N" replaces the single thumbnail by a contact sheet of N
+ * frames sampled evenly over the whole duration. Frames that show
+ * nothing (black, white, a fade) are replaced by a nearby frame of the
+ * same slice of the video. If the deadline expires while the sheet is
+ * being assembled, the sheet is emitted with the tiles obtained so far
+ * and black cells for the rest: a partial overview is more useful than
+ * a single frame, and trying to get a single frame once the budget is
+ * used up would not work anyway. Without a known duration there is
+ * nothing to spread the samples over, and the plugin falls back to the
+ * single thumbnail.
*/
#include "platform.h"
#include "extractor.h"
@@ -67,6 +84,53 @@
*/
#define JPEG_QSCALE 5
+/**
+ * Give up on reaching a gallery sample after decoding this many frames
+ * past the keyframe we seeked to.
+ */
+#define GALLERY_MAX_FRAMES 512
+
+/**
+ * Largest contact sheet we assemble, in pixels. Far below
+ * #EXTRACTOR_MEDIA_MAX_PIXELS, because the sheet has to be encoded in
+ * one go, without any I/O that would tell the parent process we are
+ * still alive, and a sheet this size already cannot be squeezed into
+ * #EXTRACTOR_MEDIA_MAX_GALLERY_BYTES as a PNG unless it is very simple.
+ */
+#define GALLERY_MAX_SHEET_PIXELS EXTRACTOR_MEDIA_MAX_GALLERY_PIXELS
+
+/**
+ * zlib level for PNG galleries; 9 costs a lot of time on a big sheet
+ * for a few percent.
+ */
+#define GALLERY_PNG_LEVEL 6
+
+/**
+ * Most distinct subtitle languages we report.
+ */
+#define MAX_LANGUAGES 64
+
+/**
+ * Longest language tag we report (e.g. "zh-Hant-TW").
+ */
+#define MAX_LANGUAGE_LEN 15
+
+/**
+ * Most subtitle cues we collect for an SRT dump.
+ */
+#define MAX_CUES 100000
+
+/**
+ * How long a cue whose end is unknown and that has no successor stays
+ * on screen, in milliseconds.
+ */
+#define DEFAULT_CUE_MS 5000
+
+/**
+ * Cap for any time stamp in an SRT dump (1000 hours), in milliseconds.
+ */
+#define MAX_CUE_TIME_MS (1000ULL * 3600 * 1000)
+
/**
* Is @a frame a flat surface? Cheap heuristic over the first plane; we
@@ -111,19 +175,30 @@ is_flat (const AVFrame *frame)
/**
- * Scale @a frame down and encode it as JPEG or PNG.
+ * Scale @a frame to @a width x @a height and encode it as JPEG or PNG.
*
- * @param frame the decoded frame to convert
- * @param opt plugin options (target size and format)
+ * @param frame the image to convert (any pixel format swscale accepts)
+ * @param width width of the result
+ * @param height height of the result
+ * @param png 1 for PNG, 0 for JPEG
+ * @param qscale JPEG quantizer scale (ignored for PNG)
+ * @param level zlib level for PNG (ignored for JPEG)
+ * @param max_bytes largest acceptable result
* @param[out] output set to the encoded image, to be freed with av_free()
* @param[out] output_size set to the number of bytes in @a output
- * @return 0 on success
+ * @return 0 on success, -2 if the result would exceed @a max_bytes,
+ * -1 on other errors
*/
static int
-encode_thumbnail (const AVFrame *frame,
- const struct EXTRACTOR_MediaOptions *opt,
- uint8_t **output,
- size_t *output_size)
+encode_image (const AVFrame *frame,
+ unsigned int width,
+ unsigned int height,
+ int png,
+ int qscale,
+ int level,
+ size_t max_bytes,
+ uint8_t **output,
+ size_t *output_size)
{
const AVCodec *codec;
AVCodecContext *ctx = NULL;
@@ -131,20 +206,17 @@ encode_thumbnail (const AVFrame *frame,
AVFrame *dst = NULL;
AVPacket *pkt = NULL;
enum AVPixelFormat dst_fmt;
- unsigned int width;
- unsigned int height;
int ret = -1;
int r;
- if (! EXTRACTOR_media_thumbnail_size_ (frame->width,
- frame->height,
- opt->size,
- &width,
- &height))
+ if ( (0 == width) ||
+ (0 == height) ||
+ (width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
+ (height > EXTRACTOR_MEDIA_MAX_DIMENSION) )
return -1;
if (! sws_isSupportedInput (frame->format))
return -1;
- if (opt->png)
+ if (png)
{
dst_fmt = AV_PIX_FMT_RGB24;
codec = avcodec_find_encoder (AV_CODEC_ID_PNG);
@@ -190,15 +262,15 @@ encode_thumbnail (const AVFrame *frame,
ctx->pix_fmt = dst_fmt;
ctx->time_base = (AVRational) { 1, 25 };
ctx->thread_count = 1;
- if (opt->png)
+ if (png)
{
- ctx->compression_level = 9;
+ ctx->compression_level = level;
}
else
{
ctx->color_range = AVCOL_RANGE_JPEG;
ctx->flags |= AV_CODEC_FLAG_QSCALE;
- ctx->global_quality = FF_QP2LAMBDA * JPEG_QSCALE;
+ ctx->global_quality = FF_QP2LAMBDA * qscale;
}
if (0 != avcodec_open2 (ctx, codec, NULL))
goto cleanup;
@@ -217,9 +289,13 @@ encode_thumbnail (const AVFrame *frame,
}
if (0 != r)
goto cleanup;
- if ( (pkt->size <= 0) ||
- (pkt->size > EXTRACTOR_MEDIA_MAX_THUMBNAIL) )
+ if (pkt->size <= 0)
goto cleanup;
+ if ((size_t) pkt->size > max_bytes)
+ {
+ ret = -2;
+ goto cleanup;
+ }
if (NULL == (*output = av_malloc (pkt->size)))
goto cleanup;
memcpy (*output,
@@ -240,14 +316,59 @@ cleanup:
/**
+ * Scale @a frame down and encode it as JPEG or PNG.
+ *
+ * @param frame the decoded frame to convert
+ * @param opt plugin options (target size and format)
+ * @param[out] output set to the encoded image, to be freed with av_free()
+ * @param[out] output_size set to the number of bytes in @a output
+ * @return 0 on success
+ */
+static int
+encode_thumbnail (const AVFrame *frame,
+ const struct EXTRACTOR_MediaOptions *opt,
+ uint8_t **output,
+ size_t *output_size)
+{
+ unsigned int width;
+ unsigned int height;
+
+ if (! EXTRACTOR_media_thumbnail_size_ (frame->width,
+ frame->height,
+ opt->size,
+ &width,
+ &height))
+ return -1;
+ return (0 == encode_image (frame,
+ width,
+ height,
+ opt->png,
+ JPEG_QSCALE,
+ 9,
+ EXTRACTOR_MEDIA_MAX_THUMBNAIL,
+ output,
+ output_size)) ? 0 : -1;
+}
+
+
+/**
* Decode the video stream until we reach @a target seconds, and hand back
* the frame we find there. If we never get there (short file, broken
* timestamps), the last frame we did decode is returned instead.
*
+ * Normally "there" is the first frame at or after @a target. With
+ * @a displayed, it is the frame on screen at @a target, i.e. the last
+ * one starting at or before it; the difference matters for streams
+ * that only carry a new frame when the picture changes.
+ *
* @param io open I/O state
* @param ctx open decoder for @a stream
* @param stream index of the video stream
* @param target position we are aiming for, in seconds
+ * @param skip_flat 1 to move on (a few times) if the frame at @a target
+ * is a flat surface
+ * @param displayed 1 to return the frame displayed at @a target
+ * @param max_frames give up after decoding this many frames
* @param[out] frame frame to fill in
* @return 0 if @a frame was filled in
*/
@@ -256,6 +377,9 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
AVCodecContext *ctx,
int stream,
double target,
+ int skip_flat,
+ int displayed,
+ unsigned int max_frames,
AVFrame *frame)
{
AVPacket *pkt;
@@ -264,6 +388,7 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
unsigned int frames = 0;
unsigned int retries = 0;
int have_last = 0;
+ int overshot = 0;
int ret = -1;
if (NULL == (pkt = av_packet_alloc ()))
@@ -273,12 +398,16 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
av_packet_free (&pkt);
return -1;
}
- while (frames < MAX_FRAMES)
+ while (frames < max_frames)
{
int r;
if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
break;
+ /* decoding can go on for a while without touching the file */
+ (void) io->ec->seek (io->ec->cls,
+ 0,
+ SEEK_CUR);
r = av_read_frame (io->fmt, pkt);
if (0 != r)
{
@@ -305,7 +434,7 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
? -1.0
: frame->best_effort_timestamp * av_q2d (tb);
if ( (ts < target) &&
- (frames < MAX_FRAMES) )
+ (frames < max_frames) )
{
/* not there yet; hold on to it in case we never arrive */
av_frame_unref (last);
@@ -314,8 +443,18 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
av_frame_unref (frame);
continue;
}
- if ( (retries < MAX_FLAT_RETRIES) &&
- is_flat (frame) )
+ if (displayed &&
+ have_last &&
+ (ts > target) )
+ {
+ /* overshot: the previous frame is still on screen at target */
+ av_frame_unref (frame);
+ overshot = 1;
+ break;
+ }
+ if (skip_flat &&
+ (retries < MAX_FLAT_RETRIES) &&
+ is_flat (frame) )
{
/* a black leader or a fade-in; try a bit further in */
retries++;
@@ -331,6 +470,8 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
av_packet_free (&pkt);
return 0;
}
+ if (overshot)
+ break;
if (0 != r)
break; /* end of file, and the decoder had nothing left either */
}
@@ -348,6 +489,994 @@ decode_frame_at (struct EXTRACTOR_FFmpegIO *io,
/**
+ * Is @a tag something we are willing to report as a language? Stream
+ * metadata comes straight from the file, so only accept what a language
+ * tag can look like (letters, digits, '-' and '_'), and skip the
+ * "undetermined" placeholder.
+ *
+ * @param tag language tag of a stream, may be NULL
+ * @return 1 if @a tag should be reported
+ */
+static int
+is_reportable_language (const char *tag)
+{
+ size_t len;
+ size_t i;
+
+ if (NULL == tag)
+ return 0;
+ len = strlen (tag);
+ if ( (0 == len) ||
+ (len > MAX_LANGUAGE_LEN) )
+ return 0;
+ for (i = 0; i < len; i++)
+ if (! ( ( (tag[i] >= 'a') && (tag[i] <= 'z') ) ||
+ ( (tag[i] >= 'A') && (tag[i] <= 'Z') ) ||
+ ( (tag[i] >= '0') && (tag[i] <= '9') ) ||
+ ('-' == tag[i]) ||
+ ('_' == tag[i]) ) )
+ return 0;
+ if (0 == strcasecmp (tag,
+ "und"))
+ return 0;
+ return 1;
+}
+
+
+/**
+ * Report the language of every subtitle stream, each distinct language
+ * once, in stream order.
+ *
+ * @param io open I/O state
+ * @return 0 to continue, 1 if the application asked us to stop
+ */
+static int
+emit_subtitle_languages (struct EXTRACTOR_FFmpegIO *io)
+{
+ const char *seen[MAX_LANGUAGES];
+ unsigned int nseen = 0;
+ unsigned int i;
+
+ for (i = 0; i < io->fmt->nb_streams; i++)
+ {
+ const AVStream *st = io->fmt->streams[i];
+ const AVDictionaryEntry *e;
+ unsigned int j;
+
+ if (AVMEDIA_TYPE_SUBTITLE != st->codecpar->codec_type)
+ continue;
+ e = av_dict_get (st->metadata,
+ "language",
+ NULL,
+ 0);
+ if ( (NULL == e) ||
+ (! is_reportable_language (e->value)) )
+ continue;
+ for (j = 0; j < nseen; j++)
+ if (0 == strcasecmp (seen[j],
+ e->value))
+ break;
+ if (j < nseen)
+ continue;
+ if (nseen == MAX_LANGUAGES)
+ break;
+ seen[nseen++] = e->value;
+ if (0 != io->ec->proc (io->ec->cls,
+ "thumbnailffmpeg",
+ EXTRACTOR_METATYPE_SUBTITLE_LANGUAGE,
+ EXTRACTOR_METAFORMAT_UTF8,
+ "text/plain",
+ e->value,
+ strlen (e->value) + 1))
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * One subtitle event, collected before we can write the SRT file.
+ */
+struct Cue
+{
+ /**
+ * Start in milliseconds from the beginning of the file.
+ */
+ uint64_t start;
+
+ /**
+ * End in milliseconds, UINT64_MAX if unknown.
+ */
+ uint64_t end;
+
+ /**
+ * Text of the cue (not 0-terminated).
+ */
+ char *text;
+
+ /**
+ * Number of bytes in @e text.
+ */
+ size_t len;
+
+ /**
+ * Position in the stream, to keep the sort stable.
+ */
+ unsigned int seq;
+
+ /**
+ * 1 if @e text is an ASS dialogue event.
+ */
+ int is_ass;
+};
+
+
+/**
+ * The cues of a subtitle stream.
+ */
+struct CueList
+{
+ /**
+ * Array of cues.
+ */
+ struct Cue *cues;
+
+ /**
+ * Number of valid entries in @e cues.
+ */
+ unsigned int count;
+
+ /**
+ * Allocated length of @e cues.
+ */
+ unsigned int cap;
+
+ /**
+ * Total number of text bytes in @e cues.
+ */
+ size_t bytes;
+
+ /**
+ * Set once we ran into one of the limits.
+ */
+ int full;
+};
+
+
+/**
+ * Append a cue to @a cl.
+ *
+ * @param[in,out] cl list to extend
+ * @param start start in milliseconds
+ * @param end end in milliseconds, UINT64_MAX if unknown
+ * @param text text of the cue
+ * @param is_ass 1 if @a text is an ASS dialogue event
+ */
+static void
+cue_add (struct CueList *cl,
+ uint64_t start,
+ uint64_t end,
+ const char *text,
+ int is_ass)
+{
+ struct Cue *c;
+ size_t len;
+
+ if ( (NULL == text) ||
+ (cl->full) )
+ return;
+ len = strlen (text);
+ if (0 == len)
+ return;
+ if ( (cl->count >= MAX_CUES) ||
+ (cl->bytes + len > EXTRACTOR_MEDIA_MAX_SRT) )
+ {
+ cl->full = 1;
+ return;
+ }
+ if (cl->count == cl->cap)
+ {
+ unsigned int ncap = (0 == cl->cap) ? 64 : 2 * cl->cap;
+ struct Cue *n;
+
+ if (ncap > MAX_CUES)
+ ncap = MAX_CUES;
+ n = realloc (cl->cues,
+ ncap * sizeof (struct Cue));
+ if (NULL == n)
+ {
+ cl->full = 1;
+ return;
+ }
+ cl->cues = n;
+ cl->cap = ncap;
+ }
+ c = &cl->cues[cl->count];
+ if (NULL == (c->text = malloc (len)))
+ {
+ cl->full = 1;
+ return;
+ }
+ memcpy (c->text,
+ text,
+ len);
+ c->len = len;
+ c->start = start;
+ c->end = end;
+ c->is_ass = is_ass;
+ c->seq = cl->count;
+ cl->bytes += len;
+ cl->count++;
+}
+
+
+/**
+ * Order cues by start time, then by position in the stream.
+ *
+ * @param a first `struct Cue`
+ * @param b second `struct Cue`
+ * @return -1, 0 or 1 as required by qsort()
+ */
+static int
+cue_cmp (const void *a,
+ const void *b)
+{
+ const struct Cue *ca = a;
+ const struct Cue *cb = b;
+
+ if (ca->start != cb->start)
+ return (ca->start < cb->start) ? -1 : 1;
+ if (ca->seq != cb->seq)
+ return (ca->seq < cb->seq) ? -1 : 1;
+ return 0;
+}
+
+
+/**
+ * Release @a cl.
+ *
+ * @param[in,out] cl list to release
+ */
+static void
+cue_clear (struct CueList *cl)
+{
+ unsigned int i;
+
+ for (i = 0; i < cl->count; i++)
+ free (cl->cues[i].text);
+ free (cl->cues);
+ memset (cl, 0, sizeof (*cl));
+}
+
+
+/**
+ * Convert @a ts in @a tb to milliseconds relative to @a origin_ms,
+ * clamped to [0, #MAX_CUE_TIME_MS].
+ *
+ * @param ts time stamp
+ * @param tb time base of @a ts
+ * @param origin_ms start of the file in milliseconds
+ * @return time in milliseconds
+ */
+static uint64_t
+to_cue_ms (int64_t ts,
+ AVRational tb,
+ int64_t origin_ms)
+{
+ int64_t ms;
+
+ if ( (tb.num <= 0) ||
+ (tb.den <= 0) )
+ return 0;
+ ms = av_rescale_q (ts,
+ tb,
+ (AVRational) { 1, 1000 });
+ if ( (ms <= 0) ||
+ (ms <= origin_ms) )
+ return 0;
+ if (origin_ms > 0)
+ ms -= origin_ms;
+ if ((uint64_t) ms > MAX_CUE_TIME_MS)
+ return MAX_CUE_TIME_MS;
+ return (uint64_t) ms;
+}
+
+
+/**
+ * Decode one subtitle packet and collect its cues.
+ *
+ * @param ctx open subtitle decoder
+ * @param pkt packet of the subtitle stream
+ * @param tb time base of the stream
+ * @param origin_ms start of the file in milliseconds
+ * @param[in,out] cl where to put the cues
+ */
+static void
+decode_subtitle_packet (AVCodecContext *ctx,
+ AVPacket *pkt,
+ AVRational tb,
+ int64_t origin_ms,
+ struct CueList *cl)
+{
+ AVSubtitle sub;
+ int got = 0;
+ int64_t pts;
+ uint64_t base;
+ uint64_t start;
+ uint64_t end;
+ unsigned int i;
+
+ pts = (AV_NOPTS_VALUE != pkt->pts) ? pkt->pts : pkt->dts;
+ if (AV_NOPTS_VALUE == pts)
+ return; /* no idea when to show it */
+ memset (&sub, 0, sizeof (sub));
+ if ( (0 > avcodec_decode_subtitle2 (ctx,
+ &sub,
+ &got,
+ pkt)) ||
+ (! got) )
+ return;
+ base = to_cue_ms (pts,
+ tb,
+ origin_ms);
+ start = base + sub.start_display_time;
+ if ( (sub.end_display_time > sub.start_display_time) &&
+ (UINT32_MAX != sub.end_display_time) )
+ end = base + sub.end_display_time;
+ else if (pkt->duration > 0)
+ end = start + to_cue_ms (pkt->duration,
+ tb,
+ 0);
+ else
+ end = UINT64_MAX;
+ for (i = 0; i < sub.num_rects; i++)
+ {
+ const AVSubtitleRect *r = sub.rects[i];
+
+ if (NULL == r)
+ continue;
+ if (SUBTITLE_ASS == r->type)
+ cue_add (cl, start, end, r->ass, 1);
+ else if (SUBTITLE_TEXT == r->type)
+ cue_add (cl, start, end, r->text, 0);
+ }
+ avsubtitle_free (&sub);
+}
+
+
+/**
+ * Find the first subtitle stream in language @a lang.
+ *
+ * @param io open I/O state
+ * @param lang requested language
+ * @return stream index, -1 if there is none
+ */
+static int
+find_subtitle_stream (struct EXTRACTOR_FFmpegIO *io,
+ const char *lang)
+{
+ unsigned int i;
+
+ for (i = 0; i < io->fmt->nb_streams; i++)
+ {
+ const AVStream *st = io->fmt->streams[i];
+ const AVDictionaryEntry *e;
+
+ if (AVMEDIA_TYPE_SUBTITLE != st->codecpar->codec_type)
+ continue;
+ e = av_dict_get (st->metadata,
+ "language",
+ NULL,
+ 0);
+ if ( (NULL != e) &&
+ EXTRACTOR_media_lang_matches_ (lang,
+ e->value) )
+ return (int) i;
+ }
+ return -1;
+}
+
+
+/**
+ * Dump the first subtitle stream in the language requested with
+ * "srt:LANG" as a SubRip file. Must be called on a freshly opened
+ * demuxer, as it reads the file from where the demuxer stands.
+ *
+ * @param io open I/O state
+ * @param opt plugin options
+ * @param[out] touched set to 1 if we read (and discarded) packets, so
+ * that the caller has to reopen the input
+ * @return 0 to continue, 1 if the application asked us to stop
+ */
+static int
+dump_srt (struct EXTRACTOR_FFmpegIO *io,
+ const struct EXTRACTOR_MediaOptions *opt,
+ int *touched)
+{
+ const AVCodecDescriptor *desc;
+ const AVCodec *codec;
+ AVCodecContext *ctx = NULL;
+ AVPacket *pkt = NULL;
+ AVStream *st;
+ struct CueList cl;
+ struct EXTRACTOR_MediaSrt srt;
+ int64_t origin_ms = 0;
+ uint64_t next_start;
+ unsigned int i;
+ int stream;
+ int ret = 0;
+
+ *touched = 0;
+ memset (&cl, 0, sizeof (cl));
+ memset (&srt, 0, sizeof (srt));
+ stream = find_subtitle_stream (io,
+ opt->srt_lang);
+ if (0 > stream)
+ return 0;
+ st = io->fmt->streams[stream];
+ desc = avcodec_descriptor_get (st->codecpar->codec_id);
+ if ( (NULL == desc) ||
+ (0 == (desc->props & AV_CODEC_PROP_TEXT_SUB)) )
+ return 0; /* bitmap subtitles; nothing we could write as text */
+ if (NULL == (codec = avcodec_find_decoder (st->codecpar->codec_id)))
+ return 0;
+ if (NULL == (ctx = avcodec_alloc_context3 (codec)))
+ return 0;
+ if (0 > avcodec_parameters_to_context (ctx,
+ st->codecpar))
+ goto cleanup;
+ ctx->pkt_timebase = st->time_base;
+ ctx->thread_count = 1;
+ if (0 != avcodec_open2 (ctx, codec, NULL))
+ goto cleanup;
+ if (NULL == (pkt = av_packet_alloc ()))
+ goto cleanup;
+ if (AV_NOPTS_VALUE != io->fmt->start_time)
+ origin_ms = io->fmt->start_time / 1000;
+ /* let the demuxer skip everything else */
+ for (i = 0; i < io->fmt->nb_streams; i++)
+ if ((int) i != stream)
+ io->fmt->streams[i]->discard = AVDISCARD_ALL;
+ *touched = 1;
+ while ( (! cl.full) &&
+ (! EXTRACTOR_media_deadline_expired_ (&io->deadline)) )
+ {
+ (void) io->ec->seek (io->ec->cls,
+ 0,
+ SEEK_CUR); /* keep-alive */
+ if (0 != av_read_frame (io->fmt,
+ pkt))
+ break;
+ if (pkt->stream_index == stream)
+ decode_subtitle_packet (ctx,
+ pkt,
+ st->time_base,
+ origin_ms,
+ &cl);
+ av_packet_unref (pkt);
+ }
+ if (0 == cl.count)
+ goto cleanup;
+ /* containers normally deliver subtitles in order, but nothing forces
+ them to; and the ends of open-ended cues depend on the order */
+ qsort (cl.cues,
+ cl.count,
+ sizeof (struct Cue),
+ &cue_cmp);
+ next_start = UINT64_MAX;
+ for (i = cl.count; i > 0; i--)
+ {
+ struct Cue *c = &cl.cues[i - 1];
+
+ if (UINT64_MAX == c->end)
+ c->end = (UINT64_MAX != next_start)
+ ? next_start
+ : c->start + DEFAULT_CUE_MS;
+ if (c->end < c->start)
+ c->end = c->start;
+ if ( (i > 1) &&
+ (cl.cues[i - 2].start < c->start) )
+ next_start = c->start;
+ }
+ for (i = 0; i < cl.count; i++)
+ EXTRACTOR_media_srt_add_ (&srt,
+ cl.cues[i].start,
+ cl.cues[i].end,
+ cl.cues[i].text,
+ cl.cues[i].len,
+ cl.cues[i].is_ass);
+ if (0 != EXTRACTOR_media_srt_emit_ (&srt,
+ io->ec,
+ "thumbnailffmpeg"))
+ ret = 1;
+cleanup:
+ EXTRACTOR_media_srt_clear_ (&srt);
+ cue_clear (&cl);
+ if (NULL != pkt)
+ av_packet_free (&pkt);
+ avcodec_free_context (&ctx);
+ return ret;
+}
+
+
+/**
+ * A contact sheet under construction.
+ */
+struct Gallery
+{
+ /**
+ * The sheet, packed RGB24, initially black.
+ */
+ uint8_t *sheet;
+
+ /**
+ * Scratch buffer for one tile, packed RGB24.
+ */
+ uint8_t *tile;
+
+ /**
+ * The least flat candidate for the current tile, packed RGB24.
+ */
+ uint8_t *best;
+
+ /**
+ * Scaler from decoded frames to tiles.
+ */
+ struct SwsContext *sws;
+
+ /**
+ * Width of a tile.
+ */
+ unsigned int tw;
+
+ /**
+ * Height of a tile.
+ */
+ unsigned int th;
+
+ /**
+ * Number of columns.
+ */
+ unsigned int cols;
+
+ /**
+ * Number of rows.
+ */
+ unsigned int rows;
+};
+
+
+/**
+ * How much is going on in a tile: the sum of the variances of its
+ * three channels. Used to pick the least boring of several flat
+ * candidates.
+ *
+ * @param rgb packed RGB24 pixels
+ * @param width width in pixels
+ * @param height height in pixels
+ * @return the score, larger means more detail
+ */
+static double
+tile_score (const uint8_t *rgb,
+ unsigned int width,
+ unsigned int height)
+{
+ double sum[3] = { 0, 0, 0 };
+ double sq[3] = { 0, 0, 0 };
+ double n = 0;
+ double score = 0;
+ unsigned int x;
+ unsigned int y;
+ unsigned int c;
+
+ for (y = 0; y < height; y += 2)
+ for (x = 0; x < width; x += 2)
+ {
+ const uint8_t *p = &rgb[((size_t) y * width + x) * 3];
+
+ for (c = 0; c < 3; c++)
+ {
+ sum[c] += p[c];
+ sq[c] += (double) p[c] * p[c];
+ }
+ n++;
+ }
+ if (0 == n)
+ return 0;
+ for (c = 0; c < 3; c++)
+ score += sq[c] / n - (sum[c] / n) * (sum[c] / n);
+ return score;
+}
+
+
+/**
+ * Seek to @a t_ms into the video, decode the frame there and scale it
+ * into the tile buffer of @a g.
+ *
+ * @param io open I/O state
+ * @param ctx open video decoder
+ * @param stream index of the video stream
+ * @param start_ts start of the stream in its time base
+ * @param t_ms position relative to @a start_ts in milliseconds
+ * @param frame scratch frame
+ * @param[in,out] g gallery whose tile buffer to fill
+ * @return 0 on success
+ */
+static int
+grab_tile (struct EXTRACTOR_FFmpegIO *io,
+ AVCodecContext *ctx,
+ int stream,
+ int64_t start_ts,
+ uint64_t t_ms,
+ AVFrame *frame,
+ struct Gallery *g)
+{
+ AVRational tb = io->fmt->streams[stream]->time_base;
+ uint8_t *dst[4] = { g->tile, NULL, NULL, NULL };
+ int dst_stride[4] = { (int) g->tw * 3, 0, 0, 0 };
+ int64_t ts;
+ int ret = -1;
+
+ if ( (tb.num <= 0) ||
+ (tb.den <= 0) )
+ return -1;
+ ts = av_rescale_q ((int64_t) t_ms,
+ (AVRational) { 1, 1000 },
+ tb);
+ if ( (ts < 0) ||
+ ( (start_ts > 0) &&
+ (ts > INT64_MAX - start_ts) ) )
+ return -1;
+ ts += start_ts;
+ if (0 > av_seek_frame (io->fmt,
+ stream,
+ ts,
+ AVSEEK_FLAG_BACKWARD))
+ return -1;
+ avcodec_flush_buffers (ctx);
+ if (0 != decode_frame_at (io,
+ ctx,
+ stream,
+ ts * av_q2d (tb),
+ 0,
+ 1,
+ GALLERY_MAX_FRAMES,
+ frame))
+ return -1;
+ if ( (frame->width <= 0) ||
+ (frame->height <= 0) ||
+ (frame->width > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
+ (frame->height > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
+ (((int64_t) frame->width) * frame->height
+ > EXTRACTOR_MEDIA_MAX_PIXELS) ||
+ (! sws_isSupportedInput (frame->format)) )
+ goto cleanup;
+ g->sws = sws_getCachedContext (g->sws,
+ frame->width,
+ frame->height,
+ frame->format,
+ (int) g->tw,
+ (int) g->th,
+ AV_PIX_FMT_RGB24,
+ SWS_BILINEAR,
+ NULL,
+ NULL,
+ NULL);
+ if (NULL == g->sws)
+ goto cleanup;
+ if ((int) g->th != sws_scale (g->sws,
+ (const uint8_t *const *) frame->data,
+ frame->linesize,
+ 0,
+ frame->height,
+ dst,
+ dst_stride))
+ goto cleanup;
+ ret = 0;
+cleanup:
+ av_frame_unref (frame);
+ return ret;
+}
+
+
+/**
+ * Copy a tile into cell @a i of the sheet.
+ *
+ * @param[in,out] g gallery to update
+ * @param i index of the cell
+ * @param tile packed RGB24 pixels of the tile
+ */
+static void
+place_tile (struct Gallery *g,
+ unsigned int i,
+ const uint8_t *tile)
+{
+ size_t sheet_stride = (size_t) g->cols * g->tw * 3;
+ size_t tile_stride = (size_t) g->tw * 3;
+ size_t x0 = (size_t) (i % g->cols) * g->tw * 3;
+ size_t y0 = (size_t) (i / g->cols) * g->th;
+ unsigned int y;
+
+ for (y = 0; y < g->th; y++)
+ memcpy (&g->sheet[(y0 + y) * sheet_stride + x0],
+ &tile[y * tile_stride],
+ tile_stride);
+}
+
+
+/**
+ * Encode the finished sheet, degrading quality and then size until it
+ * fits into #EXTRACTOR_MEDIA_MAX_GALLERY_BYTES.
+ *
+ * @param ec extraction context, kept alive between attempts
+ * @param g the gallery
+ * @param opt plugin options
+ * @param[out] output set to the image, to be freed with av_free()
+ * @param[out] output_size set to the number of bytes in @a output
+ * @return 0 on success
+ */
+static int
+encode_gallery (struct EXTRACTOR_ExtractContext *ec,
+ const struct Gallery *g,
+ const struct EXTRACTOR_MediaOptions *opt,
+ uint8_t **output,
+ size_t *output_size)
+{
+ static const int qscales[] = { JPEG_QSCALE, 10, 20, 31 };
+ AVFrame *src;
+ unsigned int w;
+ unsigned int h;
+ unsigned int shrink;
+ int r = -1;
+
+ if (NULL == (src = av_frame_alloc ()))
+ return -1;
+ src->format = AV_PIX_FMT_RGB24;
+ src->width = (int) (g->cols * g->tw);
+ src->height = (int) (g->rows * g->th);
+ src->data[0] = g->sheet;
+ src->linesize[0] = (int) (g->cols * g->tw * 3);
+ w = g->cols * g->tw;
+ h = g->rows * g->th;
+ for (shrink = 0; shrink < 4; shrink++)
+ {
+ (void) ec->seek (ec->cls, 0, SEEK_CUR); /* keep-alive */
+ if (opt->png)
+ {
+ r = encode_image (src, w, h, 1, 0, GALLERY_PNG_LEVEL,
+ EXTRACTOR_MEDIA_MAX_GALLERY_BYTES,
+ output, output_size);
+ }
+ else
+ {
+ unsigned int q;
+
+ for (q = 0; q < sizeof (qscales) / sizeof (qscales[0]); q++)
+ {
+ (void) ec->seek (ec->cls, 0, SEEK_CUR); /* keep-alive */
+ r = encode_image (src, w, h, 0, qscales[q], 0,
+ EXTRACTOR_MEDIA_MAX_GALLERY_BYTES,
+ output, output_size);
+ if (-2 != r)
+ break;
+ }
+ }
+ if (-2 != r)
+ break;
+ /* still too big: halve the sheet (i.e. every tile) */
+ w /= 2;
+ h /= 2;
+ if ( (w < 2) ||
+ (h < 2) )
+ break;
+ }
+ /* the frame only borrowed the sheet */
+ src->data[0] = NULL;
+ av_frame_free (&src);
+ return (0 == r) ? 0 : -1;
+}
+
+
+/**
+ * Build and emit a contact sheet for "gallery:N".
+ *
+ * @param io open I/O state
+ * @param ctx open video decoder
+ * @param stream index of the video stream
+ * @param opt plugin options
+ * @param[out] stop set to 1 if the application asked us to stop
+ * @param[out] touched set to 1 if we moved the demuxer
+ * @return 0 if we are done (sheet emitted, or the application asked us
+ * to stop), 1 if the caller should fall back to a single
+ * thumbnail
+ */
+static int
+make_gallery (struct EXTRACTOR_FFmpegIO *io,
+ AVCodecContext *ctx,
+ int stream,
+ const struct EXTRACTOR_MediaOptions *opt,
+ int *stop,
+ int *touched)
+{
+ /* where to look instead of a flat frame, in eighths of a slice */
+ static const int shifts[] = { 0, 2, -2, 3, -3 };
+ const AVStream *st = io->fmt->streams[stream];
+ struct Gallery g;
+ AVFrame *frame = NULL;
+ uint8_t *image = NULL;
+ size_t image_size = 0;
+ uint64_t duration_ms;
+ uint64_t slice;
+ int64_t start_ts;
+ unsigned int n = opt->gallery;
+ unsigned int box = opt->size;
+ unsigned int got = 0;
+ unsigned int i;
+ int ret = 1;
+
+ *stop = 0;
+ *touched = 0;
+ memset (&g, 0, sizeof (g));
+ if ( (AV_NOPTS_VALUE == io->fmt->duration) ||
+ (io->fmt->duration <= 0) )
+ return 1;
+ duration_ms = (uint64_t) io->fmt->duration / 1000;
+ if (duration_ms > MAX_CUE_TIME_MS)
+ duration_ms = MAX_CUE_TIME_MS;
+ if ( (0 == duration_ms) ||
+ (0 == n) )
+ return 1;
+ /* shrink the tiles until the sheet is of a size we can handle */
+ for (;;)
+ {
+ uint64_t sw;
+ uint64_t sh;
+
+ if (! EXTRACTOR_media_thumbnail_size_ (ctx->width,
+ ctx->height,
+ box,
+ &g.tw,
+ &g.th))
+ return 1;
+ EXTRACTOR_media_gallery_layout_ (n,
+ g.tw,
+ g.th,
+ &g.cols,
+ &g.rows);
+ sw = (uint64_t) g.cols * g.tw;
+ sh = (uint64_t) g.rows * g.th;
+ if ( (sw * sh <= GALLERY_MAX_SHEET_PIXELS) &&
+ (sw * sh <= EXTRACTOR_MEDIA_MAX_PIXELS) &&
+ (sw <= EXTRACTOR_MEDIA_MAX_DIMENSION) &&
+ (sh <= EXTRACTOR_MEDIA_MAX_DIMENSION) )
+ break;
+ if (box <= 8)
+ return 1;
+ box = box * 3 / 4;
+ if (box < 8)
+ box = 8;
+ }
+ g.sheet = calloc ((size_t) g.cols * g.tw * g.rows * g.th, 3);
+ g.tile = malloc ((size_t) g.tw * g.th * 3);
+ g.best = malloc ((size_t) g.tw * g.th * 3);
+ if ( (NULL == g.sheet) ||
+ (NULL == g.tile) ||
+ (NULL == g.best) ||
+ (NULL == (frame = av_frame_alloc ())) )
+ goto cleanup;
+ if (AV_NOPTS_VALUE != st->start_time)
+ start_ts = st->start_time;
+ else if (AV_NOPTS_VALUE != io->fmt->start_time)
+ start_ts = av_rescale_q (io->fmt->start_time,
+ AV_TIME_BASE_Q,
+ st->time_base);
+ else
+ start_ts = 0;
+ slice = duration_ms / n;
+ for (i = 0; i < n; i++)
+ {
+ uint64_t base = EXTRACTOR_media_gallery_time_ (i,
+ n,
+ duration_ms);
+ uint64_t tried[sizeof (shifts) / sizeof (shifts[0])];
+ double best_score = -1.0;
+ unsigned int ntried = 0;
+ unsigned int a;
+ int placed = 0;
+
+ for (a = 0; a < sizeof (shifts) / sizeof (shifts[0]); a++)
+ {
+ int64_t off = (int64_t) (slice / 8) * shifts[a];
+ uint64_t t;
+ unsigned int k;
+ double score;
+
+ if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
+ break;
+ if ( (off < 0) &&
+ ((uint64_t) -off > base) )
+ t = 0;
+ else
+ t = base + off;
+ if (t >= duration_ms)
+ t = duration_ms - 1;
+ for (k = 0; k < ntried; k++)
+ if (tried[k] == t)
+ break;
+ if (k < ntried)
+ continue;
+ tried[ntried++] = t;
+ *touched = 1;
+ if (0 != grab_tile (io,
+ ctx,
+ stream,
+ start_ts,
+ t,
+ frame,
+ &g))
+ continue;
+ if (! EXTRACTOR_media_frame_is_flat_ (g.tile,
+ g.tw,
+ g.th,
+ (size_t) g.tw * 3))
+ {
+ place_tile (&g, i, g.tile);
+ placed = 1;
+ break;
+ }
+ score = tile_score (g.tile,
+ g.tw,
+ g.th);
+ if (score > best_score)
+ {
+ best_score = score;
+ memcpy (g.best,
+ g.tile,
+ (size_t) g.tw * g.th * 3);
+ }
+ }
+ if ( (! placed) &&
+ (best_score >= 0.0) )
+ {
+ place_tile (&g, i, g.best);
+ placed = 1;
+ }
+ if (placed)
+ got++;
+ if (EXTRACTOR_media_deadline_expired_ (&io->deadline))
+ break; /* emit what we have; the other cells stay black */
+ }
+ if (0 == got)
+ goto cleanup;
+ ret = 0;
+ if (0 != encode_gallery (io->ec,
+ &g,
+ opt,
+ &image,
+ &image_size))
+ goto cleanup;
+ if (0 != io->ec->proc (io->ec->cls,
+ "thumbnailffmpeg",
+ EXTRACTOR_METATYPE_THUMBNAIL,
+ EXTRACTOR_METAFORMAT_BINARY,
+ opt->png ? "image/png" : "image/jpeg",
+ (const char *) image,
+ image_size))
+ *stop = 1;
+cleanup:
+ if (NULL != image)
+ av_free (image);
+ if (NULL != frame)
+ av_frame_free (&frame);
+ sws_freeContext (g.sws);
+ free (g.sheet);
+ free (g.tile);
+ free (g.best);
+ return ret;
+}
+
+
+/**
* Main method for the ffmpeg-thumbnailer plugin.
*
* @param ec extraction context
@@ -376,6 +1505,22 @@ EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec)
return;
if (0 != EXTRACTOR_ffmpeg_open_ (&io))
return;
+ /* subtitle metadata does not depend on there being a usable video
+ stream (think of a subtitles-only .mks), so do it first */
+ if (0 != emit_subtitle_languages (&io))
+ goto cleanup;
+ if ('\0' != opt.srt_lang[0])
+ {
+ int touched;
+
+ if (0 != dump_srt (&io,
+ &opt,
+ &touched))
+ goto cleanup;
+ if (touched &&
+ (0 != EXTRACTOR_ffmpeg_reopen_ (&io)) )
+ return;
+ }
stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
AVMEDIA_TYPE_VIDEO,
&ctx);
@@ -387,6 +1532,32 @@ EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec)
(ctx->height > EXTRACTOR_MEDIA_MAX_DIMENSION) ||
(((int64_t) ctx->width) * ctx->height > EXTRACTOR_MEDIA_MAX_PIXELS) )
goto cleanup;
+ if (0 != opt.gallery)
+ {
+ int stop;
+ int touched;
+
+ if (0 == make_gallery (&io,
+ ctx,
+ stream,
+ &opt,
+ &stop,
+ &touched))
+ goto cleanup;
+ /* no duration, or not a single frame: try for one thumbnail */
+ if (touched)
+ {
+ /* gallery seeks may have left the demuxer and decoder anywhere */
+ avcodec_free_context (&ctx);
+ if (0 != EXTRACTOR_ffmpeg_reopen_ (&io))
+ return;
+ stream = EXTRACTOR_ffmpeg_open_stream_ (&io,
+ AVMEDIA_TYPE_VIDEO,
+ &ctx);
+ if (0 > stream)
+ goto cleanup;
+ }
+ }
target = EXTRACTOR_ffmpeg_seek_ (&io,
stream,
ctx,
@@ -397,6 +1568,9 @@ EXTRACTOR_thumbnailffmpeg_extract_method (struct EXTRACTOR_ExtractContext *ec)
ctx,
stream,
target,
+ 1,
+ 0,
+ MAX_FRAMES,
frame))
goto cleanup;
if (0 != encode_thumbnail (frame,
diff --git a/src/plugins/thumbnailgst_extractor.c b/src/plugins/thumbnailgst_extractor.c
@@ -33,6 +33,30 @@
* The decoders and the image encoder are themselves GStreamer plugins,
* resolved at run time, so a missing "jpeg" or "playback" plugin cannot
* be detected when libextractor is built. We simply report nothing.
+ *
+ * Two options go beyond a single thumbnail (see mediautil.h):
+ *
+ * - "gallery:N" replaces the thumbnail by a contact sheet of N tiles,
+ * sampled at the midpoints of N equal slices of the video. A tile
+ * that shows nothing (see EXTRACTOR_media_frame_is_flat_()) is
+ * re-sampled a quarter and three eighths of a slice later and
+ * earlier; if all candidates are flat, the least flat one is used.
+ * Each tile is scaled to fit into "size" and converted to RGB by
+ * gst_video_convert_sample(), the sheet is composed in memory and
+ * encoded by another gst_video_convert_sample(). Without a known
+ * duration we fall back to the plain thumbnail.
+ *
+ * - "srt:LANG" additionally dumps the first subtitle track in language
+ * LANG as SubRip. For that the file is run through a second pipeline,
+ * appsrc ! parsebin, which demuxes (but does not decode) all streams.
+ * Text subtitles come out of it as text/x-raw (Pango markup or plain
+ * UTF-8) or as application/x-ass / application/x-ssa events. All
+ * markup is stripped (tags removed, entities decoded): SubRip only
+ * knows a handful of tags anyway, and the FFmpeg plugin, which works
+ * on ASS events, cannot keep them either, so both plugins produce the
+ * same text. The language comes from the stream's tags (sticky TAG
+ * event or the GstStream of the STREAM_START event), matched with
+ * EXTRACTOR_media_lang_matches_().
*/
#include "platform.h"
#include "extractor.h"
@@ -40,6 +64,7 @@
#include "mediagst.h"
#include <gst/video/video.h>
+#include <gst/app/gstappsink.h>
/**
@@ -93,18 +118,19 @@ encode_thumbnail (GstSample *sample,
/**
- * Main method for the gstreamer-thumbnailer plugin.
+ * Sample a single frame about opt->offset seconds into the video and
+ * emit it as a thumbnail.
*
* @param ec extraction context
+ * @param opt plugin options
+ * @param deadline when we must be done
+ * @return return value of `proc` (0 if nothing was emitted)
*/
-void
-EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec);
-
-void
-EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
+static int
+make_thumbnail (struct EXTRACTOR_ExtractContext *ec,
+ const struct EXTRACTOR_MediaOptions *opt,
+ const struct EXTRACTOR_MediaDeadline *deadline)
{
- struct EXTRACTOR_MediaOptions opt;
- struct EXTRACTOR_MediaDeadline deadline;
struct EXTRACTOR_GstIO io;
GstElement *pipeline;
GstElement *sink;
@@ -115,15 +141,8 @@ EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
GstClockTime timeout;
gint64 duration;
gint64 target;
+ int ret = 0;
- EXTRACTOR_media_options_parse_ (ec->config,
- &opt);
- EXTRACTOR_media_deadline_start_ (&deadline,
- opt.deadline);
- if (! EXTRACTOR_media_is_media_ (ec, 0))
- return;
- if (! gst_is_initialized ())
- gst_init (NULL, NULL);
sink = gst_element_factory_make ("fakesink", NULL);
if (NULL != sink)
g_object_set (sink,
@@ -136,17 +155,17 @@ EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
sink,
gst_element_factory_make ("fakesink",
NULL))))
- return;
+ return 0;
if (GST_STATE_CHANGE_FAILURE ==
gst_element_set_state (pipeline,
GST_STATE_PAUSED))
goto cleanup;
if (! EXTRACTOR_gst_wait_async_ (pipeline,
EXTRACTOR_media_deadline_remaining_ (
- &deadline) * GST_MSECOND))
+ deadline) * GST_MSECOND))
goto cleanup;
target = EXTRACTOR_gst_sample_position_ (pipeline,
- opt.offset,
+ opt->offset,
&duration);
if (target >= GST_SECOND)
{
@@ -159,7 +178,7 @@ EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
target))
(void) EXTRACTOR_gst_wait_async_ (pipeline,
EXTRACTOR_media_deadline_remaining_ (
- &deadline) * GST_MSECOND);
+ deadline) * GST_MSECOND);
}
g_object_get (sink,
"last-sample",
@@ -167,11 +186,11 @@ EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
NULL);
if (NULL == sample)
goto cleanup;
- timeout = EXTRACTOR_media_deadline_remaining_ (&deadline) * GST_MSECOND;
+ timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND;
if (0 == timeout)
goto cleanup;
if (NULL == (thumbnail = encode_thumbnail (sample,
- &opt,
+ opt,
timeout)))
goto cleanup;
if (NULL == (buffer = gst_sample_get_buffer (thumbnail)))
@@ -180,13 +199,13 @@ EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
goto cleanup;
if ( (0 < mi.size) &&
(EXTRACTOR_MEDIA_MAX_THUMBNAIL >= mi.size) )
- ec->proc (ec->cls,
- "thumbnailgst",
- EXTRACTOR_METATYPE_THUMBNAIL,
- EXTRACTOR_METAFORMAT_BINARY,
- opt.png ? "image/png" : "image/jpeg",
- (const char *) mi.data,
- mi.size);
+ ret = ec->proc (ec->cls,
+ "thumbnailgst",
+ EXTRACTOR_METATYPE_THUMBNAIL,
+ EXTRACTOR_METAFORMAT_BINARY,
+ opt->png ? "image/png" : "image/jpeg",
+ (const char *) mi.data,
+ mi.size);
gst_buffer_unmap (buffer, &mi);
cleanup:
if (NULL != thumbnail)
@@ -196,6 +215,1237 @@ cleanup:
gst_element_set_state (pipeline,
GST_STATE_NULL);
gst_object_unref (pipeline);
+ return ret;
+}
+
+
+/* ************************* gallery ************************* */
+
+
+/**
+ * How "flat" a frame is: the share (in 1/1000) of pixels inside the
+ * most populated narrow window of levels, for the channel where that
+ * share is lowest. Frames with a lower value show more; see
+ * EXTRACTOR_media_frame_is_flat_() for the yes/no version.
+ *
+ * @param rgb packed RGB24 pixels
+ * @param width width in pixels
+ * @param height height in pixels
+ * @param stride bytes per row
+ * @return flatness in [0,1000]
+ */
+static unsigned int
+flatness (const uint8_t *rgb,
+ unsigned int width,
+ unsigned int height,
+ size_t stride)
+{
+ uint32_t hist[3][256];
+ uint64_t total = 0;
+ unsigned int worst = 1000;
+ unsigned int x;
+ unsigned int y;
+ unsigned int c;
+
+ memset (hist, 0, sizeof (hist));
+ for (y = 0; y < height; y++)
+ {
+ const uint8_t *row = rgb + (size_t) y * stride;
+
+ for (x = 0; x < width; x++)
+ {
+ hist[0][row[3 * x]]++;
+ hist[1][row[3 * x + 1]]++;
+ hist[2][row[3 * x + 2]]++;
+ total++;
+ }
+ }
+ if (0 == total)
+ return 1000;
+ for (c = 0; c < 3; c++)
+ {
+ uint64_t win = 0;
+ uint64_t best = 0;
+ unsigned int i;
+
+ for (i = 0; i < 256; i++)
+ {
+ win += hist[c][i];
+ if (i >= 32)
+ win -= hist[c][i - 32];
+ if (win > best)
+ best = win;
+ }
+ if ((unsigned int) (best * 1000 / total) < worst)
+ worst = (unsigned int) (best * 1000 / total);
+ }
+ return worst;
+}
+
+
+/**
+ * Seek the paused @a pipeline to @a ms and return the frame shown there.
+ *
+ * @param pipeline paused, prerolled pipeline
+ * @param io I/O state of the pipeline
+ * @param sink the appsink receiving the video
+ * @param ms position in milliseconds
+ * @param deadline when we must be done
+ * @return the frame, NULL on error or timeout
+ */
+static GstSample *
+sample_at (GstElement *pipeline,
+ struct EXTRACTOR_GstIO *io,
+ GstElement *sink,
+ uint64_t ms,
+ const struct EXTRACTOR_MediaDeadline *deadline)
+{
+ if (! gst_element_seek_simple (pipeline,
+ GST_FORMAT_TIME,
+ GST_SEEK_FLAG_FLUSH
+ | GST_SEEK_FLAG_ACCURATE,
+ (gint64) (ms * GST_MSECOND)))
+ return NULL;
+ if (! EXTRACTOR_gst_wait_async_alive_ (pipeline,
+ EXTRACTOR_media_deadline_remaining_ (
+ deadline) * GST_MSECOND,
+ io))
+ return NULL;
+ return gst_app_sink_try_pull_preroll (GST_APP_SINK (sink),
+ EXTRACTOR_media_deadline_remaining_ (
+ deadline) * GST_MSECOND);
+}
+
+
+/**
+ * Convert @a sample to packed RGB of exactly @a width x @a height.
+ *
+ * @param sample frame to convert
+ * @param width target width
+ * @param height target height
+ * @param deadline when we must be done
+ * @return the converted frame, NULL on error
+ */
+static GstSample *
+to_rgb (GstSample *sample,
+ unsigned int width,
+ unsigned int height,
+ const struct EXTRACTOR_MediaDeadline *deadline)
+{
+ GstCaps *to_caps;
+ GstSample *out;
+ GstClockTime timeout;
+
+ timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND;
+ if (0 == timeout)
+ return NULL;
+ /* no pixel-aspect-ratio here: fixing it would make videoscale add
+ borders to keep the display aspect ratio, and a letterboxed tile
+ would look flat to the check below */
+ to_caps = gst_caps_new_simple ("video/x-raw",
+ "format", G_TYPE_STRING, "RGB",
+ "width", G_TYPE_INT, (gint) width,
+ "height", G_TYPE_INT, (gint) height,
+ NULL);
+ out = gst_video_convert_sample (sample,
+ to_caps,
+ timeout,
+ NULL);
+ gst_caps_unref (to_caps);
+ return out;
+}
+
+
+/**
+ * Map the RGB frame in @a sample, check that it has the expected size.
+ *
+ * @param sample frame from to_rgb()
+ * @param width expected width
+ * @param height expected height
+ * @param[out] frame mapped frame, to be released with
+ * gst_video_frame_unmap()
+ * @return 1 on success
+ */
+static int
+map_rgb (GstSample *sample,
+ unsigned int width,
+ unsigned int height,
+ GstVideoFrame *frame)
+{
+ GstVideoInfo vi;
+ GstCaps *caps;
+ GstBuffer *buffer;
+
+ if ( (NULL == (caps = gst_sample_get_caps (sample))) ||
+ (NULL == (buffer = gst_sample_get_buffer (sample))) ||
+ (! gst_video_info_from_caps (&vi, caps)) ||
+ (GST_VIDEO_FORMAT_RGB != GST_VIDEO_INFO_FORMAT (&vi)) ||
+ (width != (unsigned int) GST_VIDEO_INFO_WIDTH (&vi)) ||
+ (height != (unsigned int) GST_VIDEO_INFO_HEIGHT (&vi)) )
+ return 0;
+ if (! gst_video_frame_map (frame,
+ &vi,
+ buffer,
+ GST_MAP_READ))
+ return 0;
+ return 1;
+}
+
+
+/**
+ * Sample tile @a i of @a n and scale it to @a tw x @a th RGB. If the
+ * frame at the nominal time is flat, try other moments in the same
+ * slice; if they are all flat, return the least flat one.
+ *
+ * @param pipeline paused, prerolled pipeline
+ * @param io I/O state of the pipeline
+ * @param sink the appsink receiving the video
+ * @param i tile index
+ * @param n number of tiles
+ * @param duration_ms length of the video
+ * @param tw tile width
+ * @param th tile height
+ * @param deadline when we must be done
+ * @return the tile (RGB, @a tw x @a th), NULL if nothing could be sampled
+ */
+static GstSample *
+sample_tile (GstElement *pipeline,
+ struct EXTRACTOR_GstIO *io,
+ GstElement *sink,
+ unsigned int i,
+ unsigned int n,
+ uint64_t duration_ms,
+ unsigned int tw,
+ unsigned int th,
+ const struct EXTRACTOR_MediaDeadline *deadline)
+{
+ uint64_t base = EXTRACTOR_media_gallery_time_ (i, n, duration_ms);
+ int64_t slice = (int64_t) (duration_ms / n);
+ int64_t shifts[5];
+ uint64_t tried[5];
+ GstSample *best = NULL;
+ unsigned int best_score = UINT_MAX;
+ unsigned int k;
+
+ shifts[0] = 0;
+ shifts[1] = slice / 4;
+ shifts[2] = -slice / 4;
+ shifts[3] = slice * 3 / 8;
+ shifts[4] = -slice * 3 / 8;
+ for (k = 0; k < 5; k++)
+ {
+ int64_t t = (int64_t) base + shifts[k];
+ GstSample *frame;
+ GstSample *rgb;
+ GstVideoFrame vf;
+ unsigned int score;
+ int flat;
+ unsigned int j;
+ int dup = 0;
+
+ if (t < 0)
+ t = 0;
+ if ((uint64_t) t >= duration_ms)
+ t = (int64_t) duration_ms - 1;
+ for (j = 0; j < k; j++)
+ if (tried[j] == (uint64_t) t)
+ dup = 1;
+ tried[k] = (uint64_t) t;
+ if (dup)
+ continue;
+ if (EXTRACTOR_media_deadline_expired_ (deadline))
+ break;
+ EXTRACTOR_gst_keepalive_ (io);
+ if (NULL == (frame = sample_at (pipeline,
+ io,
+ sink,
+ (uint64_t) t,
+ deadline)))
+ continue;
+ rgb = to_rgb (frame,
+ tw,
+ th,
+ deadline);
+ gst_sample_unref (frame);
+ if (NULL == rgb)
+ continue;
+ if (! map_rgb (rgb,
+ tw,
+ th,
+ &vf))
+ {
+ gst_sample_unref (rgb);
+ continue;
+ }
+ score = flatness (GST_VIDEO_FRAME_PLANE_DATA (&vf, 0),
+ tw,
+ th,
+ GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0));
+ flat = EXTRACTOR_media_frame_is_flat_ (
+ GST_VIDEO_FRAME_PLANE_DATA (&vf, 0),
+ tw,
+ th,
+ GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0));
+ gst_video_frame_unmap (&vf);
+ if ( (NULL == best) ||
+ (score < best_score) )
+ {
+ if (NULL != best)
+ gst_sample_unref (best);
+ best = rgb;
+ best_score = score;
+ }
+ else
+ {
+ gst_sample_unref (rgb);
+ }
+ if (! flat)
+ break;
+ }
+ return best;
+}
+
+
+/**
+ * Emit a contact sheet of opt->gallery frames.
+ *
+ * @param ec extraction context
+ * @param opt plugin options
+ * @param deadline when we must be done
+ * @param[out] ret set to the return value of `proc`
+ * @return 1 if the gallery was dealt with (emitted, or failed for good),
+ * 0 if the caller should fall back to a single thumbnail
+ */
+static int
+make_gallery (struct EXTRACTOR_ExtractContext *ec,
+ const struct EXTRACTOR_MediaOptions *opt,
+ const struct EXTRACTOR_MediaDeadline *deadline,
+ int *ret)
+{
+ struct EXTRACTOR_GstIO io;
+ GstElement *pipeline;
+ GstElement *sink;
+ GstCaps *caps;
+ GstSample *first = NULL;
+ GstSample *sheet_sample = NULL;
+ GstSample *encoded = NULL;
+ unsigned int shrink;
+ GstBuffer *sheet = NULL;
+ GstVideoInfo sheet_info;
+ GstStructure *s;
+ GstMapInfo mi;
+ gint64 duration = 0;
+ gint width;
+ gint height;
+ unsigned int tw;
+ unsigned int th;
+ unsigned int cols;
+ unsigned int rows;
+ unsigned int n = opt->gallery;
+ unsigned int i;
+ uint64_t duration_ms;
+ int handled = 0;
+
+ *ret = 0;
+ sink = gst_element_factory_make ("appsink", NULL);
+ if (NULL != sink)
+ {
+ caps = gst_caps_new_empty_simple ("video/x-raw");
+ g_object_set (sink,
+ "sync", FALSE,
+ "caps", caps,
+ NULL);
+ gst_caps_unref (caps);
+ }
+ if (NULL == (pipeline =
+ EXTRACTOR_gst_playbin_ (&io,
+ ec,
+ EXTRACTOR_GST_PLAY_FLAG_VIDEO,
+ sink,
+ gst_element_factory_make ("fakesink",
+ NULL))))
+ return 0;
+ if (GST_STATE_CHANGE_FAILURE ==
+ gst_element_set_state (pipeline,
+ GST_STATE_PAUSED))
+ goto cleanup;
+ if (! EXTRACTOR_gst_wait_async_alive_ (pipeline,
+ EXTRACTOR_media_deadline_remaining_ (
+ deadline) * GST_MSECOND,
+ &io))
+ goto cleanup;
+ if ( (! gst_element_query_duration (pipeline,
+ GST_FORMAT_TIME,
+ &duration)) ||
+ (duration < (gint64) GST_MSECOND) )
+ goto cleanup; /* unknown length: plain thumbnail */
+ duration_ms = (uint64_t) duration / GST_MSECOND;
+ if (NULL == (first = gst_app_sink_try_pull_preroll (
+ GST_APP_SINK (sink),
+ EXTRACTOR_media_deadline_remaining_ (deadline)
+ * GST_MSECOND)))
+ goto cleanup;
+ if ( (NULL == (caps = gst_sample_get_caps (first))) ||
+ (NULL == (s = gst_caps_get_structure (caps, 0))) ||
+ (! gst_structure_get_int (s, "width", &width)) ||
+ (! gst_structure_get_int (s, "height", &height)) ||
+ (0 >= width) ||
+ (0 >= height) ||
+ (! EXTRACTOR_media_thumbnail_size_ ((unsigned int) width,
+ (unsigned int) height,
+ opt->size,
+ &tw,
+ &th)) )
+ goto cleanup;
+ /* from here on, we produce a gallery or nothing */
+ handled = 1;
+ EXTRACTOR_media_gallery_layout_ (n,
+ tw,
+ th,
+ &cols,
+ &rows);
+ while ( ((uint64_t) cols * tw) * ((uint64_t) rows * th)
+ > EXTRACTOR_MEDIA_MAX_GALLERY_PIXELS)
+ {
+ tw = (tw * 3) / 4;
+ th = (th * 3) / 4;
+ if ( (0 == tw) ||
+ (0 == th) )
+ goto cleanup;
+ }
+ if (! gst_video_info_set_format (&sheet_info,
+ GST_VIDEO_FORMAT_RGB,
+ cols * tw,
+ rows * th))
+ goto cleanup;
+ if (NULL == (sheet = gst_buffer_new_allocate (NULL,
+ GST_VIDEO_INFO_SIZE (
+ &sheet_info),
+ NULL)))
+ goto cleanup;
+ if (! gst_buffer_map (sheet, &mi, GST_MAP_WRITE))
+ goto cleanup;
+ memset (mi.data, 0, mi.size);
+ for (i = 0; i < n; i++)
+ {
+ GstSample *tile;
+ GstVideoFrame vf;
+ size_t dstride = GST_VIDEO_INFO_PLANE_STRIDE (&sheet_info, 0);
+ uint8_t *dst = mi.data
+ + GST_VIDEO_INFO_PLANE_OFFSET (&sheet_info, 0)
+ + (size_t) (i / cols) * th * dstride
+ + (size_t) (i % cols) * tw * 3;
+ unsigned int y;
+
+ tile = sample_tile (pipeline,
+ &io,
+ sink,
+ i,
+ n,
+ duration_ms,
+ tw,
+ th,
+ deadline);
+ if (EXTRACTOR_media_deadline_expired_ (deadline))
+ {
+ /* better no gallery than one with holes */
+ if (NULL != tile)
+ gst_sample_unref (tile);
+ gst_buffer_unmap (sheet, &mi);
+ goto cleanup;
+ }
+ if (NULL == tile)
+ continue; /* leave the tile black */
+ if (map_rgb (tile,
+ tw,
+ th,
+ &vf))
+ {
+ const uint8_t *src = GST_VIDEO_FRAME_PLANE_DATA (&vf, 0);
+ size_t sstride = GST_VIDEO_FRAME_PLANE_STRIDE (&vf, 0);
+
+ for (y = 0; y < th; y++)
+ memcpy (dst + (size_t) y * dstride,
+ src + (size_t) y * sstride,
+ (size_t) tw * 3);
+ gst_video_frame_unmap (&vf);
+ }
+ gst_sample_unref (tile);
+ }
+ gst_buffer_unmap (sheet, &mi);
+ /* the decoding pipeline is no longer needed; free its memory before
+ we encode */
+ gst_element_set_state (pipeline,
+ GST_STATE_NULL);
+ EXTRACTOR_gst_keepalive_ (&io);
+ caps = gst_video_info_to_caps (&sheet_info);
+ sheet_sample = gst_sample_new (sheet,
+ caps,
+ NULL,
+ NULL);
+ gst_caps_unref (caps);
+ /* encode; if the result is too large, halve the sheet and retry */
+ for (shrink = 0; shrink < 4; shrink++)
+ {
+ GstCaps *to_caps;
+ GstClockTime timeout;
+ GstBuffer *buffer;
+ unsigned int ow = (cols * tw) >> shrink;
+ unsigned int oh = (rows * th) >> shrink;
+
+ if ( (0 == ow) ||
+ (0 == oh) )
+ break;
+ EXTRACTOR_gst_keepalive_ (&io);
+ timeout = EXTRACTOR_media_deadline_remaining_ (deadline) * GST_MSECOND;
+ if (0 == timeout)
+ break;
+ to_caps = gst_caps_new_simple (opt->png ? "image/png" : "image/jpeg",
+ "width", G_TYPE_INT, (gint) ow,
+ "height", G_TYPE_INT, (gint) oh,
+ NULL);
+ encoded = gst_video_convert_sample (sheet_sample,
+ to_caps,
+ timeout,
+ NULL);
+ gst_caps_unref (to_caps);
+ if (NULL == encoded)
+ break;
+ if (NULL == (buffer = gst_sample_get_buffer (encoded)))
+ break;
+ if (! gst_buffer_map (buffer, &mi, GST_MAP_READ))
+ break;
+ if ( (0 < mi.size) &&
+ (EXTRACTOR_MEDIA_MAX_GALLERY_BYTES >= mi.size) )
+ {
+ *ret = ec->proc (ec->cls,
+ "thumbnailgst",
+ EXTRACTOR_METATYPE_THUMBNAIL,
+ EXTRACTOR_METAFORMAT_BINARY,
+ opt->png ? "image/png" : "image/jpeg",
+ (const char *) mi.data,
+ mi.size);
+ gst_buffer_unmap (buffer, &mi);
+ break;
+ }
+ gst_buffer_unmap (buffer, &mi);
+ gst_sample_unref (encoded);
+ encoded = NULL;
+ }
+cleanup:
+ if (NULL != encoded)
+ gst_sample_unref (encoded);
+ if (NULL != sheet_sample)
+ gst_sample_unref (sheet_sample);
+ if (NULL != sheet)
+ gst_buffer_unref (sheet);
+ if (NULL != first)
+ gst_sample_unref (first);
+ gst_element_set_state (pipeline,
+ GST_STATE_NULL);
+ gst_object_unref (pipeline);
+ return handled;
+}
+
+
+/* ************************* subtitles ************************* */
+
+
+/**
+ * Most subtitle tracks we keep apart while we find out their language.
+ */
+#define MAX_SUB_TRACKS 16
+
+/**
+ * Most streams of one file we connect to a sink at all.
+ */
+#define MAX_STREAMS 64
+
+
+/**
+ * How the payload of a subtitle stream is encoded.
+ */
+enum SubKind
+{
+ /**
+ * Plain UTF-8 text (text/x-raw, format=utf8).
+ */
+ SUB_PLAIN,
+
+ /**
+ * Pango markup (text/x-raw, format=pango-markup).
+ */
+ SUB_MARKUP,
+
+ /**
+ * ASS/SSA dialogue events (application/x-ass, application/x-ssa).
+ */
+ SUB_ASS
+};
+
+
+struct SubState;
+
+
+/**
+ * One subtitle track of the file.
+ */
+struct SubTrack
+{
+ /**
+ * The subtitle dump this track belongs to.
+ */
+ struct SubState *st;
+
+ /**
+ * Encoding of the payload.
+ */
+ enum SubKind kind;
+
+ /**
+ * 1 if the language matches, -1 if it does not, 0 if still unknown.
+ */
+ int match;
+
+ /**
+ * Cues collected so far (only while @e match is not -1).
+ */
+ struct EXTRACTOR_MediaSrt srt;
+};
+
+
+/**
+ * State of a subtitle dump.
+ */
+struct SubState
+{
+ /**
+ * Protects @e tracks, @e num_tracks and @e num_streams against
+ * concurrent "pad-added" signals.
+ */
+ GMutex lock;
+
+ /**
+ * The pipeline, to add sinks to.
+ */
+ GstElement *pipeline;
+
+ /**
+ * Requested language.
+ */
+ const char *want;
+
+ /**
+ * The subtitle tracks, in the order the demuxer exposed them.
+ */
+ struct SubTrack tracks[MAX_SUB_TRACKS];
+
+ /**
+ * Number of entries used in @e tracks.
+ */
+ unsigned int num_tracks;
+
+ /**
+ * Number of streams connected to a sink.
+ */
+ unsigned int num_streams;
+};
+
+
+/**
+ * Get the language tag of the stream flowing through @a pad, from the
+ * sticky TAG events or the GstStream of the STREAM_START event.
+ *
+ * @param pad pad to inspect
+ * @return the language (to be freed with g_free()), NULL if unknown
+ */
+static gchar *
+pad_language (GstPad *pad)
+{
+ GstEvent *ev;
+ gchar *lang = NULL;
+ guint i;
+
+ for (i = 0;
+ (NULL == lang) &&
+ (i < 16) &&
+ (NULL != (ev = gst_pad_get_sticky_event (pad,
+ GST_EVENT_TAG,
+ i)));
+ i++)
+ {
+ GstTagList *tags = NULL;
+
+ gst_event_parse_tag (ev,
+ &tags);
+ if ( (NULL == tags) ||
+ (! gst_tag_list_get_string (tags,
+ GST_TAG_LANGUAGE_CODE,
+ &lang)) )
+ lang = NULL;
+ gst_event_unref (ev);
+ }
+ if ( (NULL == lang) &&
+ (NULL != (ev = gst_pad_get_sticky_event (pad,
+ GST_EVENT_STREAM_START,
+ 0))) )
+ {
+ GstStream *stream = NULL;
+
+ gst_event_parse_stream (ev,
+ &stream);
+ if (NULL != stream)
+ {
+ GstTagList *tags = gst_stream_get_tags (stream);
+
+ if (NULL != tags)
+ {
+ if (! gst_tag_list_get_string (tags,
+ GST_TAG_LANGUAGE_CODE,
+ &lang))
+ lang = NULL;
+ gst_tag_list_unref (tags);
+ }
+ gst_object_unref (stream);
+ }
+ gst_event_unref (ev);
+ }
+ return lang;
+}
+
+
+/**
+ * Encode @a cp as UTF-8.
+ *
+ * @param cp code point, valid
+ * @param[out] out at least 4 bytes
+ * @return number of bytes written
+ */
+static size_t
+put_utf8 (uint32_t cp,
+ char *out)
+{
+ if (cp < 0x80)
+ {
+ out[0] = (char) cp;
+ return 1;
+ }
+ if (cp < 0x800)
+ {
+ out[0] = (char) (0xC0 | (cp >> 6));
+ out[1] = (char) (0x80 | (cp & 0x3F));
+ return 2;
+ }
+ if (cp < 0x10000)
+ {
+ out[0] = (char) (0xE0 | (cp >> 12));
+ out[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ out[2] = (char) (0x80 | (cp & 0x3F));
+ return 3;
+ }
+ out[0] = (char) (0xF0 | (cp >> 18));
+ out[1] = (char) (0x80 | ((cp >> 12) & 0x3F));
+ out[2] = (char) (0x80 | ((cp >> 6) & 0x3F));
+ out[3] = (char) (0x80 | (cp & 0x3F));
+ return 4;
+}
+
+
+/**
+ * Decode the XML entity @a name (without '&' and ';').
+ *
+ * @param name entity name
+ * @param len length of @a name
+ * @param[out] cp the code point it stands for
+ * @return 1 if the entity is known
+ */
+static int
+decode_entity (const char *name,
+ size_t len,
+ uint32_t *cp)
+{
+ static const struct
+ {
+ const char *name;
+ uint32_t cp;
+ } named[] = {
+ { "amp", '&' },
+ { "lt", '<' },
+ { "gt", '>' },
+ { "quot", '"' },
+ { "apos", '\'' },
+ { "nbsp", 0xA0 },
+ { NULL, 0 }
+ };
+ uint64_t v = 0;
+ size_t i;
+
+ if ( (len >= 2) &&
+ ('#' == name[0]) )
+ {
+ int hex = ( ('x' == name[1]) || ('X' == name[1]) );
+
+ i = hex ? 2 : 1;
+ if (i == len)
+ return 0;
+ for (; i < len; i++)
+ {
+ char c = name[i];
+ unsigned int d;
+
+ if ( (c >= '0') && (c <= '9') )
+ d = c - '0';
+ else if (hex && (c >= 'a') && (c <= 'f'))
+ d = c - 'a' + 10;
+ else if (hex && (c >= 'A') && (c <= 'F'))
+ d = c - 'A' + 10;
+ else
+ return 0;
+ v = v * (hex ? 16 : 10) + d;
+ if (v > 0x10FFFF)
+ return 0;
+ }
+ if ( (0 == v) ||
+ ( (v >= 0xD800) && (v <= 0xDFFF) ) )
+ return 0;
+ *cp = (uint32_t) v;
+ return 1;
+ }
+ for (i = 0; NULL != named[i].name; i++)
+ if ( (strlen (named[i].name) == len) &&
+ (0 == memcmp (named[i].name, name, len)) )
+ {
+ *cp = named[i].cp;
+ return 1;
+ }
+ return 0;
+}
+
+
+/**
+ * Turn Pango markup into plain text: drop all tags, decode entities.
+ * The result is never longer than the input.
+ *
+ * @param in markup
+ * @param len number of bytes in @a in
+ * @param[out] out at least @a len bytes
+ * @return number of bytes written to @a out
+ */
+static size_t
+strip_markup (const char *in,
+ size_t len,
+ char *out)
+{
+ size_t i = 0;
+ size_t o = 0;
+
+ while ( (i < len) &&
+ ('\0' != in[i]) )
+ {
+ if ('<' == in[i])
+ {
+ while ( (i < len) &&
+ ('>' != in[i]) )
+ i++;
+ i++;
+ continue;
+ }
+ if ('&' == in[i])
+ {
+ size_t e = i + 1;
+ uint32_t cp;
+
+ /* the longest entity we know is "" */
+ while ( (e < len) &&
+ (e - i < 12) &&
+ (';' != in[e]) )
+ e++;
+ if ( (e < len) &&
+ (';' == in[e]) &&
+ (decode_entity (&in[i + 1],
+ e - i - 1,
+ &cp)) )
+ {
+ /* an entity is at least as long as its UTF-8 encoding */
+ o += put_utf8 (cp,
+ &out[o]);
+ i = e + 1;
+ continue;
+ }
+ }
+ out[o++] = in[i++];
+ }
+ return o;
+}
+
+
+/**
+ * Add the subtitle in @a buf to @a t.
+ *
+ * @param t track the buffer belongs to
+ * @param pad pad the buffer arrived at (for its segment)
+ * @param buf the buffer
+ */
+static void
+add_cue (struct SubTrack *t,
+ GstPad *pad,
+ GstBuffer *buf)
+{
+ GstClockTime start = GST_BUFFER_PTS (buf);
+ GstClockTime dur = GST_BUFFER_DURATION (buf);
+ GstClockTime end;
+ GstEvent *ev;
+ GstMapInfo mi;
+
+ if (! GST_CLOCK_TIME_IS_VALID (start))
+ return;
+ if ( (! GST_CLOCK_TIME_IS_VALID (dur)) ||
+ (dur > GST_CLOCK_TIME_NONE - 1 - start) )
+ dur = 0;
+ end = start + dur;
+ if (NULL != (ev = gst_pad_get_sticky_event (pad,
+ GST_EVENT_SEGMENT,
+ 0)))
+ {
+ const GstSegment *seg;
+ guint64 st;
+
+ gst_event_parse_segment (ev,
+ &seg);
+ if ( (GST_FORMAT_TIME == seg->format) &&
+ (GST_CLOCK_TIME_IS_VALID (st =
+ gst_segment_to_stream_time (seg,
+ GST_FORMAT_TIME,
+ start))) )
+ {
+ start = st;
+ end = st + dur;
+ }
+ gst_event_unref (ev);
+ }
+ if (! gst_buffer_map (buf,
+ &mi,
+ GST_MAP_READ))
+ return;
+ if (SUB_MARKUP == t->kind)
+ {
+ char *plain = malloc (mi.size + 1);
+
+ if (NULL != plain)
+ {
+ size_t plen = strip_markup ((const char *) mi.data,
+ mi.size,
+ plain);
+
+ EXTRACTOR_media_srt_add_ (&t->srt,
+ start / GST_MSECOND,
+ end / GST_MSECOND,
+ plain,
+ plen,
+ 0);
+ free (plain);
+ }
+ }
+ else
+ {
+ EXTRACTOR_media_srt_add_ (&t->srt,
+ start / GST_MSECOND,
+ end / GST_MSECOND,
+ (const char *) mi.data,
+ mi.size,
+ (SUB_ASS == t->kind) ? 1 : 0);
+ }
+ gst_buffer_unmap (buf,
+ &mi);
+}
+
+
+/**
+ * Buffer probe on the sink of a subtitle stream: find out the language
+ * (if still unknown) and collect the cue.
+ *
+ * @param pad sink pad of the fakesink
+ * @param info the buffer
+ * @param cls the `struct SubTrack`
+ * @return #GST_PAD_PROBE_OK
+ */
+static GstPadProbeReturn
+sub_probe (GstPad *pad,
+ GstPadProbeInfo *info,
+ gpointer cls)
+{
+ struct SubTrack *t = cls;
+ GstBuffer *buf = GST_PAD_PROBE_INFO_BUFFER (info);
+
+ if (NULL == buf)
+ return GST_PAD_PROBE_OK;
+ if (0 == t->match)
+ {
+ gchar *lang = pad_language (pad);
+
+ if (NULL != lang)
+ {
+ t->match = EXTRACTOR_media_lang_matches_ (t->st->want,
+ lang) ? 1 : -1;
+ g_free (lang);
+ if (-1 == t->match)
+ EXTRACTOR_media_srt_clear_ (&t->srt);
+ }
+ }
+ if (-1 != t->match)
+ add_cue (t,
+ pad,
+ buf);
+ return GST_PAD_PROBE_OK;
+}
+
+
+/**
+ * parsebin exposed a new stream. Connect it to a fakesink; if it is a
+ * text subtitle stream, watch the buffers arriving there.
+ *
+ * @param bin the parsebin
+ * @param pad the new pad
+ * @param cls the `struct SubState`
+ */
+static void
+sub_pad_added (GstElement *bin,
+ GstPad *pad,
+ gpointer cls)
+{
+ struct SubState *st = cls;
+ struct SubTrack *track = NULL;
+ GstCaps *caps;
+ GstElement *sink;
+ GstPad *sinkpad;
+ enum SubKind kind = SUB_PLAIN;
+ int is_sub = 0;
+
+ (void) bin;
+ if (GST_PAD_SRC != GST_PAD_DIRECTION (pad))
+ return;
+ caps = gst_pad_get_current_caps (pad);
+ if (NULL == caps)
+ caps = gst_pad_query_caps (pad,
+ NULL);
+ if ( (NULL != caps) &&
+ (! gst_caps_is_empty (caps)) &&
+ (! gst_caps_is_any (caps)) )
+ {
+ const GstStructure *s = gst_caps_get_structure (caps, 0);
+ const char *name = gst_structure_get_name (s);
+
+ if (0 == strcmp (name,
+ "text/x-raw"))
+ {
+ const char *fmt = gst_structure_get_string (s,
+ "format");
+
+ is_sub = 1;
+ kind = ( (NULL != fmt) &&
+ (0 == strcmp (fmt,
+ "utf8")) )
+ ? SUB_PLAIN
+ : SUB_MARKUP;
+ }
+ else if ( (0 == strcmp (name,
+ "application/x-ass")) ||
+ (0 == strcmp (name,
+ "application/x-ssa")) )
+ {
+ is_sub = 1;
+ kind = SUB_ASS;
+ }
+ }
+ if (NULL != caps)
+ gst_caps_unref (caps);
+ g_mutex_lock (&st->lock);
+ if (st->num_streams >= MAX_STREAMS)
+ {
+ g_mutex_unlock (&st->lock);
+ return;
+ }
+ st->num_streams++;
+ if (is_sub &&
+ (st->num_tracks < MAX_SUB_TRACKS) )
+ {
+ track = &st->tracks[st->num_tracks++];
+ track->st = st;
+ track->kind = kind;
+ }
+ g_mutex_unlock (&st->lock);
+ if (NULL == (sink = gst_element_factory_make ("fakesink",
+ NULL)))
+ return;
+ g_object_set (sink,
+ "sync", FALSE,
+ "async", FALSE,
+ NULL);
+ if (! gst_bin_add (GST_BIN (st->pipeline),
+ sink))
+ {
+ gst_object_unref (sink);
+ return;
+ }
+ sinkpad = gst_element_get_static_pad (sink,
+ "sink");
+ if (NULL != track)
+ gst_pad_add_probe (sinkpad,
+ GST_PAD_PROBE_TYPE_BUFFER,
+ &sub_probe,
+ track,
+ NULL);
+ (void) gst_element_sync_state_with_parent (sink);
+ (void) gst_pad_link (pad,
+ sinkpad);
+ gst_object_unref (sinkpad);
+}
+
+
+/**
+ * Dump the first subtitle track in language opt->srt_lang as SRT.
+ * The whole file is demuxed (not decoded) for this; if we run out of
+ * time or the file turns out to be broken halfway, the cues found so
+ * far are emitted.
+ *
+ * @param ec extraction context
+ * @param opt plugin options
+ * @param deadline when we must be done
+ * @return return value of `proc` (0 if nothing was emitted)
+ */
+static int
+dump_subtitles (struct EXTRACTOR_ExtractContext *ec,
+ const struct EXTRACTOR_MediaOptions *opt,
+ const struct EXTRACTOR_MediaDeadline *deadline)
+{
+ struct EXTRACTOR_GstIO io;
+ struct SubState st;
+ GstElement *pipeline = NULL;
+ GstElement *src;
+ GstElement *parse;
+ GstBus *bus;
+ unsigned int i;
+ int ret = 0;
+
+ memset (&st, 0, sizeof (st));
+ g_mutex_init (&st.lock);
+ st.want = opt->srt_lang;
+ src = EXTRACTOR_gst_appsrc_ (&io,
+ ec);
+ parse = gst_element_factory_make ("parsebin",
+ NULL);
+ pipeline = gst_pipeline_new (NULL);
+ if ( (NULL == src) ||
+ (NULL == parse) ||
+ (NULL == pipeline) )
+ {
+ if (NULL != src)
+ gst_object_unref (src);
+ if (NULL != parse)
+ gst_object_unref (parse);
+ if (NULL != pipeline)
+ gst_object_unref (pipeline);
+ g_mutex_clear (&st.lock);
+ return 0;
+ }
+ gst_bin_add_many (GST_BIN (pipeline),
+ src,
+ parse,
+ NULL);
+ st.pipeline = pipeline;
+ g_signal_connect (parse,
+ "pad-added",
+ G_CALLBACK (&sub_pad_added),
+ &st);
+ if (! gst_element_link (src,
+ parse))
+ goto cleanup;
+ if (GST_STATE_CHANGE_FAILURE ==
+ gst_element_set_state (pipeline,
+ GST_STATE_PLAYING))
+ goto cleanup;
+ if (NULL != (bus = gst_element_get_bus (pipeline)))
+ {
+ GstClockTime timeout;
+ GstMessage *msg = NULL;
+
+ /* wait in short slices, so that we can tell the host that we are
+ still alive (see EXTRACTOR_gst_keepalive_()) */
+ while ( (NULL == msg) &&
+ (0 != (timeout = EXTRACTOR_media_deadline_remaining_ (deadline)
+ * GST_MSECOND)) )
+ {
+ if (timeout > 100 * GST_MSECOND)
+ timeout = 100 * GST_MSECOND;
+ msg = gst_bus_timed_pop_filtered (bus,
+ timeout,
+ GST_MESSAGE_ERROR
+ | GST_MESSAGE_EOS);
+ if (NULL == msg)
+ EXTRACTOR_gst_keepalive_ (&io);
+ }
+ if (NULL != msg)
+ gst_message_unref (msg);
+ gst_object_unref (bus);
+ }
+ /* this joins the streaming threads; afterwards nobody touches st */
+ gst_element_set_state (pipeline,
+ GST_STATE_NULL);
+ for (i = 0; i < st.num_tracks; i++)
+ if (1 == st.tracks[i].match)
+ {
+ ret = EXTRACTOR_media_srt_emit_ (&st.tracks[i].srt,
+ ec,
+ "thumbnailgst");
+ break;
+ }
+cleanup:
+ gst_element_set_state (pipeline,
+ GST_STATE_NULL);
+ for (i = 0; i < st.num_tracks; i++)
+ EXTRACTOR_media_srt_clear_ (&st.tracks[i].srt);
+ gst_object_unref (pipeline);
+ g_mutex_clear (&st.lock);
+ return ret;
+}
+
+
+/**
+ * Main method for the gstreamer-thumbnailer plugin.
+ *
+ * @param ec extraction context
+ */
+void
+EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec);
+
+void
+EXTRACTOR_thumbnailgst_extract_method (struct EXTRACTOR_ExtractContext *ec)
+{
+ struct EXTRACTOR_MediaOptions opt;
+ struct EXTRACTOR_MediaDeadline deadline;
+ int ret = 0;
+
+ EXTRACTOR_media_options_parse_ (ec->config,
+ &opt);
+ EXTRACTOR_media_deadline_start_ (&deadline,
+ opt.deadline);
+ if (! EXTRACTOR_media_is_media_ (ec, 0))
+ return;
+ if (! gst_is_initialized ())
+ gst_init (NULL, NULL);
+ if ( (0 == opt.gallery) ||
+ (! make_gallery (ec,
+ &opt,
+ &deadline,
+ &ret)) )
+ ret = make_thumbnail (ec,
+ &opt,
+ &deadline);
+ if ( (0 == ret) &&
+ ('\0' != opt.srt_lang[0]) )
+ (void) dump_subtitles (ec,
+ &opt,
+ &deadline);
}