libextractor

fulltext.h (9713B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2026 Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file plugins/fulltext.h
  * @brief helpers shared by the plugins implementing the "fulltext" option
  * @author Christian Grothoff
  *
  * The full text is reported as a single #EXTRACTOR_METATYPE_FULL_TEXT
  * item in UTF-8 with MIME type "text/plain".  Within a paragraph, all
  * whitespace is collapsed into a single space; paragraphs are separated
  * by exactly "\n\n"; there is no leading or trailing whitespace.
  */
 #ifndef FULLTEXT_H
 #define FULLTEXT_H
 
 #include "extractor.h"
 #include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
  * Largest full text we are willing to emit, in bytes (including the
  * terminating 0).  Must stay below MAX_META_DATA in
  * src/main/extractor_ipc.h; longer text is truncated at a character
  * boundary.
  */
 #define EXTRACTOR_FULLTEXT_MAX (16 * 1024 * 1024)
 
 
 /**
  * Check whether the comma-separated plugin configuration @a config
  * contains the token @a key (exactly, or followed by '=' or ':').
  *
  * @param config plugin configuration, may be NULL
  * @param key token to look for, e.g. "fulltext"
  * @return NULL if @a key is absent; otherwise a pointer to the
  *         character after @a key in @a config (one of '\0', ',', '='
  *         or ':')
  */
 const char *
 EXTRACTOR_config_find_ (const char *config,
                         const char *key);
 
 
 /**
  * Check whether the "fulltext" option is set in @a config.
  *
  * @param config plugin configuration, may be NULL
  * @return 1 if full-text extraction was requested, 0 if not
  */
 int
 EXTRACTOR_fulltext_requested_ (const char *config);
 
 
 /**
  * A growing UTF-8 text buffer that normalizes whitespace and paragraph
  * breaks as text is appended.
  */
 struct EXTRACTOR_FullText
 {
   /**
    * The text so far, 0-terminated if @e len > 0.
    */
   char *buf;
 
   /**
    * Number of bytes in @e buf, excluding the terminating 0.
    */
   size_t len;
 
   /**
    * Allocated size of @e buf.
    */
   size_t cap;
 
   /**
    * 1 if a space must precede the next non-space text.
    */
   int pending_space;
 
   /**
    * 1 if a paragraph break must precede the next non-space text.
    */
   int pending_par;
 
   /**
    * 1 if we ran into #EXTRACTOR_FULLTEXT_MAX or out of memory;
    * all further text is discarded.
    */
   int full;
 
   /**
    * Extraction context to keep alive while we work, may be NULL; set
    * with #EXTRACTOR_fulltext_set_context_().
    */
   struct EXTRACTOR_ExtractContext *ec;
 
   /**
    * Units of work done since the last keep-alive.
    */
   unsigned int ticks;
 };
 
 
 /**
  * Let @a ft keep the host informed that the plugin is still working
  * (out of process, a plugin that is silent for too long is presumed
  * hung): every few thousand characters appended, or calls to
  * #EXTRACTOR_fulltext_tick_(), `seek (cls, 0, SEEK_CUR)` is called
  * on @a ec.
  *
  * @param ft buffer to modify
  * @param ec extraction context of the plugin
  */
 void
 EXTRACTOR_fulltext_set_context_ (struct EXTRACTOR_FullText *ft,
                                  struct EXTRACTOR_ExtractContext *ec);
 
 
 /**
  * Account for one unit of work that did not append text (e.g. a tag
  * that was skipped), for the keep-alive.
  *
  * @param ft buffer being filled
  */
 void
 EXTRACTOR_fulltext_tick_ (struct EXTRACTOR_FullText *ft);
 
 
 /**
  * Initialize @a ft to the empty text.
  *
  * @param[out] ft buffer to initialize
  */
 void
 EXTRACTOR_fulltext_init_ (struct EXTRACTOR_FullText *ft);
 
 
 /**
  * Append @a len bytes of @a text.  Invalid UTF-8 is replaced by U+FFFD,
  * control characters are dropped, and any run of whitespace (including
  * line breaks and U+00A0) becomes a single space.
  *
  * @param ft buffer to append to
  * @param text text to append, need not be 0-terminated
  * @param len number of bytes in @a text
  */
 void
 EXTRACTOR_fulltext_append_ (struct EXTRACTOR_FullText *ft,
                             const char *text,
                             size_t len);
 
 
 /**
  * Append a 0-terminated string.
  *
  * @param ft buffer to append to
  * @param text 0-terminated text to append
  */
 void
 EXTRACTOR_fulltext_append_str_ (struct EXTRACTOR_FullText *ft,
                                 const char *text);
 
 
 /**
  * Append a single Unicode code point, subject to the same
  * normalization as #EXTRACTOR_fulltext_append_().
  *
  * @param ft buffer to append to
  * @param cp code point to append
  */
 void
 EXTRACTOR_fulltext_append_cp_ (struct EXTRACTOR_FullText *ft,
                                uint32_t cp);
 
 
 /**
  * Request a word break: the next text is separated from what came
  * before by a single space (unless a paragraph break is pending).
  *
  * @param ft buffer to modify
  */
 void
 EXTRACTOR_fulltext_space_ (struct EXTRACTOR_FullText *ft);
 
 
 /**
  * End the current paragraph: the next text is preceded by "\n\n".
  * Multiple calls without text in between have the same effect as one.
  *
  * @param ft buffer to modify
  */
 void
 EXTRACTOR_fulltext_paragraph_ (struct EXTRACTOR_FullText *ft);
 
 
 /**
  * Emit the text (if non-empty) as #EXTRACTOR_METATYPE_FULL_TEXT.
  *
  * @param ft buffer to emit
  * @param ec extraction context
  * @param plugin_name short name of the plugin
  * @return return value of the `proc` callback (0 if nothing was emitted)
  */
 int
 EXTRACTOR_fulltext_emit_ (const struct EXTRACTOR_FullText *ft,
                           struct EXTRACTOR_ExtractContext *ec,
                           const char *plugin_name);
 
 
 /**
  * Release the memory held by @a ft and reset it to the empty text.
  *
  * @param ft buffer to clean up
  */
 void
 EXTRACTOR_fulltext_clear_ (struct EXTRACTOR_FullText *ft);
 
 
 /**
  * Check whether @a line (already whitespace-trimmed or not) looks like
  * a page number or page label, e.g. "12", "- 12 -", "xiv", "Page 3",
  * "3 of 17", "[4]", "Seite 7".
  *
  * @param line text to check
  * @param len number of bytes in @a line
  * @return 1 if @a line looks like a page number, 0 if not
  */
 int
 EXTRACTOR_fulltext_is_page_number_ (const char *line,
                                     size_t len);
 
 
 /**
  * One line of text on a page, as collected by a plugin for paged
  * formats (PDF, DVI) before margin filtering.
  */
 struct EXTRACTOR_FullTextLine
 {
   /**
    * UTF-8 text of the line, 0-terminated, owned by the page.
    */
   char *text;
 
   /**
    * Vertical position of the line's baseline, relative to the page
    * height, in [0,1] with 0 at the top.  Negative if unknown.
    */
   double y;
 
   /**
    * 1 if this line ends a paragraph (the plugin decided that from the
    * layout, e.g. vertical gap, indentation, short last line).
    */
   int par_end;
 
   /**
    * Set by #EXTRACTOR_fulltext_filter_margins_() if the line is page
    * furniture (page number, running header/footer) and must be skipped.
    */
   int skip;
 };
 
 
 /**
  * All lines of one page, in reading order.
  */
 struct EXTRACTOR_FullTextPage
 {
   /**
    * Lines of the page.
    */
   struct EXTRACTOR_FullTextLine *lines;
 
   /**
    * Number of entries in @e lines.
    */
   unsigned int num_lines;
 
   /**
    * Allocated size of @e lines.
    */
   unsigned int cap_lines;
 };
 
 
 /**
  * Append a line to @a page.  @a text is copied.
  *
  * @param page page to append to
  * @param text UTF-8 text of the line
  * @param len number of bytes in @a text
  * @param y relative vertical position in [0,1], or negative if unknown
  * @param par_end 1 if the line ends a paragraph
  * @return 0 on success, -1 on allocation failure
  */
 int
 EXTRACTOR_fulltext_page_add_ (struct EXTRACTOR_FullTextPage *page,
                               const char *text,
                               size_t len,
                               double y,
                               int par_end);
 
 
 /**
  * Mark page furniture in @a pages as `skip`: the first/last lines of
  * a page within the top/bottom margin band that look like page numbers,
  * or whose text (with digits ignored) recurs in the same band on at
  * least half of the pages (when there are at least 3 pages).
  *
  * @param pages array of pages
  * @param num_pages number of entries in @a pages
  */
 void
 EXTRACTOR_fulltext_filter_margins_ (struct EXTRACTOR_FullTextPage *pages,
                                     unsigned int num_pages);
 
 
 /**
  * Append all non-skipped lines of @a pages to @a ft.  Lines are joined
  * with a space (words hyphenated across a line break are re-joined);
  * `par_end` lines end a paragraph.  A paragraph continues across page
  * boundaries unless the last line of the page has `par_end`.
  *
  * @param ft buffer to append to
  * @param pages array of pages
  * @param num_pages number of entries in @a pages
  */
 void
 EXTRACTOR_fulltext_append_pages_ (struct EXTRACTOR_FullText *ft,
                                   const struct EXTRACTOR_FullTextPage *pages,
                                   unsigned int num_pages);
 
 
 /**
  * Release the lines of @a page.
  *
  * @param page page to clean up
  */
 void
 EXTRACTOR_fulltext_page_clear_ (struct EXTRACTOR_FullTextPage *page);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif