libextractor

GNU libextractor
Log | Files | Refs | Submodules | README | LICENSE

test_paged_fulltext.h (8039B)

      1 /*
      2      This file is part of libextractor.
      3      Copyright (C) 2026 Christian Grothoff
      4 
      5      libextractor is free software; you can redistribute it and/or modify
      6      it under the terms of the GNU General Public License as published
      7      by the Free Software Foundation; either version 3, or (at your
      8      option) any later version.
      9 
     10      libextractor is distributed in the hope that it will be useful, but
     11      WITHOUT ANY WARRANTY; without even the implied warranty of
     12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13      General Public License for more details.
     14 
     15      You should have received a copy of the GNU General Public License
     16      along with libextractor; see the file COPYING.  If not, write to the
     17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18      Boston, MA 02110-1301, USA.
     19 */
     20 /**
     21  * @file plugins/test_paged_fulltext.h
     22  * @brief checks on the "fulltext" output of the paged-format plugins
     23  *        (pdf, dvi) for the documents made by
     24  *        contrib/gen_fulltext_pdf_dvi_testdata.sh
     25  * @author Christian Grothoff
     26  */
     27 #ifndef TEST_PAGED_FULLTEXT_H
     28 #define TEST_PAGED_FULLTEXT_H
     29 
     30 #include "platform.h"
     31 #include "test_lib.h"
     32 #include <ctype.h>
     33 #include <unistd.h>
     34 
     35 /**
     36  * Number of FULL_TEXT items seen by #count_fulltext().
     37  */
     38 static unsigned int fulltext_items;
     39 
     40 
     41 /**
     42  * Count FULL_TEXT items.
     43  *
     44  * @param cls unused
     45  * @param plugin_name name of the plugin
     46  * @param type type of the meta data
     47  * @param format format of the meta data
     48  * @param data_mime_type mime type of the meta data
     49  * @param data the meta data
     50  * @param data_len number of bytes in @a data
     51  * @return 0 (continue)
     52  */
     53 static int
     54 count_fulltext (void *cls,
     55                 const char *plugin_name,
     56                 enum EXTRACTOR_MetaType type,
     57                 enum EXTRACTOR_MetaFormat format,
     58                 const char *data_mime_type,
     59                 const char *data,
     60                 size_t data_len)
     61 {
     62   (void) cls;
     63   (void) plugin_name;
     64   (void) format;
     65   (void) data_mime_type;
     66   (void) data;
     67   (void) data_len;
     68   if (EXTRACTOR_METATYPE_FULL_TEXT == type)
     69     fulltext_items++;
     70   return 0;
     71 }
     72 
     73 
     74 /**
     75  * Check that running @a plugin on @a filename yields exactly
     76  * @a expected FULL_TEXT items.
     77  *
     78  * @param plugin plugin name and options
     79  * @param filename file to extract from
     80  * @param expected number of FULL_TEXT items expected
     81  * @return 0 on success, 1 on failure
     82  */
     83 static int
     84 check_fulltext_count (const char *plugin,
     85                       const char *filename,
     86                       unsigned int expected)
     87 {
     88   struct EXTRACTOR_PluginList *pl;
     89 
     90   if (0 != access (filename, R_OK))
     91     return 0;
     92   pl = EXTRACTOR_plugin_add_config (NULL,
     93                                     plugin,
     94                                     EXTRACTOR_OPTION_DEFAULT_POLICY);
     95   fulltext_items = 0;
     96   EXTRACTOR_extract (pl,
     97                      filename,
     98                      NULL, 0,
     99                      &count_fulltext,
    100                      NULL);
    101   EXTRACTOR_plugin_remove_all (pl);
    102   if (expected == fulltext_items)
    103     return 0;
    104   fprintf (stderr,
    105            "%s on %s: expected %u FULL_TEXT items, got %u\n",
    106            plugin,
    107            filename,
    108            expected,
    109            fulltext_items);
    110   return 1;
    111 }
    112 
    113 
    114 /**
    115  * Check the general form of a full text: valid UTF-8, no leading or
    116  * trailing whitespace, no control characters except the "\n\n"
    117  * paragraph separators, no double spaces.
    118  *
    119  * @param data the text, 0-terminated
    120  * @param data_len number of bytes in @a data including the 0
    121  * @return number of paragraphs, 0 if malformed
    122  */
    123 static unsigned int
    124 check_form (const char *data,
    125             size_t data_len)
    126 {
    127   const unsigned char *s = (const unsigned char *) data;
    128   size_t len;
    129   size_t i;
    130   unsigned int pars = 1;
    131 
    132   if ( (0 == data_len) ||
    133        ('\0' != data[data_len - 1]) )
    134     return 0;
    135   len = data_len - 1;
    136   if ( (0 == len) ||
    137        (strlen (data) != len) ||
    138        (isspace (s[0])) ||
    139        (isspace (s[len - 1])) )
    140     return 0;
    141   for (i = 0; i < len; i++)
    142   {
    143     unsigned char c = s[i];
    144     unsigned int n;
    145     unsigned int k;
    146 
    147     if ('\n' == c)
    148     {
    149       if ( (i + 2 >= len) ||
    150            ('\n' != s[i + 1]) ||
    151            (isspace (s[i + 2])) ||
    152            (' ' == s[i - 1]) )
    153         return 0;
    154       pars++;
    155       i++;
    156       continue;
    157     }
    158     if (' ' == c)
    159     {
    160       if (' ' == s[i + 1])
    161         return 0;
    162       continue;
    163     }
    164     if (c < 0x20)
    165       return 0;
    166     if (c < 0x80)
    167       continue;
    168     if ( (c & 0xE0) == 0xC0)
    169       n = 1;
    170     else if ( (c & 0xF0) == 0xE0)
    171       n = 2;
    172     else if ( (c & 0xF8) == 0xF0)
    173       n = 3;
    174     else
    175       return 0;
    176     if (i + n >= len)
    177       return 0;
    178     for (k = 1; k <= n; k++)
    179       if ( (s[i + k] & 0xC0) != 0x80)
    180         return 0;
    181     i += n;
    182   }
    183   return pars;
    184 }
    185 
    186 
    187 /**
    188  * Check that the paragraphs of @a data start with @a starts, in order.
    189  *
    190  * @param data the text
    191  * @param starts NULL-terminated list of paragraph prefixes
    192  * @return 1 if all match
    193  */
    194 static int
    195 check_paragraphs (const char *data,
    196                   const char *const *starts)
    197 {
    198   const char *p = data;
    199   unsigned int i;
    200 
    201   for (i = 0; NULL != starts[i]; i++)
    202   {
    203     if (NULL == p)
    204     {
    205       fprintf (stderr, "Missing paragraph `%s'\n", starts[i]);
    206       return 0;
    207     }
    208     if (0 != strncmp (p, starts[i], strlen (starts[i])))
    209     {
    210       fprintf (stderr,
    211                "Paragraph %u starts with `%.40s', expected `%s'\n",
    212                i, p, starts[i]);
    213       return 0;
    214     }
    215     p = strstr (p, "\n\n");
    216     if (NULL != p)
    217       p += 2;
    218   }
    219   if (NULL != p)
    220   {
    221     fprintf (stderr, "Unexpected paragraph `%.40s'\n", p);
    222     return 0;
    223   }
    224   return 1;
    225 }
    226 
    227 
    228 /**
    229  * Check that all of @a needles occur in @a data, and none of
    230  * @a absent.
    231  *
    232  * @param data the text
    233  * @param needles NULL-terminated list of required substrings
    234  * @param absent NULL-terminated list of forbidden substrings
    235  * @return 1 if so
    236  */
    237 static int
    238 check_contains (const char *data,
    239                 const char *const *needles,
    240                 const char *const *absent)
    241 {
    242   unsigned int i;
    243 
    244   for (i = 0; NULL != needles[i]; i++)
    245     if (NULL == strstr (data, needles[i]))
    246     {
    247       fprintf (stderr, "Full text lacks `%s'\n", needles[i]);
    248       return 0;
    249     }
    250   for (i = 0; NULL != absent[i]; i++)
    251     if (NULL != strstr (data, absent[i]))
    252     {
    253       fprintf (stderr, "Full text contains `%s'\n", absent[i]);
    254       return 0;
    255     }
    256   return 1;
    257 }
    258 
    259 
    260 /**
    261  * Check that no word of @a data consists of digits only (page and line
    262  * numbers must be gone; the test documents have no other numbers).
    263  *
    264  * @param data the text
    265  * @return 1 if there is no such word
    266  */
    267 static int
    268 check_no_numbers (const char *data)
    269 {
    270   const char *p = data;
    271 
    272   while ('\0' != *p)
    273   {
    274     const char *e = p;
    275     int digits = 1;
    276 
    277     while ( ('\0' != *e) && (' ' != *e) && ('\n' != *e) )
    278     {
    279       if (! isdigit ((unsigned char) *e))
    280         digits = 0;
    281       e++;
    282     }
    283     if ( (e > p) && digits)
    284     {
    285       fprintf (stderr, "Full text contains number `%.*s'\n",
    286                (int) (e - p), p);
    287       return 0;
    288     }
    289     p = e;
    290     while ( (' ' == *p) || ('\n' == *p) )
    291       p++;
    292   }
    293   return 1;
    294 }
    295 
    296 
    297 /**
    298  * Sentences both test documents must yield: re-joined hyphenations,
    299  * ligatures, quotes and dashes, and a paragraph across a page break.
    300  */
    301 static const char *const common_needles[] = {
    302   "put them back in reading order. This first paragraph",
    303   "the extractor should notice that indentation and start",
    304   "the office has fluffy waffles, an efficient staff and a "
    305   "\xe2\x80\x9cquoted phrase\xe2\x80\x9d with pages 10\xe2\x80\x93" "20 "
    306   "and a pause\xe2\x80\x94like this one.",
    307   "nobody will ever read, apart from the test suite, which checks every "
    308   "single word of it.",
    309   "and the extractor must join both halves",
    310   "about zebras jumping over lazy foxes.",
    311   NULL
    312 };
    313 
    314 /**
    315  * Page furniture that must be gone.
    316  */
    317 static const char *const common_absent[] = {
    318   "Notes on Paged Text",
    319   "Working Draft",
    320   "extrac- tor",
    321   "hap- pens",
    322   NULL
    323 };
    324 
    325 #endif