fulltext.c - libextractor - GNU libextractor

fulltext.c (21121B)
      1 /*
      2      This file is part of libextractor.
      3      Copyright (C) 2026 Christian Grothoff
      4 
      5      libextractor is free software; you can redistribute it and/or modify
      6      it under the terms of the GNU General Public License as published
      7      by the Free Software Foundation; either version 3, or (at your
      8      option) any later version.
      9 
     10      libextractor is distributed in the hope that it will be useful, but
     11      WITHOUT ANY WARRANTY; without even the implied warranty of
     12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13      General Public License for more details.
     14 
     15      You should have received a copy of the GNU General Public License
     16      along with libextractor; see the file COPYING.  If not, write to the
     17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18      Boston, MA 02110-1301, USA.
     19  */
     20 /**
     21  * @file plugins/fulltext.c
     22  * @brief helpers shared by the plugins implementing the "fulltext" option
     23  * @author Christian Grothoff
     24  */
     25 #include "platform.h"
     26 #include "fulltext.h"
     27 #include <strings.h>
     28 
     29 
     30 const char *
     31 EXTRACTOR_config_find_ (const char *config,
     32                         const char *key)
     33 {
     34   const char *pos;
     35   size_t klen;
     36 
     37   if ( (NULL == config) ||
     38        (NULL == key) )
     39     return NULL;
     40   klen = strlen (key);
     41   pos = config;
     42   while (1)
     43   {
     44     if ( (0 == strncmp (pos,
     45                         key,
     46                         klen)) &&
     47          ( ('\0' == pos[klen]) ||
     48            (',' == pos[klen]) ||
     49            ('=' == pos[klen]) ||
     50            (':' == pos[klen]) ) )
     51       return &pos[klen];
     52     pos = strchr (pos, ',');
     53     if (NULL == pos)
     54       return NULL;
     55     pos++;
     56   }
     57 }
     58 
     59 
     60 int
     61 EXTRACTOR_fulltext_requested_ (const char *config)
     62 {
     63   return (NULL != EXTRACTOR_config_find_ (config,
     64                                           "fulltext")) ? 1 : 0;
     65 }
     66 
     67 
     68 void
     69 EXTRACTOR_fulltext_init_ (struct EXTRACTOR_FullText *ft)
     70 {
     71   memset (ft, 0, sizeof (*ft));
     72 }
     73 
     74 
     75 void
     76 EXTRACTOR_fulltext_set_context_ (struct EXTRACTOR_FullText *ft,
     77                                  struct EXTRACTOR_ExtractContext *ec)
     78 {
     79   ft->ec = ec;
     80 }
     81 
     82 
     83 /**
     84  * Number of units of work between two keep-alives; the plugin side
     85  * of the IPC rate-limits them by time anyway, so this only bounds
     86  * the cost of asking.
     87  */
     88 #define TICKS_PER_ALIVE 4096
     89 
     90 
     91 void
     92 EXTRACTOR_fulltext_tick_ (struct EXTRACTOR_FullText *ft)
     93 {
     94   if (NULL == ft->ec)
     95     return;
     96   if (++ft->ticks < TICKS_PER_ALIVE)
     97     return;
     98   ft->ticks = 0;
     99   (void) ft->ec->seek (ft->ec->cls,
    100                        0,
    101                        SEEK_CUR);
    102 }
    103 
    104 
    105 /**
    106  * Check whether @a cp is whitespace for our purposes.
    107  *
    108  * @param cp code point to check
    109  * @return 1 if @a cp is whitespace
    110  */
    111 static int
    112 is_space_cp (uint32_t cp)
    113 {
    114   switch (cp)
    115   {
    116   case ' ':
    117   case '\t':
    118   case '\n':
    119   case '\r':
    120   case '\f':
    121   case '\v':
    122   case 0x85:
    123   case 0xA0:
    124   case 0x1680:
    125   case 0x2028:
    126   case 0x2029:
    127   case 0x202F:
    128   case 0x205F:
    129   case 0x3000:
    130     return 1;
    131   default:
    132     return ( (cp >= 0x2000) && (cp <= 0x200A) ) ? 1 : 0;
    133   }
    134 }
    135 
    136 
    137 /**
    138  * Check whether @a cp must be dropped silently: control characters,
    139  * soft hyphens, zero-width characters and byte order marks.
    140  *
    141  * @param cp code point to check
    142  * @return 1 if @a cp is to be dropped
    143  */
    144 static int
    145 is_dropped_cp (uint32_t cp)
    146 {
    147   if ( (cp < 0x20) ||
    148        ( (cp >= 0x7F) && (cp < 0xA0) ) )
    149     return 1;
    150   switch (cp)
    151   {
    152   case 0xAD:
    153   case 0x200B:
    154   case 0x200C:
    155   case 0x200D:
    156   case 0x2060:
    157   case 0xFEFF:
    158     return 1;
    159   default:
    160     return 0;
    161   }
    162 }
    163 
    164 
    165 /**
    166  * Make room for @a extra more bytes plus the terminating 0.
    167  *
    168  * @param ft buffer to grow
    169  * @param extra number of bytes we want to append
    170  * @return 0 on success, -1 if the buffer is full
    171  */
    172 static int
    173 reserve (struct EXTRACTOR_FullText *ft,
    174          size_t extra)
    175 {
    176   size_t need;
    177   size_t ncap;
    178   char *nbuf;
    179 
    180   if (ft->full)
    181     return -1;
    182   need = ft->len + extra + 1;
    183   if (need > EXTRACTOR_FULLTEXT_MAX)
    184   {
    185     ft->full = 1;
    186     return -1;
    187   }
    188   if (need <= ft->cap)
    189     return 0;
    190   ncap = (0 == ft->cap) ? 4096 : ft->cap;
    191   while (ncap < need)
    192     ncap *= 2;
    193   if (ncap > EXTRACTOR_FULLTEXT_MAX)
    194     ncap = EXTRACTOR_FULLTEXT_MAX;
    195   nbuf = realloc (ft->buf,
    196                   ncap);
    197   if (NULL == nbuf)
    198   {
    199     ft->full = 1;
    200     return -1;
    201   }
    202   ft->buf = nbuf;
    203   ft->cap = ncap;
    204   return 0;
    205 }
    206 
    207 
    208 void
    209 EXTRACTOR_fulltext_append_cp_ (struct EXTRACTOR_FullText *ft,
    210                                uint32_t cp)
    211 {
    212   char enc[4];
    213   size_t elen;
    214   const char *sep;
    215   size_t slen;
    216 
    217   if (ft->full)
    218     return;
    219   EXTRACTOR_fulltext_tick_ (ft);
    220   if (is_space_cp (cp))
    221   {
    222     EXTRACTOR_fulltext_space_ (ft);
    223     return;
    224   }
    225   if (is_dropped_cp (cp))
    226     return;
    227   if ( (cp > 0x10FFFF) ||
    228        ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
    229     cp = 0xFFFD;
    230   if (cp < 0x80)
    231   {
    232     enc[0] = (char) cp;
    233     elen = 1;
    234   }
    235   else if (cp < 0x800)
    236   {
    237     enc[0] = (char) (0xC0 | (cp >> 6));
    238     enc[1] = (char) (0x80 | (cp & 0x3F));
    239     elen = 2;
    240   }
    241   else if (cp < 0x10000)
    242   {
    243     enc[0] = (char) (0xE0 | (cp >> 12));
    244     enc[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
    245     enc[2] = (char) (0x80 | (cp & 0x3F));
    246     elen = 3;
    247   }
    248   else
    249   {
    250     enc[0] = (char) (0xF0 | (cp >> 18));
    251     enc[1] = (char) (0x80 | ((cp >> 12) & 0x3F));
    252     enc[2] = (char) (0x80 | ((cp >> 6) & 0x3F));
    253     enc[3] = (char) (0x80 | (cp & 0x3F));
    254     elen = 4;
    255   }
    256   sep = "";
    257   slen = 0;
    258   if (ft->len > 0)
    259   {
    260     if (ft->pending_par)
    261     {
    262       sep = "\n\n";
    263       slen = 2;
    264     }
    265     else if (ft->pending_space)
    266     {
    267       sep = " ";
    268       slen = 1;
    269     }
    270   }
    271   if (0 != reserve (ft,
    272                     slen + elen))
    273     return;
    274   memcpy (&ft->buf[ft->len],
    275           sep,
    276           slen);
    277   ft->len += slen;
    278   memcpy (&ft->buf[ft->len],
    279           enc,
    280           elen);
    281   ft->len += elen;
    282   ft->buf[ft->len] = '\0';
    283   ft->pending_par = 0;
    284   ft->pending_space = 0;
    285 }
    286 
    287 
    288 /**
    289  * Decode one UTF-8 sequence.
    290  *
    291  * @param s input
    292  * @param len number of bytes available at @a s, at least 1
    293  * @param[out] cp decoded code point, U+FFFD if invalid
    294  * @return number of bytes consumed (at least 1)
    295  */
    296 static size_t
    297 decode_utf8 (const unsigned char *s,
    298              size_t len,
    299              uint32_t *cp)
    300 {
    301   uint32_t c;
    302   size_t n;
    303   size_t i;
    304   uint32_t min;
    305 
    306   c = s[0];
    307   if (c < 0x80)
    308   {
    309     *cp = c;
    310     return 1;
    311   }
    312   if ( (c & 0xE0) == 0xC0)
    313   {
    314     n = 2;
    315     c &= 0x1F;
    316     min = 0x80;
    317   }
    318   else if ( (c & 0xF0) == 0xE0)
    319   {
    320     n = 3;
    321     c &= 0x0F;
    322     min = 0x800;
    323   }
    324   else if ( (c & 0xF8) == 0xF0)
    325   {
    326     n = 4;
    327     c &= 0x07;
    328     min = 0x10000;
    329   }
    330   else
    331   {
    332     *cp = 0xFFFD;
    333     return 1;
    334   }
    335   if (n > len)
    336   {
    337     *cp = 0xFFFD;
    338     return 1;
    339   }
    340   for (i = 1; i < n; i++)
    341   {
    342     if ( (s[i] & 0xC0) != 0x80)
    343     {
    344       *cp = 0xFFFD;
    345       return 1;
    346     }
    347     c = (c << 6) | (s[i] & 0x3F);
    348   }
    349   if ( (c < min) ||
    350        (c > 0x10FFFF) ||
    351        ( (c >= 0xD800) && (c <= 0xDFFF) ) )
    352   {
    353     *cp = 0xFFFD;
    354     return 1;
    355   }
    356   *cp = c;
    357   return n;
    358 }
    359 
    360 
    361 void
    362 EXTRACTOR_fulltext_append_ (struct EXTRACTOR_FullText *ft,
    363                             const char *text,
    364                             size_t len)
    365 {
    366   const unsigned char *s = (const unsigned char *) text;
    367   size_t off;
    368   uint32_t cp;
    369 
    370   off = 0;
    371   while ( (off < len) &&
    372           (! ft->full) )
    373   {
    374     off += decode_utf8 (&s[off],
    375                         len - off,
    376                         &cp);
    377     EXTRACTOR_fulltext_append_cp_ (ft,
    378                                    cp);
    379   }
    380 }
    381 
    382 
    383 void
    384 EXTRACTOR_fulltext_append_str_ (struct EXTRACTOR_FullText *ft,
    385                                 const char *text)
    386 {
    387   if (NULL == text)
    388     return;
    389   EXTRACTOR_fulltext_append_ (ft,
    390                               text,
    391                               strlen (text));
    392 }
    393 
    394 
    395 void
    396 EXTRACTOR_fulltext_space_ (struct EXTRACTOR_FullText *ft)
    397 {
    398   if (ft->len > 0)
    399     ft->pending_space = 1;
    400 }
    401 
    402 
    403 void
    404 EXTRACTOR_fulltext_paragraph_ (struct EXTRACTOR_FullText *ft)
    405 {
    406   if (ft->len > 0)
    407     ft->pending_par = 1;
    408 }
    409 
    410 
    411 int
    412 EXTRACTOR_fulltext_emit_ (const struct EXTRACTOR_FullText *ft,
    413                           struct EXTRACTOR_ExtractContext *ec,
    414                           const char *plugin_name)
    415 {
    416   if (0 == ft->len)
    417     return 0;
    418   return ec->proc (ec->cls,
    419                    plugin_name,
    420                    EXTRACTOR_METATYPE_FULL_TEXT,
    421                    EXTRACTOR_METAFORMAT_UTF8,
    422                    "text/plain",
    423                    ft->buf,
    424                    ft->len + 1);
    425 }
    426 
    427 
    428 void
    429 EXTRACTOR_fulltext_clear_ (struct EXTRACTOR_FullText *ft)
    430 {
    431   free (ft->buf);
    432   memset (ft, 0, sizeof (*ft));
    433 }
    434 
    435 
    436 /* ------------------------------------------------------------------ */
    437 /* Page furniture                                                      */
    438 /* ------------------------------------------------------------------ */
    439 
    440 /**
    441  * Words that may introduce a page number ("Page 3", "Seite 3", ...).
    442  */
    443 static const char *const page_words[] = {
    444   "page", "p", "pg", "pp", "seite", "s", "pagina", "página", "pag",
    445   "страница", "стр", "sida", "side", "sivu", "strona", "str", NULL
    446 };
    447 
    448 /**
    449  * Words that may join page number and page count ("3 of 17").
    450  */
    451 static const char *const of_words[] = {
    452   "of", "von", "de", "sur", "di", "van", "av", "af", "из", "z", "/", NULL
    453 };
    454 
    455 
    456 /**
    457  * Check whether @a tok (of @a len bytes) is in @a list, ignoring ASCII
    458  * case and a trailing '.'.
    459  *
    460  * @param tok token to check
    461  * @param len length of @a tok
    462  * @param list NULL-terminated word list
    463  * @return 1 if found
    464  */
    465 static int
    466 in_list (const char *tok,
    467          size_t len,
    468          const char *const *list)
    469 {
    470   unsigned int i;
    471 
    472   if ( (len > 1) &&
    473        ('.' == tok[len - 1]) )
    474     len--;
    475   for (i = 0; NULL != list[i]; i++)
    476     if ( (strlen (list[i]) == len) &&
    477          (0 == strncasecmp (tok,
    478                             list[i],
    479                             len)) )
    480       return 1;
    481   return 0;
    482 }
    483 
    484 
    485 /**
    486  * Check whether @a tok is a number: 1-5 decimal digits, or a roman
    487  * numeral of at most 8 characters.
    488  *
    489  * @param tok token to check
    490  * @param len length of @a tok
    491  * @return 1 if @a tok is a number
    492  */
    493 static int
    494 is_number (const char *tok,
    495            size_t len)
    496 {
    497   size_t i;
    498   int digits = 1;
    499   int roman = 1;
    500 
    501   if ( (0 == len) ||
    502        (len > 8) )
    503     return 0;
    504   for (i = 0; i < len; i++)
    505   {
    506     char c = tok[i];
    507 
    508     if (! isdigit ((unsigned char) c))
    509       digits = 0;
    510     if (NULL == strchr ("ivxlcdmIVXLCDM", c))
    511       roman = 0;
    512   }
    513   if (digits && (len <= 5))
    514     return 1;
    515   return roman;
    516 }
    517 
    518 
    519 /**
    520  * Check whether @a c is a decoration character around page numbers.
    521  *
    522  * @param c character to check
    523  * @return 1 if @a c is decoration
    524  */
    525 static int
    526 is_decoration (char c)
    527 {
    528   return (NULL != strchr (" \t\r\n-.,:;[]()|*~<>_=#", c)) ? 1 : 0;
    529 }
    530 
    531 
    532 int
    533 EXTRACTOR_fulltext_is_page_number_ (const char *line,
    534                                     size_t len)
    535 {
    536   const char *tok[4];
    537   size_t tlen[4];
    538   unsigned int ntok;
    539   size_t pos;
    540   size_t start;
    541   size_t end;
    542 
    543   /* strip ASCII decorations and UTF-8 en/em dashes at both ends */
    544   start = 0;
    545   end = len;
    546   while (start < end)
    547   {
    548     if (is_decoration (line[start]))
    549       start++;
    550     else if ( (start + 3 <= end) &&
    551               (0 == memcmp (&line[start], "\xE2\x80", 2)) &&
    552               ( ((unsigned char) line[start + 2] >= 0x93) &&
    553                 ((unsigned char) line[start + 2] <= 0x94) ) )
    554       start += 3;
    555     else
    556       break;
    557   }
    558   while (end > start)
    559   {
    560     if (is_decoration (line[end - 1]))
    561       end--;
    562     else if ( (end >= start + 3) &&
    563               (0 == memcmp (&line[end - 3], "\xE2\x80", 2)) &&
    564               ( ((unsigned char) line[end - 1] >= 0x93) &&
    565                 ((unsigned char) line[end - 1] <= 0x94) ) )
    566       end -= 3;
    567     else
    568       break;
    569   }
    570   if (start == end)
    571     return 0;
    572   /* tokenize on spaces, and split '/' into its own token */
    573   ntok = 0;
    574   pos = start;
    575   while (pos < end)
    576   {
    577     size_t ts;
    578 
    579     while ( (pos < end) &&
    580             ( (' ' == line[pos]) || ('\t' == line[pos]) ) )
    581       pos++;
    582     if (pos == end)
    583       break;
    584     if (ntok == 4)
    585       return 0;
    586     ts = pos;
    587     if ('/' == line[pos])
    588       pos++;
    589     else
    590       while ( (pos < end) &&
    591               (' ' != line[pos]) &&
    592               ('\t' != line[pos]) &&
    593               ('/' != line[pos]) )
    594         pos++;
    595     tok[ntok] = &line[ts];
    596     tlen[ntok] = pos - ts;
    597     ntok++;
    598   }
    599   /* "N" | "WORD N" | "N OF M" | "WORD N OF M" */
    600   pos = 0;
    601   if ( (ntok > 1) &&
    602        in_list (tok[0], tlen[0], page_words) )
    603     pos = 1;
    604   if ( (pos >= ntok) ||
    605        (! is_number (tok[pos], tlen[pos])) )
    606     return 0;
    607   pos++;
    608   if (pos == ntok)
    609     return 1;
    610   if ( (pos + 2 == ntok) &&
    611        in_list (tok[pos], tlen[pos], of_words) &&
    612        is_number (tok[pos + 1], tlen[pos + 1]) )
    613     return 1;
    614   return 0;
    615 }
    616 
    617 
    618 int
    619 EXTRACTOR_fulltext_page_add_ (struct EXTRACTOR_FullTextPage *page,
    620                               const char *text,
    621                               size_t len,
    622                               double y,
    623                               int par_end)
    624 {
    625   struct EXTRACTOR_FullTextLine *l;
    626 
    627   if (page->num_lines == page->cap_lines)
    628   {
    629     unsigned int ncap;
    630     struct EXTRACTOR_FullTextLine *nl;
    631 
    632     ncap = (0 == page->cap_lines) ? 32 : 2 * page->cap_lines;
    633     if (ncap > 1024 * 1024)
    634       return -1;
    635     nl = realloc (page->lines,
    636                   ncap * sizeof (*nl));
    637     if (NULL == nl)
    638       return -1;
    639     page->lines = nl;
    640     page->cap_lines = ncap;
    641   }
    642   l = &page->lines[page->num_lines];
    643   l->text = malloc (len + 1);
    644   if (NULL == l->text)
    645     return -1;
    646   memcpy (l->text,
    647           text,
    648           len);
    649   l->text[len] = '\0';
    650   l->y = y;
    651   l->par_end = par_end;
    652   l->skip = 0;
    653   page->num_lines++;
    654   return 0;
    655 }
    656 
    657 
    658 void
    659 EXTRACTOR_fulltext_page_clear_ (struct EXTRACTOR_FullTextPage *page)
    660 {
    661   unsigned int i;
    662 
    663   for (i = 0; i < page->num_lines; i++)
    664     free (page->lines[i].text);
    665   free (page->lines);
    666   memset (page, 0, sizeof (*page));
    667 }
    668 
    669 
    670 /**
    671  * Maximum number of lines at the top and at the bottom of a page we
    672  * consider as page furniture.
    673  */
    674 #define MARGIN_LINES 3
    675 
    676 /**
    677  * Relative height of the top and bottom margin bands.
    678  */
    679 #define MARGIN_BAND 0.12
    680 
    681 /**
    682  * Lines longer than this are never page furniture.
    683  */
    684 #define MARGIN_MAX_LEN 200
    685 
    686 
    687 /**
    688  * A margin line candidate for running header/footer detection.
    689  */
    690 struct Candidate
    691 {
    692   /**
    693    * Normalized text: lower-case ASCII, digits and whitespace removed.
    694    */
    695   char *key;
    696 
    697   /**
    698    * The line this candidate refers to.
    699    */
    700   struct EXTRACTOR_FullTextLine *line;
    701 
    702   /**
    703    * Page number of the line.
    704    */
    705   unsigned int page;
    706 
    707   /**
    708    * 0 for the top band, 1 for the bottom band.
    709    */
    710   int band;
    711 };
    712 
    713 
    714 /**
    715  * Order candidates by band, then key, then page.
    716  *
    717  * @param a first candidate
    718  * @param b second candidate
    719  * @return comparison result
    720  */
    721 static int
    722 cmp_candidate (const void *a,
    723                const void *b)
    724 {
    725   const struct Candidate *ca = a;
    726   const struct Candidate *cb = b;
    727   int r;
    728 
    729   if (ca->band != cb->band)
    730     return ca->band - cb->band;
    731   r = strcmp (ca->key,
    732               cb->key);
    733   if (0 != r)
    734     return r;
    735   if (ca->page != cb->page)
    736     return (ca->page < cb->page) ? -1 : 1;
    737   return 0;
    738 }
    739 
    740 
    741 /**
    742  * Normalize @a text for running header comparison.
    743  *
    744  * @param text line text
    745  * @return normalized copy, NULL on allocation failure
    746  */
    747 static char *
    748 normalize_key (const char *text)
    749 {
    750   size_t len = strlen (text);
    751   char *key;
    752   size_t i;
    753   size_t o;
    754 
    755   key = malloc (len + 1);
    756   if (NULL == key)
    757     return NULL;
    758   o = 0;
    759   for (i = 0; i < len; i++)
    760   {
    761     unsigned char c = (unsigned char) text[i];
    762 
    763     if (isdigit (c) || isspace (c))
    764       continue;
    765     key[o++] = (char) tolower (c);
    766   }
    767   key[o] = '\0';
    768   return key;
    769 }
    770 
    771 
    772 /**
    773  * Check whether line @a i of @a page is in the margin band @a band.
    774  *
    775  * @param page page to check
    776  * @param i line index
    777  * @param band 0 for top, 1 for bottom
    778  * @return 1 if in the band
    779  */
    780 static int
    781 in_band (const struct EXTRACTOR_FullTextPage *page,
    782          unsigned int i,
    783          int band)
    784 {
    785   const struct EXTRACTOR_FullTextLine *l = &page->lines[i];
    786 
    787   if (strlen (l->text) > MARGIN_MAX_LEN)
    788     return 0;
    789   if (l->y < 0)
    790   {
    791     /* position unknown: only the first/last lines qualify */
    792     return (0 == band) ? (i < 2) : (i + 2 >= page->num_lines);
    793   }
    794   return (0 == band) ? (l->y < MARGIN_BAND) : (l->y > 1.0 - MARGIN_BAND);
    795 }
    796 
    797 
    798 void
    799 EXTRACTOR_fulltext_filter_margins_ (struct EXTRACTOR_FullTextPage *pages,
    800                                     unsigned int num_pages)
    801 {
    802   struct Candidate *cands;
    803   size_t ncands;
    804   size_t i;
    805   unsigned int p;
    806 
    807   if (0 == num_pages)
    808     return;
    809   /* pass 1: page numbers */
    810   for (p = 0; p < num_pages; p++)
    811   {
    812     struct EXTRACTOR_FullTextPage *page = &pages[p];
    813     unsigned int j;
    814 
    815     /* do not strip the only line of a page */
    816     if (page->num_lines < 2)
    817       continue;
    818     for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
    819       if (in_band (page, j, 0) &&
    820           EXTRACTOR_fulltext_is_page_number_ (page->lines[j].text,
    821                                               strlen (page->lines[j].text)))
    822         page->lines[j].skip = 1;
    823     for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
    824     {
    825       unsigned int k = page->num_lines - 1 - j;
    826 
    827       if (in_band (page, k, 1) &&
    828           EXTRACTOR_fulltext_is_page_number_ (page->lines[k].text,
    829                                               strlen (page->lines[k].text)))
    830         page->lines[k].skip = 1;
    831     }
    832   }
    833   if (num_pages < 3)
    834     return;
    835   /* pass 2: running headers and footers */
    836   if (num_pages > 1000000)
    837     return;
    838   cands = calloc ((size_t) num_pages * 2 * MARGIN_LINES,
    839                   sizeof (*cands));
    840   if (NULL == cands)
    841     return;
    842   ncands = 0;
    843   for (p = 0; p < num_pages; p++)
    844   {
    845     struct EXTRACTOR_FullTextPage *page = &pages[p];
    846     unsigned int j;
    847     int band;
    848 
    849     if (page->num_lines < 2)
    850       continue;
    851     for (band = 0; band < 2; band++)
    852       for (j = 0; (j < MARGIN_LINES) && (j < page->num_lines); j++)
    853       {
    854         unsigned int k = (0 == band) ? j : page->num_lines - 1 - j;
    855         char *key;
    856 
    857         if ( (page->lines[k].skip) ||
    858              (! in_band (page, k, band)) )
    859           continue;
    860         key = normalize_key (page->lines[k].text);
    861         if (NULL == key)
    862           continue;
    863         if ('\0' == key[0])
    864         {
    865           free (key);
    866           continue;
    867         }
    868         cands[ncands].key = key;
    869         cands[ncands].line = &page->lines[k];
    870         cands[ncands].page = p;
    871         cands[ncands].band = band;
    872         ncands++;
    873       }
    874   }
    875   qsort (cands,
    876          ncands,
    877          sizeof (*cands),
    878          &cmp_candidate);
    879   i = 0;
    880   while (i < ncands)
    881   {
    882     size_t j;
    883     unsigned int distinct_pages;
    884 
    885     distinct_pages = 1;
    886     for (j = i + 1; j < ncands; j++)
    887     {
    888       if ( (cands[j].band != cands[i].band) ||
    889            (0 != strcmp (cands[j].key, cands[i].key)) )
    890         break;
    891       if (cands[j].page != cands[j - 1].page)
    892         distinct_pages++;
    893     }
    894     if ( (distinct_pages >= 2) &&
    895          (2 * distinct_pages >= num_pages) )
    896     {
    897       size_t k;
    898 
    899       for (k = i; k < j; k++)
    900         cands[k].line->skip = 1;
    901     }
    902     i = j;
    903   }
    904   for (i = 0; i < ncands; i++)
    905     free (cands[i].key);
    906   free (cands);
    907 }
    908 
    909 
    910 /**
    911  * Check whether the hyphen at the end of @a ft is part of a compound
    912  * word: the word it ends already contains a hyphen, or the word
    913  * @a next starts with does.
    914  *
    915  * @param ft text so far, ending in '-'
    916  * @param next text of the next line
    917  * @return 1 if the hyphen is to be kept
    918  */
    919 static int
    920 is_compound (const struct EXTRACTOR_FullText *ft,
    921              const char *next)
    922 {
    923   size_t i;
    924 
    925   for (i = ft->len - 1; i > 0; i--)
    926   {
    927     char c = ft->buf[i - 1];
    928 
    929     if (' ' == c)
    930       break;
    931     if ('-' == c)
    932       return 1;
    933   }
    934   for (i = 0; ('\0' != next[i]) && (' ' != next[i]); i++)
    935     if ('-' == next[i])
    936       return 1;
    937   return 0;
    938 }
    939 
    940 
    941 void
    942 EXTRACTOR_fulltext_append_pages_ (struct EXTRACTOR_FullText *ft,
    943                                   const struct EXTRACTOR_FullTextPage *pages,
    944                                   unsigned int num_pages)
    945 {
    946   unsigned int p;
    947   unsigned int j;
    948 
    949   for (p = 0; p < num_pages; p++)
    950     for (j = 0; j < pages[p].num_lines; j++)
    951     {
    952       const struct EXTRACTOR_FullTextLine *l = &pages[p].lines[j];
    953 
    954       if (l->skip)
    955         continue;
    956       /* re-join "hyphen-\nated" words; a compound broken at one of its
    957          own hyphens ("register-\nto-register") keeps the hyphen */
    958       if ( (ft->pending_space) &&
    959            (! ft->pending_par) &&
    960            (ft->len >= 2) &&
    961            ('-' == ft->buf[ft->len - 1]) &&
    962            isalpha ((unsigned char) ft->buf[ft->len - 2]) &&
    963            islower ((unsigned char) l->text[0]) )
    964       {
    965         if (! is_compound (ft,
    966                            l->text))
    967         {
    968           ft->len--;
    969           ft->buf[ft->len] = '\0';
    970         }
    971         ft->pending_space = 0;
    972       }
    973       EXTRACTOR_fulltext_append_str_ (ft,
    974                                       l->text);
    975       if (l->par_end)
    976         EXTRACTOR_fulltext_paragraph_ (ft);
    977       else
    978         EXTRACTOR_fulltext_space_ (ft);
    979     }
    980 }
    981 
    982 
    983 /* end of fulltext.c */
	libextractor GNU libextractor
	Log \| Files \| Refs \| Submodules \| README \| LICENSE