libextractor

html_extractor.c (30172B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2002, 2003, 2004, 2005, 2009, 2012 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 2, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 
  */
 /**
  * @file plugins/html_extractor.c
  * @brief plugin to support HTML files
  * @author Christian Grothoff
  *
  * With the "fulltext" option the text of the <body> is reported as one
  * #EXTRACTOR_METATYPE_FULL_TEXT item.  The file is then read as a whole
  * (up to #MAX_FULLTEXT_INPUT bytes), converted to UTF-8 according to
  * its byte order mark, XML declaration or META charset (sniffing UTF-8
  * versus windows-1252 without one) and parsed by libtidy from memory.
  * The walk over the tree drops script, style, noscript, template, the
  * page margins (nav, aside, header and footer outside of an article or
  * section, elements with an ARIA role of navigation, banner,
  * contentinfo, complementary or search) and hidden elements (hidden,
  * aria-hidden="true", display:none); block-level elements end
  * paragraphs, as do two consecutive <br>.
  */
 #include "platform.h"
 #include "extractor.h"
 #include "fulltext.h"
 #include <magic.h>
 #if HAVE_TIDY_H
 #include <tidy.h>
 #include <tidybuffio.h>
 #elif HAVE_TIDY_TIDY_H
 #include <tidy/tidy.h>
 #include <tidy/tidybuffio.h>
 #else
 Broken build, fix tidy detection.
 #endif
 
 /**
  * Mapping of HTML META names to LE types.
  */
 static struct
 {
   /**
    * HTML META name.
    */
   const char *name;
 
   /**
    * Corresponding LE type.
    */
   enum EXTRACTOR_MetaType type;
 } tagmap[] = {
   { "author", EXTRACTOR_METATYPE_AUTHOR_NAME },
   { "dc.author", EXTRACTOR_METATYPE_AUTHOR_NAME },
   { "title", EXTRACTOR_METATYPE_TITLE },
   { "dc.title", EXTRACTOR_METATYPE_TITLE},
   { "description", EXTRACTOR_METATYPE_DESCRIPTION },
   { "dc.description", EXTRACTOR_METATYPE_DESCRIPTION },
   { "subject", EXTRACTOR_METATYPE_SUBJECT},
   { "dc.subject", EXTRACTOR_METATYPE_SUBJECT},
   { "date", EXTRACTOR_METATYPE_UNKNOWN_DATE },
   { "dc.date", EXTRACTOR_METATYPE_UNKNOWN_DATE},
   { "publisher", EXTRACTOR_METATYPE_PUBLISHER },
   { "dc.publisher", EXTRACTOR_METATYPE_PUBLISHER},
   { "rights", EXTRACTOR_METATYPE_RIGHTS },
   { "dc.rights", EXTRACTOR_METATYPE_RIGHTS },
   { "copyright", EXTRACTOR_METATYPE_COPYRIGHT },
   { "language", EXTRACTOR_METATYPE_LANGUAGE },
   { "keywords", EXTRACTOR_METATYPE_KEYWORDS },
   { "abstract", EXTRACTOR_METATYPE_ABSTRACT },
   { "formatter", EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE },
   { "dc.creator", EXTRACTOR_METATYPE_CREATOR},
   { "dc.identifier", EXTRACTOR_METATYPE_URI },
   { "dc.format", EXTRACTOR_METATYPE_FORMAT },
   { NULL, EXTRACTOR_METATYPE_RESERVED }
 };
 
 
 /**
  * Global handle to MAGIC data.
  */
 static magic_t magic;
 
 
 /**
  * Map 'meta' tag to LE type.
  *
  * @param tag tag to map
  * @return EXTRACTOR_METATYPE_RESERVED if the type was not found
  */
 static enum EXTRACTOR_MetaType
 tag_to_type (const char *tag)
 {
   unsigned int i;
 
   for (i = 0; NULL != tagmap[i].name; i++)
     if (0 == strcasecmp (tag,
                          tagmap[i].name))
       return tagmap[i].type;
   return EXTRACTOR_METATYPE_RESERVED;
 }
 
 
 /**
  * Function called by libtidy for error reporting.
  *
  * @param doc tidy doc being processed
  * @param lvl report level
  * @param line input line
  * @param col input column
  * @param mssg message
  * @return FALSE (no output)
  */
 static Bool TIDY_CALL
 report_cb (TidyDoc doc,
            TidyReportLevel lvl,
            uint line,
            uint col,
            ctmbstr mssg)
 {
   return 0;
 }
 
 
 /**
  * Input callback: get next byte of input.
  *
  * @param sourceData our 'struct EXTRACTOR_ExtractContext'
  * @return next byte of input, EndOfStream on errors and EOF
  */
 static int TIDY_CALL
 get_byte_cb (void *sourceData)
 {
   struct EXTRACTOR_ExtractContext *ec = sourceData;
   void *data;
 
   if (1 !=
       ec->read (ec->cls,
                 &data, 1))
     return EndOfStream;
   return *(unsigned char*) data;
 }
 
 
 /**
  * Input callback: unget last byte of input.
  *
  * @param sourceData our 'struct EXTRACTOR_ExtractContext'
  * @param bt byte to unget (ignored)
  */
 static void TIDY_CALL
 unget_byte_cb (void *sourceData, byte bt)
 {
   struct EXTRACTOR_ExtractContext *ec = sourceData;
 
   (void) ec->seek (ec->cls, -1, SEEK_CUR);
 }
 
 
 /**
  * Input callback: check for EOF.
  *
  * @param sourceData our 'struct EXTRACTOR_ExtractContext'
  * @return true if we are at the EOF
  */
 static Bool TIDY_CALL
 eof_cb (void *sourceData)
 {
   struct EXTRACTOR_ExtractContext *ec = sourceData;
 
   return ec->seek (ec->cls, 0, SEEK_CUR) == ec->get_size (ec->cls);
 }
 
 
 /**
  * Report the TITLE and the META tags found in the document head.
  *
  * @param ec extraction context
  * @param doc parsed document
  * @return 1 if the caller should stop extracting, 0 to continue
  */
 static int
 report_head (struct EXTRACTOR_ExtractContext *ec,
              TidyDoc doc)
 {
   TidyNode head;
   TidyNode child;
   TidyNode title;
   const char *name;
   TidyBuffer tbuf;
   TidyAttr attr;
   enum EXTRACTOR_MetaType type;
 
   if (NULL == (head = tidyGetHead (doc)))
     return 0;
   for (child = tidyGetChild (head); NULL != child; child = tidyGetNext (child))
   {
     switch (tidyNodeGetType (child))
     {
     case TidyNode_Start:
     case TidyNode_StartEnd:
       name = tidyNodeGetName (child);
       if (NULL == name)
         break;
       if ( (0 == strcasecmp (name, "title")) &&
            (NULL != (title = tidyGetChild (child))) )
       {
         tidyBufInit (&tbuf);
         tidyNodeGetValue (doc, title, &tbuf);
         /* add 0-termination */
         tidyBufPutByte (&tbuf, 0);
         if (0 !=
             ec->proc (ec->cls,
                       "html",
                       EXTRACTOR_METATYPE_TITLE,
                       EXTRACTOR_METAFORMAT_UTF8,
                       "text/plain",
                       (const char *) tbuf.bp,
                       tbuf.size))
         {
           tidyBufFree (&tbuf);
           return 1;
         }
         tidyBufFree (&tbuf);
         break;
       }
       if (0 == strcasecmp (name, "meta"))
       {
         if (NULL == (attr = tidyAttrGetById (child,
                                              TidyAttr_NAME)))
           break;
         name = tidyAttrValue (attr);
         if (NULL == name)
           break;
         if (EXTRACTOR_METATYPE_RESERVED ==
             (type = tag_to_type (name)))
           break;
         if (NULL == (attr = tidyAttrGetById (child,
                                              TidyAttr_CONTENT)))
           break;
         name = tidyAttrValue (attr);
         if (NULL == name)
           break;
         if (0 !=
             ec->proc (ec->cls,
                       "html",
                       type,
                       EXTRACTOR_METAFORMAT_UTF8,
                       "text/plain",
                       name,
                       strlen (name) + 1))
           return 1;
         break;
       }
       break;
     default:
       break;
     }
   }
   return 0;
 }
 
 
 /* ------------------------------------------------------------------ */
 /* Full text                                                           */
 /* ------------------------------------------------------------------ */
 
 /**
  * Largest HTML file we read for the "fulltext" option.  libtidy keeps
  * the whole tree in memory (several times the size of the input), and
  * the text we may emit is capped at #EXTRACTOR_FULLTEXT_MAX anyway;
  * longer files are truncated.
  */
 #define MAX_FULLTEXT_INPUT (16 * 1024 * 1024)
 
 /**
  * How deep into the element tree we descend; anything nested deeper
  * is ignored.  Real documents stay far below this, and it bounds the
  * stack use of #walk_children().
  */
 #define MAX_DEPTH 256
 
 /**
  * How many attributes of one element we look at.
  */
 #define MAX_ATTRS 64
 
 /**
  * How far into the file we look for a charset declaration.
  */
 #define CHARSET_SCAN 8192
 
 
 /**
  * State of the full text walk over the tidy tree.
  */
 struct TextWalk
 {
   /**
    * Text collected so far.
    */
   struct EXTRACTOR_FullText ft;
 
   /**
    * The document.
    */
   TidyDoc doc;
 
   /**
    * Scratch buffer for text node values.
    */
   TidyBuffer tb;
 
   /**
    * 1 if the last thing we saw was a <br> (with nothing but white
    * space after it), so another one ends the paragraph.
    */
   int after_br;
 };
 
 
 /**
  * Find @a needle in @a hay, ignoring ASCII case.
  *
  * @param hay buffer to search
  * @param hlen number of bytes in @a hay
  * @param needle 0-terminated lower-case string to look for
  * @return offset of the match, or @a hlen if there is none
  */
 static size_t
 find_ci (const char *hay,
          size_t hlen,
          const char *needle)
 {
   size_t nlen = strlen (needle);
 
   if (nlen > hlen)
     return hlen;
   for (size_t i = 0; i + nlen <= hlen; i++)
   {
     size_t k;
 
     for (k = 0; k < nlen; k++)
       if (tolower ((unsigned char) hay[i + k]) != needle[k])
         break;
     if (k == nlen)
       return i;
   }
   return hlen;
 }
 
 
 /**
  * Parse a charset name following "charset" / "encoding" at @a p:
  * optional white space, '=', optional white space and quotes, then the
  * name.
  *
  * @param p where to start
  * @param end end of the buffer
  * @param[out] cs where to write the name (0-terminated)
  * @param cs_size number of bytes available in @a cs
  * @return 1 if a name was found
  */
 static int
 parse_charset_value (const char *p,
                      const char *end,
                      char *cs,
                      size_t cs_size)
 {
   size_t o = 0;
 
   while ( (p < end) && isspace ((unsigned char) *p))
     p++;
   if ( (p >= end) || ('=' != *p) )
     return 0;
   p++;
   while ( (p < end) &&
           (isspace ((unsigned char) *p) || ('"' == *p) || ('\'' == *p) ) )
     p++;
   while ( (p < end) &&
           (o + 1 < cs_size) &&
           (isalnum ((unsigned char) *p) ||
            ('-' == *p) || ('_' == *p) || ('.' == *p) || (':' == *p) ) )
     cs[o++] = (char) tolower ((unsigned char) *p++);
   cs[o] = '\0';
   return (0 != o);
 }
 
 
 /**
  * Is @a data valid UTF-8?
  *
  * @param data bytes to check
  * @param len number of bytes in @a data
  * @return 1 if valid
  */
 static int
 is_utf8 (const unsigned char *data,
          size_t len)
 {
   size_t i = 0;
 
   while (i < len)
   {
     unsigned int n;
     uint32_t cp;
 
     if (data[i] < 0x80)
     {
       i++;
       continue;
     }
     if ( (data[i] >= 0xC2) && (data[i] <= 0xDF) )
       n = 1;
     else if ( (data[i] >= 0xE0) && (data[i] <= 0xEF) )
       n = 2;
     else if ( (data[i] >= 0xF0) && (data[i] <= 0xF4) )
       n = 3;
     else
       return 0;
     if (n > len - i - 1)
       return 0;
     cp = data[i] & (0x3F >> n);
     for (unsigned int k = 1; k <= n; k++)
     {
       if (0x80 != (data[i + k] & 0xC0))
         return 0;
       cp = (cp << 6) | (data[i + k] & 0x3F);
     }
     if ( ( (2 == n) && (cp < 0x800) ) ||
          ( (3 == n) && ( (cp < 0x10000) || (cp > 0x10FFFF) ) ) ||
          ( (cp >= 0xD800) && (cp <= 0xDFFF) ) )
       return 0;
     i += n + 1;
   }
   return 1;
 }
 
 
 /**
  * Determine the character set of an HTML document: a byte order mark,
  * then an XML declaration, then a META charset / http-equiv
  * declaration; without any of those, UTF-8 if the bytes are valid
  * UTF-8 and windows-1252 otherwise.  Labels are mapped as browsers do
  * (ISO-8859-1 and ASCII mean windows-1252; a META tag claiming UTF-16
  * means UTF-8).
  *
  * @param data start of the document
  * @param len number of bytes in @a data
  * @param[out] cs where to write the iconv name of the charset
  * @param cs_size number of bytes available in @a cs
  * @return number of BOM bytes to skip
  */
 static size_t
 sniff_charset (const char *data,
                size_t len,
                char *cs,
                size_t cs_size)
 {
   const unsigned char *u = (const unsigned char *) data;
   size_t scan = (len > CHARSET_SCAN) ? CHARSET_SCAN : len;
   char found[64];
   int have = 0;
 
   if ( (len >= 3) && (0xEF == u[0]) && (0xBB == u[1]) && (0xBF == u[2]) )
   {
     snprintf (cs, cs_size, "UTF-8");
     return 3;
   }
   if ( (len >= 2) && (0xFE == u[0]) && (0xFF == u[1]) )
   {
     snprintf (cs, cs_size, "UTF-16BE");
     return 2;
   }
   if ( (len >= 2) && (0xFF == u[0]) && (0xFE == u[1]) )
   {
     snprintf (cs, cs_size, "UTF-16LE");
     return 2;
   }
   if ( (scan >= 5) && (0 == memcmp (data, "<?xml", 5)) )
   {
     const char *gt = memchr (data, '>', scan);
     size_t dlen = (NULL == gt) ? scan : (size_t) (gt - data);
     size_t off = find_ci (data, dlen, "encoding");
 
     if (off < dlen)
       have = parse_charset_value (&data[off + strlen ("encoding")],
                                   &data[dlen],
                                   found,
                                   sizeof (found));
   }
   for (size_t pos = 0; (! have) && (pos < scan); )
   {
     size_t off = find_ci (&data[pos], scan - pos, "<meta");
     const char *tag;
     const char *gt;
     size_t tlen;
     size_t c;
 
     if (off >= scan - pos)
       break;
     tag = &data[pos + off];
     gt = memchr (tag, '>', scan - pos - off);
     tlen = (NULL == gt) ? scan - pos - off : (size_t) (gt - tag);
     c = find_ci (tag, tlen, "charset");
     if (c < tlen)
       have = parse_charset_value (&tag[c + strlen ("charset")],
                                   &tag[tlen],
                                   found,
                                   sizeof (found));
     pos += off + 5;
   }
   if (have)
   {
     if ( (0 == strcmp (found, "iso-8859-1")) ||
          (0 == strcmp (found, "iso8859-1")) ||
          (0 == strcmp (found, "latin1")) ||
          (0 == strcmp (found, "l1")) ||
          (0 == strcmp (found, "us-ascii")) ||
          (0 == strcmp (found, "ascii")) ||
          (0 == strcmp (found, "cp1252")) ||
          (0 == strcmp (found, "windows-1252")) )
       snprintf (cs, cs_size, "WINDOWS-1252");
     else if ( (0 == strncmp (found, "utf-16", 6)) ||
               (0 == strncmp (found, "utf16", 5)) ||
               (0 == strcmp (found, "utf8")) ||
               (0 == strcmp (found, "utf-8")) )
       snprintf (cs, cs_size, "UTF-8");
     else
       snprintf (cs, cs_size, "%s", found);
     return 0;
   }
   if (is_utf8 (u, len))
     snprintf (cs, cs_size, "UTF-8");
   else
     snprintf (cs, cs_size, "WINDOWS-1252");
   return 0;
 }
 
 
 /**
  * Code points of windows-1252 bytes 0x80 to 0x9F.
  */
 static const uint16_t cp1252[32] = {
   0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
   0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
   0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
   0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178
 };
 
 
 /**
  * Append the UTF-8 encoding of @a cp (which must be below 0x10000)
  * to @a out.
  *
  * @param out output buffer, at least 3 bytes must be available
  * @param cp code point
  * @return number of bytes written
  */
 static size_t
 put_utf8 (char *out,
           uint32_t cp)
 {
   if (cp < 0x80)
   {
     out[0] = (char) cp;
     return 1;
   }
   if (cp < 0x800)
   {
     out[0] = (char) (0xC0 | (cp >> 6));
     out[1] = (char) (0x80 | (cp & 0x3F));
     return 2;
   }
   out[0] = (char) (0xE0 | (cp >> 12));
   out[1] = (char) (0x80 | ((cp >> 6) & 0x3F));
   out[2] = (char) (0x80 | (cp & 0x3F));
   return 3;
 }
 
 
 /**
  * Convert @a data from charset @a cs to UTF-8.  Undecodable bytes
  * become U+FFFD.  If the charset is unknown, the bytes are taken as
  * UTF-8 if they are valid UTF-8, and as windows-1252 otherwise.
  *
  * @param data input
  * @param len number of bytes in @a data
  * @param cs charset name as understood by iconv
  * @param[out] olen number of bytes in the result
  * @return converted text, NULL on allocation failure; caller frees
  */
 static char *
 to_utf8 (const char *data,
          size_t len,
          const char *cs,
          size_t *olen)
 {
   char *out;
   size_t o = 0;
 
   *olen = 0;
   if (0 == strcmp (cs, "UTF-8"))
   {
     if (NULL == (out = malloc (len + 1)))
       return NULL;
     memcpy (out, data, len);
     *olen = len;
     return out;
   }
   /* no conversion produces more than 3 bytes of UTF-8 per input byte
      (a UTF-16 surrogate pair is 4 bytes in and 4 bytes out) */
   if (NULL == (out = malloc (3 * len + 4)))
     return NULL;
 #if HAVE_ICONV
   if (0 != strcmp (cs, "WINDOWS-1252"))
   {
     iconv_t cd = iconv_open ("UTF-8", cs);
 
     if ((iconv_t) -1 != cd)
     {
       char *ib = (char *) data;
       size_t il = len;
       char *ob = out;
       size_t ol = 3 * len;
 
       while (il > 0)
       {
         if ((size_t) -1 !=
             iconv (cd, &ib, &il, &ob, &ol))
           break;
         if ( (E2BIG == errno) ||
              (ol < 3) )
           break;
         /* skip one undecodable byte */
         ob += put_utf8 (ob, 0xFFFD);
         ol -= 3;
         ib++;
         il--;
       }
       iconv_close (cd);
       *olen = (size_t) (ob - out);
       return out;
     }
     if (is_utf8 ((const unsigned char *) data, len))
     {
       memcpy (out, data, len);
       *olen = len;
       return out;
     }
   }
 #else
   if ( (0 != strcmp (cs, "WINDOWS-1252")) &&
        (is_utf8 ((const unsigned char *) data, len)) )
   {
     memcpy (out, data, len);
     *olen = len;
     return out;
   }
 #endif
   for (size_t i = 0; i < len; i++)
   {
     unsigned char c = (unsigned char) data[i];
 
     if (c < 0x80)
       out[o++] = (char) c;
     else if (c < 0xA0)
       o += put_utf8 (&out[o], cp1252[c - 0x80]);
     else
       o += put_utf8 (&out[o], c);
   }
   *olen = o;
   return out;
 }
 
 
 /**
  * Read the whole input (up to #MAX_FULLTEXT_INPUT bytes).
  *
  * @param ec extraction context, positioned at the start
  * @param[out] len number of bytes read
  * @return the data, NULL on error; caller frees
  */
 static char *
 read_all (struct EXTRACTOR_ExtractContext *ec,
           size_t *len)
 {
   char *buf = NULL;
   size_t size = 0;
   size_t cap = 0;
 
   *len = 0;
   while (size < MAX_FULLTEXT_INPUT)
   {
     void *data;
     ssize_t got;
     size_t want = MAX_FULLTEXT_INPUT - size;
 
     if (want > 64 * 1024)
       want = 64 * 1024;
     got = ec->read (ec->cls, &data, want);
     if (got <= 0)
       break;
     if ((size_t) got > want)
       got = (ssize_t) want;
     if (size + (size_t) got > cap)
     {
       size_t ncap = (0 == cap) ? 64 * 1024 : cap;
       char *nbuf;
 
       while (ncap < size + (size_t) got)
         ncap *= 2;
       if (ncap > MAX_FULLTEXT_INPUT)
         ncap = MAX_FULLTEXT_INPUT;
       if (NULL == (nbuf = realloc (buf, ncap)))
         break;
       buf = nbuf;
       cap = ncap;
     }
     memcpy (&buf[size], data, (size_t) got);
     size += (size_t) got;
   }
   *len = size;
   return buf;
 }
 
 
 /**
  * Does the space-separated token list @a list contain @a tok (ASCII
  * case-insensitively)?
  *
  * @param list token list
  * @param tok token to look for
  * @return 1 if found
  */
 static int
 has_token (const char *list,
            const char *tok)
 {
   size_t tlen = strlen (tok);
   const char *p = list;
 
   while ('\0' != *p)
   {
     size_t n;
 
     while (isspace ((unsigned char) *p))
       p++;
     n = 0;
     while ( ('\0' != p[n]) && (! isspace ((unsigned char) p[n])) )
       n++;
     if ( (n == tlen) &&
          (0 == strncasecmp (p, tok, n)) )
       return 1;
     p += n;
   }
   return 0;
 }
 
 
 /**
  * Does @a style (a CSS declaration list) hide the element, i.e. set
  * display:none or visibility:hidden?
  *
  * @param style value of the style attribute
  * @return 1 if the element is hidden
  */
 static int
 style_hides (const char *style)
 {
   char norm[256];
   size_t o = 0;
 
   for (size_t i = 0; ('\0' != style[i]) && (o + 1 < sizeof (norm)); i++)
     if (! isspace ((unsigned char) style[i]))
       norm[o++] = (char) tolower ((unsigned char) style[i]);
   norm[o] = '\0';
   return ( (NULL != strstr (norm, "display:none")) ||
            (NULL != strstr (norm, "visibility:hidden")) );
 }
 
 
 /**
  * Should the element @a node (and everything in it) be left out of the
  * full text because of its attributes: hidden, aria-hidden="true", an
  * inline style that hides it, or an ARIA landmark role for the page
  * margins (navigation, banner, contentinfo, complementary, search) or a
  * page break marker?
  *
  * @param node element to check
  * @return 1 to skip the element
  */
 static int
 attrs_skip (TidyNode node)
 {
   static const char *roles[] = {
     "navigation", "banner", "contentinfo", "complementary", "search",
     "doc-pagebreak", NULL
   };
   unsigned int n = 0;
 
   for (TidyAttr a = tidyAttrFirst (node);
        (NULL != a) && (n < MAX_ATTRS);
        a = tidyAttrNext (a), n++)
   {
     const char *name = tidyAttrName (a);
     const char *val = tidyAttrValue (a);
 
     if (NULL == name)
       continue;
     if (0 == strcasecmp (name, "hidden"))
     {
       /* hidden="until-found" is collapsed content that find-in-page
          reveals, i.e. part of the document text */
       if ( (NULL == val) ||
            (0 != strcasecmp (val, "until-found")) )
         return 1;
       continue;
     }
     if (NULL == val)
       continue;
     if ( (0 == strcasecmp (name, "aria-hidden")) &&
          (0 == strcasecmp (val, "true")) )
       return 1;
     if ( (0 == strcasecmp (name, "style")) &&
          (style_hides (val)) )
       return 1;
     if (0 == strcasecmp (name, "role"))
       for (unsigned int i = 0; NULL != roles[i]; i++)
         if (has_token (val, roles[i]))
           return 1;
   }
   return 0;
 }
 
 
 /**
  * Is @a id an element whose content is never document text?
  *
  * @param id element to check
  * @return 1 to skip the element
  */
 static int
 tag_skip (TidyTagId id)
 {
   switch (id)
   {
   case TidyTag_HEAD:
   case TidyTag_TITLE:
   case TidyTag_SCRIPT:
   case TidyTag_STYLE:
   case TidyTag_NOSCRIPT:
   case TidyTag_TEMPLATE:
   case TidyTag_NAV:
   case TidyTag_ASIDE:
   case TidyTag_SELECT:
   case TidyTag_DATALIST:
   case TidyTag_IFRAME:
   case TidyTag_OBJECT:
   case TidyTag_APPLET:
   case TidyTag_RP:
   case TidyTag_RT:
     return 1;
   default:
     return 0;
   }
 }
 
 
 /**
  * Does @a id end a paragraph (i.e. is it a block-level element)?
  *
  * @param id element to check
  * @return 1 for block-level elements
  */
 static int
 tag_is_block (TidyTagId id)
 {
   switch (id)
   {
   case TidyTag_ADDRESS:
   case TidyTag_ARTICLE:
   case TidyTag_BLOCKQUOTE:
   case TidyTag_BODY:
   case TidyTag_CAPTION:
   case TidyTag_CENTER:
   case TidyTag_DD:
   case TidyTag_DETAILS:
   case TidyTag_DIALOG:
   case TidyTag_DIR:
   case TidyTag_DIV:
   case TidyTag_DL:
   case TidyTag_DT:
   case TidyTag_FIELDSET:
   case TidyTag_FIGCAPTION:
   case TidyTag_FIGURE:
   case TidyTag_FOOTER:
   case TidyTag_FORM:
   case TidyTag_FRAMESET:
   case TidyTag_H1:
   case TidyTag_H2:
   case TidyTag_H3:
   case TidyTag_H4:
   case TidyTag_H5:
   case TidyTag_H6:
   case TidyTag_HEADER:
   case TidyTag_HGROUP:
   case TidyTag_HR:
   case TidyTag_HTML:
   case TidyTag_LEGEND:
   case TidyTag_LI:
   case TidyTag_LISTING:
   case TidyTag_MAIN:
   case TidyTag_MENU:
   case TidyTag_NOFRAMES:
   case TidyTag_OL:
   case TidyTag_P:
   case TidyTag_PLAINTEXT:
   case TidyTag_PRE:
   case TidyTag_SECTION:
   case TidyTag_SUMMARY:
   case TidyTag_TABLE:
   case TidyTag_TBODY:
   case TidyTag_TD:
   case TidyTag_TFOOT:
   case TidyTag_TH:
   case TidyTag_THEAD:
   case TidyTag_TR:
   case TidyTag_UL:
   case TidyTag_XMP:
     return 1;
   default:
     return 0;
   }
 }
 
 
 /**
  * Does @a id start a sectioning context, inside which <header> and
  * <footer> belong to that section rather than to the page?
  *
  * @param id element to check
  * @return 1 for article, aside, main, nav and section
  */
 static int
 tag_is_sectioning (TidyTagId id)
 {
   return (TidyTag_ARTICLE == id) ||
          (TidyTag_ASIDE == id) ||
          (TidyTag_MAIN == id) ||
          (TidyTag_NAV == id) ||
          (TidyTag_SECTION == id);
 }
 
 
 /**
  * Append the text of a text node.
  *
  * @param w walk state
  * @param node the text node
  */
 static void
 add_text (struct TextWalk *w,
           TidyNode node)
 {
   tidyBufClear (&w->tb);
   if (! tidyNodeGetValue (w->doc, node, &w->tb))
     return;
   if (0 == w->tb.size)
     return;
   for (uint i = 0; i < w->tb.size; i++)
     if (! isspace (w->tb.bp[i]))
     {
       w->after_br = 0;
       break;
     }
   EXTRACTOR_fulltext_append_ (&w->ft,
                               (const char *) w->tb.bp,
                               w->tb.size);
 }
 
 
 /**
  * Append the text of the children of @a node.
  *
  * @param w walk state
  * @param node parent node
  * @param depth nesting depth of @a node
  * @param in_section 1 if @a node is inside a sectioning element
  */
 static void
 walk_children (struct TextWalk *w,
                TidyNode node,
                unsigned int depth,
                int in_section)
 {
   if (depth >= MAX_DEPTH)
     return;
   for (TidyNode c = tidyGetChild (node);
        (NULL != c) && (! w->ft.full);
        c = tidyGetNext (c))
   {
     TidyTagId id;
     int block;
 
     switch (tidyNodeGetType (c))
     {
     case TidyNode_Text:
     case TidyNode_CDATA:
       add_text (w, c);
       break;
     case TidyNode_Start:
     case TidyNode_StartEnd:
       id = tidyNodeGetId (c);
       if (tag_skip (id))
         break;
       /* a page-level header or footer is page furniture; one inside
          an article or section is that section's heading or byline */
       if ( ( (TidyTag_HEADER == id) || (TidyTag_FOOTER == id) ) &&
            (! in_section) )
         break;
       if (attrs_skip (c))
         break;
       if (TidyTag_BR == id)
       {
         if (w->after_br)
         {
           EXTRACTOR_fulltext_paragraph_ (&w->ft);
           w->after_br = 0;
         }
         else
         {
           EXTRACTOR_fulltext_space_ (&w->ft);
           w->after_br = 1;
         }
         break;
       }
       block = tag_is_block (id);
       if (block)
       {
         EXTRACTOR_fulltext_paragraph_ (&w->ft);
         w->after_br = 0;
       }
       walk_children (w,
                      c,
                      depth + 1,
                      in_section || tag_is_sectioning (id));
       if (block)
       {
         EXTRACTOR_fulltext_paragraph_ (&w->ft);
         w->after_br = 0;
       }
       break;
     default:
       break;
     }
   }
 }
 
 
 /**
  * Extract meta data and the full text.  Unlike the default mode, this
  * reads the whole file, converts it to UTF-8 according to its declared
  * (or sniffed) charset and hands tidy a UTF-8 buffer.
  *
  * @param ec extraction context, positioned at the start of the file
  */
 static void
 extract_fulltext (struct EXTRACTOR_ExtractContext *ec)
 {
   struct TextWalk w;
   TidyBuffer in;
   TidyNode body;
   char cs[64];
   char *raw;
   char *utf8;
   size_t raw_len;
   size_t utf8_len;
   size_t skip;
 
   if (NULL == (raw = read_all (ec, &raw_len)))
     return;
   skip = sniff_charset (raw, raw_len, cs, sizeof (cs));
   utf8 = to_utf8 (&raw[skip], raw_len - skip, cs, &utf8_len);
   free (raw);
   if (NULL == utf8)
     return;
   if (0 == utf8_len)
   {
     free (utf8);
     return;
   }
   memset (&w, 0, sizeof (w));
   EXTRACTOR_fulltext_init_ (&w.ft);
   EXTRACTOR_fulltext_set_context_ (&w.ft, ec);
   if (NULL == (w.doc = tidyCreate ()))
   {
     free (utf8);
     return;
   }
   tidySetReportFilter (w.doc, &report_cb);
   tidySetInCharEncoding (w.doc, "utf8");
   tidyBufInit (&in);
   tidyBufAttach (&in, (byte *) utf8, (uint) utf8_len);
   tidyBufInit (&w.tb);
   if (0 > tidyParseBuffer (w.doc, &in))
     goto CLEANUP;
   /* the default mode insists on status 1 (warnings), which a document
      that no longer has an encoding problem may not produce; errors (2)
      still suppress the head items as they do there */
   if ( (2 > tidyStatus (w.doc)) &&
        (0 != report_head (ec, w.doc)) )
     goto CLEANUP;
   if (NULL != (body = tidyGetBody (w.doc)))
     walk_children (&w, body, 0, 0);
   (void) EXTRACTOR_fulltext_emit_ (&w.ft, ec, "html");
 CLEANUP:
   EXTRACTOR_fulltext_clear_ (&w.ft);
   tidyBufFree (&w.tb);
   tidyBufDetach (&in);
   tidyRelease (w.doc);
   free (utf8);
 }
 
 
 /**
  * Main entry method for the 'text/html' extraction plugin.
  *
  * @param ec extraction context provided to the plugin
  */
 void
 EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec);
 
 void
 EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
 {
   TidyDoc doc;
   TidyInputSource src;
   ssize_t iret;
   void *data;
   const char *mime;
 
   if (-1 == (iret = ec->read (ec->cls,
                               &data,
                               16 * 1024)))
     return;
   if (NULL == (mime = magic_buffer (magic, data, iret)))
     return;
   if (0 != strncmp (mime,
                     "text/html",
                     strlen ("text/html")))
     return; /* not HTML */
 
   if (0 != ec->seek (ec->cls, 0, SEEK_SET))
     return; /* seek failed !? */
 
   if (EXTRACTOR_fulltext_requested_ (ec->config))
   {
     extract_fulltext (ec);
     return;
   }
   tidyInitSource (&src, ec,
                   &get_byte_cb,
                   &unget_byte_cb,
                   &eof_cb);
   if (NULL == (doc = tidyCreate ()))
     return;
   tidySetReportFilter (doc, &report_cb);
   tidySetAppData (doc, ec);
   if (0 > tidyParseSource (doc, &src))
   {
     tidyRelease (doc);
     return;
   }
   if (1 != tidyStatus (doc))
   {
     tidyRelease (doc);
     return;
   }
   (void) report_head (ec, doc);
   tidyRelease (doc);
 }
 
 
 /**
  * Initialize glib and load magic file.
  */
 void __attribute__ ((constructor))
 html_gobject_init (void);
 
 void __attribute__ ((constructor))
 html_gobject_init ()
 {
   magic = magic_open (MAGIC_MIME_TYPE);
   if (0 != magic_load (magic, NULL))
   {
     /* FIXME: how to deal with errors? */
   }
 }
 
 
 /**
  * Destructor for the library, cleans up.
  */
 void __attribute__ ((destructor))
 html_ltdl_fini (void);
 
 void __attribute__ ((destructor))
 html_ltdl_fini ()
 {
   if (NULL != magic)
   {
     magic_close (magic);
     magic = NULL;
   }
 }
 
 
 /* end of html_extractor.c */