libextractor

ebook_extractor.c (90401B)

---

 /*
      This file is part of libextractor.
      Copyright (C) 2026 Vidyut Samanta and Christian Grothoff
 
      libextractor is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
      by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      libextractor is distributed in the hope that it will be useful, but
      WITHOUT ANY WARRANTY; without even the implied warranty of
      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
      General Public License for more details.
 
      You should have received a copy of the GNU General Public License
      along with libextractor; see the file COPYING.  If not, write to the
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
  */
 /**
  * @file plugins/ebook_extractor.c
  * @brief plugin to support EPUB and MOBI/AZW electronic books
  * @author Christian Grothoff
  *
  * Two unrelated containers share this plugin because they answer the
  * same question.  An EPUB is a ZIP whose OPF package document carries
  * Dublin Core; a MOBI is a Palm database whose EXTH header carries the
  * same fields plus, on files that came from a store, a watermark.  What
  * is worth having here is not the title but the provenance: which tool
  * built the file, when, for whom.
  *
  * With the "fulltext" option the book's text is reported as one
  * #EXTRACTOR_METATYPE_FULL_TEXT item:
  * - EPUB: the linear entries of the OPF spine, in spine order (not
  *   manifest or archive order); each (X)HTML content document is
  *   converted by a tolerant markup scanner that drops head, script,
  *   style, nav, aside, page-level header/footer, hidden elements, ARIA
  *   landmark roles for the page margins and page break markers
  *   (epub:type="pagebreak"); block elements, <br><br> and chapter
  *   boundaries end paragraphs.  Documents listed in encryption.xml
  *   (DRM) are skipped.
  * - MOBI/AZW: the text records, uncompressed or PalmDOC compressed,
  *   with the trailing entries stripped, run through the same scanner;
  *   <mbp:pagebreak/> ends a paragraph.  Encrypted books and HUFF/CDIC
  *   compression (used by some Amazon-built files) are not supported.
  *   For KF8 (AZW3) the raw markup is scanned in storage order, which
  *   is the reading order of the text.
  *
  * References:
  * - EPUB 3 OCF and Package Document, https://www.w3.org/TR/epub-33/
  * - MOBI format, https://wiki.mobileread.com/wiki/MOBI
  * - PalmDOC / PDB, https://wiki.mobileread.com/wiki/PDB
  */
 #include "platform.h"
 #include "extractor.h"
 #include "forensics.h"
 #include "fulltext.h"
 #include "unzip.h"
 
 
 /**
  * Name we report our meta data under.
  */
 #define PLUGIN_NAME "ebook"
 
 /**
  * Never read more than this from `META-INF/container.xml'.  It holds a
  * handful of paths; anything larger is padding.
  */
 #define MAX_CONTAINER (64 * 1024)
 
 /**
  * Never read more than this from the OPF package document.  Real ones
  * are a few kilobytes plus the manifest.
  */
 #define MAX_OPF (256 * 1024)
 
 /**
  * Never read more than this from record 0 of a Palm database.  The MOBI
  * and EXTH headers together are a few kilobytes.
  */
 #define MAX_RECORD0 (64 * 1024)
 
 /**
  * Upper bound on the number of EXTH records we will walk.
  */
 #define MAX_EXTH 1024
 
 /**
  * Upper bound on the number of XML elements we will walk in the OPF.
  */
 #define MAX_ELEMENTS 8192
 
 /**
  * Seconds between the Palm epoch (1904-01-01) and the Unix epoch.
  */
 #define PALM_EPOCH_OFFSET 2082844800LL
 
 
 /**
  * Find @a needle in @a hay.  `memmem()' is a GNU extension, and this is
  * only ever run over buffers of a few hundred kilobytes.
  *
  * @param hay buffer to search
  * @param hlen number of bytes in @a hay
  * @param needle bytes to look for
  * @param nlen number of bytes in @a needle
  * @return pointer into @a hay, NULL if @a needle does not occur
  */
 static const char *
 mem_find (const char *hay,
           size_t hlen,
           const char *needle,
           size_t nlen)
 {
   if ( (0 == nlen) ||
        (nlen > hlen) )
     return NULL;
   for (size_t i = 0; i + nlen <= hlen; i++)
     if (0 == memcmp (&hay[i],
                      needle,
                      nlen))
       return &hay[i];
   return NULL;
 }
 
 
 /**
  * Is @a c a character that may appear in an XML name?
  *
  * @param c character to test
  * @return 1 if @a c may be part of a name
  */
 static int
 is_name_char (char c)
 {
   return ( ( ('a' <= c) && ('z' >= c) ) ||
            ( ('A' <= c) && ('Z' >= c) ) ||
            ( ('0' <= c) && ('9' >= c) ) ||
            ('_' == c) || ('-' == c) || ('.' == c) || (':' == c) );
 }
 
 
 /**
  * Is @a c XML white space?
  *
  * @param c character to test
  * @return 1 if @a c is white space
  */
 static int
 is_space (char c)
 {
   return (' ' == c) || ('\t' == c) || ('\r' == c) || ('\n' == c);
 }
 
 
 /**
  * Append @a cp to @a out as UTF-8.
  *
  * @param out output buffer
  * @param outsz number of bytes available in @a out
  * @param pos current write position
  * @param cp code point to append
  * @return new write position, unchanged if @a cp did not fit
  */
 static size_t
 utf8_put (char *out,
           size_t outsz,
           size_t pos,
           uint32_t cp)
 {
   if ( (cp > 0x10FFFF) ||
        ( (0xD800 <= cp) && (0xDFFF >= cp) ) )
     return pos;
   if (cp < 0x80)
   {
     if (pos + 1 > outsz)
       return pos;
     out[pos++] = (char) cp;
   }
   else if (cp < 0x800)
   {
     if (pos + 2 > outsz)
       return pos;
     out[pos++] = (char) (0xC0 | (cp >> 6));
     out[pos++] = (char) (0x80 | (cp & 0x3F));
   }
   else if (cp < 0x10000)
   {
     if (pos + 3 > outsz)
       return pos;
     out[pos++] = (char) (0xE0 | (cp >> 12));
     out[pos++] = (char) (0x80 | ((cp >> 6) & 0x3F));
     out[pos++] = (char) (0x80 | (cp & 0x3F));
   }
   else
   {
     if (pos + 4 > outsz)
       return pos;
     out[pos++] = (char) (0xF0 | (cp >> 18));
     out[pos++] = (char) (0x80 | ((cp >> 12) & 0x3F));
     out[pos++] = (char) (0x80 | ((cp >> 6) & 0x3F));
     out[pos++] = (char) (0x80 | (cp & 0x3F));
   }
   return pos;
 }
 
 
 /**
  * Copy XML character data, resolving the five predefined entities and
  * numeric character references and collapsing runs of white space.
  *
  * The OPF fields we read are laid out across indented lines, so without
  * the white space collapsing every value would come back with the
  * source file's line breaks embedded in it.
  *
  * @param data the character data
  * @param len number of bytes in @a data
  * @param[out] out where to write the result, NUL-terminated
  * @param outsz number of bytes available in @a out, at least 2
  * @return number of bytes written, not counting the NUL
  */
 static size_t
 xml_text (const char *data,
           size_t len,
           char *out,
           size_t outsz)
 {
   size_t o = 0;
   size_t i = 0;
   int pending_space = 0;
 
   while ( (i < len) &&
           (o + 1 < outsz) )
   {
     if (is_space (data[i]))
     {
       if (0 != o)
         pending_space = 1;
       i++;
       continue;
     }
     if (pending_space)
     {
       out[o++] = ' ';
       pending_space = 0;
       if (o + 1 >= outsz)
         break;
     }
     if ('&' != data[i])
     {
       out[o++] = data[i++];
       continue;
     }
     {
       const char *semi = memchr (&data[i],
                                  ';',
                                  len - i);
       size_t elen;
 
       if ( (NULL == semi) ||
            ( (size_t) (semi - &data[i]) > 12) )
       {
         out[o++] = data[i++];   /* a bare '&', which is not legal XML
                                    but does occur; pass it through */
         continue;
       }
       elen = (size_t) (semi - &data[i]) - 1;   /* between '&' and ';' */
       if ( (3 == elen) && (0 == memcmp (&data[i + 1], "amp", 3)) )
         out[o++] = '&';
       else if ( (2 == elen) && (0 == memcmp (&data[i + 1], "lt", 2)) )
         out[o++] = '<';
       else if ( (2 == elen) && (0 == memcmp (&data[i + 1], "gt", 2)) )
         out[o++] = '>';
       else if ( (4 == elen) && (0 == memcmp (&data[i + 1], "quot", 4)) )
         out[o++] = '"';
       else if ( (4 == elen) && (0 == memcmp (&data[i + 1], "apos", 4)) )
         out[o++] = '\'';
       else if ( (2 <= elen) && ('#' == data[i + 1]) )
       {
         uint32_t cp = 0;
         size_t k = i + 2;
         int base = 10;
 
         if ( ('x' == data[k]) || ('X' == data[k]) )
         {
           base = 16;
           k++;
         }
         for (; k < (size_t) (semi - data); k++)
         {
           unsigned int d;
 
           if ( ('0' <= data[k]) && ('9' >= data[k]) )
             d = (unsigned int) (data[k] - '0');
           else if ( (16 == base) && ('a' <= data[k]) && ('f' >= data[k]) )
             d = (unsigned int) (data[k] - 'a') + 10;
           else if ( (16 == base) && ('A' <= data[k]) && ('F' >= data[k]) )
             d = (unsigned int) (data[k] - 'A') + 10;
           else
           {
             cp = 0;
             break;
           }
           if (cp > 0x10FFFF)
           {
             cp = 0;
             break;
           }
           cp = cp * (uint32_t) base + d;
         }
         o = utf8_put (out,
                       outsz - 1,
                       o,
                       cp);
       }
       else
       {
         /* an entity we do not know; drop it rather than guess */
       }
       i += elen + 2;
     }
   }
   out[o] = '\0';
   return o;
 }
 
 
 /**
  * Read the value of attribute @a name out of the body of a start tag.
  * A namespace prefix on the attribute is ignored, so a request for
  * `role' also matches `opf:role'.
  *
  * @param data the start tag body, after the element name
  * @param len number of bytes in @a data
  * @param name local name of the attribute
  * @param[out] out where to write the value, NUL-terminated
  * @param outsz number of bytes available in @a out, at least 2
  * @return 1 if the attribute was found, 0 if not
  */
 static int
 xml_attr (const char *data,
           size_t len,
           const char *name,
           char *out,
           size_t outsz)
 {
   size_t nlen = strlen (name);
 
   out[0] = '\0';
   if (nlen >= len)
     return 0;
   for (size_t i = 0; i + nlen < len; i++)
   {
     size_t k;
     char quote;
     const char *vend;
 
     if (0 != memcmp (&data[i],
                      name,
                      nlen))
       continue;
     /* the name must start where an attribute may start: right after
        white space, or right after a namespace prefix that does */
     if (0 == i)
       continue;
     if (is_space (data[i - 1]))
     {
       /* fine */
     }
     else if (':' == data[i - 1])
     {
       size_t j = i - 1;
 
       while ( (0 < j) &&
               is_name_char (data[j - 1]) &&
               (':' != data[j - 1]) )
         j--;
       if ( (0 == j) ||
            (! is_space (data[j - 1])) )
         continue;
     }
     else
     {
       continue;
     }
     k = i + nlen;
     while ( (k < len) &&
             is_space (data[k]) )
       k++;
     if ( (k >= len) ||
          ('=' != data[k]) )
       continue;
     k++;
     while ( (k < len) &&
             is_space (data[k]) )
       k++;
     if ( (k >= len) ||
          ( ('"' != data[k]) && ('\'' != data[k]) ) )
       continue;
     quote = data[k];
     k++;
     vend = memchr (&data[k],
                    quote,
                    len - k);
     if (NULL == vend)
       return 0;
     (void) xml_text (&data[k],
                      (size_t) (vend - &data[k]),
                      out,
                      outsz);
     return 1;
   }
   return 0;
 }
 
 
 /**
  * Does @a s, after an optional `urn:isbn:' prefix, look like an ISBN?
  *
  * @param s NUL-terminated candidate
  * @return 1 if @a s is 10 or 13 digits with optional separators
  */
 static int
 looks_like_isbn (const char *s)
 {
   size_t digits = 0;
 
   if (0 == strncasecmp (s,
                         "urn:isbn:",
                         strlen ("urn:isbn:")))
     s += strlen ("urn:isbn:");
   else if (0 == strncasecmp (s,
                              "isbn:",
                              strlen ("isbn:")))
     s += strlen ("isbn:");
   for (size_t i = 0; '\0' != s[i]; i++)
   {
     if ( ('-' == s[i]) || (' ' == s[i]) )
       continue;
     if ( ('0' <= s[i]) && ('9' >= s[i]) )
     {
       digits++;
       continue;
     }
     if ( ( ('X' == s[i]) || ('x' == s[i]) ) &&
          (9 == digits) &&
          ('\0' == s[i + 1]) )
     {
       digits++;
       continue;
     }
     return 0;
   }
   return (10 == digits) || (13 == digits);
 }
 
 
 /**
  * Read the current member of a ZIP archive.
  *
  * @param uf the archive, positioned on the member
  * @param cap never read more than this many bytes
  * @param[out] len number of bytes read
  * @return the member's contents with a NUL appended, NULL on error;
  *         caller must free
  */
 static char *
 read_current (struct EXTRACTOR_UnzipFile *uf,
               size_t cap,
               size_t *len)
 {
   struct EXTRACTOR_UnzipFileInfo fi;
   char *buf;
   size_t size;
   size_t got = 0;
 
   *len = 0;
   if (EXTRACTOR_UNZIP_OK !=
       EXTRACTOR_common_unzip_get_current_file_info (uf,
                                                     &fi,
                                                     NULL, 0,
                                                     NULL, 0,
                                                     NULL, 0))
     return NULL;
   size = (size_t) fi.uncompressed_size;
   if (0 == size)
     return NULL;
   if (size > cap)
     size = cap;
   if (NULL == (buf = malloc (size + 1)))
     return NULL;
   if (EXTRACTOR_UNZIP_OK !=
       EXTRACTOR_common_unzip_open_current_file (uf))
   {
     free (buf);
     return NULL;
   }
   while (got < size)
   {
     ssize_t ret;
 
     ret = EXTRACTOR_common_unzip_read_current_file (uf,
                                                     &buf[got],
                                                     size - got);
     if (0 >= ret)
       break;   /* error or end of member */
     if (((size_t) ret) > size - got)
       break;   /* cannot happen, but do not overrun if it does */
     got += (size_t) ret;
   }
   (void) EXTRACTOR_common_unzip_close_current_file (uf);
   if (0 == got)
   {
     free (buf);
     return NULL;
   }
   buf[got] = '\0';
   *len = got;
   return buf;
 }
 
 
 /**
  * Read a whole member out of a ZIP archive.
  *
  * @param uf the archive
  * @param name member to read
  * @param cap never read more than this many bytes
  * @param[out] len number of bytes read
  * @return the member's contents with a NUL appended, NULL on error;
  *         caller must free
  */
 static char *
 read_member (struct EXTRACTOR_UnzipFile *uf,
              const char *name,
              size_t cap,
              size_t *len)
 {
   *len = 0;
   if (EXTRACTOR_UNZIP_OK !=
       EXTRACTOR_common_unzip_go_find_local_file (uf,
                                                  name,
                                                  2))
     return NULL;
   return read_current (uf,
                        cap,
                        len);
 }
 
 
 /**
  * Does the archive contain a member with this name?
  *
  * @param uf the archive
  * @param name member to look for
  * @return 1 if present
  */
 static int
 has_member (struct EXTRACTOR_UnzipFile *uf,
             const char *name)
 {
   return (EXTRACTOR_UNZIP_OK ==
           EXTRACTOR_common_unzip_go_find_local_file (uf,
                                                      name,
                                                      2));
 }
 
 
 /**
  * Walk the `<metadata>' element of an OPF package document and report
  * what it holds.
  *
  * This is a bounded text scan, not an XML parser: the OPF is untrusted
  * input and everything we want out of it is a flat list of leaf
  * elements, so pulling in a parser would buy nothing but attack surface.
  *
  * @param ec extraction context
  * @param opf the package document
  * @param len number of bytes in @a opf
  * @return 1 if the caller should stop extracting, 0 to continue
  */
 static int
 parse_opf_metadata (struct EXTRACTOR_ExtractContext *ec,
                     const char *opf,
                     size_t len)
 {
   char value[EXTRACTOR_FORENSIC_MAX_STRING];
   char attr[256];
   const char *mstart;
   const char *mend;
   const char *p;
   unsigned int elements = 0;
   /* the repeating elements are capped separately: a file with ten
      thousand <dc:subject> entries is characterised well enough by the
      first few dozen */
   unsigned int subjects = 0;
   unsigned int identifiers = 0;
   unsigned int people = 0;
 
   mstart = mem_find (opf,
                      len,
                      "<metadata",
                      strlen ("<metadata"));
   if (NULL == mstart)
     return 0;
   mend = mem_find (mstart,
                    len - (size_t) (mstart - opf),
                    "</metadata",
                    strlen ("</metadata"));
   if (NULL == mend)
     mend = opf + len;
   p = mstart;
   while ( (p < mend) &&
           (elements++ < MAX_ELEMENTS) )
   {
     const char *name;
     const char *local;
     const char *body;
     const char *text;
     size_t name_len;
     size_t body_len;
     size_t text_len;
     int self_closing;
     char quote = '\0';
     enum EXTRACTOR_MetaType type;
 
     p = memchr (p,
                 '<',
                 (size_t) (mend - p));
     if (NULL == p)
       break;
     p++;
     if (p >= mend)
       break;
     if ( ('/' == *p) || ('?' == *p) || ('!' == *p) )
       continue;
     name = p;
     while ( (p < mend) &&
             is_name_char (*p) )
       p++;
     name_len = (size_t) (p - name);
     if (0 == name_len)
       continue;
     body = p;
     while (p < mend)
     {
       if ('\0' != quote)
       {
         if (*p == quote)
           quote = '\0';
       }
       else if ( ('"' == *p) || ('\'' == *p) )
       {
         quote = *p;
       }
       else if ('>' == *p)
       {
         break;
       }
       p++;
     }
     if (p >= mend)
       break;
     body_len = (size_t) (p - body);
     self_closing = ( (0 < body_len) && ('/' == p[-1]) );
     if (self_closing)
       body_len--;
     p++;   /* step past '>' */
     text = p;
     text_len = 0;
     if (! self_closing)
     {
       const char *e = memchr (p,
                               '<',
                               (size_t) (mend - p));
 
       text_len = (NULL == e)
                  ? (size_t) (mend - p)
                  : (size_t) (e - p);
     }
     /* strip the namespace prefix; `dc:title' and `title' are the same
        element for our purposes because we are already inside
        <metadata> */
     local = name;
     for (size_t i = 0; i < name_len; i++)
       if (':' == name[i])
         local = &name[i + 1];
     name_len -= (size_t) (local - name);
 
 #define IS(s) ( (strlen (s) == name_len) && \
                 (0 == memcmp (local, s, name_len)) )
 
     if (IS ("meta"))
     {
       if ( (xml_attr (body,
                       body_len,
                       "name",
                       attr,
                       sizeof (attr))) &&
            (xml_attr (body,
                       body_len,
                       "content",
                       value,
                       sizeof (value))) )
       {
         if (0 == strcasecmp (attr,
                              "calibre:timestamp"))
           type = EXTRACTOR_METATYPE_CREATION_DATE;
         else if ( (0 == strcasecmp (attr,
                                     "generator")) ||
                   (0 == strcasecmp (attr,
                                     "calibre:generator")) )
           type = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE;
         else
           continue;
         if (0 !=
             EXTRACTOR_forensic_emit_text_ (ec,
                                            PLUGIN_NAME,
                                            type,
                                            value,
                                            strlen (value)))
           return 1;
       }
       else if (xml_attr (body,
                          body_len,
                          "property",
                          attr,
                          sizeof (attr)))
       {
         if (0 != strcasecmp (attr,
                              "dcterms:modified"))
           continue;
         (void) xml_text (text,
                          text_len,
                          value,
                          sizeof (value));
         if (0 !=
             EXTRACTOR_forensic_emit_text_ (
               ec,
               PLUGIN_NAME,
               EXTRACTOR_METATYPE_MODIFICATION_DATE,
               value,
               strlen (value)))
           return 1;
       }
       continue;
     }
     if (0 == text_len)
       continue;
     (void) xml_text (text,
                      text_len,
                      value,
                      sizeof (value));
     if ('\0' == value[0])
       continue;
     if (IS ("title"))
       type = EXTRACTOR_METATYPE_TITLE;
     else if (IS ("creator"))
     {
       if (people++ >= EXTRACTOR_FORENSIC_MAX_ITEMS)
         continue;
       type = EXTRACTOR_METATYPE_AUTHOR_NAME;
     }
     else if (IS ("publisher"))
       type = EXTRACTOR_METATYPE_PUBLISHER;
     else if (IS ("language"))
       type = EXTRACTOR_METATYPE_LANGUAGE;
     else if (IS ("date"))
       type = EXTRACTOR_METATYPE_PUBLICATION_DATE;
     else if (IS ("subject"))
     {
       if (subjects++ >= EXTRACTOR_FORENSIC_MAX_ITEMS)
         continue;
       type = EXTRACTOR_METATYPE_KEYWORDS;
     }
     else if (IS ("description"))
       type = EXTRACTOR_METATYPE_DESCRIPTION;
     else if (IS ("rights"))
       type = EXTRACTOR_METATYPE_RIGHTS;
     else if (IS ("source"))
       type = EXTRACTOR_METATYPE_URI;
     else if (IS ("contributor"))
     {
       /* `bkp' is the MARC relator for the agent that made the file --
          which is the tool, and thus the provenance we are after */
       if ( (xml_attr (body,
                       body_len,
                       "role",
                       attr,
                       sizeof (attr))) &&
            (0 == strcasecmp (attr,
                              "bkp")) )
       {
         type = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE;
       }
       else
       {
         if (people++ >= EXTRACTOR_FORENSIC_MAX_ITEMS)
           continue;
         type = EXTRACTOR_METATYPE_CONTRIBUTOR_NAME;
       }
     }
     else if (IS ("identifier"))
     {
       int isbn = 0;
 
       if (identifiers++ >= EXTRACTOR_FORENSIC_MAX_ITEMS)
         continue;
 
       if (xml_attr (body,
                     body_len,
                     "scheme",
                     attr,
                     sizeof (attr)))
         isbn = (0 == strcasecmp (attr,
                                  "ISBN"));
       if (! isbn)
         isbn = looks_like_isbn (value);
       type = isbn
              ? EXTRACTOR_METATYPE_ISBN
              : EXTRACTOR_METATYPE_URI;
     }
     else
     {
       continue;
     }
     if (0 !=
         EXTRACTOR_forensic_emit_text_ (ec,
                                        PLUGIN_NAME,
                                        type,
                                        value,
                                        strlen (value)))
       return 1;
   }
 #undef IS
   return 0;
 }
 
 
 /* ------------------------------------------------------------------ */
 /* Full text: character sets                                           */
 /* ------------------------------------------------------------------ */
 
 /**
  * Never inflate more than this from one EPUB content document.
  */
 #define MAX_CHAPTER (8 * 1024 * 1024)
 
 /**
  * Never inflate more than this from all EPUB content documents
  * together, nor decompress more than this from the text records of a
  * MOBI file.  The full text itself is capped at #EXTRACTOR_FULLTEXT_MAX,
  * and markup rarely more than triples the size of the text.
  */
 #define MAX_BOOK_MARKUP (64 * 1024 * 1024)
 
 /**
  * Upper bound on the number of manifest items we keep.
  */
 #define MAX_MANIFEST 4096
 
 /**
  * Upper bound on the number of spine entries we follow.
  */
 #define MAX_SPINE 4096
 
 /**
  * Upper bound on the size of one MOBI text record (normally 4096 bytes
  * of text plus a few trailing bytes).
  */
 #define MAX_TEXT_RECORD (64 * 1024)
 
 /**
  * Upper bound on what one MOBI text record may decompress to.
  */
 #define MAX_TEXT_RECORD_OUT (64 * 1024)
 
 /**
  * How many attributes of one element we look at.
  */
 #define MAX_ATTRS 64
 
 
 /**
  * Code points of windows-1252 bytes 0x80 to 0x9F.
  */
 static const uint16_t cp1252[32] = {
   0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021,
   0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F,
   0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
   0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178
 };
 
 
 /**
  * Convert windows-1252 (a superset of ISO-8859-1 in the printable
  * range) to UTF-8.
  *
  * @param data input
  * @param len number of bytes in @a data
  * @param[out] olen number of bytes in the result
  * @return converted text, NULL on allocation failure; caller frees
  */
 static char *
 cp1252_to_utf8 (const char *data,
                 size_t len,
                 size_t *olen)
 {
   char *out;
   size_t o = 0;
 
   if (NULL == (out = malloc (3 * len + 1)))
     return NULL;
   for (size_t i = 0; i < len; i++)
   {
     unsigned char c = (unsigned char) data[i];
 
     if (c < 0x80)
       out[o++] = (char) c;
     else if (c < 0xA0)
       o = utf8_put (out, 3 * len, o, cp1252[c - 0x80]);
     else
       o = utf8_put (out, 3 * len, o, c);
   }
   *olen = o;
   return out;
 }
 
 
 /**
  * Convert UTF-16 to UTF-8.  Unpaired surrogates become U+FFFD.
  *
  * @param data input
  * @param len number of bytes in @a data
  * @param big_endian 1 for UTF-16BE, 0 for UTF-16LE
  * @param[out] olen number of bytes in the result
  * @return converted text, NULL on allocation failure; caller frees
  */
 static char *
 utf16_to_utf8 (const char *data,
                size_t len,
                int big_endian,
                size_t *olen)
 {
   const unsigned char *u = (const unsigned char *) data;
   char *out;
   size_t o = 0;
   size_t cap = 2 * len + 4;   /* 2 bytes in never make more than 3 out */
 
   if (NULL == (out = malloc (cap)))
     return NULL;
   for (size_t i = 0; i + 1 < len; i += 2)
   {
     uint32_t cp = big_endian
                   ? (uint32_t) ((u[i] << 8) | u[i + 1])
                   : (uint32_t) ((u[i + 1] << 8) | u[i]);
 
     if ( (cp >= 0xD800) && (cp <= 0xDBFF) && (i + 3 < len) )
     {
       uint32_t lo = big_endian
                     ? (uint32_t) ((u[i + 2] << 8) | u[i + 3])
                     : (uint32_t) ((u[i + 3] << 8) | u[i + 2]);
 
       if ( (lo >= 0xDC00) && (lo <= 0xDFFF) )
       {
         cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00);
         i += 2;
       }
     }
     if ( (cp >= 0xD800) && (cp <= 0xDFFF) )
       cp = 0xFFFD;
     o = utf8_put (out, cap, o, cp);
   }
   *olen = o;
   return out;
 }
 
 
 /**
  * Convert @a data from the named character set to UTF-8.  UTF-8 input
  * is copied as is (invalid sequences are dealt with when the text is
  * appended to the full text); UTF-16 and windows-1252 are converted
  * here, anything else by iconv if we have it.
  *
  * @param data input
  * @param len number of bytes in @a data
  * @param cs lower-case charset name
  * @param[out] olen number of bytes in the result
  * @return converted text, NULL on allocation failure; caller frees
  */
 static char *
 convert_to_utf8 (const char *data,
                  size_t len,
                  const char *cs,
                  size_t *olen)
 {
   char *out;
 
   *olen = 0;
   if ( (0 == strcmp (cs, "iso-8859-1")) ||
        (0 == strcmp (cs, "iso8859-1")) ||
        (0 == strcmp (cs, "latin1")) ||
        (0 == strcmp (cs, "us-ascii")) ||
        (0 == strcmp (cs, "ascii")) ||
        (0 == strcmp (cs, "cp1252")) ||
        (0 == strcmp (cs, "windows-1252")) )
     return cp1252_to_utf8 (data, len, olen);
   if ( (0 == strcmp (cs, "utf-16be")) ||
        (0 == strcmp (cs, "utf-16")) )
     return utf16_to_utf8 (data, len, 1, olen);
   if (0 == strcmp (cs, "utf-16le"))
     return utf16_to_utf8 (data, len, 0, olen);
 #if HAVE_ICONV
   if ( ('\0' != cs[0]) &&
        (0 != strcmp (cs, "utf-8")) &&
        (0 != strcmp (cs, "utf8")) )
   {
     iconv_t cd = iconv_open ("UTF-8", cs);
 
     if ((iconv_t) -1 != cd)
     {
       char *ib = (char *) data;
       size_t il = len;
       char *ob;
       size_t ol;
 
       /* no charset produces more than 4 bytes of UTF-8 per byte */
       if (NULL == (out = malloc (4 * len + 4)))
       {
         iconv_close (cd);
         return NULL;
       }
       ob = out;
       ol = 4 * len;
       while (il > 0)
       {
         if ((size_t) -1 !=
             iconv (cd, &ib, &il, &ob, &ol))
           break;
         if ( (E2BIG == errno) ||
              (ol < 3) )
           break;
         /* skip one undecodable byte */
         *ob++ = (char) 0xEF;
         *ob++ = (char) 0xBF;
         *ob++ = (char) 0xBD;
         ol -= 3;
         ib++;
         il--;
       }
       iconv_close (cd);
       *olen = (size_t) (ob - out);
       return out;
     }
   }
 #endif
   if (NULL == (out = malloc (len + 1)))
     return NULL;
   memcpy (out, data, len);
   *olen = len;
   return out;
 }
 
 
 /**
  * Read a charset name after `encoding' or `charset' at @a p: optional
  * white space, '=', optional white space and quotes, then the name.
  *
  * @param p where to start
  * @param end end of the buffer
  * @param[out] cs where to write the lower-case name
  * @param cs_size number of bytes available in @a cs
  * @return 1 if a name was found
  */
 static int
 charset_value (const char *p,
                const char *end,
                char *cs,
                size_t cs_size)
 {
   size_t o = 0;
 
   while ( (p < end) && is_space (*p))
     p++;
   if ( (p >= end) || ('=' != *p) )
     return 0;
   p++;
   while ( (p < end) &&
           (is_space (*p) || ('"' == *p) || ('\'' == *p) ) )
     p++;
   while ( (p < end) &&
           (o + 1 < cs_size) &&
           (isalnum ((unsigned char) *p) ||
            ('-' == *p) || ('_' == *p) || ('.' == *p) || (':' == *p) ) )
     cs[o++] = (char) tolower ((unsigned char) *p++);
   cs[o] = '\0';
   return (0 != o);
 }
 
 
 /**
  * Convert an EPUB content document to UTF-8: honour a byte order mark,
  * UTF-16 without one, and the encoding in the XML declaration or a
  * META charset declaration; default to UTF-8.
  *
  * @param data the document
  * @param len number of bytes in @a data
  * @param[out] olen number of bytes in the result
  * @return the document in UTF-8, NULL on allocation failure
  */
 static char *
 xhtml_to_utf8 (const char *data,
                size_t len,
                size_t *olen)
 {
   const unsigned char *u = (const unsigned char *) data;
   size_t scan = (len > 1024) ? 1024 : len;
   char cs[64];
 
   if ( (len >= 3) && (0xEF == u[0]) && (0xBB == u[1]) && (0xBF == u[2]) )
     return convert_to_utf8 (&data[3], len - 3, "utf-8", olen);
   if ( (len >= 2) && (0xFE == u[0]) && (0xFF == u[1]) )
     return convert_to_utf8 (&data[2], len - 2, "utf-16be", olen);
   if ( (len >= 2) && (0xFF == u[0]) && (0xFE == u[1]) )
     return convert_to_utf8 (&data[2], len - 2, "utf-16le", olen);
   if ( (len >= 4) && (0 == u[0]) && ('<' == u[1]) && (0 == u[2]) )
     return convert_to_utf8 (data, len, "utf-16be", olen);
   if ( (len >= 4) && ('<' == u[0]) && (0 == u[1]) && (0 == u[3]) )
     return convert_to_utf8 (data, len, "utf-16le", olen);
   cs[0] = '\0';
   if ( (scan > 5) && (0 == memcmp (data, "<?xml", 5)) )
   {
     const char *gt = memchr (data, '>', scan);
     size_t dlen = (NULL == gt) ? scan : (size_t) (gt - data);
     const char *e = mem_find (data, dlen, "encoding", strlen ("encoding"));
 
     if (NULL != e)
       (void) charset_value (e + strlen ("encoding"),
                             data + dlen,
                             cs,
                             sizeof (cs));
   }
   if ('\0' == cs[0])
   {
     const char *m = mem_find (data, scan, "charset", strlen ("charset"));
 
     if (NULL != m)
       (void) charset_value (m + strlen ("charset"),
                             data + scan,
                             cs,
                             sizeof (cs));
   }
   /* a document in UTF-8 that claims UTF-16 cannot be UTF-16: we would
      have seen the zero bytes */
   if (0 == strncmp (cs, "utf-16", 6))
     cs[0] = '\0';
   return convert_to_utf8 (data, len, cs, olen);
 }
 
 
 /* ------------------------------------------------------------------ */
 /* Full text: (X)HTML markup                                           */
 /* ------------------------------------------------------------------ */
 
 /**
  * HTML named character references for U+00A0 to U+00FF, in order.
  */
 static const char *const latin1_entities[96] = {
   "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect",
   "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr",
   "deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot",
   "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest",
   "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil",
   "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml",
   "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times",
   "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig",
   "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil",
   "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml",
   "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide",
   "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml"
 };
 
 
 /**
  * Other HTML named character references common in books.
  */
 static const struct
 {
   /**
    * Entity name.
    */
   const char *name;
 
   /**
    * Code point.
    */
   uint32_t cp;
 } other_entities[] = {
   { "amp", '&' }, { "lt", '<' }, { "gt", '>' }, { "quot", '"' },
   { "apos", '\'' }, { "OElig", 0x152 }, { "oelig", 0x153 },
   { "Scaron", 0x160 }, { "scaron", 0x161 }, { "Yuml", 0x178 },
   { "fnof", 0x192 }, { "circ", 0x2C6 }, { "tilde", 0x2DC },
   { "ensp", 0x2002 }, { "emsp", 0x2003 }, { "thinsp", 0x2009 },
   { "zwnj", 0x200C }, { "zwj", 0x200D }, { "lrm", 0x200E },
   { "rlm", 0x200F }, { "ndash", 0x2013 }, { "mdash", 0x2014 },
   { "lsquo", 0x2018 }, { "rsquo", 0x2019 }, { "sbquo", 0x201A },
   { "ldquo", 0x201C }, { "rdquo", 0x201D }, { "bdquo", 0x201E },
   { "dagger", 0x2020 }, { "Dagger", 0x2021 }, { "bull", 0x2022 },
   { "hellip", 0x2026 }, { "permil", 0x2030 }, { "prime", 0x2032 },
   { "Prime", 0x2033 }, { "lsaquo", 0x2039 }, { "rsaquo", 0x203A },
   { "oline", 0x203E }, { "euro", 0x20AC }, { "trade", 0x2122 },
   { "larr", 0x2190 }, { "rarr", 0x2192 }, { "minus", 0x2212 },
   { NULL, 0 }
 };
 
 
 /**
  * Decode the character reference at @a p (just after the '&').
  *
  * @param p text after the '&'
  * @param len number of bytes available at @a p
  * @param[out] cp the code point
  * @return number of bytes consumed after the '&' (including the ';'),
  *         0 if this is not a character reference we know
  */
 static size_t
 decode_entity (const char *p,
                size_t len,
                uint32_t *cp)
 {
   size_t n = 0;
 
   if ( (len >= 2) && ('#' == p[0]) )
   {
     uint32_t v = 0;
     int base = 10;
     size_t k = 1;
     size_t digits = 0;
 
     if ( ('x' == p[1]) || ('X' == p[1]) )
     {
       base = 16;
       k = 2;
     }
     for (; (k < len) && (digits < 8); k++, digits++)
     {
       unsigned int d;
 
       if ( ('0' <= p[k]) && ('9' >= p[k]) )
         d = (unsigned int) (p[k] - '0');
       else if ( (16 == base) && ('a' <= p[k]) && ('f' >= p[k]) )
         d = (unsigned int) (p[k] - 'a') + 10;
       else if ( (16 == base) && ('A' <= p[k]) && ('F' >= p[k]) )
         d = (unsigned int) (p[k] - 'A') + 10;
       else
         break;
       v = v * (uint32_t) base + d;
     }
     if ( (0 == digits) ||
          (k >= len) ||
          (';' != p[k]) )
       return 0;
     *cp = (v > 0x10FFFF) ? 0xFFFD : v;
     return k + 1;
   }
   while ( (n < len) &&
           (n < 10) &&
           isalnum ((unsigned char) p[n]) )
     n++;
   if ( (0 == n) ||
        (n >= len) ||
        (';' != p[n]) )
     return 0;
   for (unsigned int i = 0; i < 96; i++)
     if ( (strlen (latin1_entities[i]) == n) &&
          (0 == memcmp (latin1_entities[i], p, n)) )
     {
       *cp = 0xA0 + i;
       return n + 1;
     }
   for (unsigned int i = 0; NULL != other_entities[i].name; i++)
     if ( (strlen (other_entities[i].name) == n) &&
          (0 == memcmp (other_entities[i].name, p, n)) )
     {
       *cp = other_entities[i].cp;
       return n + 1;
     }
   return 0;
 }
 
 
 /**
  * State of the conversion of one (X)HTML document to text.
  */
 struct MarkupState
 {
   /**
    * Where the text goes.
    */
   struct EXTRACTOR_FullText *ft;
 
   /**
    * Local name of the element whose content we are skipping, "" if
    * none.
    */
   char skip_name[32];
 
   /**
    * Nesting level of @e skip_name elements inside the skipped one.
    */
   unsigned int skip_nest;
 
   /**
    * Number of open sectioning elements (article, section, main, ...);
    * a header or footer inside one is not page furniture.
    */
   unsigned int sections;
 
   /**
    * 1 if the last thing we saw was a <br>, so another one ends the
    * paragraph.
    */
   int after_br;
 };
 
 
 /**
  * Is @a name (lower case) in the 0-terminated list @a list?
  *
  * @param name element name
  * @param list list of names
  * @return 1 if found
  */
 static int
 name_in (const char *name,
          const char *const *list)
 {
   for (unsigned int i = 0; NULL != list[i]; i++)
     if (0 == strcmp (name, list[i]))
       return 1;
   return 0;
 }
 
 
 /**
  * Elements that end a paragraph.
  */
 static const char *const block_elements[] = {
   "address", "article", "aside", "blockquote", "body", "caption",
   "center", "dd", "details", "dialog", "dir", "div", "dl", "dt",
   "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2",
   "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "html", "legend",
   "li", "listing", "main", "menu", "nav", "ol", "p", "pre", "section",
   "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul",
   "xmp", "title", "head", "noscript",
   /* DTBook and FB2-style structure some EPUBs carry */
   "level", "level1", "level2", "level3", "level4", "level5", "level6",
   "hd", "h", "note", "annotation", "epigraph", "poem", "stanza", "v",
   NULL
 };
 
 /**
  * Elements whose content is never document text.
  */
 static const char *const skip_elements[] = {
   "head", "title", "script", "style", "noscript", "template", "nav",
   "aside", "select", "datalist", "iframe", "object", "applet", "rp", "rt",
   "guide", "svg", "math", NULL
 };
 
 /**
  * Elements whose content is not markup (the end tag is searched for
  * literally).
  */
 static const char *const raw_text_elements[] = {
   "script", "style", "xmp", NULL
 };
 
 /**
  * HTML elements without an end tag; they must never open a skipped
  * region.
  */
 static const char *const void_elements[] = {
   "area", "base", "br", "col", "embed", "hr", "img", "input", "link",
   "meta", "param", "source", "track", "wbr", NULL
 };
 
 /**
  * Elements that start a sectioning context.
  */
 static const char *const sectioning_elements[] = {
   "article", "section", "main", "aside", "nav", NULL
 };
 
 
 /**
  * Does a start tag's attribute list mark the element as not part of
  * the text: hidden, aria-hidden="true", a hiding inline style, an ARIA
  * landmark role for the page margins, or a page break marker
  * (epub:type="pagebreak", role="doc-pagebreak")?
  *
  * @param a attribute text of the start tag (after the name)
  * @param len number of bytes in @a a
  * @return 1 to skip the element
  */
 static int
 attrs_skip (const char *a,
             size_t len)
 {
   static const char *const roles[] = {
     "navigation", "banner", "contentinfo", "complementary", "search",
     "doc-pagebreak", NULL
   };
   size_t i = 0;
   unsigned int n = 0;
 
   while ( (i < len) && (n++ < MAX_ATTRS) )
   {
     const char *name;
     size_t nlen;
     const char *val = NULL;
     size_t vlen = 0;
     char lname[32];
     char lval[256];
 
     while ( (i < len) && (is_space (a[i]) || ('/' == a[i]) ) )
       i++;
     name = &a[i];
     while ( (i < len) && (! is_space (a[i])) &&
             ('=' != a[i]) && ('/' != a[i]) )
       i++;
     nlen = (size_t) (&a[i] - name);
     if (0 == nlen)
       break;
     while ( (i < len) && is_space (a[i]))
       i++;
     if ( (i < len) && ('=' == a[i]) )
     {
       i++;
       while ( (i < len) && is_space (a[i]))
         i++;
       if ( (i < len) && ( ('"' == a[i]) || ('\'' == a[i]) ) )
       {
         const char *q = memchr (&a[i + 1], a[i], len - i - 1);
 
         val = &a[i + 1];
         vlen = (NULL == q) ? len - i - 1 : (size_t) (q - val);
         i += vlen + 2;
       }
       else
       {
         val = &a[i];
         while ( (i < len) && (! is_space (a[i])) )
           i++;
         vlen = (size_t) (&a[i] - val);
       }
     }
     if (nlen >= sizeof (lname))
       continue;
     for (size_t k = 0; k < nlen; k++)
       lname[k] = (char) tolower ((unsigned char) name[k]);
     lname[nlen] = '\0';
     {
       size_t o = 0;
 
       /* lower-case, and drop white space so that `display: none'
          matches */
       for (size_t k = 0; (k < vlen) && (o + 1 < sizeof (lval)); k++)
         if (! is_space (val[k]))
           lval[o++] = (char) tolower ((unsigned char) val[k]);
         else if ( (0 != o) && (' ' != lval[o - 1]) )
           lval[o++] = ' ';
       lval[o] = '\0';
     }
     if (0 == strcmp (lname, "hidden"))
     {
       /* hidden="until-found" is collapsed content that find-in-page
          reveals, i.e. part of the text */
       if (0 != strcmp (lval, "until-found"))
         return 1;
       continue;
     }
     if (0 == strcmp (lname, "aria-hidden"))
     {
       if (0 == strcmp (lval, "true"))
         return 1;
       continue;
     }
     if (0 == strcmp (lname, "style"))
     {
       char css[256];
       size_t o = 0;
 
       for (size_t k = 0; '\0' != lval[k]; k++)
         if (' ' != lval[k])
           css[o++] = lval[k];
       css[o] = '\0';
       if ( (NULL != strstr (css, "display:none")) ||
            (NULL != strstr (css, "visibility:hidden")) )
         return 1;
       continue;
     }
     if ( (0 == strcmp (lname, "role")) ||
          (0 == strcmp (lname, "epub:type")) )
     {
       const char *p = lval;
 
       while ('\0' != *p)
       {
         size_t tl = strcspn (p, " ");
 
         if ( (tl == strlen ("pagebreak")) &&
              (0 == memcmp (p, "pagebreak", tl)) )
           return 1;
         for (unsigned int r = 0; NULL != roles[r]; r++)
           if ( (tl == strlen (roles[r])) &&
                (0 == memcmp (p, roles[r], tl)) )
             return 1;
         p += tl;
         while (' ' == *p)
           p++;
       }
     }
   }
   return 0;
 }
 
 
 /**
  * Handle a <br>: one is a space, two in a row end the paragraph.
  *
  * @param ms conversion state
  */
 static void
 markup_br (struct MarkupState *ms)
 {
   if (ms->after_br)
   {
     EXTRACTOR_fulltext_paragraph_ (ms->ft);
     ms->after_br = 0;
   }
   else
   {
     EXTRACTOR_fulltext_space_ (ms->ft);
     ms->after_br = 1;
   }
 }
 
 
 /**
  * End the current paragraph.
  *
  * @param ms conversion state
  */
 static void
 markup_par (struct MarkupState *ms)
 {
   EXTRACTOR_fulltext_paragraph_ (ms->ft);
   ms->after_br = 0;
 }
 
 
 /**
  * Append character data (between tags) to the text.
  *
  * @param ms conversion state
  * @param p the text, may contain character references
  * @param len number of bytes at @a p
  */
 static void
 markup_chars (struct MarkupState *ms,
               const char *p,
               size_t len)
 {
   size_t i = 0;
 
   while (i < len)
   {
     size_t run = i;
     uint32_t cp;
     size_t used;
 
     while ( (run < len) && ('&' != p[run]) )
     {
       if (ms->after_br && (! is_space (p[run])) )
         ms->after_br = 0;
       run++;
     }
     EXTRACTOR_fulltext_append_ (ms->ft, &p[i], run - i);
     if (run >= len)
       return;
     used = decode_entity (&p[run + 1], len - run - 1, &cp);
     if (0 == used)
     {
       EXTRACTOR_fulltext_append_ (ms->ft, "&", 1);
       i = run + 1;
       ms->after_br = 0;
       continue;
     }
     EXTRACTOR_fulltext_append_cp_ (ms->ft, cp);
     if ( (0xA0 != cp) && (' ' != cp) )
       ms->after_br = 0;
     i = run + 1 + used;
   }
 }
 
 
 /**
  * Find the end of the tag starting at @a p, honouring quoted
  * attribute values.
  *
  * @param p start of the tag body (after the name)
  * @param end end of the buffer
  * @return pointer to the closing '>', or @a end
  */
 static const char *
 tag_end (const char *p,
          const char *end)
 {
   char quote = '\0';
 
   for (; p < end; p++)
   {
     if ('\0' != quote)
     {
       if (*p == quote)
         quote = '\0';
     }
     else if ( ('"' == *p) || ('\'' == *p) )
       quote = *p;
     else if ('>' == *p)
       return p;
   }
   return end;
 }
 
 
 /**
  * Find the case-insensitive end tag `</name' at or after @a p.
  *
  * @param p where to search from
  * @param end end of the buffer
  * @param name lower-case element name
  * @return pointer to the '<' of the end tag, or @a end
  */
 static const char *
 find_end_tag (const char *p,
               const char *end,
               const char *name)
 {
   size_t nlen = strlen (name);
 
   for (; p + 2 + nlen <= end; p++)
   {
     if ( ('<' != p[0]) || ('/' != p[1]) )
       continue;
     if (0 == strncasecmp (&p[2], name, nlen))
       return p;
   }
   return end;
 }
 
 
 /**
  * Convert one (X)HTML document to text.  This is a tolerant scanner,
  * not a parser: EPUB content documents are XHTML, but MOBI text is
  * HTML 3.2 with unclosed elements, so all we track is which element's
  * content we are skipping and whether we are inside a sectioning
  * element.  Block-level elements end paragraphs, as do MOBI page
  * breaks (<mbp:pagebreak/>); one <br> is a space and two end a
  * paragraph.  Memory and time are linear in @a len; there is no
  * recursion.
  *
  * @param ft where the text goes
  * @param data the document in UTF-8
  * @param len number of bytes in @a data
  */
 static void
 markup_to_text (struct EXTRACTOR_FullText *ft,
                 const char *data,
                 size_t len)
 {
   struct MarkupState ms;
   const char *p = data;
   const char *end = data + len;
 
   memset (&ms, 0, sizeof (ms));
   ms.ft = ft;
   while ( (p < end) && (! ft->full) )
   {
     const char *lt;
     const char *name;
     const char *gt;
     char local[32];
     size_t nlen;
     int closing;
     int self_closing;
 
     lt = memchr (p, '<', (size_t) (end - p));
     if (NULL == lt)
       lt = end;
     if ('\0' == ms.skip_name[0])
       markup_chars (&ms, p, (size_t) (lt - p));
     p = lt;
     if (p >= end)
       break;
     if ( (end - p >= 4) && (0 == memcmp (p, "<!--", 4)) )
     {
       const char *c = mem_find (p + 4, (size_t) (end - p - 4), "-->", 3);
 
       p = (NULL == c) ? end : c + 3;
       continue;
     }
     if ( (end - p >= 9) && (0 == memcmp (p, "<![CDATA[", 9)) )
     {
       const char *c = mem_find (p + 9, (size_t) (end - p - 9), "]]>", 3);
       const char *ce = (NULL == c) ? end : c;
 
       if ('\0' == ms.skip_name[0])
       {
         EXTRACTOR_fulltext_append_ (ft, p + 9, (size_t) (ce - p - 9));
         ms.after_br = 0;
       }
       p = (NULL == c) ? end : c + 3;
       continue;
     }
     if ( (end - p >= 2) && ( ('!' == p[1]) || ('?' == p[1]) ) )
     {
       gt = memchr (p, '>', (size_t) (end - p));
       p = (NULL == gt) ? end : gt + 1;
       continue;
     }
     closing = ( (end - p >= 2) && ('/' == p[1]) );
     name = p + (closing ? 2 : 1);
     nlen = 0;
     while ( (name + nlen < end) &&
             is_name_char (name[nlen]) )
       nlen++;
     if ( (0 == nlen) ||
          (! isalpha ((unsigned char) name[0])) )
     {
       /* a stray '<' in the text */
       if ('\0' == ms.skip_name[0])
         markup_chars (&ms, "<", 1);
       p++;
       continue;
     }
     gt = tag_end (name + nlen, end);
     self_closing = ( (! closing) && (gt > name + nlen) && ('/' == gt[-1]) );
     {
       /* the local name, lower case, without namespace prefix */
       const char *l = name;
       size_t ll;
 
       for (size_t k = 0; k < nlen; k++)
         if (':' == name[k])
           l = &name[k + 1];
       ll = (size_t) (name + nlen - l);
       if (ll >= sizeof (local))
         ll = sizeof (local) - 1;
       for (size_t k = 0; k < ll; k++)
         local[k] = (char) tolower ((unsigned char) l[k]);
       local[ll] = '\0';
     }
     p = (gt < end) ? gt + 1 : end;
 
     if ( (! closing) && (! self_closing) &&
          name_in (local, raw_text_elements) )
     {
       /* no markup inside; the content is never text, except for the
          obsolete <xmp>, which is rare enough to skip too */
       const char *et = find_end_tag (p, end, local);
 
       if (et < end)
         et = tag_end (et, end);
       p = (et < end) ? et + 1 : end;
       if ('\0' == ms.skip_name[0])
         markup_par (&ms);
       continue;
     }
     if ('\0' != ms.skip_name[0])
     {
       if (0 != strcmp (local, ms.skip_name))
         continue;
       if (closing)
       {
         if (0 == --ms.skip_nest)
         {
           ms.skip_name[0] = '\0';
           if (name_in (local, block_elements))
             markup_par (&ms);
         }
       }
       else if (! self_closing)
       {
         ms.skip_nest++;
       }
       continue;
     }
     if (closing)
     {
       if (name_in (local, sectioning_elements) && (ms.sections > 0))
         ms.sections--;
       if (name_in (local, block_elements))
         markup_par (&ms);
       continue;
     }
     if (name_in (local, skip_elements) ||
         ( ( (0 == strcmp (local, "header")) ||
             (0 == strcmp (local, "footer")) ) &&
           (0 == ms.sections) ) ||
         attrs_skip (name + nlen, (size_t) (gt - name - nlen)) )
     {
       if (name_in (local, block_elements))
         markup_par (&ms);
       if ( (! self_closing) &&
            (! name_in (local, void_elements)) )
       {
         strcpy (ms.skip_name, local);
         ms.skip_nest = 1;
       }
       continue;
     }
     if (0 == strcmp (local, "br"))
     {
       markup_br (&ms);
       continue;
     }
     if ( (0 == strcmp (local, "pagebreak")) ||
          name_in (local, block_elements) )
       markup_par (&ms);
     if ( (! self_closing) &&
          name_in (local, sectioning_elements) )
       ms.sections++;
   }
 }
 
 
 /* ------------------------------------------------------------------ */
 /* Full text: EPUB                                                     */
 /* ------------------------------------------------------------------ */
 
 /**
  * One entry of the OPF manifest.
  */
 struct ManifestItem
 {
   /**
    * The item's id.
    */
   char *id;
 
   /**
    * The item's href, relative to the OPF.
    */
   char *href;
 };
 
 
 /**
  * Find the next start tag named @a name (not a longer name that begins
  * with it) in @a data from offset @a *pos.
  *
  * @param data buffer to search
  * @param len number of bytes in @a data
  * @param[in,out] pos where to start; set to just past the tag
  * @param name element name including the '<', e.g. "<item"
  * @param[out] body start of the attributes
  * @param[out] body_len number of bytes of attributes
  * @return 1 if found
  */
 static int
 next_tag (const char *data,
           size_t len,
           size_t *pos,
           const char *name,
           const char **body,
           size_t *body_len)
 {
   size_t nlen = strlen (name);
 
   while (*pos < len)
   {
     const char *t = mem_find (&data[*pos], len - *pos, name, nlen);
     const char *gt;
     size_t off;
 
     if (NULL == t)
     {
       *pos = len;
       return 0;
     }
     off = (size_t) (t - data) + nlen;
     if ( (off < len) &&
          is_name_char (data[off]) )
     {
       *pos = off;
       continue;
     }
     gt = tag_end (&data[off], data + len);
     *body = &data[off];
     *body_len = (size_t) (gt - &data[off]);
     *pos = (size_t) (gt - data) + ( (gt < data + len) ? 1 : 0);
     return 1;
   }
   return 0;
 }
 
 
 /**
  * Value of a hex digit.
  *
  * @param c character
  * @return value, or -1 if @a c is not a hex digit
  */
 static int
 hex_val (char c)
 {
   if ( ('0' <= c) && ('9' >= c) )
     return c - '0';
   if ( ('a' <= c) && ('f' >= c) )
     return c - 'a' + 10;
   if ( ('A' <= c) && ('F' >= c) )
     return c - 'A' + 10;
   return -1;
 }
 
 
 /**
  * Resolve @a href relative to the directory of @a base (a path inside
  * the container): drop the fragment, undo percent-encoding and
  * normalise `.' and `..' segments.
  *
  * @param base path of the document @a href appears in
  * @param href the reference
  * @param[out] out where to write the member name
  * @param outsz number of bytes available in @a out
  * @return 1 on success, 0 if @a href is not a local path we can use
  */
 static int
 resolve_href (const char *base,
               const char *href,
               char *out,
               size_t outsz)
 {
   char tmp[1024];
   size_t t = 0;
   size_t o = 0;
   const char *slash;
   const char *seg;
 
   if ('\0' == href[0])
     return 0;
   /* a scheme (http:, data:, ...) means the resource is not in the
      container */
   for (size_t i = 0; '\0' != href[i]; i++)
   {
     if (':' == href[i])
       return 0;
     if ( ('/' == href[i]) || ('#' == href[i]) || ('?' == href[i]) )
       break;
   }
   if ('/' != href[0])
   {
     slash = strrchr (base, '/');
     if (NULL != slash)
     {
       size_t bl = (size_t) (slash - base) + 1;
 
       if (bl >= sizeof (tmp))
         return 0;
       memcpy (tmp, base, bl);
       t = bl;
     }
   }
   for (size_t i = 0; ('\0' != href[i]) && ('#' != href[i]) &&
        ('?' != href[i]); i++)
   {
     char c = href[i];
 
     if ( ('%' == c) &&
          (hex_val (href[i + 1]) >= 0) &&
          (hex_val (href[i + 2]) >= 0) )
     {
       c = (char) (hex_val (href[i + 1]) * 16 + hex_val (href[i + 2]));
       i += 2;
     }
     if ('\0' == c)
       return 0;
     if (t + 1 >= sizeof (tmp))
       return 0;
     tmp[t++] = c;
   }
   tmp[t] = '\0';
   /* normalise the segments into out */
   seg = tmp;
   while ('\0' != *seg)
   {
     size_t sl = strcspn (seg, "/");
 
     if ( (0 == sl) ||
          ( (1 == sl) && ('.' == seg[0]) ) )
     {
       /* empty or `.' */
     }
     else if ( (2 == sl) && ('.' == seg[0]) && ('.' == seg[1]) )
     {
       /* `..': drop the last segment of out */
       while ( (o > 0) && ('/' != out[o - 1]) )
         o--;
       if (o > 0)
         o--;
       while ( (o > 0) && ('/' != out[o - 1]) )
         o--;
     }
     else
     {
       if (o + sl + 2 > outsz)
         return 0;
       memcpy (&out[o], seg, sl);
       o += sl;
       out[o++] = '/';
     }
     seg += sl;
     if ('/' == *seg)
       seg++;
   }
   if (0 == o)
     return 0;
   out[o - 1] = '\0';   /* replace the trailing '/' */
   return 1;
 }
 
 
 /**
  * Release a manifest.
  *
  * @param items the manifest
  * @param n number of entries in @a items
  */
 static void
 free_manifest (struct ManifestItem *items,
                unsigned int n)
 {
   for (unsigned int i = 0; i < n; i++)
   {
     free (items[i].id);
     free (items[i].href);
   }
   free (items);
 }
 
 
 /**
  * One content document in the spine.
  */
 struct SpineDoc
 {
   /**
    * Member name in the archive.
    */
   char *member;
 
   /**
    * The document's text (normalized as by #EXTRACTOR_FullText), NULL
    * if not (yet) read.  Shared between entries for the same member.
    */
   char *text;
 
   /**
    * Number of bytes in @e text.
    */
   size_t text_len;
 
   /**
    * 1 if this entry owns @e text.
    */
   int owner;
 };
 
 
 /**
  * Index entry for looking up spine documents by member name.
  */
 struct SpineKey
 {
   /**
    * Member name (owned by the `struct SpineDoc`).
    */
   const char *member;
 
   /**
    * Position in the spine.
    */
   unsigned int idx;
 };
 
 
 /**
  * Order spine keys by member name (ignoring case, as the archive
  * lookup does), then by spine position.
  *
  * @param a a `struct SpineKey`
  * @param b a `struct SpineKey`
  * @return comparison result
  */
 static int
 cmp_spine_key (const void *a,
                const void *b)
 {
   const struct SpineKey *ka = a;
   const struct SpineKey *kb = b;
   int r = strcasecmp (ka->member,
                       kb->member);
 
   if (0 != r)
     return r;
   return (ka->idx < kb->idx) ? -1 : (ka->idx > kb->idx);
 }
 
 
 /**
  * Parse the manifest and spine of the OPF: the content documents in
  * reading order.
  *
  * @param opf the package document
  * @param opf_len number of bytes in @a opf
  * @param opf_path member name of the package document
  * @param enc contents of META-INF/encryption.xml, or NULL
  * @param enc_len number of bytes in @a enc
  * @param[out] docs the spine, caller frees (also the member names)
  * @return number of entries in @a docs
  */
 static unsigned int
 parse_spine (const char *opf,
              size_t opf_len,
              const char *opf_path,
              const char *enc,
              size_t enc_len,
              struct SpineDoc **docs)
 {
   struct ManifestItem *items;
   struct SpineDoc *d;
   unsigned int n_items = 0;
   unsigned int n_docs = 0;
   unsigned int n_refs = 0;
   size_t pos;
   const char *body;
   size_t body_len;
   const char *spine;
   const char *spine_end;
 
   *docs = NULL;
   spine = mem_find (opf, opf_len, "<spine", strlen ("<spine"));
   if (NULL == spine)
     return 0;
   spine_end = mem_find (spine,
                         opf_len - (size_t) (spine - opf),
                         "</spine",
                         strlen ("</spine"));
   if (NULL == spine_end)
     spine_end = opf + opf_len;
   items = calloc (MAX_MANIFEST, sizeof (struct ManifestItem));
   d = calloc (MAX_SPINE, sizeof (struct SpineDoc));
   if ( (NULL == items) ||
        (NULL == d) )
   {
     free (items);
     free (d);
     return 0;
   }
   pos = 0;
   while ( (n_items < MAX_MANIFEST) &&
           next_tag (opf, opf_len, &pos, "<item", &body, &body_len) )
   {
     char id[256];
     char href[1024];
     char mt[128];
 
     if ( (! xml_attr (body, body_len, "id", id, sizeof (id))) ||
          (! xml_attr (body, body_len, "href", href, sizeof (href))) )
       continue;
     if (! xml_attr (body, body_len, "media-type", mt, sizeof (mt)))
       mt[0] = '\0';
     if ( (NULL == strstr (mt, "html")) &&
          (NULL == strstr (mt, "oeb1-document")) &&
          (NULL == strstr (mt, "dtbook")) )
       continue;   /* images, style sheets, fonts, the NCX */
     items[n_items].id = strdup (id);
     items[n_items].href = strdup (href);
     if ( (NULL == items[n_items].id) ||
          (NULL == items[n_items].href) )
     {
       free (items[n_items].id);
       free (items[n_items].href);
       break;
     }
     n_items++;
   }
   pos = 0;
   while ( (n_refs++ < MAX_SPINE) &&
           next_tag (spine,
                     (size_t) (spine_end - spine),
                     &pos,
                     "<itemref",
                     &body,
                     &body_len) )
   {
     char idref[256];
     char linear[16];
     char member[1024];
     unsigned int i;
 
     if (! xml_attr (body, body_len, "idref", idref, sizeof (idref)))
       continue;
     /* linear="no" marks auxiliary content (pop-up notes, answer keys)
        that is outside the reading order */
     if ( (xml_attr (body, body_len, "linear", linear, sizeof (linear))) &&
          (0 == strcmp (linear, "no")) )
       continue;
     for (i = 0; i < n_items; i++)
       if (0 == strcmp (items[i].id, idref))
         break;
     if (i == n_items)
       continue;
     if (! resolve_href (opf_path,
                         items[i].href,
                         member,
                         sizeof (member)))
       continue;
     /* content documents that are listed in encryption.xml are DRM
        protected; font obfuscation only lists fonts */
     if ( (NULL != enc) &&
          (NULL != mem_find (enc, enc_len, member, strlen (member))) )
       continue;
     if (NULL == (d[n_docs].member = strdup (member)))
       break;
     n_docs++;
   }
   free_manifest (items, n_items);
   *docs = d;
   return n_docs;
 }
 
 
 /**
  * Append @a text, which is normalized as by #EXTRACTOR_FullText
  * (paragraphs separated by "\n\n"), to @a ft.
  *
  * @param ft where to append
  * @param text the text
  * @param len number of bytes in @a text
  */
 static void
 append_normalized (struct EXTRACTOR_FullText *ft,
                    const char *text,
                    size_t len)
 {
   size_t i = 0;
 
   while ( (i < len) &&
           (! ft->full) )
   {
     const char *nl = memchr (&text[i], '\n', len - i);
     size_t pl = (NULL == nl) ? len - i : (size_t) (nl - &text[i]);
 
     EXTRACTOR_fulltext_append_ (ft, &text[i], pl);
     EXTRACTOR_fulltext_paragraph_ (ft);
     i += pl;
     while ( (i < len) && ('\n' == text[i]) )
       i++;
   }
 }
 
 
 /**
  * Collect the text of an EPUB in reading order (the linear entries of
  * the OPF spine) and emit it.
  *
  * Looking a member up by name walks the central directory, so doing
  * that once per spine entry would be quadratic in a hostile archive.
  * Instead, the archive is walked once, each content document found in
  * the spine is converted to text as it comes along, and the texts are
  * put in spine order at the end.
  *
  * @param ec extraction context
  * @param uf the archive
  * @param opf the package document
  * @param opf_len number of bytes in @a opf
  * @param opf_path member name of the package document
  * @return 1 if the caller should stop extracting, 0 to continue
  */
 static int
 epub_fulltext (struct EXTRACTOR_ExtractContext *ec,
                struct EXTRACTOR_UnzipFile *uf,
                const char *opf,
                size_t opf_len,
                const char *opf_path)
 {
   struct EXTRACTOR_FullText ft;
   struct SpineDoc *docs;
   struct SpineKey *order;
   unsigned int n_docs;
   size_t budget = MAX_BOOK_MARKUP;
   size_t text_total = 0;
   size_t enc_len = 0;
   char *enc;
   int ret;
 
   enc = read_member (uf,
                      "META-INF/encryption.xml",
                      MAX_OPF,
                      &enc_len);
   n_docs = parse_spine (opf,
                         opf_len,
                         opf_path,
                         enc,
                         enc_len,
                         &docs);
   free (enc);
   if (0 == n_docs)
   {
     free (docs);
     return 0;
   }
   if (NULL == (order = malloc (n_docs * sizeof (struct SpineKey))))
   {
     for (unsigned int i = 0; i < n_docs; i++)
       free (docs[i].member);
     free (docs);
     return 0;
   }
   for (unsigned int i = 0; i < n_docs; i++)
   {
     order[i].member = docs[i].member;
     order[i].idx = i;
   }
   qsort (order, n_docs, sizeof (struct SpineKey), &cmp_spine_key);
 
   if (EXTRACTOR_UNZIP_OK ==
       EXTRACTOR_common_unzip_go_to_first_file (uf))
   {
     do
     {
       struct EXTRACTOR_UnzipFileInfo fi;
       struct EXTRACTOR_FullText cft;
       char name[1024];
       unsigned int lo = 0;
       unsigned int hi = n_docs;
       unsigned int first;
       char *doc;
       char *utf8;
       size_t doc_len;
       size_t utf8_len;
 
       if (EXTRACTOR_UNZIP_OK !=
           EXTRACTOR_common_unzip_get_current_file_info (uf,
                                                         &fi,
                                                         name,
                                                         sizeof (name),
                                                         NULL, 0,
                                                         NULL, 0))
         break;
       name[sizeof (name) - 1] = '\0';
       /* first spine entry for this member (binary search) */
       while (lo < hi)
       {
         unsigned int mid = lo + (hi - lo) / 2;
 
         if (strcasecmp (order[mid].member, name) < 0)
           lo = mid + 1;
         else
           hi = mid;
       }
       if ( (lo >= n_docs) ||
            (0 != strcasecmp (order[lo].member, name)) )
         continue;
       first = order[lo].idx;
       if (NULL != docs[first].text)
         continue;   /* the same name twice in the archive */
       doc = read_current (uf,
                           (budget < MAX_CHAPTER) ? budget : MAX_CHAPTER,
                           &doc_len);
       if (NULL == doc)
         continue;
       budget -= doc_len;
       utf8 = xhtml_to_utf8 (doc, doc_len, &utf8_len);
       free (doc);
       if (NULL == utf8)
         continue;
       EXTRACTOR_fulltext_init_ (&cft);
       EXTRACTOR_fulltext_set_context_ (&cft, ec);
       markup_to_text (&cft, utf8, utf8_len);
       free (utf8);
       if (0 == cft.len)
       {
         EXTRACTOR_fulltext_clear_ (&cft);
         continue;
       }
       text_total += cft.len;
       /* every spine entry for this member shares the text */
       for (unsigned int k = lo;
            (k < n_docs) &&
            (0 == strcasecmp (order[k].member, name));
            k++)
       {
         docs[order[k].idx].text = cft.buf;
         docs[order[k].idx].text_len = cft.len;
       }
       docs[first].owner = 1;
     }
     while ( (budget > 0) &&
             (text_total < EXTRACTOR_FULLTEXT_MAX) &&
             (EXTRACTOR_UNZIP_OK ==
              EXTRACTOR_common_unzip_go_to_next_file (uf)) );
   }
   free (order);
 
   EXTRACTOR_fulltext_init_ (&ft);
   EXTRACTOR_fulltext_set_context_ (&ft, ec);
   for (unsigned int i = 0; (i < n_docs) && (! ft.full); i++)
   {
     /* each spine entry is a chapter, which always ends a paragraph */
     if (NULL != docs[i].text)
       append_normalized (&ft, docs[i].text, docs[i].text_len);
     EXTRACTOR_fulltext_paragraph_ (&ft);
   }
   for (unsigned int i = 0; i < n_docs; i++)
   {
     if (docs[i].owner)
       free (docs[i].text);
     free (docs[i].member);
   }
   free (docs);
   ret = EXTRACTOR_fulltext_emit_ (&ft, ec, PLUGIN_NAME);
   EXTRACTOR_fulltext_clear_ (&ft);
   return (0 != ret) ? 1 : 0;
 }
 
 
 /**
  * Handle an EPUB: read `META-INF/container.xml' to find the OPF package
  * document, then report what the package document holds.
  *
  * @param ec extraction context
  * @param uf the opened archive
  */
 static void
 extract_epub (struct EXTRACTOR_ExtractContext *ec,
               struct EXTRACTOR_UnzipFile *uf)
 {
   char path[512];
   char opf_path[512];
   char attr[64];
   char *container;
   char *opf = NULL;
   size_t container_len;
   size_t opf_len = 0;
   const char *p;
   const char *q;
   unsigned int items = 0;
 
   if (0 !=
       ec->proc (ec->cls,
                 PLUGIN_NAME,
                 EXTRACTOR_METATYPE_MIMETYPE,
                 EXTRACTOR_METAFORMAT_UTF8,
                 "text/plain",
                 "application/epub+zip",
                 strlen ("application/epub+zip") + 1))
     return;
   container = read_member (uf,
                            "META-INF/container.xml",
                            MAX_CONTAINER,
                            &container_len);
   if (NULL != container)
   {
     p = mem_find (container,
                   container_len,
                   "full-path=",
                   strlen ("full-path="));
     if (NULL != p)
     {
       p += strlen ("full-path=");
       if ( ('"' == *p) || ('\'' == *p) )
       {
         char quote = *p;
 
         p++;
         q = memchr (p,
                     quote,
                     container_len - (size_t) (p - container));
         if (NULL != q)
           (void) xml_text (p,
                            (size_t) (q - p),
                            path,
                            sizeof (path));
         else
           path[0] = '\0';
       }
       else
       {
         path[0] = '\0';
       }
       if ('\0' != path[0])
       {
         opf = read_member (uf,
                            path,
                            MAX_OPF,
                            &opf_len);
         if (NULL != opf)
           memcpy (opf_path, path, sizeof (opf_path));
       }
     }
     free (container);
   }
   if (NULL == opf)
   {
     /* no usable container.xml; the conventional locations are worth a
        try before giving up */
     static const char *guesses[] = {
       "OEBPS/content.opf",
       "content.opf",
       "OPS/content.opf",
       NULL
     };
 
     for (unsigned int i = 0; NULL != guesses[i]; i++)
     {
       opf = read_member (uf,
                          guesses[i],
                          MAX_OPF,
                          &opf_len);
       if (NULL != opf)
       {
         snprintf (opf_path, sizeof (opf_path), "%s", guesses[i]);
         break;
       }
     }
   }
   if (NULL != opf)
   {
     /* the <package version="..."> attribute distinguishes EPUB 2 from
        EPUB 3, which changes what else we may expect to find */
     p = mem_find (opf,
                   opf_len,
                   "<package",
                   strlen ("<package"));
     if (NULL != p)
     {
       size_t tail = opf_len - (size_t) (p - opf);
       const char *gt;
 
       if (tail > 1024)
         tail = 1024;
       /* stop at the end of the start tag, so that a `version=' further
          down the document cannot be mistaken for this one */
       gt = memchr (p,
                    '>',
                    tail);
       if (NULL != gt)
         tail = (size_t) (gt - p);
       if (tail <= strlen ("<package"))
         tail = strlen ("<package");
       if (xml_attr (p + strlen ("<package"),
                     tail - strlen ("<package"),
                     "version",
                     attr,
                     sizeof (attr)))
       {
         if (0 !=
             EXTRACTOR_forensic_emit_text_ (ec,
                                            PLUGIN_NAME,
                                            EXTRACTOR_METATYPE_FORMAT_VERSION,
                                            attr,
                                            strlen (attr)))
           goto CLEANUP;
       }
     }
     if (0 !=
         parse_opf_metadata (ec,
                             opf,
                             opf_len))
       goto CLEANUP;
     /* count the manifest entries; `<itemref>' does not match because
        the character after `<item' has to be a name delimiter */
     for (size_t i = 0; i + strlen ("<item") < opf_len; i++)
       if ( (0 == memcmp (&opf[i],
                          "<item",
                          strlen ("<item"))) &&
            (! is_name_char (opf[i + strlen ("<item")])) )
         items++;
     if (0 != items)
     {
       if (0 !=
           EXTRACTOR_forensic_emit_ (ec,
                                     PLUGIN_NAME,
                                     EXTRACTOR_METATYPE_ENTRY_COUNT,
                                     "%u",
                                     items))
         goto CLEANUP;
     }
   }
   if (has_member (uf,
                   "META-INF/encryption.xml"))
   {
     if (0 !=
         EXTRACTOR_forensic_emit_ (
           ec,
           PLUGIN_NAME,
           EXTRACTOR_METATYPE_ENCRYPTION,
           "META-INF/encryption.xml present"
           " (DRM or font obfuscation)"))
       goto CLEANUP;
   }
   if (has_member (uf,
                   "META-INF/rights.xml"))
   {
     if (0 !=
         EXTRACTOR_forensic_emit_ (ec,
                                   PLUGIN_NAME,
                                   EXTRACTOR_METATYPE_ENCRYPTION,
                                   "META-INF/rights.xml present"
                                   " (Adobe Content Server DRM)"))
       goto CLEANUP;
   }
   if ( (NULL != opf) &&
        (EXTRACTOR_fulltext_requested_ (ec->config)) )
     (void) epub_fulltext (ec,
                           uf,
                           opf,
                           opf_len,
                           opf_path);
 CLEANUP:
   free (opf);
 }
 
 
 /* ------------------------------------------------------------------ */
 /* Full text: MOBI                                                     */
 /* ------------------------------------------------------------------ */
 
 /**
  * Size of the trailing entries at the end of a MOBI text record, as
  * announced by the "extra data flags" of the MOBI header: bit 0 is the
  * multibyte-character overlap (its size is in the low two bits of the
  * last byte before the other entries), every other bit is an entry
  * whose size is a backward-encoded variable-length integer at its end.
  *
  * @param d the record
  * @param size number of bytes in @a d
  * @param flags extra data flags
  * @return number of bytes to strip from the end of the record
  */
 static size_t
 trailing_size (const unsigned char *d,
                size_t size,
                uint16_t flags)
 {
   size_t num = 0;
 
   for (unsigned int bit = 1; bit < 16; bit++)
   {
     size_t end;
     uint32_t v = 0;
 
     if (0 == (flags & (1u << bit)))
       continue;
     end = size - num;
     if (0 == end)
       return size;
     for (size_t k = (end >= 4) ? end - 4 : 0; k < end; k++)
     {
       if (0 != (d[k] & 0x80))
         v = 0;
       v = (v << 7) | (d[k] & 0x7F);
     }
     if (v > end)
       return size;
     num += v;
   }
   if (0 != (flags & 1))
   {
     if (num >= size)
       return size;
     num += (d[size - num - 1] & 0x3) + 1;
   }
   return (num > size) ? size : num;
 }
 
 
 /**
  * Decompress one PalmDOC (LZ77 variant) compressed record.
  *
  * @param in compressed data
  * @param len number of bytes in @a in
  * @param[out] out where to write the text
  * @param outsz number of bytes available in @a out
  * @return number of bytes written
  */
 static size_t
 palmdoc_decompress (const unsigned char *in,
                     size_t len,
                     unsigned char *out,
                     size_t outsz)
 {
   size_t i = 0;
   size_t o = 0;
 
   while ( (i < len) && (o < outsz) )
   {
     unsigned int c = in[i++];
 
     if ( (0 == c) ||
          ( (c >= 0x09) && (c <= 0x7F) ) )
     {
       out[o++] = (unsigned char) c;
     }
     else if (c <= 0x08)
     {
       /* c literal bytes follow */
       for (unsigned int k = 0; (k < c) && (i < len) && (o < outsz); k++)
         out[o++] = in[i++];
     }
     else if (c <= 0xBF)
     {
       unsigned int pair;
       size_t dist;
       unsigned int n;
 
       if (i >= len)
         break;
       pair = ((c << 8) | in[i++]) & 0x3FFF;
       dist = pair >> 3;
       n = (pair & 7) + 3;
       if ( (0 == dist) ||
            (dist > o) )
         break;   /* corrupt: points before the start */
       for (unsigned int k = 0; (k < n) && (o < outsz); k++, o++)
         out[o] = out[o - dist];
     }
     else
     {
       /* a space followed by the character c ^ 0x80 */
       out[o++] = ' ';
       if (o < outsz)
         out[o++] = (unsigned char) (c ^ 0x80);
     }
   }
   return o;
 }
 
 
 /**
  * Collect the text of a MOBI book and emit it.  Handles uncompressed
  * and PalmDOC-compressed books; HUFF/CDIC compression and encrypted
  * books are skipped.
  *
  * @param ec extraction context
  * @param rec record 0
  * @param rec_len number of bytes in @a rec (at least 16)
  * @param hdr_len length of the MOBI header, 0 if there is none
  * @param num_records number of records in the database
  * @param size size of the file
  */
 static void
 mobi_fulltext (struct EXTRACTOR_ExtractContext *ec,
                const unsigned char *rec,
                size_t rec_len,
                uint32_t hdr_len,
                uint16_t num_records,
                uint64_t size)
 {
   struct EXTRACTOR_FullText ft;
   uint16_t compression = EXTRACTOR_forensic_be16_ (&rec[0]);
   uint16_t text_records = EXTRACTOR_forensic_be16_ (&rec[8]);
   uint16_t extra_flags = 0;
   const char *cs = "windows-1252";
   unsigned char *info = NULL;
   unsigned char *in = NULL;
   unsigned char *text = NULL;
   size_t text_len = 0;
   size_t text_cap = 0;
   unsigned int n_info;
   char *utf8;
   size_t utf8_len;
 
   if (0 != EXTRACTOR_forensic_be16_ (&rec[12]))
     return;   /* encrypted */
   if ( (1 != compression) &&
        (2 != compression) )
     return;   /* HUFF/CDIC (17480) or unknown */
   if (hdr_len >= 28)
   {
     if (65001 == EXTRACTOR_forensic_be32_ (&rec[16 + 12]))
       cs = "utf-8";
   }
   /* the extra data flags exist from MOBI header length 0xE4 on */
   if ( (hdr_len >= 0xE4) &&
        (rec_len >= 16 + 0xE4) )
     extra_flags = EXTRACTOR_forensic_be16_ (&rec[16 + 0xE2]);
   if (text_records >= num_records)
     text_records = num_records - 1;
   if (0 == text_records)
     return;
   /* offsets of records 0 .. text_records + 1 */
   n_info = (unsigned int) text_records + 2;
   if (n_info > num_records)
     n_info = num_records;
   if (NULL == (info = malloc (8 * (size_t) n_info)))
     return;
   if (! EXTRACTOR_forensic_read_ (ec, 78, info, 8 * (size_t) n_info))
     goto CLEANUP;
   if ( (NULL == (in = malloc (MAX_TEXT_RECORD))) )
     goto CLEANUP;
   for (unsigned int r = 1; r <= text_records; r++)
   {
     uint64_t start = EXTRACTOR_forensic_be32_ (&info[8 * r]);
     uint64_t stop = (r + 1 < n_info)
                     ? EXTRACTOR_forensic_be32_ (&info[8 * (r + 1)])
                     : size;
     size_t len;
     size_t strip;
 
     if ( (stop <= start) ||
          (stop > size) ||
          (stop - start > MAX_TEXT_RECORD) )
       continue;
     len = (size_t) (stop - start);
     if (! EXTRACTOR_forensic_read_ (ec, (int64_t) start, in, len))
       continue;
     strip = trailing_size (in, len, extra_flags);
     len -= strip;
     if (text_len + MAX_TEXT_RECORD_OUT > text_cap)
     {
       size_t ncap = (0 == text_cap) ? 256 * 1024 : 2 * text_cap;
       unsigned char *nt;
 
       if (ncap > MAX_BOOK_MARKUP)
         break;
       if (NULL == (nt = realloc (text, ncap)))
         break;
       text = nt;
       text_cap = ncap;
     }
     if (1 == compression)
     {
       if (len > MAX_TEXT_RECORD_OUT)
         len = MAX_TEXT_RECORD_OUT;
       memcpy (&text[text_len], in, len);
       text_len += len;
     }
     else
     {
       text_len += palmdoc_decompress (in,
                                       len,
                                       &text[text_len],
                                       MAX_TEXT_RECORD_OUT);
     }
   }
   if (0 == text_len)
     goto CLEANUP;
   utf8 = convert_to_utf8 ((const char *) text, text_len, cs, &utf8_len);
   if (NULL == utf8)
     goto CLEANUP;
   free (text);
   text = NULL;
   EXTRACTOR_fulltext_init_ (&ft);
   EXTRACTOR_fulltext_set_context_ (&ft, ec);
   markup_to_text (&ft, utf8, utf8_len);
   free (utf8);
   (void) EXTRACTOR_fulltext_emit_ (&ft, ec, PLUGIN_NAME);
   EXTRACTOR_fulltext_clear_ (&ft);
 CLEANUP:
   free (text);
   free (in);
   free (info);
 }
 
 
 /**
  * Convert a Palm database time stamp.
  *
  * The format says seconds since 1904, but writers disagree and some use
  * the Unix epoch.  The two ranges do not overlap for any date between
  * 1970 and 2036, so the value itself says which convention was used.
  *
  * @param raw the 32 bit field
  * @return seconds since 1970, 0 if the field was never filled in
  */
 static int64_t
 palm_time (uint32_t raw)
 {
   if (0 == raw)
     return 0;
   if (((int64_t) raw) >= PALM_EPOCH_OFFSET)
     return ((int64_t) raw) - PALM_EPOCH_OFFSET;
   return (int64_t) raw;
 }
 
 
 /**
  * Report one EXTH record.
  *
  * @param ec extraction context
  * @param type EXTH record type
  * @param data the record's payload
  * @param len number of bytes in @a data
  * @param[in,out] have_title set to 1 once a title has been reported
  * @return 1 if the caller should stop extracting, 0 to continue
  */
 static int
 emit_exth (struct EXTRACTOR_ExtractContext *ec,
            uint32_t type,
            const unsigned char *data,
            size_t len,
            int *have_title)
 {
   enum EXTRACTOR_MetaType mt;
 
   switch (type)
   {
   case 100: mt = EXTRACTOR_METATYPE_AUTHOR_NAME; break;
   case 101: mt = EXTRACTOR_METATYPE_PUBLISHER; break;
   case 103: mt = EXTRACTOR_METATYPE_DESCRIPTION; break;
   case 104: mt = EXTRACTOR_METATYPE_ISBN; break;
   case 105: mt = EXTRACTOR_METATYPE_KEYWORDS; break;
   case 106: mt = EXTRACTOR_METATYPE_PUBLICATION_DATE; break;
   case 108:
     /* the tool that produced the file stamps itself here; calibre and
        kindlegen both do */
     mt = EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE;
     break;
   case 109: mt = EXTRACTOR_METATYPE_RIGHTS; break;
   case 112: mt = EXTRACTOR_METATYPE_URI; break;
   case 113: mt = EXTRACTOR_METATYPE_SERIAL; break;     /* ASIN */
   case 208:
     /* per-copy marking on files that came from a store */
     mt = EXTRACTOR_METATYPE_WATERMARK;
     break;
   case 502: mt = EXTRACTOR_METATYPE_MODIFICATION_DATE; break;
   case 503:
     mt = EXTRACTOR_METATYPE_TITLE;
     *have_title = 1;
     break;
   case 524: mt = EXTRACTOR_METATYPE_LANGUAGE; break;
   case 501:
     /* `EBOK' for a store book, `PDOC' for a personal document that was
        side-loaded or mailed to a device */
     return EXTRACTOR_forensic_emit_ (ec,
                                      PLUGIN_NAME,
                                      EXTRACTOR_METATYPE_ATTRIBUTES,
                                      "cdetype: %.*s",
                                      (int) ((len > 16) ? 16 : len),
                                      (const char *) data);
   case 209:
     /* "tamper proof keys": binary, and specific to the copy rather than
        to the book, so worth reporting even though we cannot decode it */
     return EXTRACTOR_forensic_emit_hex_ (ec,
                                          PLUGIN_NAME,
                                          EXTRACTOR_METATYPE_WATERMARK,
                                          data,
                                          (len > 32) ? 32 : len);
   default:
     return 0;
   }
   return EXTRACTOR_forensic_emit_text_ (ec,
                                         PLUGIN_NAME,
                                         mt,
                                         (const char *) data,
                                         len);
 }
 
 
 /**
  * Walk the EXTH header of a MOBI file.
  *
  * @param ec extraction context
  * @param rec record 0
  * @param rec_len number of bytes in @a rec
  * @param off offset of the EXTH header within @a rec
  * @param[in,out] have_title set to 1 once a title has been reported
  * @return 1 if the caller should stop extracting, 0 to continue
  */
 static int
 parse_exth (struct EXTRACTOR_ExtractContext *ec,
             const unsigned char *rec,
             size_t rec_len,
             size_t off,
             int *have_title)
 {
   uint32_t count;
   uint32_t exth_len;
   size_t end;
   size_t p;
 
   if ( (off > rec_len) ||
        (rec_len - off < 12) )
     return 0;
   if (0 != memcmp (&rec[off],
                    "EXTH",
                    4))
     return 0;
   exth_len = EXTRACTOR_forensic_be32_ (&rec[off + 4]);
   count = EXTRACTOR_forensic_be32_ (&rec[off + 8]);
   end = rec_len;
   if ( (exth_len >= 12) &&
        (exth_len <= rec_len - off) )
     end = off + exth_len;
   if (count > MAX_EXTH)
     count = MAX_EXTH;
   p = off + 12;
   for (uint32_t i = 0; i < count; i++)
   {
     uint32_t type;
     uint32_t rlen;
 
     if ( (p + 8 > end) ||
          (p + 8 < p) )
       break;
     type = EXTRACTOR_forensic_be32_ (&rec[p]);
     rlen = EXTRACTOR_forensic_be32_ (&rec[p + 4]);
     if (rlen < 8)
       break;   /* no forward progress; a malformed file must not spin */
     if (((size_t) rlen) > end - p)
       break;
     if (0 !=
         emit_exth (ec,
                    type,
                    &rec[p + 8],
                    (size_t) rlen - 8,
                    have_title))
       return 1;
     p += rlen;
   }
   return 0;
 }
 
 
 /**
  * Handle a Palm database that identifies itself as a MOBI book.
  *
  * @param ec extraction context
  * @param hdr the 78 byte PDB header
  */
 static void
 extract_mobi (struct EXTRACTOR_ExtractContext *ec,
               const unsigned char *hdr)
 {
   unsigned char info[16];
   unsigned char *rec = NULL;
   uint64_t size;
   uint32_t rec0_off;
   uint32_t rec1_off;
   uint32_t hdr_len = 0;
   uint16_t num_records;
   size_t rec_len;
   int have_title = 0;
   int rec_ok = 0;
 
   if (0 !=
       ec->proc (ec->cls,
                 PLUGIN_NAME,
                 EXTRACTOR_METATYPE_MIMETYPE,
                 EXTRACTOR_METAFORMAT_UTF8,
                 "text/plain",
                 "application/x-mobipocket-ebook",
                 strlen ("application/x-mobipocket-ebook") + 1))
     return;
   if (0 !=
       EXTRACTOR_forensic_emit_unix_time_ (ec,
                                           PLUGIN_NAME,
                                           EXTRACTOR_METATYPE_CREATION_DATE,
                                           palm_time (
                                             EXTRACTOR_forensic_be32_ (
                                               &hdr[36]))))
     return;
   if (0 !=
       EXTRACTOR_forensic_emit_unix_time_ (
         ec,
         PLUGIN_NAME,
         EXTRACTOR_METATYPE_MODIFICATION_DATE,
         palm_time (EXTRACTOR_forensic_be32_ (&hdr[40]))))
     return;
   num_records = EXTRACTOR_forensic_be16_ (&hdr[76]);
   if (0 !=
       EXTRACTOR_forensic_emit_ (ec,
                                 PLUGIN_NAME,
                                 EXTRACTOR_METATYPE_ENTRY_COUNT,
                                 "%u",
                                 (unsigned int) num_records))
     return;
   size = ec->get_size (ec->cls);
   if ( (2 > num_records) ||
        (UINT64_MAX == size) )
     goto NAME_ONLY;
   /* the record info list follows the header; entry i gives the offset
      of record i, so record 0 runs up to the start of record 1 */
   if (! EXTRACTOR_forensic_read_ (ec,
                                   78,
                                   info,
                                   16))
     goto NAME_ONLY;
   rec0_off = EXTRACTOR_forensic_be32_ (&info[0]);
   rec1_off = EXTRACTOR_forensic_be32_ (&info[8]);
   if ( (rec1_off <= rec0_off) ||
        (((uint64_t) rec1_off) > size) )
     goto NAME_ONLY;
   rec_len = (size_t) (rec1_off - rec0_off);
   if (rec_len > MAX_RECORD0)
     rec_len = MAX_RECORD0;
   if (rec_len < 16)
     goto NAME_ONLY;
   if (NULL == (rec = malloc (rec_len)))
     goto NAME_ONLY;
   if (! EXTRACTOR_forensic_read_ (ec,
                                   (int64_t) rec0_off,
                                   rec,
                                   rec_len))
     goto NAME_ONLY;
   rec_ok = 1;
   /* PalmDOC header: encryption type at offset 12 */
   switch (EXTRACTOR_forensic_be16_ (&rec[12]))
   {
   case 0:
     break;   /* no DRM; saying so would be noise */
   case 1:
     if (0 !=
         EXTRACTOR_forensic_emit_ (ec,
                                   PLUGIN_NAME,
                                   EXTRACTOR_METATYPE_ENCRYPTION,
                                   "old Mobipocket encryption"))
       goto CLEANUP;
     break;
   case 2:
     if (0 !=
         EXTRACTOR_forensic_emit_ (ec,
                                   PLUGIN_NAME,
                                   EXTRACTOR_METATYPE_ENCRYPTION,
                                   "Mobipocket DRM"))
       goto CLEANUP;
     break;
   default:
     if (0 !=
         EXTRACTOR_forensic_emit_ (ec,
                                   PLUGIN_NAME,
                                   EXTRACTOR_METATYPE_ENCRYPTION,
                                   "unknown encryption type %u",
                                   (unsigned int)
                                   EXTRACTOR_forensic_be16_ (&rec[12])))
       goto CLEANUP;
     break;
   }
   if ( (rec_len < 32) ||
        (0 != memcmp (&rec[16],
                      "MOBI",
                      4)) )
     goto NAME_ONLY;   /* PalmDOC without a MOBI header */
   hdr_len = EXTRACTOR_forensic_be32_ (&rec[20]);
   if ( (hdr_len < 24) ||
        (((uint64_t) hdr_len) + 16 > rec_len) )
   {
     hdr_len = 0;
     goto NAME_ONLY;
   }
   if (hdr_len >= 28)
   {
     const char *cs = NULL;
 
     switch (EXTRACTOR_forensic_be32_ (&rec[16 + 12]))
     {
     case 1252: cs = "windows-1252"; break;
     case 65001: cs = "UTF-8"; break;
     default: break;
     }
     if ( (NULL != cs) &&
          (0 !=
           EXTRACTOR_forensic_emit_ (ec,
                                     PLUGIN_NAME,
                                     EXTRACTOR_METATYPE_CHARACTER_SET,
                                     "%s",
                                     cs)) )
       goto CLEANUP;
   }
   /* EXTH flags live at MOBI header offset 112; bit 0x40 says that an
      EXTH header follows the MOBI header */
   if ( (hdr_len >= 116) &&
        (0 != (EXTRACTOR_forensic_be32_ (&rec[16 + 112]) & 0x40)) )
   {
     if (0 !=
         parse_exth (ec,
                     rec,
                     rec_len,
                     (size_t) hdr_len + 16,
                     &have_title))
       goto CLEANUP;
   }
   if ( (! have_title) &&
        (hdr_len >= 76) )
   {
     uint32_t fn_off = EXTRACTOR_forensic_be32_ (&rec[16 + 68]);
     uint32_t fn_len = EXTRACTOR_forensic_be32_ (&rec[16 + 72]);
 
     if ( (fn_len > 0) &&
          (fn_len <= EXTRACTOR_FORENSIC_MAX_STRING) &&
          (((uint64_t) fn_off) + fn_len <= rec_len) )
     {
       if (0 !=
           EXTRACTOR_forensic_emit_text_ (ec,
                                          PLUGIN_NAME,
                                          EXTRACTOR_METATYPE_TITLE,
                                          (const char *) &rec[fn_off],
                                          fn_len))
         goto CLEANUP;
       have_title = 1;
     }
   }
 NAME_ONLY:
   if ( (! have_title) &&
        (0 != EXTRACTOR_forensic_emit_text_ (ec,
                                             PLUGIN_NAME,
                                             EXTRACTOR_METATYPE_TITLE,
                                             (const char *) hdr,
                                             32)) )
     goto CLEANUP;
   if (rec_ok &&
       (EXTRACTOR_fulltext_requested_ (ec->config)) )
     mobi_fulltext (ec,
                    rec,
                    rec_len,
                    hdr_len,
                    num_records,
                    size);
 CLEANUP:
   free (rec);
 }
 
 
 /**
  * Main entry method for the ebook extraction plugin.
  *
  * @param ec extraction context provided to the plugin
  */
 void
 EXTRACTOR_ebook_extract_method (struct EXTRACTOR_ExtractContext *ec);
 
 void
 EXTRACTOR_ebook_extract_method (struct EXTRACTOR_ExtractContext *ec)
 {
   unsigned char hdr[78];
   struct EXTRACTOR_UnzipFile *uf;
 
   if (! EXTRACTOR_forensic_read_ (ec,
                                   0,
                                   hdr,
                                   4))
     return;
   if (0 != memcmp (hdr,
                    "PK\x03\x04",
                    4))
   {
     /* not a ZIP; the only other thing we handle is a Palm database,
        whose type/creator pair sits at offset 60 */
     if (! EXTRACTOR_forensic_read_ (ec,
                                     0,
                                     hdr,
                                     sizeof (hdr)))
       return;
     if (0 == memcmp (&hdr[60],
                      "BOOKMOBI",
                      8))
       extract_mobi (ec,
                     hdr);
     return;
   }
   if (NULL == (uf = EXTRACTOR_common_unzip_open (ec)))
     return;
   /* a plain ZIP, a docx and an ODF file all start `PK\003\004', so
      claim the file only if it carries the OCF structure */
   if (has_member (uf,
                   "META-INF/container.xml"))
   {
     extract_epub (ec,
                   uf);
   }
   else
   {
     size_t len;
     char *mt = read_member (uf,
                             "mimetype",
                             64,
                             &len);
 
     if ( (NULL != mt) &&
          (len >= strlen ("application/epub+zip")) &&
          (0 == memcmp (mt,
                        "application/epub+zip",
                        strlen ("application/epub+zip"))) )
     {
       free (mt);
       extract_epub (ec,
                     uf);
     }
     else
     {
       free (mt);
     }
   }
   (void) EXTRACTOR_common_unzip_close (uf);
 }
 
 
 /* end of ebook_extractor.c */