1 files changed, 694 insertions, 0 deletions

diff --git a/src/plugins/html_extractor.c b/src/plugins/html_extractor.c
new file mode 100644
index 0000000..65fb535
--- /dev/null
+++ b/src/plugins/html_extractor.c
@@ -0,0 +1,694 @@
+/*
+     This file is part of libextractor.
+     (C) 2002, 2003, 2004, 2005, 2009, 2012 Vidyut Samanta and Christian Grothoff
+
+     libextractor is free software; you can redistribute it and/or modify
+     it under the terms of the GNU General Public License as published
+     by the Free Software Foundation; either version 2, or (at your
+     option) any later version.
+
+     libextractor is distributed in the hope that it will be useful, but
+     WITHOUT ANY WARRANTY; without even the implied warranty of
+     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+     General Public License for more details.
+
+     You should have received a copy of the GNU General Public License
+     along with libextractor; see the file COPYING.  If not, write to the
+     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+     Boston, MA 02111-1307, USA.
+
+ */
+/**
+ * @file plugins/html_extractor.c
+ * @brief plugin to support HTML files
+ * @author Christian Grothoff
+ */
+#include "platform.h"
+#include "extractor.h"
+#include <magic.h>
+#include <tidy/tidy.h>
+#include <tidy/buffio.h>
+
+/**
+ * Mapping of HTML META names to LE types.
+ */
+static struct
+{
+  /**
+   * HTML META name.
+   */
+  const char *name;
+
+  /**
+   * Corresponding LE type.
+   */
+  enum EXTRACTOR_MetaType type;
+} tagmap[] = {
+  { "author", EXTRACTOR_METATYPE_AUTHOR_NAME },
+  { "dc.author", EXTRACTOR_METATYPE_AUTHOR_NAME },
+  { "title", EXTRACTOR_METATYPE_TITLE },
+  { "dc.title", EXTRACTOR_METATYPE_TITLE},
+  { "description", EXTRACTOR_METATYPE_DESCRIPTION },
+  { "dc.description", EXTRACTOR_METATYPE_DESCRIPTION },
+  { "subject", EXTRACTOR_METATYPE_SUBJECT},
+  { "dc.subject", EXTRACTOR_METATYPE_SUBJECT},
+  { "date", EXTRACTOR_METATYPE_UNKNOWN_DATE },
+  { "dc.date", EXTRACTOR_METATYPE_UNKNOWN_DATE},
+  { "publisher", EXTRACTOR_METATYPE_PUBLISHER },
+  { "dc.publisher", EXTRACTOR_METATYPE_PUBLISHER},
+  { "rights", EXTRACTOR_METATYPE_RIGHTS },
+  { "dc.rights", EXTRACTOR_METATYPE_RIGHTS },
+  { "copyright", EXTRACTOR_METATYPE_COPYRIGHT },
+  { "language", EXTRACTOR_METATYPE_LANGUAGE },  
+  { "keywords", EXTRACTOR_METATYPE_KEYWORDS },
+  { "abstract", EXTRACTOR_METATYPE_ABSTRACT },
+  { "formatter", EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE },
+  { "dc.creator", EXTRACTOR_METATYPE_CREATOR},
+  { "dc.identifier", EXTRACTOR_METATYPE_URI },
+  { "dc.format", EXTRACTOR_METATYPE_FORMAT },
+  { NULL, EXTRACTOR_METATYPE_RESERVED }
+};
+
+
+/**
+ * Global handle to MAGIC data.
+ */
+static magic_t magic;
+
+
+/**
+ * Map 'meta' tag to LE type.
+ *
+ * @param tag tag to map
+ * @return EXTRACTOR_METATYPE_RESERVED if the type was not found
+ */
+static enum EXTRACTOR_MetaType 
+tag_to_type (const char *tag)
+{
+  unsigned int i;
+
+  for (i=0; NULL != tagmap[i].name; i++)
+    if (0 == strcasecmp (tag,
+                         tagmap[i].name))
+      return tagmap[i].type;
+  return EXTRACTOR_METATYPE_RESERVED;
+}
+
+
+/**
+ * Function called by libtidy for error reporting.
+ *
+ * @param doc tidy doc being processed
+ * @param lvl report level
+ * @param line input line
+ * @param col input column
+ * @param mssg message
+ * @return FALSE (no output)
+ */
+static Bool
+report_cb (TidyDoc doc,
+           TidyReportLevel lvl,
+           uint line,
+           uint col,
+           ctmbstr mssg)
+{
+  return 0;
+}
+
+
+/**
+ * Input callback: get next byte of input.
+ *
+ * @param sourceData our 'struct EXTRACTOR_ExtractContext'
+ * @return next byte of input, EndOfStream on errors and EOF
+ */
+static int
+get_byte_cb (void *sourceData)
+{
+  struct EXTRACTOR_ExtractContext *ec = sourceData;
+  void *data;
+
+  if (1 !=
+      ec->read (ec->cls,
+                &data, 1))
+    return EndOfStream;
+  return *(unsigned char*) data;
+}
+
+
+/**
+ * Input callback: unget last byte of input.
+ *
+ * @param sourceData our 'struct EXTRACTOR_ExtractContext'
+ * @param bt byte to unget (ignored)
+ */
+static void
+unget_byte_cb (void *sourceData, byte bt)
+{
+  struct EXTRACTOR_ExtractContext *ec = sourceData;
+  
+  (void) ec->seek (ec->cls, -1, SEEK_CUR);
+}
+
+
+/**
+ * Input callback: check for EOF.
+ *
+ * @param sourceData our 'struct EXTRACTOR_ExtractContext'
+ * @return true if we are at the EOF
+ */
+static Bool
+eof_cb (void *sourceData)
+{
+  struct EXTRACTOR_ExtractContext *ec = sourceData;
+
+  return ec->seek (ec->cls, 0, SEEK_CUR) == ec->get_size (ec->cls);
+}
+
+
+/**
+ * Main entry method for the 'text/html' extraction plugin.  
+ *
+ * @param ec extraction context provided to the plugin
+ */
+void 
+EXTRACTOR_html_extract_method (struct EXTRACTOR_ExtractContext *ec)
+{
+  TidyDoc doc;
+  TidyNode head;
+  TidyNode child;
+  TidyNode title;
+  TidyInputSource src;
+  const char *name;
+  TidyBuffer tbuf;
+  TidyAttr attr;
+  enum EXTRACTOR_MetaType type;
+  ssize_t iret;
+  void *data;
+  const char *mime;
+
+  if (-1 == (iret = ec->read (ec->cls,
+                              &data,
+                              16 * 1024)))
+    return;
+  if (NULL == (mime = magic_buffer (magic, data, iret)))
+    return;
+  if (0 != strncmp (mime,
+                    "text/html",
+                    strlen ("text/html")))
+    return; /* not HTML */
+
+  if (0 != ec->seek (ec->cls, 0, SEEK_SET))
+    return; /* seek failed !? */
+
+  tidyInitSource (&src, ec,
+                  &get_byte_cb,
+                  &unget_byte_cb,
+                  &eof_cb);
+  if (NULL == (doc = tidyCreate ()))
+    return;
+  tidySetReportFilter (doc, &report_cb);
+  tidySetAppData (doc, ec);
+  if (0 > tidyParseSource (doc, &src))
+    {
+      tidyRelease (doc);
+      return;
+    }
+  if (1 != tidyStatus (doc))
+    {
+      tidyRelease (doc);
+      return;
+    }
+  if (NULL == (head = tidyGetHead (doc)))
+    {
+      fprintf (stderr, "no head\n");
+      tidyRelease (doc);
+      return;
+    }
+  for (child = tidyGetChild (head); NULL != child; child = tidyGetNext (child))
+    {
+      switch (tidyNodeGetType(child))
+        {
+        case TidyNode_Root:
+          break;
+        case TidyNode_DocType:
+          break;
+        case TidyNode_Comment:
+          break;
+        case TidyNode_ProcIns:
+          break;
+        case TidyNode_Text:
+          break;
+        case TidyNode_CDATA:
+          break;
+        case TidyNode_Section:
+          break;
+        case TidyNode_Asp:
+          break;
+        case TidyNode_Jste:
+          break;
+        case TidyNode_Php:
+          break;
+        case TidyNode_XmlDecl:
+          break;          
+        case TidyNode_Start:
+        case TidyNode_StartEnd: 
+          name = tidyNodeGetName (child);
+          if ( (0 == strcasecmp (name, "title")) &&
+               (NULL != (title = tidyGetChild (child))) )
+            {
+              tidyBufInit (&tbuf);
+              tidyNodeGetValue (doc, title, &tbuf);
+              /* add 0-termination */
+              tidyBufPutByte (&tbuf, 0);
+              if (0 !=
+                  ec->proc (ec->cls,
+                            "html",
+                            EXTRACTOR_METATYPE_TITLE,
+                            EXTRACTOR_METAFORMAT_UTF8,
+                            "text/plain",
+                            (const char *) tbuf.bp,
+                            tbuf.size))
+                {
+                  tidyBufFree (&tbuf);
+                  goto CLEANUP;
+                }
+              tidyBufFree (&tbuf);
+              break;
+            }
+          if (0 == strcasecmp (name, "meta"))
+            {
+              if (NULL == (attr = tidyAttrGetById (child, 
+                                                   TidyAttr_NAME)))
+                break;
+              if (EXTRACTOR_METATYPE_RESERVED == 
+                  (type = tag_to_type (tidyAttrValue (attr))))
+                break;
+              if (NULL == (attr = tidyAttrGetById (child, 
+                                                   TidyAttr_CONTENT)))
+                break;
+              name = tidyAttrValue (attr);
+              if (0 !=
+                  ec->proc (ec->cls,
+                            "html",
+                            type,
+                            EXTRACTOR_METAFORMAT_UTF8,
+                            "text/plain",
+                            name,
+                            strlen (name) + 1))
+                goto CLEANUP;
+              break;    
+            }
+          break;
+        case TidyNode_End:
+          break;          
+        default:
+          break;
+        }      
+    }
+ CLEANUP:
+  tidyRelease (doc);
+}
+
+
+
+#if OLD
+
+
+/* ******************** parser helper functions ************** */
+
+static int
+tagMatch (const char *tag, const char *s, const char *e)
+{
+  return (((e - s) == strlen (tag)) && (0 == strncasecmp (tag, s, e - s)));
+}
+
+static int
+lookFor (char c, size_t * pos, const char *data, size_t size)
+{
+  size_t p = *pos;
+
+  while ((p < size) && (data[p] != c))
+    {
+      if (data[p] == '\0')
+        return 0;
+      p++;
+    }
+  *pos = p;
+  return p < size;
+}
+
+static int
+skipWhitespace (size_t * pos, const char *data, size_t size)
+{
+  size_t p = *pos;
+
+  while ((p < size) && (isspace ( (unsigned char) data[p])))
+    {
+      if (data[p] == '\0')
+        return 0;
+      p++;
+    }
+  *pos = p;
+  return p < size;
+}
+
+static int
+skipLetters (size_t * pos, const char *data, size_t size)
+{
+  size_t p = *pos;
+
+  while ((p < size) && (isalpha ( (unsigned char) data[p])))
+    {
+      if (data[p] == '\0')
+        return 0;
+      p++;
+    }
+  *pos = p;
+  return p < size;
+}
+
+static int
+lookForMultiple (const char *c, size_t * pos, const char *data, size_t size)
+{
+  size_t p = *pos;
+
+  while ((p < size) && (strchr (c, data[p]) == NULL))
+    {
+      if (data[p] == '\0')
+        return 0;
+      p++;
+    }
+  *pos = p;
+  return p < size;
+}
+
+static void
+findEntry (const char *key,
+           const char *start,
+           const char *end, const char **mstart, const char **mend)
+{
+  size_t len;
+
+  *mstart = NULL;
+  *mend = NULL;
+  len = strlen (key);
+  while (start < end - len - 1)
+    {
+      start++;
+      if (start[len] != '=')
+        continue;
+      if (0 == strncasecmp (start, key, len))
+        {
+          start += len + 1;
+          *mstart = start;
+          if ((*start == '\"') || (*start == '\''))
+            {
+              start++;
+              while ((start < end) && (*start != **mstart))
+                start++;
+              (*mstart)++;      /* skip quote */
+            }
+          else
+            {
+              while ((start < end) && (!isspace ( (unsigned char) *start)))
+                start++;
+            }
+          *mend = start;
+          return;
+        }
+    }
+}
+
+/**
+ * Search all tags that correspond to "tagname".  Example:
+ * If the tag is <meta name="foo" desc="bar">, and
+ * tagname == "meta", keyname="name", keyvalue="foo",
+ * and searchname="desc", then this function returns a
+ * copy (!) of "bar".  Easy enough?
+ *
+ * @return NULL if nothing is found
+ */
+static char *
+findInTags (struct TagInfo * t,
+            const char *tagname,
+            const char *keyname, const char *keyvalue, const char *searchname)
+{
+  const char *pstart;
+  const char *pend;
+
+  while (t != NULL)
+    {
+      if (tagMatch (tagname, t->tagStart, t->tagEnd))
+        {
+          findEntry (keyname, t->tagEnd, t->dataStart, &pstart, &pend);
+          if ((pstart != NULL) && (tagMatch (keyvalue, pstart, pend)))
+            {
+              findEntry (searchname, t->tagEnd, t->dataStart, &pstart, &pend);
+              if (pstart != NULL)
+                {
+                  char *ret = malloc (pend - pstart + 1);
+                  if (ret == NULL)
+                    return NULL;
+                  memcpy (ret, pstart, pend - pstart);
+                  ret[pend - pstart] = '\0';
+                  return ret;
+                }
+            }
+        }
+      t = t->next;
+    }
+  return NULL;
+}
+
+
+/* mimetype = text/html */
+int 
+EXTRACTOR_html_extract (const char *data,
+                        size_t size,
+                        EXTRACTOR_MetaDataProcessor proc,
+                        void *proc_cls,
+                        const char *options)
+{
+  size_t xsize;
+  struct TagInfo *tags;
+  struct TagInfo *t;
+  struct TagInfo tag;
+  size_t pos;
+  size_t tpos;
+  int i;
+  char *charset;
+  char *tmp;
+  char *xtmp;
+  int ret;
+
+  ret = 0;
+  if (size == 0)
+    return 0;
+  /* only scan first 32k */
+  if (size > 1024 * 32)
+    xsize = 1024 * 32;
+  else
+    xsize = size;
+  tags = NULL;
+  tag.next = NULL;
+  pos = 0;
+  while (pos < xsize)
+    {
+      if (!lookFor ('<', &pos, data, size))
+        break;
+      tag.tagStart = &data[++pos];
+      if (!skipLetters (&pos, data, size))
+        break;
+      tag.tagEnd = &data[pos];
+      if (!skipWhitespace (&pos, data, size))
+        break;
+    STEP3:
+      if (!lookForMultiple (">\"\'", &pos, data, size))
+        break;
+      if (data[pos] != '>')
+        {
+          /* find end-quote, ignore escaped quotes (\') */
+          do
+            {
+              tpos = pos;
+              pos++;
+              if (!lookFor (data[tpos], &pos, data, size))
+                break;
+            }
+          while (data[pos - 1] == '\\');
+          pos++;
+          goto STEP3;
+        }
+      pos++;
+      if (!skipWhitespace (&pos, data, size))
+        break;
+      tag.dataStart = &data[pos];
+      if (!lookFor ('<', &pos, data, size))
+        break;
+      tag.dataEnd = &data[pos];
+      i = 0;
+      while (relevantTags[i] != NULL)
+        {
+          if ((strlen (relevantTags[i]) == tag.tagEnd - tag.tagStart) &&
+              (0 == strncasecmp (relevantTags[i],
+                                 tag.tagStart, tag.tagEnd - tag.tagStart)))
+            {
+              t = malloc (sizeof (struct TagInfo));
+              if (t == NULL)
+                return 0;
+              *t = tag;
+              t->next = tags;
+              tags = t;
+              break;
+            }
+          i++;
+        }
+      /* abort early if we hit the body tag */
+      if (tagMatch ("body", tag.tagStart, tag.tagEnd))
+        break;
+    }
+
+  /* fast exit */
+  if (tags == NULL)
+    return 0;
+
+  charset = NULL;
+  /* first, try to determine mime type and/or character set */
+  tmp = findInTags (tags, "meta", "http-equiv", "content-type", "content");
+  if (tmp != NULL)
+    {
+      /* ideally, tmp == "test/html; charset=ISO-XXXX-Y" or something like that;
+         if text/html is present, we take that as the mime-type; if charset=
+         is present, we try to use that for character set conversion. */
+      if (0 == strncasecmp (tmp, "text/html", strlen ("text/html")))
+        ret = proc (proc_cls, 
+                    "html",
+                    EXTRACTOR_METATYPE_MIMETYPE,
+                    EXTRACTOR_METAFORMAT_UTF8,
+                    "text/plain",
+                    "text/html",
+                    strlen ("text/html")+1);
+      charset = strcasestr (tmp, "charset=");
+      if (charset != NULL)
+        charset = strdup (&charset[strlen ("charset=")]);
+      free (tmp);
+    }
+  i = 0;
+  while (tagmap[i].name != NULL)
+    {
+      tmp = findInTags (tags, "meta", "name", tagmap[i].name, "content");
+      if ( (tmp != NULL) &&
+           (ret == 0) )
+        {
+          if (charset == NULL)
+            {
+              ret = proc (proc_cls,
+                          "html",
+                          tagmap[i].type,
+                          EXTRACTOR_METAFORMAT_C_STRING,
+                          "text/plain",
+                          tmp,
+                          strlen (tmp) + 1);
+            }
+          else
+            {
+              xtmp = EXTRACTOR_common_convert_to_utf8 (tmp,
+                                                       strlen (tmp),
+                                                       charset);
+              if (xtmp != NULL)
+                {
+                  ret = proc (proc_cls,
+                              "html",
+                              tagmap[i].type,
+                              EXTRACTOR_METAFORMAT_UTF8,
+                              "text/plain",
+                              xtmp,
+                              strlen (xtmp) + 1);
+                  free (xtmp);
+                }
+            }
+        }
+      if (tmp != NULL)
+        free (tmp);
+      i++;
+    }
+  while (tags != NULL) 
+    {
+      t = tags;
+      if ( (tagMatch ("title", t->tagStart, t->tagEnd)) &&
+           (ret == 0) )
+        {
+          if (charset == NULL)
+            {
+              xtmp = malloc (t->dataEnd - t->dataStart + 1);
+              if (xtmp != NULL)
+                {
+                  memcpy (xtmp, t->dataStart, t->dataEnd - t->dataStart);
+                  xtmp[t->dataEnd - t->dataStart] = '\0';
+                  ret = proc (proc_cls,
+                              "html",
+                              EXTRACTOR_METATYPE_TITLE,
+                              EXTRACTOR_METAFORMAT_C_STRING,
+                              "text/plain",
+                              xtmp,
+                              strlen (xtmp) + 1);
+                  free (xtmp);
+                }
+            }
+          else
+            {
+              xtmp = EXTRACTOR_common_convert_to_utf8 (t->dataStart,
+                                                       t->dataEnd - t->dataStart,
+                                                       charset);
+              if (xtmp != NULL)
+                {
+                  ret = proc (proc_cls,
+                              "html",
+                              EXTRACTOR_METATYPE_TITLE,
+                              EXTRACTOR_METAFORMAT_UTF8,
+                              "text/plain",
+                              xtmp,
+                              strlen (xtmp) + 1);
+                  free (xtmp);
+                }
+            }
+        }
+      tags = t->next;
+      free (t);
+    }
+  if (charset != NULL)
+    free (charset);
+  return ret;
+}
+#endif
+
+
+/**
+ * Initialize glib and load magic file.
+ */
+void __attribute__ ((constructor)) 
+html_gobject_init ()
+{
+  magic = magic_open (MAGIC_MIME_TYPE);
+  if (0 != magic_load (magic, NULL))
+    {
+      /* FIXME: how to deal with errors? */
+    }
+}
+
+
+/**
+ * Destructor for the library, cleans up.
+ */
+void __attribute__ ((destructor)) 
+html_ltdl_fini () 
+{
+  if (NULL != magic)
+    {
+      magic_close (magic);
+      magic = NULL;
+    }
+}
+
+/* end of html_extractor.c */