test_html.c - libextractor - GNU libextractor

test_html.c (10494B)
      1 /*
      2      This file is part of libextractor.
      3      Copyright (C) 2012 Vidyut Samanta and Christian Grothoff
      4 
      5      libextractor is free software; you can redistribute it and/or modify
      6      it under the terms of the GNU General Public License as published
      7      by the Free Software Foundation; either version 3, or (at your
      8      option) any later version.
      9 
     10      libextractor is distributed in the hope that it will be useful, but
     11      WITHOUT ANY WARRANTY; without even the implied warranty of
     12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13      General Public License for more details.
     14 
     15      You should have received a copy of the GNU General Public License
     16      along with libextractor; see the file COPYING.  If not, write to the
     17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18      Boston, MA 02110-1301, USA.
     19 */
     20 /**
     21  * @file plugins/test_html.c
     22  * @brief testcase for html plugin
     23  * @author Christian Grothoff
     24  */
     25 #include "platform.h"
     26 #include "test_lib.h"
     27 
     28 
     29 /**
     30  * Check the shape of a full text: 0-terminated, no leading or trailing
     31  * white space, paragraphs separated by exactly "\n\n", no other line
     32  * breaks, no runs of spaces, no tabs.
     33  *
     34  * @param data the full text
     35  * @param data_len number of bytes in @a data, including the 0
     36  * @return 1 if the shape is right
     37  */
     38 static int
     39 fulltext_shape_ok (const char *data,
     40                    size_t data_len)
     41 {
     42   size_t len;
     43 
     44   if ( (data_len < 2) ||
     45        ('\0' != data[data_len - 1]) ||
     46        (strlen (data) != data_len - 1) )
     47     return 0;
     48   len = data_len - 1;
     49   if ( (' ' == data[0]) || ('\n' == data[0]) ||
     50        (' ' == data[len - 1]) || ('\n' == data[len - 1]) )
     51     return 0;
     52   for (size_t i = 0; i < len; i++)
     53   {
     54     if ('\t' == data[i])
     55       return 0;
     56     if ( (' ' == data[i]) &&
     57          ( (' ' == data[i + 1]) || ('\n' == data[i + 1]) ) )
     58       return 0;
     59     if ('\n' == data[i])
     60     {
     61       if ( ('\n' != data[i + 1]) ||
     62            ('\n' == data[i + 2]) ||
     63            (' ' == data[i + 2]) )
     64         return 0;
     65       i++;
     66     }
     67   }
     68   return 1;
     69 }
     70 
     71 
     72 /**
     73  * Validate the full text of html_grothoff.html.
     74  *
     75  * @param data the full text
     76  * @param data_len number of bytes in @a data
     77  * @return 1 if acceptable
     78  */
     79 static int
     80 check_grothoff_text (const void *data,
     81                      size_t data_len)
     82 {
     83   const char *s = data;
     84 
     85   if (! fulltext_shape_ok (s, data_len))
     86     return 0;
     87   /* the heading, then the list items, each its own paragraph */
     88   if (NULL == strstr (s, "Welcome to Christian Grothoff\n\nWelcome\n\n"
     89                       "Computer Science\n\nLinux\n\n"))
     90     return 0;
     91   /* nothing from the head or its script */
     92   if ( (NULL != strstr (s, "top.location")) ||
     93        (NULL != strstr (s, "Homepage of")) )
     94     return 0;
     95   return 1;
     96 }
     97 
     98 
     99 /**
    100  * Closure for #count_cb().
    101  */
    102 struct Count
    103 {
    104   /**
    105    * Number of full text items seen.
    106    */
    107   unsigned int fulltext;
    108 
    109   /**
    110    * Number of full text items that are not normalized as promised.
    111    */
    112   unsigned int bad;
    113 };
    114 
    115 
    116 /**
    117  * Count full text items.
    118  *
    119  * @param cls a `struct Count`
    120  * @param plugin_name ignored
    121  * @param type meta data type
    122  * @param format ignored
    123  * @param data_mime_type ignored
    124  * @param data meta data
    125  * @param data_len number of bytes in @a data
    126  * @return 0 (continue)
    127  */
    128 static int
    129 count_cb (void *cls,
    130           const char *plugin_name,
    131           enum EXTRACTOR_MetaType type,
    132           enum EXTRACTOR_MetaFormat format,
    133           const char *data_mime_type,
    134           const char *data,
    135           size_t data_len)
    136 {
    137   struct Count *c = cls;
    138 
    139   if (EXTRACTOR_METATYPE_FULL_TEXT != type)
    140     return 0;
    141   c->fulltext++;
    142   if (! fulltext_shape_ok (data, data_len))
    143     c->bad++;
    144   return 0;
    145 }
    146 
    147 
    148 /**
    149  * Count the full text items the plugin reports for @a file.
    150  *
    151  * @param config plugin name with options, e.g. "html(fulltext)"
    152  * @param file file to extract from
    153  * @return number of full text items, UINT_MAX if one of them is
    154  *         malformed
    155  */
    156 static unsigned int
    157 count_fulltext (const char *config,
    158                 const char *file)
    159 {
    160   struct EXTRACTOR_PluginList *pl;
    161   struct Count c = { 0, 0 };
    162 
    163   pl = EXTRACTOR_plugin_add_config (NULL,
    164                                     config,
    165                                     EXTRACTOR_OPTION_DEFAULT_POLICY);
    166   EXTRACTOR_extract (pl, file, NULL, 0, &count_cb, &c);
    167   EXTRACTOR_plugin_remove_all (pl);
    168   if (0 != c.bad)
    169   {
    170     fprintf (stderr,
    171              "Full text of %s is not normalized\n",
    172              file);
    173     return UINT_MAX;
    174   }
    175   return c.fulltext;
    176 }
    177 
    178 
    179 /**
    180  * Main function for the HTML testcase.
    181  *
    182  * @param argc number of arguments (ignored)
    183  * @param argv arguments (ignored)
    184  * @return 0 on success
    185  */
    186 int
    187 main (int argc, char *argv[])
    188 {
    189   struct SolutionData html_grothoff_sol[] = {
    190     {
    191       EXTRACTOR_METATYPE_TITLE,
    192       EXTRACTOR_METAFORMAT_UTF8,
    193       "text/plain",
    194       "Christian Grothoff",
    195       strlen ("Christian Grothoff") + 1,
    196       0
    197     },
    198     {
    199       EXTRACTOR_METATYPE_DESCRIPTION,
    200       EXTRACTOR_METAFORMAT_UTF8,
    201       "text/plain",
    202       "Homepage of Christian Grothoff",
    203       strlen ("Homepage of Christian Grothoff") + 1,
    204       0
    205     },
    206     {
    207       EXTRACTOR_METATYPE_AUTHOR_NAME,
    208       EXTRACTOR_METAFORMAT_UTF8,
    209       "text/plain",
    210       "Christian Grothoff",
    211       strlen ("Christian Grothoff") + 1,
    212       0
    213     },
    214     {
    215       EXTRACTOR_METATYPE_KEYWORDS,
    216       EXTRACTOR_METAFORMAT_UTF8,
    217       "text/plain",
    218       "Christian,Grothoff",
    219       strlen ("Christian,Grothoff") + 1,
    220       0
    221     },
    222     {
    223       EXTRACTOR_METATYPE_TITLE,
    224       EXTRACTOR_METAFORMAT_UTF8,
    225       "text/plain",
    226       "Welcome to Christian Grothoff",
    227       strlen ("Welcome to Christian Grothoff") + 1,
    228       0
    229     },
    230     {
    231       EXTRACTOR_METATYPE_LANGUAGE,
    232       EXTRACTOR_METAFORMAT_UTF8,
    233       "text/plain",
    234       "en",
    235       strlen ("en") + 1,
    236       0
    237     },
    238     {
    239       EXTRACTOR_METATYPE_PUBLISHER,
    240       EXTRACTOR_METAFORMAT_UTF8,
    241       "text/plain",
    242       "Christian Grothoff",
    243       strlen ("Christian Grothoff") + 1,
    244       0
    245     },
    246     {
    247       EXTRACTOR_METATYPE_UNKNOWN_DATE,
    248       EXTRACTOR_METAFORMAT_UTF8,
    249       "text/plain",
    250       "2000-08-20",
    251       strlen ("2000-08-20") + 1,
    252       0
    253     },
    254     {
    255       EXTRACTOR_METATYPE_RIGHTS,
    256       EXTRACTOR_METAFORMAT_UTF8,
    257       "text/plain",
    258       "(C) 2000 by Christian Grothoff",
    259       strlen ("(C) 2000 by Christian Grothoff") + 1,
    260       0
    261     },
    262     { 0, 0, NULL, NULL, 0, -1 }
    263   };
    264   struct SolutionData html_fulltext_sol[] = {
    265     {
    266       EXTRACTOR_METATYPE_AUTHOR_NAME,
    267       EXTRACTOR_METAFORMAT_UTF8,
    268       "text/plain",
    269       "Christian Grothoff",
    270       strlen ("Christian Grothoff") + 1,
    271       0
    272     },
    273     { 0, 0, NULL, NULL, 0, -1 }
    274   };
    275   struct ProblemSet ps[] = {
    276     { "testdata/html_grothoff.html",
    277       html_grothoff_sol },
    278     { "testdata/html_fulltext.html",
    279       html_fulltext_sol },
    280     { NULL, NULL }
    281   };
    282   /* with the option: the same items, plus the full text */
    283 #define FT_TEXT \
    284         "Der K\xc3\xa4se & das Brot\n\n" \
    285         "Hello world, again here: \xc3\xa9t\xc3\xa9 \xe2\x80\x94 done.\n\n" \
    286         "Line one line two\n\n" \
    287         "New paragraph after two breaks.\n\n" \
    288         "Item A\n\nItem B\n\n" \
    289         "Cell 1\n\nCell 2\n\n" \
    290         "Quoted text\n\n" \
    291         "Softhyphen and inline elements.\n\n" \
    292         "Article byline"
    293   struct SolutionData html_grothoff_ft_sol[] = {
    294     { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    295       "Christian Grothoff", strlen ("Christian Grothoff") + 1, 0 },
    296     { EXTRACTOR_METATYPE_DESCRIPTION, EXTRACTOR_METAFORMAT_UTF8,
    297       "text/plain", "Homepage of Christian Grothoff",
    298       strlen ("Homepage of Christian Grothoff") + 1, 0 },
    299     { EXTRACTOR_METATYPE_AUTHOR_NAME, EXTRACTOR_METAFORMAT_UTF8,
    300       "text/plain", "Christian Grothoff",
    301       strlen ("Christian Grothoff") + 1, 0 },
    302     { EXTRACTOR_METATYPE_KEYWORDS, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    303       "Christian,Grothoff", strlen ("Christian,Grothoff") + 1, 0 },
    304     { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    305       "Welcome to Christian Grothoff",
    306       strlen ("Welcome to Christian Grothoff") + 1, 0 },
    307     { EXTRACTOR_METATYPE_LANGUAGE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    308       "en", strlen ("en") + 1, 0 },
    309     { EXTRACTOR_METATYPE_PUBLISHER, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    310       "Christian Grothoff", strlen ("Christian Grothoff") + 1, 0 },
    311     { EXTRACTOR_METATYPE_UNKNOWN_DATE, EXTRACTOR_METAFORMAT_UTF8,
    312       "text/plain", "2000-08-20", strlen ("2000-08-20") + 1, 0 },
    313     { EXTRACTOR_METATYPE_RIGHTS, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    314       "(C) 2000 by Christian Grothoff",
    315       strlen ("(C) 2000 by Christian Grothoff") + 1, 0 },
    316     { EXTRACTOR_METATYPE_FULL_TEXT, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    317       "^Welcome to Christian Grothoff\n", 0, 0, 1,
    318       &check_grothoff_text },
    319     { 0, 0, NULL, NULL, 0, -1 }
    320   };
    321   struct SolutionData html_fulltext_ft_sol[] = {
    322     /* in the option's code path the declared charset is honoured, so
    323        the title comes out right */
    324     { EXTRACTOR_METATYPE_TITLE, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    325       "K\xc3\xa4se und Brot", strlen ("K\xc3\xa4se und Brot") + 1, 0 },
    326     { EXTRACTOR_METATYPE_AUTHOR_NAME, EXTRACTOR_METAFORMAT_UTF8,
    327       "text/plain", "Christian Grothoff",
    328       strlen ("Christian Grothoff") + 1, 0 },
    329     { EXTRACTOR_METATYPE_FULL_TEXT, EXTRACTOR_METAFORMAT_UTF8, "text/plain",
    330       FT_TEXT, strlen (FT_TEXT) + 1, 0 },
    331     { 0, 0, NULL, NULL, 0, -1 }
    332   };
    333   struct ProblemSet ps_ft[] = {
    334     { "testdata/html_grothoff.html",
    335       html_grothoff_ft_sol },
    336     { "testdata/html_fulltext.html",
    337       html_fulltext_ft_sol },
    338     { NULL, NULL }
    339   };
    340   int ret;
    341 
    342   if (0 != (ret = ET_main ("html", ps)))
    343     return ret;
    344   if (0 != (ret = ET_main ("html(fulltext)", ps_ft)))
    345     return ret;
    346   /* ET_main only warns about extra items: make sure that the full text
    347      appears exactly when it was asked for */
    348   if ( (0 != count_fulltext ("html", "testdata/html_fulltext.html")) ||
    349        (0 != count_fulltext ("html", "testdata/html_grothoff.html")) ||
    350        (1 != count_fulltext ("html(fulltext)",
    351                              "testdata/html_fulltext.html")) ||
    352        (1 != count_fulltext ("html(fulltext)",
    353                              "testdata/html_grothoff.html")) )
    354   {
    355     fprintf (stderr,
    356              "Full text reported without the option, or not with it\n");
    357     return 1;
    358   }
    359   return 0;
    360 }
    361 
    362 
    363 /* end of test_html.c */
	libextractor GNU libextractor
	Log \| Files \| Refs \| Submodules \| README \| LICENSE