libextractor

GNU libextractor
Log | Files | Refs | Submodules | README | LICENSE

test_odf.c (11557B)

      1 /*
      2      This file is part of libextractor.
      3      Copyright (C) 2012 Vidyut Samanta and Christian Grothoff
      4 
      5      libextractor is free software; you can redistribute it and/or modify
      6      it under the terms of the GNU General Public License as published
      7      by the Free Software Foundation; either version 3, or (at your
      8      option) any later version.
      9 
     10      libextractor is distributed in the hope that it will be useful, but
     11      WITHOUT ANY WARRANTY; without even the implied warranty of
     12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     13      General Public License for more details.
     14 
     15      You should have received a copy of the GNU General Public License
     16      along with libextractor; see the file COPYING.  If not, write to the
     17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     18      Boston, MA 02110-1301, USA.
     19 */
     20 /**
     21  * @file plugins/test_odf.c
     22  * @brief testcase for odf plugin
     23  * @author Christian Grothoff
     24  */
     25 #include "platform.h"
     26 #include "test_lib.h"
     27 
     28 
     29 /**
     30  * Check the general shape of a full text: valid UTF-8, 0-terminated,
     31  * no leading or trailing white space, paragraphs separated by exactly
     32  * "\n\n", no other line breaks, no double spaces.
     33  *
     34  * @param data the full text
     35  * @param data_len number of bytes in @a data, including the 0
     36  * @return 1 if the shape is right
     37  */
     38 static int
     39 fulltext_shape_ok (const char *data,
     40                    size_t data_len)
     41 {
     42   const unsigned char *t = (const unsigned char *) data;
     43   size_t len;
     44   size_t i;
     45 
     46   if ( (data_len < 2) ||
     47        ('\0' != data[data_len - 1]) ||
     48        (strlen (data) != data_len - 1) )
     49     return 0;
     50   len = data_len - 1;
     51   if (isspace (t[0]) || isspace (t[len - 1]))
     52     return 0;
     53   for (i = 0; i < len; i++)
     54   {
     55     if ('\n' == t[i])
     56     {
     57       if ( (i + 2 >= len) ||
     58            ('\n' != t[i + 1]) ||
     59            isspace (t[i + 2]) ||
     60            isspace (t[i - 1]) )
     61         return 0;
     62       i++;
     63       continue;
     64     }
     65     if ( (' ' == t[i]) && (' ' == t[i + 1]) )
     66       return 0;
     67     if ( (t[i] < 0x20) )
     68       return 0;
     69     if (t[i] >= 0x80)
     70     {
     71       unsigned int n;
     72       unsigned int k;
     73 
     74       if ( (t[i] & 0xE0) == 0xC0)
     75         n = 1;
     76       else if ( (t[i] & 0xF0) == 0xE0)
     77         n = 2;
     78       else if ( (t[i] & 0xF8) == 0xF0)
     79         n = 3;
     80       else
     81         return 0;
     82       if (i + n >= len)
     83         return 0;
     84       for (k = 1; k <= n; k++)
     85         if ( (t[i + k] & 0xC0) != 0x80)
     86           return 0;
     87       i += n;
     88     }
     89   }
     90   return 1;
     91 }
     92 
     93 
     94 /**
     95  * Expected end of the full text of odf_cg.odt.
     96  */
     97 #define CG_TAIL \
     98         "\n\nFocus Areas of Research and Teaching\n\nProgramming languages, " \
     99         "Computer security, Networking, Software tools"
    100 
    101 
    102 /**
    103  * Validate the full text of odf_cg.odt (verified against
    104  * "soffice --headless --convert-to txt").
    105  *
    106  * @param data the full text
    107  * @param data_len number of bytes in @a data
    108  * @return 1 if acceptable
    109  */
    110 static int
    111 validate_cg (const void *data,
    112              size_t data_len)
    113 {
    114   const char *t = data;
    115 
    116   if (! fulltext_shape_ok (t, data_len))
    117     return 0;
    118   return (0 == strncmp (t,
    119                         "Christian Grothoff, PhD (UCLA)\n\n"
    120                         "TU M\xc3\xbc" "nchen, Department of Informatics, ",
    121                         strlen ("Christian Grothoff, PhD (UCLA)\n\n"
    122                                 "TU M\xc3\xbc" "nchen, Department of "
    123                                 "Informatics, "))) &&
    124          (NULL != strstr (t,
    125                           "\n\n1996\xe2\x80\x93" "2000 Diplom II in "
    126                           "Mathematik, BUGH Wuppertal\n\n")) &&
    127          (NULL != strstr (t,
    128                           "\n\nProfessional Activities\n\nDr. Grothoff has "
    129                           "a history of developing useful tools")) &&
    130          (NULL != strstr (t,
    131                           "He is also the maintainer of GNU libextractor, "
    132                           "a GNU project for metadata extraction.")) &&
    133          (data_len > strlen (CG_TAIL)) &&
    134          (0 == strcmp (t + data_len - 1 - strlen (CG_TAIL),
    135                        CG_TAIL));
    136 }
    137 
    138 
    139 /**
    140  * Called with each item from the default (no option) run; counts
    141  * #EXTRACTOR_METATYPE_FULL_TEXT items.
    142  *
    143  * @param cls pointer to an `unsigned int` counter
    144  * @param plugin_name unused
    145  * @param type type of the item
    146  * @param format unused
    147  * @param data_mime_type unused
    148  * @param data unused
    149  * @param data_len unused
    150  * @return 0 (continue)
    151  */
    152 static int
    153 count_fulltext (void *cls,
    154                 const char *plugin_name,
    155                 enum EXTRACTOR_MetaType type,
    156                 enum EXTRACTOR_MetaFormat format,
    157                 const char *data_mime_type,
    158                 const char *data,
    159                 size_t data_len)
    160 {
    161   unsigned int *cnt = cls;
    162 
    163   (void) plugin_name;
    164   (void) format;
    165   (void) data_mime_type;
    166   (void) data;
    167   (void) data_len;
    168   if (EXTRACTOR_METATYPE_FULL_TEXT == type)
    169     (*cnt)++;
    170   return 0;
    171 }
    172 
    173 
    174 /**
    175  * Check that without the "fulltext" option no full text is reported.
    176  *
    177  * @param files NULL-terminated list of files to check
    178  * @return 0 on success
    179  */
    180 static int
    181 check_no_fulltext (const char *const *files)
    182 {
    183   struct EXTRACTOR_PluginList *pl;
    184   unsigned int cnt = 0;
    185   unsigned int i;
    186 
    187   pl = EXTRACTOR_plugin_add_config (NULL,
    188                                     "odf",
    189                                     EXTRACTOR_OPTION_DEFAULT_POLICY);
    190   for (i = 0; NULL != files[i]; i++)
    191     EXTRACTOR_extract (pl,
    192                        files[i],
    193                        NULL, 0,
    194                        &count_fulltext,
    195                        &cnt);
    196   EXTRACTOR_plugin_remove_all (pl);
    197   if (0 != cnt)
    198   {
    199     fprintf (stderr,
    200              "Got full text without the fulltext option\n");
    201     return 1;
    202   }
    203   return 0;
    204 }
    205 
    206 
    207 /**
    208  * Main function for the ODF testcase.
    209  *
    210  * @param argc number of arguments (ignored)
    211  * @param argv arguments (ignored)
    212  * @return 0 on success
    213  */
    214 int
    215 main (int argc, char *argv[])
    216 {
    217   struct SolutionData odf_cg_sol[] = {
    218     {
    219       EXTRACTOR_METATYPE_MIMETYPE,
    220       EXTRACTOR_METAFORMAT_UTF8,
    221       "text/plain",
    222       "application/vnd.oasis.opendocument.text",
    223       strlen ("application/vnd.oasis.opendocument.text") + 1,
    224       0
    225     },
    226     {
    227       EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
    228       EXTRACTOR_METAFORMAT_UTF8,
    229       "text/plain",
    230       "OpenOffice.org/3.2$Unix OpenOffice.org_project/320m12$Build-9483",
    231       strlen (
    232         "OpenOffice.org/3.2$Unix OpenOffice.org_project/320m12$Build-9483") + 1,
    233       0
    234     },
    235     {
    236       EXTRACTOR_METATYPE_PAGE_COUNT,
    237       EXTRACTOR_METAFORMAT_UTF8,
    238       "text/plain",
    239       "1",
    240       strlen ("1") + 1,
    241       0
    242     },
    243     {
    244       EXTRACTOR_METATYPE_CREATION_DATE,
    245       EXTRACTOR_METAFORMAT_UTF8,
    246       "text/plain",
    247       "2005-11-22T11:44:00",
    248       strlen ("2005-11-22T11:44:00") + 1,
    249       0
    250     },
    251     {
    252       EXTRACTOR_METATYPE_UNKNOWN_DATE,
    253       EXTRACTOR_METAFORMAT_UTF8,
    254       "text/plain",
    255       "2010-06-09T13:09:34",
    256       strlen ("2010-06-09T13:09:34") + 1,
    257       0
    258     },
    259     {
    260       EXTRACTOR_METATYPE_TITLE,
    261       EXTRACTOR_METAFORMAT_UTF8,
    262       "text/plain",
    263       "Anhang 1: Profile der beteiligten Wissenschaftler",
    264       strlen ("Anhang 1: Profile der beteiligten Wissenschaftler") + 1,
    265       0
    266     },
    267     /* only expected with the "fulltext" option; its 'solved' is -1
    268        (end of list) for the default run */
    269     {
    270       EXTRACTOR_METATYPE_FULL_TEXT,
    271       EXTRACTOR_METAFORMAT_UTF8,
    272       "text/plain",
    273       "^Christian Grothoff",
    274       0,
    275       -1,
    276       1,
    277       &validate_cg
    278     },
    279     { 0, 0, NULL, NULL, 0, -1 }
    280   };
    281 #define ODT_FT \
    282         "Synthetic Heading\n\n" \
    283         "First paragraph with spaces and a tab and a break; entities & <tag> " \
    284         "\xe2\x98\xba \xe2\x82\xac and the sentence continues.\n\n" \
    285         "This is page of pages.\n\n" \
    286         "List item one\n\n" \
    287         "List item two\n\n" \
    288         "Cell A1\n\nCell B1\n\nCell A2\n\nCell B2\n\n" \
    289         "See Chapter One for details.\n\n" \
    290         "Before the frame\n\nFrame caption text\n\nafter the frame.\n\n" \
    291         "Last paragraph."
    292   struct SolutionData odt_ft_sol[] = {
    293     {
    294       EXTRACTOR_METATYPE_MIMETYPE,
    295       EXTRACTOR_METAFORMAT_UTF8,
    296       "text/plain",
    297       "application/vnd.oasis.opendocument.text",
    298       strlen ("application/vnd.oasis.opendocument.text") + 1,
    299       0
    300     },
    301     {
    302       EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
    303       EXTRACTOR_METAFORMAT_UTF8,
    304       "text/plain",
    305       "gen_odf_testdata.sh",
    306       strlen ("gen_odf_testdata.sh") + 1,
    307       0
    308     },
    309     {
    310       EXTRACTOR_METATYPE_TITLE,
    311       EXTRACTOR_METAFORMAT_UTF8,
    312       "text/plain",
    313       "Synthetic full text test",
    314       strlen ("Synthetic full text test") + 1,
    315       0
    316     },
    317     {
    318       EXTRACTOR_METATYPE_FULL_TEXT,
    319       EXTRACTOR_METAFORMAT_UTF8,
    320       "text/plain",
    321       ODT_FT,
    322       strlen (ODT_FT) + 1,
    323       0
    324     },
    325     { 0, 0, NULL, NULL, 0, -1 }
    326   };
    327 #define ODP_FT \
    328         "First slide title\n\nBullet one\n\nBullet two\n\n" \
    329         "Second slide title\n\nBullet three"
    330   struct SolutionData odp_ft_sol[] = {
    331     {
    332       EXTRACTOR_METATYPE_MIMETYPE,
    333       EXTRACTOR_METAFORMAT_UTF8,
    334       "text/plain",
    335       "application/vnd.oasis.opendocument.presentation",
    336       strlen ("application/vnd.oasis.opendocument.presentation") + 1,
    337       0
    338     },
    339     {
    340       EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
    341       EXTRACTOR_METAFORMAT_UTF8,
    342       "text/plain",
    343       "gen_odf_testdata.sh",
    344       strlen ("gen_odf_testdata.sh") + 1,
    345       0
    346     },
    347     {
    348       EXTRACTOR_METATYPE_TITLE,
    349       EXTRACTOR_METAFORMAT_UTF8,
    350       "text/plain",
    351       "Synthetic slides",
    352       strlen ("Synthetic slides") + 1,
    353       0
    354     },
    355     {
    356       EXTRACTOR_METATYPE_FULL_TEXT,
    357       EXTRACTOR_METAFORMAT_UTF8,
    358       "text/plain",
    359       ODP_FT,
    360       strlen (ODP_FT) + 1,
    361       0
    362     },
    363     { 0, 0, NULL, NULL, 0, -1 }
    364   };
    365 #define ODS_FT "Name\n\nAmount\n\nApples\n\n42"
    366   struct SolutionData ods_ft_sol[] = {
    367     {
    368       EXTRACTOR_METATYPE_MIMETYPE,
    369       EXTRACTOR_METAFORMAT_UTF8,
    370       "text/plain",
    371       "application/vnd.oasis.opendocument.spreadsheet",
    372       strlen ("application/vnd.oasis.opendocument.spreadsheet") + 1,
    373       0
    374     },
    375     {
    376       EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE,
    377       EXTRACTOR_METAFORMAT_UTF8,
    378       "text/plain",
    379       "gen_odf_testdata.sh",
    380       strlen ("gen_odf_testdata.sh") + 1,
    381       0
    382     },
    383     {
    384       EXTRACTOR_METATYPE_TITLE,
    385       EXTRACTOR_METAFORMAT_UTF8,
    386       "text/plain",
    387       "Synthetic sheet",
    388       strlen ("Synthetic sheet") + 1,
    389       0
    390     },
    391     {
    392       EXTRACTOR_METATYPE_FULL_TEXT,
    393       EXTRACTOR_METAFORMAT_UTF8,
    394       "text/plain",
    395       ODS_FT,
    396       strlen (ODS_FT) + 1,
    397       0
    398     },
    399     { 0, 0, NULL, NULL, 0, -1 }
    400   };
    401   const char *const ft_files[] = {
    402     "testdata/odf_cg.odt",
    403     "testdata/odf_fulltext.odt",
    404     "testdata/odf_fulltext.odp",
    405     "testdata/odf_fulltext.ods",
    406     NULL
    407   };
    408   struct ProblemSet ps[] = {
    409     { "testdata/odf_cg.odt",
    410       odf_cg_sol },
    411     { NULL, NULL }
    412   };
    413   struct ProblemSet ps_ft[] = {
    414     { "testdata/odf_cg.odt",
    415       odf_cg_sol },
    416     { "testdata/odf_fulltext.odt",
    417       odt_ft_sol },
    418     { "testdata/odf_fulltext.odp",
    419       odp_ft_sol },
    420     { "testdata/odf_fulltext.ods",
    421       ods_ft_sol },
    422     { NULL, NULL }
    423   };
    424   size_t ft_idx = sizeof (odf_cg_sol) / sizeof (odf_cg_sol[0]) - 2;
    425   int ret;
    426 
    427   if (0 != (ret = ET_main ("odf", ps)))
    428     return ret;
    429   if (0 != (ret = check_no_fulltext (ft_files)))
    430     return ret;
    431   odf_cg_sol[ft_idx].solved = 0;
    432   return ET_main ("odf(fulltext)", ps_ft);
    433 }
    434 
    435 
    436 /* end of test_odf.c */