odf_extractor.c (32162B)
1 /* 2 This file is part of libextractor. 3 Copyright (C) 2004, 2009, 2012 Vidyut Samanta and Christian Grothoff 4 5 libextractor is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published 7 by the Free Software Foundation; either version 3, or (at your 8 option) any later version. 9 10 libextractor is distributed in the hope that it will be useful, but 11 WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with libextractor; see the file COPYING. If not, write to the 17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 18 Boston, MA 02110-1301, USA. 19 */ 20 /** 21 * @file plugins/odf_extractor.c 22 * @brief plugin to support ODF files 23 * @author Christian Grothoff 24 */ 25 #include "platform.h" 26 #include <ctype.h> 27 #include "extractor.h" 28 #include "unzip.h" 29 #include "fulltext.h" 30 31 /** 32 * Maximum length of a filename allowed inside the ZIP archive. 33 */ 34 #define MAXFILENAME 256 35 36 /** 37 * Name of the file with the meta-data in OO documents. 38 */ 39 #define METAFILE "meta.xml" 40 41 /** 42 * Name of the file with the document body in OO documents. 43 */ 44 #define CONTENTFILE "content.xml" 45 46 /** 47 * Largest decompressed size of content.xml we are willing to look at 48 * for the "fulltext" option; anything beyond is ignored (this also 49 * bounds the damage a zip bomb can do). 50 */ 51 #define MAX_CONTENT_SIZE (64 * 1024 * 1024) 52 53 /** 54 * Maximum XML element nesting depth we keep track of. Deeper elements 55 * are still balanced correctly, but treated as having no special 56 * meaning. 57 */ 58 #define MAX_XML_DEPTH 256 59 60 /** 61 * Maximum number of namespace prefixes declared on the root element 62 * that we remember. 63 */ 64 #define MAX_NS_PREFIXES 64 65 66 67 /** 68 * Mapping from ODF meta data strings to LE types. 69 */ 70 struct Matches 71 { 72 /** 73 * ODF description. 74 */ 75 const char *text; 76 77 /** 78 * Corresponding LE type. 79 */ 80 enum EXTRACTOR_MetaType type; 81 }; 82 83 84 /** 85 * NULL-terminated map from ODF meta data strings to LE types. 86 */ 87 static struct Matches tmap[] = { 88 { "meta:generator", EXTRACTOR_METATYPE_CREATED_BY_SOFTWARE }, 89 { "meta:page-count", EXTRACTOR_METATYPE_PAGE_COUNT }, 90 { "meta:creation-date", EXTRACTOR_METATYPE_CREATION_DATE }, 91 { "dc:date", EXTRACTOR_METATYPE_UNKNOWN_DATE }, 92 { "dc:creator", EXTRACTOR_METATYPE_CREATOR }, 93 { "dc:language", EXTRACTOR_METATYPE_LANGUAGE }, 94 { "dc:title", EXTRACTOR_METATYPE_TITLE }, 95 { "dc:description", EXTRACTOR_METATYPE_DESCRIPTION }, 96 { "dc:subject", EXTRACTOR_METATYPE_SUBJECT }, 97 { "meta:keyword", EXTRACTOR_METATYPE_KEYWORDS }, 98 { "meta:user-defined meta:name=\"Info 1\"", EXTRACTOR_METATYPE_COMMENT }, 99 { "meta:user-defined meta:name=\"Info 2\"", EXTRACTOR_METATYPE_COMMENT }, 100 { "meta:user-defined meta:name=\"Info 3\"", EXTRACTOR_METATYPE_COMMENT }, 101 { "meta:user-defined meta:name=\"Info 4\"", EXTRACTOR_METATYPE_COMMENT }, 102 { NULL, 0 } 103 }; 104 105 106 /** 107 * Obtain the mimetype of the archive by reading the 'mimetype' 108 * file of the ZIP. 109 * 110 * @param uf unzip context to extract the mimetype from 111 * @return NULL if no mimetype could be found, otherwise the mime type 112 */ 113 static char * 114 libextractor_oo_getmimetype (struct EXTRACTOR_UnzipFile *uf) 115 { 116 char filename_inzip[MAXFILENAME]; 117 struct EXTRACTOR_UnzipFileInfo file_info; 118 char *buf; 119 size_t buf_size; 120 121 if (EXTRACTOR_UNZIP_OK != 122 EXTRACTOR_common_unzip_go_find_local_file (uf, 123 "mimetype", 124 2)) 125 return NULL; 126 if (EXTRACTOR_UNZIP_OK != 127 EXTRACTOR_common_unzip_get_current_file_info (uf, 128 &file_info, 129 filename_inzip, 130 sizeof (filename_inzip), 131 NULL, 132 0, 133 NULL, 134 0)) 135 return NULL; 136 if (EXTRACTOR_UNZIP_OK != 137 EXTRACTOR_common_unzip_open_current_file (uf)) 138 return NULL; 139 buf_size = file_info.uncompressed_size; 140 if (buf_size > 1024) 141 { 142 /* way too large! */ 143 EXTRACTOR_common_unzip_close_current_file (uf); 144 return NULL; 145 } 146 if (NULL == (buf = malloc (1 + buf_size))) 147 { 148 /* memory exhausted! */ 149 EXTRACTOR_common_unzip_close_current_file (uf); 150 return NULL; 151 } 152 if (buf_size != 153 (size_t) EXTRACTOR_common_unzip_read_current_file (uf, 154 buf, 155 buf_size)) 156 { 157 free (buf); 158 EXTRACTOR_common_unzip_close_current_file (uf); 159 return NULL; 160 } 161 /* found something */ 162 buf[buf_size] = '\0'; 163 while ( (0 < buf_size) && 164 isspace ( (unsigned char) buf[buf_size - 1])) 165 buf[--buf_size] = '\0'; 166 if ('\0' == buf[0]) 167 { 168 free (buf); 169 buf = NULL; 170 } 171 EXTRACTOR_common_unzip_close_current_file (uf); 172 return buf; 173 } 174 175 176 /* ******************** full text of content.xml ******************** */ 177 178 /** 179 * Namespaces the full-text scanner cares about. 180 */ 181 enum OdfNamespace 182 { 183 NS_OTHER = 0, 184 NS_OFFICE, 185 NS_TEXT, 186 NS_TABLE, 187 NS_DRAW, 188 NS_PRESENTATION, 189 NS_STYLE, 190 NS_SVG 191 }; 192 193 194 /** 195 * Map from namespace URI to #OdfNamespace. Both ODF (OASIS) and the 196 * legacy OpenOffice.org 1.x namespaces are supported. 197 */ 198 static const struct 199 { 200 const char *uri; 201 enum OdfNamespace ns; 202 } ns_uris[] = { 203 { "urn:oasis:names:tc:opendocument:xmlns:office:1.0", NS_OFFICE }, 204 { "urn:oasis:names:tc:opendocument:xmlns:text:1.0", NS_TEXT }, 205 { "urn:oasis:names:tc:opendocument:xmlns:table:1.0", NS_TABLE }, 206 { "urn:oasis:names:tc:opendocument:xmlns:drawing:1.0", NS_DRAW }, 207 { "urn:oasis:names:tc:opendocument:xmlns:presentation:1.0", 208 NS_PRESENTATION }, 209 { "urn:oasis:names:tc:opendocument:xmlns:style:1.0", NS_STYLE }, 210 { "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0", NS_SVG }, 211 { "http://openoffice.org/2000/office", NS_OFFICE }, 212 { "http://openoffice.org/2000/text", NS_TEXT }, 213 { "http://openoffice.org/2000/table", NS_TABLE }, 214 { "http://openoffice.org/2000/drawing", NS_DRAW }, 215 { "http://openoffice.org/2000/presentation", NS_PRESENTATION }, 216 { "http://openoffice.org/2000/style", NS_STYLE }, 217 { "http://www.w3.org/2000/svg", NS_SVG }, 218 { NULL, NS_OTHER } 219 }; 220 221 222 /** 223 * Conventional prefixes, used for elements whose prefix was not 224 * declared on the root element. 225 */ 226 static const struct 227 { 228 const char *prefix; 229 enum OdfNamespace ns; 230 } ns_prefixes[] = { 231 { "office", NS_OFFICE }, 232 { "text", NS_TEXT }, 233 { "table", NS_TABLE }, 234 { "draw", NS_DRAW }, 235 { "presentation", NS_PRESENTATION }, 236 { "style", NS_STYLE }, 237 { "svg", NS_SVG }, 238 { NULL, NS_OTHER } 239 }; 240 241 242 /** 243 * What an element means for the full text. 244 */ 245 enum OdfElementKind 246 { 247 /** 248 * Nothing special, its content (if any) is used. 249 */ 250 EK_OTHER = 0, 251 252 /** 253 * A paragraph (text:p, text:h): text is only taken from inside 254 * paragraphs, and each one ends with a paragraph break. 255 */ 256 EK_PARA, 257 258 /** 259 * Material that is not part of the running text (annotations, 260 * notes, deleted text, page-number fields, speaker notes, ...): 261 * the whole subtree is skipped. 262 */ 263 EK_SKIP, 264 265 /** 266 * An element standing for white space (text:s, text:tab, 267 * text:line-break). 268 */ 269 EK_SPACE, 270 271 /** 272 * The office:body element; only text inside it is used. 273 */ 274 EK_BODY 275 }; 276 277 278 /** 279 * Classification of the elements that are not #EK_OTHER. 280 */ 281 static const struct 282 { 283 enum OdfNamespace ns; 284 const char *local; 285 enum OdfElementKind kind; 286 } element_kinds[] = { 287 { NS_OFFICE, "body", EK_BODY }, 288 { NS_TEXT, "p", EK_PARA }, 289 { NS_TEXT, "h", EK_PARA }, 290 { NS_TEXT, "s", EK_SPACE }, 291 { NS_TEXT, "tab", EK_SPACE }, 292 { NS_TEXT, "tab-stop", EK_SPACE }, /* OOo 1.x name of text:tab */ 293 { NS_TEXT, "line-break", EK_SPACE }, 294 /* comments */ 295 { NS_OFFICE, "annotation", EK_SKIP }, 296 /* footnotes and endnotes (ODF, and OOo 1.x) */ 297 { NS_TEXT, "note", EK_SKIP }, 298 { NS_TEXT, "footnote", EK_SKIP }, 299 { NS_TEXT, "endnote", EK_SKIP }, 300 /* change tracking: holds the text of deletions */ 301 { NS_TEXT, "tracked-changes", EK_SKIP }, 302 /* page furniture fields */ 303 { NS_TEXT, "page-number", EK_SKIP }, 304 { NS_TEXT, "page-count", EK_SKIP }, 305 { NS_TEXT, "page-continuation", EK_SKIP }, 306 { NS_TEXT, "page-continuation-string", EK_SKIP }, 307 { NS_TEXT, "page-variable-get", EK_SKIP }, 308 /* speaker notes of presentations */ 309 { NS_PRESENTATION, "notes", EK_SKIP }, 310 /* alternative text / titles of drawing objects */ 311 { NS_SVG, "title", EK_SKIP }, 312 { NS_SVG, "desc", EK_SKIP }, 313 /* headers, footers and master pages normally live in styles.xml 314 (which we do not read); skip them should they show up here */ 315 { NS_OFFICE, "master-styles", EK_SKIP }, 316 { NS_STYLE, "master-page", EK_SKIP }, 317 { NS_STYLE, "header", EK_SKIP }, 318 { NS_STYLE, "header-left", EK_SKIP }, 319 { NS_STYLE, "header-first", EK_SKIP }, 320 { NS_STYLE, "footer", EK_SKIP }, 321 { NS_STYLE, "footer-left", EK_SKIP }, 322 { NS_STYLE, "footer-first", EK_SKIP }, 323 { NS_OTHER, NULL, EK_OTHER } 324 }; 325 326 327 /** 328 * A namespace prefix declared on the root element. 329 */ 330 struct OdfPrefix 331 { 332 /** 333 * The prefix (not 0-terminated, points into the document). 334 */ 335 const char *prefix; 336 337 /** 338 * Number of bytes in @e prefix. 339 */ 340 size_t prefix_len; 341 342 /** 343 * Namespace the prefix is bound to. 344 */ 345 enum OdfNamespace ns; 346 }; 347 348 349 /** 350 * State of the full-text scanner. 351 */ 352 struct OdfScanner 353 { 354 /** 355 * Where the text goes. 356 */ 357 struct EXTRACTOR_FullText ft; 358 359 /** 360 * Prefixes declared on the root element. 361 */ 362 struct OdfPrefix prefixes[MAX_NS_PREFIXES]; 363 364 /** 365 * Number of valid entries in @e prefixes. 366 */ 367 unsigned int num_prefixes; 368 369 /** 370 * Kinds of the currently open elements. 371 */ 372 unsigned char stack[MAX_XML_DEPTH]; 373 374 /** 375 * Number of valid entries in @e stack. 376 */ 377 unsigned int depth; 378 379 /** 380 * Number of open elements nested deeper than #MAX_XML_DEPTH. 381 */ 382 unsigned int overflow; 383 384 /** 385 * Number of open #EK_SKIP elements. 386 */ 387 unsigned int skip_depth; 388 389 /** 390 * Number of open #EK_PARA elements. 391 */ 392 unsigned int para_depth; 393 394 /** 395 * Number of open #EK_BODY elements. 396 */ 397 unsigned int body_depth; 398 399 /** 400 * 1 once we have seen the root element. 401 */ 402 int have_root; 403 }; 404 405 406 /** 407 * Is character data at the current position part of the full text? 408 * 409 * @param sc scanner state 410 * @return 1 if yes 411 */ 412 static int 413 odf_text_active (const struct OdfScanner *sc) 414 { 415 return (0 < sc->body_depth) && 416 (0 < sc->para_depth) && 417 (0 == sc->skip_depth); 418 } 419 420 421 /** 422 * Is @a c XML white space? 423 * 424 * @param c character to test 425 * @return 1 if yes 426 */ 427 static int 428 odf_is_ws (char c) 429 { 430 return (' ' == c) || ('\t' == c) || ('\n' == c) || ('\r' == c); 431 } 432 433 434 /** 435 * Find the namespace of the qualified name @a name. 436 * 437 * @param sc scanner state (for the declared prefixes) 438 * @param name qualified name 439 * @param name_len number of bytes in @a name 440 * @param[out] local set to the local part of @a name 441 * @param[out] local_len set to the length of @a local 442 * @return namespace of @a name 443 */ 444 static enum OdfNamespace 445 odf_resolve (const struct OdfScanner *sc, 446 const char *name, 447 size_t name_len, 448 const char **local, 449 size_t *local_len) 450 { 451 const char *colon; 452 size_t plen; 453 unsigned int i; 454 455 colon = memchr (name, ':', name_len); 456 if (NULL == colon) 457 { 458 *local = name; 459 *local_len = name_len; 460 return NS_OTHER; 461 } 462 plen = colon - name; 463 *local = colon + 1; 464 *local_len = name_len - plen - 1; 465 for (i = 0; i < sc->num_prefixes; i++) 466 if ( (sc->prefixes[i].prefix_len == plen) && 467 (0 == memcmp (sc->prefixes[i].prefix, name, plen)) ) 468 return sc->prefixes[i].ns; 469 for (i = 0; NULL != ns_prefixes[i].prefix; i++) 470 if ( (strlen (ns_prefixes[i].prefix) == plen) && 471 (0 == memcmp (ns_prefixes[i].prefix, name, plen)) ) 472 return ns_prefixes[i].ns; 473 return NS_OTHER; 474 } 475 476 477 /** 478 * Classify the element @a name. 479 * 480 * @param sc scanner state 481 * @param name qualified element name 482 * @param name_len number of bytes in @a name 483 * @return what the element means for the full text 484 */ 485 static enum OdfElementKind 486 odf_classify (const struct OdfScanner *sc, 487 const char *name, 488 size_t name_len) 489 { 490 enum OdfNamespace ns; 491 const char *local; 492 size_t local_len; 493 unsigned int i; 494 495 ns = odf_resolve (sc, name, name_len, &local, &local_len); 496 if (NS_OTHER == ns) 497 return EK_OTHER; 498 for (i = 0; NULL != element_kinds[i].local; i++) 499 if ( (element_kinds[i].ns == ns) && 500 (strlen (element_kinds[i].local) == local_len) && 501 (0 == memcmp (element_kinds[i].local, local, local_len)) ) 502 return element_kinds[i].kind; 503 return EK_OTHER; 504 } 505 506 507 /** 508 * Record the "xmlns:PREFIX" declarations found in the attributes of 509 * the root element. 510 * 511 * @param sc scanner state to update 512 * @param attrs start of the attributes 513 * @param end end of the attributes 514 */ 515 static void 516 odf_parse_ns_decls (struct OdfScanner *sc, 517 const char *attrs, 518 const char *end) 519 { 520 const char *p = attrs; 521 522 while (p < end) 523 { 524 const char *an; 525 size_t an_len; 526 const char *val; 527 size_t val_len; 528 char quote; 529 unsigned int i; 530 531 while ( (p < end) && odf_is_ws (*p)) 532 p++; 533 an = p; 534 while ( (p < end) && ('=' != *p) && (! odf_is_ws (*p)) ) 535 p++; 536 an_len = p - an; 537 while ( (p < end) && odf_is_ws (*p)) 538 p++; 539 if ( (p >= end) || ('=' != *p) ) 540 return; 541 p++; 542 while ( (p < end) && odf_is_ws (*p)) 543 p++; 544 if ( (p >= end) || ( ('"' != *p) && ('\'' != *p) ) ) 545 return; 546 quote = *p++; 547 val = p; 548 while ( (p < end) && (quote != *p) ) 549 p++; 550 if (p >= end) 551 return; 552 val_len = p - val; 553 p++; 554 if ( (an_len <= 6) || 555 (0 != memcmp (an, "xmlns:", 6)) || 556 (sc->num_prefixes >= MAX_NS_PREFIXES) ) 557 continue; 558 for (i = 0; NULL != ns_uris[i].uri; i++) 559 { 560 if ( (strlen (ns_uris[i].uri) != val_len) || 561 (0 != memcmp (ns_uris[i].uri, val, val_len)) ) 562 continue; 563 sc->prefixes[sc->num_prefixes].prefix = an + 6; 564 sc->prefixes[sc->num_prefixes].prefix_len = an_len - 6; 565 sc->prefixes[sc->num_prefixes].ns = ns_uris[i].ns; 566 sc->num_prefixes++; 567 break; 568 } 569 } 570 } 571 572 573 /** 574 * Append XML character data to the full text, decoding the predefined 575 * entities and numeric character references. Other entity references 576 * are kept literally (we never expand entities declared in a DTD). 577 * 578 * @param ft where to append the text 579 * @param text character data 580 * @param len number of bytes in @a text 581 */ 582 static void 583 odf_append_chardata (struct EXTRACTOR_FullText *ft, 584 const char *text, 585 size_t len) 586 { 587 static const struct 588 { 589 const char *name; 590 uint32_t cp; 591 } entities[] = { 592 { "lt", '<' }, 593 { "gt", '>' }, 594 { "amp", '&' }, 595 { "quot", '"' }, 596 { "apos", '\'' }, 597 { NULL, 0 } 598 }; 599 size_t start = 0; 600 size_t i = 0; 601 602 while (i < len) 603 { 604 const char *semi; 605 size_t elen; 606 uint32_t cp; 607 int ok; 608 609 if ('&' != text[i]) 610 { 611 i++; 612 continue; 613 } 614 /* longest valid reference is "" / "" */ 615 elen = len - i - 1; 616 if (elen > 12) 617 elen = 12; 618 semi = memchr (&text[i + 1], ';', elen); 619 if (NULL == semi) 620 { 621 i++; 622 continue; 623 } 624 elen = semi - &text[i + 1]; 625 ok = 0; 626 cp = 0; 627 if ( (elen >= 2) && 628 ('#' == text[i + 1]) ) 629 { 630 size_t j = i + 2; 631 int hex = 0; 632 633 if ( ('x' == text[j]) || ('X' == text[j]) ) 634 { 635 hex = 1; 636 j++; 637 } 638 ok = (j < i + 1 + elen); 639 for (; j < i + 1 + elen; j++) 640 { 641 unsigned char c = (unsigned char) text[j]; 642 unsigned int d; 643 644 if ( (c >= '0') && (c <= '9') ) 645 d = c - '0'; 646 else if (hex && (c >= 'a') && (c <= 'f')) 647 d = c - 'a' + 10; 648 else if (hex && (c >= 'A') && (c <= 'F')) 649 d = c - 'A' + 10; 650 else 651 { 652 ok = 0; 653 break; 654 } 655 cp = cp * (hex ? 16 : 10) + d; 656 if (cp > 0x10FFFF) 657 { 658 ok = 0; 659 break; 660 } 661 } 662 if ( (0 == cp) || 663 ( (cp >= 0xD800) && (cp <= 0xDFFF) ) ) 664 ok = 0; 665 } 666 else 667 { 668 unsigned int k; 669 670 for (k = 0; NULL != entities[k].name; k++) 671 if ( (strlen (entities[k].name) == elen) && 672 (0 == memcmp (entities[k].name, &text[i + 1], elen)) ) 673 { 674 cp = entities[k].cp; 675 ok = 1; 676 break; 677 } 678 } 679 if (! ok) 680 { 681 i++; 682 continue; 683 } 684 EXTRACTOR_fulltext_append_ (ft, &text[start], i - start); 685 EXTRACTOR_fulltext_append_cp_ (ft, cp); 686 i += elen + 2; 687 start = i; 688 } 689 EXTRACTOR_fulltext_append_ (ft, &text[start], len - start); 690 } 691 692 693 /** 694 * Find @a needle in the @a len bytes at @a hay. 695 * 696 * @param hay where to search 697 * @param len number of bytes at @a hay 698 * @param needle 0-terminated string to find 699 * @return NULL if not found 700 */ 701 static const char * 702 odf_find (const char *hay, 703 size_t len, 704 const char *needle) 705 { 706 size_t nlen = strlen (needle); 707 const char *end = hay + len; 708 const char *p = hay; 709 710 while ( (size_t) (end - p) >= nlen) 711 { 712 p = memchr (p, needle[0], (end - p) - nlen + 1); 713 if (NULL == p) 714 return NULL; 715 if (0 == memcmp (p, needle, nlen)) 716 return p; 717 p++; 718 } 719 return NULL; 720 } 721 722 723 /** 724 * Handle a start tag (or empty-element tag). 725 * 726 * @param sc scanner state 727 * @param name qualified element name 728 * @param name_len number of bytes in @a name 729 * @param attrs start of the attributes 730 * @param attrs_end end of the attributes 731 * @param empty 1 if this is an empty-element tag ("<x/>") 732 * @return 0 to continue, -1 if this is not an ODF document 733 */ 734 static int 735 odf_start_element (struct OdfScanner *sc, 736 const char *name, 737 size_t name_len, 738 const char *attrs, 739 const char *attrs_end, 740 int empty) 741 { 742 enum OdfElementKind kind; 743 744 if (! sc->have_root) 745 { 746 const char *local; 747 size_t local_len; 748 749 sc->have_root = 1; 750 odf_parse_ns_decls (sc, attrs, attrs_end); 751 if ( (NS_OFFICE != odf_resolve (sc, name, name_len, &local, &local_len)) || 752 (strlen ("document-content") != local_len) || 753 (0 != memcmp (local, "document-content", local_len)) ) 754 return -1; 755 } 756 kind = odf_classify (sc, name, name_len); 757 if (EK_SPACE == kind) 758 { 759 if (odf_text_active (sc)) 760 EXTRACTOR_fulltext_space_ (&sc->ft); 761 } 762 else if ( (EK_PARA == kind) && 763 (0 == sc->skip_depth) ) 764 { 765 /* a paragraph nested in another one (e.g. in a text frame) is a 766 paragraph of its own */ 767 EXTRACTOR_fulltext_paragraph_ (&sc->ft); 768 } 769 if (empty) 770 return 0; 771 if ( (0 < sc->overflow) || 772 (MAX_XML_DEPTH == sc->depth) ) 773 { 774 sc->overflow++; 775 return 0; 776 } 777 sc->stack[sc->depth++] = (unsigned char) kind; 778 switch (kind) 779 { 780 case EK_PARA: 781 sc->para_depth++; 782 break; 783 case EK_SKIP: 784 sc->skip_depth++; 785 break; 786 case EK_BODY: 787 sc->body_depth++; 788 break; 789 default: 790 break; 791 } 792 return 0; 793 } 794 795 796 /** 797 * Handle an end tag. We do not check that it matches the start tag; 798 * a malformed document merely yields odd text. 799 * 800 * @param sc scanner state 801 */ 802 static void 803 odf_end_element (struct OdfScanner *sc) 804 { 805 if (0 < sc->overflow) 806 { 807 sc->overflow--; 808 return; 809 } 810 if (0 == sc->depth) 811 return; 812 switch ((enum OdfElementKind) sc->stack[--sc->depth]) 813 { 814 case EK_PARA: 815 sc->para_depth--; 816 if (0 == sc->skip_depth) 817 EXTRACTOR_fulltext_paragraph_ (&sc->ft); 818 break; 819 case EK_SKIP: 820 sc->skip_depth--; 821 break; 822 case EK_BODY: 823 sc->body_depth--; 824 break; 825 default: 826 break; 827 } 828 } 829 830 831 /** 832 * Scan the XML of content.xml and collect the running text. The 833 * scanner is iterative, runs in time linear in @a len and only keeps 834 * a bounded element stack. 835 * 836 * @param sc scanner state, initialized 837 * @param xml the document 838 * @param len number of bytes in @a xml 839 */ 840 static void 841 odf_scan (struct OdfScanner *sc, 842 const char *xml, 843 size_t len) 844 { 845 const char *p = xml; 846 const char *end = xml + len; 847 848 while ( (p < end) && 849 (! sc->ft.full) ) 850 { 851 const char *lt; 852 const char *q; 853 854 EXTRACTOR_fulltext_tick_ (&sc->ft); 855 if ('<' != *p) 856 { 857 lt = memchr (p, '<', end - p); 858 if (NULL == lt) 859 lt = end; 860 if (odf_text_active (sc)) 861 odf_append_chardata (&sc->ft, p, lt - p); 862 p = lt; 863 continue; 864 } 865 if ( (end - p >= 4) && 866 (0 == memcmp (p, "<!--", 4)) ) 867 { 868 q = odf_find (p + 4, end - p - 4, "-->"); 869 if (NULL == q) 870 return; 871 p = q + 3; 872 continue; 873 } 874 if ( (end - p >= 9) && 875 (0 == memcmp (p, "<![CDATA[", 9)) ) 876 { 877 q = odf_find (p + 9, end - p - 9, "]]>"); 878 if (NULL == q) 879 return; 880 if (odf_text_active (sc)) 881 EXTRACTOR_fulltext_append_ (&sc->ft, p + 9, q - (p + 9)); 882 p = q + 3; 883 continue; 884 } 885 if ( (end - p >= 2) && 886 ('?' == p[1]) ) 887 { 888 q = odf_find (p + 2, end - p - 2, "?>"); 889 if (NULL == q) 890 return; 891 p = q + 2; 892 continue; 893 } 894 if ( (end - p >= 2) && 895 ('!' == p[1]) ) 896 { 897 /* DOCTYPE; we do not support internal subsets (and thus never 898 expand user-defined entities) */ 899 q = memchr (p, '>', end - p); 900 if (NULL == q) 901 return; 902 p = q + 1; 903 continue; 904 } 905 if ( (end - p >= 2) && 906 ('/' == p[1]) ) 907 { 908 q = memchr (p, '>', end - p); 909 if (NULL == q) 910 return; 911 odf_end_element (sc); 912 p = q + 1; 913 continue; 914 } 915 /* start tag */ 916 { 917 const char *name = p + 1; 918 const char *attrs; 919 char quote = '\0'; 920 int empty; 921 922 q = name; 923 while ( (q < end) && 924 (! odf_is_ws (*q)) && 925 ('/' != *q) && 926 ('>' != *q) ) 927 q++; 928 attrs = q; 929 /* find the closing '>', honouring quoted attribute values */ 930 while (q < end) 931 { 932 if ('\0' != quote) 933 { 934 if (quote == *q) 935 quote = '\0'; 936 } 937 else if ( ('"' == *q) || ('\'' == *q) ) 938 quote = *q; 939 else if ('>' == *q) 940 break; 941 q++; 942 } 943 if (q >= end) 944 return; 945 empty = (q > attrs) && ('/' == q[-1]); 946 if (0 != odf_start_element (sc, 947 name, 948 attrs - name, 949 attrs, 950 empty ? q - 1 : q, 951 empty)) 952 return; 953 p = q + 1; 954 } 955 } 956 } 957 958 959 /** 960 * Read the file @a name from the archive, at most @a max_size bytes of 961 * it (the rest is silently ignored). 962 * 963 * @param uf archive to read from 964 * @param name name of the file to read 965 * @param max_size maximum number of bytes to return 966 * @param[out] size set to the number of bytes read 967 * @return NULL on error, otherwise the data (0-terminated), to be freed 968 */ 969 static char * 970 odf_read_member (struct EXTRACTOR_UnzipFile *uf, 971 const char *name, 972 size_t max_size, 973 size_t *size) 974 { 975 char filename_inzip[MAXFILENAME]; 976 struct EXTRACTOR_UnzipFileInfo file_info; 977 char *buf; 978 size_t cap; 979 size_t off; 980 981 if (EXTRACTOR_UNZIP_OK != 982 EXTRACTOR_common_unzip_go_find_local_file (uf, 983 name, 984 2)) 985 return NULL; 986 if (EXTRACTOR_UNZIP_OK != 987 EXTRACTOR_common_unzip_get_current_file_info (uf, 988 &file_info, 989 filename_inzip, 990 sizeof (filename_inzip), 991 NULL, 0, NULL, 0)) 992 return NULL; 993 if (EXTRACTOR_UNZIP_OK != 994 EXTRACTOR_common_unzip_open_current_file (uf)) 995 return NULL; 996 /* the size in the header is only a hint, do not trust it */ 997 cap = 64 * 1024; 998 if ( (file_info.uncompressed_size > cap) && 999 (file_info.uncompressed_size < 4 * 1024 * 1024) ) 1000 cap = file_info.uncompressed_size; 1001 if (cap > max_size) 1002 cap = max_size; 1003 if (NULL == (buf = malloc (cap + 1))) 1004 { 1005 EXTRACTOR_common_unzip_close_current_file (uf); 1006 return NULL; 1007 } 1008 off = 0; 1009 while (off < max_size) 1010 { 1011 ssize_t got; 1012 1013 if (off == cap) 1014 { 1015 size_t ncap = (cap > max_size / 2) ? max_size : 2 * cap; 1016 char *nbuf; 1017 1018 if (NULL == (nbuf = realloc (buf, ncap + 1))) 1019 break; 1020 buf = nbuf; 1021 cap = ncap; 1022 } 1023 got = EXTRACTOR_common_unzip_read_current_file (uf, 1024 &buf[off], 1025 cap - off); 1026 if (got <= 0) 1027 break; 1028 off += (size_t) got; 1029 } 1030 EXTRACTOR_common_unzip_close_current_file (uf); 1031 buf[off] = '\0'; 1032 *size = off; 1033 return buf; 1034 } 1035 1036 1037 /** 1038 * Extract the running text of the document body (content.xml) and 1039 * report it as #EXTRACTOR_METATYPE_FULL_TEXT. 1040 * 1041 * Only content.xml is read, so headers and footers (which live in the 1042 * master pages in styles.xml) never show up. Within the body we skip 1043 * comments (office:annotation), footnotes and endnotes (text:note: 1044 * they interrupt the sentence they are anchored in, and are not part 1045 * of the running text), deleted text of tracked changes, page number 1046 * and page count fields, alternative text of drawing objects and the 1047 * speaker notes of presentations. Paragraphs and headings (text:p, 1048 * text:h) become paragraphs; this also covers list items, table cells 1049 * (spreadsheets), text frames and the text boxes on presentation 1050 * slides and drawings. 1051 * 1052 * @param uf archive to read from 1053 * @param ec extraction context 1054 * @return 0 to continue extracting, 1 to stop 1055 */ 1056 static int 1057 odf_extract_fulltext (struct EXTRACTOR_UnzipFile *uf, 1058 struct EXTRACTOR_ExtractContext *ec) 1059 { 1060 struct OdfScanner *sc; 1061 char *xml; 1062 size_t xml_size; 1063 int ret; 1064 1065 if (NULL == (xml = odf_read_member (uf, 1066 CONTENTFILE, 1067 MAX_CONTENT_SIZE, 1068 &xml_size))) 1069 return 0; 1070 if (NULL == (sc = calloc (1, sizeof (*sc)))) 1071 { 1072 free (xml); 1073 return 0; 1074 } 1075 EXTRACTOR_fulltext_init_ (&sc->ft); 1076 EXTRACTOR_fulltext_set_context_ (&sc->ft, ec); 1077 odf_scan (sc, xml, xml_size); 1078 free (xml); 1079 ret = EXTRACTOR_fulltext_emit_ (&sc->ft, ec, "odf"); 1080 EXTRACTOR_fulltext_clear_ (&sc->ft); 1081 free (sc); 1082 return (0 != ret) ? 1 : 0; 1083 } 1084 1085 1086 /** 1087 * Main entry method for the ODF extraction plugin. 1088 * 1089 * @param ec extraction context provided to the plugin 1090 */ 1091 void 1092 EXTRACTOR_odf_extract_method (struct EXTRACTOR_ExtractContext *ec) 1093 { 1094 char filename_inzip[MAXFILENAME]; 1095 struct EXTRACTOR_UnzipFile *uf; 1096 struct EXTRACTOR_UnzipFileInfo file_info; 1097 char *buf; 1098 char *pbuf; 1099 size_t buf_size; 1100 unsigned int i; 1101 char *mimetype; 1102 1103 if (NULL == (uf = EXTRACTOR_common_unzip_open (ec))) 1104 return; 1105 if (NULL != (mimetype = libextractor_oo_getmimetype (uf))) 1106 { 1107 if (0 != ec->proc (ec->cls, 1108 "odf", 1109 EXTRACTOR_METATYPE_MIMETYPE, 1110 EXTRACTOR_METAFORMAT_UTF8, 1111 "text/plain", 1112 mimetype, 1113 strlen (mimetype) + 1)) 1114 { 1115 EXTRACTOR_common_unzip_close (uf); 1116 free (mimetype); 1117 return; 1118 } 1119 free (mimetype); 1120 } 1121 if (EXTRACTOR_UNZIP_OK != 1122 EXTRACTOR_common_unzip_go_find_local_file (uf, 1123 METAFILE, 1124 2)) 1125 { 1126 /* metafile not found */ 1127 goto FULLTEXT; 1128 } 1129 if (EXTRACTOR_UNZIP_OK != 1130 EXTRACTOR_common_unzip_get_current_file_info (uf, 1131 &file_info, 1132 filename_inzip, 1133 sizeof (filename_inzip), 1134 NULL, 0, NULL, 0)) 1135 { 1136 /* problems accessing metafile */ 1137 goto FULLTEXT; 1138 } 1139 if (EXTRACTOR_UNZIP_OK != 1140 EXTRACTOR_common_unzip_open_current_file (uf)) 1141 { 1142 /* problems with unzip */ 1143 goto FULLTEXT; 1144 } 1145 1146 buf_size = file_info.uncompressed_size; 1147 if (buf_size > 128 * 1024) 1148 { 1149 /* too big to be meta-data! */ 1150 EXTRACTOR_common_unzip_close_current_file (uf); 1151 goto FULLTEXT; 1152 } 1153 if (NULL == (buf = malloc (buf_size + 1))) 1154 { 1155 /* out of memory */ 1156 EXTRACTOR_common_unzip_close_current_file (uf); 1157 goto FULLTEXT; 1158 } 1159 if (buf_size != EXTRACTOR_common_unzip_read_current_file (uf, buf, buf_size)) 1160 { 1161 EXTRACTOR_common_unzip_close_current_file (uf); 1162 free (buf); 1163 goto FULLTEXT; 1164 } 1165 EXTRACTOR_common_unzip_close_current_file (uf); 1166 /* we don't do "proper" parsing of the meta-data but rather use some heuristics 1167 to get values out that we understand */ 1168 buf[buf_size] = '\0'; 1169 /* printf("%s\n", buf); */ 1170 /* try to find some of the typical OO xml headers */ 1171 if ( (strstr (buf, "xmlns:meta=\"http://openoffice.org/2000/meta\"") != 1172 NULL) || 1173 (strstr (buf, "xmlns:dc=\"http://purl.org/dc/elements/1.1/\"") != 1174 NULL) || 1175 (strstr (buf, "xmlns:xlink=\"http://www.w3.org/1999/xlink\"") != NULL) ) 1176 { 1177 /* accept as meta-data */ 1178 for (i = 0; NULL != tmap[i].text; i++) 1179 { 1180 char *spos; 1181 char *epos; 1182 char needle[256]; 1183 int oc; 1184 1185 pbuf = buf; 1186 1187 while (1) 1188 { 1189 strcpy (needle, "<"); 1190 strcat (needle, tmap[i].text); 1191 strcat (needle, ">"); 1192 spos = strstr (pbuf, needle); 1193 if (NULL == spos) 1194 { 1195 strcpy (needle, tmap[i].text); 1196 strcat (needle, "=\""); 1197 spos = strstr (pbuf, needle); 1198 if (spos == NULL) 1199 break; 1200 spos += strlen (needle); 1201 epos = spos; 1202 while ( (epos[0] != '\0') && 1203 (epos[0] != '"') ) 1204 epos++; 1205 } 1206 else 1207 { 1208 oc = 0; 1209 spos += strlen (needle); 1210 while ( (spos[0] != '\0') && 1211 ( (spos[0] == '<') || 1212 (oc > 0) ) ) 1213 { 1214 if (spos[0] == '<') 1215 oc++; 1216 if (spos[0] == '>') 1217 oc--; 1218 spos++; 1219 } 1220 epos = spos; 1221 while ( (epos[0] != '\0') && 1222 (epos[0] != '<') && 1223 (epos[0] != '>') ) 1224 { 1225 epos++; 1226 } 1227 } 1228 if (spos != epos) 1229 { 1230 char key[epos - spos + 1]; 1231 1232 memcpy (key, spos, epos - spos); 1233 key[epos - spos] = '\0'; 1234 if (0 != ec->proc (ec->cls, 1235 "odf", 1236 tmap[i].type, 1237 EXTRACTOR_METAFORMAT_UTF8, 1238 "text/plain", 1239 key, 1240 epos - spos + 1)) 1241 { 1242 free (buf); 1243 goto CLEANUP; 1244 } 1245 pbuf = epos; 1246 } 1247 else 1248 break; 1249 } 1250 } 1251 } 1252 free (buf); 1253 FULLTEXT: 1254 if (EXTRACTOR_fulltext_requested_ (ec->config)) 1255 (void) odf_extract_fulltext (uf, 1256 ec); 1257 CLEANUP: 1258 EXTRACTOR_common_unzip_close (uf); 1259 } 1260 1261 1262 /* end of odf_extractor.c */