From 3c5249af8087b0b1a4f131a3e19a9759639ab5db Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sun, 3 Jan 2010 21:42:52 +0000 Subject: adapting main code base to work with libextractor 0.6 API; testcases will follow later --- bootstrap | 5 +- configure.ac | 11 +- m4/libltdl-external.m4 | 75 --- src/fs/fs_directory.c | 49 +- src/fs/fs_file_information.c | 4 +- src/fs/fs_getopt.c | 31 +- src/fs/fs_publish.c | 14 +- src/fs/fs_uri.c | 31 +- src/fs/gnunet-directory.c | 21 +- src/fs/gnunet-publish.c | 66 +- src/fs/gnunet-search.c | 21 +- src/include/gnunet_container_lib.h | 162 ++--- src/include/gnunet_directories.h | 34 - src/util/bio.c | 18 +- src/util/container_meta_data.c | 1214 +++++++++++++++++++++++++----------- src/util/pseudonym.c | 35 +- 16 files changed, 1109 insertions(+), 682 deletions(-) delete mode 100644 m4/libltdl-external.m4 delete mode 100644 src/include/gnunet_directories.h diff --git a/bootstrap b/bootstrap index 91516ddd9..668fac020 100755 --- a/bootstrap +++ b/bootstrap @@ -1,2 +1,5 @@ #!/bin/sh -autoreconf -fiv -I m4 +libtoolize --automake --copy --force +aclocal +automake +autoreconf -f -i diff --git a/configure.ac b/configure.ac index 4a87c0067..41157ca29 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ # This file is part of GNUnet. -# (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) +# (C) 2001--2010 Christian Grothoff (and other contributing authors) # # GNUnet is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published @@ -38,7 +38,6 @@ AC_PROG_INSTALL AC_PROG_LN_S AC_PROG_MAKE_SET AM_PROG_CC_C_O -LT_CONFIG_LTDL_DIR([libltdl]) AC_CANONICAL_HOST LT_INIT([disable-static dlopen win32-dll]) LTDL_INIT @@ -141,7 +140,7 @@ netbsd*) AC_CHECK_LIB(intl, gettext) LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols" LIBS="$LIBS -lws2_32 -lplibc" - CFLAGS="-mms-bitfields -I../../libltdl $CFLAGS" + CFLAGS="-mms-bitfields $CFLAGS" build_target="mingw" AC_PROG_CXX LIBPREFIX=lib @@ -212,14 +211,14 @@ AC_ARG_WITH(extractor, ;; yes) AC_CHECK_HEADERS(extractor.h, - AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], + AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], extractor=1)) ;; *) LDFLAGS="-L$with_extractor/lib $LDFLAGS" CPPFLAGS="-I$with_extractor/include $CPPFLAGS" AC_CHECK_HEADERS(extractor.h, - AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], + AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH" extractor=1)) ;; @@ -227,7 +226,7 @@ AC_ARG_WITH(extractor, ], [AC_MSG_RESULT([--with-extractor not specified]) AC_CHECK_HEADERS(extractor.h, - AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], + AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], extractor=1))]) if test "$extractor" != 1 then diff --git a/m4/libltdl-external.m4 b/m4/libltdl-external.m4 deleted file mode 100644 index 1c4ce09b2..000000000 --- a/m4/libltdl-external.m4 +++ /dev/null @@ -1,75 +0,0 @@ -dnl Autoconf macro for an always external libltdl -dnl Copyright (C) 2009 Heikki Lindholm -dnl -dnl This file is free software; as a special exception the author gives -dnl unlimited permission to copy and/or distribute it, with or without -dnl modifications, as long as this notice is preserved. -dnl -dnl This file is distributed in the hope that it will be useful, but -dnl WITHOUT ANY WARRANTY, to the extent permitted by law; without even the -dnl implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -dnl AM_PATH_LIBLTDL( -dnl [CHECK-SYMBOLS, [ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND ]]]) -dnl -AC_DEFUN([AM_PATH_LIBLTDL], -[ AC_ARG_WITH(libltdl-prefix, - AC_HELP_STRING([--with-libltdl-prefix=PFX], - [prefix where libltdl is installed (optional)]), - libltdl_prefix="$withval", libltdl_prefix="") - ltdl_save_CPPFLAGS="$CPPFLAGS" - ltdl_save_LDFLAGS="$LDFLAGS" - if test x$libltdl_prefix != x ; then - CPPFLAGS="-I$libltdl_prefix/include $CPPFLAGS" - LDFLAGS="-L$libltdl_prefix/lib -lltdl $LDFLAGS" - else - if test x"$LIBLTDL" = x ; then - LIBLTDL="-lltdl" - fi - CPPFLAGS="$LTDLINCL $CPPFLAGS" - LDFLAGS="$LIBLTDL $LDFLAGS" - fi - - symbols_to_check=ifelse([$1], ,"ltdl_dlopen","$1") - ltdl_found=yes - AC_CHECK_HEADER([ltdl.h], - [ - for sym in $symbols_to_check - do - AC_CHECK_DECL([$sym], - [AC_LINK_IFELSE(AC_LANG_CALL([], [$sym]), - [ltdl_found=yes], - [ltdl_found=no])], - [ltdl_found=no], - [AC_INCLUDES_DEFAULT - #include ]) - done - ], - [ltdl_found=no], - [AC_INCLUDES_DEFAULT] - ) - - if test x$libltdl_prefix != x ; then - LTDLINCL="-I$libltdl_prefix/include" - LIBLTDL="-L$libltdl_prefix/lib -lltdl" - else - if test x"$LIBLTDL" = x ; then - LIBLTDL="-lltdl" - fi - fi - CPPFLAGS="$ltdl_save_CPPFLAGS" - LDFLAGS="$ltdl_save_LDFLAGS" - - AC_MSG_CHECKING(for libltdl with symbols $symbols_to_check) - if test $ltdl_found = yes; then - AC_MSG_RESULT(yes) - ifelse([$2], , :, [$2]) - else - LTDLINCL="" - LIBLTDL="" - AC_MSG_RESULT(no) - ifelse([$3], , :, [$3]) - fi - AC_SUBST(LTDLINCL) - AC_SUBST(LIBLTDL) -]) diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index e037130c8..48578c650 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c @@ -36,10 +36,6 @@ #include "gnunet_fs_service.h" #include "fs.h" -#ifndef EXTRACTOR_GNUNET_FULL_DATA -#define EXTRACTOR_GNUNET_FULL_DATA 137 -#endif - /** * String that is used to indicate that a file * is a GNUnet directory. @@ -60,7 +56,7 @@ GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData * char *mime; int ret; - mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); + mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); if (mime == NULL) return GNUNET_SYSERR; ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO; @@ -80,7 +76,7 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md) { char *mime; - mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); + mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); if (mime != NULL) { GNUNET_break (0 == strcmp (mime, @@ -89,8 +85,12 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md) return; } GNUNET_CONTAINER_meta_data_insert (md, - EXTRACTOR_MIMETYPE, - GNUNET_FS_DIRECTORY_MIME); + "", + EXTRACTOR_METATYPE_MIMETYPE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + GNUNET_FS_DIRECTORY_MIME, + strlen (GNUNET_FS_DIRECTORY_MIME)+1); } @@ -225,9 +225,9 @@ GNUNET_FS_directory_list_contents (size_t size, } pos += mdSize; filename = GNUNET_CONTAINER_meta_data_get_by_type (md, - EXTRACTOR_FILENAME); + EXTRACTOR_METATYPE_FILENAME); file_data = GNUNET_CONTAINER_meta_data_get_by_type (md, - EXTRACTOR_GNUNET_FULL_DATA); + EXTRACTOR_METATYPE_GNUNET_FULL_DATA); if (dep != NULL) dep (dep_cls, filename, @@ -322,6 +322,7 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, size_t mdxs; char *uris; char *ser; + char *sptr; size_t slen; struct GNUNET_CONTAINER_MetaData *meta; const struct GNUNET_CONTAINER_MetaData *meta_use; @@ -340,25 +341,24 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, fsize = 0; /* not given */ if (fsize > MAX_INLINE_SIZE) fsize = 0; /* too large */ - if ( (NULL == data) || - (NULL != memchr (data, 0, fsize)) ) - fsize = 0; /* must not have 0's in data! */ uris = GNUNET_FS_uri_to_string (uri); slen = strlen (uris) + 1; mds = - GNUNET_CONTAINER_meta_data_get_serialized_size (md, - GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + GNUNET_CONTAINER_meta_data_get_serialized_size (md); meta_use = md; meta = NULL; if (fsize > 0) { meta = GNUNET_CONTAINER_meta_data_duplicate (md); GNUNET_CONTAINER_meta_data_insert (meta, - EXTRACTOR_GNUNET_FULL_DATA, - data); + "", + EXTRACTOR_METATYPE_GNUNET_FULL_DATA, + EXTRACTOR_METAFORMAT_BINARY, + NULL, + data, + fsize); mdxs = - GNUNET_CONTAINER_meta_data_get_serialized_size (meta, - GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + GNUNET_CONTAINER_meta_data_get_serialized_size (meta); if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE == (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE) { @@ -376,8 +376,9 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, ser = (char*) &e[1]; memcpy (ser, uris, slen); GNUNET_free (uris); + sptr = &ser[slen + sizeof(uint32_t)]; ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, - &ser[slen + sizeof(uint32_t)], + &sptr, mds, GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); if (NULL != meta) @@ -503,6 +504,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, void **rdata) { char *data; + char *sptr; size_t *sizes; unsigned int *perm; unsigned int i; @@ -516,8 +518,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, uint32_t big; size = 8 + sizeof (uint32_t); - size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta, - GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta); sizes = NULL; perm = NULL; bes = NULL; @@ -560,9 +561,9 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); off = 8; + sptr = &data[off + sizeof (uint32_t)]; ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta, - &data[off + - sizeof (uint32_t)], + &sptr, size - off - sizeof (uint32_t), GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); GNUNET_assert (ret != -1); diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index c28635198..9c193f88b 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c @@ -364,7 +364,7 @@ struct DirScanCls /** * Metadata extractors to use. */ - struct EXTRACTOR_Extractor *extractors; + struct EXTRACTOR_PluginList *extractors; /** * Function to call on each directory entry. @@ -518,7 +518,7 @@ GNUNET_FS_directory_scanner_default (void *cls, void *proc_cls, char **emsg) { - struct EXTRACTOR_Extractor *ex = cls; + struct EXTRACTOR_PluginList *ex = cls; struct DirScanCls dsc; dsc.extractors = ex; diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c index a0d232641..f985ed8d0 100644 --- a/src/fs/fs_getopt.c +++ b/src/fs/fs_getopt.c @@ -131,7 +131,7 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* { struct GNUNET_CONTAINER_MetaData **mm = scls; - EXTRACTOR_KeywordType type; + enum EXTRACTOR_MetaType type; const char *typename; const char *typename_i18n; struct GNUNET_CONTAINER_MetaData *meta; @@ -151,17 +151,23 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* "utf-8" #endif ); - type = EXTRACTOR_getHighestKeywordTypeNumber (); + type = EXTRACTOR_metatype_get_max (); while (type > 0) { type--; - typename = EXTRACTOR_getKeywordTypeAsString (type); + typename = EXTRACTOR_metatype_to_string (type); typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); if ((strlen (tmp) >= strlen (typename) + 1) && (tmp[strlen (typename)] == ':') && (0 == strncmp (typename, tmp, strlen (typename)))) { - GNUNET_CONTAINER_meta_data_insert (meta, type, &tmp[strlen (typename) + 1]); + GNUNET_CONTAINER_meta_data_insert (meta, + "", + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + &tmp[strlen (typename) + 1], + strlen (&tmp[strlen (typename) + 1])+1); GNUNET_free (tmp); tmp = NULL; break; @@ -170,8 +176,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* (tmp[strlen (typename_i18n)] == ':') && (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) { - GNUNET_CONTAINER_meta_data_insert (meta, type, - &tmp[strlen (typename_i18n) + 1]); + GNUNET_CONTAINER_meta_data_insert (meta, + "", + type, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + &tmp[strlen (typename_i18n) + 1], + strlen (&tmp[strlen (typename_i18n) + 1]) + 1); GNUNET_free (tmp); tmp = NULL; break; @@ -179,7 +190,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* } if (tmp != NULL) { - GNUNET_CONTAINER_meta_data_insert (meta, EXTRACTOR_UNKNOWN, tmp); + GNUNET_CONTAINER_meta_data_insert (meta, + "", + EXTRACTOR_METATYPE_UNKNOWN, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + tmp, + strlen(tmp) + 1); GNUNET_free (tmp); printf (_ ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 46b342cab..02ddb2a0d 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c @@ -842,7 +842,7 @@ do_upload (void *cls, while (NULL != p->dir) { fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, - EXTRACTOR_FILENAME); + EXTRACTOR_METATYPE_FILENAME); p = p->dir; GNUNET_asprintf (&p->emsg, _("Recursive upload failed at `%s'"), @@ -1305,6 +1305,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, char *uris; size_t size; char *kbe; + char *sptr; pkc = GNUNET_malloc (sizeof (struct PublishKskContext)); pkc->h = h; @@ -1324,8 +1325,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, return; } } - pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); GNUNET_assert (pkc->mdsize >= 0); uris = GNUNET_FS_uri_to_string (uri); pkc->slen = strlen (uris) + 1; @@ -1339,8 +1339,9 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, kbe = (char *) &pkc->kb[1]; memcpy (kbe, uris, pkc->slen); GNUNET_free (uris); + sptr = &kbe[pkc->slen]; pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, - &kbe[pkc->slen], + &sptr, pkc->mdsize, GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); if (pkc->mdsize == -1) @@ -1480,8 +1481,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, if (update == NULL) update = ""; nidlen = strlen (update) + 1; - mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); + mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); size = sizeof (struct SBlock) + slen + nidlen + mdsize; if (size > MAX_SBLOCK_SIZE) @@ -1496,7 +1496,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, memcpy (dest, uris, slen); dest += slen; mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, - dest, + &dest, mdsize, GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); if (mdsize == -1) diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c index 02310a080..16b6e26e9 100644 --- a/src/fs/fs_uri.c +++ b/src/fs/fs_uri.c @@ -1469,19 +1469,34 @@ GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri) * Adds it to the URI. * * @param cls URI to update - * @param type type of the meta data - * @param data value of the meta data - * @return GNUNET_OK (always) + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return 0 (always) */ static int gather_uri_data (void *cls, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_len) { struct GNUNET_FS_Uri *uri = cls; char *nkword; int j; + if ( (format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING) ) + return 0; for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--) if (0 == strcmp (&uri->data.ksk.keywords[j][1], data)) return GNUNET_OK; @@ -1489,7 +1504,7 @@ gather_uri_data (void *cls, strcpy (nkword, " "); /* not mandatory */ strcat (nkword, data); uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword; - return GNUNET_OK; + return 0; } @@ -1514,8 +1529,8 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData ret->data.ksk.keywords = NULL; ret->data.ksk.keywords = GNUNET_malloc (sizeof (char *) * - GNUNET_CONTAINER_meta_data_get_contents (md, NULL, NULL)); - GNUNET_CONTAINER_meta_data_get_contents (md, &gather_uri_data, ret); + GNUNET_CONTAINER_meta_data_iterate (md, NULL, NULL)); + GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret); return ret; } diff --git a/src/fs/gnunet-directory.c b/src/fs/gnunet-directory.c index 6201c5dbb..3ea76985e 100644 --- a/src/fs/gnunet-directory.c +++ b/src/fs/gnunet-directory.c @@ -36,13 +36,20 @@ static int ret; */ static int item_printer (void *cls, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_size) { + if ( (format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING) ) + return 0; printf ("\t%20s: %s\n", dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, - EXTRACTOR_getKeywordTypeAsString (type)), data); - return GNUNET_OK; + EXTRACTOR_metatype_to_string (type)), data); + return 0; } @@ -75,9 +82,9 @@ print_entry (void *cls, string = GNUNET_FS_uri_to_string (uri); printf ("%s:\n", string); GNUNET_free (string); - GNUNET_CONTAINER_meta_data_get_contents (meta, - &item_printer, - NULL); + GNUNET_CONTAINER_meta_data_iterate (meta, + &item_printer, + NULL); } diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c index e3918f6fb..65f437f11 100644 --- a/src/fs/gnunet-publish.c +++ b/src/fs/gnunet-publish.c @@ -140,43 +140,57 @@ progress_cb (void *cls, * @param cls closure * @param type type of the meta data * @param data value of the meta data - * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort + * @return always 0 */ static int meta_printer (void *cls, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_size) { - if ( (type == EXTRACTOR_FILENAME) || - (EXTRACTOR_isBinaryType (type)) ) - return GNUNET_OK; + if ( (format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING) ) + return 0; + if (type == EXTRACTOR_METATYPE_FILENAME) + return 0; fprintf (stdout, "%s - %s", - EXTRACTOR_getKeywordTypeAsString (type), + EXTRACTOR_metatype_to_string (type), data); - return GNUNET_OK; + return 0; } /** - * Merge metadata entries (except binary - * metadata). + * Merge metadata entries. * * @param cls closure, target metadata structure * @param type type of the meta data * @param data value of the meta data - * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort + * @return always 0 */ static int meta_merger (void *cls, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_size) { struct GNUNET_CONTAINER_MetaData *m = cls; + GNUNET_CONTAINER_meta_data_insert (m, + plugin_name, type, - data); - return GNUNET_OK; + format, + data_mime_type, + data, + data_size); + return 0; } @@ -227,16 +241,16 @@ publish_inspector (void *cls, } if (NULL != meta) { - GNUNET_CONTAINER_meta_data_get_contents (meta, - &meta_merger, - m); + GNUNET_CONTAINER_meta_data_iterate (meta, + &meta_merger, + m); GNUNET_CONTAINER_meta_data_destroy (meta); meta = NULL; } if (extract_only) { fn = GNUNET_CONTAINER_meta_data_get_by_type (meta, - EXTRACTOR_FILENAME); + EXTRACTOR_METATYPE_FILENAME); fs = GNUNET_STRINGS_byte_size_fancy (length); fprintf (stdout, _("Keywords for file `%s' (%s)\n"), @@ -244,9 +258,9 @@ publish_inspector (void *cls, fs); GNUNET_free (fn); GNUNET_free (fs); - GNUNET_CONTAINER_meta_data_get_contents (meta, - &meta_printer, - NULL); + GNUNET_CONTAINER_meta_data_iterate (meta, + &meta_printer, + NULL); fprintf (stdout, "\n"); } if (GNUNET_FS_meta_data_test_for_directory (meta)) @@ -275,7 +289,7 @@ run (void *cls, { struct GNUNET_FS_FileInformation *fi; struct GNUNET_FS_Namespace *namespace; - EXTRACTOR_ExtractorList *l; + struct EXTRACTOR_PluginList *l; char *ex; char *emsg; @@ -377,13 +391,13 @@ run (void *cls, l = NULL; if (! disable_extractor) { - l = EXTRACTOR_loadDefaultLibraries (); + l = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex)) { if (strlen (ex) > 0) - l = EXTRACTOR_loadConfigLibraries (l, ex); + l = EXTRACTOR_plugin_add_config (l, ex, EXTRACTOR_OPTION_DEFAULT_POLICY); GNUNET_free (ex); } } @@ -396,7 +410,7 @@ run (void *cls, priority, GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION), &emsg); - EXTRACTOR_removeAll (l); + EXTRACTOR_plugin_remove_all (l); if (fi == NULL) { fprintf (stderr, diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c index 067d77159..4836f9d83 100644 --- a/src/fs/gnunet-search.c +++ b/src/fs/gnunet-search.c @@ -45,12 +45,19 @@ static int verbose; static int item_printer (void *cls, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_size) { + if ( (format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING) ) + return 0; printf ("\t%20s: %s\n", dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, - EXTRACTOR_getKeywordTypeAsString (type)), + EXTRACTOR_metatype_to_string (type)), data); return GNUNET_OK; } @@ -86,7 +93,7 @@ progress_cb (void *cls, printf ("%s:\n", uri); filename = GNUNET_CONTAINER_meta_data_get_by_type (info->value.search.specifics.result.meta, - EXTRACTOR_FILENAME); + EXTRACTOR_METATYPE_FILENAME); if (filename != NULL) { while (NULL != (dotdot = strstr (filename, ".."))) @@ -98,9 +105,9 @@ progress_cb (void *cls, else printf ("gnunet-download %s\n", uri); if (verbose) - GNUNET_CONTAINER_meta_data_get_contents (info->value.search.specifics.result.meta, - &item_printer, - NULL); + GNUNET_CONTAINER_meta_data_iterate (info->value.search.specifics.result.meta, + &item_printer, + NULL); printf ("\n"); fflush(stdout); GNUNET_free_non_null (filename); diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h index db9e10ef8..3df9dcfc0 100644 --- a/src/include/gnunet_container_lib.h +++ b/src/include/gnunet_container_lib.h @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -195,24 +195,13 @@ void GNUNET_CONTAINER_bloomfilter_resize (struct GNUNET_CONTAINER_BloomFilter */ struct GNUNET_CONTAINER_MetaData; -/** - * Iterator over meta data. - * - * @param cls closure - * @param type type of the meta data - * @param data value of the meta data - * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort - */ -typedef int (*GNUNET_CONTAINER_MetaDataProcessor) (void *cls, - EXTRACTOR_KeywordType type, - const char *data); - /** * Create a fresh MetaData token. * * @return empty meta-data container */ -struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void); +struct GNUNET_CONTAINER_MetaData * +GNUNET_CONTAINER_meta_data_create (void); /** * Duplicate a MetaData token. @@ -220,43 +209,61 @@ struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void); * @param md what to duplicate * @return duplicate meta-data container */ -struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_duplicate (const - struct - GNUNET_CONTAINER_MetaData - *md); +struct GNUNET_CONTAINER_MetaData * +GNUNET_CONTAINER_meta_data_duplicate (const struct + GNUNET_CONTAINER_MetaData *md); /** * Free meta data. * * @param md what to free */ -void GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData - *md); +void +GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md); /** - * Test if two MDs are equal. + * Test if two MDs are equal. We consider them equal if + * the meta types, formats and content match (we do not + * include the mime types and plugins names in this + * consideration). * * @param md1 first value to check * @param md2 other value to check * @return GNUNET_YES if they are equal */ -int GNUNET_CONTAINER_meta_data_test_equal (const struct - GNUNET_CONTAINER_MetaData *md1, - const struct - GNUNET_CONTAINER_MetaData *md2); +int +GNUNET_CONTAINER_meta_data_test_equal (const struct + GNUNET_CONTAINER_MetaData *md1, + const struct + GNUNET_CONTAINER_MetaData *md2); /** * Extend metadata. * * @param md metadata to extend - * @param type type of the new entry - * @param data value for the entry + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists + * data_mime_type and plugin_name are not considered for "exists" checks */ -int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, - EXTRACTOR_KeywordType type, - const char *data); +int +GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_len); + /** * Remove an item. @@ -265,11 +272,15 @@ int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, * @param type type of the item to remove * @param data specific value to remove, NULL to remove all * entries of the given type + * @param data_len number of bytes in data * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md */ -int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, - EXTRACTOR_KeywordType type, - const char *data); +int +GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, + enum EXTRACTOR_MetaType type, + const char *data, + size_t data_len); + /** * Add the current time as the publication date @@ -277,58 +288,70 @@ int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, * * @param md metadata to modify */ -void GNUNET_CONTAINER_meta_data_add_publication_date (struct - GNUNET_CONTAINER_MetaData - *md); +void +GNUNET_CONTAINER_meta_data_add_publication_date (struct + GNUNET_CONTAINER_MetaData + *md); + /** - * Iterate over MD entries, excluding thumbnails. + * Iterate over MD entries. * * @param md metadata to inspect * @param iter function to call on each entry * @param iter_cls closure for iterator * @return number of entries */ -int GNUNET_CONTAINER_meta_data_get_contents (const struct - GNUNET_CONTAINER_MetaData *md, - GNUNET_CONTAINER_MetaDataProcessor - iter, void *iter_cls); +int GNUNET_CONTAINER_meta_data_iterate (const struct + GNUNET_CONTAINER_MetaData *md, + EXTRACTOR_MetaDataProcessor + iter, void *iter_cls); /** - * Get the first MD entry of the given type. + * Get the first MD entry of the given type. Caller + * is responsible for freeing the return value. + * Also, only meta data items that are strings (0-terminated) + * are returned by this function. * * @param md metadata to inspect * @param type type to look for - * @return NULL if we do not have any such entry, - * otherwise client is responsible for freeing the value! + * @return NULL if no entry was found */ -char *GNUNET_CONTAINER_meta_data_get_by_type (const struct - GNUNET_CONTAINER_MetaData *md, - EXTRACTOR_KeywordType type); +char * +GNUNET_CONTAINER_meta_data_get_by_type (const struct + GNUNET_CONTAINER_MetaData *md, + enum EXTRACTOR_MetaType type); + /** - * Get the first matching MD entry of the given types. + * Get the first matching MD entry of the given types. Caller is + * responsible for freeing the return value. Also, only meta data + * items that are strings (0-terminated) are returned by this + * function. * * @param md metadata to inspect * @param ... -1-terminated list of types * @return NULL if we do not have any such entry, * otherwise client is responsible for freeing the value! */ -char *GNUNET_CONTAINER_meta_data_get_first_by_types (const struct - GNUNET_CONTAINER_MetaData - *md, ...); +char * +GNUNET_CONTAINER_meta_data_get_first_by_types (const struct + GNUNET_CONTAINER_MetaData + *md, ...); /** - * Get a thumbnail from the meta-data (if present). + * Get a thumbnail from the meta-data (if present). Only matches meta + * data with mime type "image" and binary format. * * @param md metadata to inspect * @param thumb will be set to the thumbnail data. Must be * freed by the caller! * @return number of bytes in thumbnail, 0 if not available */ -size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct - GNUNET_CONTAINER_MetaData - *md, unsigned char **thumb); +size_t +GNUNET_CONTAINER_meta_data_get_thumbnail (const struct + GNUNET_CONTAINER_MetaData + *md, unsigned char **thumb); /** * Extract meta-data from a file. @@ -339,11 +362,12 @@ size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct * @return GNUNET_SYSERR on error, otherwise the number * of meta-data items obtained */ -int GNUNET_CONTAINER_meta_data_extract_from_file (struct - GNUNET_CONTAINER_MetaData - *md, const char *filename, - EXTRACTOR_ExtractorList * - extractors); +int +GNUNET_CONTAINER_meta_data_extract_from_file (struct + GNUNET_CONTAINER_MetaData + *md, const char *filename, + struct EXTRACTOR_PluginList * + extractors); /** @@ -373,7 +397,8 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions * Serialize meta-data to target. * * @param md metadata to serialize - * @param target where to write the serialized metadata + * @param target where to write the serialized metadata; + * *target can be NULL, in which case memory is allocated * @param max maximum number of bytes available * @param opt is it ok to just write SOME of the * meta-data to match the size constraint, @@ -384,7 +409,7 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions */ ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData *md, - char *target, + char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions @@ -392,22 +417,15 @@ ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct /** - * Estimate (!) the size of the meta-data in - * serialized form. The estimate MAY be higher - * than what is strictly needed. + * Get the size of the full meta-data in serialized form. * * @param md metadata to inspect - * @param opt is it ok to just write SOME of the - * meta-data to match the size constraint, - * possibly discarding some data? * @return number of bytes needed for serialization, -1 on error */ ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct GNUNET_CONTAINER_MetaData - *md, - enum - GNUNET_CONTAINER_MetaDataSerializationOptions - opt); + *md); + /** * Deserialize meta-data. Initializes md. diff --git a/src/include/gnunet_directories.h b/src/include/gnunet_directories.h deleted file mode 100644 index 05a753f44..000000000 --- a/src/include/gnunet_directories.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - This file is part of GNUnet. - (C) 2001, 2002, 2003, 2004, 2005, 2006 Christian Grothoff (and other contributing authors) - - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your - option) any later version. - - GNUnet is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. -*/ - -/** - * @file include/gnunet_directories.h - * @brief directories and files in GNUnet (default locations) - * - * @author Christian Grothoff - */ - -#ifndef GNUNET_DIRECTORIES -#define GNUNET_DIRECTORIES - -#define GNUNET_DEFAULT_CLIENT_CONFIG_FILE "~/.gnunet/gnunet.conf" -#define GNUNET_DEFAULT_DAEMON_CONFIG_FILE "/etc/gnunetd.conf" - -#endif diff --git a/src/util/bio.c b/src/util/bio.c index e1085faa6..b05746c38 100644 --- a/src/util/bio.c +++ b/src/util/bio.c @@ -434,21 +434,11 @@ GNUNET_BIO_write_meta_data (struct GNUNET_BIO_WriteHandle *h, ssize_t size; char *buf; - size = GNUNET_CONTAINER_meta_data_get_serialized_size (m, - GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL - | - GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS); - if (size == -1) - return GNUNET_SYSERR; - if (size > MAX_META_DATA) - size = MAX_META_DATA; - buf = GNUNET_malloc (size); + buf = NULL; size = GNUNET_CONTAINER_meta_data_serialize (m, - buf, - size, - GNUNET_CONTAINER_META_DATA_SERIALIZE_PART - | - GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS); + &buf, + MAX_META_DATA, + GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); if (size == -1) { GNUNET_free (buf); diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c index 912ac2684..e4d8737c8 100644 --- a/src/util/container_meta_data.c +++ b/src/util/container_meta_data.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) + (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -32,12 +32,46 @@ #include #include -#define EXTRA_CHECKS ALLOW_EXTRA_CHECKS - -struct Item +/** + * Meta data item. + */ +struct MetaItem { - EXTRACTOR_KeywordType type; + /** + * This is a linked list. + */ + struct MetaItem *next; + + /** + * Name of the extracting plugin. + */ + char *plugin_name; + + /** + * Mime-type of data. + */ + char *mime_type; + + /** + * The actual meta data. + */ char *data; + + /** + * Number of bytes in 'data'. + */ + size_t data_size; + + /** + * Type of the meta data. + */ + enum EXTRACTOR_MetaType type; + + /** + * Format of the meta data. + */ + enum EXTRACTOR_MetaFormat format; + }; /** @@ -45,86 +79,224 @@ struct Item */ struct GNUNET_CONTAINER_MetaData { - uint32_t itemCount; - struct Item *items; + /** + * Linked list of the meta data items. + */ + struct MetaItem *items; + + /** + * Complete serialized and compressed buffer of the items. + * NULL if we have not computed that buffer yet. + */ + char *sbuf; + + /** + * Number of bytes in 'sbuf'. 0 if the buffer is stale. + */ + size_t sbuf_size; + + /** + * Number of items in the linked list. + */ + unsigned int item_count; + }; + /** * Create a fresh struct CONTAINER_MetaData token. + * + * @return empty meta-data container */ struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create () { - struct GNUNET_CONTAINER_MetaData *ret; - ret = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData)); - ret->items = NULL; - ret->itemCount = 0; - return ret; + return GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData)); } + +/** + * Free meta data item. + * + * @param item item to free + */ +static void +meta_item_free (struct MetaItem *item) +{ + GNUNET_free_non_null (item->plugin_name); + GNUNET_free_non_null (item->mime_type); + GNUNET_free_non_null (item->data); + GNUNET_free (item); +} + + +/** + * The meta data has changed, invalidate its serialization + * buffer. + * + * @param md meta data that changed + */ +static void +invalidate_sbuf (struct GNUNET_CONTAINER_MetaData *md) +{ + if (md->sbuf == NULL) + return; + GNUNET_free (md->sbuf); + md->sbuf = NULL; + md->sbuf_size = 0; +} + + /** * Free meta data. + * + * @param md what to free */ void GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md) { - int i; + struct MetaItem *item; if (md == NULL) return; - for (i = 0; i < md->itemCount; i++) - GNUNET_free (md->items[i].data); - GNUNET_array_grow (md->items, md->itemCount, 0); + while (NULL != (item = md->items)) + { + md->items = item->next; + meta_item_free (item); + } + GNUNET_free_non_null (md->sbuf); GNUNET_free (md); } + /** - * Add the current time as the publication date - * to the meta-data. + * Test if two MDs are equal. We consider them equal if + * the meta types, formats and content match (we do not + * include the mime types and plugins names in this + * consideration). + * + * @param md1 first value to check + * @param md2 other value to check + * @return GNUNET_YES if they are equal */ -void -GNUNET_CONTAINER_meta_data_add_publication_date (struct - GNUNET_CONTAINER_MetaData - *md) +int +GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData + *md1, + const struct GNUNET_CONTAINER_MetaData + *md2) { - char *dat; - struct GNUNET_TIME_Absolute t; + struct MetaItem *i; + struct MetaItem *j; + int found; - t = GNUNET_TIME_absolute_get (); - GNUNET_CONTAINER_meta_data_delete (md, EXTRACTOR_PUBLICATION_DATE, NULL); - dat = GNUNET_STRINGS_absolute_time_to_string (t); - GNUNET_CONTAINER_meta_data_insert (md, EXTRACTOR_PUBLICATION_DATE, dat); - GNUNET_free (dat); + if (md1 == md2) + return GNUNET_YES; + if (md1->item_count != md2->item_count) + return GNUNET_NO; + + i = md1->items; + while (NULL != i) + { + found = GNUNET_NO; + j = md2->items; + while (NULL != j) + { + if ( (i->type == j->type) && + (i->format == j->format) && + (i->data_size == j->data_size) && + (0 == memcmp (i->data, + j->data, + i->data_size))) + { + found = GNUNET_YES; + break; + } + j = j->next; + } + if (found == GNUNET_NO) + return GNUNET_NO; + i = i->next; + } + return GNUNET_YES; } + /** - * Extend metadata. + * Extend metadata. Note that the list of meta data items is + * sorted by size (largest first). + * + * @param md metadata to extend + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists + * data_mime_type and plugin_name are not considered for "exists" checks */ int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, - EXTRACTOR_KeywordType type, - const char *data) + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_len) { - uint32_t idx; + struct MetaItem *prev; + struct MetaItem *pos; + struct MetaItem *i; char *p; - GNUNET_assert (data != NULL); - for (idx = 0; idx < md->itemCount; idx++) + prev = NULL; + pos = md->items; + while (NULL != pos) { - if ((md->items[idx].type == type) && - (0 == strcmp (md->items[idx].data, data))) - return GNUNET_SYSERR; + if (pos->data_size < data_len) + break; + if ( (pos->type == type) && + (pos->format == format) && + (pos->data_size == data_len) && + (0 == memcmp (pos->data, + data, + data_len))) + { + if ( (pos->mime_type == NULL) && + (data_mime_type != NULL) ) + { + pos->mime_type = GNUNET_strdup (data_mime_type); + invalidate_sbuf (md); + } + return GNUNET_SYSERR; + } + prev = pos; + pos = pos->next; } - idx = md->itemCount; - GNUNET_array_grow (md->items, md->itemCount, md->itemCount + 1); - md->items[idx].type = type; - md->items[idx].data = p = GNUNET_strdup (data); - + md->item_count++; + i = GNUNET_malloc (sizeof (struct MetaItem)); + i->type = type; + i->format = format; + i->data_size = data_len; + i->next = pos; + if (prev == NULL) + md->items = i; + else + prev->next = i; + i->mime_type = (data_mime_type == NULL) ? NULL : GNUNET_strdup (data_mime_type); + i->plugin_name = (plugin_name == NULL) ? NULL : GNUNET_strdup (plugin_name); + i->data = GNUNET_malloc (data_len); + memcpy (i->data, data, data_len); /* change OS native dir separators to unix '/' and others to '_' */ - if (type == EXTRACTOR_FILENAME) + if (type == EXTRACTOR_METATYPE_FILENAME) { - while (*p != '\0') + p = i->data; + while ( (*p != '\0') && + (p < i->data + data_len) ) { if (*p == DIR_SEPARATOR) *p = '/'; @@ -133,10 +305,11 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, p++; } } - + invalidate_sbuf (md); return GNUNET_OK; } + /** * Remove an item. * @@ -144,36 +317,78 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, * @param type type of the item to remove * @param data specific value to remove, NULL to remove all * entries of the given type + * @param data_len number of bytes in data * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md */ int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, - EXTRACTOR_KeywordType type, - const char *data) + enum EXTRACTOR_MetaType type, + const char *data, + size_t data_len) { - uint32_t idx; - int ret = GNUNET_SYSERR; - for (idx = 0; idx < md->itemCount; idx++) + struct MetaItem *pos; + struct MetaItem *prev; + + prev = NULL; + pos = md->items; + while (NULL != pos) { - if ((md->items[idx].type == type) && - ((data == NULL) || (0 == strcmp (md->items[idx].data, data)))) - { - GNUNET_free (md->items[idx].data); - md->items[idx] = md->items[md->itemCount - 1]; - GNUNET_array_grow (md->items, md->itemCount, md->itemCount - 1); - if (data == NULL) - { - ret = GNUNET_OK; - continue; - } - return GNUNET_OK; - } + if ( (pos->type == type) && + ( (data == NULL) || + ( (pos->data_size == data_len) && + (0 == memcmp (pos->data, + data, + data_len))) ) ) + { + if (prev == NULL) + md->items = pos->next; + else + prev->next = pos->next; + meta_item_free (pos); + md->item_count--; + invalidate_sbuf (md); + return GNUNET_OK; + } + prev = pos; + pos = pos->next; } - return ret; + return GNUNET_SYSERR; } + /** - * Iterate over MD entries, excluding thumbnails. + * Add the current time as the publication date + * to the meta-data. + * + * @param md metadata to modify + */ +void +GNUNET_CONTAINER_meta_data_add_publication_date (struct + GNUNET_CONTAINER_MetaData + *md) +{ + char *dat; + struct GNUNET_TIME_Absolute t; + + t = GNUNET_TIME_absolute_get (); + GNUNET_CONTAINER_meta_data_delete (md, + EXTRACTOR_METATYPE_PUBLICATION_DATE, + NULL, + 0); + dat = GNUNET_STRINGS_absolute_time_to_string (t); + GNUNET_CONTAINER_meta_data_insert (md, + "", + EXTRACTOR_METATYPE_PUBLICATION_DATE, + EXTRACTOR_METAFORMAT_UTF8, + "text/plain", + dat, + strlen(dat)+1); + GNUNET_free (dat); +} + + +/** + * Iterate over MD entries. * * @param md metadata to inspect * @param iter function to call on each entry @@ -181,51 +396,71 @@ GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, * @return number of entries */ int -GNUNET_CONTAINER_meta_data_get_contents (const struct - GNUNET_CONTAINER_MetaData *md, - GNUNET_CONTAINER_MetaDataProcessor - iter, void *iter_cls) +GNUNET_CONTAINER_meta_data_iterate (const struct + GNUNET_CONTAINER_MetaData *md, + EXTRACTOR_MetaDataProcessor + iter, void *iter_cls) { - uint32_t i; - uint32_t sub; + struct MetaItem *pos; - sub = 0; - for (i = 0; i < md->itemCount; i++) + if (iter == NULL) + return md->item_count; + pos = md->items; + while (NULL != pos) { - if (!EXTRACTOR_isBinaryType (md->items[i].type)) - { - if ((iter != NULL) && - (GNUNET_OK != iter (iter_cls, - md->items[i].type, md->items[i].data))) - return GNUNET_SYSERR; - } - else - sub++; + if (0 != iter (iter_cls, + pos->plugin_name, + pos->type, + pos->format, + pos->mime_type, + pos->data, + pos->data_size)) + return md->item_count; + pos = pos->next; } - return (int) (md->itemCount - sub); + return md->item_count; } + /** - * Iterate over MD entries + * Get the first MD entry of the given type. Caller + * is responsible for freeing the return value. + * Also, only meta data items that are strings (0-terminated) + * are returned by this function. * - * @return number of entries + * @param md metadata to inspect + * @param type type to look for + * @return NULL if no entry was found */ char * GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData - *md, EXTRACTOR_KeywordType type) + *md, enum EXTRACTOR_MetaType type) { - uint32_t i; + struct MetaItem *pos; - for (i = 0; i < md->itemCount; i++) - if (type == md->items[i].type) - return GNUNET_strdup (md->items[i].data); + pos = md->items; + while (NULL != pos) + { + if ( (type == pos->type) && + ( (pos->format == EXTRACTOR_METAFORMAT_UTF8) || + (pos->format == EXTRACTOR_METAFORMAT_C_STRING) ) ) + return GNUNET_strdup (pos->data); + pos = pos->next; + } return NULL; } + /** - * Iterate over MD entries + * Get the first matching MD entry of the given types. Caller is + * responsible for freeing the return value. Also, only meta data + * items that are strings (0-terminated) are returned by this + * function. * - * @return number of entries + * @param md metadata to inspect + * @param ... -1-terminated list of types + * @return NULL if we do not have any such entry, + * otherwise client is responsible for freeing the value! */ char * GNUNET_CONTAINER_meta_data_get_first_by_types (const struct @@ -234,13 +469,13 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct { char *ret; va_list args; - EXTRACTOR_KeywordType type; + enum EXTRACTOR_MetaType type; ret = NULL; va_start (args, md); while (1) { - type = va_arg (args, EXTRACTOR_KeywordType); + type = va_arg (args, enum EXTRACTOR_MetaType); if (type == -1) break; ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type); @@ -251,6 +486,7 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct return ret; } + /** * Get a thumbnail from the meta-data (if present). * @@ -264,27 +500,33 @@ GNUNET_CONTAINER_meta_data_get_thumbnail (const struct GNUNET_CONTAINER_MetaData * md, unsigned char **thumb) { - char *encoded; - int ret; - size_t size; + struct MetaItem *pos; + struct MetaItem *match; - encoded = - GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_THUMBNAIL_DATA); - if (encoded == NULL) - return 0; - if (strlen (encoded) == 0) + match = NULL; + pos = md->items; + while (NULL != pos) { - GNUNET_free (encoded); - return 0; /* invalid */ + if ( (0 == strncasecmp ("image/", pos->mime_type, + strlen("image/"))) && + (pos->format == EXTRACTOR_METAFORMAT_BINARY) ) + { + if (match == NULL) + match = pos; + else if ( (match->type != EXTRACTOR_METATYPE_THUMBNAIL) && + (pos->type == EXTRACTOR_METATYPE_THUMBNAIL) ) + match = pos; + } + pos = pos->next; } - *thumb = NULL; - ret = EXTRACTOR_binaryDecode (encoded, thumb, &size); - GNUNET_free (encoded); - if (ret != 0) + if (match == NULL) return 0; - return size; + *thumb = GNUNET_malloc (match->data_size); + memcpy (*thumb, match->data, match->data_size); + return match->data_size; } + /** * Duplicate struct GNUNET_CONTAINER_MetaData. * @@ -295,18 +537,66 @@ struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData *md) { - uint32_t i; struct GNUNET_CONTAINER_MetaData *ret; + struct MetaItem *pos; if (md == NULL) return NULL; ret = GNUNET_CONTAINER_meta_data_create (); - for (i = 0; i < md->itemCount; i++) - GNUNET_CONTAINER_meta_data_insert (ret, md->items[i].type, - md->items[i].data); + pos = md->items; + while (NULL != pos) + { + GNUNET_CONTAINER_meta_data_insert (ret, + pos->plugin_name, + pos->type, + pos->format, + pos->mime_type, + pos->data, + pos->data_size); + pos = pos->next; + } return ret; } + +/** + * Add meta data that libextractor finds to our meta data + * container. + * + * @param cls closure, our meta data container + * @param plugin_name name of the plugin that produced this value; + * special values can be used (i.e. '' for zlib being + * used in the main libextractor library and yielding + * meta data). + * @param type libextractor-type describing the meta data + * @param format basic format information about data + * @param data_mime_type mime-type of data (not of the original file); + * can be NULL (if mime-type is not known) + * @param data actual meta-data found + * @param data_len number of bytes in data + * @return always 0 to continue extracting + */ +static int +add_to_md(void *cls, + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_len) +{ + struct GNUNET_CONTAINER_MetaData *md = cls; + (void) GNUNET_CONTAINER_meta_data_insert (md, + plugin_name, + type, + format, + data_mime_type, + data, + data_len); + return 0; +} + + /** * Extract meta-data from a file. * @@ -316,37 +606,43 @@ GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData int GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData *md, const char *filename, - EXTRACTOR_ExtractorList * + struct EXTRACTOR_PluginList * extractors) { - EXTRACTOR_KeywordList *head; - EXTRACTOR_KeywordList *pos; - int ret; + unsigned int old; if (filename == NULL) return GNUNET_SYSERR; if (extractors == NULL) return 0; - head = EXTRACTOR_getKeywords (extractors, filename); - head = EXTRACTOR_removeDuplicateKeywords (head, - EXTRACTOR_DUPLICATES_REMOVE_UNKNOWN); - pos = head; - ret = 0; - while (pos != NULL) - { - if (GNUNET_OK == - GNUNET_CONTAINER_meta_data_insert (md, pos->keywordType, - pos->keyword)) - ret++; - pos = pos->next; - } - EXTRACTOR_freeKeywords (head); - return ret; + old = md->item_count; + EXTRACTOR_extract (extractors, + filename, + NULL, 0, + &add_to_md, + md); + return (int) (md->item_count - old); } -static unsigned int -tryCompression (char *data, unsigned int oldSize) +/** + * Try to compress the given block of data. + * + * @param data block to compress; if compression + * resulted in a smaller block, the first + * bytes of data are updated to the compressed + * data + * @param oldSize number of bytes in data + * @param result set to the compressed data + * @param newSize set to size of result + * @return GNUNET_YES if compression reduce the size, + * GNUNET_NO if compression did not help + */ +static int +try_compression (const char *data, + size_t oldSize, + char **result, + size_t *newSize) { char *tmp; uLongf dlen; @@ -364,62 +660,40 @@ tryCompression (char *data, unsigned int oldSize) { if (dlen < oldSize) { - memcpy (data, tmp, dlen); - GNUNET_free (tmp); - return dlen; + *result = tmp; + *newSize = dlen; + return GNUNET_YES; } } GNUNET_free (tmp); - return oldSize; + return GNUNET_NO; } -/** - * Decompress input, return the decompressed data - * as output, set outputSize to the number of bytes - * that were found. - * - * @return NULL on error - */ -static char * -decompress (const char *input, - unsigned int inputSize, unsigned int outputSize) -{ - char *output; - uLongf olen; - - olen = outputSize; - output = GNUNET_malloc (olen); - if (Z_OK == uncompress ((Bytef *) output, - &olen, (const Bytef *) input, inputSize)) - { - return output; - } - else - { - GNUNET_free (output); - return NULL; - } -} /** * Flag in 'version' that indicates compressed meta-data. */ #define HEADER_COMPRESSED 0x80000000 + /** * Bits in 'version' that give the version number. */ #define HEADER_VERSION_MASK 0x7FFFFFFF + +/** + * Header for serialized meta data. + */ struct MetaDataHeader { /** - * The version of the MD serialization. - * The highest bit is used to indicate - * compression. + * The version of the MD serialization. The highest bit is used to + * indicate compression. * - * Version 0 is the current version; - * Version is 1 for a NULL pointer. + * Version 0 is traditional (pre-0.9) meta data (unsupported) + * Version is 1 for a NULL pointer + * Version 2 is for 0.9.x (and possibly higher) * Other version numbers are not yet defined. */ uint32_t version; @@ -430,24 +704,57 @@ struct MetaDataHeader uint32_t entries; /** - * Size of the MD (decompressed) + * Size of the decompressed meta data. */ uint32_t size; /** - * This is followed by 'entries' values of type 'uint32_t' that - * correspond to EXTRACTOR_KeywordTypes. After that, the meta-data - * keywords follow (0-terminated). The MD block always ends with - * 0-termination, padding with 0 until a multiple of 8 bytes. + * This is followed by 'entries' values of type 'struct MetaDataEntry' + * and then by 'entry' plugin names, mime-types and data blocks + * as specified in those meta data entries. + */ +}; + + +/** + * Entry of serialized meta data. + */ +struct MetaDataEntry +{ + /** + * Meta data type. Corresponds to an 'enum EXTRACTOR_MetaType' + */ + uint32_t type; + + /** + * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat' */ + uint32_t format; + + /** + * Number of bytes of meta data. + */ + uint32_t data_size; + + /** + * Number of bytes in the plugin name including 0-terminator. 0 for NULL. + */ + uint32_t plugin_name_len; + + /** + * Number of bytes in the mime type including 0-terminator. 0 for NULL. + */ + uint32_t mime_type_len; }; + /** * Serialize meta-data to target. * * @param md metadata to serialize - * @param target where to write the serialized metadata + * @param target where to write the serialized metadata; + * *target can be NULL, in which case memory is allocated * @param max maximum number of bytes available in target * @param opt is it ok to just write SOME of the * meta-data to match the size constraint, @@ -458,149 +765,273 @@ struct MetaDataHeader */ ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData - *md, char *target, size_t max, + *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt) { - struct MetaDataHeader *hdr; + struct GNUNET_CONTAINER_MetaData *vmd; + struct MetaItem *pos; + struct MetaDataHeader *hdr; + struct MetaDataEntry *ent; + unsigned int i; + uint64_t msize; + size_t off; + char *mdata; + char *cdata; + size_t mlen; + size_t plen; size_t size; - size_t pos; - uint32_t i; - size_t len; - uint32_t ic; + size_t left; + size_t clen; + int comp; if (max < sizeof (struct MetaDataHeader)) return GNUNET_SYSERR; /* far too small */ - ic = md ? md->itemCount : 0; - hdr = NULL; - while (1) + if (md == NULL) + return 0; + + if (md->sbuf != NULL) { - size = sizeof (struct MetaDataHeader); - size += sizeof (uint32_t) * ic; - for (i = 0; i < ic; i++) - size += 1 + strlen (md->items[i].data); - while (size % 8 != 0) - size++; - hdr = GNUNET_malloc (size); - hdr->version = htonl (md == NULL ? 1 : 0); - hdr->entries = htonl (ic); - for (i = 0; i < ic; i++) - ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); - pos = sizeof (struct MetaDataHeader); - pos += sizeof (uint32_t) * ic; - for (i = 0; i < ic; i++) - { - len = strlen (md->items[i].data) + 1; - memcpy (&((char *) hdr)[pos], md->items[i].data, len); - pos += len; - } + /* try to use serialization cache */ + if (md->sbuf_size < max) + { + if (NULL == *target) + *target = GNUNET_malloc (md->sbuf_size); + memcpy (*target, + md->sbuf, + md->sbuf_size); + return md->sbuf_size; + } + if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART)) + return GNUNET_SYSERR; /* can say that this will fail */ + /* need to compute a partial serialization, sbuf useless ... */ + } - hdr->size = htonl (size); - if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) - { - pos = tryCompression ((char *) &hdr[1], - size - sizeof (struct MetaDataHeader)); - } + + msize = 0; + pos = md->items; + while (NULL != pos) + { + msize += sizeof (struct MetaDataEntry); + msize += pos->data_size; + if (pos->plugin_name != NULL) + msize += strlen (pos->plugin_name) + 1; + if (pos->mime_type != NULL) + msize += strlen (pos->mime_type) + 1; + pos = pos->next; + } + size = (size_t) msize; + if (size != msize) + { + GNUNET_break (0); /* integer overflow */ + return GNUNET_SYSERR; + } + if (size >= GNUNET_MAX_MALLOC_CHECKED) + { + /* too large to be processed */ + return GNUNET_SYSERR; + } + ent = GNUNET_malloc (size); + mdata = (char *) &ent[md->item_count]; + off = size - (md->item_count * sizeof(struct MetaDataEntry)); + i = 0; + pos = md->items; + while (NULL != pos) + { + ent[i].type = htonl ((uint32_t) pos->type); + ent[i].format = htonl ((uint32_t) pos->format); + ent[i].data_size = htonl ((uint32_t) pos->data_size); + if (pos->plugin_name == NULL) + plen = 0; else - { - pos = size - sizeof (struct MetaDataHeader); - } - if (pos < size - sizeof (struct MetaDataHeader)) - { - hdr->version = htonl (HEADER_COMPRESSED); - size = pos + sizeof (struct MetaDataHeader); - } - if (size <= max) - break; - GNUNET_free (hdr); - hdr = NULL; + plen = strlen (pos->plugin_name) + 1; + ent[i].plugin_name_len = htonl ( (uint32_t) plen); + if (pos->mime_type == NULL) + mlen = 0; + else + mlen = strlen (pos->mime_type) + 1; + ent[i].mime_type_len = htonl ((uint32_t) mlen); + off -= pos->data_size; + memcpy (&mdata[off], pos->data, pos->data_size); + off -= plen; + memcpy (&mdata[off], pos->plugin_name, plen); + off -= mlen; + memcpy (&mdata[off], pos->mime_type, mlen); + i++; + pos = pos->next; + } + GNUNET_assert (off == 0); - if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART) == 0) - { - return GNUNET_SYSERR; /* does not fit! */ + left = size; + for (i=0;iitem_count;i++) + { + comp = GNUNET_NO; + if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS)) + comp = try_compression ((const char*) &ent[i], + left, + &cdata, + &clen); + + if ( (md->sbuf == NULL) && + (i == 0) ) + { + /* fill 'sbuf'; this "modifies" md, but since this is only + an internal cache we will cast away the 'const' instead + of making the API look strange. */ + vmd = (struct GNUNET_CONTAINER_MetaData*) md; + hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader)); + hdr->entries = htonl (md->item_count); + if (GNUNET_YES == comp) + { + hdr->size = htonl (clen); + hdr->version = htonl (2 | HEADER_COMPRESSED); + memcpy (&hdr[1], + cdata, + clen); + vmd->sbuf_size = clen + sizeof (struct MetaDataHeader); + } + else + { + hdr->size = htonl (left); + hdr->version = htonl (2); + memcpy (&hdr[1], + &ent[0], + left); + vmd->sbuf_size = left + sizeof (struct MetaDataHeader); + } + vmd->sbuf = (char*) hdr; + } + + if ( ( (left + sizeof (struct MetaDataHeader)) <= max) || + ( (comp == GNUNET_YES) && + (clen <= max)) ) + { + /* success, this now fits! */ + if (GNUNET_YES == comp) + { + hdr = (struct MetaDataHeader*) *target; + if (hdr == NULL) + { + hdr = GNUNET_malloc (clen + sizeof (struct MetaDataHeader)); + *target = (char*) hdr; + } + hdr->version = htonl (2 | HEADER_COMPRESSED); + hdr->entries = htonl (md->item_count - i); + hdr->size = htonl (left); + memcpy (&target[sizeof(struct MetaDataHeader)], + cdata, + clen); + GNUNET_free (cdata); + GNUNET_free (ent); + return clen + sizeof (struct MetaDataHeader); + } + else + { + hdr = (struct MetaDataHeader*) target; + if (hdr == NULL) + { + hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader)); + *target = (char*) hdr; + } + hdr->version = htonl (2); + hdr->entries = htonl (md->item_count - i); + hdr->size = htonl (left); + memcpy (&target[sizeof(struct MetaDataHeader)], + &ent[i], + left); + GNUNET_free (ent); + return left + sizeof (struct MetaDataHeader); + } + } + + if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART)) + { + /* does not fit! */ + GNUNET_free (ent); + return GNUNET_SYSERR; } - /* partial serialization ok, try again with less meta-data */ - if (size > 2 * max) - ic = ic * 2 / 3; /* still far too big, make big reductions */ - else - ic--; /* small steps, we're close */ + + /* next iteration: ignore the corresponding meta data at the + end and try again without it */ + left -= sizeof (struct MetaDataEntry); + left -= pos->data_size; + if (pos->plugin_name != NULL) + left -= strlen (pos->plugin_name) + 1; + if (pos->mime_type != NULL) + left -= strlen (pos->mime_type) + 1; } - GNUNET_assert (size <= max); - memcpy (target, hdr, size); - GNUNET_free (hdr); - /* extra check: deserialize! */ -#if EXTRA_CHECKS - { - struct GNUNET_CONTAINER_MetaData *mdx; - mdx = GNUNET_CONTAINER_meta_data_deserialize (target, size); - GNUNET_assert (NULL != mdx); - GNUNET_CONTAINER_meta_data_destroy (mdx); - } -#endif - return size; + GNUNET_free (ent); + + /* nothing fit, only write header! */ + hdr = (struct MetaDataHeader*) target; + if (hdr == NULL) + { + hdr = GNUNET_malloc (sizeof (struct MetaDataHeader)); + *target = (char*) hdr; + } + hdr->version = htonl (2); + hdr->entries = htonl (0); + hdr->size = htonl (0); + return sizeof (struct MetaDataHeader); } + /** - * Estimate (!) the size of the meta-data in - * serialized form. The estimate MAY be higher - * than what is strictly needed. + * Get the size of the full meta-data in serialized form. * * @param md metadata to inspect - * @param opt is it ok to just write SOME of the - * meta-data to match the size constraint, - * possibly discarding some data? * @return number of bytes needed for serialization, -1 on error */ ssize_t -GNUNET_CONTAINER_meta_data_get_serialized_size (const struct - GNUNET_CONTAINER_MetaData * - md, - enum - GNUNET_CONTAINER_MetaDataSerializationOptions - opt) +GNUNET_CONTAINER_meta_data_get_serialized_size (const struct GNUNET_CONTAINER_MetaData *md) { - struct MetaDataHeader *hdr; - size_t size; - size_t pos; - uint32_t i; - size_t len; - uint32_t ic; + ssize_t ret; + char *ptr; + + if (md->sbuf != NULL) + return md->sbuf_size; + ptr = NULL; + ret = GNUNET_CONTAINER_meta_data_serialize (md, + &ptr, + GNUNET_MAX_MALLOC_CHECKED, + GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); + if (ret != -1) + GNUNET_free (ptr); + return ret; +} - ic = md ? md->itemCount : 0; - size = sizeof (struct MetaDataHeader); - size += sizeof (uint32_t) * ic; - for (i = 0; i < ic; i++) - size += 1 + strlen (md->items[i].data); - while (size % 8 != 0) - size++; - hdr = GNUNET_malloc (size); - hdr->version = htonl (md == NULL ? 1 : 0); - hdr->entries = htonl (ic); - for (i = 0; i < ic; i++) - ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); - pos = sizeof (struct MetaDataHeader); - pos += sizeof (uint32_t) * ic; - for (i = 0; i < ic; i++) - { - len = strlen (md->items[i].data) + 1; - memcpy (&((char *) hdr)[pos], md->items[i].data, len); - pos += len; - } - if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) + +/** + * Decompress input, return the decompressed data + * as output, set outputSize to the number of bytes + * that were found. + * + * @param input compressed data + * @param inputSize number of bytes in input + * @param outputSize expected size of the output + * @return NULL on error + */ +static char * +decompress (const char *input, + size_t inputSize, + size_t outputSize) +{ + char *output; + uLongf olen; + + olen = outputSize; + output = GNUNET_malloc (olen); + if (Z_OK == uncompress ((Bytef *) output, + &olen, (const Bytef *) input, inputSize)) { - pos = - tryCompression ((char *) &hdr[1], - size - sizeof (struct MetaDataHeader)); + return output; } else { - pos = size - sizeof (struct MetaDataHeader); + GNUNET_free (output); + return NULL; } - if (pos < size - sizeof (struct MetaDataHeader)) - size = pos + sizeof (struct MetaDataHeader); - GNUNET_free (hdr); - return size; } @@ -616,41 +1047,57 @@ struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) { struct GNUNET_CONTAINER_MetaData *md; - const struct MetaDataHeader *hdr; + struct MetaDataHeader hdr; + struct MetaDataEntry ent; uint32_t ic; + uint32_t i; char *data; const char *cdata; + uint32_t version; uint32_t dataSize; int compressed; - uint32_t i; - size_t pos; - size_t len; - uint32_t version; + size_t left; + uint32_t mlen; + uint32_t plen; + uint32_t dlen; + const char *mdata; + const char *meta_data; + const char *plugin_name; + const char *mime_type; + enum EXTRACTOR_MetaFormat format; if (size < sizeof (struct MetaDataHeader)) return NULL; - hdr = (const struct MetaDataHeader *) input; - version = ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_VERSION_MASK; + memcpy (&hdr, + input, + sizeof (struct MetaDataHeader)); + version = ntohl (hdr.version) & HEADER_VERSION_MASK; + compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0; + if (version == 1) - return NULL; /* null pointer */ - if (version != 0) + return NULL; /* null pointer */ + if (version != 2) { GNUNET_break_op (0); /* unsupported version */ return NULL; } - ic = ntohl (MAKE_UNALIGNED (hdr->entries)); - compressed = - (ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_COMPRESSED) != 0; + + ic = ntohl (hdr.entries); + dataSize = ntohl (hdr.size); + if ((sizeof (struct MetaDataEntry) * ic) > dataSize) + { + GNUNET_break_op (0); + return NULL; + } + if (compressed) { - dataSize = - ntohl (MAKE_UNALIGNED (hdr->size)) - sizeof (struct MetaDataHeader); - if (dataSize > 2 * 1042 * 1024) + if (dataSize >= GNUNET_MAX_MALLOC_CHECKED) { - GNUNET_break (0); - return NULL; /* only 2 MB allowed [to make sure we don't blow - our memory limit because of a mal-formed - message... ] */ + /* make sure we don't blow our memory limit because of a mal-formed + message... */ + GNUNET_break_op (0); + return NULL; } data = decompress ((const char *) &input[sizeof (struct MetaDataHeader)], @@ -665,84 +1112,93 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) else { data = NULL; - cdata = (const char *) &hdr[1]; - dataSize = size - sizeof (struct MetaDataHeader); - if (size != ntohl (MAKE_UNALIGNED (hdr->size))) + cdata = (const char *) &input[sizeof (struct MetaDataHeader)]; + if (dataSize != size - sizeof (struct MetaDataHeader)) { - GNUNET_break (0); + GNUNET_break_op (0); return NULL; } } - if ((sizeof (uint32_t) * ic + ic) > dataSize) - { - GNUNET_break (0); - goto FAILURE; - } - if ((ic > 0) && (cdata[dataSize - 1] != '\0')) - { - GNUNET_break (0); - goto FAILURE; - } - md = GNUNET_CONTAINER_meta_data_create (); - GNUNET_array_grow (md->items, md->itemCount, ic); - i = 0; - pos = sizeof (uint32_t) * ic; - while ((pos < dataSize) && (i < ic)) + left = dataSize - ic * sizeof (struct MetaDataEntry); + mdata = &cdata[ic * sizeof (struct MetaDataEntry)]; + for (i=0;iitems[i].type = (EXTRACTOR_KeywordType) - ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i])); - md->items[i].data = GNUNET_strdup (&cdata[pos]); - pos += len; - i++; - } - if (i < ic) - { /* oops */ - GNUNET_CONTAINER_meta_data_destroy (md); - goto FAILURE; - } - GNUNET_free_non_null (data); - return md; -FAILURE: - GNUNET_free_non_null (data); - return NULL; /* size too small */ -} - -/** - * Test if two MDs are equal. - * - * @param md1 first value to check - * @param md2 other value to check - * @return GNUNET_YES if they are equal - */ -int -GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData - *md1, - const struct GNUNET_CONTAINER_MetaData - *md2) -{ - uint32_t i; - uint32_t j; - int found; + memcpy (&ent, + &cdata[i * sizeof(struct MetaDataEntry)], + sizeof (struct MetaDataEntry)); + format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format); + if ( (format != EXTRACTOR_METAFORMAT_UTF8) && + (format != EXTRACTOR_METAFORMAT_C_STRING) && + (format != EXTRACTOR_METAFORMAT_BINARY) ) + { + GNUNET_break_op (0); + break; + } + dlen = ntohl (ent.data_size); + plen = ntohl (ent.plugin_name_len); + mlen = ntohl (ent.mime_type_len); + if (dlen > left) + { + GNUNET_break_op (0); + break; + } + left -= dlen; + meta_data = &mdata[left]; + if ( (format == EXTRACTOR_METAFORMAT_UTF8) || + (format == EXTRACTOR_METAFORMAT_C_STRING) ) + { + if ( (dlen == 0) || + (mdata[left + dlen - 1] != '\0') ) + { + GNUNET_break_op (0); + break; + } + } + if (plen > left) + { + GNUNET_break_op (0); + break; + } + left -= plen; + if ( (plen > 0) && + (mdata[left + plen - 1] != '\0') ) + { + GNUNET_break_op (0); + break; + } + if (plen == 0) + plugin_name = NULL; + else + plugin_name = &mdata[left]; - if (md1->itemCount != md2->itemCount) - return GNUNET_NO; - for (i = 0; i < md1->itemCount; i++) - { - found = GNUNET_NO; - for (j = 0; j < md2->itemCount; j++) - if ((md1->items[i].type == md2->items[j].type) && - (0 == strcmp (md1->items[i].data, md2->items[j].data))) - { - found = GNUNET_YES; - break; - } - if (found == GNUNET_NO) - return GNUNET_NO; + if (mlen > left) + { + GNUNET_break_op (0); + break; + } + left -= mlen; + if ( (mlen > 0) && + (mdata[left + mlen - 1] != '\0') ) + { + GNUNET_break_op (0); + break; + } + if (mlen == 0) + mime_type = NULL; + else + mime_type = &mdata[left]; + GNUNET_CONTAINER_meta_data_insert (md, + plugin_name, + (enum EXTRACTOR_MetaType) ntohl (ent.type), + format, + mime_type, + meta_data, + dlen); } - return GNUNET_YES; + GNUNET_free_non_null (data); + return md; } diff --git a/src/util/pseudonym.c b/src/util/pseudonym.c index c974cc5a3..626e8ca99 100644 --- a/src/util/pseudonym.c +++ b/src/util/pseudonym.c @@ -300,15 +300,14 @@ GNUNET_PSEUDONYM_id_to_name (const struct GNUNET_CONFIGURATION_Handle *cfg, { if ((meta != NULL) && (name == NULL)) name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta, - EXTRACTOR_TITLE, - EXTRACTOR_FILENAME, - EXTRACTOR_DESCRIPTION, - EXTRACTOR_SUBJECT, - EXTRACTOR_PUBLISHER, - EXTRACTOR_AUTHOR, - EXTRACTOR_COMMENT, - EXTRACTOR_SUMMARY, - EXTRACTOR_OWNER, + EXTRACTOR_METATYPE_TITLE, + EXTRACTOR_METATYPE_FILENAME, + EXTRACTOR_METATYPE_DESCRIPTION, + EXTRACTOR_METATYPE_SUBJECT, + EXTRACTOR_METATYPE_PUBLISHER, + EXTRACTOR_METATYPE_AUTHOR_NAME, + EXTRACTOR_METATYPE_COMMENT, + EXTRACTOR_METATYPE_SUMMARY, -1); if (meta != NULL) { @@ -545,11 +544,21 @@ GNUNET_PSEUDONYM_rank (const struct GNUNET_CONFIGURATION_Handle *cfg, * @param data value of entry to insert */ static int -merge_meta_helper (void *cls, EXTRACTOR_KeywordType type, const char *data) +merge_meta_helper (void *cls, + const char *plugin_name, + enum EXTRACTOR_MetaType type, + enum EXTRACTOR_MetaFormat format, + const char *data_mime_type, + const char *data, + size_t data_len) { struct GNUNET_CONTAINER_MetaData *meta = cls; - GNUNET_CONTAINER_meta_data_insert (meta, type, data); - return GNUNET_OK; + + (void) GNUNET_CONTAINER_meta_data_insert (meta, plugin_name, + type, format, + data_mime_type, + data, data_len); + return 0; } @@ -581,7 +590,7 @@ GNUNET_PSEUDONYM_add (const struct GNUNET_CONFIGURATION_Handle *cfg, if ((0 == STAT (fn, &sbuf)) && (GNUNET_OK == read_info (cfg, id, &old, &ranking, &name))) { - GNUNET_CONTAINER_meta_data_get_contents (meta, &merge_meta_helper, old); + GNUNET_CONTAINER_meta_data_iterate (meta, &merge_meta_helper, old); write_pseudonym_info (cfg, id, old, ranking, name); GNUNET_CONTAINER_meta_data_destroy (old); GNUNET_free_non_null (name); -- cgit v1.2.3