diff options
-rwxr-xr-x | bootstrap | 5 | ||||
-rw-r--r-- | configure.ac | 11 | ||||
-rw-r--r-- | m4/libltdl-external.m4 | 75 | ||||
-rw-r--r-- | src/fs/fs_directory.c | 49 | ||||
-rw-r--r-- | src/fs/fs_file_information.c | 4 | ||||
-rw-r--r-- | src/fs/fs_getopt.c | 31 | ||||
-rw-r--r-- | src/fs/fs_publish.c | 14 | ||||
-rw-r--r-- | src/fs/fs_uri.c | 31 | ||||
-rw-r--r-- | src/fs/gnunet-directory.c | 21 | ||||
-rw-r--r-- | src/fs/gnunet-publish.c | 66 | ||||
-rw-r--r-- | src/fs/gnunet-search.c | 21 | ||||
-rw-r--r-- | src/include/gnunet_container_lib.h | 162 | ||||
-rw-r--r-- | src/include/gnunet_directories.h | 34 | ||||
-rw-r--r-- | src/util/bio.c | 18 | ||||
-rw-r--r-- | src/util/container_meta_data.c | 1214 | ||||
-rw-r--r-- | src/util/pseudonym.c | 35 |
16 files changed, 1109 insertions, 682 deletions
@@ -1,2 +1,5 @@ | |||
1 | #!/bin/sh | 1 | #!/bin/sh |
2 | autoreconf -fiv -I m4 | 2 | libtoolize --automake --copy --force |
3 | aclocal | ||
4 | automake | ||
5 | autoreconf -f -i | ||
diff --git a/configure.ac b/configure.ac index 4a87c0067..41157ca29 100644 --- a/configure.ac +++ b/configure.ac | |||
@@ -1,5 +1,5 @@ | |||
1 | # This file is part of GNUnet. | 1 | # This file is part of GNUnet. |
2 | # (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) | 2 | # (C) 2001--2010 Christian Grothoff (and other contributing authors) |
3 | # | 3 | # |
4 | # GNUnet is free software; you can redistribute it and/or modify | 4 | # GNUnet is free software; you can redistribute it and/or modify |
5 | # it under the terms of the GNU General Public License as published | 5 | # it under the terms of the GNU General Public License as published |
@@ -38,7 +38,6 @@ AC_PROG_INSTALL | |||
38 | AC_PROG_LN_S | 38 | AC_PROG_LN_S |
39 | AC_PROG_MAKE_SET | 39 | AC_PROG_MAKE_SET |
40 | AM_PROG_CC_C_O | 40 | AM_PROG_CC_C_O |
41 | LT_CONFIG_LTDL_DIR([libltdl]) | ||
42 | AC_CANONICAL_HOST | 41 | AC_CANONICAL_HOST |
43 | LT_INIT([disable-static dlopen win32-dll]) | 42 | LT_INIT([disable-static dlopen win32-dll]) |
44 | LTDL_INIT | 43 | LTDL_INIT |
@@ -141,7 +140,7 @@ netbsd*) | |||
141 | AC_CHECK_LIB(intl, gettext) | 140 | AC_CHECK_LIB(intl, gettext) |
142 | LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols" | 141 | LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols" |
143 | LIBS="$LIBS -lws2_32 -lplibc" | 142 | LIBS="$LIBS -lws2_32 -lplibc" |
144 | CFLAGS="-mms-bitfields -I../../libltdl $CFLAGS" | 143 | CFLAGS="-mms-bitfields $CFLAGS" |
145 | build_target="mingw" | 144 | build_target="mingw" |
146 | AC_PROG_CXX | 145 | AC_PROG_CXX |
147 | LIBPREFIX=lib | 146 | LIBPREFIX=lib |
@@ -212,14 +211,14 @@ AC_ARG_WITH(extractor, | |||
212 | ;; | 211 | ;; |
213 | yes) | 212 | yes) |
214 | AC_CHECK_HEADERS(extractor.h, | 213 | AC_CHECK_HEADERS(extractor.h, |
215 | AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], | 214 | AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], |
216 | extractor=1)) | 215 | extractor=1)) |
217 | ;; | 216 | ;; |
218 | *) | 217 | *) |
219 | LDFLAGS="-L$with_extractor/lib $LDFLAGS" | 218 | LDFLAGS="-L$with_extractor/lib $LDFLAGS" |
220 | CPPFLAGS="-I$with_extractor/include $CPPFLAGS" | 219 | CPPFLAGS="-I$with_extractor/include $CPPFLAGS" |
221 | AC_CHECK_HEADERS(extractor.h, | 220 | AC_CHECK_HEADERS(extractor.h, |
222 | AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], | 221 | AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], |
223 | EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH" | 222 | EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH" |
224 | extractor=1)) | 223 | extractor=1)) |
225 | ;; | 224 | ;; |
@@ -227,7 +226,7 @@ AC_ARG_WITH(extractor, | |||
227 | ], | 226 | ], |
228 | [AC_MSG_RESULT([--with-extractor not specified]) | 227 | [AC_MSG_RESULT([--with-extractor not specified]) |
229 | AC_CHECK_HEADERS(extractor.h, | 228 | AC_CHECK_HEADERS(extractor.h, |
230 | AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], | 229 | AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults], |
231 | extractor=1))]) | 230 | extractor=1))]) |
232 | if test "$extractor" != 1 | 231 | if test "$extractor" != 1 |
233 | then | 232 | then |
diff --git a/m4/libltdl-external.m4 b/m4/libltdl-external.m4 deleted file mode 100644 index 1c4ce09b2..000000000 --- a/m4/libltdl-external.m4 +++ /dev/null | |||
@@ -1,75 +0,0 @@ | |||
1 | dnl Autoconf macro for an always external libltdl | ||
2 | dnl Copyright (C) 2009 Heikki Lindholm | ||
3 | dnl | ||
4 | dnl This file is free software; as a special exception the author gives | ||
5 | dnl unlimited permission to copy and/or distribute it, with or without | ||
6 | dnl modifications, as long as this notice is preserved. | ||
7 | dnl | ||
8 | dnl This file is distributed in the hope that it will be useful, but | ||
9 | dnl WITHOUT ANY WARRANTY, to the extent permitted by law; without even the | ||
10 | dnl implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | ||
11 | |||
12 | dnl AM_PATH_LIBLTDL( | ||
13 | dnl [CHECK-SYMBOLS, [ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND ]]]) | ||
14 | dnl | ||
15 | AC_DEFUN([AM_PATH_LIBLTDL], | ||
16 | [ AC_ARG_WITH(libltdl-prefix, | ||
17 | AC_HELP_STRING([--with-libltdl-prefix=PFX], | ||
18 | [prefix where libltdl is installed (optional)]), | ||
19 | libltdl_prefix="$withval", libltdl_prefix="") | ||
20 | ltdl_save_CPPFLAGS="$CPPFLAGS" | ||
21 | ltdl_save_LDFLAGS="$LDFLAGS" | ||
22 | if test x$libltdl_prefix != x ; then | ||
23 | CPPFLAGS="-I$libltdl_prefix/include $CPPFLAGS" | ||
24 | LDFLAGS="-L$libltdl_prefix/lib -lltdl $LDFLAGS" | ||
25 | else | ||
26 | if test x"$LIBLTDL" = x ; then | ||
27 | LIBLTDL="-lltdl" | ||
28 | fi | ||
29 | CPPFLAGS="$LTDLINCL $CPPFLAGS" | ||
30 | LDFLAGS="$LIBLTDL $LDFLAGS" | ||
31 | fi | ||
32 | |||
33 | symbols_to_check=ifelse([$1], ,"ltdl_dlopen","$1") | ||
34 | ltdl_found=yes | ||
35 | AC_CHECK_HEADER([ltdl.h], | ||
36 | [ | ||
37 | for sym in $symbols_to_check | ||
38 | do | ||
39 | AC_CHECK_DECL([$sym], | ||
40 | [AC_LINK_IFELSE(AC_LANG_CALL([], [$sym]), | ||
41 | [ltdl_found=yes], | ||
42 | [ltdl_found=no])], | ||
43 | [ltdl_found=no], | ||
44 | [AC_INCLUDES_DEFAULT | ||
45 | #include <ltdl.h>]) | ||
46 | done | ||
47 | ], | ||
48 | [ltdl_found=no], | ||
49 | [AC_INCLUDES_DEFAULT] | ||
50 | ) | ||
51 | |||
52 | if test x$libltdl_prefix != x ; then | ||
53 | LTDLINCL="-I$libltdl_prefix/include" | ||
54 | LIBLTDL="-L$libltdl_prefix/lib -lltdl" | ||
55 | else | ||
56 | if test x"$LIBLTDL" = x ; then | ||
57 | LIBLTDL="-lltdl" | ||
58 | fi | ||
59 | fi | ||
60 | CPPFLAGS="$ltdl_save_CPPFLAGS" | ||
61 | LDFLAGS="$ltdl_save_LDFLAGS" | ||
62 | |||
63 | AC_MSG_CHECKING(for libltdl with symbols $symbols_to_check) | ||
64 | if test $ltdl_found = yes; then | ||
65 | AC_MSG_RESULT(yes) | ||
66 | ifelse([$2], , :, [$2]) | ||
67 | else | ||
68 | LTDLINCL="" | ||
69 | LIBLTDL="" | ||
70 | AC_MSG_RESULT(no) | ||
71 | ifelse([$3], , :, [$3]) | ||
72 | fi | ||
73 | AC_SUBST(LTDLINCL) | ||
74 | AC_SUBST(LIBLTDL) | ||
75 | ]) | ||
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c index e037130c8..48578c650 100644 --- a/src/fs/fs_directory.c +++ b/src/fs/fs_directory.c | |||
@@ -36,10 +36,6 @@ | |||
36 | #include "gnunet_fs_service.h" | 36 | #include "gnunet_fs_service.h" |
37 | #include "fs.h" | 37 | #include "fs.h" |
38 | 38 | ||
39 | #ifndef EXTRACTOR_GNUNET_FULL_DATA | ||
40 | #define EXTRACTOR_GNUNET_FULL_DATA 137 | ||
41 | #endif | ||
42 | |||
43 | /** | 39 | /** |
44 | * String that is used to indicate that a file | 40 | * String that is used to indicate that a file |
45 | * is a GNUnet directory. | 41 | * is a GNUnet directory. |
@@ -60,7 +56,7 @@ GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData * | |||
60 | char *mime; | 56 | char *mime; |
61 | int ret; | 57 | int ret; |
62 | 58 | ||
63 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); | 59 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); |
64 | if (mime == NULL) | 60 | if (mime == NULL) |
65 | return GNUNET_SYSERR; | 61 | return GNUNET_SYSERR; |
66 | ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO; | 62 | ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO; |
@@ -80,7 +76,7 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md) | |||
80 | { | 76 | { |
81 | char *mime; | 77 | char *mime; |
82 | 78 | ||
83 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); | 79 | mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE); |
84 | if (mime != NULL) | 80 | if (mime != NULL) |
85 | { | 81 | { |
86 | GNUNET_break (0 == strcmp (mime, | 82 | GNUNET_break (0 == strcmp (mime, |
@@ -89,8 +85,12 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md) | |||
89 | return; | 85 | return; |
90 | } | 86 | } |
91 | GNUNET_CONTAINER_meta_data_insert (md, | 87 | GNUNET_CONTAINER_meta_data_insert (md, |
92 | EXTRACTOR_MIMETYPE, | 88 | "<gnunet>", |
93 | GNUNET_FS_DIRECTORY_MIME); | 89 | EXTRACTOR_METATYPE_MIMETYPE, |
90 | EXTRACTOR_METAFORMAT_UTF8, | ||
91 | "text/plain", | ||
92 | GNUNET_FS_DIRECTORY_MIME, | ||
93 | strlen (GNUNET_FS_DIRECTORY_MIME)+1); | ||
94 | } | 94 | } |
95 | 95 | ||
96 | 96 | ||
@@ -225,9 +225,9 @@ GNUNET_FS_directory_list_contents (size_t size, | |||
225 | } | 225 | } |
226 | pos += mdSize; | 226 | pos += mdSize; |
227 | filename = GNUNET_CONTAINER_meta_data_get_by_type (md, | 227 | filename = GNUNET_CONTAINER_meta_data_get_by_type (md, |
228 | EXTRACTOR_FILENAME); | 228 | EXTRACTOR_METATYPE_FILENAME); |
229 | file_data = GNUNET_CONTAINER_meta_data_get_by_type (md, | 229 | file_data = GNUNET_CONTAINER_meta_data_get_by_type (md, |
230 | EXTRACTOR_GNUNET_FULL_DATA); | 230 | EXTRACTOR_METATYPE_GNUNET_FULL_DATA); |
231 | if (dep != NULL) | 231 | if (dep != NULL) |
232 | dep (dep_cls, | 232 | dep (dep_cls, |
233 | filename, | 233 | filename, |
@@ -322,6 +322,7 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, | |||
322 | size_t mdxs; | 322 | size_t mdxs; |
323 | char *uris; | 323 | char *uris; |
324 | char *ser; | 324 | char *ser; |
325 | char *sptr; | ||
325 | size_t slen; | 326 | size_t slen; |
326 | struct GNUNET_CONTAINER_MetaData *meta; | 327 | struct GNUNET_CONTAINER_MetaData *meta; |
327 | const struct GNUNET_CONTAINER_MetaData *meta_use; | 328 | const struct GNUNET_CONTAINER_MetaData *meta_use; |
@@ -340,25 +341,24 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, | |||
340 | fsize = 0; /* not given */ | 341 | fsize = 0; /* not given */ |
341 | if (fsize > MAX_INLINE_SIZE) | 342 | if (fsize > MAX_INLINE_SIZE) |
342 | fsize = 0; /* too large */ | 343 | fsize = 0; /* too large */ |
343 | if ( (NULL == data) || | ||
344 | (NULL != memchr (data, 0, fsize)) ) | ||
345 | fsize = 0; /* must not have 0's in data! */ | ||
346 | uris = GNUNET_FS_uri_to_string (uri); | 344 | uris = GNUNET_FS_uri_to_string (uri); |
347 | slen = strlen (uris) + 1; | 345 | slen = strlen (uris) + 1; |
348 | mds = | 346 | mds = |
349 | GNUNET_CONTAINER_meta_data_get_serialized_size (md, | 347 | GNUNET_CONTAINER_meta_data_get_serialized_size (md); |
350 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
351 | meta_use = md; | 348 | meta_use = md; |
352 | meta = NULL; | 349 | meta = NULL; |
353 | if (fsize > 0) | 350 | if (fsize > 0) |
354 | { | 351 | { |
355 | meta = GNUNET_CONTAINER_meta_data_duplicate (md); | 352 | meta = GNUNET_CONTAINER_meta_data_duplicate (md); |
356 | GNUNET_CONTAINER_meta_data_insert (meta, | 353 | GNUNET_CONTAINER_meta_data_insert (meta, |
357 | EXTRACTOR_GNUNET_FULL_DATA, | 354 | "<gnunet>", |
358 | data); | 355 | EXTRACTOR_METATYPE_GNUNET_FULL_DATA, |
356 | EXTRACTOR_METAFORMAT_BINARY, | ||
357 | NULL, | ||
358 | data, | ||
359 | fsize); | ||
359 | mdxs = | 360 | mdxs = |
360 | GNUNET_CONTAINER_meta_data_get_serialized_size (meta, | 361 | GNUNET_CONTAINER_meta_data_get_serialized_size (meta); |
361 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
362 | if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE == | 362 | if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE == |
363 | (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE) | 363 | (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE) |
364 | { | 364 | { |
@@ -376,8 +376,9 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld, | |||
376 | ser = (char*) &e[1]; | 376 | ser = (char*) &e[1]; |
377 | memcpy (ser, uris, slen); | 377 | memcpy (ser, uris, slen); |
378 | GNUNET_free (uris); | 378 | GNUNET_free (uris); |
379 | sptr = &ser[slen + sizeof(uint32_t)]; | ||
379 | ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, | 380 | ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, |
380 | &ser[slen + sizeof(uint32_t)], | 381 | &sptr, |
381 | mds, | 382 | mds, |
382 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | 383 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); |
383 | if (NULL != meta) | 384 | if (NULL != meta) |
@@ -503,6 +504,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, | |||
503 | void **rdata) | 504 | void **rdata) |
504 | { | 505 | { |
505 | char *data; | 506 | char *data; |
507 | char *sptr; | ||
506 | size_t *sizes; | 508 | size_t *sizes; |
507 | unsigned int *perm; | 509 | unsigned int *perm; |
508 | unsigned int i; | 510 | unsigned int i; |
@@ -516,8 +518,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, | |||
516 | uint32_t big; | 518 | uint32_t big; |
517 | 519 | ||
518 | size = 8 + sizeof (uint32_t); | 520 | size = 8 + sizeof (uint32_t); |
519 | size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta, | 521 | size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta); |
520 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
521 | sizes = NULL; | 522 | sizes = NULL; |
522 | perm = NULL; | 523 | perm = NULL; |
523 | bes = NULL; | 524 | bes = NULL; |
@@ -560,9 +561,9 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld, | |||
560 | memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); | 561 | memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); |
561 | off = 8; | 562 | off = 8; |
562 | 563 | ||
564 | sptr = &data[off + sizeof (uint32_t)]; | ||
563 | ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta, | 565 | ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta, |
564 | &data[off + | 566 | &sptr, |
565 | sizeof (uint32_t)], | ||
566 | size - off - sizeof (uint32_t), | 567 | size - off - sizeof (uint32_t), |
567 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | 568 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); |
568 | GNUNET_assert (ret != -1); | 569 | GNUNET_assert (ret != -1); |
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c index c28635198..9c193f88b 100644 --- a/src/fs/fs_file_information.c +++ b/src/fs/fs_file_information.c | |||
@@ -364,7 +364,7 @@ struct DirScanCls | |||
364 | /** | 364 | /** |
365 | * Metadata extractors to use. | 365 | * Metadata extractors to use. |
366 | */ | 366 | */ |
367 | struct EXTRACTOR_Extractor *extractors; | 367 | struct EXTRACTOR_PluginList *extractors; |
368 | 368 | ||
369 | /** | 369 | /** |
370 | * Function to call on each directory entry. | 370 | * Function to call on each directory entry. |
@@ -518,7 +518,7 @@ GNUNET_FS_directory_scanner_default (void *cls, | |||
518 | void *proc_cls, | 518 | void *proc_cls, |
519 | char **emsg) | 519 | char **emsg) |
520 | { | 520 | { |
521 | struct EXTRACTOR_Extractor *ex = cls; | 521 | struct EXTRACTOR_PluginList *ex = cls; |
522 | struct DirScanCls dsc; | 522 | struct DirScanCls dsc; |
523 | 523 | ||
524 | dsc.extractors = ex; | 524 | dsc.extractors = ex; |
diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c index a0d232641..f985ed8d0 100644 --- a/src/fs/fs_getopt.c +++ b/src/fs/fs_getopt.c | |||
@@ -131,7 +131,7 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* | |||
131 | 131 | ||
132 | { | 132 | { |
133 | struct GNUNET_CONTAINER_MetaData **mm = scls; | 133 | struct GNUNET_CONTAINER_MetaData **mm = scls; |
134 | EXTRACTOR_KeywordType type; | 134 | enum EXTRACTOR_MetaType type; |
135 | const char *typename; | 135 | const char *typename; |
136 | const char *typename_i18n; | 136 | const char *typename_i18n; |
137 | struct GNUNET_CONTAINER_MetaData *meta; | 137 | struct GNUNET_CONTAINER_MetaData *meta; |
@@ -151,17 +151,23 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* | |||
151 | "utf-8" | 151 | "utf-8" |
152 | #endif | 152 | #endif |
153 | ); | 153 | ); |
154 | type = EXTRACTOR_getHighestKeywordTypeNumber (); | 154 | type = EXTRACTOR_metatype_get_max (); |
155 | while (type > 0) | 155 | while (type > 0) |
156 | { | 156 | { |
157 | type--; | 157 | type--; |
158 | typename = EXTRACTOR_getKeywordTypeAsString (type); | 158 | typename = EXTRACTOR_metatype_to_string (type); |
159 | typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); | 159 | typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); |
160 | if ((strlen (tmp) >= strlen (typename) + 1) && | 160 | if ((strlen (tmp) >= strlen (typename) + 1) && |
161 | (tmp[strlen (typename)] == ':') && | 161 | (tmp[strlen (typename)] == ':') && |
162 | (0 == strncmp (typename, tmp, strlen (typename)))) | 162 | (0 == strncmp (typename, tmp, strlen (typename)))) |
163 | { | 163 | { |
164 | GNUNET_CONTAINER_meta_data_insert (meta, type, &tmp[strlen (typename) + 1]); | 164 | GNUNET_CONTAINER_meta_data_insert (meta, |
165 | "<gnunet>", | ||
166 | type, | ||
167 | EXTRACTOR_METAFORMAT_UTF8, | ||
168 | "text/plain", | ||
169 | &tmp[strlen (typename) + 1], | ||
170 | strlen (&tmp[strlen (typename) + 1])+1); | ||
165 | GNUNET_free (tmp); | 171 | GNUNET_free (tmp); |
166 | tmp = NULL; | 172 | tmp = NULL; |
167 | break; | 173 | break; |
@@ -170,8 +176,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* | |||
170 | (tmp[strlen (typename_i18n)] == ':') && | 176 | (tmp[strlen (typename_i18n)] == ':') && |
171 | (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) | 177 | (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) |
172 | { | 178 | { |
173 | GNUNET_CONTAINER_meta_data_insert (meta, type, | 179 | GNUNET_CONTAINER_meta_data_insert (meta, |
174 | &tmp[strlen (typename_i18n) + 1]); | 180 | "<gnunet>", |
181 | type, | ||
182 | EXTRACTOR_METAFORMAT_UTF8, | ||
183 | "text/plain", | ||
184 | &tmp[strlen (typename_i18n) + 1], | ||
185 | strlen (&tmp[strlen (typename_i18n) + 1]) + 1); | ||
175 | GNUNET_free (tmp); | 186 | GNUNET_free (tmp); |
176 | tmp = NULL; | 187 | tmp = NULL; |
177 | break; | 188 | break; |
@@ -179,7 +190,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext* | |||
179 | } | 190 | } |
180 | if (tmp != NULL) | 191 | if (tmp != NULL) |
181 | { | 192 | { |
182 | GNUNET_CONTAINER_meta_data_insert (meta, EXTRACTOR_UNKNOWN, tmp); | 193 | GNUNET_CONTAINER_meta_data_insert (meta, |
194 | "<gnunet>", | ||
195 | EXTRACTOR_METATYPE_UNKNOWN, | ||
196 | EXTRACTOR_METAFORMAT_UTF8, | ||
197 | "text/plain", | ||
198 | tmp, | ||
199 | strlen(tmp) + 1); | ||
183 | GNUNET_free (tmp); | 200 | GNUNET_free (tmp); |
184 | printf (_ | 201 | printf (_ |
185 | ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), | 202 | ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), |
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c index 46b342cab..02ddb2a0d 100644 --- a/src/fs/fs_publish.c +++ b/src/fs/fs_publish.c | |||
@@ -842,7 +842,7 @@ do_upload (void *cls, | |||
842 | while (NULL != p->dir) | 842 | while (NULL != p->dir) |
843 | { | 843 | { |
844 | fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, | 844 | fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, |
845 | EXTRACTOR_FILENAME); | 845 | EXTRACTOR_METATYPE_FILENAME); |
846 | p = p->dir; | 846 | p = p->dir; |
847 | GNUNET_asprintf (&p->emsg, | 847 | GNUNET_asprintf (&p->emsg, |
848 | _("Recursive upload failed at `%s'"), | 848 | _("Recursive upload failed at `%s'"), |
@@ -1305,6 +1305,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, | |||
1305 | char *uris; | 1305 | char *uris; |
1306 | size_t size; | 1306 | size_t size; |
1307 | char *kbe; | 1307 | char *kbe; |
1308 | char *sptr; | ||
1308 | 1309 | ||
1309 | pkc = GNUNET_malloc (sizeof (struct PublishKskContext)); | 1310 | pkc = GNUNET_malloc (sizeof (struct PublishKskContext)); |
1310 | pkc->h = h; | 1311 | pkc->h = h; |
@@ -1324,8 +1325,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, | |||
1324 | return; | 1325 | return; |
1325 | } | 1326 | } |
1326 | } | 1327 | } |
1327 | pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, | 1328 | pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); |
1328 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | ||
1329 | GNUNET_assert (pkc->mdsize >= 0); | 1329 | GNUNET_assert (pkc->mdsize >= 0); |
1330 | uris = GNUNET_FS_uri_to_string (uri); | 1330 | uris = GNUNET_FS_uri_to_string (uri); |
1331 | pkc->slen = strlen (uris) + 1; | 1331 | pkc->slen = strlen (uris) + 1; |
@@ -1339,8 +1339,9 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h, | |||
1339 | kbe = (char *) &pkc->kb[1]; | 1339 | kbe = (char *) &pkc->kb[1]; |
1340 | memcpy (kbe, uris, pkc->slen); | 1340 | memcpy (kbe, uris, pkc->slen); |
1341 | GNUNET_free (uris); | 1341 | GNUNET_free (uris); |
1342 | sptr = &kbe[pkc->slen]; | ||
1342 | pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, | 1343 | pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, |
1343 | &kbe[pkc->slen], | 1344 | &sptr, |
1344 | pkc->mdsize, | 1345 | pkc->mdsize, |
1345 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | 1346 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); |
1346 | if (pkc->mdsize == -1) | 1347 | if (pkc->mdsize == -1) |
@@ -1480,8 +1481,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, | |||
1480 | if (update == NULL) | 1481 | if (update == NULL) |
1481 | update = ""; | 1482 | update = ""; |
1482 | nidlen = strlen (update) + 1; | 1483 | nidlen = strlen (update) + 1; |
1483 | mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, | 1484 | mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta); |
1484 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | ||
1485 | 1485 | ||
1486 | size = sizeof (struct SBlock) + slen + nidlen + mdsize; | 1486 | size = sizeof (struct SBlock) + slen + nidlen + mdsize; |
1487 | if (size > MAX_SBLOCK_SIZE) | 1487 | if (size > MAX_SBLOCK_SIZE) |
@@ -1496,7 +1496,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h, | |||
1496 | memcpy (dest, uris, slen); | 1496 | memcpy (dest, uris, slen); |
1497 | dest += slen; | 1497 | dest += slen; |
1498 | mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, | 1498 | mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, |
1499 | dest, | 1499 | &dest, |
1500 | mdsize, | 1500 | mdsize, |
1501 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); | 1501 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); |
1502 | if (mdsize == -1) | 1502 | if (mdsize == -1) |
diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c index 02310a080..16b6e26e9 100644 --- a/src/fs/fs_uri.c +++ b/src/fs/fs_uri.c | |||
@@ -1469,19 +1469,34 @@ GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri) | |||
1469 | * Adds it to the URI. | 1469 | * Adds it to the URI. |
1470 | * | 1470 | * |
1471 | * @param cls URI to update | 1471 | * @param cls URI to update |
1472 | * @param type type of the meta data | 1472 | * @param plugin_name name of the plugin that produced this value; |
1473 | * @param data value of the meta data | 1473 | * special values can be used (i.e. '<zlib>' for zlib being |
1474 | * @return GNUNET_OK (always) | 1474 | * used in the main libextractor library and yielding |
1475 | * meta data). | ||
1476 | * @param type libextractor-type describing the meta data | ||
1477 | * @param format basic format information about data | ||
1478 | * @param data_mime_type mime-type of data (not of the original file); | ||
1479 | * can be NULL (if mime-type is not known) | ||
1480 | * @param data actual meta-data found | ||
1481 | * @param data_len number of bytes in data | ||
1482 | * @return 0 (always) | ||
1475 | */ | 1483 | */ |
1476 | static int | 1484 | static int |
1477 | gather_uri_data (void *cls, | 1485 | gather_uri_data (void *cls, |
1478 | EXTRACTOR_KeywordType type, | 1486 | const char *plugin_name, |
1479 | const char *data) | 1487 | enum EXTRACTOR_MetaType type, |
1488 | enum EXTRACTOR_MetaFormat format, | ||
1489 | const char *data_mime_type, | ||
1490 | const char *data, | ||
1491 | size_t data_len) | ||
1480 | { | 1492 | { |
1481 | struct GNUNET_FS_Uri *uri = cls; | 1493 | struct GNUNET_FS_Uri *uri = cls; |
1482 | char *nkword; | 1494 | char *nkword; |
1483 | int j; | 1495 | int j; |
1484 | 1496 | ||
1497 | if ( (format != EXTRACTOR_METAFORMAT_UTF8) && | ||
1498 | (format != EXTRACTOR_METAFORMAT_C_STRING) ) | ||
1499 | return 0; | ||
1485 | for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--) | 1500 | for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--) |
1486 | if (0 == strcmp (&uri->data.ksk.keywords[j][1], data)) | 1501 | if (0 == strcmp (&uri->data.ksk.keywords[j][1], data)) |
1487 | return GNUNET_OK; | 1502 | return GNUNET_OK; |
@@ -1489,7 +1504,7 @@ gather_uri_data (void *cls, | |||
1489 | strcpy (nkword, " "); /* not mandatory */ | 1504 | strcpy (nkword, " "); /* not mandatory */ |
1490 | strcat (nkword, data); | 1505 | strcat (nkword, data); |
1491 | uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword; | 1506 | uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword; |
1492 | return GNUNET_OK; | 1507 | return 0; |
1493 | } | 1508 | } |
1494 | 1509 | ||
1495 | 1510 | ||
@@ -1514,8 +1529,8 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData | |||
1514 | ret->data.ksk.keywords = NULL; | 1529 | ret->data.ksk.keywords = NULL; |
1515 | ret->data.ksk.keywords | 1530 | ret->data.ksk.keywords |
1516 | = GNUNET_malloc (sizeof (char *) * | 1531 | = GNUNET_malloc (sizeof (char *) * |
1517 | GNUNET_CONTAINER_meta_data_get_contents (md, NULL, NULL)); | 1532 | GNUNET_CONTAINER_meta_data_iterate (md, NULL, NULL)); |
1518 | GNUNET_CONTAINER_meta_data_get_contents (md, &gather_uri_data, ret); | 1533 | GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret); |
1519 | return ret; | 1534 | return ret; |
1520 | 1535 | ||
1521 | } | 1536 | } |
diff --git a/src/fs/gnunet-directory.c b/src/fs/gnunet-directory.c index 6201c5dbb..3ea76985e 100644 --- a/src/fs/gnunet-directory.c +++ b/src/fs/gnunet-directory.c | |||
@@ -36,13 +36,20 @@ static int ret; | |||
36 | */ | 36 | */ |
37 | static int | 37 | static int |
38 | item_printer (void *cls, | 38 | item_printer (void *cls, |
39 | EXTRACTOR_KeywordType type, | 39 | const char *plugin_name, |
40 | const char *data) | 40 | enum EXTRACTOR_MetaType type, |
41 | enum EXTRACTOR_MetaFormat format, | ||
42 | const char *data_mime_type, | ||
43 | const char *data, | ||
44 | size_t data_size) | ||
41 | { | 45 | { |
46 | if ( (format != EXTRACTOR_METAFORMAT_UTF8) && | ||
47 | (format != EXTRACTOR_METAFORMAT_C_STRING) ) | ||
48 | return 0; | ||
42 | printf ("\t%20s: %s\n", | 49 | printf ("\t%20s: %s\n", |
43 | dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, | 50 | dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, |
44 | EXTRACTOR_getKeywordTypeAsString (type)), data); | 51 | EXTRACTOR_metatype_to_string (type)), data); |
45 | return GNUNET_OK; | 52 | return 0; |
46 | } | 53 | } |
47 | 54 | ||
48 | 55 | ||
@@ -75,9 +82,9 @@ print_entry (void *cls, | |||
75 | string = GNUNET_FS_uri_to_string (uri); | 82 | string = GNUNET_FS_uri_to_string (uri); |
76 | printf ("%s:\n", string); | 83 | printf ("%s:\n", string); |
77 | GNUNET_free (string); | 84 | GNUNET_free (string); |
78 | GNUNET_CONTAINER_meta_data_get_contents (meta, | 85 | GNUNET_CONTAINER_meta_data_iterate (meta, |
79 | &item_printer, | 86 | &item_printer, |
80 | NULL); | 87 | NULL); |
81 | } | 88 | } |
82 | 89 | ||
83 | 90 | ||
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c index e3918f6fb..65f437f11 100644 --- a/src/fs/gnunet-publish.c +++ b/src/fs/gnunet-publish.c | |||
@@ -140,43 +140,57 @@ progress_cb (void *cls, | |||
140 | * @param cls closure | 140 | * @param cls closure |
141 | * @param type type of the meta data | 141 | * @param type type of the meta data |
142 | * @param data value of the meta data | 142 | * @param data value of the meta data |
143 | * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort | 143 | * @return always 0 |
144 | */ | 144 | */ |
145 | static int | 145 | static int |
146 | meta_printer (void *cls, | 146 | meta_printer (void *cls, |
147 | EXTRACTOR_KeywordType type, | 147 | const char *plugin_name, |
148 | const char *data) | 148 | enum EXTRACTOR_MetaType type, |
149 | enum EXTRACTOR_MetaFormat format, | ||
150 | const char *data_mime_type, | ||
151 | const char *data, | ||
152 | size_t data_size) | ||
149 | { | 153 | { |
150 | if ( (type == EXTRACTOR_FILENAME) || | 154 | if ( (format != EXTRACTOR_METAFORMAT_UTF8) && |
151 | (EXTRACTOR_isBinaryType (type)) ) | 155 | (format != EXTRACTOR_METAFORMAT_C_STRING) ) |
152 | return GNUNET_OK; | 156 | return 0; |
157 | if (type == EXTRACTOR_METATYPE_FILENAME) | ||
158 | return 0; | ||
153 | fprintf (stdout, | 159 | fprintf (stdout, |
154 | "%s - %s", | 160 | "%s - %s", |
155 | EXTRACTOR_getKeywordTypeAsString (type), | 161 | EXTRACTOR_metatype_to_string (type), |
156 | data); | 162 | data); |
157 | return GNUNET_OK; | 163 | return 0; |
158 | } | 164 | } |
159 | 165 | ||
160 | 166 | ||
161 | /** | 167 | /** |
162 | * Merge metadata entries (except binary | 168 | * Merge metadata entries. |
163 | * metadata). | ||
164 | * | 169 | * |
165 | * @param cls closure, target metadata structure | 170 | * @param cls closure, target metadata structure |
166 | * @param type type of the meta data | 171 | * @param type type of the meta data |
167 | * @param data value of the meta data | 172 | * @param data value of the meta data |
168 | * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort | 173 | * @return always 0 |
169 | */ | 174 | */ |
170 | static int | 175 | static int |
171 | meta_merger (void *cls, | 176 | meta_merger (void *cls, |
172 | EXTRACTOR_KeywordType type, | 177 | const char *plugin_name, |
173 | const char *data) | 178 | enum EXTRACTOR_MetaType type, |
179 | enum EXTRACTOR_MetaFormat format, | ||
180 | const char *data_mime_type, | ||
181 | const char *data, | ||
182 | size_t data_size) | ||
174 | { | 183 | { |
175 | struct GNUNET_CONTAINER_MetaData *m = cls; | 184 | struct GNUNET_CONTAINER_MetaData *m = cls; |
185 | |||
176 | GNUNET_CONTAINER_meta_data_insert (m, | 186 | GNUNET_CONTAINER_meta_data_insert (m, |
187 | plugin_name, | ||
177 | type, | 188 | type, |
178 | data); | 189 | format, |
179 | return GNUNET_OK; | 190 | data_mime_type, |
191 | data, | ||
192 | data_size); | ||
193 | return 0; | ||
180 | } | 194 | } |
181 | 195 | ||
182 | 196 | ||
@@ -227,16 +241,16 @@ publish_inspector (void *cls, | |||
227 | } | 241 | } |
228 | if (NULL != meta) | 242 | if (NULL != meta) |
229 | { | 243 | { |
230 | GNUNET_CONTAINER_meta_data_get_contents (meta, | 244 | GNUNET_CONTAINER_meta_data_iterate (meta, |
231 | &meta_merger, | 245 | &meta_merger, |
232 | m); | 246 | m); |
233 | GNUNET_CONTAINER_meta_data_destroy (meta); | 247 | GNUNET_CONTAINER_meta_data_destroy (meta); |
234 | meta = NULL; | 248 | meta = NULL; |
235 | } | 249 | } |
236 | if (extract_only) | 250 | if (extract_only) |
237 | { | 251 | { |
238 | fn = GNUNET_CONTAINER_meta_data_get_by_type (meta, | 252 | fn = GNUNET_CONTAINER_meta_data_get_by_type (meta, |
239 | EXTRACTOR_FILENAME); | 253 | EXTRACTOR_METATYPE_FILENAME); |
240 | fs = GNUNET_STRINGS_byte_size_fancy (length); | 254 | fs = GNUNET_STRINGS_byte_size_fancy (length); |
241 | fprintf (stdout, | 255 | fprintf (stdout, |
242 | _("Keywords for file `%s' (%s)\n"), | 256 | _("Keywords for file `%s' (%s)\n"), |
@@ -244,9 +258,9 @@ publish_inspector (void *cls, | |||
244 | fs); | 258 | fs); |
245 | GNUNET_free (fn); | 259 | GNUNET_free (fn); |
246 | GNUNET_free (fs); | 260 | GNUNET_free (fs); |
247 | GNUNET_CONTAINER_meta_data_get_contents (meta, | 261 | GNUNET_CONTAINER_meta_data_iterate (meta, |
248 | &meta_printer, | 262 | &meta_printer, |
249 | NULL); | 263 | NULL); |
250 | fprintf (stdout, "\n"); | 264 | fprintf (stdout, "\n"); |
251 | } | 265 | } |
252 | if (GNUNET_FS_meta_data_test_for_directory (meta)) | 266 | if (GNUNET_FS_meta_data_test_for_directory (meta)) |
@@ -275,7 +289,7 @@ run (void *cls, | |||
275 | { | 289 | { |
276 | struct GNUNET_FS_FileInformation *fi; | 290 | struct GNUNET_FS_FileInformation *fi; |
277 | struct GNUNET_FS_Namespace *namespace; | 291 | struct GNUNET_FS_Namespace *namespace; |
278 | EXTRACTOR_ExtractorList *l; | 292 | struct EXTRACTOR_PluginList *l; |
279 | char *ex; | 293 | char *ex; |
280 | char *emsg; | 294 | char *emsg; |
281 | 295 | ||
@@ -377,13 +391,13 @@ run (void *cls, | |||
377 | l = NULL; | 391 | l = NULL; |
378 | if (! disable_extractor) | 392 | if (! disable_extractor) |
379 | { | 393 | { |
380 | l = EXTRACTOR_loadDefaultLibraries (); | 394 | l = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY); |
381 | if (GNUNET_OK == | 395 | if (GNUNET_OK == |
382 | GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", | 396 | GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", |
383 | &ex)) | 397 | &ex)) |
384 | { | 398 | { |
385 | if (strlen (ex) > 0) | 399 | if (strlen (ex) > 0) |
386 | l = EXTRACTOR_loadConfigLibraries (l, ex); | 400 | l = EXTRACTOR_plugin_add_config (l, ex, EXTRACTOR_OPTION_DEFAULT_POLICY); |
387 | GNUNET_free (ex); | 401 | GNUNET_free (ex); |
388 | } | 402 | } |
389 | } | 403 | } |
@@ -396,7 +410,7 @@ run (void *cls, | |||
396 | priority, | 410 | priority, |
397 | GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION), | 411 | GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION), |
398 | &emsg); | 412 | &emsg); |
399 | EXTRACTOR_removeAll (l); | 413 | EXTRACTOR_plugin_remove_all (l); |
400 | if (fi == NULL) | 414 | if (fi == NULL) |
401 | { | 415 | { |
402 | fprintf (stderr, | 416 | fprintf (stderr, |
diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c index 067d77159..4836f9d83 100644 --- a/src/fs/gnunet-search.c +++ b/src/fs/gnunet-search.c | |||
@@ -45,12 +45,19 @@ static int verbose; | |||
45 | 45 | ||
46 | static int | 46 | static int |
47 | item_printer (void *cls, | 47 | item_printer (void *cls, |
48 | EXTRACTOR_KeywordType type, | 48 | const char *plugin_name, |
49 | const char *data) | 49 | enum EXTRACTOR_MetaType type, |
50 | enum EXTRACTOR_MetaFormat format, | ||
51 | const char *data_mime_type, | ||
52 | const char *data, | ||
53 | size_t data_size) | ||
50 | { | 54 | { |
55 | if ( (format != EXTRACTOR_METAFORMAT_UTF8) && | ||
56 | (format != EXTRACTOR_METAFORMAT_C_STRING) ) | ||
57 | return 0; | ||
51 | printf ("\t%20s: %s\n", | 58 | printf ("\t%20s: %s\n", |
52 | dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, | 59 | dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, |
53 | EXTRACTOR_getKeywordTypeAsString (type)), | 60 | EXTRACTOR_metatype_to_string (type)), |
54 | data); | 61 | data); |
55 | return GNUNET_OK; | 62 | return GNUNET_OK; |
56 | } | 63 | } |
@@ -86,7 +93,7 @@ progress_cb (void *cls, | |||
86 | printf ("%s:\n", uri); | 93 | printf ("%s:\n", uri); |
87 | filename = | 94 | filename = |
88 | GNUNET_CONTAINER_meta_data_get_by_type (info->value.search.specifics.result.meta, | 95 | GNUNET_CONTAINER_meta_data_get_by_type (info->value.search.specifics.result.meta, |
89 | EXTRACTOR_FILENAME); | 96 | EXTRACTOR_METATYPE_FILENAME); |
90 | if (filename != NULL) | 97 | if (filename != NULL) |
91 | { | 98 | { |
92 | while (NULL != (dotdot = strstr (filename, ".."))) | 99 | while (NULL != (dotdot = strstr (filename, ".."))) |
@@ -98,9 +105,9 @@ progress_cb (void *cls, | |||
98 | else | 105 | else |
99 | printf ("gnunet-download %s\n", uri); | 106 | printf ("gnunet-download %s\n", uri); |
100 | if (verbose) | 107 | if (verbose) |
101 | GNUNET_CONTAINER_meta_data_get_contents (info->value.search.specifics.result.meta, | 108 | GNUNET_CONTAINER_meta_data_iterate (info->value.search.specifics.result.meta, |
102 | &item_printer, | 109 | &item_printer, |
103 | NULL); | 110 | NULL); |
104 | printf ("\n"); | 111 | printf ("\n"); |
105 | fflush(stdout); | 112 | fflush(stdout); |
106 | GNUNET_free_non_null (filename); | 113 | GNUNET_free_non_null (filename); |
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h index db9e10ef8..3df9dcfc0 100644 --- a/src/include/gnunet_container_lib.h +++ b/src/include/gnunet_container_lib.h | |||
@@ -1,6 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
3 | (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) | 3 | (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) |
4 | 4 | ||
5 | GNUnet is free software; you can redistribute it and/or modify | 5 | GNUnet is free software; you can redistribute it and/or modify |
6 | it under the terms of the GNU General Public License as published | 6 | it under the terms of the GNU General Public License as published |
@@ -196,23 +196,12 @@ void GNUNET_CONTAINER_bloomfilter_resize (struct GNUNET_CONTAINER_BloomFilter | |||
196 | struct GNUNET_CONTAINER_MetaData; | 196 | struct GNUNET_CONTAINER_MetaData; |
197 | 197 | ||
198 | /** | 198 | /** |
199 | * Iterator over meta data. | ||
200 | * | ||
201 | * @param cls closure | ||
202 | * @param type type of the meta data | ||
203 | * @param data value of the meta data | ||
204 | * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort | ||
205 | */ | ||
206 | typedef int (*GNUNET_CONTAINER_MetaDataProcessor) (void *cls, | ||
207 | EXTRACTOR_KeywordType type, | ||
208 | const char *data); | ||
209 | |||
210 | /** | ||
211 | * Create a fresh MetaData token. | 199 | * Create a fresh MetaData token. |
212 | * | 200 | * |
213 | * @return empty meta-data container | 201 | * @return empty meta-data container |
214 | */ | 202 | */ |
215 | struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void); | 203 | struct GNUNET_CONTAINER_MetaData * |
204 | GNUNET_CONTAINER_meta_data_create (void); | ||
216 | 205 | ||
217 | /** | 206 | /** |
218 | * Duplicate a MetaData token. | 207 | * Duplicate a MetaData token. |
@@ -220,43 +209,61 @@ struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void); | |||
220 | * @param md what to duplicate | 209 | * @param md what to duplicate |
221 | * @return duplicate meta-data container | 210 | * @return duplicate meta-data container |
222 | */ | 211 | */ |
223 | struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_duplicate (const | 212 | struct GNUNET_CONTAINER_MetaData * |
224 | struct | 213 | GNUNET_CONTAINER_meta_data_duplicate (const struct |
225 | GNUNET_CONTAINER_MetaData | 214 | GNUNET_CONTAINER_MetaData *md); |
226 | *md); | ||
227 | 215 | ||
228 | /** | 216 | /** |
229 | * Free meta data. | 217 | * Free meta data. |
230 | * | 218 | * |
231 | * @param md what to free | 219 | * @param md what to free |
232 | */ | 220 | */ |
233 | void GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData | 221 | void |
234 | *md); | 222 | GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md); |
235 | 223 | ||
236 | /** | 224 | /** |
237 | * Test if two MDs are equal. | 225 | * Test if two MDs are equal. We consider them equal if |
226 | * the meta types, formats and content match (we do not | ||
227 | * include the mime types and plugins names in this | ||
228 | * consideration). | ||
238 | * | 229 | * |
239 | * @param md1 first value to check | 230 | * @param md1 first value to check |
240 | * @param md2 other value to check | 231 | * @param md2 other value to check |
241 | * @return GNUNET_YES if they are equal | 232 | * @return GNUNET_YES if they are equal |
242 | */ | 233 | */ |
243 | int GNUNET_CONTAINER_meta_data_test_equal (const struct | 234 | int |
244 | GNUNET_CONTAINER_MetaData *md1, | 235 | GNUNET_CONTAINER_meta_data_test_equal (const struct |
245 | const struct | 236 | GNUNET_CONTAINER_MetaData *md1, |
246 | GNUNET_CONTAINER_MetaData *md2); | 237 | const struct |
238 | GNUNET_CONTAINER_MetaData *md2); | ||
247 | 239 | ||
248 | 240 | ||
249 | /** | 241 | /** |
250 | * Extend metadata. | 242 | * Extend metadata. |
251 | * | 243 | * |
252 | * @param md metadata to extend | 244 | * @param md metadata to extend |
253 | * @param type type of the new entry | 245 | * @param plugin_name name of the plugin that produced this value; |
254 | * @param data value for the entry | 246 | * special values can be used (i.e. '<zlib>' for zlib being |
247 | * used in the main libextractor library and yielding | ||
248 | * meta data). | ||
249 | * @param type libextractor-type describing the meta data | ||
250 | * @param format basic format information about data | ||
251 | * @param data_mime_type mime-type of data (not of the original file); | ||
252 | * can be NULL (if mime-type is not known) | ||
253 | * @param data actual meta-data found | ||
254 | * @param data_len number of bytes in data | ||
255 | * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists | 255 | * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists |
256 | * data_mime_type and plugin_name are not considered for "exists" checks | ||
256 | */ | 257 | */ |
257 | int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, | 258 | int |
258 | EXTRACTOR_KeywordType type, | 259 | GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, |
259 | const char *data); | 260 | const char *plugin_name, |
261 | enum EXTRACTOR_MetaType type, | ||
262 | enum EXTRACTOR_MetaFormat format, | ||
263 | const char *data_mime_type, | ||
264 | const char *data, | ||
265 | size_t data_len); | ||
266 | |||
260 | 267 | ||
261 | /** | 268 | /** |
262 | * Remove an item. | 269 | * Remove an item. |
@@ -265,11 +272,15 @@ int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, | |||
265 | * @param type type of the item to remove | 272 | * @param type type of the item to remove |
266 | * @param data specific value to remove, NULL to remove all | 273 | * @param data specific value to remove, NULL to remove all |
267 | * entries of the given type | 274 | * entries of the given type |
275 | * @param data_len number of bytes in data | ||
268 | * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md | 276 | * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md |
269 | */ | 277 | */ |
270 | int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, | 278 | int |
271 | EXTRACTOR_KeywordType type, | 279 | GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, |
272 | const char *data); | 280 | enum EXTRACTOR_MetaType type, |
281 | const char *data, | ||
282 | size_t data_len); | ||
283 | |||
273 | 284 | ||
274 | /** | 285 | /** |
275 | * Add the current time as the publication date | 286 | * Add the current time as the publication date |
@@ -277,58 +288,70 @@ int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, | |||
277 | * | 288 | * |
278 | * @param md metadata to modify | 289 | * @param md metadata to modify |
279 | */ | 290 | */ |
280 | void GNUNET_CONTAINER_meta_data_add_publication_date (struct | 291 | void |
281 | GNUNET_CONTAINER_MetaData | 292 | GNUNET_CONTAINER_meta_data_add_publication_date (struct |
282 | *md); | 293 | GNUNET_CONTAINER_MetaData |
294 | *md); | ||
295 | |||
283 | 296 | ||
284 | /** | 297 | /** |
285 | * Iterate over MD entries, excluding thumbnails. | 298 | * Iterate over MD entries. |
286 | * | 299 | * |
287 | * @param md metadata to inspect | 300 | * @param md metadata to inspect |
288 | * @param iter function to call on each entry | 301 | * @param iter function to call on each entry |
289 | * @param iter_cls closure for iterator | 302 | * @param iter_cls closure for iterator |
290 | * @return number of entries | 303 | * @return number of entries |
291 | */ | 304 | */ |
292 | int GNUNET_CONTAINER_meta_data_get_contents (const struct | 305 | int GNUNET_CONTAINER_meta_data_iterate (const struct |
293 | GNUNET_CONTAINER_MetaData *md, | 306 | GNUNET_CONTAINER_MetaData *md, |
294 | GNUNET_CONTAINER_MetaDataProcessor | 307 | EXTRACTOR_MetaDataProcessor |
295 | iter, void *iter_cls); | 308 | iter, void *iter_cls); |
296 | 309 | ||
297 | /** | 310 | /** |
298 | * Get the first MD entry of the given type. | 311 | * Get the first MD entry of the given type. Caller |
312 | * is responsible for freeing the return value. | ||
313 | * Also, only meta data items that are strings (0-terminated) | ||
314 | * are returned by this function. | ||
299 | * | 315 | * |
300 | * @param md metadata to inspect | 316 | * @param md metadata to inspect |
301 | * @param type type to look for | 317 | * @param type type to look for |
302 | * @return NULL if we do not have any such entry, | 318 | * @return NULL if no entry was found |
303 | * otherwise client is responsible for freeing the value! | ||
304 | */ | 319 | */ |
305 | char *GNUNET_CONTAINER_meta_data_get_by_type (const struct | 320 | char * |
306 | GNUNET_CONTAINER_MetaData *md, | 321 | GNUNET_CONTAINER_meta_data_get_by_type (const struct |
307 | EXTRACTOR_KeywordType type); | 322 | GNUNET_CONTAINER_MetaData *md, |
323 | enum EXTRACTOR_MetaType type); | ||
324 | |||
308 | 325 | ||
309 | /** | 326 | /** |
310 | * Get the first matching MD entry of the given types. | 327 | * Get the first matching MD entry of the given types. Caller is |
328 | * responsible for freeing the return value. Also, only meta data | ||
329 | * items that are strings (0-terminated) are returned by this | ||
330 | * function. | ||
311 | * | 331 | * |
312 | * @param md metadata to inspect | 332 | * @param md metadata to inspect |
313 | * @param ... -1-terminated list of types | 333 | * @param ... -1-terminated list of types |
314 | * @return NULL if we do not have any such entry, | 334 | * @return NULL if we do not have any such entry, |
315 | * otherwise client is responsible for freeing the value! | 335 | * otherwise client is responsible for freeing the value! |
316 | */ | 336 | */ |
317 | char *GNUNET_CONTAINER_meta_data_get_first_by_types (const struct | 337 | char * |
318 | GNUNET_CONTAINER_MetaData | 338 | GNUNET_CONTAINER_meta_data_get_first_by_types (const struct |
319 | *md, ...); | 339 | GNUNET_CONTAINER_MetaData |
340 | *md, ...); | ||
320 | 341 | ||
321 | /** | 342 | /** |
322 | * Get a thumbnail from the meta-data (if present). | 343 | * Get a thumbnail from the meta-data (if present). Only matches meta |
344 | * data with mime type "image" and binary format. | ||
323 | * | 345 | * |
324 | * @param md metadata to inspect | 346 | * @param md metadata to inspect |
325 | * @param thumb will be set to the thumbnail data. Must be | 347 | * @param thumb will be set to the thumbnail data. Must be |
326 | * freed by the caller! | 348 | * freed by the caller! |
327 | * @return number of bytes in thumbnail, 0 if not available | 349 | * @return number of bytes in thumbnail, 0 if not available |
328 | */ | 350 | */ |
329 | size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct | 351 | size_t |
330 | GNUNET_CONTAINER_MetaData | 352 | GNUNET_CONTAINER_meta_data_get_thumbnail (const struct |
331 | *md, unsigned char **thumb); | 353 | GNUNET_CONTAINER_MetaData |
354 | *md, unsigned char **thumb); | ||
332 | 355 | ||
333 | /** | 356 | /** |
334 | * Extract meta-data from a file. | 357 | * Extract meta-data from a file. |
@@ -339,11 +362,12 @@ size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct | |||
339 | * @return GNUNET_SYSERR on error, otherwise the number | 362 | * @return GNUNET_SYSERR on error, otherwise the number |
340 | * of meta-data items obtained | 363 | * of meta-data items obtained |
341 | */ | 364 | */ |
342 | int GNUNET_CONTAINER_meta_data_extract_from_file (struct | 365 | int |
343 | GNUNET_CONTAINER_MetaData | 366 | GNUNET_CONTAINER_meta_data_extract_from_file (struct |
344 | *md, const char *filename, | 367 | GNUNET_CONTAINER_MetaData |
345 | EXTRACTOR_ExtractorList * | 368 | *md, const char *filename, |
346 | extractors); | 369 | struct EXTRACTOR_PluginList * |
370 | extractors); | ||
347 | 371 | ||
348 | 372 | ||
349 | /** | 373 | /** |
@@ -373,7 +397,8 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions | |||
373 | * Serialize meta-data to target. | 397 | * Serialize meta-data to target. |
374 | * | 398 | * |
375 | * @param md metadata to serialize | 399 | * @param md metadata to serialize |
376 | * @param target where to write the serialized metadata | 400 | * @param target where to write the serialized metadata; |
401 | * *target can be NULL, in which case memory is allocated | ||
377 | * @param max maximum number of bytes available | 402 | * @param max maximum number of bytes available |
378 | * @param opt is it ok to just write SOME of the | 403 | * @param opt is it ok to just write SOME of the |
379 | * meta-data to match the size constraint, | 404 | * meta-data to match the size constraint, |
@@ -384,7 +409,7 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions | |||
384 | */ | 409 | */ |
385 | ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct | 410 | ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct |
386 | GNUNET_CONTAINER_MetaData *md, | 411 | GNUNET_CONTAINER_MetaData *md, |
387 | char *target, | 412 | char **target, |
388 | size_t max, | 413 | size_t max, |
389 | enum | 414 | enum |
390 | GNUNET_CONTAINER_MetaDataSerializationOptions | 415 | GNUNET_CONTAINER_MetaDataSerializationOptions |
@@ -392,22 +417,15 @@ ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct | |||
392 | 417 | ||
393 | 418 | ||
394 | /** | 419 | /** |
395 | * Estimate (!) the size of the meta-data in | 420 | * Get the size of the full meta-data in serialized form. |
396 | * serialized form. The estimate MAY be higher | ||
397 | * than what is strictly needed. | ||
398 | * | 421 | * |
399 | * @param md metadata to inspect | 422 | * @param md metadata to inspect |
400 | * @param opt is it ok to just write SOME of the | ||
401 | * meta-data to match the size constraint, | ||
402 | * possibly discarding some data? | ||
403 | * @return number of bytes needed for serialization, -1 on error | 423 | * @return number of bytes needed for serialization, -1 on error |
404 | */ | 424 | */ |
405 | ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct | 425 | ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct |
406 | GNUNET_CONTAINER_MetaData | 426 | GNUNET_CONTAINER_MetaData |
407 | *md, | 427 | *md); |
408 | enum | 428 | |
409 | GNUNET_CONTAINER_MetaDataSerializationOptions | ||
410 | opt); | ||
411 | 429 | ||
412 | /** | 430 | /** |
413 | * Deserialize meta-data. Initializes md. | 431 | * Deserialize meta-data. Initializes md. |
diff --git a/src/include/gnunet_directories.h b/src/include/gnunet_directories.h deleted file mode 100644 index 05a753f44..000000000 --- a/src/include/gnunet_directories.h +++ /dev/null | |||
@@ -1,34 +0,0 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | (C) 2001, 2002, 2003, 2004, 2005, 2006 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 2, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file include/gnunet_directories.h | ||
23 | * @brief directories and files in GNUnet (default locations) | ||
24 | * | ||
25 | * @author Christian Grothoff | ||
26 | */ | ||
27 | |||
28 | #ifndef GNUNET_DIRECTORIES | ||
29 | #define GNUNET_DIRECTORIES | ||
30 | |||
31 | #define GNUNET_DEFAULT_CLIENT_CONFIG_FILE "~/.gnunet/gnunet.conf" | ||
32 | #define GNUNET_DEFAULT_DAEMON_CONFIG_FILE "/etc/gnunetd.conf" | ||
33 | |||
34 | #endif | ||
diff --git a/src/util/bio.c b/src/util/bio.c index e1085faa6..b05746c38 100644 --- a/src/util/bio.c +++ b/src/util/bio.c | |||
@@ -434,21 +434,11 @@ GNUNET_BIO_write_meta_data (struct GNUNET_BIO_WriteHandle *h, | |||
434 | ssize_t size; | 434 | ssize_t size; |
435 | char *buf; | 435 | char *buf; |
436 | 436 | ||
437 | size = GNUNET_CONTAINER_meta_data_get_serialized_size (m, | 437 | buf = NULL; |
438 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL | ||
439 | | | ||
440 | GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS); | ||
441 | if (size == -1) | ||
442 | return GNUNET_SYSERR; | ||
443 | if (size > MAX_META_DATA) | ||
444 | size = MAX_META_DATA; | ||
445 | buf = GNUNET_malloc (size); | ||
446 | size = GNUNET_CONTAINER_meta_data_serialize (m, | 438 | size = GNUNET_CONTAINER_meta_data_serialize (m, |
447 | buf, | 439 | &buf, |
448 | size, | 440 | MAX_META_DATA, |
449 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART | 441 | GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); |
450 | | | ||
451 | GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS); | ||
452 | if (size == -1) | 442 | if (size == -1) |
453 | { | 443 | { |
454 | GNUNET_free (buf); | 444 | GNUNET_free (buf); |
diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c index 912ac2684..e4d8737c8 100644 --- a/src/util/container_meta_data.c +++ b/src/util/container_meta_data.c | |||
@@ -1,6 +1,6 @@ | |||
1 | /* | 1 | /* |
2 | This file is part of GNUnet. | 2 | This file is part of GNUnet. |
3 | (C) 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) | 3 | (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors) |
4 | 4 | ||
5 | GNUnet is free software; you can redistribute it and/or modify | 5 | GNUnet is free software; you can redistribute it and/or modify |
6 | it under the terms of the GNU General Public License as published | 6 | it under the terms of the GNU General Public License as published |
@@ -32,12 +32,46 @@ | |||
32 | #include <extractor.h> | 32 | #include <extractor.h> |
33 | #include <zlib.h> | 33 | #include <zlib.h> |
34 | 34 | ||
35 | #define EXTRA_CHECKS ALLOW_EXTRA_CHECKS | 35 | /** |
36 | 36 | * Meta data item. | |
37 | struct Item | 37 | */ |
38 | struct MetaItem | ||
38 | { | 39 | { |
39 | EXTRACTOR_KeywordType type; | 40 | /** |
41 | * This is a linked list. | ||
42 | */ | ||
43 | struct MetaItem *next; | ||
44 | |||
45 | /** | ||
46 | * Name of the extracting plugin. | ||
47 | */ | ||
48 | char *plugin_name; | ||
49 | |||
50 | /** | ||
51 | * Mime-type of data. | ||
52 | */ | ||
53 | char *mime_type; | ||
54 | |||
55 | /** | ||
56 | * The actual meta data. | ||
57 | */ | ||
40 | char *data; | 58 | char *data; |
59 | |||
60 | /** | ||
61 | * Number of bytes in 'data'. | ||
62 | */ | ||
63 | size_t data_size; | ||
64 | |||
65 | /** | ||
66 | * Type of the meta data. | ||
67 | */ | ||
68 | enum EXTRACTOR_MetaType type; | ||
69 | |||
70 | /** | ||
71 | * Format of the meta data. | ||
72 | */ | ||
73 | enum EXTRACTOR_MetaFormat format; | ||
74 | |||
41 | }; | 75 | }; |
42 | 76 | ||
43 | /** | 77 | /** |
@@ -45,86 +79,224 @@ struct Item | |||
45 | */ | 79 | */ |
46 | struct GNUNET_CONTAINER_MetaData | 80 | struct GNUNET_CONTAINER_MetaData |
47 | { | 81 | { |
48 | uint32_t itemCount; | 82 | /** |
49 | struct Item *items; | 83 | * Linked list of the meta data items. |
84 | */ | ||
85 | struct MetaItem *items; | ||
86 | |||
87 | /** | ||
88 | * Complete serialized and compressed buffer of the items. | ||
89 | * NULL if we have not computed that buffer yet. | ||
90 | */ | ||
91 | char *sbuf; | ||
92 | |||
93 | /** | ||
94 | * Number of bytes in 'sbuf'. 0 if the buffer is stale. | ||
95 | */ | ||
96 | size_t sbuf_size; | ||
97 | |||
98 | /** | ||
99 | * Number of items in the linked list. | ||
100 | */ | ||
101 | unsigned int item_count; | ||
102 | |||
50 | }; | 103 | }; |
51 | 104 | ||
105 | |||
52 | /** | 106 | /** |
53 | * Create a fresh struct CONTAINER_MetaData token. | 107 | * Create a fresh struct CONTAINER_MetaData token. |
108 | * | ||
109 | * @return empty meta-data container | ||
54 | */ | 110 | */ |
55 | struct GNUNET_CONTAINER_MetaData * | 111 | struct GNUNET_CONTAINER_MetaData * |
56 | GNUNET_CONTAINER_meta_data_create () | 112 | GNUNET_CONTAINER_meta_data_create () |
57 | { | 113 | { |
58 | struct GNUNET_CONTAINER_MetaData *ret; | 114 | return GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData)); |
59 | ret = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData)); | ||
60 | ret->items = NULL; | ||
61 | ret->itemCount = 0; | ||
62 | return ret; | ||
63 | } | 115 | } |
64 | 116 | ||
117 | |||
118 | /** | ||
119 | * Free meta data item. | ||
120 | * | ||
121 | * @param item item to free | ||
122 | */ | ||
123 | static void | ||
124 | meta_item_free (struct MetaItem *item) | ||
125 | { | ||
126 | GNUNET_free_non_null (item->plugin_name); | ||
127 | GNUNET_free_non_null (item->mime_type); | ||
128 | GNUNET_free_non_null (item->data); | ||
129 | GNUNET_free (item); | ||
130 | } | ||
131 | |||
132 | |||
133 | /** | ||
134 | * The meta data has changed, invalidate its serialization | ||
135 | * buffer. | ||
136 | * | ||
137 | * @param md meta data that changed | ||
138 | */ | ||
139 | static void | ||
140 | invalidate_sbuf (struct GNUNET_CONTAINER_MetaData *md) | ||
141 | { | ||
142 | if (md->sbuf == NULL) | ||
143 | return; | ||
144 | GNUNET_free (md->sbuf); | ||
145 | md->sbuf = NULL; | ||
146 | md->sbuf_size = 0; | ||
147 | } | ||
148 | |||
149 | |||
65 | /** | 150 | /** |
66 | * Free meta data. | 151 | * Free meta data. |
152 | * | ||
153 | * @param md what to free | ||
67 | */ | 154 | */ |
68 | void | 155 | void |
69 | GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md) | 156 | GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md) |
70 | { | 157 | { |
71 | int i; | 158 | struct MetaItem *item; |
72 | 159 | ||
73 | if (md == NULL) | 160 | if (md == NULL) |
74 | return; | 161 | return; |
75 | for (i = 0; i < md->itemCount; i++) | 162 | while (NULL != (item = md->items)) |
76 | GNUNET_free (md->items[i].data); | 163 | { |
77 | GNUNET_array_grow (md->items, md->itemCount, 0); | 164 | md->items = item->next; |
165 | meta_item_free (item); | ||
166 | } | ||
167 | GNUNET_free_non_null (md->sbuf); | ||
78 | GNUNET_free (md); | 168 | GNUNET_free (md); |
79 | } | 169 | } |
80 | 170 | ||
171 | |||
81 | /** | 172 | /** |
82 | * Add the current time as the publication date | 173 | * Test if two MDs are equal. We consider them equal if |
83 | * to the meta-data. | 174 | * the meta types, formats and content match (we do not |
175 | * include the mime types and plugins names in this | ||
176 | * consideration). | ||
177 | * | ||
178 | * @param md1 first value to check | ||
179 | * @param md2 other value to check | ||
180 | * @return GNUNET_YES if they are equal | ||
84 | */ | 181 | */ |
85 | void | 182 | int |
86 | GNUNET_CONTAINER_meta_data_add_publication_date (struct | 183 | GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData |
87 | GNUNET_CONTAINER_MetaData | 184 | *md1, |
88 | *md) | 185 | const struct GNUNET_CONTAINER_MetaData |
186 | *md2) | ||
89 | { | 187 | { |
90 | char *dat; | 188 | struct MetaItem *i; |
91 | struct GNUNET_TIME_Absolute t; | 189 | struct MetaItem *j; |
190 | int found; | ||
92 | 191 | ||
93 | t = GNUNET_TIME_absolute_get (); | 192 | if (md1 == md2) |
94 | GNUNET_CONTAINER_meta_data_delete (md, EXTRACTOR_PUBLICATION_DATE, NULL); | 193 | return GNUNET_YES; |
95 | dat = GNUNET_STRINGS_absolute_time_to_string (t); | 194 | if (md1->item_count != md2->item_count) |
96 | GNUNET_CONTAINER_meta_data_insert (md, EXTRACTOR_PUBLICATION_DATE, dat); | 195 | return GNUNET_NO; |
97 | GNUNET_free (dat); | 196 | |
197 | i = md1->items; | ||
198 | while (NULL != i) | ||
199 | { | ||
200 | found = GNUNET_NO; | ||
201 | j = md2->items; | ||
202 | while (NULL != j) | ||
203 | { | ||
204 | if ( (i->type == j->type) && | ||
205 | (i->format == j->format) && | ||
206 | (i->data_size == j->data_size) && | ||
207 | (0 == memcmp (i->data, | ||
208 | j->data, | ||
209 | i->data_size))) | ||
210 | { | ||
211 | found = GNUNET_YES; | ||
212 | break; | ||
213 | } | ||
214 | j = j->next; | ||
215 | } | ||
216 | if (found == GNUNET_NO) | ||
217 | return GNUNET_NO; | ||
218 | i = i->next; | ||
219 | } | ||
220 | return GNUNET_YES; | ||
98 | } | 221 | } |
99 | 222 | ||
223 | |||
100 | /** | 224 | /** |
101 | * Extend metadata. | 225 | * Extend metadata. Note that the list of meta data items is |
226 | * sorted by size (largest first). | ||
227 | * | ||
228 | * @param md metadata to extend | ||
229 | * @param plugin_name name of the plugin that produced this value; | ||
230 | * special values can be used (i.e. '<zlib>' for zlib being | ||
231 | * used in the main libextractor library and yielding | ||
232 | * meta data). | ||
233 | * @param type libextractor-type describing the meta data | ||
234 | * @param format basic format information about data | ||
235 | * @param data_mime_type mime-type of data (not of the original file); | ||
236 | * can be NULL (if mime-type is not known) | ||
237 | * @param data actual meta-data found | ||
238 | * @param data_len number of bytes in data | ||
102 | * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists | 239 | * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists |
240 | * data_mime_type and plugin_name are not considered for "exists" checks | ||
103 | */ | 241 | */ |
104 | int | 242 | int |
105 | GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, | 243 | GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, |
106 | EXTRACTOR_KeywordType type, | 244 | const char *plugin_name, |
107 | const char *data) | 245 | enum EXTRACTOR_MetaType type, |
246 | enum EXTRACTOR_MetaFormat format, | ||
247 | const char *data_mime_type, | ||
248 | const char *data, | ||
249 | size_t data_len) | ||
108 | { | 250 | { |
109 | uint32_t idx; | 251 | struct MetaItem *prev; |
252 | struct MetaItem *pos; | ||
253 | struct MetaItem *i; | ||
110 | char *p; | 254 | char *p; |
111 | 255 | ||
112 | GNUNET_assert (data != NULL); | 256 | prev = NULL; |
113 | for (idx = 0; idx < md->itemCount; idx++) | 257 | pos = md->items; |
258 | while (NULL != pos) | ||
114 | { | 259 | { |
115 | if ((md->items[idx].type == type) && | 260 | if (pos->data_size < data_len) |
116 | (0 == strcmp (md->items[idx].data, data))) | 261 | break; |
117 | return GNUNET_SYSERR; | 262 | if ( (pos->type == type) && |
263 | (pos->format == format) && | ||
264 | (pos->data_size == data_len) && | ||
265 | (0 == memcmp (pos->data, | ||
266 | data, | ||
267 | data_len))) | ||
268 | { | ||
269 | if ( (pos->mime_type == NULL) && | ||
270 | (data_mime_type != NULL) ) | ||
271 | { | ||
272 | pos->mime_type = GNUNET_strdup (data_mime_type); | ||
273 | invalidate_sbuf (md); | ||
274 | } | ||
275 | return GNUNET_SYSERR; | ||
276 | } | ||
277 | prev = pos; | ||
278 | pos = pos->next; | ||
118 | } | 279 | } |
119 | idx = md->itemCount; | 280 | md->item_count++; |
120 | GNUNET_array_grow (md->items, md->itemCount, md->itemCount + 1); | 281 | i = GNUNET_malloc (sizeof (struct MetaItem)); |
121 | md->items[idx].type = type; | 282 | i->type = type; |
122 | md->items[idx].data = p = GNUNET_strdup (data); | 283 | i->format = format; |
123 | 284 | i->data_size = data_len; | |
285 | i->next = pos; | ||
286 | if (prev == NULL) | ||
287 | md->items = i; | ||
288 | else | ||
289 | prev->next = i; | ||
290 | i->mime_type = (data_mime_type == NULL) ? NULL : GNUNET_strdup (data_mime_type); | ||
291 | i->plugin_name = (plugin_name == NULL) ? NULL : GNUNET_strdup (plugin_name); | ||
292 | i->data = GNUNET_malloc (data_len); | ||
293 | memcpy (i->data, data, data_len); | ||
124 | /* change OS native dir separators to unix '/' and others to '_' */ | 294 | /* change OS native dir separators to unix '/' and others to '_' */ |
125 | if (type == EXTRACTOR_FILENAME) | 295 | if (type == EXTRACTOR_METATYPE_FILENAME) |
126 | { | 296 | { |
127 | while (*p != '\0') | 297 | p = i->data; |
298 | while ( (*p != '\0') && | ||
299 | (p < i->data + data_len) ) | ||
128 | { | 300 | { |
129 | if (*p == DIR_SEPARATOR) | 301 | if (*p == DIR_SEPARATOR) |
130 | *p = '/'; | 302 | *p = '/'; |
@@ -133,10 +305,11 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, | |||
133 | p++; | 305 | p++; |
134 | } | 306 | } |
135 | } | 307 | } |
136 | 308 | invalidate_sbuf (md); | |
137 | return GNUNET_OK; | 309 | return GNUNET_OK; |
138 | } | 310 | } |
139 | 311 | ||
312 | |||
140 | /** | 313 | /** |
141 | * Remove an item. | 314 | * Remove an item. |
142 | * | 315 | * |
@@ -144,36 +317,78 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, | |||
144 | * @param type type of the item to remove | 317 | * @param type type of the item to remove |
145 | * @param data specific value to remove, NULL to remove all | 318 | * @param data specific value to remove, NULL to remove all |
146 | * entries of the given type | 319 | * entries of the given type |
320 | * @param data_len number of bytes in data | ||
147 | * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md | 321 | * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md |
148 | */ | 322 | */ |
149 | int | 323 | int |
150 | GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, | 324 | GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, |
151 | EXTRACTOR_KeywordType type, | 325 | enum EXTRACTOR_MetaType type, |
152 | const char *data) | 326 | const char *data, |
327 | size_t data_len) | ||
153 | { | 328 | { |
154 | uint32_t idx; | 329 | struct MetaItem *pos; |
155 | int ret = GNUNET_SYSERR; | 330 | struct MetaItem *prev; |
156 | for (idx = 0; idx < md->itemCount; idx++) | 331 | |
332 | prev = NULL; | ||
333 | pos = md->items; | ||
334 | while (NULL != pos) | ||
157 | { | 335 | { |
158 | if ((md->items[idx].type == type) && | 336 | if ( (pos->type == type) && |
159 | ((data == NULL) || (0 == strcmp (md->items[idx].data, data)))) | 337 | ( (data == NULL) || |
160 | { | 338 | ( (pos->data_size == data_len) && |
161 | GNUNET_free (md->items[idx].data); | 339 | (0 == memcmp (pos->data, |
162 | md->items[idx] = md->items[md->itemCount - 1]; | 340 | data, |
163 | GNUNET_array_grow (md->items, md->itemCount, md->itemCount - 1); | 341 | data_len))) ) ) |
164 | if (data == NULL) | 342 | { |
165 | { | 343 | if (prev == NULL) |
166 | ret = GNUNET_OK; | 344 | md->items = pos->next; |
167 | continue; | 345 | else |
168 | } | 346 | prev->next = pos->next; |
169 | return GNUNET_OK; | 347 | meta_item_free (pos); |
170 | } | 348 | md->item_count--; |
349 | invalidate_sbuf (md); | ||
350 | return GNUNET_OK; | ||
351 | } | ||
352 | prev = pos; | ||
353 | pos = pos->next; | ||
171 | } | 354 | } |
172 | return ret; | 355 | return GNUNET_SYSERR; |
173 | } | 356 | } |
174 | 357 | ||
358 | |||
175 | /** | 359 | /** |
176 | * Iterate over MD entries, excluding thumbnails. | 360 | * Add the current time as the publication date |
361 | * to the meta-data. | ||
362 | * | ||
363 | * @param md metadata to modify | ||
364 | */ | ||
365 | void | ||
366 | GNUNET_CONTAINER_meta_data_add_publication_date (struct | ||
367 | GNUNET_CONTAINER_MetaData | ||
368 | *md) | ||
369 | { | ||
370 | char *dat; | ||
371 | struct GNUNET_TIME_Absolute t; | ||
372 | |||
373 | t = GNUNET_TIME_absolute_get (); | ||
374 | GNUNET_CONTAINER_meta_data_delete (md, | ||
375 | EXTRACTOR_METATYPE_PUBLICATION_DATE, | ||
376 | NULL, | ||
377 | 0); | ||
378 | dat = GNUNET_STRINGS_absolute_time_to_string (t); | ||
379 | GNUNET_CONTAINER_meta_data_insert (md, | ||
380 | "<gnunet>", | ||
381 | EXTRACTOR_METATYPE_PUBLICATION_DATE, | ||
382 | EXTRACTOR_METAFORMAT_UTF8, | ||
383 | "text/plain", | ||
384 | dat, | ||
385 | strlen(dat)+1); | ||
386 | GNUNET_free (dat); | ||
387 | } | ||
388 | |||
389 | |||
390 | /** | ||
391 | * Iterate over MD entries. | ||
177 | * | 392 | * |
178 | * @param md metadata to inspect | 393 | * @param md metadata to inspect |
179 | * @param iter function to call on each entry | 394 | * @param iter function to call on each entry |
@@ -181,51 +396,71 @@ GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, | |||
181 | * @return number of entries | 396 | * @return number of entries |
182 | */ | 397 | */ |
183 | int | 398 | int |
184 | GNUNET_CONTAINER_meta_data_get_contents (const struct | 399 | GNUNET_CONTAINER_meta_data_iterate (const struct |
185 | GNUNET_CONTAINER_MetaData *md, | 400 | GNUNET_CONTAINER_MetaData *md, |
186 | GNUNET_CONTAINER_MetaDataProcessor | 401 | EXTRACTOR_MetaDataProcessor |
187 | iter, void *iter_cls) | 402 | iter, void *iter_cls) |
188 | { | 403 | { |
189 | uint32_t i; | 404 | struct MetaItem *pos; |
190 | uint32_t sub; | ||
191 | 405 | ||
192 | sub = 0; | 406 | if (iter == NULL) |
193 | for (i = 0; i < md->itemCount; i++) | 407 | return md->item_count; |
408 | pos = md->items; | ||
409 | while (NULL != pos) | ||
194 | { | 410 | { |
195 | if (!EXTRACTOR_isBinaryType (md->items[i].type)) | 411 | if (0 != iter (iter_cls, |
196 | { | 412 | pos->plugin_name, |
197 | if ((iter != NULL) && | 413 | pos->type, |
198 | (GNUNET_OK != iter (iter_cls, | 414 | pos->format, |
199 | md->items[i].type, md->items[i].data))) | 415 | pos->mime_type, |
200 | return GNUNET_SYSERR; | 416 | pos->data, |
201 | } | 417 | pos->data_size)) |
202 | else | 418 | return md->item_count; |
203 | sub++; | 419 | pos = pos->next; |
204 | } | 420 | } |
205 | return (int) (md->itemCount - sub); | 421 | return md->item_count; |
206 | } | 422 | } |
207 | 423 | ||
424 | |||
208 | /** | 425 | /** |
209 | * Iterate over MD entries | 426 | * Get the first MD entry of the given type. Caller |
427 | * is responsible for freeing the return value. | ||
428 | * Also, only meta data items that are strings (0-terminated) | ||
429 | * are returned by this function. | ||
210 | * | 430 | * |
211 | * @return number of entries | 431 | * @param md metadata to inspect |
432 | * @param type type to look for | ||
433 | * @return NULL if no entry was found | ||
212 | */ | 434 | */ |
213 | char * | 435 | char * |
214 | GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData | 436 | GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData |
215 | *md, EXTRACTOR_KeywordType type) | 437 | *md, enum EXTRACTOR_MetaType type) |
216 | { | 438 | { |
217 | uint32_t i; | 439 | struct MetaItem *pos; |
218 | 440 | ||
219 | for (i = 0; i < md->itemCount; i++) | 441 | pos = md->items; |
220 | if (type == md->items[i].type) | 442 | while (NULL != pos) |
221 | return GNUNET_strdup (md->items[i].data); | 443 | { |
444 | if ( (type == pos->type) && | ||
445 | ( (pos->format == EXTRACTOR_METAFORMAT_UTF8) || | ||
446 | (pos->format == EXTRACTOR_METAFORMAT_C_STRING) ) ) | ||
447 | return GNUNET_strdup (pos->data); | ||
448 | pos = pos->next; | ||
449 | } | ||
222 | return NULL; | 450 | return NULL; |
223 | } | 451 | } |
224 | 452 | ||
453 | |||
225 | /** | 454 | /** |
226 | * Iterate over MD entries | 455 | * Get the first matching MD entry of the given types. Caller is |
456 | * responsible for freeing the return value. Also, only meta data | ||
457 | * items that are strings (0-terminated) are returned by this | ||
458 | * function. | ||
227 | * | 459 | * |
228 | * @return number of entries | 460 | * @param md metadata to inspect |
461 | * @param ... -1-terminated list of types | ||
462 | * @return NULL if we do not have any such entry, | ||
463 | * otherwise client is responsible for freeing the value! | ||
229 | */ | 464 | */ |
230 | char * | 465 | char * |
231 | GNUNET_CONTAINER_meta_data_get_first_by_types (const struct | 466 | GNUNET_CONTAINER_meta_data_get_first_by_types (const struct |
@@ -234,13 +469,13 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct | |||
234 | { | 469 | { |
235 | char *ret; | 470 | char *ret; |
236 | va_list args; | 471 | va_list args; |
237 | EXTRACTOR_KeywordType type; | 472 | enum EXTRACTOR_MetaType type; |
238 | 473 | ||
239 | ret = NULL; | 474 | ret = NULL; |
240 | va_start (args, md); | 475 | va_start (args, md); |
241 | while (1) | 476 | while (1) |
242 | { | 477 | { |
243 | type = va_arg (args, EXTRACTOR_KeywordType); | 478 | type = va_arg (args, enum EXTRACTOR_MetaType); |
244 | if (type == -1) | 479 | if (type == -1) |
245 | break; | 480 | break; |
246 | ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type); | 481 | ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type); |
@@ -251,6 +486,7 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct | |||
251 | return ret; | 486 | return ret; |
252 | } | 487 | } |
253 | 488 | ||
489 | |||
254 | /** | 490 | /** |
255 | * Get a thumbnail from the meta-data (if present). | 491 | * Get a thumbnail from the meta-data (if present). |
256 | * | 492 | * |
@@ -264,27 +500,33 @@ GNUNET_CONTAINER_meta_data_get_thumbnail (const struct | |||
264 | GNUNET_CONTAINER_MetaData * md, | 500 | GNUNET_CONTAINER_MetaData * md, |
265 | unsigned char **thumb) | 501 | unsigned char **thumb) |
266 | { | 502 | { |
267 | char *encoded; | 503 | struct MetaItem *pos; |
268 | int ret; | 504 | struct MetaItem *match; |
269 | size_t size; | ||
270 | 505 | ||
271 | encoded = | 506 | match = NULL; |
272 | GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_THUMBNAIL_DATA); | 507 | pos = md->items; |
273 | if (encoded == NULL) | 508 | while (NULL != pos) |
274 | return 0; | ||
275 | if (strlen (encoded) == 0) | ||
276 | { | 509 | { |
277 | GNUNET_free (encoded); | 510 | if ( (0 == strncasecmp ("image/", pos->mime_type, |
278 | return 0; /* invalid */ | 511 | strlen("image/"))) && |
512 | (pos->format == EXTRACTOR_METAFORMAT_BINARY) ) | ||
513 | { | ||
514 | if (match == NULL) | ||
515 | match = pos; | ||
516 | else if ( (match->type != EXTRACTOR_METATYPE_THUMBNAIL) && | ||
517 | (pos->type == EXTRACTOR_METATYPE_THUMBNAIL) ) | ||
518 | match = pos; | ||
519 | } | ||
520 | pos = pos->next; | ||
279 | } | 521 | } |
280 | *thumb = NULL; | 522 | if (match == NULL) |
281 | ret = EXTRACTOR_binaryDecode (encoded, thumb, &size); | ||
282 | GNUNET_free (encoded); | ||
283 | if (ret != 0) | ||
284 | return 0; | 523 | return 0; |
285 | return size; | 524 | *thumb = GNUNET_malloc (match->data_size); |
525 | memcpy (*thumb, match->data, match->data_size); | ||
526 | return match->data_size; | ||
286 | } | 527 | } |
287 | 528 | ||
529 | |||
288 | /** | 530 | /** |
289 | * Duplicate struct GNUNET_CONTAINER_MetaData. | 531 | * Duplicate struct GNUNET_CONTAINER_MetaData. |
290 | * | 532 | * |
@@ -295,18 +537,66 @@ struct GNUNET_CONTAINER_MetaData * | |||
295 | GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData | 537 | GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData |
296 | *md) | 538 | *md) |
297 | { | 539 | { |
298 | uint32_t i; | ||
299 | struct GNUNET_CONTAINER_MetaData *ret; | 540 | struct GNUNET_CONTAINER_MetaData *ret; |
541 | struct MetaItem *pos; | ||
300 | 542 | ||
301 | if (md == NULL) | 543 | if (md == NULL) |
302 | return NULL; | 544 | return NULL; |
303 | ret = GNUNET_CONTAINER_meta_data_create (); | 545 | ret = GNUNET_CONTAINER_meta_data_create (); |
304 | for (i = 0; i < md->itemCount; i++) | 546 | pos = md->items; |
305 | GNUNET_CONTAINER_meta_data_insert (ret, md->items[i].type, | 547 | while (NULL != pos) |
306 | md->items[i].data); | 548 | { |
549 | GNUNET_CONTAINER_meta_data_insert (ret, | ||
550 | pos->plugin_name, | ||
551 | pos->type, | ||
552 | pos->format, | ||
553 | pos->mime_type, | ||
554 | pos->data, | ||
555 | pos->data_size); | ||
556 | pos = pos->next; | ||
557 | } | ||
307 | return ret; | 558 | return ret; |
308 | } | 559 | } |
309 | 560 | ||
561 | |||
562 | /** | ||
563 | * Add meta data that libextractor finds to our meta data | ||
564 | * container. | ||
565 | * | ||
566 | * @param cls closure, our meta data container | ||
567 | * @param plugin_name name of the plugin that produced this value; | ||
568 | * special values can be used (i.e. '<zlib>' for zlib being | ||
569 | * used in the main libextractor library and yielding | ||
570 | * meta data). | ||
571 | * @param type libextractor-type describing the meta data | ||
572 | * @param format basic format information about data | ||
573 | * @param data_mime_type mime-type of data (not of the original file); | ||
574 | * can be NULL (if mime-type is not known) | ||
575 | * @param data actual meta-data found | ||
576 | * @param data_len number of bytes in data | ||
577 | * @return always 0 to continue extracting | ||
578 | */ | ||
579 | static int | ||
580 | add_to_md(void *cls, | ||
581 | const char *plugin_name, | ||
582 | enum EXTRACTOR_MetaType type, | ||
583 | enum EXTRACTOR_MetaFormat format, | ||
584 | const char *data_mime_type, | ||
585 | const char *data, | ||
586 | size_t data_len) | ||
587 | { | ||
588 | struct GNUNET_CONTAINER_MetaData *md = cls; | ||
589 | (void) GNUNET_CONTAINER_meta_data_insert (md, | ||
590 | plugin_name, | ||
591 | type, | ||
592 | format, | ||
593 | data_mime_type, | ||
594 | data, | ||
595 | data_len); | ||
596 | return 0; | ||
597 | } | ||
598 | |||
599 | |||
310 | /** | 600 | /** |
311 | * Extract meta-data from a file. | 601 | * Extract meta-data from a file. |
312 | * | 602 | * |
@@ -316,37 +606,43 @@ GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData | |||
316 | int | 606 | int |
317 | GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData | 607 | GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData |
318 | *md, const char *filename, | 608 | *md, const char *filename, |
319 | EXTRACTOR_ExtractorList * | 609 | struct EXTRACTOR_PluginList * |
320 | extractors) | 610 | extractors) |
321 | { | 611 | { |
322 | EXTRACTOR_KeywordList *head; | 612 | unsigned int old; |
323 | EXTRACTOR_KeywordList *pos; | ||
324 | int ret; | ||
325 | 613 | ||
326 | if (filename == NULL) | 614 | if (filename == NULL) |
327 | return GNUNET_SYSERR; | 615 | return GNUNET_SYSERR; |
328 | if (extractors == NULL) | 616 | if (extractors == NULL) |
329 | return 0; | 617 | return 0; |
330 | head = EXTRACTOR_getKeywords (extractors, filename); | 618 | old = md->item_count; |
331 | head = EXTRACTOR_removeDuplicateKeywords (head, | 619 | EXTRACTOR_extract (extractors, |
332 | EXTRACTOR_DUPLICATES_REMOVE_UNKNOWN); | 620 | filename, |
333 | pos = head; | 621 | NULL, 0, |
334 | ret = 0; | 622 | &add_to_md, |
335 | while (pos != NULL) | 623 | md); |
336 | { | 624 | return (int) (md->item_count - old); |
337 | if (GNUNET_OK == | ||
338 | GNUNET_CONTAINER_meta_data_insert (md, pos->keywordType, | ||
339 | pos->keyword)) | ||
340 | ret++; | ||
341 | pos = pos->next; | ||
342 | } | ||
343 | EXTRACTOR_freeKeywords (head); | ||
344 | return ret; | ||
345 | } | 625 | } |
346 | 626 | ||
347 | 627 | ||
348 | static unsigned int | 628 | /** |
349 | tryCompression (char *data, unsigned int oldSize) | 629 | * Try to compress the given block of data. |
630 | * | ||
631 | * @param data block to compress; if compression | ||
632 | * resulted in a smaller block, the first | ||
633 | * bytes of data are updated to the compressed | ||
634 | * data | ||
635 | * @param oldSize number of bytes in data | ||
636 | * @param result set to the compressed data | ||
637 | * @param newSize set to size of result | ||
638 | * @return GNUNET_YES if compression reduce the size, | ||
639 | * GNUNET_NO if compression did not help | ||
640 | */ | ||
641 | static int | ||
642 | try_compression (const char *data, | ||
643 | size_t oldSize, | ||
644 | char **result, | ||
645 | size_t *newSize) | ||
350 | { | 646 | { |
351 | char *tmp; | 647 | char *tmp; |
352 | uLongf dlen; | 648 | uLongf dlen; |
@@ -364,62 +660,40 @@ tryCompression (char *data, unsigned int oldSize) | |||
364 | { | 660 | { |
365 | if (dlen < oldSize) | 661 | if (dlen < oldSize) |
366 | { | 662 | { |
367 | memcpy (data, tmp, dlen); | 663 | *result = tmp; |
368 | GNUNET_free (tmp); | 664 | *newSize = dlen; |
369 | return dlen; | 665 | return GNUNET_YES; |
370 | } | 666 | } |
371 | } | 667 | } |
372 | GNUNET_free (tmp); | 668 | GNUNET_free (tmp); |
373 | return oldSize; | 669 | return GNUNET_NO; |
374 | } | 670 | } |
375 | 671 | ||
376 | /** | ||
377 | * Decompress input, return the decompressed data | ||
378 | * as output, set outputSize to the number of bytes | ||
379 | * that were found. | ||
380 | * | ||
381 | * @return NULL on error | ||
382 | */ | ||
383 | static char * | ||
384 | decompress (const char *input, | ||
385 | unsigned int inputSize, unsigned int outputSize) | ||
386 | { | ||
387 | char *output; | ||
388 | uLongf olen; | ||
389 | |||
390 | olen = outputSize; | ||
391 | output = GNUNET_malloc (olen); | ||
392 | if (Z_OK == uncompress ((Bytef *) output, | ||
393 | &olen, (const Bytef *) input, inputSize)) | ||
394 | { | ||
395 | return output; | ||
396 | } | ||
397 | else | ||
398 | { | ||
399 | GNUNET_free (output); | ||
400 | return NULL; | ||
401 | } | ||
402 | } | ||
403 | 672 | ||
404 | /** | 673 | /** |
405 | * Flag in 'version' that indicates compressed meta-data. | 674 | * Flag in 'version' that indicates compressed meta-data. |
406 | */ | 675 | */ |
407 | #define HEADER_COMPRESSED 0x80000000 | 676 | #define HEADER_COMPRESSED 0x80000000 |
408 | 677 | ||
678 | |||
409 | /** | 679 | /** |
410 | * Bits in 'version' that give the version number. | 680 | * Bits in 'version' that give the version number. |
411 | */ | 681 | */ |
412 | #define HEADER_VERSION_MASK 0x7FFFFFFF | 682 | #define HEADER_VERSION_MASK 0x7FFFFFFF |
413 | 683 | ||
684 | |||
685 | /** | ||
686 | * Header for serialized meta data. | ||
687 | */ | ||
414 | struct MetaDataHeader | 688 | struct MetaDataHeader |
415 | { | 689 | { |
416 | /** | 690 | /** |
417 | * The version of the MD serialization. | 691 | * The version of the MD serialization. The highest bit is used to |
418 | * The highest bit is used to indicate | 692 | * indicate compression. |
419 | * compression. | ||
420 | * | 693 | * |
421 | * Version 0 is the current version; | 694 | * Version 0 is traditional (pre-0.9) meta data (unsupported) |
422 | * Version is 1 for a NULL pointer. | 695 | * Version is 1 for a NULL pointer |
696 | * Version 2 is for 0.9.x (and possibly higher) | ||
423 | * Other version numbers are not yet defined. | 697 | * Other version numbers are not yet defined. |
424 | */ | 698 | */ |
425 | uint32_t version; | 699 | uint32_t version; |
@@ -430,24 +704,57 @@ struct MetaDataHeader | |||
430 | uint32_t entries; | 704 | uint32_t entries; |
431 | 705 | ||
432 | /** | 706 | /** |
433 | * Size of the MD (decompressed) | 707 | * Size of the decompressed meta data. |
434 | */ | 708 | */ |
435 | uint32_t size; | 709 | uint32_t size; |
436 | 710 | ||
437 | /** | 711 | /** |
438 | * This is followed by 'entries' values of type 'uint32_t' that | 712 | * This is followed by 'entries' values of type 'struct MetaDataEntry' |
439 | * correspond to EXTRACTOR_KeywordTypes. After that, the meta-data | 713 | * and then by 'entry' plugin names, mime-types and data blocks |
440 | * keywords follow (0-terminated). The MD block always ends with | 714 | * as specified in those meta data entries. |
441 | * 0-termination, padding with 0 until a multiple of 8 bytes. | 715 | */ |
716 | }; | ||
717 | |||
718 | |||
719 | /** | ||
720 | * Entry of serialized meta data. | ||
721 | */ | ||
722 | struct MetaDataEntry | ||
723 | { | ||
724 | /** | ||
725 | * Meta data type. Corresponds to an 'enum EXTRACTOR_MetaType' | ||
726 | */ | ||
727 | uint32_t type; | ||
728 | |||
729 | /** | ||
730 | * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat' | ||
442 | */ | 731 | */ |
732 | uint32_t format; | ||
733 | |||
734 | /** | ||
735 | * Number of bytes of meta data. | ||
736 | */ | ||
737 | uint32_t data_size; | ||
738 | |||
739 | /** | ||
740 | * Number of bytes in the plugin name including 0-terminator. 0 for NULL. | ||
741 | */ | ||
742 | uint32_t plugin_name_len; | ||
743 | |||
744 | /** | ||
745 | * Number of bytes in the mime type including 0-terminator. 0 for NULL. | ||
746 | */ | ||
747 | uint32_t mime_type_len; | ||
443 | 748 | ||
444 | }; | 749 | }; |
445 | 750 | ||
751 | |||
446 | /** | 752 | /** |
447 | * Serialize meta-data to target. | 753 | * Serialize meta-data to target. |
448 | * | 754 | * |
449 | * @param md metadata to serialize | 755 | * @param md metadata to serialize |
450 | * @param target where to write the serialized metadata | 756 | * @param target where to write the serialized metadata; |
757 | * *target can be NULL, in which case memory is allocated | ||
451 | * @param max maximum number of bytes available in target | 758 | * @param max maximum number of bytes available in target |
452 | * @param opt is it ok to just write SOME of the | 759 | * @param opt is it ok to just write SOME of the |
453 | * meta-data to match the size constraint, | 760 | * meta-data to match the size constraint, |
@@ -458,149 +765,273 @@ struct MetaDataHeader | |||
458 | */ | 765 | */ |
459 | ssize_t | 766 | ssize_t |
460 | GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData | 767 | GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData |
461 | *md, char *target, size_t max, | 768 | *md, char **target, size_t max, |
462 | enum | 769 | enum |
463 | GNUNET_CONTAINER_MetaDataSerializationOptions | 770 | GNUNET_CONTAINER_MetaDataSerializationOptions |
464 | opt) | 771 | opt) |
465 | { | 772 | { |
466 | struct MetaDataHeader *hdr; | 773 | struct GNUNET_CONTAINER_MetaData *vmd; |
774 | struct MetaItem *pos; | ||
775 | struct MetaDataHeader *hdr; | ||
776 | struct MetaDataEntry *ent; | ||
777 | unsigned int i; | ||
778 | uint64_t msize; | ||
779 | size_t off; | ||
780 | char *mdata; | ||
781 | char *cdata; | ||
782 | size_t mlen; | ||
783 | size_t plen; | ||
467 | size_t size; | 784 | size_t size; |
468 | size_t pos; | 785 | size_t left; |
469 | uint32_t i; | 786 | size_t clen; |
470 | size_t len; | 787 | int comp; |
471 | uint32_t ic; | ||
472 | 788 | ||
473 | if (max < sizeof (struct MetaDataHeader)) | 789 | if (max < sizeof (struct MetaDataHeader)) |
474 | return GNUNET_SYSERR; /* far too small */ | 790 | return GNUNET_SYSERR; /* far too small */ |
475 | ic = md ? md->itemCount : 0; | 791 | if (md == NULL) |
476 | hdr = NULL; | 792 | return 0; |
477 | while (1) | 793 | |
794 | if (md->sbuf != NULL) | ||
478 | { | 795 | { |
479 | size = sizeof (struct MetaDataHeader); | 796 | /* try to use serialization cache */ |
480 | size += sizeof (uint32_t) * ic; | 797 | if (md->sbuf_size < max) |
481 | for (i = 0; i < ic; i++) | 798 | { |
482 | size += 1 + strlen (md->items[i].data); | 799 | if (NULL == *target) |
483 | while (size % 8 != 0) | 800 | *target = GNUNET_malloc (md->sbuf_size); |
484 | size++; | 801 | memcpy (*target, |
485 | hdr = GNUNET_malloc (size); | 802 | md->sbuf, |
486 | hdr->version = htonl (md == NULL ? 1 : 0); | 803 | md->sbuf_size); |
487 | hdr->entries = htonl (ic); | 804 | return md->sbuf_size; |
488 | for (i = 0; i < ic; i++) | 805 | } |
489 | ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); | 806 | if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART)) |
490 | pos = sizeof (struct MetaDataHeader); | 807 | return GNUNET_SYSERR; /* can say that this will fail */ |
491 | pos += sizeof (uint32_t) * ic; | 808 | /* need to compute a partial serialization, sbuf useless ... */ |
492 | for (i = 0; i < ic; i++) | 809 | } |
493 | { | ||
494 | len = strlen (md->items[i].data) + 1; | ||
495 | memcpy (&((char *) hdr)[pos], md->items[i].data, len); | ||
496 | pos += len; | ||
497 | } | ||
498 | 810 | ||
499 | hdr->size = htonl (size); | 811 | |
500 | if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) | 812 | msize = 0; |
501 | { | 813 | pos = md->items; |
502 | pos = tryCompression ((char *) &hdr[1], | 814 | while (NULL != pos) |
503 | size - sizeof (struct MetaDataHeader)); | 815 | { |
504 | } | 816 | msize += sizeof (struct MetaDataEntry); |
817 | msize += pos->data_size; | ||
818 | if (pos->plugin_name != NULL) | ||
819 | msize += strlen (pos->plugin_name) + 1; | ||
820 | if (pos->mime_type != NULL) | ||
821 | msize += strlen (pos->mime_type) + 1; | ||
822 | pos = pos->next; | ||
823 | } | ||
824 | size = (size_t) msize; | ||
825 | if (size != msize) | ||
826 | { | ||
827 | GNUNET_break (0); /* integer overflow */ | ||
828 | return GNUNET_SYSERR; | ||
829 | } | ||
830 | if (size >= GNUNET_MAX_MALLOC_CHECKED) | ||
831 | { | ||
832 | /* too large to be processed */ | ||
833 | return GNUNET_SYSERR; | ||
834 | } | ||
835 | ent = GNUNET_malloc (size); | ||
836 | mdata = (char *) &ent[md->item_count]; | ||
837 | off = size - (md->item_count * sizeof(struct MetaDataEntry)); | ||
838 | i = 0; | ||
839 | pos = md->items; | ||
840 | while (NULL != pos) | ||
841 | { | ||
842 | ent[i].type = htonl ((uint32_t) pos->type); | ||
843 | ent[i].format = htonl ((uint32_t) pos->format); | ||
844 | ent[i].data_size = htonl ((uint32_t) pos->data_size); | ||
845 | if (pos->plugin_name == NULL) | ||
846 | plen = 0; | ||
505 | else | 847 | else |
506 | { | 848 | plen = strlen (pos->plugin_name) + 1; |
507 | pos = size - sizeof (struct MetaDataHeader); | 849 | ent[i].plugin_name_len = htonl ( (uint32_t) plen); |
508 | } | 850 | if (pos->mime_type == NULL) |
509 | if (pos < size - sizeof (struct MetaDataHeader)) | 851 | mlen = 0; |
510 | { | 852 | else |
511 | hdr->version = htonl (HEADER_COMPRESSED); | 853 | mlen = strlen (pos->mime_type) + 1; |
512 | size = pos + sizeof (struct MetaDataHeader); | 854 | ent[i].mime_type_len = htonl ((uint32_t) mlen); |
513 | } | 855 | off -= pos->data_size; |
514 | if (size <= max) | 856 | memcpy (&mdata[off], pos->data, pos->data_size); |
515 | break; | 857 | off -= plen; |
516 | GNUNET_free (hdr); | 858 | memcpy (&mdata[off], pos->plugin_name, plen); |
517 | hdr = NULL; | 859 | off -= mlen; |
860 | memcpy (&mdata[off], pos->mime_type, mlen); | ||
861 | i++; | ||
862 | pos = pos->next; | ||
863 | } | ||
864 | GNUNET_assert (off == 0); | ||
518 | 865 | ||
519 | if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART) == 0) | 866 | left = size; |
520 | { | 867 | for (i=0;i<md->item_count;i++) |
521 | return GNUNET_SYSERR; /* does not fit! */ | 868 | { |
869 | comp = GNUNET_NO; | ||
870 | if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS)) | ||
871 | comp = try_compression ((const char*) &ent[i], | ||
872 | left, | ||
873 | &cdata, | ||
874 | &clen); | ||
875 | |||
876 | if ( (md->sbuf == NULL) && | ||
877 | (i == 0) ) | ||
878 | { | ||
879 | /* fill 'sbuf'; this "modifies" md, but since this is only | ||
880 | an internal cache we will cast away the 'const' instead | ||
881 | of making the API look strange. */ | ||
882 | vmd = (struct GNUNET_CONTAINER_MetaData*) md; | ||
883 | hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader)); | ||
884 | hdr->entries = htonl (md->item_count); | ||
885 | if (GNUNET_YES == comp) | ||
886 | { | ||
887 | hdr->size = htonl (clen); | ||
888 | hdr->version = htonl (2 | HEADER_COMPRESSED); | ||
889 | memcpy (&hdr[1], | ||
890 | cdata, | ||
891 | clen); | ||
892 | vmd->sbuf_size = clen + sizeof (struct MetaDataHeader); | ||
893 | } | ||
894 | else | ||
895 | { | ||
896 | hdr->size = htonl (left); | ||
897 | hdr->version = htonl (2); | ||
898 | memcpy (&hdr[1], | ||
899 | &ent[0], | ||
900 | left); | ||
901 | vmd->sbuf_size = left + sizeof (struct MetaDataHeader); | ||
902 | } | ||
903 | vmd->sbuf = (char*) hdr; | ||
904 | } | ||
905 | |||
906 | if ( ( (left + sizeof (struct MetaDataHeader)) <= max) || | ||
907 | ( (comp == GNUNET_YES) && | ||
908 | (clen <= max)) ) | ||
909 | { | ||
910 | /* success, this now fits! */ | ||
911 | if (GNUNET_YES == comp) | ||
912 | { | ||
913 | hdr = (struct MetaDataHeader*) *target; | ||
914 | if (hdr == NULL) | ||
915 | { | ||
916 | hdr = GNUNET_malloc (clen + sizeof (struct MetaDataHeader)); | ||
917 | *target = (char*) hdr; | ||
918 | } | ||
919 | hdr->version = htonl (2 | HEADER_COMPRESSED); | ||
920 | hdr->entries = htonl (md->item_count - i); | ||
921 | hdr->size = htonl (left); | ||
922 | memcpy (&target[sizeof(struct MetaDataHeader)], | ||
923 | cdata, | ||
924 | clen); | ||
925 | GNUNET_free (cdata); | ||
926 | GNUNET_free (ent); | ||
927 | return clen + sizeof (struct MetaDataHeader); | ||
928 | } | ||
929 | else | ||
930 | { | ||
931 | hdr = (struct MetaDataHeader*) target; | ||
932 | if (hdr == NULL) | ||
933 | { | ||
934 | hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader)); | ||
935 | *target = (char*) hdr; | ||
936 | } | ||
937 | hdr->version = htonl (2); | ||
938 | hdr->entries = htonl (md->item_count - i); | ||
939 | hdr->size = htonl (left); | ||
940 | memcpy (&target[sizeof(struct MetaDataHeader)], | ||
941 | &ent[i], | ||
942 | left); | ||
943 | GNUNET_free (ent); | ||
944 | return left + sizeof (struct MetaDataHeader); | ||
945 | } | ||
946 | } | ||
947 | |||
948 | if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART)) | ||
949 | { | ||
950 | /* does not fit! */ | ||
951 | GNUNET_free (ent); | ||
952 | return GNUNET_SYSERR; | ||
522 | } | 953 | } |
523 | /* partial serialization ok, try again with less meta-data */ | 954 | |
524 | if (size > 2 * max) | 955 | /* next iteration: ignore the corresponding meta data at the |
525 | ic = ic * 2 / 3; /* still far too big, make big reductions */ | 956 | end and try again without it */ |
526 | else | 957 | left -= sizeof (struct MetaDataEntry); |
527 | ic--; /* small steps, we're close */ | 958 | left -= pos->data_size; |
959 | if (pos->plugin_name != NULL) | ||
960 | left -= strlen (pos->plugin_name) + 1; | ||
961 | if (pos->mime_type != NULL) | ||
962 | left -= strlen (pos->mime_type) + 1; | ||
528 | } | 963 | } |
529 | GNUNET_assert (size <= max); | 964 | GNUNET_free (ent); |
530 | memcpy (target, hdr, size); | 965 | |
531 | GNUNET_free (hdr); | 966 | /* nothing fit, only write header! */ |
532 | /* extra check: deserialize! */ | 967 | hdr = (struct MetaDataHeader*) target; |
533 | #if EXTRA_CHECKS | 968 | if (hdr == NULL) |
534 | { | 969 | { |
535 | struct GNUNET_CONTAINER_MetaData *mdx; | 970 | hdr = GNUNET_malloc (sizeof (struct MetaDataHeader)); |
536 | mdx = GNUNET_CONTAINER_meta_data_deserialize (target, size); | 971 | *target = (char*) hdr; |
537 | GNUNET_assert (NULL != mdx); | 972 | } |
538 | GNUNET_CONTAINER_meta_data_destroy (mdx); | 973 | hdr->version = htonl (2); |
539 | } | 974 | hdr->entries = htonl (0); |
540 | #endif | 975 | hdr->size = htonl (0); |
541 | return size; | 976 | return sizeof (struct MetaDataHeader); |
542 | } | 977 | } |
543 | 978 | ||
979 | |||
544 | /** | 980 | /** |
545 | * Estimate (!) the size of the meta-data in | 981 | * Get the size of the full meta-data in serialized form. |
546 | * serialized form. The estimate MAY be higher | ||
547 | * than what is strictly needed. | ||
548 | * | 982 | * |
549 | * @param md metadata to inspect | 983 | * @param md metadata to inspect |
550 | * @param opt is it ok to just write SOME of the | ||
551 | * meta-data to match the size constraint, | ||
552 | * possibly discarding some data? | ||
553 | * @return number of bytes needed for serialization, -1 on error | 984 | * @return number of bytes needed for serialization, -1 on error |
554 | */ | 985 | */ |
555 | ssize_t | 986 | ssize_t |
556 | GNUNET_CONTAINER_meta_data_get_serialized_size (const struct | 987 | GNUNET_CONTAINER_meta_data_get_serialized_size (const struct GNUNET_CONTAINER_MetaData *md) |
557 | GNUNET_CONTAINER_MetaData * | ||
558 | md, | ||
559 | enum | ||
560 | GNUNET_CONTAINER_MetaDataSerializationOptions | ||
561 | opt) | ||
562 | { | 988 | { |
563 | struct MetaDataHeader *hdr; | 989 | ssize_t ret; |
564 | size_t size; | 990 | char *ptr; |
565 | size_t pos; | 991 | |
566 | uint32_t i; | 992 | if (md->sbuf != NULL) |
567 | size_t len; | 993 | return md->sbuf_size; |
568 | uint32_t ic; | 994 | ptr = NULL; |
995 | ret = GNUNET_CONTAINER_meta_data_serialize (md, | ||
996 | &ptr, | ||
997 | GNUNET_MAX_MALLOC_CHECKED, | ||
998 | GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); | ||
999 | if (ret != -1) | ||
1000 | GNUNET_free (ptr); | ||
1001 | return ret; | ||
1002 | } | ||
569 | 1003 | ||
570 | ic = md ? md->itemCount : 0; | 1004 | |
571 | size = sizeof (struct MetaDataHeader); | 1005 | /** |
572 | size += sizeof (uint32_t) * ic; | 1006 | * Decompress input, return the decompressed data |
573 | for (i = 0; i < ic; i++) | 1007 | * as output, set outputSize to the number of bytes |
574 | size += 1 + strlen (md->items[i].data); | 1008 | * that were found. |
575 | while (size % 8 != 0) | 1009 | * |
576 | size++; | 1010 | * @param input compressed data |
577 | hdr = GNUNET_malloc (size); | 1011 | * @param inputSize number of bytes in input |
578 | hdr->version = htonl (md == NULL ? 1 : 0); | 1012 | * @param outputSize expected size of the output |
579 | hdr->entries = htonl (ic); | 1013 | * @return NULL on error |
580 | for (i = 0; i < ic; i++) | 1014 | */ |
581 | ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); | 1015 | static char * |
582 | pos = sizeof (struct MetaDataHeader); | 1016 | decompress (const char *input, |
583 | pos += sizeof (uint32_t) * ic; | 1017 | size_t inputSize, |
584 | for (i = 0; i < ic; i++) | 1018 | size_t outputSize) |
585 | { | 1019 | { |
586 | len = strlen (md->items[i].data) + 1; | 1020 | char *output; |
587 | memcpy (&((char *) hdr)[pos], md->items[i].data, len); | 1021 | uLongf olen; |
588 | pos += len; | 1022 | |
589 | } | 1023 | olen = outputSize; |
590 | if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) | 1024 | output = GNUNET_malloc (olen); |
1025 | if (Z_OK == uncompress ((Bytef *) output, | ||
1026 | &olen, (const Bytef *) input, inputSize)) | ||
591 | { | 1027 | { |
592 | pos = | 1028 | return output; |
593 | tryCompression ((char *) &hdr[1], | ||
594 | size - sizeof (struct MetaDataHeader)); | ||
595 | } | 1029 | } |
596 | else | 1030 | else |
597 | { | 1031 | { |
598 | pos = size - sizeof (struct MetaDataHeader); | 1032 | GNUNET_free (output); |
1033 | return NULL; | ||
599 | } | 1034 | } |
600 | if (pos < size - sizeof (struct MetaDataHeader)) | ||
601 | size = pos + sizeof (struct MetaDataHeader); | ||
602 | GNUNET_free (hdr); | ||
603 | return size; | ||
604 | } | 1035 | } |
605 | 1036 | ||
606 | 1037 | ||
@@ -616,41 +1047,57 @@ struct GNUNET_CONTAINER_MetaData * | |||
616 | GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) | 1047 | GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) |
617 | { | 1048 | { |
618 | struct GNUNET_CONTAINER_MetaData *md; | 1049 | struct GNUNET_CONTAINER_MetaData *md; |
619 | const struct MetaDataHeader *hdr; | 1050 | struct MetaDataHeader hdr; |
1051 | struct MetaDataEntry ent; | ||
620 | uint32_t ic; | 1052 | uint32_t ic; |
1053 | uint32_t i; | ||
621 | char *data; | 1054 | char *data; |
622 | const char *cdata; | 1055 | const char *cdata; |
1056 | uint32_t version; | ||
623 | uint32_t dataSize; | 1057 | uint32_t dataSize; |
624 | int compressed; | 1058 | int compressed; |
625 | uint32_t i; | 1059 | size_t left; |
626 | size_t pos; | 1060 | uint32_t mlen; |
627 | size_t len; | 1061 | uint32_t plen; |
628 | uint32_t version; | 1062 | uint32_t dlen; |
1063 | const char *mdata; | ||
1064 | const char *meta_data; | ||
1065 | const char *plugin_name; | ||
1066 | const char *mime_type; | ||
1067 | enum EXTRACTOR_MetaFormat format; | ||
629 | 1068 | ||
630 | if (size < sizeof (struct MetaDataHeader)) | 1069 | if (size < sizeof (struct MetaDataHeader)) |
631 | return NULL; | 1070 | return NULL; |
632 | hdr = (const struct MetaDataHeader *) input; | 1071 | memcpy (&hdr, |
633 | version = ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_VERSION_MASK; | 1072 | input, |
1073 | sizeof (struct MetaDataHeader)); | ||
1074 | version = ntohl (hdr.version) & HEADER_VERSION_MASK; | ||
1075 | compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0; | ||
1076 | |||
634 | if (version == 1) | 1077 | if (version == 1) |
635 | return NULL; /* null pointer */ | 1078 | return NULL; /* null pointer */ |
636 | if (version != 0) | 1079 | if (version != 2) |
637 | { | 1080 | { |
638 | GNUNET_break_op (0); /* unsupported version */ | 1081 | GNUNET_break_op (0); /* unsupported version */ |
639 | return NULL; | 1082 | return NULL; |
640 | } | 1083 | } |
641 | ic = ntohl (MAKE_UNALIGNED (hdr->entries)); | 1084 | |
642 | compressed = | 1085 | ic = ntohl (hdr.entries); |
643 | (ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_COMPRESSED) != 0; | 1086 | dataSize = ntohl (hdr.size); |
1087 | if ((sizeof (struct MetaDataEntry) * ic) > dataSize) | ||
1088 | { | ||
1089 | GNUNET_break_op (0); | ||
1090 | return NULL; | ||
1091 | } | ||
1092 | |||
644 | if (compressed) | 1093 | if (compressed) |
645 | { | 1094 | { |
646 | dataSize = | 1095 | if (dataSize >= GNUNET_MAX_MALLOC_CHECKED) |
647 | ntohl (MAKE_UNALIGNED (hdr->size)) - sizeof (struct MetaDataHeader); | ||
648 | if (dataSize > 2 * 1042 * 1024) | ||
649 | { | 1096 | { |
650 | GNUNET_break (0); | 1097 | /* make sure we don't blow our memory limit because of a mal-formed |
651 | return NULL; /* only 2 MB allowed [to make sure we don't blow | 1098 | message... */ |
652 | our memory limit because of a mal-formed | 1099 | GNUNET_break_op (0); |
653 | message... ] */ | 1100 | return NULL; |
654 | } | 1101 | } |
655 | data = | 1102 | data = |
656 | decompress ((const char *) &input[sizeof (struct MetaDataHeader)], | 1103 | decompress ((const char *) &input[sizeof (struct MetaDataHeader)], |
@@ -665,84 +1112,93 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) | |||
665 | else | 1112 | else |
666 | { | 1113 | { |
667 | data = NULL; | 1114 | data = NULL; |
668 | cdata = (const char *) &hdr[1]; | 1115 | cdata = (const char *) &input[sizeof (struct MetaDataHeader)]; |
669 | dataSize = size - sizeof (struct MetaDataHeader); | 1116 | if (dataSize != size - sizeof (struct MetaDataHeader)) |
670 | if (size != ntohl (MAKE_UNALIGNED (hdr->size))) | ||
671 | { | 1117 | { |
672 | GNUNET_break (0); | 1118 | GNUNET_break_op (0); |
673 | return NULL; | 1119 | return NULL; |
674 | } | 1120 | } |
675 | } | 1121 | } |
676 | 1122 | ||
677 | if ((sizeof (uint32_t) * ic + ic) > dataSize) | ||
678 | { | ||
679 | GNUNET_break (0); | ||
680 | goto FAILURE; | ||
681 | } | ||
682 | if ((ic > 0) && (cdata[dataSize - 1] != '\0')) | ||
683 | { | ||
684 | GNUNET_break (0); | ||
685 | goto FAILURE; | ||
686 | } | ||
687 | |||
688 | md = GNUNET_CONTAINER_meta_data_create (); | 1123 | md = GNUNET_CONTAINER_meta_data_create (); |
689 | GNUNET_array_grow (md->items, md->itemCount, ic); | 1124 | left = dataSize - ic * sizeof (struct MetaDataEntry); |
690 | i = 0; | 1125 | mdata = &cdata[ic * sizeof (struct MetaDataEntry)]; |
691 | pos = sizeof (uint32_t) * ic; | 1126 | for (i=0;i<ic;i++) |
692 | while ((pos < dataSize) && (i < ic)) | ||
693 | { | 1127 | { |
694 | len = strlen (&cdata[pos]) + 1; | 1128 | memcpy (&ent, |
695 | md->items[i].type = (EXTRACTOR_KeywordType) | 1129 | &cdata[i * sizeof(struct MetaDataEntry)], |
696 | ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i])); | 1130 | sizeof (struct MetaDataEntry)); |
697 | md->items[i].data = GNUNET_strdup (&cdata[pos]); | 1131 | format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format); |
698 | pos += len; | 1132 | if ( (format != EXTRACTOR_METAFORMAT_UTF8) && |
699 | i++; | 1133 | (format != EXTRACTOR_METAFORMAT_C_STRING) && |
700 | } | 1134 | (format != EXTRACTOR_METAFORMAT_BINARY) ) |
701 | if (i < ic) | 1135 | { |
702 | { /* oops */ | 1136 | GNUNET_break_op (0); |
703 | GNUNET_CONTAINER_meta_data_destroy (md); | 1137 | break; |
704 | goto FAILURE; | 1138 | } |
705 | } | 1139 | dlen = ntohl (ent.data_size); |
706 | GNUNET_free_non_null (data); | 1140 | plen = ntohl (ent.plugin_name_len); |
707 | return md; | 1141 | mlen = ntohl (ent.mime_type_len); |
708 | FAILURE: | 1142 | if (dlen > left) |
709 | GNUNET_free_non_null (data); | 1143 | { |
710 | return NULL; /* size too small */ | 1144 | GNUNET_break_op (0); |
711 | } | 1145 | break; |
712 | 1146 | } | |
713 | /** | 1147 | left -= dlen; |
714 | * Test if two MDs are equal. | 1148 | meta_data = &mdata[left]; |
715 | * | 1149 | if ( (format == EXTRACTOR_METAFORMAT_UTF8) || |
716 | * @param md1 first value to check | 1150 | (format == EXTRACTOR_METAFORMAT_C_STRING) ) |
717 | * @param md2 other value to check | 1151 | { |
718 | * @return GNUNET_YES if they are equal | 1152 | if ( (dlen == 0) || |
719 | */ | 1153 | (mdata[left + dlen - 1] != '\0') ) |
720 | int | 1154 | { |
721 | GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData | 1155 | GNUNET_break_op (0); |
722 | *md1, | 1156 | break; |
723 | const struct GNUNET_CONTAINER_MetaData | 1157 | } |
724 | *md2) | 1158 | } |
725 | { | 1159 | if (plen > left) |
726 | uint32_t i; | 1160 | { |
727 | uint32_t j; | 1161 | GNUNET_break_op (0); |
728 | int found; | 1162 | break; |
1163 | } | ||
1164 | left -= plen; | ||
1165 | if ( (plen > 0) && | ||
1166 | (mdata[left + plen - 1] != '\0') ) | ||
1167 | { | ||
1168 | GNUNET_break_op (0); | ||
1169 | break; | ||
1170 | } | ||
1171 | if (plen == 0) | ||
1172 | plugin_name = NULL; | ||
1173 | else | ||
1174 | plugin_name = &mdata[left]; | ||
729 | 1175 | ||
730 | if (md1->itemCount != md2->itemCount) | 1176 | if (mlen > left) |
731 | return GNUNET_NO; | 1177 | { |
732 | for (i = 0; i < md1->itemCount; i++) | 1178 | GNUNET_break_op (0); |
733 | { | 1179 | break; |
734 | found = GNUNET_NO; | 1180 | } |
735 | for (j = 0; j < md2->itemCount; j++) | 1181 | left -= mlen; |
736 | if ((md1->items[i].type == md2->items[j].type) && | 1182 | if ( (mlen > 0) && |
737 | (0 == strcmp (md1->items[i].data, md2->items[j].data))) | 1183 | (mdata[left + mlen - 1] != '\0') ) |
738 | { | 1184 | { |
739 | found = GNUNET_YES; | 1185 | GNUNET_break_op (0); |
740 | break; | 1186 | break; |
741 | } | 1187 | } |
742 | if (found == GNUNET_NO) | 1188 | if (mlen == 0) |
743 | return GNUNET_NO; | 1189 | mime_type = NULL; |
1190 | else | ||
1191 | mime_type = &mdata[left]; | ||
1192 | GNUNET_CONTAINER_meta_data_insert (md, | ||
1193 | plugin_name, | ||
1194 | (enum EXTRACTOR_MetaType) ntohl (ent.type), | ||
1195 | format, | ||
1196 | mime_type, | ||
1197 | meta_data, | ||
1198 | dlen); | ||
744 | } | 1199 | } |
745 | return GNUNET_YES; | 1200 | GNUNET_free_non_null (data); |
1201 | return md; | ||
746 | } | 1202 | } |
747 | 1203 | ||
748 | 1204 | ||
diff --git a/src/util/pseudonym.c b/src/util/pseudonym.c index c974cc5a3..626e8ca99 100644 --- a/src/util/pseudonym.c +++ b/src/util/pseudonym.c | |||
@@ -300,15 +300,14 @@ GNUNET_PSEUDONYM_id_to_name (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
300 | { | 300 | { |
301 | if ((meta != NULL) && (name == NULL)) | 301 | if ((meta != NULL) && (name == NULL)) |
302 | name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta, | 302 | name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta, |
303 | EXTRACTOR_TITLE, | 303 | EXTRACTOR_METATYPE_TITLE, |
304 | EXTRACTOR_FILENAME, | 304 | EXTRACTOR_METATYPE_FILENAME, |
305 | EXTRACTOR_DESCRIPTION, | 305 | EXTRACTOR_METATYPE_DESCRIPTION, |
306 | EXTRACTOR_SUBJECT, | 306 | EXTRACTOR_METATYPE_SUBJECT, |
307 | EXTRACTOR_PUBLISHER, | 307 | EXTRACTOR_METATYPE_PUBLISHER, |
308 | EXTRACTOR_AUTHOR, | 308 | EXTRACTOR_METATYPE_AUTHOR_NAME, |
309 | EXTRACTOR_COMMENT, | 309 | EXTRACTOR_METATYPE_COMMENT, |
310 | EXTRACTOR_SUMMARY, | 310 | EXTRACTOR_METATYPE_SUMMARY, |
311 | EXTRACTOR_OWNER, | ||
312 | -1); | 311 | -1); |
313 | if (meta != NULL) | 312 | if (meta != NULL) |
314 | { | 313 | { |
@@ -545,11 +544,21 @@ GNUNET_PSEUDONYM_rank (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
545 | * @param data value of entry to insert | 544 | * @param data value of entry to insert |
546 | */ | 545 | */ |
547 | static int | 546 | static int |
548 | merge_meta_helper (void *cls, EXTRACTOR_KeywordType type, const char *data) | 547 | merge_meta_helper (void *cls, |
548 | const char *plugin_name, | ||
549 | enum EXTRACTOR_MetaType type, | ||
550 | enum EXTRACTOR_MetaFormat format, | ||
551 | const char *data_mime_type, | ||
552 | const char *data, | ||
553 | size_t data_len) | ||
549 | { | 554 | { |
550 | struct GNUNET_CONTAINER_MetaData *meta = cls; | 555 | struct GNUNET_CONTAINER_MetaData *meta = cls; |
551 | GNUNET_CONTAINER_meta_data_insert (meta, type, data); | 556 | |
552 | return GNUNET_OK; | 557 | (void) GNUNET_CONTAINER_meta_data_insert (meta, plugin_name, |
558 | type, format, | ||
559 | data_mime_type, | ||
560 | data, data_len); | ||
561 | return 0; | ||
553 | } | 562 | } |
554 | 563 | ||
555 | 564 | ||
@@ -581,7 +590,7 @@ GNUNET_PSEUDONYM_add (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
581 | if ((0 == STAT (fn, &sbuf)) && | 590 | if ((0 == STAT (fn, &sbuf)) && |
582 | (GNUNET_OK == read_info (cfg, id, &old, &ranking, &name))) | 591 | (GNUNET_OK == read_info (cfg, id, &old, &ranking, &name))) |
583 | { | 592 | { |
584 | GNUNET_CONTAINER_meta_data_get_contents (meta, &merge_meta_helper, old); | 593 | GNUNET_CONTAINER_meta_data_iterate (meta, &merge_meta_helper, old); |
585 | write_pseudonym_info (cfg, id, old, ranking, name); | 594 | write_pseudonym_info (cfg, id, old, ranking, name); |
586 | GNUNET_CONTAINER_meta_data_destroy (old); | 595 | GNUNET_CONTAINER_meta_data_destroy (old); |
587 | GNUNET_free_non_null (name); | 596 | GNUNET_free_non_null (name); |