aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2010-01-03 21:42:52 +0000
committerChristian Grothoff <christian@grothoff.org>2010-01-03 21:42:52 +0000
commit3c5249af8087b0b1a4f131a3e19a9759639ab5db (patch)
tree697ff622324a7d259dc59da63205f398661a56fa
parentebd1325d27e2062f3fb8e92766178ab0660c2dac (diff)
downloadgnunet-3c5249af8087b0b1a4f131a3e19a9759639ab5db.tar.gz
gnunet-3c5249af8087b0b1a4f131a3e19a9759639ab5db.zip
adapting main code base to work with libextractor 0.6 API; testcases will follow later
-rwxr-xr-xbootstrap5
-rw-r--r--configure.ac11
-rw-r--r--m4/libltdl-external.m475
-rw-r--r--src/fs/fs_directory.c49
-rw-r--r--src/fs/fs_file_information.c4
-rw-r--r--src/fs/fs_getopt.c31
-rw-r--r--src/fs/fs_publish.c14
-rw-r--r--src/fs/fs_uri.c31
-rw-r--r--src/fs/gnunet-directory.c21
-rw-r--r--src/fs/gnunet-publish.c66
-rw-r--r--src/fs/gnunet-search.c21
-rw-r--r--src/include/gnunet_container_lib.h162
-rw-r--r--src/include/gnunet_directories.h34
-rw-r--r--src/util/bio.c18
-rw-r--r--src/util/container_meta_data.c1214
-rw-r--r--src/util/pseudonym.c35
16 files changed, 1109 insertions, 682 deletions
diff --git a/bootstrap b/bootstrap
index 91516ddd9..668fac020 100755
--- a/bootstrap
+++ b/bootstrap
@@ -1,2 +1,5 @@
1#!/bin/sh 1#!/bin/sh
2autoreconf -fiv -I m4 2libtoolize --automake --copy --force
3aclocal
4automake
5autoreconf -f -i
diff --git a/configure.ac b/configure.ac
index 4a87c0067..41157ca29 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,5 +1,5 @@
1# This file is part of GNUnet. 1# This file is part of GNUnet.
2# (C) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Christian Grothoff (and other contributing authors) 2# (C) 2001--2010 Christian Grothoff (and other contributing authors)
3# 3#
4# GNUnet is free software; you can redistribute it and/or modify 4# GNUnet is free software; you can redistribute it and/or modify
5# it under the terms of the GNU General Public License as published 5# it under the terms of the GNU General Public License as published
@@ -38,7 +38,6 @@ AC_PROG_INSTALL
38AC_PROG_LN_S 38AC_PROG_LN_S
39AC_PROG_MAKE_SET 39AC_PROG_MAKE_SET
40AM_PROG_CC_C_O 40AM_PROG_CC_C_O
41LT_CONFIG_LTDL_DIR([libltdl])
42AC_CANONICAL_HOST 41AC_CANONICAL_HOST
43LT_INIT([disable-static dlopen win32-dll]) 42LT_INIT([disable-static dlopen win32-dll])
44LTDL_INIT 43LTDL_INIT
@@ -141,7 +140,7 @@ netbsd*)
141 AC_CHECK_LIB(intl, gettext) 140 AC_CHECK_LIB(intl, gettext)
142 LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols" 141 LDFLAGS="$LDFLAGS -no-undefined -Wl,--export-all-symbols"
143 LIBS="$LIBS -lws2_32 -lplibc" 142 LIBS="$LIBS -lws2_32 -lplibc"
144 CFLAGS="-mms-bitfields -I../../libltdl $CFLAGS" 143 CFLAGS="-mms-bitfields $CFLAGS"
145 build_target="mingw" 144 build_target="mingw"
146 AC_PROG_CXX 145 AC_PROG_CXX
147 LIBPREFIX=lib 146 LIBPREFIX=lib
@@ -212,14 +211,14 @@ AC_ARG_WITH(extractor,
212 ;; 211 ;;
213 yes) 212 yes)
214 AC_CHECK_HEADERS(extractor.h, 213 AC_CHECK_HEADERS(extractor.h,
215 AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], 214 AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
216 extractor=1)) 215 extractor=1))
217 ;; 216 ;;
218 *) 217 *)
219 LDFLAGS="-L$with_extractor/lib $LDFLAGS" 218 LDFLAGS="-L$with_extractor/lib $LDFLAGS"
220 CPPFLAGS="-I$with_extractor/include $CPPFLAGS" 219 CPPFLAGS="-I$with_extractor/include $CPPFLAGS"
221 AC_CHECK_HEADERS(extractor.h, 220 AC_CHECK_HEADERS(extractor.h,
222 AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], 221 AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
223 EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH" 222 EXT_LIB_PATH="-L$with_extractor/lib $EXT_LIB_PATH"
224 extractor=1)) 223 extractor=1))
225 ;; 224 ;;
@@ -227,7 +226,7 @@ AC_ARG_WITH(extractor,
227 ], 226 ],
228 [AC_MSG_RESULT([--with-extractor not specified]) 227 [AC_MSG_RESULT([--with-extractor not specified])
229 AC_CHECK_HEADERS(extractor.h, 228 AC_CHECK_HEADERS(extractor.h,
230 AC_CHECK_LIB([extractor], [EXTRACTOR_loadDefaultLibraries], 229 AC_CHECK_LIB([extractor], [EXTRACTOR_plugin_add_defaults],
231 extractor=1))]) 230 extractor=1))])
232if test "$extractor" != 1 231if test "$extractor" != 1
233then 232then
diff --git a/m4/libltdl-external.m4 b/m4/libltdl-external.m4
deleted file mode 100644
index 1c4ce09b2..000000000
--- a/m4/libltdl-external.m4
+++ /dev/null
@@ -1,75 +0,0 @@
1dnl Autoconf macro for an always external libltdl
2dnl Copyright (C) 2009 Heikki Lindholm
3dnl
4dnl This file is free software; as a special exception the author gives
5dnl unlimited permission to copy and/or distribute it, with or without
6dnl modifications, as long as this notice is preserved.
7dnl
8dnl This file is distributed in the hope that it will be useful, but
9dnl WITHOUT ANY WARRANTY, to the extent permitted by law; without even the
10dnl implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
11
12dnl AM_PATH_LIBLTDL(
13dnl [CHECK-SYMBOLS, [ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND ]]])
14dnl
15AC_DEFUN([AM_PATH_LIBLTDL],
16[ AC_ARG_WITH(libltdl-prefix,
17 AC_HELP_STRING([--with-libltdl-prefix=PFX],
18 [prefix where libltdl is installed (optional)]),
19 libltdl_prefix="$withval", libltdl_prefix="")
20 ltdl_save_CPPFLAGS="$CPPFLAGS"
21 ltdl_save_LDFLAGS="$LDFLAGS"
22 if test x$libltdl_prefix != x ; then
23 CPPFLAGS="-I$libltdl_prefix/include $CPPFLAGS"
24 LDFLAGS="-L$libltdl_prefix/lib -lltdl $LDFLAGS"
25 else
26 if test x"$LIBLTDL" = x ; then
27 LIBLTDL="-lltdl"
28 fi
29 CPPFLAGS="$LTDLINCL $CPPFLAGS"
30 LDFLAGS="$LIBLTDL $LDFLAGS"
31 fi
32
33 symbols_to_check=ifelse([$1], ,"ltdl_dlopen","$1")
34 ltdl_found=yes
35 AC_CHECK_HEADER([ltdl.h],
36 [
37 for sym in $symbols_to_check
38 do
39 AC_CHECK_DECL([$sym],
40 [AC_LINK_IFELSE(AC_LANG_CALL([], [$sym]),
41 [ltdl_found=yes],
42 [ltdl_found=no])],
43 [ltdl_found=no],
44 [AC_INCLUDES_DEFAULT
45 #include <ltdl.h>])
46 done
47 ],
48 [ltdl_found=no],
49 [AC_INCLUDES_DEFAULT]
50 )
51
52 if test x$libltdl_prefix != x ; then
53 LTDLINCL="-I$libltdl_prefix/include"
54 LIBLTDL="-L$libltdl_prefix/lib -lltdl"
55 else
56 if test x"$LIBLTDL" = x ; then
57 LIBLTDL="-lltdl"
58 fi
59 fi
60 CPPFLAGS="$ltdl_save_CPPFLAGS"
61 LDFLAGS="$ltdl_save_LDFLAGS"
62
63 AC_MSG_CHECKING(for libltdl with symbols $symbols_to_check)
64 if test $ltdl_found = yes; then
65 AC_MSG_RESULT(yes)
66 ifelse([$2], , :, [$2])
67 else
68 LTDLINCL=""
69 LIBLTDL=""
70 AC_MSG_RESULT(no)
71 ifelse([$3], , :, [$3])
72 fi
73 AC_SUBST(LTDLINCL)
74 AC_SUBST(LIBLTDL)
75])
diff --git a/src/fs/fs_directory.c b/src/fs/fs_directory.c
index e037130c8..48578c650 100644
--- a/src/fs/fs_directory.c
+++ b/src/fs/fs_directory.c
@@ -36,10 +36,6 @@
36#include "gnunet_fs_service.h" 36#include "gnunet_fs_service.h"
37#include "fs.h" 37#include "fs.h"
38 38
39#ifndef EXTRACTOR_GNUNET_FULL_DATA
40#define EXTRACTOR_GNUNET_FULL_DATA 137
41#endif
42
43/** 39/**
44 * String that is used to indicate that a file 40 * String that is used to indicate that a file
45 * is a GNUnet directory. 41 * is a GNUnet directory.
@@ -60,7 +56,7 @@ GNUNET_FS_meta_data_test_for_directory (const struct GNUNET_CONTAINER_MetaData *
60 char *mime; 56 char *mime;
61 int ret; 57 int ret;
62 58
63 mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); 59 mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
64 if (mime == NULL) 60 if (mime == NULL)
65 return GNUNET_SYSERR; 61 return GNUNET_SYSERR;
66 ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO; 62 ret = (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO;
@@ -80,7 +76,7 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md)
80{ 76{
81 char *mime; 77 char *mime;
82 78
83 mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_MIMETYPE); 79 mime = GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
84 if (mime != NULL) 80 if (mime != NULL)
85 { 81 {
86 GNUNET_break (0 == strcmp (mime, 82 GNUNET_break (0 == strcmp (mime,
@@ -89,8 +85,12 @@ GNUNET_FS_meta_data_make_directory (struct GNUNET_CONTAINER_MetaData *md)
89 return; 85 return;
90 } 86 }
91 GNUNET_CONTAINER_meta_data_insert (md, 87 GNUNET_CONTAINER_meta_data_insert (md,
92 EXTRACTOR_MIMETYPE, 88 "<gnunet>",
93 GNUNET_FS_DIRECTORY_MIME); 89 EXTRACTOR_METATYPE_MIMETYPE,
90 EXTRACTOR_METAFORMAT_UTF8,
91 "text/plain",
92 GNUNET_FS_DIRECTORY_MIME,
93 strlen (GNUNET_FS_DIRECTORY_MIME)+1);
94} 94}
95 95
96 96
@@ -225,9 +225,9 @@ GNUNET_FS_directory_list_contents (size_t size,
225 } 225 }
226 pos += mdSize; 226 pos += mdSize;
227 filename = GNUNET_CONTAINER_meta_data_get_by_type (md, 227 filename = GNUNET_CONTAINER_meta_data_get_by_type (md,
228 EXTRACTOR_FILENAME); 228 EXTRACTOR_METATYPE_FILENAME);
229 file_data = GNUNET_CONTAINER_meta_data_get_by_type (md, 229 file_data = GNUNET_CONTAINER_meta_data_get_by_type (md,
230 EXTRACTOR_GNUNET_FULL_DATA); 230 EXTRACTOR_METATYPE_GNUNET_FULL_DATA);
231 if (dep != NULL) 231 if (dep != NULL)
232 dep (dep_cls, 232 dep (dep_cls,
233 filename, 233 filename,
@@ -322,6 +322,7 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
322 size_t mdxs; 322 size_t mdxs;
323 char *uris; 323 char *uris;
324 char *ser; 324 char *ser;
325 char *sptr;
325 size_t slen; 326 size_t slen;
326 struct GNUNET_CONTAINER_MetaData *meta; 327 struct GNUNET_CONTAINER_MetaData *meta;
327 const struct GNUNET_CONTAINER_MetaData *meta_use; 328 const struct GNUNET_CONTAINER_MetaData *meta_use;
@@ -340,25 +341,24 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
340 fsize = 0; /* not given */ 341 fsize = 0; /* not given */
341 if (fsize > MAX_INLINE_SIZE) 342 if (fsize > MAX_INLINE_SIZE)
342 fsize = 0; /* too large */ 343 fsize = 0; /* too large */
343 if ( (NULL == data) ||
344 (NULL != memchr (data, 0, fsize)) )
345 fsize = 0; /* must not have 0's in data! */
346 uris = GNUNET_FS_uri_to_string (uri); 344 uris = GNUNET_FS_uri_to_string (uri);
347 slen = strlen (uris) + 1; 345 slen = strlen (uris) + 1;
348 mds = 346 mds =
349 GNUNET_CONTAINER_meta_data_get_serialized_size (md, 347 GNUNET_CONTAINER_meta_data_get_serialized_size (md);
350 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
351 meta_use = md; 348 meta_use = md;
352 meta = NULL; 349 meta = NULL;
353 if (fsize > 0) 350 if (fsize > 0)
354 { 351 {
355 meta = GNUNET_CONTAINER_meta_data_duplicate (md); 352 meta = GNUNET_CONTAINER_meta_data_duplicate (md);
356 GNUNET_CONTAINER_meta_data_insert (meta, 353 GNUNET_CONTAINER_meta_data_insert (meta,
357 EXTRACTOR_GNUNET_FULL_DATA, 354 "<gnunet>",
358 data); 355 EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
356 EXTRACTOR_METAFORMAT_BINARY,
357 NULL,
358 data,
359 fsize);
359 mdxs = 360 mdxs =
360 GNUNET_CONTAINER_meta_data_get_serialized_size (meta, 361 GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
361 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
362 if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE == 362 if ( (slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
363 (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE) 363 (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE)
364 { 364 {
@@ -376,8 +376,9 @@ GNUNET_FS_directory_builder_add (struct GNUNET_FS_DirectoryBuilder *bld,
376 ser = (char*) &e[1]; 376 ser = (char*) &e[1];
377 memcpy (ser, uris, slen); 377 memcpy (ser, uris, slen);
378 GNUNET_free (uris); 378 GNUNET_free (uris);
379 sptr = &ser[slen + sizeof(uint32_t)];
379 ret = GNUNET_CONTAINER_meta_data_serialize (meta_use, 380 ret = GNUNET_CONTAINER_meta_data_serialize (meta_use,
380 &ser[slen + sizeof(uint32_t)], 381 &sptr,
381 mds, 382 mds,
382 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); 383 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
383 if (NULL != meta) 384 if (NULL != meta)
@@ -503,6 +504,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
503 void **rdata) 504 void **rdata)
504{ 505{
505 char *data; 506 char *data;
507 char *sptr;
506 size_t *sizes; 508 size_t *sizes;
507 unsigned int *perm; 509 unsigned int *perm;
508 unsigned int i; 510 unsigned int i;
@@ -516,8 +518,7 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
516 uint32_t big; 518 uint32_t big;
517 519
518 size = 8 + sizeof (uint32_t); 520 size = 8 + sizeof (uint32_t);
519 size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta, 521 size += GNUNET_CONTAINER_meta_data_get_serialized_size (bld->meta);
520 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
521 sizes = NULL; 522 sizes = NULL;
522 perm = NULL; 523 perm = NULL;
523 bes = NULL; 524 bes = NULL;
@@ -560,9 +561,9 @@ GNUNET_FS_directory_builder_finish (struct GNUNET_FS_DirectoryBuilder *bld,
560 memcpy (data, GNUNET_DIRECTORY_MAGIC, 8); 561 memcpy (data, GNUNET_DIRECTORY_MAGIC, 8);
561 off = 8; 562 off = 8;
562 563
564 sptr = &data[off + sizeof (uint32_t)];
563 ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta, 565 ret = GNUNET_CONTAINER_meta_data_serialize (bld->meta,
564 &data[off + 566 &sptr,
565 sizeof (uint32_t)],
566 size - off - sizeof (uint32_t), 567 size - off - sizeof (uint32_t),
567 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL); 568 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
568 GNUNET_assert (ret != -1); 569 GNUNET_assert (ret != -1);
diff --git a/src/fs/fs_file_information.c b/src/fs/fs_file_information.c
index c28635198..9c193f88b 100644
--- a/src/fs/fs_file_information.c
+++ b/src/fs/fs_file_information.c
@@ -364,7 +364,7 @@ struct DirScanCls
364 /** 364 /**
365 * Metadata extractors to use. 365 * Metadata extractors to use.
366 */ 366 */
367 struct EXTRACTOR_Extractor *extractors; 367 struct EXTRACTOR_PluginList *extractors;
368 368
369 /** 369 /**
370 * Function to call on each directory entry. 370 * Function to call on each directory entry.
@@ -518,7 +518,7 @@ GNUNET_FS_directory_scanner_default (void *cls,
518 void *proc_cls, 518 void *proc_cls,
519 char **emsg) 519 char **emsg)
520{ 520{
521 struct EXTRACTOR_Extractor *ex = cls; 521 struct EXTRACTOR_PluginList *ex = cls;
522 struct DirScanCls dsc; 522 struct DirScanCls dsc;
523 523
524 dsc.extractors = ex; 524 dsc.extractors = ex;
diff --git a/src/fs/fs_getopt.c b/src/fs/fs_getopt.c
index a0d232641..f985ed8d0 100644
--- a/src/fs/fs_getopt.c
+++ b/src/fs/fs_getopt.c
@@ -131,7 +131,7 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext*
131 131
132{ 132{
133 struct GNUNET_CONTAINER_MetaData **mm = scls; 133 struct GNUNET_CONTAINER_MetaData **mm = scls;
134 EXTRACTOR_KeywordType type; 134 enum EXTRACTOR_MetaType type;
135 const char *typename; 135 const char *typename;
136 const char *typename_i18n; 136 const char *typename_i18n;
137 struct GNUNET_CONTAINER_MetaData *meta; 137 struct GNUNET_CONTAINER_MetaData *meta;
@@ -151,17 +151,23 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext*
151 "utf-8" 151 "utf-8"
152#endif 152#endif
153 ); 153 );
154 type = EXTRACTOR_getHighestKeywordTypeNumber (); 154 type = EXTRACTOR_metatype_get_max ();
155 while (type > 0) 155 while (type > 0)
156 { 156 {
157 type--; 157 type--;
158 typename = EXTRACTOR_getKeywordTypeAsString (type); 158 typename = EXTRACTOR_metatype_to_string (type);
159 typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename); 159 typename_i18n = dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, typename);
160 if ((strlen (tmp) >= strlen (typename) + 1) && 160 if ((strlen (tmp) >= strlen (typename) + 1) &&
161 (tmp[strlen (typename)] == ':') && 161 (tmp[strlen (typename)] == ':') &&
162 (0 == strncmp (typename, tmp, strlen (typename)))) 162 (0 == strncmp (typename, tmp, strlen (typename))))
163 { 163 {
164 GNUNET_CONTAINER_meta_data_insert (meta, type, &tmp[strlen (typename) + 1]); 164 GNUNET_CONTAINER_meta_data_insert (meta,
165 "<gnunet>",
166 type,
167 EXTRACTOR_METAFORMAT_UTF8,
168 "text/plain",
169 &tmp[strlen (typename) + 1],
170 strlen (&tmp[strlen (typename) + 1])+1);
165 GNUNET_free (tmp); 171 GNUNET_free (tmp);
166 tmp = NULL; 172 tmp = NULL;
167 break; 173 break;
@@ -170,8 +176,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext*
170 (tmp[strlen (typename_i18n)] == ':') && 176 (tmp[strlen (typename_i18n)] == ':') &&
171 (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n)))) 177 (0 == strncmp (typename_i18n, tmp, strlen (typename_i18n))))
172 { 178 {
173 GNUNET_CONTAINER_meta_data_insert (meta, type, 179 GNUNET_CONTAINER_meta_data_insert (meta,
174 &tmp[strlen (typename_i18n) + 1]); 180 "<gnunet>",
181 type,
182 EXTRACTOR_METAFORMAT_UTF8,
183 "text/plain",
184 &tmp[strlen (typename_i18n) + 1],
185 strlen (&tmp[strlen (typename_i18n) + 1]) + 1);
175 GNUNET_free (tmp); 186 GNUNET_free (tmp);
176 tmp = NULL; 187 tmp = NULL;
177 break; 188 break;
@@ -179,7 +190,13 @@ GNUNET_FS_getopt_set_metadata (struct GNUNET_GETOPT_CommandLineProcessorContext*
179 } 190 }
180 if (tmp != NULL) 191 if (tmp != NULL)
181 { 192 {
182 GNUNET_CONTAINER_meta_data_insert (meta, EXTRACTOR_UNKNOWN, tmp); 193 GNUNET_CONTAINER_meta_data_insert (meta,
194 "<gnunet>",
195 EXTRACTOR_METATYPE_UNKNOWN,
196 EXTRACTOR_METAFORMAT_UTF8,
197 "text/plain",
198 tmp,
199 strlen(tmp) + 1);
183 GNUNET_free (tmp); 200 GNUNET_free (tmp);
184 printf (_ 201 printf (_
185 ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"), 202 ("Unknown metadata type in metadata option `%s'. Using metadata type `unknown' instead.\n"),
diff --git a/src/fs/fs_publish.c b/src/fs/fs_publish.c
index 46b342cab..02ddb2a0d 100644
--- a/src/fs/fs_publish.c
+++ b/src/fs/fs_publish.c
@@ -842,7 +842,7 @@ do_upload (void *cls,
842 while (NULL != p->dir) 842 while (NULL != p->dir)
843 { 843 {
844 fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta, 844 fn = GNUNET_CONTAINER_meta_data_get_by_type (p->meta,
845 EXTRACTOR_FILENAME); 845 EXTRACTOR_METATYPE_FILENAME);
846 p = p->dir; 846 p = p->dir;
847 GNUNET_asprintf (&p->emsg, 847 GNUNET_asprintf (&p->emsg,
848 _("Recursive upload failed at `%s'"), 848 _("Recursive upload failed at `%s'"),
@@ -1305,6 +1305,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1305 char *uris; 1305 char *uris;
1306 size_t size; 1306 size_t size;
1307 char *kbe; 1307 char *kbe;
1308 char *sptr;
1308 1309
1309 pkc = GNUNET_malloc (sizeof (struct PublishKskContext)); 1310 pkc = GNUNET_malloc (sizeof (struct PublishKskContext));
1310 pkc->h = h; 1311 pkc->h = h;
@@ -1324,8 +1325,7 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1324 return; 1325 return;
1325 } 1326 }
1326 } 1327 }
1327 pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, 1328 pkc->mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1328 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1329 GNUNET_assert (pkc->mdsize >= 0); 1329 GNUNET_assert (pkc->mdsize >= 0);
1330 uris = GNUNET_FS_uri_to_string (uri); 1330 uris = GNUNET_FS_uri_to_string (uri);
1331 pkc->slen = strlen (uris) + 1; 1331 pkc->slen = strlen (uris) + 1;
@@ -1339,8 +1339,9 @@ GNUNET_FS_publish_ksk (struct GNUNET_FS_Handle *h,
1339 kbe = (char *) &pkc->kb[1]; 1339 kbe = (char *) &pkc->kb[1];
1340 memcpy (kbe, uris, pkc->slen); 1340 memcpy (kbe, uris, pkc->slen);
1341 GNUNET_free (uris); 1341 GNUNET_free (uris);
1342 sptr = &kbe[pkc->slen];
1342 pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, 1343 pkc->mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1343 &kbe[pkc->slen], 1344 &sptr,
1344 pkc->mdsize, 1345 pkc->mdsize,
1345 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); 1346 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1346 if (pkc->mdsize == -1) 1347 if (pkc->mdsize == -1)
@@ -1480,8 +1481,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
1480 if (update == NULL) 1481 if (update == NULL)
1481 update = ""; 1482 update = "";
1482 nidlen = strlen (update) + 1; 1483 nidlen = strlen (update) + 1;
1483 mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta, 1484 mdsize = GNUNET_CONTAINER_meta_data_get_serialized_size (meta);
1484 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1485 1485
1486 size = sizeof (struct SBlock) + slen + nidlen + mdsize; 1486 size = sizeof (struct SBlock) + slen + nidlen + mdsize;
1487 if (size > MAX_SBLOCK_SIZE) 1487 if (size > MAX_SBLOCK_SIZE)
@@ -1496,7 +1496,7 @@ GNUNET_FS_publish_sks (struct GNUNET_FS_Handle *h,
1496 memcpy (dest, uris, slen); 1496 memcpy (dest, uris, slen);
1497 dest += slen; 1497 dest += slen;
1498 mdsize = GNUNET_CONTAINER_meta_data_serialize (meta, 1498 mdsize = GNUNET_CONTAINER_meta_data_serialize (meta,
1499 dest, 1499 &dest,
1500 mdsize, 1500 mdsize,
1501 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART); 1501 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
1502 if (mdsize == -1) 1502 if (mdsize == -1)
diff --git a/src/fs/fs_uri.c b/src/fs/fs_uri.c
index 02310a080..16b6e26e9 100644
--- a/src/fs/fs_uri.c
+++ b/src/fs/fs_uri.c
@@ -1469,19 +1469,34 @@ GNUNET_FS_uri_test_loc (const struct GNUNET_FS_Uri *uri)
1469 * Adds it to the URI. 1469 * Adds it to the URI.
1470 * 1470 *
1471 * @param cls URI to update 1471 * @param cls URI to update
1472 * @param type type of the meta data 1472 * @param plugin_name name of the plugin that produced this value;
1473 * @param data value of the meta data 1473 * special values can be used (i.e. '<zlib>' for zlib being
1474 * @return GNUNET_OK (always) 1474 * used in the main libextractor library and yielding
1475 * meta data).
1476 * @param type libextractor-type describing the meta data
1477 * @param format basic format information about data
1478 * @param data_mime_type mime-type of data (not of the original file);
1479 * can be NULL (if mime-type is not known)
1480 * @param data actual meta-data found
1481 * @param data_len number of bytes in data
1482 * @return 0 (always)
1475 */ 1483 */
1476static int 1484static int
1477gather_uri_data (void *cls, 1485gather_uri_data (void *cls,
1478 EXTRACTOR_KeywordType type, 1486 const char *plugin_name,
1479 const char *data) 1487 enum EXTRACTOR_MetaType type,
1488 enum EXTRACTOR_MetaFormat format,
1489 const char *data_mime_type,
1490 const char *data,
1491 size_t data_len)
1480{ 1492{
1481 struct GNUNET_FS_Uri *uri = cls; 1493 struct GNUNET_FS_Uri *uri = cls;
1482 char *nkword; 1494 char *nkword;
1483 int j; 1495 int j;
1484 1496
1497 if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
1498 (format != EXTRACTOR_METAFORMAT_C_STRING) )
1499 return 0;
1485 for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--) 1500 for (j = uri->data.ksk.keywordCount - 1; j >= 0; j--)
1486 if (0 == strcmp (&uri->data.ksk.keywords[j][1], data)) 1501 if (0 == strcmp (&uri->data.ksk.keywords[j][1], data))
1487 return GNUNET_OK; 1502 return GNUNET_OK;
@@ -1489,7 +1504,7 @@ gather_uri_data (void *cls,
1489 strcpy (nkword, " "); /* not mandatory */ 1504 strcpy (nkword, " "); /* not mandatory */
1490 strcat (nkword, data); 1505 strcat (nkword, data);
1491 uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword; 1506 uri->data.ksk.keywords[uri->data.ksk.keywordCount++] = nkword;
1492 return GNUNET_OK; 1507 return 0;
1493} 1508}
1494 1509
1495 1510
@@ -1514,8 +1529,8 @@ GNUNET_FS_uri_ksk_create_from_meta_data (const struct GNUNET_CONTAINER_MetaData
1514 ret->data.ksk.keywords = NULL; 1529 ret->data.ksk.keywords = NULL;
1515 ret->data.ksk.keywords 1530 ret->data.ksk.keywords
1516 = GNUNET_malloc (sizeof (char *) * 1531 = GNUNET_malloc (sizeof (char *) *
1517 GNUNET_CONTAINER_meta_data_get_contents (md, NULL, NULL)); 1532 GNUNET_CONTAINER_meta_data_iterate (md, NULL, NULL));
1518 GNUNET_CONTAINER_meta_data_get_contents (md, &gather_uri_data, ret); 1533 GNUNET_CONTAINER_meta_data_iterate (md, &gather_uri_data, ret);
1519 return ret; 1534 return ret;
1520 1535
1521} 1536}
diff --git a/src/fs/gnunet-directory.c b/src/fs/gnunet-directory.c
index 6201c5dbb..3ea76985e 100644
--- a/src/fs/gnunet-directory.c
+++ b/src/fs/gnunet-directory.c
@@ -36,13 +36,20 @@ static int ret;
36 */ 36 */
37static int 37static int
38item_printer (void *cls, 38item_printer (void *cls,
39 EXTRACTOR_KeywordType type, 39 const char *plugin_name,
40 const char *data) 40 enum EXTRACTOR_MetaType type,
41 enum EXTRACTOR_MetaFormat format,
42 const char *data_mime_type,
43 const char *data,
44 size_t data_size)
41{ 45{
46 if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
47 (format != EXTRACTOR_METAFORMAT_C_STRING) )
48 return 0;
42 printf ("\t%20s: %s\n", 49 printf ("\t%20s: %s\n",
43 dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, 50 dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN,
44 EXTRACTOR_getKeywordTypeAsString (type)), data); 51 EXTRACTOR_metatype_to_string (type)), data);
45 return GNUNET_OK; 52 return 0;
46} 53}
47 54
48 55
@@ -75,9 +82,9 @@ print_entry (void *cls,
75 string = GNUNET_FS_uri_to_string (uri); 82 string = GNUNET_FS_uri_to_string (uri);
76 printf ("%s:\n", string); 83 printf ("%s:\n", string);
77 GNUNET_free (string); 84 GNUNET_free (string);
78 GNUNET_CONTAINER_meta_data_get_contents (meta, 85 GNUNET_CONTAINER_meta_data_iterate (meta,
79 &item_printer, 86 &item_printer,
80 NULL); 87 NULL);
81} 88}
82 89
83 90
diff --git a/src/fs/gnunet-publish.c b/src/fs/gnunet-publish.c
index e3918f6fb..65f437f11 100644
--- a/src/fs/gnunet-publish.c
+++ b/src/fs/gnunet-publish.c
@@ -140,43 +140,57 @@ progress_cb (void *cls,
140 * @param cls closure 140 * @param cls closure
141 * @param type type of the meta data 141 * @param type type of the meta data
142 * @param data value of the meta data 142 * @param data value of the meta data
143 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort 143 * @return always 0
144 */ 144 */
145static int 145static int
146meta_printer (void *cls, 146meta_printer (void *cls,
147 EXTRACTOR_KeywordType type, 147 const char *plugin_name,
148 const char *data) 148 enum EXTRACTOR_MetaType type,
149 enum EXTRACTOR_MetaFormat format,
150 const char *data_mime_type,
151 const char *data,
152 size_t data_size)
149{ 153{
150 if ( (type == EXTRACTOR_FILENAME) || 154 if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
151 (EXTRACTOR_isBinaryType (type)) ) 155 (format != EXTRACTOR_METAFORMAT_C_STRING) )
152 return GNUNET_OK; 156 return 0;
157 if (type == EXTRACTOR_METATYPE_FILENAME)
158 return 0;
153 fprintf (stdout, 159 fprintf (stdout,
154 "%s - %s", 160 "%s - %s",
155 EXTRACTOR_getKeywordTypeAsString (type), 161 EXTRACTOR_metatype_to_string (type),
156 data); 162 data);
157 return GNUNET_OK; 163 return 0;
158} 164}
159 165
160 166
161/** 167/**
162 * Merge metadata entries (except binary 168 * Merge metadata entries.
163 * metadata).
164 * 169 *
165 * @param cls closure, target metadata structure 170 * @param cls closure, target metadata structure
166 * @param type type of the meta data 171 * @param type type of the meta data
167 * @param data value of the meta data 172 * @param data value of the meta data
168 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort 173 * @return always 0
169 */ 174 */
170static int 175static int
171meta_merger (void *cls, 176meta_merger (void *cls,
172 EXTRACTOR_KeywordType type, 177 const char *plugin_name,
173 const char *data) 178 enum EXTRACTOR_MetaType type,
179 enum EXTRACTOR_MetaFormat format,
180 const char *data_mime_type,
181 const char *data,
182 size_t data_size)
174{ 183{
175 struct GNUNET_CONTAINER_MetaData *m = cls; 184 struct GNUNET_CONTAINER_MetaData *m = cls;
185
176 GNUNET_CONTAINER_meta_data_insert (m, 186 GNUNET_CONTAINER_meta_data_insert (m,
187 plugin_name,
177 type, 188 type,
178 data); 189 format,
179 return GNUNET_OK; 190 data_mime_type,
191 data,
192 data_size);
193 return 0;
180} 194}
181 195
182 196
@@ -227,16 +241,16 @@ publish_inspector (void *cls,
227 } 241 }
228 if (NULL != meta) 242 if (NULL != meta)
229 { 243 {
230 GNUNET_CONTAINER_meta_data_get_contents (meta, 244 GNUNET_CONTAINER_meta_data_iterate (meta,
231 &meta_merger, 245 &meta_merger,
232 m); 246 m);
233 GNUNET_CONTAINER_meta_data_destroy (meta); 247 GNUNET_CONTAINER_meta_data_destroy (meta);
234 meta = NULL; 248 meta = NULL;
235 } 249 }
236 if (extract_only) 250 if (extract_only)
237 { 251 {
238 fn = GNUNET_CONTAINER_meta_data_get_by_type (meta, 252 fn = GNUNET_CONTAINER_meta_data_get_by_type (meta,
239 EXTRACTOR_FILENAME); 253 EXTRACTOR_METATYPE_FILENAME);
240 fs = GNUNET_STRINGS_byte_size_fancy (length); 254 fs = GNUNET_STRINGS_byte_size_fancy (length);
241 fprintf (stdout, 255 fprintf (stdout,
242 _("Keywords for file `%s' (%s)\n"), 256 _("Keywords for file `%s' (%s)\n"),
@@ -244,9 +258,9 @@ publish_inspector (void *cls,
244 fs); 258 fs);
245 GNUNET_free (fn); 259 GNUNET_free (fn);
246 GNUNET_free (fs); 260 GNUNET_free (fs);
247 GNUNET_CONTAINER_meta_data_get_contents (meta, 261 GNUNET_CONTAINER_meta_data_iterate (meta,
248 &meta_printer, 262 &meta_printer,
249 NULL); 263 NULL);
250 fprintf (stdout, "\n"); 264 fprintf (stdout, "\n");
251 } 265 }
252 if (GNUNET_FS_meta_data_test_for_directory (meta)) 266 if (GNUNET_FS_meta_data_test_for_directory (meta))
@@ -275,7 +289,7 @@ run (void *cls,
275{ 289{
276 struct GNUNET_FS_FileInformation *fi; 290 struct GNUNET_FS_FileInformation *fi;
277 struct GNUNET_FS_Namespace *namespace; 291 struct GNUNET_FS_Namespace *namespace;
278 EXTRACTOR_ExtractorList *l; 292 struct EXTRACTOR_PluginList *l;
279 char *ex; 293 char *ex;
280 char *emsg; 294 char *emsg;
281 295
@@ -377,13 +391,13 @@ run (void *cls,
377 l = NULL; 391 l = NULL;
378 if (! disable_extractor) 392 if (! disable_extractor)
379 { 393 {
380 l = EXTRACTOR_loadDefaultLibraries (); 394 l = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
381 if (GNUNET_OK == 395 if (GNUNET_OK ==
382 GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", 396 GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS",
383 &ex)) 397 &ex))
384 { 398 {
385 if (strlen (ex) > 0) 399 if (strlen (ex) > 0)
386 l = EXTRACTOR_loadConfigLibraries (l, ex); 400 l = EXTRACTOR_plugin_add_config (l, ex, EXTRACTOR_OPTION_DEFAULT_POLICY);
387 GNUNET_free (ex); 401 GNUNET_free (ex);
388 } 402 }
389 } 403 }
@@ -396,7 +410,7 @@ run (void *cls,
396 priority, 410 priority,
397 GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION), 411 GNUNET_TIME_relative_to_absolute (DEFAULT_EXPIRATION),
398 &emsg); 412 &emsg);
399 EXTRACTOR_removeAll (l); 413 EXTRACTOR_plugin_remove_all (l);
400 if (fi == NULL) 414 if (fi == NULL)
401 { 415 {
402 fprintf (stderr, 416 fprintf (stderr,
diff --git a/src/fs/gnunet-search.c b/src/fs/gnunet-search.c
index 067d77159..4836f9d83 100644
--- a/src/fs/gnunet-search.c
+++ b/src/fs/gnunet-search.c
@@ -45,12 +45,19 @@ static int verbose;
45 45
46static int 46static int
47item_printer (void *cls, 47item_printer (void *cls,
48 EXTRACTOR_KeywordType type, 48 const char *plugin_name,
49 const char *data) 49 enum EXTRACTOR_MetaType type,
50 enum EXTRACTOR_MetaFormat format,
51 const char *data_mime_type,
52 const char *data,
53 size_t data_size)
50{ 54{
55 if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
56 (format != EXTRACTOR_METAFORMAT_C_STRING) )
57 return 0;
51 printf ("\t%20s: %s\n", 58 printf ("\t%20s: %s\n",
52 dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN, 59 dgettext (LIBEXTRACTOR_GETTEXT_DOMAIN,
53 EXTRACTOR_getKeywordTypeAsString (type)), 60 EXTRACTOR_metatype_to_string (type)),
54 data); 61 data);
55 return GNUNET_OK; 62 return GNUNET_OK;
56} 63}
@@ -86,7 +93,7 @@ progress_cb (void *cls,
86 printf ("%s:\n", uri); 93 printf ("%s:\n", uri);
87 filename = 94 filename =
88 GNUNET_CONTAINER_meta_data_get_by_type (info->value.search.specifics.result.meta, 95 GNUNET_CONTAINER_meta_data_get_by_type (info->value.search.specifics.result.meta,
89 EXTRACTOR_FILENAME); 96 EXTRACTOR_METATYPE_FILENAME);
90 if (filename != NULL) 97 if (filename != NULL)
91 { 98 {
92 while (NULL != (dotdot = strstr (filename, ".."))) 99 while (NULL != (dotdot = strstr (filename, "..")))
@@ -98,9 +105,9 @@ progress_cb (void *cls,
98 else 105 else
99 printf ("gnunet-download %s\n", uri); 106 printf ("gnunet-download %s\n", uri);
100 if (verbose) 107 if (verbose)
101 GNUNET_CONTAINER_meta_data_get_contents (info->value.search.specifics.result.meta, 108 GNUNET_CONTAINER_meta_data_iterate (info->value.search.specifics.result.meta,
102 &item_printer, 109 &item_printer,
103 NULL); 110 NULL);
104 printf ("\n"); 111 printf ("\n");
105 fflush(stdout); 112 fflush(stdout);
106 GNUNET_free_non_null (filename); 113 GNUNET_free_non_null (filename);
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h
index db9e10ef8..3df9dcfc0 100644
--- a/src/include/gnunet_container_lib.h
+++ b/src/include/gnunet_container_lib.h
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) 3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -196,23 +196,12 @@ void GNUNET_CONTAINER_bloomfilter_resize (struct GNUNET_CONTAINER_BloomFilter
196struct GNUNET_CONTAINER_MetaData; 196struct GNUNET_CONTAINER_MetaData;
197 197
198/** 198/**
199 * Iterator over meta data.
200 *
201 * @param cls closure
202 * @param type type of the meta data
203 * @param data value of the meta data
204 * @return GNUNET_OK to continue to iterate, GNUNET_SYSERR to abort
205 */
206typedef int (*GNUNET_CONTAINER_MetaDataProcessor) (void *cls,
207 EXTRACTOR_KeywordType type,
208 const char *data);
209
210/**
211 * Create a fresh MetaData token. 199 * Create a fresh MetaData token.
212 * 200 *
213 * @return empty meta-data container 201 * @return empty meta-data container
214 */ 202 */
215struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void); 203struct GNUNET_CONTAINER_MetaData *
204GNUNET_CONTAINER_meta_data_create (void);
216 205
217/** 206/**
218 * Duplicate a MetaData token. 207 * Duplicate a MetaData token.
@@ -220,43 +209,61 @@ struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_create (void);
220 * @param md what to duplicate 209 * @param md what to duplicate
221 * @return duplicate meta-data container 210 * @return duplicate meta-data container
222 */ 211 */
223struct GNUNET_CONTAINER_MetaData *GNUNET_CONTAINER_meta_data_duplicate (const 212struct GNUNET_CONTAINER_MetaData *
224 struct 213GNUNET_CONTAINER_meta_data_duplicate (const struct
225 GNUNET_CONTAINER_MetaData 214 GNUNET_CONTAINER_MetaData *md);
226 *md);
227 215
228/** 216/**
229 * Free meta data. 217 * Free meta data.
230 * 218 *
231 * @param md what to free 219 * @param md what to free
232 */ 220 */
233void GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData 221void
234 *md); 222GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md);
235 223
236/** 224/**
237 * Test if two MDs are equal. 225 * Test if two MDs are equal. We consider them equal if
226 * the meta types, formats and content match (we do not
227 * include the mime types and plugins names in this
228 * consideration).
238 * 229 *
239 * @param md1 first value to check 230 * @param md1 first value to check
240 * @param md2 other value to check 231 * @param md2 other value to check
241 * @return GNUNET_YES if they are equal 232 * @return GNUNET_YES if they are equal
242 */ 233 */
243int GNUNET_CONTAINER_meta_data_test_equal (const struct 234int
244 GNUNET_CONTAINER_MetaData *md1, 235GNUNET_CONTAINER_meta_data_test_equal (const struct
245 const struct 236 GNUNET_CONTAINER_MetaData *md1,
246 GNUNET_CONTAINER_MetaData *md2); 237 const struct
238 GNUNET_CONTAINER_MetaData *md2);
247 239
248 240
249/** 241/**
250 * Extend metadata. 242 * Extend metadata.
251 * 243 *
252 * @param md metadata to extend 244 * @param md metadata to extend
253 * @param type type of the new entry 245 * @param plugin_name name of the plugin that produced this value;
254 * @param data value for the entry 246 * special values can be used (i.e. '<zlib>' for zlib being
247 * used in the main libextractor library and yielding
248 * meta data).
249 * @param type libextractor-type describing the meta data
250 * @param format basic format information about data
251 * @param data_mime_type mime-type of data (not of the original file);
252 * can be NULL (if mime-type is not known)
253 * @param data actual meta-data found
254 * @param data_len number of bytes in data
255 * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists 255 * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists
256 * data_mime_type and plugin_name are not considered for "exists" checks
256 */ 257 */
257int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, 258int
258 EXTRACTOR_KeywordType type, 259GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
259 const char *data); 260 const char *plugin_name,
261 enum EXTRACTOR_MetaType type,
262 enum EXTRACTOR_MetaFormat format,
263 const char *data_mime_type,
264 const char *data,
265 size_t data_len);
266
260 267
261/** 268/**
262 * Remove an item. 269 * Remove an item.
@@ -265,11 +272,15 @@ int GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
265 * @param type type of the item to remove 272 * @param type type of the item to remove
266 * @param data specific value to remove, NULL to remove all 273 * @param data specific value to remove, NULL to remove all
267 * entries of the given type 274 * entries of the given type
275 * @param data_len number of bytes in data
268 * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md 276 * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md
269 */ 277 */
270int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, 278int
271 EXTRACTOR_KeywordType type, 279GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
272 const char *data); 280 enum EXTRACTOR_MetaType type,
281 const char *data,
282 size_t data_len);
283
273 284
274/** 285/**
275 * Add the current time as the publication date 286 * Add the current time as the publication date
@@ -277,58 +288,70 @@ int GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
277 * 288 *
278 * @param md metadata to modify 289 * @param md metadata to modify
279 */ 290 */
280void GNUNET_CONTAINER_meta_data_add_publication_date (struct 291void
281 GNUNET_CONTAINER_MetaData 292GNUNET_CONTAINER_meta_data_add_publication_date (struct
282 *md); 293 GNUNET_CONTAINER_MetaData
294 *md);
295
283 296
284/** 297/**
285 * Iterate over MD entries, excluding thumbnails. 298 * Iterate over MD entries.
286 * 299 *
287 * @param md metadata to inspect 300 * @param md metadata to inspect
288 * @param iter function to call on each entry 301 * @param iter function to call on each entry
289 * @param iter_cls closure for iterator 302 * @param iter_cls closure for iterator
290 * @return number of entries 303 * @return number of entries
291 */ 304 */
292int GNUNET_CONTAINER_meta_data_get_contents (const struct 305int GNUNET_CONTAINER_meta_data_iterate (const struct
293 GNUNET_CONTAINER_MetaData *md, 306 GNUNET_CONTAINER_MetaData *md,
294 GNUNET_CONTAINER_MetaDataProcessor 307 EXTRACTOR_MetaDataProcessor
295 iter, void *iter_cls); 308 iter, void *iter_cls);
296 309
297/** 310/**
298 * Get the first MD entry of the given type. 311 * Get the first MD entry of the given type. Caller
312 * is responsible for freeing the return value.
313 * Also, only meta data items that are strings (0-terminated)
314 * are returned by this function.
299 * 315 *
300 * @param md metadata to inspect 316 * @param md metadata to inspect
301 * @param type type to look for 317 * @param type type to look for
302 * @return NULL if we do not have any such entry, 318 * @return NULL if no entry was found
303 * otherwise client is responsible for freeing the value!
304 */ 319 */
305char *GNUNET_CONTAINER_meta_data_get_by_type (const struct 320char *
306 GNUNET_CONTAINER_MetaData *md, 321GNUNET_CONTAINER_meta_data_get_by_type (const struct
307 EXTRACTOR_KeywordType type); 322 GNUNET_CONTAINER_MetaData *md,
323 enum EXTRACTOR_MetaType type);
324
308 325
309/** 326/**
310 * Get the first matching MD entry of the given types. 327 * Get the first matching MD entry of the given types. Caller is
328 * responsible for freeing the return value. Also, only meta data
329 * items that are strings (0-terminated) are returned by this
330 * function.
311 * 331 *
312 * @param md metadata to inspect 332 * @param md metadata to inspect
313 * @param ... -1-terminated list of types 333 * @param ... -1-terminated list of types
314 * @return NULL if we do not have any such entry, 334 * @return NULL if we do not have any such entry,
315 * otherwise client is responsible for freeing the value! 335 * otherwise client is responsible for freeing the value!
316 */ 336 */
317char *GNUNET_CONTAINER_meta_data_get_first_by_types (const struct 337char *
318 GNUNET_CONTAINER_MetaData 338GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
319 *md, ...); 339 GNUNET_CONTAINER_MetaData
340 *md, ...);
320 341
321/** 342/**
322 * Get a thumbnail from the meta-data (if present). 343 * Get a thumbnail from the meta-data (if present). Only matches meta
344 * data with mime type "image" and binary format.
323 * 345 *
324 * @param md metadata to inspect 346 * @param md metadata to inspect
325 * @param thumb will be set to the thumbnail data. Must be 347 * @param thumb will be set to the thumbnail data. Must be
326 * freed by the caller! 348 * freed by the caller!
327 * @return number of bytes in thumbnail, 0 if not available 349 * @return number of bytes in thumbnail, 0 if not available
328 */ 350 */
329size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct 351size_t
330 GNUNET_CONTAINER_MetaData 352GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
331 *md, unsigned char **thumb); 353 GNUNET_CONTAINER_MetaData
354 *md, unsigned char **thumb);
332 355
333/** 356/**
334 * Extract meta-data from a file. 357 * Extract meta-data from a file.
@@ -339,11 +362,12 @@ size_t GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
339 * @return GNUNET_SYSERR on error, otherwise the number 362 * @return GNUNET_SYSERR on error, otherwise the number
340 * of meta-data items obtained 363 * of meta-data items obtained
341 */ 364 */
342int GNUNET_CONTAINER_meta_data_extract_from_file (struct 365int
343 GNUNET_CONTAINER_MetaData 366GNUNET_CONTAINER_meta_data_extract_from_file (struct
344 *md, const char *filename, 367 GNUNET_CONTAINER_MetaData
345 EXTRACTOR_ExtractorList * 368 *md, const char *filename,
346 extractors); 369 struct EXTRACTOR_PluginList *
370 extractors);
347 371
348 372
349/** 373/**
@@ -373,7 +397,8 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions
373 * Serialize meta-data to target. 397 * Serialize meta-data to target.
374 * 398 *
375 * @param md metadata to serialize 399 * @param md metadata to serialize
376 * @param target where to write the serialized metadata 400 * @param target where to write the serialized metadata;
401 * *target can be NULL, in which case memory is allocated
377 * @param max maximum number of bytes available 402 * @param max maximum number of bytes available
378 * @param opt is it ok to just write SOME of the 403 * @param opt is it ok to just write SOME of the
379 * meta-data to match the size constraint, 404 * meta-data to match the size constraint,
@@ -384,7 +409,7 @@ enum GNUNET_CONTAINER_MetaDataSerializationOptions
384 */ 409 */
385ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct 410ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct
386 GNUNET_CONTAINER_MetaData *md, 411 GNUNET_CONTAINER_MetaData *md,
387 char *target, 412 char **target,
388 size_t max, 413 size_t max,
389 enum 414 enum
390 GNUNET_CONTAINER_MetaDataSerializationOptions 415 GNUNET_CONTAINER_MetaDataSerializationOptions
@@ -392,22 +417,15 @@ ssize_t GNUNET_CONTAINER_meta_data_serialize (const struct
392 417
393 418
394/** 419/**
395 * Estimate (!) the size of the meta-data in 420 * Get the size of the full meta-data in serialized form.
396 * serialized form. The estimate MAY be higher
397 * than what is strictly needed.
398 * 421 *
399 * @param md metadata to inspect 422 * @param md metadata to inspect
400 * @param opt is it ok to just write SOME of the
401 * meta-data to match the size constraint,
402 * possibly discarding some data?
403 * @return number of bytes needed for serialization, -1 on error 423 * @return number of bytes needed for serialization, -1 on error
404 */ 424 */
405ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct 425ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size (const struct
406 GNUNET_CONTAINER_MetaData 426 GNUNET_CONTAINER_MetaData
407 *md, 427 *md);
408 enum 428
409 GNUNET_CONTAINER_MetaDataSerializationOptions
410 opt);
411 429
412/** 430/**
413 * Deserialize meta-data. Initializes md. 431 * Deserialize meta-data. Initializes md.
diff --git a/src/include/gnunet_directories.h b/src/include/gnunet_directories.h
deleted file mode 100644
index 05a753f44..000000000
--- a/src/include/gnunet_directories.h
+++ /dev/null
@@ -1,34 +0,0 @@
1/*
2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006 Christian Grothoff (and other contributing authors)
4
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
19*/
20
21/**
22 * @file include/gnunet_directories.h
23 * @brief directories and files in GNUnet (default locations)
24 *
25 * @author Christian Grothoff
26 */
27
28#ifndef GNUNET_DIRECTORIES
29#define GNUNET_DIRECTORIES
30
31#define GNUNET_DEFAULT_CLIENT_CONFIG_FILE "~/.gnunet/gnunet.conf"
32#define GNUNET_DEFAULT_DAEMON_CONFIG_FILE "/etc/gnunetd.conf"
33
34#endif
diff --git a/src/util/bio.c b/src/util/bio.c
index e1085faa6..b05746c38 100644
--- a/src/util/bio.c
+++ b/src/util/bio.c
@@ -434,21 +434,11 @@ GNUNET_BIO_write_meta_data (struct GNUNET_BIO_WriteHandle *h,
434 ssize_t size; 434 ssize_t size;
435 char *buf; 435 char *buf;
436 436
437 size = GNUNET_CONTAINER_meta_data_get_serialized_size (m, 437 buf = NULL;
438 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL
439 |
440 GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS);
441 if (size == -1)
442 return GNUNET_SYSERR;
443 if (size > MAX_META_DATA)
444 size = MAX_META_DATA;
445 buf = GNUNET_malloc (size);
446 size = GNUNET_CONTAINER_meta_data_serialize (m, 438 size = GNUNET_CONTAINER_meta_data_serialize (m,
447 buf, 439 &buf,
448 size, 440 MAX_META_DATA,
449 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART 441 GNUNET_CONTAINER_META_DATA_SERIALIZE_PART);
450 |
451 GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS);
452 if (size == -1) 442 if (size == -1)
453 { 443 {
454 GNUNET_free (buf); 444 GNUNET_free (buf);
diff --git a/src/util/container_meta_data.c b/src/util/container_meta_data.c
index 912ac2684..e4d8737c8 100644
--- a/src/util/container_meta_data.c
+++ b/src/util/container_meta_data.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 (C) 2003, 2004, 2005, 2006, 2008, 2009 Christian Grothoff (and other contributing authors) 3 (C) 2003, 2004, 2005, 2006, 2008, 2009, 2010 Christian Grothoff (and other contributing authors)
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -32,12 +32,46 @@
32#include <extractor.h> 32#include <extractor.h>
33#include <zlib.h> 33#include <zlib.h>
34 34
35#define EXTRA_CHECKS ALLOW_EXTRA_CHECKS 35/**
36 36 * Meta data item.
37struct Item 37 */
38struct MetaItem
38{ 39{
39 EXTRACTOR_KeywordType type; 40 /**
41 * This is a linked list.
42 */
43 struct MetaItem *next;
44
45 /**
46 * Name of the extracting plugin.
47 */
48 char *plugin_name;
49
50 /**
51 * Mime-type of data.
52 */
53 char *mime_type;
54
55 /**
56 * The actual meta data.
57 */
40 char *data; 58 char *data;
59
60 /**
61 * Number of bytes in 'data'.
62 */
63 size_t data_size;
64
65 /**
66 * Type of the meta data.
67 */
68 enum EXTRACTOR_MetaType type;
69
70 /**
71 * Format of the meta data.
72 */
73 enum EXTRACTOR_MetaFormat format;
74
41}; 75};
42 76
43/** 77/**
@@ -45,86 +79,224 @@ struct Item
45 */ 79 */
46struct GNUNET_CONTAINER_MetaData 80struct GNUNET_CONTAINER_MetaData
47{ 81{
48 uint32_t itemCount; 82 /**
49 struct Item *items; 83 * Linked list of the meta data items.
84 */
85 struct MetaItem *items;
86
87 /**
88 * Complete serialized and compressed buffer of the items.
89 * NULL if we have not computed that buffer yet.
90 */
91 char *sbuf;
92
93 /**
94 * Number of bytes in 'sbuf'. 0 if the buffer is stale.
95 */
96 size_t sbuf_size;
97
98 /**
99 * Number of items in the linked list.
100 */
101 unsigned int item_count;
102
50}; 103};
51 104
105
52/** 106/**
53 * Create a fresh struct CONTAINER_MetaData token. 107 * Create a fresh struct CONTAINER_MetaData token.
108 *
109 * @return empty meta-data container
54 */ 110 */
55struct GNUNET_CONTAINER_MetaData * 111struct GNUNET_CONTAINER_MetaData *
56GNUNET_CONTAINER_meta_data_create () 112GNUNET_CONTAINER_meta_data_create ()
57{ 113{
58 struct GNUNET_CONTAINER_MetaData *ret; 114 return GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData));
59 ret = GNUNET_malloc (sizeof (struct GNUNET_CONTAINER_MetaData));
60 ret->items = NULL;
61 ret->itemCount = 0;
62 return ret;
63} 115}
64 116
117
118/**
119 * Free meta data item.
120 *
121 * @param item item to free
122 */
123static void
124meta_item_free (struct MetaItem *item)
125{
126 GNUNET_free_non_null (item->plugin_name);
127 GNUNET_free_non_null (item->mime_type);
128 GNUNET_free_non_null (item->data);
129 GNUNET_free (item);
130}
131
132
133/**
134 * The meta data has changed, invalidate its serialization
135 * buffer.
136 *
137 * @param md meta data that changed
138 */
139static void
140invalidate_sbuf (struct GNUNET_CONTAINER_MetaData *md)
141{
142 if (md->sbuf == NULL)
143 return;
144 GNUNET_free (md->sbuf);
145 md->sbuf = NULL;
146 md->sbuf_size = 0;
147}
148
149
65/** 150/**
66 * Free meta data. 151 * Free meta data.
152 *
153 * @param md what to free
67 */ 154 */
68void 155void
69GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md) 156GNUNET_CONTAINER_meta_data_destroy (struct GNUNET_CONTAINER_MetaData *md)
70{ 157{
71 int i; 158 struct MetaItem *item;
72 159
73 if (md == NULL) 160 if (md == NULL)
74 return; 161 return;
75 for (i = 0; i < md->itemCount; i++) 162 while (NULL != (item = md->items))
76 GNUNET_free (md->items[i].data); 163 {
77 GNUNET_array_grow (md->items, md->itemCount, 0); 164 md->items = item->next;
165 meta_item_free (item);
166 }
167 GNUNET_free_non_null (md->sbuf);
78 GNUNET_free (md); 168 GNUNET_free (md);
79} 169}
80 170
171
81/** 172/**
82 * Add the current time as the publication date 173 * Test if two MDs are equal. We consider them equal if
83 * to the meta-data. 174 * the meta types, formats and content match (we do not
175 * include the mime types and plugins names in this
176 * consideration).
177 *
178 * @param md1 first value to check
179 * @param md2 other value to check
180 * @return GNUNET_YES if they are equal
84 */ 181 */
85void 182int
86GNUNET_CONTAINER_meta_data_add_publication_date (struct 183GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData
87 GNUNET_CONTAINER_MetaData 184 *md1,
88 *md) 185 const struct GNUNET_CONTAINER_MetaData
186 *md2)
89{ 187{
90 char *dat; 188 struct MetaItem *i;
91 struct GNUNET_TIME_Absolute t; 189 struct MetaItem *j;
190 int found;
92 191
93 t = GNUNET_TIME_absolute_get (); 192 if (md1 == md2)
94 GNUNET_CONTAINER_meta_data_delete (md, EXTRACTOR_PUBLICATION_DATE, NULL); 193 return GNUNET_YES;
95 dat = GNUNET_STRINGS_absolute_time_to_string (t); 194 if (md1->item_count != md2->item_count)
96 GNUNET_CONTAINER_meta_data_insert (md, EXTRACTOR_PUBLICATION_DATE, dat); 195 return GNUNET_NO;
97 GNUNET_free (dat); 196
197 i = md1->items;
198 while (NULL != i)
199 {
200 found = GNUNET_NO;
201 j = md2->items;
202 while (NULL != j)
203 {
204 if ( (i->type == j->type) &&
205 (i->format == j->format) &&
206 (i->data_size == j->data_size) &&
207 (0 == memcmp (i->data,
208 j->data,
209 i->data_size)))
210 {
211 found = GNUNET_YES;
212 break;
213 }
214 j = j->next;
215 }
216 if (found == GNUNET_NO)
217 return GNUNET_NO;
218 i = i->next;
219 }
220 return GNUNET_YES;
98} 221}
99 222
223
100/** 224/**
101 * Extend metadata. 225 * Extend metadata. Note that the list of meta data items is
226 * sorted by size (largest first).
227 *
228 * @param md metadata to extend
229 * @param plugin_name name of the plugin that produced this value;
230 * special values can be used (i.e. '<zlib>' for zlib being
231 * used in the main libextractor library and yielding
232 * meta data).
233 * @param type libextractor-type describing the meta data
234 * @param format basic format information about data
235 * @param data_mime_type mime-type of data (not of the original file);
236 * can be NULL (if mime-type is not known)
237 * @param data actual meta-data found
238 * @param data_len number of bytes in data
102 * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists 239 * @return GNUNET_OK on success, GNUNET_SYSERR if this entry already exists
240 * data_mime_type and plugin_name are not considered for "exists" checks
103 */ 241 */
104int 242int
105GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md, 243GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
106 EXTRACTOR_KeywordType type, 244 const char *plugin_name,
107 const char *data) 245 enum EXTRACTOR_MetaType type,
246 enum EXTRACTOR_MetaFormat format,
247 const char *data_mime_type,
248 const char *data,
249 size_t data_len)
108{ 250{
109 uint32_t idx; 251 struct MetaItem *prev;
252 struct MetaItem *pos;
253 struct MetaItem *i;
110 char *p; 254 char *p;
111 255
112 GNUNET_assert (data != NULL); 256 prev = NULL;
113 for (idx = 0; idx < md->itemCount; idx++) 257 pos = md->items;
258 while (NULL != pos)
114 { 259 {
115 if ((md->items[idx].type == type) && 260 if (pos->data_size < data_len)
116 (0 == strcmp (md->items[idx].data, data))) 261 break;
117 return GNUNET_SYSERR; 262 if ( (pos->type == type) &&
263 (pos->format == format) &&
264 (pos->data_size == data_len) &&
265 (0 == memcmp (pos->data,
266 data,
267 data_len)))
268 {
269 if ( (pos->mime_type == NULL) &&
270 (data_mime_type != NULL) )
271 {
272 pos->mime_type = GNUNET_strdup (data_mime_type);
273 invalidate_sbuf (md);
274 }
275 return GNUNET_SYSERR;
276 }
277 prev = pos;
278 pos = pos->next;
118 } 279 }
119 idx = md->itemCount; 280 md->item_count++;
120 GNUNET_array_grow (md->items, md->itemCount, md->itemCount + 1); 281 i = GNUNET_malloc (sizeof (struct MetaItem));
121 md->items[idx].type = type; 282 i->type = type;
122 md->items[idx].data = p = GNUNET_strdup (data); 283 i->format = format;
123 284 i->data_size = data_len;
285 i->next = pos;
286 if (prev == NULL)
287 md->items = i;
288 else
289 prev->next = i;
290 i->mime_type = (data_mime_type == NULL) ? NULL : GNUNET_strdup (data_mime_type);
291 i->plugin_name = (plugin_name == NULL) ? NULL : GNUNET_strdup (plugin_name);
292 i->data = GNUNET_malloc (data_len);
293 memcpy (i->data, data, data_len);
124 /* change OS native dir separators to unix '/' and others to '_' */ 294 /* change OS native dir separators to unix '/' and others to '_' */
125 if (type == EXTRACTOR_FILENAME) 295 if (type == EXTRACTOR_METATYPE_FILENAME)
126 { 296 {
127 while (*p != '\0') 297 p = i->data;
298 while ( (*p != '\0') &&
299 (p < i->data + data_len) )
128 { 300 {
129 if (*p == DIR_SEPARATOR) 301 if (*p == DIR_SEPARATOR)
130 *p = '/'; 302 *p = '/';
@@ -133,10 +305,11 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
133 p++; 305 p++;
134 } 306 }
135 } 307 }
136 308 invalidate_sbuf (md);
137 return GNUNET_OK; 309 return GNUNET_OK;
138} 310}
139 311
312
140/** 313/**
141 * Remove an item. 314 * Remove an item.
142 * 315 *
@@ -144,36 +317,78 @@ GNUNET_CONTAINER_meta_data_insert (struct GNUNET_CONTAINER_MetaData *md,
144 * @param type type of the item to remove 317 * @param type type of the item to remove
145 * @param data specific value to remove, NULL to remove all 318 * @param data specific value to remove, NULL to remove all
146 * entries of the given type 319 * entries of the given type
320 * @param data_len number of bytes in data
147 * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md 321 * @return GNUNET_OK on success, GNUNET_SYSERR if the item does not exist in md
148 */ 322 */
149int 323int
150GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md, 324GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
151 EXTRACTOR_KeywordType type, 325 enum EXTRACTOR_MetaType type,
152 const char *data) 326 const char *data,
327 size_t data_len)
153{ 328{
154 uint32_t idx; 329 struct MetaItem *pos;
155 int ret = GNUNET_SYSERR; 330 struct MetaItem *prev;
156 for (idx = 0; idx < md->itemCount; idx++) 331
332 prev = NULL;
333 pos = md->items;
334 while (NULL != pos)
157 { 335 {
158 if ((md->items[idx].type == type) && 336 if ( (pos->type == type) &&
159 ((data == NULL) || (0 == strcmp (md->items[idx].data, data)))) 337 ( (data == NULL) ||
160 { 338 ( (pos->data_size == data_len) &&
161 GNUNET_free (md->items[idx].data); 339 (0 == memcmp (pos->data,
162 md->items[idx] = md->items[md->itemCount - 1]; 340 data,
163 GNUNET_array_grow (md->items, md->itemCount, md->itemCount - 1); 341 data_len))) ) )
164 if (data == NULL) 342 {
165 { 343 if (prev == NULL)
166 ret = GNUNET_OK; 344 md->items = pos->next;
167 continue; 345 else
168 } 346 prev->next = pos->next;
169 return GNUNET_OK; 347 meta_item_free (pos);
170 } 348 md->item_count--;
349 invalidate_sbuf (md);
350 return GNUNET_OK;
351 }
352 prev = pos;
353 pos = pos->next;
171 } 354 }
172 return ret; 355 return GNUNET_SYSERR;
173} 356}
174 357
358
175/** 359/**
176 * Iterate over MD entries, excluding thumbnails. 360 * Add the current time as the publication date
361 * to the meta-data.
362 *
363 * @param md metadata to modify
364 */
365void
366GNUNET_CONTAINER_meta_data_add_publication_date (struct
367 GNUNET_CONTAINER_MetaData
368 *md)
369{
370 char *dat;
371 struct GNUNET_TIME_Absolute t;
372
373 t = GNUNET_TIME_absolute_get ();
374 GNUNET_CONTAINER_meta_data_delete (md,
375 EXTRACTOR_METATYPE_PUBLICATION_DATE,
376 NULL,
377 0);
378 dat = GNUNET_STRINGS_absolute_time_to_string (t);
379 GNUNET_CONTAINER_meta_data_insert (md,
380 "<gnunet>",
381 EXTRACTOR_METATYPE_PUBLICATION_DATE,
382 EXTRACTOR_METAFORMAT_UTF8,
383 "text/plain",
384 dat,
385 strlen(dat)+1);
386 GNUNET_free (dat);
387}
388
389
390/**
391 * Iterate over MD entries.
177 * 392 *
178 * @param md metadata to inspect 393 * @param md metadata to inspect
179 * @param iter function to call on each entry 394 * @param iter function to call on each entry
@@ -181,51 +396,71 @@ GNUNET_CONTAINER_meta_data_delete (struct GNUNET_CONTAINER_MetaData *md,
181 * @return number of entries 396 * @return number of entries
182 */ 397 */
183int 398int
184GNUNET_CONTAINER_meta_data_get_contents (const struct 399GNUNET_CONTAINER_meta_data_iterate (const struct
185 GNUNET_CONTAINER_MetaData *md, 400 GNUNET_CONTAINER_MetaData *md,
186 GNUNET_CONTAINER_MetaDataProcessor 401 EXTRACTOR_MetaDataProcessor
187 iter, void *iter_cls) 402 iter, void *iter_cls)
188{ 403{
189 uint32_t i; 404 struct MetaItem *pos;
190 uint32_t sub;
191 405
192 sub = 0; 406 if (iter == NULL)
193 for (i = 0; i < md->itemCount; i++) 407 return md->item_count;
408 pos = md->items;
409 while (NULL != pos)
194 { 410 {
195 if (!EXTRACTOR_isBinaryType (md->items[i].type)) 411 if (0 != iter (iter_cls,
196 { 412 pos->plugin_name,
197 if ((iter != NULL) && 413 pos->type,
198 (GNUNET_OK != iter (iter_cls, 414 pos->format,
199 md->items[i].type, md->items[i].data))) 415 pos->mime_type,
200 return GNUNET_SYSERR; 416 pos->data,
201 } 417 pos->data_size))
202 else 418 return md->item_count;
203 sub++; 419 pos = pos->next;
204 } 420 }
205 return (int) (md->itemCount - sub); 421 return md->item_count;
206} 422}
207 423
424
208/** 425/**
209 * Iterate over MD entries 426 * Get the first MD entry of the given type. Caller
427 * is responsible for freeing the return value.
428 * Also, only meta data items that are strings (0-terminated)
429 * are returned by this function.
210 * 430 *
211 * @return number of entries 431 * @param md metadata to inspect
432 * @param type type to look for
433 * @return NULL if no entry was found
212 */ 434 */
213char * 435char *
214GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData 436GNUNET_CONTAINER_meta_data_get_by_type (const struct GNUNET_CONTAINER_MetaData
215 *md, EXTRACTOR_KeywordType type) 437 *md, enum EXTRACTOR_MetaType type)
216{ 438{
217 uint32_t i; 439 struct MetaItem *pos;
218 440
219 for (i = 0; i < md->itemCount; i++) 441 pos = md->items;
220 if (type == md->items[i].type) 442 while (NULL != pos)
221 return GNUNET_strdup (md->items[i].data); 443 {
444 if ( (type == pos->type) &&
445 ( (pos->format == EXTRACTOR_METAFORMAT_UTF8) ||
446 (pos->format == EXTRACTOR_METAFORMAT_C_STRING) ) )
447 return GNUNET_strdup (pos->data);
448 pos = pos->next;
449 }
222 return NULL; 450 return NULL;
223} 451}
224 452
453
225/** 454/**
226 * Iterate over MD entries 455 * Get the first matching MD entry of the given types. Caller is
456 * responsible for freeing the return value. Also, only meta data
457 * items that are strings (0-terminated) are returned by this
458 * function.
227 * 459 *
228 * @return number of entries 460 * @param md metadata to inspect
461 * @param ... -1-terminated list of types
462 * @return NULL if we do not have any such entry,
463 * otherwise client is responsible for freeing the value!
229 */ 464 */
230char * 465char *
231GNUNET_CONTAINER_meta_data_get_first_by_types (const struct 466GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
@@ -234,13 +469,13 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
234{ 469{
235 char *ret; 470 char *ret;
236 va_list args; 471 va_list args;
237 EXTRACTOR_KeywordType type; 472 enum EXTRACTOR_MetaType type;
238 473
239 ret = NULL; 474 ret = NULL;
240 va_start (args, md); 475 va_start (args, md);
241 while (1) 476 while (1)
242 { 477 {
243 type = va_arg (args, EXTRACTOR_KeywordType); 478 type = va_arg (args, enum EXTRACTOR_MetaType);
244 if (type == -1) 479 if (type == -1)
245 break; 480 break;
246 ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type); 481 ret = GNUNET_CONTAINER_meta_data_get_by_type (md, type);
@@ -251,6 +486,7 @@ GNUNET_CONTAINER_meta_data_get_first_by_types (const struct
251 return ret; 486 return ret;
252} 487}
253 488
489
254/** 490/**
255 * Get a thumbnail from the meta-data (if present). 491 * Get a thumbnail from the meta-data (if present).
256 * 492 *
@@ -264,27 +500,33 @@ GNUNET_CONTAINER_meta_data_get_thumbnail (const struct
264 GNUNET_CONTAINER_MetaData * md, 500 GNUNET_CONTAINER_MetaData * md,
265 unsigned char **thumb) 501 unsigned char **thumb)
266{ 502{
267 char *encoded; 503 struct MetaItem *pos;
268 int ret; 504 struct MetaItem *match;
269 size_t size;
270 505
271 encoded = 506 match = NULL;
272 GNUNET_CONTAINER_meta_data_get_by_type (md, EXTRACTOR_THUMBNAIL_DATA); 507 pos = md->items;
273 if (encoded == NULL) 508 while (NULL != pos)
274 return 0;
275 if (strlen (encoded) == 0)
276 { 509 {
277 GNUNET_free (encoded); 510 if ( (0 == strncasecmp ("image/", pos->mime_type,
278 return 0; /* invalid */ 511 strlen("image/"))) &&
512 (pos->format == EXTRACTOR_METAFORMAT_BINARY) )
513 {
514 if (match == NULL)
515 match = pos;
516 else if ( (match->type != EXTRACTOR_METATYPE_THUMBNAIL) &&
517 (pos->type == EXTRACTOR_METATYPE_THUMBNAIL) )
518 match = pos;
519 }
520 pos = pos->next;
279 } 521 }
280 *thumb = NULL; 522 if (match == NULL)
281 ret = EXTRACTOR_binaryDecode (encoded, thumb, &size);
282 GNUNET_free (encoded);
283 if (ret != 0)
284 return 0; 523 return 0;
285 return size; 524 *thumb = GNUNET_malloc (match->data_size);
525 memcpy (*thumb, match->data, match->data_size);
526 return match->data_size;
286} 527}
287 528
529
288/** 530/**
289 * Duplicate struct GNUNET_CONTAINER_MetaData. 531 * Duplicate struct GNUNET_CONTAINER_MetaData.
290 * 532 *
@@ -295,18 +537,66 @@ struct GNUNET_CONTAINER_MetaData *
295GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData 537GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData
296 *md) 538 *md)
297{ 539{
298 uint32_t i;
299 struct GNUNET_CONTAINER_MetaData *ret; 540 struct GNUNET_CONTAINER_MetaData *ret;
541 struct MetaItem *pos;
300 542
301 if (md == NULL) 543 if (md == NULL)
302 return NULL; 544 return NULL;
303 ret = GNUNET_CONTAINER_meta_data_create (); 545 ret = GNUNET_CONTAINER_meta_data_create ();
304 for (i = 0; i < md->itemCount; i++) 546 pos = md->items;
305 GNUNET_CONTAINER_meta_data_insert (ret, md->items[i].type, 547 while (NULL != pos)
306 md->items[i].data); 548 {
549 GNUNET_CONTAINER_meta_data_insert (ret,
550 pos->plugin_name,
551 pos->type,
552 pos->format,
553 pos->mime_type,
554 pos->data,
555 pos->data_size);
556 pos = pos->next;
557 }
307 return ret; 558 return ret;
308} 559}
309 560
561
562/**
563 * Add meta data that libextractor finds to our meta data
564 * container.
565 *
566 * @param cls closure, our meta data container
567 * @param plugin_name name of the plugin that produced this value;
568 * special values can be used (i.e. '<zlib>' for zlib being
569 * used in the main libextractor library and yielding
570 * meta data).
571 * @param type libextractor-type describing the meta data
572 * @param format basic format information about data
573 * @param data_mime_type mime-type of data (not of the original file);
574 * can be NULL (if mime-type is not known)
575 * @param data actual meta-data found
576 * @param data_len number of bytes in data
577 * @return always 0 to continue extracting
578 */
579static int
580add_to_md(void *cls,
581 const char *plugin_name,
582 enum EXTRACTOR_MetaType type,
583 enum EXTRACTOR_MetaFormat format,
584 const char *data_mime_type,
585 const char *data,
586 size_t data_len)
587{
588 struct GNUNET_CONTAINER_MetaData *md = cls;
589 (void) GNUNET_CONTAINER_meta_data_insert (md,
590 plugin_name,
591 type,
592 format,
593 data_mime_type,
594 data,
595 data_len);
596 return 0;
597}
598
599
310/** 600/**
311 * Extract meta-data from a file. 601 * Extract meta-data from a file.
312 * 602 *
@@ -316,37 +606,43 @@ GNUNET_CONTAINER_meta_data_duplicate (const struct GNUNET_CONTAINER_MetaData
316int 606int
317GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData 607GNUNET_CONTAINER_meta_data_extract_from_file (struct GNUNET_CONTAINER_MetaData
318 *md, const char *filename, 608 *md, const char *filename,
319 EXTRACTOR_ExtractorList * 609 struct EXTRACTOR_PluginList *
320 extractors) 610 extractors)
321{ 611{
322 EXTRACTOR_KeywordList *head; 612 unsigned int old;
323 EXTRACTOR_KeywordList *pos;
324 int ret;
325 613
326 if (filename == NULL) 614 if (filename == NULL)
327 return GNUNET_SYSERR; 615 return GNUNET_SYSERR;
328 if (extractors == NULL) 616 if (extractors == NULL)
329 return 0; 617 return 0;
330 head = EXTRACTOR_getKeywords (extractors, filename); 618 old = md->item_count;
331 head = EXTRACTOR_removeDuplicateKeywords (head, 619 EXTRACTOR_extract (extractors,
332 EXTRACTOR_DUPLICATES_REMOVE_UNKNOWN); 620 filename,
333 pos = head; 621 NULL, 0,
334 ret = 0; 622 &add_to_md,
335 while (pos != NULL) 623 md);
336 { 624 return (int) (md->item_count - old);
337 if (GNUNET_OK ==
338 GNUNET_CONTAINER_meta_data_insert (md, pos->keywordType,
339 pos->keyword))
340 ret++;
341 pos = pos->next;
342 }
343 EXTRACTOR_freeKeywords (head);
344 return ret;
345} 625}
346 626
347 627
348static unsigned int 628/**
349tryCompression (char *data, unsigned int oldSize) 629 * Try to compress the given block of data.
630 *
631 * @param data block to compress; if compression
632 * resulted in a smaller block, the first
633 * bytes of data are updated to the compressed
634 * data
635 * @param oldSize number of bytes in data
636 * @param result set to the compressed data
637 * @param newSize set to size of result
638 * @return GNUNET_YES if compression reduce the size,
639 * GNUNET_NO if compression did not help
640 */
641static int
642try_compression (const char *data,
643 size_t oldSize,
644 char **result,
645 size_t *newSize)
350{ 646{
351 char *tmp; 647 char *tmp;
352 uLongf dlen; 648 uLongf dlen;
@@ -364,62 +660,40 @@ tryCompression (char *data, unsigned int oldSize)
364 { 660 {
365 if (dlen < oldSize) 661 if (dlen < oldSize)
366 { 662 {
367 memcpy (data, tmp, dlen); 663 *result = tmp;
368 GNUNET_free (tmp); 664 *newSize = dlen;
369 return dlen; 665 return GNUNET_YES;
370 } 666 }
371 } 667 }
372 GNUNET_free (tmp); 668 GNUNET_free (tmp);
373 return oldSize; 669 return GNUNET_NO;
374} 670}
375 671
376/**
377 * Decompress input, return the decompressed data
378 * as output, set outputSize to the number of bytes
379 * that were found.
380 *
381 * @return NULL on error
382 */
383static char *
384decompress (const char *input,
385 unsigned int inputSize, unsigned int outputSize)
386{
387 char *output;
388 uLongf olen;
389
390 olen = outputSize;
391 output = GNUNET_malloc (olen);
392 if (Z_OK == uncompress ((Bytef *) output,
393 &olen, (const Bytef *) input, inputSize))
394 {
395 return output;
396 }
397 else
398 {
399 GNUNET_free (output);
400 return NULL;
401 }
402}
403 672
404/** 673/**
405 * Flag in 'version' that indicates compressed meta-data. 674 * Flag in 'version' that indicates compressed meta-data.
406 */ 675 */
407#define HEADER_COMPRESSED 0x80000000 676#define HEADER_COMPRESSED 0x80000000
408 677
678
409/** 679/**
410 * Bits in 'version' that give the version number. 680 * Bits in 'version' that give the version number.
411 */ 681 */
412#define HEADER_VERSION_MASK 0x7FFFFFFF 682#define HEADER_VERSION_MASK 0x7FFFFFFF
413 683
684
685/**
686 * Header for serialized meta data.
687 */
414struct MetaDataHeader 688struct MetaDataHeader
415{ 689{
416 /** 690 /**
417 * The version of the MD serialization. 691 * The version of the MD serialization. The highest bit is used to
418 * The highest bit is used to indicate 692 * indicate compression.
419 * compression.
420 * 693 *
421 * Version 0 is the current version; 694 * Version 0 is traditional (pre-0.9) meta data (unsupported)
422 * Version is 1 for a NULL pointer. 695 * Version is 1 for a NULL pointer
696 * Version 2 is for 0.9.x (and possibly higher)
423 * Other version numbers are not yet defined. 697 * Other version numbers are not yet defined.
424 */ 698 */
425 uint32_t version; 699 uint32_t version;
@@ -430,24 +704,57 @@ struct MetaDataHeader
430 uint32_t entries; 704 uint32_t entries;
431 705
432 /** 706 /**
433 * Size of the MD (decompressed) 707 * Size of the decompressed meta data.
434 */ 708 */
435 uint32_t size; 709 uint32_t size;
436 710
437 /** 711 /**
438 * This is followed by 'entries' values of type 'uint32_t' that 712 * This is followed by 'entries' values of type 'struct MetaDataEntry'
439 * correspond to EXTRACTOR_KeywordTypes. After that, the meta-data 713 * and then by 'entry' plugin names, mime-types and data blocks
440 * keywords follow (0-terminated). The MD block always ends with 714 * as specified in those meta data entries.
441 * 0-termination, padding with 0 until a multiple of 8 bytes. 715 */
716};
717
718
719/**
720 * Entry of serialized meta data.
721 */
722struct MetaDataEntry
723{
724 /**
725 * Meta data type. Corresponds to an 'enum EXTRACTOR_MetaType'
726 */
727 uint32_t type;
728
729 /**
730 * Meta data format. Corresponds to an 'enum EXTRACTOR_MetaFormat'
442 */ 731 */
732 uint32_t format;
733
734 /**
735 * Number of bytes of meta data.
736 */
737 uint32_t data_size;
738
739 /**
740 * Number of bytes in the plugin name including 0-terminator. 0 for NULL.
741 */
742 uint32_t plugin_name_len;
743
744 /**
745 * Number of bytes in the mime type including 0-terminator. 0 for NULL.
746 */
747 uint32_t mime_type_len;
443 748
444}; 749};
445 750
751
446/** 752/**
447 * Serialize meta-data to target. 753 * Serialize meta-data to target.
448 * 754 *
449 * @param md metadata to serialize 755 * @param md metadata to serialize
450 * @param target where to write the serialized metadata 756 * @param target where to write the serialized metadata;
757 * *target can be NULL, in which case memory is allocated
451 * @param max maximum number of bytes available in target 758 * @param max maximum number of bytes available in target
452 * @param opt is it ok to just write SOME of the 759 * @param opt is it ok to just write SOME of the
453 * meta-data to match the size constraint, 760 * meta-data to match the size constraint,
@@ -458,149 +765,273 @@ struct MetaDataHeader
458 */ 765 */
459ssize_t 766ssize_t
460GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData 767GNUNET_CONTAINER_meta_data_serialize (const struct GNUNET_CONTAINER_MetaData
461 *md, char *target, size_t max, 768 *md, char **target, size_t max,
462 enum 769 enum
463 GNUNET_CONTAINER_MetaDataSerializationOptions 770 GNUNET_CONTAINER_MetaDataSerializationOptions
464 opt) 771 opt)
465{ 772{
466 struct MetaDataHeader *hdr; 773 struct GNUNET_CONTAINER_MetaData *vmd;
774 struct MetaItem *pos;
775 struct MetaDataHeader *hdr;
776 struct MetaDataEntry *ent;
777 unsigned int i;
778 uint64_t msize;
779 size_t off;
780 char *mdata;
781 char *cdata;
782 size_t mlen;
783 size_t plen;
467 size_t size; 784 size_t size;
468 size_t pos; 785 size_t left;
469 uint32_t i; 786 size_t clen;
470 size_t len; 787 int comp;
471 uint32_t ic;
472 788
473 if (max < sizeof (struct MetaDataHeader)) 789 if (max < sizeof (struct MetaDataHeader))
474 return GNUNET_SYSERR; /* far too small */ 790 return GNUNET_SYSERR; /* far too small */
475 ic = md ? md->itemCount : 0; 791 if (md == NULL)
476 hdr = NULL; 792 return 0;
477 while (1) 793
794 if (md->sbuf != NULL)
478 { 795 {
479 size = sizeof (struct MetaDataHeader); 796 /* try to use serialization cache */
480 size += sizeof (uint32_t) * ic; 797 if (md->sbuf_size < max)
481 for (i = 0; i < ic; i++) 798 {
482 size += 1 + strlen (md->items[i].data); 799 if (NULL == *target)
483 while (size % 8 != 0) 800 *target = GNUNET_malloc (md->sbuf_size);
484 size++; 801 memcpy (*target,
485 hdr = GNUNET_malloc (size); 802 md->sbuf,
486 hdr->version = htonl (md == NULL ? 1 : 0); 803 md->sbuf_size);
487 hdr->entries = htonl (ic); 804 return md->sbuf_size;
488 for (i = 0; i < ic; i++) 805 }
489 ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); 806 if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART))
490 pos = sizeof (struct MetaDataHeader); 807 return GNUNET_SYSERR; /* can say that this will fail */
491 pos += sizeof (uint32_t) * ic; 808 /* need to compute a partial serialization, sbuf useless ... */
492 for (i = 0; i < ic; i++) 809 }
493 {
494 len = strlen (md->items[i].data) + 1;
495 memcpy (&((char *) hdr)[pos], md->items[i].data, len);
496 pos += len;
497 }
498 810
499 hdr->size = htonl (size); 811
500 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) 812 msize = 0;
501 { 813 pos = md->items;
502 pos = tryCompression ((char *) &hdr[1], 814 while (NULL != pos)
503 size - sizeof (struct MetaDataHeader)); 815 {
504 } 816 msize += sizeof (struct MetaDataEntry);
817 msize += pos->data_size;
818 if (pos->plugin_name != NULL)
819 msize += strlen (pos->plugin_name) + 1;
820 if (pos->mime_type != NULL)
821 msize += strlen (pos->mime_type) + 1;
822 pos = pos->next;
823 }
824 size = (size_t) msize;
825 if (size != msize)
826 {
827 GNUNET_break (0); /* integer overflow */
828 return GNUNET_SYSERR;
829 }
830 if (size >= GNUNET_MAX_MALLOC_CHECKED)
831 {
832 /* too large to be processed */
833 return GNUNET_SYSERR;
834 }
835 ent = GNUNET_malloc (size);
836 mdata = (char *) &ent[md->item_count];
837 off = size - (md->item_count * sizeof(struct MetaDataEntry));
838 i = 0;
839 pos = md->items;
840 while (NULL != pos)
841 {
842 ent[i].type = htonl ((uint32_t) pos->type);
843 ent[i].format = htonl ((uint32_t) pos->format);
844 ent[i].data_size = htonl ((uint32_t) pos->data_size);
845 if (pos->plugin_name == NULL)
846 plen = 0;
505 else 847 else
506 { 848 plen = strlen (pos->plugin_name) + 1;
507 pos = size - sizeof (struct MetaDataHeader); 849 ent[i].plugin_name_len = htonl ( (uint32_t) plen);
508 } 850 if (pos->mime_type == NULL)
509 if (pos < size - sizeof (struct MetaDataHeader)) 851 mlen = 0;
510 { 852 else
511 hdr->version = htonl (HEADER_COMPRESSED); 853 mlen = strlen (pos->mime_type) + 1;
512 size = pos + sizeof (struct MetaDataHeader); 854 ent[i].mime_type_len = htonl ((uint32_t) mlen);
513 } 855 off -= pos->data_size;
514 if (size <= max) 856 memcpy (&mdata[off], pos->data, pos->data_size);
515 break; 857 off -= plen;
516 GNUNET_free (hdr); 858 memcpy (&mdata[off], pos->plugin_name, plen);
517 hdr = NULL; 859 off -= mlen;
860 memcpy (&mdata[off], pos->mime_type, mlen);
861 i++;
862 pos = pos->next;
863 }
864 GNUNET_assert (off == 0);
518 865
519 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART) == 0) 866 left = size;
520 { 867 for (i=0;i<md->item_count;i++)
521 return GNUNET_SYSERR; /* does not fit! */ 868 {
869 comp = GNUNET_NO;
870 if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS))
871 comp = try_compression ((const char*) &ent[i],
872 left,
873 &cdata,
874 &clen);
875
876 if ( (md->sbuf == NULL) &&
877 (i == 0) )
878 {
879 /* fill 'sbuf'; this "modifies" md, but since this is only
880 an internal cache we will cast away the 'const' instead
881 of making the API look strange. */
882 vmd = (struct GNUNET_CONTAINER_MetaData*) md;
883 hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader));
884 hdr->entries = htonl (md->item_count);
885 if (GNUNET_YES == comp)
886 {
887 hdr->size = htonl (clen);
888 hdr->version = htonl (2 | HEADER_COMPRESSED);
889 memcpy (&hdr[1],
890 cdata,
891 clen);
892 vmd->sbuf_size = clen + sizeof (struct MetaDataHeader);
893 }
894 else
895 {
896 hdr->size = htonl (left);
897 hdr->version = htonl (2);
898 memcpy (&hdr[1],
899 &ent[0],
900 left);
901 vmd->sbuf_size = left + sizeof (struct MetaDataHeader);
902 }
903 vmd->sbuf = (char*) hdr;
904 }
905
906 if ( ( (left + sizeof (struct MetaDataHeader)) <= max) ||
907 ( (comp == GNUNET_YES) &&
908 (clen <= max)) )
909 {
910 /* success, this now fits! */
911 if (GNUNET_YES == comp)
912 {
913 hdr = (struct MetaDataHeader*) *target;
914 if (hdr == NULL)
915 {
916 hdr = GNUNET_malloc (clen + sizeof (struct MetaDataHeader));
917 *target = (char*) hdr;
918 }
919 hdr->version = htonl (2 | HEADER_COMPRESSED);
920 hdr->entries = htonl (md->item_count - i);
921 hdr->size = htonl (left);
922 memcpy (&target[sizeof(struct MetaDataHeader)],
923 cdata,
924 clen);
925 GNUNET_free (cdata);
926 GNUNET_free (ent);
927 return clen + sizeof (struct MetaDataHeader);
928 }
929 else
930 {
931 hdr = (struct MetaDataHeader*) target;
932 if (hdr == NULL)
933 {
934 hdr = GNUNET_malloc (left + sizeof (struct MetaDataHeader));
935 *target = (char*) hdr;
936 }
937 hdr->version = htonl (2);
938 hdr->entries = htonl (md->item_count - i);
939 hdr->size = htonl (left);
940 memcpy (&target[sizeof(struct MetaDataHeader)],
941 &ent[i],
942 left);
943 GNUNET_free (ent);
944 return left + sizeof (struct MetaDataHeader);
945 }
946 }
947
948 if (0 == (opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_PART))
949 {
950 /* does not fit! */
951 GNUNET_free (ent);
952 return GNUNET_SYSERR;
522 } 953 }
523 /* partial serialization ok, try again with less meta-data */ 954
524 if (size > 2 * max) 955 /* next iteration: ignore the corresponding meta data at the
525 ic = ic * 2 / 3; /* still far too big, make big reductions */ 956 end and try again without it */
526 else 957 left -= sizeof (struct MetaDataEntry);
527 ic--; /* small steps, we're close */ 958 left -= pos->data_size;
959 if (pos->plugin_name != NULL)
960 left -= strlen (pos->plugin_name) + 1;
961 if (pos->mime_type != NULL)
962 left -= strlen (pos->mime_type) + 1;
528 } 963 }
529 GNUNET_assert (size <= max); 964 GNUNET_free (ent);
530 memcpy (target, hdr, size); 965
531 GNUNET_free (hdr); 966 /* nothing fit, only write header! */
532 /* extra check: deserialize! */ 967 hdr = (struct MetaDataHeader*) target;
533#if EXTRA_CHECKS 968 if (hdr == NULL)
534 { 969 {
535 struct GNUNET_CONTAINER_MetaData *mdx; 970 hdr = GNUNET_malloc (sizeof (struct MetaDataHeader));
536 mdx = GNUNET_CONTAINER_meta_data_deserialize (target, size); 971 *target = (char*) hdr;
537 GNUNET_assert (NULL != mdx); 972 }
538 GNUNET_CONTAINER_meta_data_destroy (mdx); 973 hdr->version = htonl (2);
539 } 974 hdr->entries = htonl (0);
540#endif 975 hdr->size = htonl (0);
541 return size; 976 return sizeof (struct MetaDataHeader);
542} 977}
543 978
979
544/** 980/**
545 * Estimate (!) the size of the meta-data in 981 * Get the size of the full meta-data in serialized form.
546 * serialized form. The estimate MAY be higher
547 * than what is strictly needed.
548 * 982 *
549 * @param md metadata to inspect 983 * @param md metadata to inspect
550 * @param opt is it ok to just write SOME of the
551 * meta-data to match the size constraint,
552 * possibly discarding some data?
553 * @return number of bytes needed for serialization, -1 on error 984 * @return number of bytes needed for serialization, -1 on error
554 */ 985 */
555ssize_t 986ssize_t
556GNUNET_CONTAINER_meta_data_get_serialized_size (const struct 987GNUNET_CONTAINER_meta_data_get_serialized_size (const struct GNUNET_CONTAINER_MetaData *md)
557 GNUNET_CONTAINER_MetaData *
558 md,
559 enum
560 GNUNET_CONTAINER_MetaDataSerializationOptions
561 opt)
562{ 988{
563 struct MetaDataHeader *hdr; 989 ssize_t ret;
564 size_t size; 990 char *ptr;
565 size_t pos; 991
566 uint32_t i; 992 if (md->sbuf != NULL)
567 size_t len; 993 return md->sbuf_size;
568 uint32_t ic; 994 ptr = NULL;
995 ret = GNUNET_CONTAINER_meta_data_serialize (md,
996 &ptr,
997 GNUNET_MAX_MALLOC_CHECKED,
998 GNUNET_CONTAINER_META_DATA_SERIALIZE_FULL);
999 if (ret != -1)
1000 GNUNET_free (ptr);
1001 return ret;
1002}
569 1003
570 ic = md ? md->itemCount : 0; 1004
571 size = sizeof (struct MetaDataHeader); 1005/**
572 size += sizeof (uint32_t) * ic; 1006 * Decompress input, return the decompressed data
573 for (i = 0; i < ic; i++) 1007 * as output, set outputSize to the number of bytes
574 size += 1 + strlen (md->items[i].data); 1008 * that were found.
575 while (size % 8 != 0) 1009 *
576 size++; 1010 * @param input compressed data
577 hdr = GNUNET_malloc (size); 1011 * @param inputSize number of bytes in input
578 hdr->version = htonl (md == NULL ? 1 : 0); 1012 * @param outputSize expected size of the output
579 hdr->entries = htonl (ic); 1013 * @return NULL on error
580 for (i = 0; i < ic; i++) 1014 */
581 ((uint32_t *) & hdr[1])[i] = htonl ((uint32_t) md->items[i].type); 1015static char *
582 pos = sizeof (struct MetaDataHeader); 1016decompress (const char *input,
583 pos += sizeof (uint32_t) * ic; 1017 size_t inputSize,
584 for (i = 0; i < ic; i++) 1018 size_t outputSize)
585 { 1019{
586 len = strlen (md->items[i].data) + 1; 1020 char *output;
587 memcpy (&((char *) hdr)[pos], md->items[i].data, len); 1021 uLongf olen;
588 pos += len; 1022
589 } 1023 olen = outputSize;
590 if ((opt & GNUNET_CONTAINER_META_DATA_SERIALIZE_NO_COMPRESS) == 0) 1024 output = GNUNET_malloc (olen);
1025 if (Z_OK == uncompress ((Bytef *) output,
1026 &olen, (const Bytef *) input, inputSize))
591 { 1027 {
592 pos = 1028 return output;
593 tryCompression ((char *) &hdr[1],
594 size - sizeof (struct MetaDataHeader));
595 } 1029 }
596 else 1030 else
597 { 1031 {
598 pos = size - sizeof (struct MetaDataHeader); 1032 GNUNET_free (output);
1033 return NULL;
599 } 1034 }
600 if (pos < size - sizeof (struct MetaDataHeader))
601 size = pos + sizeof (struct MetaDataHeader);
602 GNUNET_free (hdr);
603 return size;
604} 1035}
605 1036
606 1037
@@ -616,41 +1047,57 @@ struct GNUNET_CONTAINER_MetaData *
616GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size) 1047GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
617{ 1048{
618 struct GNUNET_CONTAINER_MetaData *md; 1049 struct GNUNET_CONTAINER_MetaData *md;
619 const struct MetaDataHeader *hdr; 1050 struct MetaDataHeader hdr;
1051 struct MetaDataEntry ent;
620 uint32_t ic; 1052 uint32_t ic;
1053 uint32_t i;
621 char *data; 1054 char *data;
622 const char *cdata; 1055 const char *cdata;
1056 uint32_t version;
623 uint32_t dataSize; 1057 uint32_t dataSize;
624 int compressed; 1058 int compressed;
625 uint32_t i; 1059 size_t left;
626 size_t pos; 1060 uint32_t mlen;
627 size_t len; 1061 uint32_t plen;
628 uint32_t version; 1062 uint32_t dlen;
1063 const char *mdata;
1064 const char *meta_data;
1065 const char *plugin_name;
1066 const char *mime_type;
1067 enum EXTRACTOR_MetaFormat format;
629 1068
630 if (size < sizeof (struct MetaDataHeader)) 1069 if (size < sizeof (struct MetaDataHeader))
631 return NULL; 1070 return NULL;
632 hdr = (const struct MetaDataHeader *) input; 1071 memcpy (&hdr,
633 version = ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_VERSION_MASK; 1072 input,
1073 sizeof (struct MetaDataHeader));
1074 version = ntohl (hdr.version) & HEADER_VERSION_MASK;
1075 compressed = (ntohl (hdr.version) & HEADER_COMPRESSED) != 0;
1076
634 if (version == 1) 1077 if (version == 1)
635 return NULL; /* null pointer */ 1078 return NULL; /* null pointer */
636 if (version != 0) 1079 if (version != 2)
637 { 1080 {
638 GNUNET_break_op (0); /* unsupported version */ 1081 GNUNET_break_op (0); /* unsupported version */
639 return NULL; 1082 return NULL;
640 } 1083 }
641 ic = ntohl (MAKE_UNALIGNED (hdr->entries)); 1084
642 compressed = 1085 ic = ntohl (hdr.entries);
643 (ntohl (MAKE_UNALIGNED (hdr->version)) & HEADER_COMPRESSED) != 0; 1086 dataSize = ntohl (hdr.size);
1087 if ((sizeof (struct MetaDataEntry) * ic) > dataSize)
1088 {
1089 GNUNET_break_op (0);
1090 return NULL;
1091 }
1092
644 if (compressed) 1093 if (compressed)
645 { 1094 {
646 dataSize = 1095 if (dataSize >= GNUNET_MAX_MALLOC_CHECKED)
647 ntohl (MAKE_UNALIGNED (hdr->size)) - sizeof (struct MetaDataHeader);
648 if (dataSize > 2 * 1042 * 1024)
649 { 1096 {
650 GNUNET_break (0); 1097 /* make sure we don't blow our memory limit because of a mal-formed
651 return NULL; /* only 2 MB allowed [to make sure we don't blow 1098 message... */
652 our memory limit because of a mal-formed 1099 GNUNET_break_op (0);
653 message... ] */ 1100 return NULL;
654 } 1101 }
655 data = 1102 data =
656 decompress ((const char *) &input[sizeof (struct MetaDataHeader)], 1103 decompress ((const char *) &input[sizeof (struct MetaDataHeader)],
@@ -665,84 +1112,93 @@ GNUNET_CONTAINER_meta_data_deserialize (const char *input, size_t size)
665 else 1112 else
666 { 1113 {
667 data = NULL; 1114 data = NULL;
668 cdata = (const char *) &hdr[1]; 1115 cdata = (const char *) &input[sizeof (struct MetaDataHeader)];
669 dataSize = size - sizeof (struct MetaDataHeader); 1116 if (dataSize != size - sizeof (struct MetaDataHeader))
670 if (size != ntohl (MAKE_UNALIGNED (hdr->size)))
671 { 1117 {
672 GNUNET_break (0); 1118 GNUNET_break_op (0);
673 return NULL; 1119 return NULL;
674 } 1120 }
675 } 1121 }
676 1122
677 if ((sizeof (uint32_t) * ic + ic) > dataSize)
678 {
679 GNUNET_break (0);
680 goto FAILURE;
681 }
682 if ((ic > 0) && (cdata[dataSize - 1] != '\0'))
683 {
684 GNUNET_break (0);
685 goto FAILURE;
686 }
687
688 md = GNUNET_CONTAINER_meta_data_create (); 1123 md = GNUNET_CONTAINER_meta_data_create ();
689 GNUNET_array_grow (md->items, md->itemCount, ic); 1124 left = dataSize - ic * sizeof (struct MetaDataEntry);
690 i = 0; 1125 mdata = &cdata[ic * sizeof (struct MetaDataEntry)];
691 pos = sizeof (uint32_t) * ic; 1126 for (i=0;i<ic;i++)
692 while ((pos < dataSize) && (i < ic))
693 { 1127 {
694 len = strlen (&cdata[pos]) + 1; 1128 memcpy (&ent,
695 md->items[i].type = (EXTRACTOR_KeywordType) 1129 &cdata[i * sizeof(struct MetaDataEntry)],
696 ntohl (MAKE_UNALIGNED (((const uint32_t *) cdata)[i])); 1130 sizeof (struct MetaDataEntry));
697 md->items[i].data = GNUNET_strdup (&cdata[pos]); 1131 format = (enum EXTRACTOR_MetaFormat) ntohl (ent.format);
698 pos += len; 1132 if ( (format != EXTRACTOR_METAFORMAT_UTF8) &&
699 i++; 1133 (format != EXTRACTOR_METAFORMAT_C_STRING) &&
700 } 1134 (format != EXTRACTOR_METAFORMAT_BINARY) )
701 if (i < ic) 1135 {
702 { /* oops */ 1136 GNUNET_break_op (0);
703 GNUNET_CONTAINER_meta_data_destroy (md); 1137 break;
704 goto FAILURE; 1138 }
705 } 1139 dlen = ntohl (ent.data_size);
706 GNUNET_free_non_null (data); 1140 plen = ntohl (ent.plugin_name_len);
707 return md; 1141 mlen = ntohl (ent.mime_type_len);
708FAILURE: 1142 if (dlen > left)
709 GNUNET_free_non_null (data); 1143 {
710 return NULL; /* size too small */ 1144 GNUNET_break_op (0);
711} 1145 break;
712 1146 }
713/** 1147 left -= dlen;
714 * Test if two MDs are equal. 1148 meta_data = &mdata[left];
715 * 1149 if ( (format == EXTRACTOR_METAFORMAT_UTF8) ||
716 * @param md1 first value to check 1150 (format == EXTRACTOR_METAFORMAT_C_STRING) )
717 * @param md2 other value to check 1151 {
718 * @return GNUNET_YES if they are equal 1152 if ( (dlen == 0) ||
719 */ 1153 (mdata[left + dlen - 1] != '\0') )
720int 1154 {
721GNUNET_CONTAINER_meta_data_test_equal (const struct GNUNET_CONTAINER_MetaData 1155 GNUNET_break_op (0);
722 *md1, 1156 break;
723 const struct GNUNET_CONTAINER_MetaData 1157 }
724 *md2) 1158 }
725{ 1159 if (plen > left)
726 uint32_t i; 1160 {
727 uint32_t j; 1161 GNUNET_break_op (0);
728 int found; 1162 break;
1163 }
1164 left -= plen;
1165 if ( (plen > 0) &&
1166 (mdata[left + plen - 1] != '\0') )
1167 {
1168 GNUNET_break_op (0);
1169 break;
1170 }
1171 if (plen == 0)
1172 plugin_name = NULL;
1173 else
1174 plugin_name = &mdata[left];
729 1175
730 if (md1->itemCount != md2->itemCount) 1176 if (mlen > left)
731 return GNUNET_NO; 1177 {
732 for (i = 0; i < md1->itemCount; i++) 1178 GNUNET_break_op (0);
733 { 1179 break;
734 found = GNUNET_NO; 1180 }
735 for (j = 0; j < md2->itemCount; j++) 1181 left -= mlen;
736 if ((md1->items[i].type == md2->items[j].type) && 1182 if ( (mlen > 0) &&
737 (0 == strcmp (md1->items[i].data, md2->items[j].data))) 1183 (mdata[left + mlen - 1] != '\0') )
738 { 1184 {
739 found = GNUNET_YES; 1185 GNUNET_break_op (0);
740 break; 1186 break;
741 } 1187 }
742 if (found == GNUNET_NO) 1188 if (mlen == 0)
743 return GNUNET_NO; 1189 mime_type = NULL;
1190 else
1191 mime_type = &mdata[left];
1192 GNUNET_CONTAINER_meta_data_insert (md,
1193 plugin_name,
1194 (enum EXTRACTOR_MetaType) ntohl (ent.type),
1195 format,
1196 mime_type,
1197 meta_data,
1198 dlen);
744 } 1199 }
745 return GNUNET_YES; 1200 GNUNET_free_non_null (data);
1201 return md;
746} 1202}
747 1203
748 1204
diff --git a/src/util/pseudonym.c b/src/util/pseudonym.c
index c974cc5a3..626e8ca99 100644
--- a/src/util/pseudonym.c
+++ b/src/util/pseudonym.c
@@ -300,15 +300,14 @@ GNUNET_PSEUDONYM_id_to_name (const struct GNUNET_CONFIGURATION_Handle *cfg,
300 { 300 {
301 if ((meta != NULL) && (name == NULL)) 301 if ((meta != NULL) && (name == NULL))
302 name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta, 302 name = GNUNET_CONTAINER_meta_data_get_first_by_types (meta,
303 EXTRACTOR_TITLE, 303 EXTRACTOR_METATYPE_TITLE,
304 EXTRACTOR_FILENAME, 304 EXTRACTOR_METATYPE_FILENAME,
305 EXTRACTOR_DESCRIPTION, 305 EXTRACTOR_METATYPE_DESCRIPTION,
306 EXTRACTOR_SUBJECT, 306 EXTRACTOR_METATYPE_SUBJECT,
307 EXTRACTOR_PUBLISHER, 307 EXTRACTOR_METATYPE_PUBLISHER,
308 EXTRACTOR_AUTHOR, 308 EXTRACTOR_METATYPE_AUTHOR_NAME,
309 EXTRACTOR_COMMENT, 309 EXTRACTOR_METATYPE_COMMENT,
310 EXTRACTOR_SUMMARY, 310 EXTRACTOR_METATYPE_SUMMARY,
311 EXTRACTOR_OWNER,
312 -1); 311 -1);
313 if (meta != NULL) 312 if (meta != NULL)
314 { 313 {
@@ -545,11 +544,21 @@ GNUNET_PSEUDONYM_rank (const struct GNUNET_CONFIGURATION_Handle *cfg,
545 * @param data value of entry to insert 544 * @param data value of entry to insert
546 */ 545 */
547static int 546static int
548merge_meta_helper (void *cls, EXTRACTOR_KeywordType type, const char *data) 547merge_meta_helper (void *cls,
548 const char *plugin_name,
549 enum EXTRACTOR_MetaType type,
550 enum EXTRACTOR_MetaFormat format,
551 const char *data_mime_type,
552 const char *data,
553 size_t data_len)
549{ 554{
550 struct GNUNET_CONTAINER_MetaData *meta = cls; 555 struct GNUNET_CONTAINER_MetaData *meta = cls;
551 GNUNET_CONTAINER_meta_data_insert (meta, type, data); 556
552 return GNUNET_OK; 557 (void) GNUNET_CONTAINER_meta_data_insert (meta, plugin_name,
558 type, format,
559 data_mime_type,
560 data, data_len);
561 return 0;
553} 562}
554 563
555 564
@@ -581,7 +590,7 @@ GNUNET_PSEUDONYM_add (const struct GNUNET_CONFIGURATION_Handle *cfg,
581 if ((0 == STAT (fn, &sbuf)) && 590 if ((0 == STAT (fn, &sbuf)) &&
582 (GNUNET_OK == read_info (cfg, id, &old, &ranking, &name))) 591 (GNUNET_OK == read_info (cfg, id, &old, &ranking, &name)))
583 { 592 {
584 GNUNET_CONTAINER_meta_data_get_contents (meta, &merge_meta_helper, old); 593 GNUNET_CONTAINER_meta_data_iterate (meta, &merge_meta_helper, old);
585 write_pseudonym_info (cfg, id, old, ranking, name); 594 write_pseudonym_info (cfg, id, old, ranking, name);
586 GNUNET_CONTAINER_meta_data_destroy (old); 595 GNUNET_CONTAINER_meta_data_destroy (old);
587 GNUNET_free_non_null (name); 596 GNUNET_free_non_null (name);