aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2017-02-20 17:19:47 +0100
committerChristian Grothoff <christian@grothoff.org>2017-02-20 17:19:47 +0100
commitf6f7fbbe98c110867febbcca647da8308be123c7 (patch)
treeaf69447cf4f08c417197685855c097c132aea8a1
parenta3882b58f1c5976677aa65b0af8a48e8e946b06e (diff)
downloadgnunet-f6f7fbbe98c110867febbcca647da8308be123c7.tar.gz
gnunet-f6f7fbbe98c110867febbcca647da8308be123c7.zip
completed big block refactoring in preparation for SET-BLOCK integration
-rw-r--r--src/block/Makefile.am6
-rw-r--r--src/block/bg_bf.c64
-rw-r--r--src/block/block.c145
-rw-r--r--src/block/plugin_block_template.c88
-rw-r--r--src/block/plugin_block_test.c80
-rw-r--r--src/dht/Makefile.am3
-rw-r--r--src/dht/gnunet-service-dht_clients.c30
-rw-r--r--src/dht/gnunet-service-dht_datacache.c39
-rw-r--r--src/dht/gnunet-service-dht_datacache.h6
-rw-r--r--src/dht/gnunet-service-dht_neighbours.c196
-rw-r--r--src/dht/gnunet-service-dht_neighbours.h14
-rw-r--r--src/dht/gnunet-service-dht_routing.c56
-rw-r--r--src/dht/gnunet-service-dht_routing.h11
-rw-r--r--src/dht/plugin_block_dht.c34
-rw-r--r--src/dns/plugin_block_dns.c6
-rw-r--r--src/fs/gnunet-service-fs_pr.c139
-rw-r--r--src/fs/plugin_block_fs.c95
-rw-r--r--src/fs/test_plugin_block_fs.c6
-rw-r--r--src/gns/plugin_block_gns.c33
-rw-r--r--src/include/gnunet_block_group_lib.h14
-rw-r--r--src/include/gnunet_block_lib.h49
-rw-r--r--src/include/gnunet_block_plugin.h52
-rw-r--r--src/regex/plugin_block_regex.c82
23 files changed, 707 insertions, 541 deletions
diff --git a/src/block/Makefile.am b/src/block/Makefile.am
index 4a6d8e71e..da1d8257c 100644
--- a/src/block/Makefile.am
+++ b/src/block/Makefile.am
@@ -26,7 +26,7 @@ noinst_LTLIBRARIES = \
26libgnunet_plugin_block_template_la_SOURCES = \ 26libgnunet_plugin_block_template_la_SOURCES = \
27 plugin_block_template.c 27 plugin_block_template.c
28libgnunet_plugin_block_template_la_LIBADD = \ 28libgnunet_plugin_block_template_la_LIBADD = \
29 libgnunetblock.la \ 29 libgnunetblockgroup.la \
30 $(top_builddir)/src/util/libgnunetutil.la \ 30 $(top_builddir)/src/util/libgnunetutil.la \
31 $(LTLIBINTL) 31 $(LTLIBINTL)
32libgnunet_plugin_block_template_la_LDFLAGS = \ 32libgnunet_plugin_block_template_la_LDFLAGS = \
@@ -35,8 +35,8 @@ libgnunet_plugin_block_template_la_LDFLAGS = \
35libgnunet_plugin_block_test_la_SOURCES = \ 35libgnunet_plugin_block_test_la_SOURCES = \
36 plugin_block_test.c 36 plugin_block_test.c
37libgnunet_plugin_block_test_la_LIBADD = \ 37libgnunet_plugin_block_test_la_LIBADD = \
38 libgnunetblock.la \ 38 libgnunetblockgroup.la \
39 $(top_builddir)/src/util/libgnunetutil.la \ 39$(top_builddir)/src/util/libgnunetutil.la \
40 $(LTLIBINTL) 40 $(LTLIBINTL)
41libgnunet_plugin_block_test_la_LDFLAGS = \ 41libgnunet_plugin_block_test_la_LDFLAGS = \
42 $(GN_PLUGIN_LDFLAGS) 42 $(GN_PLUGIN_LDFLAGS)
diff --git a/src/block/bg_bf.c b/src/block/bg_bf.c
index f03ae5247..9c4dc9060 100644
--- a/src/block/bg_bf.c
+++ b/src/block/bg_bf.c
@@ -56,6 +56,7 @@ struct BfGroupInternals
56 * Serialize state of a block group. 56 * Serialize state of a block group.
57 * 57 *
58 * @param bg group to serialize 58 * @param bg group to serialize
59 * @param[out] nonce set to the nonce of the @a bg
59 * @param[out] raw_data set to the serialized state 60 * @param[out] raw_data set to the serialized state
60 * @param[out] raw_data_size set to the number of bytes in @a raw_data 61 * @param[out] raw_data_size set to the number of bytes in @a raw_data
61 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not 62 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
@@ -63,6 +64,7 @@ struct BfGroupInternals
63 */ 64 */
64static int 65static int
65bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg, 66bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg,
67 uint32_t *nonce,
66 void **raw_data, 68 void **raw_data,
67 size_t *raw_data_size) 69 size_t *raw_data_size)
68{ 70{
@@ -78,6 +80,7 @@ bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg,
78 GNUNET_break (0); 80 GNUNET_break (0);
79 return GNUNET_SYSERR; 81 return GNUNET_SYSERR;
80 } 82 }
83 *nonce = gi->bf_mutator;
81 *raw_data = raw; 84 *raw_data = raw;
82 *raw_data_size = gi->bf_size; 85 *raw_data_size = gi->bf_size;
83 return GNUNET_OK; 86 return GNUNET_OK;
@@ -85,6 +88,60 @@ bf_group_serialize_cb (struct GNUNET_BLOCK_Group *bg,
85 88
86 89
87/** 90/**
91 * Mark elements as "seen" using a hash of the element. Not supported
92 * by all block plugins.
93 *
94 * @param bg group to update
95 * @param seen_results results already seen
96 * @param seen_results_count number of entries in @a seen_results
97 */
98static void
99bf_group_mark_seen_cb (struct GNUNET_BLOCK_Group *bg,
100 const struct GNUNET_HashCode *seen_results,
101 unsigned int seen_results_count)
102{
103 struct BfGroupInternals *gi = bg->internal_cls;
104
105 for (unsigned int i=0;i<seen_results_count;i++)
106 {
107 struct GNUNET_HashCode mhash;
108
109 GNUNET_BLOCK_mingle_hash (&seen_results[i],
110 gi->bf_mutator,
111 &mhash);
112 GNUNET_CONTAINER_bloomfilter_add (gi->bf,
113 &mhash);
114 }
115}
116
117
118/**
119 * Merge two groups, if possible. Not supported by all block plugins,
120 * can also fail if the nonces were different.
121 *
122 * @param bg1 group to update
123 * @param bg2 group to merge into @a bg1
124 * @return #GNUNET_OK on success, #GNUNET_NO if the nonces were different and thus
125 * we failed.
126 */
127static int
128bf_group_merge_cb (struct GNUNET_BLOCK_Group *bg1,
129 const struct GNUNET_BLOCK_Group *bg2)
130{
131 struct BfGroupInternals *gi1 = bg1->internal_cls;
132 struct BfGroupInternals *gi2 = bg2->internal_cls;
133
134 if (gi1->bf_mutator != gi2->bf_mutator)
135 return GNUNET_NO;
136 if (gi1->bf_size != gi2->bf_size)
137 return GNUNET_NO;
138 GNUNET_CONTAINER_bloomfilter_or2 (gi1->bf,
139 gi2->bf);
140 return GNUNET_OK;
141}
142
143
144/**
88 * Destroy resources used by a block group. 145 * Destroy resources used by a block group.
89 * 146 *
90 * @param bg group to destroy, NULL is allowed 147 * @param bg group to destroy, NULL is allowed
@@ -134,6 +191,8 @@ GNUNET_BLOCK_GROUP_bf_create (void *cls,
134 bg = GNUNET_new (struct GNUNET_BLOCK_Group); 191 bg = GNUNET_new (struct GNUNET_BLOCK_Group);
135 bg->type = type; 192 bg->type = type;
136 bg->serialize_cb = &bf_group_serialize_cb; 193 bg->serialize_cb = &bf_group_serialize_cb;
194 bg->mark_seen_cb = &bf_group_mark_seen_cb;
195 bg->merge_cb = &bf_group_merge_cb;
137 bg->destroy_cb = &bf_group_destroy_cb; 196 bg->destroy_cb = &bf_group_destroy_cb;
138 bg->internal_cls = gi; 197 bg->internal_cls = gi;
139 return bg; 198 return bg;
@@ -154,9 +213,12 @@ int
154GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg, 213GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg,
155 const struct GNUNET_HashCode *hc) 214 const struct GNUNET_HashCode *hc)
156{ 215{
157 struct BfGroupInternals *gi = bg->internal_cls; 216 struct BfGroupInternals *gi;
158 struct GNUNET_HashCode mhash; 217 struct GNUNET_HashCode mhash;
159 218
219 if (NULL == bg)
220 return GNUNET_NO;
221 gi = bg->internal_cls;
160 GNUNET_BLOCK_mingle_hash (hc, 222 GNUNET_BLOCK_mingle_hash (hc,
161 gi->bf_mutator, 223 gi->bf_mutator,
162 &mhash); 224 &mhash);
diff --git a/src/block/block.c b/src/block/block.c
index d4f5462dd..b7a19ae90 100644
--- a/src/block/block.c
+++ b/src/block/block.c
@@ -162,6 +162,7 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx)
162 * Serialize state of a block group. 162 * Serialize state of a block group.
163 * 163 *
164 * @param bg group to serialize 164 * @param bg group to serialize
165 * @param[out] nonce set to the nonce of the @a bg
165 * @param[out] raw_data set to the serialized state 166 * @param[out] raw_data set to the serialized state
166 * @param[out] raw_data_size set to the number of bytes in @a raw_data 167 * @param[out] raw_data_size set to the number of bytes in @a raw_data
167 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not 168 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
@@ -169,9 +170,11 @@ GNUNET_BLOCK_context_destroy (struct GNUNET_BLOCK_Context *ctx)
169 */ 170 */
170int 171int
171GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg, 172GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg,
173 uint32_t *nonce,
172 void **raw_data, 174 void **raw_data,
173 size_t *raw_data_size) 175 size_t *raw_data_size)
174{ 176{
177 *nonce = 0;
175 *raw_data = NULL; 178 *raw_data = NULL;
176 *raw_data_size = 0; 179 *raw_data_size = 0;
177 if (NULL == bg) 180 if (NULL == bg)
@@ -179,6 +182,7 @@ GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg,
179 if (NULL == bg->serialize_cb) 182 if (NULL == bg->serialize_cb)
180 return GNUNET_NO; 183 return GNUNET_NO;
181 return bg->serialize_cb (bg, 184 return bg->serialize_cb (bg,
185 nonce,
182 raw_data, 186 raw_data,
183 raw_data_size); 187 raw_data_size);
184} 188}
@@ -199,6 +203,41 @@ GNUNET_BLOCK_group_destroy (struct GNUNET_BLOCK_Group *bg)
199 203
200 204
201/** 205/**
206 * Try merging two block groups. Afterwards, @a bg1 should remain
207 * valid and contain the rules from both @a bg1 and @bg2, and
208 * @a bg2 should be destroyed (as part of this call). The latter
209 * should happen even if merging is not supported.
210 *
211 * @param[in,out] bg1 first group to merge, is updated
212 * @param bg2 second group to merge, is destroyed
213 * @return #GNUNET_OK on success,
214 * #GNUNET_NO if merge failed due to different nonce
215 * #GNUNET_SYSERR if merging is not supported
216 */
217int
218GNUNET_BLOCK_group_merge (struct GNUNET_BLOCK_Group *bg1,
219 struct GNUNET_BLOCK_Group *bg2)
220{
221 int ret;
222
223 if (NULL == bg2)
224 return GNUNET_OK;
225 if (NULL == bg1)
226 {
227 bg2->destroy_cb (bg2);
228 return GNUNET_OK;
229 }
230 if (NULL == bg1->merge_cb)
231 return GNUNET_SYSERR;
232 GNUNET_assert (bg1->merge_cb == bg1->merge_cb);
233 ret = bg1->merge_cb (bg1,
234 bg2);
235 bg2->destroy_cb (bg2);
236 return ret;
237}
238
239
240/**
202 * Find a plugin for the given type. 241 * Find a plugin for the given type.
203 * 242 *
204 * @param ctx context to search 243 * @param ctx context to search
@@ -244,19 +283,26 @@ GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
244 enum GNUNET_BLOCK_Type type, 283 enum GNUNET_BLOCK_Type type,
245 uint32_t nonce, 284 uint32_t nonce,
246 const void *raw_data, 285 const void *raw_data,
247 size_t raw_data_size) 286 size_t raw_data_size,
287 ...)
248{ 288{
249 struct GNUNET_BLOCK_PluginFunctions *plugin; 289 struct GNUNET_BLOCK_PluginFunctions *plugin;
290 struct GNUNET_BLOCK_Group *bg;
291 va_list ap;
250 292
251 plugin = find_plugin (ctx, 293 plugin = find_plugin (ctx,
252 type); 294 type);
253 if (NULL == plugin->create_group) 295 if (NULL == plugin->create_group)
254 return NULL; 296 return NULL;
255 return plugin->create_group (plugin->cls, 297 va_start (ap, raw_data_size);
256 type, 298 bg = plugin->create_group (plugin->cls,
257 nonce, 299 type,
258 raw_data, 300 nonce,
259 raw_data_size); 301 raw_data,
302 raw_data_size,
303 ap);
304 va_end (ap);
305 return bg;
260} 306}
261 307
262 308
@@ -269,10 +315,9 @@ GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
269 * 315 *
270 * @param ctx block contxt 316 * @param ctx block contxt
271 * @param type block type 317 * @param type block type
318 * @param block block group to use
272 * @param eo control flags 319 * @param eo control flags
273 * @param query original query (hash) 320 * @param query original query (hash)
274 * @param bf pointer to bloom filter associated with query; possibly updated (!)
275 * @param bf_mutator mutation value for @a bf
276 * @param xquery extended query data (can be NULL, depending on type) 321 * @param xquery extended query data (can be NULL, depending on type)
277 * @param xquery_size number of bytes in @a xquery 322 * @param xquery_size number of bytes in @a xquery
278 * @param reply_block response to validate 323 * @param reply_block response to validate
@@ -282,25 +327,24 @@ GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
282enum GNUNET_BLOCK_EvaluationResult 327enum GNUNET_BLOCK_EvaluationResult
283GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx, 328GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx,
284 enum GNUNET_BLOCK_Type type, 329 enum GNUNET_BLOCK_Type type,
330 struct GNUNET_BLOCK_Group *group,
285 enum GNUNET_BLOCK_EvaluationOptions eo, 331 enum GNUNET_BLOCK_EvaluationOptions eo,
286 const struct GNUNET_HashCode *query, 332 const struct GNUNET_HashCode *query,
287 struct GNUNET_CONTAINER_BloomFilter **bf,
288 int32_t bf_mutator,
289 const void *xquery, 333 const void *xquery,
290 size_t xquery_size, 334 size_t xquery_size,
291 const void *reply_block, 335 const void *reply_block,
292 size_t reply_block_size) 336 size_t reply_block_size)
293{ 337{
294 struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx, type); 338 struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx,
339 type);
295 340
296 if (plugin == NULL) 341 if (NULL == plugin)
297 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 342 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
298 return plugin->evaluate (plugin->cls, 343 return plugin->evaluate (plugin->cls,
299 type, 344 type,
345 group,
300 eo, 346 eo,
301 query, 347 query,
302 bf,
303 bf_mutator,
304 xquery, 348 xquery,
305 xquery_size, 349 xquery_size,
306 reply_block, 350 reply_block,
@@ -326,7 +370,8 @@ GNUNET_BLOCK_get_key (struct GNUNET_BLOCK_Context *ctx,
326 size_t block_size, 370 size_t block_size,
327 struct GNUNET_HashCode *key) 371 struct GNUNET_HashCode *key)
328{ 372{
329 struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx, type); 373 struct GNUNET_BLOCK_PluginFunctions *plugin = find_plugin (ctx,
374 type);
330 375
331 if (plugin == NULL) 376 if (plugin == NULL)
332 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 377 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -335,65 +380,29 @@ GNUNET_BLOCK_get_key (struct GNUNET_BLOCK_Context *ctx,
335 380
336 381
337/** 382/**
338 * How many bytes should a bloomfilter be if we have already seen 383 * Update block group to filter out the given results. Note that the
339 * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K 384 * use of a hash for seen results implies that the caller magically
340 * gives us the number of bits set per entry. Furthermore, we should 385 * knows how the specific block engine hashes for filtering
341 * not re-size the filter too often (to keep it cheap). 386 * duplicates, so this API may not always apply.
342 *
343 * Since other peers will also add entries but not resize the filter,
344 * we should generally pick a slightly larger size than what the
345 * strict math would suggest.
346 *
347 * @param entry_count expected number of entries in the Bloom filter
348 * @return must be a power of two and smaller or equal to 2^15.
349 */
350static size_t
351compute_bloomfilter_size (unsigned int entry_count)
352{
353 size_t size;
354 unsigned int ideal = (entry_count * GNUNET_CONSTANTS_BLOOMFILTER_K) / 4;
355 uint16_t max = 1 << 15;
356
357 if (entry_count > max)
358 return max;
359 size = 8;
360 while ((size < max) && (size < ideal))
361 size *= 2;
362 if (size > max)
363 return max;
364 return size;
365}
366
367
368/**
369 * Construct a bloom filter that would filter out the given
370 * results.
371 * 387 *
372 * @param bf_mutator mutation value to use 388 * @param bf_mutator mutation value to use
373 * @param seen_results results already seen 389 * @param seen_results results already seen
374 * @param seen_results_count number of entries in @a seen_results 390 * @param seen_results_count number of entries in @a seen_results
375 * @return NULL if seen_results_count is 0, otherwise a BF 391 * @return #GNUNET_SYSERR if not supported, #GNUNET_OK on success
376 * that would match the given results.
377 */ 392 */
378struct GNUNET_CONTAINER_BloomFilter * 393int
379GNUNET_BLOCK_construct_bloomfilter (int32_t bf_mutator, 394GNUNET_BLOCK_group_set_seen (struct GNUNET_BLOCK_Group *bg,
380 const struct GNUNET_HashCode *seen_results, 395 const struct GNUNET_HashCode *seen_results,
381 unsigned int seen_results_count) 396 unsigned int seen_results_count)
382{ 397{
383 struct GNUNET_CONTAINER_BloomFilter *bf; 398 if (NULL == bg)
384 struct GNUNET_HashCode mhash; 399 return GNUNET_OK;
385 unsigned int i; 400 if (NULL == bg->mark_seen_cb)
386 size_t nsize; 401 return GNUNET_SYSERR;
387 402 bg->mark_seen_cb (bg,
388 nsize = compute_bloomfilter_size (seen_results_count); 403 seen_results,
389 bf = GNUNET_CONTAINER_bloomfilter_init (NULL, nsize, 404 seen_results_count);
390 GNUNET_CONSTANTS_BLOOMFILTER_K); 405 return GNUNET_OK;
391 for (i = 0; i < seen_results_count; i++)
392 {
393 GNUNET_BLOCK_mingle_hash (&seen_results[i], bf_mutator, &mhash);
394 GNUNET_CONTAINER_bloomfilter_add (bf, &mhash);
395 }
396 return bf;
397} 406}
398 407
399 408
diff --git a/src/block/plugin_block_template.c b/src/block/plugin_block_template.c
index 6cb69ef5f..0e8107af2 100644
--- a/src/block/plugin_block_template.c
+++ b/src/block/plugin_block_template.c
@@ -26,9 +26,52 @@
26 26
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_block_plugin.h" 28#include "gnunet_block_plugin.h"
29#include "gnunet_block_group_lib.h"
29 30
30#define DEBUG_TEMPLATE GNUNET_EXTRA_LOGGING 31#define DEBUG_TEMPLATE GNUNET_EXTRA_LOGGING
31 32
33/**
34 * Number of bits we set per entry in the bloomfilter.
35 * Do not change!
36 */
37#define BLOOMFILTER_K 16
38
39
40/**
41 * How big is the BF we use for DHT blocks?
42 */
43#define TEMPLATE_BF_SIZE 8
44
45
46/**
47 * Create a new block group.
48 *
49 * @param ctx block context in which the block group is created
50 * @param type type of the block for which we are creating the group
51 * @param nonce random value used to seed the group creation
52 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
53 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
54 * @param va variable arguments specific to @a type
55 * @return block group handle, NULL if block groups are not supported
56 * by this @a type of block (this is not an error)
57 */
58static struct GNUNET_BLOCK_Group *
59block_plugin_template_create_group (void *cls,
60 enum GNUNET_BLOCK_Type type,
61 uint32_t nonce,
62 const void *raw_data,
63 size_t raw_data_size,
64 va_list va)
65{
66 return GNUNET_BLOCK_GROUP_bf_create (cls,
67 TEMPLATE_BF_SIZE,
68 BLOOMFILTER_K,
69 type,
70 nonce,
71 raw_data,
72 raw_data_size);
73}
74
32 75
33/** 76/**
34 * Function called to validate a reply or a request. For 77 * Function called to validate a reply or a request. For
@@ -36,10 +79,9 @@
36 * 79 *
37 * @param cls closure 80 * @param cls closure
38 * @param type block type 81 * @param type block type
82 * @param group block group to use
39 * @param eo control flags 83 * @param eo control flags
40 * @param query original query (hash) 84 * @param query original query (hash)
41 * @param bf pointer to bloom filter associated with query; possibly updated (!)
42 * @param bf_mutator mutation value for bf
43 * @param xquery extrended query data (can be NULL, depending on type) 85 * @param xquery extrended query data (can be NULL, depending on type)
44 * @param xquery_size number of bytes in xquery 86 * @param xquery_size number of bytes in xquery
45 * @param reply_block response to validate 87 * @param reply_block response to validate
@@ -49,36 +91,25 @@
49static enum GNUNET_BLOCK_EvaluationResult 91static enum GNUNET_BLOCK_EvaluationResult
50block_plugin_template_evaluate (void *cls, 92block_plugin_template_evaluate (void *cls,
51 enum GNUNET_BLOCK_Type type, 93 enum GNUNET_BLOCK_Type type,
94 struct GNUNET_BLOCK_Group *group,
52 enum GNUNET_BLOCK_EvaluationOptions eo, 95 enum GNUNET_BLOCK_EvaluationOptions eo,
53 const struct GNUNET_HashCode *query, 96 const struct GNUNET_HashCode *query,
54 struct GNUNET_CONTAINER_BloomFilter **bf,
55 int32_t bf_mutator,
56 const void *xquery, 97 const void *xquery,
57 size_t xquery_size, 98 size_t xquery_size,
58 const void *reply_block, 99 const void *reply_block,
59 size_t reply_block_size) 100 size_t reply_block_size)
60{ 101{
61 struct GNUNET_HashCode chash; 102 struct GNUNET_HashCode chash;
62 struct GNUNET_HashCode mhash;
63 /* FIXME: check validity first... */
64 103
65 /* mandatory duplicate-detection code... */ 104 if (NULL == reply_block)
66 if (NULL != bf) 105 return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
67 { 106 GNUNET_CRYPTO_hash (reply_block,
68 GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); 107 reply_block_size,
69 GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); 108 &chash);
70 if (NULL != *bf) 109 if (GNUNET_YES ==
71 { 110 GNUNET_BLOCK_GROUP_bf_test_and_set (group,
72 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash)) 111 &chash))
73 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; 112 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
74 }
75 else
76 {
77 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, 64 /* BLOOMFILTER_K */);
78 }
79 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
80 }
81 /* FIXME: other stuff here... */
82 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 113 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
83} 114}
84 115
@@ -91,13 +122,15 @@ block_plugin_template_evaluate (void *cls,
91 * @param block block to get the key for 122 * @param block block to get the key for
92 * @param block_size number of bytes in block 123 * @param block_size number of bytes in block
93 * @param key set to the key (query) for the given block 124 * @param key set to the key (query) for the given block
94 * @return GNUNET_OK on success, GNUNET_SYSERR if type not supported 125 * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
95 * (or if extracting a key from a block of this type does not work) 126 * (or if extracting a key from a block of this type does not work)
96 */ 127 */
97static int 128static int
98block_plugin_template_get_key (void *cls, enum GNUNET_BLOCK_Type type, 129block_plugin_template_get_key (void *cls,
99 const void *block, size_t block_size, 130 enum GNUNET_BLOCK_Type type,
100 struct GNUNET_HashCode * key) 131 const void *block,
132 size_t block_size,
133 struct GNUNET_HashCode *key)
101{ 134{
102 return GNUNET_SYSERR; 135 return GNUNET_SYSERR;
103} 136}
@@ -119,6 +152,7 @@ libgnunet_plugin_block_template_init (void *cls)
119 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); 152 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
120 api->evaluate = &block_plugin_template_evaluate; 153 api->evaluate = &block_plugin_template_evaluate;
121 api->get_key = &block_plugin_template_get_key; 154 api->get_key = &block_plugin_template_get_key;
155 api->create_group = &block_plugin_template_create_group;
122 api->types = types; 156 api->types = types;
123 return api; 157 return api;
124} 158}
diff --git a/src/block/plugin_block_test.c b/src/block/plugin_block_test.c
index b692d6230..615f1571b 100644
--- a/src/block/plugin_block_test.c
+++ b/src/block/plugin_block_test.c
@@ -27,7 +27,7 @@
27 27
28#include "platform.h" 28#include "platform.h"
29#include "gnunet_block_plugin.h" 29#include "gnunet_block_plugin.h"
30 30#include "gnunet_block_group_lib.h"
31 31
32/** 32/**
33 * Number of bits we set per entry in the bloomfilter. 33 * Number of bits we set per entry in the bloomfilter.
@@ -36,15 +36,50 @@
36#define BLOOMFILTER_K 16 36#define BLOOMFILTER_K 16
37 37
38/** 38/**
39 * How big is the BF we use for DHT blocks?
40 */
41#define TEST_BF_SIZE 8
42
43
44/**
45 * Create a new block group.
46 *
47 * @param ctx block context in which the block group is created
48 * @param type type of the block for which we are creating the group
49 * @param nonce random value used to seed the group creation
50 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
51 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
52 * @param va variable arguments specific to @a type
53 * @return block group handle, NULL if block groups are not supported
54 * by this @a type of block (this is not an error)
55 */
56static struct GNUNET_BLOCK_Group *
57block_plugin_test_create_group (void *cls,
58 enum GNUNET_BLOCK_Type type,
59 uint32_t nonce,
60 const void *raw_data,
61 size_t raw_data_size,
62 va_list va)
63{
64 return GNUNET_BLOCK_GROUP_bf_create (cls,
65 TEST_BF_SIZE,
66 BLOOMFILTER_K,
67 type,
68 nonce,
69 raw_data,
70 raw_data_size);
71}
72
73
74/**
39 * Function called to validate a reply or a request. For 75 * Function called to validate a reply or a request. For
40 * request evaluation, simply pass "NULL" for the reply_block. 76 * request evaluation, simply pass "NULL" for the reply_block.
41 * 77 *
42 * @param cls closure 78 * @param cls closure
43 * @param type block type 79 * @param type block type
80 * @param group group to check against
44 * @param eo control flags 81 * @param eo control flags
45 * @param query original query (hash) 82 * @param query original query (hash)
46 * @param bf pointer to bloom filter associated with query; possibly updated (!)
47 * @param bf_mutator mutation value for @a bf
48 * @param xquery extrended query data (can be NULL, depending on type) 83 * @param xquery extrended query data (can be NULL, depending on type)
49 * @param xquery_size number of bytes in @a xquery 84 * @param xquery_size number of bytes in @a xquery
50 * @param reply_block response to validate 85 * @param reply_block response to validate
@@ -54,20 +89,21 @@
54static enum GNUNET_BLOCK_EvaluationResult 89static enum GNUNET_BLOCK_EvaluationResult
55block_plugin_test_evaluate (void *cls, 90block_plugin_test_evaluate (void *cls,
56 enum GNUNET_BLOCK_Type type, 91 enum GNUNET_BLOCK_Type type,
92 struct GNUNET_BLOCK_Group *group,
57 enum GNUNET_BLOCK_EvaluationOptions eo, 93 enum GNUNET_BLOCK_EvaluationOptions eo,
58 const struct GNUNET_HashCode *query, 94 const struct GNUNET_HashCode *query,
59 struct GNUNET_CONTAINER_BloomFilter **bf,
60 int32_t bf_mutator,
61 const void *xquery, 95 const void *xquery,
62 size_t xquery_size, 96 size_t xquery_size,
63 const void *reply_block, 97 const void *reply_block,
64 size_t reply_block_size) 98 size_t reply_block_size)
65{ 99{
66 struct GNUNET_HashCode chash; 100 struct GNUNET_HashCode chash;
67 struct GNUNET_HashCode mhash;
68 101
69 if ( GNUNET_BLOCK_TYPE_TEST != type) 102 if ( GNUNET_BLOCK_TYPE_TEST != type)
103 {
104 GNUNET_break (0);
70 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 105 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
106 }
71 if (0 != xquery_size) 107 if (0 != xquery_size)
72 { 108 {
73 GNUNET_break_op (0); 109 GNUNET_break_op (0);
@@ -75,22 +111,13 @@ block_plugin_test_evaluate (void *cls,
75 } 111 }
76 if (NULL == reply_block) 112 if (NULL == reply_block)
77 return GNUNET_BLOCK_EVALUATION_REQUEST_VALID; 113 return GNUNET_BLOCK_EVALUATION_REQUEST_VALID;
78 114 GNUNET_CRYPTO_hash (reply_block,
79 if (NULL != bf) 115 reply_block_size,
80 { 116 &chash);
81 GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); 117 if (GNUNET_YES ==
82 GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); 118 GNUNET_BLOCK_GROUP_bf_test_and_set (group,
83 if (NULL != *bf) 119 &chash))
84 { 120 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
85 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
86 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
87 }
88 else
89 {
90 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K);
91 }
92 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
93 }
94 return GNUNET_BLOCK_EVALUATION_OK_MORE; 121 return GNUNET_BLOCK_EVALUATION_OK_MORE;
95} 122}
96 123
@@ -107,9 +134,11 @@ block_plugin_test_evaluate (void *cls,
107 * (or if extracting a key from a block of this type does not work) 134 * (or if extracting a key from a block of this type does not work)
108 */ 135 */
109static int 136static int
110block_plugin_test_get_key (void *cls, enum GNUNET_BLOCK_Type type, 137block_plugin_test_get_key (void *cls,
111 const void *block, size_t block_size, 138 enum GNUNET_BLOCK_Type type,
112 struct GNUNET_HashCode * key) 139 const void *block,
140 size_t block_size,
141 struct GNUNET_HashCode *key)
113{ 142{
114 /* always fails since there is no fixed relationship between 143 /* always fails since there is no fixed relationship between
115 * keys and values for test values */ 144 * keys and values for test values */
@@ -136,6 +165,7 @@ libgnunet_plugin_block_test_init (void *cls)
136 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); 165 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
137 api->evaluate = &block_plugin_test_evaluate; 166 api->evaluate = &block_plugin_test_evaluate;
138 api->get_key = &block_plugin_test_get_key; 167 api->get_key = &block_plugin_test_get_key;
168 api->create_group = &block_plugin_test_create_group;
139 api->types = types; 169 api->types = types;
140 return api; 170 return api;
141} 171}
diff --git a/src/dht/Makefile.am b/src/dht/Makefile.am
index 93dae9f6e..4216af400 100644
--- a/src/dht/Makefile.am
+++ b/src/dht/Makefile.am
@@ -82,6 +82,7 @@ gnunet_service_dht_LDADD = \
82 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \ 82 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
83 $(top_builddir)/src/hello/libgnunethello.la \ 83 $(top_builddir)/src/hello/libgnunethello.la \
84 $(top_builddir)/src/block/libgnunetblock.la \ 84 $(top_builddir)/src/block/libgnunetblock.la \
85 $(top_builddir)/src/block/libgnunetblockgroup.la \
85 $(top_builddir)/src/datacache/libgnunetdatacache.la \ 86 $(top_builddir)/src/datacache/libgnunetdatacache.la \
86 $(top_builddir)/src/util/libgnunetutil.la \ 87 $(top_builddir)/src/util/libgnunetutil.la \
87 -lm 88 -lm
@@ -102,6 +103,7 @@ gnunet_service_dht_xvine_LDADD = \
102 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \ 103 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
103 $(top_builddir)/src/hello/libgnunethello.la \ 104 $(top_builddir)/src/hello/libgnunethello.la \
104 $(top_builddir)/src/block/libgnunetblock.la \ 105 $(top_builddir)/src/block/libgnunetblock.la \
106 $(top_builddir)/src/block/libgnunetblockgroup.la \
105 $(top_builddir)/src/datacache/libgnunetdatacache.la \ 107 $(top_builddir)/src/datacache/libgnunetdatacache.la \
106 $(top_builddir)/src/util/libgnunetutil.la \ 108 $(top_builddir)/src/util/libgnunetutil.la \
107 -lm 109 -lm
@@ -120,6 +122,7 @@ gnunet_service_dht_whanau_LDADD = \
120 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \ 122 $(top_builddir)/src/peerinfo/libgnunetpeerinfo.la \
121 $(top_builddir)/src/hello/libgnunethello.la \ 123 $(top_builddir)/src/hello/libgnunethello.la \
122 $(top_builddir)/src/block/libgnunetblock.la \ 124 $(top_builddir)/src/block/libgnunetblock.la \
125 $(top_builddir)/src/block/libgnunetblockgroup.la \
123 $(top_builddir)/src/datacache/libgnunetdatacache.la \ 126 $(top_builddir)/src/datacache/libgnunetdatacache.la \
124 $(top_builddir)/src/util/libgnunetutil.la \ 127 $(top_builddir)/src/util/libgnunetutil.la \
125 -lm 128 -lm
diff --git a/src/dht/gnunet-service-dht_clients.c b/src/dht/gnunet-service-dht_clients.c
index 5ba4e5820..a42356e5f 100644
--- a/src/dht/gnunet-service-dht_clients.c
+++ b/src/dht/gnunet-service-dht_clients.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 Copyright (C) 2009, 2010, 2011, 2016 GNUnet e.V. 3 Copyright (C) 2009, 2010, 2011, 2016, 2017 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -362,21 +362,22 @@ client_disconnect_cb (void *cls,
362static void 362static void
363transmit_request (struct ClientQueryRecord *cqr) 363transmit_request (struct ClientQueryRecord *cqr)
364{ 364{
365 int32_t reply_bf_mutator; 365 struct GNUNET_BLOCK_Group *bg;
366 struct GNUNET_CONTAINER_BloomFilter *reply_bf;
367 struct GNUNET_CONTAINER_BloomFilter *peer_bf; 366 struct GNUNET_CONTAINER_BloomFilter *peer_bf;
368 367
369 GNUNET_STATISTICS_update (GDS_stats, 368 GNUNET_STATISTICS_update (GDS_stats,
370 gettext_noop ("# GET requests from clients injected"), 369 gettext_noop ("# GET requests from clients injected"),
371 1, 370 1,
372 GNUNET_NO); 371 GNUNET_NO);
373 reply_bf_mutator = 372 bg = GNUNET_BLOCK_group_create (GDS_block_context,
374 (int32_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 373 cqr->type,
375 UINT32_MAX); 374 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
376 reply_bf 375 UINT32_MAX),
377 = GNUNET_BLOCK_construct_bloomfilter (reply_bf_mutator, 376 NULL,
378 cqr->seen_replies, 377 0);
379 cqr->seen_replies_count); 378 GNUNET_BLOCK_group_set_seen (bg,
379 cqr->seen_replies,
380 cqr->seen_replies_count);
380 peer_bf 381 peer_bf
381 = GNUNET_CONTAINER_bloomfilter_init (NULL, 382 = GNUNET_CONTAINER_bloomfilter_init (NULL,
382 DHT_BLOOM_SIZE, 383 DHT_BLOOM_SIZE,
@@ -393,10 +394,9 @@ transmit_request (struct ClientQueryRecord *cqr)
393 &cqr->key, 394 &cqr->key,
394 cqr->xquery, 395 cqr->xquery,
395 cqr->xquery_size, 396 cqr->xquery_size,
396 reply_bf, 397 bg,
397 reply_bf_mutator,
398 peer_bf); 398 peer_bf);
399 GNUNET_CONTAINER_bloomfilter_free (reply_bf); 399 GNUNET_BLOCK_group_destroy (bg);
400 GNUNET_CONTAINER_bloomfilter_free (peer_bf); 400 GNUNET_CONTAINER_bloomfilter_free (peer_bf);
401 401
402 /* exponential back-off for retries. 402 /* exponential back-off for retries.
@@ -668,7 +668,6 @@ handle_dht_local_get (void *cls,
668 cqr->xquery, 668 cqr->xquery,
669 xquery_size, 669 xquery_size,
670 NULL, 670 NULL,
671 0,
672 &handle_local_result, 671 &handle_local_result,
673 ch); 672 ch);
674 GNUNET_SERVICE_client_continue (ch->client); 673 GNUNET_SERVICE_client_continue (ch->client);
@@ -1052,10 +1051,9 @@ forward_reply (void *cls,
1052 eval 1051 eval
1053 = GNUNET_BLOCK_evaluate (GDS_block_context, 1052 = GNUNET_BLOCK_evaluate (GDS_block_context,
1054 record->type, 1053 record->type,
1054 NULL,
1055 GNUNET_BLOCK_EO_NONE, 1055 GNUNET_BLOCK_EO_NONE,
1056 key, 1056 key,
1057 NULL,
1058 0,
1059 record->xquery, 1057 record->xquery,
1060 record->xquery_size, 1058 record->xquery_size,
1061 frc->data, 1059 frc->data,
diff --git a/src/dht/gnunet-service-dht_datacache.c b/src/dht/gnunet-service-dht_datacache.c
index 74fa1cc29..fef637cad 100644
--- a/src/dht/gnunet-service-dht_datacache.c
+++ b/src/dht/gnunet-service-dht_datacache.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 Copyright (C) 2009, 2010, 2011, 2015 GNUnet e.V. 3 Copyright (C) 2009, 2010, 2011, 2015, 2017 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -109,39 +109,34 @@ struct GetRequestContext
109 const void *xquery; 109 const void *xquery;
110 110
111 /** 111 /**
112 * Bloomfilter to filter out duplicate replies (updated)
113 */
114 struct GNUNET_CONTAINER_BloomFilter **reply_bf;
115
116 /**
117 * The key this request was about 112 * The key this request was about
118 */ 113 */
119 struct GNUNET_HashCode key; 114 struct GNUNET_HashCode key;
120 115
121 /** 116 /**
122 * Number of bytes in xquery. 117 * Block group to use to evaluate replies (updated)
123 */ 118 */
124 size_t xquery_size; 119 struct GNUNET_BLOCK_Group *bg;
125 120
126 /** 121 /**
127 * Mutator value for the @e reply_bf, see gnunet_block_lib.h 122 * Function to call on results.
128 */ 123 */
129 uint32_t reply_bf_mutator; 124 GDS_DATACACHE_GetCallback gc;
130 125
131 /** 126 /**
132 * Return value to give back. 127 * Closure for @e gc.
133 */ 128 */
134 enum GNUNET_BLOCK_EvaluationResult eval; 129 void *gc_cls;
135 130
136 /** 131 /**
137 * Function to call on results. 132 * Number of bytes in xquery.
138 */ 133 */
139 GDS_DATACACHE_GetCallback gc; 134 size_t xquery_size;
140 135
141 /** 136 /**
142 * Closure for @e gc. 137 * Return value to give back.
143 */ 138 */
144 void *gc_cls; 139 enum GNUNET_BLOCK_EvaluationResult eval;
145 140
146}; 141};
147 142
@@ -181,10 +176,9 @@ datacache_get_iterator (void *cls,
181 eval 176 eval
182 = GNUNET_BLOCK_evaluate (GDS_block_context, 177 = GNUNET_BLOCK_evaluate (GDS_block_context,
183 type, 178 type,
179 ctx->bg,
184 GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO, 180 GNUNET_BLOCK_EO_LOCAL_SKIP_CRYPTO,
185 key, 181 key,
186 ctx->reply_bf,
187 ctx->reply_bf_mutator,
188 ctx->xquery, 182 ctx->xquery,
189 ctx->xquery_size, 183 ctx->xquery_size,
190 data, 184 data,
@@ -256,8 +250,7 @@ datacache_get_iterator (void *cls,
256 * @param type requested data type 250 * @param type requested data type
257 * @param xquery extended query 251 * @param xquery extended query
258 * @param xquery_size number of bytes in @a xquery 252 * @param xquery_size number of bytes in @a xquery
259 * @param reply_bf where the reply bf is (to be) stored, possibly updated, can be NULL 253 * @param bg block group to use for reply evaluation
260 * @param reply_bf_mutator mutation value for @a reply_bf
261 * @param gc function to call on the results 254 * @param gc function to call on the results
262 * @param gc_cls closure for @a gc 255 * @param gc_cls closure for @a gc
263 * @return evaluation result for the local replies 256 * @return evaluation result for the local replies
@@ -267,8 +260,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
267 enum GNUNET_BLOCK_Type type, 260 enum GNUNET_BLOCK_Type type,
268 const void *xquery, 261 const void *xquery,
269 size_t xquery_size, 262 size_t xquery_size,
270 struct GNUNET_CONTAINER_BloomFilter **reply_bf, 263 struct GNUNET_BLOCK_Group *bg,
271 uint32_t reply_bf_mutator,
272 GDS_DATACACHE_GetCallback gc, 264 GDS_DATACACHE_GetCallback gc,
273 void *gc_cls) 265 void *gc_cls)
274{ 266{
@@ -285,8 +277,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
285 ctx.key = *key; 277 ctx.key = *key;
286 ctx.xquery = xquery; 278 ctx.xquery = xquery;
287 ctx.xquery_size = xquery_size; 279 ctx.xquery_size = xquery_size;
288 ctx.reply_bf = reply_bf; 280 ctx.bg = bg;
289 ctx.reply_bf_mutator = reply_bf_mutator;
290 ctx.gc = gc; 281 ctx.gc = gc;
291 ctx.gc_cls = gc_cls; 282 ctx.gc_cls = gc_cls;
292 r = GNUNET_DATACACHE_get (datacache, 283 r = GNUNET_DATACACHE_get (datacache,
diff --git a/src/dht/gnunet-service-dht_datacache.h b/src/dht/gnunet-service-dht_datacache.h
index 5069883c7..ff6ae23da 100644
--- a/src/dht/gnunet-service-dht_datacache.h
+++ b/src/dht/gnunet-service-dht_datacache.h
@@ -87,8 +87,7 @@ typedef void
87 * @param type requested data type 87 * @param type requested data type
88 * @param xquery extended query 88 * @param xquery extended query
89 * @param xquery_size number of bytes in xquery 89 * @param xquery_size number of bytes in xquery
90 * @param reply_bf where the reply bf is (to be) stored, possibly updated!, can be NULL 90 * @param bg block group to use for evaluation of replies
91 * @param reply_bf_mutator mutation value for reply_bf
92 * @param gc function to call on the results 91 * @param gc function to call on the results
93 * @param gc_cls closure for @a gc 92 * @param gc_cls closure for @a gc
94 * @return evaluation result for the local replies 93 * @return evaluation result for the local replies
@@ -98,8 +97,7 @@ GDS_DATACACHE_handle_get (const struct GNUNET_HashCode *key,
98 enum GNUNET_BLOCK_Type type, 97 enum GNUNET_BLOCK_Type type,
99 const void *xquery, 98 const void *xquery,
100 size_t xquery_size, 99 size_t xquery_size,
101 struct GNUNET_CONTAINER_BloomFilter **reply_bf, 100 struct GNUNET_BLOCK_Group *bg,
102 uint32_t reply_bf_mutator,
103 GDS_DATACACHE_GetCallback gc, 101 GDS_DATACACHE_GetCallback gc,
104 void *gc_cls); 102 void *gc_cls);
105 103
diff --git a/src/dht/gnunet-service-dht_neighbours.c b/src/dht/gnunet-service-dht_neighbours.c
index 7f3a44588..1bbc95a06 100644
--- a/src/dht/gnunet-service-dht_neighbours.c
+++ b/src/dht/gnunet-service-dht_neighbours.c
@@ -1,6 +1,6 @@
1/* 1/*
2 This file is part of GNUnet. 2 This file is part of GNUnet.
3 Copyright (C) 2009-2016 GNUnet e.V. 3 Copyright (C) 2009-2017 GNUnet e.V.
4 4
5 GNUnet is free software; you can redistribute it and/or modify 5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published 6 it under the terms of the GNU General Public License as published
@@ -27,6 +27,7 @@
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_util_lib.h" 28#include "gnunet_util_lib.h"
29#include "gnunet_block_lib.h" 29#include "gnunet_block_lib.h"
30#include "gnunet_block_group_lib.h"
30#include "gnunet_hello_lib.h" 31#include "gnunet_hello_lib.h"
31#include "gnunet_constants.h" 32#include "gnunet_constants.h"
32#include "gnunet_protocols.h" 33#include "gnunet_protocols.h"
@@ -50,7 +51,7 @@
50 51
51/** 52/**
52 * Enable slow sanity checks to debug issues. 53 * Enable slow sanity checks to debug issues.
53 */ 54 */
54#define SANITY_CHECKS 1 55#define SANITY_CHECKS 1
55 56
56/** 57/**
@@ -602,27 +603,10 @@ update_connect_preferences ()
602 603
603 604
604/** 605/**
605 * Closure for #add_known_to_bloom().
606 */
607struct BloomConstructorContext
608{
609 /**
610 * Bloom filter under construction.
611 */
612 struct GNUNET_CONTAINER_BloomFilter *bloom;
613
614 /**
615 * Mutator to use.
616 */
617 uint32_t bf_mutator;
618};
619
620
621/**
622 * Add each of the peers we already know to the bloom filter of 606 * Add each of the peers we already know to the bloom filter of
623 * the request so that we don't get duplicate HELLOs. 607 * the request so that we don't get duplicate HELLOs.
624 * 608 *
625 * @param cls the 'struct BloomConstructorContext'. 609 * @param cls the `struct GNUNET_BLOCK_Group`
626 * @param key peer identity to add to the bloom filter 610 * @param key peer identity to add to the bloom filter
627 * @param value value the peer information (unused) 611 * @param value value the peer information (unused)
628 * @return #GNUNET_YES (we should continue to iterate) 612 * @return #GNUNET_YES (we should continue to iterate)
@@ -632,22 +616,17 @@ add_known_to_bloom (void *cls,
632 const struct GNUNET_PeerIdentity *key, 616 const struct GNUNET_PeerIdentity *key,
633 void *value) 617 void *value)
634{ 618{
635 struct BloomConstructorContext *ctx = cls; 619 struct GNUNET_BLOCK_Group *bg = cls;
636 struct GNUNET_HashCode key_hash; 620 struct GNUNET_HashCode key_hash;
637 struct GNUNET_HashCode mh;
638 621
639 GNUNET_CRYPTO_hash (key, 622 GNUNET_CRYPTO_hash (key,
640 sizeof (struct GNUNET_PeerIdentity), 623 sizeof (struct GNUNET_PeerIdentity),
641 &key_hash); 624 &key_hash);
642 GNUNET_BLOCK_mingle_hash (&key_hash, 625 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
643 ctx->bf_mutator, 626 &key_hash);
644 &mh);
645 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 627 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
646 "Adding known peer (%s) to bloomfilter for FIND PEER with mutation %u\n", 628 "Adding known peer (%s) to bloomfilter for FIND PEER\n",
647 GNUNET_i2s (key), 629 GNUNET_i2s (key));
648 ctx->bf_mutator);
649 GNUNET_CONTAINER_bloomfilter_add (ctx->bloom,
650 &mh);
651 return GNUNET_YES; 630 return GNUNET_YES;
652} 631}
653 632
@@ -663,7 +642,7 @@ static void
663send_find_peer_message (void *cls) 642send_find_peer_message (void *cls)
664{ 643{
665 struct GNUNET_TIME_Relative next_send_time; 644 struct GNUNET_TIME_Relative next_send_time;
666 struct BloomConstructorContext bcc; 645 struct GNUNET_BLOCK_Group *bg;
667 struct GNUNET_CONTAINER_BloomFilter *peer_bf; 646 struct GNUNET_CONTAINER_BloomFilter *peer_bf;
668 647
669 find_peer_task = NULL; 648 find_peer_task = NULL;
@@ -677,30 +656,37 @@ send_find_peer_message (void *cls)
677 newly_found_peers = 0; 656 newly_found_peers = 0;
678 return; 657 return;
679 } 658 }
680 bcc.bf_mutator = 659 bg = GNUNET_BLOCK_GROUP_bf_create (NULL,
681 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 660 DHT_BLOOM_SIZE,
682 UINT32_MAX); 661 GNUNET_CONSTANTS_BLOOMFILTER_K,
683 bcc.bloom = 662 GNUNET_BLOCK_TYPE_DHT_HELLO,
684 GNUNET_CONTAINER_bloomfilter_init (NULL, DHT_BLOOM_SIZE, 663 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
685 GNUNET_CONSTANTS_BLOOMFILTER_K); 664 UINT32_MAX),
665 NULL,
666 0);
686 GNUNET_CONTAINER_multipeermap_iterate (all_connected_peers, 667 GNUNET_CONTAINER_multipeermap_iterate (all_connected_peers,
687 &add_known_to_bloom, 668 &add_known_to_bloom,
688 &bcc); 669 bg);
689 GNUNET_STATISTICS_update (GDS_stats, 670 GNUNET_STATISTICS_update (GDS_stats,
690 gettext_noop ("# FIND PEER messages initiated"), 671 gettext_noop ("# FIND PEER messages initiated"),
691 1, 672 1,
692 GNUNET_NO); 673 GNUNET_NO);
693 peer_bf = 674 peer_bf
694 GNUNET_CONTAINER_bloomfilter_init (NULL, DHT_BLOOM_SIZE, 675 = GNUNET_CONTAINER_bloomfilter_init (NULL,
676 DHT_BLOOM_SIZE,
695 GNUNET_CONSTANTS_BLOOMFILTER_K); 677 GNUNET_CONSTANTS_BLOOMFILTER_K);
696 // FIXME: pass priority!? 678 // FIXME: pass priority!?
697 GDS_NEIGHBOURS_handle_get (GNUNET_BLOCK_TYPE_DHT_HELLO, 679 GDS_NEIGHBOURS_handle_get (GNUNET_BLOCK_TYPE_DHT_HELLO,
698 GNUNET_DHT_RO_FIND_PEER, 680 GNUNET_DHT_RO_FIND_PEER,
699 FIND_PEER_REPLICATION_LEVEL, 0, 681 FIND_PEER_REPLICATION_LEVEL,
700 &my_identity_hash, NULL, 0, bcc.bloom, 682 0,
701 bcc.bf_mutator, peer_bf); 683 &my_identity_hash,
684 NULL,
685 0,
686 bg,
687 peer_bf);
702 GNUNET_CONTAINER_bloomfilter_free (peer_bf); 688 GNUNET_CONTAINER_bloomfilter_free (peer_bf);
703 GNUNET_CONTAINER_bloomfilter_free (bcc.bloom); 689 GNUNET_BLOCK_group_destroy (bg);
704 /* schedule next round */ 690 /* schedule next round */
705 next_send_time.rel_value_us = 691 next_send_time.rel_value_us =
706 DHT_MINIMUM_FIND_PEER_INTERVAL.rel_value_us + 692 DHT_MINIMUM_FIND_PEER_INTERVAL.rel_value_us +
@@ -1357,8 +1343,7 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
1357 * @param key key for the content 1343 * @param key key for the content
1358 * @param xquery extended query 1344 * @param xquery extended query
1359 * @param xquery_size number of bytes in @a xquery 1345 * @param xquery_size number of bytes in @a xquery
1360 * @param reply_bf bloomfilter to filter duplicates 1346 * @param bg group to use for filtering replies
1361 * @param reply_bf_mutator mutator for @a reply_bf
1362 * @param peer_bf filter for peers not to select (again) 1347 * @param peer_bf filter for peers not to select (again)
1363 * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not 1348 * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
1364 */ 1349 */
@@ -1366,14 +1351,14 @@ int
1366GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type, 1351GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1367 enum GNUNET_DHT_RouteOption options, 1352 enum GNUNET_DHT_RouteOption options,
1368 uint32_t desired_replication_level, 1353 uint32_t desired_replication_level,
1369 uint32_t hop_count, const struct GNUNET_HashCode * key, 1354 uint32_t hop_count,
1370 const void *xquery, size_t xquery_size, 1355 const struct GNUNET_HashCode *key,
1371 const struct GNUNET_CONTAINER_BloomFilter *reply_bf, 1356 const void *xquery,
1372 uint32_t reply_bf_mutator, 1357 size_t xquery_size,
1358 struct GNUNET_BLOCK_Group *bg,
1373 struct GNUNET_CONTAINER_BloomFilter *peer_bf) 1359 struct GNUNET_CONTAINER_BloomFilter *peer_bf)
1374{ 1360{
1375 unsigned int target_count; 1361 unsigned int target_count;
1376 unsigned int i;
1377 struct PeerInfo **targets; 1362 struct PeerInfo **targets;
1378 struct PeerInfo *target; 1363 struct PeerInfo *target;
1379 struct GNUNET_MQ_Envelope *env; 1364 struct GNUNET_MQ_Envelope *env;
@@ -1381,7 +1366,9 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1381 struct PeerGetMessage *pgm; 1366 struct PeerGetMessage *pgm;
1382 char *xq; 1367 char *xq;
1383 size_t reply_bf_size; 1368 size_t reply_bf_size;
1369 void *reply_bf;
1384 unsigned int skip_count; 1370 unsigned int skip_count;
1371 uint32_t bf_nonce;
1385 1372
1386 GNUNET_assert (NULL != peer_bf); 1373 GNUNET_assert (NULL != peer_bf);
1387 GNUNET_STATISTICS_update (GDS_stats, 1374 GNUNET_STATISTICS_update (GDS_stats,
@@ -1408,11 +1395,22 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1408 GNUNET_i2s (&my_identity)); 1395 GNUNET_i2s (&my_identity));
1409 return GNUNET_NO; 1396 return GNUNET_NO;
1410 } 1397 }
1411 reply_bf_size = GNUNET_CONTAINER_bloomfilter_get_size (reply_bf); 1398 if (GNUNET_OK !=
1399 GNUNET_BLOCK_group_serialize (bg,
1400 &bf_nonce,
1401 &reply_bf,
1402 &reply_bf_size))
1403 {
1404 reply_bf = NULL;
1405 reply_bf_size = 0;
1406 bf_nonce = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1407 UINT32_MAX);
1408 }
1412 msize = xquery_size + reply_bf_size; 1409 msize = xquery_size + reply_bf_size;
1413 if (msize + sizeof (struct PeerGetMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE) 1410 if (msize + sizeof (struct PeerGetMessage) >= GNUNET_SERVER_MAX_MESSAGE_SIZE)
1414 { 1411 {
1415 GNUNET_break (0); 1412 GNUNET_break (0);
1413 GNUNET_free_non_null (reply_bf);
1416 GNUNET_free (targets); 1414 GNUNET_free (targets);
1417 return GNUNET_NO; 1415 return GNUNET_NO;
1418 } 1416 }
@@ -1422,7 +1420,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1422 GNUNET_NO); 1420 GNUNET_NO);
1423 /* forward request */ 1421 /* forward request */
1424 skip_count = 0; 1422 skip_count = 0;
1425 for (i = 0; i < target_count; i++) 1423 for (unsigned int i = 0; i < target_count; i++)
1426 { 1424 {
1427 target = targets[i]; 1425 target = targets[i];
1428 if (GNUNET_MQ_get_length (target->mq) >= MAXIMUM_PENDING_PER_PEER) 1426 if (GNUNET_MQ_get_length (target->mq) >= MAXIMUM_PENDING_PER_PEER)
@@ -1447,7 +1445,7 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1447 pgm->hop_count = htonl (hop_count + 1); 1445 pgm->hop_count = htonl (hop_count + 1);
1448 pgm->desired_replication_level = htonl (desired_replication_level); 1446 pgm->desired_replication_level = htonl (desired_replication_level);
1449 pgm->xquery_size = htonl (xquery_size); 1447 pgm->xquery_size = htonl (xquery_size);
1450 pgm->bf_mutator = reply_bf_mutator; 1448 pgm->bf_mutator = bf_nonce;
1451 GNUNET_break (GNUNET_YES == 1449 GNUNET_break (GNUNET_YES ==
1452 GNUNET_CONTAINER_bloomfilter_test (peer_bf, 1450 GNUNET_CONTAINER_bloomfilter_test (peer_bf,
1453 &target->phash)); 1451 &target->phash));
@@ -1460,16 +1458,14 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
1460 GNUNET_memcpy (xq, 1458 GNUNET_memcpy (xq,
1461 xquery, 1459 xquery,
1462 xquery_size); 1460 xquery_size);
1463 if (NULL != reply_bf) 1461 GNUNET_memcpy (&xq[xquery_size],
1464 GNUNET_assert (GNUNET_OK == 1462 reply_bf,
1465 GNUNET_CONTAINER_bloomfilter_get_raw_data (reply_bf, 1463 reply_bf_size);
1466 &xq
1467 [xquery_size],
1468 reply_bf_size));
1469 GNUNET_MQ_send (target->mq, 1464 GNUNET_MQ_send (target->mq,
1470 env); 1465 env);
1471 } 1466 }
1472 GNUNET_free (targets); 1467 GNUNET_free (targets);
1468 GNUNET_free_non_null (reply_bf);
1473 return (skip_count < target_count) ? GNUNET_OK : GNUNET_NO; 1469 return (skip_count < target_count) ? GNUNET_OK : GNUNET_NO;
1474} 1470}
1475 1471
@@ -1717,11 +1713,12 @@ handle_dht_p2p_put (void *cls,
1717 { 1713 {
1718 switch (GNUNET_BLOCK_evaluate (GDS_block_context, 1714 switch (GNUNET_BLOCK_evaluate (GDS_block_context,
1719 ntohl (put->type), 1715 ntohl (put->type),
1716 NULL, /* query group */
1720 GNUNET_BLOCK_EO_NONE, 1717 GNUNET_BLOCK_EO_NONE,
1721 NULL, /* query */ 1718 NULL, /* query */
1722 NULL, 0, /* bloom filer */
1723 NULL, 0, /* xquery */ 1719 NULL, 0, /* xquery */
1724 payload, payload_size)) 1720 payload,
1721 payload_size))
1725 { 1722 {
1726 case GNUNET_BLOCK_EVALUATION_OK_MORE: 1723 case GNUNET_BLOCK_EVALUATION_OK_MORE:
1727 case GNUNET_BLOCK_EVALUATION_OK_LAST: 1724 case GNUNET_BLOCK_EVALUATION_OK_LAST:
@@ -1762,7 +1759,7 @@ handle_dht_p2p_put (void *cls,
1762 } 1759 }
1763 GNUNET_break (0 != memcmp (&pp[i], 1760 GNUNET_break (0 != memcmp (&pp[i],
1764 peer->id, 1761 peer->id,
1765 sizeof (struct GNUNET_PeerIdentity))); 1762 sizeof (struct GNUNET_PeerIdentity)));
1766 } 1763 }
1767#endif 1764#endif
1768 GNUNET_memcpy (pp, 1765 GNUNET_memcpy (pp,
@@ -1830,30 +1827,25 @@ handle_dht_p2p_put (void *cls,
1830 * 1827 *
1831 * @param sender sender of the FIND PEER request 1828 * @param sender sender of the FIND PEER request
1832 * @param key peers close to this key are desired 1829 * @param key peers close to this key are desired
1833 * @param bf peers matching this bf are excluded 1830 * @param bg group for filtering peers
1834 * @param bf_mutator mutator for bf
1835 */ 1831 */
1836static void 1832static void
1837handle_find_peer (const struct GNUNET_PeerIdentity *sender, 1833handle_find_peer (const struct GNUNET_PeerIdentity *sender,
1838 const struct GNUNET_HashCode *key, 1834 const struct GNUNET_HashCode *key,
1839 struct GNUNET_CONTAINER_BloomFilter *bf, 1835 struct GNUNET_BLOCK_Group *bg)
1840 uint32_t bf_mutator)
1841{ 1836{
1842 int bucket_idx; 1837 int bucket_idx;
1843 struct PeerBucket *bucket; 1838 struct PeerBucket *bucket;
1844 struct PeerInfo *peer; 1839 struct PeerInfo *peer;
1845 unsigned int choice; 1840 unsigned int choice;
1846 struct GNUNET_HashCode mhash;
1847 const struct GNUNET_HELLO_Message *hello; 1841 const struct GNUNET_HELLO_Message *hello;
1848 1842
1849 /* first, check about our own HELLO */ 1843 /* first, check about our own HELLO */
1850 if (NULL != GDS_my_hello) 1844 if (NULL != GDS_my_hello)
1851 { 1845 {
1852 GNUNET_BLOCK_mingle_hash (&my_identity_hash, 1846 if (GNUNET_YES !=
1853 bf_mutator, 1847 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
1854 &mhash); 1848 &my_identity_hash))
1855 if ((NULL == bf) ||
1856 (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (bf, &mhash)))
1857 { 1849 {
1858 size_t hello_size; 1850 size_t hello_size;
1859 1851
@@ -1913,18 +1905,15 @@ handle_find_peer (const struct GNUNET_PeerIdentity *sender,
1913 do 1905 do
1914 { 1906 {
1915 peer = peer->next; 1907 peer = peer->next;
1916 if (choice-- == 0) 1908 if (0 == choice--)
1917 return; /* no non-masked peer available */ 1909 return; /* no non-masked peer available */
1918 if (NULL == peer) 1910 if (NULL == peer)
1919 peer = bucket->head; 1911 peer = bucket->head;
1920 GNUNET_BLOCK_mingle_hash (&peer->phash,
1921 bf_mutator,
1922 &mhash);
1923 hello = GDS_HELLO_get (peer->id); 1912 hello = GDS_HELLO_get (peer->id);
1924 } while ( (hello == NULL) || 1913 } while ( (NULL == hello) ||
1925 (GNUNET_YES == 1914 (GNUNET_YES ==
1926 GNUNET_CONTAINER_bloomfilter_test (bf, 1915 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
1927 &mhash)) ); 1916 &peer->phash)) );
1928 GDS_NEIGHBOURS_handle_reply (sender, 1917 GDS_NEIGHBOURS_handle_reply (sender,
1929 GNUNET_BLOCK_TYPE_DHT_HELLO, 1918 GNUNET_BLOCK_TYPE_DHT_HELLO,
1930 GNUNET_TIME_relative_to_absolute 1919 GNUNET_TIME_relative_to_absolute
@@ -2019,7 +2008,7 @@ handle_dht_p2p_get (void *cls,
2019 enum GNUNET_BLOCK_Type type; 2008 enum GNUNET_BLOCK_Type type;
2020 enum GNUNET_DHT_RouteOption options; 2009 enum GNUNET_DHT_RouteOption options;
2021 enum GNUNET_BLOCK_EvaluationResult eval; 2010 enum GNUNET_BLOCK_EvaluationResult eval;
2022 struct GNUNET_CONTAINER_BloomFilter *reply_bf; 2011 struct GNUNET_BLOCK_Group *bg;
2023 struct GNUNET_CONTAINER_BloomFilter *peer_bf; 2012 struct GNUNET_CONTAINER_BloomFilter *peer_bf;
2024 const char *xquery; 2013 const char *xquery;
2025 int forwarded; 2014 int forwarded;
@@ -2036,7 +2025,6 @@ handle_dht_p2p_get (void *cls,
2036 type = ntohl (get->type); 2025 type = ntohl (get->type);
2037 options = ntohl (get->options); 2026 options = ntohl (get->options);
2038 xquery = (const char *) &get[1]; 2027 xquery = (const char *) &get[1];
2039 reply_bf = NULL;
2040 GNUNET_STATISTICS_update (GDS_stats, 2028 GNUNET_STATISTICS_update (GDS_stats,
2041 gettext_noop ("# P2P GET requests received"), 2029 gettext_noop ("# P2P GET requests received"),
2042 1, 2030 1,
@@ -2064,19 +2052,17 @@ handle_dht_p2p_get (void *cls,
2064 xquery); 2052 xquery);
2065 GNUNET_free (tmp); 2053 GNUNET_free (tmp);
2066 } 2054 }
2067 2055 bg = GNUNET_BLOCK_group_create (GDS_block_context,
2068 if (reply_bf_size > 0) 2056 type,
2069 reply_bf = 2057 get->bf_mutator,
2070 GNUNET_CONTAINER_bloomfilter_init (&xquery[xquery_size], 2058 &xquery[xquery_size],
2071 reply_bf_size, 2059 reply_bf_size);
2072 GNUNET_CONSTANTS_BLOOMFILTER_K); 2060 eval
2073 eval = 2061 = GNUNET_BLOCK_evaluate (GDS_block_context,
2074 GNUNET_BLOCK_evaluate (GDS_block_context,
2075 type, 2062 type,
2063 bg,
2076 GNUNET_BLOCK_EO_NONE, 2064 GNUNET_BLOCK_EO_NONE,
2077 &get->key, 2065 &get->key,
2078 &reply_bf,
2079 get->bf_mutator,
2080 xquery, 2066 xquery,
2081 xquery_size, 2067 xquery_size,
2082 NULL, 2068 NULL,
@@ -2085,8 +2071,7 @@ handle_dht_p2p_get (void *cls,
2085 { 2071 {
2086 /* request invalid or block type not supported */ 2072 /* request invalid or block type not supported */
2087 GNUNET_break_op (eval == GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED); 2073 GNUNET_break_op (eval == GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED);
2088 if (NULL != reply_bf) 2074 GNUNET_BLOCK_group_destroy (bg);
2089 GNUNET_CONTAINER_bloomfilter_free (reply_bf);
2090 return; 2075 return;
2091 } 2076 }
2092 peer_bf = GNUNET_CONTAINER_bloomfilter_init (get->bloomfilter, 2077 peer_bf = GNUNET_CONTAINER_bloomfilter_init (get->bloomfilter,
@@ -2098,12 +2083,11 @@ handle_dht_p2p_get (void *cls,
2098 /* remember request for routing replies */ 2083 /* remember request for routing replies */
2099 GDS_ROUTING_add (peer->id, 2084 GDS_ROUTING_add (peer->id,
2100 type, 2085 type,
2086 bg, /* bg now owned by routing, but valid at least until end of this function! */
2101 options, 2087 options,
2102 &get->key, 2088 &get->key,
2103 xquery, 2089 xquery,
2104 xquery_size, 2090 xquery_size);
2105 reply_bf,
2106 get->bf_mutator);
2107 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 2091 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2108 "GET for %s at %s after %u hops\n", 2092 "GET for %s at %s after %u hops\n",
2109 GNUNET_h2s (&get->key), 2093 GNUNET_h2s (&get->key),
@@ -2122,8 +2106,7 @@ handle_dht_p2p_get (void *cls,
2122 GNUNET_NO); 2106 GNUNET_NO);
2123 handle_find_peer (peer->id, 2107 handle_find_peer (peer->id,
2124 &get->key, 2108 &get->key,
2125 reply_bf, 2109 bg);
2126 get->bf_mutator);
2127 } 2110 }
2128 else 2111 else
2129 { 2112 {
@@ -2131,8 +2114,7 @@ handle_dht_p2p_get (void *cls,
2131 type, 2114 type,
2132 xquery, 2115 xquery,
2133 xquery_size, 2116 xquery_size,
2134 &reply_bf, 2117 bg,
2135 get->bf_mutator,
2136 &handle_local_result, 2118 &handle_local_result,
2137 NULL); 2119 NULL);
2138 } 2120 }
@@ -2155,8 +2137,7 @@ handle_dht_p2p_get (void *cls,
2155 &get->key, 2137 &get->key,
2156 xquery, 2138 xquery,
2157 xquery_size, 2139 xquery_size,
2158 reply_bf, 2140 bg,
2159 get->bf_mutator,
2160 peer_bf); 2141 peer_bf);
2161 GDS_CLIENTS_process_get (options 2142 GDS_CLIENTS_process_get (options
2162 | (GNUNET_OK == forwarded) 2143 | (GNUNET_OK == forwarded)
@@ -2168,10 +2149,7 @@ handle_dht_p2p_get (void *cls,
2168 NULL, 2149 NULL,
2169 &get->key); 2150 &get->key);
2170 2151
2171 2152 /* clean up; note that 'bg' is owned by routing now! */
2172 /* clean up */
2173 if (NULL != reply_bf)
2174 GNUNET_CONTAINER_bloomfilter_free (reply_bf);
2175 GNUNET_CONTAINER_bloomfilter_free (peer_bf); 2153 GNUNET_CONTAINER_bloomfilter_free (peer_bf);
2176} 2154}
2177 2155
@@ -2310,7 +2288,7 @@ handle_dht_p2p_result (void *cls,
2310 } 2288 }
2311 GNUNET_break (0 != memcmp (&get_path[i], 2289 GNUNET_break (0 != memcmp (&get_path[i],
2312 peer->id, 2290 peer->id,
2313 sizeof (struct GNUNET_PeerIdentity))); 2291 sizeof (struct GNUNET_PeerIdentity)));
2314 } 2292 }
2315#endif 2293#endif
2316 GNUNET_memcpy (xget_path, 2294 GNUNET_memcpy (xget_path,
diff --git a/src/dht/gnunet-service-dht_neighbours.h b/src/dht/gnunet-service-dht_neighbours.h
index d89e5c54f..34b76ee8a 100644
--- a/src/dht/gnunet-service-dht_neighbours.h
+++ b/src/dht/gnunet-service-dht_neighbours.h
@@ -77,8 +77,7 @@ GDS_NEIGHBOURS_handle_put (enum GNUNET_BLOCK_Type type,
77 * @param key key for the content 77 * @param key key for the content
78 * @param xquery extended query 78 * @param xquery extended query
79 * @param xquery_size number of bytes in @a xquery 79 * @param xquery_size number of bytes in @a xquery
80 * @param reply_bf bloomfilter to filter duplicates 80 * @param bg block group to filter replies
81 * @param reply_bf_mutator mutator for @a reply_bf
82 * @param peer_bf filter for peers not to select (again, updated) 81 * @param peer_bf filter for peers not to select (again, updated)
83 * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not 82 * @return #GNUNET_OK if the request was forwarded, #GNUNET_NO if not
84 */ 83 */
@@ -88,9 +87,9 @@ GDS_NEIGHBOURS_handle_get (enum GNUNET_BLOCK_Type type,
88 uint32_t desired_replication_level, 87 uint32_t desired_replication_level,
89 uint32_t hop_count, 88 uint32_t hop_count,
90 const struct GNUNET_HashCode *key, 89 const struct GNUNET_HashCode *key,
91 const void *xquery, size_t xquery_size, 90 const void *xquery,
92 const struct GNUNET_CONTAINER_BloomFilter *reply_bf, 91 size_t xquery_size,
93 uint32_t reply_bf_mutator, 92 struct GNUNET_BLOCK_Group *bg,
94 struct GNUNET_CONTAINER_BloomFilter *peer_bf); 93 struct GNUNET_CONTAINER_BloomFilter *peer_bf);
95 94
96 95
@@ -114,12 +113,13 @@ void
114GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target, 113GDS_NEIGHBOURS_handle_reply (const struct GNUNET_PeerIdentity *target,
115 enum GNUNET_BLOCK_Type type, 114 enum GNUNET_BLOCK_Type type,
116 struct GNUNET_TIME_Absolute expiration_time, 115 struct GNUNET_TIME_Absolute expiration_time,
117 const struct GNUNET_HashCode * key, 116 const struct GNUNET_HashCode *key,
118 unsigned int put_path_length, 117 unsigned int put_path_length,
119 const struct GNUNET_PeerIdentity *put_path, 118 const struct GNUNET_PeerIdentity *put_path,
120 unsigned int get_path_length, 119 unsigned int get_path_length,
121 const struct GNUNET_PeerIdentity *get_path, 120 const struct GNUNET_PeerIdentity *get_path,
122 const void *data, size_t data_size); 121 const void *data,
122 size_t data_size);
123 123
124 124
125/** 125/**
diff --git a/src/dht/gnunet-service-dht_routing.c b/src/dht/gnunet-service-dht_routing.c
index 71240a503..252995737 100644
--- a/src/dht/gnunet-service-dht_routing.c
+++ b/src/dht/gnunet-service-dht_routing.c
@@ -58,9 +58,9 @@ struct RecentRequest
58 struct GNUNET_CONTAINER_HeapNode *heap_node; 58 struct GNUNET_CONTAINER_HeapNode *heap_node;
59 59
60 /** 60 /**
61 * Bloomfilter for replies to drop. 61 * Block group for filtering replies.
62 */ 62 */
63 struct GNUNET_CONTAINER_BloomFilter *reply_bf; 63 struct GNUNET_BLOCK_Group *bg;
64 64
65 /** 65 /**
66 * Type of the requested block. 66 * Type of the requested block.
@@ -79,11 +79,6 @@ struct RecentRequest
79 size_t xquery_size; 79 size_t xquery_size;
80 80
81 /** 81 /**
82 * Mutator value for the reply_bf, see gnunet_block_lib.h
83 */
84 uint32_t reply_bf_mutator;
85
86 /**
87 * Request options. 82 * Request options.
88 */ 83 */
89 enum GNUNET_DHT_RouteOption options; 84 enum GNUNET_DHT_RouteOption options;
@@ -207,10 +202,9 @@ process (void *cls,
207 eval 202 eval
208 = GNUNET_BLOCK_evaluate (GDS_block_context, 203 = GNUNET_BLOCK_evaluate (GDS_block_context,
209 pc->type, 204 pc->type,
205 rr->bg,
210 GNUNET_BLOCK_EO_NONE, 206 GNUNET_BLOCK_EO_NONE,
211 eval_key, 207 eval_key,
212 &rr->reply_bf,
213 rr->reply_bf_mutator,
214 rr->xquery, 208 rr->xquery,
215 rr->xquery_size, 209 rr->xquery_size,
216 pc->data, 210 pc->data,
@@ -343,7 +337,7 @@ expire_oldest_entry ()
343 recent_req = GNUNET_CONTAINER_heap_peek (recent_heap); 337 recent_req = GNUNET_CONTAINER_heap_peek (recent_heap);
344 GNUNET_assert (recent_req != NULL); 338 GNUNET_assert (recent_req != NULL);
345 GNUNET_CONTAINER_heap_remove_node (recent_req->heap_node); 339 GNUNET_CONTAINER_heap_remove_node (recent_req->heap_node);
346 GNUNET_CONTAINER_bloomfilter_free (recent_req->reply_bf); 340 GNUNET_BLOCK_group_destroy (recent_req->bg);
347 GNUNET_assert (GNUNET_YES == 341 GNUNET_assert (GNUNET_YES ==
348 GNUNET_CONTAINER_multihashmap_remove (recent_map, 342 GNUNET_CONTAINER_multihashmap_remove (recent_map,
349 &recent_req->key, 343 &recent_req->key,
@@ -379,18 +373,10 @@ try_combine_recent (void *cls,
379 rr->xquery, 373 rr->xquery,
380 in->xquery_size)) ) 374 in->xquery_size)) )
381 return GNUNET_OK; 375 return GNUNET_OK;
382 if (in->reply_bf_mutator != rr->reply_bf_mutator) 376 GNUNET_break (GNUNET_SYSERR !=
383 { 377 GNUNET_BLOCK_group_merge (in->bg,
384 rr->reply_bf_mutator = in->reply_bf_mutator; 378 rr->bg));
385 GNUNET_CONTAINER_bloomfilter_free (rr->reply_bf); 379 rr->bg = in->bg;
386 rr->reply_bf = in->reply_bf;
387 }
388 else
389 {
390 GNUNET_CONTAINER_bloomfilter_or2 (rr->reply_bf,
391 in->reply_bf);
392 GNUNET_CONTAINER_bloomfilter_free (in->reply_bf);
393 }
394 GNUNET_free (in); 380 GNUNET_free (in);
395 return GNUNET_SYSERR; 381 return GNUNET_SYSERR;
396} 382}
@@ -411,12 +397,11 @@ try_combine_recent (void *cls,
411void 397void
412GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender, 398GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
413 enum GNUNET_BLOCK_Type type, 399 enum GNUNET_BLOCK_Type type,
400 struct GNUNET_BLOCK_Group *bg,
414 enum GNUNET_DHT_RouteOption options, 401 enum GNUNET_DHT_RouteOption options,
415 const struct GNUNET_HashCode *key, 402 const struct GNUNET_HashCode *key,
416 const void *xquery, 403 const void *xquery,
417 size_t xquery_size, 404 size_t xquery_size)
418 const struct GNUNET_CONTAINER_BloomFilter *reply_bf,
419 uint32_t reply_bf_mutator)
420{ 405{
421 struct RecentRequest *recent_req; 406 struct RecentRequest *recent_req;
422 407
@@ -424,17 +409,19 @@ GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
424 expire_oldest_entry (); 409 expire_oldest_entry ();
425 GNUNET_STATISTICS_update (GDS_stats, 410 GNUNET_STATISTICS_update (GDS_stats,
426 gettext_noop ("# Entries added to routing table"), 411 gettext_noop ("# Entries added to routing table"),
427 1, GNUNET_NO); 412 1,
413 GNUNET_NO);
428 recent_req = GNUNET_malloc (sizeof (struct RecentRequest) + xquery_size); 414 recent_req = GNUNET_malloc (sizeof (struct RecentRequest) + xquery_size);
429 recent_req->peer = *sender; 415 recent_req->peer = *sender;
430 recent_req->key = *key; 416 recent_req->key = *key;
431 recent_req->reply_bf = GNUNET_CONTAINER_bloomfilter_copy (reply_bf); 417 recent_req->bg = bg;
432 recent_req->type = type; 418 recent_req->type = type;
433 recent_req->options = options; 419 recent_req->options = options;
434 recent_req->xquery = &recent_req[1]; 420 recent_req->xquery = &recent_req[1];
435 GNUNET_memcpy (&recent_req[1], xquery, xquery_size); 421 GNUNET_memcpy (&recent_req[1],
422 xquery,
423 xquery_size);
436 recent_req->xquery_size = xquery_size; 424 recent_req->xquery_size = xquery_size;
437 recent_req->reply_bf_mutator = reply_bf_mutator;
438 if (GNUNET_SYSERR == 425 if (GNUNET_SYSERR ==
439 GNUNET_CONTAINER_multihashmap_get_multiple (recent_map, 426 GNUNET_CONTAINER_multihashmap_get_multiple (recent_map,
440 key, 427 key,
@@ -447,13 +434,14 @@ GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
447 1, GNUNET_NO); 434 1, GNUNET_NO);
448 return; 435 return;
449 } 436 }
450 recent_req->heap_node = 437 recent_req->heap_node
451 GNUNET_CONTAINER_heap_insert (recent_heap, recent_req, 438 = GNUNET_CONTAINER_heap_insert (recent_heap,
439 recent_req,
452 GNUNET_TIME_absolute_get ().abs_value_us); 440 GNUNET_TIME_absolute_get ().abs_value_us);
453 GNUNET_CONTAINER_multihashmap_put (recent_map, key, recent_req, 441 GNUNET_CONTAINER_multihashmap_put (recent_map,
442 key,
443 recent_req,
454 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 444 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
455
456
457} 445}
458 446
459 447
diff --git a/src/dht/gnunet-service-dht_routing.h b/src/dht/gnunet-service-dht_routing.h
index 7c57361dc..1d9284164 100644
--- a/src/dht/gnunet-service-dht_routing.h
+++ b/src/dht/gnunet-service-dht_routing.h
@@ -67,21 +67,20 @@ GDS_ROUTING_process (void *cls,
67 * 67 *
68 * @param sender peer that originated the request 68 * @param sender peer that originated the request
69 * @param type type of the block 69 * @param type type of the block
70 * @param bg block group to evaluate replies, henceforth owned by routing
70 * @param options options for processing 71 * @param options options for processing
71 * @param key key for the content 72 * @param key key for the content
72 * @param xquery extended query 73 * @param xquery extended query
73 * @param xquery_size number of bytes in @a xquery 74 * @param xquery_size number of bytes in @a xquery
74 * @param reply_bf bloomfilter to filter duplicates
75 * @param reply_bf_mutator mutator for @a reply_bf
76*/ 75*/
77void 76void
78GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender, 77GDS_ROUTING_add (const struct GNUNET_PeerIdentity *sender,
79 enum GNUNET_BLOCK_Type type, 78 enum GNUNET_BLOCK_Type type,
79 struct GNUNET_BLOCK_Group *bg,
80 enum GNUNET_DHT_RouteOption options, 80 enum GNUNET_DHT_RouteOption options,
81 const struct GNUNET_HashCode * key, const void *xquery, 81 const struct GNUNET_HashCode * key,
82 size_t xquery_size, 82 const void *xquery,
83 const struct GNUNET_CONTAINER_BloomFilter *reply_bf, 83 size_t xquery_size);
84 uint32_t reply_bf_mutator);
85 84
86 85
87/** 86/**
diff --git a/src/dht/plugin_block_dht.c b/src/dht/plugin_block_dht.c
index 4c5f122a4..0304dad87 100644
--- a/src/dht/plugin_block_dht.c
+++ b/src/dht/plugin_block_dht.c
@@ -47,6 +47,7 @@
47 * @param nonce random value used to seed the group creation 47 * @param nonce random value used to seed the group creation
48 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 48 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
49 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 49 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
50 * @param va variable arguments specific to @a type
50 * @return block group handle, NULL if block groups are not supported 51 * @return block group handle, NULL if block groups are not supported
51 * by this @a type of block (this is not an error) 52 * by this @a type of block (this is not an error)
52 */ 53 */
@@ -55,7 +56,8 @@ block_plugin_dht_create_group (void *cls,
55 enum GNUNET_BLOCK_Type type, 56 enum GNUNET_BLOCK_Type type,
56 uint32_t nonce, 57 uint32_t nonce,
57 const void *raw_data, 58 const void *raw_data,
58 size_t raw_data_size) 59 size_t raw_data_size,
60 va_list va)
59{ 61{
60 return GNUNET_BLOCK_GROUP_bf_create (cls, 62 return GNUNET_BLOCK_GROUP_bf_create (cls,
61 DHT_BF_SIZE, 63 DHT_BF_SIZE,
@@ -73,10 +75,9 @@ block_plugin_dht_create_group (void *cls,
73 * 75 *
74 * @param cls closure 76 * @param cls closure
75 * @param type block type 77 * @param type block type
78 * @param group block group to check against
76 * @param eo control flags 79 * @param eo control flags
77 * @param query original query (hash) 80 * @param query original query (hash)
78 * @param bf pointer to bloom filter associated with query; possibly updated (!)
79 * @param bf_mutator mutation value for @a bf
80 * @param xquery extended query data (can be NULL, depending on type) 81 * @param xquery extended query data (can be NULL, depending on type)
81 * @param xquery_size number of bytes in @a xquery 82 * @param xquery_size number of bytes in @a xquery
82 * @param reply_block response to validate 83 * @param reply_block response to validate
@@ -86,16 +87,14 @@ block_plugin_dht_create_group (void *cls,
86static enum GNUNET_BLOCK_EvaluationResult 87static enum GNUNET_BLOCK_EvaluationResult
87block_plugin_dht_evaluate (void *cls, 88block_plugin_dht_evaluate (void *cls,
88 enum GNUNET_BLOCK_Type type, 89 enum GNUNET_BLOCK_Type type,
90 struct GNUNET_BLOCK_Group *group,
89 enum GNUNET_BLOCK_EvaluationOptions eo, 91 enum GNUNET_BLOCK_EvaluationOptions eo,
90 const struct GNUNET_HashCode *query, 92 const struct GNUNET_HashCode *query,
91 struct GNUNET_CONTAINER_BloomFilter **bf,
92 int32_t bf_mutator,
93 const void *xquery, 93 const void *xquery,
94 size_t xquery_size, 94 size_t xquery_size,
95 const void *reply_block, 95 const void *reply_block,
96 size_t reply_block_size) 96 size_t reply_block_size)
97{ 97{
98 struct GNUNET_HashCode mhash;
99 const struct GNUNET_HELLO_Message *hello; 98 const struct GNUNET_HELLO_Message *hello;
100 struct GNUNET_PeerIdentity pid; 99 struct GNUNET_PeerIdentity pid;
101 const struct GNUNET_MessageHeader *msg; 100 const struct GNUNET_MessageHeader *msg;
@@ -127,22 +126,13 @@ block_plugin_dht_evaluate (void *cls,
127 GNUNET_break_op (0); 126 GNUNET_break_op (0);
128 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; 127 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
129 } 128 }
130 if (NULL != bf) 129 GNUNET_CRYPTO_hash (&pid,
131 { 130 sizeof (pid),
132 GNUNET_CRYPTO_hash (&pid, sizeof (pid), &phash); 131 &phash);
133 GNUNET_BLOCK_mingle_hash (&phash, bf_mutator, &mhash); 132 if (GNUNET_YES ==
134 if (NULL != *bf) 133 GNUNET_BLOCK_GROUP_bf_test_and_set (group,
135 { 134 &phash))
136 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash)) 135 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
137 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
138 }
139 else
140 {
141 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8,
142 GNUNET_CONSTANTS_BLOOMFILTER_K);
143 }
144 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
145 }
146 return GNUNET_BLOCK_EVALUATION_OK_MORE; 136 return GNUNET_BLOCK_EVALUATION_OK_MORE;
147} 137}
148 138
diff --git a/src/dns/plugin_block_dns.c b/src/dns/plugin_block_dns.c
index e4bc9209c..65da0de63 100644
--- a/src/dns/plugin_block_dns.c
+++ b/src/dns/plugin_block_dns.c
@@ -39,10 +39,9 @@
39 * 39 *
40 * @param cls closure 40 * @param cls closure
41 * @param type block type 41 * @param type block type
42 * @param bg group to evaluate against
42 * @param eo control flags 43 * @param eo control flags
43 * @param query original query (hash) 44 * @param query original query (hash)
44 * @param bf pointer to bloom filter associated with query; possibly updated (!)
45 * @param bf_mutator mutation value for bf
46 * @param xquery extended query data (can be NULL, depending on type) 45 * @param xquery extended query data (can be NULL, depending on type)
47 * @param xquery_size number of bytes in @a xquery 46 * @param xquery_size number of bytes in @a xquery
48 * @param reply_block response to validate 47 * @param reply_block response to validate
@@ -52,10 +51,9 @@
52static enum GNUNET_BLOCK_EvaluationResult 51static enum GNUNET_BLOCK_EvaluationResult
53block_plugin_dns_evaluate (void *cls, 52block_plugin_dns_evaluate (void *cls,
54 enum GNUNET_BLOCK_Type type, 53 enum GNUNET_BLOCK_Type type,
54 struct GNUNET_BLOCK_Group *bg,
55 enum GNUNET_BLOCK_EvaluationOptions eo, 55 enum GNUNET_BLOCK_EvaluationOptions eo,
56 const struct GNUNET_HashCode * query, 56 const struct GNUNET_HashCode * query,
57 struct GNUNET_CONTAINER_BloomFilter **bf,
58 int32_t bf_mutator,
59 const void *xquery, 57 const void *xquery,
60 size_t xquery_size, 58 size_t xquery_size,
61 const void *reply_block, 59 const void *reply_block,
diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c
index 63462f7dc..87e2d2ee1 100644
--- a/src/fs/gnunet-service-fs_pr.c
+++ b/src/fs/gnunet-service-fs_pr.c
@@ -97,9 +97,9 @@ struct GSF_PendingRequest
97 struct GNUNET_HashCode *replies_seen; 97 struct GNUNET_HashCode *replies_seen;
98 98
99 /** 99 /**
100 * Bloomfilter masking replies we've already seen. 100 * Block group for filtering replies we've already seen.
101 */ 101 */
102 struct GNUNET_CONTAINER_BloomFilter *bf; 102 struct GNUNET_BLOCK_Group *bg;
103 103
104 /** 104 /**
105 * Entry for this pending request in the expiration heap, or NULL. 105 * Entry for this pending request in the expiration heap, or NULL.
@@ -190,11 +190,6 @@ struct GSF_PendingRequest
190 unsigned int replies_seen_size; 190 unsigned int replies_seen_size;
191 191
192 /** 192 /**
193 * Mingle value we currently use for the bf.
194 */
195 uint32_t mingle;
196
197 /**
198 * Do we have a first UID yet? 193 * Do we have a first UID yet?
199 */ 194 */
200 unsigned int have_first_uid; 195 unsigned int have_first_uid;
@@ -248,18 +243,35 @@ static unsigned long long max_pending_requests = (32 * 1024);
248 * fresh one of minimal size without problems) OR if our peer is the 243 * fresh one of minimal size without problems) OR if our peer is the
249 * initiator (in which case we may resize to larger than mimimum size). 244 * initiator (in which case we may resize to larger than mimimum size).
250 * 245 *
246 * @param type type of the request
251 * @param pr request for which the BF is to be recomputed 247 * @param pr request for which the BF is to be recomputed
252 */ 248 */
253static void 249static void
254refresh_bloomfilter (struct GSF_PendingRequest *pr) 250refresh_bloomfilter (enum GNUNET_BLOCK_Type type,
251 struct GSF_PendingRequest *pr)
255{ 252{
256 if (pr->bf != NULL) 253 if (NULL != pr->bg)
257 GNUNET_CONTAINER_bloomfilter_free (pr->bf); 254 {
258 pr->mingle = 255 GNUNET_BLOCK_group_destroy (pr->bg);
259 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, UINT32_MAX); 256 pr->bg = NULL;
260 pr->bf = 257 }
261 GNUNET_BLOCK_construct_bloomfilter (pr->mingle, pr->replies_seen, 258 if (GNUNET_BLOCK_TYPE_FS_UBLOCK != type)
262 pr->replies_seen_count); 259 return; /* no need */
260 pr->bg
261 = GNUNET_BLOCK_group_create (GSF_block_ctx,
262 type,
263 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
264 UINT32_MAX),
265 NULL,
266 0,
267 "fs-seen-set-size",
268 pr->replies_seen_count);
269 if (NULL == pr->bg)
270 return;
271 GNUNET_break (GNUNET_OK ==
272 GNUNET_BLOCK_group_set_seen (pr->bg,
273 pr->replies_seen,
274 pr->replies_seen_count));
263} 275}
264 276
265 277
@@ -355,25 +367,30 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options,
355 if (replies_seen_count > 0) 367 if (replies_seen_count > 0)
356 { 368 {
357 pr->replies_seen_size = replies_seen_count; 369 pr->replies_seen_size = replies_seen_count;
358 pr->replies_seen = 370 pr->replies_seen = GNUNET_new_array (pr->replies_seen_size,
359 GNUNET_malloc (sizeof (struct GNUNET_HashCode) * pr->replies_seen_size); 371 struct GNUNET_HashCode);
360 GNUNET_memcpy (pr->replies_seen, 372 GNUNET_memcpy (pr->replies_seen,
361 replies_seen, 373 replies_seen,
362 replies_seen_count * sizeof (struct GNUNET_HashCode)); 374 replies_seen_count * sizeof (struct GNUNET_HashCode));
363 pr->replies_seen_count = replies_seen_count; 375 pr->replies_seen_count = replies_seen_count;
364 } 376 }
365 if (NULL != bf_data) 377 if ( (NULL != bf_data) &&
378 (GNUNET_BLOCK_TYPE_FS_UBLOCK == pr->public_data.type) )
366 { 379 {
367 pr->bf = 380 pr->bg
368 GNUNET_CONTAINER_bloomfilter_init (bf_data, 381 = GNUNET_BLOCK_group_create (GSF_block_ctx,
369 bf_size, 382 pr->public_data.type,
370 GNUNET_CONSTANTS_BLOOMFILTER_K); 383 mingle,
371 pr->mingle = mingle; 384 bf_data,
385 bf_size,
386 "fs-seen-set-size",
387 0);
372 } 388 }
373 else if ((replies_seen_count > 0) && 389 else if ((replies_seen_count > 0) &&
374 (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))) 390 (0 != (options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)))
375 { 391 {
376 refresh_bloomfilter (pr); 392 refresh_bloomfilter (pr->public_data.type,
393 pr);
377 } 394 }
378 GNUNET_CONTAINER_multihashmap_put (pr_map, 395 GNUNET_CONTAINER_multihashmap_put (pr_map,
379 &pr->public_data.query, 396 &pr->public_data.query,
@@ -461,46 +478,37 @@ GSF_pending_request_update_ (struct GSF_PendingRequest *pr,
461 const struct GNUNET_HashCode * replies_seen, 478 const struct GNUNET_HashCode * replies_seen,
462 unsigned int replies_seen_count) 479 unsigned int replies_seen_count)
463{ 480{
464 unsigned int i;
465 struct GNUNET_HashCode mhash;
466
467 if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count) 481 if (replies_seen_count + pr->replies_seen_count < pr->replies_seen_count)
468 return; /* integer overflow */ 482 return; /* integer overflow */
469 if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH)) 483 if (0 != (pr->public_data.options & GSF_PRO_BLOOMFILTER_FULL_REFRESH))
470 { 484 {
471 /* we're responsible for the BF, full refresh */ 485 /* we're responsible for the BF, full refresh */
472 if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size) 486 if (replies_seen_count + pr->replies_seen_count > pr->replies_seen_size)
473 GNUNET_array_grow (pr->replies_seen, pr->replies_seen_size, 487 GNUNET_array_grow (pr->replies_seen,
488 pr->replies_seen_size,
474 replies_seen_count + pr->replies_seen_count); 489 replies_seen_count + pr->replies_seen_count);
475 GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count], replies_seen, 490 GNUNET_memcpy (&pr->replies_seen[pr->replies_seen_count],
476 sizeof (struct GNUNET_HashCode) * replies_seen_count); 491 replies_seen,
492 sizeof (struct GNUNET_HashCode) * replies_seen_count);
477 pr->replies_seen_count += replies_seen_count; 493 pr->replies_seen_count += replies_seen_count;
478 refresh_bloomfilter (pr); 494 refresh_bloomfilter (pr->public_data.type,
495 pr);
479 } 496 }
480 else 497 else
481 { 498 {
482 if (NULL == pr->bf) 499 if (NULL == pr->bg)
483 { 500 {
484 /* we're not the initiator, but the initiator did not give us 501 /* we're not the initiator, but the initiator did not give us
485 * any bloom-filter, so we need to create one on-the-fly */ 502 * any bloom-filter, so we need to create one on-the-fly */
486 pr->mingle = 503 refresh_bloomfilter (pr->public_data.type,
487 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 504 pr);
488 UINT32_MAX);
489 pr->bf =
490 GNUNET_BLOCK_construct_bloomfilter (pr->mingle,
491 replies_seen,
492 replies_seen_count);
493 } 505 }
494 else 506 else
495 { 507 {
496 for (i = 0; i < pr->replies_seen_count; i++) 508 GNUNET_break (GNUNET_OK ==
497 { 509 GNUNET_BLOCK_group_set_seen (pr->bg,
498 GNUNET_BLOCK_mingle_hash (&replies_seen[i], 510 replies_seen,
499 pr->mingle, 511 pr->replies_seen_count));
500 &mhash);
501 GNUNET_CONTAINER_bloomfilter_add (pr->bf,
502 &mhash);
503 }
504 } 512 }
505 } 513 }
506 if (NULL != pr->gh) 514 if (NULL != pr->gh)
@@ -530,6 +538,8 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
530 struct GNUNET_TIME_Absolute now; 538 struct GNUNET_TIME_Absolute now;
531 int64_t ttl; 539 int64_t ttl;
532 int do_route; 540 int do_route;
541 void *bf_data;
542 uint32_t bf_nonce;
533 543
534 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 544 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
535 "Building request message for `%s' of type %d\n", 545 "Building request message for `%s' of type %d\n",
@@ -553,7 +563,15 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
553 bm |= GET_MESSAGE_BIT_TRANSMIT_TO; 563 bm |= GET_MESSAGE_BIT_TRANSMIT_TO;
554 k++; 564 k++;
555 } 565 }
556 bf_size = GNUNET_CONTAINER_bloomfilter_get_size (pr->bf); 566 if (GNUNET_OK !=
567 GNUNET_BLOCK_group_serialize (pr->bg,
568 &bf_nonce,
569 &bf_data,
570 &bf_size))
571 {
572 bf_size = 0;
573 bf_data = NULL;
574 }
557 env = GNUNET_MQ_msg_extra (gm, 575 env = GNUNET_MQ_msg_extra (gm,
558 bf_size + k * sizeof (struct GNUNET_PeerIdentity), 576 bf_size + k * sizeof (struct GNUNET_PeerIdentity),
559 GNUNET_MESSAGE_TYPE_FS_GET); 577 GNUNET_MESSAGE_TYPE_FS_GET);
@@ -571,7 +589,7 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
571 now = GNUNET_TIME_absolute_get (); 589 now = GNUNET_TIME_absolute_get ();
572 ttl = (int64_t) (pr->public_data.ttl.abs_value_us - now.abs_value_us); 590 ttl = (int64_t) (pr->public_data.ttl.abs_value_us - now.abs_value_us);
573 gm->ttl = htonl (ttl / 1000LL / 1000LL); 591 gm->ttl = htonl (ttl / 1000LL / 1000LL);
574 gm->filter_mutator = htonl (pr->mingle); 592 gm->filter_mutator = htonl (bf_nonce);
575 gm->hash_bitmap = htonl (bm); 593 gm->hash_bitmap = htonl (bm);
576 gm->query = pr->public_data.query; 594 gm->query = pr->public_data.query;
577 ext = (struct GNUNET_PeerIdentity *) &gm[1]; 595 ext = (struct GNUNET_PeerIdentity *) &gm[1];
@@ -581,11 +599,10 @@ GSF_pending_request_get_message_ (struct GSF_PendingRequest *pr)
581 &ext[k++]); 599 &ext[k++]);
582 if (NULL != pr->public_data.target) 600 if (NULL != pr->public_data.target)
583 ext[k++] = *pr->public_data.target; 601 ext[k++] = *pr->public_data.target;
584 if (NULL != pr->bf) 602 GNUNET_memcpy (&ext[k],
585 GNUNET_assert (GNUNET_SYSERR != 603 bf_data,
586 GNUNET_CONTAINER_bloomfilter_get_raw_data (pr->bf, 604 bf_size);
587 (char *) &ext[k], 605 GNUNET_free_non_null (bf_data);
588 bf_size));
589 return env; 606 return env;
590} 607}
591 608
@@ -624,11 +641,8 @@ clean_request (void *cls,
624 } 641 }
625 GSF_plan_notify_request_done_ (pr); 642 GSF_plan_notify_request_done_ (pr);
626 GNUNET_free_non_null (pr->replies_seen); 643 GNUNET_free_non_null (pr->replies_seen);
627 if (NULL != pr->bf) 644 GNUNET_BLOCK_group_destroy (pr->bg);
628 { 645 pr->bg = NULL;
629 GNUNET_CONTAINER_bloomfilter_free (pr->bf);
630 pr->bf = NULL;
631 }
632 GNUNET_PEER_change_rc (pr->sender_pid, -1); 646 GNUNET_PEER_change_rc (pr->sender_pid, -1);
633 pr->sender_pid = 0; 647 pr->sender_pid = 0;
634 GNUNET_PEER_change_rc (pr->origin_pid, -1); 648 GNUNET_PEER_change_rc (pr->origin_pid, -1);
@@ -844,10 +858,9 @@ process_reply (void *cls,
844 prq->eval = 858 prq->eval =
845 GNUNET_BLOCK_evaluate (GSF_block_ctx, 859 GNUNET_BLOCK_evaluate (GSF_block_ctx,
846 prq->type, 860 prq->type,
861 pr->bg,
847 prq->eo, 862 prq->eo,
848 key, 863 key,
849 &pr->bf,
850 pr->mingle,
851 NULL, 864 NULL,
852 0, 865 0,
853 prq->data, 866 prq->data,
diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c
index 038734082..6c574fca2 100644
--- a/src/fs/plugin_block_fs.c
+++ b/src/fs/plugin_block_fs.c
@@ -28,6 +28,7 @@
28#include "gnunet_fs_service.h" 28#include "gnunet_fs_service.h"
29#include "block_fs.h" 29#include "block_fs.h"
30#include "gnunet_signatures.h" 30#include "gnunet_signatures.h"
31#include "gnunet_constants.h"
31#include "gnunet_block_group_lib.h" 32#include "gnunet_block_group_lib.h"
32 33
33 34
@@ -37,10 +38,36 @@
37 */ 38 */
38#define BLOOMFILTER_K 16 39#define BLOOMFILTER_K 16
39 40
41
40/** 42/**
41 * How big is the BF we use for FS blocks? 43 * How many bytes should a bloomfilter be if we have already seen
44 * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K
45 * gives us the number of bits set per entry. Furthermore, we should
46 * not re-size the filter too often (to keep it cheap).
47 *
48 * Since other peers will also add entries but not resize the filter,
49 * we should generally pick a slightly larger size than what the
50 * strict math would suggest.
51 *
52 * @param entry_count expected number of entries in the Bloom filter
53 * @return must be a power of two and smaller or equal to 2^15.
42 */ 54 */
43#define FS_BF_SIZE 8 55static size_t
56compute_bloomfilter_size (unsigned int entry_count)
57{
58 size_t size;
59 unsigned int ideal = (entry_count * GNUNET_CONSTANTS_BLOOMFILTER_K) / 4;
60 uint16_t max = 1 << 15;
61
62 if (entry_count > max)
63 return max;
64 size = 8;
65 while ((size < max) && (size < ideal))
66 size *= 2;
67 if (size > max)
68 return max;
69 return size;
70}
44 71
45 72
46/** 73/**
@@ -51,16 +78,21 @@
51 * @param nonce random value used to seed the group creation 78 * @param nonce random value used to seed the group creation
52 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 79 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
53 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 80 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
81 * @param va variable arguments specific to @a type
54 * @return block group handle, NULL if block groups are not supported 82 * @return block group handle, NULL if block groups are not supported
55 * by this @a type of block (this is not an error) 83 * by this @a type of block (this is not an error)
56 */ 84 */
57static struct GNUNET_BLOCK_Group * 85static struct GNUNET_BLOCK_Group *
58block_plugin_fs_create_group (void *cls, 86block_plugin_fs_create_group (void *cls,
59 enum GNUNET_BLOCK_Type type, 87 enum GNUNET_BLOCK_Type type,
60 uint32_t nonce, 88 uint32_t nonce,
61 const void *raw_data, 89 const void *raw_data,
62 size_t raw_data_size) 90 size_t raw_data_size,
91 va_list va)
63{ 92{
93 unsigned int size;
94 const char *guard;
95
64 switch (type) 96 switch (type)
65 { 97 {
66 case GNUNET_BLOCK_TYPE_FS_DBLOCK: 98 case GNUNET_BLOCK_TYPE_FS_DBLOCK:
@@ -68,8 +100,23 @@ block_plugin_fs_create_group (void *cls,
68 case GNUNET_BLOCK_TYPE_FS_IBLOCK: 100 case GNUNET_BLOCK_TYPE_FS_IBLOCK:
69 return NULL; 101 return NULL;
70 case GNUNET_BLOCK_TYPE_FS_UBLOCK: 102 case GNUNET_BLOCK_TYPE_FS_UBLOCK:
103 guard = va_arg (va, const char *);
104 if (0 != memcmp (guard,
105 "fs-seen-set-size",
106 strlen ("fs-seen-set-size")))
107 {
108 /* va-args invalid! bad bug, complain! */
109 GNUNET_break (0);
110 size = 8;
111 }
112 else
113 {
114 size = compute_bloomfilter_size (va_arg (va, unsigned int));
115 }
116 if (0 == size)
117 size = raw_data_size; /* not for us to determine, use what we got! */
71 return GNUNET_BLOCK_GROUP_bf_create (cls, 118 return GNUNET_BLOCK_GROUP_bf_create (cls,
72 FS_BF_SIZE, 119 size,
73 BLOOMFILTER_K, 120 BLOOMFILTER_K,
74 type, 121 type,
75 nonce, 122 nonce,
@@ -91,10 +138,9 @@ block_plugin_fs_create_group (void *cls,
91 * 138 *
92 * @param cls closure 139 * @param cls closure
93 * @param type block type 140 * @param type block type
141 * @param bg group to use for evaluation
94 * @param eo control flags 142 * @param eo control flags
95 * @param query original query (hash) 143 * @param query original query (hash)
96 * @param bf pointer to bloom filter associated with query; possibly updated (!)
97 * @param bf_mutator mutation value for @a bf
98 * @param xquery extrended query data (can be NULL, depending on type) 144 * @param xquery extrended query data (can be NULL, depending on type)
99 * @param xquery_size number of bytes in @a xquery 145 * @param xquery_size number of bytes in @a xquery
100 * @param reply_block response to validate 146 * @param reply_block response to validate
@@ -104,10 +150,9 @@ block_plugin_fs_create_group (void *cls,
104static enum GNUNET_BLOCK_EvaluationResult 150static enum GNUNET_BLOCK_EvaluationResult
105block_plugin_fs_evaluate (void *cls, 151block_plugin_fs_evaluate (void *cls,
106 enum GNUNET_BLOCK_Type type, 152 enum GNUNET_BLOCK_Type type,
153 struct GNUNET_BLOCK_Group *bg,
107 enum GNUNET_BLOCK_EvaluationOptions eo, 154 enum GNUNET_BLOCK_EvaluationOptions eo,
108 const struct GNUNET_HashCode *query, 155 const struct GNUNET_HashCode *query,
109 struct GNUNET_CONTAINER_BloomFilter **bf,
110 int32_t bf_mutator,
111 const void *xquery, 156 const void *xquery,
112 size_t xquery_size, 157 size_t xquery_size,
113 const void *reply_block, 158 const void *reply_block,
@@ -116,7 +161,6 @@ block_plugin_fs_evaluate (void *cls,
116 const struct UBlock *ub; 161 const struct UBlock *ub;
117 struct GNUNET_HashCode hc; 162 struct GNUNET_HashCode hc;
118 struct GNUNET_HashCode chash; 163 struct GNUNET_HashCode chash;
119 struct GNUNET_HashCode mhash;
120 164
121 switch (type) 165 switch (type)
122 { 166 {
@@ -170,26 +214,13 @@ block_plugin_fs_evaluate (void *cls,
170 GNUNET_break_op (0); 214 GNUNET_break_op (0);
171 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; 215 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
172 } 216 }
173 if (NULL != bf) 217 GNUNET_CRYPTO_hash (reply_block,
174 { 218 reply_block_size,
175 GNUNET_CRYPTO_hash (reply_block, 219 &chash);
176 reply_block_size, 220 if (GNUNET_YES ==
177 &chash); 221 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
178 GNUNET_BLOCK_mingle_hash (&chash, 222 &chash))
179 bf_mutator, 223 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
180 &mhash);
181 if (NULL != *bf)
182 {
183 if (GNUNET_YES ==
184 GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
185 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
186 }
187 else
188 {
189 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, BLOOMFILTER_K);
190 }
191 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
192 }
193 return GNUNET_BLOCK_EVALUATION_OK_MORE; 224 return GNUNET_BLOCK_EVALUATION_OK_MORE;
194 default: 225 default:
195 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 226 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
diff --git a/src/fs/test_plugin_block_fs.c b/src/fs/test_plugin_block_fs.c
index 1fc9f2110..ba4f28bc5 100644
--- a/src/fs/test_plugin_block_fs.c
+++ b/src/fs/test_plugin_block_fs.c
@@ -40,28 +40,28 @@ test_fs (struct GNUNET_BLOCK_Context *ctx)
40 if (GNUNET_BLOCK_EVALUATION_OK_LAST != 40 if (GNUNET_BLOCK_EVALUATION_OK_LAST !=
41 GNUNET_BLOCK_evaluate (ctx, 41 GNUNET_BLOCK_evaluate (ctx,
42 GNUNET_BLOCK_TYPE_FS_DBLOCK, 42 GNUNET_BLOCK_TYPE_FS_DBLOCK,
43 NULL,
43 GNUNET_BLOCK_EO_NONE, 44 GNUNET_BLOCK_EO_NONE,
44 &key, 45 &key,
45 NULL, 0, 46 NULL, 0,
46 NULL, 0,
47 block, sizeof (block))) 47 block, sizeof (block)))
48 return 2; 48 return 2;
49 if (GNUNET_BLOCK_EVALUATION_REQUEST_VALID != 49 if (GNUNET_BLOCK_EVALUATION_REQUEST_VALID !=
50 GNUNET_BLOCK_evaluate (ctx, 50 GNUNET_BLOCK_evaluate (ctx,
51 GNUNET_BLOCK_TYPE_FS_DBLOCK, 51 GNUNET_BLOCK_TYPE_FS_DBLOCK,
52 NULL,
52 GNUNET_BLOCK_EO_NONE, 53 GNUNET_BLOCK_EO_NONE,
53 &key, 54 &key,
54 NULL, 0, 55 NULL, 0,
55 NULL, 0,
56 NULL, 0)) 56 NULL, 0))
57 return 4; 57 return 4;
58 GNUNET_log_skip (1, GNUNET_NO); 58 GNUNET_log_skip (1, GNUNET_NO);
59 if (GNUNET_BLOCK_EVALUATION_REQUEST_INVALID != 59 if (GNUNET_BLOCK_EVALUATION_REQUEST_INVALID !=
60 GNUNET_BLOCK_evaluate (ctx, 60 GNUNET_BLOCK_evaluate (ctx,
61 GNUNET_BLOCK_TYPE_FS_DBLOCK, 61 GNUNET_BLOCK_TYPE_FS_DBLOCK,
62 NULL,
62 GNUNET_BLOCK_EO_NONE, 63 GNUNET_BLOCK_EO_NONE,
63 &key, 64 &key,
64 NULL, 0,
65 "bogus", 5, 65 "bogus", 5,
66 NULL, 0)) 66 NULL, 0))
67 return 8; 67 return 8;
diff --git a/src/gns/plugin_block_gns.c b/src/gns/plugin_block_gns.c
index 8d3e84042..94222e32b 100644
--- a/src/gns/plugin_block_gns.c
+++ b/src/gns/plugin_block_gns.c
@@ -51,6 +51,7 @@
51 * @param nonce random value used to seed the group creation 51 * @param nonce random value used to seed the group creation
52 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 52 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
53 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 53 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
54 * @param va variable arguments specific to @a type
54 * @return block group handle, NULL if block groups are not supported 55 * @return block group handle, NULL if block groups are not supported
55 * by this @a type of block (this is not an error) 56 * by this @a type of block (this is not an error)
56 */ 57 */
@@ -59,7 +60,8 @@ block_plugin_gns_create_group (void *cls,
59 enum GNUNET_BLOCK_Type type, 60 enum GNUNET_BLOCK_Type type,
60 uint32_t nonce, 61 uint32_t nonce,
61 const void *raw_data, 62 const void *raw_data,
62 size_t raw_data_size) 63 size_t raw_data_size,
64 va_list va)
63{ 65{
64 return GNUNET_BLOCK_GROUP_bf_create (cls, 66 return GNUNET_BLOCK_GROUP_bf_create (cls,
65 GNS_BF_SIZE, 67 GNS_BF_SIZE,
@@ -80,10 +82,9 @@ block_plugin_gns_create_group (void *cls,
80 * 82 *
81 * @param cls closure 83 * @param cls closure
82 * @param type block type 84 * @param type block type
85 * @param bg block group to use for evaluation
83 * @param eo control flags 86 * @param eo control flags
84 * @param query original query (hash) 87 * @param query original query (hash)
85 * @param bf pointer to bloom filter associated with @a query; possibly updated (!)
86 * @param bf_mutator mutation value for @a bf
87 * @param xquery extrended query data (can be NULL, depending on @a type) 88 * @param xquery extrended query data (can be NULL, depending on @a type)
88 * @param xquery_size number of bytes in @a xquery 89 * @param xquery_size number of bytes in @a xquery
89 * @param reply_block response to validate 90 * @param reply_block response to validate
@@ -93,10 +94,9 @@ block_plugin_gns_create_group (void *cls,
93static enum GNUNET_BLOCK_EvaluationResult 94static enum GNUNET_BLOCK_EvaluationResult
94block_plugin_gns_evaluate (void *cls, 95block_plugin_gns_evaluate (void *cls,
95 enum GNUNET_BLOCK_Type type, 96 enum GNUNET_BLOCK_Type type,
97 struct GNUNET_BLOCK_Group *bg,
96 enum GNUNET_BLOCK_EvaluationOptions eo, 98 enum GNUNET_BLOCK_EvaluationOptions eo,
97 const struct GNUNET_HashCode *query, 99 const struct GNUNET_HashCode *query,
98 struct GNUNET_CONTAINER_BloomFilter **bf,
99 int32_t bf_mutator,
100 const void *xquery, 100 const void *xquery,
101 size_t xquery_size, 101 size_t xquery_size,
102 const void *reply_block, 102 const void *reply_block,
@@ -105,7 +105,6 @@ block_plugin_gns_evaluate (void *cls,
105 const struct GNUNET_GNSRECORD_Block *block; 105 const struct GNUNET_GNSRECORD_Block *block;
106 struct GNUNET_HashCode h; 106 struct GNUNET_HashCode h;
107 struct GNUNET_HashCode chash; 107 struct GNUNET_HashCode chash;
108 struct GNUNET_HashCode mhash;
109 108
110 if (type != GNUNET_BLOCK_TYPE_GNS_NAMERECORD) 109 if (type != GNUNET_BLOCK_TYPE_GNS_NAMERECORD)
111 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; 110 return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED;
@@ -146,21 +145,13 @@ block_plugin_gns_evaluate (void *cls,
146 GNUNET_break_op (0); 145 GNUNET_break_op (0);
147 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; 146 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
148 } 147 }
149 if (NULL != bf) 148 GNUNET_CRYPTO_hash (reply_block,
150 { 149 reply_block_size,
151 GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); 150 &chash);
152 GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); 151 if (GNUNET_YES ==
153 if (NULL != *bf) 152 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
154 { 153 &chash))
155 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test(*bf, &mhash)) 154 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
156 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
157 }
158 else
159 {
160 *bf = GNUNET_CONTAINER_bloomfilter_init(NULL, 8, BLOOMFILTER_K);
161 }
162 GNUNET_CONTAINER_bloomfilter_add(*bf, &mhash);
163 }
164 return GNUNET_BLOCK_EVALUATION_OK_MORE; 155 return GNUNET_BLOCK_EVALUATION_OK_MORE;
165} 156}
166 157
diff --git a/src/include/gnunet_block_group_lib.h b/src/include/gnunet_block_group_lib.h
index 5fa14ce00..a1ea807f6 100644
--- a/src/include/gnunet_block_group_lib.h
+++ b/src/include/gnunet_block_group_lib.h
@@ -66,6 +66,20 @@ GNUNET_BLOCK_GROUP_bf_create (void *cls,
66 size_t raw_data_size); 66 size_t raw_data_size);
67 67
68 68
69/**
70 * Test if @a hc is contained in the Bloom filter of @a bg. If so,
71 * return #GNUNET_YES. If not, add @a hc to the Bloom filter and
72 * return #GNUNET_NO.
73 *
74 * @param bg block group to use for testing
75 * @param hc hash of element to evaluate
76 * @return #GNUNET_YES if @a hc is (likely) a duplicate
77 * #GNUNET_NO if @a hc was definitively not in @bg (but now is)
78 */
79int
80GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg,
81 const struct GNUNET_HashCode *hc);
82
69 83
70#if 0 /* keep Emacsens' auto-indent happy */ 84#if 0 /* keep Emacsens' auto-indent happy */
71{ 85{
diff --git a/src/include/gnunet_block_lib.h b/src/include/gnunet_block_lib.h
index 0f0fee499..a40f33699 100644
--- a/src/include/gnunet_block_lib.h
+++ b/src/include/gnunet_block_lib.h
@@ -245,6 +245,7 @@ struct GNUNET_BLOCK_Group;
245 * @param nonce random value used to seed the group creation 245 * @param nonce random value used to seed the group creation
246 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 246 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
247 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 247 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
248 * @param ... type-specific additional data, can be empty
248 * @return block group handle, NULL if block groups are not supported 249 * @return block group handle, NULL if block groups are not supported
249 * by this @a type of block (this is not an error) 250 * by this @a type of block (this is not an error)
250 */ 251 */
@@ -253,13 +254,15 @@ GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
253 enum GNUNET_BLOCK_Type type, 254 enum GNUNET_BLOCK_Type type,
254 uint32_t nonce, 255 uint32_t nonce,
255 const void *raw_data, 256 const void *raw_data,
256 size_t raw_data_size); 257 size_t raw_data_size,
258 ...);
257 259
258 260
259/** 261/**
260 * Serialize state of a block group. 262 * Serialize state of a block group.
261 * 263 *
262 * @param bg group to serialize 264 * @param bg group to serialize
265 * @param[out] nonce set to the nonce of the @a bg
263 * @param[out] raw_data set to the serialized state 266 * @param[out] raw_data set to the serialized state
264 * @param[out] raw_data_size set to the number of bytes in @a raw_data 267 * @param[out] raw_data_size set to the number of bytes in @a raw_data
265 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not 268 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
@@ -267,6 +270,7 @@ GNUNET_BLOCK_group_create (struct GNUNET_BLOCK_Context *ctx,
267 */ 270 */
268int 271int
269GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg, 272GNUNET_BLOCK_group_serialize (struct GNUNET_BLOCK_Group *bg,
273 uint32_t *nonce,
270 void **raw_data, 274 void **raw_data,
271 size_t *raw_data_size); 275 size_t *raw_data_size);
272 276
@@ -289,10 +293,9 @@ GNUNET_BLOCK_group_destroy (struct GNUNET_BLOCK_Group *bg);
289 * 293 *
290 * @param ctx block contxt 294 * @param ctx block contxt
291 * @param type block type 295 * @param type block type
296 * @param group block group to use for evaluation
292 * @param eo evaluation options to control evaluation 297 * @param eo evaluation options to control evaluation
293 * @param query original query (hash) 298 * @param query original query (hash)
294 * @param bf pointer to bloom filter associated with query; possibly updated (!)
295 * @param bf_mutator mutation value for @a bf
296 * @param xquery extrended query data (can be NULL, depending on type) 299 * @param xquery extrended query data (can be NULL, depending on type)
297 * @param xquery_size number of bytes in @a xquery 300 * @param xquery_size number of bytes in @a xquery
298 * @param reply_block response to validate 301 * @param reply_block response to validate
@@ -302,10 +305,9 @@ GNUNET_BLOCK_group_destroy (struct GNUNET_BLOCK_Group *bg);
302enum GNUNET_BLOCK_EvaluationResult 305enum GNUNET_BLOCK_EvaluationResult
303GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx, 306GNUNET_BLOCK_evaluate (struct GNUNET_BLOCK_Context *ctx,
304 enum GNUNET_BLOCK_Type type, 307 enum GNUNET_BLOCK_Type type,
308 struct GNUNET_BLOCK_Group *group,
305 enum GNUNET_BLOCK_EvaluationOptions eo, 309 enum GNUNET_BLOCK_EvaluationOptions eo,
306 const struct GNUNET_HashCode *query, 310 const struct GNUNET_HashCode *query,
307 struct GNUNET_CONTAINER_BloomFilter **bf,
308 int32_t bf_mutator,
309 const void *xquery, 311 const void *xquery,
310 size_t xquery_size, 312 size_t xquery_size,
311 const void *reply_block, 313 const void *reply_block,
@@ -330,24 +332,41 @@ GNUNET_BLOCK_get_key (struct GNUNET_BLOCK_Context *ctx,
330 enum GNUNET_BLOCK_Type type, 332 enum GNUNET_BLOCK_Type type,
331 const void *block, 333 const void *block,
332 size_t block_size, 334 size_t block_size,
333 struct GNUNET_HashCode * key); 335 struct GNUNET_HashCode *key);
334
335 336
336 337
337/** 338/**
338 * Construct a bloom filter that would filter out the given 339 * Update block group to filter out the given results. Note that the
339 * results. 340 * use of a hash for seen results implies that the caller magically
341 * knows how the specific block engine hashes for filtering
342 * duplicates, so this API may not always apply.
340 * 343 *
341 * @param bf_mutator mutation value to use 344 * @param bf_mutator mutation value to use
342 * @param seen_results results already seen 345 * @param seen_results results already seen
343 * @param seen_results_count number of entries in @a seen_results 346 * @param seen_results_count number of entries in @a seen_results
344 * @return NULL if seen_results_count is 0, otherwise a BF 347 * @return #GNUNET_SYSERR if not supported, #GNUNET_OK on success
345 * that would match the given results. 348 */
349int
350GNUNET_BLOCK_group_set_seen (struct GNUNET_BLOCK_Group *bg,
351 const struct GNUNET_HashCode *seen_results,
352 unsigned int seen_results_count);
353
354
355/**
356 * Try merging two block groups. Afterwards, @a bg1 should remain
357 * valid and contain the rules from both @a bg1 and @bg2, and
358 * @a bg2 should be destroyed (as part of this call). The latter
359 * should happen even if merging is not supported.
360 *
361 * @param[in,out] bg1 first group to merge, is updated
362 * @param bg2 second group to merge, is destroyed
363 * @return #GNUNET_OK on success,
364 * #GNUNET_NO if merge failed due to different nonce
365 * #GNUNET_SYSERR if merging is not supported
346 */ 366 */
347struct GNUNET_CONTAINER_BloomFilter * 367int
348GNUNET_BLOCK_construct_bloomfilter (int32_t bf_mutator, 368GNUNET_BLOCK_group_merge (struct GNUNET_BLOCK_Group *bg1,
349 const struct GNUNET_HashCode *seen_results, 369 struct GNUNET_BLOCK_Group *bg2);
350 unsigned int seen_results_count);
351 370
352 371
353#if 0 /* keep Emacsens' auto-indent happy */ 372#if 0 /* keep Emacsens' auto-indent happy */
diff --git a/src/include/gnunet_block_plugin.h b/src/include/gnunet_block_plugin.h
index d7454b5d5..171b6cfc0 100644
--- a/src/include/gnunet_block_plugin.h
+++ b/src/include/gnunet_block_plugin.h
@@ -40,9 +40,38 @@
40 40
41 41
42/** 42/**
43 * Mark elements as "seen" using a hash of the element. Not supported
44 * by all block plugins.
45 *
46 * @param bg group to update
47 * @param seen_results results already seen
48 * @param seen_results_count number of entries in @a seen_results
49 */
50typedef void
51(*GNUNET_BLOCK_GroupMarkSeenFunction)(struct GNUNET_BLOCK_Group *bg,
52 const struct GNUNET_HashCode *seen_results,
53 unsigned int seen_results_count);
54
55
56/**
57 * Merge two groups, if possible. Not supported by all block plugins,
58 * can also fail if the nonces were different.
59 *
60 * @param bg1 group to update
61 * @param bg2 group to merge into @a bg1
62 * @return #GNUNET_OK on success, #GNUNET_NO if the nonces were different and thus
63 * we failed.
64 */
65typedef int
66(*GNUNET_BLOCK_GroupMergeFunction)(struct GNUNET_BLOCK_Group *bg1,
67 const struct GNUNET_BLOCK_Group *bg2);
68
69
70/**
43 * Serialize state of a block group. 71 * Serialize state of a block group.
44 * 72 *
45 * @param bg group to serialize 73 * @param bg group to serialize
74 * @param[out] nonce set to the nonce of the @a bg
46 * @param[out] raw_data set to the serialized state 75 * @param[out] raw_data set to the serialized state
47 * @param[out] raw_data_size set to the number of bytes in @a raw_data 76 * @param[out] raw_data_size set to the number of bytes in @a raw_data
48 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not 77 * @return #GNUNET_OK on success, #GNUNET_NO if serialization is not
@@ -50,6 +79,7 @@
50 */ 79 */
51typedef int 80typedef int
52(*GNUNET_BLOCK_GroupSerializeFunction)(struct GNUNET_BLOCK_Group *bg, 81(*GNUNET_BLOCK_GroupSerializeFunction)(struct GNUNET_BLOCK_Group *bg,
82 uint32_t *nonce,
53 void **raw_data, 83 void **raw_data,
54 size_t *raw_data_size); 84 size_t *raw_data_size);
55 85
@@ -87,6 +117,18 @@ struct GNUNET_BLOCK_Group
87 GNUNET_BLOCK_GroupSerializeFunction serialize_cb; 117 GNUNET_BLOCK_GroupSerializeFunction serialize_cb;
88 118
89 /** 119 /**
120 * Function to call to mark elements as seen in the group.
121 * Can be NULL if not supported.
122 */
123 GNUNET_BLOCK_GroupMarkSeenFunction mark_seen_cb;
124
125 /**
126 * Function to call to merge two groups.
127 * Can be NULL if not supported.
128 */
129 GNUNET_BLOCK_GroupMergeFunction merge_cb;
130
131 /**
90 * Function to call to destroy the block group. 132 * Function to call to destroy the block group.
91 * Must not be NULL. 133 * Must not be NULL.
92 */ 134 */
@@ -108,6 +150,7 @@ struct GNUNET_BLOCK_Group
108 * @param nonce random value used to seed the group creation 150 * @param nonce random value used to seed the group creation
109 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 151 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
110 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 152 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
153 * @param va variable arguments specific to @a type
111 * @return block group handle, NULL if block groups are not supported 154 * @return block group handle, NULL if block groups are not supported
112 * by this @a type of block (this is not an error) 155 * by this @a type of block (this is not an error)
113 */ 156 */
@@ -116,7 +159,8 @@ typedef struct GNUNET_BLOCK_Group *
116 enum GNUNET_BLOCK_Type type, 159 enum GNUNET_BLOCK_Type type,
117 uint32_t nonce, 160 uint32_t nonce,
118 const void *raw_data, 161 const void *raw_data,
119 size_t raw_data_size); 162 size_t raw_data_size,
163 va_list va);
120 164
121 165
122/** 166/**
@@ -128,10 +172,9 @@ typedef struct GNUNET_BLOCK_Group *
128 * 172 *
129 * @param cls closure 173 * @param cls closure
130 * @param type block type 174 * @param type block type
175 * @param group which block group to use for evaluation
131 * @param eo evaluation options to control evaluation 176 * @param eo evaluation options to control evaluation
132 * @param query original query (hash) 177 * @param query original query (hash)
133 * @param bf pointer to bloom filter associated with query; possibly updated (!)
134 * @param bf_mutator mutation value for @a bf
135 * @param xquery extrended query data (can be NULL, depending on type) 178 * @param xquery extrended query data (can be NULL, depending on type)
136 * @param xquery_size number of bytes in @a xquery 179 * @param xquery_size number of bytes in @a xquery
137 * @param reply_block response to validate 180 * @param reply_block response to validate
@@ -141,10 +184,9 @@ typedef struct GNUNET_BLOCK_Group *
141typedef enum GNUNET_BLOCK_EvaluationResult 184typedef enum GNUNET_BLOCK_EvaluationResult
142(*GNUNET_BLOCK_EvaluationFunction) (void *cls, 185(*GNUNET_BLOCK_EvaluationFunction) (void *cls,
143 enum GNUNET_BLOCK_Type type, 186 enum GNUNET_BLOCK_Type type,
187 struct GNUNET_BLOCK_Group *group,
144 enum GNUNET_BLOCK_EvaluationOptions eo, 188 enum GNUNET_BLOCK_EvaluationOptions eo,
145 const struct GNUNET_HashCode *query, 189 const struct GNUNET_HashCode *query,
146 struct GNUNET_CONTAINER_BloomFilter **bf,
147 int32_t bf_mutator,
148 const void *xquery, 190 const void *xquery,
149 size_t xquery_size, 191 size_t xquery_size,
150 const void *reply_block, 192 const void *reply_block,
diff --git a/src/regex/plugin_block_regex.c b/src/regex/plugin_block_regex.c
index 11511a71b..6636f3cdb 100644
--- a/src/regex/plugin_block_regex.c
+++ b/src/regex/plugin_block_regex.c
@@ -46,6 +46,7 @@
46 * @param nonce random value used to seed the group creation 46 * @param nonce random value used to seed the group creation
47 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh 47 * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh
48 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh 48 * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh
49 * @param va variable arguments specific to @a type
49 * @return block group handle, NULL if block groups are not supported 50 * @return block group handle, NULL if block groups are not supported
50 * by this @a type of block (this is not an error) 51 * by this @a type of block (this is not an error)
51 */ 52 */
@@ -54,7 +55,8 @@ block_plugin_regex_create_group (void *cls,
54 enum GNUNET_BLOCK_Type type, 55 enum GNUNET_BLOCK_Type type,
55 uint32_t nonce, 56 uint32_t nonce,
56 const void *raw_data, 57 const void *raw_data,
57 size_t raw_data_size) 58 size_t raw_data_size,
59 va_list va)
58{ 60{
59 return GNUNET_BLOCK_GROUP_bf_create (cls, 61 return GNUNET_BLOCK_GROUP_bf_create (cls,
60 REGEX_BF_SIZE, 62 REGEX_BF_SIZE,
@@ -76,10 +78,9 @@ block_plugin_regex_create_group (void *cls,
76 * 78 *
77 * @param cls closure 79 * @param cls closure
78 * @param type block type 80 * @param type block type
81 * @param bg block group to evaluate against
79 * @param eo control flags 82 * @param eo control flags
80 * @param query original query (hash) 83 * @param query original query (hash)
81 * @param bf pointer to bloom filter associated with query; possibly updated (!)
82 * @param bf_mutator mutation value for bf
83 * @param xquery extrended query data (can be NULL, depending on type) 84 * @param xquery extrended query data (can be NULL, depending on type)
84 * @param xquery_size number of bytes in @a xquery 85 * @param xquery_size number of bytes in @a xquery
85 * @param reply_block response to validate 86 * @param reply_block response to validate
@@ -89,15 +90,16 @@ block_plugin_regex_create_group (void *cls,
89static enum GNUNET_BLOCK_EvaluationResult 90static enum GNUNET_BLOCK_EvaluationResult
90evaluate_block_regex (void *cls, 91evaluate_block_regex (void *cls,
91 enum GNUNET_BLOCK_Type type, 92 enum GNUNET_BLOCK_Type type,
93 struct GNUNET_BLOCK_Group *bg,
92 enum GNUNET_BLOCK_EvaluationOptions eo, 94 enum GNUNET_BLOCK_EvaluationOptions eo,
93 const struct GNUNET_HashCode *query, 95 const struct GNUNET_HashCode *query,
94 struct GNUNET_CONTAINER_BloomFilter **bf,
95 int32_t bf_mutator,
96 const void *xquery, 96 const void *xquery,
97 size_t xquery_size, 97 size_t xquery_size,
98 const void *reply_block, 98 const void *reply_block,
99 size_t reply_block_size) 99 size_t reply_block_size)
100{ 100{
101 struct GNUNET_HashCode chash;
102
101 if (NULL == reply_block) 103 if (NULL == reply_block)
102 { 104 {
103 if (0 != xquery_size) 105 if (0 != xquery_size)
@@ -146,24 +148,13 @@ evaluate_block_regex (void *cls,
146 default: 148 default:
147 break; 149 break;
148 } 150 }
149 if (NULL != bf) 151 GNUNET_CRYPTO_hash (reply_block,
150 { 152 reply_block_size,
151 struct GNUNET_HashCode chash; 153 &chash);
152 struct GNUNET_HashCode mhash; 154 if (GNUNET_YES ==
153 155 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
154 GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); 156 &chash))
155 GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); 157 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
156 if (NULL != *bf)
157 {
158 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
159 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
160 }
161 else
162 {
163 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, GNUNET_CONSTANTS_BLOOMFILTER_K);
164 }
165 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
166 }
167 return GNUNET_BLOCK_EVALUATION_OK_MORE; 158 return GNUNET_BLOCK_EVALUATION_OK_MORE;
168} 159}
169 160
@@ -178,10 +169,9 @@ evaluate_block_regex (void *cls,
178 * 169 *
179 * @param cls closure 170 * @param cls closure
180 * @param type block type 171 * @param type block type
172 * @param bg block group to evaluate against
181 * @param eo control flags 173 * @param eo control flags
182 * @param query original query (hash) 174 * @param query original query (hash)
183 * @param bf pointer to bloom filter associated with query; possibly updated (!)
184 * @param bf_mutator mutation value for bf
185 * @param xquery extrended query data (can be NULL, depending on type) 175 * @param xquery extrended query data (can be NULL, depending on type)
186 * @param xquery_size number of bytes in @a xquery 176 * @param xquery_size number of bytes in @a xquery
187 * @param reply_block response to validate 177 * @param reply_block response to validate
@@ -191,14 +181,15 @@ evaluate_block_regex (void *cls,
191static enum GNUNET_BLOCK_EvaluationResult 181static enum GNUNET_BLOCK_EvaluationResult
192evaluate_block_regex_accept (void *cls, 182evaluate_block_regex_accept (void *cls,
193 enum GNUNET_BLOCK_Type type, 183 enum GNUNET_BLOCK_Type type,
184 struct GNUNET_BLOCK_Group *bg,
194 enum GNUNET_BLOCK_EvaluationOptions eo, 185 enum GNUNET_BLOCK_EvaluationOptions eo,
195 const struct GNUNET_HashCode * query, 186 const struct GNUNET_HashCode *query,
196 struct GNUNET_CONTAINER_BloomFilter **bf, 187 const void *xquery,
197 int32_t bf_mutator, const void *xquery,
198 size_t xquery_size, const void *reply_block, 188 size_t xquery_size, const void *reply_block,
199 size_t reply_block_size) 189 size_t reply_block_size)
200{ 190{
201 const struct RegexAcceptBlock *rba; 191 const struct RegexAcceptBlock *rba;
192 struct GNUNET_HashCode chash;
202 193
203 if (0 != xquery_size) 194 if (0 != xquery_size)
204 { 195 {
@@ -236,24 +227,13 @@ evaluate_block_regex_accept (void *cls,
236 GNUNET_break_op(0); 227 GNUNET_break_op(0);
237 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; 228 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
238 } 229 }
239 if (NULL != bf) 230 GNUNET_CRYPTO_hash (reply_block,
240 { 231 reply_block_size,
241 struct GNUNET_HashCode chash; 232 &chash);
242 struct GNUNET_HashCode mhash; 233 if (GNUNET_YES ==
243 234 GNUNET_BLOCK_GROUP_bf_test_and_set (bg,
244 GNUNET_CRYPTO_hash (reply_block, reply_block_size, &chash); 235 &chash))
245 GNUNET_BLOCK_mingle_hash (&chash, bf_mutator, &mhash); 236 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
246 if (NULL != *bf)
247 {
248 if (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (*bf, &mhash))
249 return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE;
250 }
251 else
252 {
253 *bf = GNUNET_CONTAINER_bloomfilter_init (NULL, 8, GNUNET_CONSTANTS_BLOOMFILTER_K);
254 }
255 GNUNET_CONTAINER_bloomfilter_add (*bf, &mhash);
256 }
257 return GNUNET_BLOCK_EVALUATION_OK_MORE; 237 return GNUNET_BLOCK_EVALUATION_OK_MORE;
258} 238}
259 239
@@ -267,10 +247,9 @@ evaluate_block_regex_accept (void *cls,
267 * 247 *
268 * @param cls closure 248 * @param cls closure
269 * @param type block type 249 * @param type block type
250 * @param bg group to evaluate against
270 * @param eo control flags 251 * @param eo control flags
271 * @param query original query (hash) 252 * @param query original query (hash)
272 * @param bf pointer to bloom filter associated with query; possibly updated (!)
273 * @param bf_mutator mutation value for bf
274 * @param xquery extrended query data (can be NULL, depending on type) 253 * @param xquery extrended query data (can be NULL, depending on type)
275 * @param xquery_size number of bytes in xquery 254 * @param xquery_size number of bytes in xquery
276 * @param reply_block response to validate 255 * @param reply_block response to validate
@@ -280,10 +259,9 @@ evaluate_block_regex_accept (void *cls,
280static enum GNUNET_BLOCK_EvaluationResult 259static enum GNUNET_BLOCK_EvaluationResult
281block_plugin_regex_evaluate (void *cls, 260block_plugin_regex_evaluate (void *cls,
282 enum GNUNET_BLOCK_Type type, 261 enum GNUNET_BLOCK_Type type,
262 struct GNUNET_BLOCK_Group *bg,
283 enum GNUNET_BLOCK_EvaluationOptions eo, 263 enum GNUNET_BLOCK_EvaluationOptions eo,
284 const struct GNUNET_HashCode *query, 264 const struct GNUNET_HashCode *query,
285 struct GNUNET_CONTAINER_BloomFilter **bf,
286 int32_t bf_mutator,
287 const void *xquery, 265 const void *xquery,
288 size_t xquery_size, 266 size_t xquery_size,
289 const void *reply_block, 267 const void *reply_block,
@@ -296,18 +274,18 @@ block_plugin_regex_evaluate (void *cls,
296 case GNUNET_BLOCK_TYPE_REGEX: 274 case GNUNET_BLOCK_TYPE_REGEX:
297 result = evaluate_block_regex (cls, 275 result = evaluate_block_regex (cls,
298 type, 276 type,
277 bg,
299 eo, 278 eo,
300 query, 279 query,
301 bf, bf_mutator,
302 xquery, xquery_size, 280 xquery, xquery_size,
303 reply_block, reply_block_size); 281 reply_block, reply_block_size);
304 break; 282 break;
305 case GNUNET_BLOCK_TYPE_REGEX_ACCEPT: 283 case GNUNET_BLOCK_TYPE_REGEX_ACCEPT:
306 result = evaluate_block_regex_accept (cls, 284 result = evaluate_block_regex_accept (cls,
307 type, 285 type,
286 bg,
308 eo, 287 eo,
309 query, 288 query,
310 bf, bf_mutator,
311 xquery, xquery_size, 289 xquery, xquery_size,
312 reply_block, reply_block_size); 290 reply_block, reply_block_size);
313 break; 291 break;