aboutsummaryrefslogtreecommitdiff
path: root/src/set
diff options
context:
space:
mode:
Diffstat (limited to 'src/set')
-rw-r--r--src/set/.gitignore7
-rw-r--r--src/set/Makefile.am124
-rw-r--r--src/set/gnunet-service-set.c1984
-rw-r--r--src/set/gnunet-service-set.h665
-rw-r--r--src/set/gnunet-service-set_intersection.c1331
-rw-r--r--src/set/gnunet-service-set_intersection.h79
-rw-r--r--src/set/gnunet-service-set_protocol.h226
-rw-r--r--src/set/gnunet-service-set_union.c2469
-rw-r--r--src/set/gnunet-service-set_union.h246
-rw-r--r--src/set/gnunet-service-set_union_strata_estimator.c303
-rw-r--r--src/set/gnunet-service-set_union_strata_estimator.h169
-rw-r--r--src/set/gnunet-set-ibf-profiler.c308
-rw-r--r--src/set/gnunet-set-profiler.c508
-rw-r--r--src/set/ibf.c409
-rw-r--r--src/set/ibf.h256
-rw-r--r--src/set/ibf_sim.c142
-rw-r--r--src/set/plugin_block_set_test.c167
-rw-r--r--src/set/set.conf.in12
-rw-r--r--src/set/set.h400
-rw-r--r--src/set/set_api.c1260
-rw-r--r--src/set/test_set.conf33
-rw-r--r--src/set/test_set_api.c403
-rw-r--r--src/set/test_set_intersection_result_full.c393
-rw-r--r--src/set/test_set_union_copy.c311
-rw-r--r--src/set/test_set_union_result_symmetric.c455
25 files changed, 0 insertions, 12660 deletions
diff --git a/src/set/.gitignore b/src/set/.gitignore
deleted file mode 100644
index f1c958639..000000000
--- a/src/set/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
1gnunet-set-profiler
2gnunet-service-set
3gnunet-set-ibf-profiler
4test_set_api
5test_set_intersection_result_full
6test_set_union_copy
7test_set_union_result_symmetric
diff --git a/src/set/Makefile.am b/src/set/Makefile.am
deleted file mode 100644
index 837d42ff5..000000000
--- a/src/set/Makefile.am
+++ /dev/null
@@ -1,124 +0,0 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4pkgcfgdir= $(pkgdatadir)/config.d/
5
6libexecdir= $(pkglibdir)/libexec/
7
8plugindir = $(libdir)/gnunet
9
10pkgcfg_DATA = \
11 set.conf
12
13if USE_COVERAGE
14 AM_CFLAGS = -fprofile-arcs -ftest-coverage
15endif
16
17bin_PROGRAMS = \
18 gnunet-set-profiler
19
20noinst_PROGRAMS = \
21 gnunet-set-ibf-profiler
22
23libexec_PROGRAMS = \
24 gnunet-service-set
25
26lib_LTLIBRARIES = \
27 libgnunetset.la
28
29gnunet_set_profiler_SOURCES = \
30 gnunet-set-profiler.c
31gnunet_set_profiler_LDADD = \
32 $(top_builddir)/src/util/libgnunetutil.la \
33 $(top_builddir)/src/statistics/libgnunetstatistics.la \
34 libgnunetset.la \
35 $(top_builddir)/src/testing/libgnunettesting.la \
36 $(GN_LIBINTL)
37
38
39gnunet_set_ibf_profiler_SOURCES = \
40 gnunet-set-ibf-profiler.c \
41 ibf.c
42gnunet_set_ibf_profiler_LDADD = \
43 $(top_builddir)/src/util/libgnunetutil.la \
44 $(GN_LIBINTL)
45
46gnunet_service_set_SOURCES = \
47 gnunet-service-set.c gnunet-service-set.h \
48 gnunet-service-set_union.c gnunet-service-set_union.h \
49 gnunet-service-set_intersection.c gnunet-service-set_intersection.h \
50 ibf.c ibf.h \
51 gnunet-service-set_union_strata_estimator.c gnunet-service-set_union_strata_estimator.h \
52 gnunet-service-set_protocol.h
53gnunet_service_set_LDADD = \
54 $(top_builddir)/src/util/libgnunetutil.la \
55 $(top_builddir)/src/statistics/libgnunetstatistics.la \
56 $(top_builddir)/src/core/libgnunetcore.la \
57 $(top_builddir)/src/cadet/libgnunetcadet.la \
58 $(top_builddir)/src/block/libgnunetblock.la \
59 libgnunetset.la \
60 $(GN_LIBINTL)
61
62libgnunetset_la_SOURCES = \
63 set_api.c set.h
64libgnunetset_la_LIBADD = \
65 $(top_builddir)/src/util/libgnunetutil.la \
66 $(LTLIBINTL)
67libgnunetset_la_LDFLAGS = \
68 $(GN_LIB_LDFLAGS)
69
70check_PROGRAMS = \
71 test_set_api \
72 test_set_union_result_symmetric \
73 test_set_intersection_result_full \
74 test_set_union_copy
75
76if ENABLE_TEST_RUN
77AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
78TESTS = $(check_PROGRAMS)
79endif
80
81test_set_api_SOURCES = \
82 test_set_api.c
83test_set_api_LDADD = \
84 $(top_builddir)/src/util/libgnunetutil.la \
85 $(top_builddir)/src/testing/libgnunettesting.la \
86 libgnunetset.la
87
88test_set_union_result_symmetric_SOURCES = \
89 test_set_union_result_symmetric.c
90test_set_union_result_symmetric_LDADD = \
91 $(top_builddir)/src/util/libgnunetutil.la \
92 $(top_builddir)/src/testing/libgnunettesting.la \
93 libgnunetset.la
94
95test_set_intersection_result_full_SOURCES = \
96 test_set_intersection_result_full.c
97test_set_intersection_result_full_LDADD = \
98 $(top_builddir)/src/util/libgnunetutil.la \
99 $(top_builddir)/src/testing/libgnunettesting.la \
100 libgnunetset.la
101
102test_set_union_copy_SOURCES = \
103 test_set_union_copy.c
104test_set_union_copy_LDADD = \
105 $(top_builddir)/src/util/libgnunetutil.la \
106 $(top_builddir)/src/testing/libgnunettesting.la \
107 libgnunetset.la
108
109plugin_LTLIBRARIES = \
110 libgnunet_plugin_block_set_test.la
111
112libgnunet_plugin_block_set_test_la_SOURCES = \
113 plugin_block_set_test.c
114libgnunet_plugin_block_set_test_la_LIBADD = \
115 $(top_builddir)/src/block/libgnunetblock.la \
116 $(top_builddir)/src/block/libgnunetblockgroup.la \
117 $(top_builddir)/src/util/libgnunetutil.la \
118 $(LTLIBINTL)
119libgnunet_plugin_block_set_test_la_LDFLAGS = \
120 $(GN_PLUGIN_LDFLAGS)
121
122
123EXTRA_DIST = \
124 test_set.conf
diff --git a/src/set/gnunet-service-set.c b/src/set/gnunet-service-set.c
deleted file mode 100644
index 2b859d81a..000000000
--- a/src/set/gnunet-service-set.c
+++ /dev/null
@@ -1,1984 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set.c
22 * @brief two-peer set operations
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#include "gnunet-service-set.h"
27#include "gnunet-service-set_union.h"
28#include "gnunet-service-set_intersection.h"
29#include "gnunet-service-set_protocol.h"
30#include "gnunet_statistics_service.h"
31
32/**
33 * How long do we hold on to an incoming channel if there is
34 * no local listener before giving up?
35 */
36#define INCOMING_CHANNEL_TIMEOUT GNUNET_TIME_UNIT_MINUTES
37
38
39/**
40 * Lazy copy requests made by a client.
41 */
42struct LazyCopyRequest
43{
44 /**
45 * Kept in a DLL.
46 */
47 struct LazyCopyRequest *prev;
48
49 /**
50 * Kept in a DLL.
51 */
52 struct LazyCopyRequest *next;
53
54 /**
55 * Which set are we supposed to copy?
56 */
57 struct Set *source_set;
58
59 /**
60 * Cookie identifying the request.
61 */
62 uint32_t cookie;
63};
64
65
66/**
67 * A listener is inhabited by a client, and waits for evaluation
68 * requests from remote peers.
69 */
70struct Listener
71{
72 /**
73 * Listeners are held in a doubly linked list.
74 */
75 struct Listener *next;
76
77 /**
78 * Listeners are held in a doubly linked list.
79 */
80 struct Listener *prev;
81
82 /**
83 * Head of DLL of operations this listener is responsible for.
84 * Once the client has accepted/declined the operation, the
85 * operation is moved to the respective set's operation DLLS.
86 */
87 struct Operation *op_head;
88
89 /**
90 * Tail of DLL of operations this listener is responsible for.
91 * Once the client has accepted/declined the operation, the
92 * operation is moved to the respective set's operation DLLS.
93 */
94 struct Operation *op_tail;
95
96 /**
97 * Client that owns the listener.
98 * Only one client may own a listener.
99 */
100 struct ClientState *cs;
101
102 /**
103 * The port we are listening on with CADET.
104 */
105 struct GNUNET_CADET_Port *open_port;
106
107 /**
108 * Application ID for the operation, used to distinguish
109 * multiple operations of the same type with the same peer.
110 */
111 struct GNUNET_HashCode app_id;
112
113 /**
114 * The type of the operation.
115 */
116 enum GNUNET_SET_OperationType operation;
117};
118
119
120/**
121 * Handle to the cadet service, used to listen for and connect to
122 * remote peers.
123 */
124static struct GNUNET_CADET_Handle *cadet;
125
126/**
127 * DLL of lazy copy requests by this client.
128 */
129static struct LazyCopyRequest *lazy_copy_head;
130
131/**
132 * DLL of lazy copy requests by this client.
133 */
134static struct LazyCopyRequest *lazy_copy_tail;
135
136/**
137 * Generator for unique cookie we set per lazy copy request.
138 */
139static uint32_t lazy_copy_cookie;
140
141/**
142 * Statistics handle.
143 */
144struct GNUNET_STATISTICS_Handle *_GSS_statistics;
145
146/**
147 * Listeners are held in a doubly linked list.
148 */
149static struct Listener *listener_head;
150
151/**
152 * Listeners are held in a doubly linked list.
153 */
154static struct Listener *listener_tail;
155
156/**
157 * Number of active clients.
158 */
159static unsigned int num_clients;
160
161/**
162 * Are we in shutdown? if #GNUNET_YES and the number of clients
163 * drops to zero, disconnect from CADET.
164 */
165static int in_shutdown;
166
167/**
168 * Counter for allocating unique IDs for clients, used to identify
169 * incoming operation requests from remote peers, that the client can
170 * choose to accept or refuse. 0 must not be used (reserved for
171 * uninitialized).
172 */
173static uint32_t suggest_id;
174
175
176/**
177 * Get the incoming socket associated with the given id.
178 *
179 * @param listener the listener to look in
180 * @param id id to look for
181 * @return the incoming socket associated with the id,
182 * or NULL if there is none
183 */
184static struct Operation *
185get_incoming (uint32_t id)
186{
187 for (struct Listener *listener = listener_head; NULL != listener;
188 listener = listener->next)
189 {
190 for (struct Operation *op = listener->op_head; NULL != op; op = op->next)
191 if (op->suggest_id == id)
192 return op;
193 }
194 return NULL;
195}
196
197
198/**
199 * Destroy an incoming request from a remote peer
200 *
201 * @param op remote request to destroy
202 */
203static void
204incoming_destroy (struct Operation *op)
205{
206 struct Listener *listener;
207
208 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
209 "Destroying incoming operation %p\n",
210 op);
211 if (NULL != (listener = op->listener))
212 {
213 GNUNET_CONTAINER_DLL_remove (listener->op_head, listener->op_tail, op);
214 op->listener = NULL;
215 }
216 if (NULL != op->timeout_task)
217 {
218 GNUNET_SCHEDULER_cancel (op->timeout_task);
219 op->timeout_task = NULL;
220 }
221 _GSS_operation_destroy2 (op);
222}
223
224
225/**
226 * Context for the #garbage_collect_cb().
227 */
228struct GarbageContext
229{
230 /**
231 * Map for which we are garbage collecting removed elements.
232 */
233 struct GNUNET_CONTAINER_MultiHashMap *map;
234
235 /**
236 * Lowest generation for which an operation is still pending.
237 */
238 unsigned int min_op_generation;
239
240 /**
241 * Largest generation for which an operation is still pending.
242 */
243 unsigned int max_op_generation;
244};
245
246
247/**
248 * Function invoked to check if an element can be removed from
249 * the set's history because it is no longer needed.
250 *
251 * @param cls the `struct GarbageContext *`
252 * @param key key of the element in the map
253 * @param value the `struct ElementEntry *`
254 * @return #GNUNET_OK (continue to iterate)
255 */
256static int
257garbage_collect_cb (void *cls, const struct GNUNET_HashCode *key, void *value)
258{
259 // struct GarbageContext *gc = cls;
260 // struct ElementEntry *ee = value;
261
262 // if (GNUNET_YES != ee->removed)
263 // return GNUNET_OK;
264 // if ( (gc->max_op_generation < ee->generation_added) ||
265 // (ee->generation_removed > gc->min_op_generation) )
266 // {
267 // GNUNET_assert (GNUNET_YES ==
268 // GNUNET_CONTAINER_multihashmap_remove (gc->map,
269 // key,
270 // ee));
271 // GNUNET_free (ee);
272 // }
273 return GNUNET_OK;
274}
275
276
277/**
278 * Collect and destroy elements that are not needed anymore, because
279 * their lifetime (as determined by their generation) does not overlap
280 * with any active set operation.
281 *
282 * @param set set to garbage collect
283 */
284static void
285collect_generation_garbage (struct Set *set)
286{
287 struct GarbageContext gc;
288
289 gc.min_op_generation = UINT_MAX;
290 gc.max_op_generation = 0;
291 for (struct Operation *op = set->ops_head; NULL != op; op = op->next)
292 {
293 gc.min_op_generation =
294 GNUNET_MIN (gc.min_op_generation, op->generation_created);
295 gc.max_op_generation =
296 GNUNET_MAX (gc.max_op_generation, op->generation_created);
297 }
298 gc.map = set->content->elements;
299 GNUNET_CONTAINER_multihashmap_iterate (set->content->elements,
300 &garbage_collect_cb,
301 &gc);
302}
303
304
305/**
306 * Is @a generation in the range of exclusions?
307 *
308 * @param generation generation to query
309 * @param excluded array of generations where the element is excluded
310 * @param excluded_size length of the @a excluded array
311 * @return #GNUNET_YES if @a generation is in any of the ranges
312 */
313static int
314is_excluded_generation (unsigned int generation,
315 struct GenerationRange *excluded,
316 unsigned int excluded_size)
317{
318 for (unsigned int i = 0; i < excluded_size; i++)
319 if ((generation >= excluded[i].start) && (generation < excluded[i].end))
320 return GNUNET_YES;
321 return GNUNET_NO;
322}
323
324
325/**
326 * Is element @a ee part of the set during @a query_generation?
327 *
328 * @param ee element to test
329 * @param query_generation generation to query
330 * @param excluded array of generations where the element is excluded
331 * @param excluded_size length of the @a excluded array
332 * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
333 */
334static int
335is_element_of_generation (struct ElementEntry *ee,
336 unsigned int query_generation,
337 struct GenerationRange *excluded,
338 unsigned int excluded_size)
339{
340 struct MutationEvent *mut;
341 int is_present;
342
343 GNUNET_assert (NULL != ee->mutations);
344 if (GNUNET_YES ==
345 is_excluded_generation (query_generation, excluded, excluded_size))
346 {
347 GNUNET_break (0);
348 return GNUNET_NO;
349 }
350
351 is_present = GNUNET_NO;
352
353 /* Could be made faster with binary search, but lists
354 are small, so why bother. */
355 for (unsigned int i = 0; i < ee->mutations_size; i++)
356 {
357 mut = &ee->mutations[i];
358
359 if (mut->generation > query_generation)
360 {
361 /* The mutation doesn't apply to our generation
362 anymore. We can'b break here, since mutations aren't
363 sorted by generation. */
364 continue;
365 }
366
367 if (GNUNET_YES ==
368 is_excluded_generation (mut->generation, excluded, excluded_size))
369 {
370 /* The generation is excluded (because it belongs to another
371 fork via a lazy copy) and thus mutations aren't considered
372 for membership testing. */
373 continue;
374 }
375
376 /* This would be an inconsistency in how we manage mutations. */
377 if ((GNUNET_YES == is_present) && (GNUNET_YES == mut->added))
378 GNUNET_assert (0);
379 /* Likewise. */
380 if ((GNUNET_NO == is_present) && (GNUNET_NO == mut->added))
381 GNUNET_assert (0);
382
383 is_present = mut->added;
384 }
385
386 return is_present;
387}
388
389
390/**
391 * Is element @a ee part of the set used by @a op?
392 *
393 * @param ee element to test
394 * @param op operation the defines the set and its generation
395 * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
396 */
397int
398_GSS_is_element_of_operation (struct ElementEntry *ee, struct Operation *op)
399{
400 return is_element_of_generation (ee,
401 op->generation_created,
402 op->set->excluded_generations,
403 op->set->excluded_generations_size);
404}
405
406
407/**
408 * Destroy the given operation. Used for any operation where both
409 * peers were known and that thus actually had a vt and channel. Must
410 * not be used for operations where 'listener' is still set and we do
411 * not know the other peer.
412 *
413 * Call the implementation-specific cancel function of the operation.
414 * Disconnects from the remote peer. Does not disconnect the client,
415 * as there may be multiple operations per set.
416 *
417 * @param op operation to destroy
418 * @param gc #GNUNET_YES to perform garbage collection on the set
419 */
420void
421_GSS_operation_destroy (struct Operation *op, int gc)
422{
423 struct Set *set = op->set;
424 struct GNUNET_CADET_Channel *channel;
425
426 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Destroying operation %p\n", op);
427 GNUNET_assert (NULL == op->listener);
428 if (NULL != op->state)
429 {
430 set->vt->cancel (op);
431 op->state = NULL;
432 }
433 if (NULL != set)
434 {
435 GNUNET_CONTAINER_DLL_remove (set->ops_head, set->ops_tail, op);
436 op->set = NULL;
437 }
438 if (NULL != op->context_msg)
439 {
440 GNUNET_free (op->context_msg);
441 op->context_msg = NULL;
442 }
443 if (NULL != (channel = op->channel))
444 {
445 /* This will free op; called conditionally as this helper function
446 is also called from within the channel disconnect handler. */
447 op->channel = NULL;
448 GNUNET_CADET_channel_destroy (channel);
449 }
450 if ((NULL != set) && (GNUNET_YES == gc))
451 collect_generation_garbage (set);
452 /* We rely on the channel end handler to free 'op'. When 'op->channel' was NULL,
453 * there was a channel end handler that will free 'op' on the call stack. */
454}
455
456
457/**
458 * Callback called when a client connects to the service.
459 *
460 * @param cls closure for the service
461 * @param c the new client that connected to the service
462 * @param mq the message queue used to send messages to the client
463 * @return @a `struct ClientState`
464 */
465static void *
466client_connect_cb (void *cls,
467 struct GNUNET_SERVICE_Client *c,
468 struct GNUNET_MQ_Handle *mq)
469{
470 struct ClientState *cs;
471
472 num_clients++;
473 cs = GNUNET_new (struct ClientState);
474 cs->client = c;
475 cs->mq = mq;
476 return cs;
477}
478
479
480/**
481 * Iterator over hash map entries to free element entries.
482 *
483 * @param cls closure
484 * @param key current key code
485 * @param value a `struct ElementEntry *` to be free'd
486 * @return #GNUNET_YES (continue to iterate)
487 */
488static int
489destroy_elements_iterator (void *cls,
490 const struct GNUNET_HashCode *key,
491 void *value)
492{
493 struct ElementEntry *ee = value;
494
495 GNUNET_free (ee->mutations);
496 GNUNET_free (ee);
497 return GNUNET_YES;
498}
499
500
501/**
502 * Clean up after a client has disconnected
503 *
504 * @param cls closure, unused
505 * @param client the client to clean up after
506 * @param internal_cls the `struct ClientState`
507 */
508static void
509client_disconnect_cb (void *cls,
510 struct GNUNET_SERVICE_Client *client,
511 void *internal_cls)
512{
513 struct ClientState *cs = internal_cls;
514 struct Operation *op;
515 struct Listener *listener;
516 struct Set *set;
517
518 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Client disconnected, cleaning up\n");
519 if (NULL != (set = cs->set))
520 {
521 struct SetContent *content = set->content;
522 struct PendingMutation *pm;
523 struct PendingMutation *pm_current;
524 struct LazyCopyRequest *lcr;
525
526 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Destroying client's set\n");
527 /* Destroy pending set operations */
528 while (NULL != set->ops_head)
529 _GSS_operation_destroy (set->ops_head, GNUNET_NO);
530
531 /* Destroy operation-specific state */
532 GNUNET_assert (NULL != set->state);
533 set->vt->destroy_set (set->state);
534 set->state = NULL;
535
536 /* Clean up ongoing iterations */
537 if (NULL != set->iter)
538 {
539 GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
540 set->iter = NULL;
541 set->iteration_id++;
542 }
543
544 /* discard any pending mutations that reference this set */
545 pm = content->pending_mutations_head;
546 while (NULL != pm)
547 {
548 pm_current = pm;
549 pm = pm->next;
550 if (pm_current->set == set)
551 {
552 GNUNET_CONTAINER_DLL_remove (content->pending_mutations_head,
553 content->pending_mutations_tail,
554 pm_current);
555 GNUNET_free (pm_current);
556 }
557 }
558
559 /* free set content (or at least decrement RC) */
560 set->content = NULL;
561 GNUNET_assert (0 != content->refcount);
562 content->refcount--;
563 if (0 == content->refcount)
564 {
565 GNUNET_assert (NULL != content->elements);
566 GNUNET_CONTAINER_multihashmap_iterate (content->elements,
567 &destroy_elements_iterator,
568 NULL);
569 GNUNET_CONTAINER_multihashmap_destroy (content->elements);
570 content->elements = NULL;
571 GNUNET_free (content);
572 }
573 GNUNET_free (set->excluded_generations);
574 set->excluded_generations = NULL;
575
576 /* remove set from pending copy requests */
577 lcr = lazy_copy_head;
578 while (NULL != lcr)
579 {
580 struct LazyCopyRequest *lcr_current = lcr;
581
582 lcr = lcr->next;
583 if (lcr_current->source_set == set)
584 {
585 GNUNET_CONTAINER_DLL_remove (lazy_copy_head,
586 lazy_copy_tail,
587 lcr_current);
588 GNUNET_free (lcr_current);
589 }
590 }
591 GNUNET_free (set);
592 }
593
594 if (NULL != (listener = cs->listener))
595 {
596 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Destroying client's listener\n");
597 GNUNET_CADET_close_port (listener->open_port);
598 listener->open_port = NULL;
599 while (NULL != (op = listener->op_head))
600 {
601 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
602 "Destroying incoming operation `%u' from peer `%s'\n",
603 (unsigned int) op->client_request_id,
604 GNUNET_i2s (&op->peer));
605 incoming_destroy (op);
606 }
607 GNUNET_CONTAINER_DLL_remove (listener_head, listener_tail, listener);
608 GNUNET_free (listener);
609 }
610 GNUNET_free (cs);
611 num_clients--;
612 if ((GNUNET_YES == in_shutdown) && (0 == num_clients))
613 {
614 if (NULL != cadet)
615 {
616 GNUNET_CADET_disconnect (cadet);
617 cadet = NULL;
618 }
619 }
620}
621
622
623/**
624 * Check a request for a set operation from another peer.
625 *
626 * @param cls the operation state
627 * @param msg the received message
628 * @return #GNUNET_OK if the channel should be kept alive,
629 * #GNUNET_SYSERR to destroy the channel
630 */
631static int
632check_incoming_msg (void *cls, const struct OperationRequestMessage *msg)
633{
634 struct Operation *op = cls;
635 struct Listener *listener = op->listener;
636 const struct GNUNET_MessageHeader *nested_context;
637
638 /* double operation request */
639 if (0 != op->suggest_id)
640 {
641 GNUNET_break_op (0);
642 return GNUNET_SYSERR;
643 }
644 /* This should be equivalent to the previous condition, but can't hurt to check twice */
645 if (NULL == op->listener)
646 {
647 GNUNET_break (0);
648 return GNUNET_SYSERR;
649 }
650 if (listener->operation !=
651 (enum GNUNET_SET_OperationType) ntohl (msg->operation))
652 {
653 GNUNET_break_op (0);
654 return GNUNET_SYSERR;
655 }
656 nested_context = GNUNET_MQ_extract_nested_mh (msg);
657 if ((NULL != nested_context) &&
658 (ntohs (nested_context->size) > GNUNET_SET_CONTEXT_MESSAGE_MAX_SIZE))
659 {
660 GNUNET_break_op (0);
661 return GNUNET_SYSERR;
662 }
663 return GNUNET_OK;
664}
665
666
667/**
668 * Handle a request for a set operation from another peer. Checks if we
669 * have a listener waiting for such a request (and in that case initiates
670 * asking the listener about accepting the connection). If no listener
671 * is waiting, we queue the operation request in hope that a listener
672 * shows up soon (before timeout).
673 *
674 * This msg is expected as the first and only msg handled through the
675 * non-operation bound virtual table, acceptance of this operation replaces
676 * our virtual table and subsequent msgs would be routed differently (as
677 * we then know what type of operation this is).
678 *
679 * @param cls the operation state
680 * @param msg the received message
681 * @return #GNUNET_OK if the channel should be kept alive,
682 * #GNUNET_SYSERR to destroy the channel
683 */
684static void
685handle_incoming_msg (void *cls, const struct OperationRequestMessage *msg)
686{
687 struct Operation *op = cls;
688 struct Listener *listener = op->listener;
689 const struct GNUNET_MessageHeader *nested_context;
690 struct GNUNET_MQ_Envelope *env;
691 struct GNUNET_SET_RequestMessage *cmsg;
692
693 nested_context = GNUNET_MQ_extract_nested_mh (msg);
694 /* Make a copy of the nested_context (application-specific context
695 information that is opaque to set) so we can pass it to the
696 listener later on */
697 if (NULL != nested_context)
698 op->context_msg = GNUNET_copy_message (nested_context);
699 op->remote_element_count = ntohl (msg->element_count);
700 GNUNET_log (
701 GNUNET_ERROR_TYPE_DEBUG,
702 "Received P2P operation request (op %u, port %s) for active listener\n",
703 (uint32_t) ntohl (msg->operation),
704 GNUNET_h2s (&op->listener->app_id));
705 GNUNET_assert (0 == op->suggest_id);
706 if (0 == suggest_id)
707 suggest_id++;
708 op->suggest_id = suggest_id++;
709 GNUNET_assert (NULL != op->timeout_task);
710 GNUNET_SCHEDULER_cancel (op->timeout_task);
711 op->timeout_task = NULL;
712 env = GNUNET_MQ_msg_nested_mh (cmsg,
713 GNUNET_MESSAGE_TYPE_SET_REQUEST,
714 op->context_msg);
715 GNUNET_log (
716 GNUNET_ERROR_TYPE_DEBUG,
717 "Suggesting incoming request with accept id %u to listener %p of client %p\n",
718 op->suggest_id,
719 listener,
720 listener->cs);
721 cmsg->accept_id = htonl (op->suggest_id);
722 cmsg->peer_id = op->peer;
723 GNUNET_MQ_send (listener->cs->mq, env);
724 /* NOTE: GNUNET_CADET_receive_done() will be called in
725 #handle_client_accept() */
726}
727
728
729/**
730 * Add an element to @a set as specified by @a msg
731 *
732 * @param set set to manipulate
733 * @param msg message specifying the change
734 */
735static void
736execute_add (struct Set *set, const struct GNUNET_SET_ElementMessage *msg)
737{
738 struct GNUNET_SET_Element el;
739 struct ElementEntry *ee;
740 struct GNUNET_HashCode hash;
741
742 GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_ADD == ntohs (msg->header.type));
743 el.size = ntohs (msg->header.size) - sizeof(*msg);
744 el.data = &msg[1];
745 el.element_type = ntohs (msg->element_type);
746 GNUNET_SET_element_hash (&el, &hash);
747 ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements, &hash);
748 if (NULL == ee)
749 {
750 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
751 "Client inserts element %s of size %u\n",
752 GNUNET_h2s (&hash),
753 el.size);
754 ee = GNUNET_malloc (el.size + sizeof(*ee));
755 ee->element.size = el.size;
756 GNUNET_memcpy (&ee[1], el.data, el.size);
757 ee->element.data = &ee[1];
758 ee->element.element_type = el.element_type;
759 ee->remote = GNUNET_NO;
760 ee->mutations = NULL;
761 ee->mutations_size = 0;
762 ee->element_hash = hash;
763 GNUNET_break (GNUNET_YES ==
764 GNUNET_CONTAINER_multihashmap_put (
765 set->content->elements,
766 &ee->element_hash,
767 ee,
768 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
769 }
770 else if (GNUNET_YES ==
771 is_element_of_generation (ee,
772 set->current_generation,
773 set->excluded_generations,
774 set->excluded_generations_size))
775 {
776 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
777 "Client inserted element %s of size %u twice (ignored)\n",
778 GNUNET_h2s (&hash),
779 el.size);
780
781 /* same element inserted twice */
782 return;
783 }
784
785 {
786 struct MutationEvent mut = { .generation = set->current_generation,
787 .added = GNUNET_YES };
788 GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
789 }
790 set->vt->add (set->state, ee);
791}
792
793
794/**
795 * Remove an element from @a set as specified by @a msg
796 *
797 * @param set set to manipulate
798 * @param msg message specifying the change
799 */
800static void
801execute_remove (struct Set *set, const struct GNUNET_SET_ElementMessage *msg)
802{
803 struct GNUNET_SET_Element el;
804 struct ElementEntry *ee;
805 struct GNUNET_HashCode hash;
806
807 GNUNET_assert (GNUNET_MESSAGE_TYPE_SET_REMOVE == ntohs (msg->header.type));
808 el.size = ntohs (msg->header.size) - sizeof(*msg);
809 el.data = &msg[1];
810 el.element_type = ntohs (msg->element_type);
811 GNUNET_SET_element_hash (&el, &hash);
812 ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements, &hash);
813 if (NULL == ee)
814 {
815 /* Client tried to remove non-existing element. */
816 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
817 "Client removes non-existing element of size %u\n",
818 el.size);
819 return;
820 }
821 if (GNUNET_NO == is_element_of_generation (ee,
822 set->current_generation,
823 set->excluded_generations,
824 set->excluded_generations_size))
825 {
826 /* Client tried to remove element twice */
827 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
828 "Client removed element of size %u twice (ignored)\n",
829 el.size);
830 return;
831 }
832 else
833 {
834 struct MutationEvent mut = { .generation = set->current_generation,
835 .added = GNUNET_NO };
836
837 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
838 "Client removes element of size %u\n",
839 el.size);
840
841 GNUNET_array_append (ee->mutations, ee->mutations_size, mut);
842 }
843 set->vt->remove (set->state, ee);
844}
845
846
847/**
848 * Perform a mutation on a set as specified by the @a msg
849 *
850 * @param set the set to mutate
851 * @param msg specification of what to change
852 */
853static void
854execute_mutation (struct Set *set, const struct GNUNET_SET_ElementMessage *msg)
855{
856 switch (ntohs (msg->header.type))
857 {
858 case GNUNET_MESSAGE_TYPE_SET_ADD:
859 execute_add (set, msg);
860 break;
861
862 case GNUNET_MESSAGE_TYPE_SET_REMOVE:
863 execute_remove (set, msg);
864 break;
865
866 default:
867 GNUNET_break (0);
868 }
869}
870
871
872/**
873 * Execute mutations that were delayed on a set because of
874 * pending operations.
875 *
876 * @param set the set to execute mutations on
877 */
878static void
879execute_delayed_mutations (struct Set *set)
880{
881 struct PendingMutation *pm;
882
883 if (0 != set->content->iterator_count)
884 return; /* still cannot do this */
885 while (NULL != (pm = set->content->pending_mutations_head))
886 {
887 GNUNET_CONTAINER_DLL_remove (set->content->pending_mutations_head,
888 set->content->pending_mutations_tail,
889 pm);
890 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
891 "Executing pending mutation on %p.\n",
892 pm->set);
893 execute_mutation (pm->set, pm->msg);
894 GNUNET_free (pm->msg);
895 GNUNET_free (pm);
896 }
897}
898
899
900/**
901 * Send the next element of a set to the set's client. The next element is given by
902 * the set's current hashmap iterator. The set's iterator will be set to NULL if there
903 * are no more elements in the set. The caller must ensure that the set's iterator is
904 * valid.
905 *
906 * The client will acknowledge each received element with a
907 * #GNUNET_MESSAGE_TYPE_SET_ITER_ACK message. Our
908 * #handle_client_iter_ack() will then trigger the next transmission.
909 * Note that the #GNUNET_MESSAGE_TYPE_SET_ITER_DONE is not acknowledged.
910 *
911 * @param set set that should send its next element to its client
912 */
913static void
914send_client_element (struct Set *set)
915{
916 int ret;
917 struct ElementEntry *ee;
918 struct GNUNET_MQ_Envelope *ev;
919 struct GNUNET_SET_IterResponseMessage *msg;
920
921 GNUNET_assert (NULL != set->iter);
922 do
923 {
924 ret = GNUNET_CONTAINER_multihashmap_iterator_next (set->iter,
925 NULL,
926 (const void **) &ee);
927 if (GNUNET_NO == ret)
928 {
929 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Iteration on %p done.\n", set);
930 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_DONE);
931 GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
932 set->iter = NULL;
933 set->iteration_id++;
934 GNUNET_assert (set->content->iterator_count > 0);
935 set->content->iterator_count--;
936 execute_delayed_mutations (set);
937 GNUNET_MQ_send (set->cs->mq, ev);
938 return;
939 }
940 GNUNET_assert (NULL != ee);
941 }
942 while (GNUNET_NO ==
943 is_element_of_generation (ee,
944 set->iter_generation,
945 set->excluded_generations,
946 set->excluded_generations_size));
947 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
948 "Sending iteration element on %p.\n",
949 set);
950 ev = GNUNET_MQ_msg_extra (msg,
951 ee->element.size,
952 GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT);
953 GNUNET_memcpy (&msg[1], ee->element.data, ee->element.size);
954 msg->element_type = htons (ee->element.element_type);
955 msg->iteration_id = htons (set->iteration_id);
956 GNUNET_MQ_send (set->cs->mq, ev);
957}
958
959
960/**
961 * Called when a client wants to iterate the elements of a set.
962 * Checks if we have a set associated with the client and if we
963 * can right now start an iteration. If all checks out, starts
964 * sending the elements of the set to the client.
965 *
966 * @param cls client that sent the message
967 * @param m message sent by the client
968 */
969static void
970handle_client_iterate (void *cls, const struct GNUNET_MessageHeader *m)
971{
972 struct ClientState *cs = cls;
973 struct Set *set;
974
975 if (NULL == (set = cs->set))
976 {
977 /* attempt to iterate over a non existing set */
978 GNUNET_break (0);
979 GNUNET_SERVICE_client_drop (cs->client);
980 return;
981 }
982 if (NULL != set->iter)
983 {
984 /* Only one concurrent iterate-action allowed per set */
985 GNUNET_break (0);
986 GNUNET_SERVICE_client_drop (cs->client);
987 return;
988 }
989 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
990 "Iterating set %p in gen %u with %u content elements\n",
991 (void *) set,
992 set->current_generation,
993 GNUNET_CONTAINER_multihashmap_size (set->content->elements));
994 GNUNET_SERVICE_client_continue (cs->client);
995 set->content->iterator_count++;
996 set->iter =
997 GNUNET_CONTAINER_multihashmap_iterator_create (set->content->elements);
998 set->iter_generation = set->current_generation;
999 send_client_element (set);
1000}
1001
1002
1003/**
1004 * Called when a client wants to create a new set. This is typically
1005 * the first request from a client, and includes the type of set
1006 * operation to be performed.
1007 *
1008 * @param cls client that sent the message
1009 * @param m message sent by the client
1010 */
1011static void
1012handle_client_create_set (void *cls, const struct GNUNET_SET_CreateMessage *msg)
1013{
1014 struct ClientState *cs = cls;
1015 struct Set *set;
1016
1017 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1018 "Client created new set (operation %u)\n",
1019 (uint32_t) ntohl (msg->operation));
1020 if (NULL != cs->set)
1021 {
1022 /* There can only be one set per client */
1023 GNUNET_break (0);
1024 GNUNET_SERVICE_client_drop (cs->client);
1025 return;
1026 }
1027 set = GNUNET_new (struct Set);
1028 switch (ntohl (msg->operation))
1029 {
1030 case GNUNET_SET_OPERATION_INTERSECTION:
1031 set->vt = _GSS_intersection_vt ();
1032 break;
1033
1034 case GNUNET_SET_OPERATION_UNION:
1035 set->vt = _GSS_union_vt ();
1036 break;
1037
1038 default:
1039 GNUNET_free (set);
1040 GNUNET_break (0);
1041 GNUNET_SERVICE_client_drop (cs->client);
1042 return;
1043 }
1044 set->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
1045 set->state = set->vt->create ();
1046 if (NULL == set->state)
1047 {
1048 /* initialization failed (i.e. out of memory) */
1049 GNUNET_free (set);
1050 GNUNET_SERVICE_client_drop (cs->client);
1051 return;
1052 }
1053 set->content = GNUNET_new (struct SetContent);
1054 set->content->refcount = 1;
1055 set->content->elements = GNUNET_CONTAINER_multihashmap_create (1, GNUNET_YES);
1056 set->cs = cs;
1057 cs->set = set;
1058 GNUNET_SERVICE_client_continue (cs->client);
1059}
1060
1061
1062/**
1063 * Timeout happens iff:
1064 * - we suggested an operation to our listener,
1065 * but did not receive a response in time
1066 * - we got the channel from a peer but no #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
1067 *
1068 * @param cls channel context
1069 * @param tc context information (why was this task triggered now)
1070 */
1071static void
1072incoming_timeout_cb (void *cls)
1073{
1074 struct Operation *op = cls;
1075
1076 op->timeout_task = NULL;
1077 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1078 "Remote peer's incoming request timed out\n");
1079 incoming_destroy (op);
1080}
1081
1082
1083/**
1084 * Method called whenever another peer has added us to a channel the
1085 * other peer initiated. Only called (once) upon reception of data
1086 * from a channel we listen on.
1087 *
1088 * The channel context represents the operation itself and gets added
1089 * to a DLL, from where it gets looked up when our local listener
1090 * client responds to a proposed/suggested operation or connects and
1091 * associates with this operation.
1092 *
1093 * @param cls closure
1094 * @param channel new handle to the channel
1095 * @param source peer that started the channel
1096 * @return initial channel context for the channel
1097 * returns NULL on error
1098 */
1099static void *
1100channel_new_cb (void *cls,
1101 struct GNUNET_CADET_Channel *channel,
1102 const struct GNUNET_PeerIdentity *source)
1103{
1104 struct Listener *listener = cls;
1105 struct Operation *op;
1106
1107 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "New incoming channel\n");
1108 op = GNUNET_new (struct Operation);
1109 op->listener = listener;
1110 op->peer = *source;
1111 op->channel = channel;
1112 op->mq = GNUNET_CADET_get_mq (op->channel);
1113 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
1114 op->timeout_task = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
1115 &incoming_timeout_cb,
1116 op);
1117 GNUNET_CONTAINER_DLL_insert (listener->op_head, listener->op_tail, op);
1118 return op;
1119}
1120
1121
1122/**
1123 * Function called whenever a channel is destroyed. Should clean up
1124 * any associated state. It must NOT call
1125 * GNUNET_CADET_channel_destroy() on the channel.
1126 *
1127 * The peer_disconnect function is part of a a virtual table set initially either
1128 * when a peer creates a new channel with us, or once we create
1129 * a new channel ourselves (evaluate).
1130 *
1131 * Once we know the exact type of operation (union/intersection), the vt is
1132 * replaced with an operation specific instance (_GSS_[op]_vt).
1133 *
1134 * @param channel_ctx place where local state associated
1135 * with the channel is stored
1136 * @param channel connection to the other end (henceforth invalid)
1137 */
1138static void
1139channel_end_cb (void *channel_ctx, const struct GNUNET_CADET_Channel *channel)
1140{
1141 struct Operation *op = channel_ctx;
1142
1143 op->channel = NULL;
1144 _GSS_operation_destroy2 (op);
1145}
1146
1147
1148/**
1149 * This function probably should not exist
1150 * and be replaced by inlining more specific
1151 * logic in the various places where it is called.
1152 */
1153void
1154_GSS_operation_destroy2 (struct Operation *op)
1155{
1156 struct GNUNET_CADET_Channel *channel;
1157
1158 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "channel_end_cb called\n");
1159 if (NULL != (channel = op->channel))
1160 {
1161 /* This will free op; called conditionally as this helper function
1162 is also called from within the channel disconnect handler. */
1163 op->channel = NULL;
1164 GNUNET_CADET_channel_destroy (channel);
1165 }
1166 if (NULL != op->listener)
1167 {
1168 incoming_destroy (op);
1169 return;
1170 }
1171 if (NULL != op->set)
1172 op->set->vt->channel_death (op);
1173 else
1174 _GSS_operation_destroy (op, GNUNET_YES);
1175 GNUNET_free (op);
1176}
1177
1178
1179/**
1180 * Function called whenever an MQ-channel's transmission window size changes.
1181 *
1182 * The first callback in an outgoing channel will be with a non-zero value
1183 * and will mean the channel is connected to the destination.
1184 *
1185 * For an incoming channel it will be called immediately after the
1186 * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
1187 *
1188 * @param cls Channel closure.
1189 * @param channel Connection to the other end (henceforth invalid).
1190 * @param window_size New window size. If the is more messages than buffer size
1191 * this value will be negative..
1192 */
1193static void
1194channel_window_cb (void *cls,
1195 const struct GNUNET_CADET_Channel *channel,
1196 int window_size)
1197{
1198 /* FIXME: not implemented, we could do flow control here... */
1199}
1200
1201
1202/**
1203 * Called when a client wants to create a new listener.
1204 *
1205 * @param cls client that sent the message
1206 * @param msg message sent by the client
1207 */
1208static void
1209handle_client_listen (void *cls, const struct GNUNET_SET_ListenMessage *msg)
1210{
1211 struct ClientState *cs = cls;
1212 struct GNUNET_MQ_MessageHandler cadet_handlers[] =
1213 { GNUNET_MQ_hd_var_size (incoming_msg,
1214 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
1215 struct OperationRequestMessage,
1216 NULL),
1217 GNUNET_MQ_hd_var_size (union_p2p_ibf,
1218 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
1219 struct IBFMessage,
1220 NULL),
1221 GNUNET_MQ_hd_var_size (union_p2p_elements,
1222 GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
1223 struct GNUNET_SET_ElementMessage,
1224 NULL),
1225 GNUNET_MQ_hd_var_size (union_p2p_offer,
1226 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
1227 struct GNUNET_MessageHeader,
1228 NULL),
1229 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
1230 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
1231 struct InquiryMessage,
1232 NULL),
1233 GNUNET_MQ_hd_var_size (union_p2p_demand,
1234 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
1235 struct GNUNET_MessageHeader,
1236 NULL),
1237 GNUNET_MQ_hd_fixed_size (union_p2p_done,
1238 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
1239 struct GNUNET_MessageHeader,
1240 NULL),
1241 GNUNET_MQ_hd_fixed_size (union_p2p_over,
1242 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OVER,
1243 struct GNUNET_MessageHeader,
1244 NULL),
1245 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
1246 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
1247 struct GNUNET_MessageHeader,
1248 NULL),
1249 GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
1250 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
1251 struct GNUNET_MessageHeader,
1252 NULL),
1253 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
1254 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
1255 struct StrataEstimatorMessage,
1256 NULL),
1257 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
1258 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
1259 struct StrataEstimatorMessage,
1260 NULL),
1261 GNUNET_MQ_hd_var_size (union_p2p_full_element,
1262 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
1263 struct GNUNET_SET_ElementMessage,
1264 NULL),
1265 GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
1266 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
1267 struct IntersectionElementInfoMessage,
1268 NULL),
1269 GNUNET_MQ_hd_var_size (intersection_p2p_bf,
1270 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
1271 struct BFMessage,
1272 NULL),
1273 GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
1274 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
1275 struct IntersectionDoneMessage,
1276 NULL),
1277 GNUNET_MQ_handler_end () };
1278 struct Listener *listener;
1279
1280 if (NULL != cs->listener)
1281 {
1282 /* max. one active listener per client! */
1283 GNUNET_break (0);
1284 GNUNET_SERVICE_client_drop (cs->client);
1285 return;
1286 }
1287 listener = GNUNET_new (struct Listener);
1288 listener->cs = cs;
1289 cs->listener = listener;
1290 listener->app_id = msg->app_id;
1291 listener->operation = (enum GNUNET_SET_OperationType) ntohl (msg->operation);
1292 GNUNET_CONTAINER_DLL_insert (listener_head, listener_tail, listener);
1293 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1294 "New listener created (op %u, port %s)\n",
1295 listener->operation,
1296 GNUNET_h2s (&listener->app_id));
1297 listener->open_port = GNUNET_CADET_open_port (cadet,
1298 &msg->app_id,
1299 &channel_new_cb,
1300 listener,
1301 &channel_window_cb,
1302 &channel_end_cb,
1303 cadet_handlers);
1304 GNUNET_SERVICE_client_continue (cs->client);
1305}
1306
1307
1308/**
1309 * Called when the listening client rejects an operation
1310 * request by another peer.
1311 *
1312 * @param cls client that sent the message
1313 * @param msg message sent by the client
1314 */
1315static void
1316handle_client_reject (void *cls, const struct GNUNET_SET_RejectMessage *msg)
1317{
1318 struct ClientState *cs = cls;
1319 struct Operation *op;
1320
1321 op = get_incoming (ntohl (msg->accept_reject_id));
1322 if (NULL == op)
1323 {
1324 /* no matching incoming operation for this reject;
1325 could be that the other peer already disconnected... */
1326 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1327 "Client rejected unknown operation %u\n",
1328 (unsigned int) ntohl (msg->accept_reject_id));
1329 GNUNET_SERVICE_client_continue (cs->client);
1330 return;
1331 }
1332 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1333 "Peer request (op %u, app %s) rejected by client\n",
1334 op->listener->operation,
1335 GNUNET_h2s (&cs->listener->app_id));
1336 _GSS_operation_destroy2 (op);
1337 GNUNET_SERVICE_client_continue (cs->client);
1338}
1339
1340
1341/**
1342 * Called when a client wants to add or remove an element to a set it inhabits.
1343 *
1344 * @param cls client that sent the message
1345 * @param msg message sent by the client
1346 */
1347static int
1348check_client_mutation (void *cls, const struct GNUNET_SET_ElementMessage *msg)
1349{
1350 /* NOTE: Technically, we should probably check with the
1351 block library whether the element we are given is well-formed */
1352 return GNUNET_OK;
1353}
1354
1355
1356/**
1357 * Called when a client wants to add or remove an element to a set it inhabits.
1358 *
1359 * @param cls client that sent the message
1360 * @param msg message sent by the client
1361 */
1362static void
1363handle_client_mutation (void *cls, const struct GNUNET_SET_ElementMessage *msg)
1364{
1365 struct ClientState *cs = cls;
1366 struct Set *set;
1367
1368 if (NULL == (set = cs->set))
1369 {
1370 /* client without a set requested an operation */
1371 GNUNET_break (0);
1372 GNUNET_SERVICE_client_drop (cs->client);
1373 return;
1374 }
1375 GNUNET_SERVICE_client_continue (cs->client);
1376
1377 if (0 != set->content->iterator_count)
1378 {
1379 struct PendingMutation *pm;
1380
1381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Scheduling mutation on set\n");
1382 pm = GNUNET_new (struct PendingMutation);
1383 pm->msg =
1384 (struct GNUNET_SET_ElementMessage *) GNUNET_copy_message (&msg->header);
1385 pm->set = set;
1386 GNUNET_CONTAINER_DLL_insert_tail (set->content->pending_mutations_head,
1387 set->content->pending_mutations_tail,
1388 pm);
1389 return;
1390 }
1391 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing mutation on set\n");
1392 execute_mutation (set, msg);
1393}
1394
1395
1396/**
1397 * Advance the current generation of a set,
1398 * adding exclusion ranges if necessary.
1399 *
1400 * @param set the set where we want to advance the generation
1401 */
1402static void
1403advance_generation (struct Set *set)
1404{
1405 struct GenerationRange r;
1406
1407 if (set->current_generation == set->content->latest_generation)
1408 {
1409 set->content->latest_generation++;
1410 set->current_generation++;
1411 return;
1412 }
1413
1414 GNUNET_assert (set->current_generation < set->content->latest_generation);
1415
1416 r.start = set->current_generation + 1;
1417 r.end = set->content->latest_generation + 1;
1418 set->content->latest_generation = r.end;
1419 set->current_generation = r.end;
1420 GNUNET_array_append (set->excluded_generations,
1421 set->excluded_generations_size,
1422 r);
1423}
1424
1425
1426/**
1427 * Called when a client wants to initiate a set operation with another
1428 * peer. Initiates the CADET connection to the listener and sends the
1429 * request.
1430 *
1431 * @param cls client that sent the message
1432 * @param msg message sent by the client
1433 * @return #GNUNET_OK if the message is well-formed
1434 */
1435static int
1436check_client_evaluate (void *cls, const struct GNUNET_SET_EvaluateMessage *msg)
1437{
1438 /* FIXME: suboptimal, even if the context below could be NULL,
1439 there are malformed messages this does not check for... */
1440 return GNUNET_OK;
1441}
1442
1443
1444/**
1445 * Called when a client wants to initiate a set operation with another
1446 * peer. Initiates the CADET connection to the listener and sends the
1447 * request.
1448 *
1449 * @param cls client that sent the message
1450 * @param msg message sent by the client
1451 */
1452static void
1453handle_client_evaluate (void *cls, const struct GNUNET_SET_EvaluateMessage *msg)
1454{
1455 struct ClientState *cs = cls;
1456 struct Operation *op = GNUNET_new (struct Operation);
1457 const struct GNUNET_MQ_MessageHandler cadet_handlers[] =
1458 { GNUNET_MQ_hd_var_size (incoming_msg,
1459 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
1460 struct OperationRequestMessage,
1461 op),
1462 GNUNET_MQ_hd_var_size (union_p2p_ibf,
1463 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF,
1464 struct IBFMessage,
1465 op),
1466 GNUNET_MQ_hd_var_size (union_p2p_elements,
1467 GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS,
1468 struct GNUNET_SET_ElementMessage,
1469 op),
1470 GNUNET_MQ_hd_var_size (union_p2p_offer,
1471 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER,
1472 struct GNUNET_MessageHeader,
1473 op),
1474 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
1475 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY,
1476 struct InquiryMessage,
1477 op),
1478 GNUNET_MQ_hd_var_size (union_p2p_demand,
1479 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND,
1480 struct GNUNET_MessageHeader,
1481 op),
1482 GNUNET_MQ_hd_fixed_size (union_p2p_done,
1483 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE,
1484 struct GNUNET_MessageHeader,
1485 op),
1486 GNUNET_MQ_hd_fixed_size (union_p2p_over,
1487 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OVER,
1488 struct GNUNET_MessageHeader,
1489 op),
1490 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
1491 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE,
1492 struct GNUNET_MessageHeader,
1493 op),
1494 GNUNET_MQ_hd_fixed_size (union_p2p_request_full,
1495 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL,
1496 struct GNUNET_MessageHeader,
1497 op),
1498 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
1499 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE,
1500 struct StrataEstimatorMessage,
1501 op),
1502 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
1503 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC,
1504 struct StrataEstimatorMessage,
1505 op),
1506 GNUNET_MQ_hd_var_size (union_p2p_full_element,
1507 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT,
1508 struct GNUNET_SET_ElementMessage,
1509 op),
1510 GNUNET_MQ_hd_fixed_size (intersection_p2p_element_info,
1511 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO,
1512 struct IntersectionElementInfoMessage,
1513 op),
1514 GNUNET_MQ_hd_var_size (intersection_p2p_bf,
1515 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF,
1516 struct BFMessage,
1517 op),
1518 GNUNET_MQ_hd_fixed_size (intersection_p2p_done,
1519 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE,
1520 struct IntersectionDoneMessage,
1521 op),
1522 GNUNET_MQ_handler_end () };
1523 struct Set *set;
1524 const struct GNUNET_MessageHeader *context;
1525
1526 if (NULL == (set = cs->set))
1527 {
1528 GNUNET_break (0);
1529 GNUNET_free (op);
1530 GNUNET_SERVICE_client_drop (cs->client);
1531 return;
1532 }
1533 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
1534 op->peer = msg->target_peer;
1535 op->result_mode = ntohl (msg->result_mode);
1536 op->client_request_id = ntohl (msg->request_id);
1537 op->byzantine = msg->byzantine;
1538 op->byzantine_lower_bound = msg->byzantine_lower_bound;
1539 op->force_full = msg->force_full;
1540 op->force_delta = msg->force_delta;
1541 context = GNUNET_MQ_extract_nested_mh (msg);
1542
1543 /* Advance generation values, so that
1544 mutations won't interfer with the running operation. */
1545 op->set = set;
1546 op->generation_created = set->current_generation;
1547 advance_generation (set);
1548 GNUNET_CONTAINER_DLL_insert (set->ops_head, set->ops_tail, op);
1549 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1550 "Creating new CADET channel to port %s for set operation type %u\n",
1551 GNUNET_h2s (&msg->app_id),
1552 set->operation);
1553 op->channel = GNUNET_CADET_channel_create (cadet,
1554 op,
1555 &msg->target_peer,
1556 &msg->app_id,
1557 &channel_window_cb,
1558 &channel_end_cb,
1559 cadet_handlers);
1560 op->mq = GNUNET_CADET_get_mq (op->channel);
1561 op->state = set->vt->evaluate (op, context);
1562 if (NULL == op->state)
1563 {
1564 GNUNET_break (0);
1565 GNUNET_SERVICE_client_drop (cs->client);
1566 return;
1567 }
1568 GNUNET_SERVICE_client_continue (cs->client);
1569}
1570
1571
1572/**
1573 * Handle an ack from a client, and send the next element. Note
1574 * that we only expect acks for set elements, not after the
1575 * #GNUNET_MESSAGE_TYPE_SET_ITER_DONE message.
1576 *
1577 * @param cls client the client
1578 * @param ack the message
1579 */
1580static void
1581handle_client_iter_ack (void *cls, const struct GNUNET_SET_IterAckMessage *ack)
1582{
1583 struct ClientState *cs = cls;
1584 struct Set *set;
1585
1586 if (NULL == (set = cs->set))
1587 {
1588 /* client without a set acknowledged receiving a value */
1589 GNUNET_break (0);
1590 GNUNET_SERVICE_client_drop (cs->client);
1591 return;
1592 }
1593 if (NULL == set->iter)
1594 {
1595 /* client sent an ack, but we were not expecting one (as
1596 set iteration has finished) */
1597 GNUNET_break (0);
1598 GNUNET_SERVICE_client_drop (cs->client);
1599 return;
1600 }
1601 GNUNET_SERVICE_client_continue (cs->client);
1602 if (ntohl (ack->send_more))
1603 {
1604 send_client_element (set);
1605 }
1606 else
1607 {
1608 GNUNET_CONTAINER_multihashmap_iterator_destroy (set->iter);
1609 set->iter = NULL;
1610 set->iteration_id++;
1611 }
1612}
1613
1614
1615/**
1616 * Handle a request from the client to copy a set.
1617 *
1618 * @param cls the client
1619 * @param mh the message
1620 */
1621static void
1622handle_client_copy_lazy_prepare (void *cls,
1623 const struct GNUNET_MessageHeader *mh)
1624{
1625 struct ClientState *cs = cls;
1626 struct Set *set;
1627 struct LazyCopyRequest *cr;
1628 struct GNUNET_MQ_Envelope *ev;
1629 struct GNUNET_SET_CopyLazyResponseMessage *resp_msg;
1630
1631 if (NULL == (set = cs->set))
1632 {
1633 /* client without a set requested an operation */
1634 GNUNET_break (0);
1635 GNUNET_SERVICE_client_drop (cs->client);
1636 return;
1637 }
1638 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1639 "Client requested creation of lazy copy\n");
1640 cr = GNUNET_new (struct LazyCopyRequest);
1641 cr->cookie = ++lazy_copy_cookie;
1642 cr->source_set = set;
1643 GNUNET_CONTAINER_DLL_insert (lazy_copy_head, lazy_copy_tail, cr);
1644 ev = GNUNET_MQ_msg (resp_msg, GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE);
1645 resp_msg->cookie = cr->cookie;
1646 GNUNET_MQ_send (set->cs->mq, ev);
1647 GNUNET_SERVICE_client_continue (cs->client);
1648}
1649
1650
1651/**
1652 * Handle a request from the client to connect to a copy of a set.
1653 *
1654 * @param cls the client
1655 * @param msg the message
1656 */
1657static void
1658handle_client_copy_lazy_connect (
1659 void *cls,
1660 const struct GNUNET_SET_CopyLazyConnectMessage *msg)
1661{
1662 struct ClientState *cs = cls;
1663 struct LazyCopyRequest *cr;
1664 struct Set *set;
1665 int found;
1666
1667 if (NULL != cs->set)
1668 {
1669 /* There can only be one set per client */
1670 GNUNET_break (0);
1671 GNUNET_SERVICE_client_drop (cs->client);
1672 return;
1673 }
1674 found = GNUNET_NO;
1675 for (cr = lazy_copy_head; NULL != cr; cr = cr->next)
1676 {
1677 if (cr->cookie == msg->cookie)
1678 {
1679 found = GNUNET_YES;
1680 break;
1681 }
1682 }
1683 if (GNUNET_NO == found)
1684 {
1685 /* client asked for copy with cookie we don't know */
1686 GNUNET_break (0);
1687 GNUNET_SERVICE_client_drop (cs->client);
1688 return;
1689 }
1690 GNUNET_CONTAINER_DLL_remove (lazy_copy_head, lazy_copy_tail, cr);
1691 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1692 "Client %p requested use of lazy copy\n",
1693 cs);
1694 set = GNUNET_new (struct Set);
1695 switch (cr->source_set->operation)
1696 {
1697 case GNUNET_SET_OPERATION_INTERSECTION:
1698 set->vt = _GSS_intersection_vt ();
1699 break;
1700
1701 case GNUNET_SET_OPERATION_UNION:
1702 set->vt = _GSS_union_vt ();
1703 break;
1704
1705 default:
1706 GNUNET_assert (0);
1707 return;
1708 }
1709
1710 if (NULL == set->vt->copy_state)
1711 {
1712 /* Lazy copy not supported for this set operation */
1713 GNUNET_break (0);
1714 GNUNET_free (set);
1715 GNUNET_free (cr);
1716 GNUNET_SERVICE_client_drop (cs->client);
1717 return;
1718 }
1719
1720 set->operation = cr->source_set->operation;
1721 set->state = set->vt->copy_state (cr->source_set->state);
1722 set->content = cr->source_set->content;
1723 set->content->refcount++;
1724
1725 set->current_generation = cr->source_set->current_generation;
1726 set->excluded_generations_size = cr->source_set->excluded_generations_size;
1727 set->excluded_generations =
1728 GNUNET_memdup (cr->source_set->excluded_generations,
1729 set->excluded_generations_size
1730 * sizeof(struct GenerationRange));
1731
1732 /* Advance the generation of the new set, so that mutations to the
1733 of the cloned set and the source set are independent. */
1734 advance_generation (set);
1735 set->cs = cs;
1736 cs->set = set;
1737 GNUNET_free (cr);
1738 GNUNET_SERVICE_client_continue (cs->client);
1739}
1740
1741
1742/**
1743 * Handle a request from the client to cancel a running set operation.
1744 *
1745 * @param cls the client
1746 * @param msg the message
1747 */
1748static void
1749handle_client_cancel (void *cls, const struct GNUNET_SET_CancelMessage *msg)
1750{
1751 struct ClientState *cs = cls;
1752 struct Set *set;
1753 struct Operation *op;
1754 int found;
1755
1756 if (NULL == (set = cs->set))
1757 {
1758 /* client without a set requested an operation */
1759 GNUNET_break (0);
1760 GNUNET_SERVICE_client_drop (cs->client);
1761 return;
1762 }
1763 found = GNUNET_NO;
1764 for (op = set->ops_head; NULL != op; op = op->next)
1765 {
1766 if (op->client_request_id == ntohl (msg->request_id))
1767 {
1768 found = GNUNET_YES;
1769 break;
1770 }
1771 }
1772 if (GNUNET_NO == found)
1773 {
1774 /* It may happen that the operation was already destroyed due to
1775 * the other peer disconnecting. The client may not know about this
1776 * yet and try to cancel the (just barely non-existent) operation.
1777 * So this is not a hard error.
1778 */GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1779 "Client canceled non-existent op %u\n",
1780 (uint32_t) ntohl (msg->request_id));
1781 }
1782 else
1783 {
1784 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1785 "Client requested cancel for op %u\n",
1786 (uint32_t) ntohl (msg->request_id));
1787 _GSS_operation_destroy (op, GNUNET_YES);
1788 }
1789 GNUNET_SERVICE_client_continue (cs->client);
1790}
1791
1792
1793/**
1794 * Handle a request from the client to accept a set operation that
1795 * came from a remote peer. We forward the accept to the associated
1796 * operation for handling
1797 *
1798 * @param cls the client
1799 * @param msg the message
1800 */
1801static void
1802handle_client_accept (void *cls, const struct GNUNET_SET_AcceptMessage *msg)
1803{
1804 struct ClientState *cs = cls;
1805 struct Set *set;
1806 struct Operation *op;
1807 struct GNUNET_SET_ResultMessage *result_message;
1808 struct GNUNET_MQ_Envelope *ev;
1809 struct Listener *listener;
1810
1811 if (NULL == (set = cs->set))
1812 {
1813 /* client without a set requested to accept */
1814 GNUNET_break (0);
1815 GNUNET_SERVICE_client_drop (cs->client);
1816 return;
1817 }
1818 op = get_incoming (ntohl (msg->accept_reject_id));
1819 if (NULL == op)
1820 {
1821 /* It is not an error if the set op does not exist -- it may
1822 * have been destroyed when the partner peer disconnected. */
1823 GNUNET_log (
1824 GNUNET_ERROR_TYPE_INFO,
1825 "Client %p accepted request %u of listener %p that is no longer active\n",
1826 cs,
1827 ntohl (msg->accept_reject_id),
1828 cs->listener);
1829 ev = GNUNET_MQ_msg (result_message, GNUNET_MESSAGE_TYPE_SET_RESULT);
1830 result_message->request_id = msg->request_id;
1831 result_message->result_status = htons (GNUNET_SET_STATUS_FAILURE);
1832 GNUNET_MQ_send (set->cs->mq, ev);
1833 GNUNET_SERVICE_client_continue (cs->client);
1834 return;
1835 }
1836 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1837 "Client accepting request %u\n",
1838 (uint32_t) ntohl (msg->accept_reject_id));
1839 listener = op->listener;
1840 op->listener = NULL;
1841 GNUNET_CONTAINER_DLL_remove (listener->op_head, listener->op_tail, op);
1842 op->set = set;
1843 GNUNET_CONTAINER_DLL_insert (set->ops_head, set->ops_tail, op);
1844 op->client_request_id = ntohl (msg->request_id);
1845 op->result_mode = ntohl (msg->result_mode);
1846 op->byzantine = msg->byzantine;
1847 op->byzantine_lower_bound = msg->byzantine_lower_bound;
1848 op->force_full = msg->force_full;
1849 op->force_delta = msg->force_delta;
1850
1851 /* Advance generation values, so that future mutations do not
1852 interfer with the running operation. */
1853 op->generation_created = set->current_generation;
1854 advance_generation (set);
1855 GNUNET_assert (NULL == op->state);
1856 op->state = set->vt->accept (op);
1857 if (NULL == op->state)
1858 {
1859 GNUNET_break (0);
1860 GNUNET_SERVICE_client_drop (cs->client);
1861 return;
1862 }
1863 /* Now allow CADET to continue, as we did not do this in
1864 #handle_incoming_msg (as we wanted to first see if the
1865 local client would accept the request). */
1866 GNUNET_CADET_receive_done (op->channel);
1867 GNUNET_SERVICE_client_continue (cs->client);
1868}
1869
1870
1871/**
1872 * Called to clean up, after a shutdown has been requested.
1873 *
1874 * @param cls closure, NULL
1875 */
1876static void
1877shutdown_task (void *cls)
1878{
1879 /* Delay actual shutdown to allow service to disconnect clients */
1880 in_shutdown = GNUNET_YES;
1881 if (0 == num_clients)
1882 {
1883 if (NULL != cadet)
1884 {
1885 GNUNET_CADET_disconnect (cadet);
1886 cadet = NULL;
1887 }
1888 }
1889 GNUNET_STATISTICS_destroy (_GSS_statistics, GNUNET_YES);
1890 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "handled shutdown request\n");
1891}
1892
1893
1894/**
1895 * Function called by the service's run
1896 * method to run service-specific setup code.
1897 *
1898 * @param cls closure
1899 * @param cfg configuration to use
1900 * @param service the initialized service
1901 */
1902static void
1903run (void *cls,
1904 const struct GNUNET_CONFIGURATION_Handle *cfg,
1905 struct GNUNET_SERVICE_Handle *service)
1906{
1907 /* FIXME: need to modify SERVICE (!) API to allow
1908 us to run a shutdown task *after* clients were
1909 forcefully disconnected! */
1910 GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL);
1911 _GSS_statistics = GNUNET_STATISTICS_create ("set", cfg);
1912 cadet = GNUNET_CADET_connect (cfg);
1913 if (NULL == cadet)
1914 {
1915 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1916 _ ("Could not connect to CADET service\n"));
1917 GNUNET_SCHEDULER_shutdown ();
1918 return;
1919 }
1920}
1921
1922
1923/**
1924 * Define "main" method using service macro.
1925 */
1926GNUNET_SERVICE_MAIN (
1927 "set",
1928 GNUNET_SERVICE_OPTION_NONE,
1929 &run,
1930 &client_connect_cb,
1931 &client_disconnect_cb,
1932 NULL,
1933 GNUNET_MQ_hd_fixed_size (client_accept,
1934 GNUNET_MESSAGE_TYPE_SET_ACCEPT,
1935 struct GNUNET_SET_AcceptMessage,
1936 NULL),
1937 GNUNET_MQ_hd_fixed_size (client_iter_ack,
1938 GNUNET_MESSAGE_TYPE_SET_ITER_ACK,
1939 struct GNUNET_SET_IterAckMessage,
1940 NULL),
1941 GNUNET_MQ_hd_var_size (client_mutation,
1942 GNUNET_MESSAGE_TYPE_SET_ADD,
1943 struct GNUNET_SET_ElementMessage,
1944 NULL),
1945 GNUNET_MQ_hd_fixed_size (client_create_set,
1946 GNUNET_MESSAGE_TYPE_SET_CREATE,
1947 struct GNUNET_SET_CreateMessage,
1948 NULL),
1949 GNUNET_MQ_hd_fixed_size (client_iterate,
1950 GNUNET_MESSAGE_TYPE_SET_ITER_REQUEST,
1951 struct GNUNET_MessageHeader,
1952 NULL),
1953 GNUNET_MQ_hd_var_size (client_evaluate,
1954 GNUNET_MESSAGE_TYPE_SET_EVALUATE,
1955 struct GNUNET_SET_EvaluateMessage,
1956 NULL),
1957 GNUNET_MQ_hd_fixed_size (client_listen,
1958 GNUNET_MESSAGE_TYPE_SET_LISTEN,
1959 struct GNUNET_SET_ListenMessage,
1960 NULL),
1961 GNUNET_MQ_hd_fixed_size (client_reject,
1962 GNUNET_MESSAGE_TYPE_SET_REJECT,
1963 struct GNUNET_SET_RejectMessage,
1964 NULL),
1965 GNUNET_MQ_hd_var_size (client_mutation,
1966 GNUNET_MESSAGE_TYPE_SET_REMOVE,
1967 struct GNUNET_SET_ElementMessage,
1968 NULL),
1969 GNUNET_MQ_hd_fixed_size (client_cancel,
1970 GNUNET_MESSAGE_TYPE_SET_CANCEL,
1971 struct GNUNET_SET_CancelMessage,
1972 NULL),
1973 GNUNET_MQ_hd_fixed_size (client_copy_lazy_prepare,
1974 GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE,
1975 struct GNUNET_MessageHeader,
1976 NULL),
1977 GNUNET_MQ_hd_fixed_size (client_copy_lazy_connect,
1978 GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT,
1979 struct GNUNET_SET_CopyLazyConnectMessage,
1980 NULL),
1981 GNUNET_MQ_handler_end ());
1982
1983
1984/* end of gnunet-service-set.c */
diff --git a/src/set/gnunet-service-set.h b/src/set/gnunet-service-set.h
deleted file mode 100644
index abdec7f08..000000000
--- a/src/set/gnunet-service-set.h
+++ /dev/null
@@ -1,665 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set.h
22 * @brief common components for the implementation the different set operations
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef GNUNET_SERVICE_SET_H_PRIVATE
27#define GNUNET_SERVICE_SET_H_PRIVATE
28
29#include "platform.h"
30#include "gnunet_util_lib.h"
31#include "gnunet_protocols.h"
32#include "gnunet_applications.h"
33#include "gnunet_core_service.h"
34#include "gnunet_cadet_service.h"
35#include "gnunet_set_service.h"
36#include "set.h"
37
38
39/**
40 * Implementation-specific set state. Used as opaque pointer, and
41 * specified further in the respective implementation.
42 */
43struct SetState;
44
45/**
46 * Implementation-specific set operation. Used as opaque pointer, and
47 * specified further in the respective implementation.
48 */
49struct OperationState;
50
51/**
52 * A set that supports a specific operation with other peers.
53 */
54struct Set;
55
56/**
57 * Information about an element element in the set. All elements are
58 * stored in a hash-table from their hash-code to their 'struct
59 * Element', so that the remove and add operations are reasonably
60 * fast.
61 */
62struct ElementEntry;
63
64/**
65 * Operation context used to execute a set operation.
66 */
67struct Operation;
68
69
70/**
71 * Signature of functions that create the implementation-specific
72 * state for a set supporting a specific operation.
73 *
74 * @return a set state specific to the supported operation, NULL on error
75 */
76typedef struct SetState *
77(*SetCreateImpl) (void);
78
79
80/**
81 * Signature of functions that implement the add/remove functionality
82 * for a set supporting a specific operation.
83 *
84 * @param set implementation-specific set state
85 * @param ee element message from the client
86 */
87typedef void
88(*SetAddRemoveImpl) (struct SetState *state,
89 struct ElementEntry *ee);
90
91
92/**
93 * Make a copy of a set's internal state.
94 *
95 * @param state set state to copy
96 * @return copy of the internal state
97 */
98typedef struct SetState *
99(*SetCopyStateImpl) (struct SetState *state);
100
101
102/**
103 * Signature of functions that implement the destruction of the
104 * implementation-specific set state.
105 *
106 * @param state the set state, contains implementation-specific data
107 */
108typedef void
109(*SetDestroyImpl) (struct SetState *state);
110
111
112/**
113 * Signature of functions that implement accepting a set operation.
114 *
115 * @param op operation that is created by accepting the operation,
116 * should be initialized by the implementation
117 * @return operation-specific state to keep in @a op
118 */
119typedef struct OperationState *
120(*OpAcceptImpl) (struct Operation *op);
121
122
123/**
124 * Signature of functions that implement starting the evaluation of
125 * set operations.
126 *
127 * @param op operation that is created, should be initialized to
128 * begin the evaluation
129 * @param opaque_context message to be transmitted to the listener
130 * to convince it to accept, may be NULL
131 * @return operation-specific state to keep in @a op
132 */
133typedef struct OperationState *
134(*OpEvaluateImpl) (struct Operation *op,
135 const struct GNUNET_MessageHeader *opaque_context);
136
137/**
138 * Signature of functions that implement operation cancellation.
139 * This includes notifying the client about the operation's final
140 * state.
141 *
142 * @param op operation state
143 */
144typedef void
145(*OpCancelImpl) (struct Operation *op);
146
147
148/**
149 * Signature of functions called when the CADET channel died.
150 *
151 * @param op operation state
152 */
153typedef void
154(*OpChannelDeathImpl) (struct Operation *op);
155
156
157/**
158 * Dispatch table for a specific set operation. Every set operation
159 * has to implement the callback in this struct.
160 */
161struct SetVT
162{
163 /**
164 * Callback for the set creation.
165 */
166 SetCreateImpl create;
167
168 /**
169 * Callback for element insertion
170 */
171 SetAddRemoveImpl add;
172
173 /**
174 * Callback for element removal.
175 */
176 SetAddRemoveImpl remove;
177
178 /**
179 * Callback for making a copy of a set's internal state.
180 */
181 SetCopyStateImpl copy_state;
182
183 /**
184 * Callback for destruction of the set state.
185 */
186 SetDestroyImpl destroy_set;
187
188 /**
189 * Callback for accepting a set operation request
190 */
191 OpAcceptImpl accept;
192
193 /**
194 * Callback for starting evaluation with a remote peer.
195 */
196 OpEvaluateImpl evaluate;
197
198 /**
199 * Callback for canceling an operation.
200 */
201 OpCancelImpl cancel;
202
203 /**
204 * Callback called in case the CADET channel died.
205 */
206 OpChannelDeathImpl channel_death;
207};
208
209
210/**
211 * MutationEvent gives information about changes
212 * to an element (removal / addition) in a set content.
213 */
214struct MutationEvent
215{
216 /**
217 * First generation affected by this mutation event.
218 *
219 * If @a generation is 0, this mutation event is a list
220 * sentinel element.
221 */
222 unsigned int generation;
223
224 /**
225 * If @a added is #GNUNET_YES, then this is a
226 * `remove` event, otherwise it is an `add` event.
227 */
228 int added;
229};
230
231
232/**
233 * Information about an element element in the set. All elements are
234 * stored in a hash-table from their hash-code to their `struct
235 * Element`, so that the remove and add operations are reasonably
236 * fast.
237 */
238struct ElementEntry
239{
240 /**
241 * The actual element. The data for the element
242 * should be allocated at the end of this struct.
243 */
244 struct GNUNET_SET_Element element;
245
246 /**
247 * Hash of the element. For set union: Will be used to derive the
248 * different IBF keys for different salts.
249 */
250 struct GNUNET_HashCode element_hash;
251
252 /**
253 * If @a mutations is not NULL, it contains
254 * a list of mutations, ordered by increasing generation.
255 * The list is terminated by a sentinel event with `generation`
256 * set to 0.
257 *
258 * If @a mutations is NULL, then this element exists in all generations
259 * of the respective set content this element belongs to.
260 */
261 struct MutationEvent *mutations;
262
263 /**
264 * Number of elements in the array @a mutations.
265 */
266 unsigned int mutations_size;
267
268 /**
269 * #GNUNET_YES if the element is a remote element, and does not belong
270 * to the operation's set.
271 */
272 int remote;
273};
274
275
276/**
277 * A listener is inhabited by a client, and waits for evaluation
278 * requests from remote peers.
279 */
280struct Listener;
281
282
283/**
284 * State we keep per client.
285 */
286struct ClientState
287{
288 /**
289 * Set, if associated with the client, otherwise NULL.
290 */
291 struct Set *set;
292
293 /**
294 * Listener, if associated with the client, otherwise NULL.
295 */
296 struct Listener *listener;
297
298 /**
299 * Client handle.
300 */
301 struct GNUNET_SERVICE_Client *client;
302
303 /**
304 * Message queue.
305 */
306 struct GNUNET_MQ_Handle *mq;
307};
308
309
310/**
311 * Operation context used to execute a set operation.
312 */
313struct Operation
314{
315 /**
316 * Kept in a DLL of the listener, if @e listener is non-NULL.
317 */
318 struct Operation *next;
319
320 /**
321 * Kept in a DLL of the listener, if @e listener is non-NULL.
322 */
323 struct Operation *prev;
324
325 /**
326 * Channel to the peer.
327 */
328 struct GNUNET_CADET_Channel *channel;
329
330 /**
331 * Port this operation runs on.
332 */
333 struct Listener *listener;
334
335 /**
336 * Message queue for the channel.
337 */
338 struct GNUNET_MQ_Handle *mq;
339
340 /**
341 * Context message, may be NULL.
342 */
343 struct GNUNET_MessageHeader *context_msg;
344
345 /**
346 * Set associated with the operation, NULL until the spec has been
347 * associated with a set.
348 */
349 struct Set *set;
350
351 /**
352 * Operation-specific operation state. Note that the exact
353 * type depends on this being a union or intersection operation
354 * (and thus on @e vt).
355 */
356 struct OperationState *state;
357
358 /**
359 * The identity of the requesting peer. Needs to
360 * be stored here as the op spec might not have been created yet.
361 */
362 struct GNUNET_PeerIdentity peer;
363
364 /**
365 * Timeout task, if the incoming peer has not been accepted
366 * after the timeout, it will be disconnected.
367 */
368 struct GNUNET_SCHEDULER_Task *timeout_task;
369
370 /**
371 * Salt to use for the operation.
372 */
373 uint32_t salt;
374
375 /**
376 * Remote peers element count
377 */
378 uint32_t remote_element_count;
379
380 /**
381 * ID used to identify an operation between service and client
382 */
383 uint32_t client_request_id;
384
385 /**
386 * When are elements sent to the client, and which elements are sent?
387 */
388 enum GNUNET_SET_ResultMode result_mode;
389
390 /**
391 * Always use delta operation instead of sending full sets,
392 * even it it's less efficient.
393 */
394 int force_delta;
395
396 /**
397 * Always send full sets, even if delta operations would
398 * be more efficient.
399 */
400 int force_full;
401
402 /**
403 * #GNUNET_YES to fail operations where Byzantine faults
404 * are suspected
405 */
406 int byzantine;
407
408 /**
409 * Lower bound for the set size, used only when
410 * byzantine mode is enabled.
411 */
412 int byzantine_lower_bound;
413
414 /**
415 * Unique request id for the request from a remote peer, sent to the
416 * client, which will accept or reject the request. Set to '0' iff
417 * the request has not been suggested yet.
418 */
419 uint32_t suggest_id;
420
421 /**
422 * Generation in which the operation handle
423 * was created.
424 */
425 unsigned int generation_created;
426};
427
428
429/**
430 * SetContent stores the actual set elements, which may be shared by
431 * multiple generations derived from one set.
432 */
433struct SetContent
434{
435 /**
436 * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *`.
437 */
438 struct GNUNET_CONTAINER_MultiHashMap *elements;
439
440 /**
441 * Mutations requested by the client that we're
442 * unable to execute right now because we're iterating
443 * over the underlying hash map of elements.
444 */
445 struct PendingMutation *pending_mutations_head;
446
447 /**
448 * Mutations requested by the client that we're
449 * unable to execute right now because we're iterating
450 * over the underlying hash map of elements.
451 */
452 struct PendingMutation *pending_mutations_tail;
453
454 /**
455 * Number of references to the content.
456 */
457 unsigned int refcount;
458
459 /**
460 * FIXME: document!
461 */
462 unsigned int latest_generation;
463
464 /**
465 * Number of concurrently active iterators.
466 */
467 int iterator_count;
468};
469
470
471struct GenerationRange
472{
473 /**
474 * First generation that is excluded.
475 */
476 unsigned int start;
477
478 /**
479 * Generation after the last excluded generation.
480 */
481 unsigned int end;
482};
483
484
485/**
486 * Information about a mutation to apply to a set.
487 */
488struct PendingMutation
489{
490 /**
491 * Mutations are kept in a DLL.
492 */
493 struct PendingMutation *prev;
494
495 /**
496 * Mutations are kept in a DLL.
497 */
498 struct PendingMutation *next;
499
500 /**
501 * Set this mutation is about.
502 */
503 struct Set *set;
504
505 /**
506 * Message that describes the desired mutation.
507 * May only be a #GNUNET_MESSAGE_TYPE_SET_ADD or
508 * #GNUNET_MESSAGE_TYPE_SET_REMOVE.
509 */
510 struct GNUNET_SET_ElementMessage *msg;
511};
512
513
514/**
515 * A set that supports a specific operation with other peers.
516 */
517struct Set
518{
519 /**
520 * Sets are held in a doubly linked list (in `sets_head` and `sets_tail`).
521 */
522 struct Set *next;
523
524 /**
525 * Sets are held in a doubly linked list.
526 */
527 struct Set *prev;
528
529 /**
530 * Client that owns the set. Only one client may own a set,
531 * and there can only be one set per client.
532 */
533 struct ClientState *cs;
534
535 /**
536 * Content, possibly shared by multiple sets,
537 * and thus reference counted.
538 */
539 struct SetContent *content;
540
541 /**
542 * Virtual table for this set. Determined by the operation type of
543 * this set.
544 *
545 * Used only for Add/remove of elements and when receiving an incoming
546 * operation from a remote peer.
547 */
548 const struct SetVT *vt;
549
550 /**
551 * Implementation-specific state.
552 */
553 struct SetState *state;
554
555 /**
556 * Current state of iterating elements for the client.
557 * NULL if we are not currently iterating.
558 */
559 struct GNUNET_CONTAINER_MultiHashMapIterator *iter;
560
561 /**
562 * Evaluate operations are held in a linked list.
563 */
564 struct Operation *ops_head;
565
566 /**
567 * Evaluate operations are held in a linked list.
568 */
569 struct Operation *ops_tail;
570
571 /**
572 * List of generations we have to exclude, due to lazy copies.
573 */
574 struct GenerationRange *excluded_generations;
575
576 /**
577 * Current generation, that is, number of previously executed
578 * operations and lazy copies on the underlying set content.
579 */
580 unsigned int current_generation;
581
582 /**
583 * Number of elements in array @a excluded_generations.
584 */
585 unsigned int excluded_generations_size;
586
587 /**
588 * Type of operation supported for this set
589 */
590 enum GNUNET_SET_OperationType operation;
591
592 /**
593 * Generation we're currently iteration over.
594 */
595 unsigned int iter_generation;
596
597 /**
598 * Each @e iter is assigned a unique number, so that the client
599 * can distinguish iterations.
600 */
601 uint16_t iteration_id;
602};
603
604
605extern struct GNUNET_STATISTICS_Handle *_GSS_statistics;
606
607
608/**
609 * Destroy the given operation. Used for any operation where both
610 * peers were known and that thus actually had a vt and channel. Must
611 * not be used for operations where 'listener' is still set and we do
612 * not know the other peer.
613 *
614 * Call the implementation-specific cancel function of the operation.
615 * Disconnects from the remote peer. Does not disconnect the client,
616 * as there may be multiple operations per set.
617 *
618 * @param op operation to destroy
619 * @param gc #GNUNET_YES to perform garbage collection on the set
620 */
621void
622_GSS_operation_destroy (struct Operation *op,
623 int gc);
624
625
626/**
627 * This function probably should not exist
628 * and be replaced by inlining more specific
629 * logic in the various places where it is called.
630 */
631void
632_GSS_operation_destroy2 (struct Operation *op);
633
634
635/**
636 * Get the table with implementing functions for set union.
637 *
638 * @return the operation specific VTable
639 */
640const struct SetVT *
641_GSS_union_vt (void);
642
643
644/**
645 * Get the table with implementing functions for set intersection.
646 *
647 * @return the operation specific VTable
648 */
649const struct SetVT *
650_GSS_intersection_vt (void);
651
652
653/**
654 * Is element @a ee part of the set used by @a op?
655 *
656 * @param ee element to test
657 * @param op operation the defines the set and its generation
658 * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
659 */
660int
661_GSS_is_element_of_operation (struct ElementEntry *ee,
662 struct Operation *op);
663
664
665#endif
diff --git a/src/set/gnunet-service-set_intersection.c b/src/set/gnunet-service-set_intersection.c
deleted file mode 100644
index 9313483bb..000000000
--- a/src/set/gnunet-service-set_intersection.c
+++ /dev/null
@@ -1,1331 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set_intersection.c
22 * @brief two-peer set intersection
23 * @author Christian Fuchs
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_statistics_service.h"
29#include "gnunet-service-set.h"
30#include "gnunet_block_lib.h"
31#include "gnunet-service-set_protocol.h"
32#include "gnunet-service-set_intersection.h"
33#include <gcrypt.h>
34
35
36/**
37 * Current phase we are in for a intersection operation.
38 */
39enum IntersectionOperationPhase
40{
41 /**
42 * We are just starting.
43 */
44 PHASE_INITIAL,
45
46 /**
47 * We have send the number of our elements to the other
48 * peer, but did not setup our element set yet.
49 */
50 PHASE_COUNT_SENT,
51
52 /**
53 * We have initialized our set and are now reducing it by exchanging
54 * Bloom filters until one party notices the their element hashes
55 * are equal.
56 */
57 PHASE_BF_EXCHANGE,
58
59 /**
60 * We must next send the P2P DONE message (after finishing mostly
61 * with the local client). Then we will wait for the channel to close.
62 */
63 PHASE_MUST_SEND_DONE,
64
65 /**
66 * We have received the P2P DONE message, and must finish with the
67 * local client before terminating the channel.
68 */
69 PHASE_DONE_RECEIVED,
70
71 /**
72 * The protocol is over. Results may still have to be sent to the
73 * client.
74 */
75 PHASE_FINISHED
76};
77
78
79/**
80 * State of an evaluate operation with another peer.
81 */
82struct OperationState
83{
84 /**
85 * The bf we currently receive
86 */
87 struct GNUNET_CONTAINER_BloomFilter *remote_bf;
88
89 /**
90 * BF of the set's element.
91 */
92 struct GNUNET_CONTAINER_BloomFilter *local_bf;
93
94 /**
95 * Remaining elements in the intersection operation.
96 * Maps element-id-hashes to 'elements in our set'.
97 */
98 struct GNUNET_CONTAINER_MultiHashMap *my_elements;
99
100 /**
101 * Iterator for sending the final set of @e my_elements to the client.
102 */
103 struct GNUNET_CONTAINER_MultiHashMapIterator *full_result_iter;
104
105 /**
106 * Evaluate operations are held in a linked list.
107 */
108 struct OperationState *next;
109
110 /**
111 * Evaluate operations are held in a linked list.
112 */
113 struct OperationState *prev;
114
115 /**
116 * For multipart BF transmissions, we have to store the
117 * bloomfilter-data until we fully received it.
118 */
119 char *bf_data;
120
121 /**
122 * XOR of the keys of all of the elements (remaining) in my set.
123 * Always updated when elements are added or removed to
124 * @e my_elements.
125 */
126 struct GNUNET_HashCode my_xor;
127
128 /**
129 * XOR of the keys of all of the elements (remaining) in
130 * the other peer's set. Updated when we receive the
131 * other peer's Bloom filter.
132 */
133 struct GNUNET_HashCode other_xor;
134
135 /**
136 * How many bytes of @e bf_data are valid?
137 */
138 uint32_t bf_data_offset;
139
140 /**
141 * Current element count contained within @e my_elements.
142 * (May differ briefly during initialization.)
143 */
144 uint32_t my_element_count;
145
146 /**
147 * size of the bloomfilter in @e bf_data.
148 */
149 uint32_t bf_data_size;
150
151 /**
152 * size of the bloomfilter
153 */
154 uint32_t bf_bits_per_element;
155
156 /**
157 * Salt currently used for BF construction (by us or the other peer,
158 * depending on where we are in the code).
159 */
160 uint32_t salt;
161
162 /**
163 * Current state of the operation.
164 */
165 enum IntersectionOperationPhase phase;
166
167 /**
168 * Generation in which the operation handle
169 * was created.
170 */
171 unsigned int generation_created;
172
173 /**
174 * Did we send the client that we are done?
175 */
176 int client_done_sent;
177
178 /**
179 * Set whenever we reach the state where the death of the
180 * channel is perfectly find and should NOT result in the
181 * operation being cancelled.
182 */
183 int channel_death_expected;
184};
185
186
187/**
188 * Extra state required for efficient set intersection.
189 * Merely tracks the total number of elements.
190 */
191struct SetState
192{
193 /**
194 * Number of currently valid elements in the set which have not been
195 * removed.
196 */
197 uint32_t current_set_element_count;
198};
199
200
201/**
202 * If applicable in the current operation mode, send a result message
203 * to the client indicating we removed an element.
204 *
205 * @param op intersection operation
206 * @param element element to send
207 */
208static void
209send_client_removed_element (struct Operation *op,
210 struct GNUNET_SET_Element *element)
211{
212 struct GNUNET_MQ_Envelope *ev;
213 struct GNUNET_SET_ResultMessage *rm;
214
215 if (GNUNET_SET_RESULT_REMOVED != op->result_mode)
216 return; /* Wrong mode for transmitting removed elements */
217 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
218 "Sending removed element (size %u) to client\n",
219 element->size);
220 GNUNET_STATISTICS_update (_GSS_statistics,
221 "# Element removed messages sent",
222 1,
223 GNUNET_NO);
224 GNUNET_assert (0 != op->client_request_id);
225 ev = GNUNET_MQ_msg_extra (rm,
226 element->size,
227 GNUNET_MESSAGE_TYPE_SET_RESULT);
228 if (NULL == ev)
229 {
230 GNUNET_break (0);
231 return;
232 }
233 rm->result_status = htons (GNUNET_SET_STATUS_OK);
234 rm->request_id = htonl (op->client_request_id);
235 rm->element_type = element->element_type;
236 GNUNET_memcpy (&rm[1],
237 element->data,
238 element->size);
239 GNUNET_MQ_send (op->set->cs->mq,
240 ev);
241}
242
243
244/**
245 * Fills the "my_elements" hashmap with all relevant elements.
246 *
247 * @param cls the `struct Operation *` we are performing
248 * @param key current key code
249 * @param value the `struct ElementEntry *` from the hash map
250 * @return #GNUNET_YES (we should continue to iterate)
251 */
252static int
253filtered_map_initialization (void *cls,
254 const struct GNUNET_HashCode *key,
255 void *value)
256{
257 struct Operation *op = cls;
258 struct ElementEntry *ee = value;
259 struct GNUNET_HashCode mutated_hash;
260
261
262 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
263 "FIMA called for %s:%u\n",
264 GNUNET_h2s (&ee->element_hash),
265 ee->element.size);
266
267 if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
268 {
269 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
270 "Reduced initialization, not starting with %s:%u (wrong generation)\n",
271 GNUNET_h2s (&ee->element_hash),
272 ee->element.size);
273 return GNUNET_YES; /* element not valid in our operation's generation */
274 }
275
276 /* Test if element is in other peer's bloomfilter */
277 GNUNET_BLOCK_mingle_hash (&ee->element_hash,
278 op->state->salt,
279 &mutated_hash);
280 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
281 "Testing mingled hash %s with salt %u\n",
282 GNUNET_h2s (&mutated_hash),
283 op->state->salt);
284 if (GNUNET_NO ==
285 GNUNET_CONTAINER_bloomfilter_test (op->state->remote_bf,
286 &mutated_hash))
287 {
288 /* remove this element */
289 send_client_removed_element (op,
290 &ee->element);
291 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
292 "Reduced initialization, not starting with %s:%u\n",
293 GNUNET_h2s (&ee->element_hash),
294 ee->element.size);
295 return GNUNET_YES;
296 }
297 op->state->my_element_count++;
298 GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
299 &ee->element_hash,
300 &op->state->my_xor);
301 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
302 "Filtered initialization of my_elements, adding %s:%u\n",
303 GNUNET_h2s (&ee->element_hash),
304 ee->element.size);
305 GNUNET_break (GNUNET_YES ==
306 GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
307 &ee->element_hash,
308 ee,
309 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
310
311 return GNUNET_YES;
312}
313
314
315/**
316 * Removes elements from our hashmap if they are not contained within the
317 * provided remote bloomfilter.
318 *
319 * @param cls closure with the `struct Operation *`
320 * @param key current key code
321 * @param value value in the hash map
322 * @return #GNUNET_YES (we should continue to iterate)
323 */
324static int
325iterator_bf_reduce (void *cls,
326 const struct GNUNET_HashCode *key,
327 void *value)
328{
329 struct Operation *op = cls;
330 struct ElementEntry *ee = value;
331 struct GNUNET_HashCode mutated_hash;
332
333 GNUNET_BLOCK_mingle_hash (&ee->element_hash,
334 op->state->salt,
335 &mutated_hash);
336 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
337 "Testing mingled hash %s with salt %u\n",
338 GNUNET_h2s (&mutated_hash),
339 op->state->salt);
340 if (GNUNET_NO ==
341 GNUNET_CONTAINER_bloomfilter_test (op->state->remote_bf,
342 &mutated_hash))
343 {
344 GNUNET_break (0 < op->state->my_element_count);
345 op->state->my_element_count--;
346 GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
347 &ee->element_hash,
348 &op->state->my_xor);
349 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
350 "Bloom filter reduction of my_elements, removing %s:%u\n",
351 GNUNET_h2s (&ee->element_hash),
352 ee->element.size);
353 GNUNET_assert (GNUNET_YES ==
354 GNUNET_CONTAINER_multihashmap_remove (op->state->my_elements,
355 &ee->element_hash,
356 ee));
357 send_client_removed_element (op,
358 &ee->element);
359 }
360 else
361 {
362 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
363 "Bloom filter reduction of my_elements, keeping %s:%u\n",
364 GNUNET_h2s (&ee->element_hash),
365 ee->element.size);
366 }
367 return GNUNET_YES;
368}
369
370
371/**
372 * Create initial bloomfilter based on all the elements given.
373 *
374 * @param cls the `struct Operation *`
375 * @param key current key code
376 * @param value the `struct ElementEntry` to process
377 * @return #GNUNET_YES (we should continue to iterate)
378 */
379static int
380iterator_bf_create (void *cls,
381 const struct GNUNET_HashCode *key,
382 void *value)
383{
384 struct Operation *op = cls;
385 struct ElementEntry *ee = value;
386 struct GNUNET_HashCode mutated_hash;
387
388 GNUNET_BLOCK_mingle_hash (&ee->element_hash,
389 op->state->salt,
390 &mutated_hash);
391 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
392 "Initializing BF with hash %s with salt %u\n",
393 GNUNET_h2s (&mutated_hash),
394 op->state->salt);
395 GNUNET_CONTAINER_bloomfilter_add (op->state->local_bf,
396 &mutated_hash);
397 return GNUNET_YES;
398}
399
400
401/**
402 * Inform the client that the intersection operation has failed,
403 * and proceed to destroy the evaluate operation.
404 *
405 * @param op the intersection operation to fail
406 */
407static void
408fail_intersection_operation (struct Operation *op)
409{
410 struct GNUNET_MQ_Envelope *ev;
411 struct GNUNET_SET_ResultMessage *msg;
412
413 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
414 "Intersection operation failed\n");
415 GNUNET_STATISTICS_update (_GSS_statistics,
416 "# Intersection operations failed",
417 1,
418 GNUNET_NO);
419 if (NULL != op->state->my_elements)
420 {
421 GNUNET_CONTAINER_multihashmap_destroy (op->state->my_elements);
422 op->state->my_elements = NULL;
423 }
424 ev = GNUNET_MQ_msg (msg,
425 GNUNET_MESSAGE_TYPE_SET_RESULT);
426 msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
427 msg->request_id = htonl (op->client_request_id);
428 msg->element_type = htons (0);
429 GNUNET_MQ_send (op->set->cs->mq,
430 ev);
431 _GSS_operation_destroy (op,
432 GNUNET_YES);
433}
434
435
436/**
437 * Send a bloomfilter to our peer. After the result done message has
438 * been sent to the client, destroy the evaluate operation.
439 *
440 * @param op intersection operation
441 */
442static void
443send_bloomfilter (struct Operation *op)
444{
445 struct GNUNET_MQ_Envelope *ev;
446 struct BFMessage *msg;
447 uint32_t bf_size;
448 uint32_t bf_elementbits;
449 uint32_t chunk_size;
450 char *bf_data;
451 uint32_t offset;
452
453 /* We consider the ratio of the set sizes to determine
454 the number of bits per element, as the smaller set
455 should use more bits to maximize its set reduction
456 potential and minimize overall bandwidth consumption. */
457 bf_elementbits = 2 + ceil (log2 ((double)
458 (op->remote_element_count
459 / (double) op->state->my_element_count)));
460 if (bf_elementbits < 1)
461 bf_elementbits = 1; /* make sure k is not 0 */
462 /* optimize BF-size to ~50% of bits set */
463 bf_size = ceil ((double) (op->state->my_element_count
464 * bf_elementbits / log (2)));
465 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
466 "Sending Bloom filter (%u) of size %u bytes\n",
467 (unsigned int) bf_elementbits,
468 (unsigned int) bf_size);
469 op->state->local_bf = GNUNET_CONTAINER_bloomfilter_init (NULL,
470 bf_size,
471 bf_elementbits);
472 op->state->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
473 UINT32_MAX);
474 GNUNET_CONTAINER_multihashmap_iterate (op->state->my_elements,
475 &iterator_bf_create,
476 op);
477
478 /* send our Bloom filter */
479 GNUNET_STATISTICS_update (_GSS_statistics,
480 "# Intersection Bloom filters sent",
481 1,
482 GNUNET_NO);
483 chunk_size = 60 * 1024 - sizeof(struct BFMessage);
484 if (bf_size <= chunk_size)
485 {
486 /* singlepart */
487 chunk_size = bf_size;
488 ev = GNUNET_MQ_msg_extra (msg,
489 chunk_size,
490 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF);
491 GNUNET_assert (GNUNET_SYSERR !=
492 GNUNET_CONTAINER_bloomfilter_get_raw_data (
493 op->state->local_bf,
494 (char *) &msg[1],
495 bf_size));
496 msg->sender_element_count = htonl (op->state->my_element_count);
497 msg->bloomfilter_total_length = htonl (bf_size);
498 msg->bits_per_element = htonl (bf_elementbits);
499 msg->sender_mutator = htonl (op->state->salt);
500 msg->element_xor_hash = op->state->my_xor;
501 GNUNET_MQ_send (op->mq, ev);
502 }
503 else
504 {
505 /* multipart */
506 bf_data = GNUNET_malloc (bf_size);
507 GNUNET_assert (GNUNET_SYSERR !=
508 GNUNET_CONTAINER_bloomfilter_get_raw_data (
509 op->state->local_bf,
510 bf_data,
511 bf_size));
512 offset = 0;
513 while (offset < bf_size)
514 {
515 if (bf_size - chunk_size < offset)
516 chunk_size = bf_size - offset;
517 ev = GNUNET_MQ_msg_extra (msg,
518 chunk_size,
519 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF);
520 GNUNET_memcpy (&msg[1],
521 &bf_data[offset],
522 chunk_size);
523 offset += chunk_size;
524 msg->sender_element_count = htonl (op->state->my_element_count);
525 msg->bloomfilter_total_length = htonl (bf_size);
526 msg->bits_per_element = htonl (bf_elementbits);
527 msg->sender_mutator = htonl (op->state->salt);
528 msg->element_xor_hash = op->state->my_xor;
529 GNUNET_MQ_send (op->mq, ev);
530 }
531 GNUNET_free (bf_data);
532 }
533 GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
534 op->state->local_bf = NULL;
535}
536
537
538/**
539 * Signal to the client that the operation has finished and
540 * destroy the operation.
541 *
542 * @param cls operation to destroy
543 */
544static void
545send_client_done_and_destroy (void *cls)
546{
547 struct Operation *op = cls;
548 struct GNUNET_MQ_Envelope *ev;
549 struct GNUNET_SET_ResultMessage *rm;
550
551 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
552 "Intersection succeeded, sending DONE to local client\n");
553 GNUNET_STATISTICS_update (_GSS_statistics,
554 "# Intersection operations succeeded",
555 1,
556 GNUNET_NO);
557 ev = GNUNET_MQ_msg (rm,
558 GNUNET_MESSAGE_TYPE_SET_RESULT);
559 rm->request_id = htonl (op->client_request_id);
560 rm->result_status = htons (GNUNET_SET_STATUS_DONE);
561 rm->element_type = htons (0);
562 GNUNET_MQ_send (op->set->cs->mq,
563 ev);
564 _GSS_operation_destroy (op,
565 GNUNET_YES);
566}
567
568
569/**
570 * Remember that we are done dealing with the local client
571 * AND have sent the other peer our message that we are done,
572 * so we are not just waiting for the channel to die before
573 * telling the local client that we are done as our last act.
574 *
575 * @param cls the `struct Operation`.
576 */
577static void
578finished_local_operations (void *cls)
579{
580 struct Operation *op = cls;
581
582 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
583 "DONE sent to other peer, now waiting for other end to close the channel\n");
584 op->state->phase = PHASE_FINISHED;
585 op->state->channel_death_expected = GNUNET_YES;
586}
587
588
589/**
590 * Notify the other peer that we are done. Once this message
591 * is out, we still need to notify the local client that we
592 * are done.
593 *
594 * @param op operation to notify for.
595 */
596static void
597send_p2p_done (struct Operation *op)
598{
599 struct GNUNET_MQ_Envelope *ev;
600 struct IntersectionDoneMessage *idm;
601
602 GNUNET_assert (PHASE_MUST_SEND_DONE == op->state->phase);
603 GNUNET_assert (GNUNET_NO == op->state->channel_death_expected);
604 ev = GNUNET_MQ_msg (idm,
605 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE);
606 idm->final_element_count = htonl (op->state->my_element_count);
607 idm->element_xor_hash = op->state->my_xor;
608 GNUNET_MQ_notify_sent (ev,
609 &finished_local_operations,
610 op);
611 GNUNET_MQ_send (op->mq,
612 ev);
613}
614
615
616/**
617 * Send all elements in the full result iterator.
618 *
619 * @param cls the `struct Operation *`
620 */
621static void
622send_remaining_elements (void *cls)
623{
624 struct Operation *op = cls;
625 const void *nxt;
626 const struct ElementEntry *ee;
627 struct GNUNET_MQ_Envelope *ev;
628 struct GNUNET_SET_ResultMessage *rm;
629 const struct GNUNET_SET_Element *element;
630 int res;
631
632 res = GNUNET_CONTAINER_multihashmap_iterator_next (
633 op->state->full_result_iter,
634 NULL,
635 &nxt);
636 if (GNUNET_NO == res)
637 {
638 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
639 "Sending done and destroy because iterator ran out\n");
640 GNUNET_CONTAINER_multihashmap_iterator_destroy (
641 op->state->full_result_iter);
642 op->state->full_result_iter = NULL;
643 if (PHASE_DONE_RECEIVED == op->state->phase)
644 {
645 op->state->phase = PHASE_FINISHED;
646 send_client_done_and_destroy (op);
647 }
648 else if (PHASE_MUST_SEND_DONE == op->state->phase)
649 {
650 send_p2p_done (op);
651 }
652 else
653 {
654 GNUNET_assert (0);
655 }
656 return;
657 }
658 ee = nxt;
659 element = &ee->element;
660 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
661 "Sending element %s:%u to client (full set)\n",
662 GNUNET_h2s (&ee->element_hash),
663 element->size);
664 GNUNET_assert (0 != op->client_request_id);
665 ev = GNUNET_MQ_msg_extra (rm,
666 element->size,
667 GNUNET_MESSAGE_TYPE_SET_RESULT);
668 GNUNET_assert (NULL != ev);
669 rm->result_status = htons (GNUNET_SET_STATUS_OK);
670 rm->request_id = htonl (op->client_request_id);
671 rm->element_type = element->element_type;
672 GNUNET_memcpy (&rm[1],
673 element->data,
674 element->size);
675 GNUNET_MQ_notify_sent (ev,
676 &send_remaining_elements,
677 op);
678 GNUNET_MQ_send (op->set->cs->mq,
679 ev);
680}
681
682
683/**
684 * Fills the "my_elements" hashmap with the initial set of
685 * (non-deleted) elements from the set of the specification.
686 *
687 * @param cls closure with the `struct Operation *`
688 * @param key current key code for the element
689 * @param value value in the hash map with the `struct ElementEntry *`
690 * @return #GNUNET_YES (we should continue to iterate)
691 */
692static int
693initialize_map_unfiltered (void *cls,
694 const struct GNUNET_HashCode *key,
695 void *value)
696{
697 struct ElementEntry *ee = value;
698 struct Operation *op = cls;
699
700 if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
701 return GNUNET_YES; /* element not live in operation's generation */
702 GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
703 &ee->element_hash,
704 &op->state->my_xor);
705 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
706 "Initial full initialization of my_elements, adding %s:%u\n",
707 GNUNET_h2s (&ee->element_hash),
708 ee->element.size);
709 GNUNET_break (GNUNET_YES ==
710 GNUNET_CONTAINER_multihashmap_put (op->state->my_elements,
711 &ee->element_hash,
712 ee,
713 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
714 return GNUNET_YES;
715}
716
717
718/**
719 * Send our element count to the peer, in case our element count is
720 * lower than theirs.
721 *
722 * @param op intersection operation
723 */
724static void
725send_element_count (struct Operation *op)
726{
727 struct GNUNET_MQ_Envelope *ev;
728 struct IntersectionElementInfoMessage *msg;
729
730 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
731 "Sending our element count (%u)\n",
732 op->state->my_element_count);
733 ev = GNUNET_MQ_msg (msg,
734 GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO);
735 msg->sender_element_count = htonl (op->state->my_element_count);
736 GNUNET_MQ_send (op->mq, ev);
737}
738
739
740/**
741 * We go first, initialize our map with all elements and
742 * send the first Bloom filter.
743 *
744 * @param op operation to start exchange for
745 */
746static void
747begin_bf_exchange (struct Operation *op)
748{
749 op->state->phase = PHASE_BF_EXCHANGE;
750 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
751 &initialize_map_unfiltered,
752 op);
753 send_bloomfilter (op);
754}
755
756
757/**
758 * Handle the initial `struct IntersectionElementInfoMessage` from a
759 * remote peer.
760 *
761 * @param cls the intersection operation
762 * @param mh the header of the message
763 */
764void
765handle_intersection_p2p_element_info (void *cls,
766 const struct
767 IntersectionElementInfoMessage *msg)
768{
769 struct Operation *op = cls;
770
771 if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
772 {
773 GNUNET_break_op (0);
774 fail_intersection_operation (op);
775 return;
776 }
777 op->remote_element_count = ntohl (msg->sender_element_count);
778 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
779 "Received remote element count (%u), I have %u\n",
780 op->remote_element_count,
781 op->state->my_element_count);
782 if (((PHASE_INITIAL != op->state->phase) &&
783 (PHASE_COUNT_SENT != op->state->phase)) ||
784 (op->state->my_element_count > op->remote_element_count) ||
785 (0 == op->state->my_element_count) ||
786 (0 == op->remote_element_count))
787 {
788 GNUNET_break_op (0);
789 fail_intersection_operation (op);
790 return;
791 }
792 GNUNET_break (NULL == op->state->remote_bf);
793 begin_bf_exchange (op);
794 GNUNET_CADET_receive_done (op->channel);
795}
796
797
798/**
799 * Process a Bloomfilter once we got all the chunks.
800 *
801 * @param op the intersection operation
802 */
803static void
804process_bf (struct Operation *op)
805{
806 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
807 "Received BF in phase %u, foreign count is %u, my element count is %u/%u\n",
808 op->state->phase,
809 op->remote_element_count,
810 op->state->my_element_count,
811 GNUNET_CONTAINER_multihashmap_size (op->set->content->elements));
812 switch (op->state->phase)
813 {
814 case PHASE_INITIAL:
815 GNUNET_break_op (0);
816 fail_intersection_operation (op);
817 return;
818
819 case PHASE_COUNT_SENT:
820 /* This is the first BF being sent, build our initial map with
821 filtering in place */
822 op->state->my_element_count = 0;
823 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
824 &filtered_map_initialization,
825 op);
826 break;
827
828 case PHASE_BF_EXCHANGE:
829 /* Update our set by reduction */
830 GNUNET_CONTAINER_multihashmap_iterate (op->state->my_elements,
831 &iterator_bf_reduce,
832 op);
833 break;
834
835 case PHASE_MUST_SEND_DONE:
836 GNUNET_break_op (0);
837 fail_intersection_operation (op);
838 return;
839
840 case PHASE_DONE_RECEIVED:
841 GNUNET_break_op (0);
842 fail_intersection_operation (op);
843 return;
844
845 case PHASE_FINISHED:
846 GNUNET_break_op (0);
847 fail_intersection_operation (op);
848 return;
849 }
850 GNUNET_CONTAINER_bloomfilter_free (op->state->remote_bf);
851 op->state->remote_bf = NULL;
852
853 if ((0 == op->state->my_element_count) || /* fully disjoint */
854 ((op->state->my_element_count == op->remote_element_count) &&
855 (0 == GNUNET_memcmp (&op->state->my_xor,
856 &op->state->other_xor))))
857 {
858 /* we are done */
859 op->state->phase = PHASE_MUST_SEND_DONE;
860 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
861 "Intersection succeeded, sending DONE to other peer\n");
862 GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
863 op->state->local_bf = NULL;
864 if (GNUNET_SET_RESULT_FULL == op->result_mode)
865 {
866 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
867 "Sending full result set (%u elements)\n",
868 GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
869 op->state->full_result_iter
870 = GNUNET_CONTAINER_multihashmap_iterator_create (
871 op->state->my_elements);
872 send_remaining_elements (op);
873 return;
874 }
875 send_p2p_done (op);
876 return;
877 }
878 op->state->phase = PHASE_BF_EXCHANGE;
879 send_bloomfilter (op);
880}
881
882
883/**
884 * Check an BF message from a remote peer.
885 *
886 * @param cls the intersection operation
887 * @param msg the header of the message
888 * @return #GNUNET_OK if @a msg is well-formed
889 */
890int
891check_intersection_p2p_bf (void *cls,
892 const struct BFMessage *msg)
893{
894 struct Operation *op = cls;
895
896 if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
897 {
898 GNUNET_break_op (0);
899 return GNUNET_SYSERR;
900 }
901 return GNUNET_OK;
902}
903
904
905/**
906 * Handle an BF message from a remote peer.
907 *
908 * @param cls the intersection operation
909 * @param msg the header of the message
910 */
911void
912handle_intersection_p2p_bf (void *cls,
913 const struct BFMessage *msg)
914{
915 struct Operation *op = cls;
916 uint32_t bf_size;
917 uint32_t chunk_size;
918 uint32_t bf_bits_per_element;
919
920 switch (op->state->phase)
921 {
922 case PHASE_INITIAL:
923 GNUNET_break_op (0);
924 fail_intersection_operation (op);
925 return;
926
927 case PHASE_COUNT_SENT:
928 case PHASE_BF_EXCHANGE:
929 bf_size = ntohl (msg->bloomfilter_total_length);
930 bf_bits_per_element = ntohl (msg->bits_per_element);
931 chunk_size = htons (msg->header.size) - sizeof(struct BFMessage);
932 op->state->other_xor = msg->element_xor_hash;
933 if (bf_size == chunk_size)
934 {
935 if (NULL != op->state->bf_data)
936 {
937 GNUNET_break_op (0);
938 fail_intersection_operation (op);
939 return;
940 }
941 /* single part, done here immediately */
942 op->state->remote_bf
943 = GNUNET_CONTAINER_bloomfilter_init ((const char *) &msg[1],
944 bf_size,
945 bf_bits_per_element);
946 op->state->salt = ntohl (msg->sender_mutator);
947 op->remote_element_count = ntohl (msg->sender_element_count);
948 process_bf (op);
949 break;
950 }
951 /* multipart chunk */
952 if (NULL == op->state->bf_data)
953 {
954 /* first chunk, initialize */
955 op->state->bf_data = GNUNET_malloc (bf_size);
956 op->state->bf_data_size = bf_size;
957 op->state->bf_bits_per_element = bf_bits_per_element;
958 op->state->bf_data_offset = 0;
959 op->state->salt = ntohl (msg->sender_mutator);
960 op->remote_element_count = ntohl (msg->sender_element_count);
961 }
962 else
963 {
964 /* increment */
965 if ((op->state->bf_data_size != bf_size) ||
966 (op->state->bf_bits_per_element != bf_bits_per_element) ||
967 (op->state->bf_data_offset + chunk_size > bf_size) ||
968 (op->state->salt != ntohl (msg->sender_mutator)) ||
969 (op->remote_element_count != ntohl (msg->sender_element_count)))
970 {
971 GNUNET_break_op (0);
972 fail_intersection_operation (op);
973 return;
974 }
975 }
976 GNUNET_memcpy (&op->state->bf_data[op->state->bf_data_offset],
977 (const char *) &msg[1],
978 chunk_size);
979 op->state->bf_data_offset += chunk_size;
980 if (op->state->bf_data_offset == bf_size)
981 {
982 /* last chunk, run! */
983 op->state->remote_bf
984 = GNUNET_CONTAINER_bloomfilter_init (op->state->bf_data,
985 bf_size,
986 bf_bits_per_element);
987 GNUNET_free (op->state->bf_data);
988 op->state->bf_data = NULL;
989 op->state->bf_data_size = 0;
990 process_bf (op);
991 }
992 break;
993
994 default:
995 GNUNET_break_op (0);
996 fail_intersection_operation (op);
997 return;
998 }
999 GNUNET_CADET_receive_done (op->channel);
1000}
1001
1002
1003/**
1004 * Remove all elements from our hashmap.
1005 *
1006 * @param cls closure with the `struct Operation *`
1007 * @param key current key code
1008 * @param value value in the hash map
1009 * @return #GNUNET_YES (we should continue to iterate)
1010 */
1011static int
1012filter_all (void *cls,
1013 const struct GNUNET_HashCode *key,
1014 void *value)
1015{
1016 struct Operation *op = cls;
1017 struct ElementEntry *ee = value;
1018
1019 GNUNET_break (0 < op->state->my_element_count);
1020 op->state->my_element_count--;
1021 GNUNET_CRYPTO_hash_xor (&op->state->my_xor,
1022 &ee->element_hash,
1023 &op->state->my_xor);
1024 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1025 "Final reduction of my_elements, removing %s:%u\n",
1026 GNUNET_h2s (&ee->element_hash),
1027 ee->element.size);
1028 GNUNET_assert (GNUNET_YES ==
1029 GNUNET_CONTAINER_multihashmap_remove (op->state->my_elements,
1030 &ee->element_hash,
1031 ee));
1032 send_client_removed_element (op,
1033 &ee->element);
1034 return GNUNET_YES;
1035}
1036
1037
1038/**
1039 * Handle a done message from a remote peer
1040 *
1041 * @param cls the intersection operation
1042 * @param mh the message
1043 */
1044void
1045handle_intersection_p2p_done (void *cls,
1046 const struct IntersectionDoneMessage *idm)
1047{
1048 struct Operation *op = cls;
1049
1050 if (GNUNET_SET_OPERATION_INTERSECTION != op->set->operation)
1051 {
1052 GNUNET_break_op (0);
1053 fail_intersection_operation (op);
1054 return;
1055 }
1056 if (PHASE_BF_EXCHANGE != op->state->phase)
1057 {
1058 /* wrong phase to conclude? FIXME: Or should we allow this
1059 if the other peer has _initially_ already an empty set? */
1060 GNUNET_break_op (0);
1061 fail_intersection_operation (op);
1062 return;
1063 }
1064 if (0 == ntohl (idm->final_element_count))
1065 {
1066 /* other peer determined empty set is the intersection,
1067 remove all elements */
1068 GNUNET_CONTAINER_multihashmap_iterate (op->state->my_elements,
1069 &filter_all,
1070 op);
1071 }
1072 if ((op->state->my_element_count != ntohl (idm->final_element_count)) ||
1073 (0 != GNUNET_memcmp (&op->state->my_xor,
1074 &idm->element_xor_hash)))
1075 {
1076 /* Other peer thinks we are done, but we disagree on the result! */
1077 GNUNET_break_op (0);
1078 fail_intersection_operation (op);
1079 return;
1080 }
1081 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1082 "Got IntersectionDoneMessage, have %u elements in intersection\n",
1083 op->state->my_element_count);
1084 op->state->phase = PHASE_DONE_RECEIVED;
1085 GNUNET_CADET_receive_done (op->channel);
1086
1087 GNUNET_assert (GNUNET_NO == op->state->client_done_sent);
1088 if (GNUNET_SET_RESULT_FULL == op->result_mode)
1089 {
1090 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1091 "Sending full result set to client (%u elements)\n",
1092 GNUNET_CONTAINER_multihashmap_size (op->state->my_elements));
1093 op->state->full_result_iter
1094 = GNUNET_CONTAINER_multihashmap_iterator_create (op->state->my_elements);
1095 send_remaining_elements (op);
1096 return;
1097 }
1098 op->state->phase = PHASE_FINISHED;
1099 send_client_done_and_destroy (op);
1100}
1101
1102
1103/**
1104 * Initiate a set intersection operation with a remote peer.
1105 *
1106 * @param op operation that is created, should be initialized to
1107 * begin the evaluation
1108 * @param opaque_context message to be transmitted to the listener
1109 * to convince it to accept, may be NULL
1110 * @return operation-specific state to keep in @a op
1111 */
1112static struct OperationState *
1113intersection_evaluate (struct Operation *op,
1114 const struct GNUNET_MessageHeader *opaque_context)
1115{
1116 struct OperationState *state;
1117 struct GNUNET_MQ_Envelope *ev;
1118 struct OperationRequestMessage *msg;
1119
1120 ev = GNUNET_MQ_msg_nested_mh (msg,
1121 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
1122 opaque_context);
1123 if (NULL == ev)
1124 {
1125 /* the context message is too large!? */
1126 GNUNET_break (0);
1127 return NULL;
1128 }
1129 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1130 "Initiating intersection operation evaluation\n");
1131 state = GNUNET_new (struct OperationState);
1132 /* we started the operation, thus we have to send the operation request */
1133 state->phase = PHASE_INITIAL;
1134 state->my_element_count = op->set->state->current_set_element_count;
1135 state->my_elements
1136 = GNUNET_CONTAINER_multihashmap_create (state->my_element_count,
1137 GNUNET_YES);
1138
1139 msg->operation = htonl (GNUNET_SET_OPERATION_INTERSECTION);
1140 msg->element_count = htonl (state->my_element_count);
1141 GNUNET_MQ_send (op->mq,
1142 ev);
1143 state->phase = PHASE_COUNT_SENT;
1144 if (NULL != opaque_context)
1145 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1146 "Sent op request with context message\n");
1147 else
1148 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1149 "Sent op request without context message\n");
1150 return state;
1151}
1152
1153
1154/**
1155 * Accept an intersection operation request from a remote peer. Only
1156 * initializes the private operation state.
1157 *
1158 * @param op operation that will be accepted as an intersection operation
1159 */
1160static struct OperationState *
1161intersection_accept (struct Operation *op)
1162{
1163 struct OperationState *state;
1164
1165 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1166 "Accepting set intersection operation\n");
1167 state = GNUNET_new (struct OperationState);
1168 state->phase = PHASE_INITIAL;
1169 state->my_element_count
1170 = op->set->state->current_set_element_count;
1171 state->my_elements
1172 = GNUNET_CONTAINER_multihashmap_create (GNUNET_MIN (state->my_element_count,
1173 op->remote_element_count),
1174 GNUNET_YES);
1175 op->state = state;
1176 if (op->remote_element_count < state->my_element_count)
1177 {
1178 /* If the other peer (Alice) has fewer elements than us (Bob),
1179 we just send the count as Alice should send the first BF */
1180 send_element_count (op);
1181 state->phase = PHASE_COUNT_SENT;
1182 return state;
1183 }
1184 /* We have fewer elements, so we start with the BF */
1185 begin_bf_exchange (op);
1186 return state;
1187}
1188
1189
1190/**
1191 * Destroy the intersection operation. Only things specific to the
1192 * intersection operation are destroyed.
1193 *
1194 * @param op intersection operation to destroy
1195 */
1196static void
1197intersection_op_cancel (struct Operation *op)
1198{
1199 /* check if the op was canceled twice */
1200 GNUNET_assert (NULL != op->state);
1201 if (NULL != op->state->remote_bf)
1202 {
1203 GNUNET_CONTAINER_bloomfilter_free (op->state->remote_bf);
1204 op->state->remote_bf = NULL;
1205 }
1206 if (NULL != op->state->local_bf)
1207 {
1208 GNUNET_CONTAINER_bloomfilter_free (op->state->local_bf);
1209 op->state->local_bf = NULL;
1210 }
1211 if (NULL != op->state->my_elements)
1212 {
1213 GNUNET_CONTAINER_multihashmap_destroy (op->state->my_elements);
1214 op->state->my_elements = NULL;
1215 }
1216 if (NULL != op->state->full_result_iter)
1217 {
1218 GNUNET_CONTAINER_multihashmap_iterator_destroy (
1219 op->state->full_result_iter);
1220 op->state->full_result_iter = NULL;
1221 }
1222 GNUNET_free (op->state);
1223 op->state = NULL;
1224 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1225 "Destroying intersection op state done\n");
1226}
1227
1228
1229/**
1230 * Create a new set supporting the intersection operation.
1231 *
1232 * @return the newly created set
1233 */
1234static struct SetState *
1235intersection_set_create ()
1236{
1237 struct SetState *set_state;
1238
1239 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1240 "Intersection set created\n");
1241 set_state = GNUNET_new (struct SetState);
1242 set_state->current_set_element_count = 0;
1243
1244 return set_state;
1245}
1246
1247
1248/**
1249 * Add the element from the given element message to the set.
1250 *
1251 * @param set_state state of the set want to add to
1252 * @param ee the element to add to the set
1253 */
1254static void
1255intersection_add (struct SetState *set_state,
1256 struct ElementEntry *ee)
1257{
1258 set_state->current_set_element_count++;
1259}
1260
1261
1262/**
1263 * Destroy a set that supports the intersection operation
1264 *
1265 * @param set_state the set to destroy
1266 */
1267static void
1268intersection_set_destroy (struct SetState *set_state)
1269{
1270 GNUNET_free (set_state);
1271}
1272
1273
1274/**
1275 * Remove the element given in the element message from the set.
1276 *
1277 * @param set_state state of the set to remove from
1278 * @param element set element to remove
1279 */
1280static void
1281intersection_remove (struct SetState *set_state,
1282 struct ElementEntry *element)
1283{
1284 GNUNET_assert (0 < set_state->current_set_element_count);
1285 set_state->current_set_element_count--;
1286}
1287
1288
1289/**
1290 * Callback for channel death for the intersection operation.
1291 *
1292 * @param op operation that lost the channel
1293 */
1294static void
1295intersection_channel_death (struct Operation *op)
1296{
1297 if (GNUNET_YES == op->state->channel_death_expected)
1298 {
1299 /* oh goodie, we are done! */
1300 send_client_done_and_destroy (op);
1301 }
1302 else
1303 {
1304 /* sorry, channel went down early, too bad. */
1305 _GSS_operation_destroy (op,
1306 GNUNET_YES);
1307 }
1308}
1309
1310
1311/**
1312 * Get the table with implementing functions for set intersection.
1313 *
1314 * @return the operation specific VTable
1315 */
1316const struct SetVT *
1317_GSS_intersection_vt ()
1318{
1319 static const struct SetVT intersection_vt = {
1320 .create = &intersection_set_create,
1321 .add = &intersection_add,
1322 .remove = &intersection_remove,
1323 .destroy_set = &intersection_set_destroy,
1324 .evaluate = &intersection_evaluate,
1325 .accept = &intersection_accept,
1326 .cancel = &intersection_op_cancel,
1327 .channel_death = &intersection_channel_death,
1328 };
1329
1330 return &intersection_vt;
1331}
diff --git a/src/set/gnunet-service-set_intersection.h b/src/set/gnunet-service-set_intersection.h
deleted file mode 100644
index 200e8f5ff..000000000
--- a/src/set/gnunet-service-set_intersection.h
+++ /dev/null
@@ -1,79 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set_intersection.h
22 * @brief two-peer set operations
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef GNUNET_SERVICE_SET_INTERSECTION_H
27#define GNUNET_SERVICE_SET_INTERSECTION_H
28
29#include "gnunet-service-set.h"
30
31
32/**
33 * Check an BF message from a remote peer.
34 *
35 * @param cls the intersection operation
36 * @param msg the header of the message
37 * @return #GNUNET_OK if @a msg is well-formed
38 */
39int
40check_intersection_p2p_bf (void *cls,
41 const struct BFMessage *msg);
42
43
44/**
45 * Handle an BF message from a remote peer.
46 *
47 * @param cls the intersection operation
48 * @param msg the header of the message
49 */
50void
51handle_intersection_p2p_bf (void *cls,
52 const struct BFMessage *msg);
53
54
55/**
56 * Handle the initial `struct IntersectionElementInfoMessage` from a
57 * remote peer.
58 *
59 * @param cls the intersection operation
60 * @param mh the header of the message
61 */
62void
63handle_intersection_p2p_element_info (void *cls,
64 const struct
65 IntersectionElementInfoMessage *msg);
66
67
68/**
69 * Handle a done message from a remote peer
70 *
71 * @param cls the intersection operation
72 * @param mh the message
73 */
74void
75handle_intersection_p2p_done (void *cls,
76 const struct IntersectionDoneMessage *idm);
77
78
79#endif
diff --git a/src/set/gnunet-service-set_protocol.h b/src/set/gnunet-service-set_protocol.h
deleted file mode 100644
index a2803ee47..000000000
--- a/src/set/gnunet-service-set_protocol.h
+++ /dev/null
@@ -1,226 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2013, 2014 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @author Florian Dold
22 * @author Christian Grothoff
23 * @file set/gnunet-service-set_protocol.h
24 * @brief Peer-to-Peer messages for gnunet set
25 */
26#ifndef SET_PROTOCOL_H
27#define SET_PROTOCOL_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35struct OperationRequestMessage
36{
37 /**
38 * Type: #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
39 */
40 struct GNUNET_MessageHeader header;
41
42 /**
43 * Operation to request, values from `enum GNUNET_SET_OperationType`
44 */
45 uint32_t operation GNUNET_PACKED;
46
47 /**
48 * For Intersection: my element count
49 */
50 uint32_t element_count GNUNET_PACKED;
51
52 /**
53 * Application-specific identifier of the request.
54 */
55 struct GNUNET_HashCode app_idX;
56
57 /* rest: optional message */
58};
59
60
61/**
62 * Message containing buckets of an invertible bloom filter.
63 *
64 * If an IBF has too many buckets for an IBF message,
65 * it is split into multiple messages.
66 */
67struct IBFMessage
68{
69 /**
70 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
71 */
72 struct GNUNET_MessageHeader header;
73
74 /**
75 * Order of the whole ibf, where
76 * num_buckets = 2^order
77 */
78 uint8_t order;
79
80 /**
81 * Padding, must be 0.
82 */
83 uint8_t reserved1;
84
85 /**
86 * Padding, must be 0.
87 */
88 uint16_t reserved2 GNUNET_PACKED;
89
90 /**
91 * Offset of the strata in the rest of the message
92 */
93 uint32_t offset GNUNET_PACKED;
94
95 /**
96 * Salt used when hashing elements for this IBF.
97 */
98 uint32_t salt GNUNET_PACKED;
99
100 /* rest: buckets */
101};
102
103
104struct InquiryMessage
105{
106 /**
107 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
108 */
109 struct GNUNET_MessageHeader header;
110
111 /**
112 * Salt used when hashing elements for this inquiry.
113 */
114 uint32_t salt GNUNET_PACKED;
115
116 /**
117 * Reserved, set to 0.
118 */
119 uint32_t reserved GNUNET_PACKED;
120
121 /* rest: inquiry IBF keys */
122};
123
124
125/**
126 * During intersection, the first (and possibly second) message
127 * send it the number of elements in the set, to allow the peers
128 * to decide who should start with the Bloom filter.
129 */
130struct IntersectionElementInfoMessage
131{
132 /**
133 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO
134 */
135 struct GNUNET_MessageHeader header;
136
137 /**
138 * mutator used with this bloomfilter.
139 */
140 uint32_t sender_element_count GNUNET_PACKED;
141};
142
143
144/**
145 * Bloom filter messages exchanged for set intersection calculation.
146 */
147struct BFMessage
148{
149 /**
150 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF
151 */
152 struct GNUNET_MessageHeader header;
153
154 /**
155 * Number of elements the sender still has in the set.
156 */
157 uint32_t sender_element_count GNUNET_PACKED;
158
159 /**
160 * XOR of all hashes over all elements remaining in the set.
161 * Used to determine termination.
162 */
163 struct GNUNET_HashCode element_xor_hash;
164
165 /**
166 * Mutator used with this bloomfilter.
167 */
168 uint32_t sender_mutator GNUNET_PACKED;
169
170 /**
171 * Total length of the bloomfilter data.
172 */
173 uint32_t bloomfilter_total_length GNUNET_PACKED;
174
175 /**
176 * Number of bits (k-value) used in encoding the bloomfilter.
177 */
178 uint32_t bits_per_element GNUNET_PACKED;
179
180 /**
181 * rest: the sender's bloomfilter
182 */
183};
184
185
186/**
187 * Last message, send to confirm the final set. Contains the element
188 * count as it is possible that the peer determined that we were done
189 * by getting the empty set, which in that case also needs to be
190 * communicated.
191 */
192struct IntersectionDoneMessage
193{
194 /**
195 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE
196 */
197 struct GNUNET_MessageHeader header;
198
199 /**
200 * Final number of elements in intersection.
201 */
202 uint32_t final_element_count GNUNET_PACKED;
203
204 /**
205 * XOR of all hashes over all elements remaining in the set.
206 */
207 struct GNUNET_HashCode element_xor_hash;
208};
209
210
211/**
212 * Strata estimator together with the peer's overall set size.
213 */
214struct StrataEstimatorMessage
215{
216 /**
217 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE(C)
218 */
219 struct GNUNET_MessageHeader header;
220
221 uint64_t set_size;
222};
223
224GNUNET_NETWORK_STRUCT_END
225
226#endif
diff --git a/src/set/gnunet-service-set_union.c b/src/set/gnunet-service-set_union.c
deleted file mode 100644
index d762e4bba..000000000
--- a/src/set/gnunet-service-set_union.c
+++ /dev/null
@@ -1,2469 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set_union.c
22 * @brief two-peer set operations
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_statistics_service.h"
29#include "gnunet-service-set.h"
30#include "ibf.h"
31#include "gnunet-service-set_union.h"
32#include "gnunet-service-set_union_strata_estimator.h"
33#include "gnunet-service-set_protocol.h"
34#include <gcrypt.h>
35
36
37#define LOG(kind, ...) GNUNET_log_from (kind, "set-union", __VA_ARGS__)
38
39
40/**
41 * Number of IBFs in a strata estimator.
42 */
43#define SE_STRATA_COUNT 32
44
45/**
46 * Size of the IBFs in the strata estimator.
47 */
48#define SE_IBF_SIZE 80
49
50/**
51 * The hash num parameter for the difference digests and strata estimators.
52 */
53#define SE_IBF_HASH_NUM 4
54
55/**
56 * Number of buckets that can be transmitted in one message.
57 */
58#define MAX_BUCKETS_PER_MESSAGE ((1 << 15) / IBF_BUCKET_SIZE)
59
60/**
61 * The maximum size of an ibf we use is 2^(MAX_IBF_ORDER).
62 * Choose this value so that computing the IBF is still cheaper
63 * than transmitting all values.
64 */
65#define MAX_IBF_ORDER (20)
66
67/**
68 * Number of buckets used in the ibf per estimated
69 * difference.
70 */
71#define IBF_ALPHA 4
72
73
74/**
75 * Current phase we are in for a union operation.
76 */
77enum UnionOperationPhase
78{
79 /**
80 * We sent the request message, and expect a strata estimator.
81 */
82 PHASE_EXPECT_SE,
83
84 /**
85 * We sent the strata estimator, and expect an IBF. This phase is entered once
86 * upon initialization and later via #PHASE_EXPECT_ELEMENTS_AND_REQUESTS.
87 *
88 * XXX: could use better wording.
89 * XXX: repurposed to also expect a "request full set" message, should be renamed
90 *
91 * After receiving the complete IBF, we enter #PHASE_EXPECT_ELEMENTS
92 */
93 PHASE_EXPECT_IBF,
94
95 /**
96 * Continuation for multi part IBFs.
97 */
98 PHASE_EXPECT_IBF_CONT,
99
100 /**
101 * We are decoding an IBF.
102 */
103 PHASE_INVENTORY_ACTIVE,
104
105 /**
106 * The other peer is decoding the IBF we just sent.
107 */
108 PHASE_INVENTORY_PASSIVE,
109
110 /**
111 * The protocol is almost finished, but we still have to flush our message
112 * queue and/or expect some elements.
113 */
114 PHASE_FINISH_CLOSING,
115
116 /**
117 * In the penultimate phase,
118 * we wait until all our demands
119 * are satisfied. Then we send a done
120 * message, and wait for another done message.
121 */
122 PHASE_FINISH_WAITING,
123
124 /**
125 * In the ultimate phase, we wait until
126 * our demands are satisfied and then
127 * quit (sending another DONE message).
128 */
129 PHASE_DONE,
130
131 /**
132 * After sending the full set, wait for responses with the elements
133 * that the local peer is missing.
134 */
135 PHASE_FULL_SENDING,
136};
137
138
139/**
140 * State of an evaluate operation with another peer.
141 */
142struct OperationState
143{
144 /**
145 * Copy of the set's strata estimator at the time of
146 * creation of this operation.
147 */
148 struct StrataEstimator *se;
149
150 /**
151 * The IBF we currently receive.
152 */
153 struct InvertibleBloomFilter *remote_ibf;
154
155 /**
156 * The IBF with the local set's element.
157 */
158 struct InvertibleBloomFilter *local_ibf;
159
160 /**
161 * Maps unsalted IBF-Keys to elements.
162 * Used as a multihashmap, the keys being the lower 32bit of the IBF-Key.
163 * Colliding IBF-Keys are linked.
164 */
165 struct GNUNET_CONTAINER_MultiHashMap32 *key_to_element;
166
167 /**
168 * Current state of the operation.
169 */
170 enum UnionOperationPhase phase;
171
172 /**
173 * Did we send the client that we are done?
174 */
175 int client_done_sent;
176
177 /**
178 * Number of ibf buckets already received into the @a remote_ibf.
179 */
180 unsigned int ibf_buckets_received;
181
182 /**
183 * Hashes for elements that we have demanded from the other peer.
184 */
185 struct GNUNET_CONTAINER_MultiHashMap *demanded_hashes;
186
187 /**
188 * Salt that we're using for sending IBFs
189 */
190 uint32_t salt_send;
191
192 /**
193 * Salt for the IBF we've received and that we're currently decoding.
194 */
195 uint32_t salt_receive;
196
197 /**
198 * Number of elements we received from the other peer
199 * that were not in the local set yet.
200 */
201 uint32_t received_fresh;
202
203 /**
204 * Total number of elements received from the other peer.
205 */
206 uint32_t received_total;
207
208 /**
209 * Initial size of our set, just before
210 * the operation started.
211 */
212 uint64_t initial_size;
213};
214
215
216/**
217 * The key entry is used to associate an ibf key with an element.
218 */
219struct KeyEntry
220{
221 /**
222 * IBF key for the entry, derived from the current salt.
223 */
224 struct IBF_Key ibf_key;
225
226 /**
227 * The actual element associated with the key.
228 *
229 * Only owned by the union operation if element->operation
230 * is #GNUNET_YES.
231 */
232 struct ElementEntry *element;
233
234 /**
235 * Did we receive this element?
236 * Even if element->is_foreign is false, we might
237 * have received the element, so this indicates that
238 * the other peer has it.
239 */
240 int received;
241};
242
243
244/**
245 * Used as a closure for sending elements
246 * with a specific IBF key.
247 */
248struct SendElementClosure
249{
250 /**
251 * The IBF key whose matching elements should be
252 * sent.
253 */
254 struct IBF_Key ibf_key;
255
256 /**
257 * Operation for which the elements
258 * should be sent.
259 */
260 struct Operation *op;
261};
262
263
264/**
265 * Extra state required for efficient set union.
266 */
267struct SetState
268{
269 /**
270 * The strata estimator is only generated once for
271 * each set.
272 * The IBF keys are derived from the element hashes with
273 * salt=0.
274 */
275 struct StrataEstimator *se;
276};
277
278
279/**
280 * Iterator over hash map entries, called to
281 * destroy the linked list of colliding ibf key entries.
282 *
283 * @param cls closure
284 * @param key current key code
285 * @param value value in the hash map
286 * @return #GNUNET_YES if we should continue to iterate,
287 * #GNUNET_NO if not.
288 */
289static int
290destroy_key_to_element_iter (void *cls,
291 uint32_t key,
292 void *value)
293{
294 struct KeyEntry *k = value;
295
296 GNUNET_assert (NULL != k);
297 if (GNUNET_YES == k->element->remote)
298 {
299 GNUNET_free (k->element);
300 k->element = NULL;
301 }
302 GNUNET_free (k);
303 return GNUNET_YES;
304}
305
306
307/**
308 * Destroy the union operation. Only things specific to the union
309 * operation are destroyed.
310 *
311 * @param op union operation to destroy
312 */
313static void
314union_op_cancel (struct Operation *op)
315{
316 LOG (GNUNET_ERROR_TYPE_DEBUG,
317 "destroying union op\n");
318 /* check if the op was canceled twice */
319 GNUNET_assert (NULL != op->state);
320 if (NULL != op->state->remote_ibf)
321 {
322 ibf_destroy (op->state->remote_ibf);
323 op->state->remote_ibf = NULL;
324 }
325 if (NULL != op->state->demanded_hashes)
326 {
327 GNUNET_CONTAINER_multihashmap_destroy (op->state->demanded_hashes);
328 op->state->demanded_hashes = NULL;
329 }
330 if (NULL != op->state->local_ibf)
331 {
332 ibf_destroy (op->state->local_ibf);
333 op->state->local_ibf = NULL;
334 }
335 if (NULL != op->state->se)
336 {
337 strata_estimator_destroy (op->state->se);
338 op->state->se = NULL;
339 }
340 if (NULL != op->state->key_to_element)
341 {
342 GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
343 &destroy_key_to_element_iter,
344 NULL);
345 GNUNET_CONTAINER_multihashmap32_destroy (op->state->key_to_element);
346 op->state->key_to_element = NULL;
347 }
348 GNUNET_free (op->state);
349 op->state = NULL;
350 LOG (GNUNET_ERROR_TYPE_DEBUG,
351 "destroying union op done\n");
352}
353
354
355/**
356 * Inform the client that the union operation has failed,
357 * and proceed to destroy the evaluate operation.
358 *
359 * @param op the union operation to fail
360 */
361static void
362fail_union_operation (struct Operation *op)
363{
364 struct GNUNET_MQ_Envelope *ev;
365 struct GNUNET_SET_ResultMessage *msg;
366
367 LOG (GNUNET_ERROR_TYPE_WARNING,
368 "union operation failed\n");
369 ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_RESULT);
370 msg->result_status = htons (GNUNET_SET_STATUS_FAILURE);
371 msg->request_id = htonl (op->client_request_id);
372 msg->element_type = htons (0);
373 GNUNET_MQ_send (op->set->cs->mq,
374 ev);
375 _GSS_operation_destroy (op, GNUNET_YES);
376}
377
378
379/**
380 * Derive the IBF key from a hash code and
381 * a salt.
382 *
383 * @param src the hash code
384 * @return the derived IBF key
385 */
386static struct IBF_Key
387get_ibf_key (const struct GNUNET_HashCode *src)
388{
389 struct IBF_Key key;
390 uint16_t salt = 0;
391
392 GNUNET_assert (GNUNET_OK ==
393 GNUNET_CRYPTO_kdf (&key, sizeof(key),
394 src, sizeof *src,
395 &salt, sizeof(salt),
396 NULL, 0));
397 return key;
398}
399
400
401/**
402 * Context for #op_get_element_iterator
403 */
404struct GetElementContext
405{
406 /**
407 * FIXME.
408 */
409 struct GNUNET_HashCode hash;
410
411 /**
412 * FIXME.
413 */
414 struct KeyEntry *k;
415};
416
417
418/**
419 * Iterator over the mapping from IBF keys to element entries. Checks if we
420 * have an element with a given GNUNET_HashCode.
421 *
422 * @param cls closure
423 * @param key current key code
424 * @param value value in the hash map
425 * @return #GNUNET_YES if we should search further,
426 * #GNUNET_NO if we've found the element.
427 */
428static int
429op_get_element_iterator (void *cls,
430 uint32_t key,
431 void *value)
432{
433 struct GetElementContext *ctx = cls;
434 struct KeyEntry *k = value;
435
436 GNUNET_assert (NULL != k);
437 if (0 == GNUNET_CRYPTO_hash_cmp (&k->element->element_hash,
438 &ctx->hash))
439 {
440 ctx->k = k;
441 return GNUNET_NO;
442 }
443 return GNUNET_YES;
444}
445
446
447/**
448 * Determine whether the given element is already in the operation's element
449 * set.
450 *
451 * @param op operation that should be tested for 'element_hash'
452 * @param element_hash hash of the element to look for
453 * @return #GNUNET_YES if the element has been found, #GNUNET_NO otherwise
454 */
455static struct KeyEntry *
456op_get_element (struct Operation *op,
457 const struct GNUNET_HashCode *element_hash)
458{
459 int ret;
460 struct IBF_Key ibf_key;
461 struct GetElementContext ctx = { { { 0 } }, 0 };
462
463 ctx.hash = *element_hash;
464
465 ibf_key = get_ibf_key (element_hash);
466 ret = GNUNET_CONTAINER_multihashmap32_get_multiple (op->state->key_to_element,
467 (uint32_t) ibf_key.key_val,
468 op_get_element_iterator,
469 &ctx);
470
471 /* was the iteration aborted because we found the element? */
472 if (GNUNET_SYSERR == ret)
473 {
474 GNUNET_assert (NULL != ctx.k);
475 return ctx.k;
476 }
477 return NULL;
478}
479
480
481/**
482 * Insert an element into the union operation's
483 * key-to-element mapping. Takes ownership of 'ee'.
484 * Note that this does not insert the element in the set,
485 * only in the operation's key-element mapping.
486 * This is done to speed up re-tried operations, if some elements
487 * were transmitted, and then the IBF fails to decode.
488 *
489 * XXX: clarify ownership, doesn't sound right.
490 *
491 * @param op the union operation
492 * @param ee the element entry
493 * @param received was this element received from the remote peer?
494 */
495static void
496op_register_element (struct Operation *op,
497 struct ElementEntry *ee,
498 int received)
499{
500 struct IBF_Key ibf_key;
501 struct KeyEntry *k;
502
503 ibf_key = get_ibf_key (&ee->element_hash);
504 k = GNUNET_new (struct KeyEntry);
505 k->element = ee;
506 k->ibf_key = ibf_key;
507 k->received = received;
508 GNUNET_assert (GNUNET_OK ==
509 GNUNET_CONTAINER_multihashmap32_put (op->state->key_to_element,
510 (uint32_t) ibf_key.key_val,
511 k,
512 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
513}
514
515
516/**
517 * FIXME.
518 */
519static void
520salt_key (const struct IBF_Key *k_in,
521 uint32_t salt,
522 struct IBF_Key *k_out)
523{
524 int s = salt % 64;
525 uint64_t x = k_in->key_val;
526
527 /* rotate ibf key */
528 x = (x >> s) | (x << (64 - s));
529 k_out->key_val = x;
530}
531
532
533/**
534 * FIXME.
535 */
536static void
537unsalt_key (const struct IBF_Key *k_in,
538 uint32_t salt,
539 struct IBF_Key *k_out)
540{
541 int s = salt % 64;
542 uint64_t x = k_in->key_val;
543
544 x = (x << s) | (x >> (64 - s));
545 k_out->key_val = x;
546}
547
548
549/**
550 * Insert a key into an ibf.
551 *
552 * @param cls the ibf
553 * @param key unused
554 * @param value the key entry to get the key from
555 */
556static int
557prepare_ibf_iterator (void *cls,
558 uint32_t key,
559 void *value)
560{
561 struct Operation *op = cls;
562 struct KeyEntry *ke = value;
563 struct IBF_Key salted_key;
564
565 LOG (GNUNET_ERROR_TYPE_DEBUG,
566 "[OP %p] inserting %lx (hash %s) into ibf\n",
567 op,
568 (unsigned long) ke->ibf_key.key_val,
569 GNUNET_h2s (&ke->element->element_hash));
570 salt_key (&ke->ibf_key,
571 op->state->salt_send,
572 &salted_key);
573 ibf_insert (op->state->local_ibf, salted_key);
574 return GNUNET_YES;
575}
576
577
578/**
579 * Iterator for initializing the
580 * key-to-element mapping of a union operation
581 *
582 * @param cls the union operation `struct Operation *`
583 * @param key unused
584 * @param value the `struct ElementEntry *` to insert
585 * into the key-to-element mapping
586 * @return #GNUNET_YES (to continue iterating)
587 */
588static int
589init_key_to_element_iterator (void *cls,
590 const struct GNUNET_HashCode *key,
591 void *value)
592{
593 struct Operation *op = cls;
594 struct ElementEntry *ee = value;
595
596 /* make sure that the element belongs to the set at the time
597 * of creating the operation */
598 if (GNUNET_NO ==
599 _GSS_is_element_of_operation (ee,
600 op))
601 return GNUNET_YES;
602 GNUNET_assert (GNUNET_NO == ee->remote);
603 op_register_element (op,
604 ee,
605 GNUNET_NO);
606 return GNUNET_YES;
607}
608
609
610/**
611 * Initialize the IBF key to element mapping local to this set
612 * operation.
613 *
614 * @param op the set union operation
615 */
616static void
617initialize_key_to_element (struct Operation *op)
618{
619 unsigned int len;
620
621 GNUNET_assert (NULL == op->state->key_to_element);
622 len = GNUNET_CONTAINER_multihashmap_size (op->set->content->elements);
623 op->state->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
624 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
625 &init_key_to_element_iterator,
626 op);
627}
628
629
630/**
631 * Create an ibf with the operation's elements
632 * of the specified size
633 *
634 * @param op the union operation
635 * @param size size of the ibf to create
636 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
637 */
638static int
639prepare_ibf (struct Operation *op,
640 uint32_t size)
641{
642 GNUNET_assert (NULL != op->state->key_to_element);
643
644 if (NULL != op->state->local_ibf)
645 ibf_destroy (op->state->local_ibf);
646 op->state->local_ibf = ibf_create (size, SE_IBF_HASH_NUM);
647 if (NULL == op->state->local_ibf)
648 {
649 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
650 "Failed to allocate local IBF\n");
651 return GNUNET_SYSERR;
652 }
653 GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
654 &prepare_ibf_iterator,
655 op);
656 return GNUNET_OK;
657}
658
659
660/**
661 * Send an ibf of appropriate size.
662 *
663 * Fragments the IBF into multiple messages if necessary.
664 *
665 * @param op the union operation
666 * @param ibf_order order of the ibf to send, size=2^order
667 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
668 */
669static int
670send_ibf (struct Operation *op,
671 uint16_t ibf_order)
672{
673 unsigned int buckets_sent = 0;
674 struct InvertibleBloomFilter *ibf;
675
676 if (GNUNET_OK !=
677 prepare_ibf (op, 1 << ibf_order))
678 {
679 /* allocation failed */
680 return GNUNET_SYSERR;
681 }
682
683 LOG (GNUNET_ERROR_TYPE_DEBUG,
684 "sending ibf of size %u\n",
685 1 << ibf_order);
686
687 {
688 char name[64] = { 0 };
689 snprintf (name, sizeof(name), "# sent IBF (order %u)", ibf_order);
690 GNUNET_STATISTICS_update (_GSS_statistics, name, 1, GNUNET_NO);
691 }
692
693 ibf = op->state->local_ibf;
694
695 while (buckets_sent < (1 << ibf_order))
696 {
697 unsigned int buckets_in_message;
698 struct GNUNET_MQ_Envelope *ev;
699 struct IBFMessage *msg;
700
701 buckets_in_message = (1 << ibf_order) - buckets_sent;
702 /* limit to maximum */
703 if (buckets_in_message > MAX_BUCKETS_PER_MESSAGE)
704 buckets_in_message = MAX_BUCKETS_PER_MESSAGE;
705
706 ev = GNUNET_MQ_msg_extra (msg,
707 buckets_in_message * IBF_BUCKET_SIZE,
708 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF);
709 msg->reserved1 = 0;
710 msg->reserved2 = 0;
711 msg->order = ibf_order;
712 msg->offset = htonl (buckets_sent);
713 msg->salt = htonl (op->state->salt_send);
714 ibf_write_slice (ibf, buckets_sent,
715 buckets_in_message, &msg[1]);
716 buckets_sent += buckets_in_message;
717 LOG (GNUNET_ERROR_TYPE_DEBUG,
718 "ibf chunk size %u, %u/%u sent\n",
719 buckets_in_message,
720 buckets_sent,
721 1 << ibf_order);
722 GNUNET_MQ_send (op->mq, ev);
723 }
724
725 /* The other peer must decode the IBF, so
726 * we're passive. */
727 op->state->phase = PHASE_INVENTORY_PASSIVE;
728 return GNUNET_OK;
729}
730
731
732/**
733 * Compute the necessary order of an ibf
734 * from the size of the symmetric set difference.
735 *
736 * @param diff the difference
737 * @return the required size of the ibf
738 */
739static unsigned int
740get_order_from_difference (unsigned int diff)
741{
742 unsigned int ibf_order;
743
744 ibf_order = 2;
745 while (((1 << ibf_order) < (IBF_ALPHA * diff) ||
746 ((1 << ibf_order) < SE_IBF_HASH_NUM)) &&
747 (ibf_order < MAX_IBF_ORDER))
748 ibf_order++;
749 // add one for correction
750 return ibf_order + 1;
751}
752
753
754/**
755 * Send a set element.
756 *
757 * @param cls the union operation `struct Operation *`
758 * @param key unused
759 * @param value the `struct ElementEntry *` to insert
760 * into the key-to-element mapping
761 * @return #GNUNET_YES (to continue iterating)
762 */
763static int
764send_full_element_iterator (void *cls,
765 const struct GNUNET_HashCode *key,
766 void *value)
767{
768 struct Operation *op = cls;
769 struct GNUNET_SET_ElementMessage *emsg;
770 struct ElementEntry *ee = value;
771 struct GNUNET_SET_Element *el = &ee->element;
772 struct GNUNET_MQ_Envelope *ev;
773
774 LOG (GNUNET_ERROR_TYPE_DEBUG,
775 "Sending element %s\n",
776 GNUNET_h2s (key));
777 ev = GNUNET_MQ_msg_extra (emsg,
778 el->size,
779 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
780 emsg->element_type = htons (el->element_type);
781 GNUNET_memcpy (&emsg[1],
782 el->data,
783 el->size);
784 GNUNET_MQ_send (op->mq,
785 ev);
786 return GNUNET_YES;
787}
788
789
790/**
791 * Switch to full set transmission for @a op.
792 *
793 * @param op operation to switch to full set transmission.
794 */
795static void
796send_full_set (struct Operation *op)
797{
798 struct GNUNET_MQ_Envelope *ev;
799
800 op->state->phase = PHASE_FULL_SENDING;
801 LOG (GNUNET_ERROR_TYPE_DEBUG,
802 "Dedicing to transmit the full set\n");
803 /* FIXME: use a more memory-friendly way of doing this with an
804 iterator, just as we do in the non-full case! */
805 (void) GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
806 &send_full_element_iterator,
807 op);
808 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
809 GNUNET_MQ_send (op->mq,
810 ev);
811}
812
813
814/**
815 * Handle a strata estimator from a remote peer
816 *
817 * @param cls the union operation
818 * @param msg the message
819 */
820int
821check_union_p2p_strata_estimator (void *cls,
822 const struct StrataEstimatorMessage *msg)
823{
824 struct Operation *op = cls;
825 int is_compressed;
826 size_t len;
827
828 if (op->state->phase != PHASE_EXPECT_SE)
829 {
830 GNUNET_break (0);
831 return GNUNET_SYSERR;
832 }
833 is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (
834 msg->header.type));
835 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
836 if ((GNUNET_NO == is_compressed) &&
837 (len != SE_STRATA_COUNT * SE_IBF_SIZE * IBF_BUCKET_SIZE))
838 {
839 GNUNET_break (0);
840 return GNUNET_SYSERR;
841 }
842 return GNUNET_OK;
843}
844
845
846/**
847 * Handle a strata estimator from a remote peer
848 *
849 * @param cls the union operation
850 * @param msg the message
851 */
852void
853handle_union_p2p_strata_estimator (void *cls,
854 const struct StrataEstimatorMessage *msg)
855{
856 struct Operation *op = cls;
857 struct StrataEstimator *remote_se;
858 unsigned int diff;
859 uint64_t other_size;
860 size_t len;
861 int is_compressed;
862
863 is_compressed = (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC == htons (
864 msg->header.type));
865 GNUNET_STATISTICS_update (_GSS_statistics,
866 "# bytes of SE received",
867 ntohs (msg->header.size),
868 GNUNET_NO);
869 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
870 other_size = GNUNET_ntohll (msg->set_size);
871 remote_se = strata_estimator_create (SE_STRATA_COUNT,
872 SE_IBF_SIZE,
873 SE_IBF_HASH_NUM);
874 if (NULL == remote_se)
875 {
876 /* insufficient resources, fail */
877 fail_union_operation (op);
878 return;
879 }
880 if (GNUNET_OK !=
881 strata_estimator_read (&msg[1],
882 len,
883 is_compressed,
884 remote_se))
885 {
886 /* decompression failed */
887 strata_estimator_destroy (remote_se);
888 fail_union_operation (op);
889 return;
890 }
891 GNUNET_assert (NULL != op->state->se);
892 diff = strata_estimator_difference (remote_se,
893 op->state->se);
894
895 if (diff > 200)
896 diff = diff * 3 / 2;
897
898 strata_estimator_destroy (remote_se);
899 strata_estimator_destroy (op->state->se);
900 op->state->se = NULL;
901 LOG (GNUNET_ERROR_TYPE_DEBUG,
902 "got se diff=%d, using ibf size %d\n",
903 diff,
904 1U << get_order_from_difference (diff));
905
906 {
907 char *set_debug;
908
909 set_debug = getenv ("GNUNET_SET_BENCHMARK");
910 if ((NULL != set_debug) &&
911 (0 == strcmp (set_debug, "1")))
912 {
913 FILE *f = fopen ("set.log", "a");
914 fprintf (f, "%llu\n", (unsigned long long) diff);
915 fclose (f);
916 }
917 }
918
919 if ((GNUNET_YES == op->byzantine) &&
920 (other_size < op->byzantine_lower_bound))
921 {
922 GNUNET_break (0);
923 fail_union_operation (op);
924 return;
925 }
926
927 if ((GNUNET_YES == op->force_full) ||
928 (diff > op->state->initial_size / 4) ||
929 (0 == other_size))
930 {
931 LOG (GNUNET_ERROR_TYPE_DEBUG,
932 "Deciding to go for full set transmission (diff=%d, own set=%llu)\n",
933 diff,
934 (unsigned long long) op->state->initial_size);
935 GNUNET_STATISTICS_update (_GSS_statistics,
936 "# of full sends",
937 1,
938 GNUNET_NO);
939 if ((op->state->initial_size <= other_size) ||
940 (0 == other_size))
941 {
942 send_full_set (op);
943 }
944 else
945 {
946 struct GNUNET_MQ_Envelope *ev;
947
948 LOG (GNUNET_ERROR_TYPE_DEBUG,
949 "Telling other peer that we expect its full set\n");
950 op->state->phase = PHASE_EXPECT_IBF;
951 ev = GNUNET_MQ_msg_header (
952 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_REQUEST_FULL);
953 GNUNET_MQ_send (op->mq,
954 ev);
955 }
956 }
957 else
958 {
959 GNUNET_STATISTICS_update (_GSS_statistics,
960 "# of ibf sends",
961 1,
962 GNUNET_NO);
963 if (GNUNET_OK !=
964 send_ibf (op,
965 get_order_from_difference (diff)))
966 {
967 /* Internal error, best we can do is shut the connection */
968 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
969 "Failed to send IBF, closing connection\n");
970 fail_union_operation (op);
971 return;
972 }
973 }
974 GNUNET_CADET_receive_done (op->channel);
975}
976
977
978/**
979 * Iterator to send elements to a remote peer
980 *
981 * @param cls closure with the element key and the union operation
982 * @param key ignored
983 * @param value the key entry
984 */
985static int
986send_offers_iterator (void *cls,
987 uint32_t key,
988 void *value)
989{
990 struct SendElementClosure *sec = cls;
991 struct Operation *op = sec->op;
992 struct KeyEntry *ke = value;
993 struct GNUNET_MQ_Envelope *ev;
994 struct GNUNET_MessageHeader *mh;
995
996 /* Detect 32-bit key collision for the 64-bit IBF keys. */
997 if (ke->ibf_key.key_val != sec->ibf_key.key_val)
998 return GNUNET_YES;
999
1000 ev = GNUNET_MQ_msg_header_extra (mh,
1001 sizeof(struct GNUNET_HashCode),
1002 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_OFFER);
1003
1004 GNUNET_assert (NULL != ev);
1005 *(struct GNUNET_HashCode *) &mh[1] = ke->element->element_hash;
1006 LOG (GNUNET_ERROR_TYPE_DEBUG,
1007 "[OP %p] sending element offer (%s) to peer\n",
1008 op,
1009 GNUNET_h2s (&ke->element->element_hash));
1010 GNUNET_MQ_send (op->mq, ev);
1011 return GNUNET_YES;
1012}
1013
1014
1015/**
1016 * Send offers (in the form of GNUNET_Hash-es) to the remote peer for the given IBF key.
1017 *
1018 * @param op union operation
1019 * @param ibf_key IBF key of interest
1020 */
1021static void
1022send_offers_for_key (struct Operation *op,
1023 struct IBF_Key ibf_key)
1024{
1025 struct SendElementClosure send_cls;
1026
1027 send_cls.ibf_key = ibf_key;
1028 send_cls.op = op;
1029 (void) GNUNET_CONTAINER_multihashmap32_get_multiple (
1030 op->state->key_to_element,
1031 (uint32_t) ibf_key.
1032 key_val,
1033 &send_offers_iterator,
1034 &send_cls);
1035}
1036
1037
1038/**
1039 * Decode which elements are missing on each side, and
1040 * send the appropriate offers and inquiries.
1041 *
1042 * @param op union operation
1043 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
1044 */
1045static int
1046decode_and_send (struct Operation *op)
1047{
1048 struct IBF_Key key;
1049 struct IBF_Key last_key;
1050 int side;
1051 unsigned int num_decoded;
1052 struct InvertibleBloomFilter *diff_ibf;
1053
1054 GNUNET_assert (PHASE_INVENTORY_ACTIVE == op->state->phase);
1055
1056 if (GNUNET_OK !=
1057 prepare_ibf (op,
1058 op->state->remote_ibf->size))
1059 {
1060 GNUNET_break (0);
1061 /* allocation failed */
1062 return GNUNET_SYSERR;
1063 }
1064 diff_ibf = ibf_dup (op->state->local_ibf);
1065 ibf_subtract (diff_ibf,
1066 op->state->remote_ibf);
1067
1068 ibf_destroy (op->state->remote_ibf);
1069 op->state->remote_ibf = NULL;
1070
1071 LOG (GNUNET_ERROR_TYPE_DEBUG,
1072 "decoding IBF (size=%u)\n",
1073 diff_ibf->size);
1074
1075 num_decoded = 0;
1076 key.key_val = 0; /* just to avoid compiler thinking we use undef'ed variable */
1077
1078 while (1)
1079 {
1080 int res;
1081 int cycle_detected = GNUNET_NO;
1082
1083 last_key = key;
1084
1085 res = ibf_decode (diff_ibf, &side, &key);
1086 if (res == GNUNET_OK)
1087 {
1088 LOG (GNUNET_ERROR_TYPE_DEBUG,
1089 "decoded ibf key %lx\n",
1090 (unsigned long) key.key_val);
1091 num_decoded += 1;
1092 if ((num_decoded > diff_ibf->size) ||
1093 ((num_decoded > 1) &&
1094 (last_key.key_val == key.key_val)))
1095 {
1096 LOG (GNUNET_ERROR_TYPE_DEBUG,
1097 "detected cyclic ibf (decoded %u/%u)\n",
1098 num_decoded,
1099 diff_ibf->size);
1100 cycle_detected = GNUNET_YES;
1101 }
1102 }
1103 if ((GNUNET_SYSERR == res) ||
1104 (GNUNET_YES == cycle_detected))
1105 {
1106 int next_order;
1107 next_order = 0;
1108 while (1 << next_order < diff_ibf->size)
1109 next_order++;
1110 next_order++;
1111 if (next_order <= MAX_IBF_ORDER)
1112 {
1113 LOG (GNUNET_ERROR_TYPE_DEBUG,
1114 "decoding failed, sending larger ibf (size %u)\n",
1115 1 << next_order);
1116 GNUNET_STATISTICS_update (_GSS_statistics,
1117 "# of IBF retries",
1118 1,
1119 GNUNET_NO);
1120 op->state->salt_send++;
1121 if (GNUNET_OK !=
1122 send_ibf (op, next_order))
1123 {
1124 /* Internal error, best we can do is shut the connection */
1125 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1126 "Failed to send IBF, closing connection\n");
1127 fail_union_operation (op);
1128 ibf_destroy (diff_ibf);
1129 return GNUNET_SYSERR;
1130 }
1131 }
1132 else
1133 {
1134 GNUNET_STATISTICS_update (_GSS_statistics,
1135 "# of failed union operations (too large)",
1136 1,
1137 GNUNET_NO);
1138 // XXX: Send the whole set, element-by-element
1139 LOG (GNUNET_ERROR_TYPE_ERROR,
1140 "set union failed: reached ibf limit\n");
1141 fail_union_operation (op);
1142 ibf_destroy (diff_ibf);
1143 return GNUNET_SYSERR;
1144 }
1145 break;
1146 }
1147 if (GNUNET_NO == res)
1148 {
1149 struct GNUNET_MQ_Envelope *ev;
1150
1151 LOG (GNUNET_ERROR_TYPE_DEBUG,
1152 "transmitted all values, sending DONE\n");
1153 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE);
1154 GNUNET_MQ_send (op->mq, ev);
1155 /* We now wait until we get a DONE message back
1156 * and then wait for our MQ to be flushed and all our
1157 * demands be delivered. */
1158 break;
1159 }
1160 if (1 == side)
1161 {
1162 struct IBF_Key unsalted_key;
1163
1164 unsalt_key (&key,
1165 op->state->salt_receive,
1166 &unsalted_key);
1167 send_offers_for_key (op,
1168 unsalted_key);
1169 }
1170 else if (-1 == side)
1171 {
1172 struct GNUNET_MQ_Envelope *ev;
1173 struct InquiryMessage *msg;
1174
1175 /* It may be nice to merge multiple requests, but with CADET's corking it is not worth
1176 * the effort additional complexity. */
1177 ev = GNUNET_MQ_msg_extra (msg,
1178 sizeof(struct IBF_Key),
1179 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_INQUIRY);
1180 msg->salt = htonl (op->state->salt_receive);
1181 GNUNET_memcpy (&msg[1],
1182 &key,
1183 sizeof(struct IBF_Key));
1184 LOG (GNUNET_ERROR_TYPE_DEBUG,
1185 "sending element inquiry for IBF key %lx\n",
1186 (unsigned long) key.key_val);
1187 GNUNET_MQ_send (op->mq, ev);
1188 }
1189 else
1190 {
1191 GNUNET_assert (0);
1192 }
1193 }
1194 ibf_destroy (diff_ibf);
1195 return GNUNET_OK;
1196}
1197
1198
1199/**
1200 * Check an IBF message from a remote peer.
1201 *
1202 * Reassemble the IBF from multiple pieces, and
1203 * process the whole IBF once possible.
1204 *
1205 * @param cls the union operation
1206 * @param msg the header of the message
1207 * @return #GNUNET_OK if @a msg is well-formed
1208 */
1209int
1210check_union_p2p_ibf (void *cls,
1211 const struct IBFMessage *msg)
1212{
1213 struct Operation *op = cls;
1214 unsigned int buckets_in_message;
1215
1216 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1217 {
1218 GNUNET_break_op (0);
1219 return GNUNET_SYSERR;
1220 }
1221 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
1222 / IBF_BUCKET_SIZE;
1223 if (0 == buckets_in_message)
1224 {
1225 GNUNET_break_op (0);
1226 return GNUNET_SYSERR;
1227 }
1228 if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message
1229 * IBF_BUCKET_SIZE)
1230 {
1231 GNUNET_break_op (0);
1232 return GNUNET_SYSERR;
1233 }
1234 if (op->state->phase == PHASE_EXPECT_IBF_CONT)
1235 {
1236 if (ntohl (msg->offset) != op->state->ibf_buckets_received)
1237 {
1238 GNUNET_break_op (0);
1239 return GNUNET_SYSERR;
1240 }
1241 if (1 << msg->order != op->state->remote_ibf->size)
1242 {
1243 GNUNET_break_op (0);
1244 return GNUNET_SYSERR;
1245 }
1246 if (ntohl (msg->salt) != op->state->salt_receive)
1247 {
1248 GNUNET_break_op (0);
1249 return GNUNET_SYSERR;
1250 }
1251 }
1252 else if ((op->state->phase != PHASE_INVENTORY_PASSIVE) &&
1253 (op->state->phase != PHASE_EXPECT_IBF))
1254 {
1255 GNUNET_break_op (0);
1256 return GNUNET_SYSERR;
1257 }
1258
1259 return GNUNET_OK;
1260}
1261
1262
1263/**
1264 * Handle an IBF message from a remote peer.
1265 *
1266 * Reassemble the IBF from multiple pieces, and
1267 * process the whole IBF once possible.
1268 *
1269 * @param cls the union operation
1270 * @param msg the header of the message
1271 */
1272void
1273handle_union_p2p_ibf (void *cls,
1274 const struct IBFMessage *msg)
1275{
1276 struct Operation *op = cls;
1277 unsigned int buckets_in_message;
1278
1279 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
1280 / IBF_BUCKET_SIZE;
1281 if ((op->state->phase == PHASE_INVENTORY_PASSIVE) ||
1282 (op->state->phase == PHASE_EXPECT_IBF))
1283 {
1284 op->state->phase = PHASE_EXPECT_IBF_CONT;
1285 GNUNET_assert (NULL == op->state->remote_ibf);
1286 LOG (GNUNET_ERROR_TYPE_DEBUG,
1287 "Creating new ibf of size %u\n",
1288 1 << msg->order);
1289 op->state->remote_ibf = ibf_create (1 << msg->order, SE_IBF_HASH_NUM);
1290 op->state->salt_receive = ntohl (msg->salt);
1291 LOG (GNUNET_ERROR_TYPE_DEBUG,
1292 "Receiving new IBF with salt %u\n",
1293 op->state->salt_receive);
1294 if (NULL == op->state->remote_ibf)
1295 {
1296 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1297 "Failed to parse remote IBF, closing connection\n");
1298 fail_union_operation (op);
1299 return;
1300 }
1301 op->state->ibf_buckets_received = 0;
1302 if (0 != ntohl (msg->offset))
1303 {
1304 GNUNET_break_op (0);
1305 fail_union_operation (op);
1306 return;
1307 }
1308 }
1309 else
1310 {
1311 GNUNET_assert (op->state->phase == PHASE_EXPECT_IBF_CONT);
1312 LOG (GNUNET_ERROR_TYPE_DEBUG,
1313 "Received more of IBF\n");
1314 }
1315 GNUNET_assert (NULL != op->state->remote_ibf);
1316
1317 ibf_read_slice (&msg[1],
1318 op->state->ibf_buckets_received,
1319 buckets_in_message,
1320 op->state->remote_ibf);
1321 op->state->ibf_buckets_received += buckets_in_message;
1322
1323 if (op->state->ibf_buckets_received == op->state->remote_ibf->size)
1324 {
1325 LOG (GNUNET_ERROR_TYPE_DEBUG,
1326 "received full ibf\n");
1327 op->state->phase = PHASE_INVENTORY_ACTIVE;
1328 if (GNUNET_OK !=
1329 decode_and_send (op))
1330 {
1331 /* Internal error, best we can do is shut down */
1332 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1333 "Failed to decode IBF, closing connection\n");
1334 fail_union_operation (op);
1335 return;
1336 }
1337 }
1338 GNUNET_CADET_receive_done (op->channel);
1339}
1340
1341
1342/**
1343 * Send a result message to the client indicating
1344 * that there is a new element.
1345 *
1346 * @param op union operation
1347 * @param element element to send
1348 * @param status status to send with the new element
1349 */
1350static void
1351send_client_element (struct Operation *op,
1352 struct GNUNET_SET_Element *element,
1353 int status)
1354{
1355 struct GNUNET_MQ_Envelope *ev;
1356 struct GNUNET_SET_ResultMessage *rm;
1357
1358 LOG (GNUNET_ERROR_TYPE_DEBUG,
1359 "sending element (size %u) to client\n",
1360 element->size);
1361 GNUNET_assert (0 != op->client_request_id);
1362 ev = GNUNET_MQ_msg_extra (rm, element->size, GNUNET_MESSAGE_TYPE_SET_RESULT);
1363 if (NULL == ev)
1364 {
1365 GNUNET_MQ_discard (ev);
1366 GNUNET_break (0);
1367 return;
1368 }
1369 rm->result_status = htons (status);
1370 rm->request_id = htonl (op->client_request_id);
1371 rm->element_type = htons (element->element_type);
1372 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
1373 op->state->key_to_element));
1374 GNUNET_memcpy (&rm[1],
1375 element->data,
1376 element->size);
1377 GNUNET_MQ_send (op->set->cs->mq,
1378 ev);
1379}
1380
1381
1382/**
1383 * Signal to the client that the operation has finished and
1384 * destroy the operation.
1385 *
1386 * @param cls operation to destroy
1387 */
1388static void
1389send_client_done (void *cls)
1390{
1391 struct Operation *op = cls;
1392 struct GNUNET_MQ_Envelope *ev;
1393 struct GNUNET_SET_ResultMessage *rm;
1394
1395 if (GNUNET_YES == op->state->client_done_sent)
1396 {
1397 return;
1398 }
1399
1400 if (PHASE_DONE != op->state->phase)
1401 {
1402 LOG (GNUNET_ERROR_TYPE_WARNING,
1403 "Union operation failed\n");
1404 GNUNET_STATISTICS_update (_GSS_statistics,
1405 "# Union operations failed",
1406 1,
1407 GNUNET_NO);
1408 ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SET_RESULT);
1409 rm->result_status = htons (GNUNET_SET_STATUS_FAILURE);
1410 rm->request_id = htonl (op->client_request_id);
1411 rm->element_type = htons (0);
1412 GNUNET_MQ_send (op->set->cs->mq,
1413 ev);
1414 return;
1415 }
1416
1417 op->state->client_done_sent = GNUNET_YES;
1418
1419 GNUNET_STATISTICS_update (_GSS_statistics,
1420 "# Union operations succeeded",
1421 1,
1422 GNUNET_NO);
1423 LOG (GNUNET_ERROR_TYPE_INFO,
1424 "Signalling client that union operation is done\n");
1425 ev = GNUNET_MQ_msg (rm,
1426 GNUNET_MESSAGE_TYPE_SET_RESULT);
1427 rm->request_id = htonl (op->client_request_id);
1428 rm->result_status = htons (GNUNET_SET_STATUS_DONE);
1429 rm->element_type = htons (0);
1430 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
1431 op->state->key_to_element));
1432 GNUNET_MQ_send (op->set->cs->mq,
1433 ev);
1434}
1435
1436
1437/**
1438 * Tests if the operation is finished, and if so notify.
1439 *
1440 * @param op operation to check
1441 */
1442static void
1443maybe_finish (struct Operation *op)
1444{
1445 unsigned int num_demanded;
1446
1447 num_demanded = GNUNET_CONTAINER_multihashmap_size (
1448 op->state->demanded_hashes);
1449
1450 if (PHASE_FINISH_WAITING == op->state->phase)
1451 {
1452 LOG (GNUNET_ERROR_TYPE_DEBUG,
1453 "In PHASE_FINISH_WAITING, pending %u demands\n",
1454 num_demanded);
1455 if (0 == num_demanded)
1456 {
1457 struct GNUNET_MQ_Envelope *ev;
1458
1459 op->state->phase = PHASE_DONE;
1460 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DONE);
1461 GNUNET_MQ_send (op->mq,
1462 ev);
1463 /* We now wait until the other peer sends P2P_OVER
1464 * after it got all elements from us. */
1465 }
1466 }
1467 if (PHASE_FINISH_CLOSING == op->state->phase)
1468 {
1469 LOG (GNUNET_ERROR_TYPE_DEBUG,
1470 "In PHASE_FINISH_CLOSING, pending %u demands\n",
1471 num_demanded);
1472 if (0 == num_demanded)
1473 {
1474 op->state->phase = PHASE_DONE;
1475 send_client_done (op);
1476 _GSS_operation_destroy2 (op);
1477 }
1478 }
1479}
1480
1481
1482/**
1483 * Check an element message from a remote peer.
1484 *
1485 * @param cls the union operation
1486 * @param emsg the message
1487 */
1488int
1489check_union_p2p_elements (void *cls,
1490 const struct GNUNET_SET_ElementMessage *emsg)
1491{
1492 struct Operation *op = cls;
1493
1494 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1495 {
1496 GNUNET_break_op (0);
1497 return GNUNET_SYSERR;
1498 }
1499 if (0 == GNUNET_CONTAINER_multihashmap_size (op->state->demanded_hashes))
1500 {
1501 GNUNET_break_op (0);
1502 return GNUNET_SYSERR;
1503 }
1504 return GNUNET_OK;
1505}
1506
1507
1508/**
1509 * Handle an element message from a remote peer.
1510 * Sent by the other peer either because we decoded an IBF and placed a demand,
1511 * or because the other peer switched to full set transmission.
1512 *
1513 * @param cls the union operation
1514 * @param emsg the message
1515 */
1516void
1517handle_union_p2p_elements (void *cls,
1518 const struct GNUNET_SET_ElementMessage *emsg)
1519{
1520 struct Operation *op = cls;
1521 struct ElementEntry *ee;
1522 struct KeyEntry *ke;
1523 uint16_t element_size;
1524
1525 element_size = ntohs (emsg->header.size) - sizeof(struct
1526 GNUNET_SET_ElementMessage);
1527 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
1528 GNUNET_memcpy (&ee[1],
1529 &emsg[1],
1530 element_size);
1531 ee->element.size = element_size;
1532 ee->element.data = &ee[1];
1533 ee->element.element_type = ntohs (emsg->element_type);
1534 ee->remote = GNUNET_YES;
1535 GNUNET_SET_element_hash (&ee->element,
1536 &ee->element_hash);
1537 if (GNUNET_NO ==
1538 GNUNET_CONTAINER_multihashmap_remove (op->state->demanded_hashes,
1539 &ee->element_hash,
1540 NULL))
1541 {
1542 /* We got something we didn't demand, since it's not in our map. */
1543 GNUNET_break_op (0);
1544 fail_union_operation (op);
1545 return;
1546 }
1547
1548 LOG (GNUNET_ERROR_TYPE_DEBUG,
1549 "Got element (size %u, hash %s) from peer\n",
1550 (unsigned int) element_size,
1551 GNUNET_h2s (&ee->element_hash));
1552
1553 GNUNET_STATISTICS_update (_GSS_statistics,
1554 "# received elements",
1555 1,
1556 GNUNET_NO);
1557 GNUNET_STATISTICS_update (_GSS_statistics,
1558 "# exchanged elements",
1559 1,
1560 GNUNET_NO);
1561
1562 op->state->received_total++;
1563
1564 ke = op_get_element (op, &ee->element_hash);
1565 if (NULL != ke)
1566 {
1567 /* Got repeated element. Should not happen since
1568 * we track demands. */
1569 GNUNET_STATISTICS_update (_GSS_statistics,
1570 "# repeated elements",
1571 1,
1572 GNUNET_NO);
1573 ke->received = GNUNET_YES;
1574 GNUNET_free (ee);
1575 }
1576 else
1577 {
1578 LOG (GNUNET_ERROR_TYPE_DEBUG,
1579 "Registering new element from remote peer\n");
1580 op->state->received_fresh++;
1581 op_register_element (op, ee, GNUNET_YES);
1582 /* only send results immediately if the client wants it */
1583 switch (op->result_mode)
1584 {
1585 case GNUNET_SET_RESULT_ADDED:
1586 send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
1587 break;
1588
1589 case GNUNET_SET_RESULT_SYMMETRIC:
1590 send_client_element (op, &ee->element, GNUNET_SET_STATUS_ADD_LOCAL);
1591 break;
1592
1593 default:
1594 /* Result mode not supported, should have been caught earlier. */
1595 GNUNET_break (0);
1596 break;
1597 }
1598 }
1599
1600 if ((op->state->received_total > 8) &&
1601 (op->state->received_fresh < op->state->received_total / 3))
1602 {
1603 /* The other peer gave us lots of old elements, there's something wrong. */
1604 GNUNET_break_op (0);
1605 fail_union_operation (op);
1606 return;
1607 }
1608 GNUNET_CADET_receive_done (op->channel);
1609 maybe_finish (op);
1610}
1611
1612
1613/**
1614 * Check a full element message from a remote peer.
1615 *
1616 * @param cls the union operation
1617 * @param emsg the message
1618 */
1619int
1620check_union_p2p_full_element (void *cls,
1621 const struct GNUNET_SET_ElementMessage *emsg)
1622{
1623 struct Operation *op = cls;
1624
1625 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1626 {
1627 GNUNET_break_op (0);
1628 return GNUNET_SYSERR;
1629 }
1630 // FIXME: check that we expect full elements here?
1631 return GNUNET_OK;
1632}
1633
1634
1635/**
1636 * Handle an element message from a remote peer.
1637 *
1638 * @param cls the union operation
1639 * @param emsg the message
1640 */
1641void
1642handle_union_p2p_full_element (void *cls,
1643 const struct GNUNET_SET_ElementMessage *emsg)
1644{
1645 struct Operation *op = cls;
1646 struct ElementEntry *ee;
1647 struct KeyEntry *ke;
1648 uint16_t element_size;
1649
1650 element_size = ntohs (emsg->header.size) - sizeof(struct
1651 GNUNET_SET_ElementMessage);
1652 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
1653 GNUNET_memcpy (&ee[1], &emsg[1], element_size);
1654 ee->element.size = element_size;
1655 ee->element.data = &ee[1];
1656 ee->element.element_type = ntohs (emsg->element_type);
1657 ee->remote = GNUNET_YES;
1658 GNUNET_SET_element_hash (&ee->element, &ee->element_hash);
1659
1660 LOG (GNUNET_ERROR_TYPE_DEBUG,
1661 "Got element (full diff, size %u, hash %s) from peer\n",
1662 (unsigned int) element_size,
1663 GNUNET_h2s (&ee->element_hash));
1664
1665 GNUNET_STATISTICS_update (_GSS_statistics,
1666 "# received elements",
1667 1,
1668 GNUNET_NO);
1669 GNUNET_STATISTICS_update (_GSS_statistics,
1670 "# exchanged elements",
1671 1,
1672 GNUNET_NO);
1673
1674 op->state->received_total++;
1675
1676 ke = op_get_element (op, &ee->element_hash);
1677 if (NULL != ke)
1678 {
1679 /* Got repeated element. Should not happen since
1680 * we track demands. */
1681 GNUNET_STATISTICS_update (_GSS_statistics,
1682 "# repeated elements",
1683 1,
1684 GNUNET_NO);
1685 ke->received = GNUNET_YES;
1686 GNUNET_free (ee);
1687 }
1688 else
1689 {
1690 LOG (GNUNET_ERROR_TYPE_DEBUG,
1691 "Registering new element from remote peer\n");
1692 op->state->received_fresh++;
1693 op_register_element (op, ee, GNUNET_YES);
1694 /* only send results immediately if the client wants it */
1695 switch (op->result_mode)
1696 {
1697 case GNUNET_SET_RESULT_ADDED:
1698 send_client_element (op, &ee->element, GNUNET_SET_STATUS_OK);
1699 break;
1700
1701 case GNUNET_SET_RESULT_SYMMETRIC:
1702 send_client_element (op, &ee->element, GNUNET_SET_STATUS_ADD_LOCAL);
1703 break;
1704
1705 default:
1706 /* Result mode not supported, should have been caught earlier. */
1707 GNUNET_break (0);
1708 break;
1709 }
1710 }
1711
1712 if ((GNUNET_YES == op->byzantine) &&
1713 (op->state->received_total > 384 + op->state->received_fresh * 4) &&
1714 (op->state->received_fresh < op->state->received_total / 6))
1715 {
1716 /* The other peer gave us lots of old elements, there's something wrong. */
1717 LOG (GNUNET_ERROR_TYPE_ERROR,
1718 "Other peer sent only %llu/%llu fresh elements, failing operation\n",
1719 (unsigned long long) op->state->received_fresh,
1720 (unsigned long long) op->state->received_total);
1721 GNUNET_break_op (0);
1722 fail_union_operation (op);
1723 return;
1724 }
1725 GNUNET_CADET_receive_done (op->channel);
1726}
1727
1728
1729/**
1730 * Send offers (for GNUNET_Hash-es) in response
1731 * to inquiries (for IBF_Key-s).
1732 *
1733 * @param cls the union operation
1734 * @param msg the message
1735 */
1736int
1737check_union_p2p_inquiry (void *cls,
1738 const struct InquiryMessage *msg)
1739{
1740 struct Operation *op = cls;
1741 unsigned int num_keys;
1742
1743 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1744 {
1745 GNUNET_break_op (0);
1746 return GNUNET_SYSERR;
1747 }
1748 if (op->state->phase != PHASE_INVENTORY_PASSIVE)
1749 {
1750 GNUNET_break_op (0);
1751 return GNUNET_SYSERR;
1752 }
1753 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
1754 / sizeof(struct IBF_Key);
1755 if ((ntohs (msg->header.size) - sizeof(struct InquiryMessage))
1756 != num_keys * sizeof(struct IBF_Key))
1757 {
1758 GNUNET_break_op (0);
1759 return GNUNET_SYSERR;
1760 }
1761 return GNUNET_OK;
1762}
1763
1764
1765/**
1766 * Send offers (for GNUNET_Hash-es) in response
1767 * to inquiries (for IBF_Key-s).
1768 *
1769 * @param cls the union operation
1770 * @param msg the message
1771 */
1772void
1773handle_union_p2p_inquiry (void *cls,
1774 const struct InquiryMessage *msg)
1775{
1776 struct Operation *op = cls;
1777 const struct IBF_Key *ibf_key;
1778 unsigned int num_keys;
1779
1780 LOG (GNUNET_ERROR_TYPE_DEBUG,
1781 "Received union inquiry\n");
1782 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
1783 / sizeof(struct IBF_Key);
1784 ibf_key = (const struct IBF_Key *) &msg[1];
1785 while (0 != num_keys--)
1786 {
1787 struct IBF_Key unsalted_key;
1788
1789 unsalt_key (ibf_key,
1790 ntohl (msg->salt),
1791 &unsalted_key);
1792 send_offers_for_key (op,
1793 unsalted_key);
1794 ibf_key++;
1795 }
1796 GNUNET_CADET_receive_done (op->channel);
1797}
1798
1799
1800/**
1801 * Iterator over hash map entries, called to
1802 * destroy the linked list of colliding ibf key entries.
1803 *
1804 * @param cls closure
1805 * @param key current key code
1806 * @param value value in the hash map
1807 * @return #GNUNET_YES if we should continue to iterate,
1808 * #GNUNET_NO if not.
1809 */
1810static int
1811send_missing_full_elements_iter (void *cls,
1812 uint32_t key,
1813 void *value)
1814{
1815 struct Operation *op = cls;
1816 struct KeyEntry *ke = value;
1817 struct GNUNET_MQ_Envelope *ev;
1818 struct GNUNET_SET_ElementMessage *emsg;
1819 struct ElementEntry *ee = ke->element;
1820
1821 if (GNUNET_YES == ke->received)
1822 return GNUNET_YES;
1823 ev = GNUNET_MQ_msg_extra (emsg,
1824 ee->element.size,
1825 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_ELEMENT);
1826 GNUNET_memcpy (&emsg[1],
1827 ee->element.data,
1828 ee->element.size);
1829 emsg->element_type = htons (ee->element.element_type);
1830 GNUNET_MQ_send (op->mq,
1831 ev);
1832 return GNUNET_YES;
1833}
1834
1835
1836/**
1837 * Handle a request for full set transmission.
1838 *
1839 * @param cls closure, a set union operation
1840 * @param mh the demand message
1841 */
1842void
1843handle_union_p2p_request_full (void *cls,
1844 const struct GNUNET_MessageHeader *mh)
1845{
1846 struct Operation *op = cls;
1847
1848 LOG (GNUNET_ERROR_TYPE_DEBUG,
1849 "Received request for full set transmission\n");
1850 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1851 {
1852 GNUNET_break_op (0);
1853 fail_union_operation (op);
1854 return;
1855 }
1856 if (PHASE_EXPECT_IBF != op->state->phase)
1857 {
1858 GNUNET_break_op (0);
1859 fail_union_operation (op);
1860 return;
1861 }
1862
1863 // FIXME: we need to check that our set is larger than the
1864 // byzantine_lower_bound by some threshold
1865 send_full_set (op);
1866 GNUNET_CADET_receive_done (op->channel);
1867}
1868
1869
1870/**
1871 * Handle a "full done" message.
1872 *
1873 * @param cls closure, a set union operation
1874 * @param mh the demand message
1875 */
1876void
1877handle_union_p2p_full_done (void *cls,
1878 const struct GNUNET_MessageHeader *mh)
1879{
1880 struct Operation *op = cls;
1881
1882 switch (op->state->phase)
1883 {
1884 case PHASE_EXPECT_IBF:
1885 {
1886 struct GNUNET_MQ_Envelope *ev;
1887
1888 LOG (GNUNET_ERROR_TYPE_DEBUG,
1889 "got FULL DONE, sending elements that other peer is missing\n");
1890
1891 /* send all the elements that did not come from the remote peer */
1892 GNUNET_CONTAINER_multihashmap32_iterate (op->state->key_to_element,
1893 &send_missing_full_elements_iter,
1894 op);
1895
1896 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_UNION_P2P_FULL_DONE);
1897 GNUNET_MQ_send (op->mq,
1898 ev);
1899 op->state->phase = PHASE_DONE;
1900 /* we now wait until the other peer sends us the OVER message*/
1901 }
1902 break;
1903
1904 case PHASE_FULL_SENDING:
1905 {
1906 LOG (GNUNET_ERROR_TYPE_DEBUG,
1907 "got FULL DONE, finishing\n");
1908 /* We sent the full set, and got the response for that. We're done. */
1909 op->state->phase = PHASE_DONE;
1910 GNUNET_CADET_receive_done (op->channel);
1911 send_client_done (op);
1912 _GSS_operation_destroy2 (op);
1913 return;
1914 }
1915 break;
1916
1917 default:
1918 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1919 "Handle full done phase is %u\n",
1920 (unsigned) op->state->phase);
1921 GNUNET_break_op (0);
1922 fail_union_operation (op);
1923 return;
1924 }
1925 GNUNET_CADET_receive_done (op->channel);
1926}
1927
1928
1929/**
1930 * Check a demand by the other peer for elements based on a list
1931 * of `struct GNUNET_HashCode`s.
1932 *
1933 * @param cls closure, a set union operation
1934 * @param mh the demand message
1935 * @return #GNUNET_OK if @a mh is well-formed
1936 */
1937int
1938check_union_p2p_demand (void *cls,
1939 const struct GNUNET_MessageHeader *mh)
1940{
1941 struct Operation *op = cls;
1942 unsigned int num_hashes;
1943
1944 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
1945 {
1946 GNUNET_break_op (0);
1947 return GNUNET_SYSERR;
1948 }
1949 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
1950 / sizeof(struct GNUNET_HashCode);
1951 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
1952 != num_hashes * sizeof(struct GNUNET_HashCode))
1953 {
1954 GNUNET_break_op (0);
1955 return GNUNET_SYSERR;
1956 }
1957 return GNUNET_OK;
1958}
1959
1960
1961/**
1962 * Handle a demand by the other peer for elements based on a list
1963 * of `struct GNUNET_HashCode`s.
1964 *
1965 * @param cls closure, a set union operation
1966 * @param mh the demand message
1967 */
1968void
1969handle_union_p2p_demand (void *cls,
1970 const struct GNUNET_MessageHeader *mh)
1971{
1972 struct Operation *op = cls;
1973 struct ElementEntry *ee;
1974 struct GNUNET_SET_ElementMessage *emsg;
1975 const struct GNUNET_HashCode *hash;
1976 unsigned int num_hashes;
1977 struct GNUNET_MQ_Envelope *ev;
1978
1979 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
1980 / sizeof(struct GNUNET_HashCode);
1981 for (hash = (const struct GNUNET_HashCode *) &mh[1];
1982 num_hashes > 0;
1983 hash++, num_hashes--)
1984 {
1985 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
1986 hash);
1987 if (NULL == ee)
1988 {
1989 /* Demand for non-existing element. */
1990 GNUNET_break_op (0);
1991 fail_union_operation (op);
1992 return;
1993 }
1994 if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
1995 {
1996 /* Probably confused lazily copied sets. */
1997 GNUNET_break_op (0);
1998 fail_union_operation (op);
1999 return;
2000 }
2001 ev = GNUNET_MQ_msg_extra (emsg, ee->element.size,
2002 GNUNET_MESSAGE_TYPE_SET_P2P_ELEMENTS);
2003 GNUNET_memcpy (&emsg[1], ee->element.data, ee->element.size);
2004 emsg->reserved = htons (0);
2005 emsg->element_type = htons (ee->element.element_type);
2006 LOG (GNUNET_ERROR_TYPE_DEBUG,
2007 "[OP %p] Sending demanded element (size %u, hash %s) to peer\n",
2008 op,
2009 (unsigned int) ee->element.size,
2010 GNUNET_h2s (&ee->element_hash));
2011 GNUNET_MQ_send (op->mq, ev);
2012 GNUNET_STATISTICS_update (_GSS_statistics,
2013 "# exchanged elements",
2014 1,
2015 GNUNET_NO);
2016
2017 switch (op->result_mode)
2018 {
2019 case GNUNET_SET_RESULT_ADDED:
2020 /* Nothing to do. */
2021 break;
2022
2023 case GNUNET_SET_RESULT_SYMMETRIC:
2024 send_client_element (op, &ee->element, GNUNET_SET_STATUS_ADD_REMOTE);
2025 break;
2026
2027 default:
2028 /* Result mode not supported, should have been caught earlier. */
2029 GNUNET_break (0);
2030 break;
2031 }
2032 }
2033 GNUNET_CADET_receive_done (op->channel);
2034}
2035
2036
2037/**
2038 * Check offer (of `struct GNUNET_HashCode`s).
2039 *
2040 * @param cls the union operation
2041 * @param mh the message
2042 * @return #GNUNET_OK if @a mh is well-formed
2043 */
2044int
2045check_union_p2p_offer (void *cls,
2046 const struct GNUNET_MessageHeader *mh)
2047{
2048 struct Operation *op = cls;
2049 unsigned int num_hashes;
2050
2051 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
2052 {
2053 GNUNET_break_op (0);
2054 return GNUNET_SYSERR;
2055 }
2056 /* look up elements and send them */
2057 if ((op->state->phase != PHASE_INVENTORY_PASSIVE) &&
2058 (op->state->phase != PHASE_INVENTORY_ACTIVE))
2059 {
2060 GNUNET_break_op (0);
2061 return GNUNET_SYSERR;
2062 }
2063 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
2064 / sizeof(struct GNUNET_HashCode);
2065 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader)) !=
2066 num_hashes * sizeof(struct GNUNET_HashCode))
2067 {
2068 GNUNET_break_op (0);
2069 return GNUNET_SYSERR;
2070 }
2071 return GNUNET_OK;
2072}
2073
2074
2075/**
2076 * Handle offers (of `struct GNUNET_HashCode`s) and
2077 * respond with demands (of `struct GNUNET_HashCode`s).
2078 *
2079 * @param cls the union operation
2080 * @param mh the message
2081 */
2082void
2083handle_union_p2p_offer (void *cls,
2084 const struct GNUNET_MessageHeader *mh)
2085{
2086 struct Operation *op = cls;
2087 const struct GNUNET_HashCode *hash;
2088 unsigned int num_hashes;
2089
2090 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
2091 / sizeof(struct GNUNET_HashCode);
2092 for (hash = (const struct GNUNET_HashCode *) &mh[1];
2093 num_hashes > 0;
2094 hash++, num_hashes--)
2095 {
2096 struct ElementEntry *ee;
2097 struct GNUNET_MessageHeader *demands;
2098 struct GNUNET_MQ_Envelope *ev;
2099
2100 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
2101 hash);
2102 if (NULL != ee)
2103 if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
2104 continue;
2105
2106 if (GNUNET_YES ==
2107 GNUNET_CONTAINER_multihashmap_contains (op->state->demanded_hashes,
2108 hash))
2109 {
2110 LOG (GNUNET_ERROR_TYPE_DEBUG,
2111 "Skipped sending duplicate demand\n");
2112 continue;
2113 }
2114
2115 GNUNET_assert (GNUNET_OK ==
2116 GNUNET_CONTAINER_multihashmap_put (
2117 op->state->demanded_hashes,
2118 hash,
2119 NULL,
2120 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST));
2121
2122 LOG (GNUNET_ERROR_TYPE_DEBUG,
2123 "[OP %p] Requesting element (hash %s)\n",
2124 op, GNUNET_h2s (hash));
2125 ev = GNUNET_MQ_msg_header_extra (demands,
2126 sizeof(struct GNUNET_HashCode),
2127 GNUNET_MESSAGE_TYPE_SET_UNION_P2P_DEMAND);
2128 GNUNET_memcpy (&demands[1],
2129 hash,
2130 sizeof(struct GNUNET_HashCode));
2131 GNUNET_MQ_send (op->mq, ev);
2132 }
2133 GNUNET_CADET_receive_done (op->channel);
2134}
2135
2136
2137/**
2138 * Handle a done message from a remote peer
2139 *
2140 * @param cls the union operation
2141 * @param mh the message
2142 */
2143void
2144handle_union_p2p_done (void *cls,
2145 const struct GNUNET_MessageHeader *mh)
2146{
2147 struct Operation *op = cls;
2148
2149 if (GNUNET_SET_OPERATION_UNION != op->set->operation)
2150 {
2151 GNUNET_break_op (0);
2152 fail_union_operation (op);
2153 return;
2154 }
2155 switch (op->state->phase)
2156 {
2157 case PHASE_INVENTORY_PASSIVE:
2158 /* We got all requests, but still have to send our elements in response. */
2159 op->state->phase = PHASE_FINISH_WAITING;
2160
2161 LOG (GNUNET_ERROR_TYPE_DEBUG,
2162 "got DONE (as passive partner), waiting for our demands to be satisfied\n");
2163 /* The active peer is done sending offers
2164 * and inquiries. This means that all
2165 * our responses to that (demands and offers)
2166 * must be in flight (queued or in mesh).
2167 *
2168 * We should notify the active peer once
2169 * all our demands are satisfied, so that the active
2170 * peer can quit if we gave it everything.
2171 */GNUNET_CADET_receive_done (op->channel);
2172 maybe_finish (op);
2173 return;
2174
2175 case PHASE_INVENTORY_ACTIVE:
2176 LOG (GNUNET_ERROR_TYPE_DEBUG,
2177 "got DONE (as active partner), waiting to finish\n");
2178 /* All demands of the other peer are satisfied,
2179 * and we processed all offers, thus we know
2180 * exactly what our demands must be.
2181 *
2182 * We'll close the channel
2183 * to the other peer once our demands are met.
2184 */op->state->phase = PHASE_FINISH_CLOSING;
2185 GNUNET_CADET_receive_done (op->channel);
2186 maybe_finish (op);
2187 return;
2188
2189 default:
2190 GNUNET_break_op (0);
2191 fail_union_operation (op);
2192 return;
2193 }
2194}
2195
2196
2197/**
2198 * Handle a over message from a remote peer
2199 *
2200 * @param cls the union operation
2201 * @param mh the message
2202 */
2203void
2204handle_union_p2p_over (void *cls,
2205 const struct GNUNET_MessageHeader *mh)
2206{
2207 send_client_done (cls);
2208}
2209
2210
2211/**
2212 * Initiate operation to evaluate a set union with a remote peer.
2213 *
2214 * @param op operation to perform (to be initialized)
2215 * @param opaque_context message to be transmitted to the listener
2216 * to convince it to accept, may be NULL
2217 */
2218static struct OperationState *
2219union_evaluate (struct Operation *op,
2220 const struct GNUNET_MessageHeader *opaque_context)
2221{
2222 struct OperationState *state;
2223 struct GNUNET_MQ_Envelope *ev;
2224 struct OperationRequestMessage *msg;
2225
2226 ev = GNUNET_MQ_msg_nested_mh (msg,
2227 GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST,
2228 opaque_context);
2229 if (NULL == ev)
2230 {
2231 /* the context message is too large */
2232 GNUNET_break (0);
2233 return NULL;
2234 }
2235 state = GNUNET_new (struct OperationState);
2236 state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
2237 GNUNET_NO);
2238 /* copy the current generation's strata estimator for this operation */
2239 state->se = strata_estimator_dup (op->set->state->se);
2240 /* we started the operation, thus we have to send the operation request */
2241 state->phase = PHASE_EXPECT_SE;
2242 state->salt_receive = state->salt_send = 42; // FIXME?????
2243 LOG (GNUNET_ERROR_TYPE_DEBUG,
2244 "Initiating union operation evaluation\n");
2245 GNUNET_STATISTICS_update (_GSS_statistics,
2246 "# of total union operations",
2247 1,
2248 GNUNET_NO);
2249 GNUNET_STATISTICS_update (_GSS_statistics,
2250 "# of initiated union operations",
2251 1,
2252 GNUNET_NO);
2253 msg->operation = htonl (GNUNET_SET_OPERATION_UNION);
2254 GNUNET_MQ_send (op->mq,
2255 ev);
2256
2257 if (NULL != opaque_context)
2258 LOG (GNUNET_ERROR_TYPE_DEBUG,
2259 "sent op request with context message\n");
2260 else
2261 LOG (GNUNET_ERROR_TYPE_DEBUG,
2262 "sent op request without context message\n");
2263
2264 op->state = state;
2265 initialize_key_to_element (op);
2266 state->initial_size = GNUNET_CONTAINER_multihashmap32_size (
2267 state->key_to_element);
2268 return state;
2269}
2270
2271
2272/**
2273 * Accept an union operation request from a remote peer.
2274 * Only initializes the private operation state.
2275 *
2276 * @param op operation that will be accepted as a union operation
2277 */
2278static struct OperationState *
2279union_accept (struct Operation *op)
2280{
2281 struct OperationState *state;
2282 const struct StrataEstimator *se;
2283 struct GNUNET_MQ_Envelope *ev;
2284 struct StrataEstimatorMessage *strata_msg;
2285 char *buf;
2286 size_t len;
2287 uint16_t type;
2288
2289 LOG (GNUNET_ERROR_TYPE_DEBUG,
2290 "accepting set union operation\n");
2291 GNUNET_STATISTICS_update (_GSS_statistics,
2292 "# of accepted union operations",
2293 1,
2294 GNUNET_NO);
2295 GNUNET_STATISTICS_update (_GSS_statistics,
2296 "# of total union operations",
2297 1,
2298 GNUNET_NO);
2299
2300 state = GNUNET_new (struct OperationState);
2301 state->se = strata_estimator_dup (op->set->state->se);
2302 state->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
2303 GNUNET_NO);
2304 state->salt_receive = state->salt_send = 42; // FIXME?????
2305 op->state = state;
2306 initialize_key_to_element (op);
2307 state->initial_size = GNUNET_CONTAINER_multihashmap32_size (
2308 state->key_to_element);
2309
2310 /* kick off the operation */
2311 se = state->se;
2312 buf = GNUNET_malloc (se->strata_count * IBF_BUCKET_SIZE * se->ibf_size);
2313 len = strata_estimator_write (se,
2314 buf);
2315 if (len < se->strata_count * IBF_BUCKET_SIZE * se->ibf_size)
2316 type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SEC;
2317 else
2318 type = GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE;
2319 ev = GNUNET_MQ_msg_extra (strata_msg,
2320 len,
2321 type);
2322 GNUNET_memcpy (&strata_msg[1],
2323 buf,
2324 len);
2325 GNUNET_free (buf);
2326 strata_msg->set_size
2327 = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (
2328 op->set->content->elements));
2329 GNUNET_MQ_send (op->mq,
2330 ev);
2331 state->phase = PHASE_EXPECT_IBF;
2332 return state;
2333}
2334
2335
2336/**
2337 * Create a new set supporting the union operation
2338 *
2339 * We maintain one strata estimator per set and then manipulate it over the
2340 * lifetime of the set, as recreating a strata estimator would be expensive.
2341 *
2342 * @return the newly created set, NULL on error
2343 */
2344static struct SetState *
2345union_set_create (void)
2346{
2347 struct SetState *set_state;
2348
2349 LOG (GNUNET_ERROR_TYPE_DEBUG,
2350 "union set created\n");
2351 set_state = GNUNET_new (struct SetState);
2352 set_state->se = strata_estimator_create (SE_STRATA_COUNT,
2353 SE_IBF_SIZE, SE_IBF_HASH_NUM);
2354 if (NULL == set_state->se)
2355 {
2356 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2357 "Failed to allocate strata estimator\n");
2358 GNUNET_free (set_state);
2359 return NULL;
2360 }
2361 return set_state;
2362}
2363
2364
2365/**
2366 * Add the element from the given element message to the set.
2367 *
2368 * @param set_state state of the set want to add to
2369 * @param ee the element to add to the set
2370 */
2371static void
2372union_add (struct SetState *set_state,
2373 struct ElementEntry *ee)
2374{
2375 strata_estimator_insert (set_state->se,
2376 get_ibf_key (&ee->element_hash));
2377}
2378
2379
2380/**
2381 * Remove the element given in the element message from the set.
2382 * Only marks the element as removed, so that older set operations can still exchange it.
2383 *
2384 * @param set_state state of the set to remove from
2385 * @param ee set element to remove
2386 */
2387static void
2388union_remove (struct SetState *set_state,
2389 struct ElementEntry *ee)
2390{
2391 strata_estimator_remove (set_state->se,
2392 get_ibf_key (&ee->element_hash));
2393}
2394
2395
2396/**
2397 * Destroy a set that supports the union operation.
2398 *
2399 * @param set_state the set to destroy
2400 */
2401static void
2402union_set_destroy (struct SetState *set_state)
2403{
2404 if (NULL != set_state->se)
2405 {
2406 strata_estimator_destroy (set_state->se);
2407 set_state->se = NULL;
2408 }
2409 GNUNET_free (set_state);
2410}
2411
2412
2413/**
2414 * Copy union-specific set state.
2415 *
2416 * @param state source state for copying the union state
2417 * @return a copy of the union-specific set state
2418 */
2419static struct SetState *
2420union_copy_state (struct SetState *state)
2421{
2422 struct SetState *new_state;
2423
2424 GNUNET_assert ((NULL != state) &&
2425 (NULL != state->se));
2426 new_state = GNUNET_new (struct SetState);
2427 new_state->se = strata_estimator_dup (state->se);
2428
2429 return new_state;
2430}
2431
2432
2433/**
2434 * Handle case where channel went down for an operation.
2435 *
2436 * @param op operation that lost the channel
2437 */
2438static void
2439union_channel_death (struct Operation *op)
2440{
2441 send_client_done (op);
2442 _GSS_operation_destroy (op,
2443 GNUNET_YES);
2444}
2445
2446
2447/**
2448 * Get the table with implementing functions for
2449 * set union.
2450 *
2451 * @return the operation specific VTable
2452 */
2453const struct SetVT *
2454_GSS_union_vt ()
2455{
2456 static const struct SetVT union_vt = {
2457 .create = &union_set_create,
2458 .add = &union_add,
2459 .remove = &union_remove,
2460 .destroy_set = &union_set_destroy,
2461 .evaluate = &union_evaluate,
2462 .accept = &union_accept,
2463 .cancel = &union_op_cancel,
2464 .copy_state = &union_copy_state,
2465 .channel_death = &union_channel_death
2466 };
2467
2468 return &union_vt;
2469}
diff --git a/src/set/gnunet-service-set_union.h b/src/set/gnunet-service-set_union.h
deleted file mode 100644
index 68301c96b..000000000
--- a/src/set/gnunet-service-set_union.h
+++ /dev/null
@@ -1,246 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set_union.h
22 * @brief two-peer set operations
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef GNUNET_SERVICE_SET_UNION_H
27#define GNUNET_SERVICE_SET_UNION_H
28
29#include "gnunet-service-set.h"
30#include "gnunet-service-set_protocol.h"
31
32
33/**
34 * Handle a strata estimator from a remote peer
35 *
36 * @param cls the union operation
37 * @param msg the message
38 */
39int
40check_union_p2p_strata_estimator (void *cls,
41 const struct StrataEstimatorMessage *msg);
42
43
44/**
45 * Handle a strata estimator from a remote peer
46 *
47 * @param cls the union operation
48 * @param msg the message
49 */
50void
51handle_union_p2p_strata_estimator (void *cls,
52 const struct StrataEstimatorMessage *msg);
53
54
55/**
56 * Check an IBF message from a remote peer.
57 *
58 * Reassemble the IBF from multiple pieces, and
59 * process the whole IBF once possible.
60 *
61 * @param cls the union operation
62 * @param msg the header of the message
63 * @return #GNUNET_OK if @a msg is well-formed
64 */
65int
66check_union_p2p_ibf (void *cls,
67 const struct IBFMessage *msg);
68
69
70/**
71 * Handle an IBF message from a remote peer.
72 *
73 * Reassemble the IBF from multiple pieces, and
74 * process the whole IBF once possible.
75 *
76 * @param cls the union operation
77 * @param msg the header of the message
78 */
79void
80handle_union_p2p_ibf (void *cls,
81 const struct IBFMessage *msg);
82
83
84/**
85 * Check an element message from a remote peer.
86 *
87 * @param cls the union operation
88 * @param emsg the message
89 */
90int
91check_union_p2p_elements (void *cls,
92 const struct GNUNET_SET_ElementMessage *emsg);
93
94
95/**
96 * Handle an element message from a remote peer.
97 * Sent by the other peer either because we decoded an IBF and placed a demand,
98 * or because the other peer switched to full set transmission.
99 *
100 * @param cls the union operation
101 * @param emsg the message
102 */
103void
104handle_union_p2p_elements (void *cls,
105 const struct GNUNET_SET_ElementMessage *emsg);
106
107
108/**
109 * Check a full element message from a remote peer.
110 *
111 * @param cls the union operation
112 * @param emsg the message
113 */
114int
115check_union_p2p_full_element (void *cls,
116 const struct GNUNET_SET_ElementMessage *emsg);
117
118
119/**
120 * Handle an element message from a remote peer.
121 *
122 * @param cls the union operation
123 * @param emsg the message
124 */
125void
126handle_union_p2p_full_element (void *cls,
127 const struct GNUNET_SET_ElementMessage *emsg);
128
129
130/**
131 * Send offers (for GNUNET_Hash-es) in response
132 * to inquiries (for IBF_Key-s).
133 *
134 * @param cls the union operation
135 * @param msg the message
136 */
137int
138check_union_p2p_inquiry (void *cls,
139 const struct InquiryMessage *msg);
140
141
142/**
143 * Send offers (for GNUNET_Hash-es) in response
144 * to inquiries (for IBF_Key-s).
145 *
146 * @param cls the union operation
147 * @param msg the message
148 */
149void
150handle_union_p2p_inquiry (void *cls,
151 const struct InquiryMessage *msg);
152
153
154/**
155 * Handle a request for full set transmission.
156 *
157 * @param cls closure, a set union operation
158 * @param mh the demand message
159 */
160void
161handle_union_p2p_request_full (void *cls,
162 const struct GNUNET_MessageHeader *mh);
163
164
165/**
166 * Handle a "full done" message.
167 *
168 * @param cls closure, a set union operation
169 * @param mh the demand message
170 */
171void
172handle_union_p2p_full_done (void *cls,
173 const struct GNUNET_MessageHeader *mh);
174
175
176/**
177 * Check a demand by the other peer for elements based on a list
178 * of `struct GNUNET_HashCode`s.
179 *
180 * @param cls closure, a set union operation
181 * @param mh the demand message
182 * @return #GNUNET_OK if @a mh is well-formed
183 */
184int
185check_union_p2p_demand (void *cls,
186 const struct GNUNET_MessageHeader *mh);
187
188
189/**
190 * Handle a demand by the other peer for elements based on a list
191 * of `struct GNUNET_HashCode`s.
192 *
193 * @param cls closure, a set union operation
194 * @param mh the demand message
195 */
196void
197handle_union_p2p_demand (void *cls,
198 const struct GNUNET_MessageHeader *mh);
199
200
201/**
202 * Check offer (of `struct GNUNET_HashCode`s).
203 *
204 * @param cls the union operation
205 * @param mh the message
206 * @return #GNUNET_OK if @a mh is well-formed
207 */
208int
209check_union_p2p_offer (void *cls,
210 const struct GNUNET_MessageHeader *mh);
211
212
213/**
214 * Handle offers (of `struct GNUNET_HashCode`s) and
215 * respond with demands (of `struct GNUNET_HashCode`s).
216 *
217 * @param cls the union operation
218 * @param mh the message
219 */
220void
221handle_union_p2p_offer (void *cls,
222 const struct GNUNET_MessageHeader *mh);
223
224
225/**
226 * Handle a done message from a remote peer
227 *
228 * @param cls the union operation
229 * @param mh the message
230 */
231void
232handle_union_p2p_done (void *cls,
233 const struct GNUNET_MessageHeader *mh);
234
235/**
236 * Handle an over message from a remote peer
237 *
238 * @param cls the union operation
239 * @param mh the message
240 */
241void
242handle_union_p2p_over (void *cls,
243 const struct GNUNET_MessageHeader *mh);
244
245
246#endif
diff --git a/src/set/gnunet-service-set_union_strata_estimator.c b/src/set/gnunet-service-set_union_strata_estimator.c
deleted file mode 100644
index 97b4a1f98..000000000
--- a/src/set/gnunet-service-set_union_strata_estimator.c
+++ /dev/null
@@ -1,303 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-set_union_strata_estimator.c
22 * @brief invertible bloom filter
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "ibf.h"
29#include "gnunet-service-set_union_strata_estimator.h"
30
31
32/**
33 * Should we try compressing the strata estimator? This will
34 * break compatibility with the 0.10.1-network.
35 */
36#define FAIL_10_1_COMPATIBILTIY 1
37
38
39/**
40 * Write the given strata estimator to the buffer.
41 *
42 * @param se strata estimator to serialize
43 * @param[out] buf buffer to write to, must be of appropriate size
44 * @return number of bytes written to @a buf
45 */
46size_t
47strata_estimator_write (const struct StrataEstimator *se,
48 void *buf)
49{
50 char *sbuf = buf;
51 unsigned int i;
52 size_t osize;
53
54 GNUNET_assert (NULL != se);
55 for (i = 0; i < se->strata_count; i++)
56 {
57 ibf_write_slice (se->strata[i],
58 0,
59 se->ibf_size,
60 &sbuf[se->ibf_size * IBF_BUCKET_SIZE * i]);
61 }
62 osize = se->ibf_size * IBF_BUCKET_SIZE * se->strata_count;
63#if FAIL_10_1_COMPATIBILTIY
64 {
65 char *cbuf;
66 size_t nsize;
67
68 if (GNUNET_YES ==
69 GNUNET_try_compression (buf,
70 osize,
71 &cbuf,
72 &nsize))
73 {
74 GNUNET_memcpy (buf, cbuf, nsize);
75 osize = nsize;
76 GNUNET_free (cbuf);
77 }
78 }
79#endif
80 return osize;
81}
82
83
84/**
85 * Read strata from the buffer into the given strata
86 * estimator. The strata estimator must already be allocated.
87 *
88 * @param buf buffer to read from
89 * @param buf_len number of bytes in @a buf
90 * @param is_compressed is the data compressed?
91 * @param[out] se strata estimator to write to
92 * @return #GNUNET_OK on success
93 */
94int
95strata_estimator_read (const void *buf,
96 size_t buf_len,
97 int is_compressed,
98 struct StrataEstimator *se)
99{
100 unsigned int i;
101 size_t osize;
102 char *dbuf;
103
104 dbuf = NULL;
105 if (GNUNET_YES == is_compressed)
106 {
107 osize = se->ibf_size * IBF_BUCKET_SIZE * se->strata_count;
108 dbuf = GNUNET_decompress (buf,
109 buf_len,
110 osize);
111 if (NULL == dbuf)
112 {
113 GNUNET_break_op (0); /* bad compressed input data */
114 return GNUNET_SYSERR;
115 }
116 buf = dbuf;
117 buf_len = osize;
118 }
119
120 if (buf_len != se->strata_count * se->ibf_size * IBF_BUCKET_SIZE)
121 {
122 GNUNET_break (0); /* very odd error */
123 GNUNET_free (dbuf);
124 return GNUNET_SYSERR;
125 }
126
127 for (i = 0; i < se->strata_count; i++)
128 {
129 ibf_read_slice (buf, 0, se->ibf_size, se->strata[i]);
130 buf += se->ibf_size * IBF_BUCKET_SIZE;
131 }
132 GNUNET_free (dbuf);
133 return GNUNET_OK;
134}
135
136
137/**
138 * Add a key to the strata estimator.
139 *
140 * @param se strata estimator to add the key to
141 * @param key key to add
142 */
143void
144strata_estimator_insert (struct StrataEstimator *se,
145 struct IBF_Key key)
146{
147 uint64_t v;
148 unsigned int i;
149
150 v = key.key_val;
151 /* count trailing '1'-bits of v */
152 for (i = 0; v & 1; v >>= 1, i++)
153 /* empty */;
154 ibf_insert (se->strata[i], key);
155}
156
157
158/**
159 * Remove a key from the strata estimator.
160 *
161 * @param se strata estimator to remove the key from
162 * @param key key to remove
163 */
164void
165strata_estimator_remove (struct StrataEstimator *se,
166 struct IBF_Key key)
167{
168 uint64_t v;
169 unsigned int i;
170
171 v = key.key_val;
172 /* count trailing '1'-bits of v */
173 for (i = 0; v & 1; v >>= 1, i++)
174 /* empty */;
175 ibf_remove (se->strata[i], key);
176}
177
178
179/**
180 * Create a new strata estimator with the given parameters.
181 *
182 * @param strata_count number of stratas, that is, number of ibfs in the estimator
183 * @param ibf_size size of each ibf stratum
184 * @param ibf_hashnum hashnum parameter of each ibf
185 * @return a freshly allocated, empty strata estimator, NULL on error
186 */
187struct StrataEstimator *
188strata_estimator_create (unsigned int strata_count,
189 uint32_t ibf_size,
190 uint8_t ibf_hashnum)
191{
192 struct StrataEstimator *se;
193 unsigned int i;
194 unsigned int j;
195
196 se = GNUNET_new (struct StrataEstimator);
197 se->strata_count = strata_count;
198 se->ibf_size = ibf_size;
199 se->strata = GNUNET_new_array (strata_count,
200 struct InvertibleBloomFilter *);
201 for (i = 0; i < strata_count; i++)
202 {
203 se->strata[i] = ibf_create (ibf_size, ibf_hashnum);
204 if (NULL == se->strata[i])
205 {
206 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
207 "Failed to allocate memory for strata estimator\n");
208 for (j = 0; j < i; j++)
209 ibf_destroy (se->strata[i]);
210 GNUNET_free (se);
211 return NULL;
212 }
213 }
214 return se;
215}
216
217
218/**
219 * Estimate set difference with two strata estimators,
220 * i.e. arrays of IBFs.
221 * Does not not modify its arguments.
222 *
223 * @param se1 first strata estimator
224 * @param se2 second strata estimator
225 * @return the estimated difference
226 */
227unsigned int
228strata_estimator_difference (const struct StrataEstimator *se1,
229 const struct StrataEstimator *se2)
230{
231 unsigned int count;
232
233 GNUNET_assert (se1->strata_count == se2->strata_count);
234 count = 0;
235 for (int i = se1->strata_count - 1; i >= 0; i--)
236 {
237 struct InvertibleBloomFilter *diff;
238 /* number of keys decoded from the ibf */
239
240 /* FIXME: implement this without always allocating new IBFs */
241 diff = ibf_dup (se1->strata[i]);
242 ibf_subtract (diff, se2->strata[i]);
243 for (int ibf_count = 0; GNUNET_YES; ibf_count++)
244 {
245 int more;
246
247 more = ibf_decode (diff, NULL, NULL);
248 if (GNUNET_NO == more)
249 {
250 count += ibf_count;
251 break;
252 }
253 /* Estimate if decoding fails or would not terminate */
254 if ((GNUNET_SYSERR == more) || (ibf_count > diff->size))
255 {
256 ibf_destroy (diff);
257 return count * (1 << (i + 1));
258 }
259 }
260 ibf_destroy (diff);
261 }
262 return count;
263}
264
265
266/**
267 * Make a copy of a strata estimator.
268 *
269 * @param se the strata estimator to copy
270 * @return the copy
271 */
272struct StrataEstimator *
273strata_estimator_dup (struct StrataEstimator *se)
274{
275 struct StrataEstimator *c;
276 unsigned int i;
277
278 c = GNUNET_new (struct StrataEstimator);
279 c->strata_count = se->strata_count;
280 c->ibf_size = se->ibf_size;
281 c->strata = GNUNET_new_array (se->strata_count,
282 struct InvertibleBloomFilter *);
283 for (i = 0; i < se->strata_count; i++)
284 c->strata[i] = ibf_dup (se->strata[i]);
285 return c;
286}
287
288
289/**
290 * Destroy a strata estimator, free all of its resources.
291 *
292 * @param se strata estimator to destroy.
293 */
294void
295strata_estimator_destroy (struct StrataEstimator *se)
296{
297 unsigned int i;
298
299 for (i = 0; i < se->strata_count; i++)
300 ibf_destroy (se->strata[i]);
301 GNUNET_free (se->strata);
302 GNUNET_free (se);
303}
diff --git a/src/set/gnunet-service-set_union_strata_estimator.h b/src/set/gnunet-service-set_union_strata_estimator.h
deleted file mode 100644
index 94267a2dc..000000000
--- a/src/set/gnunet-service-set_union_strata_estimator.h
+++ /dev/null
@@ -1,169 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-service-set_union_strata_estimator.h
23 * @brief estimator of set difference
24 * @author Florian Dold
25 */
26
27#ifndef GNUNET_CONSENSUS_STRATA_ESTIMATOR_H
28#define GNUNET_CONSENSUS_STRATA_ESTIMATOR_H
29
30#include "platform.h"
31#include "gnunet_common.h"
32#include "gnunet_util_lib.h"
33
34#ifdef __cplusplus
35extern "C"
36{
37#if 0 /* keep Emacsens' auto-indent happy */
38}
39#endif
40#endif
41
42
43/**
44 * A handle to a strata estimator.
45 */
46struct StrataEstimator
47{
48 /**
49 * The IBFs of this strata estimator.
50 */
51 struct InvertibleBloomFilter **strata;
52
53 /**
54 * Size of the IBF array in @e strata
55 */
56 unsigned int strata_count;
57
58 /**
59 * Size of each IBF stratum (in bytes)
60 */
61 unsigned int ibf_size;
62};
63
64
65/**
66 * Write the given strata estimator to the buffer.
67 *
68 * @param se strata estimator to serialize
69 * @param[out] buf buffer to write to, must be of appropriate size
70 * @return number of bytes written to @a buf
71 */
72size_t
73strata_estimator_write (const struct StrataEstimator *se,
74 void *buf);
75
76
77/**
78 * Read strata from the buffer into the given strata
79 * estimator. The strata estimator must already be allocated.
80 *
81 * @param buf buffer to read from
82 * @param buf_len number of bytes in @a buf
83 * @param is_compressed is the data compressed?
84 * @param[out] se strata estimator to write to
85 * @return #GNUNET_OK on success
86 */
87int
88strata_estimator_read (const void *buf,
89 size_t buf_len,
90 int is_compressed,
91 struct StrataEstimator *se);
92
93
94/**
95 * Create a new strata estimator with the given parameters.
96 *
97 * @param strata_count number of stratas, that is, number of ibfs in the estimator
98 * @param ibf_size size of each ibf stratum
99 * @param ibf_hashnum hashnum parameter of each ibf
100 * @return a freshly allocated, empty strata estimator, NULL on error
101 */
102struct StrataEstimator *
103strata_estimator_create (unsigned int strata_count,
104 uint32_t ibf_size,
105 uint8_t ibf_hashnum);
106
107
108/**
109 * Get an estimation of the symmetric difference of the elements
110 * contained in both strata estimators.
111 *
112 * @param se1 first strata estimator
113 * @param se2 second strata estimator
114 * @return abs(|se1| - |se2|)
115 */
116unsigned int
117strata_estimator_difference (const struct StrataEstimator *se1,
118 const struct StrataEstimator *se2);
119
120
121/**
122 * Add a key to the strata estimator.
123 *
124 * @param se strata estimator to add the key to
125 * @param key key to add
126 */
127void
128strata_estimator_insert (struct StrataEstimator *se,
129 struct IBF_Key key);
130
131
132/**
133 * Remove a key from the strata estimator.
134 *
135 * @param se strata estimator to remove the key from
136 * @param key key to remove
137 */
138void
139strata_estimator_remove (struct StrataEstimator *se,
140 struct IBF_Key key);
141
142
143/**
144 * Destroy a strata estimator, free all of its resources.
145 *
146 * @param se strata estimator to destroy.
147 */
148void
149strata_estimator_destroy (struct StrataEstimator *se);
150
151
152/**
153 * Make a copy of a strata estimator.
154 *
155 * @param se the strata estimator to copy
156 * @return the copy
157 */
158struct StrataEstimator *
159strata_estimator_dup (struct StrataEstimator *se);
160
161
162#if 0 /* keep Emacsens' auto-indent happy */
163{
164#endif
165#ifdef __cplusplus
166}
167#endif
168
169#endif
diff --git a/src/set/gnunet-set-ibf-profiler.c b/src/set/gnunet-set-ibf-profiler.c
deleted file mode 100644
index 944b63d30..000000000
--- a/src/set/gnunet-set-ibf-profiler.c
+++ /dev/null
@@ -1,308 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-set-ibf-profiler.c
23 * @brief tool for profiling the invertible bloom filter implementation
24 * @author Florian Dold
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29
30#include "ibf.h"
31
32static unsigned int asize = 10;
33static unsigned int bsize = 10;
34static unsigned int csize = 10;
35static unsigned int hash_num = 4;
36static unsigned int ibf_size = 80;
37
38/* FIXME: add parameter for this */
39static enum GNUNET_CRYPTO_Quality random_quality = GNUNET_CRYPTO_QUALITY_WEAK;
40
41static struct GNUNET_CONTAINER_MultiHashMap *set_a;
42static struct GNUNET_CONTAINER_MultiHashMap *set_b;
43/* common elements in a and b */
44static struct GNUNET_CONTAINER_MultiHashMap *set_c;
45
46static struct GNUNET_CONTAINER_MultiHashMap *key_to_hashcode;
47
48static struct InvertibleBloomFilter *ibf_a;
49static struct InvertibleBloomFilter *ibf_b;
50
51
52static void
53register_hashcode (struct GNUNET_HashCode *hash)
54{
55 struct GNUNET_HashCode replicated;
56 struct IBF_Key key;
57
58 key = ibf_key_from_hashcode (hash);
59 ibf_hashcode_from_key (key, &replicated);
60 (void) GNUNET_CONTAINER_multihashmap_put (
61 key_to_hashcode,
62 &replicated,
63 GNUNET_memdup (hash, sizeof *hash),
64 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
65}
66
67
68static void
69iter_hashcodes (struct IBF_Key key,
70 GNUNET_CONTAINER_MulitHashMapIteratorCallback iter,
71 void *cls)
72{
73 struct GNUNET_HashCode replicated;
74
75 ibf_hashcode_from_key (key, &replicated);
76 GNUNET_CONTAINER_multihashmap_get_multiple (key_to_hashcode,
77 &replicated,
78 iter,
79 cls);
80}
81
82
83static int
84insert_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
85{
86 struct InvertibleBloomFilter *ibf = cls;
87
88 ibf_insert (ibf, ibf_key_from_hashcode (key));
89 return GNUNET_YES;
90}
91
92
93static int
94remove_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
95{
96 struct GNUNET_CONTAINER_MultiHashMap *hashmap = cls;
97
98 /* if remove fails, there just was a collision with another key */
99 (void) GNUNET_CONTAINER_multihashmap_remove (hashmap, value, NULL);
100 return GNUNET_YES;
101}
102
103
104static void
105run (void *cls,
106 char *const *args,
107 const char *cfgfile,
108 const struct GNUNET_CONFIGURATION_Handle *cfg)
109{
110 struct GNUNET_HashCode id;
111 struct IBF_Key ibf_key;
112 int i;
113 int side;
114 int res;
115 struct GNUNET_TIME_Absolute start_time;
116 struct GNUNET_TIME_Relative delta_time;
117
118 set_a =
119 GNUNET_CONTAINER_multihashmap_create (((asize == 0) ? 1 : (asize + csize)),
120 GNUNET_NO);
121 set_b =
122 GNUNET_CONTAINER_multihashmap_create (((bsize == 0) ? 1 : (bsize + csize)),
123 GNUNET_NO);
124 set_c = GNUNET_CONTAINER_multihashmap_create (((csize == 0) ? 1 : csize),
125 GNUNET_NO);
126
127 key_to_hashcode =
128 GNUNET_CONTAINER_multihashmap_create (((asize + bsize + csize == 0)
129 ? 1
130 : (asize + bsize + csize)),
131 GNUNET_NO);
132
133 printf ("hash-num=%u, size=%u, #(A-B)=%u, #(B-A)=%u, #(A&B)=%u\n",
134 hash_num,
135 ibf_size,
136 asize,
137 bsize,
138 csize);
139
140 i = 0;
141 while (i < asize)
142 {
143 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
144 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
145 continue;
146 GNUNET_break (GNUNET_OK ==
147 GNUNET_CONTAINER_multihashmap_put (
148 set_a,
149 &id,
150 NULL,
151 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
152 register_hashcode (&id);
153 i++;
154 }
155 i = 0;
156 while (i < bsize)
157 {
158 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
159 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
160 continue;
161 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
162 continue;
163 GNUNET_break (GNUNET_OK ==
164 GNUNET_CONTAINER_multihashmap_put (
165 set_b,
166 &id,
167 NULL,
168 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
169 register_hashcode (&id);
170 i++;
171 }
172 i = 0;
173 while (i < csize)
174 {
175 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
176 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
177 continue;
178 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
179 continue;
180 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_c, &id))
181 continue;
182 GNUNET_break (GNUNET_OK ==
183 GNUNET_CONTAINER_multihashmap_put (
184 set_c,
185 &id,
186 NULL,
187 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
188 register_hashcode (&id);
189 i++;
190 }
191
192 ibf_a = ibf_create (ibf_size, hash_num);
193 ibf_b = ibf_create (ibf_size, hash_num);
194 if ((NULL == ibf_a) || (NULL == ibf_b))
195 {
196 /* insufficient memory */
197 GNUNET_break (0);
198 GNUNET_SCHEDULER_shutdown ();
199 return;
200 }
201
202
203 printf ("generated sets\n");
204
205 start_time = GNUNET_TIME_absolute_get ();
206
207 GNUNET_CONTAINER_multihashmap_iterate (set_a, &insert_iterator, ibf_a);
208 GNUNET_CONTAINER_multihashmap_iterate (set_b, &insert_iterator, ibf_b);
209 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_a);
210 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_b);
211
212 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
213
214 printf ("encoded in: %s\n",
215 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
216
217 ibf_subtract (ibf_a, ibf_b);
218
219
220 start_time = GNUNET_TIME_absolute_get ();
221
222 for (i = 0; i <= asize + bsize; i++)
223 {
224 res = ibf_decode (ibf_a, &side, &ibf_key);
225 if (GNUNET_SYSERR == res)
226 {
227 printf ("decode failed, %u/%u elements left\n",
228 GNUNET_CONTAINER_multihashmap_size (set_a)
229 + GNUNET_CONTAINER_multihashmap_size (set_b),
230 asize + bsize);
231 return;
232 }
233 if (GNUNET_NO == res)
234 {
235 if ((0 == GNUNET_CONTAINER_multihashmap_size (set_b)) &&
236 (0 == GNUNET_CONTAINER_multihashmap_size (set_a)))
237 {
238 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
239 printf ("decoded successfully in: %s\n",
240 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
241 }
242 else
243 {
244 printf ("decode missed elements (should never happen)\n");
245 }
246 return;
247 }
248
249 if (side == 1)
250 iter_hashcodes (ibf_key, remove_iterator, set_a);
251 if (side == -1)
252 iter_hashcodes (ibf_key, remove_iterator, set_b);
253 }
254 printf ("cyclic IBF, %u/%u elements left\n",
255 GNUNET_CONTAINER_multihashmap_size (set_a)
256 + GNUNET_CONTAINER_multihashmap_size (set_b),
257 asize + bsize);
258}
259
260
261int
262main (int argc, char **argv)
263{
264 struct GNUNET_GETOPT_CommandLineOption options[] = {
265 GNUNET_GETOPT_option_uint ('A',
266 "asize",
267 NULL,
268 gettext_noop ("number of element in set A-B"),
269 &asize),
270
271 GNUNET_GETOPT_option_uint ('B',
272 "bsize",
273 NULL,
274 gettext_noop ("number of element in set B-A"),
275 &bsize),
276
277 GNUNET_GETOPT_option_uint ('C',
278 "csize",
279 NULL,
280 gettext_noop (
281 "number of common elements in A and B"),
282 &csize),
283
284 GNUNET_GETOPT_option_uint ('k',
285 "hash-num",
286 NULL,
287 gettext_noop ("hash num"),
288 &hash_num),
289
290 GNUNET_GETOPT_option_uint ('s',
291 "ibf-size",
292 NULL,
293 gettext_noop ("ibf size"),
294 &ibf_size),
295
296 GNUNET_GETOPT_OPTION_END
297 };
298
299 GNUNET_PROGRAM_run2 (argc,
300 argv,
301 "gnunet-consensus-ibf",
302 "help",
303 options,
304 &run,
305 NULL,
306 GNUNET_YES);
307 return 0;
308}
diff --git a/src/set/gnunet-set-profiler.c b/src/set/gnunet-set-profiler.c
deleted file mode 100644
index 3014861a6..000000000
--- a/src/set/gnunet-set-profiler.c
+++ /dev/null
@@ -1,508 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-set-profiler.c
23 * @brief profiling tool for set
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_statistics_service.h"
29#include "gnunet_set_service.h"
30#include "gnunet_testbed_service.h"
31
32
33static int ret;
34
35static unsigned int num_a = 5;
36static unsigned int num_b = 5;
37static unsigned int num_c = 20;
38
39static char *op_str = "union";
40
41const static struct GNUNET_CONFIGURATION_Handle *config;
42
43struct SetInfo
44{
45 char *id;
46 struct GNUNET_SET_Handle *set;
47 struct GNUNET_SET_OperationHandle *oh;
48 struct GNUNET_CONTAINER_MultiHashMap *sent;
49 struct GNUNET_CONTAINER_MultiHashMap *received;
50 int done;
51} info1, info2;
52
53static struct GNUNET_CONTAINER_MultiHashMap *common_sent;
54
55static struct GNUNET_HashCode app_id;
56
57static struct GNUNET_PeerIdentity local_peer;
58
59static struct GNUNET_SET_ListenHandle *set_listener;
60
61static int byzantine;
62static unsigned int force_delta;
63static unsigned int force_full;
64static unsigned int element_size = 32;
65
66/**
67 * Handle to the statistics service.
68 */
69static struct GNUNET_STATISTICS_Handle *statistics;
70
71/**
72 * The profiler will write statistics
73 * for all peers to the file with this name.
74 */
75static char *statistics_filename;
76
77/**
78 * The profiler will write statistics
79 * for all peers to this file.
80 */
81static FILE *statistics_file;
82
83
84static int
85map_remove_iterator (void *cls,
86 const struct GNUNET_HashCode *key,
87 void *value)
88{
89 struct GNUNET_CONTAINER_MultiHashMap *m = cls;
90 int ret;
91
92 GNUNET_assert (NULL != key);
93
94 ret = GNUNET_CONTAINER_multihashmap_remove_all (m, key);
95 if (GNUNET_OK != ret)
96 printf ("spurious element\n");
97 return GNUNET_YES;
98}
99
100
101/**
102 * Callback function to process statistic values.
103 *
104 * @param cls closure
105 * @param subsystem name of subsystem that created the statistic
106 * @param name the name of the datum
107 * @param value the current value
108 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
109 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
110 */
111static int
112statistics_result (void *cls,
113 const char *subsystem,
114 const char *name,
115 uint64_t value,
116 int is_persistent)
117{
118 if (NULL != statistics_file)
119 {
120 fprintf (statistics_file, "%s\t%s\t%lu\n", subsystem, name, (unsigned
121 long) value);
122 }
123 return GNUNET_OK;
124}
125
126
127static void
128statistics_done (void *cls,
129 int success)
130{
131 GNUNET_assert (GNUNET_YES == success);
132 if (NULL != statistics_file)
133 fclose (statistics_file);
134 GNUNET_SCHEDULER_shutdown ();
135}
136
137
138static void
139check_all_done (void)
140{
141 if ((info1.done == GNUNET_NO) || (info2.done == GNUNET_NO))
142 return;
143
144 GNUNET_CONTAINER_multihashmap_iterate (info1.received, map_remove_iterator,
145 info2.sent);
146 GNUNET_CONTAINER_multihashmap_iterate (info2.received, map_remove_iterator,
147 info1.sent);
148
149 printf ("set a: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
150 info1.sent));
151 printf ("set b: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
152 info2.sent));
153
154 if (NULL == statistics_filename)
155 {
156 GNUNET_SCHEDULER_shutdown ();
157 return;
158 }
159
160 statistics_file = fopen (statistics_filename, "w");
161 GNUNET_STATISTICS_get (statistics, NULL, NULL,
162 &statistics_done,
163 &statistics_result, NULL);
164}
165
166
167static void
168set_result_cb (void *cls,
169 const struct GNUNET_SET_Element *element,
170 uint64_t current_size,
171 enum GNUNET_SET_Status status)
172{
173 struct SetInfo *info = cls;
174 struct GNUNET_HashCode hash;
175
176 GNUNET_assert (GNUNET_NO == info->done);
177 switch (status)
178 {
179 case GNUNET_SET_STATUS_DONE:
180 case GNUNET_SET_STATUS_HALF_DONE:
181 info->done = GNUNET_YES;
182 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s done\n", info->id);
183 check_all_done ();
184 info->oh = NULL;
185 return;
186
187 case GNUNET_SET_STATUS_FAILURE:
188 info->oh = NULL;
189 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "failure\n");
190 GNUNET_SCHEDULER_shutdown ();
191 return;
192
193 case GNUNET_SET_STATUS_ADD_LOCAL:
194 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: local element\n", info->id);
195 break;
196
197 case GNUNET_SET_STATUS_ADD_REMOTE:
198 GNUNET_CRYPTO_hash (element->data, element->size, &hash);
199 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: remote element %s\n", info->id,
200 GNUNET_h2s (&hash));
201 // XXX: record and check
202 return;
203
204 default:
205 GNUNET_assert (0);
206 }
207
208 if (element->size != element_size)
209 {
210 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
211 "wrong element size: %u, expected %u\n",
212 element->size,
213 (unsigned int) sizeof(struct GNUNET_HashCode));
214 GNUNET_assert (0);
215 }
216
217 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: got element (%s)\n",
218 info->id, GNUNET_h2s (element->data));
219 GNUNET_assert (NULL != element->data);
220 struct GNUNET_HashCode data_hash;
221 GNUNET_CRYPTO_hash (element->data, element_size, &data_hash);
222 GNUNET_CONTAINER_multihashmap_put (info->received,
223 &data_hash, NULL,
224 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
225}
226
227
228static void
229set_listen_cb (void *cls,
230 const struct GNUNET_PeerIdentity *other_peer,
231 const struct GNUNET_MessageHeader *context_msg,
232 struct GNUNET_SET_Request *request)
233{
234 /* max. 2 options plus terminator */
235 struct GNUNET_SET_Option opts[3] = { { 0 } };
236 unsigned int n_opts = 0;
237
238 if (NULL == request)
239 {
240 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
241 "listener failed\n");
242 return;
243 }
244 GNUNET_assert (NULL == info2.oh);
245 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
246 "set listen cb called\n");
247 if (byzantine)
248 {
249 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
250 GNUNET_SET_OPTION_BYZANTINE };
251 }
252 GNUNET_assert (! (force_full && force_delta));
253 if (force_full)
254 {
255 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
256 GNUNET_SET_OPTION_FORCE_FULL };
257 }
258 if (force_delta)
259 {
260 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
261 GNUNET_SET_OPTION_FORCE_DELTA };
262 }
263
264 opts[n_opts].type = 0;
265 info2.oh = GNUNET_SET_accept (request, GNUNET_SET_RESULT_SYMMETRIC,
266 opts,
267 set_result_cb, &info2);
268 GNUNET_SET_commit (info2.oh, info2.set);
269}
270
271
272static int
273set_insert_iterator (void *cls,
274 const struct GNUNET_HashCode *key,
275 void *value)
276{
277 struct GNUNET_SET_Handle *set = cls;
278 struct GNUNET_SET_Element el;
279
280 el.element_type = 0;
281 el.data = value;
282 el.size = element_size;
283 GNUNET_SET_add_element (set, &el, NULL, NULL);
284 return GNUNET_YES;
285}
286
287
288static void
289handle_shutdown (void *cls)
290{
291 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
292 "Shutting down set profiler\n");
293 if (NULL != set_listener)
294 {
295 GNUNET_SET_listen_cancel (set_listener);
296 set_listener = NULL;
297 }
298 if (NULL != info1.oh)
299 {
300 GNUNET_SET_operation_cancel (info1.oh);
301 info1.oh = NULL;
302 }
303 if (NULL != info2.oh)
304 {
305 GNUNET_SET_operation_cancel (info2.oh);
306 info2.oh = NULL;
307 }
308 if (NULL != info1.set)
309 {
310 GNUNET_SET_destroy (info1.set);
311 info1.set = NULL;
312 }
313 if (NULL != info2.set)
314 {
315 GNUNET_SET_destroy (info2.set);
316 info2.set = NULL;
317 }
318 GNUNET_STATISTICS_destroy (statistics, GNUNET_NO);
319}
320
321
322static void
323run (void *cls,
324 const struct GNUNET_CONFIGURATION_Handle *cfg,
325 struct GNUNET_TESTING_Peer *peer)
326{
327 unsigned int i;
328 struct GNUNET_HashCode hash;
329 /* max. 2 options plus terminator */
330 struct GNUNET_SET_Option opts[3] = { { 0 } };
331 unsigned int n_opts = 0;
332
333 config = cfg;
334
335 GNUNET_assert (element_size > 0);
336
337 if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &local_peer))
338 {
339 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
340 ret = 0;
341 return;
342 }
343
344 statistics = GNUNET_STATISTICS_create ("set-profiler", cfg);
345
346 GNUNET_SCHEDULER_add_shutdown (&handle_shutdown, NULL);
347
348 info1.id = "a";
349 info2.id = "b";
350
351 info1.sent = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
352 info2.sent = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
353 common_sent = GNUNET_CONTAINER_multihashmap_create (num_c + 1, GNUNET_NO);
354
355 info1.received = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
356 info2.received = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
357
358 for (i = 0; i < num_a; i++)
359 {
360 char *data = GNUNET_malloc (element_size);
361 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
362 GNUNET_CRYPTO_hash (data, element_size, &hash);
363 GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, data,
364 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
365 }
366
367 for (i = 0; i < num_b; i++)
368 {
369 char *data = GNUNET_malloc (element_size);
370 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
371 GNUNET_CRYPTO_hash (data, element_size, &hash);
372 GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, data,
373 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
374 }
375
376 for (i = 0; i < num_c; i++)
377 {
378 char *data = GNUNET_malloc (element_size);
379 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
380 GNUNET_CRYPTO_hash (data, element_size, &hash);
381 GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, data,
382 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
383 }
384
385 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &app_id);
386
387 /* FIXME: also implement intersection etc. */
388 info1.set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
389 info2.set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
390
391 GNUNET_CONTAINER_multihashmap_iterate (info1.sent, set_insert_iterator,
392 info1.set);
393 GNUNET_CONTAINER_multihashmap_iterate (info2.sent, set_insert_iterator,
394 info2.set);
395 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
396 info1.set);
397 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
398 info2.set);
399
400 set_listener = GNUNET_SET_listen (config, GNUNET_SET_OPERATION_UNION,
401 &app_id, set_listen_cb, NULL);
402
403
404 if (byzantine)
405 {
406 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
407 GNUNET_SET_OPTION_BYZANTINE };
408 }
409 GNUNET_assert (! (force_full && force_delta));
410 if (force_full)
411 {
412 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
413 GNUNET_SET_OPTION_FORCE_FULL };
414 }
415 if (force_delta)
416 {
417 opts[n_opts++] = (struct GNUNET_SET_Option) { .type =
418 GNUNET_SET_OPTION_FORCE_DELTA };
419 }
420
421 opts[n_opts].type = 0;
422
423 info1.oh = GNUNET_SET_prepare (&local_peer, &app_id, NULL,
424 GNUNET_SET_RESULT_SYMMETRIC,
425 opts,
426 set_result_cb, &info1);
427 GNUNET_SET_commit (info1.oh, info1.set);
428 GNUNET_SET_destroy (info1.set);
429 info1.set = NULL;
430}
431
432
433static void
434pre_run (void *cls, char *const *args, const char *cfgfile,
435 const struct GNUNET_CONFIGURATION_Handle *cfg)
436{
437 if (0 != GNUNET_TESTING_peer_run ("set-profiler",
438 cfgfile,
439 &run, NULL))
440 ret = 2;
441}
442
443
444int
445main (int argc, char **argv)
446{
447 struct GNUNET_GETOPT_CommandLineOption options[] = {
448 GNUNET_GETOPT_option_uint ('A',
449 "num-first",
450 NULL,
451 gettext_noop ("number of values"),
452 &num_a),
453
454 GNUNET_GETOPT_option_uint ('B',
455 "num-second",
456 NULL,
457 gettext_noop ("number of values"),
458 &num_b),
459
460 GNUNET_GETOPT_option_flag ('b',
461 "byzantine",
462 gettext_noop ("use byzantine mode"),
463 &byzantine),
464
465 GNUNET_GETOPT_option_uint ('f',
466 "force-full",
467 NULL,
468 gettext_noop ("force sending full set"),
469 &force_full),
470
471 GNUNET_GETOPT_option_uint ('d',
472 "force-delta",
473 NULL,
474 gettext_noop ("number delta operation"),
475 &force_delta),
476
477 GNUNET_GETOPT_option_uint ('C',
478 "num-common",
479 NULL,
480 gettext_noop ("number of values"),
481 &num_c),
482
483 GNUNET_GETOPT_option_string ('x',
484 "operation",
485 NULL,
486 gettext_noop ("operation to execute"),
487 &op_str),
488
489 GNUNET_GETOPT_option_uint ('w',
490 "element-size",
491 NULL,
492 gettext_noop ("element size"),
493 &element_size),
494
495 GNUNET_GETOPT_option_filename ('s',
496 "statistics",
497 "FILENAME",
498 gettext_noop ("write statistics to file"),
499 &statistics_filename),
500
501 GNUNET_GETOPT_OPTION_END
502 };
503
504 GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler",
505 "help",
506 options, &pre_run, NULL, GNUNET_YES);
507 return ret;
508}
diff --git a/src/set/ibf.c b/src/set/ibf.c
deleted file mode 100644
index 0f7eb6a9f..000000000
--- a/src/set/ibf.c
+++ /dev/null
@@ -1,409 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.c
23 * @brief implementation of the invertible bloom filter
24 * @author Florian Dold
25 */
26
27#include "ibf.h"
28
29/**
30 * Compute the key's hash from the key.
31 * Redefine to use a different hash function.
32 */
33#define IBF_KEY_HASH_VAL(k) (GNUNET_CRYPTO_crc32_n (&(k), sizeof(struct \
34 IBF_KeyHash)))
35
36/**
37 * Create a key from a hashcode.
38 *
39 * @param hash the hashcode
40 * @return a key
41 */
42struct IBF_Key
43ibf_key_from_hashcode (const struct GNUNET_HashCode *hash)
44{
45 return *(struct IBF_Key *) hash;
46}
47
48
49/**
50 * Create a hashcode from a key, by replicating the key
51 * until the hascode is filled
52 *
53 * @param key the key
54 * @param dst hashcode to store the result in
55 */
56void
57ibf_hashcode_from_key (struct IBF_Key key,
58 struct GNUNET_HashCode *dst)
59{
60 struct IBF_Key *p;
61 unsigned int i;
62 const unsigned int keys_per_hashcode = sizeof(struct GNUNET_HashCode)
63 / sizeof(struct IBF_Key);
64
65 p = (struct IBF_Key *) dst;
66 for (i = 0; i < keys_per_hashcode; i++)
67 *p++ = key;
68}
69
70
71/**
72 * Create an invertible bloom filter.
73 *
74 * @param size number of IBF buckets
75 * @param hash_num number of buckets one element is hashed in
76 * @return the newly created invertible bloom filter, NULL on error
77 */
78struct InvertibleBloomFilter *
79ibf_create (uint32_t size, uint8_t hash_num)
80{
81 struct InvertibleBloomFilter *ibf;
82
83 GNUNET_assert (0 != size);
84
85 ibf = GNUNET_new (struct InvertibleBloomFilter);
86 ibf->count = GNUNET_malloc_large (size * sizeof(uint8_t));
87 if (NULL == ibf->count)
88 {
89 GNUNET_free (ibf);
90 return NULL;
91 }
92 ibf->key_sum = GNUNET_malloc_large (size * sizeof(struct IBF_Key));
93 if (NULL == ibf->key_sum)
94 {
95 GNUNET_free (ibf->count);
96 GNUNET_free (ibf);
97 return NULL;
98 }
99 ibf->key_hash_sum = GNUNET_malloc_large (size * sizeof(struct IBF_KeyHash));
100 if (NULL == ibf->key_hash_sum)
101 {
102 GNUNET_free (ibf->key_sum);
103 GNUNET_free (ibf->count);
104 GNUNET_free (ibf);
105 return NULL;
106 }
107 ibf->size = size;
108 ibf->hash_num = hash_num;
109
110 return ibf;
111}
112
113
114/**
115 * Store unique bucket indices for the specified key in dst.
116 */
117static void
118ibf_get_indices (const struct InvertibleBloomFilter *ibf,
119 struct IBF_Key key,
120 int *dst)
121{
122 uint32_t filled;
123 uint32_t i;
124 uint32_t bucket;
125
126 bucket = GNUNET_CRYPTO_crc32_n (&key, sizeof key);
127 for (i = 0, filled = 0; filled < ibf->hash_num; i++)
128 {
129 unsigned int j;
130 uint64_t x;
131 for (j = 0; j < filled; j++)
132 if (dst[j] == bucket)
133 goto try_next;
134 dst[filled++] = bucket % ibf->size;
135try_next:;
136 x = ((uint64_t) bucket << 32) | i;
137 bucket = GNUNET_CRYPTO_crc32_n (&x, sizeof x);
138 }
139}
140
141
142static void
143ibf_insert_into (struct InvertibleBloomFilter *ibf,
144 struct IBF_Key key,
145 const int *buckets, int side)
146{
147 int i;
148
149 for (i = 0; i < ibf->hash_num; i++)
150 {
151 const int bucket = buckets[i];
152 ibf->count[bucket].count_val += side;
153 ibf->key_sum[bucket].key_val ^= key.key_val;
154 ibf->key_hash_sum[bucket].key_hash_val
155 ^= IBF_KEY_HASH_VAL (key);
156 }
157}
158
159
160/**
161 * Insert a key into an IBF.
162 *
163 * @param ibf the IBF
164 * @param key the element's hash code
165 */
166void
167ibf_insert (struct InvertibleBloomFilter *ibf, struct IBF_Key key)
168{
169 int buckets[ibf->hash_num];
170
171 GNUNET_assert (ibf->hash_num <= ibf->size);
172 ibf_get_indices (ibf, key, buckets);
173 ibf_insert_into (ibf, key, buckets, 1);
174}
175
176
177/**
178 * Remove a key from an IBF.
179 *
180 * @param ibf the IBF
181 * @param key the element's hash code
182 */
183void
184ibf_remove (struct InvertibleBloomFilter *ibf, struct IBF_Key key)
185{
186 int buckets[ibf->hash_num];
187
188 GNUNET_assert (ibf->hash_num <= ibf->size);
189 ibf_get_indices (ibf, key, buckets);
190 ibf_insert_into (ibf, key, buckets, -1);
191}
192
193
194/**
195 * Test is the IBF is empty, i.e. all counts, keys and key hashes are zero.
196 */
197static int
198ibf_is_empty (struct InvertibleBloomFilter *ibf)
199{
200 int i;
201
202 for (i = 0; i < ibf->size; i++)
203 {
204 if (0 != ibf->count[i].count_val)
205 return GNUNET_NO;
206 if (0 != ibf->key_hash_sum[i].key_hash_val)
207 return GNUNET_NO;
208 if (0 != ibf->key_sum[i].key_val)
209 return GNUNET_NO;
210 }
211 return GNUNET_YES;
212}
213
214
215/**
216 * Decode and remove an element from the IBF, if possible.
217 *
218 * @param ibf the invertible bloom filter to decode
219 * @param ret_side sign of the cell's count where the decoded element came from.
220 * A negative sign indicates that the element was recovered
221 * resides in an IBF that was previously subtracted from.
222 * @param ret_id receives the hash code of the decoded element, if successful
223 * @return GNUNET_YES if decoding an element was successful,
224 * GNUNET_NO if the IBF is empty,
225 * GNUNET_SYSERR if the decoding has failed
226 */
227int
228ibf_decode (struct InvertibleBloomFilter *ibf,
229 int *ret_side, struct IBF_Key *ret_id)
230{
231 struct IBF_KeyHash hash;
232 int i;
233 int buckets[ibf->hash_num];
234
235 GNUNET_assert (NULL != ibf);
236
237 for (i = 0; i < ibf->size; i++)
238 {
239 int j;
240 int hit;
241
242 /* we can only decode from pure buckets */
243 if ((1 != ibf->count[i].count_val) && (-1 != ibf->count[i].count_val))
244 continue;
245
246 hash.key_hash_val = IBF_KEY_HASH_VAL (ibf->key_sum[i]);
247
248 /* test if the hash matches the key */
249 if (hash.key_hash_val != ibf->key_hash_sum[i].key_hash_val)
250 continue;
251
252 /* test if key in bucket hits its own location,
253 * if not, the key hash was subject to collision */
254 hit = GNUNET_NO;
255 ibf_get_indices (ibf, ibf->key_sum[i], buckets);
256 for (j = 0; j < ibf->hash_num; j++)
257 if (buckets[j] == i)
258 hit = GNUNET_YES;
259
260 if (GNUNET_NO == hit)
261 continue;
262
263 if (NULL != ret_side)
264 *ret_side = ibf->count[i].count_val;
265 if (NULL != ret_id)
266 *ret_id = ibf->key_sum[i];
267
268 /* insert on the opposite side, effectively removing the element */
269 ibf_insert_into (ibf, ibf->key_sum[i], buckets, -ibf->count[i].count_val);
270
271 return GNUNET_YES;
272 }
273
274 if (GNUNET_YES == ibf_is_empty (ibf))
275 return GNUNET_NO;
276 return GNUNET_SYSERR;
277}
278
279
280/**
281 * Write buckets from an ibf to a buffer.
282 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
283 *
284 * @param ibf the ibf to write
285 * @param start with which bucket to start
286 * @param count how many buckets to write
287 * @param buf buffer to write the data to
288 */
289void
290ibf_write_slice (const struct InvertibleBloomFilter *ibf, uint32_t start,
291 uint32_t count, void *buf)
292{
293 struct IBF_Key *key_dst;
294 struct IBF_KeyHash *key_hash_dst;
295 struct IBF_Count *count_dst;
296
297 GNUNET_assert (start + count <= ibf->size);
298
299 /* copy keys */
300 key_dst = (struct IBF_Key *) buf;
301 GNUNET_memcpy (key_dst, ibf->key_sum + start, count * sizeof *key_dst);
302 key_dst += count;
303 /* copy key hashes */
304 key_hash_dst = (struct IBF_KeyHash *) key_dst;
305 GNUNET_memcpy (key_hash_dst, ibf->key_hash_sum + start, count
306 * sizeof *key_hash_dst);
307 key_hash_dst += count;
308 /* copy counts */
309 count_dst = (struct IBF_Count *) key_hash_dst;
310 GNUNET_memcpy (count_dst, ibf->count + start, count * sizeof *count_dst);
311}
312
313
314/**
315 * Read buckets from a buffer into an ibf.
316 *
317 * @param buf pointer to the buffer to read from
318 * @param start which bucket to start at
319 * @param count how many buckets to read
320 * @param ibf the ibf to read from
321 */
322void
323ibf_read_slice (const void *buf, uint32_t start, uint32_t count, struct
324 InvertibleBloomFilter *ibf)
325{
326 struct IBF_Key *key_src;
327 struct IBF_KeyHash *key_hash_src;
328 struct IBF_Count *count_src;
329
330 GNUNET_assert (count > 0);
331 GNUNET_assert (start + count <= ibf->size);
332
333 /* copy keys */
334 key_src = (struct IBF_Key *) buf;
335 GNUNET_memcpy (ibf->key_sum + start, key_src, count * sizeof *key_src);
336 key_src += count;
337 /* copy key hashes */
338 key_hash_src = (struct IBF_KeyHash *) key_src;
339 GNUNET_memcpy (ibf->key_hash_sum + start, key_hash_src, count
340 * sizeof *key_hash_src);
341 key_hash_src += count;
342 /* copy counts */
343 count_src = (struct IBF_Count *) key_hash_src;
344 GNUNET_memcpy (ibf->count + start, count_src, count * sizeof *count_src);
345}
346
347
348/**
349 * Subtract ibf2 from ibf1, storing the result in ibf1.
350 * The two IBF's must have the same parameters size and hash_num.
351 *
352 * @param ibf1 IBF that is subtracted from
353 * @param ibf2 IBF that will be subtracted from ibf1
354 */
355void
356ibf_subtract (struct InvertibleBloomFilter *ibf1, const struct
357 InvertibleBloomFilter *ibf2)
358{
359 int i;
360
361 GNUNET_assert (ibf1->size == ibf2->size);
362 GNUNET_assert (ibf1->hash_num == ibf2->hash_num);
363
364 for (i = 0; i < ibf1->size; i++)
365 {
366 ibf1->count[i].count_val -= ibf2->count[i].count_val;
367 ibf1->key_hash_sum[i].key_hash_val ^= ibf2->key_hash_sum[i].key_hash_val;
368 ibf1->key_sum[i].key_val ^= ibf2->key_sum[i].key_val;
369 }
370}
371
372
373/**
374 * Create a copy of an IBF, the copy has to be destroyed properly.
375 *
376 * @param ibf the IBF to copy
377 */
378struct InvertibleBloomFilter *
379ibf_dup (const struct InvertibleBloomFilter *ibf)
380{
381 struct InvertibleBloomFilter *copy;
382
383 copy = GNUNET_malloc (sizeof *copy);
384 copy->hash_num = ibf->hash_num;
385 copy->size = ibf->size;
386 copy->key_hash_sum = GNUNET_memdup (ibf->key_hash_sum, ibf->size
387 * sizeof(struct IBF_KeyHash));
388 copy->key_sum = GNUNET_memdup (ibf->key_sum, ibf->size * sizeof(struct
389 IBF_Key));
390 copy->count = GNUNET_memdup (ibf->count, ibf->size * sizeof(struct
391 IBF_Count));
392 return copy;
393}
394
395
396/**
397 * Destroy all resources associated with the invertible bloom filter.
398 * No more ibf_*-functions may be called on ibf after calling destroy.
399 *
400 * @param ibf the intertible bloom filter to destroy
401 */
402void
403ibf_destroy (struct InvertibleBloomFilter *ibf)
404{
405 GNUNET_free (ibf->key_sum);
406 GNUNET_free (ibf->key_hash_sum);
407 GNUNET_free (ibf->count);
408 GNUNET_free (ibf);
409}
diff --git a/src/set/ibf.h b/src/set/ibf.h
deleted file mode 100644
index 334a797ef..000000000
--- a/src/set/ibf.h
+++ /dev/null
@@ -1,256 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.h
23 * @brief invertible bloom filter
24 * @author Florian Dold
25 */
26
27#ifndef GNUNET_CONSENSUS_IBF_H
28#define GNUNET_CONSENSUS_IBF_H
29
30#include "platform.h"
31#include "gnunet_util_lib.h"
32
33#ifdef __cplusplus
34extern "C"
35{
36#if 0 /* keep Emacsens' auto-indent happy */
37}
38#endif
39#endif
40
41
42/**
43 * Keys that can be inserted into and removed from an IBF.
44 */
45struct IBF_Key
46{
47 uint64_t key_val;
48};
49
50
51/**
52 * Hash of an IBF key.
53 */
54struct IBF_KeyHash
55{
56 uint32_t key_hash_val;
57};
58
59
60/**
61 * Type of the count field of IBF buckets.
62 */
63struct IBF_Count
64{
65 int8_t count_val;
66};
67
68
69/**
70 * Size of one ibf bucket in bytes
71 */
72#define IBF_BUCKET_SIZE (sizeof(struct IBF_Count) + sizeof(struct IBF_Key) \
73 + sizeof(struct IBF_KeyHash))
74
75
76/**
77 * Invertible bloom filter (IBF).
78 *
79 * An IBF is a counting bloom filter that has the ability to restore
80 * the hashes of its stored elements with high probability.
81 */
82struct InvertibleBloomFilter
83{
84 /**
85 * How many cells does this IBF have?
86 */
87 uint32_t size;
88
89 /**
90 * In how many cells do we hash one element?
91 * Usually 4 or 3.
92 */
93 uint8_t hash_num;
94
95 /**
96 * Xor sums of the elements' keys, used to identify the elements.
97 * Array of 'size' elements.
98 */
99 struct IBF_Key *key_sum;
100
101 /**
102 * Xor sums of the hashes of the keys of inserted elements.
103 * Array of 'size' elements.
104 */
105 struct IBF_KeyHash *key_hash_sum;
106
107 /**
108 * How many times has a bucket been hit?
109 * Can be negative, as a result of IBF subtraction.
110 * Array of 'size' elements.
111 */
112 struct IBF_Count *count;
113};
114
115
116/**
117 * Write buckets from an ibf to a buffer.
118 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
119 *
120 * @param ibf the ibf to write
121 * @param start with which bucket to start
122 * @param count how many buckets to write
123 * @param buf buffer to write the data to
124 */
125void
126ibf_write_slice (const struct InvertibleBloomFilter *ibf,
127 uint32_t start,
128 uint32_t count,
129 void *buf);
130
131
132/**
133 * Read buckets from a buffer into an ibf.
134 *
135 * @param buf pointer to the buffer to read from
136 * @param start which bucket to start at
137 * @param count how many buckets to read
138 * @param ibf the ibf to write to
139 */
140void
141ibf_read_slice (const void *buf,
142 uint32_t start,
143 uint32_t count,
144 struct InvertibleBloomFilter *ibf);
145
146
147/**
148 * Create a key from a hashcode.
149 *
150 * @param hash the hashcode
151 * @return a key
152 */
153struct IBF_Key
154ibf_key_from_hashcode (const struct GNUNET_HashCode *hash);
155
156
157/**
158 * Create a hashcode from a key, by replicating the key
159 * until the hascode is filled
160 *
161 * @param key the key
162 * @param dst hashcode to store the result in
163 */
164void
165ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst);
166
167
168/**
169 * Create an invertible bloom filter.
170 *
171 * @param size number of IBF buckets
172 * @param hash_num number of buckets one element is hashed in, usually 3 or 4
173 * @return the newly created invertible bloom filter, NULL on error
174 */
175struct InvertibleBloomFilter *
176ibf_create (uint32_t size, uint8_t hash_num);
177
178
179/**
180 * Insert a key into an IBF.
181 *
182 * @param ibf the IBF
183 * @param key the element's hash code
184 */
185void
186ibf_insert (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
187
188
189/**
190 * Remove a key from an IBF.
191 *
192 * @param ibf the IBF
193 * @param key the element's hash code
194 */
195void
196ibf_remove (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
197
198
199/**
200 * Subtract ibf2 from ibf1, storing the result in ibf1.
201 * The two IBF's must have the same parameters size and hash_num.
202 *
203 * @param ibf1 IBF that is subtracted from
204 * @param ibf2 IBF that will be subtracted from ibf1
205 */
206void
207ibf_subtract (struct InvertibleBloomFilter *ibf1,
208 const struct InvertibleBloomFilter *ibf2);
209
210
211/**
212 * Decode and remove an element from the IBF, if possible.
213 *
214 * @param ibf the invertible bloom filter to decode
215 * @param ret_side sign of the cell's count where the decoded element came from.
216 * A negative sign indicates that the element was recovered
217 * resides in an IBF that was previously subtracted from.
218 * @param ret_id receives the hash code of the decoded element, if successful
219 * @return #GNUNET_YES if decoding an element was successful,
220 * #GNUNET_NO if the IBF is empty,
221 * #GNUNET_SYSERR if the decoding has failed
222 */
223int
224ibf_decode (struct InvertibleBloomFilter *ibf,
225 int *ret_side,
226 struct IBF_Key *ret_id);
227
228
229/**
230 * Create a copy of an IBF, the copy has to be destroyed properly.
231 *
232 * @param ibf the IBF to copy
233 */
234struct InvertibleBloomFilter *
235ibf_dup (const struct InvertibleBloomFilter *ibf);
236
237
238/**
239 * Destroy all resources associated with the invertible bloom filter.
240 * No more ibf_*-functions may be called on ibf after calling destroy.
241 *
242 * @param ibf the intertible bloom filter to destroy
243 */
244void
245ibf_destroy (struct InvertibleBloomFilter *ibf);
246
247
248
249#if 0 /* keep Emacsens' auto-indent happy */
250{
251#endif
252#ifdef __cplusplus
253}
254#endif
255
256#endif
diff --git a/src/set/ibf_sim.c b/src/set/ibf_sim.c
deleted file mode 100644
index 6415d00e1..000000000
--- a/src/set/ibf_sim.c
+++ /dev/null
@@ -1,142 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf_sim.c
23 * @brief implementation of simulation for invertible bloom filter
24 * @author Florian Dold
25 *
26 * This code was used for some internal experiments, it is not
27 * build or shipped as part of the GNUnet system.
28 */
29#include <stdlib.h>
30#include <stdio.h>
31#include <string.h>
32
33#define MAX_IBF_DECODE 16
34
35/* report average over how many rounds? */
36#define ROUNDS 100000
37
38/* enable one of the three below */
39// simple fix
40#define FIX1 0
41// possibly slightly better fix for large IBF_DECODE values
42#define FIX2 1
43
44// SIGCOMM algorithm
45#define STRATA 0
46
47// print each value?
48#define VERBOSE 0
49// avoid assembly? (ASM is about 50% faster)
50#define SLOW 0
51
52int
53main (int argc, char **argv)
54{
55 unsigned int round;
56 unsigned int buckets[31]; // max is 2^31 as 'random' returns only between 0 and 2^31
57 unsigned int i;
58 int j;
59 unsigned int r;
60 unsigned int ret;
61 unsigned long long total;
62 unsigned int want;
63 double predict;
64
65 srandom (time (NULL));
66 total = 0;
67 want = atoi (argv[1]);
68 for (round = 0; round < ROUNDS; round++)
69 {
70 memset (buckets, 0, sizeof(buckets));
71 for (i = 0; i < want; i++)
72 {
73 /* FIXME: might want to use 'better' PRNG to avoid
74 PRNG-induced biases */
75 r = random ();
76 if (0 == r)
77 continue;
78#if SLOW
79 for (j = 0; (j < 31) && (0 == (r & (1 << j))); j++)
80 ;
81#else
82 /* use assembly / gcc */
83 j = __builtin_ffs (r) - 1;
84#endif
85 buckets[j]++;
86 }
87 ret = 0;
88 predict = 0.0;
89 for (j = 31; j >= 0; j--)
90 {
91#if FIX1
92 /* improved algorithm, for 1000 elements with IBF-DECODE 8, I
93 get 990/1000 elements on average over 1 million runs; key
94 idea being to stop short of the 'last' possible IBF as
95 otherwise a "lowball" per-chance would unduely influence the
96 result */if ((j > 0) &&
97 (buckets[j - 1] > MAX_IBF_DECODE))
98 {
99 ret *= (1 << (j + 1));
100 break;
101 }
102#endif
103#if FIX2
104 /* another improvement: don't just always cut off the last one,
105 but rather try to predict based on all previous values where
106 that "last" one is; additional prediction can only really
107 work if MAX_IBF_DECODE is sufficiently high */
108 if ((j > 0) &&
109 ((buckets[j - 1] > MAX_IBF_DECODE) ||
110 (predict > MAX_IBF_DECODE)))
111 {
112 ret *= (1 << (j + 1));
113 break;
114 }
115#endif
116#if STRATA
117 /* original algorithm, for 1000 elements with IBF-DECODE 8,
118 I get 920/1000 elements on average over 1 million runs */
119 if (buckets[j] > MAX_IBF_DECODE)
120 {
121 ret *= (1 << (j + 1));
122 break;
123 }
124#endif
125 ret += buckets[j];
126 predict = (buckets[j] + 2.0 * predict) / 2.0;
127 }
128#if VERBOSE
129 fprintf (stderr, "%u ", ret);
130#endif
131 total += ret;
132 }
133 fprintf (stderr, "\n");
134 fprintf (stdout, "average %llu\n", total / ROUNDS);
135 return 0;
136}
137
138
139/* TODO: should calculate stddev of the results to also be able to
140 say something about the stability of the results, outside of
141 large-scale averages -- gaining 8% precision at the expense of
142 50% additional variance might not be worth it... */
diff --git a/src/set/plugin_block_set_test.c b/src/set/plugin_block_set_test.c
deleted file mode 100644
index cb5cef5ad..000000000
--- a/src/set/plugin_block_set_test.c
+++ /dev/null
@@ -1,167 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/plugin_block_set_test.c
23 * @brief set test block, recognizes elements with non-zero first byte as invalid
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_block_plugin.h"
29#include "gnunet_block_group_lib.h"
30
31
32/**
33 * Function called to validate a query.
34 *
35 * @param cls closure
36 * @param type block type
37 * @param query original query (hash)
38 * @param xquery extrended query data (can be NULL, depending on type)
39 * @param xquery_size number of bytes in @a xquery
40 * @return #GNUNET_OK if the query is fine, #GNUNET_NO if not
41 */
42static enum GNUNET_GenericReturnValue
43block_plugin_set_test_check_query (void *cls,
44 enum GNUNET_BLOCK_Type type,
45 const struct GNUNET_HashCode *query,
46 const void *xquery,
47 size_t xquery_size)
48{
49 return GNUNET_OK;
50}
51
52
53/**
54 * Function called to validate a block for storage.
55 *
56 * @param cls closure
57 * @param type block type
58 * @param block block data to validate
59 * @param block_size number of bytes in @a block
60 * @return #GNUNET_OK if the block is fine, #GNUNET_NO if not
61 */
62static enum GNUNET_GenericReturnValue
63block_plugin_set_test_check_block (void *cls,
64 enum GNUNET_BLOCK_Type type,
65 const void *block,
66 size_t block_size)
67{
68 if ((NULL == block) ||
69 (0 == block_size) ||
70 (0 != ((char *) block)[0]))
71 return GNUNET_SYSERR;
72 return GNUNET_OK;
73}
74
75
76/**
77 * Function called to validate a reply to a request. Note that it is assumed
78 * that the reply has already been matched to the key (and signatures checked)
79 * as it would be done with the GetKeyFunction and the
80 * BlockEvaluationFunction.
81 *
82 * @param cls closure
83 * @param type block type
84 * @param group which block group to use for evaluation
85 * @param query original query (hash)
86 * @param xquery extrended query data (can be NULL, depending on type)
87 * @param xquery_size number of bytes in @a xquery
88 * @param reply_block response to validate
89 * @param reply_block_size number of bytes in @a reply_block
90 * @return characterization of result
91 */
92static enum GNUNET_BLOCK_ReplyEvaluationResult
93block_plugin_set_test_check_reply (void *cls,
94 enum GNUNET_BLOCK_Type type,
95 struct GNUNET_BLOCK_Group *group,
96 const struct GNUNET_HashCode *query,
97 const void *xquery,
98 size_t xquery_size,
99 const void *reply_block,
100 size_t reply_block_size)
101{
102 if ((NULL == reply_block) ||
103 (0 == reply_block_size) ||
104 (0 != ((char *) reply_block)[0]))
105 GNUNET_assert (0);
106 return GNUNET_BLOCK_REPLY_OK_MORE;
107}
108
109
110/**
111 * Function called to obtain the key for a block.
112 *
113 * @param cls closure
114 * @param type block type
115 * @param block block to get the key for
116 * @param block_size number of bytes in block
117 * @param key set to the key (query) for the given block
118 * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
119 * (or if extracting a key from a block of this type does not work)
120 */
121static enum GNUNET_GenericReturnValue
122block_plugin_set_test_get_key (void *cls,
123 enum GNUNET_BLOCK_Type type,
124 const void *block,
125 size_t block_size,
126 struct GNUNET_HashCode *key)
127{
128 return GNUNET_NO;
129}
130
131
132/**
133 * Entry point for the plugin.
134 */
135void *
136libgnunet_plugin_block_set_test_init (void *cls)
137{
138 static const enum GNUNET_BLOCK_Type types[] = {
139 GNUNET_BLOCK_TYPE_SET_TEST,
140 GNUNET_BLOCK_TYPE_ANY /* end of list */
141 };
142 struct GNUNET_BLOCK_PluginFunctions *api;
143
144 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
145 api->get_key = &block_plugin_set_test_get_key;
146 api->check_query = &block_plugin_set_test_check_query;
147 api->check_block = &block_plugin_set_test_check_block;
148 api->check_reply = &block_plugin_set_test_check_reply;
149 api->types = types;
150 return api;
151}
152
153
154/**
155 * Exit point from the plugin.
156 */
157void *
158libgnunet_plugin_block_set_test_done (void *cls)
159{
160 struct GNUNET_BLOCK_PluginFunctions *api = cls;
161
162 GNUNET_free (api);
163 return NULL;
164}
165
166
167/* end of plugin_block_set_test.c */
diff --git a/src/set/set.conf.in b/src/set/set.conf.in
deleted file mode 100644
index 66bcfa169..000000000
--- a/src/set/set.conf.in
+++ /dev/null
@@ -1,12 +0,0 @@
1[set]
2START_ON_DEMAND = @START_ON_DEMAND@
3@UNIXONLY@PORT = 2106
4HOSTNAME = localhost
5BINARY = gnunet-service-set
6ACCEPT_FROM = 127.0.0.1;
7ACCEPT_FROM6 = ::1;
8UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-set.sock
9UNIX_MATCH_UID = YES
10UNIX_MATCH_GID = YES
11
12#PREFIX = valgrind
diff --git a/src/set/set.h b/src/set/set.h
deleted file mode 100644
index e9d10ea22..000000000
--- a/src/set/set.h
+++ /dev/null
@@ -1,400 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2014 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/set.h
22 * @brief messages used for the set api
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef SET_H
27#define SET_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31#include "gnunet_set_service.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message sent by the client to the service to ask starting
37 * a new set to perform operations with. Includes the desired
38 * set operation type.
39 */
40struct GNUNET_SET_CreateMessage
41{
42 /**
43 * Type: #GNUNET_MESSAGE_TYPE_SET_CREATE
44 */
45 struct GNUNET_MessageHeader header;
46
47 /**
48 * Operation type, values of `enum GNUNET_SET_OperationType`
49 */
50 uint32_t operation GNUNET_PACKED;
51};
52
53
54/**
55 * Message sent by the client to the service to start listening for
56 * incoming requests to perform a certain type of set operation for a
57 * certain type of application.
58 */
59struct GNUNET_SET_ListenMessage
60{
61 /**
62 * Type: #GNUNET_MESSAGE_TYPE_SET_LISTEN
63 */
64 struct GNUNET_MessageHeader header;
65
66 /**
67 * Operation type, values of `enum GNUNET_SET_OperationType`
68 */
69 uint32_t operation GNUNET_PACKED;
70
71 /**
72 * application id
73 */
74 struct GNUNET_HashCode app_id;
75};
76
77
78/**
79 * Message sent by a listening client to the service to accept
80 * performing the operation with the other peer.
81 */
82struct GNUNET_SET_AcceptMessage
83{
84 /**
85 * Type: #GNUNET_MESSAGE_TYPE_SET_ACCEPT
86 */
87 struct GNUNET_MessageHeader header;
88
89 /**
90 * ID of the incoming request we want to accept.
91 */
92 uint32_t accept_reject_id GNUNET_PACKED;
93
94 /**
95 * Request ID to identify responses.
96 */
97 uint32_t request_id GNUNET_PACKED;
98
99 /**
100 * How should results be sent to us?
101 * See `enum GNUNET_SET_ResultMode`.
102 */
103 uint32_t result_mode GNUNET_PACKED;
104
105 /**
106 * Always use delta operation instead of sending full sets,
107 * even it it's less efficient.
108 */
109 uint8_t force_delta;
110
111 /**
112 * Always send full sets, even if delta operations would
113 * be more efficient.
114 */
115 uint8_t force_full;
116
117 /**
118 * #GNUNET_YES to fail operations where Byzantine faults
119 * are suspected
120 */
121 uint8_t byzantine;
122
123 /**
124 * Lower bound for the set size, used only when
125 * byzantine mode is enabled.
126 */
127 uint8_t byzantine_lower_bound;
128};
129
130
131/**
132 * Message sent by a listening client to the service to reject
133 * performing the operation with the other peer.
134 */
135struct GNUNET_SET_RejectMessage
136{
137 /**
138 * Type: #GNUNET_MESSAGE_TYPE_SET_REJECT
139 */
140 struct GNUNET_MessageHeader header;
141
142 /**
143 * ID of the incoming request we want to reject.
144 */
145 uint32_t accept_reject_id GNUNET_PACKED;
146};
147
148
149/**
150 * A request for an operation with another client.
151 */
152struct GNUNET_SET_RequestMessage
153{
154 /**
155 * Type: #GNUNET_MESSAGE_TYPE_SET_REQUEST.
156 */
157 struct GNUNET_MessageHeader header;
158
159 /**
160 * ID of the to identify the request when accepting or
161 * rejecting it.
162 */
163 uint32_t accept_id GNUNET_PACKED;
164
165 /**
166 * Identity of the requesting peer.
167 */
168 struct GNUNET_PeerIdentity peer_id;
169
170 /* rest: context message, that is, application-specific
171 message to convince listener to pick up */
172};
173
174
175/**
176 * Message sent by client to service to initiate a set operation as a
177 * client (not as listener). A set (which determines the operation
178 * type) must already exist in association with this client.
179 */
180struct GNUNET_SET_EvaluateMessage
181{
182 /**
183 * Type: #GNUNET_MESSAGE_TYPE_SET_EVALUATE
184 */
185 struct GNUNET_MessageHeader header;
186
187 /**
188 * How should results be sent to us?
189 * See `enum GNUNET_SET_ResultMode`.
190 */
191 uint32_t result_mode GNUNET_PACKED;
192
193 /**
194 * Peer to evaluate the operation with
195 */
196 struct GNUNET_PeerIdentity target_peer;
197
198 /**
199 * Application id
200 */
201 struct GNUNET_HashCode app_id;
202
203 /**
204 * Id of our set to evaluate, chosen implicitly by the client when it
205 * calls #GNUNET_SET_commit().
206 */
207 uint32_t request_id GNUNET_PACKED;
208
209 /**
210 * Always use delta operation instead of sending full sets,
211 * even it it's less efficient.
212 */
213 uint8_t force_delta;
214
215 /**
216 * Always send full sets, even if delta operations would
217 * be more efficient.
218 */
219 uint8_t force_full;
220
221 /**
222 * #GNUNET_YES to fail operations where Byzantine faults
223 * are suspected
224 */
225 uint8_t byzantine;
226
227 /**
228 * Lower bound for the set size, used only when
229 * byzantine mode is enabled.
230 */
231 uint8_t byzantine_lower_bound;
232
233 /* rest: context message, that is, application-specific
234 message to convince listener to pick up */
235};
236
237
238/**
239 * Message sent by the service to the client to indicate an
240 * element that is removed (set intersection) or added
241 * (set union) or part of the final result, depending on
242 * options specified for the operation.
243 */
244struct GNUNET_SET_ResultMessage
245{
246 /**
247 * Type: #GNUNET_MESSAGE_TYPE_SET_RESULT
248 */
249 struct GNUNET_MessageHeader header;
250
251 /**
252 * Current set size.
253 */
254 uint64_t current_size;
255
256 /**
257 * id the result belongs to
258 */
259 uint32_t request_id GNUNET_PACKED;
260
261 /**
262 * Was the evaluation successful? Contains
263 * an `enum GNUNET_SET_Status` in NBO.
264 */
265 uint16_t result_status GNUNET_PACKED;
266
267 /**
268 * Type of the element attached to the message, if any.
269 */
270 uint16_t element_type GNUNET_PACKED;
271
272 /* rest: the actual element */
273};
274
275
276/**
277 * Message sent by client to the service to add or remove
278 * an element to/from the set.
279 */
280struct GNUNET_SET_ElementMessage
281{
282 /**
283 * Type: #GNUNET_MESSAGE_TYPE_SET_ADD or
284 * #GNUNET_MESSAGE_TYPE_SET_REMOVE
285 */
286 struct GNUNET_MessageHeader header;
287
288 /**
289 * Type of the element to add or remove.
290 */
291 uint16_t element_type GNUNET_PACKED;
292
293 /**
294 * For alignment, always zero.
295 */
296 uint16_t reserved GNUNET_PACKED;
297
298 /* rest: the actual element */
299};
300
301
302/**
303 * Sent to the service by the client
304 * in order to cancel a set operation.
305 */
306struct GNUNET_SET_CancelMessage
307{
308 /**
309 * Type: #GNUNET_MESSAGE_TYPE_SET_CANCEL
310 */
311 struct GNUNET_MessageHeader header;
312
313 /**
314 * ID of the request we want to cancel.
315 */
316 uint32_t request_id GNUNET_PACKED;
317};
318
319
320/**
321 * Set element transmitted by service to client in response to a set
322 * iteration request.
323 */
324struct GNUNET_SET_IterResponseMessage
325{
326 /**
327 * Type: #GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT
328 */
329 struct GNUNET_MessageHeader header;
330
331 /**
332 * To which set iteration does this response belong to? First
333 * iteration (per client) has counter zero. Wraps around.
334 */
335 uint16_t iteration_id GNUNET_PACKED;
336
337 /**
338 * Type of the element attached to the message,
339 * if any.
340 */
341 uint16_t element_type GNUNET_PACKED;
342
343 /* rest: element */
344};
345
346
347/**
348 * Client acknowledges receiving element in iteration.
349 */
350struct GNUNET_SET_IterAckMessage
351{
352 /**
353 * Type: #GNUNET_MESSAGE_TYPE_SET_ITER_ACK
354 */
355 struct GNUNET_MessageHeader header;
356
357 /**
358 * Non-zero if the service should continue sending elements.
359 */
360 uint32_t send_more;
361};
362
363
364/**
365 * Server responds to a lazy copy request.
366 */
367struct GNUNET_SET_CopyLazyResponseMessage
368{
369 /**
370 * Type: #GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE
371 */
372 struct GNUNET_MessageHeader header;
373
374 /**
375 * Temporary name for the copied set.
376 */
377 uint32_t cookie;
378};
379
380
381/**
382 * Client connects to a lazily copied set.
383 */
384struct GNUNET_SET_CopyLazyConnectMessage
385{
386 /**
387 * Type: #GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT
388 */
389 struct GNUNET_MessageHeader header;
390
391 /**
392 * Temporary name for the copied set.
393 */
394 uint32_t cookie;
395};
396
397
398GNUNET_NETWORK_STRUCT_END
399
400#endif
diff --git a/src/set/set_api.c b/src/set/set_api.c
deleted file mode 100644
index 6b3dc940c..000000000
--- a/src/set/set_api.c
+++ /dev/null
@@ -1,1260 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/set_api.c
22 * @brief api for the set service
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_protocols.h"
29#include "gnunet_set_service.h"
30#include "set.h"
31
32
33#define LOG(kind, ...) GNUNET_log_from (kind, "set-api", __VA_ARGS__)
34
35struct SetCopyRequest
36{
37 struct SetCopyRequest *next;
38
39 struct SetCopyRequest *prev;
40
41 void *cls;
42
43 GNUNET_SET_CopyReadyCallback cb;
44};
45
46/**
47 * Opaque handle to a set.
48 */
49struct GNUNET_SET_Handle
50{
51 /**
52 * Message queue for @e client.
53 */
54 struct GNUNET_MQ_Handle *mq;
55
56 /**
57 * Linked list of operations on the set.
58 */
59 struct GNUNET_SET_OperationHandle *ops_head;
60
61 /**
62 * Linked list of operations on the set.
63 */
64 struct GNUNET_SET_OperationHandle *ops_tail;
65
66 /**
67 * Callback for the current iteration over the set,
68 * NULL if no iterator is active.
69 */
70 GNUNET_SET_ElementIterator iterator;
71
72 /**
73 * Closure for @e iterator
74 */
75 void *iterator_cls;
76
77 /**
78 * Should the set be destroyed once all operations are gone?
79 * #GNUNET_SYSERR if #GNUNET_SET_destroy() must raise this flag,
80 * #GNUNET_YES if #GNUNET_SET_destroy() did raise this flag.
81 */
82 int destroy_requested;
83
84 /**
85 * Has the set become invalid (e.g. service died)?
86 */
87 int invalid;
88
89 /**
90 * Both client and service count the number of iterators
91 * created so far to match replies with iterators.
92 */
93 uint16_t iteration_id;
94
95 /**
96 * Configuration, needed when creating (lazy) copies.
97 */
98 const struct GNUNET_CONFIGURATION_Handle *cfg;
99
100 /**
101 * Doubly linked list of copy requests.
102 */
103 struct SetCopyRequest *copy_req_head;
104
105 /**
106 * Doubly linked list of copy requests.
107 */
108 struct SetCopyRequest *copy_req_tail;
109};
110
111
112/**
113 * Handle for a set operation request from another peer.
114 */
115struct GNUNET_SET_Request
116{
117 /**
118 * Id of the request, used to identify the request when
119 * accepting/rejecting it.
120 */
121 uint32_t accept_id;
122
123 /**
124 * Has the request been accepted already?
125 * #GNUNET_YES/#GNUNET_NO
126 */
127 int accepted;
128};
129
130
131/**
132 * Handle to an operation. Only known to the service after committing
133 * the handle with a set.
134 */
135struct GNUNET_SET_OperationHandle
136{
137 /**
138 * Function to be called when we have a result,
139 * or an error.
140 */
141 GNUNET_SET_ResultIterator result_cb;
142
143 /**
144 * Closure for @e result_cb.
145 */
146 void *result_cls;
147
148 /**
149 * Local set used for the operation,
150 * NULL if no set has been provided by conclude yet.
151 */
152 struct GNUNET_SET_Handle *set;
153
154 /**
155 * Message sent to the server on calling conclude,
156 * NULL if conclude has been called.
157 */
158 struct GNUNET_MQ_Envelope *conclude_mqm;
159
160 /**
161 * Address of the request if in the conclude message,
162 * used to patch the request id into the message when the set is known.
163 */
164 uint32_t *request_id_addr;
165
166 /**
167 * Handles are kept in a linked list.
168 */
169 struct GNUNET_SET_OperationHandle *prev;
170
171 /**
172 * Handles are kept in a linked list.
173 */
174 struct GNUNET_SET_OperationHandle *next;
175
176 /**
177 * Request ID to identify the operation within the set.
178 */
179 uint32_t request_id;
180};
181
182
183/**
184 * Opaque handle to a listen operation.
185 */
186struct GNUNET_SET_ListenHandle
187{
188 /**
189 * Message queue for the client.
190 */
191 struct GNUNET_MQ_Handle*mq;
192
193 /**
194 * Configuration handle for the listener, stored
195 * here to be able to reconnect transparently on
196 * connection failure.
197 */
198 const struct GNUNET_CONFIGURATION_Handle *cfg;
199
200 /**
201 * Function to call on a new incoming request,
202 * or on error.
203 */
204 GNUNET_SET_ListenCallback listen_cb;
205
206 /**
207 * Closure for @e listen_cb.
208 */
209 void *listen_cls;
210
211 /**
212 * Application ID we listen for.
213 */
214 struct GNUNET_HashCode app_id;
215
216 /**
217 * Time to wait until we try to reconnect on failure.
218 */
219 struct GNUNET_TIME_Relative reconnect_backoff;
220
221 /**
222 * Task for reconnecting when the listener fails.
223 */
224 struct GNUNET_SCHEDULER_Task *reconnect_task;
225
226 /**
227 * Operation we listen for.
228 */
229 enum GNUNET_SET_OperationType operation;
230};
231
232
233/* mutual recursion with handle_copy_lazy */
234static struct GNUNET_SET_Handle *
235create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
236 enum GNUNET_SET_OperationType op,
237 const uint32_t *cookie);
238
239
240/**
241 * Handle element for iteration over the set. Notifies the
242 * iterator and sends an acknowledgement to the service.
243 *
244 * @param cls the `struct GNUNET_SET_Handle *`
245 * @param msg the message
246 */
247static void
248handle_copy_lazy (void *cls,
249 const struct GNUNET_SET_CopyLazyResponseMessage *msg)
250{
251 struct GNUNET_SET_Handle *set = cls;
252 struct SetCopyRequest *req;
253 struct GNUNET_SET_Handle *new_set;
254
255 req = set->copy_req_head;
256 if (NULL == req)
257 {
258 /* Service sent us unsolicited lazy copy response */
259 GNUNET_break (0);
260 return;
261 }
262
263 LOG (GNUNET_ERROR_TYPE_DEBUG,
264 "Handling response to lazy copy\n");
265 GNUNET_CONTAINER_DLL_remove (set->copy_req_head,
266 set->copy_req_tail,
267 req);
268 // We pass none as operation here, since it doesn't matter when
269 // cloning.
270 new_set = create_internal (set->cfg,
271 GNUNET_SET_OPERATION_NONE,
272 &msg->cookie);
273 req->cb (req->cls, new_set);
274 GNUNET_free (req);
275}
276
277
278/**
279 * Check that the given @a msg is well-formed.
280 *
281 * @param cls closure
282 * @param msg message to check
283 * @return #GNUNET_OK if message is well-formed
284 */
285static int
286check_iter_element (void *cls,
287 const struct GNUNET_SET_IterResponseMessage *msg)
288{
289 /* minimum size was already checked, everything else is OK! */
290 return GNUNET_OK;
291}
292
293
294/**
295 * Handle element for iteration over the set. Notifies the
296 * iterator and sends an acknowledgement to the service.
297 *
298 * @param cls the `struct GNUNET_SET_Handle *`
299 * @param mh the message
300 */
301static void
302handle_iter_element (void *cls,
303 const struct GNUNET_SET_IterResponseMessage *msg)
304{
305 struct GNUNET_SET_Handle *set = cls;
306 GNUNET_SET_ElementIterator iter = set->iterator;
307 struct GNUNET_SET_Element element;
308 struct GNUNET_SET_IterAckMessage *ack_msg;
309 struct GNUNET_MQ_Envelope *ev;
310 uint16_t msize;
311
312 LOG (GNUNET_ERROR_TYPE_DEBUG,
313 "Received element in set iteration\n");
314 msize = ntohs (msg->header.size);
315 if (set->iteration_id != ntohs (msg->iteration_id))
316 {
317 /* element from a previous iteration, skip! */
318 iter = NULL;
319 }
320 if (NULL != iter)
321 {
322 element.size = msize - sizeof(struct GNUNET_SET_IterResponseMessage);
323 element.element_type = ntohs (msg->element_type);
324 element.data = &msg[1];
325 iter (set->iterator_cls,
326 &element);
327 }
328 ev = GNUNET_MQ_msg (ack_msg,
329 GNUNET_MESSAGE_TYPE_SET_ITER_ACK);
330 ack_msg->send_more = htonl ((NULL != iter));
331 GNUNET_MQ_send (set->mq, ev);
332}
333
334
335/**
336 * Handle message signalling conclusion of iteration over the set.
337 * Notifies the iterator that we are done.
338 *
339 * @param cls the set
340 * @param mh the message
341 */
342static void
343handle_iter_done (void *cls,
344 const struct GNUNET_MessageHeader *mh)
345{
346 struct GNUNET_SET_Handle *set = cls;
347 GNUNET_SET_ElementIterator iter = set->iterator;
348
349 if (NULL == iter)
350 {
351 /* FIXME: if this is true, could cancel+start a fresh one
352 cause elements to go to the wrong iteration? */
353 LOG (GNUNET_ERROR_TYPE_INFO,
354 "Service completed set iteration that was already cancelled\n");
355 return;
356 }
357 LOG (GNUNET_ERROR_TYPE_DEBUG,
358 "Set iteration completed\n");
359 set->destroy_requested = GNUNET_SYSERR;
360 set->iterator = NULL;
361 set->iteration_id++;
362 iter (set->iterator_cls,
363 NULL);
364 if (GNUNET_SYSERR == set->destroy_requested)
365 set->destroy_requested = GNUNET_NO;
366 if (GNUNET_YES == set->destroy_requested)
367 GNUNET_SET_destroy (set);
368}
369
370
371/**
372 * Check that the given @a msg is well-formed.
373 *
374 * @param cls closure
375 * @param msg message to check
376 * @return #GNUNET_OK if message is well-formed
377 */
378static int
379check_result (void *cls,
380 const struct GNUNET_SET_ResultMessage *msg)
381{
382 /* minimum size was already checked, everything else is OK! */
383 return GNUNET_OK;
384}
385
386
387/**
388 * Handle result message for a set operation.
389 *
390 * @param cls the set
391 * @param mh the message
392 */
393static void
394handle_result (void *cls,
395 const struct GNUNET_SET_ResultMessage *msg)
396{
397 struct GNUNET_SET_Handle *set = cls;
398 struct GNUNET_SET_OperationHandle *oh;
399 struct GNUNET_SET_Element e;
400 enum GNUNET_SET_Status result_status;
401 int destroy_set;
402
403 GNUNET_assert (NULL != set->mq);
404 result_status = (enum GNUNET_SET_Status) ntohs (msg->result_status);
405 LOG (GNUNET_ERROR_TYPE_DEBUG,
406 "Got result message with status %d\n",
407 result_status);
408
409 oh = GNUNET_MQ_assoc_get (set->mq,
410 ntohl (msg->request_id));
411 if (NULL == oh)
412 {
413 /* 'oh' can be NULL if we canceled the operation, but the service
414 did not get the cancel message yet. */
415 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
416 "Ignoring result from canceled operation\n");
417 return;
418 }
419
420 switch (result_status)
421 {
422 case GNUNET_SET_STATUS_OK:
423 case GNUNET_SET_STATUS_ADD_LOCAL:
424 case GNUNET_SET_STATUS_ADD_REMOTE:
425 goto do_element;
426
427 case GNUNET_SET_STATUS_FAILURE:
428 case GNUNET_SET_STATUS_DONE:
429 goto do_final;
430
431 case GNUNET_SET_STATUS_HALF_DONE:
432 /* not used anymore */
433 GNUNET_assert (0);
434 }
435
436do_final:
437 LOG (GNUNET_ERROR_TYPE_DEBUG,
438 "Treating result as final status\n");
439 GNUNET_MQ_assoc_remove (set->mq,
440 ntohl (msg->request_id));
441 GNUNET_CONTAINER_DLL_remove (set->ops_head,
442 set->ops_tail,
443 oh);
444 /* Need to do this calculation _before_ the result callback,
445 as IF the application still has a valid set handle, it
446 may trigger destruction of the set during the callback. */
447 destroy_set = (GNUNET_YES == set->destroy_requested) &&
448 (NULL == set->ops_head);
449 if (NULL != oh->result_cb)
450 {
451 oh->result_cb (oh->result_cls,
452 NULL,
453 GNUNET_ntohll (msg->current_size),
454 result_status);
455 }
456 else
457 {
458 LOG (GNUNET_ERROR_TYPE_DEBUG,
459 "No callback for final status\n");
460 }
461 if (destroy_set)
462 GNUNET_SET_destroy (set);
463 GNUNET_free (oh);
464 return;
465
466do_element:
467 LOG (GNUNET_ERROR_TYPE_DEBUG,
468 "Treating result as element\n");
469 e.data = &msg[1];
470 e.size = ntohs (msg->header.size) - sizeof(struct GNUNET_SET_ResultMessage);
471 e.element_type = ntohs (msg->element_type);
472 if (NULL != oh->result_cb)
473 oh->result_cb (oh->result_cls,
474 &e,
475 GNUNET_ntohll (msg->current_size),
476 result_status);
477}
478
479
480/**
481 * Destroy the given set operation.
482 *
483 * @param oh set operation to destroy
484 */
485static void
486set_operation_destroy (struct GNUNET_SET_OperationHandle *oh)
487{
488 struct GNUNET_SET_Handle *set = oh->set;
489 struct GNUNET_SET_OperationHandle *h_assoc;
490
491 if (NULL != oh->conclude_mqm)
492 GNUNET_MQ_discard (oh->conclude_mqm);
493 /* is the operation already committed? */
494 if (NULL != set)
495 {
496 GNUNET_CONTAINER_DLL_remove (set->ops_head,
497 set->ops_tail,
498 oh);
499 h_assoc = GNUNET_MQ_assoc_remove (set->mq,
500 oh->request_id);
501 GNUNET_assert ((NULL == h_assoc) ||
502 (h_assoc == oh));
503 }
504 GNUNET_free (oh);
505}
506
507
508/**
509 * Cancel the given set operation. We need to send an explicit cancel
510 * message, as all operations one one set communicate using one
511 * handle.
512 *
513 * @param oh set operation to cancel
514 */
515void
516GNUNET_SET_operation_cancel (struct GNUNET_SET_OperationHandle *oh)
517{
518 struct GNUNET_SET_Handle *set = oh->set;
519 struct GNUNET_SET_CancelMessage *m;
520 struct GNUNET_MQ_Envelope *mqm;
521
522 LOG (GNUNET_ERROR_TYPE_DEBUG,
523 "Cancelling SET operation\n");
524 if (NULL != set)
525 {
526 mqm = GNUNET_MQ_msg (m, GNUNET_MESSAGE_TYPE_SET_CANCEL);
527 m->request_id = htonl (oh->request_id);
528 GNUNET_MQ_send (set->mq, mqm);
529 }
530 set_operation_destroy (oh);
531 if ((NULL != set) &&
532 (GNUNET_YES == set->destroy_requested) &&
533 (NULL == set->ops_head))
534 {
535 LOG (GNUNET_ERROR_TYPE_DEBUG,
536 "Destroying set after operation cancel\n");
537 GNUNET_SET_destroy (set);
538 }
539}
540
541
542/**
543 * We encountered an error communicating with the set service while
544 * performing a set operation. Report to the application.
545 *
546 * @param cls the `struct GNUNET_SET_Handle`
547 * @param error error code
548 */
549static void
550handle_client_set_error (void *cls,
551 enum GNUNET_MQ_Error error)
552{
553 struct GNUNET_SET_Handle *set = cls;
554 GNUNET_SET_ElementIterator iter = set->iterator;
555
556 LOG (GNUNET_ERROR_TYPE_ERROR,
557 "Handling client set error %d\n",
558 error);
559 while (NULL != set->ops_head)
560 {
561 if ((NULL != set->ops_head->result_cb) &&
562 (GNUNET_NO == set->destroy_requested))
563 set->ops_head->result_cb (set->ops_head->result_cls,
564 NULL,
565 0,
566 GNUNET_SET_STATUS_FAILURE);
567 set_operation_destroy (set->ops_head);
568 }
569 set->iterator = NULL;
570 set->iteration_id++;
571 set->invalid = GNUNET_YES;
572 if (NULL != iter)
573 iter (set->iterator_cls,
574 NULL);
575}
576
577
578/**
579 * FIXME.
580 */
581static struct GNUNET_SET_Handle *
582create_internal (const struct GNUNET_CONFIGURATION_Handle *cfg,
583 enum GNUNET_SET_OperationType op,
584 const uint32_t *cookie)
585{
586 struct GNUNET_SET_Handle *set = GNUNET_new (struct GNUNET_SET_Handle);
587 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
588 GNUNET_MQ_hd_var_size (result,
589 GNUNET_MESSAGE_TYPE_SET_RESULT,
590 struct GNUNET_SET_ResultMessage,
591 set),
592 GNUNET_MQ_hd_var_size (iter_element,
593 GNUNET_MESSAGE_TYPE_SET_ITER_ELEMENT,
594 struct GNUNET_SET_IterResponseMessage,
595 set),
596 GNUNET_MQ_hd_fixed_size (iter_done,
597 GNUNET_MESSAGE_TYPE_SET_ITER_DONE,
598 struct GNUNET_MessageHeader,
599 set),
600 GNUNET_MQ_hd_fixed_size (copy_lazy,
601 GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_RESPONSE,
602 struct GNUNET_SET_CopyLazyResponseMessage,
603 set),
604 GNUNET_MQ_handler_end ()
605 };
606 struct GNUNET_MQ_Envelope *mqm;
607 struct GNUNET_SET_CreateMessage *create_msg;
608 struct GNUNET_SET_CopyLazyConnectMessage *copy_msg;
609
610 set->cfg = cfg;
611 set->mq = GNUNET_CLIENT_connect (cfg,
612 "set",
613 mq_handlers,
614 &handle_client_set_error,
615 set);
616 if (NULL == set->mq)
617 {
618 GNUNET_free (set);
619 return NULL;
620 }
621 if (NULL == cookie)
622 {
623 LOG (GNUNET_ERROR_TYPE_DEBUG,
624 "Creating new set (operation %u)\n",
625 op);
626 mqm = GNUNET_MQ_msg (create_msg,
627 GNUNET_MESSAGE_TYPE_SET_CREATE);
628 create_msg->operation = htonl (op);
629 }
630 else
631 {
632 LOG (GNUNET_ERROR_TYPE_DEBUG,
633 "Creating new set (lazy copy)\n");
634 mqm = GNUNET_MQ_msg (copy_msg,
635 GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_CONNECT);
636 copy_msg->cookie = *cookie;
637 }
638 GNUNET_MQ_send (set->mq,
639 mqm);
640 return set;
641}
642
643
644/**
645 * Create an empty set, supporting the specified operation.
646 *
647 * @param cfg configuration to use for connecting to the
648 * set service
649 * @param op operation supported by the set
650 * Note that the operation has to be specified
651 * beforehand, as certain set operations need to maintain
652 * data structures specific to the operation
653 * @return a handle to the set
654 */
655struct GNUNET_SET_Handle *
656GNUNET_SET_create (const struct GNUNET_CONFIGURATION_Handle *cfg,
657 enum GNUNET_SET_OperationType op)
658{
659 struct GNUNET_SET_Handle *set;
660
661 set = create_internal (cfg,
662 op,
663 NULL);
664 LOG (GNUNET_ERROR_TYPE_DEBUG,
665 "Creating set %p for operation %d\n",
666 set,
667 op);
668 return set;
669}
670
671
672/**
673 * Add an element to the given set. After the element has been added
674 * (in the sense of being transmitted to the set service), @a cont
675 * will be called. Multiple calls to GNUNET_SET_add_element() can be
676 * queued.
677 *
678 * @param set set to add element to
679 * @param element element to add to the set
680 * @param cont continuation called after the element has been added
681 * @param cont_cls closure for @a cont
682 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
683 * set is invalid (e.g. the set service crashed)
684 */
685int
686GNUNET_SET_add_element (struct GNUNET_SET_Handle *set,
687 const struct GNUNET_SET_Element *element,
688 GNUNET_SET_Continuation cont,
689 void *cont_cls)
690{
691 struct GNUNET_MQ_Envelope *mqm;
692 struct GNUNET_SET_ElementMessage *msg;
693
694 LOG (GNUNET_ERROR_TYPE_DEBUG,
695 "adding element of type %u to set %p\n",
696 (unsigned int) element->element_type,
697 set);
698 GNUNET_assert (NULL != set);
699 if (GNUNET_YES == set->invalid)
700 {
701 if (NULL != cont)
702 cont (cont_cls);
703 return GNUNET_SYSERR;
704 }
705 mqm = GNUNET_MQ_msg_extra (msg,
706 element->size,
707 GNUNET_MESSAGE_TYPE_SET_ADD);
708 msg->element_type = htons (element->element_type);
709 GNUNET_memcpy (&msg[1],
710 element->data,
711 element->size);
712 GNUNET_MQ_notify_sent (mqm,
713 cont, cont_cls);
714 GNUNET_MQ_send (set->mq, mqm);
715 return GNUNET_OK;
716}
717
718
719/**
720 * Remove an element to the given set. After the element has been
721 * removed (in the sense of the request being transmitted to the set
722 * service), @a cont will be called. Multiple calls to
723 * GNUNET_SET_remove_element() can be queued
724 *
725 * @param set set to remove element from
726 * @param element element to remove from the set
727 * @param cont continuation called after the element has been removed
728 * @param cont_cls closure for @a cont
729 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
730 * set is invalid (e.g. the set service crashed)
731 */
732int
733GNUNET_SET_remove_element (struct GNUNET_SET_Handle *set,
734 const struct GNUNET_SET_Element *element,
735 GNUNET_SET_Continuation cont,
736 void *cont_cls)
737{
738 struct GNUNET_MQ_Envelope *mqm;
739 struct GNUNET_SET_ElementMessage *msg;
740
741 LOG (GNUNET_ERROR_TYPE_DEBUG,
742 "Removing element from set %p\n",
743 set);
744 if (GNUNET_YES == set->invalid)
745 {
746 if (NULL != cont)
747 cont (cont_cls);
748 return GNUNET_SYSERR;
749 }
750 mqm = GNUNET_MQ_msg_extra (msg,
751 element->size,
752 GNUNET_MESSAGE_TYPE_SET_REMOVE);
753 msg->element_type = htons (element->element_type);
754 GNUNET_memcpy (&msg[1],
755 element->data,
756 element->size);
757 GNUNET_MQ_notify_sent (mqm,
758 cont, cont_cls);
759 GNUNET_MQ_send (set->mq, mqm);
760 return GNUNET_OK;
761}
762
763
764/**
765 * Destroy the set handle if no operations are left, mark the set
766 * for destruction otherwise.
767 *
768 * @param set set handle to destroy
769 */
770void
771GNUNET_SET_destroy (struct GNUNET_SET_Handle *set)
772{
773 /* destroying set while iterator is active is currently
774 not supported; we should expand the API to allow
775 clients to explicitly cancel the iteration! */
776 GNUNET_assert (NULL != set);
777 if ((NULL != set->ops_head) ||
778 (NULL != set->iterator) ||
779 (GNUNET_SYSERR == set->destroy_requested))
780 {
781 LOG (GNUNET_ERROR_TYPE_DEBUG,
782 "Set operations are pending, delaying set destruction\n");
783 set->destroy_requested = GNUNET_YES;
784 return;
785 }
786 LOG (GNUNET_ERROR_TYPE_DEBUG,
787 "Really destroying set\n");
788 if (NULL != set->mq)
789 {
790 GNUNET_MQ_destroy (set->mq);
791 set->mq = NULL;
792 }
793 GNUNET_free (set);
794}
795
796
797/**
798 * Prepare a set operation to be evaluated with another peer.
799 * The evaluation will not start until the client provides
800 * a local set with #GNUNET_SET_commit().
801 *
802 * @param other_peer peer with the other set
803 * @param app_id hash for the application using the set
804 * @param context_msg additional information for the request
805 * @param result_mode specified how results will be returned,
806 * see `enum GNUNET_SET_ResultMode`.
807 * @param result_cb called on error or success
808 * @param result_cls closure for @e result_cb
809 * @return a handle to cancel the operation
810 */
811struct GNUNET_SET_OperationHandle *
812GNUNET_SET_prepare (const struct GNUNET_PeerIdentity *other_peer,
813 const struct GNUNET_HashCode *app_id,
814 const struct GNUNET_MessageHeader *context_msg,
815 enum GNUNET_SET_ResultMode result_mode,
816 struct GNUNET_SET_Option options[],
817 GNUNET_SET_ResultIterator result_cb,
818 void *result_cls)
819{
820 struct GNUNET_MQ_Envelope *mqm;
821 struct GNUNET_SET_OperationHandle *oh;
822 struct GNUNET_SET_EvaluateMessage *msg;
823 struct GNUNET_SET_Option *opt;
824
825 LOG (GNUNET_ERROR_TYPE_DEBUG,
826 "Client prepares set operation (%d)\n",
827 result_mode);
828 oh = GNUNET_new (struct GNUNET_SET_OperationHandle);
829 oh->result_cb = result_cb;
830 oh->result_cls = result_cls;
831 mqm = GNUNET_MQ_msg_nested_mh (msg,
832 GNUNET_MESSAGE_TYPE_SET_EVALUATE,
833 context_msg);
834 msg->app_id = *app_id;
835 msg->result_mode = htonl (result_mode);
836 msg->target_peer = *other_peer;
837 for (opt = options; opt->type != 0; opt++)
838 {
839 switch (opt->type)
840 {
841 case GNUNET_SET_OPTION_BYZANTINE:
842 msg->byzantine = GNUNET_YES;
843 msg->byzantine_lower_bound = opt->v.num;
844 break;
845
846 case GNUNET_SET_OPTION_FORCE_FULL:
847 msg->force_full = GNUNET_YES;
848 break;
849
850 case GNUNET_SET_OPTION_FORCE_DELTA:
851 msg->force_delta = GNUNET_YES;
852 break;
853
854 default:
855 LOG (GNUNET_ERROR_TYPE_ERROR,
856 "Option with type %d not recognized\n", (int) opt->type);
857 }
858 }
859 oh->conclude_mqm = mqm;
860 oh->request_id_addr = &msg->request_id;
861
862 return oh;
863}
864
865
866/**
867 * Connect to the set service in order to listen for requests.
868 *
869 * @param cls the `struct GNUNET_SET_ListenHandle *` to connect
870 */
871static void
872listen_connect (void *cls);
873
874
875/**
876 * Check validity of request message for a listen operation
877 *
878 * @param cls the listen handle
879 * @param msg the message
880 * @return #GNUNET_OK if the message is well-formed
881 */
882static int
883check_request (void *cls,
884 const struct GNUNET_SET_RequestMessage *msg)
885{
886 const struct GNUNET_MessageHeader *context_msg;
887
888 if (ntohs (msg->header.size) == sizeof(*msg))
889 return GNUNET_OK; /* no context message is OK */
890 context_msg = GNUNET_MQ_extract_nested_mh (msg);
891 if (NULL == context_msg)
892 {
893 /* malformed context message is NOT ok */
894 GNUNET_break_op (0);
895 return GNUNET_SYSERR;
896 }
897 return GNUNET_OK;
898}
899
900
901/**
902 * Handle request message for a listen operation
903 *
904 * @param cls the listen handle
905 * @param msg the message
906 */
907static void
908handle_request (void *cls,
909 const struct GNUNET_SET_RequestMessage *msg)
910{
911 struct GNUNET_SET_ListenHandle *lh = cls;
912 struct GNUNET_SET_Request req;
913 const struct GNUNET_MessageHeader *context_msg;
914 struct GNUNET_MQ_Envelope *mqm;
915 struct GNUNET_SET_RejectMessage *rmsg;
916
917 LOG (GNUNET_ERROR_TYPE_DEBUG,
918 "Processing incoming operation request with id %u\n",
919 ntohl (msg->accept_id));
920 /* we got another valid request => reset the backoff */
921 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
922 req.accept_id = ntohl (msg->accept_id);
923 req.accepted = GNUNET_NO;
924 context_msg = GNUNET_MQ_extract_nested_mh (msg);
925 /* calling #GNUNET_SET_accept() in the listen cb will set req->accepted */
926 lh->listen_cb (lh->listen_cls,
927 &msg->peer_id,
928 context_msg,
929 &req);
930 if (GNUNET_YES == req.accepted)
931 return; /* the accept-case is handled in #GNUNET_SET_accept() */
932 LOG (GNUNET_ERROR_TYPE_DEBUG,
933 "Rejected request %u\n",
934 ntohl (msg->accept_id));
935 mqm = GNUNET_MQ_msg (rmsg,
936 GNUNET_MESSAGE_TYPE_SET_REJECT);
937 rmsg->accept_reject_id = msg->accept_id;
938 GNUNET_MQ_send (lh->mq, mqm);
939}
940
941
942/**
943 * Our connection with the set service encountered an error,
944 * re-initialize with exponential back-off.
945 *
946 * @param cls the `struct GNUNET_SET_ListenHandle *`
947 * @param error reason for the disconnect
948 */
949static void
950handle_client_listener_error (void *cls,
951 enum GNUNET_MQ_Error error)
952{
953 struct GNUNET_SET_ListenHandle *lh = cls;
954
955 LOG (GNUNET_ERROR_TYPE_DEBUG,
956 "Listener broke down (%d), re-connecting\n",
957 (int) error);
958 GNUNET_MQ_destroy (lh->mq);
959 lh->mq = NULL;
960 lh->reconnect_task = GNUNET_SCHEDULER_add_delayed (lh->reconnect_backoff,
961 &listen_connect,
962 lh);
963 lh->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (lh->reconnect_backoff);
964}
965
966
967/**
968 * Connect to the set service in order to listen for requests.
969 *
970 * @param cls the `struct GNUNET_SET_ListenHandle *` to connect
971 */
972static void
973listen_connect (void *cls)
974{
975 struct GNUNET_SET_ListenHandle *lh = cls;
976 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
977 GNUNET_MQ_hd_var_size (request,
978 GNUNET_MESSAGE_TYPE_SET_REQUEST,
979 struct GNUNET_SET_RequestMessage,
980 lh),
981 GNUNET_MQ_handler_end ()
982 };
983 struct GNUNET_MQ_Envelope *mqm;
984 struct GNUNET_SET_ListenMessage *msg;
985
986 lh->reconnect_task = NULL;
987 GNUNET_assert (NULL == lh->mq);
988 lh->mq = GNUNET_CLIENT_connect (lh->cfg,
989 "set",
990 mq_handlers,
991 &handle_client_listener_error,
992 lh);
993 if (NULL == lh->mq)
994 return;
995 mqm = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SET_LISTEN);
996 msg->operation = htonl (lh->operation);
997 msg->app_id = lh->app_id;
998 GNUNET_MQ_send (lh->mq,
999 mqm);
1000}
1001
1002
1003/**
1004 * Wait for set operation requests for the given application id
1005 *
1006 * @param cfg configuration to use for connecting to
1007 * the set service, needs to be valid for the lifetime of the listen handle
1008 * @param operation operation we want to listen for
1009 * @param app_id id of the application that handles set operation requests
1010 * @param listen_cb called for each incoming request matching the operation
1011 * and application id
1012 * @param listen_cls handle for @a listen_cb
1013 * @return a handle that can be used to cancel the listen operation
1014 */
1015struct GNUNET_SET_ListenHandle *
1016GNUNET_SET_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
1017 enum GNUNET_SET_OperationType operation,
1018 const struct GNUNET_HashCode *app_id,
1019 GNUNET_SET_ListenCallback listen_cb,
1020 void *listen_cls)
1021{
1022 struct GNUNET_SET_ListenHandle *lh;
1023
1024 LOG (GNUNET_ERROR_TYPE_DEBUG,
1025 "Starting listener for app %s\n",
1026 GNUNET_h2s (app_id));
1027 lh = GNUNET_new (struct GNUNET_SET_ListenHandle);
1028 lh->listen_cb = listen_cb;
1029 lh->listen_cls = listen_cls;
1030 lh->cfg = cfg;
1031 lh->operation = operation;
1032 lh->app_id = *app_id;
1033 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
1034 listen_connect (lh);
1035 if (NULL == lh->mq)
1036 {
1037 GNUNET_free (lh);
1038 return NULL;
1039 }
1040 return lh;
1041}
1042
1043
1044/**
1045 * Cancel the given listen operation.
1046 *
1047 * @param lh handle for the listen operation
1048 */
1049void
1050GNUNET_SET_listen_cancel (struct GNUNET_SET_ListenHandle *lh)
1051{
1052 LOG (GNUNET_ERROR_TYPE_DEBUG,
1053 "Canceling listener %s\n",
1054 GNUNET_h2s (&lh->app_id));
1055 if (NULL != lh->mq)
1056 {
1057 GNUNET_MQ_destroy (lh->mq);
1058 lh->mq = NULL;
1059 }
1060 if (NULL != lh->reconnect_task)
1061 {
1062 GNUNET_SCHEDULER_cancel (lh->reconnect_task);
1063 lh->reconnect_task = NULL;
1064 }
1065 GNUNET_free (lh);
1066}
1067
1068
1069/**
1070 * Accept a request we got via #GNUNET_SET_listen. Must be called during
1071 * #GNUNET_SET_listen, as the 'struct GNUNET_SET_Request' becomes invalid
1072 * afterwards.
1073 * Call #GNUNET_SET_commit to provide the local set to use for the operation,
1074 * and to begin the exchange with the remote peer.
1075 *
1076 * @param request request to accept
1077 * @param result_mode specified how results will be returned,
1078 * see `enum GNUNET_SET_ResultMode`.
1079 * @param result_cb callback for the results
1080 * @param result_cls closure for @a result_cb
1081 * @return a handle to cancel the operation
1082 */
1083struct GNUNET_SET_OperationHandle *
1084GNUNET_SET_accept (struct GNUNET_SET_Request *request,
1085 enum GNUNET_SET_ResultMode result_mode,
1086 struct GNUNET_SET_Option options[],
1087 GNUNET_SET_ResultIterator result_cb,
1088 void *result_cls)
1089{
1090 struct GNUNET_MQ_Envelope *mqm;
1091 struct GNUNET_SET_OperationHandle *oh;
1092 struct GNUNET_SET_AcceptMessage *msg;
1093
1094 GNUNET_assert (GNUNET_NO == request->accepted);
1095 LOG (GNUNET_ERROR_TYPE_DEBUG,
1096 "Client accepts set operation (%d) with id %u\n",
1097 result_mode,
1098 request->accept_id);
1099 request->accepted = GNUNET_YES;
1100 mqm = GNUNET_MQ_msg (msg,
1101 GNUNET_MESSAGE_TYPE_SET_ACCEPT);
1102 msg->accept_reject_id = htonl (request->accept_id);
1103 msg->result_mode = htonl (result_mode);
1104 oh = GNUNET_new (struct GNUNET_SET_OperationHandle);
1105 oh->result_cb = result_cb;
1106 oh->result_cls = result_cls;
1107 oh->conclude_mqm = mqm;
1108 oh->request_id_addr = &msg->request_id;
1109 return oh;
1110}
1111
1112
1113/**
1114 * Commit a set to be used with a set operation.
1115 * This function is called once we have fully constructed
1116 * the set that we want to use for the operation. At this
1117 * time, the P2P protocol can then begin to exchange the
1118 * set information and call the result callback with the
1119 * result information.
1120 *
1121 * @param oh handle to the set operation
1122 * @param set the set to use for the operation
1123 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
1124 * set is invalid (e.g. the set service crashed)
1125 */
1126int
1127GNUNET_SET_commit (struct GNUNET_SET_OperationHandle *oh,
1128 struct GNUNET_SET_Handle *set)
1129{
1130 if (NULL != oh->set)
1131 {
1132 /* Some other set was already committed for this
1133 * operation, there is a logic bug in the client of this API */
1134 GNUNET_break (0);
1135 return GNUNET_OK;
1136 }
1137 GNUNET_assert (NULL != set);
1138 if (GNUNET_YES == set->invalid)
1139 return GNUNET_SYSERR;
1140 LOG (GNUNET_ERROR_TYPE_DEBUG,
1141 "Client commits to SET\n");
1142 GNUNET_assert (NULL != oh->conclude_mqm);
1143 oh->set = set;
1144 GNUNET_CONTAINER_DLL_insert (set->ops_head,
1145 set->ops_tail,
1146 oh);
1147 oh->request_id = GNUNET_MQ_assoc_add (set->mq,
1148 oh);
1149 *oh->request_id_addr = htonl (oh->request_id);
1150 GNUNET_MQ_send (set->mq,
1151 oh->conclude_mqm);
1152 oh->conclude_mqm = NULL;
1153 oh->request_id_addr = NULL;
1154 return GNUNET_OK;
1155}
1156
1157
1158/**
1159 * Iterate over all elements in the given set. Note that this
1160 * operation involves transferring every element of the set from the
1161 * service to the client, and is thus costly.
1162 *
1163 * @param set the set to iterate over
1164 * @param iter the iterator to call for each element
1165 * @param iter_cls closure for @a iter
1166 * @return #GNUNET_YES if the iteration started successfully,
1167 * #GNUNET_NO if another iteration is active
1168 * #GNUNET_SYSERR if the set is invalid (e.g. the server crashed, disconnected)
1169 */
1170int
1171GNUNET_SET_iterate (struct GNUNET_SET_Handle *set,
1172 GNUNET_SET_ElementIterator iter,
1173 void *iter_cls)
1174{
1175 struct GNUNET_MQ_Envelope *ev;
1176
1177 GNUNET_assert (NULL != iter);
1178 if (GNUNET_YES == set->invalid)
1179 return GNUNET_SYSERR;
1180 if (NULL != set->iterator)
1181 return GNUNET_NO;
1182 LOG (GNUNET_ERROR_TYPE_DEBUG,
1183 "Iterating over set\n");
1184 set->iterator = iter;
1185 set->iterator_cls = iter_cls;
1186 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_ITER_REQUEST);
1187 GNUNET_MQ_send (set->mq, ev);
1188 return GNUNET_YES;
1189}
1190
1191
1192void
1193GNUNET_SET_copy_lazy (struct GNUNET_SET_Handle *set,
1194 GNUNET_SET_CopyReadyCallback cb,
1195 void *cls)
1196{
1197 struct GNUNET_MQ_Envelope *ev;
1198 struct SetCopyRequest *req;
1199
1200 LOG (GNUNET_ERROR_TYPE_DEBUG,
1201 "Creating lazy copy of set\n");
1202 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SET_COPY_LAZY_PREPARE);
1203 GNUNET_MQ_send (set->mq, ev);
1204
1205 req = GNUNET_new (struct SetCopyRequest);
1206 req->cb = cb;
1207 req->cls = cls;
1208 GNUNET_CONTAINER_DLL_insert (set->copy_req_head,
1209 set->copy_req_tail,
1210 req);
1211}
1212
1213
1214/**
1215 * Create a copy of an element. The copy
1216 * must be GNUNET_free-d by the caller.
1217 *
1218 * @param element the element to copy
1219 * @return the copied element
1220 */
1221struct GNUNET_SET_Element *
1222GNUNET_SET_element_dup (const struct GNUNET_SET_Element *element)
1223{
1224 struct GNUNET_SET_Element *copy;
1225
1226 copy = GNUNET_malloc (element->size + sizeof(struct GNUNET_SET_Element));
1227 copy->size = element->size;
1228 copy->element_type = element->element_type;
1229 copy->data = &copy[1];
1230 GNUNET_memcpy (&copy[1],
1231 element->data,
1232 copy->size);
1233 return copy;
1234}
1235
1236
1237/**
1238 * Hash a set element.
1239 *
1240 * @param element the element that should be hashed
1241 * @param[out] ret_hash a pointer to where the hash of @a element
1242 * should be stored
1243 */
1244void
1245GNUNET_SET_element_hash (const struct GNUNET_SET_Element *element,
1246 struct GNUNET_HashCode *ret_hash)
1247{
1248 struct GNUNET_HashContext *ctx = GNUNET_CRYPTO_hash_context_start ();
1249
1250 /* It's not guaranteed that the element data is always after the element header,
1251 so we need to hash the chunks separately. */
1252 GNUNET_CRYPTO_hash_context_read (ctx, &element->size, sizeof(uint16_t));
1253 GNUNET_CRYPTO_hash_context_read (ctx, &element->element_type,
1254 sizeof(uint16_t));
1255 GNUNET_CRYPTO_hash_context_read (ctx, element->data, element->size);
1256 GNUNET_CRYPTO_hash_context_finish (ctx, ret_hash);
1257}
1258
1259
1260/* end of set_api.c */
diff --git a/src/set/test_set.conf b/src/set/test_set.conf
deleted file mode 100644
index f9b4547e9..000000000
--- a/src/set/test_set.conf
+++ /dev/null
@@ -1,33 +0,0 @@
1@INLINE@ ../../contrib/conf/gnunet/no_forcestart.conf
2
3[PATHS]
4GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-set/
5
6[set]
7START_ON_DEMAND = YES
8#PREFIX = valgrind --leak-check=full
9#PREFIX = gdbserver :1234
10OPTIONS = -L INFO
11
12[transport]
13PLUGINS = unix
14OPTIONS = -LERROR
15
16[nat]
17RETURN_LOCAL_ADDRESSES = YES
18DISABLEV6 = YES
19USE_LOCALADDR = YES
20
21[peerinfo]
22NO_IO = YES
23
24[nat]
25# Use addresses from the local network interfaces (including loopback, but also others)
26USE_LOCALADDR = YES
27
28# Disable IPv6 support
29DISABLEV6 = NO
30
31# Do we use addresses from localhost address ranges? (::1, 127.0.0.0/8)
32RETURN_LOCAL_ADDRESSES = YES
33
diff --git a/src/set/test_set_api.c b/src/set/test_set_api.c
deleted file mode 100644
index d1afdd354..000000000
--- a/src/set/test_set_api.c
+++ /dev/null
@@ -1,403 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_set_api.c
23 * @brief testcase for set_api.c
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_testing_lib.h"
29#include "gnunet_set_service.h"
30
31
32static struct GNUNET_PeerIdentity local_id;
33
34static struct GNUNET_HashCode app_id;
35
36static struct GNUNET_SET_Handle *set1;
37
38static struct GNUNET_SET_Handle *set2;
39
40static struct GNUNET_SET_ListenHandle *listen_handle;
41
42static struct GNUNET_SET_OperationHandle *oh1;
43
44static struct GNUNET_SET_OperationHandle *oh2;
45
46static const struct GNUNET_CONFIGURATION_Handle *config;
47
48static unsigned int iter_count;
49
50static int ret;
51
52static struct GNUNET_SCHEDULER_Task *tt;
53
54
55static void
56result_cb_set1 (void *cls,
57 const struct GNUNET_SET_Element *element,
58 uint64_t size,
59 enum GNUNET_SET_Status status)
60{
61 switch (status)
62 {
63 case GNUNET_SET_STATUS_OK:
64 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: got element\n");
65 break;
66
67 case GNUNET_SET_STATUS_FAILURE:
68 GNUNET_break (0);
69 oh1 = NULL;
70 fprintf (stderr, "set 1: received failure status!\n");
71 ret = 1;
72 if (NULL != tt)
73 {
74 GNUNET_SCHEDULER_cancel (tt);
75 tt = NULL;
76 }
77 GNUNET_SCHEDULER_shutdown ();
78 break;
79
80 case GNUNET_SET_STATUS_DONE:
81 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: done\n");
82 oh1 = NULL;
83 if (NULL != set1)
84 {
85 GNUNET_SET_destroy (set1);
86 set1 = NULL;
87 }
88 if (NULL == set2)
89 {
90 GNUNET_SCHEDULER_cancel (tt);
91 tt = NULL;
92 GNUNET_SCHEDULER_shutdown ();
93 }
94 break;
95
96 default:
97 GNUNET_assert (0);
98 }
99}
100
101
102static void
103result_cb_set2 (void *cls,
104 const struct GNUNET_SET_Element *element,
105 uint64_t size,
106 enum GNUNET_SET_Status status)
107{
108 switch (status)
109 {
110 case GNUNET_SET_STATUS_OK:
111 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: got element\n");
112 break;
113
114 case GNUNET_SET_STATUS_FAILURE:
115 GNUNET_break (0);
116 oh2 = NULL;
117 fprintf (stderr, "set 2: received failure status\n");
118 GNUNET_SCHEDULER_shutdown ();
119 ret = 1;
120 break;
121
122 case GNUNET_SET_STATUS_DONE:
123 oh2 = NULL;
124 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: done\n");
125 GNUNET_SET_destroy (set2);
126 set2 = NULL;
127 if (NULL == set1)
128 {
129 GNUNET_SCHEDULER_cancel (tt);
130 tt = NULL;
131 GNUNET_SCHEDULER_shutdown ();
132 }
133 break;
134
135 default:
136 GNUNET_assert (0);
137 }
138}
139
140
141static void
142listen_cb (void *cls,
143 const struct GNUNET_PeerIdentity *other_peer,
144 const struct GNUNET_MessageHeader *context_msg,
145 struct GNUNET_SET_Request *request)
146{
147 GNUNET_assert (NULL != context_msg);
148 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
149 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "listen cb called\n");
150 oh2 = GNUNET_SET_accept (request,
151 GNUNET_SET_RESULT_ADDED,
152 (struct GNUNET_SET_Option[]){ 0 },
153 &result_cb_set2,
154 NULL);
155 GNUNET_SET_commit (oh2, set2);
156}
157
158
159/**
160 * Start the set operation.
161 *
162 * @param cls closure, unused
163 */
164static void
165start (void *cls)
166{
167 struct GNUNET_MessageHeader context_msg;
168
169 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting reconciliation\n");
170 context_msg.size = htons (sizeof context_msg);
171 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
172 listen_handle = GNUNET_SET_listen (config,
173 GNUNET_SET_OPERATION_UNION,
174 &app_id,
175 &listen_cb,
176 NULL);
177 oh1 = GNUNET_SET_prepare (&local_id,
178 &app_id,
179 &context_msg,
180 GNUNET_SET_RESULT_ADDED,
181 (struct GNUNET_SET_Option[]){ 0 },
182 &result_cb_set1,
183 NULL);
184 GNUNET_SET_commit (oh1, set1);
185}
186
187
188/**
189 * Initialize the second set, continue
190 *
191 * @param cls closure, unused
192 */
193static void
194init_set2 (void *cls)
195{
196 struct GNUNET_SET_Element element;
197
198 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initializing set 2\n");
199
200 element.element_type = 0;
201 element.data = "hello";
202 element.size = strlen (element.data);
203 GNUNET_SET_add_element (set2, &element, NULL, NULL);
204 element.data = "quux";
205 element.size = strlen (element.data);
206 GNUNET_SET_add_element (set2, &element, NULL, NULL);
207 element.data = "baz";
208 element.size = strlen (element.data);
209 GNUNET_SET_add_element (set2, &element, &start, NULL);
210}
211
212
213/**
214 * Initialize the first set, continue.
215 */
216static void
217init_set1 (void)
218{
219 struct GNUNET_SET_Element element;
220
221 element.element_type = 0;
222 element.data = "hello";
223 element.size = strlen (element.data);
224 GNUNET_SET_add_element (set1, &element, NULL, NULL);
225 element.data = "bar";
226 element.size = strlen (element.data);
227 GNUNET_SET_add_element (set1, &element, &init_set2, NULL);
228 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized set 1\n");
229}
230
231
232static int
233iter_cb (void *cls, const struct GNUNET_SET_Element *element)
234{
235 struct GNUNET_SET_Handle *set = cls;
236
237 if (NULL == element)
238 {
239 GNUNET_assert (3 == iter_count);
240 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
241 "Iteration finished, destroying set %p\n",
242 set);
243 GNUNET_SET_destroy (set);
244 return GNUNET_YES;
245 }
246 iter_count++;
247 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "iter: got element %u\n", iter_count);
248 return GNUNET_YES;
249}
250
251
252static void
253test_iter ()
254{
255 struct GNUNET_SET_Element element;
256 struct GNUNET_SET_Handle *iter_set;
257
258 iter_set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
259 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
260 "Testing iteration over 3 elements on set %p\n",
261 iter_set);
262 element.element_type = 0;
263
264 element.data = "hello";
265 element.size = strlen (element.data);
266 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
267 element.data = "bar";
268 element.size = strlen (element.data);
269 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
270 element.data = "quux";
271 element.size = strlen (element.data);
272 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
273 GNUNET_SET_iterate (iter_set, &iter_cb, iter_set);
274}
275
276
277/**
278 * Function run on timeout.
279 *
280 * @param cls closure
281 */
282static void
283timeout_fail (void *cls)
284{
285 tt = NULL;
286 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, "Testcase failed with timeout\n");
287 GNUNET_SCHEDULER_shutdown ();
288 ret = 1;
289}
290
291
292/**
293 * Function run on shutdown.
294 *
295 * @param cls closure
296 */
297static void
298do_shutdown (void *cls)
299{
300 if (NULL != tt)
301 {
302 GNUNET_SCHEDULER_cancel (tt);
303 tt = NULL;
304 }
305 if (NULL != oh1)
306 {
307 GNUNET_SET_operation_cancel (oh1);
308 oh1 = NULL;
309 }
310 if (NULL != oh2)
311 {
312 GNUNET_SET_operation_cancel (oh2);
313 oh2 = NULL;
314 }
315 if (NULL != set1)
316 {
317 GNUNET_SET_destroy (set1);
318 set1 = NULL;
319 }
320 if (NULL != set2)
321 {
322 GNUNET_SET_destroy (set2);
323 set2 = NULL;
324 }
325 if (NULL != listen_handle)
326 {
327 GNUNET_SET_listen_cancel (listen_handle);
328 listen_handle = NULL;
329 }
330}
331
332
333/**
334 * Signature of the 'main' function for a (single-peer) testcase that
335 * is run using 'GNUNET_TESTING_peer_run'.
336 *
337 * @param cls closure
338 * @param cfg configuration of the peer that was started
339 * @param peer identity of the peer that was created
340 */
341static void
342run (void *cls,
343 const struct GNUNET_CONFIGURATION_Handle *cfg,
344 struct GNUNET_TESTING_Peer *peer)
345{
346 struct GNUNET_SET_OperationHandle *my_oh;
347
348 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Running preparatory tests\n");
349 tt = GNUNET_SCHEDULER_add_delayed (
350 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
351 &timeout_fail,
352 NULL);
353 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
354
355 config = cfg;
356 GNUNET_assert (GNUNET_OK == GNUNET_CRYPTO_get_peer_identity (cfg, &local_id));
357 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
358 "my id (from CRYPTO): %s\n",
359 GNUNET_i2s (&local_id));
360 GNUNET_TESTING_peer_get_identity (peer, &local_id);
361 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
362 "my id (from TESTING): %s\n",
363 GNUNET_i2s (&local_id));
364 test_iter ();
365
366 set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
367 set2 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
368 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
369 "Created sets %p and %p for union operation\n",
370 set1,
371 set2);
372 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
373
374 /* test if canceling an uncommitted request works! */
375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
376 "Launching and instantly stopping set operation\n");
377 my_oh = GNUNET_SET_prepare (&local_id,
378 &app_id,
379 NULL,
380 GNUNET_SET_RESULT_ADDED,
381 (struct GNUNET_SET_Option[]){ 0 },
382 NULL,
383 NULL);
384 GNUNET_SET_operation_cancel (my_oh);
385
386 /* test the real set reconciliation */
387 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Running real set-reconciliation\n");
388 init_set1 ();
389}
390
391
392int
393main (int argc, char **argv)
394{
395 GNUNET_log_setup ("test_set_api", "WARNING", NULL);
396 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Launching peer\n");
397 if (0 !=
398 GNUNET_TESTING_peer_run ("test_set_api", "test_set.conf", &run, NULL))
399 {
400 return 1;
401 }
402 return ret;
403}
diff --git a/src/set/test_set_intersection_result_full.c b/src/set/test_set_intersection_result_full.c
deleted file mode 100644
index 42dedb846..000000000
--- a/src/set/test_set_intersection_result_full.c
+++ /dev/null
@@ -1,393 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2014 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_set_intersection_result_full.c
23 * @brief testcase for full result mode of the intersection set operation
24 * @author Christian Fuchs
25 * @author Christian Grothoff
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_testing_lib.h"
30#include "gnunet_set_service.h"
31
32
33static int ret;
34
35static struct GNUNET_PeerIdentity local_id;
36
37static struct GNUNET_HashCode app_id;
38
39static struct GNUNET_SET_Handle *set1;
40
41static struct GNUNET_SET_Handle *set2;
42
43static struct GNUNET_SET_ListenHandle *listen_handle;
44
45static const struct GNUNET_CONFIGURATION_Handle *config;
46
47static int iter_count;
48
49static struct GNUNET_SCHEDULER_Task *tt;
50
51static struct GNUNET_SET_OperationHandle *oh1;
52
53static struct GNUNET_SET_OperationHandle *oh2;
54
55
56static void
57result_cb_set1 (void *cls,
58 const struct GNUNET_SET_Element *element,
59 uint64_t current_size,
60 enum GNUNET_SET_Status status)
61{
62 static int count;
63
64 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
65 "Processing result set 1 (%d)\n",
66 status);
67 switch (status)
68 {
69 case GNUNET_SET_STATUS_OK:
70 count++;
71 break;
72
73 case GNUNET_SET_STATUS_FAILURE:
74 oh1 = NULL;
75 ret = 1;
76 break;
77
78 case GNUNET_SET_STATUS_DONE:
79 oh1 = NULL;
80 GNUNET_assert (1 == count);
81 GNUNET_SET_destroy (set1);
82 set1 = NULL;
83 if (NULL == set2)
84 GNUNET_SCHEDULER_shutdown ();
85 break;
86
87 default:
88 GNUNET_assert (0);
89 }
90}
91
92
93static void
94result_cb_set2 (void *cls,
95 const struct GNUNET_SET_Element *element,
96 uint64_t current_size,
97 enum GNUNET_SET_Status status)
98{
99 static int count;
100
101 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
102 "Processing result set 2 (%d)\n",
103 status);
104 switch (status)
105 {
106 case GNUNET_SET_STATUS_OK:
107 count++;
108 break;
109
110 case GNUNET_SET_STATUS_FAILURE:
111 oh2 = NULL;
112 ret = 1;
113 break;
114
115 case GNUNET_SET_STATUS_DONE:
116 oh2 = NULL;
117 GNUNET_assert (1 == count);
118 GNUNET_SET_destroy (set2);
119 set2 = NULL;
120 if (NULL == set1)
121 GNUNET_SCHEDULER_shutdown ();
122 break;
123
124 default:
125 GNUNET_assert (0);
126 }
127}
128
129
130static void
131listen_cb (void *cls,
132 const struct GNUNET_PeerIdentity *other_peer,
133 const struct GNUNET_MessageHeader *context_msg,
134 struct GNUNET_SET_Request *request)
135{
136 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
137 "starting intersection by accepting and committing\n");
138 GNUNET_assert (NULL != context_msg);
139 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
140 oh2 = GNUNET_SET_accept (request,
141 GNUNET_SET_RESULT_FULL,
142 (struct GNUNET_SET_Option[]) { 0 },
143 &result_cb_set2,
144 NULL);
145 GNUNET_SET_commit (oh2,
146 set2);
147}
148
149
150/**
151 * Start the set operation.
152 *
153 * @param cls closure, unused
154 */
155static void
156start (void *cls)
157{
158 struct GNUNET_MessageHeader context_msg;
159
160 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
161 "starting listener\n");
162 context_msg.size = htons (sizeof context_msg);
163 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
164 listen_handle = GNUNET_SET_listen (config,
165 GNUNET_SET_OPERATION_INTERSECTION,
166 &app_id,
167 &listen_cb,
168 NULL);
169 oh1 = GNUNET_SET_prepare (&local_id,
170 &app_id,
171 &context_msg,
172 GNUNET_SET_RESULT_FULL,
173 (struct GNUNET_SET_Option[]) { 0 },
174 &result_cb_set1,
175 NULL);
176 GNUNET_SET_commit (oh1,
177 set1);
178}
179
180
181/**
182 * Initialize the second set, continue
183 *
184 * @param cls closure, unused
185 */
186static void
187init_set2 (void *cls)
188{
189 struct GNUNET_SET_Element element;
190
191 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
192 "initializing set 2\n");
193 element.element_type = 0;
194 element.data = "hello";
195 element.size = strlen (element.data);
196 GNUNET_SET_add_element (set2,
197 &element,
198 NULL,
199 NULL);
200 element.data = "quux";
201 element.size = strlen (element.data);
202 GNUNET_SET_add_element (set2,
203 &element,
204 NULL,
205 NULL);
206 element.data = "baz";
207 element.size = strlen (element.data);
208 GNUNET_SET_add_element (set2,
209 &element,
210 &start,
211 NULL);
212}
213
214
215/**
216 * Initialize the first set, continue.
217 */
218static void
219init_set1 (void)
220{
221 struct GNUNET_SET_Element element;
222
223 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
224 "initializing set 1\n");
225 element.element_type = 0;
226 element.data = "hello";
227 element.size = strlen (element.data);
228 GNUNET_SET_add_element (set1,
229 &element,
230 NULL,
231 NULL);
232 element.data = "bar";
233 element.size = strlen (element.data);
234 GNUNET_SET_add_element (set1,
235 &element,
236 &init_set2,
237 NULL);
238}
239
240
241static int
242iter_cb (void *cls,
243 const struct GNUNET_SET_Element *element)
244{
245 if (NULL == element)
246 {
247 GNUNET_assert (iter_count == 3);
248 GNUNET_SET_destroy (cls);
249 return GNUNET_YES;
250 }
251 iter_count++;
252 return GNUNET_YES;
253}
254
255
256static void
257test_iter ()
258{
259 struct GNUNET_SET_Element element;
260 struct GNUNET_SET_Handle *iter_set;
261
262 iter_set = GNUNET_SET_create (config,
263 GNUNET_SET_OPERATION_INTERSECTION);
264 element.element_type = 0;
265 element.data = "hello";
266 element.size = strlen (element.data);
267 GNUNET_SET_add_element (iter_set,
268 &element,
269 NULL,
270 NULL);
271 element.data = "bar";
272 element.size = strlen (element.data);
273 GNUNET_SET_add_element (iter_set,
274 &element,
275 NULL,
276 NULL);
277 element.data = "quux";
278 element.size = strlen (element.data);
279 GNUNET_SET_add_element (iter_set,
280 &element,
281 NULL,
282 NULL);
283 GNUNET_SET_iterate (iter_set,
284 &iter_cb,
285 iter_set);
286}
287
288
289/**
290 * Function run on shutdown.
291 *
292 * @param cls closure
293 */
294static void
295do_shutdown (void *cls)
296{
297 if (NULL != tt)
298 {
299 GNUNET_SCHEDULER_cancel (tt);
300 tt = NULL;
301 }
302 if (NULL != oh1)
303 {
304 GNUNET_SET_operation_cancel (oh1);
305 oh1 = NULL;
306 }
307 if (NULL != oh2)
308 {
309 GNUNET_SET_operation_cancel (oh2);
310 oh2 = NULL;
311 }
312 if (NULL != set1)
313 {
314 GNUNET_SET_destroy (set1);
315 set1 = NULL;
316 }
317 if (NULL != set2)
318 {
319 GNUNET_SET_destroy (set2);
320 set2 = NULL;
321 }
322 if (NULL != listen_handle)
323 {
324 GNUNET_SET_listen_cancel (listen_handle);
325 listen_handle = NULL;
326 }
327}
328
329
330/**
331 * Function run on timeout.
332 *
333 * @param cls closure
334 */
335static void
336timeout_fail (void *cls)
337{
338 tt = NULL;
339 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
340 "Testcase failed with timeout\n");
341 GNUNET_SCHEDULER_shutdown ();
342 ret = 1;
343}
344
345
346/**
347 * Signature of the 'main' function for a (single-peer) testcase that
348 * is run using 'GNUNET_TESTING_peer_run'.
349 *
350 * @param cls closure
351 * @param cfg configuration of the peer that was started
352 * @param peer identity of the peer that was created
353 */
354static void
355run (void *cls,
356 const struct GNUNET_CONFIGURATION_Handle *cfg,
357 struct GNUNET_TESTING_Peer *peer)
358{
359 config = cfg;
360 GNUNET_TESTING_peer_get_identity (peer,
361 &local_id);
362 if (0)
363 test_iter ();
364
365 tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (
366 GNUNET_TIME_UNIT_SECONDS, 5),
367 &timeout_fail,
368 NULL);
369 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
370 NULL);
371
372 set1 = GNUNET_SET_create (cfg,
373 GNUNET_SET_OPERATION_INTERSECTION);
374 set2 = GNUNET_SET_create (cfg,
375 GNUNET_SET_OPERATION_INTERSECTION);
376 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
377 &app_id);
378
379 /* test the real set reconciliation */
380 init_set1 ();
381}
382
383
384int
385main (int argc,
386 char **argv)
387{
388 if (0 != GNUNET_TESTING_peer_run ("test_set_intersection_result_full",
389 "test_set.conf",
390 &run, NULL))
391 return 1;
392 return ret;
393}
diff --git a/src/set/test_set_union_copy.c b/src/set/test_set_union_copy.c
deleted file mode 100644
index 908527017..000000000
--- a/src/set/test_set_union_copy.c
+++ /dev/null
@@ -1,311 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2015, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_set_union_copy.c
23 * @brief testcase for lazy copying of union sets
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_common.h"
29#include "gnunet_testing_lib.h"
30#include "gnunet_set_service.h"
31
32
33/**
34 * Value to return from #main().
35 */
36static int ret;
37
38static struct GNUNET_PeerIdentity local_id;
39
40static struct GNUNET_SET_Handle *set1;
41
42static struct GNUNET_SET_Handle *set2;
43
44static const struct GNUNET_CONFIGURATION_Handle *config;
45
46static struct GNUNET_SCHEDULER_Task *tt;
47
48
49static void
50add_element_str (struct GNUNET_SET_Handle *set,
51 char *str)
52{
53 struct GNUNET_SET_Element element;
54
55 element.element_type = 0;
56 element.data = str;
57 element.size = strlen (str);
58 GNUNET_SET_add_element (set,
59 &element,
60 NULL,
61 NULL);
62}
63
64
65static void
66remove_element_str (struct GNUNET_SET_Handle *set,
67 char *str)
68{
69 struct GNUNET_SET_Element element;
70
71 element.element_type = 0;
72 element.data = str;
73 element.size = strlen (str);
74 GNUNET_SET_remove_element (set,
75 &element,
76 NULL,
77 NULL);
78}
79
80
81/**
82 * Signature of the main function of a task.
83 *
84 * @param cls closure
85 */
86static void
87timeout_fail (void *cls)
88{
89 tt = NULL;
90 GNUNET_SCHEDULER_shutdown ();
91 ret = 1;
92}
93
94
95struct CountIterClosure
96{
97 unsigned int expected_count;
98 unsigned int ongoing_count;
99 GNUNET_SCHEDULER_TaskCallback cont;
100 void *cont_cls;
101 char *what;
102};
103
104
105static int
106check_count_iter (void *cls,
107 const struct GNUNET_SET_Element *element)
108{
109 struct CountIterClosure *ci_cls = cls;
110
111 if (NULL == element)
112 {
113 if (ci_cls->expected_count != ci_cls->ongoing_count)
114 {
115 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
116 "Expected count (what: %s) to be %u, but it's actually %u\n",
117 ci_cls->what,
118 ci_cls->expected_count,
119 ci_cls->ongoing_count);
120 ret = 1;
121 GNUNET_SCHEDULER_shutdown ();
122 return GNUNET_NO;
123 }
124 ci_cls->cont (ci_cls->cont_cls);
125 GNUNET_free (ci_cls);
126 return GNUNET_NO;
127 }
128 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
129 "Set `%s' has element %.*s\n",
130 ci_cls->what,
131 (int) element->size,
132 (const char *) element->data);
133
134 ci_cls->ongoing_count++;
135 return GNUNET_YES;
136}
137
138
139static void
140check_count (struct GNUNET_SET_Handle *set,
141 char *what,
142 unsigned int expected_count,
143 GNUNET_SCHEDULER_TaskCallback cont,
144 void *cont_cls)
145{
146 struct CountIterClosure *ci_cls = GNUNET_new (struct CountIterClosure);
147
148 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
149 "Checking count of %s\n",
150 what);
151
152 ci_cls->expected_count = expected_count;
153 ci_cls->ongoing_count = 0;
154 ci_cls->cont = cont;
155 ci_cls->cont_cls = cont_cls;
156 ci_cls->what = what;
157
158 GNUNET_assert (GNUNET_YES ==
159 GNUNET_SET_iterate (set,
160 &check_count_iter,
161 ci_cls));
162}
163
164
165static void
166test_done (void *cls)
167{
168 GNUNET_SCHEDULER_shutdown ();
169}
170
171
172static void
173check_new_set_count (void *cls)
174{
175 check_count (set2,
176 "new set",
177 3,
178 &test_done,
179 NULL);
180}
181
182
183static void
184copy_done (void *cls,
185 struct GNUNET_SET_Handle *new_set)
186{
187 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
188 "copy done\n");
189 set2 = new_set;
190 remove_element_str (set2,
191 "k5555");
192 add_element_str (set2,
193 "n66666");
194 add_element_str (set2,
195 "new2butremoved");
196 remove_element_str (set2,
197 "new2butremoved");
198 remove_element_str (set2,
199 "new3justremoved");
200 // Check that set1 didn't change.
201 check_count (set1,
202 "old set",
203 3,
204 &check_new_set_count,
205 NULL);
206}
207
208
209static void
210test_copy (void *cls)
211{
212 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
213 "about to copy\n");
214 GNUNET_SET_copy_lazy (set1,
215 &copy_done,
216 NULL);
217}
218
219
220/**
221 * Function run on shutdown.
222 *
223 * @param cls closure
224 */
225static void
226do_shutdown (void *cls)
227{
228 if (NULL != tt)
229 {
230 GNUNET_SCHEDULER_cancel (tt);
231 tt = NULL;
232 }
233 if (NULL != set1)
234 {
235 GNUNET_SET_destroy (set1);
236 set1 = NULL;
237 }
238 if (NULL != set2)
239 {
240 GNUNET_SET_destroy (set2);
241 set2 = NULL;
242 }
243}
244
245
246/**
247 * Signature of the 'main' function for a (single-peer) testcase that
248 * is run using #GNUNET_TESTING_peer_run().
249 *
250 * @param cls closure
251 * @param cfg configuration of the peer that was started
252 * @param peer identity of the peer that was created
253 */
254static void
255run (void *cls,
256 const struct GNUNET_CONFIGURATION_Handle *cfg,
257 struct GNUNET_TESTING_Peer *peer)
258{
259 tt = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (
260 GNUNET_TIME_UNIT_SECONDS, 5),
261 &timeout_fail,
262 NULL);
263 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
264 NULL);
265 config = cfg;
266 GNUNET_TESTING_peer_get_identity (peer,
267 &local_id);
268
269 set1 = GNUNET_SET_create (cfg,
270 GNUNET_SET_OPERATION_UNION);
271 add_element_str (set1,
272 "333");
273 add_element_str (set1,
274 "k444");
275 /* duplicate -- ignored */
276 add_element_str (set1,
277 "k444");
278 remove_element_str (set1,
279 "333");
280 /* non-existent -- ignored */
281 remove_element_str (set1,
282 "999999999");
283 add_element_str (set1,
284 "k5555");
285 /* duplicate -- ignored */
286 remove_element_str (set1,
287 "333");
288 add_element_str (set1,
289 "k2");
290
291 check_count (set1,
292 "initial test",
293 3,
294 &test_copy,
295 NULL);
296}
297
298
299int
300main (int argc, char **argv)
301{
302 if (0 != GNUNET_TESTING_peer_run ("test_set_union_copy",
303 "test_set.conf",
304 &run, NULL))
305 {
306 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
307 "failed to start testing peer\n");
308 return 1;
309 }
310 return ret;
311}
diff --git a/src/set/test_set_union_result_symmetric.c b/src/set/test_set_union_result_symmetric.c
deleted file mode 100644
index b6c7a82f6..000000000
--- a/src/set/test_set_union_result_symmetric.c
+++ /dev/null
@@ -1,455 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012, 2016 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_set_union_result_smmetric
23 * @brief testcase for symmetric result mode of the union set operation
24 * @author Florian Dold
25 * @author Christian Grothoff
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_testing_lib.h"
30#include "gnunet_set_service.h"
31
32
33/**
34 * Value to return from #main().
35 */
36static int ret;
37
38static struct GNUNET_PeerIdentity local_id;
39
40static struct GNUNET_HashCode app_id;
41
42static struct GNUNET_SET_Handle *set1;
43
44static struct GNUNET_SET_Handle *set2;
45
46static struct GNUNET_SET_ListenHandle *listen_handle;
47
48static const struct GNUNET_CONFIGURATION_Handle *config;
49
50static struct GNUNET_SET_OperationHandle *oh1;
51
52static struct GNUNET_SET_OperationHandle *oh2;
53
54static int iter_count;
55
56/**
57 * Are we testing correctness for the empty set union?
58 */
59static int empty;
60
61/**
62 * Number of elements found in set 1
63 */
64static unsigned int count_set1;
65
66/**
67 * Number of elements found in set 2
68 */
69static unsigned int count_set2;
70
71/**
72 * Task that is run when the test times out.
73 */
74static struct GNUNET_SCHEDULER_Task *timeout_task;
75
76
77static void
78result_cb_set1 (void *cls,
79 const struct GNUNET_SET_Element *element,
80 uint64_t current_size,
81 enum GNUNET_SET_Status status)
82{
83 switch (status)
84 {
85 case GNUNET_SET_STATUS_ADD_LOCAL:
86 count_set1++;
87 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
88 "set 1: got element\n");
89 break;
90
91 case GNUNET_SET_STATUS_FAILURE:
92 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
93 "set 1: failure\n");
94 oh1 = NULL;
95 ret = 1;
96 if (NULL != timeout_task)
97 {
98 GNUNET_SCHEDULER_cancel (timeout_task);
99 timeout_task = NULL;
100 }
101 GNUNET_SCHEDULER_shutdown ();
102 break;
103
104 case GNUNET_SET_STATUS_DONE:
105 oh1 = NULL;
106 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
107 "set 1: done\n");
108 GNUNET_SET_destroy (set1);
109 set1 = NULL;
110 if (NULL == set2)
111 {
112 if (NULL != timeout_task)
113 {
114 GNUNET_SCHEDULER_cancel (timeout_task);
115 timeout_task = NULL;
116 }
117 GNUNET_SCHEDULER_shutdown ();
118 }
119 break;
120
121 case GNUNET_SET_STATUS_ADD_REMOTE:
122 break;
123
124 default:
125 GNUNET_assert (0);
126 }
127}
128
129
130static void
131result_cb_set2 (void *cls,
132 const struct GNUNET_SET_Element *element,
133 uint64_t current_size,
134 enum GNUNET_SET_Status status)
135{
136 switch (status)
137 {
138 case GNUNET_SET_STATUS_ADD_LOCAL:
139 count_set2++;
140 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
141 "set 2: got element\n");
142 break;
143
144 case GNUNET_SET_STATUS_FAILURE:
145 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
146 "set 2: failure\n");
147 oh2 = NULL;
148 ret = 1;
149 if (NULL != timeout_task)
150 {
151 GNUNET_SCHEDULER_cancel (timeout_task);
152 timeout_task = NULL;
153 }
154 GNUNET_SCHEDULER_shutdown ();
155 break;
156
157 case GNUNET_SET_STATUS_DONE:
158 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
159 "set 2: done\n");
160 oh2 = NULL;
161 GNUNET_SET_destroy (set2);
162 set2 = NULL;
163 if (NULL == set1)
164 {
165 if (NULL != timeout_task)
166 {
167 GNUNET_SCHEDULER_cancel (timeout_task);
168 timeout_task = NULL;
169 }
170 GNUNET_SCHEDULER_shutdown ();
171 }
172 break;
173
174 case GNUNET_SET_STATUS_ADD_REMOTE:
175 break;
176
177 default:
178 GNUNET_assert (0);
179 }
180}
181
182
183static void
184listen_cb (void *cls,
185 const struct GNUNET_PeerIdentity *other_peer,
186 const struct GNUNET_MessageHeader *context_msg,
187 struct GNUNET_SET_Request *request)
188{
189 GNUNET_assert (NULL != context_msg);
190 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
191 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
192 "listen cb called\n");
193 oh2 = GNUNET_SET_accept (request,
194 GNUNET_SET_RESULT_SYMMETRIC,
195 (struct GNUNET_SET_Option[]) { 0 },
196 &result_cb_set2,
197 NULL);
198 GNUNET_SET_commit (oh2,
199 set2);
200}
201
202
203/**
204 * Start the set operation.
205 *
206 * @param cls closure, unused
207 */
208static void
209start (void *cls)
210{
211 struct GNUNET_MessageHeader context_msg;
212
213 context_msg.size = htons (sizeof context_msg);
214 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
215
216 listen_handle = GNUNET_SET_listen (config,
217 GNUNET_SET_OPERATION_UNION,
218 &app_id,
219 &listen_cb, NULL);
220 oh1 = GNUNET_SET_prepare (&local_id,
221 &app_id,
222 &context_msg,
223 GNUNET_SET_RESULT_SYMMETRIC,
224 (struct GNUNET_SET_Option[]) { 0 },
225 &result_cb_set1, NULL);
226 GNUNET_SET_commit (oh1, set1);
227}
228
229
230/**
231 * Initialize the second set, continue
232 *
233 * @param cls closure, unused
234 */
235static void
236init_set2 (void *cls)
237{
238 struct GNUNET_SET_Element element;
239
240 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
241 "initializing set 2\n");
242 if (empty)
243 {
244 start (NULL);
245 return;
246 }
247 element.element_type = 0;
248 element.data = "hello";
249 element.size = strlen (element.data);
250 GNUNET_SET_add_element (set2,
251 &element,
252 NULL,
253 NULL);
254 element.data = "quux";
255 element.size = strlen (element.data);
256 GNUNET_SET_add_element (set2,
257 &element,
258 NULL,
259 NULL);
260 element.data = "baz";
261 element.size = strlen (element.data);
262 GNUNET_SET_add_element (set2,
263 &element,
264 &start, NULL);
265}
266
267
268/**
269 * Initialize the first set, continue.
270 */
271static void
272init_set1 (void)
273{
274 struct GNUNET_SET_Element element;
275
276 if (empty)
277 {
278 init_set2 (NULL);
279 return;
280 }
281 element.element_type = 0;
282 element.data = "hello";
283 element.size = strlen (element.data);
284 GNUNET_SET_add_element (set1,
285 &element,
286 NULL,
287 NULL);
288 element.data = "bar";
289 element.size = strlen (element.data);
290 GNUNET_SET_add_element (set1,
291 &element,
292 &init_set2,
293 NULL);
294 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
295 "initialized set 1\n");
296}
297
298
299static int
300iter_cb (void *cls,
301 const struct GNUNET_SET_Element *element)
302{
303 if (NULL == element)
304 {
305 GNUNET_assert (iter_count == 3);
306 GNUNET_SET_destroy (cls);
307 return GNUNET_YES;
308 }
309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
310 "iter: got element\n");
311 iter_count++;
312 return GNUNET_YES;
313}
314
315
316static void
317test_iter ()
318{
319 struct GNUNET_SET_Element element;
320 struct GNUNET_SET_Handle *iter_set;
321
322 iter_count = 0;
323 iter_set = GNUNET_SET_create (config, GNUNET_SET_OPERATION_UNION);
324 element.element_type = 0;
325 element.data = "hello";
326 element.size = strlen (element.data);
327 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
328 element.data = "bar";
329 element.size = strlen (element.data);
330 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
331 element.data = "quux";
332 element.size = strlen (element.data);
333 GNUNET_SET_add_element (iter_set, &element, NULL, NULL);
334
335 GNUNET_SET_iterate (iter_set,
336 &iter_cb,
337 iter_set);
338}
339
340
341/**
342 * Signature of the main function of a task.
343 *
344 * @param cls closure
345 */
346static void
347timeout_fail (void *cls)
348{
349 timeout_task = NULL;
350 GNUNET_SCHEDULER_shutdown ();
351 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
352 "test timed out\n");
353 ret = 1;
354}
355
356
357/**
358 * Function run on shutdown.
359 *
360 * @param cls closure
361 */
362static void
363do_shutdown (void *cls)
364{
365 if (NULL != timeout_task)
366 {
367 GNUNET_SCHEDULER_cancel (timeout_task);
368 timeout_task = NULL;
369 }
370 if (NULL != oh1)
371 {
372 GNUNET_SET_operation_cancel (oh1);
373 oh1 = NULL;
374 }
375 if (NULL != oh2)
376 {
377 GNUNET_SET_operation_cancel (oh2);
378 oh2 = NULL;
379 }
380 if (NULL != set1)
381 {
382 GNUNET_SET_destroy (set1);
383 set1 = NULL;
384 }
385 if (NULL != set2)
386 {
387 GNUNET_SET_destroy (set2);
388 set2 = NULL;
389 }
390 if (NULL != listen_handle)
391 {
392 GNUNET_SET_listen_cancel (listen_handle);
393 listen_handle = NULL;
394 }
395}
396
397
398/**
399 * Signature of the 'main' function for a (single-peer) testcase that
400 * is run using 'GNUNET_TESTING_peer_run'.
401 *
402 * @param cls closure
403 * @param cfg configuration of the peer that was started
404 * @param peer identity of the peer that was created
405 */
406static void
407run (void *cls,
408 const struct GNUNET_CONFIGURATION_Handle *cfg,
409 struct GNUNET_TESTING_Peer *peer)
410{
411 timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply (
412 GNUNET_TIME_UNIT_SECONDS, 5),
413 &timeout_fail,
414 NULL);
415 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
416 NULL);
417 config = cfg;
418 GNUNET_TESTING_peer_get_identity (peer,
419 &local_id);
420
421 if (0)
422 test_iter ();
423
424 set1 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
425 set2 = GNUNET_SET_create (cfg, GNUNET_SET_OPERATION_UNION);
426 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
427
428 /* test the real set reconciliation */
429 init_set1 ();
430}
431
432
433int
434main (int argc, char **argv)
435{
436 empty = 1;
437 if (0 != GNUNET_TESTING_peer_run ("test_set_api",
438 "test_set.conf",
439 &run, NULL))
440 {
441 return 1;
442 }
443 GNUNET_assert (0 == count_set1);
444 GNUNET_assert (0 == count_set2);
445 empty = 0;
446 if (0 != GNUNET_TESTING_peer_run ("test_set_api",
447 "test_set.conf",
448 &run, NULL))
449 {
450 return 1;
451 }
452 GNUNET_break (2 == count_set1);
453 GNUNET_break (1 == count_set2);
454 return ret;
455}