aboutsummaryrefslogtreecommitdiff
path: root/src/setu
diff options
context:
space:
mode:
Diffstat (limited to 'src/setu')
-rw-r--r--src/setu/.gitignore7
-rw-r--r--src/setu/Makefile.am108
-rw-r--r--src/setu/gnunet-service-setu.c5437
-rw-r--r--src/setu/gnunet-service-setu_protocol.h256
-rw-r--r--src/setu/gnunet-service-setu_strata_estimator.c467
-rw-r--r--src/setu/gnunet-service-setu_strata_estimator.h199
-rw-r--r--src/setu/gnunet-setu-ibf-profiler.c308
-rw-r--r--src/setu/gnunet-setu-profiler.c499
-rw-r--r--src/setu/ibf.c614
-rw-r--r--src/setu/ibf.h310
-rw-r--r--src/setu/ibf_sim.c142
-rw-r--r--src/setu/perf_setu_api.c473
-rw-r--r--src/setu/plugin_block_setu_test.c123
-rw-r--r--src/setu/setu.conf.in12
-rw-r--r--src/setu/setu.h365
-rw-r--r--src/setu/setu_api.c937
-rw-r--r--src/setu/test_setu.conf32
-rw-r--r--src/setu/test_setu_api.c360
18 files changed, 0 insertions, 10649 deletions
diff --git a/src/setu/.gitignore b/src/setu/.gitignore
deleted file mode 100644
index 0ac10295e..000000000
--- a/src/setu/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
1gnunet-setu-profiler
2gnunet-service-setu
3gnunet-setu-ibf-profiler
4test_setu_api
5test_setu_copy
6test_setu_result_symmetric
7perf_setu_api
diff --git a/src/setu/Makefile.am b/src/setu/Makefile.am
deleted file mode 100644
index 897bf3f86..000000000
--- a/src/setu/Makefile.am
+++ /dev/null
@@ -1,108 +0,0 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4pkgcfgdir= $(pkgdatadir)/config.d/
5
6libexecdir= $(pkglibdir)/libexec/
7
8plugindir = $(libdir)/gnunet
9
10pkgcfg_DATA = \
11 setu.conf
12
13if USE_COVERAGE
14 AM_CFLAGS = -fprofile-arcs -ftest-coverage
15endif
16
17bin_PROGRAMS = \
18 gnunet-setu-profiler
19
20noinst_PROGRAMS = \
21 gnunet-setu-ibf-profiler
22
23libexec_PROGRAMS = \
24 gnunet-service-setu
25
26lib_LTLIBRARIES = \
27 libgnunetsetu.la
28
29gnunet_setu_profiler_SOURCES = \
30 gnunet-setu-profiler.c
31gnunet_setu_profiler_LDADD = \
32 $(top_builddir)/src/util/libgnunetutil.la \
33 $(top_builddir)/src/statistics/libgnunetstatistics.la \
34 libgnunetsetu.la \
35 $(top_builddir)/src/testing/libgnunettesting.la \
36 $(GN_LIBINTL)
37
38
39gnunet_setu_ibf_profiler_SOURCES = \
40 gnunet-setu-ibf-profiler.c \
41 ibf.c
42gnunet_setu_ibf_profiler_LDADD = \
43 $(top_builddir)/src/util/libgnunetutil.la \
44 $(GN_LIBINTL)
45
46gnunet_service_setu_SOURCES = \
47 gnunet-service-setu.c gnunet-service-setu_protocol.h \
48 ibf.c ibf.h \
49 gnunet-service-setu_strata_estimator.c gnunet-service-setu_strata_estimator.h \
50 gnunet-service-setu_protocol.h
51gnunet_service_setu_LDADD = \
52 $(top_builddir)/src/util/libgnunetutil.la \
53 $(top_builddir)/src/statistics/libgnunetstatistics.la \
54 $(top_builddir)/src/core/libgnunetcore.la \
55 $(top_builddir)/src/cadet/libgnunetcadet.la \
56 $(top_builddir)/src/block/libgnunetblock.la \
57 libgnunetsetu.la \
58 $(GN_LIBINTL)
59
60libgnunetsetu_la_SOURCES = \
61 setu_api.c setu.h
62libgnunetsetu_la_LIBADD = \
63 $(top_builddir)/src/util/libgnunetutil.la \
64 $(LTLIBINTL)
65libgnunetsetu_la_LDFLAGS = \
66 $(GN_LIB_LDFLAGS)
67
68check_PROGRAMS = \
69 test_setu_api \
70 perf_setu_api
71
72if ENABLE_TEST_RUN
73AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
74TESTS = $(check_PROGRAMS)
75endif
76
77test_setu_api_SOURCES = \
78 test_setu_api.c
79test_setu_api_LDADD = \
80 $(top_builddir)/src/util/libgnunetutil.la \
81 $(top_builddir)/src/testing/libgnunettesting.la \
82 libgnunetsetu.la
83
84
85perf_setu_api_SOURCES = \
86 perf_setu_api.c
87perf_setu_api_LDADD = \
88 $(top_builddir)/src/util/libgnunetutil.la \
89 $(top_builddir)/src/testing/libgnunettesting.la \
90 libgnunetsetu.la
91
92
93plugin_LTLIBRARIES = \
94 libgnunet_plugin_block_setu_test.la
95
96libgnunet_plugin_block_setu_test_la_SOURCES = \
97 plugin_block_setu_test.c
98libgnunet_plugin_block_setu_test_la_LIBADD = \
99 $(top_builddir)/src/block/libgnunetblock.la \
100 $(top_builddir)/src/block/libgnunetblockgroup.la \
101 $(top_builddir)/src/util/libgnunetutil.la \
102 $(LTLIBINTL)
103libgnunet_plugin_block_setu_test_la_LDFLAGS = \
104 $(GN_PLUGIN_LDFLAGS)
105
106
107EXTRA_DIST = \
108 test_setu.conf
diff --git a/src/setu/gnunet-service-setu.c b/src/setu/gnunet-service-setu.c
deleted file mode 100644
index 339d347f8..000000000
--- a/src/setu/gnunet-service-setu.c
+++ /dev/null
@@ -1,5437 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file setu/gnunet-service-setu.c
22 * @brief set union operation
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_statistics_service.h"
30#include "ibf.h"
31#include "gnunet_protocols.h"
32#include "gnunet_applications.h"
33#include "gnunet_cadet_service.h"
34#include "gnunet-service-setu_strata_estimator.h"
35#include "gnunet-service-setu_protocol.h"
36#include "gnunet_statistics_service.h"
37#include <gcrypt.h>
38#include "gnunet_setu_service.h"
39#include "setu.h"
40
41#define LOG(kind, ...) GNUNET_log_from (kind, "setu", __VA_ARGS__)
42
43/**
44 * How long do we hold on to an incoming channel if there is
45 * no local listener before giving up?
46 */
47#define INCOMING_CHANNEL_TIMEOUT GNUNET_TIME_UNIT_MINUTES
48
49/**
50 * Number of IBFs in a strata estimator.
51 */
52#define SE_STRATA_COUNT 32
53
54
55/**
56 * Primes for all 4 different strata estimators 61,67,71,73,79,83,89,97 348
57 * Based on the bsc thesis of Elias Summermatter (2021)
58 */
59#define SE_IBFS_TOTAL_SIZE 632
60
61/**
62 * The hash num parameter for the difference digests and strata estimators.
63 */
64#define SE_IBF_HASH_NUM 3
65
66/**
67 * Number of buckets that can be transmitted in one message.
68 */
69#define MAX_BUCKETS_PER_MESSAGE ((1 << 16) / IBF_BUCKET_SIZE)
70
71/**
72 * The maximum size of an ibf we use is MAX_IBF_SIZE=2^20.
73 * Choose this value so that computing the IBF is still cheaper
74 * than transmitting all values.
75 */
76#define MAX_IBF_SIZE 1048576
77
78
79/**
80 * Minimal size of an ibf
81 * Based on the bsc thesis of Elias Summermatter (2021)
82 */
83#define IBF_MIN_SIZE 37
84
85/**
86 * AVG RTT for differential sync when k=2 and Factor = 2
87 * Based on the bsc thesis of Elias Summermatter (2021)
88 */
89#define DIFFERENTIAL_RTT_MEAN 3.65145
90
91/**
92 * Security level used for byzantine checks (2^80)
93 */
94
95#define SECURITY_LEVEL 80
96
97/**
98 * Is the estimated probability for a new round this values
99 * is based on the bsc thesis of Elias Summermatter (2021)
100 */
101
102#define PROBABILITY_FOR_NEW_ROUND 0.15
103
104/**
105 * Measure the performance in a csv
106 */
107
108#define MEASURE_PERFORMANCE 0
109
110
111/**
112 * Current phase we are in for a union operation.
113 */
114enum UnionOperationPhase
115{
116 /**
117 * We sent the request message, and expect a strata estimator.
118 */
119 PHASE_EXPECT_SE,
120
121 /**
122 * We sent the strata estimator, and expect an IBF. This phase is entered once
123 * upon initialization and later via #PHASE_EXPECT_ELEMENTS_AND_REQUESTS.
124 *
125 * XXX: could use better wording.
126 * XXX: repurposed to also expect a "request full set" message, should be renamed
127 *
128 * After receiving the complete IBF, we enter #PHASE_EXPECT_ELEMENTS
129 */
130 PHASE_EXPECT_IBF,
131
132 /**
133 * Continuation for multi part IBFs.
134 */
135 PHASE_EXPECT_IBF_LAST,
136
137 /**
138 * We are decoding an IBF.
139 */
140 PHASE_ACTIVE_DECODING,
141
142 /**
143 * The other peer is decoding the IBF we just sent.
144 */
145 PHASE_PASSIVE_DECODING,
146
147 /**
148 * The protocol is almost finished, but we still have to flush our message
149 * queue and/or expect some elements.
150 */
151 PHASE_FINISH_CLOSING,
152
153 /**
154 * In the penultimate phase, we wait until all our demands are satisfied.
155 * Then we send a done message, and wait for another done message.
156 */
157 PHASE_FINISH_WAITING,
158
159 /**
160 * In the ultimate phase, we wait until our demands are satisfied and then
161 * quit (sending another DONE message).
162 */
163 PHASE_FINISHED,
164
165 /**
166 * After sending the full set, wait for responses with the elements
167 * that the local peer is missing.
168 */
169 PHASE_FULL_SENDING,
170
171 /**
172 * Phase that receives full set first and then sends elements that are
173 * the local peer missing
174 */
175 PHASE_FULL_RECEIVING
176};
177
178/**
179 * Different modes of operations
180 */
181
182enum MODE_OF_OPERATION
183{
184 /**
185 * Mode just synchronizes the difference between sets
186 */
187 DIFFERENTIAL_SYNC,
188
189 /**
190 * Mode send full set sending local set first
191 */
192 FULL_SYNC_LOCAL_SENDING_FIRST,
193
194 /**
195 * Mode request full set from remote peer
196 */
197 FULL_SYNC_REMOTE_SENDING_FIRST
198};
199
200
201/**
202 * Information about an element element in the set. All elements are
203 * stored in a hash-table from their hash-code to their `struct
204 * Element`, so that the remove and add operations are reasonably
205 * fast.
206 */
207struct ElementEntry
208{
209 /**
210 * The actual element. The data for the element
211 * should be allocated at the end of this struct.
212 */
213 struct GNUNET_SETU_Element element;
214
215 /**
216 * Hash of the element. For set union: Will be used to derive the
217 * different IBF keys for different salts.
218 */
219 struct GNUNET_HashCode element_hash;
220
221 /**
222 * First generation that includes this element.
223 */
224 unsigned int generation;
225
226 /**
227 * #GNUNET_YES if the element is a remote element, and does not belong
228 * to the operation's set.
229 */
230 int remote;
231};
232
233
234/**
235 * A listener is inhabited by a client, and waits for evaluation
236 * requests from remote peers.
237 */
238struct Listener;
239
240
241/**
242 * A set that supports a specific operation with other peers.
243 */
244struct Set;
245
246
247/**
248 * State we keep per client.
249 */
250struct ClientState
251{
252 /**
253 * Set, if associated with the client, otherwise NULL.
254 */
255 struct Set *set;
256
257 /**
258 * Listener, if associated with the client, otherwise NULL.
259 */
260 struct Listener *listener;
261
262 /**
263 * Client handle.
264 */
265 struct GNUNET_SERVICE_Client *client;
266
267 /**
268 * Message queue.
269 */
270 struct GNUNET_MQ_Handle *mq;
271};
272
273
274/**
275 * Operation context used to execute a set operation.
276 */
277struct Operation
278{
279
280 /**
281 * The identity of the requesting peer. Needs to
282 * be stored here as the op spec might not have been created yet.
283 */
284 struct GNUNET_PeerIdentity peer;
285
286 /**
287 * Initial size of our set, just before the operation started.
288 */
289 uint64_t initial_size;
290
291 /**
292 * Kept in a DLL of the listener, if @e listener is non-NULL.
293 */
294 struct Operation *next;
295
296 /**
297 * Kept in a DLL of the listener, if @e listener is non-NULL.
298 */
299 struct Operation *prev;
300
301 /**
302 * Channel to the peer.
303 */
304 struct GNUNET_CADET_Channel *channel;
305
306 /**
307 * Port this operation runs on.
308 */
309 struct Listener *listener;
310
311 /**
312 * Message queue for the channel.
313 */
314 struct GNUNET_MQ_Handle *mq;
315
316 /**
317 * Context message, may be NULL.
318 */
319 struct GNUNET_MessageHeader *context_msg;
320
321 /**
322 * Set associated with the operation, NULL until the spec has been
323 * associated with a set.
324 */
325 struct Set *set;
326
327 /**
328 * Copy of the set's strata estimator at the time of
329 * creation of this operation.
330 */
331 struct MultiStrataEstimator *se;
332
333 /**
334 * The IBF we currently receive.
335 */
336 struct InvertibleBloomFilter *remote_ibf;
337
338 /**
339 * The IBF with the local set's element.
340 */
341 struct InvertibleBloomFilter *local_ibf;
342
343 /**
344 * Maps unsalted IBF-Keys to elements.
345 * Used as a multihashmap, the keys being the lower 32bit of the IBF-Key.
346 * Colliding IBF-Keys are linked.
347 */
348 struct GNUNET_CONTAINER_MultiHashMap32 *key_to_element;
349
350 /**
351 * Timeout task, if the incoming peer has not been accepted
352 * after the timeout, it will be disconnected.
353 */
354 struct GNUNET_SCHEDULER_Task *timeout_task;
355
356 /**
357 * Hashes for elements that we have demanded from the other peer.
358 */
359 struct GNUNET_CONTAINER_MultiHashMap *demanded_hashes;
360
361 /**
362 * Current state of the operation.
363 */
364 enum UnionOperationPhase phase;
365
366 /**
367 * Did we send the client that we are done?
368 */
369 int client_done_sent;
370
371 /**
372 * Number of ibf buckets already received into the @a remote_ibf.
373 */
374 uint64_t ibf_buckets_received;
375
376 /**
377 * Salt that we're using for sending IBFs
378 */
379 uint32_t salt_send;
380
381 /**
382 * Salt for the IBF we've received and that we're currently decoding.
383 */
384 uint32_t salt_receive;
385
386 /**
387 * Number of elements we received from the other peer
388 * that were not in the local set yet.
389 */
390 uint32_t received_fresh;
391
392 /**
393 * Total number of elements received from the other peer.
394 */
395 uint32_t received_total;
396
397 /**
398 * Salt to use for the operation.
399 */
400 uint32_t salt;
401
402 /**
403 * Remote peers element count
404 */
405 uint32_t remote_element_count;
406
407 /**
408 * ID used to identify an operation between service and client
409 */
410 uint32_t client_request_id;
411
412 /**
413 * Always use delta operation instead of sending full sets,
414 * even it it's less efficient.
415 */
416 int force_delta;
417
418 /**
419 * Always send full sets, even if delta operations would
420 * be more efficient.
421 */
422 int force_full;
423
424 /**
425 * #GNUNET_YES to fail operations where Byzantine faults
426 * are suspected
427 */
428 int byzantine;
429
430 /**
431 * #GNUNET_YES to also send back set elements we are sending to
432 * the remote peer.
433 */
434 int symmetric;
435
436 /**
437 * Lower bound for the set size, used only when
438 * byzantine mode is enabled.
439 */
440 uint64_t byzantine_lower_bound;
441
442 /**
443 * Unique request id for the request from a remote peer, sent to the
444 * client, which will accept or reject the request. Set to '0' iff
445 * the request has not been suggested yet.
446 */
447 uint32_t suggest_id;
448
449 /**
450 * Generation in which the operation handle
451 * was created.
452 */
453 unsigned int generation_created;
454
455
456 /**
457 * User defined Bandwidth Round Trips Tradeoff
458 */
459 uint64_t rtt_bandwidth_tradeoff;
460
461
462 /**
463 * Number of Element per bucket in IBF
464 */
465 uint8_t ibf_number_buckets_per_element;
466
467
468 /**
469 * Set difference is multiplied with this factor
470 * to gennerate large enough IBF
471 */
472 uint8_t ibf_bucket_number_factor;
473
474 /**
475 * Defines which site a client is
476 * 0 = Initiating peer
477 * 1 = Receiving peer
478 */
479 uint8_t peer_site;
480
481
482 /**
483 * Local peer element count
484 */
485 uint64_t local_element_count;
486
487 /**
488 * Mode of operation that was chosen by the algorithm
489 */
490 uint8_t mode_of_operation;
491
492 /**
493 * Hashmap to keep track of the send/received messages
494 */
495 struct GNUNET_CONTAINER_MultiHashMap *message_control_flow;
496
497 /**
498 * Hashmap to keep track of the send/received inquiries (ibf keys)
499 */
500 struct GNUNET_CONTAINER_MultiHashMap *inquiries_sent;
501
502
503 /**
504 * Total size of local set
505 */
506 uint64_t total_elements_size_local;
507
508 /**
509 * Limit of number of elements in set
510 */
511 uint64_t byzantine_upper_bound;
512
513 /**
514 * is the count of already passed differential sync iterations
515 */
516 uint8_t differential_sync_iterations;
517
518 /**
519 * Estimated or committed set difference at the start
520 */
521 uint64_t remote_set_diff;
522
523 /**
524 * Estimated or committed set difference at the start
525 */
526 uint64_t local_set_diff;
527
528 /**
529 * Boolean to enforce an active passive switch
530 */
531 bool active_passive_switch_required;
532};
533
534
535/**
536 * SetContent stores the actual set elements, which may be shared by
537 * multiple generations derived from one set.
538 */
539struct SetContent
540{
541 /**
542 * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *`.
543 */
544 struct GNUNET_CONTAINER_MultiHashMap *elements;
545
546 /**
547 * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *` randomized.
548 */
549 struct GNUNET_CONTAINER_MultiHashMap *elements_randomized;
550
551 /**
552 * Salt to construct the randomized element map
553 */
554 uint64_t elements_randomized_salt;
555
556 /**
557 * Number of references to the content.
558 */
559 unsigned int refcount;
560
561 /**
562 * FIXME: document!
563 */
564 unsigned int latest_generation;
565
566 /**
567 * Number of concurrently active iterators.
568 */
569 int iterator_count;
570};
571
572
573/**
574 * A set that supports a specific operation with other peers.
575 */
576struct Set
577{
578 /**
579 * Sets are held in a doubly linked list (in `sets_head` and `sets_tail`).
580 */
581 struct Set *next;
582
583 /**
584 * Sets are held in a doubly linked list.
585 */
586 struct Set *prev;
587
588 /**
589 * Client that owns the set. Only one client may own a set,
590 * and there can only be one set per client.
591 */
592 struct ClientState *cs;
593
594 /**
595 * Content, possibly shared by multiple sets,
596 * and thus reference counted.
597 */
598 struct SetContent *content;
599
600 /**
601 * The strata estimator is only generated once for each set. The IBF keys
602 * are derived from the element hashes with salt=0.
603 */
604 struct MultiStrataEstimator *se;
605
606 /**
607 * Evaluate operations are held in a linked list.
608 */
609 struct Operation *ops_head;
610
611 /**
612 * Evaluate operations are held in a linked list.
613 */
614 struct Operation *ops_tail;
615
616 /**
617 * Current generation, that is, number of previously executed
618 * operations and lazy copies on the underlying set content.
619 */
620 unsigned int current_generation;
621
622};
623
624
625/**
626 * The key entry is used to associate an ibf key with an element.
627 */
628struct KeyEntry
629{
630 /**
631 * IBF key for the entry, derived from the current salt.
632 */
633 struct IBF_Key ibf_key;
634
635 /**
636 * The actual element associated with the key.
637 *
638 * Only owned by the union operation if element->operation
639 * is #GNUNET_YES.
640 */
641 struct ElementEntry *element;
642
643 /**
644 * Did we receive this element? Even if element->is_foreign is false, we
645 * might have received the element, so this indicates that the other peer
646 * has it.
647 */
648 int received;
649};
650
651
652/**
653 * Used as a closure for sending elements
654 * with a specific IBF key.
655 */
656struct SendElementClosure
657{
658 /**
659 * The IBF key whose matching elements should be
660 * sent.
661 */
662 struct IBF_Key ibf_key;
663
664 /**
665 * Operation for which the elements
666 * should be sent.
667 */
668 struct Operation *op;
669};
670
671
672/**
673 * A listener is inhabited by a client, and waits for evaluation
674 * requests from remote peers.
675 */
676struct Listener
677{
678 /**
679 * Listeners are held in a doubly linked list.
680 */
681 struct Listener *next;
682
683 /**
684 * Listeners are held in a doubly linked list.
685 */
686 struct Listener *prev;
687
688 /**
689 * Head of DLL of operations this listener is responsible for.
690 * Once the client has accepted/declined the operation, the
691 * operation is moved to the respective set's operation DLLS.
692 */
693 struct Operation *op_head;
694
695 /**
696 * Tail of DLL of operations this listener is responsible for.
697 * Once the client has accepted/declined the operation, the
698 * operation is moved to the respective set's operation DLLS.
699 */
700 struct Operation *op_tail;
701
702 /**
703 * Client that owns the listener.
704 * Only one client may own a listener.
705 */
706 struct ClientState *cs;
707
708 /**
709 * The port we are listening on with CADET.
710 */
711 struct GNUNET_CADET_Port *open_port;
712
713 /**
714 * Application ID for the operation, used to distinguish
715 * multiple operations of the same type with the same peer.
716 */
717 struct GNUNET_HashCode app_id;
718
719};
720
721
722/**
723 * Handle to the cadet service, used to listen for and connect to
724 * remote peers.
725 */
726static struct GNUNET_CADET_Handle *cadet;
727
728/**
729 * Statistics handle.
730 */
731static struct GNUNET_STATISTICS_Handle *_GSS_statistics;
732
733/**
734 * Listeners are held in a doubly linked list.
735 */
736static struct Listener *listener_head;
737
738/**
739 * Listeners are held in a doubly linked list.
740 */
741static struct Listener *listener_tail;
742
743/**
744 * Number of active clients.
745 */
746static unsigned int num_clients;
747
748/**
749 * Are we in shutdown? if #GNUNET_YES and the number of clients
750 * drops to zero, disconnect from CADET.
751 */
752static int in_shutdown;
753
754/**
755 * Counter for allocating unique IDs for clients, used to identify incoming
756 * operation requests from remote peers, that the client can choose to accept
757 * or refuse. 0 must not be used (reserved for uninitialized).
758 */
759static uint32_t suggest_id;
760
761#if MEASURE_PERFORMANCE
762/**
763 * Handles configuration file for setu performance test
764 *
765 */
766static const struct GNUNET_CONFIGURATION_Handle *setu_cfg;
767
768
769/**
770 * Stores the performance data for induvidual message
771 */
772
773
774struct perf_num_send_received_msg
775{
776 uint64_t sent;
777 uint64_t sent_var_bytes;
778 uint64_t received;
779 uint64_t received_var_bytes;
780};
781
782/**
783 * Main struct to measure performance (data/rtts)
784 */
785struct per_store_struct
786{
787 struct perf_num_send_received_msg operation_request;
788 struct perf_num_send_received_msg se;
789 struct perf_num_send_received_msg request_full;
790 struct perf_num_send_received_msg element_full;
791 struct perf_num_send_received_msg full_done;
792 struct perf_num_send_received_msg ibf;
793 struct perf_num_send_received_msg inquery;
794 struct perf_num_send_received_msg element;
795 struct perf_num_send_received_msg demand;
796 struct perf_num_send_received_msg offer;
797 struct perf_num_send_received_msg done;
798 struct perf_num_send_received_msg over;
799 uint64_t se_diff;
800 uint64_t se_diff_remote;
801 uint64_t se_diff_local;
802 uint64_t active_passive_switches;
803 uint8_t mode_of_operation;
804};
805
806struct per_store_struct perf_store;
807#endif
808
809/**
810 * Different states to control the messages flow in differential mode
811 */
812
813enum MESSAGE_CONTROL_FLOW_STATE
814{
815 /**
816 * Initial message state
817 */
818 MSG_CFS_UNINITIALIZED,
819
820 /**
821 * Track that a message has been sent
822 */
823 MSG_CFS_SENT,
824
825 /**
826 * Track that receiving this message is expected
827 */
828 MSG_CFS_EXPECTED,
829
830 /**
831 * Track that message has been received
832 */
833 MSG_CFS_RECEIVED,
834};
835
836/**
837 * Message types to track in message control flow
838 */
839
840enum MESSAGE_TYPE
841{
842 /**
843 * Offer message type
844 */
845 OFFER_MESSAGE,
846
847 /**
848 * Demand message type
849 */
850 DEMAND_MESSAGE,
851
852 /**
853 * Element message type
854 */
855 ELEMENT_MESSAGE,
856};
857
858
859/**
860 * Struct to tracked messages in message control flow
861 */
862struct messageControlFlowElement
863{
864 /**
865 * Track the message control state of the offer message
866 */
867 enum MESSAGE_CONTROL_FLOW_STATE offer;
868 /**
869 * Track the message control state of the demand message
870 */
871 enum MESSAGE_CONTROL_FLOW_STATE demand;
872 /**
873 * Track the message control state of the element message
874 */
875 enum MESSAGE_CONTROL_FLOW_STATE element;
876};
877
878
879#if MEASURE_PERFORMANCE
880
881/**
882 * Loads different configuration to perform performance tests
883 *
884 * @param op operation handle
885 */
886static void
887load_config (struct Operation *op)
888{
889 long long number;
890 float fl;
891
892 setu_cfg = GNUNET_CONFIGURATION_create ();
893 GNUNET_CONFIGURATION_load (setu_cfg,
894 "perf_setu.conf");
895 GNUNET_CONFIGURATION_get_value_float (setu_cfg,
896 "IBF",
897 "BUCKET_NUMBER_FACTOR",
898 &fl);
899 op->ibf_bucket_number_factor = fl;
900 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
901 "IBF",
902 "NUMBER_PER_BUCKET",
903 &number);
904 op->ibf_number_buckets_per_element = number;
905 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
906 "PERFORMANCE",
907 "TRADEOFF",
908 &number);
909 op->rtt_bandwidth_tradeoff = number;
910 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
911 "BOUNDARIES",
912 "UPPER_ELEMENT",
913 &number);
914 op->byzantine_upper_bound = number;
915 op->peer_site = 0;
916}
917
918
919/**
920 * Function to calculate total bytes used for performance measurement
921 * @param size
922 * @param perf_num_send_received_msg
923 * @return bytes used
924 */
925static int
926sum_sent_received_bytes (uint64_t size,
927 struct perf_num_send_received_msg
928 perf_num_send_received_msg)
929{
930 return (size * perf_num_send_received_msg.sent)
931 + (size * perf_num_send_received_msg.received)
932 + perf_num_send_received_msg.sent_var_bytes
933 + perf_num_send_received_msg.received_var_bytes;
934}
935
936
937/**
938 * Function that calculates the perfmance values and writes them down
939 */
940static void
941calculate_perf_store ()
942{
943
944 /**
945 * Calculate RTT of init phase normally always 1
946 */
947 float rtt = 1;
948 int bytes_transmitted = 0;
949
950 /**
951 * Calculate RGNUNET_SETU_AcceptMessageRT of Fullsync normally 1 or 1.5 depending
952 */
953 if ((perf_store.element_full.received != 0) ||
954 (perf_store.element_full.sent != 0)
955 )
956 rtt += 1;
957
958 if ((perf_store.request_full.received != 0) ||
959 (perf_store.request_full.sent != 0)
960 )
961 rtt += 0.5;
962
963 /**
964 * In case of a differential sync 3 rtt's are needed.
965 * for every active/passive switch additional 3.5 rtt's are used
966 */
967 if ((perf_store.element.received != 0) ||
968 (perf_store.element.sent != 0))
969 {
970 int iterations = perf_store.active_passive_switches;
971
972 if (iterations > 0)
973 rtt += iterations * 0.5;
974 rtt += 2.5;
975 }
976
977
978 /**
979 * Calculate data sended size
980 */
981 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
982 GNUNET_SETU_ResultMessage),
983 perf_store.request_full);
984
985 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
986 GNUNET_SETU_ElementMessage),
987 perf_store.element_full);
988 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
989 GNUNET_SETU_ElementMessage),
990 perf_store.element);
991 // bytes_transmitted += sum_sent_received_bytes(sizeof(GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST), perf_store.operation_request);
992 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
993 StrataEstimatorMessage),
994 perf_store.se);
995 bytes_transmitted += sum_sent_received_bytes (4, perf_store.full_done);
996 bytes_transmitted += sum_sent_received_bytes (sizeof(struct IBFMessage),
997 perf_store.ibf);
998 bytes_transmitted += sum_sent_received_bytes (sizeof(struct InquiryMessage),
999 perf_store.inquery);
1000 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
1001 GNUNET_MessageHeader),
1002 perf_store.demand);
1003 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
1004 GNUNET_MessageHeader),
1005 perf_store.offer);
1006 bytes_transmitted += sum_sent_received_bytes (4, perf_store.done);
1007
1008 /**
1009 * Write IBF failure rate for different BUCKET_NUMBER_FACTOR
1010 */
1011 float factor;
1012 GNUNET_CONFIGURATION_get_value_float (setu_cfg,"IBF", "BUCKET_NUMBER_FACTOR",
1013 &factor);
1014 long long num_per_bucket;
1015 GNUNET_CONFIGURATION_get_value_number (setu_cfg,"IBF", "NUMBER_PER_BUCKET",
1016 &num_per_bucket);
1017
1018
1019 int decoded = 0;
1020 if (perf_store.active_passive_switches == 0)
1021 decoded = 1;
1022 int ibf_bytes_transmitted = sum_sent_received_bytes (sizeof(struct
1023 IBFMessage),
1024 perf_store.ibf);
1025
1026 FILE *out1 = fopen ("perf_data.csv", "a");
1027 fprintf (out1, "%d,%f,%d,%d,%f,%d,%d,%d,%d,%d\n",num_per_bucket,factor,
1028 decoded,ibf_bytes_transmitted,rtt,perf_store.se_diff,
1029 bytes_transmitted,
1030 perf_store.se_diff_local,perf_store.se_diff_remote,
1031 perf_store.mode_of_operation);
1032 fclose (out1);
1033
1034}
1035
1036
1037#endif
1038/**
1039 * Function that chooses the optimal mode of operation depending on
1040 * operation parameters.
1041 * @param avg_element_size
1042 * @param local_set_size
1043 * @param remote_set_size
1044 * @param est_set_diff_remote
1045 * @param est_set_diff_local
1046 * @param bandwith_latency_tradeoff
1047 * @param ibf_bucket_number_factor
1048 * @return calcuated mode of operation
1049 */
1050static uint8_t
1051estimate_best_mode_of_operation (uint64_t avg_element_size,
1052 uint64_t local_set_size,
1053 uint64_t remote_set_size,
1054 uint64_t est_set_diff_remote,
1055 uint64_t est_set_diff_local,
1056 uint64_t bandwith_latency_tradeoff,
1057 uint64_t ibf_bucket_number_factor)
1058{
1059
1060 /*
1061 * In case of initial sync fall to predefined states
1062 */
1063
1064 if (0 == local_set_size)
1065 return FULL_SYNC_REMOTE_SENDING_FIRST;
1066 if (0 == remote_set_size)
1067 return FULL_SYNC_LOCAL_SENDING_FIRST;
1068
1069 /*
1070 * Calculate bytes for full Sync
1071 */
1072
1073 uint8_t sizeof_full_done_header = 4;
1074 uint8_t sizeof_done_header = 4;
1075 uint8_t rtt_min_full = 2;
1076 uint8_t sizeof_request_full = 4;
1077 uint64_t estimated_total_diff = (est_set_diff_remote + est_set_diff_local);
1078
1079 /* Estimate byte required if we send first */
1080 uint64_t total_elements_to_send_local_send_first = est_set_diff_remote
1081 + local_set_size;
1082
1083 uint64_t total_bytes_full_local_send_first = (avg_element_size
1084 *
1085 total_elements_to_send_local_send_first) \
1086 + (
1087 total_elements_to_send_local_send_first * sizeof(struct
1088 GNUNET_SETU_ElementMessage)) \
1089 + (sizeof_full_done_header * 2) \
1090 + rtt_min_full
1091 * bandwith_latency_tradeoff;
1092
1093 /* Estimate bytes required if we request from remote peer */
1094 uint64_t total_elements_to_send_remote_send_first = est_set_diff_local
1095 + remote_set_size;
1096
1097 uint64_t total_bytes_full_remote_send_first = (avg_element_size
1098 *
1099 total_elements_to_send_remote_send_first) \
1100 + (
1101 total_elements_to_send_remote_send_first * sizeof(struct
1102 GNUNET_SETU_ElementMessage)) \
1103 + (sizeof_full_done_header * 2) \
1104 + (rtt_min_full + 0.5)
1105 * bandwith_latency_tradeoff \
1106 + sizeof_request_full;
1107
1108 /*
1109 * Calculate bytes for differential Sync
1110 */
1111
1112 /* Estimate bytes required by IBF transmission*/
1113
1114 long double ibf_bucket_count = estimated_total_diff
1115 * ibf_bucket_number_factor;
1116
1117 if (ibf_bucket_count <= IBF_MIN_SIZE)
1118 {
1119 ibf_bucket_count = IBF_MIN_SIZE;
1120 }
1121 uint64_t ibf_message_count = ceil ( ((float) ibf_bucket_count)
1122 / ((float) MAX_BUCKETS_PER_MESSAGE));
1123
1124 uint64_t estimated_counter_size = ceil (
1125 MIN (2 * log2l (((float) local_set_size)
1126 / ((float) ibf_bucket_count)),
1127 log2l (local_set_size)));
1128
1129 long double counter_bytes = (float) estimated_counter_size / 8;
1130
1131 uint64_t ibf_bytes = ceil ((sizeof (struct IBFMessage) * ibf_message_count)
1132 * 1.2 \
1133 + (ibf_bucket_count * sizeof(struct IBF_Key)) * 1.2 \
1134 + (ibf_bucket_count * sizeof(struct IBF_KeyHash))
1135 * 1.2 \
1136 + (ibf_bucket_count * counter_bytes) * 1.2);
1137
1138 /* Estimate full byte count for differential sync */
1139 uint64_t element_size = (avg_element_size
1140 + sizeof (struct GNUNET_SETU_ElementMessage)) \
1141 * estimated_total_diff;
1142 uint64_t done_size = sizeof_done_header;
1143 uint64_t inquery_size = (sizeof (struct IBF_Key)
1144 + sizeof (struct InquiryMessage))
1145 * estimated_total_diff;
1146 uint64_t demand_size =
1147 (sizeof(struct GNUNET_HashCode) + sizeof(struct GNUNET_MessageHeader))
1148 * estimated_total_diff;
1149 uint64_t offer_size = (sizeof (struct GNUNET_HashCode)
1150 + sizeof (struct GNUNET_MessageHeader))
1151 * estimated_total_diff;
1152
1153 uint64_t total_bytes_diff = (element_size + done_size + inquery_size
1154 + demand_size + offer_size + ibf_bytes) \
1155 + (DIFFERENTIAL_RTT_MEAN
1156 * bandwith_latency_tradeoff);
1157
1158 uint64_t full_min = MIN (total_bytes_full_local_send_first,
1159 total_bytes_full_remote_send_first);
1160
1161 /* Decide between full and differential sync */
1162
1163 if (full_min < total_bytes_diff)
1164 {
1165 /* Decide between sending all element first or receiving all elements */
1166 if (total_bytes_full_remote_send_first > total_bytes_full_local_send_first)
1167 {
1168 return FULL_SYNC_LOCAL_SENDING_FIRST;
1169 }
1170 else
1171 {
1172 return FULL_SYNC_REMOTE_SENDING_FIRST;
1173 }
1174 }
1175 else
1176 {
1177 return DIFFERENTIAL_SYNC;
1178 }
1179}
1180
1181
1182/**
1183 * Validates the if a message is received in a correct phase
1184 * @param allowed_phases
1185 * @param size_phases
1186 * @param op
1187 * @return #GNUNET_YES if message permitted in phase and #GNUNET_NO if not permitted in given
1188 * phase
1189 */
1190static enum GNUNET_GenericReturnValue
1191check_valid_phase (const uint8_t allowed_phases[],
1192 size_t size_phases,
1193 struct Operation *op)
1194{
1195 /**
1196 * Iterate over allowed phases
1197 */
1198 for (uint32_t phase_ctr = 0; phase_ctr < size_phases; phase_ctr++)
1199 {
1200 uint8_t phase = allowed_phases[phase_ctr];
1201 if (phase == op->phase)
1202 {
1203 LOG (GNUNET_ERROR_TYPE_DEBUG,
1204 "Message received in valid phase\n");
1205 return GNUNET_YES;
1206 }
1207 }
1208 LOG (GNUNET_ERROR_TYPE_ERROR,
1209 "Received message in invalid phase: %u\n", op->phase);
1210 return GNUNET_NO;
1211}
1212
1213
1214/**
1215 * Function to update, track and validate message received in differential
1216 * sync. This function tracks states of messages and check it against different
1217 * constraints as described in Summermatter's BSc Thesis (2021)
1218 * @param hash_map: Hashmap to store message control flow
1219 * @param new_mcfs: The new message control flow state an given message type should be set to
1220 * @param hash_code: Hash code of the element
1221 * @param mt: The message type for which the message control flow state should be set
1222 * @return GNUNET_YES message is valid in message control flow GNUNET_NO when message is not valid
1223 * at this point in message flow
1224 */
1225static int
1226update_message_control_flow (struct GNUNET_CONTAINER_MultiHashMap *hash_map,
1227 enum MESSAGE_CONTROL_FLOW_STATE new_mcfs,
1228 const struct GNUNET_HashCode *hash_code,
1229 enum MESSAGE_TYPE mt)
1230{
1231 struct messageControlFlowElement *cfe = NULL;
1232 enum MESSAGE_CONTROL_FLOW_STATE *mcfs;
1233
1234 /**
1235 * Check logic for forbidden messages
1236 */
1237
1238 cfe = GNUNET_CONTAINER_multihashmap_get (hash_map, hash_code);
1239 if ((ELEMENT_MESSAGE == mt) && (cfe != NULL))
1240 {
1241 if ((new_mcfs != MSG_CFS_SENT) && (MSG_CFS_RECEIVED != cfe->offer))
1242 {
1243 LOG (GNUNET_ERROR_TYPE_ERROR,
1244 "Received an element without sent offer!\n");
1245 return GNUNET_NO;
1246 }
1247 /* Check that only requested elements are received! */
1248 if ((ELEMENT_MESSAGE == mt) && (new_mcfs != MSG_CFS_SENT) && (cfe->demand !=
1249 MSG_CFS_SENT))
1250 {
1251 LOG (GNUNET_ERROR_TYPE_ERROR,
1252 "Received an element that was not demanded\n");
1253 return GNUNET_NO;
1254 }
1255 }
1256
1257 /**
1258 * In case the element hash is not in the hashmap create a new entry
1259 */
1260
1261 if (NULL == cfe)
1262 {
1263 cfe = GNUNET_new (struct messageControlFlowElement);
1264 if (GNUNET_SYSERR == GNUNET_CONTAINER_multihashmap_put (hash_map, hash_code,
1265 cfe,
1266 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1267 {
1268 GNUNET_free (cfe);
1269 return GNUNET_SYSERR;
1270 }
1271 }
1272
1273 /**
1274 * Set state of message
1275 */
1276
1277 if (OFFER_MESSAGE == mt)
1278 {
1279 mcfs = &cfe->offer;
1280 }
1281 else if (DEMAND_MESSAGE == mt)
1282 {
1283 mcfs = &cfe->demand;
1284 }
1285 else if (ELEMENT_MESSAGE == mt)
1286 {
1287 mcfs = &cfe->element;
1288 }
1289 else
1290 {
1291 return GNUNET_SYSERR;
1292 }
1293
1294 /**
1295 * Check if state is allowed
1296 */
1297
1298 if (new_mcfs <= *mcfs)
1299 {
1300 return GNUNET_NO;
1301 }
1302
1303 *mcfs = new_mcfs;
1304 return GNUNET_YES;
1305}
1306
1307
1308/**
1309 * Validate if a message in differential sync si already received before.
1310 * @param hash_map
1311 * @param hash_code
1312 * @param mt
1313 * @return GNUNET_YES when message is already in store if message is not in store return GNUNET_NO
1314 */
1315static int
1316is_message_in_message_control_flow (struct
1317 GNUNET_CONTAINER_MultiHashMap *hash_map,
1318 struct GNUNET_HashCode *hash_code,
1319 enum MESSAGE_TYPE mt)
1320{
1321 struct messageControlFlowElement *cfe = NULL;
1322 enum MESSAGE_CONTROL_FLOW_STATE *mcfs;
1323
1324 cfe = GNUNET_CONTAINER_multihashmap_get (hash_map, hash_code);
1325
1326 /**
1327 * Set state of message
1328 */
1329
1330 if (cfe != NULL)
1331 {
1332 if (OFFER_MESSAGE == mt)
1333 {
1334 mcfs = &cfe->offer;
1335 }
1336 else if (DEMAND_MESSAGE == mt)
1337 {
1338 mcfs = &cfe->demand;
1339 }
1340 else if (ELEMENT_MESSAGE == mt)
1341 {
1342 mcfs = &cfe->element;
1343 }
1344 else
1345 {
1346 return GNUNET_SYSERR;
1347 }
1348
1349 /**
1350 * Evaluate if set is in message
1351 */
1352 if (*mcfs != MSG_CFS_UNINITIALIZED)
1353 {
1354 return GNUNET_YES;
1355 }
1356 }
1357 return GNUNET_NO;
1358}
1359
1360
1361/**
1362 * Iterator for determining if all demands have been
1363 * satisfied
1364 *
1365 * @param cls the union operation `struct Operation *`
1366 * @param key unused
1367 * @param value the `struct ElementEntry *` to insert
1368 * into the key-to-element mapping
1369 * @return #GNUNET_YES (to continue iterating)
1370 */
1371static int
1372determinate_done_message_iterator (void *cls,
1373 const struct GNUNET_HashCode *key,
1374 void *value)
1375{
1376 struct messageControlFlowElement *mcfe = value;
1377
1378 if (((mcfe->element == MSG_CFS_SENT) || (mcfe->element == MSG_CFS_RECEIVED) ))
1379 {
1380 return GNUNET_YES;
1381 }
1382 return GNUNET_NO;
1383}
1384
1385
1386/**
1387 * Iterator for determining average size
1388 *
1389 * @param cls the union operation `struct Operation *`
1390 * @param key unused
1391 * @param value the `struct ElementEntry *` to insert
1392 * into the key-to-element mapping
1393 * @return #GNUNET_YES (to continue iterating)
1394 */
1395static int
1396determinate_avg_element_size_iterator (void *cls,
1397 const struct GNUNET_HashCode *key,
1398 void *value)
1399{
1400 struct Operation *op = cls;
1401 struct GNUNET_SETU_Element *element = value;
1402 op->total_elements_size_local += element->size;
1403 return GNUNET_YES;
1404}
1405
1406
1407/**
1408 * Create randomized element hashmap for full sending
1409 *
1410 * @param cls the union operation `struct Operation *`
1411 * @param key unused
1412 * @param value the `struct ElementEntry *` to insert
1413 * into the key-to-element mapping
1414 * @return #GNUNET_YES (to continue iterating)
1415 */
1416static int
1417create_randomized_element_iterator (void *cls,
1418 const struct GNUNET_HashCode *key,
1419 void *value)
1420{
1421 struct Operation *op = cls;
1422
1423 struct GNUNET_HashContext *hashed_key_context =
1424 GNUNET_CRYPTO_hash_context_start ();
1425 struct GNUNET_HashCode new_key;
1426
1427 /**
1428 * Hash element with new salt to randomize hashmap
1429 */
1430 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
1431 &key,
1432 sizeof(struct IBF_Key));
1433 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
1434 &op->set->content->elements_randomized_salt,
1435 sizeof(uint32_t));
1436 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
1437 &new_key);
1438 GNUNET_CONTAINER_multihashmap_put (op->set->content->elements_randomized,
1439 &new_key,value,
1440 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
1441 return GNUNET_YES;
1442}
1443
1444
1445/**
1446 * Iterator over hash map entries, called to
1447 * destroy the linked list of colliding ibf key entries.
1448 *
1449 * @param cls closure
1450 * @param key current key code
1451 * @param value value in the hash map
1452 * @return #GNUNET_YES if we should continue to iterate,
1453 * #GNUNET_NO if not.
1454 */
1455static int
1456destroy_key_to_element_iter (void *cls,
1457 uint32_t key,
1458 void *value)
1459{
1460 struct KeyEntry *k = value;
1461
1462 GNUNET_assert (NULL != k);
1463 if (GNUNET_YES == k->element->remote)
1464 {
1465 GNUNET_free (k->element);
1466 k->element = NULL;
1467 }
1468 GNUNET_free (k);
1469 return GNUNET_YES;
1470}
1471
1472
1473/**
1474 * Signal to the client that the operation has finished and
1475 * destroy the operation.
1476 *
1477 * @param cls operation to destroy
1478 */
1479static void
1480send_client_done (void *cls)
1481{
1482 struct Operation *op = cls;
1483 struct GNUNET_MQ_Envelope *ev;
1484 struct GNUNET_SETU_ResultMessage *rm;
1485
1486 if (GNUNET_YES == op->client_done_sent)
1487 return;
1488 if (PHASE_FINISHED != op->phase)
1489 {
1490 LOG (GNUNET_ERROR_TYPE_WARNING,
1491 "Union operation failed\n");
1492 GNUNET_STATISTICS_update (_GSS_statistics,
1493 "# Union operations failed",
1494 1,
1495 GNUNET_NO);
1496 ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SETU_RESULT);
1497 rm->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
1498 rm->request_id = htonl (op->client_request_id);
1499 rm->element_type = htons (0);
1500 GNUNET_MQ_send (op->set->cs->mq,
1501 ev);
1502 return;
1503 }
1504
1505 op->client_done_sent = GNUNET_YES;
1506
1507 GNUNET_STATISTICS_update (_GSS_statistics,
1508 "# Union operations succeeded",
1509 1,
1510 GNUNET_NO);
1511 LOG (GNUNET_ERROR_TYPE_INFO,
1512 "Signalling client that union operation is done\n");
1513 ev = GNUNET_MQ_msg (rm,
1514 GNUNET_MESSAGE_TYPE_SETU_RESULT);
1515 rm->request_id = htonl (op->client_request_id);
1516 rm->result_status = htons (GNUNET_SETU_STATUS_DONE);
1517 rm->element_type = htons (0);
1518 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
1519 op->key_to_element));
1520 GNUNET_MQ_send (op->set->cs->mq,
1521 ev);
1522}
1523
1524
1525/**
1526 * Check if all given byzantine parameters are in given boundaries
1527 * @param op
1528 * @return indicator if all given byzantine parameters are in given boundaries
1529 */
1530
1531static int
1532check_byzantine_bounds (struct Operation *op)
1533{
1534 if (op->byzantine != GNUNET_YES)
1535 return GNUNET_OK;
1536
1537 /**
1538 * Check upper byzantine bounds
1539 */
1540 if (op->remote_element_count + op->remote_set_diff >
1541 op->byzantine_upper_bound)
1542 return GNUNET_SYSERR;
1543 if (op->local_element_count + op->local_set_diff > op->byzantine_upper_bound)
1544 return GNUNET_SYSERR;
1545
1546 /**
1547 * Check lower byzantine bounds
1548 */
1549 if (op->remote_element_count < op->byzantine_lower_bound)
1550 return GNUNET_SYSERR;
1551 return GNUNET_OK;
1552}
1553
1554
1555/* FIXME: the destroy logic is a mess and should be cleaned up! */
1556
1557/**
1558 * Destroy the given operation. Used for any operation where both
1559 * peers were known and that thus actually had a vt and channel. Must
1560 * not be used for operations where 'listener' is still set and we do
1561 * not know the other peer.
1562 *
1563 * Call the implementation-specific cancel function of the operation.
1564 * Disconnects from the remote peer. Does not disconnect the client,
1565 * as there may be multiple operations per set.
1566 *
1567 * @param op operation to destroy
1568 */
1569static void
1570_GSS_operation_destroy (struct Operation *op)
1571{
1572 struct Set *set = op->set;
1573 struct GNUNET_CADET_Channel *channel;
1574
1575 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1576 "Destroying union operation %p\n",
1577 op);
1578 GNUNET_assert (NULL == op->listener);
1579 /* check if the op was canceled twice */
1580 if (NULL != op->remote_ibf)
1581 {
1582 ibf_destroy (op->remote_ibf);
1583 op->remote_ibf = NULL;
1584 }
1585 if (NULL != op->demanded_hashes)
1586 {
1587 GNUNET_CONTAINER_multihashmap_destroy (op->demanded_hashes);
1588 op->demanded_hashes = NULL;
1589 }
1590 if (NULL != op->local_ibf)
1591 {
1592 ibf_destroy (op->local_ibf);
1593 op->local_ibf = NULL;
1594 }
1595 if (NULL != op->se)
1596 {
1597 strata_estimator_destroy (op->se);
1598 op->se = NULL;
1599 }
1600 if (NULL != op->key_to_element)
1601 {
1602 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
1603 &destroy_key_to_element_iter,
1604 NULL);
1605 GNUNET_CONTAINER_multihashmap32_destroy (op->key_to_element);
1606 op->key_to_element = NULL;
1607 }
1608 if (NULL != set)
1609 {
1610 GNUNET_CONTAINER_DLL_remove (set->ops_head,
1611 set->ops_tail,
1612 op);
1613 op->set = NULL;
1614 }
1615 if (NULL != op->context_msg)
1616 {
1617 GNUNET_free (op->context_msg);
1618 op->context_msg = NULL;
1619 }
1620 if (NULL != (channel = op->channel))
1621 {
1622 /* This will free op; called conditionally as this helper function
1623 is also called from within the channel disconnect handler. */
1624 op->channel = NULL;
1625 GNUNET_CADET_channel_destroy (channel);
1626 }
1627 /* We rely on the channel end handler to free 'op'. When 'op->channel' was NULL,
1628 * there was a channel end handler that will free 'op' on the call stack. */
1629}
1630
1631
1632/**
1633 * This function probably should not exist
1634 * and be replaced by inlining more specific
1635 * logic in the various places where it is called.
1636 */
1637static void
1638_GSS_operation_destroy2 (struct Operation *op);
1639
1640
1641/**
1642 * Destroy an incoming request from a remote peer
1643 *
1644 * @param op remote request to destroy
1645 */
1646static void
1647incoming_destroy (struct Operation *op)
1648{
1649 struct Listener *listener;
1650
1651 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1652 "Destroying incoming operation %p\n",
1653 op);
1654 if (NULL != (listener = op->listener))
1655 {
1656 GNUNET_CONTAINER_DLL_remove (listener->op_head,
1657 listener->op_tail,
1658 op);
1659 op->listener = NULL;
1660 }
1661 if (NULL != op->timeout_task)
1662 {
1663 GNUNET_SCHEDULER_cancel (op->timeout_task);
1664 op->timeout_task = NULL;
1665 }
1666 _GSS_operation_destroy2 (op);
1667}
1668
1669
1670/**
1671 * This function probably should not exist
1672 * and be replaced by inlining more specific
1673 * logic in the various places where it is called.
1674 */
1675static void
1676_GSS_operation_destroy2 (struct Operation *op)
1677{
1678 struct GNUNET_CADET_Channel *channel;
1679
1680 if (NULL != (channel = op->channel))
1681 {
1682 /* This will free op; called conditionally as this helper function
1683 is also called from within the channel disconnect handler. */
1684 op->channel = NULL;
1685 GNUNET_CADET_channel_destroy (channel);
1686 }
1687 if (NULL != op->listener)
1688 {
1689 incoming_destroy (op);
1690 return;
1691 }
1692 if (NULL != op->set)
1693 send_client_done (op);
1694 _GSS_operation_destroy (op);
1695 GNUNET_free (op);
1696}
1697
1698
1699/**
1700 * Inform the client that the union operation has failed,
1701 * and proceed to destroy the evaluate operation.
1702 *
1703 * @param op the union operation to fail
1704 */
1705static void
1706fail_union_operation (struct Operation *op)
1707{
1708 struct GNUNET_MQ_Envelope *ev;
1709 struct GNUNET_SETU_ResultMessage *msg;
1710
1711 LOG (GNUNET_ERROR_TYPE_WARNING,
1712 "union operation failed\n");
1713 ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SETU_RESULT);
1714 msg->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
1715 msg->request_id = htonl (op->client_request_id);
1716 msg->element_type = htons (0);
1717 GNUNET_MQ_send (op->set->cs->mq,
1718 ev);
1719 _GSS_operation_destroy (op);
1720}
1721
1722
1723/**
1724 * Function that checks if full sync is plausible
1725 * @param initial_local_elements_in_set
1726 * @param estimated_set_difference
1727 * @param repeated_elements
1728 * @param fresh_elements
1729 * @param op
1730 * @return GNUNET_OK if
1731 */
1732
1733static void
1734full_sync_plausibility_check (struct Operation *op)
1735{
1736 if (GNUNET_YES != op->byzantine)
1737 return;
1738
1739 int security_level_lb = -1 * SECURITY_LEVEL;
1740 uint64_t duplicates = op->received_fresh - op->received_total;
1741
1742 /*
1743 * Protect full sync from receiving double element when in FULL SENDING
1744 */
1745 if (PHASE_FULL_SENDING == op->phase)
1746 {
1747 if (duplicates > 0)
1748 {
1749 LOG (GNUNET_ERROR_TYPE_ERROR,
1750 "PROTOCOL VIOLATION: Received duplicate element in full receiving "
1751 "mode of operation this is not allowed! Duplicates: %llu\n",
1752 (unsigned long long) duplicates);
1753 GNUNET_break_op (0);
1754 fail_union_operation (op);
1755 return;
1756 }
1757
1758 }
1759
1760 /*
1761 * Protect full sync with probabilistic algorithm
1762 */
1763 if (PHASE_FULL_RECEIVING == op->phase)
1764 {
1765 if (0 == op->remote_set_diff)
1766 op->remote_set_diff = 1;
1767
1768 long double base = (1 - (long double) (op->remote_set_diff
1769 / (long double) (op->initial_size
1770 + op->
1771 remote_set_diff)));
1772 long double exponent = (op->received_total - (op->received_fresh * ((long
1773 double)
1774 op->
1775 initial_size
1776 / (long
1777 double)
1778 op->
1779 remote_set_diff)));
1780 long double value = exponent * (log2l (base) / log2l (2));
1781 if ((value < security_level_lb) || (value > SECURITY_LEVEL) )
1782 {
1783 LOG (GNUNET_ERROR_TYPE_ERROR,
1784 "PROTOCOL VIOLATION: Other peer violated probabilistic rule for receiving "
1785 "to many duplicated full element : %LF\n",
1786 value);
1787 GNUNET_break_op (0);
1788 fail_union_operation (op);
1789 return;
1790 }
1791 }
1792}
1793
1794
1795/**
1796 * Limit active passive switches in differential sync to configured security level
1797 * @param op
1798 */
1799static void
1800check_max_differential_rounds (struct Operation *op)
1801{
1802 double probability = op->differential_sync_iterations * (log2l (
1803 PROBABILITY_FOR_NEW_ROUND)
1804 / log2l (2));
1805 if ((-1 * SECURITY_LEVEL) > probability)
1806 {
1807 LOG (GNUNET_ERROR_TYPE_ERROR,
1808 "PROTOCOL VIOLATION: Other peer violated probabilistic rule for to many active passive "
1809 "switches in differential sync: %u\n",
1810 op->differential_sync_iterations);
1811 GNUNET_break_op (0);
1812 fail_union_operation (op);
1813 return;
1814 }
1815}
1816
1817
1818/**
1819 * Derive the IBF key from a hash code and
1820 * a salt.
1821 *
1822 * @param src the hash code
1823 * @return the derived IBF key
1824 */
1825static struct IBF_Key
1826get_ibf_key (const struct GNUNET_HashCode *src)
1827{
1828 struct IBF_Key key;
1829 uint16_t salt = 0;
1830
1831 GNUNET_assert (GNUNET_OK ==
1832 GNUNET_CRYPTO_kdf (&key, sizeof(key),
1833 src, sizeof *src,
1834 &salt, sizeof(salt),
1835 NULL, 0));
1836 return key;
1837}
1838
1839
1840/**
1841 * Context for #op_get_element_iterator
1842 */
1843struct GetElementContext
1844{
1845 /**
1846 * Gnunet hash code in context
1847 */
1848 struct GNUNET_HashCode hash;
1849
1850 /**
1851 * Pointer to the key entry
1852 */
1853 struct KeyEntry *k;
1854};
1855
1856
1857/**
1858 * Iterator over the mapping from IBF keys to element entries. Checks if we
1859 * have an element with a given GNUNET_HashCode.
1860 *
1861 * @param cls closure
1862 * @param key current key code
1863 * @param value value in the hash map
1864 * @return #GNUNET_YES if we should search further,
1865 * #GNUNET_NO if we've found the element.
1866 */
1867static int
1868op_get_element_iterator (void *cls,
1869 uint32_t key,
1870 void *value)
1871{
1872 struct GetElementContext *ctx = cls;
1873 struct KeyEntry *k = value;
1874
1875 GNUNET_assert (NULL != k);
1876 if (0 == GNUNET_CRYPTO_hash_cmp (&k->element->element_hash,
1877 &ctx->hash))
1878 {
1879 ctx->k = k;
1880 return GNUNET_NO;
1881 }
1882 return GNUNET_YES;
1883}
1884
1885
1886/**
1887 * Determine whether the given element is already in the operation's element
1888 * set.
1889 *
1890 * @param op operation that should be tested for 'element_hash'
1891 * @param element_hash hash of the element to look for
1892 * @return #GNUNET_YES if the element has been found, #GNUNET_NO otherwise
1893 */
1894static struct KeyEntry *
1895op_get_element (struct Operation *op,
1896 const struct GNUNET_HashCode *element_hash)
1897{
1898 int ret;
1899 struct IBF_Key ibf_key;
1900 struct GetElementContext ctx = { { { 0 } }, 0 };
1901
1902 ctx.hash = *element_hash;
1903
1904 ibf_key = get_ibf_key (element_hash);
1905 ret = GNUNET_CONTAINER_multihashmap32_get_multiple (op->key_to_element,
1906 (uint32_t) ibf_key.key_val,
1907 &op_get_element_iterator,
1908 &ctx);
1909
1910 /* was the iteration aborted because we found the element? */
1911 if (GNUNET_SYSERR == ret)
1912 {
1913 GNUNET_assert (NULL != ctx.k);
1914 return ctx.k;
1915 }
1916 return NULL;
1917}
1918
1919
1920/**
1921 * Insert an element into the union operation's
1922 * key-to-element mapping. Takes ownership of 'ee'.
1923 * Note that this does not insert the element in the set,
1924 * only in the operation's key-element mapping.
1925 * This is done to speed up re-tried operations, if some elements
1926 * were transmitted, and then the IBF fails to decode.
1927 *
1928 * XXX: clarify ownership, doesn't sound right.
1929 *
1930 * @param op the union operation
1931 * @param ee the element entry
1932 * @param received was this element received from the remote peer?
1933 */
1934static void
1935op_register_element (struct Operation *op,
1936 struct ElementEntry *ee,
1937 int received)
1938{
1939 struct IBF_Key ibf_key;
1940 struct KeyEntry *k;
1941
1942 ibf_key = get_ibf_key (&ee->element_hash);
1943 k = GNUNET_new (struct KeyEntry);
1944 k->element = ee;
1945 k->ibf_key = ibf_key;
1946 k->received = received;
1947 GNUNET_assert (GNUNET_OK ==
1948 GNUNET_CONTAINER_multihashmap32_put (op->key_to_element,
1949 (uint32_t) ibf_key.key_val,
1950 k,
1951 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
1952}
1953
1954
1955/**
1956 * Modify an IBF key @a k_in based on the @a salt, returning a
1957 * salted key in @a k_out.
1958 */
1959static void
1960salt_key (const struct IBF_Key *k_in,
1961 uint32_t salt,
1962 struct IBF_Key *k_out)
1963{
1964 int s = (salt * 7) % 64;
1965 uint64_t x = k_in->key_val;
1966
1967 /* rotate ibf key */
1968 x = (x >> s) | (x << (64 - s));
1969 k_out->key_val = x;
1970}
1971
1972
1973/**
1974 * Reverse modification done in the salt_key function
1975 */
1976static void
1977unsalt_key (const struct IBF_Key *k_in,
1978 uint32_t salt,
1979 struct IBF_Key *k_out)
1980{
1981 int s = (salt * 7) % 64;
1982 uint64_t x = k_in->key_val;
1983
1984 x = (x << s) | (x >> (64 - s));
1985 k_out->key_val = x;
1986}
1987
1988
1989/**
1990 * Insert a key into an ibf.
1991 *
1992 * @param cls the ibf
1993 * @param key unused
1994 * @param value the key entry to get the key from
1995 */
1996static int
1997prepare_ibf_iterator (void *cls,
1998 uint32_t key,
1999 void *value)
2000{
2001 struct Operation *op = cls;
2002 struct KeyEntry *ke = value;
2003 struct IBF_Key salted_key;
2004
2005 LOG (GNUNET_ERROR_TYPE_DEBUG,
2006 "[OP %p] inserting %lx (hash %s) into ibf\n",
2007 op,
2008 (unsigned long) ke->ibf_key.key_val,
2009 GNUNET_h2s (&ke->element->element_hash));
2010 salt_key (&ke->ibf_key,
2011 op->salt_send,
2012 &salted_key);
2013 ibf_insert (op->local_ibf, salted_key);
2014 return GNUNET_YES;
2015}
2016
2017
2018/**
2019 * Is element @a ee part of the set used by @a op?
2020 *
2021 * @param ee element to test
2022 * @param op operation the defines the set and its generation
2023 * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
2024 */
2025static int
2026_GSS_is_element_of_operation (struct ElementEntry *ee,
2027 struct Operation *op)
2028{
2029 return ee->generation >= op->generation_created;
2030}
2031
2032
2033/**
2034 * Iterator for initializing the
2035 * key-to-element mapping of a union operation
2036 *
2037 * @param cls the union operation `struct Operation *`
2038 * @param key unused
2039 * @param value the `struct ElementEntry *` to insert
2040 * into the key-to-element mapping
2041 * @return #GNUNET_YES (to continue iterating)
2042 */
2043static int
2044init_key_to_element_iterator (void *cls,
2045 const struct GNUNET_HashCode *key,
2046 void *value)
2047{
2048 struct Operation *op = cls;
2049 struct ElementEntry *ee = value;
2050
2051 /* make sure that the element belongs to the set at the time
2052 * of creating the operation */
2053 if (GNUNET_NO ==
2054 _GSS_is_element_of_operation (ee,
2055 op))
2056 return GNUNET_YES;
2057 GNUNET_assert (GNUNET_NO == ee->remote);
2058 op_register_element (op,
2059 ee,
2060 GNUNET_NO);
2061 return GNUNET_YES;
2062}
2063
2064
2065/**
2066 * Initialize the IBF key to element mapping local to this set operation.
2067 *
2068 * @param op the set union operation
2069 */
2070static void
2071initialize_key_to_element (struct Operation *op)
2072{
2073 unsigned int len;
2074
2075 GNUNET_assert (NULL == op->key_to_element);
2076 len = GNUNET_CONTAINER_multihashmap_size (op->set->content->elements);
2077 op->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
2078 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2079 &init_key_to_element_iterator,
2080 op);
2081}
2082
2083
2084/**
2085 * Create an ibf with the operation's elements
2086 * of the specified size
2087 *
2088 * @param op the union operation
2089 * @param size size of the ibf to create
2090 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2091 */
2092static int
2093prepare_ibf (struct Operation *op,
2094 uint32_t size)
2095{
2096 GNUNET_assert (NULL != op->key_to_element);
2097
2098 if (NULL != op->local_ibf)
2099 ibf_destroy (op->local_ibf);
2100 // op->local_ibf = ibf_create (size, SE_IBF_HASH_NUM);
2101 op->local_ibf = ibf_create (size,
2102 ((uint8_t) op->ibf_number_buckets_per_element));
2103 if (NULL == op->local_ibf)
2104 {
2105 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2106 "Failed to allocate local IBF\n");
2107 return GNUNET_SYSERR;
2108 }
2109 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
2110 &prepare_ibf_iterator,
2111 op);
2112 return GNUNET_OK;
2113}
2114
2115
2116/**
2117 * Send an ibf of appropriate size.
2118 *
2119 * Fragments the IBF into multiple messages if necessary.
2120 *
2121 * @param op the union operation
2122 * @param ibf_order order of the ibf to send, size=2^order
2123 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2124 */
2125static int
2126send_ibf (struct Operation *op,
2127 uint32_t ibf_size)
2128{
2129 uint64_t buckets_sent = 0;
2130 struct InvertibleBloomFilter *ibf;
2131 op->differential_sync_iterations++;
2132
2133 /**
2134 * Enforce min size of IBF
2135 */
2136 uint32_t ibf_min_size = IBF_MIN_SIZE;
2137
2138 if (ibf_size < ibf_min_size)
2139 {
2140 ibf_size = ibf_min_size;
2141 }
2142 if (GNUNET_OK !=
2143 prepare_ibf (op, ibf_size))
2144 {
2145 /* allocation failed */
2146 return GNUNET_SYSERR;
2147 }
2148
2149 LOG (GNUNET_ERROR_TYPE_DEBUG,
2150 "sending ibf of size %u\n",
2151 (unsigned int) ibf_size);
2152
2153 {
2154 char name[64];
2155
2156 GNUNET_snprintf (name,
2157 sizeof(name),
2158 "# sent IBF (order %u)",
2159 ibf_size);
2160 GNUNET_STATISTICS_update (_GSS_statistics, name, 1, GNUNET_NO);
2161 }
2162
2163 ibf = op->local_ibf;
2164
2165 while (buckets_sent < ibf_size)
2166 {
2167 unsigned int buckets_in_message;
2168 struct GNUNET_MQ_Envelope *ev;
2169 struct IBFMessage *msg;
2170
2171 buckets_in_message = ibf_size - buckets_sent;
2172 /* limit to maximum */
2173 if (buckets_in_message > MAX_BUCKETS_PER_MESSAGE)
2174 buckets_in_message = MAX_BUCKETS_PER_MESSAGE;
2175
2176#if MEASURE_PERFORMANCE
2177 perf_store.ibf.sent += 1;
2178 perf_store.ibf.sent_var_bytes += (buckets_in_message * IBF_BUCKET_SIZE);
2179#endif
2180 ev = GNUNET_MQ_msg_extra (msg,
2181 buckets_in_message * IBF_BUCKET_SIZE,
2182 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF);
2183 msg->ibf_size = ibf_size;
2184 msg->offset = htonl (buckets_sent);
2185 msg->salt = htonl (op->salt_send);
2186 msg->ibf_counter_bit_length = ibf_get_max_counter (ibf);
2187
2188
2189 ibf_write_slice (ibf, buckets_sent,
2190 buckets_in_message, &msg[1], msg->ibf_counter_bit_length);
2191 buckets_sent += buckets_in_message;
2192 LOG (GNUNET_ERROR_TYPE_DEBUG,
2193 "ibf chunk size %u, %llu/%u sent\n",
2194 (unsigned int) buckets_in_message,
2195 (unsigned long long) buckets_sent,
2196 (unsigned int) ibf_size);
2197 GNUNET_MQ_send (op->mq, ev);
2198 }
2199
2200 /* The other peer must decode the IBF, so
2201 * we're passive. */
2202 op->phase = PHASE_PASSIVE_DECODING;
2203 return GNUNET_OK;
2204}
2205
2206
2207/**
2208 * Compute the necessary order of an ibf
2209 * from the size of the symmetric set difference.
2210 *
2211 * @param diff the difference
2212 * @return the required size of the ibf
2213 */
2214static unsigned int
2215get_size_from_difference (unsigned int diff, int number_buckets_per_element,
2216 float ibf_bucket_number_factor)
2217{
2218 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2219 * Elias Summermatter (2021) in section 3.11 **/
2220 return (((int) (diff * ibf_bucket_number_factor)) | 1);
2221
2222}
2223
2224
2225static unsigned int
2226get_next_ibf_size (float ibf_bucket_number_factor, unsigned int
2227 decoded_elements, unsigned int last_ibf_size)
2228{
2229 unsigned int next_size = (unsigned int) ((last_ibf_size * 2)
2230 - (ibf_bucket_number_factor
2231 * decoded_elements));
2232 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2233 * Elias Summermatter (2021) in section 3.11 **/
2234 return next_size | 1;
2235}
2236
2237
2238/**
2239 * Send a set element.
2240 *
2241 * @param cls the union operation `struct Operation *`
2242 * @param key unused
2243 * @param value the `struct ElementEntry *` to insert
2244 * into the key-to-element mapping
2245 * @return #GNUNET_YES (to continue iterating)
2246 */
2247static int
2248send_full_element_iterator (void *cls,
2249 const struct GNUNET_HashCode *key,
2250 void *value)
2251{
2252 struct Operation *op = cls;
2253 struct GNUNET_SETU_ElementMessage *emsg;
2254 struct ElementEntry *ee = value;
2255 struct GNUNET_SETU_Element *el = &ee->element;
2256 struct GNUNET_MQ_Envelope *ev;
2257
2258 LOG (GNUNET_ERROR_TYPE_DEBUG,
2259 "Sending element %s\n",
2260 GNUNET_h2s (key));
2261#if MEASURE_PERFORMANCE
2262 perf_store.element_full.received += 1;
2263 perf_store.element_full.received_var_bytes += el->size;
2264#endif
2265 ev = GNUNET_MQ_msg_extra (emsg,
2266 el->size,
2267 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT);
2268 emsg->element_type = htons (el->element_type);
2269 GNUNET_memcpy (&emsg[1],
2270 el->data,
2271 el->size);
2272 GNUNET_MQ_send (op->mq,
2273 ev);
2274 return GNUNET_YES;
2275}
2276
2277
2278/**
2279 * Switch to full set transmission for @a op.
2280 *
2281 * @param op operation to switch to full set transmission.
2282 */
2283static void
2284send_full_set (struct Operation *op)
2285{
2286 struct GNUNET_MQ_Envelope *ev;
2287
2288 op->phase = PHASE_FULL_SENDING;
2289 LOG (GNUNET_ERROR_TYPE_DEBUG,
2290 "Dedicing to transmit the full set\n");
2291 /* FIXME: use a more memory-friendly way of doing this with an
2292 iterator, just as we do in the non-full case! */
2293
2294 // Randomize Elements to send
2295 op->set->content->elements_randomized = GNUNET_CONTAINER_multihashmap_create (
2296 32,GNUNET_NO);
2297 op->set->content->elements_randomized_salt = GNUNET_CRYPTO_random_u64 (
2298 GNUNET_CRYPTO_QUALITY_NONCE,
2299 UINT64_MAX);
2300 (void) GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2301 &
2302 create_randomized_element_iterator,
2303 op);
2304
2305 (void) GNUNET_CONTAINER_multihashmap_iterate (
2306 op->set->content->elements_randomized,
2307 &send_full_element_iterator,
2308 op);
2309#if MEASURE_PERFORMANCE
2310 perf_store.full_done.sent += 1;
2311#endif
2312 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE);
2313 GNUNET_MQ_send (op->mq,
2314 ev);
2315}
2316
2317
2318/**
2319 * Handle a strata estimator from a remote peer
2320 *
2321 * @param cls the union operation
2322 * @param msg the message
2323 */
2324static int
2325check_union_p2p_strata_estimator (void *cls,
2326 const struct StrataEstimatorMessage *msg)
2327{
2328 struct Operation *op = cls;
2329 int is_compressed;
2330 size_t len;
2331
2332 if (op->phase != PHASE_EXPECT_SE)
2333 {
2334 GNUNET_break (0);
2335 return GNUNET_SYSERR;
2336 }
2337 is_compressed = (GNUNET_MESSAGE_TYPE_SETU_P2P_SEC == htons (
2338 msg->header.type));
2339 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
2340 if ((GNUNET_NO == is_compressed) &&
2341 (len != SE_STRATA_COUNT * SE_IBFS_TOTAL_SIZE * IBF_BUCKET_SIZE))
2342 {
2343 GNUNET_break (0);
2344 return GNUNET_SYSERR;
2345 }
2346 return GNUNET_OK;
2347}
2348
2349
2350/**
2351 * Handle a strata estimator from a remote peer
2352 *
2353 * @param cls the union operation
2354 * @param msg the message
2355 */
2356static void
2357handle_union_p2p_strata_estimator (void *cls,
2358 const struct StrataEstimatorMessage *msg)
2359{
2360#if MEASURE_PERFORMANCE
2361 perf_store.se.received += 1;
2362 perf_store.se.received_var_bytes += ntohs (msg->header.size) - sizeof(struct
2363 StrataEstimatorMessage);
2364#endif
2365 struct Operation *op = cls;
2366 struct MultiStrataEstimator *remote_se;
2367 unsigned int diff;
2368 uint64_t other_size;
2369 size_t len;
2370 int is_compressed;
2371 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
2372 op->set->content->elements);
2373 // Setting peer site to receiving peer
2374 op->peer_site = 1;
2375
2376 /**
2377 * Check that the message is received only in supported phase
2378 */
2379 uint8_t allowed_phases[] = {PHASE_EXPECT_SE};
2380 if (GNUNET_OK !=
2381 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
2382 {
2383 GNUNET_break (0);
2384 fail_union_operation (op);
2385 return;
2386 }
2387
2388 /** Only allow 1,2,4,8 SEs **/
2389 if ((msg->se_count > 8) || (__builtin_popcount ((int) msg->se_count) != 1))
2390 {
2391 LOG (GNUNET_ERROR_TYPE_ERROR,
2392 "PROTOCOL VIOLATION: Invalid number of se transmitted by other peer %u\n",
2393 msg->se_count);
2394 GNUNET_break_op (0);
2395 fail_union_operation (op);
2396 return;
2397 }
2398
2399 is_compressed = (GNUNET_MESSAGE_TYPE_SETU_P2P_SEC == htons (
2400 msg->header.type));
2401 GNUNET_STATISTICS_update (_GSS_statistics,
2402 "# bytes of SE received",
2403 ntohs (msg->header.size),
2404 GNUNET_NO);
2405 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
2406 other_size = GNUNET_ntohll (msg->set_size);
2407 op->remote_element_count = other_size;
2408
2409 if (op->byzantine_upper_bound < op->remote_element_count)
2410 {
2411 LOG (GNUNET_ERROR_TYPE_ERROR,
2412 "Exceeded configured upper bound <%lu> of element: %u\n",
2413 op->byzantine_upper_bound,
2414 op->remote_element_count);
2415 fail_union_operation (op);
2416 return;
2417 }
2418
2419 remote_se = strata_estimator_create (SE_STRATA_COUNT,
2420 SE_IBFS_TOTAL_SIZE,
2421 SE_IBF_HASH_NUM);
2422 if (NULL == remote_se)
2423 {
2424 /* insufficient resources, fail */
2425 fail_union_operation (op);
2426 return;
2427 }
2428 if (GNUNET_OK !=
2429 strata_estimator_read (&msg[1],
2430 len,
2431 is_compressed,
2432 msg->se_count,
2433 SE_IBFS_TOTAL_SIZE,
2434 remote_se))
2435 {
2436 /* decompression failed */
2437 strata_estimator_destroy (remote_se);
2438 fail_union_operation (op);
2439 return;
2440 }
2441 GNUNET_assert (NULL != op->se);
2442 strata_estimator_difference (remote_se,
2443 op->se);
2444
2445 /* Calculate remote local diff */
2446 long diff_remote = remote_se->stratas[0]->strata[0]->remote_decoded_count;
2447 long diff_local = remote_se->stratas[0]->strata[0]->local_decoded_count;
2448
2449 /* Prevent estimations from overshooting max element */
2450 if (diff_remote + op->remote_element_count > op->byzantine_upper_bound)
2451 diff_remote = op->byzantine_upper_bound - op->remote_element_count;
2452 if (diff_local + op->local_element_count > op->byzantine_upper_bound)
2453 diff_local = op->byzantine_upper_bound - op->local_element_count;
2454 if ((diff_remote < 0) || (diff_local < 0))
2455 {
2456 strata_estimator_destroy (remote_se);
2457 LOG (GNUNET_ERROR_TYPE_ERROR,
2458 "PROTOCOL VIOLATION: More element is set as upper boundary or other peer is "
2459 "malicious: remote diff %ld, local diff: %ld\n",
2460 diff_remote, diff_local);
2461 GNUNET_break_op (0);
2462 fail_union_operation (op);
2463 return;
2464 }
2465
2466 /* Make estimation more precise in initial sync cases */
2467 if (0 == op->remote_element_count)
2468 {
2469 diff_remote = 0;
2470 diff_local = op->local_element_count;
2471 }
2472 if (0 == op->local_element_count)
2473 {
2474 diff_local = 0;
2475 diff_remote = op->remote_element_count;
2476 }
2477
2478 diff = diff_remote + diff_local;
2479 op->remote_set_diff = diff_remote;
2480
2481 /** Calculate avg element size if not initial sync **/
2482 uint64_t avg_element_size = 0;
2483 if (0 < op->local_element_count)
2484 {
2485 op->total_elements_size_local = 0;
2486 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2487 &
2488 determinate_avg_element_size_iterator,
2489 op);
2490 avg_element_size = op->total_elements_size_local / op->local_element_count;
2491 }
2492
2493 op->mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
2494 GNUNET_CONTAINER_multihashmap_size (
2495 op->set->content->
2496 elements),
2497 op->
2498 remote_element_count,
2499 diff_remote,
2500 diff_local,
2501 op->
2502 rtt_bandwidth_tradeoff,
2503 op->
2504 ibf_bucket_number_factor);
2505
2506#if MEASURE_PERFORMANCE
2507 perf_store.se_diff_local = diff_local;
2508 perf_store.se_diff_remote = diff_remote;
2509 perf_store.se_diff = diff;
2510 perf_store.mode_of_operation = op->mode_of_operation;
2511#endif
2512
2513 strata_estimator_destroy (remote_se);
2514 strata_estimator_destroy (op->se);
2515 op->se = NULL;
2516 LOG (GNUNET_ERROR_TYPE_DEBUG,
2517 "got se diff=%d, using ibf size %d\n",
2518 diff,
2519 1U << get_size_from_difference (diff, op->ibf_number_buckets_per_element,
2520 op->ibf_bucket_number_factor));
2521
2522 {
2523 char *set_debug;
2524
2525 set_debug = getenv ("GNUNET_SETU_BENCHMARK");
2526 if ((NULL != set_debug) &&
2527 (0 == strcmp (set_debug, "1")))
2528 {
2529 FILE *f = fopen ("set.log", "a");
2530 fprintf (f, "%llu\n", (unsigned long long) diff);
2531 fclose (f);
2532 }
2533 }
2534
2535 if ((GNUNET_YES == op->byzantine) &&
2536 (other_size < op->byzantine_lower_bound))
2537 {
2538 GNUNET_break (0);
2539 fail_union_operation (op);
2540 return;
2541 }
2542
2543 if ((GNUNET_YES == op->force_full) ||
2544 (op->mode_of_operation != DIFFERENTIAL_SYNC))
2545 {
2546 LOG (GNUNET_ERROR_TYPE_DEBUG,
2547 "Deciding to go for full set transmission (diff=%d, own set=%llu)\n",
2548 diff,
2549 (unsigned long long) op->initial_size);
2550 GNUNET_STATISTICS_update (_GSS_statistics,
2551 "# of full sends",
2552 1,
2553 GNUNET_NO);
2554 if (FULL_SYNC_LOCAL_SENDING_FIRST == op->mode_of_operation)
2555 {
2556 struct TransmitFullMessage *signal_msg;
2557 struct GNUNET_MQ_Envelope *ev;
2558 ev = GNUNET_MQ_msg_extra (signal_msg,sizeof(struct TransmitFullMessage),
2559 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL);
2560 signal_msg->remote_set_difference = htonl (diff_local);
2561 signal_msg->remote_set_size = htonl (op->local_element_count);
2562 signal_msg->local_set_difference = htonl (diff_remote);
2563 GNUNET_MQ_send (op->mq,
2564 ev);
2565 send_full_set (op);
2566 }
2567 else
2568 {
2569 struct GNUNET_MQ_Envelope *ev;
2570
2571 LOG (GNUNET_ERROR_TYPE_DEBUG,
2572 "Telling other peer that we expect its full set\n");
2573 op->phase = PHASE_FULL_RECEIVING;
2574#if MEASURE_PERFORMANCE
2575 perf_store.request_full.sent += 1;
2576#endif
2577 struct TransmitFullMessage *signal_msg;
2578 ev = GNUNET_MQ_msg_extra (signal_msg,sizeof(struct TransmitFullMessage),
2579 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL);
2580 signal_msg->remote_set_difference = htonl (diff_local);
2581 signal_msg->remote_set_size = htonl (op->local_element_count);
2582 signal_msg->local_set_difference = htonl (diff_remote);
2583 GNUNET_MQ_send (op->mq,
2584 ev);
2585 }
2586 }
2587 else
2588 {
2589 GNUNET_STATISTICS_update (_GSS_statistics,
2590 "# of ibf sends",
2591 1,
2592 GNUNET_NO);
2593 if (GNUNET_OK !=
2594 send_ibf (op,
2595 get_size_from_difference (diff,
2596 op->ibf_number_buckets_per_element,
2597 op->ibf_bucket_number_factor)))
2598 {
2599 /* Internal error, best we can do is shut the connection */
2600 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2601 "Failed to send IBF, closing connection\n");
2602 fail_union_operation (op);
2603 return;
2604 }
2605 }
2606 GNUNET_CADET_receive_done (op->channel);
2607}
2608
2609
2610/**
2611 * Iterator to send elements to a remote peer
2612 *
2613 * @param cls closure with the element key and the union operation
2614 * @param key ignored
2615 * @param value the key entry
2616 */
2617static int
2618send_offers_iterator (void *cls,
2619 uint32_t key,
2620 void *value)
2621{
2622 struct SendElementClosure *sec = cls;
2623 struct Operation *op = sec->op;
2624 struct KeyEntry *ke = value;
2625 struct GNUNET_MQ_Envelope *ev;
2626 struct GNUNET_MessageHeader *mh;
2627
2628 /* Detect 32-bit key collision for the 64-bit IBF keys. */
2629 if (ke->ibf_key.key_val != sec->ibf_key.key_val)
2630 {
2631 op->active_passive_switch_required = true;
2632 return GNUNET_YES;
2633 }
2634
2635 /* Prevent implementation from sending a offer multiple times in case of roll switch */
2636 if (GNUNET_YES ==
2637 is_message_in_message_control_flow (
2638 op->message_control_flow,
2639 &ke->element->element_hash,
2640 OFFER_MESSAGE)
2641 )
2642 {
2643 LOG (GNUNET_ERROR_TYPE_DEBUG,
2644 "Skipping already sent processed element offer!\n");
2645 return GNUNET_YES;
2646 }
2647
2648 /* Save send offer message for message control */
2649 if (GNUNET_YES !=
2650 update_message_control_flow (
2651 op->message_control_flow,
2652 MSG_CFS_SENT,
2653 &ke->element->element_hash,
2654 OFFER_MESSAGE)
2655 )
2656 {
2657 LOG (GNUNET_ERROR_TYPE_ERROR,
2658 "Double offer message sent found!\n");
2659 GNUNET_break (0);
2660 fail_union_operation (op);
2661 return GNUNET_NO;
2662 }
2663 ;
2664
2665 /* Mark element to be expected to received */
2666 if (GNUNET_YES !=
2667 update_message_control_flow (
2668 op->message_control_flow,
2669 MSG_CFS_EXPECTED,
2670 &ke->element->element_hash,
2671 DEMAND_MESSAGE)
2672 )
2673 {
2674 LOG (GNUNET_ERROR_TYPE_ERROR,
2675 "Double demand received found!\n");
2676 GNUNET_break (0);
2677 fail_union_operation (op);
2678 return GNUNET_NO;
2679 }
2680 ;
2681#if MEASURE_PERFORMANCE
2682 perf_store.offer.sent += 1;
2683 perf_store.offer.sent_var_bytes += sizeof(struct GNUNET_HashCode);
2684#endif
2685 ev = GNUNET_MQ_msg_header_extra (mh,
2686 sizeof(struct GNUNET_HashCode),
2687 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER);
2688 GNUNET_assert (NULL != ev);
2689 *(struct GNUNET_HashCode *) &mh[1] = ke->element->element_hash;
2690 LOG (GNUNET_ERROR_TYPE_DEBUG,
2691 "[OP %p] sending element offer (%s) to peer\n",
2692 op,
2693 GNUNET_h2s (&ke->element->element_hash));
2694 GNUNET_MQ_send (op->mq, ev);
2695 return GNUNET_YES;
2696}
2697
2698
2699/**
2700 * Send offers (in the form of GNUNET_Hash-es) to the remote peer for the given IBF key.
2701 *
2702 * @param op union operation
2703 * @param ibf_key IBF key of interest
2704 */
2705void
2706send_offers_for_key (struct Operation *op,
2707 struct IBF_Key ibf_key)
2708{
2709 struct SendElementClosure send_cls;
2710
2711 send_cls.ibf_key = ibf_key;
2712 send_cls.op = op;
2713 (void) GNUNET_CONTAINER_multihashmap32_get_multiple (
2714 op->key_to_element,
2715 (uint32_t) ibf_key.
2716 key_val,
2717 &send_offers_iterator,
2718 &send_cls);
2719}
2720
2721
2722/**
2723 * Decode which elements are missing on each side, and
2724 * send the appropriate offers and inquiries.
2725 *
2726 * @param op union operation
2727 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2728 */
2729static int
2730decode_and_send (struct Operation *op)
2731{
2732 struct IBF_Key key;
2733 struct IBF_Key last_key;
2734 int side;
2735 unsigned int num_decoded;
2736 struct InvertibleBloomFilter *diff_ibf;
2737
2738 GNUNET_assert (PHASE_ACTIVE_DECODING == op->phase);
2739
2740 if (GNUNET_OK !=
2741 prepare_ibf (op,
2742 op->remote_ibf->size))
2743 {
2744 GNUNET_break (0);
2745 /* allocation failed */
2746 return GNUNET_SYSERR;
2747 }
2748
2749 diff_ibf = ibf_dup (op->local_ibf);
2750 ibf_subtract (diff_ibf,
2751 op->remote_ibf);
2752
2753 ibf_destroy (op->remote_ibf);
2754 op->remote_ibf = NULL;
2755
2756 LOG (GNUNET_ERROR_TYPE_DEBUG,
2757 "decoding IBF (size=%u)\n",
2758 diff_ibf->size);
2759
2760 num_decoded = 0;
2761 key.key_val = 0; /* just to avoid compiler thinking we use undef'ed variable */
2762
2763 while (1)
2764 {
2765 int res;
2766 int cycle_detected = GNUNET_NO;
2767
2768 last_key = key;
2769
2770 res = ibf_decode (diff_ibf,
2771 &side,
2772 &key);
2773 if (res == GNUNET_OK)
2774 {
2775 LOG (GNUNET_ERROR_TYPE_DEBUG,
2776 "decoded ibf key %lx\n",
2777 (unsigned long) key.key_val);
2778 num_decoded += 1;
2779 if ((num_decoded > diff_ibf->size) ||
2780 ((num_decoded > 1) &&
2781 (last_key.key_val == key.key_val)))
2782 {
2783 LOG (GNUNET_ERROR_TYPE_DEBUG,
2784 "detected cyclic ibf (decoded %u/%u)\n",
2785 num_decoded,
2786 diff_ibf->size);
2787 cycle_detected = GNUNET_YES;
2788 }
2789 }
2790 if ((GNUNET_SYSERR == res) ||
2791 (GNUNET_YES == cycle_detected))
2792 {
2793 uint32_t next_size;
2794 /** Enforce odd ibf size **/
2795
2796 next_size = get_next_ibf_size (op->ibf_bucket_number_factor, num_decoded,
2797 diff_ibf->size);
2798 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2799 * Elias Summermatter (2021) in section 3.11 **/
2800 uint32_t ibf_min_size = IBF_MIN_SIZE | 1;
2801
2802 if (next_size<ibf_min_size)
2803 next_size = ibf_min_size;
2804
2805
2806 if (next_size <= MAX_IBF_SIZE)
2807 {
2808 LOG (GNUNET_ERROR_TYPE_DEBUG,
2809 "decoding failed, sending larger ibf (size %u)\n",
2810 next_size);
2811 GNUNET_STATISTICS_update (_GSS_statistics,
2812 "# of IBF retries",
2813 1,
2814 GNUNET_NO);
2815#if MEASURE_PERFORMANCE
2816 perf_store.active_passive_switches += 1;
2817#endif
2818
2819 op->salt_send = op->salt_receive++;
2820
2821 if (GNUNET_OK !=
2822 send_ibf (op, next_size))
2823 {
2824 /* Internal error, best we can do is shut the connection */
2825 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2826 "Failed to send IBF, closing connection\n");
2827 fail_union_operation (op);
2828 ibf_destroy (diff_ibf);
2829 return GNUNET_SYSERR;
2830 }
2831 }
2832 else
2833 {
2834 GNUNET_STATISTICS_update (_GSS_statistics,
2835 "# of failed union operations (too large)",
2836 1,
2837 GNUNET_NO);
2838 // XXX: Send the whole set, element-by-element
2839 LOG (GNUNET_ERROR_TYPE_ERROR,
2840 "set union failed: reached ibf limit\n");
2841 fail_union_operation (op);
2842 ibf_destroy (diff_ibf);
2843 return GNUNET_SYSERR;
2844 }
2845 break;
2846 }
2847 if (GNUNET_NO == res)
2848 {
2849 struct GNUNET_MQ_Envelope *ev;
2850
2851 LOG (GNUNET_ERROR_TYPE_DEBUG,
2852 "transmitted all values, sending DONE\n");
2853
2854#if MEASURE_PERFORMANCE
2855 perf_store.done.sent += 1;
2856#endif
2857 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_DONE);
2858 GNUNET_MQ_send (op->mq, ev);
2859 /* We now wait until we get a DONE message back
2860 * and then wait for our MQ to be flushed and all our
2861 * demands be delivered. */
2862 break;
2863 }
2864 if (1 == side)
2865 {
2866 struct IBF_Key unsalted_key;
2867 unsalt_key (&key,
2868 op->salt_receive,
2869 &unsalted_key);
2870 send_offers_for_key (op,
2871 unsalted_key);
2872 }
2873 else if (-1 == side)
2874 {
2875 struct GNUNET_MQ_Envelope *ev;
2876 struct InquiryMessage *msg;
2877
2878#if MEASURE_PERFORMANCE
2879 perf_store.inquery.sent += 1;
2880 perf_store.inquery.sent_var_bytes += sizeof(struct IBF_Key);
2881#endif
2882
2883 /** Add sent inquiries to hashmap for flow control **/
2884 struct GNUNET_HashContext *hashed_key_context =
2885 GNUNET_CRYPTO_hash_context_start ();
2886 struct GNUNET_HashCode *hashed_key = (struct
2887 GNUNET_HashCode*) GNUNET_malloc (
2888 sizeof(struct GNUNET_HashCode));
2889 enum MESSAGE_CONTROL_FLOW_STATE mcfs = MSG_CFS_SENT;
2890 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
2891 &key,
2892 sizeof(struct IBF_Key));
2893 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
2894 hashed_key);
2895 GNUNET_CONTAINER_multihashmap_put (op->inquiries_sent,
2896 hashed_key,
2897 &mcfs,
2898 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE
2899 );
2900
2901 /* It may be nice to merge multiple requests, but with CADET's corking it is not worth
2902 * the effort additional complexity. */
2903 ev = GNUNET_MQ_msg_extra (msg,
2904 sizeof(struct IBF_Key),
2905 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY);
2906 msg->salt = htonl (op->salt_receive);
2907 GNUNET_memcpy (&msg[1],
2908 &key,
2909 sizeof(struct IBF_Key));
2910 LOG (GNUNET_ERROR_TYPE_DEBUG,
2911 "sending element inquiry for IBF key %lx\n",
2912 (unsigned long) key.key_val);
2913 GNUNET_MQ_send (op->mq, ev);
2914 }
2915 else
2916 {
2917 GNUNET_assert (0);
2918 }
2919 }
2920 ibf_destroy (diff_ibf);
2921 return GNUNET_OK;
2922}
2923
2924
2925/**
2926 * Check send full message received from other peer
2927 * @param cls
2928 * @param msg
2929 * @return
2930 */
2931
2932static int
2933check_union_p2p_send_full (void *cls,
2934 const struct TransmitFullMessage *msg)
2935{
2936 return GNUNET_OK;
2937}
2938
2939
2940/**
2941 * Handle send full message received from other peer
2942 *
2943 * @param cls
2944 * @param msg
2945 */
2946static void
2947handle_union_p2p_send_full (void *cls,
2948 const struct TransmitFullMessage *msg)
2949{
2950 struct Operation *op = cls;
2951
2952 /**
2953 * Check that the message is received only in supported phase
2954 */
2955 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF};
2956 if (GNUNET_OK !=
2957 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
2958 {
2959 GNUNET_break (0);
2960 fail_union_operation (op);
2961 return;
2962 }
2963
2964 /** write received values to operator**/
2965 op->remote_element_count = ntohl (msg->remote_set_size);
2966 op->remote_set_diff = ntohl (msg->remote_set_difference);
2967 op->local_set_diff = ntohl (msg->local_set_difference);
2968
2969 /** Check byzantine limits **/
2970 if (check_byzantine_bounds (op) != GNUNET_OK)
2971 {
2972 LOG (GNUNET_ERROR_TYPE_ERROR,
2973 "PROTOCOL VIOLATION: Parameters transmitted from other peer do not satisfie byzantine "
2974 "criteria\n");
2975 GNUNET_break_op (0);
2976 fail_union_operation (op);
2977 return;
2978 }
2979
2980 /** Calculate avg element size if not initial sync **/
2981 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
2982 op->set->content->elements);
2983 uint64_t avg_element_size = 0;
2984 if (0 < op->local_element_count)
2985 {
2986 op->total_elements_size_local = 0;
2987 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2988 &
2989 determinate_avg_element_size_iterator,
2990 op);
2991 avg_element_size = op->total_elements_size_local / op->local_element_count;
2992 }
2993
2994 /** Validate mode of operation **/
2995 int mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
2996 op->
2997 remote_element_count,
2998 op->
2999 local_element_count,
3000 op->local_set_diff,
3001 op->remote_set_diff,
3002 op->
3003 rtt_bandwidth_tradeoff,
3004 op->
3005 ibf_bucket_number_factor);
3006 if (FULL_SYNC_LOCAL_SENDING_FIRST != mode_of_operation)
3007 {
3008 LOG (GNUNET_ERROR_TYPE_ERROR,
3009 "PROTOCOL VIOLATION: Remote peer choose to send his full set first but correct mode would have been"
3010 " : %d\n", mode_of_operation);
3011 GNUNET_break_op (0);
3012 fail_union_operation (op);
3013 return;
3014 }
3015 op->phase = PHASE_FULL_RECEIVING;
3016}
3017
3018
3019/**
3020 * Check an IBF message from a remote peer.
3021 *
3022 * Reassemble the IBF from multiple pieces, and
3023 * process the whole IBF once possible.
3024 *
3025 * @param cls the union operation
3026 * @param msg the header of the message
3027 * @return #GNUNET_OK if @a msg is well-formed
3028 */
3029static int
3030check_union_p2p_ibf (void *cls,
3031 const struct IBFMessage *msg)
3032{
3033 struct Operation *op = cls;
3034 unsigned int buckets_in_message;
3035
3036 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
3037 / IBF_BUCKET_SIZE;
3038 if (0 == buckets_in_message)
3039 {
3040 GNUNET_break_op (0);
3041 return GNUNET_SYSERR;
3042 }
3043 if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message
3044 * IBF_BUCKET_SIZE)
3045 {
3046 GNUNET_break_op (0);
3047 return GNUNET_SYSERR;
3048 }
3049 if (op->phase == PHASE_EXPECT_IBF_LAST)
3050 {
3051 if (ntohl (msg->offset) != op->ibf_buckets_received)
3052 {
3053 GNUNET_break_op (0);
3054 return GNUNET_SYSERR;
3055 }
3056
3057 if (msg->ibf_size != op->remote_ibf->size)
3058 {
3059 GNUNET_break_op (0);
3060 return GNUNET_SYSERR;
3061 }
3062 if (ntohl (msg->salt) != op->salt_receive)
3063 {
3064 GNUNET_break_op (0);
3065 return GNUNET_SYSERR;
3066 }
3067 }
3068 else if ((op->phase != PHASE_PASSIVE_DECODING) &&
3069 (op->phase != PHASE_EXPECT_IBF))
3070 {
3071 GNUNET_break_op (0);
3072 return GNUNET_SYSERR;
3073 }
3074
3075 return GNUNET_OK;
3076}
3077
3078
3079/**
3080 * Handle an IBF message from a remote peer.
3081 *
3082 * Reassemble the IBF from multiple pieces, and
3083 * process the whole IBF once possible.
3084 *
3085 * @param cls the union operation
3086 * @param msg the header of the message
3087 */
3088static void
3089handle_union_p2p_ibf (void *cls,
3090 const struct IBFMessage *msg)
3091{
3092 struct Operation *op = cls;
3093 unsigned int buckets_in_message;
3094 /**
3095 * Check that the message is received only in supported phase
3096 */
3097 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF, PHASE_EXPECT_IBF_LAST,
3098 PHASE_PASSIVE_DECODING};
3099 if (GNUNET_OK !=
3100 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3101 {
3102 GNUNET_break (0);
3103 fail_union_operation (op);
3104 return;
3105 }
3106 op->differential_sync_iterations++;
3107 check_max_differential_rounds (op);
3108 op->active_passive_switch_required = false;
3109
3110#if MEASURE_PERFORMANCE
3111 perf_store.ibf.received += 1;
3112 perf_store.ibf.received_var_bytes += (ntohs (msg->header.size) - sizeof *msg);
3113#endif
3114
3115 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
3116 / IBF_BUCKET_SIZE;
3117 if ((op->phase == PHASE_PASSIVE_DECODING) ||
3118 (op->phase == PHASE_EXPECT_IBF))
3119 {
3120 op->phase = PHASE_EXPECT_IBF_LAST;
3121 GNUNET_assert (NULL == op->remote_ibf);
3122 LOG (GNUNET_ERROR_TYPE_DEBUG,
3123 "Creating new ibf of size %u\n",
3124 ntohl (msg->ibf_size));
3125 // op->remote_ibf = ibf_create (1 << msg->order, SE_IBF_HASH_NUM);
3126 op->remote_ibf = ibf_create (msg->ibf_size,
3127 ((uint8_t) op->ibf_number_buckets_per_element));
3128 op->salt_receive = ntohl (msg->salt);
3129 LOG (GNUNET_ERROR_TYPE_DEBUG,
3130 "Receiving new IBF with salt %u\n",
3131 op->salt_receive);
3132 if (NULL == op->remote_ibf)
3133 {
3134 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3135 "Failed to parse remote IBF, closing connection\n");
3136 fail_union_operation (op);
3137 return;
3138 }
3139 op->ibf_buckets_received = 0;
3140 if (0 != ntohl (msg->offset))
3141 {
3142 GNUNET_break_op (0);
3143 fail_union_operation (op);
3144 return;
3145 }
3146 }
3147 else
3148 {
3149 GNUNET_assert (op->phase == PHASE_EXPECT_IBF_LAST);
3150 LOG (GNUNET_ERROR_TYPE_DEBUG,
3151 "Received more of IBF\n");
3152 }
3153 GNUNET_assert (NULL != op->remote_ibf);
3154
3155 ibf_read_slice (&msg[1],
3156 op->ibf_buckets_received,
3157 buckets_in_message,
3158 op->remote_ibf, msg->ibf_counter_bit_length);
3159 op->ibf_buckets_received += buckets_in_message;
3160
3161 if (op->ibf_buckets_received == op->remote_ibf->size)
3162 {
3163 LOG (GNUNET_ERROR_TYPE_DEBUG,
3164 "received full ibf\n");
3165 op->phase = PHASE_ACTIVE_DECODING;
3166 if (GNUNET_OK !=
3167 decode_and_send (op))
3168 {
3169 /* Internal error, best we can do is shut down */
3170 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3171 "Failed to decode IBF, closing connection\n");
3172 fail_union_operation (op);
3173 return;
3174 }
3175 }
3176 GNUNET_CADET_receive_done (op->channel);
3177}
3178
3179
3180/**
3181 * Send a result message to the client indicating
3182 * that there is a new element.
3183 *
3184 * @param op union operation
3185 * @param element element to send
3186 * @param status status to send with the new element
3187 */
3188static void
3189send_client_element (struct Operation *op,
3190 const struct GNUNET_SETU_Element *element,
3191 enum GNUNET_SETU_Status status)
3192{
3193 struct GNUNET_MQ_Envelope *ev;
3194 struct GNUNET_SETU_ResultMessage *rm;
3195
3196 LOG (GNUNET_ERROR_TYPE_DEBUG,
3197 "sending element (size %u) to client\n",
3198 element->size);
3199 GNUNET_assert (0 != op->client_request_id);
3200 ev = GNUNET_MQ_msg_extra (rm,
3201 element->size,
3202 GNUNET_MESSAGE_TYPE_SETU_RESULT);
3203 if (NULL == ev)
3204 {
3205 GNUNET_MQ_discard (ev);
3206 GNUNET_break (0);
3207 return;
3208 }
3209 rm->result_status = htons (status);
3210 rm->request_id = htonl (op->client_request_id);
3211 rm->element_type = htons (element->element_type);
3212 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
3213 op->key_to_element));
3214 GNUNET_memcpy (&rm[1],
3215 element->data,
3216 element->size);
3217 GNUNET_MQ_send (op->set->cs->mq,
3218 ev);
3219}
3220
3221
3222/**
3223 * Tests if the operation is finished, and if so notify.
3224 *
3225 * @param op operation to check
3226 */
3227static void
3228maybe_finish (struct Operation *op)
3229{
3230 unsigned int num_demanded;
3231
3232 num_demanded = GNUNET_CONTAINER_multihashmap_size (
3233 op->demanded_hashes);
3234 int send_done = GNUNET_CONTAINER_multihashmap_iterate (
3235 op->message_control_flow,
3236 &
3237 determinate_done_message_iterator,
3238 op);
3239 if (PHASE_FINISH_WAITING == op->phase)
3240 {
3241 LOG (GNUNET_ERROR_TYPE_DEBUG,
3242 "In PHASE_FINISH_WAITING, pending %u demands -> %d\n",
3243 num_demanded, op->peer_site);
3244 if (-1 != send_done)
3245 {
3246 struct GNUNET_MQ_Envelope *ev;
3247
3248 op->phase = PHASE_FINISHED;
3249#if MEASURE_PERFORMANCE
3250 perf_store.done.sent += 1;
3251#endif
3252 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_DONE);
3253 GNUNET_MQ_send (op->mq,
3254 ev);
3255 /* We now wait until the other peer sends P2P_OVER
3256 * after it got all elements from us. */
3257 }
3258 }
3259 if (PHASE_FINISH_CLOSING == op->phase)
3260 {
3261 LOG (GNUNET_ERROR_TYPE_DEBUG,
3262 "In PHASE_FINISH_CLOSING, pending %u demands %d\n",
3263 num_demanded, op->peer_site);
3264 if (-1 != send_done)
3265 {
3266 op->phase = PHASE_FINISHED;
3267 send_client_done (op);
3268 _GSS_operation_destroy2 (op);
3269 }
3270 }
3271}
3272
3273
3274/**
3275 * Check an element message from a remote peer.
3276 *
3277 * @param cls the union operation
3278 * @param emsg the message
3279 */
3280static int
3281check_union_p2p_elements (void *cls,
3282 const struct GNUNET_SETU_ElementMessage *emsg)
3283{
3284 struct Operation *op = cls;
3285
3286 if (0 == GNUNET_CONTAINER_multihashmap_size (op->demanded_hashes))
3287 {
3288 GNUNET_break_op (0);
3289 return GNUNET_SYSERR;
3290 }
3291 return GNUNET_OK;
3292}
3293
3294
3295/**
3296 * Handle an element message from a remote peer.
3297 * Sent by the other peer either because we decoded an IBF and placed a demand,
3298 * or because the other peer switched to full set transmission.
3299 *
3300 * @param cls the union operation
3301 * @param emsg the message
3302 */
3303static void
3304handle_union_p2p_elements (void *cls,
3305 const struct GNUNET_SETU_ElementMessage *emsg)
3306{
3307 struct Operation *op = cls;
3308 struct ElementEntry *ee;
3309 struct KeyEntry *ke;
3310 uint16_t element_size;
3311
3312 /**
3313 * Check that the message is received only in supported phase
3314 */
3315 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING,
3316 PHASE_FINISH_WAITING, PHASE_FINISH_CLOSING};
3317 if (GNUNET_OK !=
3318 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3319 {
3320 GNUNET_break (0);
3321 fail_union_operation (op);
3322 return;
3323 }
3324
3325 element_size = ntohs (emsg->header.size) - sizeof(struct
3326 GNUNET_SETU_ElementMessage);
3327#if MEASURE_PERFORMANCE
3328 perf_store.element.received += 1;
3329 perf_store.element.received_var_bytes += element_size;
3330#endif
3331
3332 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
3333 GNUNET_memcpy (&ee[1],
3334 &emsg[1],
3335 element_size);
3336 ee->element.size = element_size;
3337 ee->element.data = &ee[1];
3338 ee->element.element_type = ntohs (emsg->element_type);
3339 ee->remote = GNUNET_YES;
3340 GNUNET_SETU_element_hash (&ee->element,
3341 &ee->element_hash);
3342 if (GNUNET_NO ==
3343 GNUNET_CONTAINER_multihashmap_remove (op->demanded_hashes,
3344 &ee->element_hash,
3345 NULL))
3346 {
3347 /* We got something we didn't demand, since it's not in our map. */
3348 GNUNET_break_op (0);
3349 fail_union_operation (op);
3350 return;
3351 }
3352
3353 if (GNUNET_OK !=
3354 update_message_control_flow (
3355 op->message_control_flow,
3356 MSG_CFS_RECEIVED,
3357 &ee->element_hash,
3358 ELEMENT_MESSAGE)
3359 )
3360 {
3361 LOG (GNUNET_ERROR_TYPE_ERROR,
3362 "An element has been received more than once!\n");
3363 GNUNET_break (0);
3364 fail_union_operation (op);
3365 return;
3366 }
3367
3368 LOG (GNUNET_ERROR_TYPE_DEBUG,
3369 "Got element (size %u, hash %s) from peer\n",
3370 (unsigned int) element_size,
3371 GNUNET_h2s (&ee->element_hash));
3372
3373 GNUNET_STATISTICS_update (_GSS_statistics,
3374 "# received elements",
3375 1,
3376 GNUNET_NO);
3377 GNUNET_STATISTICS_update (_GSS_statistics,
3378 "# exchanged elements",
3379 1,
3380 GNUNET_NO);
3381
3382 op->received_total++;
3383
3384 ke = op_get_element (op,
3385 &ee->element_hash);
3386 if (NULL != ke)
3387 {
3388 /* Got repeated element. Should not happen since
3389 * we track demands. */
3390 GNUNET_STATISTICS_update (_GSS_statistics,
3391 "# repeated elements",
3392 1,
3393 GNUNET_NO);
3394 ke->received = GNUNET_YES;
3395 GNUNET_free (ee);
3396 }
3397 else
3398 {
3399 LOG (GNUNET_ERROR_TYPE_DEBUG,
3400 "Registering new element from remote peer\n");
3401 op->received_fresh++;
3402 op_register_element (op, ee, GNUNET_YES);
3403 /* only send results immediately if the client wants it */
3404 send_client_element (op,
3405 &ee->element,
3406 GNUNET_SETU_STATUS_ADD_LOCAL);
3407 }
3408
3409 if ((op->received_total > 8) &&
3410 (op->received_fresh < op->received_total / 3))
3411 {
3412 /* The other peer gave us lots of old elements, there's something wrong. */
3413 GNUNET_break_op (0);
3414 fail_union_operation (op);
3415 return;
3416 }
3417 GNUNET_CADET_receive_done (op->channel);
3418 maybe_finish (op);
3419}
3420
3421
3422/**
3423 * Check a full element message from a remote peer.
3424 *
3425 * @param cls the union operation
3426 * @param emsg the message
3427 */
3428static int
3429check_union_p2p_full_element (void *cls,
3430 const struct GNUNET_SETU_ElementMessage *emsg)
3431{
3432 struct Operation *op = cls;
3433
3434 (void) op;
3435
3436 // FIXME: check that we expect full elements here?
3437 return GNUNET_OK;
3438}
3439
3440
3441/**
3442 * Handle an element message from a remote peer.
3443 *
3444 * @param cls the union operation
3445 * @param emsg the message
3446 */
3447static void
3448handle_union_p2p_full_element (void *cls,
3449 const struct GNUNET_SETU_ElementMessage *emsg)
3450{
3451 struct Operation *op = cls;
3452 struct ElementEntry *ee;
3453 struct KeyEntry *ke;
3454 uint16_t element_size;
3455
3456 /**
3457 * Check that the message is received only in supported phase
3458 */
3459 uint8_t allowed_phases[] = {PHASE_FULL_RECEIVING, PHASE_FULL_SENDING};
3460 if (GNUNET_OK !=
3461 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3462 {
3463 GNUNET_break (0);
3464 fail_union_operation (op);
3465 return;
3466 }
3467
3468 element_size = ntohs (emsg->header.size)
3469 - sizeof(struct GNUNET_SETU_ElementMessage);
3470
3471#if MEASURE_PERFORMANCE
3472 perf_store.element_full.received += 1;
3473 perf_store.element_full.received_var_bytes += element_size;
3474#endif
3475
3476 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
3477 GNUNET_memcpy (&ee[1], &emsg[1], element_size);
3478 ee->element.size = element_size;
3479 ee->element.data = &ee[1];
3480 ee->element.element_type = ntohs (emsg->element_type);
3481 ee->remote = GNUNET_YES;
3482 GNUNET_SETU_element_hash (&ee->element,
3483 &ee->element_hash);
3484 LOG (GNUNET_ERROR_TYPE_DEBUG,
3485 "Got element (full diff, size %u, hash %s) from peer\n",
3486 (unsigned int) element_size,
3487 GNUNET_h2s (&ee->element_hash));
3488
3489 GNUNET_STATISTICS_update (_GSS_statistics,
3490 "# received elements",
3491 1,
3492 GNUNET_NO);
3493 GNUNET_STATISTICS_update (_GSS_statistics,
3494 "# exchanged elements",
3495 1,
3496 GNUNET_NO);
3497
3498 op->received_total++;
3499 ke = op_get_element (op,
3500 &ee->element_hash);
3501 if (NULL != ke)
3502 {
3503 GNUNET_STATISTICS_update (_GSS_statistics,
3504 "# repeated elements",
3505 1,
3506 GNUNET_NO);
3507 full_sync_plausibility_check (op);
3508 ke->received = GNUNET_YES;
3509 GNUNET_free (ee);
3510 }
3511 else
3512 {
3513 LOG (GNUNET_ERROR_TYPE_DEBUG,
3514 "Registering new element from remote peer\n");
3515 op->received_fresh++;
3516 op_register_element (op, ee, GNUNET_YES);
3517 /* only send results immediately if the client wants it */
3518 send_client_element (op,
3519 &ee->element,
3520 GNUNET_SETU_STATUS_ADD_LOCAL);
3521 }
3522
3523
3524 if ((GNUNET_YES == op->byzantine) &&
3525 (op->received_total > op->remote_element_count) )
3526 {
3527 /* The other peer gave us lots of old elements, there's something wrong. */
3528 LOG (GNUNET_ERROR_TYPE_ERROR,
3529 "Other peer sent %llu elements while pretending to have %llu elements, failing operation\n",
3530 (unsigned long long) op->received_total,
3531 (unsigned long long) op->remote_element_count);
3532 GNUNET_break_op (0);
3533 fail_union_operation (op);
3534 return;
3535 }
3536 GNUNET_CADET_receive_done (op->channel);
3537}
3538
3539
3540/**
3541 * Send offers (for GNUNET_Hash-es) in response
3542 * to inquiries (for IBF_Key-s).
3543 *
3544 * @param cls the union operation
3545 * @param msg the message
3546 */
3547static int
3548check_union_p2p_inquiry (void *cls,
3549 const struct InquiryMessage *msg)
3550{
3551 struct Operation *op = cls;
3552 unsigned int num_keys;
3553
3554 if (op->phase != PHASE_PASSIVE_DECODING)
3555 {
3556 GNUNET_break_op (0);
3557 return GNUNET_SYSERR;
3558 }
3559 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3560 / sizeof(struct IBF_Key);
3561 if ((ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3562 != num_keys * sizeof(struct IBF_Key))
3563 {
3564 GNUNET_break_op (0);
3565 return GNUNET_SYSERR;
3566 }
3567 return GNUNET_OK;
3568}
3569
3570
3571/**
3572 * Send offers (for GNUNET_Hash-es) in response to inquiries (for IBF_Key-s).
3573 *
3574 * @param cls the union operation
3575 * @param msg the message
3576 */
3577static void
3578handle_union_p2p_inquiry (void *cls,
3579 const struct InquiryMessage *msg)
3580{
3581 struct Operation *op = cls;
3582 const struct IBF_Key *ibf_key;
3583 unsigned int num_keys;
3584
3585 /**
3586 * Check that the message is received only in supported phase
3587 */
3588 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
3589 if (GNUNET_OK !=
3590 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3591 {
3592 GNUNET_break (0);
3593 fail_union_operation (op);
3594 return;
3595 }
3596
3597#if MEASURE_PERFORMANCE
3598 perf_store.inquery.received += 1;
3599 perf_store.inquery.received_var_bytes += (ntohs (msg->header.size)
3600 - sizeof(struct InquiryMessage));
3601#endif
3602
3603 LOG (GNUNET_ERROR_TYPE_DEBUG,
3604 "Received union inquiry\n");
3605 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3606 / sizeof(struct IBF_Key);
3607 ibf_key = (const struct IBF_Key *) &msg[1];
3608
3609 /** Add received inquiries to hashmap for flow control **/
3610 struct GNUNET_HashContext *hashed_key_context =
3611 GNUNET_CRYPTO_hash_context_start ();
3612 struct GNUNET_HashCode *hashed_key = (struct GNUNET_HashCode*) GNUNET_malloc (
3613 sizeof(struct GNUNET_HashCode));;
3614 enum MESSAGE_CONTROL_FLOW_STATE mcfs = MSG_CFS_RECEIVED;
3615 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
3616 &ibf_key,
3617 sizeof(struct IBF_Key));
3618 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
3619 hashed_key);
3620 GNUNET_CONTAINER_multihashmap_put (op->inquiries_sent,
3621 hashed_key,
3622 &mcfs,
3623 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE
3624 );
3625
3626 while (0 != num_keys--)
3627 {
3628 struct IBF_Key unsalted_key;
3629 unsalt_key (ibf_key,
3630 ntohl (msg->salt),
3631 &unsalted_key);
3632 send_offers_for_key (op,
3633 unsalted_key);
3634 ibf_key++;
3635 }
3636 GNUNET_CADET_receive_done (op->channel);
3637}
3638
3639
3640/**
3641 * Iterator over hash map entries, called to destroy the linked list of
3642 * colliding ibf key entries.
3643 *
3644 * @param cls closure
3645 * @param key current key code
3646 * @param value value in the hash map
3647 * @return #GNUNET_YES if we should continue to iterate,
3648 * #GNUNET_NO if not.
3649 */
3650static int
3651send_missing_full_elements_iter (void *cls,
3652 uint32_t key,
3653 void *value)
3654{
3655 struct Operation *op = cls;
3656 struct KeyEntry *ke = value;
3657 struct GNUNET_MQ_Envelope *ev;
3658 struct GNUNET_SETU_ElementMessage *emsg;
3659 struct ElementEntry *ee = ke->element;
3660
3661 if (GNUNET_YES == ke->received)
3662 return GNUNET_YES;
3663#if MEASURE_PERFORMANCE
3664 perf_store.element_full.received += 1;
3665#endif
3666 ev = GNUNET_MQ_msg_extra (emsg,
3667 ee->element.size,
3668 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT);
3669 GNUNET_memcpy (&emsg[1],
3670 ee->element.data,
3671 ee->element.size);
3672 emsg->element_type = htons (ee->element.element_type);
3673 GNUNET_MQ_send (op->mq,
3674 ev);
3675 return GNUNET_YES;
3676}
3677
3678
3679/**
3680 * Handle a request for full set transmission.
3681 *
3682 * @param cls closure, a set union operation
3683 * @param mh the demand message
3684 */
3685static int
3686check_union_p2p_request_full (void *cls,
3687 const struct TransmitFullMessage *mh)
3688{
3689 return GNUNET_OK;
3690}
3691
3692
3693static void
3694handle_union_p2p_request_full (void *cls,
3695 const struct TransmitFullMessage *msg)
3696{
3697 struct Operation *op = cls;
3698
3699 /**
3700 * Check that the message is received only in supported phase
3701 */
3702 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF};
3703 if (GNUNET_OK !=
3704 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3705 {
3706 GNUNET_break (0);
3707 fail_union_operation (op);
3708 return;
3709 }
3710
3711 op->remote_element_count = ntohl (msg->remote_set_size);
3712 op->remote_set_diff = ntohl (msg->remote_set_difference);
3713 op->local_set_diff = ntohl (msg->local_set_difference);
3714
3715
3716 if (check_byzantine_bounds (op) != GNUNET_OK)
3717 {
3718 LOG (GNUNET_ERROR_TYPE_ERROR,
3719 "PROTOCOL VIOLATION: Parameters transmitted from other peer do not satisfie byzantine "
3720 "criteria\n");
3721 GNUNET_break_op (0);
3722 fail_union_operation (op);
3723 return;
3724 }
3725
3726#if MEASURE_PERFORMANCE
3727 perf_store.request_full.received += 1;
3728#endif
3729
3730 LOG (GNUNET_ERROR_TYPE_DEBUG,
3731 "Received request for full set transmission\n");
3732
3733 /** Calculate avg element size if not initial sync **/
3734 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
3735 op->set->content->elements);
3736 uint64_t avg_element_size = 0;
3737 if (0 < op->local_element_count)
3738 {
3739 op->total_elements_size_local = 0;
3740 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
3741 &
3742 determinate_avg_element_size_iterator,
3743 op);
3744 avg_element_size = op->total_elements_size_local / op->local_element_count;
3745 }
3746
3747 int mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
3748 op->
3749 remote_element_count,
3750 op->
3751 local_element_count,
3752 op->local_set_diff,
3753 op->remote_set_diff,
3754 op->
3755 rtt_bandwidth_tradeoff,
3756 op->
3757 ibf_bucket_number_factor);
3758 if (FULL_SYNC_REMOTE_SENDING_FIRST != mode_of_operation)
3759 {
3760 LOG (GNUNET_ERROR_TYPE_ERROR,
3761 "PROTOCOL VIOLATION: Remote peer choose to request the full set first but correct mode would have been"
3762 " : %d\n", mode_of_operation);
3763 GNUNET_break_op (0);
3764 fail_union_operation (op);
3765 return;
3766 }
3767
3768 // FIXME: we need to check that our set is larger than the
3769 // byzantine_lower_bound by some threshold
3770 send_full_set (op);
3771 GNUNET_CADET_receive_done (op->channel);
3772}
3773
3774
3775/**
3776 * Handle a "full done" message.
3777 *
3778 * @param cls closure, a set union operation
3779 * @param mh the demand message
3780 */
3781static void
3782handle_union_p2p_full_done (void *cls,
3783 const struct GNUNET_MessageHeader *mh)
3784{
3785 struct Operation *op = cls;
3786
3787 /**
3788 * Check that the message is received only in supported phase
3789 */
3790 uint8_t allowed_phases[] = {PHASE_FULL_SENDING, PHASE_FULL_RECEIVING};
3791 if (GNUNET_OK !=
3792 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3793 {
3794 GNUNET_break (0);
3795 fail_union_operation (op);
3796 return;
3797 }
3798
3799#if MEASURE_PERFORMANCE
3800 perf_store.full_done.received += 1;
3801#endif
3802
3803 switch (op->phase)
3804 {
3805 case PHASE_FULL_RECEIVING:
3806 {
3807 struct GNUNET_MQ_Envelope *ev;
3808
3809 if ((GNUNET_YES == op->byzantine) &&
3810 (op->received_total != op->remote_element_count) )
3811 {
3812 /* The other peer gave not enough elements before sending full done, there's something wrong. */
3813 LOG (GNUNET_ERROR_TYPE_ERROR,
3814 "Other peer sent only %llu/%llu fresh elements, failing operation\n",
3815 (unsigned long long) op->received_total,
3816 (unsigned long long) op->remote_element_count);
3817 GNUNET_break_op (0);
3818 fail_union_operation (op);
3819 return;
3820 }
3821
3822 LOG (GNUNET_ERROR_TYPE_DEBUG,
3823 "got FULL DONE, sending elements that other peer is missing\n");
3824
3825 /* send all the elements that did not come from the remote peer */
3826 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
3827 &send_missing_full_elements_iter,
3828 op);
3829#if MEASURE_PERFORMANCE
3830 perf_store.full_done.sent += 1;
3831#endif
3832 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE);
3833 GNUNET_MQ_send (op->mq,
3834 ev);
3835 op->phase = PHASE_FINISHED;
3836 /* we now wait until the other peer sends us the OVER message*/
3837 }
3838 break;
3839
3840 case PHASE_FULL_SENDING:
3841 {
3842 LOG (GNUNET_ERROR_TYPE_DEBUG,
3843 "got FULL DONE, finishing\n");
3844 /* We sent the full set, and got the response for that. We're done. */
3845 op->phase = PHASE_FINISHED;
3846 GNUNET_CADET_receive_done (op->channel);
3847 send_client_done (op);
3848 _GSS_operation_destroy2 (op);
3849 return;
3850 }
3851
3852 default:
3853 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3854 "Handle full done phase is %u\n",
3855 (unsigned) op->phase);
3856 GNUNET_break_op (0);
3857 fail_union_operation (op);
3858 return;
3859 }
3860 GNUNET_CADET_receive_done (op->channel);
3861}
3862
3863
3864/**
3865 * Check a demand by the other peer for elements based on a list
3866 * of `struct GNUNET_HashCode`s.
3867 *
3868 * @param cls closure, a set union operation
3869 * @param mh the demand message
3870 * @return #GNUNET_OK if @a mh is well-formed
3871 */
3872static int
3873check_union_p2p_demand (void *cls,
3874 const struct GNUNET_MessageHeader *mh)
3875{
3876 struct Operation *op = cls;
3877 unsigned int num_hashes;
3878
3879 (void) op;
3880 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3881 / sizeof(struct GNUNET_HashCode);
3882 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3883 != num_hashes * sizeof(struct GNUNET_HashCode))
3884 {
3885 GNUNET_break_op (0);
3886 return GNUNET_SYSERR;
3887 }
3888 return GNUNET_OK;
3889}
3890
3891
3892/**
3893 * Handle a demand by the other peer for elements based on a list
3894 * of `struct GNUNET_HashCode`s.
3895 *
3896 * @param cls closure, a set union operation
3897 * @param mh the demand message
3898 */
3899static void
3900handle_union_p2p_demand (void *cls,
3901 const struct GNUNET_MessageHeader *mh)
3902{
3903 struct Operation *op = cls;
3904 struct ElementEntry *ee;
3905 struct GNUNET_SETU_ElementMessage *emsg;
3906 const struct GNUNET_HashCode *hash;
3907 unsigned int num_hashes;
3908 struct GNUNET_MQ_Envelope *ev;
3909
3910 /**
3911 * Check that the message is received only in supported phase
3912 */
3913 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING,
3914 PHASE_FINISH_WAITING};
3915 if (GNUNET_OK !=
3916 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3917 {
3918 GNUNET_break (0);
3919 fail_union_operation (op);
3920 return;
3921 }
3922#if MEASURE_PERFORMANCE
3923 perf_store.demand.received += 1;
3924 perf_store.demand.received_var_bytes += (ntohs (mh->size) - sizeof(struct
3925 GNUNET_MessageHeader));
3926#endif
3927
3928 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3929 / sizeof(struct GNUNET_HashCode);
3930 for (hash = (const struct GNUNET_HashCode *) &mh[1];
3931 num_hashes > 0;
3932 hash++, num_hashes--)
3933 {
3934 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
3935 hash);
3936 if (NULL == ee)
3937 {
3938 /* Demand for non-existing element. */
3939 GNUNET_break_op (0);
3940 fail_union_operation (op);
3941 return;
3942 }
3943
3944 /* Save send demand message for message control */
3945 if (GNUNET_YES !=
3946 update_message_control_flow (
3947 op->message_control_flow,
3948 MSG_CFS_RECEIVED,
3949 &ee->element_hash,
3950 DEMAND_MESSAGE)
3951 )
3952 {
3953 LOG (GNUNET_ERROR_TYPE_ERROR,
3954 "Double demand message received found!\n");
3955 GNUNET_break (0);
3956 fail_union_operation (op);
3957 return;
3958 }
3959 ;
3960
3961 /* Mark element to be expected to received */
3962 if (GNUNET_YES !=
3963 update_message_control_flow (
3964 op->message_control_flow,
3965 MSG_CFS_SENT,
3966 &ee->element_hash,
3967 ELEMENT_MESSAGE)
3968 )
3969 {
3970 LOG (GNUNET_ERROR_TYPE_ERROR,
3971 "Double element message sent found!\n");
3972 GNUNET_break (0);
3973 fail_union_operation (op);
3974 return;
3975 }
3976 if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
3977 {
3978 /* Probably confused lazily copied sets. */
3979 GNUNET_break_op (0);
3980 fail_union_operation (op);
3981 return;
3982 }
3983#if MEASURE_PERFORMANCE
3984 perf_store.element.sent += 1;
3985 perf_store.element.sent_var_bytes += ee->element.size;
3986#endif
3987 ev = GNUNET_MQ_msg_extra (emsg,
3988 ee->element.size,
3989 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS);
3990 GNUNET_memcpy (&emsg[1],
3991 ee->element.data,
3992 ee->element.size);
3993 emsg->reserved = htons (0);
3994 emsg->element_type = htons (ee->element.element_type);
3995 LOG (GNUNET_ERROR_TYPE_DEBUG,
3996 "[OP %p] Sending demanded element (size %u, hash %s) to peer\n",
3997 op,
3998 (unsigned int) ee->element.size,
3999 GNUNET_h2s (&ee->element_hash));
4000 GNUNET_MQ_send (op->mq, ev);
4001 GNUNET_STATISTICS_update (_GSS_statistics,
4002 "# exchanged elements",
4003 1,
4004 GNUNET_NO);
4005 if (op->symmetric)
4006 send_client_element (op,
4007 &ee->element,
4008 GNUNET_SETU_STATUS_ADD_REMOTE);
4009 }
4010 GNUNET_CADET_receive_done (op->channel);
4011 maybe_finish (op);
4012}
4013
4014
4015/**
4016 * Check offer (of `struct GNUNET_HashCode`s).
4017 *
4018 * @param cls the union operation
4019 * @param mh the message
4020 * @return #GNUNET_OK if @a mh is well-formed
4021 */
4022static int
4023check_union_p2p_offer (void *cls,
4024 const struct GNUNET_MessageHeader *mh)
4025{
4026 struct Operation *op = cls;
4027 unsigned int num_hashes;
4028
4029 /* look up elements and send them */
4030 if ((op->phase != PHASE_PASSIVE_DECODING) &&
4031 (op->phase != PHASE_ACTIVE_DECODING))
4032 {
4033 GNUNET_break_op (0);
4034 return GNUNET_SYSERR;
4035 }
4036 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
4037 / sizeof(struct GNUNET_HashCode);
4038 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader)) !=
4039 num_hashes * sizeof(struct GNUNET_HashCode))
4040 {
4041 GNUNET_break_op (0);
4042 return GNUNET_SYSERR;
4043 }
4044 return GNUNET_OK;
4045}
4046
4047
4048/**
4049 * Handle offers (of `struct GNUNET_HashCode`s) and
4050 * respond with demands (of `struct GNUNET_HashCode`s).
4051 *
4052 * @param cls the union operation
4053 * @param mh the message
4054 */
4055static void
4056handle_union_p2p_offer (void *cls,
4057 const struct GNUNET_MessageHeader *mh)
4058{
4059 struct Operation *op = cls;
4060 const struct GNUNET_HashCode *hash;
4061 unsigned int num_hashes;
4062 /**
4063 * Check that the message is received only in supported phase
4064 */
4065 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
4066 if (GNUNET_OK !=
4067 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
4068 {
4069 GNUNET_break (0);
4070 fail_union_operation (op);
4071 return;
4072 }
4073
4074#if MEASURE_PERFORMANCE
4075 perf_store.offer.received += 1;
4076 perf_store.offer.received_var_bytes += (ntohs (mh->size) - sizeof(struct
4077 GNUNET_MessageHeader));
4078#endif
4079
4080 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
4081 / sizeof(struct GNUNET_HashCode);
4082 for (hash = (const struct GNUNET_HashCode *) &mh[1];
4083 num_hashes > 0;
4084 hash++, num_hashes--)
4085 {
4086 struct ElementEntry *ee;
4087 struct GNUNET_MessageHeader *demands;
4088 struct GNUNET_MQ_Envelope *ev;
4089
4090 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
4091 hash);
4092 if (NULL != ee)
4093 if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
4094 continue;
4095
4096 if (GNUNET_YES ==
4097 GNUNET_CONTAINER_multihashmap_contains (op->demanded_hashes,
4098 hash))
4099 {
4100 LOG (GNUNET_ERROR_TYPE_DEBUG,
4101 "Skipped sending duplicate demand\n");
4102 continue;
4103 }
4104
4105 GNUNET_assert (GNUNET_OK ==
4106 GNUNET_CONTAINER_multihashmap_put (
4107 op->demanded_hashes,
4108 hash,
4109 NULL,
4110 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST));
4111
4112 LOG (GNUNET_ERROR_TYPE_DEBUG,
4113 "[OP %p] Requesting element (hash %s)\n",
4114 op, GNUNET_h2s (hash));
4115
4116#if MEASURE_PERFORMANCE
4117 perf_store.demand.sent += 1;
4118 perf_store.demand.sent_var_bytes += sizeof(struct GNUNET_HashCode);
4119#endif
4120 /* Save send demand message for message control */
4121 if (GNUNET_YES !=
4122 update_message_control_flow (
4123 op->message_control_flow,
4124 MSG_CFS_SENT,
4125 hash,
4126 DEMAND_MESSAGE))
4127 {
4128 LOG (GNUNET_ERROR_TYPE_ERROR,
4129 "Double demand message sent found!\n");
4130 GNUNET_break (0);
4131 fail_union_operation (op);
4132 return;
4133 }
4134
4135 /* Mark offer as received received */
4136 if (GNUNET_YES !=
4137 update_message_control_flow (
4138 op->message_control_flow,
4139 MSG_CFS_RECEIVED,
4140 hash,
4141 OFFER_MESSAGE))
4142 {
4143 LOG (GNUNET_ERROR_TYPE_ERROR,
4144 "Double offer message received found!\n");
4145 GNUNET_break (0);
4146 fail_union_operation (op);
4147 return;
4148 }
4149 /* Mark element to be expected to received */
4150 if (GNUNET_YES !=
4151 update_message_control_flow (
4152 op->message_control_flow,
4153 MSG_CFS_EXPECTED,
4154 hash,
4155 ELEMENT_MESSAGE))
4156 {
4157 LOG (GNUNET_ERROR_TYPE_ERROR,
4158 "Element already expected!\n");
4159 GNUNET_break (0);
4160 fail_union_operation (op);
4161 return;
4162 }
4163 ev = GNUNET_MQ_msg_header_extra (demands,
4164 sizeof(struct GNUNET_HashCode),
4165 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND);
4166 GNUNET_memcpy (&demands[1],
4167 hash,
4168 sizeof(struct GNUNET_HashCode));
4169 GNUNET_MQ_send (op->mq, ev);
4170 }
4171 GNUNET_CADET_receive_done (op->channel);
4172}
4173
4174
4175/**
4176 * Handle a done message from a remote peer
4177 *
4178 * @param cls the union operation
4179 * @param mh the message
4180 */
4181static void
4182handle_union_p2p_done (void *cls,
4183 const struct GNUNET_MessageHeader *mh)
4184{
4185 struct Operation *op = cls;
4186
4187 /**
4188 * Check that the message is received only in supported phase
4189 */
4190 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
4191 if (GNUNET_OK !=
4192 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
4193 {
4194 GNUNET_break (0);
4195 fail_union_operation (op);
4196 return;
4197 }
4198
4199 if (op->active_passive_switch_required)
4200 {
4201 LOG (GNUNET_ERROR_TYPE_ERROR,
4202 "PROTOCOL VIOLATION: Received done but role change is necessary\n");
4203 GNUNET_break (0);
4204 fail_union_operation (op);
4205 return;
4206 }
4207
4208#if MEASURE_PERFORMANCE
4209 perf_store.done.received += 1;
4210#endif
4211 switch (op->phase)
4212 {
4213 case PHASE_PASSIVE_DECODING:
4214 /* We got all requests, but still have to send our elements in response. */
4215 op->phase = PHASE_FINISH_WAITING;
4216 LOG (GNUNET_ERROR_TYPE_DEBUG,
4217 "got DONE (as passive partner), waiting for our demands to be satisfied\n");
4218 /* The active peer is done sending offers
4219 * and inquiries. This means that all
4220 * our responses to that (demands and offers)
4221 * must be in flight (queued or in mesh).
4222 *
4223 * We should notify the active peer once
4224 * all our demands are satisfied, so that the active
4225 * peer can quit if we gave it everything.
4226 */GNUNET_CADET_receive_done (op->channel);
4227 maybe_finish (op);
4228 return;
4229 case PHASE_ACTIVE_DECODING:
4230 LOG (GNUNET_ERROR_TYPE_DEBUG,
4231 "got DONE (as active partner), waiting to finish\n");
4232 /* All demands of the other peer are satisfied,
4233 * and we processed all offers, thus we know
4234 * exactly what our demands must be.
4235 *
4236 * We'll close the channel
4237 * to the other peer once our demands are met.
4238 */op->phase = PHASE_FINISH_CLOSING;
4239 GNUNET_CADET_receive_done (op->channel);
4240 maybe_finish (op);
4241 return;
4242 default:
4243 GNUNET_break_op (0);
4244 fail_union_operation (op);
4245 return;
4246 }
4247}
4248
4249
4250/**
4251 * Handle a over message from a remote peer
4252 *
4253 * @param cls the union operation
4254 * @param mh the message
4255 */
4256static void
4257handle_union_p2p_over (void *cls,
4258 const struct GNUNET_MessageHeader *mh)
4259{
4260#if MEASURE_PERFORMANCE
4261 perf_store.over.received += 1;
4262#endif
4263 send_client_done (cls);
4264}
4265
4266
4267/**
4268 * Get the incoming socket associated with the given id.
4269 *
4270 * @param listener the listener to look in
4271 * @param id id to look for
4272 * @return the incoming socket associated with the id,
4273 * or NULL if there is none
4274 */
4275static struct Operation *
4276get_incoming (uint32_t id)
4277{
4278 for (struct Listener *listener = listener_head;
4279 NULL != listener;
4280 listener = listener->next)
4281 {
4282 for (struct Operation *op = listener->op_head;
4283 NULL != op;
4284 op = op->next)
4285 if (op->suggest_id == id)
4286 return op;
4287 }
4288 return NULL;
4289}
4290
4291
4292/**
4293 * Callback called when a client connects to the service.
4294 *
4295 * @param cls closure for the service
4296 * @param c the new client that connected to the service
4297 * @param mq the message queue used to send messages to the client
4298 * @return @a `struct ClientState`
4299 */
4300static void *
4301client_connect_cb (void *cls,
4302 struct GNUNET_SERVICE_Client *c,
4303 struct GNUNET_MQ_Handle *mq)
4304{
4305 struct ClientState *cs;
4306
4307 num_clients++;
4308 cs = GNUNET_new (struct ClientState);
4309 cs->client = c;
4310 cs->mq = mq;
4311 return cs;
4312}
4313
4314
4315/**
4316 * Iterator over hash map entries to free element entries.
4317 *
4318 * @param cls closure
4319 * @param key current key code
4320 * @param value a `struct ElementEntry *` to be free'd
4321 * @return #GNUNET_YES (continue to iterate)
4322 */
4323static int
4324destroy_elements_iterator (void *cls,
4325 const struct GNUNET_HashCode *key,
4326 void *value)
4327{
4328 struct ElementEntry *ee = value;
4329
4330 GNUNET_free (ee);
4331 return GNUNET_YES;
4332}
4333
4334
4335/**
4336 * Clean up after a client has disconnected
4337 *
4338 * @param cls closure, unused
4339 * @param client the client to clean up after
4340 * @param internal_cls the `struct ClientState`
4341 */
4342static void
4343client_disconnect_cb (void *cls,
4344 struct GNUNET_SERVICE_Client *client,
4345 void *internal_cls)
4346{
4347 struct ClientState *cs = internal_cls;
4348 struct Operation *op;
4349 struct Listener *listener;
4350 struct Set *set;
4351
4352 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4353 "Client disconnected, cleaning up\n");
4354 if (NULL != (set = cs->set))
4355 {
4356 struct SetContent *content = set->content;
4357
4358 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4359 "Destroying client's set\n");
4360 /* Destroy pending set operations */
4361 while (NULL != set->ops_head)
4362 _GSS_operation_destroy (set->ops_head);
4363
4364 /* Destroy operation-specific state */
4365 if (NULL != set->se)
4366 {
4367 strata_estimator_destroy (set->se);
4368 set->se = NULL;
4369 }
4370 /* free set content (or at least decrement RC) */
4371 set->content = NULL;
4372 GNUNET_assert (0 != content->refcount);
4373 content->refcount--;
4374 if (0 == content->refcount)
4375 {
4376 GNUNET_assert (NULL != content->elements);
4377 GNUNET_CONTAINER_multihashmap_iterate (content->elements,
4378 &destroy_elements_iterator,
4379 NULL);
4380 GNUNET_CONTAINER_multihashmap_destroy (content->elements);
4381 content->elements = NULL;
4382 GNUNET_free (content);
4383 }
4384 GNUNET_free (set);
4385 }
4386
4387 if (NULL != (listener = cs->listener))
4388 {
4389 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4390 "Destroying client's listener\n");
4391 GNUNET_CADET_close_port (listener->open_port);
4392 listener->open_port = NULL;
4393 while (NULL != (op = listener->op_head))
4394 {
4395 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
4396 "Destroying incoming operation `%u' from peer `%s'\n",
4397 (unsigned int) op->client_request_id,
4398 GNUNET_i2s (&op->peer));
4399 incoming_destroy (op);
4400 }
4401 GNUNET_CONTAINER_DLL_remove (listener_head,
4402 listener_tail,
4403 listener);
4404 GNUNET_free (listener);
4405 }
4406 GNUNET_free (cs);
4407 num_clients--;
4408 if ( (GNUNET_YES == in_shutdown) &&
4409 (0 == num_clients) )
4410 {
4411 if (NULL != cadet)
4412 {
4413 GNUNET_CADET_disconnect (cadet);
4414 cadet = NULL;
4415 }
4416 }
4417}
4418
4419
4420/**
4421 * Check a request for a set operation from another peer.
4422 *
4423 * @param cls the operation state
4424 * @param msg the received message
4425 * @return #GNUNET_OK if the channel should be kept alive,
4426 * #GNUNET_SYSERR to destroy the channel
4427 */
4428static int
4429check_incoming_msg (void *cls,
4430 const struct OperationRequestMessage *msg)
4431{
4432 struct Operation *op = cls;
4433 struct Listener *listener = op->listener;
4434 const struct GNUNET_MessageHeader *nested_context;
4435
4436 /* double operation request */
4437 if (0 != op->suggest_id)
4438 {
4439 GNUNET_break_op (0);
4440 return GNUNET_SYSERR;
4441 }
4442 /* This should be equivalent to the previous condition, but can't hurt to check twice */
4443 if (NULL == listener)
4444 {
4445 GNUNET_break (0);
4446 return GNUNET_SYSERR;
4447 }
4448 nested_context = GNUNET_MQ_extract_nested_mh (msg);
4449 if ((NULL != nested_context) &&
4450 (ntohs (nested_context->size) > GNUNET_SETU_CONTEXT_MESSAGE_MAX_SIZE))
4451 {
4452 GNUNET_break_op (0);
4453 return GNUNET_SYSERR;
4454 }
4455 return GNUNET_OK;
4456}
4457
4458
4459/**
4460 * Handle a request for a set operation from another peer. Checks if we
4461 * have a listener waiting for such a request (and in that case initiates
4462 * asking the listener about accepting the connection). If no listener
4463 * is waiting, we queue the operation request in hope that a listener
4464 * shows up soon (before timeout).
4465 *
4466 * This msg is expected as the first and only msg handled through the
4467 * non-operation bound virtual table, acceptance of this operation replaces
4468 * our virtual table and subsequent msgs would be routed differently (as
4469 * we then know what type of operation this is).
4470 *
4471 * @param cls the operation state
4472 * @param msg the received message
4473 */
4474static void
4475handle_incoming_msg (void *cls,
4476 const struct OperationRequestMessage *msg)
4477{
4478 struct Operation *op = cls;
4479 struct Listener *listener = op->listener;
4480 const struct GNUNET_MessageHeader *nested_context;
4481 struct GNUNET_MQ_Envelope *env;
4482 struct GNUNET_SETU_RequestMessage *cmsg;
4483
4484 nested_context = GNUNET_MQ_extract_nested_mh (msg);
4485 /* Make a copy of the nested_context (application-specific context
4486 information that is opaque to set) so we can pass it to the
4487 listener later on */
4488 if (NULL != nested_context)
4489 op->context_msg = GNUNET_copy_message (nested_context);
4490 op->remote_element_count = ntohl (msg->element_count);
4491 GNUNET_log (
4492 GNUNET_ERROR_TYPE_DEBUG,
4493 "Received P2P operation request (port %s) for active listener\n",
4494 GNUNET_h2s (&op->listener->app_id));
4495 GNUNET_assert (0 == op->suggest_id);
4496 if (0 == suggest_id)
4497 suggest_id++;
4498 op->suggest_id = suggest_id++;
4499 GNUNET_assert (NULL != op->timeout_task);
4500 GNUNET_SCHEDULER_cancel (op->timeout_task);
4501 op->timeout_task = NULL;
4502 env = GNUNET_MQ_msg_nested_mh (cmsg,
4503 GNUNET_MESSAGE_TYPE_SETU_REQUEST,
4504 op->context_msg);
4505 GNUNET_log (
4506 GNUNET_ERROR_TYPE_DEBUG,
4507 "Suggesting incoming request with accept id %u to listener %p of client %p\n",
4508 op->suggest_id,
4509 listener,
4510 listener->cs);
4511 cmsg->accept_id = htonl (op->suggest_id);
4512 cmsg->peer_id = op->peer;
4513 GNUNET_MQ_send (listener->cs->mq,
4514 env);
4515 /* NOTE: GNUNET_CADET_receive_done() will be called in
4516 #handle_client_accept() */
4517}
4518
4519
4520/**
4521 * Called when a client wants to create a new set. This is typically
4522 * the first request from a client, and includes the type of set
4523 * operation to be performed.
4524 *
4525 * @param cls client that sent the message
4526 * @param m message sent by the client
4527 */
4528static void
4529handle_client_create_set (void *cls,
4530 const struct GNUNET_SETU_CreateMessage *msg)
4531{
4532 struct ClientState *cs = cls;
4533 struct Set *set;
4534
4535 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4536 "Client created new set for union operation\n");
4537 if (NULL != cs->set)
4538 {
4539 /* There can only be one set per client */
4540 GNUNET_break (0);
4541 GNUNET_SERVICE_client_drop (cs->client);
4542 return;
4543 }
4544 set = GNUNET_new (struct Set);
4545 {
4546 struct MultiStrataEstimator *se;
4547
4548 se = strata_estimator_create (SE_STRATA_COUNT,
4549 SE_IBFS_TOTAL_SIZE,
4550 SE_IBF_HASH_NUM);
4551 if (NULL == se)
4552 {
4553 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
4554 "Failed to allocate strata estimator\n");
4555 GNUNET_free (set);
4556 GNUNET_SERVICE_client_drop (cs->client);
4557 return;
4558 }
4559 set->se = se;
4560 }
4561 set->content = GNUNET_new (struct SetContent);
4562 set->content->refcount = 1;
4563 set->content->elements = GNUNET_CONTAINER_multihashmap_create (1,
4564 GNUNET_YES);
4565 set->cs = cs;
4566 cs->set = set;
4567 GNUNET_SERVICE_client_continue (cs->client);
4568}
4569
4570
4571/**
4572 * Timeout happens iff:
4573 * - we suggested an operation to our listener,
4574 * but did not receive a response in time
4575 * - we got the channel from a peer but no #GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST
4576 *
4577 * @param cls channel context
4578 * @param tc context information (why was this task triggered now)
4579 */
4580static void
4581incoming_timeout_cb (void *cls)
4582{
4583 struct Operation *op = cls;
4584
4585 op->timeout_task = NULL;
4586 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4587 "Remote peer's incoming request timed out\n");
4588 incoming_destroy (op);
4589}
4590
4591
4592/**
4593 * Method called whenever another peer has added us to a channel the
4594 * other peer initiated. Only called (once) upon reception of data
4595 * from a channel we listen on.
4596 *
4597 * The channel context represents the operation itself and gets added
4598 * to a DLL, from where it gets looked up when our local listener
4599 * client responds to a proposed/suggested operation or connects and
4600 * associates with this operation.
4601 *
4602 * @param cls closure
4603 * @param channel new handle to the channel
4604 * @param source peer that started the channel
4605 * @return initial channel context for the channel
4606 * returns NULL on error
4607 */
4608static void *
4609channel_new_cb (void *cls,
4610 struct GNUNET_CADET_Channel *channel,
4611 const struct GNUNET_PeerIdentity *source)
4612{
4613 struct Listener *listener = cls;
4614 struct Operation *op;
4615
4616 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4617 "New incoming channel\n");
4618 op = GNUNET_new (struct Operation);
4619 op->listener = listener;
4620 op->peer = *source;
4621 op->channel = channel;
4622 op->mq = GNUNET_CADET_get_mq (op->channel);
4623 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
4624 UINT32_MAX);
4625 op->timeout_task = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
4626 &incoming_timeout_cb,
4627 op);
4628 GNUNET_CONTAINER_DLL_insert (listener->op_head,
4629 listener->op_tail,
4630 op);
4631 return op;
4632}
4633
4634
4635/**
4636 * Function called whenever a channel is destroyed. Should clean up
4637 * any associated state. It must NOT call
4638 * GNUNET_CADET_channel_destroy() on the channel.
4639 *
4640 * The peer_disconnect function is part of a a virtual table set initially either
4641 * when a peer creates a new channel with us, or once we create
4642 * a new channel ourselves (evaluate).
4643 *
4644 * Once we know the exact type of operation (union/intersection), the vt is
4645 * replaced with an operation specific instance (_GSS_[op]_vt).
4646 *
4647 * @param channel_ctx place where local state associated
4648 * with the channel is stored
4649 * @param channel connection to the other end (henceforth invalid)
4650 */
4651static void
4652channel_end_cb (void *channel_ctx,
4653 const struct GNUNET_CADET_Channel *channel)
4654{
4655 struct Operation *op = channel_ctx;
4656
4657 op->channel = NULL;
4658 _GSS_operation_destroy2 (op);
4659}
4660
4661
4662/**
4663 * Function called whenever an MQ-channel's transmission window size changes.
4664 *
4665 * The first callback in an outgoing channel will be with a non-zero value
4666 * and will mean the channel is connected to the destination.
4667 *
4668 * For an incoming channel it will be called immediately after the
4669 * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
4670 *
4671 * @param cls Channel closure.
4672 * @param channel Connection to the other end (henceforth invalid).
4673 * @param window_size New window size. If the is more messages than buffer size
4674 * this value will be negative..
4675 */
4676static void
4677channel_window_cb (void *cls,
4678 const struct GNUNET_CADET_Channel *channel,
4679 int window_size)
4680{
4681 /* FIXME: not implemented, we could do flow control here... */
4682}
4683
4684
4685/**
4686 * Called when a client wants to create a new listener.
4687 *
4688 * @param cls client that sent the message
4689 * @param msg message sent by the client
4690 */
4691
4692static void
4693handle_client_listen (void *cls,
4694 const struct GNUNET_SETU_ListenMessage *msg)
4695{
4696 struct ClientState *cs = cls;
4697 struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
4698 GNUNET_MQ_hd_var_size (incoming_msg,
4699 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
4700 struct OperationRequestMessage,
4701 NULL),
4702 GNUNET_MQ_hd_var_size (union_p2p_ibf,
4703 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF,
4704 struct IBFMessage,
4705 NULL),
4706 GNUNET_MQ_hd_var_size (union_p2p_elements,
4707 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS,
4708 struct GNUNET_SETU_ElementMessage,
4709 NULL),
4710 GNUNET_MQ_hd_var_size (union_p2p_offer,
4711 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER,
4712 struct GNUNET_MessageHeader,
4713 NULL),
4714 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
4715 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY,
4716 struct InquiryMessage,
4717 NULL),
4718 GNUNET_MQ_hd_var_size (union_p2p_demand,
4719 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND,
4720 struct GNUNET_MessageHeader,
4721 NULL),
4722 GNUNET_MQ_hd_fixed_size (union_p2p_done,
4723 GNUNET_MESSAGE_TYPE_SETU_P2P_DONE,
4724 struct GNUNET_MessageHeader,
4725 NULL),
4726 GNUNET_MQ_hd_fixed_size (union_p2p_over,
4727 GNUNET_MESSAGE_TYPE_SETU_P2P_OVER,
4728 struct GNUNET_MessageHeader,
4729 NULL),
4730 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
4731 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE,
4732 struct GNUNET_MessageHeader,
4733 NULL),
4734 GNUNET_MQ_hd_var_size (union_p2p_request_full,
4735 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL,
4736 struct TransmitFullMessage,
4737 NULL),
4738 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4739 GNUNET_MESSAGE_TYPE_SETU_P2P_SE,
4740 struct StrataEstimatorMessage,
4741 NULL),
4742 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4743 GNUNET_MESSAGE_TYPE_SETU_P2P_SEC,
4744 struct StrataEstimatorMessage,
4745 NULL),
4746 GNUNET_MQ_hd_var_size (union_p2p_full_element,
4747 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT,
4748 struct GNUNET_SETU_ElementMessage,
4749 NULL),
4750 GNUNET_MQ_hd_var_size (union_p2p_send_full,
4751 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL,
4752 struct TransmitFullMessage,
4753 NULL),
4754 GNUNET_MQ_handler_end ()
4755 };
4756 struct Listener *listener;
4757
4758 if (NULL != cs->listener)
4759 {
4760 /* max. one active listener per client! */
4761 GNUNET_break (0);
4762 GNUNET_SERVICE_client_drop (cs->client);
4763 return;
4764 }
4765 listener = GNUNET_new (struct Listener);
4766 listener->cs = cs;
4767 cs->listener = listener;
4768 listener->app_id = msg->app_id;
4769 GNUNET_CONTAINER_DLL_insert (listener_head,
4770 listener_tail,
4771 listener);
4772 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4773 "New listener created (port %s)\n",
4774 GNUNET_h2s (&listener->app_id));
4775 listener->open_port = GNUNET_CADET_open_port (cadet,
4776 &msg->app_id,
4777 &channel_new_cb,
4778 listener,
4779 &channel_window_cb,
4780 &channel_end_cb,
4781 cadet_handlers);
4782 GNUNET_SERVICE_client_continue (cs->client);
4783}
4784
4785
4786/**
4787 * Called when the listening client rejects an operation
4788 * request by another peer.
4789 *
4790 * @param cls client that sent the message
4791 * @param msg message sent by the client
4792 */
4793static void
4794handle_client_reject (void *cls,
4795 const struct GNUNET_SETU_RejectMessage *msg)
4796{
4797 struct ClientState *cs = cls;
4798 struct Operation *op;
4799
4800 op = get_incoming (ntohl (msg->accept_reject_id));
4801 if (NULL == op)
4802 {
4803 /* no matching incoming operation for this reject;
4804 could be that the other peer already disconnected... */
4805 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
4806 "Client rejected unknown operation %u\n",
4807 (unsigned int) ntohl (msg->accept_reject_id));
4808 GNUNET_SERVICE_client_continue (cs->client);
4809 return;
4810 }
4811 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4812 "Peer request (app %s) rejected by client\n",
4813 GNUNET_h2s (&cs->listener->app_id));
4814 _GSS_operation_destroy2 (op);
4815 GNUNET_SERVICE_client_continue (cs->client);
4816}
4817
4818
4819/**
4820 * Called when a client wants to add or remove an element to a set it inhabits.
4821 *
4822 * @param cls client that sent the message
4823 * @param msg message sent by the client
4824 */
4825static int
4826check_client_set_add (void *cls,
4827 const struct GNUNET_SETU_ElementMessage *msg)
4828{
4829 /* NOTE: Technically, we should probably check with the
4830 block library whether the element we are given is well-formed */
4831 return GNUNET_OK;
4832}
4833
4834
4835/**
4836 * Called when a client wants to add or remove an element to a set it inhabits.
4837 *
4838 * @param cls client that sent the message
4839 * @param msg message sent by the client
4840 */
4841static void
4842handle_client_set_add (void *cls,
4843 const struct GNUNET_SETU_ElementMessage *msg)
4844{
4845 struct ClientState *cs = cls;
4846 struct Set *set;
4847 struct GNUNET_SETU_Element el;
4848 struct ElementEntry *ee;
4849 struct GNUNET_HashCode hash;
4850
4851 if (NULL == (set = cs->set))
4852 {
4853 /* client without a set requested an operation */
4854 GNUNET_break (0);
4855 GNUNET_SERVICE_client_drop (cs->client);
4856 return;
4857 }
4858 GNUNET_SERVICE_client_continue (cs->client);
4859 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing mutation on set\n");
4860 el.size = ntohs (msg->header.size) - sizeof(*msg);
4861 el.data = &msg[1];
4862 el.element_type = ntohs (msg->element_type);
4863 GNUNET_SETU_element_hash (&el,
4864 &hash);
4865 ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
4866 &hash);
4867 if (NULL == ee)
4868 {
4869 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4870 "Client inserts element %s of size %u\n",
4871 GNUNET_h2s (&hash),
4872 el.size);
4873 ee = GNUNET_malloc (el.size + sizeof(*ee));
4874 ee->element.size = el.size;
4875 GNUNET_memcpy (&ee[1], el.data, el.size);
4876 ee->element.data = &ee[1];
4877 ee->element.element_type = el.element_type;
4878 ee->remote = GNUNET_NO;
4879 ee->generation = set->current_generation;
4880 ee->element_hash = hash;
4881 GNUNET_break (GNUNET_YES ==
4882 GNUNET_CONTAINER_multihashmap_put (
4883 set->content->elements,
4884 &ee->element_hash,
4885 ee,
4886 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
4887 }
4888 else
4889 {
4890 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4891 "Client inserted element %s of size %u twice (ignored)\n",
4892 GNUNET_h2s (&hash),
4893 el.size);
4894 /* same element inserted twice */
4895 return;
4896 }
4897 strata_estimator_insert (set->se,
4898 get_ibf_key (&ee->element_hash));
4899}
4900
4901
4902/**
4903 * Advance the current generation of a set,
4904 * adding exclusion ranges if necessary.
4905 *
4906 * @param set the set where we want to advance the generation
4907 */
4908static void
4909advance_generation (struct Set *set)
4910{
4911 set->content->latest_generation++;
4912 set->current_generation++;
4913}
4914
4915
4916/**
4917 * Called when a client wants to initiate a set operation with another
4918 * peer. Initiates the CADET connection to the listener and sends the
4919 * request.
4920 *
4921 * @param cls client that sent the message
4922 * @param msg message sent by the client
4923 * @return #GNUNET_OK if the message is well-formed
4924 */
4925static int
4926check_client_evaluate (void *cls,
4927 const struct GNUNET_SETU_EvaluateMessage *msg)
4928{
4929 /* FIXME: suboptimal, even if the context below could be NULL,
4930 there are malformed messages this does not check for... */
4931 return GNUNET_OK;
4932}
4933
4934
4935/**
4936 * Called when a client wants to initiate a set operation with another
4937 * peer. Initiates the CADET connection to the listener and sends the
4938 * request.
4939 *
4940 * @param cls client that sent the message
4941 * @param msg message sent by the client
4942 */
4943static void
4944handle_client_evaluate (void *cls,
4945 const struct GNUNET_SETU_EvaluateMessage *msg)
4946{
4947 struct ClientState *cs = cls;
4948 struct Operation *op = GNUNET_new (struct Operation);
4949
4950 const struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
4951 GNUNET_MQ_hd_var_size (incoming_msg,
4952 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
4953 struct OperationRequestMessage,
4954 op),
4955 GNUNET_MQ_hd_var_size (union_p2p_ibf,
4956 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF,
4957 struct IBFMessage,
4958 op),
4959 GNUNET_MQ_hd_var_size (union_p2p_elements,
4960 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS,
4961 struct GNUNET_SETU_ElementMessage,
4962 op),
4963 GNUNET_MQ_hd_var_size (union_p2p_offer,
4964 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER,
4965 struct GNUNET_MessageHeader,
4966 op),
4967 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
4968 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY,
4969 struct InquiryMessage,
4970 op),
4971 GNUNET_MQ_hd_var_size (union_p2p_demand,
4972 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND,
4973 struct GNUNET_MessageHeader,
4974 op),
4975 GNUNET_MQ_hd_fixed_size (union_p2p_done,
4976 GNUNET_MESSAGE_TYPE_SETU_P2P_DONE,
4977 struct GNUNET_MessageHeader,
4978 op),
4979 GNUNET_MQ_hd_fixed_size (union_p2p_over,
4980 GNUNET_MESSAGE_TYPE_SETU_P2P_OVER,
4981 struct GNUNET_MessageHeader,
4982 op),
4983 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
4984 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE,
4985 struct GNUNET_MessageHeader,
4986 op),
4987 GNUNET_MQ_hd_var_size (union_p2p_request_full,
4988 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL,
4989 struct TransmitFullMessage,
4990 op),
4991 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4992 GNUNET_MESSAGE_TYPE_SETU_P2P_SE,
4993 struct StrataEstimatorMessage,
4994 op),
4995 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4996 GNUNET_MESSAGE_TYPE_SETU_P2P_SEC,
4997 struct StrataEstimatorMessage,
4998 op),
4999 GNUNET_MQ_hd_var_size (union_p2p_full_element,
5000 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT,
5001 struct GNUNET_SETU_ElementMessage,
5002 op),
5003 GNUNET_MQ_hd_var_size (union_p2p_send_full,
5004 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL,
5005 struct TransmitFullMessage,
5006 NULL),
5007 GNUNET_MQ_handler_end ()
5008 };
5009 struct Set *set;
5010 const struct GNUNET_MessageHeader *context;
5011
5012 if (NULL == (set = cs->set))
5013 {
5014 GNUNET_break (0);
5015 GNUNET_free (op);
5016 GNUNET_SERVICE_client_drop (cs->client);
5017 return;
5018 }
5019 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
5020 UINT32_MAX);
5021 op->peer = msg->target_peer;
5022 op->client_request_id = ntohl (msg->request_id);
5023 op->byzantine = msg->byzantine;
5024 op->byzantine_lower_bound = ntohl (msg->byzantine_lower_bound);
5025 op->force_full = msg->force_full;
5026 op->force_delta = msg->force_delta;
5027 op->symmetric = msg->symmetric;
5028 op->rtt_bandwidth_tradeoff = msg->bandwidth_latency_tradeoff;
5029 op->ibf_bucket_number_factor = msg->ibf_bucket_number_factor;
5030 op->ibf_number_buckets_per_element = msg->ibf_number_of_buckets_per_element;
5031 op->byzantine_upper_bound = msg->byzantine_upper_bond;
5032 op->active_passive_switch_required = false;
5033 context = GNUNET_MQ_extract_nested_mh (msg);
5034
5035 /* create hashmap for message control */
5036 op->message_control_flow = GNUNET_CONTAINER_multihashmap_create (32,
5037 GNUNET_NO);
5038 op->inquiries_sent = GNUNET_CONTAINER_multihashmap_create (32,GNUNET_NO);
5039
5040#if MEASURE_PERFORMANCE
5041 /* load config */
5042 load_config (op);
5043#endif
5044
5045 /* Advance generation values, so that
5046 mutations won't interfer with the running operation. */
5047 op->set = set;
5048 op->generation_created = set->current_generation;
5049 advance_generation (set);
5050 GNUNET_CONTAINER_DLL_insert (set->ops_head,
5051 set->ops_tail,
5052 op);
5053 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5054 "Creating new CADET channel to port %s for set union\n",
5055 GNUNET_h2s (&msg->app_id));
5056 op->channel = GNUNET_CADET_channel_create (cadet,
5057 op,
5058 &msg->target_peer,
5059 &msg->app_id,
5060 &channel_window_cb,
5061 &channel_end_cb,
5062 cadet_handlers);
5063 op->mq = GNUNET_CADET_get_mq (op->channel);
5064 {
5065 struct GNUNET_MQ_Envelope *ev;
5066 struct OperationRequestMessage *msg;
5067
5068#if MEASURE_PERFORMANCE
5069 perf_store.operation_request.sent += 1;
5070#endif
5071 ev = GNUNET_MQ_msg_nested_mh (msg,
5072 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
5073 context);
5074 if (NULL == ev)
5075 {
5076 /* the context message is too large */
5077 GNUNET_break (0);
5078 GNUNET_SERVICE_client_drop (cs->client);
5079 return;
5080 }
5081 op->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
5082 GNUNET_NO);
5083 /* copy the current generation's strata estimator for this operation */
5084 op->se = strata_estimator_dup (op->set->se);
5085 /* we started the operation, thus we have to send the operation request */
5086 op->phase = PHASE_EXPECT_SE;
5087
5088 op->salt_receive = (op->peer_site + 1) % 2;
5089 op->salt_send = op->peer_site; // FIXME?????
5090
5091
5092 LOG (GNUNET_ERROR_TYPE_DEBUG,
5093 "Initiating union operation evaluation\n");
5094 GNUNET_STATISTICS_update (_GSS_statistics,
5095 "# of total union operations",
5096 1,
5097 GNUNET_NO);
5098 GNUNET_STATISTICS_update (_GSS_statistics,
5099 "# of initiated union operations",
5100 1,
5101 GNUNET_NO);
5102 GNUNET_MQ_send (op->mq,
5103 ev);
5104 if (NULL != context)
5105 LOG (GNUNET_ERROR_TYPE_DEBUG,
5106 "sent op request with context message\n");
5107 else
5108 LOG (GNUNET_ERROR_TYPE_DEBUG,
5109 "sent op request without context message\n");
5110 initialize_key_to_element (op);
5111 op->initial_size = GNUNET_CONTAINER_multihashmap32_size (
5112 op->key_to_element);
5113
5114 }
5115 GNUNET_SERVICE_client_continue (cs->client);
5116}
5117
5118
5119/**
5120 * Handle a request from the client to cancel a running set operation.
5121 *
5122 * @param cls the client
5123 * @param msg the message
5124 */
5125static void
5126handle_client_cancel (void *cls,
5127 const struct GNUNET_SETU_CancelMessage *msg)
5128{
5129 struct ClientState *cs = cls;
5130 struct Set *set;
5131 struct Operation *op;
5132 int found;
5133
5134 if (NULL == (set = cs->set))
5135 {
5136 /* client without a set requested an operation */
5137 GNUNET_break (0);
5138 GNUNET_SERVICE_client_drop (cs->client);
5139 return;
5140 }
5141 found = GNUNET_NO;
5142 for (op = set->ops_head; NULL != op; op = op->next)
5143 {
5144 if (op->client_request_id == ntohl (msg->request_id))
5145 {
5146 found = GNUNET_YES;
5147 break;
5148 }
5149 }
5150 if (GNUNET_NO == found)
5151 {
5152 /* It may happen that the operation was already destroyed due to
5153 * the other peer disconnecting. The client may not know about this
5154 * yet and try to cancel the (just barely non-existent) operation.
5155 * So this is not a hard error.
5156 *///
5157 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
5158 "Client canceled non-existent op %u\n",
5159 (uint32_t) ntohl (msg->request_id));
5160 }
5161 else
5162 {
5163 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5164 "Client requested cancel for op %u\n",
5165 (uint32_t) ntohl (msg->request_id));
5166 _GSS_operation_destroy (op);
5167 }
5168 GNUNET_SERVICE_client_continue (cs->client);
5169}
5170
5171
5172/**
5173 * Handle a request from the client to accept a set operation that
5174 * came from a remote peer. We forward the accept to the associated
5175 * operation for handling
5176 *
5177 * @param cls the client
5178 * @param msg the message
5179 */
5180static void
5181handle_client_accept (void *cls,
5182 const struct GNUNET_SETU_AcceptMessage *msg)
5183{
5184 struct ClientState *cs = cls;
5185 struct Set *set;
5186 struct Operation *op;
5187 struct GNUNET_SETU_ResultMessage *result_message;
5188 struct GNUNET_MQ_Envelope *ev;
5189 struct Listener *listener;
5190
5191 if (NULL == (set = cs->set))
5192 {
5193 /* client without a set requested to accept */
5194 GNUNET_break (0);
5195 GNUNET_SERVICE_client_drop (cs->client);
5196 return;
5197 }
5198 op = get_incoming (ntohl (msg->accept_reject_id));
5199 if (NULL == op)
5200 {
5201 /* It is not an error if the set op does not exist -- it may
5202 * have been destroyed when the partner peer disconnected. */
5203 GNUNET_log (
5204 GNUNET_ERROR_TYPE_INFO,
5205 "Client %p accepted request %u of listener %p that is no longer active\n",
5206 cs,
5207 ntohl (msg->accept_reject_id),
5208 cs->listener);
5209 ev = GNUNET_MQ_msg (result_message,
5210 GNUNET_MESSAGE_TYPE_SETU_RESULT);
5211 result_message->request_id = msg->request_id;
5212 result_message->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
5213 GNUNET_MQ_send (set->cs->mq, ev);
5214 GNUNET_SERVICE_client_continue (cs->client);
5215 return;
5216 }
5217 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5218 "Client accepting request %u\n",
5219 (uint32_t) ntohl (msg->accept_reject_id));
5220 listener = op->listener;
5221 op->listener = NULL;
5222 GNUNET_CONTAINER_DLL_remove (listener->op_head,
5223 listener->op_tail,
5224 op);
5225 op->set = set;
5226 GNUNET_CONTAINER_DLL_insert (set->ops_head,
5227 set->ops_tail,
5228 op);
5229 op->client_request_id = ntohl (msg->request_id);
5230 op->byzantine = msg->byzantine;
5231 op->byzantine_lower_bound = ntohl (msg->byzantine_lower_bound);
5232 op->force_full = msg->force_full;
5233 op->force_delta = msg->force_delta;
5234 op->symmetric = msg->symmetric;
5235 op->rtt_bandwidth_tradeoff = msg->bandwidth_latency_tradeoff;
5236 op->ibf_bucket_number_factor = msg->ibf_bucket_number_factor;
5237 op->ibf_number_buckets_per_element = msg->ibf_number_of_buckets_per_element;
5238 op->byzantine_upper_bound = msg->byzantine_upper_bond;
5239 op->active_passive_switch_required = false;
5240 /* create hashmap for message control */
5241 op->message_control_flow = GNUNET_CONTAINER_multihashmap_create (32,
5242 GNUNET_NO);
5243 op->inquiries_sent = GNUNET_CONTAINER_multihashmap_create (32,GNUNET_NO);
5244
5245#if MEASURE_PERFORMANCE
5246 /* load config */
5247 load_config (op);
5248#endif
5249
5250 /* Advance generation values, so that future mutations do not
5251 interfer with the running operation. */
5252 op->generation_created = set->current_generation;
5253 advance_generation (set);
5254 GNUNET_assert (NULL == op->se);
5255
5256 LOG (GNUNET_ERROR_TYPE_DEBUG,
5257 "accepting set union operation\n");
5258 GNUNET_STATISTICS_update (_GSS_statistics,
5259 "# of accepted union operations",
5260 1,
5261 GNUNET_NO);
5262 GNUNET_STATISTICS_update (_GSS_statistics,
5263 "# of total union operations",
5264 1,
5265 GNUNET_NO);
5266 {
5267 struct MultiStrataEstimator *se;
5268 struct GNUNET_MQ_Envelope *ev;
5269 struct StrataEstimatorMessage *strata_msg;
5270 char *buf;
5271 size_t len;
5272 uint16_t type;
5273
5274 op->se = strata_estimator_dup (op->set->se);
5275 op->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
5276 GNUNET_NO);
5277 op->salt_receive = (op->peer_site + 1) % 2;
5278 op->salt_send = op->peer_site; // FIXME?????
5279 initialize_key_to_element (op);
5280 op->initial_size = GNUNET_CONTAINER_multihashmap32_size (
5281 op->key_to_element);
5282
5283 /* kick off the operation */
5284 se = op->se;
5285
5286 uint8_t se_count = 1;
5287 if (op->initial_size > 0)
5288 {
5289 op->total_elements_size_local = 0;
5290 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
5291 &
5292 determinate_avg_element_size_iterator,
5293 op);
5294 se_count = determine_strata_count (
5295 op->total_elements_size_local / op->initial_size,
5296 op->initial_size);
5297 }
5298 buf = GNUNET_malloc (se->stratas[0]->strata_count * IBF_BUCKET_SIZE
5299 * ((SE_IBFS_TOTAL_SIZE / 8) * se_count));
5300 len = strata_estimator_write (se,
5301 SE_IBFS_TOTAL_SIZE,
5302 se_count,
5303 buf);
5304#if MEASURE_PERFORMANCE
5305 perf_store.se.sent += 1;
5306 perf_store.se.sent_var_bytes += len;
5307#endif
5308
5309 if (len < se->stratas[0]->strata_count * IBF_BUCKET_SIZE
5310 * SE_IBFS_TOTAL_SIZE)
5311 type = GNUNET_MESSAGE_TYPE_SETU_P2P_SEC;
5312 else
5313 type = GNUNET_MESSAGE_TYPE_SETU_P2P_SE;
5314 ev = GNUNET_MQ_msg_extra (strata_msg,
5315 len,
5316 type);
5317 GNUNET_memcpy (&strata_msg[1],
5318 buf,
5319 len);
5320 GNUNET_free (buf);
5321 strata_msg->set_size
5322 = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (
5323 op->set->content->elements));
5324 strata_msg->se_count = se_count;
5325 GNUNET_MQ_send (op->mq,
5326 ev);
5327 op->phase = PHASE_EXPECT_IBF;
5328 }
5329 /* Now allow CADET to continue, as we did not do this in
5330 #handle_incoming_msg (as we wanted to first see if the
5331 local client would accept the request). */
5332 GNUNET_CADET_receive_done (op->channel);
5333 GNUNET_SERVICE_client_continue (cs->client);
5334}
5335
5336
5337/**
5338 * Called to clean up, after a shutdown has been requested.
5339 *
5340 * @param cls closure, NULL
5341 */
5342static void
5343shutdown_task (void *cls)
5344{
5345 /* Delay actual shutdown to allow service to disconnect clients */
5346 in_shutdown = GNUNET_YES;
5347 if (0 == num_clients)
5348 {
5349 if (NULL != cadet)
5350 {
5351 GNUNET_CADET_disconnect (cadet);
5352 cadet = NULL;
5353 }
5354 }
5355 GNUNET_STATISTICS_destroy (_GSS_statistics,
5356 GNUNET_YES);
5357 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5358 "handled shutdown request\n");
5359#if MEASURE_PERFORMANCE
5360 calculate_perf_store ();
5361#endif
5362}
5363
5364
5365/**
5366 * Function called by the service's run
5367 * method to run service-specific setup code.
5368 *
5369 * @param cls closure
5370 * @param cfg configuration to use
5371 * @param service the initialized service
5372 */
5373static void
5374run (void *cls,
5375 const struct GNUNET_CONFIGURATION_Handle *cfg,
5376 struct GNUNET_SERVICE_Handle *service)
5377{
5378 /* FIXME: need to modify SERVICE (!) API to allow
5379 us to run a shutdown task *after* clients were
5380 forcefully disconnected! */
5381 GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
5382 NULL);
5383 _GSS_statistics = GNUNET_STATISTICS_create ("setu",
5384 cfg);
5385 cadet = GNUNET_CADET_connect (cfg);
5386 if (NULL == cadet)
5387 {
5388 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
5389 _ ("Could not connect to CADET service\n"));
5390 GNUNET_SCHEDULER_shutdown ();
5391 return;
5392 }
5393}
5394
5395
5396/**
5397 * Define "main" method using service macro.
5398 */
5399GNUNET_SERVICE_MAIN (
5400 "set",
5401 GNUNET_SERVICE_OPTION_NONE,
5402 &run,
5403 &client_connect_cb,
5404 &client_disconnect_cb,
5405 NULL,
5406 GNUNET_MQ_hd_fixed_size (client_accept,
5407 GNUNET_MESSAGE_TYPE_SETU_ACCEPT,
5408 struct GNUNET_SETU_AcceptMessage,
5409 NULL),
5410 GNUNET_MQ_hd_var_size (client_set_add,
5411 GNUNET_MESSAGE_TYPE_SETU_ADD,
5412 struct GNUNET_SETU_ElementMessage,
5413 NULL),
5414 GNUNET_MQ_hd_fixed_size (client_create_set,
5415 GNUNET_MESSAGE_TYPE_SETU_CREATE,
5416 struct GNUNET_SETU_CreateMessage,
5417 NULL),
5418 GNUNET_MQ_hd_var_size (client_evaluate,
5419 GNUNET_MESSAGE_TYPE_SETU_EVALUATE,
5420 struct GNUNET_SETU_EvaluateMessage,
5421 NULL),
5422 GNUNET_MQ_hd_fixed_size (client_listen,
5423 GNUNET_MESSAGE_TYPE_SETU_LISTEN,
5424 struct GNUNET_SETU_ListenMessage,
5425 NULL),
5426 GNUNET_MQ_hd_fixed_size (client_reject,
5427 GNUNET_MESSAGE_TYPE_SETU_REJECT,
5428 struct GNUNET_SETU_RejectMessage,
5429 NULL),
5430 GNUNET_MQ_hd_fixed_size (client_cancel,
5431 GNUNET_MESSAGE_TYPE_SETU_CANCEL,
5432 struct GNUNET_SETU_CancelMessage,
5433 NULL),
5434 GNUNET_MQ_handler_end ());
5435
5436
5437/* end of gnunet-service-setu.c */
diff --git a/src/setu/gnunet-service-setu_protocol.h b/src/setu/gnunet-service-setu_protocol.h
deleted file mode 100644
index c2a166e60..000000000
--- a/src/setu/gnunet-service-setu_protocol.h
+++ /dev/null
@@ -1,256 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2013, 2014 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @author Florian Dold
22 * @author Christian Grothoff
23 * @file set/gnunet-service-set_protocol.h
24 * @brief Peer-to-Peer messages for gnunet set
25 */
26#ifndef SET_PROTOCOL_H
27#define SET_PROTOCOL_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35struct OperationRequestMessage
36{
37 /**
38 * Type: #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
39 */
40 struct GNUNET_MessageHeader header;
41
42 /**
43 * For Intersection: my element count
44 */
45 uint32_t element_count GNUNET_PACKED;
46
47 /**
48 * Application-specific identifier of the request.
49 */
50 struct GNUNET_HashCode app_idX;
51
52 /* rest: optional message */
53};
54
55
56/**
57 * Message containing buckets of an invertible bloom filter.
58 *
59 * If an IBF has too many buckets for an IBF message,
60 * it is split into multiple messages.
61 */
62struct IBFMessage
63{
64 /**
65 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
66 */
67 struct GNUNET_MessageHeader header;
68
69 /**
70 * Size of the whole ibf (number of buckets)
71 */
72 uint32_t ibf_size;
73
74 /**
75 * Offset of the strata in the rest of the message
76 */
77 uint32_t offset GNUNET_PACKED;
78
79 /**
80 * Salt used when hashing elements for this IBF.
81 */
82 uint16_t salt GNUNET_PACKED;
83
84 /**
85 * The bit length of the counter
86 */
87 uint16_t ibf_counter_bit_length;
88 /* rest: buckets */
89};
90
91
92/**
93estimate_best_mode_of_operation (uint64_t avg_element_size,
94uint64_t local_set_size,
95 uint64_t remote_set_size,
96uint64_t est_set_diff_remote,
97 uint64_t est_set_diff_local,)
98 **/
99
100
101struct InquiryMessage
102{
103 /**
104 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
105 */
106 struct GNUNET_MessageHeader header;
107
108 /**
109 * Salt used when hashing elements for this inquiry.
110 */
111 uint32_t salt GNUNET_PACKED;
112
113 /* rest: inquiry IBF keys */
114};
115
116
117/**
118 * During intersection, the first (and possibly second) message
119 * send it the number of elements in the set, to allow the peers
120 * to decide who should start with the Bloom filter.
121 */
122struct IntersectionElementInfoMessage
123{
124 /**
125 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO
126 */
127 struct GNUNET_MessageHeader header;
128
129 /**
130 * mutator used with this bloomfilter.
131 */
132 uint32_t sender_element_count GNUNET_PACKED;
133};
134
135
136/**
137 * Bloom filter messages exchanged for set intersection calculation.
138 */
139struct BFMessage
140{
141 /**
142 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF
143 */
144 struct GNUNET_MessageHeader header;
145
146 /**
147 * Number of elements the sender still has in the set.
148 */
149 uint32_t sender_element_count GNUNET_PACKED;
150
151 /**
152 * XOR of all hashes over all elements remaining in the set.
153 * Used to determine termination.
154 */
155 struct GNUNET_HashCode element_xor_hash;
156
157 /**
158 * Mutator used with this bloomfilter.
159 */
160 uint32_t sender_mutator GNUNET_PACKED;
161
162 /**
163 * Total length of the bloomfilter data.
164 */
165 uint32_t bloomfilter_total_length GNUNET_PACKED;
166
167 /**
168 * Number of bits (k-value) used in encoding the bloomfilter.
169 */
170 uint32_t bits_per_element GNUNET_PACKED;
171
172 /**
173 * rest: the sender's bloomfilter
174 */
175};
176
177
178/**
179 * Last message, send to confirm the final set. Contains the element
180 * count as it is possible that the peer determined that we were done
181 * by getting the empty set, which in that case also needs to be
182 * communicated.
183 */
184struct IntersectionDoneMessage
185{
186 /**
187 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE
188 */
189 struct GNUNET_MessageHeader header;
190
191 /**
192 * Final number of elements in intersection.
193 */
194 uint32_t final_element_count GNUNET_PACKED;
195
196 /**
197 * XOR of all hashes over all elements remaining in the set.
198 */
199 struct GNUNET_HashCode element_xor_hash;
200};
201
202
203/**
204 * Strata estimator together with the peer's overall set size.
205 */
206struct StrataEstimatorMessage
207{
208 /**
209 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE(C)
210 */
211 struct GNUNET_MessageHeader header;
212
213 /**
214 * The number of ses transmitted
215 */
216 uint8_t se_count;
217
218 /**
219 * Size of the local set
220 */
221 uint64_t set_size;
222};
223
224
225/**
226 * Message which signals to other peer that we are sending full set
227 *
228 */
229struct TransmitFullMessage
230{
231 /**
232 * Type: #GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL
233 */
234 struct GNUNET_MessageHeader header;
235
236 /**
237 * Remote set difference calculated with strata estimator
238 */
239 uint32_t remote_set_difference;
240
241 /**
242 * Total remote set size
243 */
244 uint32_t remote_set_size;
245
246 /**
247 * Local set difference calculated with strata estimator
248 */
249 uint32_t local_set_difference;
250
251};
252
253
254GNUNET_NETWORK_STRUCT_END
255
256#endif
diff --git a/src/setu/gnunet-service-setu_strata_estimator.c b/src/setu/gnunet-service-setu_strata_estimator.c
deleted file mode 100644
index 7981cc847..000000000
--- a/src/setu/gnunet-service-setu_strata_estimator.c
+++ /dev/null
@@ -1,467 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-setu_strata_estimator.c
22 * @brief invertible bloom filter
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "ibf.h"
30#include "gnunet-service-setu_strata_estimator.h"
31
32
33/**
34 * Should we try compressing the strata estimator? This will
35 * break compatibility with the 0.10.1-network.
36 */
37#define FAIL_10_1_COMPATIBILTIY 1
38
39/**
40 * Number of strata estimators in memory NOT transmitted
41 */
42
43#define MULTI_SE_BASE_COUNT 8
44
45/**
46 * The avg size of 1 se
47 * Based on the bsc thesis of Elias Summermatter (2021)
48 */
49
50#define AVG_BYTE_SIZE_SE 4221
51
52/**
53 * Calculates the optimal number of strata Estimators to send
54 * @param avg_element_size
55 * @param element_count
56 * @return
57 */
58uint8_t
59determine_strata_count (uint64_t avg_element_size, uint64_t element_count)
60{
61 uint64_t base_size = avg_element_size * element_count;
62 /* >67kb total size of elements in set */
63 if (base_size < AVG_BYTE_SIZE_SE * 16)
64 return 1;
65 /* >270kb total size of elements in set */
66 if (base_size < AVG_BYTE_SIZE_SE * 64)
67 return 2;
68 /* >1mb total size of elements in set */
69 if (base_size < AVG_BYTE_SIZE_SE * 256)
70 return 4;
71 return 8;
72}
73
74
75/**
76 * Modify an IBF key @a k_in based on the @a salt, returning a
77 * salted key in @a k_out.
78 */
79static void
80salt_key (const struct IBF_Key *k_in,
81 uint32_t salt,
82 struct IBF_Key *k_out)
83{
84 int s = (salt * 7) % 64;
85 uint64_t x = k_in->key_val;
86
87 /* rotate ibf key */
88 x = (x >> s) | (x << (64 - s));
89 k_out->key_val = x;
90}
91
92
93/**
94 * Reverse modification done in the salt_key function
95 */
96static void
97unsalt_key (const struct IBF_Key *k_in,
98 uint32_t salt,
99 struct IBF_Key *k_out)
100{
101 int s = (salt * 7) % 64;
102 uint64_t x = k_in->key_val;
103
104 x = (x << s) | (x >> (64 - s));
105 k_out->key_val = x;
106}
107
108
109/**
110 * Write the given strata estimator to the buffer.
111 *
112 * @param se strata estimator to serialize
113 * @param[out] buf buffer to write to, must be of appropriate size
114 * @return number of bytes written to @a buf
115 */
116size_t
117strata_estimator_write (struct MultiStrataEstimator *se,
118 uint16_t se_ibf_total_size,
119 uint8_t number_se_send,
120 void *buf)
121{
122 char *sbuf = buf;
123 unsigned int i;
124 size_t osize;
125 uint64_t sbuf_offset = 0;
126 se->size = number_se_send;
127
128 GNUNET_assert (NULL != se);
129 for (uint8_t strata_ctr = 0; strata_ctr < number_se_send; strata_ctr++)
130 {
131 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
132 {
133 ibf_write_slice (se->stratas[strata_ctr]->strata[i],
134 0,
135 se->stratas[strata_ctr]->ibf_size,
136 &sbuf[sbuf_offset],
137 8);
138 sbuf_offset += se->stratas[strata_ctr]->ibf_size * IBF_BUCKET_SIZE;
139 }
140 }
141 osize = ((se_ibf_total_size / 8) * number_se_send) * IBF_BUCKET_SIZE
142 * se->stratas[0]->strata_count;
143#if FAIL_10_1_COMPATIBILTIY
144 {
145 char *cbuf;
146 size_t nsize;
147
148 if (GNUNET_YES ==
149 GNUNET_try_compression (buf,
150 osize,
151 &cbuf,
152 &nsize))
153 {
154 GNUNET_memcpy (buf, cbuf, nsize);
155 osize = nsize;
156 GNUNET_free (cbuf);
157 }
158 }
159#endif
160 return osize;
161}
162
163
164/**
165 * Read strata from the buffer into the given strata
166 * estimator. The strata estimator must already be allocated.
167 *
168 * @param buf buffer to read from
169 * @param buf_len number of bytes in @a buf
170 * @param is_compressed is the data compressed?
171 * @param[out] se strata estimator to write to
172 * @return #GNUNET_OK on success
173 */
174int
175strata_estimator_read (const void *buf,
176 size_t buf_len,
177 int is_compressed,
178 uint8_t number_se_received,
179 uint16_t se_ibf_total_size,
180 struct MultiStrataEstimator *se)
181{
182 unsigned int i;
183 size_t osize;
184 char *dbuf;
185
186 dbuf = NULL;
187 if (GNUNET_YES == is_compressed)
188 {
189 osize = ((se_ibf_total_size / 8) * number_se_received) * IBF_BUCKET_SIZE
190 * se->stratas[0]->strata_count;
191 dbuf = GNUNET_decompress (buf,
192 buf_len,
193 osize);
194 if (NULL == dbuf)
195 {
196 GNUNET_break_op (0); /* bad compressed input data */
197 return GNUNET_SYSERR;
198 }
199 buf = dbuf;
200 buf_len = osize;
201 }
202
203 if (buf_len != se->stratas[0]->strata_count * ((se_ibf_total_size / 8)
204 * number_se_received)
205 * IBF_BUCKET_SIZE)
206 {
207 GNUNET_break (0); /* very odd error */
208 GNUNET_free (dbuf);
209 return GNUNET_SYSERR;
210 }
211
212 for (uint8_t strata_ctr = 0; strata_ctr < number_se_received; strata_ctr++)
213 {
214 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
215 {
216 ibf_read_slice (buf, 0, se->stratas[strata_ctr]->ibf_size,
217 se->stratas[strata_ctr]->strata[i], 8);
218 buf += se->stratas[strata_ctr]->ibf_size * IBF_BUCKET_SIZE;
219 }
220 }
221 se->size = number_se_received;
222 GNUNET_free (dbuf);
223 return GNUNET_OK;
224}
225
226
227/**
228 * Add a key to the strata estimator.
229 *
230 * @param se strata estimator to add the key to
231 * @param key key to add
232 */
233void
234strata_estimator_insert (struct MultiStrataEstimator *se,
235 struct IBF_Key key)
236{
237
238
239 /* count trailing '1'-bits of v */
240 for (int strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
241 {
242 unsigned int i;
243 uint64_t v;
244
245 struct IBF_Key salted_key;
246 salt_key (&key,
247 strata_ctr * (64 / MULTI_SE_BASE_COUNT),
248 &salted_key);
249 v = salted_key.key_val;
250 for (i = 0; v & 1; v >>= 1, i++)
251 {
252 ibf_insert (se->stratas[strata_ctr]->strata[i], salted_key);
253 }
254 }
255 /* empty */;
256
257}
258
259
260/**
261 * Remove a key from the strata estimator. (NOT USED)
262 *
263 * @param se strata estimator to remove the key from
264 * @param key key to remove
265 */
266void
267strata_estimator_remove (struct MultiStrataEstimator *se,
268 struct IBF_Key key)
269{
270
271 /* count trailing '1'-bits of v */
272 for (int strata_ctr = 0; strata_ctr < se->size; strata_ctr++)
273 {
274 uint64_t v;
275 unsigned int i;
276
277 struct IBF_Key unsalted_key;
278 unsalt_key (&key,
279 strata_ctr * (64 / MULTI_SE_BASE_COUNT),
280 &unsalted_key);
281
282 v = unsalted_key.key_val;
283 for (i = 0; v & 1; v >>= 1, i++)
284 {
285 /* empty */;
286 ibf_remove (se->stratas[strata_ctr]->strata[i], unsalted_key);
287 }
288 }
289}
290
291
292/**
293 * Create a new strata estimator with the given parameters.
294 *
295 * @param strata_count number of stratas, that is, number of ibfs in the estimator
296 * @param ibf_size size of each ibf stratum
297 * @param ibf_hashnum hashnum parameter of each ibf
298 * @return a freshly allocated, empty strata estimator, NULL on error
299 */
300struct MultiStrataEstimator *
301strata_estimator_create (unsigned int strata_count,
302 uint32_t ibf_size,
303 uint8_t ibf_hashnum)
304{
305 struct MultiStrataEstimator *se;
306 unsigned int i;
307 unsigned int j;
308 se = GNUNET_new (struct MultiStrataEstimator);
309
310 se->size = MULTI_SE_BASE_COUNT;
311 se->stratas = GNUNET_new_array (MULTI_SE_BASE_COUNT,struct StrataEstimator *);
312
313 uint8_t ibf_prime_sizes[] = {79,79,79,79,79,79,79,79};
314
315 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
316 {
317 se->stratas[strata_ctr] = GNUNET_new (struct StrataEstimator);
318 se->stratas[strata_ctr]->strata_count = strata_count;
319 se->stratas[strata_ctr]->ibf_size = ibf_prime_sizes[strata_ctr];
320 se->stratas[strata_ctr]->strata = GNUNET_new_array (strata_count * 4,
321 struct
322 InvertibleBloomFilter *);
323 for (i = 0; i < strata_count; i++)
324 {
325 se->stratas[strata_ctr]->strata[i] = ibf_create (
326 ibf_prime_sizes[strata_ctr], ibf_hashnum);
327 if (NULL == se->stratas[strata_ctr]->strata[i])
328 {
329 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
330 "Failed to allocate memory for strata estimator\n");
331 for (j = 0; j < i; j++)
332 ibf_destroy (se->stratas[strata_ctr]->strata[i]);
333 GNUNET_free (se);
334 return NULL;
335 }
336 }
337 }
338 return se;
339}
340
341
342/**
343 * Estimate set difference with two strata estimators,
344 * i.e. arrays of IBFs.
345 * Does not not modify its arguments.
346 *
347 * @param se1 first strata estimator
348 * @param se2 second strata estimator
349 * @return the estimated difference
350 */
351void
352strata_estimator_difference (const struct MultiStrataEstimator *se1,
353 const struct MultiStrataEstimator *se2)
354{
355 int avg_local_diff = 0;
356 int avg_remote_diff = 0;
357 uint8_t number_of_estimators = se1->size;
358
359 for (uint8_t strata_ctr = 0; strata_ctr < number_of_estimators; strata_ctr++)
360 {
361 GNUNET_assert (se1->stratas[strata_ctr]->strata_count ==
362 se2->stratas[strata_ctr]->strata_count);
363
364
365 for (int i = se1->stratas[strata_ctr]->strata_count - 1; i >= 0; i--)
366 {
367 struct InvertibleBloomFilter *diff;
368 /* number of keys decoded from the ibf */
369
370 /* FIXME: implement this without always allocating new IBFs */
371 diff = ibf_dup (se1->stratas[strata_ctr]->strata[i]);
372 diff->local_decoded_count = 0;
373 diff->remote_decoded_count = 0;
374
375 ibf_subtract (diff, se2->stratas[strata_ctr]->strata[i]);
376
377 for (int ibf_count = 0; GNUNET_YES; ibf_count++)
378 {
379 int more;
380
381 more = ibf_decode (diff, NULL, NULL);
382 if (GNUNET_NO == more)
383 {
384 se1->stratas[strata_ctr]->strata[0]->local_decoded_count +=
385 diff->local_decoded_count;
386 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count +=
387 diff->remote_decoded_count;
388 break;
389 }
390 /* Estimate if decoding fails or would not terminate */
391 if ((GNUNET_SYSERR == more) || (ibf_count > diff->size))
392 {
393 se1->stratas[strata_ctr]->strata[0]->local_decoded_count =
394 se1->stratas[strata_ctr]->strata[0]->local_decoded_count * (1 << (i
395 +
396 1));
397 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count =
398 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count * (1 << (i
399 +
400 1));
401 ibf_destroy (diff);
402 goto break_all_counting_loops;
403 }
404 }
405 ibf_destroy (diff);
406 }
407break_all_counting_loops:;
408 avg_local_diff += se1->stratas[strata_ctr]->strata[0]->local_decoded_count;
409 avg_remote_diff +=
410 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count;
411 }
412 se1->stratas[0]->strata[0]->local_decoded_count = avg_local_diff
413 / number_of_estimators;
414 se1->stratas[0]->strata[0]->remote_decoded_count = avg_remote_diff
415 / number_of_estimators;
416}
417
418
419/**
420 * Make a copy of a strata estimator.
421 *
422 * @param se the strata estimator to copy
423 * @return the copy
424 */
425struct MultiStrataEstimator *
426strata_estimator_dup (struct MultiStrataEstimator *se)
427{
428 struct MultiStrataEstimator *c;
429 unsigned int i;
430
431 c = GNUNET_new (struct MultiStrataEstimator);
432 c->stratas = GNUNET_new_array (MULTI_SE_BASE_COUNT,struct StrataEstimator *);
433 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
434 {
435 c->stratas[strata_ctr] = GNUNET_new (struct StrataEstimator);
436 c->stratas[strata_ctr]->strata_count =
437 se->stratas[strata_ctr]->strata_count;
438 c->stratas[strata_ctr]->ibf_size = se->stratas[strata_ctr]->ibf_size;
439 c->stratas[strata_ctr]->strata = GNUNET_new_array (
440 se->stratas[strata_ctr]->strata_count,
441 struct
442 InvertibleBloomFilter *);
443 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
444 c->stratas[strata_ctr]->strata[i] = ibf_dup (
445 se->stratas[strata_ctr]->strata[i]);
446 }
447 return c;
448}
449
450
451/**
452 * Destroy a strata estimator, free all of its resources.
453 *
454 * @param se strata estimator to destroy.
455 */
456void
457strata_estimator_destroy (struct MultiStrataEstimator *se)
458{
459 unsigned int i;
460 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
461 {
462 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
463 ibf_destroy (se->stratas[strata_ctr]->strata[i]);
464 GNUNET_free (se->stratas[strata_ctr]->strata);
465 }
466 GNUNET_free (se);
467}
diff --git a/src/setu/gnunet-service-setu_strata_estimator.h b/src/setu/gnunet-service-setu_strata_estimator.h
deleted file mode 100644
index 4871a7fcd..000000000
--- a/src/setu/gnunet-service-setu_strata_estimator.h
+++ /dev/null
@@ -1,199 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-service-setu_strata_estimator.h
23 * @brief estimator of set difference
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#ifndef GNUNET_SERVICE_SETU_STRATA_ESTIMATOR_H
29#define GNUNET_SERVICE_SETU_STRATA_ESTIMATOR_H
30
31#include "platform.h"
32#include "gnunet_common.h"
33#include "gnunet_util_lib.h"
34
35#ifdef __cplusplus
36extern "C"
37{
38#if 0 /* keep Emacsens' auto-indent happy */
39}
40#endif
41#endif
42
43
44/**
45 * A handle to a strata estimator.
46 */
47struct StrataEstimator
48{
49 /**
50 * The IBFs of this strata estimator.
51 */
52 struct InvertibleBloomFilter **strata;
53
54 /**
55 * Size of the IBF array in @e strata
56 */
57 unsigned int strata_count;
58
59 /**
60 * Size of each IBF stratum (in bytes)
61 */
62 unsigned int ibf_size;
63};
64
65struct MultiStrataEstimator
66{
67 /**
68 * Array of strata estimators
69 */
70 struct StrataEstimator **stratas;
71
72 /**
73 * Number of strata estimators in struct
74 */
75 uint8_t size;
76
77};
78
79/**
80 * Deteminate how many strata estimators in the message are necessary
81 * @param avg_element_size
82 * @param element_count
83 * @return number of strata's
84 */
85
86uint8_t
87determine_strata_count (uint64_t avg_element_size,
88 uint64_t element_count);
89
90
91/**
92 * Write the given strata estimator to the buffer.
93 *
94 * @param se strata estimator to serialize
95 * @param[out] buf buffer to write to, must be of appropriate size
96 * @return number of bytes written to @a buf
97 */
98size_t
99strata_estimator_write (struct MultiStrataEstimator *se,
100 uint16_t se_ibf_total_size,
101 uint8_t number_se_send,
102 void *buf);
103
104
105/**
106 * Read strata from the buffer into the given strata
107 * estimator. The strata estimator must already be allocated.
108 *
109 * @param buf buffer to read from
110 * @param buf_len number of bytes in @a buf
111 * @param is_compressed is the data compressed?
112 * @param[out] se strata estimator to write to
113 * @return #GNUNET_OK on success
114 */
115int
116strata_estimator_read (const void *buf,
117 size_t buf_len,
118 int is_compressed,
119 uint8_t number_se_received,
120 uint16_t se_ibf_total_size,
121 struct MultiStrataEstimator *se);
122
123
124/**
125 * Create a new strata estimator with the given parameters.
126 *
127 * @param strata_count number of stratas, that is, number of ibfs in the estimator
128 * @param ibf_size size of each ibf stratum
129 * @param ibf_hashnum hashnum parameter of each ibf
130 * @return a freshly allocated, empty strata estimator, NULL on error
131 */
132struct MultiStrataEstimator *
133strata_estimator_create (unsigned int strata_count,
134 uint32_t ibf_size,
135 uint8_t ibf_hashnum);
136
137
138/**
139 * Get an estimation of the symmetric difference of the elements
140 * contained in both strata estimators.
141 *
142 * @param se1 first strata estimator
143 * @param se2 second strata estimator
144 * @return nothing
145 */
146void
147strata_estimator_difference (const struct MultiStrataEstimator *se1,
148 const struct MultiStrataEstimator *se2);
149
150
151/**
152 * Add a key to the strata estimator.
153 *
154 * @param se strata estimator to add the key to
155 * @param key key to add
156 */
157void
158strata_estimator_insert (struct MultiStrataEstimator *se,
159 struct IBF_Key key);
160
161
162/**
163 * Remove a key from the strata estimator.
164 *
165 * @param se strata estimator to remove the key from
166 * @param key key to remove
167 */
168void
169strata_estimator_remove (struct MultiStrataEstimator *se,
170 struct IBF_Key key);
171
172
173/**
174 * Destroy a strata estimator, free all of its resources.
175 *
176 * @param se strata estimator to destroy.
177 */
178void
179strata_estimator_destroy (struct MultiStrataEstimator *se);
180
181
182/**
183 * Make a copy of a strata estimator.
184 *
185 * @param se the strata estimator to copy
186 * @return the copy
187 */
188struct MultiStrataEstimator *
189strata_estimator_dup (struct MultiStrataEstimator *se);
190
191
192#if 0 /* keep Emacsens' auto-indent happy */
193{
194#endif
195#ifdef __cplusplus
196}
197#endif
198
199#endif
diff --git a/src/setu/gnunet-setu-ibf-profiler.c b/src/setu/gnunet-setu-ibf-profiler.c
deleted file mode 100644
index 944b63d30..000000000
--- a/src/setu/gnunet-setu-ibf-profiler.c
+++ /dev/null
@@ -1,308 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-set-ibf-profiler.c
23 * @brief tool for profiling the invertible bloom filter implementation
24 * @author Florian Dold
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29
30#include "ibf.h"
31
32static unsigned int asize = 10;
33static unsigned int bsize = 10;
34static unsigned int csize = 10;
35static unsigned int hash_num = 4;
36static unsigned int ibf_size = 80;
37
38/* FIXME: add parameter for this */
39static enum GNUNET_CRYPTO_Quality random_quality = GNUNET_CRYPTO_QUALITY_WEAK;
40
41static struct GNUNET_CONTAINER_MultiHashMap *set_a;
42static struct GNUNET_CONTAINER_MultiHashMap *set_b;
43/* common elements in a and b */
44static struct GNUNET_CONTAINER_MultiHashMap *set_c;
45
46static struct GNUNET_CONTAINER_MultiHashMap *key_to_hashcode;
47
48static struct InvertibleBloomFilter *ibf_a;
49static struct InvertibleBloomFilter *ibf_b;
50
51
52static void
53register_hashcode (struct GNUNET_HashCode *hash)
54{
55 struct GNUNET_HashCode replicated;
56 struct IBF_Key key;
57
58 key = ibf_key_from_hashcode (hash);
59 ibf_hashcode_from_key (key, &replicated);
60 (void) GNUNET_CONTAINER_multihashmap_put (
61 key_to_hashcode,
62 &replicated,
63 GNUNET_memdup (hash, sizeof *hash),
64 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
65}
66
67
68static void
69iter_hashcodes (struct IBF_Key key,
70 GNUNET_CONTAINER_MulitHashMapIteratorCallback iter,
71 void *cls)
72{
73 struct GNUNET_HashCode replicated;
74
75 ibf_hashcode_from_key (key, &replicated);
76 GNUNET_CONTAINER_multihashmap_get_multiple (key_to_hashcode,
77 &replicated,
78 iter,
79 cls);
80}
81
82
83static int
84insert_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
85{
86 struct InvertibleBloomFilter *ibf = cls;
87
88 ibf_insert (ibf, ibf_key_from_hashcode (key));
89 return GNUNET_YES;
90}
91
92
93static int
94remove_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
95{
96 struct GNUNET_CONTAINER_MultiHashMap *hashmap = cls;
97
98 /* if remove fails, there just was a collision with another key */
99 (void) GNUNET_CONTAINER_multihashmap_remove (hashmap, value, NULL);
100 return GNUNET_YES;
101}
102
103
104static void
105run (void *cls,
106 char *const *args,
107 const char *cfgfile,
108 const struct GNUNET_CONFIGURATION_Handle *cfg)
109{
110 struct GNUNET_HashCode id;
111 struct IBF_Key ibf_key;
112 int i;
113 int side;
114 int res;
115 struct GNUNET_TIME_Absolute start_time;
116 struct GNUNET_TIME_Relative delta_time;
117
118 set_a =
119 GNUNET_CONTAINER_multihashmap_create (((asize == 0) ? 1 : (asize + csize)),
120 GNUNET_NO);
121 set_b =
122 GNUNET_CONTAINER_multihashmap_create (((bsize == 0) ? 1 : (bsize + csize)),
123 GNUNET_NO);
124 set_c = GNUNET_CONTAINER_multihashmap_create (((csize == 0) ? 1 : csize),
125 GNUNET_NO);
126
127 key_to_hashcode =
128 GNUNET_CONTAINER_multihashmap_create (((asize + bsize + csize == 0)
129 ? 1
130 : (asize + bsize + csize)),
131 GNUNET_NO);
132
133 printf ("hash-num=%u, size=%u, #(A-B)=%u, #(B-A)=%u, #(A&B)=%u\n",
134 hash_num,
135 ibf_size,
136 asize,
137 bsize,
138 csize);
139
140 i = 0;
141 while (i < asize)
142 {
143 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
144 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
145 continue;
146 GNUNET_break (GNUNET_OK ==
147 GNUNET_CONTAINER_multihashmap_put (
148 set_a,
149 &id,
150 NULL,
151 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
152 register_hashcode (&id);
153 i++;
154 }
155 i = 0;
156 while (i < bsize)
157 {
158 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
159 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
160 continue;
161 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
162 continue;
163 GNUNET_break (GNUNET_OK ==
164 GNUNET_CONTAINER_multihashmap_put (
165 set_b,
166 &id,
167 NULL,
168 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
169 register_hashcode (&id);
170 i++;
171 }
172 i = 0;
173 while (i < csize)
174 {
175 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
176 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
177 continue;
178 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
179 continue;
180 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_c, &id))
181 continue;
182 GNUNET_break (GNUNET_OK ==
183 GNUNET_CONTAINER_multihashmap_put (
184 set_c,
185 &id,
186 NULL,
187 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
188 register_hashcode (&id);
189 i++;
190 }
191
192 ibf_a = ibf_create (ibf_size, hash_num);
193 ibf_b = ibf_create (ibf_size, hash_num);
194 if ((NULL == ibf_a) || (NULL == ibf_b))
195 {
196 /* insufficient memory */
197 GNUNET_break (0);
198 GNUNET_SCHEDULER_shutdown ();
199 return;
200 }
201
202
203 printf ("generated sets\n");
204
205 start_time = GNUNET_TIME_absolute_get ();
206
207 GNUNET_CONTAINER_multihashmap_iterate (set_a, &insert_iterator, ibf_a);
208 GNUNET_CONTAINER_multihashmap_iterate (set_b, &insert_iterator, ibf_b);
209 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_a);
210 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_b);
211
212 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
213
214 printf ("encoded in: %s\n",
215 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
216
217 ibf_subtract (ibf_a, ibf_b);
218
219
220 start_time = GNUNET_TIME_absolute_get ();
221
222 for (i = 0; i <= asize + bsize; i++)
223 {
224 res = ibf_decode (ibf_a, &side, &ibf_key);
225 if (GNUNET_SYSERR == res)
226 {
227 printf ("decode failed, %u/%u elements left\n",
228 GNUNET_CONTAINER_multihashmap_size (set_a)
229 + GNUNET_CONTAINER_multihashmap_size (set_b),
230 asize + bsize);
231 return;
232 }
233 if (GNUNET_NO == res)
234 {
235 if ((0 == GNUNET_CONTAINER_multihashmap_size (set_b)) &&
236 (0 == GNUNET_CONTAINER_multihashmap_size (set_a)))
237 {
238 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
239 printf ("decoded successfully in: %s\n",
240 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
241 }
242 else
243 {
244 printf ("decode missed elements (should never happen)\n");
245 }
246 return;
247 }
248
249 if (side == 1)
250 iter_hashcodes (ibf_key, remove_iterator, set_a);
251 if (side == -1)
252 iter_hashcodes (ibf_key, remove_iterator, set_b);
253 }
254 printf ("cyclic IBF, %u/%u elements left\n",
255 GNUNET_CONTAINER_multihashmap_size (set_a)
256 + GNUNET_CONTAINER_multihashmap_size (set_b),
257 asize + bsize);
258}
259
260
261int
262main (int argc, char **argv)
263{
264 struct GNUNET_GETOPT_CommandLineOption options[] = {
265 GNUNET_GETOPT_option_uint ('A',
266 "asize",
267 NULL,
268 gettext_noop ("number of element in set A-B"),
269 &asize),
270
271 GNUNET_GETOPT_option_uint ('B',
272 "bsize",
273 NULL,
274 gettext_noop ("number of element in set B-A"),
275 &bsize),
276
277 GNUNET_GETOPT_option_uint ('C',
278 "csize",
279 NULL,
280 gettext_noop (
281 "number of common elements in A and B"),
282 &csize),
283
284 GNUNET_GETOPT_option_uint ('k',
285 "hash-num",
286 NULL,
287 gettext_noop ("hash num"),
288 &hash_num),
289
290 GNUNET_GETOPT_option_uint ('s',
291 "ibf-size",
292 NULL,
293 gettext_noop ("ibf size"),
294 &ibf_size),
295
296 GNUNET_GETOPT_OPTION_END
297 };
298
299 GNUNET_PROGRAM_run2 (argc,
300 argv,
301 "gnunet-consensus-ibf",
302 "help",
303 options,
304 &run,
305 NULL,
306 GNUNET_YES);
307 return 0;
308}
diff --git a/src/setu/gnunet-setu-profiler.c b/src/setu/gnunet-setu-profiler.c
deleted file mode 100644
index 8d6a2dc8c..000000000
--- a/src/setu/gnunet-setu-profiler.c
+++ /dev/null
@@ -1,499 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file setu/gnunet-setu-profiler.c
23 * @brief profiling tool for set
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_statistics_service.h"
29#include "gnunet_setu_service.h"
30#include "gnunet_testbed_service.h"
31
32
33static int ret;
34
35static unsigned int num_a = 5;
36static unsigned int num_b = 5;
37static unsigned int num_c = 20;
38
39static char *op_str = "union";
40
41const static struct GNUNET_CONFIGURATION_Handle *config;
42
43struct SetInfo
44{
45 char *id;
46 struct GNUNET_SETU_Handle *set;
47 struct GNUNET_SETU_OperationHandle *oh;
48 struct GNUNET_CONTAINER_MultiHashMap *sent;
49 struct GNUNET_CONTAINER_MultiHashMap *received;
50 int done;
51} info1, info2;
52
53static struct GNUNET_CONTAINER_MultiHashMap *common_sent;
54
55static struct GNUNET_HashCode app_id;
56
57static struct GNUNET_PeerIdentity local_peer;
58
59static struct GNUNET_SETU_ListenHandle *set_listener;
60
61static int byzantine;
62static unsigned int force_delta;
63static unsigned int force_full;
64static unsigned int element_size = 32;
65
66/**
67 * Handle to the statistics service.
68 */
69static struct GNUNET_STATISTICS_Handle *statistics;
70
71/**
72 * The profiler will write statistics
73 * for all peers to the file with this name.
74 */
75static char *statistics_filename;
76
77/**
78 * The profiler will write statistics
79 * for all peers to this file.
80 */
81static FILE *statistics_file;
82
83
84static int
85map_remove_iterator (void *cls,
86 const struct GNUNET_HashCode *key,
87 void *value)
88{
89 struct GNUNET_CONTAINER_MultiHashMap *m = cls;
90 int ret;
91
92 GNUNET_assert (NULL != key);
93
94 ret = GNUNET_CONTAINER_multihashmap_remove_all (m, key);
95 if (GNUNET_OK != ret)
96 printf ("spurious element\n");
97 return GNUNET_YES;
98}
99
100
101/**
102 * Callback function to process statistic values.
103 *
104 * @param cls closure
105 * @param subsystem name of subsystem that created the statistic
106 * @param name the name of the datum
107 * @param value the current value
108 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
109 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
110 */
111static int
112statistics_result (void *cls,
113 const char *subsystem,
114 const char *name,
115 uint64_t value,
116 int is_persistent)
117{
118 if (NULL != statistics_file)
119 {
120 fprintf (statistics_file, "%s\t%s\t%lu\n", subsystem, name, (unsigned
121 long) value);
122 }
123 return GNUNET_OK;
124}
125
126
127static void
128statistics_done (void *cls,
129 int success)
130{
131 GNUNET_assert (GNUNET_YES == success);
132 if (NULL != statistics_file)
133 fclose (statistics_file);
134 GNUNET_SCHEDULER_shutdown ();
135}
136
137
138static void
139check_all_done (void)
140{
141 if ((info1.done == GNUNET_NO) || (info2.done == GNUNET_NO))
142 return;
143
144 GNUNET_CONTAINER_multihashmap_iterate (info1.received, map_remove_iterator,
145 info2.sent);
146 GNUNET_CONTAINER_multihashmap_iterate (info2.received, map_remove_iterator,
147 info1.sent);
148
149 printf ("set a: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
150 info1.sent));
151 printf ("set b: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
152 info2.sent));
153
154 if (NULL == statistics_filename)
155 {
156 GNUNET_SCHEDULER_shutdown ();
157 return;
158 }
159
160 statistics_file = fopen (statistics_filename, "w");
161 GNUNET_STATISTICS_get (statistics, NULL, NULL,
162 &statistics_done,
163 &statistics_result, NULL);
164}
165
166
167static void
168set_result_cb (void *cls,
169 const struct GNUNET_SETU_Element *element,
170 uint64_t current_size,
171 enum GNUNET_SETU_Status status)
172{
173 struct SetInfo *info = cls;
174
175 GNUNET_assert (GNUNET_NO == info->done);
176 switch (status)
177 {
178 case GNUNET_SETU_STATUS_DONE:
179 info->done = GNUNET_YES;
180 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s done\n", info->id);
181 check_all_done ();
182 info->oh = NULL;
183 return;
184
185 case GNUNET_SETU_STATUS_FAILURE:
186 info->oh = NULL;
187 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "failure\n");
188 GNUNET_SCHEDULER_shutdown ();
189 return;
190
191 case GNUNET_SETU_STATUS_ADD_LOCAL:
192 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: local element\n", info->id);
193 break;
194 default:
195 GNUNET_assert (0);
196 }
197
198 if (element->size != element_size)
199 {
200 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
201 "wrong element size: %u, expected %u\n",
202 element->size,
203 (unsigned int) sizeof(struct GNUNET_HashCode));
204 GNUNET_assert (0);
205 }
206
207 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: got element (%s)\n",
208 info->id, GNUNET_h2s (element->data));
209 GNUNET_assert (NULL != element->data);
210 struct GNUNET_HashCode data_hash;
211 GNUNET_CRYPTO_hash (element->data, element_size, &data_hash);
212 GNUNET_CONTAINER_multihashmap_put (info->received,
213 &data_hash, NULL,
214 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
215}
216
217
218static void
219set_listen_cb (void *cls,
220 const struct GNUNET_PeerIdentity *other_peer,
221 const struct GNUNET_MessageHeader *context_msg,
222 struct GNUNET_SETU_Request *request)
223{
224 /* max. 2 options plus terminator */
225 struct GNUNET_SETU_Option opts[3] = { { 0 } };
226 unsigned int n_opts = 0;
227
228 if (NULL == request)
229 {
230 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
231 "listener failed\n");
232 return;
233 }
234 GNUNET_assert (NULL == info2.oh);
235 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
236 "set listen cb called\n");
237 if (byzantine)
238 {
239 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
240 GNUNET_SETU_OPTION_BYZANTINE };
241 }
242 GNUNET_assert (! (force_full && force_delta));
243 if (force_full)
244 {
245 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
246 GNUNET_SETU_OPTION_FORCE_FULL };
247 }
248 if (force_delta)
249 {
250 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
251 GNUNET_SETU_OPTION_FORCE_DELTA };
252 }
253
254 opts[n_opts].type = 0;
255 info2.oh = GNUNET_SETU_accept (request,
256 opts,
257 set_result_cb, &info2);
258 GNUNET_SETU_commit (info2.oh, info2.set);
259}
260
261
262static int
263set_insert_iterator (void *cls,
264 const struct GNUNET_HashCode *key,
265 void *value)
266{
267 struct GNUNET_SETU_Handle *set = cls;
268 struct GNUNET_SETU_Element el;
269
270 el.element_type = 0;
271 el.data = value;
272 el.size = element_size;
273 GNUNET_SETU_add_element (set, &el, NULL, NULL);
274 return GNUNET_YES;
275}
276
277
278static void
279handle_shutdown (void *cls)
280{
281 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
282 "Shutting down set profiler\n");
283 if (NULL != set_listener)
284 {
285 GNUNET_SETU_listen_cancel (set_listener);
286 set_listener = NULL;
287 }
288 if (NULL != info1.oh)
289 {
290 GNUNET_SETU_operation_cancel (info1.oh);
291 info1.oh = NULL;
292 }
293 if (NULL != info2.oh)
294 {
295 GNUNET_SETU_operation_cancel (info2.oh);
296 info2.oh = NULL;
297 }
298 if (NULL != info1.set)
299 {
300 GNUNET_SETU_destroy (info1.set);
301 info1.set = NULL;
302 }
303 if (NULL != info2.set)
304 {
305 GNUNET_SETU_destroy (info2.set);
306 info2.set = NULL;
307 }
308 GNUNET_STATISTICS_destroy (statistics, GNUNET_NO);
309}
310
311
312static void
313run (void *cls,
314 const struct GNUNET_CONFIGURATION_Handle *cfg,
315 struct GNUNET_TESTING_Peer *peer)
316{
317 unsigned int i;
318 struct GNUNET_HashCode hash;
319 /* max. 2 options plus terminator */
320 struct GNUNET_SETU_Option opts[3] = { { 0 } };
321 unsigned int n_opts = 0;
322
323 config = cfg;
324
325 GNUNET_assert (element_size > 0);
326
327 if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &local_peer))
328 {
329 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
330 ret = 0;
331 return;
332 }
333
334 statistics = GNUNET_STATISTICS_create ("set-profiler", cfg);
335
336 GNUNET_SCHEDULER_add_shutdown (&handle_shutdown, NULL);
337
338 info1.id = "a";
339 info2.id = "b";
340
341 info1.sent = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
342 info2.sent = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
343 common_sent = GNUNET_CONTAINER_multihashmap_create (num_c + 1, GNUNET_NO);
344
345 info1.received = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
346 info2.received = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
347
348 for (i = 0; i < num_a; i++)
349 {
350 char *data = GNUNET_malloc (element_size);
351 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
352 GNUNET_CRYPTO_hash (data, element_size, &hash);
353 GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, data,
354 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
355 }
356
357 for (i = 0; i < num_b; i++)
358 {
359 char *data = GNUNET_malloc (element_size);
360 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
361 GNUNET_CRYPTO_hash (data, element_size, &hash);
362 GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, data,
363 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
364 }
365
366 for (i = 0; i < num_c; i++)
367 {
368 char *data = GNUNET_malloc (element_size);
369 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
370 GNUNET_CRYPTO_hash (data, element_size, &hash);
371 GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, data,
372 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
373 }
374
375 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &app_id);
376
377 /* FIXME: also implement intersection etc. */
378 info1.set = GNUNET_SETU_create (config);
379 info2.set = GNUNET_SETU_create (config);
380
381 GNUNET_CONTAINER_multihashmap_iterate (info1.sent, set_insert_iterator,
382 info1.set);
383 GNUNET_CONTAINER_multihashmap_iterate (info2.sent, set_insert_iterator,
384 info2.set);
385 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
386 info1.set);
387 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
388 info2.set);
389
390 set_listener = GNUNET_SETU_listen (config,
391 &app_id,
392 &set_listen_cb,
393 NULL);
394
395
396 if (byzantine)
397 {
398 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
399 GNUNET_SETU_OPTION_BYZANTINE };
400 }
401 GNUNET_assert (! (force_full && force_delta));
402 if (force_full)
403 {
404 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
405 GNUNET_SETU_OPTION_FORCE_FULL };
406 }
407 if (force_delta)
408 {
409 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
410 GNUNET_SETU_OPTION_FORCE_DELTA };
411 }
412
413 opts[n_opts].type = 0;
414
415 info1.oh = GNUNET_SETU_prepare (&local_peer, &app_id, NULL,
416 opts,
417 set_result_cb, &info1);
418 GNUNET_SETU_commit (info1.oh, info1.set);
419 GNUNET_SETU_destroy (info1.set);
420 info1.set = NULL;
421}
422
423
424static void
425pre_run (void *cls, char *const *args, const char *cfgfile,
426 const struct GNUNET_CONFIGURATION_Handle *cfg)
427{
428 if (0 != GNUNET_TESTING_peer_run ("set-profiler",
429 cfgfile,
430 &run, NULL))
431 ret = 2;
432}
433
434
435int
436main (int argc, char **argv)
437{
438 struct GNUNET_GETOPT_CommandLineOption options[] = {
439 GNUNET_GETOPT_option_uint ('A',
440 "num-first",
441 NULL,
442 gettext_noop ("number of values"),
443 &num_a),
444
445 GNUNET_GETOPT_option_uint ('B',
446 "num-second",
447 NULL,
448 gettext_noop ("number of values"),
449 &num_b),
450
451 GNUNET_GETOPT_option_flag ('b',
452 "byzantine",
453 gettext_noop ("use byzantine mode"),
454 &byzantine),
455
456 GNUNET_GETOPT_option_uint ('f',
457 "force-full",
458 NULL,
459 gettext_noop ("force sending full set"),
460 &force_full),
461
462 GNUNET_GETOPT_option_uint ('d',
463 "force-delta",
464 NULL,
465 gettext_noop ("number delta operation"),
466 &force_delta),
467
468 GNUNET_GETOPT_option_uint ('C',
469 "num-common",
470 NULL,
471 gettext_noop ("number of values"),
472 &num_c),
473
474 GNUNET_GETOPT_option_string ('x',
475 "operation",
476 NULL,
477 gettext_noop ("operation to execute"),
478 &op_str),
479
480 GNUNET_GETOPT_option_uint ('w',
481 "element-size",
482 NULL,
483 gettext_noop ("element size"),
484 &element_size),
485
486 GNUNET_GETOPT_option_filename ('s',
487 "statistics",
488 "FILENAME",
489 gettext_noop ("write statistics to file"),
490 &statistics_filename),
491
492 GNUNET_GETOPT_OPTION_END
493 };
494
495 GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler",
496 "help",
497 options, &pre_run, NULL, GNUNET_YES);
498 return ret;
499}
diff --git a/src/setu/ibf.c b/src/setu/ibf.c
deleted file mode 100644
index dbd23c320..000000000
--- a/src/setu/ibf.c
+++ /dev/null
@@ -1,614 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.c
23 * @brief implementation of the invertible bloom filter
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#include "ibf.h"
29#include "gnunet_util_lib.h"
30#define LOG(kind, ...) GNUNET_log_from (kind, "setu", __VA_ARGS__)
31
32
33/**
34 * Compute the key's hash from the key.
35 * Redefine to use a different hash function.
36 */
37#define IBF_KEY_HASH_VAL(k) (GNUNET_CRYPTO_crc32_n (&(k), sizeof(struct \
38 IBF_KeyHash)))
39
40/**
41 * Create a key from a hashcode.
42 *
43 * @param hash the hashcode
44 * @return a key
45 */
46struct IBF_Key
47ibf_key_from_hashcode (const struct GNUNET_HashCode *hash)
48{
49 return *(struct IBF_Key *) hash;
50}
51
52
53/**
54 * Create a hashcode from a key, by replicating the key
55 * until the hascode is filled
56 *
57 * @param key the key
58 * @param dst hashcode to store the result in
59 */
60void
61ibf_hashcode_from_key (struct IBF_Key key,
62 struct GNUNET_HashCode *dst)
63{
64 struct IBF_Key *p;
65 unsigned int i;
66 const unsigned int keys_per_hashcode = sizeof(struct GNUNET_HashCode)
67 / sizeof(struct IBF_Key);
68
69 p = (struct IBF_Key *) dst;
70 for (i = 0; i < keys_per_hashcode; i++)
71 *p++ = key;
72}
73
74
75/**
76 * Create an invertible bloom filter.
77 *
78 * @param size number of IBF buckets
79 * @param hash_num number of buckets one element is hashed in
80 * @return the newly created invertible bloom filter, NULL on error
81 */
82struct InvertibleBloomFilter *
83ibf_create (uint32_t size, uint8_t hash_num)
84{
85 struct InvertibleBloomFilter *ibf;
86
87 GNUNET_assert (0 != size);
88
89 ibf = GNUNET_new (struct InvertibleBloomFilter);
90 ibf->count = GNUNET_malloc_large (size * sizeof(uint64_t));
91 if (NULL == ibf->count)
92 {
93 GNUNET_free (ibf);
94 return NULL;
95 }
96 ibf->key_sum = GNUNET_malloc_large (size * sizeof(struct IBF_Key));
97 if (NULL == ibf->key_sum)
98 {
99 GNUNET_free (ibf->count);
100 GNUNET_free (ibf);
101 return NULL;
102 }
103 ibf->key_hash_sum = GNUNET_malloc_large (size * sizeof(struct IBF_KeyHash));
104 if (NULL == ibf->key_hash_sum)
105 {
106 GNUNET_free (ibf->key_sum);
107 GNUNET_free (ibf->count);
108 GNUNET_free (ibf);
109 return NULL;
110 }
111 ibf->size = size;
112 ibf->hash_num = hash_num;
113
114 return ibf;
115}
116
117
118/**
119 * Store unique bucket indices for the specified @a key in @a dst.
120 */
121static void
122ibf_get_indices (const struct InvertibleBloomFilter *ibf,
123 struct IBF_Key key,
124 int *dst)
125{
126 uint32_t filled;
127 uint32_t i;
128 uint32_t bucket;
129
130 bucket = GNUNET_CRYPTO_crc32_n (&key, sizeof key);
131 for (i = 0, filled = 0; filled < ibf->hash_num; i++)
132 {
133 uint64_t x;
134
135 for (unsigned int j = 0; j < filled; j++)
136 if (dst[j] == bucket % ibf->size)
137 goto try_next;
138 dst[filled++] = bucket % ibf->size;
139try_next:
140 x = ((uint64_t) bucket << 32) | i;
141 bucket = GNUNET_CRYPTO_crc32_n (&x, sizeof x);
142 }
143}
144
145
146static void
147ibf_insert_into (struct InvertibleBloomFilter *ibf,
148 struct IBF_Key key,
149 const int *buckets,
150 int side)
151{
152 for (unsigned int i = 0; i < ibf->hash_num; i++)
153 {
154 const int bucket = buckets[i];
155
156 ibf->count[bucket].count_val += side;
157 ibf->key_sum[bucket].key_val ^= key.key_val;
158 ibf->key_hash_sum[bucket].key_hash_val
159 ^= IBF_KEY_HASH_VAL (key);
160 }
161}
162
163
164/**
165 * Insert a key into an IBF.
166 *
167 * @param ibf the IBF
168 * @param key the element's hash code
169 */
170void
171ibf_insert (struct InvertibleBloomFilter *ibf,
172 struct IBF_Key key)
173{
174 int buckets[ibf->hash_num];
175
176 GNUNET_assert (ibf->hash_num <= ibf->size);
177 ibf_get_indices (ibf, key, buckets);
178 ibf_insert_into (ibf, key, buckets, 1);
179}
180
181
182/**
183 * Remove a key from an IBF.
184 *
185 * @param ibf the IBF
186 * @param key the element's hash code
187 */
188void
189ibf_remove (struct InvertibleBloomFilter *ibf,
190 struct IBF_Key key)
191{
192 int buckets[ibf->hash_num];
193
194 GNUNET_assert (ibf->hash_num <= ibf->size);
195 ibf_get_indices (ibf, key, buckets);
196 ibf_insert_into (ibf, key, buckets, -1);
197}
198
199
200/**
201 * Test is the IBF is empty, i.e. all counts, keys and key hashes are zero.
202 */
203static int
204ibf_is_empty (struct InvertibleBloomFilter *ibf)
205{
206 for (uint32_t i = 0; i < ibf->size; i++)
207 {
208 if (0 != ibf->count[i].count_val)
209 return GNUNET_NO;
210 if (0 != ibf->key_hash_sum[i].key_hash_val)
211 return GNUNET_NO;
212 if (0 != ibf->key_sum[i].key_val)
213 return GNUNET_NO;
214 }
215 return GNUNET_YES;
216}
217
218
219/**
220 * Decode and remove an element from the IBF, if possible.
221 *
222 * @param ibf the invertible bloom filter to decode
223 * @param ret_side sign of the cell's count where the decoded element came from.
224 * A negative sign indicates that the element was recovered
225 * resides in an IBF that was previously subtracted from.
226 * @param ret_id receives the hash code of the decoded element, if successful
227 * @return #GNUNET_YES if decoding an element was successful,
228 * #GNUNET_NO if the IBF is empty,
229 * #GNUNET_SYSERR if the decoding has failed
230 */
231int
232ibf_decode (struct InvertibleBloomFilter *ibf,
233 int *ret_side,
234 struct IBF_Key *ret_id)
235{
236 struct IBF_KeyHash hash;
237 int buckets[ibf->hash_num];
238
239 for (uint32_t i = 0; i < ibf->size; i++)
240 {
241 int hit;
242
243 /* we can only decode from pure buckets */
244 if ( (1 != ibf->count[i].count_val) &&
245 (-1 != ibf->count[i].count_val) )
246 continue;
247
248 hash.key_hash_val = IBF_KEY_HASH_VAL (ibf->key_sum[i]);
249
250 /* test if the hash matches the key */
251 if (hash.key_hash_val != ibf->key_hash_sum[i].key_hash_val)
252 continue;
253
254 /* test if key in bucket hits its own location,
255 * if not, the key hash was subject to collision */
256 hit = GNUNET_NO;
257 ibf_get_indices (ibf, ibf->key_sum[i], buckets);
258 for (int j = 0; j < ibf->hash_num; j++)
259 if (buckets[j] == i)
260 hit = GNUNET_YES;
261
262 if (GNUNET_NO == hit)
263 continue;
264
265 if (1 == ibf->count[i].count_val)
266 {
267 ibf->remote_decoded_count++;
268 }
269 else
270 {
271 ibf->local_decoded_count++;
272 }
273
274
275 if (NULL != ret_side)
276 *ret_side = ibf->count[i].count_val;
277 if (NULL != ret_id)
278 *ret_id = ibf->key_sum[i];
279
280 /* insert on the opposite side, effectively removing the element */
281 ibf_insert_into (ibf, ibf->key_sum[i], buckets, -ibf->count[i].count_val);
282
283 return GNUNET_YES;
284 }
285
286 if (GNUNET_YES == ibf_is_empty (ibf))
287 return GNUNET_NO;
288 return GNUNET_SYSERR;
289}
290
291
292/**
293 * Returns the minimal bytes needed to store the counter of the IBF
294 *
295 * @param ibf the IBF
296 */
297uint8_t
298ibf_get_max_counter (struct InvertibleBloomFilter *ibf)
299{
300 long long max_counter = 0;
301 for (uint64_t i = 0; i < ibf->size; i++)
302 {
303 if (ibf->count[i].count_val > max_counter)
304 {
305 max_counter = ibf->count[i].count_val;
306 }
307 }
308 return 64 - __builtin_clzll (max_counter);
309}
310
311
312/**
313 * Write buckets from an ibf to a buffer.
314 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
315 *
316 * @param ibf the ibf to write
317 * @param start with which bucket to start
318 * @param count how many buckets to write
319 * @param buf buffer to write the data to
320 * @param max bit length of a counter for unpacking
321 */
322void
323ibf_write_slice (const struct InvertibleBloomFilter *ibf,
324 uint32_t start,
325 uint64_t count,
326 void *buf,
327 uint8_t counter_max_length)
328{
329 struct IBF_Key *key_dst;
330 struct IBF_KeyHash *key_hash_dst;
331
332 GNUNET_assert (start + count <= ibf->size);
333
334 /* copy keys */
335 key_dst = (struct IBF_Key *) buf;
336 GNUNET_memcpy (key_dst,
337 ibf->key_sum + start,
338 count * sizeof(*key_dst));
339 key_dst += count;
340 /* copy key hashes */
341 key_hash_dst = (struct IBF_KeyHash *) key_dst;
342 GNUNET_memcpy (key_hash_dst,
343 ibf->key_hash_sum + start,
344 count * sizeof(*key_hash_dst));
345 key_hash_dst += count;
346
347 /* pack and copy counter */
348 pack_counter (ibf,
349 start,
350 count,
351 (uint8_t *) key_hash_dst,
352 counter_max_length);
353
354
355}
356
357
358/**
359 * Packs the counter to transmit only the smallest possible amount of bytes and
360 * preventing overflow of the counter
361 * @param ibf the ibf to write
362 * @param start with which bucket to start
363 * @param count how many buckets to write
364 * @param buf buffer to write the data to
365 * @param max bit length of a counter for unpacking
366 */
367
368void
369pack_counter (const struct InvertibleBloomFilter *ibf,
370 uint32_t start,
371 uint64_t count,
372 uint8_t *buf,
373 uint8_t counter_max_length)
374{
375 uint8_t store_size = 0;
376 uint8_t store = 0;
377 uint16_t byte_ctr = 0;
378
379 /**
380 * Iterate over IBF bucket
381 */
382 for (uint64_t i = start; i< (count + start);)
383 {
384 uint64_t count_val_to_write = ibf->count[i].count_val;
385 uint8_t count_len_to_write = counter_max_length;
386
387 /**
388 * Pack and compose counters to byte values
389 */
390 while ((count_len_to_write + store_size) >= 8)
391 {
392 uint8_t bit_shift = 0;
393
394 /**
395 * Shift bits if more than a byte has to be written
396 * or the store size is not empty
397 */
398 if ((store_size > 0) || (count_len_to_write > 8))
399 {
400 uint8_t bit_unused = 8 - store_size;
401 bit_shift = count_len_to_write - bit_unused;
402 store = store << bit_unused;
403 }
404
405 buf[byte_ctr] = ((count_val_to_write >> bit_shift) | store) & 0xFF;
406 byte_ctr++;
407 count_len_to_write -= (8 - store_size);
408 count_val_to_write = count_val_to_write & ((1ULL <<
409 count_len_to_write) - 1);
410 store = 0;
411 store_size = 0;
412 }
413 store = (store << count_len_to_write) | count_val_to_write;
414 store_size = store_size + count_len_to_write;
415 count_len_to_write = 0;
416 i++;
417 }
418
419 /**
420 * Pack data left in story before finishing
421 */
422 if (store_size > 0)
423 {
424 buf[byte_ctr] = store << (8 - store_size);
425 byte_ctr++;
426 }
427
428}
429
430
431/**
432 * Unpacks the counter to transmit only the smallest possible amount of bytes and
433 * preventing overflow of the counter
434 * @param ibf the ibf to write
435 * @param start with which bucket to start
436 * @param count how many buckets to write
437 * @param buf buffer to write the data to
438 * @param max bit length of a counter for unpacking
439 */
440
441void
442unpack_counter (const struct InvertibleBloomFilter *ibf,
443 uint32_t start,
444 uint64_t count,
445 uint8_t *buf,
446 uint8_t counter_max_length)
447{
448 uint64_t ibf_counter_ctr = 0;
449 uint64_t store = 0;
450 uint64_t store_bit_ctr = 0;
451 uint64_t byte_ctr = 0;
452
453 /**
454 * Iterate over received bytes
455 */
456 while (true)
457 {
458 uint8_t byte_read = buf[byte_ctr];
459 uint8_t bit_to_read_left = 8;
460 byte_ctr++;
461
462 /**
463 * Pack data left in story before finishing
464 */
465 while (true)
466 {
467 /**
468 * Stop decoding when end is reached
469 */
470 if (ibf_counter_ctr > (count - 1))
471 return;
472
473 /*
474 * Unpack the counter
475 */
476 if ((store_bit_ctr + bit_to_read_left) >= counter_max_length)
477 {
478 uint8_t bytes_used = counter_max_length - store_bit_ctr;
479 if (store_bit_ctr > 0)
480 {
481 store = store << bytes_used;
482 }
483
484 uint8_t bytes_to_shift = bit_to_read_left - bytes_used;
485 uint64_t counter_part = byte_read >> bytes_to_shift;
486 store = store | counter_part;
487 ibf->count[ibf_counter_ctr + start].count_val = store;
488 byte_read = byte_read & ((1 << bytes_to_shift) - 1);
489 bit_to_read_left -= bytes_used;
490 ibf_counter_ctr++;
491 store = 0;
492 store_bit_ctr = 0;
493 }
494 else
495 {
496 store_bit_ctr += bit_to_read_left;
497 if (0 == store)
498 {
499 store = byte_read;
500 }
501 else
502 {
503 store = store << bit_to_read_left;
504 store = store | byte_read;
505 }
506 break;
507 }
508 }
509 }
510}
511
512
513/**
514 * Read buckets from a buffer into an ibf.
515 *
516 * @param buf pointer to the buffer to read from
517 * @param start which bucket to start at
518 * @param count how many buckets to read
519 * @param ibf the ibf to read from
520 * @param max bit length of a counter for unpacking
521 */
522void
523ibf_read_slice (const void *buf,
524 uint32_t start,
525 uint64_t count,
526 struct InvertibleBloomFilter *ibf,
527 uint8_t counter_max_length)
528{
529 struct IBF_Key *key_src;
530 struct IBF_KeyHash *key_hash_src;
531 struct IBF_Count *count_src;
532
533 GNUNET_assert (count > 0);
534 GNUNET_assert (start + count <= ibf->size);
535
536 /* copy keys */
537 key_src = (struct IBF_Key *) buf;
538 GNUNET_memcpy (ibf->key_sum + start,
539 key_src,
540 count * sizeof *key_src);
541 key_src += count;
542 /* copy key hashes */
543 key_hash_src = (struct IBF_KeyHash *) key_src;
544 GNUNET_memcpy (ibf->key_hash_sum + start,
545 key_hash_src,
546 count * sizeof *key_hash_src);
547 key_hash_src += count;
548
549 /* copy and unpack counts */
550 count_src = (struct IBF_Count *) key_hash_src;
551 unpack_counter (ibf,start,count,(uint8_t *) count_src,counter_max_length);
552}
553
554
555/**
556 * Subtract ibf2 from ibf1, storing the result in ibf1.
557 * The two IBF's must have the same parameters size and hash_num.
558 *
559 * @param ibf1 IBF that is subtracted from
560 * @param ibf2 IBF that will be subtracted from ibf1
561 */
562void
563ibf_subtract (struct InvertibleBloomFilter *ibf1,
564 const struct InvertibleBloomFilter *ibf2)
565{
566 GNUNET_assert (ibf1->size == ibf2->size);
567 GNUNET_assert (ibf1->hash_num == ibf2->hash_num);
568
569 for (uint32_t i = 0; i < ibf1->size; i++)
570 {
571 ibf1->count[i].count_val -= ibf2->count[i].count_val;
572 ibf1->key_hash_sum[i].key_hash_val ^= ibf2->key_hash_sum[i].key_hash_val;
573 ibf1->key_sum[i].key_val ^= ibf2->key_sum[i].key_val;
574 }
575}
576
577
578/**
579 * Create a copy of an IBF, the copy has to be destroyed properly.
580 *
581 * @param ibf the IBF to copy
582 */
583struct InvertibleBloomFilter *
584ibf_dup (const struct InvertibleBloomFilter *ibf)
585{
586 struct InvertibleBloomFilter *copy;
587
588 copy = GNUNET_malloc (sizeof *copy);
589 copy->hash_num = ibf->hash_num;
590 copy->size = ibf->size;
591 copy->key_hash_sum = GNUNET_memdup (ibf->key_hash_sum,
592 ibf->size * sizeof(struct IBF_KeyHash));
593 copy->key_sum = GNUNET_memdup (ibf->key_sum,
594 ibf->size * sizeof(struct IBF_Key));
595 copy->count = GNUNET_memdup (ibf->count,
596 ibf->size * sizeof(struct IBF_Count));
597 return copy;
598}
599
600
601/**
602 * Destroy all resources associated with the invertible bloom filter.
603 * No more ibf_*-functions may be called on ibf after calling destroy.
604 *
605 * @param ibf the intertible bloom filter to destroy
606 */
607void
608ibf_destroy (struct InvertibleBloomFilter *ibf)
609{
610 GNUNET_free (ibf->key_sum);
611 GNUNET_free (ibf->key_hash_sum);
612 GNUNET_free (ibf->count);
613 GNUNET_free (ibf);
614}
diff --git a/src/setu/ibf.h b/src/setu/ibf.h
deleted file mode 100644
index 5628405dc..000000000
--- a/src/setu/ibf.h
+++ /dev/null
@@ -1,310 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.h
23 * @brief invertible bloom filter
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#ifndef GNUNET_CONSENSUS_IBF_H
29#define GNUNET_CONSENSUS_IBF_H
30
31#include "platform.h"
32#include "gnunet_util_lib.h"
33
34#ifdef __cplusplus
35extern "C"
36{
37#if 0 /* keep Emacsens' auto-indent happy */
38}
39#endif
40#endif
41
42
43/**
44 * Keys that can be inserted into and removed from an IBF.
45 */
46struct IBF_Key
47{
48 uint64_t key_val;
49};
50
51
52/**
53 * Hash of an IBF key.
54 */
55struct IBF_KeyHash
56{
57 uint32_t key_hash_val;
58};
59
60
61/**
62 * Type of the count field of IBF buckets.
63 */
64struct IBF_Count
65{
66 int64_t count_val;
67};
68
69
70/**
71 * Size of one ibf bucket in bytes
72 */
73#define IBF_BUCKET_SIZE (sizeof(struct IBF_Count) + sizeof(struct IBF_Key) \
74 + sizeof(struct IBF_KeyHash))
75
76
77/**
78 * Invertible bloom filter (IBF).
79 *
80 * An IBF is a counting bloom filter that has the ability to restore
81 * the hashes of its stored elements with high probability.
82 */
83struct InvertibleBloomFilter
84{
85 /**
86 * How many cells does this IBF have?
87 */
88 uint32_t size;
89
90 /**
91 * In how many cells do we hash one element?
92 * Usually 4 or 3.
93 */
94 uint8_t hash_num;
95
96 /**
97 * If an IBF is decoded this count stores how many
98 * elements are on the local site. This is used
99 * to estimate the set difference on a site
100 */
101 int local_decoded_count;
102
103 /**
104 * If an IBF is decoded this count stores how many
105 * elements are on the remote site. This is used
106 * to estimate the set difference on a site
107 */
108 int remote_decoded_count;
109
110 /**
111 * Xor sums of the elements' keys, used to identify the elements.
112 * Array of 'size' elements.
113 */
114 struct IBF_Key *key_sum;
115
116 /**
117 * Xor sums of the hashes of the keys of inserted elements.
118 * Array of 'size' elements.
119 */
120 struct IBF_KeyHash *key_hash_sum;
121
122 /**
123 * How many times has a bucket been hit?
124 * Can be negative, as a result of IBF subtraction.
125 * Array of 'size' elements.
126 */
127 struct IBF_Count *count;
128};
129
130
131/**
132 * Write buckets from an ibf to a buffer.
133 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
134 *
135 * @param ibf the ibf to write
136 * @param start with which bucket to start
137 * @param count how many buckets to write
138 * @param buf buffer to write the data to
139 */
140void
141ibf_write_slice (const struct InvertibleBloomFilter *ibf,
142 uint32_t start,
143 uint64_t count,
144 void *buf,
145 uint8_t counter_max_length);
146
147
148/**
149 * Read buckets from a buffer into an ibf.
150 *
151 * @param buf pointer to the buffer to read from
152 * @param start which bucket to start at
153 * @param count how many buckets to read
154 * @param ibf the ibf to write to
155 */
156void
157ibf_read_slice (const void *buf,
158 uint32_t start,
159 uint64_t count,
160 struct InvertibleBloomFilter *ibf,
161 uint8_t counter_max_length);
162
163
164/**
165 * Create a key from a hashcode.
166 *
167 * @param hash the hashcode
168 * @return a key
169 */
170struct IBF_Key
171ibf_key_from_hashcode (const struct GNUNET_HashCode *hash);
172
173
174/**
175 * Create a hashcode from a key, by replicating the key
176 * until the hascode is filled
177 *
178 * @param key the key
179 * @param dst hashcode to store the result in
180 */
181void
182ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst);
183
184
185/**
186 * Create an invertible bloom filter.
187 *
188 * @param size number of IBF buckets
189 * @param hash_num number of buckets one element is hashed in, usually 3 or 4
190 * @return the newly created invertible bloom filter, NULL on error
191 */
192struct InvertibleBloomFilter *
193ibf_create (uint32_t size, uint8_t hash_num);
194
195
196/**
197 * Insert a key into an IBF.
198 *
199 * @param ibf the IBF
200 * @param key the element's hash code
201 */
202void
203ibf_insert (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
204
205
206/**
207 * Remove a key from an IBF.
208 *
209 * @param ibf the IBF
210 * @param key the element's hash code
211 */
212void
213ibf_remove (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
214
215
216/**
217 * Subtract ibf2 from ibf1, storing the result in ibf1.
218 * The two IBF's must have the same parameters size and hash_num.
219 *
220 * @param ibf1 IBF that is subtracted from
221 * @param ibf2 IBF that will be subtracted from ibf1
222 */
223void
224ibf_subtract (struct InvertibleBloomFilter *ibf1,
225 const struct InvertibleBloomFilter *ibf2);
226
227
228/**
229 * Decode and remove an element from the IBF, if possible.
230 *
231 * @param ibf the invertible bloom filter to decode
232 * @param ret_side sign of the cell's count where the decoded element came from.
233 * A negative sign indicates that the element was recovered
234 * resides in an IBF that was previously subtracted from.
235 * @param ret_id receives the hash code of the decoded element, if successful
236 * @return #GNUNET_YES if decoding an element was successful,
237 * #GNUNET_NO if the IBF is empty,
238 * #GNUNET_SYSERR if the decoding has failed
239 */
240int
241ibf_decode (struct InvertibleBloomFilter *ibf,
242 int *ret_side,
243 struct IBF_Key *ret_id);
244
245
246/**
247 * Create a copy of an IBF, the copy has to be destroyed properly.
248 *
249 * @param ibf the IBF to copy
250 */
251struct InvertibleBloomFilter *
252ibf_dup (const struct InvertibleBloomFilter *ibf);
253
254
255/**
256 * Destroy all resources associated with the invertible bloom filter.
257 * No more ibf_*-functions may be called on ibf after calling destroy.
258 *
259 * @param ibf the intertible bloom filter to destroy
260 */
261void
262ibf_destroy (struct InvertibleBloomFilter *ibf);
263
264uint8_t
265ibf_get_max_counter (struct InvertibleBloomFilter *ibf);
266
267
268/**
269 * Packs the counter to transmit only the smallest possible amount of bytes and
270 * preventing overflow of the counter
271 * @param ibf the ibf to write
272 * @param start with which bucket to start
273 * @param count how many buckets to write
274 * @param buf buffer to write the data to
275 * @param max bit length of a counter for unpacking
276 */
277
278void
279pack_counter (const struct InvertibleBloomFilter *ibf,
280 uint32_t start,
281 uint64_t count,
282 uint8_t *buf,
283 uint8_t counter_max_length);
284
285/**
286 * Unpacks the counter to transmit only the smallest possible amount of bytes and
287 * preventing overflow of the counter
288 * @param ibf the ibf to write
289 * @param start with which bucket to start
290 * @param count how many buckets to write
291 * @param buf buffer to write the data to
292 * @param max bit length of a counter for unpacking
293 */
294
295void
296unpack_counter (const struct InvertibleBloomFilter *ibf,
297 uint32_t start,
298 uint64_t count,
299 uint8_t *buf,
300 uint8_t counter_max_length);
301
302
303#if 0 /* keep Emacsens' auto-indent happy */
304{
305#endif
306#ifdef __cplusplus
307}
308#endif
309
310#endif
diff --git a/src/setu/ibf_sim.c b/src/setu/ibf_sim.c
deleted file mode 100644
index 6415d00e1..000000000
--- a/src/setu/ibf_sim.c
+++ /dev/null
@@ -1,142 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf_sim.c
23 * @brief implementation of simulation for invertible bloom filter
24 * @author Florian Dold
25 *
26 * This code was used for some internal experiments, it is not
27 * build or shipped as part of the GNUnet system.
28 */
29#include <stdlib.h>
30#include <stdio.h>
31#include <string.h>
32
33#define MAX_IBF_DECODE 16
34
35/* report average over how many rounds? */
36#define ROUNDS 100000
37
38/* enable one of the three below */
39// simple fix
40#define FIX1 0
41// possibly slightly better fix for large IBF_DECODE values
42#define FIX2 1
43
44// SIGCOMM algorithm
45#define STRATA 0
46
47// print each value?
48#define VERBOSE 0
49// avoid assembly? (ASM is about 50% faster)
50#define SLOW 0
51
52int
53main (int argc, char **argv)
54{
55 unsigned int round;
56 unsigned int buckets[31]; // max is 2^31 as 'random' returns only between 0 and 2^31
57 unsigned int i;
58 int j;
59 unsigned int r;
60 unsigned int ret;
61 unsigned long long total;
62 unsigned int want;
63 double predict;
64
65 srandom (time (NULL));
66 total = 0;
67 want = atoi (argv[1]);
68 for (round = 0; round < ROUNDS; round++)
69 {
70 memset (buckets, 0, sizeof(buckets));
71 for (i = 0; i < want; i++)
72 {
73 /* FIXME: might want to use 'better' PRNG to avoid
74 PRNG-induced biases */
75 r = random ();
76 if (0 == r)
77 continue;
78#if SLOW
79 for (j = 0; (j < 31) && (0 == (r & (1 << j))); j++)
80 ;
81#else
82 /* use assembly / gcc */
83 j = __builtin_ffs (r) - 1;
84#endif
85 buckets[j]++;
86 }
87 ret = 0;
88 predict = 0.0;
89 for (j = 31; j >= 0; j--)
90 {
91#if FIX1
92 /* improved algorithm, for 1000 elements with IBF-DECODE 8, I
93 get 990/1000 elements on average over 1 million runs; key
94 idea being to stop short of the 'last' possible IBF as
95 otherwise a "lowball" per-chance would unduely influence the
96 result */if ((j > 0) &&
97 (buckets[j - 1] > MAX_IBF_DECODE))
98 {
99 ret *= (1 << (j + 1));
100 break;
101 }
102#endif
103#if FIX2
104 /* another improvement: don't just always cut off the last one,
105 but rather try to predict based on all previous values where
106 that "last" one is; additional prediction can only really
107 work if MAX_IBF_DECODE is sufficiently high */
108 if ((j > 0) &&
109 ((buckets[j - 1] > MAX_IBF_DECODE) ||
110 (predict > MAX_IBF_DECODE)))
111 {
112 ret *= (1 << (j + 1));
113 break;
114 }
115#endif
116#if STRATA
117 /* original algorithm, for 1000 elements with IBF-DECODE 8,
118 I get 920/1000 elements on average over 1 million runs */
119 if (buckets[j] > MAX_IBF_DECODE)
120 {
121 ret *= (1 << (j + 1));
122 break;
123 }
124#endif
125 ret += buckets[j];
126 predict = (buckets[j] + 2.0 * predict) / 2.0;
127 }
128#if VERBOSE
129 fprintf (stderr, "%u ", ret);
130#endif
131 total += ret;
132 }
133 fprintf (stderr, "\n");
134 fprintf (stdout, "average %llu\n", total / ROUNDS);
135 return 0;
136}
137
138
139/* TODO: should calculate stddev of the results to also be able to
140 say something about the stability of the results, outside of
141 large-scale averages -- gaining 8% precision at the expense of
142 50% additional variance might not be worth it... */
diff --git a/src/setu/perf_setu_api.c b/src/setu/perf_setu_api.c
deleted file mode 100644
index 7f4d64f74..000000000
--- a/src/setu/perf_setu_api.c
+++ /dev/null
@@ -1,473 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_setu_api.c
23 * @brief testcase for setu_api.c
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_testing_lib.h"
30#include "gnunet_setu_service.h"
31#include <sys/sysinfo.h>
32#include <pthread.h>
33
34
35static struct GNUNET_PeerIdentity local_id;
36
37static struct GNUNET_HashCode app_id;
38
39static struct GNUNET_SETU_Handle *set1;
40
41static struct GNUNET_SETU_Handle *set2;
42
43static struct GNUNET_SETU_ListenHandle *listen_handle;
44
45static struct GNUNET_SETU_OperationHandle *oh1;
46
47static struct GNUNET_SETU_OperationHandle *oh2;
48
49static const struct GNUNET_CONFIGURATION_Handle *config;
50
51static int ret;
52
53static struct GNUNET_SCHEDULER_Task *tt;
54
55
56/**
57 * Handles configuration file for setu performance test
58 *
59 */
60static struct GNUNET_CONFIGURATION_Handle *setu_cfg;
61
62
63static void
64result_cb_set1 (void *cls,
65 const struct GNUNET_SETU_Element *element,
66 uint64_t size,
67 enum GNUNET_SETU_Status status)
68{
69 switch (status)
70 {
71 case GNUNET_SETU_STATUS_ADD_LOCAL:
72 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: got element\n");
73 break;
74
75 case GNUNET_SETU_STATUS_FAILURE:
76 GNUNET_break (0);
77 oh1 = NULL;
78 fprintf (stderr, "set 1: received failure status!\n");
79 ret = 1;
80 if (NULL != tt)
81 {
82 GNUNET_SCHEDULER_cancel (tt);
83 tt = NULL;
84 }
85 GNUNET_SCHEDULER_shutdown ();
86 break;
87
88 case GNUNET_SETU_STATUS_DONE:
89 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: done\n");
90 oh1 = NULL;
91 if (NULL != set1)
92 {
93 GNUNET_SETU_destroy (set1);
94 set1 = NULL;
95 }
96 if (NULL == set2)
97 {
98 GNUNET_SCHEDULER_cancel (tt);
99 tt = NULL;
100 GNUNET_SCHEDULER_shutdown ();
101 }
102 break;
103
104 default:
105 GNUNET_assert (0);
106 }
107}
108
109
110static void
111result_cb_set2 (void *cls,
112 const struct GNUNET_SETU_Element *element,
113 uint64_t size,
114 enum GNUNET_SETU_Status status)
115{
116 switch (status)
117 {
118 case GNUNET_SETU_STATUS_ADD_LOCAL:
119 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: got element\n");
120 break;
121
122 case GNUNET_SETU_STATUS_FAILURE:
123 GNUNET_break (0);
124 oh2 = NULL;
125 fprintf (stderr, "set 2: received failure status\n");
126 GNUNET_SCHEDULER_shutdown ();
127 ret = 1;
128 break;
129
130 case GNUNET_SETU_STATUS_DONE:
131 oh2 = NULL;
132 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: done\n");
133 GNUNET_SETU_destroy (set2);
134 set2 = NULL;
135 if (NULL == set1)
136 {
137 GNUNET_SCHEDULER_cancel (tt);
138 tt = NULL;
139 GNUNET_SCHEDULER_shutdown ();
140 }
141 break;
142
143 default:
144 GNUNET_assert (0);
145 }
146}
147
148
149static void
150listen_cb (void *cls,
151 const struct GNUNET_PeerIdentity *other_peer,
152 const struct GNUNET_MessageHeader *context_msg,
153 struct GNUNET_SETU_Request *request)
154{
155 GNUNET_assert (NULL != context_msg);
156 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
157 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "listen cb called\n");
158 oh2 = GNUNET_SETU_accept (request,
159 (struct GNUNET_SETU_Option[]){ 0 },
160 &result_cb_set2,
161 NULL);
162 GNUNET_SETU_commit (oh2, set2);
163}
164
165
166/**
167 * Start the set operation.
168 *
169 * @param cls closure, unused
170 */
171static void
172start (void *cls)
173{
174 struct GNUNET_MessageHeader context_msg;
175
176 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting reconciliation\n");
177 context_msg.size = htons (sizeof context_msg);
178 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
179 listen_handle = GNUNET_SETU_listen (config,
180 &app_id,
181 &listen_cb,
182 NULL);
183 oh1 = GNUNET_SETU_prepare (&local_id,
184 &app_id,
185 &context_msg,
186 (struct GNUNET_SETU_Option[]){ 0 },
187 &result_cb_set1,
188 NULL);
189 GNUNET_SETU_commit (oh1, set1);
190}
191
192
193/**
194 * Generate random byte stream
195 */
196
197unsigned char *
198gen_rdm_bytestream (size_t num_bytes)
199{
200 unsigned char *stream = GNUNET_malloc (num_bytes);
201 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, stream, num_bytes);
202 return stream;
203}
204
205
206/**
207 * Generate random sets
208 */
209
210static void
211initRandomSets (int overlap, int set1_size, int set2_size, int
212 element_size_in_bytes)
213{
214 struct GNUNET_SETU_Element element;
215 element.element_type = 0;
216
217 // Add elements to both sets
218 for (int i = 0; i < overlap; i++)
219 {
220 element.data = gen_rdm_bytestream (element_size_in_bytes);
221 element.size = element_size_in_bytes;
222 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
223 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
224 set1_size--;
225 set2_size--;
226 }
227 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in both sets\n");
228
229 // Add other elements to set 1
230 while (set1_size>0)
231 {
232 element.data = gen_rdm_bytestream (element_size_in_bytes);
233 element.size = element_size_in_bytes;
234 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
235 set1_size--;
236 }
237 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in set1\n");
238
239 // Add other elements to set 2
240 while (set2_size > 0)
241 {
242 element.data = gen_rdm_bytestream (element_size_in_bytes);
243 element.size = element_size_in_bytes;
244
245 if (set2_size != 1)
246 {
247 GNUNET_SETU_add_element (set2, &element,NULL, NULL);
248 }
249 else
250 {
251 GNUNET_SETU_add_element (set2, &element,&start, NULL);
252 }
253
254 set2_size--;
255 }
256 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in set2\n");
257}
258
259
260/**
261 * Function run on timeout.
262 *
263 * @param cls closure
264 */
265static void
266timeout_fail (void *cls)
267{
268 tt = NULL;
269 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, "Testcase failed with timeout\n");
270 GNUNET_SCHEDULER_shutdown ();
271 ret = 1;
272}
273
274
275/**
276 * Function run on shutdown.
277 *
278 * @param cls closure
279 */
280static void
281do_shutdown (void *cls)
282{
283 if (NULL != tt)
284 {
285 GNUNET_SCHEDULER_cancel (tt);
286 tt = NULL;
287 }
288 if (NULL != oh1)
289 {
290 GNUNET_SETU_operation_cancel (oh1);
291 oh1 = NULL;
292 }
293 if (NULL != oh2)
294 {
295 GNUNET_SETU_operation_cancel (oh2);
296 oh2 = NULL;
297 }
298 if (NULL != set1)
299 {
300 GNUNET_SETU_destroy (set1);
301 set1 = NULL;
302 }
303 if (NULL != set2)
304 {
305 GNUNET_SETU_destroy (set2);
306 set2 = NULL;
307 }
308 if (NULL != listen_handle)
309 {
310 GNUNET_SETU_listen_cancel (listen_handle);
311 listen_handle = NULL;
312 }
313}
314
315
316/**
317 * Signature of the 'main' function for a (single-peer) testcase that
318 * is run using 'GNUNET_TESTING_peer_run'.
319 *
320 * @param cls closure
321 * @param cfg configuration of the peer that was started
322 * @param peer identity of the peer that was created
323 */
324static void
325run (void *cls,
326 const struct GNUNET_CONFIGURATION_Handle *cfg,
327 struct GNUNET_TESTING_Peer *peer)
328{
329 struct GNUNET_SETU_OperationHandle *my_oh;
330
331 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
332 "Running preparatory tests\n");
333 tt = GNUNET_SCHEDULER_add_delayed (
334 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
335 &timeout_fail,
336 NULL);
337 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
338
339 config = cfg;
340 GNUNET_assert (GNUNET_OK == GNUNET_CRYPTO_get_peer_identity (cfg,
341 &local_id));
342 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
343 "my id (from CRYPTO): %s\n",
344 GNUNET_i2s (&local_id));
345 GNUNET_TESTING_peer_get_identity (peer,
346 &local_id);
347 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
348 "my id (from TESTING): %s\n",
349 GNUNET_i2s (&local_id));
350 set1 = GNUNET_SETU_create (cfg);
351 set2 = GNUNET_SETU_create (cfg);
352 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
353 "Created sets %p and %p for union operation\n",
354 set1,
355 set2);
356 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
357
358 /* test if canceling an uncommitted request works! */
359 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
360 "Launching and instantly stopping set operation\n");
361 my_oh = GNUNET_SETU_prepare (&local_id,
362 &app_id,
363 NULL,
364 (struct GNUNET_SETU_Option[]){ 0 },
365 NULL,
366 NULL);
367 GNUNET_SETU_operation_cancel (my_oh);
368
369 /* test the real set reconciliation */
370 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
371 "Running real set-reconciliation\n");
372 // init_set1 ();
373 // limit ~23800 element total
374 initRandomSets (490, 500,500,32);
375}
376
377
378void
379perf_thread ()
380{
381 GNUNET_TESTING_service_run ("perf_setu_api",
382 "arm",
383 "test_setu.conf",
384 &run,
385 NULL);
386
387}
388
389
390static void
391run_petf_thread (int total_runs)
392{
393 int core_count = get_nprocs_conf ();
394 pid_t child_pid, wpid;
395 int status = 0;
396
397// Father code (before child processes start)
398 for (int processed = 0; processed < total_runs;)
399 {
400 for (int id = 0; id < core_count; id++)
401 {
402 if (processed >= total_runs)
403 break;
404
405 if ((child_pid = fork ()) == 0)
406 {
407 perf_thread ();
408 exit (0);
409 }
410 processed += 1;
411 }
412 while ((wpid = wait (&status)) > 0)
413 ;
414
415 }
416}
417
418
419static void
420execute_perf ()
421{
422
423 /**
424 * Erase statfile
425 */
426 remove ("perf_stats.csv");
427 remove ("perf_failure_bucket_number_factor.csv");
428 for (int out_out_ctr = 3; out_out_ctr <= 3; out_out_ctr++)
429 {
430
431 for (int out_ctr = 20; out_ctr <= 20; out_ctr++)
432 {
433 float base = 0.1;
434 float x = out_ctr * base;
435 char factor[10];
436 char *buffer = gcvt (x, 4, factor);
437 setu_cfg = GNUNET_CONFIGURATION_create ();
438 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "IBF",
439 "BUCKET_NUMBER_FACTOR",
440 buffer); // Factor default=4
441 GNUNET_CONFIGURATION_set_value_number (setu_cfg, "IBF",
442 "NUMBER_PER_BUCKET", 3); // K default=4
443 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "PERFORMANCE",
444 "TRADEOFF", "2"); // default=0.25
445 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "PERFORMANCE",
446 "MAX_SET_DIFF_FACTOR_DIFFERENTIAL",
447 "20000"); // default=0.25
448 GNUNET_CONFIGURATION_set_value_number (setu_cfg, "BOUNDARIES",
449 "UPPER_ELEMENT", 5000);
450
451
452 if (GNUNET_OK != GNUNET_CONFIGURATION_write (setu_cfg, "perf_setu.conf"))
453 GNUNET_log (
454 GNUNET_ERROR_TYPE_ERROR,
455 _ ("Failed to write subsystem default identifier map'.\n"));
456 run_petf_thread (100);
457 }
458
459 }
460 return;
461}
462
463
464int
465main (int argc, char **argv)
466{
467
468 GNUNET_log_setup ("perf_setu_api",
469 "WARNING",
470 NULL);
471 execute_perf ();
472 return 0;
473}
diff --git a/src/setu/plugin_block_setu_test.c b/src/setu/plugin_block_setu_test.c
deleted file mode 100644
index fd0c8a680..000000000
--- a/src/setu/plugin_block_setu_test.c
+++ /dev/null
@@ -1,123 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file setu/plugin_block_setu_test.c
23 * @brief set test block, recognizes elements with non-zero first byte as invalid
24 * @author Christian Grothoff
25 */
26
27#include "platform.h"
28#include "gnunet_block_plugin.h"
29#include "gnunet_block_group_lib.h"
30
31
32/**
33 * Function called to validate a reply or a request. For
34 * request evaluation, simply pass "NULL" for the reply_block.
35 *
36 * @param cls closure
37 * @param ctx block context
38 * @param type block type
39 * @param group block group to use
40 * @param eo control flags
41 * @param query original query (hash)
42 * @param xquery extrended query data (can be NULL, depending on type)
43 * @param xquery_size number of bytes in xquery
44 * @param reply_block response to validate
45 * @param reply_block_size number of bytes in reply block
46 * @return characterization of result
47 */
48static enum GNUNET_BLOCK_EvaluationResult
49block_plugin_setu_test_evaluate (void *cls,
50 struct GNUNET_BLOCK_Context *ctx,
51 enum GNUNET_BLOCK_Type type,
52 struct GNUNET_BLOCK_Group *group,
53 enum GNUNET_BLOCK_EvaluationOptions eo,
54 const struct GNUNET_HashCode *query,
55 const void *xquery,
56 size_t xquery_size,
57 const void *reply_block,
58 size_t reply_block_size)
59{
60 if ((NULL == reply_block) ||
61 (reply_block_size == 0) ||
62 (0 != ((char *) reply_block)[0]))
63 return GNUNET_BLOCK_EVALUATION_RESULT_INVALID;
64 return GNUNET_BLOCK_EVALUATION_OK_MORE;
65}
66
67
68/**
69 * Function called to obtain the key for a block.
70 *
71 * @param cls closure
72 * @param type block type
73 * @param block block to get the key for
74 * @param block_size number of bytes in block
75 * @param key set to the key (query) for the given block
76 * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
77 * (or if extracting a key from a block of this type does not work)
78 */
79static int
80block_plugin_setu_test_get_key (void *cls,
81 enum GNUNET_BLOCK_Type type,
82 const void *block,
83 size_t block_size,
84 struct GNUNET_HashCode *key)
85{
86 return GNUNET_SYSERR;
87}
88
89
90/**
91 * Entry point for the plugin.
92 */
93void *
94libgnunet_plugin_block_setu_test_init (void *cls)
95{
96 static enum GNUNET_BLOCK_Type types[] = {
97 GNUNET_BLOCK_TYPE_SETU_TEST,
98 GNUNET_BLOCK_TYPE_ANY /* end of list */
99 };
100 struct GNUNET_BLOCK_PluginFunctions *api;
101
102 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
103 api->evaluate = &block_plugin_setu_test_evaluate;
104 api->get_key = &block_plugin_setu_test_get_key;
105 api->types = types;
106 return api;
107}
108
109
110/**
111 * Exit point from the plugin.
112 */
113void *
114libgnunet_plugin_block_setu_test_done (void *cls)
115{
116 struct GNUNET_BLOCK_PluginFunctions *api = cls;
117
118 GNUNET_free (api);
119 return NULL;
120}
121
122
123/* end of plugin_block_setu_test.c */
diff --git a/src/setu/setu.conf.in b/src/setu/setu.conf.in
deleted file mode 100644
index 6c48f6156..000000000
--- a/src/setu/setu.conf.in
+++ /dev/null
@@ -1,12 +0,0 @@
1[setu]
2START_ON_DEMAND = @START_ON_DEMAND@
3@UNIXONLY@PORT = 2106
4HOSTNAME = localhost
5BINARY = gnunet-service-setu
6ACCEPT_FROM = 127.0.0.1;
7ACCEPT_FROM6 = ::1;
8UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-setu.sock
9UNIX_MATCH_UID = YES
10UNIX_MATCH_GID = YES
11
12#PREFIX = valgrind
diff --git a/src/setu/setu.h b/src/setu/setu.h
deleted file mode 100644
index 7b606f12c..000000000
--- a/src/setu/setu.h
+++ /dev/null
@@ -1,365 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2014, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/set.h
22 * @brief messages used for the set union api
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef SET_H
27#define SET_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31#include "gnunet_set_service.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message sent by the client to the service to ask starting
37 * a new set to perform operations with. Includes the desired
38 * set operation type.
39 */
40struct GNUNET_SETU_CreateMessage
41{
42 /**
43 * Type: #GNUNET_MESSAGE_TYPE_SETU_CREATE
44 */
45 struct GNUNET_MessageHeader header;
46
47};
48
49
50/**
51 * Message sent by the client to the service to start listening for
52 * incoming requests to perform a certain type of set operation for a
53 * certain type of application.
54 */
55struct GNUNET_SETU_ListenMessage
56{
57 /**
58 * Type: #GNUNET_MESSAGE_TYPE_SETU_LISTEN
59 */
60 struct GNUNET_MessageHeader header;
61
62 /**
63 * Always zero.
64 */
65 uint32_t reserved GNUNET_PACKED;
66
67 /**
68 * application id
69 */
70 struct GNUNET_HashCode app_id;
71};
72
73
74/**
75 * Message sent by a listening client to the service to accept
76 * performing the operation with the other peer.
77 */
78struct GNUNET_SETU_AcceptMessage
79{
80 /**
81 * Type: #GNUNET_MESSAGE_TYPE_SETU_ACCEPT
82 */
83 struct GNUNET_MessageHeader header;
84
85 /**
86 * ID of the incoming request we want to accept.
87 */
88 uint32_t accept_reject_id GNUNET_PACKED;
89
90 /**
91 * Request ID to identify responses.
92 */
93 uint32_t request_id GNUNET_PACKED;
94
95 /**
96 * Always use delta operation instead of sending full sets,
97 * even it it's less efficient.
98 */
99 uint8_t force_delta;
100
101 /**
102 * Always send full sets, even if delta operations would
103 * be more efficient.
104 */
105 uint8_t force_full;
106
107 /**
108 * #GNUNET_YES to fail operations where Byzantine faults
109 * are suspected
110 */
111 uint8_t byzantine;
112
113 /**
114 * #GNUNET_YES to also send back set elements we are sending to
115 * the remote peer.
116 */
117 uint8_t symmetric;
118
119 /**
120 * Lower bound for the set size, used only when
121 * byzantine mode is enabled.
122 */
123 uint32_t byzantine_lower_bound;
124
125
126 /**
127 * Upper bound for the set size, used only when
128 * byzantine mode is enabled.
129 */
130 uint64_t byzantine_upper_bond;
131
132 /**
133 * Bandwidth latency tradeoff determines how much bytes a single RTT is
134 * worth, which is a performance setting
135 */
136 uint64_t bandwidth_latency_tradeoff;
137
138 /**
139 * The factor determines the number of buckets an IBF has which is
140 * multiplied by the estimated setsize default: 2
141 */
142 uint64_t ibf_bucket_number_factor;
143
144 /**
145 * This setting determines to how many IBF buckets an single elements
146 * is mapped to.
147 */
148 uint64_t ibf_number_of_buckets_per_element;
149
150};
151
152
153/**
154 * Message sent by a listening client to the service to reject
155 * performing the operation with the other peer.
156 */
157struct GNUNET_SETU_RejectMessage
158{
159 /**
160 * Type: #GNUNET_MESSAGE_TYPE_SETU_REJECT
161 */
162 struct GNUNET_MessageHeader header;
163
164 /**
165 * ID of the incoming request we want to reject.
166 */
167 uint32_t accept_reject_id GNUNET_PACKED;
168};
169
170
171/**
172 * A request for an operation with another client.
173 */
174struct GNUNET_SETU_RequestMessage
175{
176 /**
177 * Type: #GNUNET_MESSAGE_TYPE_SETU_REQUEST.
178 */
179 struct GNUNET_MessageHeader header;
180
181 /**
182 * ID of the to identify the request when accepting or
183 * rejecting it.
184 */
185 uint32_t accept_id GNUNET_PACKED;
186
187 /**
188 * Identity of the requesting peer.
189 */
190 struct GNUNET_PeerIdentity peer_id;
191
192 /* rest: context message, that is, application-specific
193 message to convince listener to pick up */
194};
195
196
197/**
198 * Message sent by client to service to initiate a set operation as a
199 * client (not as listener). A set (which determines the operation
200 * type) must already exist in association with this client.
201 */
202struct GNUNET_SETU_EvaluateMessage
203{
204 /**
205 * Type: #GNUNET_MESSAGE_TYPE_SETU_EVALUATE
206 */
207 struct GNUNET_MessageHeader header;
208
209 /**
210 * Id of our set to evaluate, chosen implicitly by the client when it
211 * calls #GNUNET_SETU_commit().
212 */
213 uint32_t request_id GNUNET_PACKED;
214
215 /**
216 * Peer to evaluate the operation with
217 */
218 struct GNUNET_PeerIdentity target_peer;
219
220 /**
221 * Application id
222 */
223 struct GNUNET_HashCode app_id;
224
225 /**
226 * Always use delta operation instead of sending full sets,
227 * even it it's less efficient.
228 */
229 uint8_t force_delta;
230
231 /**
232 * Always send full sets, even if delta operations would
233 * be more efficient.
234 */
235 uint8_t force_full;
236
237 /**
238 * #GNUNET_YES to fail operations where Byzantine faults
239 * are suspected
240 */
241 uint8_t byzantine;
242
243 /**
244 * Also return set elements we are sending to the remote peer.
245 */
246 uint8_t symmetric;
247
248 /**
249 * Lower bound for the set size, used only when
250 * byzantine mode is enabled.
251 */
252 uint32_t byzantine_lower_bound;
253
254 /**
255 * Upper bound for the set size, used only when
256 * byzantine mode is enabled.
257 */
258 uint64_t byzantine_upper_bond;
259
260 /**
261 * Bandwidth latency tradeoff determines how much bytes a single RTT is
262 * worth, which is a performance setting
263 */
264 uint64_t bandwidth_latency_tradeoff;
265
266 /**
267 * The factor determines the number of buckets an IBF has which is
268 * multiplied by the estimated setsize default: 2
269 */
270 uint64_t ibf_bucket_number_factor;
271
272 /**
273 * This setting determines to how many IBF buckets an single elements
274 * is mapped to.
275 */
276 uint64_t ibf_number_of_buckets_per_element;
277
278 /* rest: context message, that is, application-specific
279 message to convince listener to pick up */
280};
281
282
283/**
284 * Message sent by the service to the client to indicate an
285 * element that is removed (set intersection) or added
286 * (set union) or part of the final result, depending on
287 * options specified for the operation.
288 */
289struct GNUNET_SETU_ResultMessage
290{
291 /**
292 * Type: #GNUNET_MESSAGE_TYPE_SETU_RESULT
293 */
294 struct GNUNET_MessageHeader header;
295
296 /**
297 * Current set size.
298 */
299 uint64_t current_size;
300
301 /**
302 * id the result belongs to
303 */
304 uint32_t request_id GNUNET_PACKED;
305
306 /**
307 * Was the evaluation successful? Contains
308 * an `enum GNUNET_SETU_Status` in NBO.
309 */
310 uint16_t result_status GNUNET_PACKED;
311
312 /**
313 * Type of the element attached to the message, if any.
314 */
315 uint16_t element_type GNUNET_PACKED;
316
317 /* rest: the actual element */
318};
319
320
321/**
322 * Message sent by client to the service to add
323 * an element to the set.
324 */
325struct GNUNET_SETU_ElementMessage
326{
327 /**
328 * Type: #GNUNET_MESSAGE_TYPE_SETU_ADD
329 */
330 struct GNUNET_MessageHeader header;
331
332 /**
333 * Type of the element to add or remove.
334 */
335 uint16_t element_type GNUNET_PACKED;
336
337 /**
338 * For alignment, always zero.
339 */
340 uint16_t reserved GNUNET_PACKED;
341
342 /* rest: the actual element */
343};
344
345
346/**
347 * Sent to the service by the client in order to cancel a set operation.
348 */
349struct GNUNET_SETU_CancelMessage
350{
351 /**
352 * Type: #GNUNET_MESSAGE_TYPE_SETU_CANCEL
353 */
354 struct GNUNET_MessageHeader header;
355
356 /**
357 * ID of the request we want to cancel.
358 */
359 uint32_t request_id GNUNET_PACKED;
360};
361
362
363GNUNET_NETWORK_STRUCT_END
364
365#endif
diff --git a/src/setu/setu_api.c b/src/setu/setu_api.c
deleted file mode 100644
index faa57aaba..000000000
--- a/src/setu/setu_api.c
+++ /dev/null
@@ -1,937 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2016, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/setu_api.c
22 * @brief api for the set union service
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_protocols.h"
30#include "gnunet_setu_service.h"
31#include "setu.h"
32
33
34#define LOG(kind, ...) GNUNET_log_from (kind, "set-api", __VA_ARGS__)
35
36/**
37 * Opaque handle to a set.
38 */
39struct GNUNET_SETU_Handle
40{
41 /**
42 * Message queue for @e client.
43 */
44 struct GNUNET_MQ_Handle *mq;
45
46 /**
47 * Linked list of operations on the set.
48 */
49 struct GNUNET_SETU_OperationHandle *ops_head;
50
51 /**
52 * Linked list of operations on the set.
53 */
54 struct GNUNET_SETU_OperationHandle *ops_tail;
55
56 /**
57 * Should the set be destroyed once all operations are gone?
58 * #GNUNET_SYSERR if #GNUNET_SETU_destroy() must raise this flag,
59 * #GNUNET_YES if #GNUNET_SETU_destroy() did raise this flag.
60 */
61 int destroy_requested;
62
63 /**
64 * Has the set become invalid (e.g. service died)?
65 */
66 int invalid;
67
68};
69
70
71/**
72 * Handle for a set operation request from another peer.
73 */
74struct GNUNET_SETU_Request
75{
76 /**
77 * Id of the request, used to identify the request when
78 * accepting/rejecting it.
79 */
80 uint32_t accept_id;
81
82 /**
83 * Has the request been accepted already?
84 * #GNUNET_YES/#GNUNET_NO
85 */
86 int accepted;
87};
88
89
90/**
91 * Handle to an operation. Only known to the service after committing
92 * the handle with a set.
93 */
94struct GNUNET_SETU_OperationHandle
95{
96 /**
97 * Function to be called when we have a result,
98 * or an error.
99 */
100 GNUNET_SETU_ResultIterator result_cb;
101
102 /**
103 * Closure for @e result_cb.
104 */
105 void *result_cls;
106
107 /**
108 * Local set used for the operation,
109 * NULL if no set has been provided by conclude yet.
110 */
111 struct GNUNET_SETU_Handle *set;
112
113 /**
114 * Message sent to the server on calling conclude,
115 * NULL if conclude has been called.
116 */
117 struct GNUNET_MQ_Envelope *conclude_mqm;
118
119 /**
120 * Address of the request if in the conclude message,
121 * used to patch the request id into the message when the set is known.
122 */
123 uint32_t *request_id_addr;
124
125 /**
126 * Handles are kept in a linked list.
127 */
128 struct GNUNET_SETU_OperationHandle *prev;
129
130 /**
131 * Handles are kept in a linked list.
132 */
133 struct GNUNET_SETU_OperationHandle *next;
134
135 /**
136 * Request ID to identify the operation within the set.
137 */
138 uint32_t request_id;
139};
140
141
142/**
143 * Opaque handle to a listen operation.
144 */
145struct GNUNET_SETU_ListenHandle
146{
147 /**
148 * Message queue for the client.
149 */
150 struct GNUNET_MQ_Handle*mq;
151
152 /**
153 * Configuration handle for the listener, stored
154 * here to be able to reconnect transparently on
155 * connection failure.
156 */
157 const struct GNUNET_CONFIGURATION_Handle *cfg;
158
159 /**
160 * Function to call on a new incoming request,
161 * or on error.
162 */
163 GNUNET_SETU_ListenCallback listen_cb;
164
165 /**
166 * Closure for @e listen_cb.
167 */
168 void *listen_cls;
169
170 /**
171 * Application ID we listen for.
172 */
173 struct GNUNET_HashCode app_id;
174
175 /**
176 * Time to wait until we try to reconnect on failure.
177 */
178 struct GNUNET_TIME_Relative reconnect_backoff;
179
180 /**
181 * Task for reconnecting when the listener fails.
182 */
183 struct GNUNET_SCHEDULER_Task *reconnect_task;
184
185};
186
187
188/**
189 * Check that the given @a msg is well-formed.
190 *
191 * @param cls closure
192 * @param msg message to check
193 * @return #GNUNET_OK if message is well-formed
194 */
195static int
196check_result (void *cls,
197 const struct GNUNET_SETU_ResultMessage *msg)
198{
199 /* minimum size was already checked, everything else is OK! */
200 return GNUNET_OK;
201}
202
203
204/**
205 * Handle result message for a set operation.
206 *
207 * @param cls the set
208 * @param mh the message
209 */
210static void
211handle_result (void *cls,
212 const struct GNUNET_SETU_ResultMessage *msg)
213{
214 struct GNUNET_SETU_Handle *set = cls;
215 struct GNUNET_SETU_OperationHandle *oh;
216 struct GNUNET_SETU_Element e;
217 enum GNUNET_SETU_Status result_status;
218 int destroy_set;
219
220 GNUNET_assert (NULL != set->mq);
221 result_status = (enum GNUNET_SETU_Status) ntohs (msg->result_status);
222 LOG (GNUNET_ERROR_TYPE_DEBUG,
223 "Got result message with status %d\n",
224 result_status);
225 oh = GNUNET_MQ_assoc_get (set->mq,
226 ntohl (msg->request_id));
227 if (NULL == oh)
228 {
229 /* 'oh' can be NULL if we canceled the operation, but the service
230 did not get the cancel message yet. */
231 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
232 "Ignoring result from canceled operation\n");
233 return;
234 }
235
236 switch (result_status)
237 {
238 case GNUNET_SETU_STATUS_ADD_LOCAL:
239 case GNUNET_SETU_STATUS_ADD_REMOTE:
240 e.data = &msg[1];
241 e.size = ntohs (msg->header.size)
242 - sizeof(struct GNUNET_SETU_ResultMessage);
243 e.element_type = ntohs (msg->element_type);
244 if (NULL != oh->result_cb)
245 oh->result_cb (oh->result_cls,
246 &e,
247 GNUNET_ntohll (msg->current_size),
248 result_status);
249 return;
250 case GNUNET_SETU_STATUS_FAILURE:
251 case GNUNET_SETU_STATUS_DONE:
252 LOG (GNUNET_ERROR_TYPE_DEBUG,
253 "Treating result as final status\n");
254 GNUNET_MQ_assoc_remove (set->mq,
255 ntohl (msg->request_id));
256 GNUNET_CONTAINER_DLL_remove (set->ops_head,
257 set->ops_tail,
258 oh);
259 /* Need to do this calculation _before_ the result callback,
260 as IF the application still has a valid set handle, it
261 may trigger destruction of the set during the callback. */
262 destroy_set = (GNUNET_YES == set->destroy_requested) &&
263 (NULL == set->ops_head);
264 if (NULL != oh->result_cb)
265 {
266 oh->result_cb (oh->result_cls,
267 NULL,
268 GNUNET_ntohll (msg->current_size),
269 result_status);
270 }
271 else
272 {
273 LOG (GNUNET_ERROR_TYPE_DEBUG,
274 "No callback for final status\n");
275 }
276 if (destroy_set)
277 GNUNET_SETU_destroy (set);
278 GNUNET_free (oh);
279 return;
280 }
281}
282
283
284/**
285 * Destroy the given set operation.
286 *
287 * @param oh set operation to destroy
288 */
289static void
290set_operation_destroy (struct GNUNET_SETU_OperationHandle *oh)
291{
292 struct GNUNET_SETU_Handle *set = oh->set;
293 struct GNUNET_SETU_OperationHandle *h_assoc;
294
295 if (NULL != oh->conclude_mqm)
296 GNUNET_MQ_discard (oh->conclude_mqm);
297 /* is the operation already committed? */
298 if (NULL != set)
299 {
300 GNUNET_CONTAINER_DLL_remove (set->ops_head,
301 set->ops_tail,
302 oh);
303 h_assoc = GNUNET_MQ_assoc_remove (set->mq,
304 oh->request_id);
305 GNUNET_assert ((NULL == h_assoc) ||
306 (h_assoc == oh));
307 }
308 GNUNET_free (oh);
309}
310
311
312/**
313 * Cancel the given set operation. We need to send an explicit cancel
314 * message, as all operations one one set communicate using one
315 * handle.
316 *
317 * @param oh set operation to cancel
318 */
319void
320GNUNET_SETU_operation_cancel (struct GNUNET_SETU_OperationHandle *oh)
321{
322 struct GNUNET_SETU_Handle *set = oh->set;
323 struct GNUNET_SETU_CancelMessage *m;
324 struct GNUNET_MQ_Envelope *mqm;
325
326 LOG (GNUNET_ERROR_TYPE_DEBUG,
327 "Cancelling SET operation\n");
328 if (NULL != set)
329 {
330 mqm = GNUNET_MQ_msg (m, GNUNET_MESSAGE_TYPE_SETU_CANCEL);
331 m->request_id = htonl (oh->request_id);
332 GNUNET_MQ_send (set->mq, mqm);
333 }
334 set_operation_destroy (oh);
335 if ((NULL != set) &&
336 (GNUNET_YES == set->destroy_requested) &&
337 (NULL == set->ops_head))
338 {
339 LOG (GNUNET_ERROR_TYPE_DEBUG,
340 "Destroying set after operation cancel\n");
341 GNUNET_SETU_destroy (set);
342 }
343}
344
345
346/**
347 * We encountered an error communicating with the set service while
348 * performing a set operation. Report to the application.
349 *
350 * @param cls the `struct GNUNET_SETU_Handle`
351 * @param error error code
352 */
353static void
354handle_client_set_error (void *cls,
355 enum GNUNET_MQ_Error error)
356{
357 struct GNUNET_SETU_Handle *set = cls;
358
359 LOG (GNUNET_ERROR_TYPE_ERROR,
360 "Handling client set error %d\n",
361 error);
362 while (NULL != set->ops_head)
363 {
364 if ((NULL != set->ops_head->result_cb) &&
365 (GNUNET_NO == set->destroy_requested))
366 set->ops_head->result_cb (set->ops_head->result_cls,
367 NULL,
368 0,
369 GNUNET_SETU_STATUS_FAILURE);
370 set_operation_destroy (set->ops_head);
371 }
372 set->invalid = GNUNET_YES;
373}
374
375
376/**
377 * Create an empty set, supporting the specified operation.
378 *
379 * @param cfg configuration to use for connecting to the
380 * set service
381 * @return a handle to the set
382 */
383struct GNUNET_SETU_Handle *
384GNUNET_SETU_create (const struct GNUNET_CONFIGURATION_Handle *cfg)
385{
386 struct GNUNET_SETU_Handle *set = GNUNET_new (struct GNUNET_SETU_Handle);
387 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
388 GNUNET_MQ_hd_var_size (result,
389 GNUNET_MESSAGE_TYPE_SETU_RESULT,
390 struct GNUNET_SETU_ResultMessage,
391 set),
392 GNUNET_MQ_handler_end ()
393 };
394 struct GNUNET_MQ_Envelope *mqm;
395 struct GNUNET_SETU_CreateMessage *create_msg;
396
397 set->mq = GNUNET_CLIENT_connect (cfg,
398 "setu",
399 mq_handlers,
400 &handle_client_set_error,
401 set);
402 if (NULL == set->mq)
403 {
404 GNUNET_free (set);
405 return NULL;
406 }
407 mqm = GNUNET_MQ_msg (create_msg,
408 GNUNET_MESSAGE_TYPE_SETU_CREATE);
409 GNUNET_MQ_send (set->mq,
410 mqm);
411 return set;
412}
413
414
415/**
416 * Add an element to the given set. After the element has been added
417 * (in the sense of being transmitted to the set service), @a cont
418 * will be called. Multiple calls to GNUNET_SETU_add_element() can be
419 * queued.
420 *
421 * @param set set to add element to
422 * @param element element to add to the set
423 * @param cb continuation called after the element has been added
424 * @param cb_cls closure for @a cb
425 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
426 * set is invalid (e.g. the set service crashed)
427 */
428int
429GNUNET_SETU_add_element (struct GNUNET_SETU_Handle *set,
430 const struct GNUNET_SETU_Element *element,
431 GNUNET_SCHEDULER_TaskCallback cb,
432 void *cb_cls)
433{
434 struct GNUNET_MQ_Envelope *mqm;
435 struct GNUNET_SETU_ElementMessage *msg;
436
437 LOG (GNUNET_ERROR_TYPE_DEBUG,
438 "adding element of type %u to set %p\n",
439 (unsigned int) element->element_type,
440 set);
441 GNUNET_assert (NULL != set);
442 if (GNUNET_YES == set->invalid)
443 {
444 if (NULL != cb)
445 cb (cb_cls);
446 return GNUNET_SYSERR;
447 }
448 mqm = GNUNET_MQ_msg_extra (msg,
449 element->size,
450 GNUNET_MESSAGE_TYPE_SETU_ADD);
451 msg->element_type = htons (element->element_type);
452 GNUNET_memcpy (&msg[1],
453 element->data,
454 element->size);
455 GNUNET_MQ_notify_sent (mqm,
456 cb,
457 cb_cls);
458 GNUNET_MQ_send (set->mq,
459 mqm);
460 return GNUNET_OK;
461}
462
463
464/**
465 * Destroy the set handle if no operations are left, mark the set
466 * for destruction otherwise.
467 *
468 * @param set set handle to destroy
469 */
470void
471GNUNET_SETU_destroy (struct GNUNET_SETU_Handle *set)
472{
473 /* destroying set while iterator is active is currently
474 not supported; we should expand the API to allow
475 clients to explicitly cancel the iteration! */
476 GNUNET_assert (NULL != set);
477 if ((NULL != set->ops_head) ||
478 (GNUNET_SYSERR == set->destroy_requested))
479 {
480 LOG (GNUNET_ERROR_TYPE_DEBUG,
481 "Set operations are pending, delaying set destruction\n");
482 set->destroy_requested = GNUNET_YES;
483 return;
484 }
485 LOG (GNUNET_ERROR_TYPE_DEBUG,
486 "Really destroying set\n");
487 if (NULL != set->mq)
488 {
489 GNUNET_MQ_destroy (set->mq);
490 set->mq = NULL;
491 }
492 GNUNET_free (set);
493}
494
495
496/**
497 * Prepare a set operation to be evaluated with another peer.
498 * The evaluation will not start until the client provides
499 * a local set with #GNUNET_SETU_commit().
500 *
501 * @param other_peer peer with the other set
502 * @param app_id hash for the application using the set
503 * @param context_msg additional information for the request
504 * @param result_cb called on error or success
505 * @param result_cls closure for @e result_cb
506 * @return a handle to cancel the operation
507 */
508struct GNUNET_SETU_OperationHandle *
509GNUNET_SETU_prepare (const struct GNUNET_PeerIdentity *other_peer,
510 const struct GNUNET_HashCode *app_id,
511 const struct GNUNET_MessageHeader *context_msg,
512 const struct GNUNET_SETU_Option options[],
513 GNUNET_SETU_ResultIterator result_cb,
514 void *result_cls)
515{
516 struct GNUNET_MQ_Envelope *mqm;
517 struct GNUNET_SETU_OperationHandle *oh;
518 struct GNUNET_SETU_EvaluateMessage *msg;
519
520 LOG (GNUNET_ERROR_TYPE_DEBUG,
521 "Client prepares set union operation\n");
522 oh = GNUNET_new (struct GNUNET_SETU_OperationHandle);
523 oh->result_cb = result_cb;
524 oh->result_cls = result_cls;
525 mqm = GNUNET_MQ_msg_nested_mh (msg,
526 GNUNET_MESSAGE_TYPE_SETU_EVALUATE,
527 context_msg);
528 msg->app_id = *app_id;
529 msg->target_peer = *other_peer;
530
531 /* Set default values */
532 msg->byzantine_upper_bond = UINT64_MAX;
533 msg->bandwidth_latency_tradeoff = 0;
534 msg->ibf_bucket_number_factor = 2;
535 msg->ibf_number_of_buckets_per_element = 3;
536
537
538 for (const struct GNUNET_SETU_Option *opt = options; opt->type != 0; opt++)
539 {
540 switch (opt->type)
541 {
542 case GNUNET_SETU_OPTION_BYZANTINE:
543 msg->byzantine = GNUNET_YES;
544 msg->byzantine_lower_bound = htonl (opt->v.num);
545 break;
546 case GNUNET_SETU_OPTION_CUSTOM_BYZANTINE_UPPER_BOUND:
547 msg->byzantine_upper_bond = htonl (opt->v.num);
548 break;
549 case GNUNET_SETU_OPTION_CUSTOM_BANDWIDTH_LATENCY_TRADEOFF:
550 msg->bandwidth_latency_tradeoff = htonl (opt->v.num);
551 break;
552 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKET_NUMBER_FACTOR:
553 msg->ibf_bucket_number_factor = htonl (opt->v.num);
554 break;
555 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKETS_PER_ELEMENT:
556 msg->ibf_number_of_buckets_per_element = htonl (opt->v.num);
557 break;
558 case GNUNET_SETU_OPTION_FORCE_FULL:
559 msg->force_full = GNUNET_YES;
560 break;
561 case GNUNET_SETU_OPTION_FORCE_DELTA:
562 msg->force_delta = GNUNET_YES;
563 break;
564 case GNUNET_SETU_OPTION_SYMMETRIC:
565 msg->symmetric = GNUNET_YES;
566 break;
567 default:
568 LOG (GNUNET_ERROR_TYPE_ERROR,
569 "Option with type %d not recognized\n",
570 (int) opt->type);
571 }
572 }
573 oh->conclude_mqm = mqm;
574 oh->request_id_addr = &msg->request_id;
575 return oh;
576}
577
578
579/**
580 * Connect to the set service in order to listen for requests.
581 *
582 * @param cls the `struct GNUNET_SETU_ListenHandle *` to connect
583 */
584static void
585listen_connect (void *cls);
586
587
588/**
589 * Check validity of request message for a listen operation
590 *
591 * @param cls the listen handle
592 * @param msg the message
593 * @return #GNUNET_OK if the message is well-formed
594 */
595static int
596check_request (void *cls,
597 const struct GNUNET_SETU_RequestMessage *msg)
598{
599 const struct GNUNET_MessageHeader *context_msg;
600
601 if (ntohs (msg->header.size) == sizeof(*msg))
602 return GNUNET_OK; /* no context message is OK */
603 context_msg = GNUNET_MQ_extract_nested_mh (msg);
604 if (NULL == context_msg)
605 {
606 /* malformed context message is NOT ok */
607 GNUNET_break_op (0);
608 return GNUNET_SYSERR;
609 }
610 return GNUNET_OK;
611}
612
613
614/**
615 * Handle request message for a listen operation
616 *
617 * @param cls the listen handle
618 * @param msg the message
619 */
620static void
621handle_request (void *cls,
622 const struct GNUNET_SETU_RequestMessage *msg)
623{
624 struct GNUNET_SETU_ListenHandle *lh = cls;
625 struct GNUNET_SETU_Request req;
626 const struct GNUNET_MessageHeader *context_msg;
627 struct GNUNET_MQ_Envelope *mqm;
628 struct GNUNET_SETU_RejectMessage *rmsg;
629
630 LOG (GNUNET_ERROR_TYPE_DEBUG,
631 "Processing incoming operation request with id %u\n",
632 ntohl (msg->accept_id));
633 /* we got another valid request => reset the backoff */
634 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
635 req.accept_id = ntohl (msg->accept_id);
636 req.accepted = GNUNET_NO;
637 context_msg = GNUNET_MQ_extract_nested_mh (msg);
638 /* calling #GNUNET_SETU_accept() in the listen cb will set req->accepted */
639 lh->listen_cb (lh->listen_cls,
640 &msg->peer_id,
641 context_msg,
642 &req);
643 if (GNUNET_YES == req.accepted)
644 return; /* the accept-case is handled in #GNUNET_SETU_accept() */
645 LOG (GNUNET_ERROR_TYPE_DEBUG,
646 "Rejected request %u\n",
647 ntohl (msg->accept_id));
648 mqm = GNUNET_MQ_msg (rmsg,
649 GNUNET_MESSAGE_TYPE_SETU_REJECT);
650 rmsg->accept_reject_id = msg->accept_id;
651 GNUNET_MQ_send (lh->mq,
652 mqm);
653}
654
655
656/**
657 * Our connection with the set service encountered an error,
658 * re-initialize with exponential back-off.
659 *
660 * @param cls the `struct GNUNET_SETU_ListenHandle *`
661 * @param error reason for the disconnect
662 */
663static void
664handle_client_listener_error (void *cls,
665 enum GNUNET_MQ_Error error)
666{
667 struct GNUNET_SETU_ListenHandle *lh = cls;
668
669 LOG (GNUNET_ERROR_TYPE_DEBUG,
670 "Listener broke down (%d), re-connecting\n",
671 (int) error);
672 GNUNET_MQ_destroy (lh->mq);
673 lh->mq = NULL;
674 lh->reconnect_task = GNUNET_SCHEDULER_add_delayed (lh->reconnect_backoff,
675 &listen_connect,
676 lh);
677 lh->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (lh->reconnect_backoff);
678}
679
680
681/**
682 * Connect to the set service in order to listen for requests.
683 *
684 * @param cls the `struct GNUNET_SETU_ListenHandle *` to connect
685 */
686static void
687listen_connect (void *cls)
688{
689 struct GNUNET_SETU_ListenHandle *lh = cls;
690 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
691 GNUNET_MQ_hd_var_size (request,
692 GNUNET_MESSAGE_TYPE_SETU_REQUEST,
693 struct GNUNET_SETU_RequestMessage,
694 lh),
695 GNUNET_MQ_handler_end ()
696 };
697 struct GNUNET_MQ_Envelope *mqm;
698 struct GNUNET_SETU_ListenMessage *msg;
699
700 lh->reconnect_task = NULL;
701 GNUNET_assert (NULL == lh->mq);
702 lh->mq = GNUNET_CLIENT_connect (lh->cfg,
703 "setu",
704 mq_handlers,
705 &handle_client_listener_error,
706 lh);
707 if (NULL == lh->mq)
708 return;
709 mqm = GNUNET_MQ_msg (msg,
710 GNUNET_MESSAGE_TYPE_SETU_LISTEN);
711 msg->app_id = lh->app_id;
712 GNUNET_MQ_send (lh->mq,
713 mqm);
714}
715
716
717/**
718 * Wait for set operation requests for the given application id
719 *
720 * @param cfg configuration to use for connecting to
721 * the set service, needs to be valid for the lifetime of the listen handle
722 * @param app_id id of the application that handles set operation requests
723 * @param listen_cb called for each incoming request matching the operation
724 * and application id
725 * @param listen_cls handle for @a listen_cb
726 * @return a handle that can be used to cancel the listen operation
727 */
728struct GNUNET_SETU_ListenHandle *
729GNUNET_SETU_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
730 const struct GNUNET_HashCode *app_id,
731 GNUNET_SETU_ListenCallback listen_cb,
732 void *listen_cls)
733{
734 struct GNUNET_SETU_ListenHandle *lh;
735
736 LOG (GNUNET_ERROR_TYPE_DEBUG,
737 "Starting listener for app %s\n",
738 GNUNET_h2s (app_id));
739 lh = GNUNET_new (struct GNUNET_SETU_ListenHandle);
740 lh->listen_cb = listen_cb;
741 lh->listen_cls = listen_cls;
742 lh->cfg = cfg;
743 lh->app_id = *app_id;
744 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
745 listen_connect (lh);
746 if (NULL == lh->mq)
747 {
748 GNUNET_free (lh);
749 return NULL;
750 }
751 return lh;
752}
753
754
755/**
756 * Cancel the given listen operation.
757 *
758 * @param lh handle for the listen operation
759 */
760void
761GNUNET_SETU_listen_cancel (struct GNUNET_SETU_ListenHandle *lh)
762{
763 LOG (GNUNET_ERROR_TYPE_DEBUG,
764 "Canceling listener %s\n",
765 GNUNET_h2s (&lh->app_id));
766 if (NULL != lh->mq)
767 {
768 GNUNET_MQ_destroy (lh->mq);
769 lh->mq = NULL;
770 }
771 if (NULL != lh->reconnect_task)
772 {
773 GNUNET_SCHEDULER_cancel (lh->reconnect_task);
774 lh->reconnect_task = NULL;
775 }
776 GNUNET_free (lh);
777}
778
779
780/**
781 * Accept a request we got via #GNUNET_SETU_listen. Must be called during
782 * #GNUNET_SETU_listen, as the 'struct GNUNET_SETU_Request' becomes invalid
783 * afterwards.
784 * Call #GNUNET_SETU_commit to provide the local set to use for the operation,
785 * and to begin the exchange with the remote peer.
786 *
787 * @param request request to accept
788 * @param result_mode specified how results will be returned,
789 * see `enum GNUNET_SETU_ResultMode`.
790 * @param result_cb callback for the results
791 * @param result_cls closure for @a result_cb
792 * @return a handle to cancel the operation
793 */
794struct GNUNET_SETU_OperationHandle *
795GNUNET_SETU_accept (struct GNUNET_SETU_Request *request,
796 const struct GNUNET_SETU_Option options[],
797 GNUNET_SETU_ResultIterator result_cb,
798 void *result_cls)
799{
800 struct GNUNET_MQ_Envelope *mqm;
801 struct GNUNET_SETU_OperationHandle *oh;
802 struct GNUNET_SETU_AcceptMessage *msg;
803
804 GNUNET_assert (GNUNET_NO == request->accepted);
805 LOG (GNUNET_ERROR_TYPE_DEBUG,
806 "Client accepts set union operation with id %u\n",
807 request->accept_id);
808 request->accepted = GNUNET_YES;
809 mqm = GNUNET_MQ_msg (msg,
810 GNUNET_MESSAGE_TYPE_SETU_ACCEPT);
811 msg->accept_reject_id = htonl (request->accept_id);
812
813 /* Set default values */
814 msg->byzantine_upper_bond = UINT64_MAX;
815 msg->bandwidth_latency_tradeoff = 0;
816 msg->ibf_bucket_number_factor = 2;
817 msg->ibf_number_of_buckets_per_element = 3;
818
819 for (const struct GNUNET_SETU_Option *opt = options; opt->type != 0; opt++)
820 {
821 switch (opt->type)
822 {
823 case GNUNET_SETU_OPTION_BYZANTINE:
824 msg->byzantine = GNUNET_YES;
825 msg->byzantine_lower_bound = htonl (opt->v.num);
826 break;
827 case GNUNET_SETU_OPTION_CUSTOM_BYZANTINE_UPPER_BOUND:
828 msg->byzantine_upper_bond = htonl (opt->v.num);
829 break;
830 case GNUNET_SETU_OPTION_CUSTOM_BANDWIDTH_LATENCY_TRADEOFF:
831 msg->bandwidth_latency_tradeoff = htonl (opt->v.num);
832 break;
833 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKET_NUMBER_FACTOR:
834 msg->ibf_bucket_number_factor = htonl (opt->v.num);
835 break;
836 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKETS_PER_ELEMENT:
837 msg->ibf_number_of_buckets_per_element = htonl (opt->v.num);
838 break;
839 case GNUNET_SETU_OPTION_FORCE_FULL:
840 msg->force_full = GNUNET_YES;
841 break;
842 case GNUNET_SETU_OPTION_FORCE_DELTA:
843 msg->force_delta = GNUNET_YES;
844 break;
845 case GNUNET_SETU_OPTION_SYMMETRIC:
846 msg->symmetric = GNUNET_YES;
847 break;
848 default:
849 LOG (GNUNET_ERROR_TYPE_ERROR,
850 "Option with type %d not recognized\n",
851 (int) opt->type);
852 }
853 }
854 oh = GNUNET_new (struct GNUNET_SETU_OperationHandle);
855 oh->result_cb = result_cb;
856 oh->result_cls = result_cls;
857 oh->conclude_mqm = mqm;
858 oh->request_id_addr = &msg->request_id;
859 return oh;
860}
861
862
863/**
864 * Commit a set to be used with a set operation.
865 * This function is called once we have fully constructed
866 * the set that we want to use for the operation. At this
867 * time, the P2P protocol can then begin to exchange the
868 * set information and call the result callback with the
869 * result information.
870 *
871 * @param oh handle to the set operation
872 * @param set the set to use for the operation
873 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
874 * set is invalid (e.g. the set service crashed)
875 */
876int
877GNUNET_SETU_commit (struct GNUNET_SETU_OperationHandle *oh,
878 struct GNUNET_SETU_Handle *set)
879{
880 if (NULL != oh->set)
881 {
882 /* Some other set was already committed for this
883 * operation, there is a logic bug in the client of this API */
884 GNUNET_break (0);
885 return GNUNET_OK;
886 }
887 GNUNET_assert (NULL != set);
888 if (GNUNET_YES == set->invalid)
889 return GNUNET_SYSERR;
890 LOG (GNUNET_ERROR_TYPE_DEBUG,
891 "Client commits to SET\n");
892 GNUNET_assert (NULL != oh->conclude_mqm);
893 oh->set = set;
894 GNUNET_CONTAINER_DLL_insert (set->ops_head,
895 set->ops_tail,
896 oh);
897 oh->request_id = GNUNET_MQ_assoc_add (set->mq,
898 oh);
899 *oh->request_id_addr = htonl (oh->request_id);
900 GNUNET_MQ_send (set->mq,
901 oh->conclude_mqm);
902 oh->conclude_mqm = NULL;
903 oh->request_id_addr = NULL;
904 return GNUNET_OK;
905}
906
907
908/**
909 * Hash a set element.
910 *
911 * @param element the element that should be hashed
912 * @param[out] ret_hash a pointer to where the hash of @a element
913 * should be stored
914 */
915void
916GNUNET_SETU_element_hash (const struct GNUNET_SETU_Element *element,
917 struct GNUNET_HashCode *ret_hash)
918{
919 struct GNUNET_HashContext *ctx = GNUNET_CRYPTO_hash_context_start ();
920
921 /* It's not guaranteed that the element data is always after the element header,
922 so we need to hash the chunks separately. */
923 GNUNET_CRYPTO_hash_context_read (ctx,
924 &element->size,
925 sizeof(uint16_t));
926 GNUNET_CRYPTO_hash_context_read (ctx,
927 &element->element_type,
928 sizeof(uint16_t));
929 GNUNET_CRYPTO_hash_context_read (ctx,
930 element->data,
931 element->size);
932 GNUNET_CRYPTO_hash_context_finish (ctx,
933 ret_hash);
934}
935
936
937/* end of setu_api.c */
diff --git a/src/setu/test_setu.conf b/src/setu/test_setu.conf
deleted file mode 100644
index 6d132fc1f..000000000
--- a/src/setu/test_setu.conf
+++ /dev/null
@@ -1,32 +0,0 @@
1@INLINE@ ../../contrib/conf/gnunet/no_forcestart.conf
2
3[PATHS]
4GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-set/
5
6[setu]
7START_ON_DEMAND = YES
8#PREFIX = valgrind --leak-check=full
9#PREFIX = gdbserver :1234
10OPTIONS = -L INFO
11
12[transport]
13PLUGINS = unix
14OPTIONS = -LERROR
15
16[nat]
17RETURN_LOCAL_ADDRESSES = YES
18DISABLEV6 = YES
19USE_LOCALADDR = YES
20
21[peerinfo]
22NO_IO = YES
23
24[nat]
25# Use addresses from the local network interfaces (including loopback, but also others)
26USE_LOCALADDR = YES
27
28# Disable IPv6 support
29DISABLEV6 = NO
30
31# Do we use addresses from localhost address ranges? (::1, 127.0.0.0/8)
32RETURN_LOCAL_ADDRESSES = YES
diff --git a/src/setu/test_setu_api.c b/src/setu/test_setu_api.c
deleted file mode 100644
index 5a0c9d70d..000000000
--- a/src/setu/test_setu_api.c
+++ /dev/null
@@ -1,360 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_setu_api.c
23 * @brief testcase for setu_api.c
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_testing_lib.h"
29#include "gnunet_setu_service.h"
30
31
32static struct GNUNET_PeerIdentity local_id;
33
34static struct GNUNET_HashCode app_id;
35
36static struct GNUNET_SETU_Handle *set1;
37
38static struct GNUNET_SETU_Handle *set2;
39
40static struct GNUNET_SETU_ListenHandle *listen_handle;
41
42static struct GNUNET_SETU_OperationHandle *oh1;
43
44static struct GNUNET_SETU_OperationHandle *oh2;
45
46static const struct GNUNET_CONFIGURATION_Handle *config;
47
48static int ret;
49
50static struct GNUNET_SCHEDULER_Task *tt;
51
52
53static void
54result_cb_set1 (void *cls,
55 const struct GNUNET_SETU_Element *element,
56 uint64_t size,
57 enum GNUNET_SETU_Status status)
58{
59 switch (status)
60 {
61 case GNUNET_SETU_STATUS_ADD_LOCAL:
62 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: got element\n");
63 break;
64
65 case GNUNET_SETU_STATUS_FAILURE:
66 GNUNET_break (0);
67 oh1 = NULL;
68 fprintf (stderr, "set 1: received failure status!\n");
69 ret = 1;
70 if (NULL != tt)
71 {
72 GNUNET_SCHEDULER_cancel (tt);
73 tt = NULL;
74 }
75 GNUNET_SCHEDULER_shutdown ();
76 break;
77
78 case GNUNET_SETU_STATUS_DONE:
79 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: done\n");
80 oh1 = NULL;
81 if (NULL != set1)
82 {
83 GNUNET_SETU_destroy (set1);
84 set1 = NULL;
85 }
86 if (NULL == set2)
87 {
88 GNUNET_SCHEDULER_cancel (tt);
89 tt = NULL;
90 GNUNET_SCHEDULER_shutdown ();
91 }
92 break;
93
94 default:
95 GNUNET_assert (0);
96 }
97}
98
99
100static void
101result_cb_set2 (void *cls,
102 const struct GNUNET_SETU_Element *element,
103 uint64_t size,
104 enum GNUNET_SETU_Status status)
105{
106 switch (status)
107 {
108 case GNUNET_SETU_STATUS_ADD_LOCAL:
109 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: got element\n");
110 break;
111
112 case GNUNET_SETU_STATUS_FAILURE:
113 GNUNET_break (0);
114 oh2 = NULL;
115 fprintf (stderr, "set 2: received failure status\n");
116 GNUNET_SCHEDULER_shutdown ();
117 ret = 1;
118 break;
119
120 case GNUNET_SETU_STATUS_DONE:
121 oh2 = NULL;
122 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: done\n");
123 GNUNET_SETU_destroy (set2);
124 set2 = NULL;
125 if (NULL == set1)
126 {
127 GNUNET_SCHEDULER_cancel (tt);
128 tt = NULL;
129 GNUNET_SCHEDULER_shutdown ();
130 }
131 break;
132
133 default:
134 GNUNET_assert (0);
135 }
136}
137
138
139static void
140listen_cb (void *cls,
141 const struct GNUNET_PeerIdentity *other_peer,
142 const struct GNUNET_MessageHeader *context_msg,
143 struct GNUNET_SETU_Request *request)
144{
145 GNUNET_assert (NULL != context_msg);
146 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
147 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "listen cb called\n");
148 oh2 = GNUNET_SETU_accept (request,
149 (struct GNUNET_SETU_Option[]){ 0 },
150 &result_cb_set2,
151 NULL);
152 GNUNET_SETU_commit (oh2, set2);
153}
154
155
156/**
157 * Start the set operation.
158 *
159 * @param cls closure, unused
160 */
161static void
162start (void *cls)
163{
164 struct GNUNET_MessageHeader context_msg;
165
166 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting reconciliation\n");
167 context_msg.size = htons (sizeof context_msg);
168 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
169 listen_handle = GNUNET_SETU_listen (config,
170 &app_id,
171 &listen_cb,
172 NULL);
173 oh1 = GNUNET_SETU_prepare (&local_id,
174 &app_id,
175 &context_msg,
176 (struct GNUNET_SETU_Option[]){ 0 },
177 &result_cb_set1,
178 NULL);
179 GNUNET_SETU_commit (oh1, set1);
180}
181
182
183/**
184 * Initialize the second set, continue
185 *
186 * @param cls closure, unused
187 */
188static void
189init_set2 (void *cls)
190{
191 struct GNUNET_SETU_Element element;
192
193 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initializing set 2\n");
194
195 element.element_type = 0;
196 element.data = "hello1";
197 element.size = strlen (element.data);
198 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
199 element.data = "quux";
200 element.size = strlen (element.data);
201 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
202 element.data = "baz";
203 element.size = strlen (element.data);
204 GNUNET_SETU_add_element (set2, &element, &start, NULL);
205}
206
207
208/**
209 * Initialize the first set, continue.
210 */
211static void
212init_set1 (void)
213{
214 struct GNUNET_SETU_Element element;
215
216 element.element_type = 0;
217 element.data = "hello";
218 element.size = strlen (element.data);
219 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
220 element.data = "bar";
221 element.size = strlen (element.data);
222 GNUNET_SETU_add_element (set1, &element, &init_set2, NULL);
223 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized set 1\n");
224}
225
226
227/**
228 * Function run on timeout.
229 *
230 * @param cls closure
231 */
232static void
233timeout_fail (void *cls)
234{
235 tt = NULL;
236 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, "Testcase failed with timeout\n");
237 GNUNET_SCHEDULER_shutdown ();
238 ret = 1;
239}
240
241
242/**
243 * Function run on shutdown.
244 *
245 * @param cls closure
246 */
247static void
248do_shutdown (void *cls)
249{
250 if (NULL != tt)
251 {
252 GNUNET_SCHEDULER_cancel (tt);
253 tt = NULL;
254 }
255 if (NULL != oh1)
256 {
257 GNUNET_SETU_operation_cancel (oh1);
258 oh1 = NULL;
259 }
260 if (NULL != oh2)
261 {
262 GNUNET_SETU_operation_cancel (oh2);
263 oh2 = NULL;
264 }
265 if (NULL != set1)
266 {
267 GNUNET_SETU_destroy (set1);
268 set1 = NULL;
269 }
270 if (NULL != set2)
271 {
272 GNUNET_SETU_destroy (set2);
273 set2 = NULL;
274 }
275 if (NULL != listen_handle)
276 {
277 GNUNET_SETU_listen_cancel (listen_handle);
278 listen_handle = NULL;
279 }
280}
281
282
283/**
284 * Signature of the 'main' function for a (single-peer) testcase that
285 * is run using 'GNUNET_TESTING_peer_run'.
286 *
287 * @param cls closure
288 * @param cfg configuration of the peer that was started
289 * @param peer identity of the peer that was created
290 */
291static void
292run (void *cls,
293 const struct GNUNET_CONFIGURATION_Handle *cfg,
294 struct GNUNET_TESTING_Peer *peer)
295{
296 struct GNUNET_SETU_OperationHandle *my_oh;
297
298 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
299 "Running preparatory tests\n");
300 tt = GNUNET_SCHEDULER_add_delayed (
301 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
302 &timeout_fail,
303 NULL);
304 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
305
306 config = cfg;
307 GNUNET_assert (GNUNET_OK == GNUNET_CRYPTO_get_peer_identity (cfg,
308 &local_id));
309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
310 "my id (from CRYPTO): %s\n",
311 GNUNET_i2s (&local_id));
312 GNUNET_TESTING_peer_get_identity (peer,
313 &local_id);
314 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
315 "my id (from TESTING): %s\n",
316 GNUNET_i2s (&local_id));
317 set1 = GNUNET_SETU_create (cfg);
318 set2 = GNUNET_SETU_create (cfg);
319 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
320 "Created sets %p and %p for union operation\n",
321 set1,
322 set2);
323 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
324
325 /* test if canceling an uncommitted request works! */
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Launching and instantly stopping set operation\n");
328 my_oh = GNUNET_SETU_prepare (&local_id,
329 &app_id,
330 NULL,
331 (struct GNUNET_SETU_Option[]){ 0 },
332 NULL,
333 NULL);
334 GNUNET_SETU_operation_cancel (my_oh);
335
336 /* test the real set reconciliation */
337 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
338 "Running real set-reconciliation\n");
339 init_set1 ();
340}
341
342
343int
344main (int argc, char **argv)
345{
346 GNUNET_log_setup ("test_setu_api",
347 "WARNING",
348 NULL);
349 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
350 "Launching peer\n");
351 if (0 !=
352 GNUNET_TESTING_peer_run ("test_setu_api",
353 "test_setu.conf",
354 &run,
355 NULL))
356 {
357 return 1;
358 }
359 return ret;
360}
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-27 21:20:05 +0000