aboutsummaryrefslogtreecommitdiff
path: root/src/setu
diff options
context:
space:
mode:
authorMartin Schanzenbach <schanzen@gnunet.org>2023-10-19 10:06:52 +0200
committerMartin Schanzenbach <schanzen@gnunet.org>2023-10-19 10:06:52 +0200
commit3bfe2d52dcac5df09783d82b3a80899ae5b5d69a (patch)
treefd082362504b71f71ee58db5d1c9b562b3b0203b /src/setu
parenta93ccc1ebb57f50ce6e90c7f0e57a3e58020b05e (diff)
downloadgnunet-3bfe2d52dcac5df09783d82b3a80899ae5b5d69a.tar.gz
gnunet-3bfe2d52dcac5df09783d82b3a80899ae5b5d69a.zip
BUILD: Move seti/setu to service
Diffstat (limited to 'src/setu')
-rw-r--r--src/setu/.gitignore7
-rw-r--r--src/setu/Makefile.am106
-rw-r--r--src/setu/gnunet-service-setu.c5460
-rw-r--r--src/setu/gnunet-service-setu_protocol.h256
-rw-r--r--src/setu/gnunet-service-setu_strata_estimator.c468
-rw-r--r--src/setu/gnunet-service-setu_strata_estimator.h199
-rw-r--r--src/setu/gnunet-setu-ibf-profiler.c308
-rw-r--r--src/setu/gnunet-setu-profiler.c499
-rw-r--r--src/setu/ibf.c585
-rw-r--r--src/setu/ibf.h310
-rw-r--r--src/setu/ibf_sim.c143
-rw-r--r--src/setu/meson.build50
-rw-r--r--src/setu/perf_setu_api.c473
-rw-r--r--src/setu/plugin_block_setu_test.c195
-rw-r--r--src/setu/setu.conf.in12
-rw-r--r--src/setu/setu.h365
-rw-r--r--src/setu/setu_api.c911
-rw-r--r--src/setu/test_setu.conf32
-rw-r--r--src/setu/test_setu_api.c360
19 files changed, 0 insertions, 10739 deletions
diff --git a/src/setu/.gitignore b/src/setu/.gitignore
deleted file mode 100644
index 0ac10295e..000000000
--- a/src/setu/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
1gnunet-setu-profiler
2gnunet-service-setu
3gnunet-setu-ibf-profiler
4test_setu_api
5test_setu_copy
6test_setu_result_symmetric
7perf_setu_api
diff --git a/src/setu/Makefile.am b/src/setu/Makefile.am
deleted file mode 100644
index f7f8ed394..000000000
--- a/src/setu/Makefile.am
+++ /dev/null
@@ -1,106 +0,0 @@
1# This Makefile.am is in the public domain
2AM_CPPFLAGS = -I$(top_srcdir)/src/include
3
4pkgcfgdir= $(pkgdatadir)/config.d/
5
6libexecdir= $(pkglibdir)/libexec/
7
8plugindir = $(libdir)/gnunet
9
10pkgcfg_DATA = \
11 setu.conf
12
13if USE_COVERAGE
14 AM_CFLAGS = -fprofile-arcs -ftest-coverage
15endif
16
17noinst_PROGRAMS = \
18 gnunet-setu-ibf-profiler \
19 gnunet-setu-profiler
20
21libexec_PROGRAMS = \
22 gnunet-service-setu
23
24lib_LTLIBRARIES = \
25 libgnunetsetu.la
26
27gnunet_setu_profiler_SOURCES = \
28 gnunet-setu-profiler.c
29gnunet_setu_profiler_LDADD = \
30 $(top_builddir)/src/lib/util/libgnunetutil.la \
31 $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
32 libgnunetsetu.la \
33 $(top_builddir)/src/service/testing/libgnunettesting.la \
34 $(GN_LIBINTL)
35
36
37gnunet_setu_ibf_profiler_SOURCES = \
38 gnunet-setu-ibf-profiler.c \
39 ibf.c
40gnunet_setu_ibf_profiler_LDADD = \
41 $(top_builddir)/src/lib/util/libgnunetutil.la \
42 $(GN_LIBINTL)
43
44gnunet_service_setu_SOURCES = \
45 gnunet-service-setu.c gnunet-service-setu_protocol.h \
46 ibf.c ibf.h \
47 gnunet-service-setu_strata_estimator.c gnunet-service-setu_strata_estimator.h \
48 gnunet-service-setu_protocol.h
49gnunet_service_setu_LDADD = \
50 $(top_builddir)/src/lib/util/libgnunetutil.la \
51 $(top_builddir)/src/service/statistics/libgnunetstatistics.la \
52 $(top_builddir)/src/service/core/libgnunetcore.la \
53 $(top_builddir)/src/service/cadet/libgnunetcadet.la \
54 $(top_builddir)/src/lib/block/libgnunetblock.la \
55 libgnunetsetu.la \
56 $(GN_LIBINTL)
57
58libgnunetsetu_la_SOURCES = \
59 setu_api.c setu.h
60libgnunetsetu_la_LIBADD = \
61 $(top_builddir)/src/lib/util/libgnunetutil.la \
62 $(LTLIBINTL)
63libgnunetsetu_la_LDFLAGS = \
64 $(GN_LIB_LDFLAGS)
65
66check_PROGRAMS = \
67 # test_setu_api \
68 # perf_setu_api
69
70if ENABLE_TEST_RUN
71AM_TESTS_ENVIRONMENT=export GNUNET_PREFIX=$${GNUNET_PREFIX:-@libdir@};export PATH=$${GNUNET_PREFIX:-@prefix@}/bin:$$PATH;unset XDG_DATA_HOME;unset XDG_CONFIG_HOME;
72TESTS = $(check_PROGRAMS)
73endif
74
75test_setu_api_SOURCES = \
76 test_setu_api.c
77test_setu_api_LDADD = \
78 $(top_builddir)/src/lib/util/libgnunetutil.la \
79 $(top_builddir)/src/service/testing/libgnunettesting.la \
80 libgnunetsetu.la
81
82
83perf_setu_api_SOURCES = \
84 perf_setu_api.c
85perf_setu_api_LDADD = \
86 $(top_builddir)/src/lib/util/libgnunetutil.la \
87 $(top_builddir)/src/service/testing/libgnunettesting.la \
88 libgnunetsetu.la
89
90
91plugin_LTLIBRARIES = \
92 libgnunet_plugin_block_setu_test.la
93
94libgnunet_plugin_block_setu_test_la_SOURCES = \
95 plugin_block_setu_test.c
96libgnunet_plugin_block_setu_test_la_LIBADD = \
97 $(top_builddir)/src/lib/block/libgnunetblock.la \
98 $(top_builddir)/src/lib/block/libgnunetblockgroup.la \
99 $(top_builddir)/src/lib/util/libgnunetutil.la \
100 $(LTLIBINTL)
101libgnunet_plugin_block_setu_test_la_LDFLAGS = \
102 $(GN_PLUGIN_LDFLAGS)
103
104
105EXTRA_DIST = \
106 test_setu.conf
diff --git a/src/setu/gnunet-service-setu.c b/src/setu/gnunet-service-setu.c
deleted file mode 100644
index 46d027cca..000000000
--- a/src/setu/gnunet-service-setu.c
+++ /dev/null
@@ -1,5460 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013-2017, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file setu/gnunet-service-setu.c
22 * @brief set union operation
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_statistics_service.h"
30#include "ibf.h"
31#include "gnunet_protocols.h"
32#include "gnunet_applications.h"
33#include "gnunet_cadet_service.h"
34#include "gnunet-service-setu_strata_estimator.h"
35#include "gnunet-service-setu_protocol.h"
36#include "gnunet_statistics_service.h"
37#include <gcrypt.h>
38#include "gnunet_setu_service.h"
39#include "setu.h"
40
41#define LOG(kind, ...) GNUNET_log_from (kind, "setu", __VA_ARGS__)
42
43/**
44 * How long do we hold on to an incoming channel if there is
45 * no local listener before giving up?
46 */
47#define INCOMING_CHANNEL_TIMEOUT GNUNET_TIME_UNIT_MINUTES
48
49/**
50 * Number of IBFs in a strata estimator.
51 */
52#define SE_STRATA_COUNT 32
53
54
55/**
56 * Primes for all 4 different strata estimators 61,67,71,73,79,83,89,97 348
57 * Based on the bsc thesis of Elias Summermatter (2021)
58 */
59#define SE_IBFS_TOTAL_SIZE 632
60
61/**
62 * The hash num parameter for the difference digests and strata estimators.
63 */
64#define SE_IBF_HASH_NUM 3
65
66/**
67 * Number of buckets that can be transmitted in one message.
68 */
69#define MAX_BUCKETS_PER_MESSAGE ((1 << 16) / IBF_BUCKET_SIZE)
70
71/**
72 * The maximum size of an ibf we use is MAX_IBF_SIZE=2^20.
73 * Choose this value so that computing the IBF is still cheaper
74 * than transmitting all values.
75 */
76#define MAX_IBF_SIZE 1048576
77
78
79/**
80 * Minimal size of an ibf
81 * Based on the bsc thesis of Elias Summermatter (2021)
82 */
83#define IBF_MIN_SIZE 37
84
85/**
86 * AVG RTT for differential sync when k=2 and Factor = 2
87 * Based on the bsc thesis of Elias Summermatter (2021)
88 */
89#define DIFFERENTIAL_RTT_MEAN 3.65145
90
91/**
92 * Security level used for byzantine checks (2^80)
93 */
94
95#define SECURITY_LEVEL 80
96
97/**
98 * Is the estimated probability for a new round this values
99 * is based on the bsc thesis of Elias Summermatter (2021)
100 */
101
102#define PROBABILITY_FOR_NEW_ROUND 0.15
103
104/**
105 * Measure the performance in a csv
106 */
107
108#define MEASURE_PERFORMANCE 0
109
110
111/**
112 * Current phase we are in for a union operation.
113 */
114enum UnionOperationPhase
115{
116 /**
117 * We sent the request message, and expect a strata estimator.
118 */
119 PHASE_EXPECT_SE,
120
121 /**
122 * We sent the strata estimator, and expect an IBF. This phase is entered once
123 * upon initialization and later via #PHASE_EXPECT_ELEMENTS_AND_REQUESTS.
124 *
125 * XXX: could use better wording.
126 * XXX: repurposed to also expect a "request full set" message, should be renamed
127 *
128 * After receiving the complete IBF, we enter #PHASE_EXPECT_ELEMENTS
129 */
130 PHASE_EXPECT_IBF,
131
132 /**
133 * Continuation for multi part IBFs.
134 */
135 PHASE_EXPECT_IBF_LAST,
136
137 /**
138 * We are decoding an IBF.
139 */
140 PHASE_ACTIVE_DECODING,
141
142 /**
143 * The other peer is decoding the IBF we just sent.
144 */
145 PHASE_PASSIVE_DECODING,
146
147 /**
148 * The protocol is almost finished, but we still have to flush our message
149 * queue and/or expect some elements.
150 */
151 PHASE_FINISH_CLOSING,
152
153 /**
154 * In the penultimate phase, we wait until all our demands are satisfied.
155 * Then we send a done message, and wait for another done message.
156 */
157 PHASE_FINISH_WAITING,
158
159 /**
160 * In the ultimate phase, we wait until our demands are satisfied and then
161 * quit (sending another DONE message).
162 */
163 PHASE_FINISHED,
164
165 /**
166 * After sending the full set, wait for responses with the elements
167 * that the local peer is missing.
168 */
169 PHASE_FULL_SENDING,
170
171 /**
172 * Phase that receives full set first and then sends elements that are
173 * the local peer missing
174 */
175 PHASE_FULL_RECEIVING
176};
177
178/**
179 * Different modes of operations
180 */
181
182enum MODE_OF_OPERATION
183{
184 /**
185 * Mode just synchronizes the difference between sets
186 */
187 DIFFERENTIAL_SYNC,
188
189 /**
190 * Mode send full set sending local set first
191 */
192 FULL_SYNC_LOCAL_SENDING_FIRST,
193
194 /**
195 * Mode request full set from remote peer
196 */
197 FULL_SYNC_REMOTE_SENDING_FIRST
198};
199
200
201/**
202 * Information about an element element in the set. All elements are
203 * stored in a hash-table from their hash-code to their `struct
204 * Element`, so that the remove and add operations are reasonably
205 * fast.
206 */
207struct ElementEntry
208{
209 /**
210 * The actual element. The data for the element
211 * should be allocated at the end of this struct.
212 */
213 struct GNUNET_SETU_Element element;
214
215 /**
216 * Hash of the element. For set union: Will be used to derive the
217 * different IBF keys for different salts.
218 */
219 struct GNUNET_HashCode element_hash;
220
221 /**
222 * First generation that includes this element.
223 */
224 unsigned int generation;
225
226 /**
227 * #GNUNET_YES if the element is a remote element, and does not belong
228 * to the operation's set.
229 */
230 int remote;
231};
232
233
234/**
235 * A listener is inhabited by a client, and waits for evaluation
236 * requests from remote peers.
237 */
238struct Listener;
239
240
241/**
242 * A set that supports a specific operation with other peers.
243 */
244struct Set;
245
246
247/**
248 * State we keep per client.
249 */
250struct ClientState
251{
252 /**
253 * Set, if associated with the client, otherwise NULL.
254 */
255 struct Set *set;
256
257 /**
258 * Listener, if associated with the client, otherwise NULL.
259 */
260 struct Listener *listener;
261
262 /**
263 * Client handle.
264 */
265 struct GNUNET_SERVICE_Client *client;
266
267 /**
268 * Message queue.
269 */
270 struct GNUNET_MQ_Handle *mq;
271};
272
273
274/**
275 * Operation context used to execute a set operation.
276 */
277struct Operation
278{
279
280 /**
281 * The identity of the requesting peer. Needs to
282 * be stored here as the op spec might not have been created yet.
283 */
284 struct GNUNET_PeerIdentity peer;
285
286 /**
287 * Initial size of our set, just before the operation started.
288 */
289 uint64_t initial_size;
290
291 /**
292 * Kept in a DLL of the listener, if @e listener is non-NULL.
293 */
294 struct Operation *next;
295
296 /**
297 * Kept in a DLL of the listener, if @e listener is non-NULL.
298 */
299 struct Operation *prev;
300
301 /**
302 * Channel to the peer.
303 */
304 struct GNUNET_CADET_Channel *channel;
305
306 /**
307 * Port this operation runs on.
308 */
309 struct Listener *listener;
310
311 /**
312 * Message queue for the channel.
313 */
314 struct GNUNET_MQ_Handle *mq;
315
316 /**
317 * Context message, may be NULL.
318 */
319 struct GNUNET_MessageHeader *context_msg;
320
321 /**
322 * Set associated with the operation, NULL until the spec has been
323 * associated with a set.
324 */
325 struct Set *set;
326
327 /**
328 * Copy of the set's strata estimator at the time of
329 * creation of this operation.
330 */
331 struct MultiStrataEstimator *se;
332
333 /**
334 * The IBF we currently receive.
335 */
336 struct InvertibleBloomFilter *remote_ibf;
337
338 /**
339 * The IBF with the local set's element.
340 */
341 struct InvertibleBloomFilter *local_ibf;
342
343 /**
344 * Maps unsalted IBF-Keys to elements.
345 * Used as a multihashmap, the keys being the lower 32bit of the IBF-Key.
346 * Colliding IBF-Keys are linked.
347 */
348 struct GNUNET_CONTAINER_MultiHashMap32 *key_to_element;
349
350 /**
351 * Timeout task, if the incoming peer has not been accepted
352 * after the timeout, it will be disconnected.
353 */
354 struct GNUNET_SCHEDULER_Task *timeout_task;
355
356 /**
357 * Hashes for elements that we have demanded from the other peer.
358 */
359 struct GNUNET_CONTAINER_MultiHashMap *demanded_hashes;
360
361 /**
362 * Current state of the operation.
363 */
364 enum UnionOperationPhase phase;
365
366 /**
367 * Did we send the client that we are done?
368 */
369 int client_done_sent;
370
371 /**
372 * Number of ibf buckets already received into the @a remote_ibf.
373 */
374 uint64_t ibf_buckets_received;
375
376 /**
377 * Salt that we're using for sending IBFs
378 */
379 uint32_t salt_send;
380
381 /**
382 * Salt for the IBF we've received and that we're currently decoding.
383 */
384 uint32_t salt_receive;
385
386 /**
387 * Number of elements we received from the other peer
388 * that were not in the local set yet.
389 */
390 uint32_t received_fresh;
391
392 /**
393 * Total number of elements received from the other peer.
394 */
395 uint32_t received_total;
396
397 /**
398 * Salt to use for the operation.
399 */
400 uint32_t salt;
401
402 /**
403 * Remote peers element count
404 */
405 uint32_t remote_element_count;
406
407 /**
408 * ID used to identify an operation between service and client
409 */
410 uint32_t client_request_id;
411
412 /**
413 * Always use delta operation instead of sending full sets,
414 * even it it's less efficient.
415 */
416 int force_delta;
417
418 /**
419 * Always send full sets, even if delta operations would
420 * be more efficient.
421 */
422 int force_full;
423
424 /**
425 * #GNUNET_YES to fail operations where Byzantine faults
426 * are suspected
427 */
428 int byzantine;
429
430 /**
431 * #GNUNET_YES to also send back set elements we are sending to
432 * the remote peer.
433 */
434 int symmetric;
435
436 /**
437 * Lower bound for the set size, used only when
438 * byzantine mode is enabled.
439 */
440 uint64_t byzantine_lower_bound;
441
442 /**
443 * Unique request id for the request from a remote peer, sent to the
444 * client, which will accept or reject the request. Set to '0' iff
445 * the request has not been suggested yet.
446 */
447 uint32_t suggest_id;
448
449 /**
450 * Generation in which the operation handle
451 * was created.
452 */
453 unsigned int generation_created;
454
455
456 /**
457 * User defined Bandwidth Round Trips Tradeoff
458 */
459 uint64_t rtt_bandwidth_tradeoff;
460
461
462 /**
463 * Number of Element per bucket in IBF
464 */
465 uint8_t ibf_number_buckets_per_element;
466
467
468 /**
469 * Set difference is multiplied with this factor
470 * to gennerate large enough IBF
471 */
472 uint8_t ibf_bucket_number_factor;
473
474 /**
475 * Defines which site a client is
476 * 0 = Initiating peer
477 * 1 = Receiving peer
478 */
479 uint8_t peer_site;
480
481
482 /**
483 * Local peer element count
484 */
485 uint64_t local_element_count;
486
487 /**
488 * Mode of operation that was chosen by the algorithm
489 */
490 uint8_t mode_of_operation;
491
492 /**
493 * Hashmap to keep track of the send/received messages
494 */
495 struct GNUNET_CONTAINER_MultiHashMap *message_control_flow;
496
497 /**
498 * Hashmap to keep track of the send/received inquiries (ibf keys)
499 */
500 struct GNUNET_CONTAINER_MultiHashMap *inquiries_sent;
501
502
503 /**
504 * Total size of local set
505 */
506 uint64_t total_elements_size_local;
507
508 /**
509 * Limit of number of elements in set
510 */
511 uint64_t byzantine_upper_bound;
512
513 /**
514 * is the count of already passed differential sync iterations
515 */
516 uint8_t differential_sync_iterations;
517
518 /**
519 * Estimated or committed set difference at the start
520 */
521 uint64_t remote_set_diff;
522
523 /**
524 * Estimated or committed set difference at the start
525 */
526 uint64_t local_set_diff;
527
528 /**
529 * Boolean to enforce an active passive switch
530 */
531 bool active_passive_switch_required;
532};
533
534
535/**
536 * SetContent stores the actual set elements, which may be shared by
537 * multiple generations derived from one set.
538 */
539struct SetContent
540{
541 /**
542 * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *`.
543 */
544 struct GNUNET_CONTAINER_MultiHashMap *elements;
545
546 /**
547 * Maps `struct GNUNET_HashCode *` to `struct ElementEntry *` randomized.
548 */
549 struct GNUNET_CONTAINER_MultiHashMap *elements_randomized;
550
551 /**
552 * Salt to construct the randomized element map
553 */
554 uint64_t elements_randomized_salt;
555
556 /**
557 * Number of references to the content.
558 */
559 unsigned int refcount;
560
561 /**
562 * FIXME: document!
563 */
564 unsigned int latest_generation;
565
566 /**
567 * Number of concurrently active iterators.
568 */
569 int iterator_count;
570};
571
572
573/**
574 * A set that supports a specific operation with other peers.
575 */
576struct Set
577{
578 /**
579 * Sets are held in a doubly linked list (in `sets_head` and `sets_tail`).
580 */
581 struct Set *next;
582
583 /**
584 * Sets are held in a doubly linked list.
585 */
586 struct Set *prev;
587
588 /**
589 * Client that owns the set. Only one client may own a set,
590 * and there can only be one set per client.
591 */
592 struct ClientState *cs;
593
594 /**
595 * Content, possibly shared by multiple sets,
596 * and thus reference counted.
597 */
598 struct SetContent *content;
599
600 /**
601 * The strata estimator is only generated once for each set. The IBF keys
602 * are derived from the element hashes with salt=0.
603 */
604 struct MultiStrataEstimator *se;
605
606 /**
607 * Evaluate operations are held in a linked list.
608 */
609 struct Operation *ops_head;
610
611 /**
612 * Evaluate operations are held in a linked list.
613 */
614 struct Operation *ops_tail;
615
616 /**
617 * Current generation, that is, number of previously executed
618 * operations and lazy copies on the underlying set content.
619 */
620 unsigned int current_generation;
621
622};
623
624
625/**
626 * The key entry is used to associate an ibf key with an element.
627 */
628struct KeyEntry
629{
630 /**
631 * IBF key for the entry, derived from the current salt.
632 */
633 struct IBF_Key ibf_key;
634
635 /**
636 * The actual element associated with the key.
637 *
638 * Only owned by the union operation if element->operation
639 * is #GNUNET_YES.
640 */
641 struct ElementEntry *element;
642
643 /**
644 * Did we receive this element? Even if element->is_foreign is false, we
645 * might have received the element, so this indicates that the other peer
646 * has it.
647 */
648 int received;
649};
650
651
652/**
653 * Used as a closure for sending elements
654 * with a specific IBF key.
655 */
656struct SendElementClosure
657{
658 /**
659 * The IBF key whose matching elements should be
660 * sent.
661 */
662 struct IBF_Key ibf_key;
663
664 /**
665 * Operation for which the elements
666 * should be sent.
667 */
668 struct Operation *op;
669};
670
671
672/**
673 * A listener is inhabited by a client, and waits for evaluation
674 * requests from remote peers.
675 */
676struct Listener
677{
678 /**
679 * Listeners are held in a doubly linked list.
680 */
681 struct Listener *next;
682
683 /**
684 * Listeners are held in a doubly linked list.
685 */
686 struct Listener *prev;
687
688 /**
689 * Head of DLL of operations this listener is responsible for.
690 * Once the client has accepted/declined the operation, the
691 * operation is moved to the respective set's operation DLLS.
692 */
693 struct Operation *op_head;
694
695 /**
696 * Tail of DLL of operations this listener is responsible for.
697 * Once the client has accepted/declined the operation, the
698 * operation is moved to the respective set's operation DLLS.
699 */
700 struct Operation *op_tail;
701
702 /**
703 * Client that owns the listener.
704 * Only one client may own a listener.
705 */
706 struct ClientState *cs;
707
708 /**
709 * The port we are listening on with CADET.
710 */
711 struct GNUNET_CADET_Port *open_port;
712
713 /**
714 * Application ID for the operation, used to distinguish
715 * multiple operations of the same type with the same peer.
716 */
717 struct GNUNET_HashCode app_id;
718
719};
720
721
722/**
723 * Handle to the cadet service, used to listen for and connect to
724 * remote peers.
725 */
726static struct GNUNET_CADET_Handle *cadet;
727
728/**
729 * Statistics handle.
730 */
731static struct GNUNET_STATISTICS_Handle *_GSS_statistics;
732
733/**
734 * Listeners are held in a doubly linked list.
735 */
736static struct Listener *listener_head;
737
738/**
739 * Listeners are held in a doubly linked list.
740 */
741static struct Listener *listener_tail;
742
743/**
744 * Number of active clients.
745 */
746static unsigned int num_clients;
747
748/**
749 * Are we in shutdown? if #GNUNET_YES and the number of clients
750 * drops to zero, disconnect from CADET.
751 */
752static int in_shutdown;
753
754/**
755 * Counter for allocating unique IDs for clients, used to identify incoming
756 * operation requests from remote peers, that the client can choose to accept
757 * or refuse. 0 must not be used (reserved for uninitialized).
758 */
759static uint32_t suggest_id;
760
761#if MEASURE_PERFORMANCE
762/**
763 * Handles configuration file for setu performance test
764 *
765 */
766static const struct GNUNET_CONFIGURATION_Handle *setu_cfg;
767
768
769/**
770 * Stores the performance data for induvidual message
771 */
772
773
774struct perf_num_send_received_msg
775{
776 uint64_t sent;
777 uint64_t sent_var_bytes;
778 uint64_t received;
779 uint64_t received_var_bytes;
780};
781
782/**
783 * Main struct to measure performance (data/rtts)
784 */
785struct per_store_struct
786{
787 struct perf_num_send_received_msg operation_request;
788 struct perf_num_send_received_msg se;
789 struct perf_num_send_received_msg request_full;
790 struct perf_num_send_received_msg element_full;
791 struct perf_num_send_received_msg full_done;
792 struct perf_num_send_received_msg ibf;
793 struct perf_num_send_received_msg inquery;
794 struct perf_num_send_received_msg element;
795 struct perf_num_send_received_msg demand;
796 struct perf_num_send_received_msg offer;
797 struct perf_num_send_received_msg done;
798 struct perf_num_send_received_msg over;
799 uint64_t se_diff;
800 uint64_t se_diff_remote;
801 uint64_t se_diff_local;
802 uint64_t active_passive_switches;
803 uint8_t mode_of_operation;
804};
805
806struct per_store_struct perf_store;
807#endif
808
809/**
810 * Different states to control the messages flow in differential mode
811 */
812
813enum MESSAGE_CONTROL_FLOW_STATE
814{
815 /**
816 * Initial message state
817 */
818 MSG_CFS_UNINITIALIZED,
819
820 /**
821 * Track that a message has been sent
822 */
823 MSG_CFS_SENT,
824
825 /**
826 * Track that receiving this message is expected
827 */
828 MSG_CFS_EXPECTED,
829
830 /**
831 * Track that message has been received
832 */
833 MSG_CFS_RECEIVED,
834};
835
836/**
837 * Message types to track in message control flow
838 */
839
840enum MESSAGE_TYPE
841{
842 /**
843 * Offer message type
844 */
845 OFFER_MESSAGE,
846
847 /**
848 * Demand message type
849 */
850 DEMAND_MESSAGE,
851
852 /**
853 * Element message type
854 */
855 ELEMENT_MESSAGE,
856};
857
858
859/**
860 * Struct to tracked messages in message control flow
861 */
862struct messageControlFlowElement
863{
864 /**
865 * Track the message control state of the offer message
866 */
867 enum MESSAGE_CONTROL_FLOW_STATE offer;
868 /**
869 * Track the message control state of the demand message
870 */
871 enum MESSAGE_CONTROL_FLOW_STATE demand;
872 /**
873 * Track the message control state of the element message
874 */
875 enum MESSAGE_CONTROL_FLOW_STATE element;
876};
877
878
879#if MEASURE_PERFORMANCE
880
881/**
882 * Loads different configuration to perform performance tests
883 *
884 * @param op operation handle
885 */
886static void
887load_config (struct Operation *op)
888{
889 long long number;
890 float fl;
891
892 setu_cfg = GNUNET_CONFIGURATION_create ();
893 GNUNET_CONFIGURATION_load (setu_cfg,
894 "perf_setu.conf");
895 GNUNET_CONFIGURATION_get_value_float (setu_cfg,
896 "IBF",
897 "BUCKET_NUMBER_FACTOR",
898 &fl);
899 op->ibf_bucket_number_factor = fl;
900 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
901 "IBF",
902 "NUMBER_PER_BUCKET",
903 &number);
904 op->ibf_number_buckets_per_element = number;
905 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
906 "PERFORMANCE",
907 "TRADEOFF",
908 &number);
909 op->rtt_bandwidth_tradeoff = number;
910 GNUNET_CONFIGURATION_get_value_number (setu_cfg,
911 "BOUNDARIES",
912 "UPPER_ELEMENT",
913 &number);
914 op->byzantine_upper_bound = number;
915 op->peer_site = 0;
916}
917
918
919/**
920 * Function to calculate total bytes used for performance measurement
921 * @param size
922 * @param perf_num_send_received_msg
923 * @return bytes used
924 */
925static int
926sum_sent_received_bytes (uint64_t size,
927 struct perf_num_send_received_msg
928 perf_num_send_received_msg)
929{
930 return (size * perf_num_send_received_msg.sent)
931 + (size * perf_num_send_received_msg.received)
932 + perf_num_send_received_msg.sent_var_bytes
933 + perf_num_send_received_msg.received_var_bytes;
934}
935
936
937/**
938 * Function that calculates the perfmance values and writes them down
939 */
940static void
941calculate_perf_store ()
942{
943
944 /**
945 * Calculate RTT of init phase normally always 1
946 */
947 float rtt = 1;
948 int bytes_transmitted = 0;
949
950 /**
951 * Calculate RGNUNET_SETU_AcceptMessageRT of Fullsync normally 1 or 1.5 depending
952 */
953 if ((perf_store.element_full.received != 0) ||
954 (perf_store.element_full.sent != 0)
955 )
956 rtt += 1;
957
958 if ((perf_store.request_full.received != 0) ||
959 (perf_store.request_full.sent != 0)
960 )
961 rtt += 0.5;
962
963 /**
964 * In case of a differential sync 3 rtt's are needed.
965 * for every active/passive switch additional 3.5 rtt's are used
966 */
967 if ((perf_store.element.received != 0) ||
968 (perf_store.element.sent != 0))
969 {
970 int iterations = perf_store.active_passive_switches;
971
972 if (iterations > 0)
973 rtt += iterations * 0.5;
974 rtt += 2.5;
975 }
976
977
978 /**
979 * Calculate data sended size
980 */
981 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
982 GNUNET_SETU_ResultMessage),
983 perf_store.request_full);
984
985 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
986 GNUNET_SETU_ElementMessage),
987 perf_store.element_full);
988 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
989 GNUNET_SETU_ElementMessage),
990 perf_store.element);
991 // bytes_transmitted += sum_sent_received_bytes(sizeof(GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST), perf_store.operation_request);
992 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
993 StrataEstimatorMessage),
994 perf_store.se);
995 bytes_transmitted += sum_sent_received_bytes (4, perf_store.full_done);
996 bytes_transmitted += sum_sent_received_bytes (sizeof(struct IBFMessage),
997 perf_store.ibf);
998 bytes_transmitted += sum_sent_received_bytes (sizeof(struct InquiryMessage),
999 perf_store.inquery);
1000 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
1001 GNUNET_MessageHeader),
1002 perf_store.demand);
1003 bytes_transmitted += sum_sent_received_bytes (sizeof(struct
1004 GNUNET_MessageHeader),
1005 perf_store.offer);
1006 bytes_transmitted += sum_sent_received_bytes (4, perf_store.done);
1007
1008 /**
1009 * Write IBF failure rate for different BUCKET_NUMBER_FACTOR
1010 */
1011 float factor;
1012 GNUNET_CONFIGURATION_get_value_float (setu_cfg,"IBF", "BUCKET_NUMBER_FACTOR",
1013 &factor);
1014 long long num_per_bucket;
1015 GNUNET_CONFIGURATION_get_value_number (setu_cfg,"IBF", "NUMBER_PER_BUCKET",
1016 &num_per_bucket);
1017
1018
1019 int decoded = 0;
1020 if (perf_store.active_passive_switches == 0)
1021 decoded = 1;
1022 int ibf_bytes_transmitted = sum_sent_received_bytes (sizeof(struct
1023 IBFMessage),
1024 perf_store.ibf);
1025
1026 FILE *out1 = fopen ("perf_data.csv", "a");
1027 fprintf (out1, "%d,%f,%d,%d,%f,%d,%d,%d,%d,%d\n",num_per_bucket,factor,
1028 decoded,ibf_bytes_transmitted,rtt,perf_store.se_diff,
1029 bytes_transmitted,
1030 perf_store.se_diff_local,perf_store.se_diff_remote,
1031 perf_store.mode_of_operation);
1032 fclose (out1);
1033
1034}
1035
1036
1037#endif
1038/**
1039 * Function that chooses the optimal mode of operation depending on
1040 * operation parameters.
1041 * @param avg_element_size
1042 * @param local_set_size
1043 * @param remote_set_size
1044 * @param est_set_diff_remote
1045 * @param est_set_diff_local
1046 * @param bandwith_latency_tradeoff
1047 * @param ibf_bucket_number_factor
1048 * @return calcuated mode of operation
1049 */
1050static uint8_t
1051estimate_best_mode_of_operation (uint64_t avg_element_size,
1052 uint64_t local_set_size,
1053 uint64_t remote_set_size,
1054 uint64_t est_set_diff_remote,
1055 uint64_t est_set_diff_local,
1056 uint64_t bandwith_latency_tradeoff,
1057 uint64_t ibf_bucket_number_factor)
1058{
1059
1060 /*
1061 * In case of initial sync fall to predefined states
1062 */
1063
1064 if (0 == local_set_size)
1065 return FULL_SYNC_REMOTE_SENDING_FIRST;
1066 if (0 == remote_set_size)
1067 return FULL_SYNC_LOCAL_SENDING_FIRST;
1068
1069 /*
1070 * Calculate bytes for full Sync
1071 */
1072
1073 uint8_t sizeof_full_done_header = 4;
1074 uint8_t sizeof_done_header = 4;
1075 uint8_t rtt_min_full = 2;
1076 uint8_t sizeof_request_full = 4;
1077 uint64_t estimated_total_diff = (est_set_diff_remote + est_set_diff_local);
1078
1079 /* Estimate byte required if we send first */
1080 uint64_t total_elements_to_send_local_send_first = est_set_diff_remote
1081 + local_set_size;
1082
1083 uint64_t total_bytes_full_local_send_first = (avg_element_size
1084 *
1085 total_elements_to_send_local_send_first) \
1086 + (
1087 total_elements_to_send_local_send_first * sizeof(struct
1088 GNUNET_SETU_ElementMessage)) \
1089 + (sizeof_full_done_header * 2) \
1090 + rtt_min_full
1091 * bandwith_latency_tradeoff;
1092
1093 /* Estimate bytes required if we request from remote peer */
1094 uint64_t total_elements_to_send_remote_send_first = est_set_diff_local
1095 + remote_set_size;
1096
1097 uint64_t total_bytes_full_remote_send_first = (avg_element_size
1098 *
1099 total_elements_to_send_remote_send_first) \
1100 + (
1101 total_elements_to_send_remote_send_first * sizeof(struct
1102 GNUNET_SETU_ElementMessage)) \
1103 + (sizeof_full_done_header * 2) \
1104 + (rtt_min_full + 0.5)
1105 * bandwith_latency_tradeoff \
1106 + sizeof_request_full;
1107
1108 /*
1109 * Calculate bytes for differential Sync
1110 */
1111
1112 /* Estimate bytes required by IBF transmission*/
1113
1114 long double ibf_bucket_count = estimated_total_diff
1115 * ibf_bucket_number_factor;
1116
1117 if (ibf_bucket_count <= IBF_MIN_SIZE)
1118 {
1119 ibf_bucket_count = IBF_MIN_SIZE;
1120 }
1121 uint64_t ibf_message_count = ceil ( ((float) ibf_bucket_count)
1122 / ((float) MAX_BUCKETS_PER_MESSAGE));
1123
1124 uint64_t estimated_counter_size = ceil (
1125 MIN (2 * log2l (((float) local_set_size)
1126 / ((float) ibf_bucket_count)),
1127 log2l (local_set_size)));
1128
1129 long double counter_bytes = (float) estimated_counter_size / 8;
1130
1131 uint64_t ibf_bytes = ceil ((sizeof (struct IBFMessage) * ibf_message_count)
1132 * 1.2 \
1133 + (ibf_bucket_count * sizeof(struct IBF_Key)) * 1.2 \
1134 + (ibf_bucket_count * sizeof(struct IBF_KeyHash))
1135 * 1.2 \
1136 + (ibf_bucket_count * counter_bytes) * 1.2);
1137
1138 /* Estimate full byte count for differential sync */
1139 uint64_t element_size = (avg_element_size
1140 + sizeof (struct GNUNET_SETU_ElementMessage)) \
1141 * estimated_total_diff;
1142 uint64_t done_size = sizeof_done_header;
1143 uint64_t inquery_size = (sizeof (struct IBF_Key)
1144 + sizeof (struct InquiryMessage))
1145 * estimated_total_diff;
1146 uint64_t demand_size =
1147 (sizeof(struct GNUNET_HashCode) + sizeof(struct GNUNET_MessageHeader))
1148 * estimated_total_diff;
1149 uint64_t offer_size = (sizeof (struct GNUNET_HashCode)
1150 + sizeof (struct GNUNET_MessageHeader))
1151 * estimated_total_diff;
1152
1153 uint64_t total_bytes_diff = (element_size + done_size + inquery_size
1154 + demand_size + offer_size + ibf_bytes) \
1155 + (DIFFERENTIAL_RTT_MEAN
1156 * bandwith_latency_tradeoff);
1157
1158 uint64_t full_min = MIN (total_bytes_full_local_send_first,
1159 total_bytes_full_remote_send_first);
1160
1161 /* Decide between full and differential sync */
1162
1163 if (full_min < total_bytes_diff)
1164 {
1165 /* Decide between sending all element first or receiving all elements */
1166 if (total_bytes_full_remote_send_first > total_bytes_full_local_send_first)
1167 {
1168 return FULL_SYNC_LOCAL_SENDING_FIRST;
1169 }
1170 else
1171 {
1172 return FULL_SYNC_REMOTE_SENDING_FIRST;
1173 }
1174 }
1175 else
1176 {
1177 return DIFFERENTIAL_SYNC;
1178 }
1179}
1180
1181
1182/**
1183 * Validates the if a message is received in a correct phase
1184 * @param allowed_phases
1185 * @param size_phases
1186 * @param op
1187 * @return #GNUNET_YES if message permitted in phase and #GNUNET_NO if not permitted in given
1188 * phase
1189 */
1190static enum GNUNET_GenericReturnValue
1191check_valid_phase (const uint8_t allowed_phases[],
1192 size_t size_phases,
1193 struct Operation *op)
1194{
1195 /**
1196 * Iterate over allowed phases
1197 */
1198 for (uint32_t phase_ctr = 0; phase_ctr < size_phases; phase_ctr++)
1199 {
1200 uint8_t phase = allowed_phases[phase_ctr];
1201 if (phase == op->phase)
1202 {
1203 LOG (GNUNET_ERROR_TYPE_DEBUG,
1204 "Message received in valid phase\n");
1205 return GNUNET_YES;
1206 }
1207 }
1208 LOG (GNUNET_ERROR_TYPE_ERROR,
1209 "Received message in invalid phase: %u\n", op->phase);
1210 return GNUNET_NO;
1211}
1212
1213
1214/**
1215 * Function to update, track and validate message received in differential
1216 * sync. This function tracks states of messages and check it against different
1217 * constraints as described in Summermatter's BSc Thesis (2021)
1218 * @param hash_map: Hashmap to store message control flow
1219 * @param new_mcfs: The new message control flow state an given message type should be set to
1220 * @param hash_code: Hash code of the element
1221 * @param mt: The message type for which the message control flow state should be set
1222 * @return GNUNET_YES message is valid in message control flow GNUNET_NO when message is not valid
1223 * at this point in message flow
1224 */
1225static int
1226update_message_control_flow (struct GNUNET_CONTAINER_MultiHashMap *hash_map,
1227 enum MESSAGE_CONTROL_FLOW_STATE new_mcfs,
1228 const struct GNUNET_HashCode *hash_code,
1229 enum MESSAGE_TYPE mt)
1230{
1231 struct messageControlFlowElement *cfe = NULL;
1232 enum MESSAGE_CONTROL_FLOW_STATE *mcfs;
1233
1234 /**
1235 * Check logic for forbidden messages
1236 */
1237
1238 cfe = GNUNET_CONTAINER_multihashmap_get (hash_map, hash_code);
1239 if ((ELEMENT_MESSAGE == mt) && (cfe != NULL))
1240 {
1241 if ((new_mcfs != MSG_CFS_SENT) && (MSG_CFS_RECEIVED != cfe->offer))
1242 {
1243 LOG (GNUNET_ERROR_TYPE_ERROR,
1244 "Received an element without sent offer!\n");
1245 return GNUNET_NO;
1246 }
1247 /* Check that only requested elements are received! */
1248 if ((ELEMENT_MESSAGE == mt) && (new_mcfs != MSG_CFS_SENT) && (cfe->demand !=
1249 MSG_CFS_SENT))
1250 {
1251 LOG (GNUNET_ERROR_TYPE_ERROR,
1252 "Received an element that was not demanded\n");
1253 return GNUNET_NO;
1254 }
1255 }
1256
1257 /**
1258 * In case the element hash is not in the hashmap create a new entry
1259 */
1260
1261 if (NULL == cfe)
1262 {
1263 cfe = GNUNET_new (struct messageControlFlowElement);
1264 if (GNUNET_SYSERR == GNUNET_CONTAINER_multihashmap_put (hash_map, hash_code,
1265 cfe,
1266 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1267 {
1268 GNUNET_free (cfe);
1269 return GNUNET_SYSERR;
1270 }
1271 }
1272
1273 /**
1274 * Set state of message
1275 */
1276
1277 if (OFFER_MESSAGE == mt)
1278 {
1279 mcfs = &cfe->offer;
1280 }
1281 else if (DEMAND_MESSAGE == mt)
1282 {
1283 mcfs = &cfe->demand;
1284 }
1285 else if (ELEMENT_MESSAGE == mt)
1286 {
1287 mcfs = &cfe->element;
1288 }
1289 else
1290 {
1291 return GNUNET_SYSERR;
1292 }
1293
1294 /**
1295 * Check if state is allowed
1296 */
1297
1298 if (new_mcfs <= *mcfs)
1299 {
1300 return GNUNET_NO;
1301 }
1302
1303 *mcfs = new_mcfs;
1304 return GNUNET_YES;
1305}
1306
1307
1308/**
1309 * Validate if a message in differential sync si already received before.
1310 * @param hash_map
1311 * @param hash_code
1312 * @param mt
1313 * @return GNUNET_YES when message is already in store if message is not in store return GNUNET_NO
1314 */
1315static int
1316is_message_in_message_control_flow (struct
1317 GNUNET_CONTAINER_MultiHashMap *hash_map,
1318 struct GNUNET_HashCode *hash_code,
1319 enum MESSAGE_TYPE mt)
1320{
1321 struct messageControlFlowElement *cfe = NULL;
1322 enum MESSAGE_CONTROL_FLOW_STATE *mcfs;
1323
1324 cfe = GNUNET_CONTAINER_multihashmap_get (hash_map, hash_code);
1325
1326 /**
1327 * Set state of message
1328 */
1329
1330 if (cfe != NULL)
1331 {
1332 if (OFFER_MESSAGE == mt)
1333 {
1334 mcfs = &cfe->offer;
1335 }
1336 else if (DEMAND_MESSAGE == mt)
1337 {
1338 mcfs = &cfe->demand;
1339 }
1340 else if (ELEMENT_MESSAGE == mt)
1341 {
1342 mcfs = &cfe->element;
1343 }
1344 else
1345 {
1346 return GNUNET_SYSERR;
1347 }
1348
1349 /**
1350 * Evaluate if set is in message
1351 */
1352 if (*mcfs != MSG_CFS_UNINITIALIZED)
1353 {
1354 return GNUNET_YES;
1355 }
1356 }
1357 return GNUNET_NO;
1358}
1359
1360
1361/**
1362 * Iterator for determining if all demands have been
1363 * satisfied
1364 *
1365 * @param cls the union operation `struct Operation *`
1366 * @param key unused
1367 * @param value the `struct ElementEntry *` to insert
1368 * into the key-to-element mapping
1369 * @return #GNUNET_YES (to continue iterating)
1370 */
1371static int
1372determinate_done_message_iterator (void *cls,
1373 const struct GNUNET_HashCode *key,
1374 void *value)
1375{
1376 struct messageControlFlowElement *mcfe = value;
1377
1378 if (((mcfe->element == MSG_CFS_SENT) || (mcfe->element == MSG_CFS_RECEIVED) ))
1379 {
1380 return GNUNET_YES;
1381 }
1382 return GNUNET_NO;
1383}
1384
1385
1386/**
1387 * Iterator for determining average size
1388 *
1389 * @param cls the union operation `struct Operation *`
1390 * @param key unused
1391 * @param value the `struct ElementEntry *` to insert
1392 * into the key-to-element mapping
1393 * @return #GNUNET_YES (to continue iterating)
1394 */
1395static int
1396determinate_avg_element_size_iterator (void *cls,
1397 const struct GNUNET_HashCode *key,
1398 void *value)
1399{
1400 struct Operation *op = cls;
1401 struct GNUNET_SETU_Element *element = value;
1402 op->total_elements_size_local += element->size;
1403 return GNUNET_YES;
1404}
1405
1406
1407/**
1408 * Create randomized element hashmap for full sending
1409 *
1410 * @param cls the union operation `struct Operation *`
1411 * @param key unused
1412 * @param value the `struct ElementEntry *` to insert
1413 * into the key-to-element mapping
1414 * @return #GNUNET_YES (to continue iterating)
1415 */
1416static int
1417create_randomized_element_iterator (void *cls,
1418 const struct GNUNET_HashCode *key,
1419 void *value)
1420{
1421 struct Operation *op = cls;
1422
1423 struct GNUNET_HashContext *hashed_key_context =
1424 GNUNET_CRYPTO_hash_context_start ();
1425 struct GNUNET_HashCode new_key;
1426
1427 /**
1428 * Hash element with new salt to randomize hashmap
1429 */
1430 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
1431 &key,
1432 sizeof(struct IBF_Key));
1433 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
1434 &op->set->content->elements_randomized_salt,
1435 sizeof(uint32_t));
1436 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
1437 &new_key);
1438 GNUNET_CONTAINER_multihashmap_put (op->set->content->elements_randomized,
1439 &new_key,value,
1440 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
1441 return GNUNET_YES;
1442}
1443
1444
1445/**
1446 * Iterator over hash map entries, called to
1447 * destroy the linked list of colliding ibf key entries.
1448 *
1449 * @param cls closure
1450 * @param key current key code
1451 * @param value value in the hash map
1452 * @return #GNUNET_YES if we should continue to iterate,
1453 * #GNUNET_NO if not.
1454 */
1455static int
1456destroy_key_to_element_iter (void *cls,
1457 uint32_t key,
1458 void *value)
1459{
1460 struct KeyEntry *k = value;
1461
1462 GNUNET_assert (NULL != k);
1463 if (GNUNET_YES == k->element->remote)
1464 {
1465 GNUNET_free (k->element);
1466 k->element = NULL;
1467 }
1468 GNUNET_free (k);
1469 return GNUNET_YES;
1470}
1471
1472
1473/**
1474 * Signal to the client that the operation has finished and
1475 * destroy the operation.
1476 *
1477 * @param cls operation to destroy
1478 */
1479static void
1480send_client_done (void *cls)
1481{
1482 struct Operation *op = cls;
1483 struct GNUNET_MQ_Envelope *ev;
1484 struct GNUNET_SETU_ResultMessage *rm;
1485
1486 if (GNUNET_YES == op->client_done_sent)
1487 return;
1488 if (PHASE_FINISHED != op->phase)
1489 {
1490 LOG (GNUNET_ERROR_TYPE_WARNING,
1491 "Union operation failed\n");
1492 GNUNET_STATISTICS_update (_GSS_statistics,
1493 "# Union operations failed",
1494 1,
1495 GNUNET_NO);
1496 ev = GNUNET_MQ_msg (rm, GNUNET_MESSAGE_TYPE_SETU_RESULT);
1497 rm->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
1498 rm->request_id = htonl (op->client_request_id);
1499 rm->element_type = htons (0);
1500 GNUNET_MQ_send (op->set->cs->mq,
1501 ev);
1502 return;
1503 }
1504
1505 op->client_done_sent = GNUNET_YES;
1506
1507 GNUNET_STATISTICS_update (_GSS_statistics,
1508 "# Union operations succeeded",
1509 1,
1510 GNUNET_NO);
1511 LOG (GNUNET_ERROR_TYPE_INFO,
1512 "Signalling client that union operation is done\n");
1513 ev = GNUNET_MQ_msg (rm,
1514 GNUNET_MESSAGE_TYPE_SETU_RESULT);
1515 rm->request_id = htonl (op->client_request_id);
1516 rm->result_status = htons (GNUNET_SETU_STATUS_DONE);
1517 rm->element_type = htons (0);
1518 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
1519 op->key_to_element));
1520 GNUNET_MQ_send (op->set->cs->mq,
1521 ev);
1522}
1523
1524
1525/**
1526 * Check if all given byzantine parameters are in given boundaries
1527 * @param op
1528 * @return indicator if all given byzantine parameters are in given boundaries
1529 */
1530
1531static int
1532check_byzantine_bounds (struct Operation *op)
1533{
1534 if (op->byzantine != GNUNET_YES)
1535 return GNUNET_OK;
1536
1537 /**
1538 * Check upper byzantine bounds
1539 */
1540 if (op->remote_element_count + op->remote_set_diff >
1541 op->byzantine_upper_bound)
1542 return GNUNET_SYSERR;
1543 if (op->local_element_count + op->local_set_diff > op->byzantine_upper_bound)
1544 return GNUNET_SYSERR;
1545
1546 /**
1547 * Check lower byzantine bounds
1548 */
1549 if (op->remote_element_count < op->byzantine_lower_bound)
1550 return GNUNET_SYSERR;
1551 return GNUNET_OK;
1552}
1553
1554
1555static enum GNUNET_GenericReturnValue
1556free_values_iter(void *cls,
1557 const struct GNUNET_HashCode *key,
1558 void *value)
1559{
1560 GNUNET_free (value);
1561 return GNUNET_YES;
1562}
1563
1564
1565/* FIXME: the destroy logic is a mess and should be cleaned up! */
1566
1567/**
1568 * Destroy the given operation. Used for any operation where both
1569 * peers were known and that thus actually had a vt and channel. Must
1570 * not be used for operations where 'listener' is still set and we do
1571 * not know the other peer.
1572 *
1573 * Call the implementation-specific cancel function of the operation.
1574 * Disconnects from the remote peer. Does not disconnect the client,
1575 * as there may be multiple operations per set.
1576 *
1577 * @param op operation to destroy
1578 */
1579static void
1580_GSS_operation_destroy (struct Operation *op)
1581{
1582 struct Set *set = op->set;
1583 struct GNUNET_CADET_Channel *channel;
1584
1585 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1586 "Destroying union operation %p\n",
1587 op);
1588 GNUNET_assert (NULL == op->listener);
1589 /* check if the op was canceled twice */
1590 if (NULL != op->remote_ibf)
1591 {
1592 ibf_destroy (op->remote_ibf);
1593 op->remote_ibf = NULL;
1594 }
1595 if (NULL != op->demanded_hashes)
1596 {
1597 GNUNET_CONTAINER_multihashmap_destroy (op->demanded_hashes);
1598 op->demanded_hashes = NULL;
1599 }
1600 if (NULL != op->local_ibf)
1601 {
1602 ibf_destroy (op->local_ibf);
1603 op->local_ibf = NULL;
1604 }
1605 if (NULL != op->se)
1606 {
1607 strata_estimator_destroy (op->se);
1608 op->se = NULL;
1609 }
1610 if (NULL != op->key_to_element)
1611 {
1612 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
1613 &destroy_key_to_element_iter,
1614 NULL);
1615 GNUNET_CONTAINER_multihashmap32_destroy (op->key_to_element);
1616 op->key_to_element = NULL;
1617 }
1618 if (NULL != op->message_control_flow)
1619 {
1620 GNUNET_CONTAINER_multihashmap_iterate (op->message_control_flow,
1621 &free_values_iter,
1622 NULL);
1623 GNUNET_CONTAINER_multihashmap_destroy (op->message_control_flow);
1624 op->message_control_flow = NULL;
1625 }
1626 if (NULL != op->inquiries_sent)
1627 {
1628 GNUNET_CONTAINER_multihashmap_destroy (op->inquiries_sent);
1629 op->inquiries_sent = NULL;
1630 }
1631 if (NULL != set)
1632 {
1633 GNUNET_CONTAINER_DLL_remove (set->ops_head,
1634 set->ops_tail,
1635 op);
1636 op->set = NULL;
1637 }
1638 if (NULL != op->context_msg)
1639 {
1640 GNUNET_free (op->context_msg);
1641 op->context_msg = NULL;
1642 }
1643 if (NULL != (channel = op->channel))
1644 {
1645 /* This will free op; called conditionally as this helper function
1646 is also called from within the channel disconnect handler. */
1647 op->channel = NULL;
1648 GNUNET_CADET_channel_destroy (channel);
1649 }
1650 /* We rely on the channel end handler to free 'op'. When 'op->channel' was NULL,
1651 * there was a channel end handler that will free 'op' on the call stack. */
1652}
1653
1654
1655/**
1656 * This function probably should not exist
1657 * and be replaced by inlining more specific
1658 * logic in the various places where it is called.
1659 */
1660static void
1661_GSS_operation_destroy2 (struct Operation *op);
1662
1663
1664/**
1665 * Destroy an incoming request from a remote peer
1666 *
1667 * @param op remote request to destroy
1668 */
1669static void
1670incoming_destroy (struct Operation *op)
1671{
1672 struct Listener *listener;
1673
1674 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1675 "Destroying incoming operation %p\n",
1676 op);
1677 if (NULL != (listener = op->listener))
1678 {
1679 GNUNET_CONTAINER_DLL_remove (listener->op_head,
1680 listener->op_tail,
1681 op);
1682 op->listener = NULL;
1683 }
1684 if (NULL != op->timeout_task)
1685 {
1686 GNUNET_SCHEDULER_cancel (op->timeout_task);
1687 op->timeout_task = NULL;
1688 }
1689 _GSS_operation_destroy2 (op);
1690}
1691
1692
1693/**
1694 * This function probably should not exist
1695 * and be replaced by inlining more specific
1696 * logic in the various places where it is called.
1697 */
1698static void
1699_GSS_operation_destroy2 (struct Operation *op)
1700{
1701 struct GNUNET_CADET_Channel *channel;
1702
1703 if (NULL != (channel = op->channel))
1704 {
1705 /* This will free op; called conditionally as this helper function
1706 is also called from within the channel disconnect handler. */
1707 op->channel = NULL;
1708 GNUNET_CADET_channel_destroy (channel);
1709 }
1710 if (NULL != op->listener)
1711 {
1712 incoming_destroy (op);
1713 return;
1714 }
1715 if (NULL != op->set)
1716 send_client_done (op);
1717 _GSS_operation_destroy (op);
1718 GNUNET_free (op);
1719}
1720
1721
1722/**
1723 * Inform the client that the union operation has failed,
1724 * and proceed to destroy the evaluate operation.
1725 *
1726 * @param op the union operation to fail
1727 */
1728static void
1729fail_union_operation (struct Operation *op)
1730{
1731 struct GNUNET_MQ_Envelope *ev;
1732 struct GNUNET_SETU_ResultMessage *msg;
1733
1734 LOG (GNUNET_ERROR_TYPE_WARNING,
1735 "union operation failed\n");
1736 ev = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_SETU_RESULT);
1737 msg->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
1738 msg->request_id = htonl (op->client_request_id);
1739 msg->element_type = htons (0);
1740 GNUNET_MQ_send (op->set->cs->mq,
1741 ev);
1742 _GSS_operation_destroy (op);
1743}
1744
1745
1746/**
1747 * Function that checks if full sync is plausible
1748 * @param initial_local_elements_in_set
1749 * @param estimated_set_difference
1750 * @param repeated_elements
1751 * @param fresh_elements
1752 * @param op
1753 * @return GNUNET_OK if
1754 */
1755
1756static void
1757full_sync_plausibility_check (struct Operation *op)
1758{
1759 if (GNUNET_YES != op->byzantine)
1760 return;
1761
1762 int security_level_lb = -1 * SECURITY_LEVEL;
1763 uint64_t duplicates = op->received_fresh - op->received_total;
1764
1765 /*
1766 * Protect full sync from receiving double element when in FULL SENDING
1767 */
1768 if (PHASE_FULL_SENDING == op->phase)
1769 {
1770 if (duplicates > 0)
1771 {
1772 LOG (GNUNET_ERROR_TYPE_ERROR,
1773 "PROTOCOL VIOLATION: Received duplicate element in full receiving "
1774 "mode of operation this is not allowed! Duplicates: %llu\n",
1775 (unsigned long long) duplicates);
1776 GNUNET_break_op (0);
1777 fail_union_operation (op);
1778 return;
1779 }
1780
1781 }
1782
1783 /*
1784 * Protect full sync with probabilistic algorithm
1785 */
1786 if (PHASE_FULL_RECEIVING == op->phase)
1787 {
1788 if (0 == op->remote_set_diff)
1789 op->remote_set_diff = 1;
1790
1791 long double base = (1 - (long double) (op->remote_set_diff
1792 / (long double) (op->initial_size
1793 + op->
1794 remote_set_diff)));
1795 long double exponent = (op->received_total - (op->received_fresh * ((long
1796 double)
1797 op->
1798 initial_size
1799 / (long
1800 double)
1801 op->
1802 remote_set_diff)));
1803 long double value = exponent * (log2l (base) / log2l (2));
1804 if ((value < security_level_lb) || (value > SECURITY_LEVEL) )
1805 {
1806 LOG (GNUNET_ERROR_TYPE_ERROR,
1807 "PROTOCOL VIOLATION: Other peer violated probabilistic rule for receiving "
1808 "to many duplicated full element : %LF\n",
1809 value);
1810 GNUNET_break_op (0);
1811 fail_union_operation (op);
1812 return;
1813 }
1814 }
1815}
1816
1817
1818/**
1819 * Limit active passive switches in differential sync to configured security level
1820 * @param op
1821 */
1822static void
1823check_max_differential_rounds (struct Operation *op)
1824{
1825 double probability = op->differential_sync_iterations * (log2l (
1826 PROBABILITY_FOR_NEW_ROUND)
1827 / log2l (2));
1828 if ((-1 * SECURITY_LEVEL) > probability)
1829 {
1830 LOG (GNUNET_ERROR_TYPE_ERROR,
1831 "PROTOCOL VIOLATION: Other peer violated probabilistic rule for to many active passive "
1832 "switches in differential sync: %u\n",
1833 op->differential_sync_iterations);
1834 GNUNET_break_op (0);
1835 fail_union_operation (op);
1836 return;
1837 }
1838}
1839
1840
1841/**
1842 * Derive the IBF key from a hash code and
1843 * a salt.
1844 *
1845 * @param src the hash code
1846 * @return the derived IBF key
1847 */
1848static struct IBF_Key
1849get_ibf_key (const struct GNUNET_HashCode *src)
1850{
1851 struct IBF_Key key;
1852 uint16_t salt = 0;
1853
1854 GNUNET_assert (GNUNET_OK ==
1855 GNUNET_CRYPTO_kdf (&key, sizeof(key),
1856 src, sizeof *src,
1857 &salt, sizeof(salt),
1858 NULL, 0));
1859 return key;
1860}
1861
1862
1863/**
1864 * Context for #op_get_element_iterator
1865 */
1866struct GetElementContext
1867{
1868 /**
1869 * Gnunet hash code in context
1870 */
1871 struct GNUNET_HashCode hash;
1872
1873 /**
1874 * Pointer to the key entry
1875 */
1876 struct KeyEntry *k;
1877};
1878
1879
1880/**
1881 * Iterator over the mapping from IBF keys to element entries. Checks if we
1882 * have an element with a given GNUNET_HashCode.
1883 *
1884 * @param cls closure
1885 * @param key current key code
1886 * @param value value in the hash map
1887 * @return #GNUNET_YES if we should search further,
1888 * #GNUNET_NO if we've found the element.
1889 */
1890static int
1891op_get_element_iterator (void *cls,
1892 uint32_t key,
1893 void *value)
1894{
1895 struct GetElementContext *ctx = cls;
1896 struct KeyEntry *k = value;
1897
1898 GNUNET_assert (NULL != k);
1899 if (0 == GNUNET_CRYPTO_hash_cmp (&k->element->element_hash,
1900 &ctx->hash))
1901 {
1902 ctx->k = k;
1903 return GNUNET_NO;
1904 }
1905 return GNUNET_YES;
1906}
1907
1908
1909/**
1910 * Determine whether the given element is already in the operation's element
1911 * set.
1912 *
1913 * @param op operation that should be tested for 'element_hash'
1914 * @param element_hash hash of the element to look for
1915 * @return #GNUNET_YES if the element has been found, #GNUNET_NO otherwise
1916 */
1917static struct KeyEntry *
1918op_get_element (struct Operation *op,
1919 const struct GNUNET_HashCode *element_hash)
1920{
1921 int ret;
1922 struct IBF_Key ibf_key;
1923 struct GetElementContext ctx = { { { 0 } }, 0 };
1924
1925 ctx.hash = *element_hash;
1926
1927 ibf_key = get_ibf_key (element_hash);
1928 ret = GNUNET_CONTAINER_multihashmap32_get_multiple (op->key_to_element,
1929 (uint32_t) ibf_key.key_val,
1930 &op_get_element_iterator,
1931 &ctx);
1932
1933 /* was the iteration aborted because we found the element? */
1934 if (GNUNET_SYSERR == ret)
1935 {
1936 GNUNET_assert (NULL != ctx.k);
1937 return ctx.k;
1938 }
1939 return NULL;
1940}
1941
1942
1943/**
1944 * Insert an element into the union operation's
1945 * key-to-element mapping. Takes ownership of 'ee'.
1946 * Note that this does not insert the element in the set,
1947 * only in the operation's key-element mapping.
1948 * This is done to speed up re-tried operations, if some elements
1949 * were transmitted, and then the IBF fails to decode.
1950 *
1951 * XXX: clarify ownership, doesn't sound right.
1952 *
1953 * @param op the union operation
1954 * @param ee the element entry
1955 * @param received was this element received from the remote peer?
1956 */
1957static void
1958op_register_element (struct Operation *op,
1959 struct ElementEntry *ee,
1960 int received)
1961{
1962 struct IBF_Key ibf_key;
1963 struct KeyEntry *k;
1964
1965 ibf_key = get_ibf_key (&ee->element_hash);
1966 k = GNUNET_new (struct KeyEntry);
1967 k->element = ee;
1968 k->ibf_key = ibf_key;
1969 k->received = received;
1970 GNUNET_assert (GNUNET_OK ==
1971 GNUNET_CONTAINER_multihashmap32_put (op->key_to_element,
1972 (uint32_t) ibf_key.key_val,
1973 k,
1974 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
1975}
1976
1977
1978/**
1979 * Modify an IBF key @a k_in based on the @a salt, returning a
1980 * salted key in @a k_out.
1981 */
1982static void
1983salt_key (const struct IBF_Key *k_in,
1984 uint32_t salt,
1985 struct IBF_Key *k_out)
1986{
1987 int s = (salt * 7) % 64;
1988 uint64_t x = k_in->key_val;
1989
1990 /* rotate ibf key */
1991 x = (x >> s) | (x << (64 - s));
1992 k_out->key_val = x;
1993}
1994
1995
1996/**
1997 * Reverse modification done in the salt_key function
1998 */
1999static void
2000unsalt_key (const struct IBF_Key *k_in,
2001 uint32_t salt,
2002 struct IBF_Key *k_out)
2003{
2004 int s = (salt * 7) % 64;
2005 uint64_t x = k_in->key_val;
2006
2007 x = (x << s) | (x >> (64 - s));
2008 k_out->key_val = x;
2009}
2010
2011
2012/**
2013 * Insert a key into an ibf.
2014 *
2015 * @param cls the ibf
2016 * @param key unused
2017 * @param value the key entry to get the key from
2018 */
2019static int
2020prepare_ibf_iterator (void *cls,
2021 uint32_t key,
2022 void *value)
2023{
2024 struct Operation *op = cls;
2025 struct KeyEntry *ke = value;
2026 struct IBF_Key salted_key;
2027
2028 LOG (GNUNET_ERROR_TYPE_DEBUG,
2029 "[OP %p] inserting %lx (hash %s) into ibf\n",
2030 op,
2031 (unsigned long) ke->ibf_key.key_val,
2032 GNUNET_h2s (&ke->element->element_hash));
2033 salt_key (&ke->ibf_key,
2034 op->salt_send,
2035 &salted_key);
2036 ibf_insert (op->local_ibf, salted_key);
2037 return GNUNET_YES;
2038}
2039
2040
2041/**
2042 * Is element @a ee part of the set used by @a op?
2043 *
2044 * @param ee element to test
2045 * @param op operation the defines the set and its generation
2046 * @return #GNUNET_YES if the element is in the set, #GNUNET_NO if not
2047 */
2048static int
2049_GSS_is_element_of_operation (struct ElementEntry *ee,
2050 struct Operation *op)
2051{
2052 return ee->generation >= op->generation_created;
2053}
2054
2055
2056/**
2057 * Iterator for initializing the
2058 * key-to-element mapping of a union operation
2059 *
2060 * @param cls the union operation `struct Operation *`
2061 * @param key unused
2062 * @param value the `struct ElementEntry *` to insert
2063 * into the key-to-element mapping
2064 * @return #GNUNET_YES (to continue iterating)
2065 */
2066static int
2067init_key_to_element_iterator (void *cls,
2068 const struct GNUNET_HashCode *key,
2069 void *value)
2070{
2071 struct Operation *op = cls;
2072 struct ElementEntry *ee = value;
2073
2074 /* make sure that the element belongs to the set at the time
2075 * of creating the operation */
2076 if (GNUNET_NO ==
2077 _GSS_is_element_of_operation (ee,
2078 op))
2079 return GNUNET_YES;
2080 GNUNET_assert (GNUNET_NO == ee->remote);
2081 op_register_element (op,
2082 ee,
2083 GNUNET_NO);
2084 return GNUNET_YES;
2085}
2086
2087
2088/**
2089 * Initialize the IBF key to element mapping local to this set operation.
2090 *
2091 * @param op the set union operation
2092 */
2093static void
2094initialize_key_to_element (struct Operation *op)
2095{
2096 unsigned int len;
2097
2098 GNUNET_assert (NULL == op->key_to_element);
2099 len = GNUNET_CONTAINER_multihashmap_size (op->set->content->elements);
2100 op->key_to_element = GNUNET_CONTAINER_multihashmap32_create (len + 1);
2101 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2102 &init_key_to_element_iterator,
2103 op);
2104}
2105
2106
2107/**
2108 * Create an ibf with the operation's elements
2109 * of the specified size
2110 *
2111 * @param op the union operation
2112 * @param size size of the ibf to create
2113 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2114 */
2115static int
2116prepare_ibf (struct Operation *op,
2117 uint32_t size)
2118{
2119 GNUNET_assert (NULL != op->key_to_element);
2120
2121 if (NULL != op->local_ibf)
2122 ibf_destroy (op->local_ibf);
2123 // op->local_ibf = ibf_create (size, SE_IBF_HASH_NUM);
2124 op->local_ibf = ibf_create (size,
2125 ((uint8_t) op->ibf_number_buckets_per_element));
2126 if (NULL == op->local_ibf)
2127 {
2128 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2129 "Failed to allocate local IBF\n");
2130 return GNUNET_SYSERR;
2131 }
2132 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
2133 &prepare_ibf_iterator,
2134 op);
2135 return GNUNET_OK;
2136}
2137
2138
2139/**
2140 * Send an ibf of appropriate size.
2141 *
2142 * Fragments the IBF into multiple messages if necessary.
2143 *
2144 * @param op the union operation
2145 * @param ibf_order order of the ibf to send, size=2^order
2146 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2147 */
2148static int
2149send_ibf (struct Operation *op,
2150 uint32_t ibf_size)
2151{
2152 uint64_t buckets_sent = 0;
2153 struct InvertibleBloomFilter *ibf;
2154 op->differential_sync_iterations++;
2155
2156 /**
2157 * Enforce min size of IBF
2158 */
2159 uint32_t ibf_min_size = IBF_MIN_SIZE;
2160
2161 if (ibf_size < ibf_min_size)
2162 {
2163 ibf_size = ibf_min_size;
2164 }
2165 if (GNUNET_OK !=
2166 prepare_ibf (op, ibf_size))
2167 {
2168 /* allocation failed */
2169 return GNUNET_SYSERR;
2170 }
2171
2172 LOG (GNUNET_ERROR_TYPE_DEBUG,
2173 "sending ibf of size %u\n",
2174 (unsigned int) ibf_size);
2175
2176 {
2177 char name[64];
2178
2179 GNUNET_snprintf (name,
2180 sizeof(name),
2181 "# sent IBF (order %u)",
2182 ibf_size);
2183 GNUNET_STATISTICS_update (_GSS_statistics, name, 1, GNUNET_NO);
2184 }
2185
2186 ibf = op->local_ibf;
2187
2188 while (buckets_sent < ibf_size)
2189 {
2190 unsigned int buckets_in_message;
2191 struct GNUNET_MQ_Envelope *ev;
2192 struct IBFMessage *msg;
2193
2194 buckets_in_message = ibf_size - buckets_sent;
2195 /* limit to maximum */
2196 if (buckets_in_message > MAX_BUCKETS_PER_MESSAGE)
2197 buckets_in_message = MAX_BUCKETS_PER_MESSAGE;
2198
2199#if MEASURE_PERFORMANCE
2200 perf_store.ibf.sent += 1;
2201 perf_store.ibf.sent_var_bytes += (buckets_in_message * IBF_BUCKET_SIZE);
2202#endif
2203 ev = GNUNET_MQ_msg_extra (msg,
2204 buckets_in_message * IBF_BUCKET_SIZE,
2205 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF);
2206 msg->ibf_size = ibf_size;
2207 msg->offset = htonl (buckets_sent);
2208 msg->salt = htonl (op->salt_send);
2209 msg->ibf_counter_bit_length = ibf_get_max_counter (ibf);
2210
2211
2212 ibf_write_slice (ibf, buckets_sent,
2213 buckets_in_message, &msg[1], msg->ibf_counter_bit_length);
2214 buckets_sent += buckets_in_message;
2215 LOG (GNUNET_ERROR_TYPE_DEBUG,
2216 "ibf chunk size %u, %llu/%u sent\n",
2217 (unsigned int) buckets_in_message,
2218 (unsigned long long) buckets_sent,
2219 (unsigned int) ibf_size);
2220 GNUNET_MQ_send (op->mq, ev);
2221 }
2222
2223 /* The other peer must decode the IBF, so
2224 * we're passive. */
2225 op->phase = PHASE_PASSIVE_DECODING;
2226 return GNUNET_OK;
2227}
2228
2229
2230/**
2231 * Compute the necessary order of an ibf
2232 * from the size of the symmetric set difference.
2233 *
2234 * @param diff the difference
2235 * @return the required size of the ibf
2236 */
2237static unsigned int
2238get_size_from_difference (unsigned int diff, int number_buckets_per_element,
2239 float ibf_bucket_number_factor)
2240{
2241 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2242 * Elias Summermatter (2021) in section 3.11 **/
2243 return (((int) (diff * ibf_bucket_number_factor)) | 1);
2244
2245}
2246
2247
2248static unsigned int
2249get_next_ibf_size (float ibf_bucket_number_factor, unsigned int
2250 decoded_elements, unsigned int last_ibf_size)
2251{
2252 unsigned int next_size = (unsigned int) ((last_ibf_size * 2)
2253 - (ibf_bucket_number_factor
2254 * decoded_elements));
2255 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2256 * Elias Summermatter (2021) in section 3.11 **/
2257 return next_size | 1;
2258}
2259
2260
2261/**
2262 * Send a set element.
2263 *
2264 * @param cls the union operation `struct Operation *`
2265 * @param key unused
2266 * @param value the `struct ElementEntry *` to insert
2267 * into the key-to-element mapping
2268 * @return #GNUNET_YES (to continue iterating)
2269 */
2270static int
2271send_full_element_iterator (void *cls,
2272 const struct GNUNET_HashCode *key,
2273 void *value)
2274{
2275 struct Operation *op = cls;
2276 struct GNUNET_SETU_ElementMessage *emsg;
2277 struct ElementEntry *ee = value;
2278 struct GNUNET_SETU_Element *el = &ee->element;
2279 struct GNUNET_MQ_Envelope *ev;
2280
2281 LOG (GNUNET_ERROR_TYPE_DEBUG,
2282 "Sending element %s\n",
2283 GNUNET_h2s (key));
2284#if MEASURE_PERFORMANCE
2285 perf_store.element_full.received += 1;
2286 perf_store.element_full.received_var_bytes += el->size;
2287#endif
2288 ev = GNUNET_MQ_msg_extra (emsg,
2289 el->size,
2290 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT);
2291 emsg->element_type = htons (el->element_type);
2292 GNUNET_memcpy (&emsg[1],
2293 el->data,
2294 el->size);
2295 GNUNET_MQ_send (op->mq,
2296 ev);
2297 return GNUNET_YES;
2298}
2299
2300
2301/**
2302 * Switch to full set transmission for @a op.
2303 *
2304 * @param op operation to switch to full set transmission.
2305 */
2306static void
2307send_full_set (struct Operation *op)
2308{
2309 struct GNUNET_MQ_Envelope *ev;
2310
2311 op->phase = PHASE_FULL_SENDING;
2312 LOG (GNUNET_ERROR_TYPE_DEBUG,
2313 "Dedicing to transmit the full set\n");
2314 /* FIXME: use a more memory-friendly way of doing this with an
2315 iterator, just as we do in the non-full case! */
2316
2317 // Randomize Elements to send
2318 op->set->content->elements_randomized = GNUNET_CONTAINER_multihashmap_create (
2319 32,GNUNET_NO);
2320 op->set->content->elements_randomized_salt = GNUNET_CRYPTO_random_u64 (
2321 GNUNET_CRYPTO_QUALITY_NONCE,
2322 UINT64_MAX);
2323 (void) GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2324 &
2325 create_randomized_element_iterator,
2326 op);
2327
2328 (void) GNUNET_CONTAINER_multihashmap_iterate (
2329 op->set->content->elements_randomized,
2330 &send_full_element_iterator,
2331 op);
2332#if MEASURE_PERFORMANCE
2333 perf_store.full_done.sent += 1;
2334#endif
2335 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE);
2336 GNUNET_MQ_send (op->mq,
2337 ev);
2338}
2339
2340
2341/**
2342 * Handle a strata estimator from a remote peer
2343 *
2344 * @param cls the union operation
2345 * @param msg the message
2346 */
2347static int
2348check_union_p2p_strata_estimator (void *cls,
2349 const struct StrataEstimatorMessage *msg)
2350{
2351 struct Operation *op = cls;
2352 int is_compressed;
2353 size_t len;
2354
2355 if (op->phase != PHASE_EXPECT_SE)
2356 {
2357 GNUNET_break (0);
2358 return GNUNET_SYSERR;
2359 }
2360 is_compressed = (GNUNET_MESSAGE_TYPE_SETU_P2P_SEC == htons (
2361 msg->header.type));
2362 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
2363 if ((GNUNET_NO == is_compressed) &&
2364 (len != SE_STRATA_COUNT * SE_IBFS_TOTAL_SIZE * IBF_BUCKET_SIZE))
2365 {
2366 GNUNET_break (0);
2367 return GNUNET_SYSERR;
2368 }
2369 return GNUNET_OK;
2370}
2371
2372
2373/**
2374 * Handle a strata estimator from a remote peer
2375 *
2376 * @param cls the union operation
2377 * @param msg the message
2378 */
2379static void
2380handle_union_p2p_strata_estimator (void *cls,
2381 const struct StrataEstimatorMessage *msg)
2382{
2383#if MEASURE_PERFORMANCE
2384 perf_store.se.received += 1;
2385 perf_store.se.received_var_bytes += ntohs (msg->header.size) - sizeof(struct
2386 StrataEstimatorMessage);
2387#endif
2388 struct Operation *op = cls;
2389 struct MultiStrataEstimator *remote_se;
2390 unsigned int diff;
2391 uint64_t other_size;
2392 size_t len;
2393 int is_compressed;
2394 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
2395 op->set->content->elements);
2396 // Setting peer site to receiving peer
2397 op->peer_site = 1;
2398
2399 /**
2400 * Check that the message is received only in supported phase
2401 */
2402 uint8_t allowed_phases[] = {PHASE_EXPECT_SE};
2403 if (GNUNET_OK !=
2404 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
2405 {
2406 GNUNET_break (0);
2407 fail_union_operation (op);
2408 return;
2409 }
2410
2411 /** Only allow 1,2,4,8 SEs **/
2412 if ((msg->se_count > 8) || (__builtin_popcount ((int) msg->se_count) != 1))
2413 {
2414 LOG (GNUNET_ERROR_TYPE_ERROR,
2415 "PROTOCOL VIOLATION: Invalid number of se transmitted by other peer %u\n",
2416 msg->se_count);
2417 GNUNET_break_op (0);
2418 fail_union_operation (op);
2419 return;
2420 }
2421
2422 is_compressed = (GNUNET_MESSAGE_TYPE_SETU_P2P_SEC == htons (
2423 msg->header.type));
2424 GNUNET_STATISTICS_update (_GSS_statistics,
2425 "# bytes of SE received",
2426 ntohs (msg->header.size),
2427 GNUNET_NO);
2428 len = ntohs (msg->header.size) - sizeof(struct StrataEstimatorMessage);
2429 other_size = GNUNET_ntohll (msg->set_size);
2430 op->remote_element_count = other_size;
2431
2432 if (op->byzantine_upper_bound < op->remote_element_count)
2433 {
2434 LOG (GNUNET_ERROR_TYPE_ERROR,
2435 "Exceeded configured upper bound <%"PRIu64"> of element: %u\n",
2436 op->byzantine_upper_bound,
2437 op->remote_element_count);
2438 fail_union_operation (op);
2439 return;
2440 }
2441
2442 remote_se = strata_estimator_create (SE_STRATA_COUNT,
2443 SE_IBFS_TOTAL_SIZE,
2444 SE_IBF_HASH_NUM);
2445 if (NULL == remote_se)
2446 {
2447 /* insufficient resources, fail */
2448 fail_union_operation (op);
2449 return;
2450 }
2451 if (GNUNET_OK !=
2452 strata_estimator_read (&msg[1],
2453 len,
2454 is_compressed,
2455 msg->se_count,
2456 SE_IBFS_TOTAL_SIZE,
2457 remote_se))
2458 {
2459 /* decompression failed */
2460 strata_estimator_destroy (remote_se);
2461 fail_union_operation (op);
2462 return;
2463 }
2464 GNUNET_assert (NULL != op->se);
2465 strata_estimator_difference (remote_se,
2466 op->se);
2467
2468 /* Calculate remote local diff */
2469 long diff_remote = remote_se->stratas[0]->strata[0]->remote_decoded_count;
2470 long diff_local = remote_se->stratas[0]->strata[0]->local_decoded_count;
2471
2472 /* Prevent estimations from overshooting max element */
2473 if (diff_remote + op->remote_element_count > op->byzantine_upper_bound)
2474 diff_remote = op->byzantine_upper_bound - op->remote_element_count;
2475 if (diff_local + op->local_element_count > op->byzantine_upper_bound)
2476 diff_local = op->byzantine_upper_bound - op->local_element_count;
2477 if ((diff_remote < 0) || (diff_local < 0))
2478 {
2479 strata_estimator_destroy (remote_se);
2480 LOG (GNUNET_ERROR_TYPE_ERROR,
2481 "PROTOCOL VIOLATION: More element is set as upper boundary or other peer is "
2482 "malicious: remote diff %ld, local diff: %ld\n",
2483 diff_remote, diff_local);
2484 GNUNET_break_op (0);
2485 fail_union_operation (op);
2486 return;
2487 }
2488
2489 /* Make estimation more precise in initial sync cases */
2490 if (0 == op->remote_element_count)
2491 {
2492 diff_remote = 0;
2493 diff_local = op->local_element_count;
2494 }
2495 if (0 == op->local_element_count)
2496 {
2497 diff_local = 0;
2498 diff_remote = op->remote_element_count;
2499 }
2500
2501 diff = diff_remote + diff_local;
2502 op->remote_set_diff = diff_remote;
2503
2504 /** Calculate avg element size if not initial sync **/
2505 uint64_t avg_element_size = 0;
2506 if (0 < op->local_element_count)
2507 {
2508 op->total_elements_size_local = 0;
2509 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
2510 &
2511 determinate_avg_element_size_iterator,
2512 op);
2513 avg_element_size = op->total_elements_size_local / op->local_element_count;
2514 }
2515
2516 op->mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
2517 GNUNET_CONTAINER_multihashmap_size (
2518 op->set->content->
2519 elements),
2520 op->
2521 remote_element_count,
2522 diff_remote,
2523 diff_local,
2524 op->
2525 rtt_bandwidth_tradeoff,
2526 op->
2527 ibf_bucket_number_factor);
2528
2529#if MEASURE_PERFORMANCE
2530 perf_store.se_diff_local = diff_local;
2531 perf_store.se_diff_remote = diff_remote;
2532 perf_store.se_diff = diff;
2533 perf_store.mode_of_operation = op->mode_of_operation;
2534#endif
2535
2536 strata_estimator_destroy (remote_se);
2537 strata_estimator_destroy (op->se);
2538 op->se = NULL;
2539 LOG (GNUNET_ERROR_TYPE_DEBUG,
2540 "got se diff=%d, using ibf size %d\n",
2541 diff,
2542 1U << get_size_from_difference (diff, op->ibf_number_buckets_per_element,
2543 op->ibf_bucket_number_factor));
2544
2545 {
2546 char *set_debug;
2547
2548 set_debug = getenv ("GNUNET_SETU_BENCHMARK");
2549 if ((NULL != set_debug) &&
2550 (0 == strcmp (set_debug, "1")))
2551 {
2552 FILE *f = fopen ("set.log", "a");
2553 fprintf (f, "%llu\n", (unsigned long long) diff);
2554 fclose (f);
2555 }
2556 }
2557
2558 if ((GNUNET_YES == op->byzantine) &&
2559 (other_size < op->byzantine_lower_bound))
2560 {
2561 GNUNET_break (0);
2562 fail_union_operation (op);
2563 return;
2564 }
2565
2566 if ((GNUNET_YES == op->force_full) ||
2567 (op->mode_of_operation != DIFFERENTIAL_SYNC))
2568 {
2569 LOG (GNUNET_ERROR_TYPE_DEBUG,
2570 "Deciding to go for full set transmission (diff=%d, own set=%llu)\n",
2571 diff,
2572 (unsigned long long) op->initial_size);
2573 GNUNET_STATISTICS_update (_GSS_statistics,
2574 "# of full sends",
2575 1,
2576 GNUNET_NO);
2577 if (FULL_SYNC_LOCAL_SENDING_FIRST == op->mode_of_operation)
2578 {
2579 struct TransmitFullMessage *signal_msg;
2580 struct GNUNET_MQ_Envelope *ev;
2581 ev = GNUNET_MQ_msg_extra (signal_msg,sizeof(struct TransmitFullMessage),
2582 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL);
2583 signal_msg->remote_set_difference = htonl (diff_local);
2584 signal_msg->remote_set_size = htonl (op->local_element_count);
2585 signal_msg->local_set_difference = htonl (diff_remote);
2586 GNUNET_MQ_send (op->mq,
2587 ev);
2588 send_full_set (op);
2589 }
2590 else
2591 {
2592 struct GNUNET_MQ_Envelope *ev;
2593
2594 LOG (GNUNET_ERROR_TYPE_DEBUG,
2595 "Telling other peer that we expect its full set\n");
2596 op->phase = PHASE_FULL_RECEIVING;
2597#if MEASURE_PERFORMANCE
2598 perf_store.request_full.sent += 1;
2599#endif
2600 struct TransmitFullMessage *signal_msg;
2601 ev = GNUNET_MQ_msg_extra (signal_msg,sizeof(struct TransmitFullMessage),
2602 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL);
2603 signal_msg->remote_set_difference = htonl (diff_local);
2604 signal_msg->remote_set_size = htonl (op->local_element_count);
2605 signal_msg->local_set_difference = htonl (diff_remote);
2606 GNUNET_MQ_send (op->mq,
2607 ev);
2608 }
2609 }
2610 else
2611 {
2612 GNUNET_STATISTICS_update (_GSS_statistics,
2613 "# of ibf sends",
2614 1,
2615 GNUNET_NO);
2616 if (GNUNET_OK !=
2617 send_ibf (op,
2618 get_size_from_difference (diff,
2619 op->ibf_number_buckets_per_element,
2620 op->ibf_bucket_number_factor)))
2621 {
2622 /* Internal error, best we can do is shut the connection */
2623 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2624 "Failed to send IBF, closing connection\n");
2625 fail_union_operation (op);
2626 return;
2627 }
2628 }
2629 GNUNET_CADET_receive_done (op->channel);
2630}
2631
2632
2633/**
2634 * Iterator to send elements to a remote peer
2635 *
2636 * @param cls closure with the element key and the union operation
2637 * @param key ignored
2638 * @param value the key entry
2639 */
2640static int
2641send_offers_iterator (void *cls,
2642 uint32_t key,
2643 void *value)
2644{
2645 struct SendElementClosure *sec = cls;
2646 struct Operation *op = sec->op;
2647 struct KeyEntry *ke = value;
2648 struct GNUNET_MQ_Envelope *ev;
2649 struct GNUNET_MessageHeader *mh;
2650
2651 /* Detect 32-bit key collision for the 64-bit IBF keys. */
2652 if (ke->ibf_key.key_val != sec->ibf_key.key_val)
2653 {
2654 op->active_passive_switch_required = true;
2655 return GNUNET_YES;
2656 }
2657
2658 /* Prevent implementation from sending a offer multiple times in case of roll switch */
2659 if (GNUNET_YES ==
2660 is_message_in_message_control_flow (
2661 op->message_control_flow,
2662 &ke->element->element_hash,
2663 OFFER_MESSAGE)
2664 )
2665 {
2666 LOG (GNUNET_ERROR_TYPE_DEBUG,
2667 "Skipping already sent processed element offer!\n");
2668 return GNUNET_YES;
2669 }
2670
2671 /* Save send offer message for message control */
2672 if (GNUNET_YES !=
2673 update_message_control_flow (
2674 op->message_control_flow,
2675 MSG_CFS_SENT,
2676 &ke->element->element_hash,
2677 OFFER_MESSAGE)
2678 )
2679 {
2680 LOG (GNUNET_ERROR_TYPE_ERROR,
2681 "Double offer message sent found!\n");
2682 GNUNET_break (0);
2683 fail_union_operation (op);
2684 return GNUNET_NO;
2685 }
2686 ;
2687
2688 /* Mark element to be expected to received */
2689 if (GNUNET_YES !=
2690 update_message_control_flow (
2691 op->message_control_flow,
2692 MSG_CFS_EXPECTED,
2693 &ke->element->element_hash,
2694 DEMAND_MESSAGE)
2695 )
2696 {
2697 LOG (GNUNET_ERROR_TYPE_ERROR,
2698 "Double demand received found!\n");
2699 GNUNET_break (0);
2700 fail_union_operation (op);
2701 return GNUNET_NO;
2702 }
2703 ;
2704#if MEASURE_PERFORMANCE
2705 perf_store.offer.sent += 1;
2706 perf_store.offer.sent_var_bytes += sizeof(struct GNUNET_HashCode);
2707#endif
2708 ev = GNUNET_MQ_msg_header_extra (mh,
2709 sizeof(struct GNUNET_HashCode),
2710 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER);
2711 GNUNET_assert (NULL != ev);
2712 *(struct GNUNET_HashCode *) &mh[1] = ke->element->element_hash;
2713 LOG (GNUNET_ERROR_TYPE_DEBUG,
2714 "[OP %p] sending element offer (%s) to peer\n",
2715 op,
2716 GNUNET_h2s (&ke->element->element_hash));
2717 GNUNET_MQ_send (op->mq, ev);
2718 return GNUNET_YES;
2719}
2720
2721
2722/**
2723 * Send offers (in the form of GNUNET_Hash-es) to the remote peer for the given IBF key.
2724 *
2725 * @param op union operation
2726 * @param ibf_key IBF key of interest
2727 */
2728void
2729send_offers_for_key (struct Operation *op,
2730 struct IBF_Key ibf_key)
2731{
2732 struct SendElementClosure send_cls;
2733
2734 send_cls.ibf_key = ibf_key;
2735 send_cls.op = op;
2736 (void) GNUNET_CONTAINER_multihashmap32_get_multiple (
2737 op->key_to_element,
2738 (uint32_t) ibf_key.
2739 key_val,
2740 &send_offers_iterator,
2741 &send_cls);
2742}
2743
2744
2745/**
2746 * Decode which elements are missing on each side, and
2747 * send the appropriate offers and inquiries.
2748 *
2749 * @param op union operation
2750 * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure
2751 */
2752static int
2753decode_and_send (struct Operation *op)
2754{
2755 struct IBF_Key key;
2756 struct IBF_Key last_key;
2757 int side;
2758 unsigned int num_decoded;
2759 struct InvertibleBloomFilter *diff_ibf;
2760
2761 GNUNET_assert (PHASE_ACTIVE_DECODING == op->phase);
2762
2763 if (GNUNET_OK !=
2764 prepare_ibf (op,
2765 op->remote_ibf->size))
2766 {
2767 GNUNET_break (0);
2768 /* allocation failed */
2769 return GNUNET_SYSERR;
2770 }
2771
2772 diff_ibf = ibf_dup (op->local_ibf);
2773 ibf_subtract (diff_ibf,
2774 op->remote_ibf);
2775
2776 ibf_destroy (op->remote_ibf);
2777 op->remote_ibf = NULL;
2778
2779 LOG (GNUNET_ERROR_TYPE_DEBUG,
2780 "decoding IBF (size=%u)\n",
2781 diff_ibf->size);
2782
2783 num_decoded = 0;
2784 key.key_val = 0; /* just to avoid compiler thinking we use undef'ed variable */
2785
2786 while (1)
2787 {
2788 int res;
2789 int cycle_detected = GNUNET_NO;
2790
2791 last_key = key;
2792
2793 res = ibf_decode (diff_ibf,
2794 &side,
2795 &key);
2796 if (res == GNUNET_OK)
2797 {
2798 LOG (GNUNET_ERROR_TYPE_DEBUG,
2799 "decoded ibf key %lx\n",
2800 (unsigned long) key.key_val);
2801 num_decoded += 1;
2802 if ((num_decoded > diff_ibf->size) ||
2803 ((num_decoded > 1) &&
2804 (last_key.key_val == key.key_val)))
2805 {
2806 LOG (GNUNET_ERROR_TYPE_DEBUG,
2807 "detected cyclic ibf (decoded %u/%u)\n",
2808 num_decoded,
2809 diff_ibf->size);
2810 cycle_detected = GNUNET_YES;
2811 }
2812 }
2813 if ((GNUNET_SYSERR == res) ||
2814 (GNUNET_YES == cycle_detected))
2815 {
2816 uint32_t next_size;
2817 /** Enforce odd ibf size **/
2818
2819 next_size = get_next_ibf_size (op->ibf_bucket_number_factor, num_decoded,
2820 diff_ibf->size);
2821 /** Make ibf estimation size odd reasoning can be found in BSc Thesis of
2822 * Elias Summermatter (2021) in section 3.11 **/
2823 uint32_t ibf_min_size = IBF_MIN_SIZE | 1;
2824
2825 if (next_size<ibf_min_size)
2826 next_size = ibf_min_size;
2827
2828
2829 if (next_size <= MAX_IBF_SIZE)
2830 {
2831 LOG (GNUNET_ERROR_TYPE_DEBUG,
2832 "decoding failed, sending larger ibf (size %u)\n",
2833 next_size);
2834 GNUNET_STATISTICS_update (_GSS_statistics,
2835 "# of IBF retries",
2836 1,
2837 GNUNET_NO);
2838#if MEASURE_PERFORMANCE
2839 perf_store.active_passive_switches += 1;
2840#endif
2841
2842 op->salt_send = op->salt_receive++;
2843
2844 if (GNUNET_OK !=
2845 send_ibf (op, next_size))
2846 {
2847 /* Internal error, best we can do is shut the connection */
2848 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
2849 "Failed to send IBF, closing connection\n");
2850 fail_union_operation (op);
2851 ibf_destroy (diff_ibf);
2852 return GNUNET_SYSERR;
2853 }
2854 }
2855 else
2856 {
2857 GNUNET_STATISTICS_update (_GSS_statistics,
2858 "# of failed union operations (too large)",
2859 1,
2860 GNUNET_NO);
2861 // XXX: Send the whole set, element-by-element
2862 LOG (GNUNET_ERROR_TYPE_ERROR,
2863 "set union failed: reached ibf limit\n");
2864 fail_union_operation (op);
2865 ibf_destroy (diff_ibf);
2866 return GNUNET_SYSERR;
2867 }
2868 break;
2869 }
2870 if (GNUNET_NO == res)
2871 {
2872 struct GNUNET_MQ_Envelope *ev;
2873
2874 LOG (GNUNET_ERROR_TYPE_DEBUG,
2875 "transmitted all values, sending DONE\n");
2876
2877#if MEASURE_PERFORMANCE
2878 perf_store.done.sent += 1;
2879#endif
2880 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_DONE);
2881 GNUNET_MQ_send (op->mq, ev);
2882 /* We now wait until we get a DONE message back
2883 * and then wait for our MQ to be flushed and all our
2884 * demands be delivered. */
2885 break;
2886 }
2887 if (1 == side)
2888 {
2889 struct IBF_Key unsalted_key;
2890 unsalt_key (&key,
2891 op->salt_receive,
2892 &unsalted_key);
2893 send_offers_for_key (op,
2894 unsalted_key);
2895 }
2896 else if (-1 == side)
2897 {
2898 struct GNUNET_MQ_Envelope *ev;
2899 struct InquiryMessage *msg;
2900
2901#if MEASURE_PERFORMANCE
2902 perf_store.inquery.sent += 1;
2903 perf_store.inquery.sent_var_bytes += sizeof(struct IBF_Key);
2904#endif
2905
2906 /** Add sent inquiries to hashmap for flow control **/
2907 struct GNUNET_HashContext *hashed_key_context =
2908 GNUNET_CRYPTO_hash_context_start ();
2909 struct GNUNET_HashCode *hashed_key = (struct
2910 GNUNET_HashCode*) GNUNET_malloc (
2911 sizeof(struct GNUNET_HashCode));
2912 enum MESSAGE_CONTROL_FLOW_STATE mcfs = MSG_CFS_SENT;
2913 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
2914 &key,
2915 sizeof(struct IBF_Key));
2916 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
2917 hashed_key);
2918 GNUNET_CONTAINER_multihashmap_put (op->inquiries_sent,
2919 hashed_key,
2920 &mcfs,
2921 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE
2922 );
2923
2924 /* It may be nice to merge multiple requests, but with CADET's corking it is not worth
2925 * the effort additional complexity. */
2926 ev = GNUNET_MQ_msg_extra (msg,
2927 sizeof(struct IBF_Key),
2928 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY);
2929 msg->salt = htonl (op->salt_receive);
2930 GNUNET_memcpy (&msg[1],
2931 &key,
2932 sizeof(struct IBF_Key));
2933 LOG (GNUNET_ERROR_TYPE_DEBUG,
2934 "sending element inquiry for IBF key %lx\n",
2935 (unsigned long) key.key_val);
2936 GNUNET_MQ_send (op->mq, ev);
2937 }
2938 else
2939 {
2940 GNUNET_assert (0);
2941 }
2942 }
2943 ibf_destroy (diff_ibf);
2944 return GNUNET_OK;
2945}
2946
2947
2948/**
2949 * Check send full message received from other peer
2950 * @param cls
2951 * @param msg
2952 * @return
2953 */
2954
2955static int
2956check_union_p2p_send_full (void *cls,
2957 const struct TransmitFullMessage *msg)
2958{
2959 return GNUNET_OK;
2960}
2961
2962
2963/**
2964 * Handle send full message received from other peer
2965 *
2966 * @param cls
2967 * @param msg
2968 */
2969static void
2970handle_union_p2p_send_full (void *cls,
2971 const struct TransmitFullMessage *msg)
2972{
2973 struct Operation *op = cls;
2974
2975 /**
2976 * Check that the message is received only in supported phase
2977 */
2978 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF};
2979 if (GNUNET_OK !=
2980 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
2981 {
2982 GNUNET_break (0);
2983 fail_union_operation (op);
2984 return;
2985 }
2986
2987 /** write received values to operator**/
2988 op->remote_element_count = ntohl (msg->remote_set_size);
2989 op->remote_set_diff = ntohl (msg->remote_set_difference);
2990 op->local_set_diff = ntohl (msg->local_set_difference);
2991
2992 /** Check byzantine limits **/
2993 if (check_byzantine_bounds (op) != GNUNET_OK)
2994 {
2995 LOG (GNUNET_ERROR_TYPE_ERROR,
2996 "PROTOCOL VIOLATION: Parameters transmitted from other peer do not satisfie byzantine "
2997 "criteria\n");
2998 GNUNET_break_op (0);
2999 fail_union_operation (op);
3000 return;
3001 }
3002
3003 /** Calculate avg element size if not initial sync **/
3004 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
3005 op->set->content->elements);
3006 uint64_t avg_element_size = 0;
3007 if (0 < op->local_element_count)
3008 {
3009 op->total_elements_size_local = 0;
3010 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
3011 &
3012 determinate_avg_element_size_iterator,
3013 op);
3014 avg_element_size = op->total_elements_size_local / op->local_element_count;
3015 }
3016
3017 /** Validate mode of operation **/
3018 int mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
3019 op->
3020 remote_element_count,
3021 op->
3022 local_element_count,
3023 op->local_set_diff,
3024 op->remote_set_diff,
3025 op->
3026 rtt_bandwidth_tradeoff,
3027 op->
3028 ibf_bucket_number_factor);
3029 if (FULL_SYNC_LOCAL_SENDING_FIRST != mode_of_operation)
3030 {
3031 LOG (GNUNET_ERROR_TYPE_ERROR,
3032 "PROTOCOL VIOLATION: Remote peer choose to send his full set first but correct mode would have been"
3033 " : %d\n", mode_of_operation);
3034 GNUNET_break_op (0);
3035 fail_union_operation (op);
3036 return;
3037 }
3038 op->phase = PHASE_FULL_RECEIVING;
3039}
3040
3041
3042/**
3043 * Check an IBF message from a remote peer.
3044 *
3045 * Reassemble the IBF from multiple pieces, and
3046 * process the whole IBF once possible.
3047 *
3048 * @param cls the union operation
3049 * @param msg the header of the message
3050 * @return #GNUNET_OK if @a msg is well-formed
3051 */
3052static int
3053check_union_p2p_ibf (void *cls,
3054 const struct IBFMessage *msg)
3055{
3056 struct Operation *op = cls;
3057 unsigned int buckets_in_message;
3058
3059 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
3060 / IBF_BUCKET_SIZE;
3061 if (0 == buckets_in_message)
3062 {
3063 GNUNET_break_op (0);
3064 return GNUNET_SYSERR;
3065 }
3066 if ((ntohs (msg->header.size) - sizeof *msg) != buckets_in_message
3067 * IBF_BUCKET_SIZE)
3068 {
3069 GNUNET_break_op (0);
3070 return GNUNET_SYSERR;
3071 }
3072 if (op->phase == PHASE_EXPECT_IBF_LAST)
3073 {
3074 if (ntohl (msg->offset) != op->ibf_buckets_received)
3075 {
3076 GNUNET_break_op (0);
3077 return GNUNET_SYSERR;
3078 }
3079
3080 if (msg->ibf_size != op->remote_ibf->size)
3081 {
3082 GNUNET_break_op (0);
3083 return GNUNET_SYSERR;
3084 }
3085 if (ntohl (msg->salt) != op->salt_receive)
3086 {
3087 GNUNET_break_op (0);
3088 return GNUNET_SYSERR;
3089 }
3090 }
3091 else if ((op->phase != PHASE_PASSIVE_DECODING) &&
3092 (op->phase != PHASE_EXPECT_IBF))
3093 {
3094 GNUNET_break_op (0);
3095 return GNUNET_SYSERR;
3096 }
3097
3098 return GNUNET_OK;
3099}
3100
3101
3102/**
3103 * Handle an IBF message from a remote peer.
3104 *
3105 * Reassemble the IBF from multiple pieces, and
3106 * process the whole IBF once possible.
3107 *
3108 * @param cls the union operation
3109 * @param msg the header of the message
3110 */
3111static void
3112handle_union_p2p_ibf (void *cls,
3113 const struct IBFMessage *msg)
3114{
3115 struct Operation *op = cls;
3116 unsigned int buckets_in_message;
3117 /**
3118 * Check that the message is received only in supported phase
3119 */
3120 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF, PHASE_EXPECT_IBF_LAST,
3121 PHASE_PASSIVE_DECODING};
3122 if (GNUNET_OK !=
3123 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3124 {
3125 GNUNET_break (0);
3126 fail_union_operation (op);
3127 return;
3128 }
3129 op->differential_sync_iterations++;
3130 check_max_differential_rounds (op);
3131 op->active_passive_switch_required = false;
3132
3133#if MEASURE_PERFORMANCE
3134 perf_store.ibf.received += 1;
3135 perf_store.ibf.received_var_bytes += (ntohs (msg->header.size) - sizeof *msg);
3136#endif
3137
3138 buckets_in_message = (ntohs (msg->header.size) - sizeof *msg)
3139 / IBF_BUCKET_SIZE;
3140 if ((op->phase == PHASE_PASSIVE_DECODING) ||
3141 (op->phase == PHASE_EXPECT_IBF))
3142 {
3143 op->phase = PHASE_EXPECT_IBF_LAST;
3144 GNUNET_assert (NULL == op->remote_ibf);
3145 LOG (GNUNET_ERROR_TYPE_DEBUG,
3146 "Creating new ibf of size %u\n",
3147 ntohl (msg->ibf_size));
3148 // op->remote_ibf = ibf_create (1 << msg->order, SE_IBF_HASH_NUM);
3149 op->remote_ibf = ibf_create (msg->ibf_size,
3150 ((uint8_t) op->ibf_number_buckets_per_element));
3151 op->salt_receive = ntohl (msg->salt);
3152 LOG (GNUNET_ERROR_TYPE_DEBUG,
3153 "Receiving new IBF with salt %u\n",
3154 op->salt_receive);
3155 if (NULL == op->remote_ibf)
3156 {
3157 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3158 "Failed to parse remote IBF, closing connection\n");
3159 fail_union_operation (op);
3160 return;
3161 }
3162 op->ibf_buckets_received = 0;
3163 if (0 != ntohl (msg->offset))
3164 {
3165 GNUNET_break_op (0);
3166 fail_union_operation (op);
3167 return;
3168 }
3169 }
3170 else
3171 {
3172 GNUNET_assert (op->phase == PHASE_EXPECT_IBF_LAST);
3173 LOG (GNUNET_ERROR_TYPE_DEBUG,
3174 "Received more of IBF\n");
3175 }
3176 GNUNET_assert (NULL != op->remote_ibf);
3177
3178 ibf_read_slice (&msg[1],
3179 op->ibf_buckets_received,
3180 buckets_in_message,
3181 op->remote_ibf, msg->ibf_counter_bit_length);
3182 op->ibf_buckets_received += buckets_in_message;
3183
3184 if (op->ibf_buckets_received == op->remote_ibf->size)
3185 {
3186 LOG (GNUNET_ERROR_TYPE_DEBUG,
3187 "received full ibf\n");
3188 op->phase = PHASE_ACTIVE_DECODING;
3189 if (GNUNET_OK !=
3190 decode_and_send (op))
3191 {
3192 /* Internal error, best we can do is shut down */
3193 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3194 "Failed to decode IBF, closing connection\n");
3195 fail_union_operation (op);
3196 return;
3197 }
3198 }
3199 GNUNET_CADET_receive_done (op->channel);
3200}
3201
3202
3203/**
3204 * Send a result message to the client indicating
3205 * that there is a new element.
3206 *
3207 * @param op union operation
3208 * @param element element to send
3209 * @param status status to send with the new element
3210 */
3211static void
3212send_client_element (struct Operation *op,
3213 const struct GNUNET_SETU_Element *element,
3214 enum GNUNET_SETU_Status status)
3215{
3216 struct GNUNET_MQ_Envelope *ev;
3217 struct GNUNET_SETU_ResultMessage *rm;
3218
3219 LOG (GNUNET_ERROR_TYPE_DEBUG,
3220 "sending element (size %u) to client\n",
3221 element->size);
3222 GNUNET_assert (0 != op->client_request_id);
3223 ev = GNUNET_MQ_msg_extra (rm,
3224 element->size,
3225 GNUNET_MESSAGE_TYPE_SETU_RESULT);
3226 if (NULL == ev)
3227 {
3228 GNUNET_MQ_discard (ev);
3229 GNUNET_break (0);
3230 return;
3231 }
3232 rm->result_status = htons (status);
3233 rm->request_id = htonl (op->client_request_id);
3234 rm->element_type = htons (element->element_type);
3235 rm->current_size = GNUNET_htonll (GNUNET_CONTAINER_multihashmap32_size (
3236 op->key_to_element));
3237 GNUNET_memcpy (&rm[1],
3238 element->data,
3239 element->size);
3240 GNUNET_MQ_send (op->set->cs->mq,
3241 ev);
3242}
3243
3244
3245/**
3246 * Tests if the operation is finished, and if so notify.
3247 *
3248 * @param op operation to check
3249 */
3250static void
3251maybe_finish (struct Operation *op)
3252{
3253 unsigned int num_demanded;
3254
3255 num_demanded = GNUNET_CONTAINER_multihashmap_size (
3256 op->demanded_hashes);
3257 int send_done = GNUNET_CONTAINER_multihashmap_iterate (
3258 op->message_control_flow,
3259 &
3260 determinate_done_message_iterator,
3261 op);
3262 if (PHASE_FINISH_WAITING == op->phase)
3263 {
3264 LOG (GNUNET_ERROR_TYPE_DEBUG,
3265 "In PHASE_FINISH_WAITING, pending %u demands -> %d\n",
3266 num_demanded, op->peer_site);
3267 if (-1 != send_done)
3268 {
3269 struct GNUNET_MQ_Envelope *ev;
3270
3271 op->phase = PHASE_FINISHED;
3272#if MEASURE_PERFORMANCE
3273 perf_store.done.sent += 1;
3274#endif
3275 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_DONE);
3276 GNUNET_MQ_send (op->mq,
3277 ev);
3278 /* We now wait until the other peer sends P2P_OVER
3279 * after it got all elements from us. */
3280 }
3281 }
3282 if (PHASE_FINISH_CLOSING == op->phase)
3283 {
3284 LOG (GNUNET_ERROR_TYPE_DEBUG,
3285 "In PHASE_FINISH_CLOSING, pending %u demands %d\n",
3286 num_demanded, op->peer_site);
3287 if (-1 != send_done)
3288 {
3289 op->phase = PHASE_FINISHED;
3290 send_client_done (op);
3291 _GSS_operation_destroy2 (op);
3292 }
3293 }
3294}
3295
3296
3297/**
3298 * Check an element message from a remote peer.
3299 *
3300 * @param cls the union operation
3301 * @param emsg the message
3302 */
3303static int
3304check_union_p2p_elements (void *cls,
3305 const struct GNUNET_SETU_ElementMessage *emsg)
3306{
3307 struct Operation *op = cls;
3308
3309 if (0 == GNUNET_CONTAINER_multihashmap_size (op->demanded_hashes))
3310 {
3311 GNUNET_break_op (0);
3312 return GNUNET_SYSERR;
3313 }
3314 return GNUNET_OK;
3315}
3316
3317
3318/**
3319 * Handle an element message from a remote peer.
3320 * Sent by the other peer either because we decoded an IBF and placed a demand,
3321 * or because the other peer switched to full set transmission.
3322 *
3323 * @param cls the union operation
3324 * @param emsg the message
3325 */
3326static void
3327handle_union_p2p_elements (void *cls,
3328 const struct GNUNET_SETU_ElementMessage *emsg)
3329{
3330 struct Operation *op = cls;
3331 struct ElementEntry *ee;
3332 struct KeyEntry *ke;
3333 uint16_t element_size;
3334
3335 /**
3336 * Check that the message is received only in supported phase
3337 */
3338 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING,
3339 PHASE_FINISH_WAITING, PHASE_FINISH_CLOSING};
3340 if (GNUNET_OK !=
3341 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3342 {
3343 GNUNET_break (0);
3344 fail_union_operation (op);
3345 return;
3346 }
3347
3348 element_size = ntohs (emsg->header.size) - sizeof(struct
3349 GNUNET_SETU_ElementMessage);
3350#if MEASURE_PERFORMANCE
3351 perf_store.element.received += 1;
3352 perf_store.element.received_var_bytes += element_size;
3353#endif
3354
3355 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
3356 GNUNET_memcpy (&ee[1],
3357 &emsg[1],
3358 element_size);
3359 ee->element.size = element_size;
3360 ee->element.data = &ee[1];
3361 ee->element.element_type = ntohs (emsg->element_type);
3362 ee->remote = GNUNET_YES;
3363 GNUNET_SETU_element_hash (&ee->element,
3364 &ee->element_hash);
3365 if (GNUNET_NO ==
3366 GNUNET_CONTAINER_multihashmap_remove (op->demanded_hashes,
3367 &ee->element_hash,
3368 NULL))
3369 {
3370 /* We got something we didn't demand, since it's not in our map. */
3371 GNUNET_break_op (0);
3372 fail_union_operation (op);
3373 return;
3374 }
3375
3376 if (GNUNET_OK !=
3377 update_message_control_flow (
3378 op->message_control_flow,
3379 MSG_CFS_RECEIVED,
3380 &ee->element_hash,
3381 ELEMENT_MESSAGE)
3382 )
3383 {
3384 LOG (GNUNET_ERROR_TYPE_ERROR,
3385 "An element has been received more than once!\n");
3386 GNUNET_break (0);
3387 fail_union_operation (op);
3388 return;
3389 }
3390
3391 LOG (GNUNET_ERROR_TYPE_DEBUG,
3392 "Got element (size %u, hash %s) from peer\n",
3393 (unsigned int) element_size,
3394 GNUNET_h2s (&ee->element_hash));
3395
3396 GNUNET_STATISTICS_update (_GSS_statistics,
3397 "# received elements",
3398 1,
3399 GNUNET_NO);
3400 GNUNET_STATISTICS_update (_GSS_statistics,
3401 "# exchanged elements",
3402 1,
3403 GNUNET_NO);
3404
3405 op->received_total++;
3406
3407 ke = op_get_element (op,
3408 &ee->element_hash);
3409 if (NULL != ke)
3410 {
3411 /* Got repeated element. Should not happen since
3412 * we track demands. */
3413 GNUNET_STATISTICS_update (_GSS_statistics,
3414 "# repeated elements",
3415 1,
3416 GNUNET_NO);
3417 ke->received = GNUNET_YES;
3418 GNUNET_free (ee);
3419 }
3420 else
3421 {
3422 LOG (GNUNET_ERROR_TYPE_DEBUG,
3423 "Registering new element from remote peer\n");
3424 op->received_fresh++;
3425 op_register_element (op, ee, GNUNET_YES);
3426 /* only send results immediately if the client wants it */
3427 send_client_element (op,
3428 &ee->element,
3429 GNUNET_SETU_STATUS_ADD_LOCAL);
3430 }
3431
3432 if ((op->received_total > 8) &&
3433 (op->received_fresh < op->received_total / 3))
3434 {
3435 /* The other peer gave us lots of old elements, there's something wrong. */
3436 GNUNET_break_op (0);
3437 fail_union_operation (op);
3438 return;
3439 }
3440 GNUNET_CADET_receive_done (op->channel);
3441 maybe_finish (op);
3442}
3443
3444
3445/**
3446 * Check a full element message from a remote peer.
3447 *
3448 * @param cls the union operation
3449 * @param emsg the message
3450 */
3451static int
3452check_union_p2p_full_element (void *cls,
3453 const struct GNUNET_SETU_ElementMessage *emsg)
3454{
3455 struct Operation *op = cls;
3456
3457 (void) op;
3458
3459 // FIXME: check that we expect full elements here?
3460 return GNUNET_OK;
3461}
3462
3463
3464/**
3465 * Handle an element message from a remote peer.
3466 *
3467 * @param cls the union operation
3468 * @param emsg the message
3469 */
3470static void
3471handle_union_p2p_full_element (void *cls,
3472 const struct GNUNET_SETU_ElementMessage *emsg)
3473{
3474 struct Operation *op = cls;
3475 struct ElementEntry *ee;
3476 struct KeyEntry *ke;
3477 uint16_t element_size;
3478
3479 /**
3480 * Check that the message is received only in supported phase
3481 */
3482 uint8_t allowed_phases[] = {PHASE_FULL_RECEIVING, PHASE_FULL_SENDING};
3483 if (GNUNET_OK !=
3484 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3485 {
3486 GNUNET_break (0);
3487 fail_union_operation (op);
3488 return;
3489 }
3490
3491 element_size = ntohs (emsg->header.size)
3492 - sizeof(struct GNUNET_SETU_ElementMessage);
3493
3494#if MEASURE_PERFORMANCE
3495 perf_store.element_full.received += 1;
3496 perf_store.element_full.received_var_bytes += element_size;
3497#endif
3498
3499 ee = GNUNET_malloc (sizeof(struct ElementEntry) + element_size);
3500 GNUNET_memcpy (&ee[1], &emsg[1], element_size);
3501 ee->element.size = element_size;
3502 ee->element.data = &ee[1];
3503 ee->element.element_type = ntohs (emsg->element_type);
3504 ee->remote = GNUNET_YES;
3505 GNUNET_SETU_element_hash (&ee->element,
3506 &ee->element_hash);
3507 LOG (GNUNET_ERROR_TYPE_DEBUG,
3508 "Got element (full diff, size %u, hash %s) from peer\n",
3509 (unsigned int) element_size,
3510 GNUNET_h2s (&ee->element_hash));
3511
3512 GNUNET_STATISTICS_update (_GSS_statistics,
3513 "# received elements",
3514 1,
3515 GNUNET_NO);
3516 GNUNET_STATISTICS_update (_GSS_statistics,
3517 "# exchanged elements",
3518 1,
3519 GNUNET_NO);
3520
3521 op->received_total++;
3522 ke = op_get_element (op,
3523 &ee->element_hash);
3524 if (NULL != ke)
3525 {
3526 GNUNET_STATISTICS_update (_GSS_statistics,
3527 "# repeated elements",
3528 1,
3529 GNUNET_NO);
3530 full_sync_plausibility_check (op);
3531 ke->received = GNUNET_YES;
3532 GNUNET_free (ee);
3533 }
3534 else
3535 {
3536 LOG (GNUNET_ERROR_TYPE_DEBUG,
3537 "Registering new element from remote peer\n");
3538 op->received_fresh++;
3539 op_register_element (op, ee, GNUNET_YES);
3540 /* only send results immediately if the client wants it */
3541 send_client_element (op,
3542 &ee->element,
3543 GNUNET_SETU_STATUS_ADD_LOCAL);
3544 }
3545
3546
3547 if ((GNUNET_YES == op->byzantine) &&
3548 (op->received_total > op->remote_element_count) )
3549 {
3550 /* The other peer gave us lots of old elements, there's something wrong. */
3551 LOG (GNUNET_ERROR_TYPE_ERROR,
3552 "Other peer sent %llu elements while pretending to have %llu elements, failing operation\n",
3553 (unsigned long long) op->received_total,
3554 (unsigned long long) op->remote_element_count);
3555 GNUNET_break_op (0);
3556 fail_union_operation (op);
3557 return;
3558 }
3559 GNUNET_CADET_receive_done (op->channel);
3560}
3561
3562
3563/**
3564 * Send offers (for GNUNET_Hash-es) in response
3565 * to inquiries (for IBF_Key-s).
3566 *
3567 * @param cls the union operation
3568 * @param msg the message
3569 */
3570static int
3571check_union_p2p_inquiry (void *cls,
3572 const struct InquiryMessage *msg)
3573{
3574 struct Operation *op = cls;
3575 unsigned int num_keys;
3576
3577 if (op->phase != PHASE_PASSIVE_DECODING)
3578 {
3579 GNUNET_break_op (0);
3580 return GNUNET_SYSERR;
3581 }
3582 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3583 / sizeof(struct IBF_Key);
3584 if ((ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3585 != num_keys * sizeof(struct IBF_Key))
3586 {
3587 GNUNET_break_op (0);
3588 return GNUNET_SYSERR;
3589 }
3590 return GNUNET_OK;
3591}
3592
3593
3594/**
3595 * Send offers (for GNUNET_Hash-es) in response to inquiries (for IBF_Key-s).
3596 *
3597 * @param cls the union operation
3598 * @param msg the message
3599 */
3600static void
3601handle_union_p2p_inquiry (void *cls,
3602 const struct InquiryMessage *msg)
3603{
3604 struct Operation *op = cls;
3605 const struct IBF_Key *ibf_key;
3606 unsigned int num_keys;
3607
3608 /**
3609 * Check that the message is received only in supported phase
3610 */
3611 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
3612 if (GNUNET_OK !=
3613 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3614 {
3615 GNUNET_break (0);
3616 fail_union_operation (op);
3617 return;
3618 }
3619
3620#if MEASURE_PERFORMANCE
3621 perf_store.inquery.received += 1;
3622 perf_store.inquery.received_var_bytes += (ntohs (msg->header.size)
3623 - sizeof(struct InquiryMessage));
3624#endif
3625
3626 LOG (GNUNET_ERROR_TYPE_DEBUG,
3627 "Received union inquiry\n");
3628 num_keys = (ntohs (msg->header.size) - sizeof(struct InquiryMessage))
3629 / sizeof(struct IBF_Key);
3630 ibf_key = (const struct IBF_Key *) &msg[1];
3631
3632 /** Add received inquiries to hashmap for flow control **/
3633 struct GNUNET_HashContext *hashed_key_context =
3634 GNUNET_CRYPTO_hash_context_start ();
3635 struct GNUNET_HashCode *hashed_key = (struct GNUNET_HashCode*) GNUNET_malloc (
3636 sizeof(struct GNUNET_HashCode));;
3637 enum MESSAGE_CONTROL_FLOW_STATE mcfs = MSG_CFS_RECEIVED;
3638 GNUNET_CRYPTO_hash_context_read (hashed_key_context,
3639 &ibf_key,
3640 sizeof(struct IBF_Key));
3641 GNUNET_CRYPTO_hash_context_finish (hashed_key_context,
3642 hashed_key);
3643 GNUNET_CONTAINER_multihashmap_put (op->inquiries_sent,
3644 hashed_key,
3645 &mcfs,
3646 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE
3647 );
3648
3649 while (0 != num_keys--)
3650 {
3651 struct IBF_Key unsalted_key;
3652 unsalt_key (ibf_key,
3653 ntohl (msg->salt),
3654 &unsalted_key);
3655 send_offers_for_key (op,
3656 unsalted_key);
3657 ibf_key++;
3658 }
3659 GNUNET_CADET_receive_done (op->channel);
3660}
3661
3662
3663/**
3664 * Iterator over hash map entries, called to destroy the linked list of
3665 * colliding ibf key entries.
3666 *
3667 * @param cls closure
3668 * @param key current key code
3669 * @param value value in the hash map
3670 * @return #GNUNET_YES if we should continue to iterate,
3671 * #GNUNET_NO if not.
3672 */
3673static int
3674send_missing_full_elements_iter (void *cls,
3675 uint32_t key,
3676 void *value)
3677{
3678 struct Operation *op = cls;
3679 struct KeyEntry *ke = value;
3680 struct GNUNET_MQ_Envelope *ev;
3681 struct GNUNET_SETU_ElementMessage *emsg;
3682 struct ElementEntry *ee = ke->element;
3683
3684 if (GNUNET_YES == ke->received)
3685 return GNUNET_YES;
3686#if MEASURE_PERFORMANCE
3687 perf_store.element_full.received += 1;
3688#endif
3689 ev = GNUNET_MQ_msg_extra (emsg,
3690 ee->element.size,
3691 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT);
3692 GNUNET_memcpy (&emsg[1],
3693 ee->element.data,
3694 ee->element.size);
3695 emsg->element_type = htons (ee->element.element_type);
3696 GNUNET_MQ_send (op->mq,
3697 ev);
3698 return GNUNET_YES;
3699}
3700
3701
3702/**
3703 * Handle a request for full set transmission.
3704 *
3705 * @param cls closure, a set union operation
3706 * @param mh the demand message
3707 */
3708static int
3709check_union_p2p_request_full (void *cls,
3710 const struct TransmitFullMessage *mh)
3711{
3712 return GNUNET_OK;
3713}
3714
3715
3716static void
3717handle_union_p2p_request_full (void *cls,
3718 const struct TransmitFullMessage *msg)
3719{
3720 struct Operation *op = cls;
3721
3722 /**
3723 * Check that the message is received only in supported phase
3724 */
3725 uint8_t allowed_phases[] = {PHASE_EXPECT_IBF};
3726 if (GNUNET_OK !=
3727 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3728 {
3729 GNUNET_break (0);
3730 fail_union_operation (op);
3731 return;
3732 }
3733
3734 op->remote_element_count = ntohl (msg->remote_set_size);
3735 op->remote_set_diff = ntohl (msg->remote_set_difference);
3736 op->local_set_diff = ntohl (msg->local_set_difference);
3737
3738
3739 if (check_byzantine_bounds (op) != GNUNET_OK)
3740 {
3741 LOG (GNUNET_ERROR_TYPE_ERROR,
3742 "PROTOCOL VIOLATION: Parameters transmitted from other peer do not satisfie byzantine "
3743 "criteria\n");
3744 GNUNET_break_op (0);
3745 fail_union_operation (op);
3746 return;
3747 }
3748
3749#if MEASURE_PERFORMANCE
3750 perf_store.request_full.received += 1;
3751#endif
3752
3753 LOG (GNUNET_ERROR_TYPE_DEBUG,
3754 "Received request for full set transmission\n");
3755
3756 /** Calculate avg element size if not initial sync **/
3757 op->local_element_count = GNUNET_CONTAINER_multihashmap_size (
3758 op->set->content->elements);
3759 uint64_t avg_element_size = 0;
3760 if (0 < op->local_element_count)
3761 {
3762 op->total_elements_size_local = 0;
3763 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
3764 &
3765 determinate_avg_element_size_iterator,
3766 op);
3767 avg_element_size = op->total_elements_size_local / op->local_element_count;
3768 }
3769
3770 int mode_of_operation = estimate_best_mode_of_operation (avg_element_size,
3771 op->
3772 remote_element_count,
3773 op->
3774 local_element_count,
3775 op->local_set_diff,
3776 op->remote_set_diff,
3777 op->
3778 rtt_bandwidth_tradeoff,
3779 op->
3780 ibf_bucket_number_factor);
3781 if (FULL_SYNC_REMOTE_SENDING_FIRST != mode_of_operation)
3782 {
3783 LOG (GNUNET_ERROR_TYPE_ERROR,
3784 "PROTOCOL VIOLATION: Remote peer choose to request the full set first but correct mode would have been"
3785 " : %d\n", mode_of_operation);
3786 GNUNET_break_op (0);
3787 fail_union_operation (op);
3788 return;
3789 }
3790
3791 // FIXME: we need to check that our set is larger than the
3792 // byzantine_lower_bound by some threshold
3793 send_full_set (op);
3794 GNUNET_CADET_receive_done (op->channel);
3795}
3796
3797
3798/**
3799 * Handle a "full done" message.
3800 *
3801 * @param cls closure, a set union operation
3802 * @param mh the demand message
3803 */
3804static void
3805handle_union_p2p_full_done (void *cls,
3806 const struct GNUNET_MessageHeader *mh)
3807{
3808 struct Operation *op = cls;
3809
3810 /**
3811 * Check that the message is received only in supported phase
3812 */
3813 uint8_t allowed_phases[] = {PHASE_FULL_SENDING, PHASE_FULL_RECEIVING};
3814 if (GNUNET_OK !=
3815 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3816 {
3817 GNUNET_break (0);
3818 fail_union_operation (op);
3819 return;
3820 }
3821
3822#if MEASURE_PERFORMANCE
3823 perf_store.full_done.received += 1;
3824#endif
3825
3826 switch (op->phase)
3827 {
3828 case PHASE_FULL_RECEIVING:
3829 {
3830 struct GNUNET_MQ_Envelope *ev;
3831
3832 if ((GNUNET_YES == op->byzantine) &&
3833 (op->received_total != op->remote_element_count) )
3834 {
3835 /* The other peer gave not enough elements before sending full done, there's something wrong. */
3836 LOG (GNUNET_ERROR_TYPE_ERROR,
3837 "Other peer sent only %llu/%llu fresh elements, failing operation\n",
3838 (unsigned long long) op->received_total,
3839 (unsigned long long) op->remote_element_count);
3840 GNUNET_break_op (0);
3841 fail_union_operation (op);
3842 return;
3843 }
3844
3845 LOG (GNUNET_ERROR_TYPE_DEBUG,
3846 "got FULL DONE, sending elements that other peer is missing\n");
3847
3848 /* send all the elements that did not come from the remote peer */
3849 GNUNET_CONTAINER_multihashmap32_iterate (op->key_to_element,
3850 &send_missing_full_elements_iter,
3851 op);
3852#if MEASURE_PERFORMANCE
3853 perf_store.full_done.sent += 1;
3854#endif
3855 ev = GNUNET_MQ_msg_header (GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE);
3856 GNUNET_MQ_send (op->mq,
3857 ev);
3858 op->phase = PHASE_FINISHED;
3859 /* we now wait until the other peer sends us the OVER message*/
3860 }
3861 break;
3862
3863 case PHASE_FULL_SENDING:
3864 {
3865 LOG (GNUNET_ERROR_TYPE_DEBUG,
3866 "got FULL DONE, finishing\n");
3867 /* We sent the full set, and got the response for that. We're done. */
3868 op->phase = PHASE_FINISHED;
3869 GNUNET_CADET_receive_done (op->channel);
3870 send_client_done (op);
3871 _GSS_operation_destroy2 (op);
3872 return;
3873 }
3874
3875 default:
3876 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3877 "Handle full done phase is %u\n",
3878 (unsigned) op->phase);
3879 GNUNET_break_op (0);
3880 fail_union_operation (op);
3881 return;
3882 }
3883 GNUNET_CADET_receive_done (op->channel);
3884}
3885
3886
3887/**
3888 * Check a demand by the other peer for elements based on a list
3889 * of `struct GNUNET_HashCode`s.
3890 *
3891 * @param cls closure, a set union operation
3892 * @param mh the demand message
3893 * @return #GNUNET_OK if @a mh is well-formed
3894 */
3895static int
3896check_union_p2p_demand (void *cls,
3897 const struct GNUNET_MessageHeader *mh)
3898{
3899 struct Operation *op = cls;
3900 unsigned int num_hashes;
3901
3902 (void) op;
3903 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3904 / sizeof(struct GNUNET_HashCode);
3905 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3906 != num_hashes * sizeof(struct GNUNET_HashCode))
3907 {
3908 GNUNET_break_op (0);
3909 return GNUNET_SYSERR;
3910 }
3911 return GNUNET_OK;
3912}
3913
3914
3915/**
3916 * Handle a demand by the other peer for elements based on a list
3917 * of `struct GNUNET_HashCode`s.
3918 *
3919 * @param cls closure, a set union operation
3920 * @param mh the demand message
3921 */
3922static void
3923handle_union_p2p_demand (void *cls,
3924 const struct GNUNET_MessageHeader *mh)
3925{
3926 struct Operation *op = cls;
3927 struct ElementEntry *ee;
3928 struct GNUNET_SETU_ElementMessage *emsg;
3929 const struct GNUNET_HashCode *hash;
3930 unsigned int num_hashes;
3931 struct GNUNET_MQ_Envelope *ev;
3932
3933 /**
3934 * Check that the message is received only in supported phase
3935 */
3936 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING,
3937 PHASE_FINISH_WAITING};
3938 if (GNUNET_OK !=
3939 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
3940 {
3941 GNUNET_break (0);
3942 fail_union_operation (op);
3943 return;
3944 }
3945#if MEASURE_PERFORMANCE
3946 perf_store.demand.received += 1;
3947 perf_store.demand.received_var_bytes += (ntohs (mh->size) - sizeof(struct
3948 GNUNET_MessageHeader));
3949#endif
3950
3951 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
3952 / sizeof(struct GNUNET_HashCode);
3953 for (hash = (const struct GNUNET_HashCode *) &mh[1];
3954 num_hashes > 0;
3955 hash++, num_hashes--)
3956 {
3957 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
3958 hash);
3959 if (NULL == ee)
3960 {
3961 /* Demand for non-existing element. */
3962 GNUNET_break_op (0);
3963 fail_union_operation (op);
3964 return;
3965 }
3966
3967 /* Save send demand message for message control */
3968 if (GNUNET_YES !=
3969 update_message_control_flow (
3970 op->message_control_flow,
3971 MSG_CFS_RECEIVED,
3972 &ee->element_hash,
3973 DEMAND_MESSAGE)
3974 )
3975 {
3976 LOG (GNUNET_ERROR_TYPE_ERROR,
3977 "Double demand message received found!\n");
3978 GNUNET_break (0);
3979 fail_union_operation (op);
3980 return;
3981 }
3982 ;
3983
3984 /* Mark element to be expected to received */
3985 if (GNUNET_YES !=
3986 update_message_control_flow (
3987 op->message_control_flow,
3988 MSG_CFS_SENT,
3989 &ee->element_hash,
3990 ELEMENT_MESSAGE)
3991 )
3992 {
3993 LOG (GNUNET_ERROR_TYPE_ERROR,
3994 "Double element message sent found!\n");
3995 GNUNET_break (0);
3996 fail_union_operation (op);
3997 return;
3998 }
3999 if (GNUNET_NO == _GSS_is_element_of_operation (ee, op))
4000 {
4001 /* Probably confused lazily copied sets. */
4002 GNUNET_break_op (0);
4003 fail_union_operation (op);
4004 return;
4005 }
4006#if MEASURE_PERFORMANCE
4007 perf_store.element.sent += 1;
4008 perf_store.element.sent_var_bytes += ee->element.size;
4009#endif
4010 ev = GNUNET_MQ_msg_extra (emsg,
4011 ee->element.size,
4012 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS);
4013 GNUNET_memcpy (&emsg[1],
4014 ee->element.data,
4015 ee->element.size);
4016 emsg->reserved = htons (0);
4017 emsg->element_type = htons (ee->element.element_type);
4018 LOG (GNUNET_ERROR_TYPE_DEBUG,
4019 "[OP %p] Sending demanded element (size %u, hash %s) to peer\n",
4020 op,
4021 (unsigned int) ee->element.size,
4022 GNUNET_h2s (&ee->element_hash));
4023 GNUNET_MQ_send (op->mq, ev);
4024 GNUNET_STATISTICS_update (_GSS_statistics,
4025 "# exchanged elements",
4026 1,
4027 GNUNET_NO);
4028 if (op->symmetric)
4029 send_client_element (op,
4030 &ee->element,
4031 GNUNET_SETU_STATUS_ADD_REMOTE);
4032 }
4033 GNUNET_CADET_receive_done (op->channel);
4034 maybe_finish (op);
4035}
4036
4037
4038/**
4039 * Check offer (of `struct GNUNET_HashCode`s).
4040 *
4041 * @param cls the union operation
4042 * @param mh the message
4043 * @return #GNUNET_OK if @a mh is well-formed
4044 */
4045static int
4046check_union_p2p_offer (void *cls,
4047 const struct GNUNET_MessageHeader *mh)
4048{
4049 struct Operation *op = cls;
4050 unsigned int num_hashes;
4051
4052 /* look up elements and send them */
4053 if ((op->phase != PHASE_PASSIVE_DECODING) &&
4054 (op->phase != PHASE_ACTIVE_DECODING))
4055 {
4056 GNUNET_break_op (0);
4057 return GNUNET_SYSERR;
4058 }
4059 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
4060 / sizeof(struct GNUNET_HashCode);
4061 if ((ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader)) !=
4062 num_hashes * sizeof(struct GNUNET_HashCode))
4063 {
4064 GNUNET_break_op (0);
4065 return GNUNET_SYSERR;
4066 }
4067 return GNUNET_OK;
4068}
4069
4070
4071/**
4072 * Handle offers (of `struct GNUNET_HashCode`s) and
4073 * respond with demands (of `struct GNUNET_HashCode`s).
4074 *
4075 * @param cls the union operation
4076 * @param mh the message
4077 */
4078static void
4079handle_union_p2p_offer (void *cls,
4080 const struct GNUNET_MessageHeader *mh)
4081{
4082 struct Operation *op = cls;
4083 const struct GNUNET_HashCode *hash;
4084 unsigned int num_hashes;
4085 /**
4086 * Check that the message is received only in supported phase
4087 */
4088 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
4089 if (GNUNET_OK !=
4090 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
4091 {
4092 GNUNET_break (0);
4093 fail_union_operation (op);
4094 return;
4095 }
4096
4097#if MEASURE_PERFORMANCE
4098 perf_store.offer.received += 1;
4099 perf_store.offer.received_var_bytes += (ntohs (mh->size) - sizeof(struct
4100 GNUNET_MessageHeader));
4101#endif
4102
4103 num_hashes = (ntohs (mh->size) - sizeof(struct GNUNET_MessageHeader))
4104 / sizeof(struct GNUNET_HashCode);
4105 for (hash = (const struct GNUNET_HashCode *) &mh[1];
4106 num_hashes > 0;
4107 hash++, num_hashes--)
4108 {
4109 struct ElementEntry *ee;
4110 struct GNUNET_MessageHeader *demands;
4111 struct GNUNET_MQ_Envelope *ev;
4112
4113 ee = GNUNET_CONTAINER_multihashmap_get (op->set->content->elements,
4114 hash);
4115 if (NULL != ee)
4116 if (GNUNET_YES == _GSS_is_element_of_operation (ee, op))
4117 continue;
4118
4119 if (GNUNET_YES ==
4120 GNUNET_CONTAINER_multihashmap_contains (op->demanded_hashes,
4121 hash))
4122 {
4123 LOG (GNUNET_ERROR_TYPE_DEBUG,
4124 "Skipped sending duplicate demand\n");
4125 continue;
4126 }
4127
4128 GNUNET_assert (GNUNET_OK ==
4129 GNUNET_CONTAINER_multihashmap_put (
4130 op->demanded_hashes,
4131 hash,
4132 NULL,
4133 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST));
4134
4135 LOG (GNUNET_ERROR_TYPE_DEBUG,
4136 "[OP %p] Requesting element (hash %s)\n",
4137 op, GNUNET_h2s (hash));
4138
4139#if MEASURE_PERFORMANCE
4140 perf_store.demand.sent += 1;
4141 perf_store.demand.sent_var_bytes += sizeof(struct GNUNET_HashCode);
4142#endif
4143 /* Save send demand message for message control */
4144 if (GNUNET_YES !=
4145 update_message_control_flow (
4146 op->message_control_flow,
4147 MSG_CFS_SENT,
4148 hash,
4149 DEMAND_MESSAGE))
4150 {
4151 LOG (GNUNET_ERROR_TYPE_ERROR,
4152 "Double demand message sent found!\n");
4153 GNUNET_break (0);
4154 fail_union_operation (op);
4155 return;
4156 }
4157
4158 /* Mark offer as received received */
4159 if (GNUNET_YES !=
4160 update_message_control_flow (
4161 op->message_control_flow,
4162 MSG_CFS_RECEIVED,
4163 hash,
4164 OFFER_MESSAGE))
4165 {
4166 LOG (GNUNET_ERROR_TYPE_ERROR,
4167 "Double offer message received found!\n");
4168 GNUNET_break (0);
4169 fail_union_operation (op);
4170 return;
4171 }
4172 /* Mark element to be expected to received */
4173 if (GNUNET_YES !=
4174 update_message_control_flow (
4175 op->message_control_flow,
4176 MSG_CFS_EXPECTED,
4177 hash,
4178 ELEMENT_MESSAGE))
4179 {
4180 LOG (GNUNET_ERROR_TYPE_ERROR,
4181 "Element already expected!\n");
4182 GNUNET_break (0);
4183 fail_union_operation (op);
4184 return;
4185 }
4186 ev = GNUNET_MQ_msg_header_extra (demands,
4187 sizeof(struct GNUNET_HashCode),
4188 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND);
4189 GNUNET_memcpy (&demands[1],
4190 hash,
4191 sizeof(struct GNUNET_HashCode));
4192 GNUNET_MQ_send (op->mq, ev);
4193 }
4194 GNUNET_CADET_receive_done (op->channel);
4195}
4196
4197
4198/**
4199 * Handle a done message from a remote peer
4200 *
4201 * @param cls the union operation
4202 * @param mh the message
4203 */
4204static void
4205handle_union_p2p_done (void *cls,
4206 const struct GNUNET_MessageHeader *mh)
4207{
4208 struct Operation *op = cls;
4209
4210 /**
4211 * Check that the message is received only in supported phase
4212 */
4213 uint8_t allowed_phases[] = {PHASE_ACTIVE_DECODING, PHASE_PASSIVE_DECODING};
4214 if (GNUNET_OK !=
4215 check_valid_phase (allowed_phases,sizeof(allowed_phases),op))
4216 {
4217 GNUNET_break (0);
4218 fail_union_operation (op);
4219 return;
4220 }
4221
4222 if (op->active_passive_switch_required)
4223 {
4224 LOG (GNUNET_ERROR_TYPE_ERROR,
4225 "PROTOCOL VIOLATION: Received done but role change is necessary\n");
4226 GNUNET_break (0);
4227 fail_union_operation (op);
4228 return;
4229 }
4230
4231#if MEASURE_PERFORMANCE
4232 perf_store.done.received += 1;
4233#endif
4234 switch (op->phase)
4235 {
4236 case PHASE_PASSIVE_DECODING:
4237 /* We got all requests, but still have to send our elements in response. */
4238 op->phase = PHASE_FINISH_WAITING;
4239 LOG (GNUNET_ERROR_TYPE_DEBUG,
4240 "got DONE (as passive partner), waiting for our demands to be satisfied\n");
4241 /* The active peer is done sending offers
4242 * and inquiries. This means that all
4243 * our responses to that (demands and offers)
4244 * must be in flight (queued or in mesh).
4245 *
4246 * We should notify the active peer once
4247 * all our demands are satisfied, so that the active
4248 * peer can quit if we gave it everything.
4249 */GNUNET_CADET_receive_done (op->channel);
4250 maybe_finish (op);
4251 return;
4252 case PHASE_ACTIVE_DECODING:
4253 LOG (GNUNET_ERROR_TYPE_DEBUG,
4254 "got DONE (as active partner), waiting to finish\n");
4255 /* All demands of the other peer are satisfied,
4256 * and we processed all offers, thus we know
4257 * exactly what our demands must be.
4258 *
4259 * We'll close the channel
4260 * to the other peer once our demands are met.
4261 */op->phase = PHASE_FINISH_CLOSING;
4262 GNUNET_CADET_receive_done (op->channel);
4263 maybe_finish (op);
4264 return;
4265 default:
4266 GNUNET_break_op (0);
4267 fail_union_operation (op);
4268 return;
4269 }
4270}
4271
4272
4273/**
4274 * Handle a over message from a remote peer
4275 *
4276 * @param cls the union operation
4277 * @param mh the message
4278 */
4279static void
4280handle_union_p2p_over (void *cls,
4281 const struct GNUNET_MessageHeader *mh)
4282{
4283#if MEASURE_PERFORMANCE
4284 perf_store.over.received += 1;
4285#endif
4286 send_client_done (cls);
4287}
4288
4289
4290/**
4291 * Get the incoming socket associated with the given id.
4292 *
4293 * @param listener the listener to look in
4294 * @param id id to look for
4295 * @return the incoming socket associated with the id,
4296 * or NULL if there is none
4297 */
4298static struct Operation *
4299get_incoming (uint32_t id)
4300{
4301 for (struct Listener *listener = listener_head;
4302 NULL != listener;
4303 listener = listener->next)
4304 {
4305 for (struct Operation *op = listener->op_head;
4306 NULL != op;
4307 op = op->next)
4308 if (op->suggest_id == id)
4309 return op;
4310 }
4311 return NULL;
4312}
4313
4314
4315/**
4316 * Callback called when a client connects to the service.
4317 *
4318 * @param cls closure for the service
4319 * @param c the new client that connected to the service
4320 * @param mq the message queue used to send messages to the client
4321 * @return @a `struct ClientState`
4322 */
4323static void *
4324client_connect_cb (void *cls,
4325 struct GNUNET_SERVICE_Client *c,
4326 struct GNUNET_MQ_Handle *mq)
4327{
4328 struct ClientState *cs;
4329
4330 num_clients++;
4331 cs = GNUNET_new (struct ClientState);
4332 cs->client = c;
4333 cs->mq = mq;
4334 return cs;
4335}
4336
4337
4338/**
4339 * Iterator over hash map entries to free element entries.
4340 *
4341 * @param cls closure
4342 * @param key current key code
4343 * @param value a `struct ElementEntry *` to be free'd
4344 * @return #GNUNET_YES (continue to iterate)
4345 */
4346static int
4347destroy_elements_iterator (void *cls,
4348 const struct GNUNET_HashCode *key,
4349 void *value)
4350{
4351 struct ElementEntry *ee = value;
4352
4353 GNUNET_free (ee);
4354 return GNUNET_YES;
4355}
4356
4357
4358/**
4359 * Clean up after a client has disconnected
4360 *
4361 * @param cls closure, unused
4362 * @param client the client to clean up after
4363 * @param internal_cls the `struct ClientState`
4364 */
4365static void
4366client_disconnect_cb (void *cls,
4367 struct GNUNET_SERVICE_Client *client,
4368 void *internal_cls)
4369{
4370 struct ClientState *cs = internal_cls;
4371 struct Operation *op;
4372 struct Listener *listener;
4373 struct Set *set;
4374
4375 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4376 "Client disconnected, cleaning up\n");
4377 if (NULL != (set = cs->set))
4378 {
4379 struct SetContent *content = set->content;
4380
4381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4382 "Destroying client's set\n");
4383 /* Destroy pending set operations */
4384 while (NULL != set->ops_head)
4385 _GSS_operation_destroy (set->ops_head);
4386
4387 /* Destroy operation-specific state */
4388 if (NULL != set->se)
4389 {
4390 strata_estimator_destroy (set->se);
4391 set->se = NULL;
4392 }
4393 /* free set content (or at least decrement RC) */
4394 set->content = NULL;
4395 GNUNET_assert (0 != content->refcount);
4396 content->refcount--;
4397 if (0 == content->refcount)
4398 {
4399 GNUNET_assert (NULL != content->elements);
4400 GNUNET_CONTAINER_multihashmap_iterate (content->elements,
4401 &destroy_elements_iterator,
4402 NULL);
4403 GNUNET_CONTAINER_multihashmap_destroy (content->elements);
4404 content->elements = NULL;
4405 GNUNET_free (content);
4406 }
4407 GNUNET_free (set);
4408 }
4409
4410 if (NULL != (listener = cs->listener))
4411 {
4412 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4413 "Destroying client's listener\n");
4414 GNUNET_CADET_close_port (listener->open_port);
4415 listener->open_port = NULL;
4416 while (NULL != (op = listener->op_head))
4417 {
4418 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
4419 "Destroying incoming operation `%u' from peer `%s'\n",
4420 (unsigned int) op->client_request_id,
4421 GNUNET_i2s (&op->peer));
4422 incoming_destroy (op);
4423 }
4424 GNUNET_CONTAINER_DLL_remove (listener_head,
4425 listener_tail,
4426 listener);
4427 GNUNET_free (listener);
4428 }
4429 GNUNET_free (cs);
4430 num_clients--;
4431 if ( (GNUNET_YES == in_shutdown) &&
4432 (0 == num_clients) )
4433 {
4434 if (NULL != cadet)
4435 {
4436 GNUNET_CADET_disconnect (cadet);
4437 cadet = NULL;
4438 }
4439 }
4440}
4441
4442
4443/**
4444 * Check a request for a set operation from another peer.
4445 *
4446 * @param cls the operation state
4447 * @param msg the received message
4448 * @return #GNUNET_OK if the channel should be kept alive,
4449 * #GNUNET_SYSERR to destroy the channel
4450 */
4451static int
4452check_incoming_msg (void *cls,
4453 const struct OperationRequestMessage *msg)
4454{
4455 struct Operation *op = cls;
4456 struct Listener *listener = op->listener;
4457 const struct GNUNET_MessageHeader *nested_context;
4458
4459 /* double operation request */
4460 if (0 != op->suggest_id)
4461 {
4462 GNUNET_break_op (0);
4463 return GNUNET_SYSERR;
4464 }
4465 /* This should be equivalent to the previous condition, but can't hurt to check twice */
4466 if (NULL == listener)
4467 {
4468 GNUNET_break (0);
4469 return GNUNET_SYSERR;
4470 }
4471 nested_context = GNUNET_MQ_extract_nested_mh (msg);
4472 if ((NULL != nested_context) &&
4473 (ntohs (nested_context->size) > GNUNET_SETU_CONTEXT_MESSAGE_MAX_SIZE))
4474 {
4475 GNUNET_break_op (0);
4476 return GNUNET_SYSERR;
4477 }
4478 return GNUNET_OK;
4479}
4480
4481
4482/**
4483 * Handle a request for a set operation from another peer. Checks if we
4484 * have a listener waiting for such a request (and in that case initiates
4485 * asking the listener about accepting the connection). If no listener
4486 * is waiting, we queue the operation request in hope that a listener
4487 * shows up soon (before timeout).
4488 *
4489 * This msg is expected as the first and only msg handled through the
4490 * non-operation bound virtual table, acceptance of this operation replaces
4491 * our virtual table and subsequent msgs would be routed differently (as
4492 * we then know what type of operation this is).
4493 *
4494 * @param cls the operation state
4495 * @param msg the received message
4496 */
4497static void
4498handle_incoming_msg (void *cls,
4499 const struct OperationRequestMessage *msg)
4500{
4501 struct Operation *op = cls;
4502 struct Listener *listener = op->listener;
4503 const struct GNUNET_MessageHeader *nested_context;
4504 struct GNUNET_MQ_Envelope *env;
4505 struct GNUNET_SETU_RequestMessage *cmsg;
4506
4507 nested_context = GNUNET_MQ_extract_nested_mh (msg);
4508 /* Make a copy of the nested_context (application-specific context
4509 information that is opaque to set) so we can pass it to the
4510 listener later on */
4511 if (NULL != nested_context)
4512 op->context_msg = GNUNET_copy_message (nested_context);
4513 op->remote_element_count = ntohl (msg->element_count);
4514 GNUNET_log (
4515 GNUNET_ERROR_TYPE_DEBUG,
4516 "Received P2P operation request (port %s) for active listener\n",
4517 GNUNET_h2s (&op->listener->app_id));
4518 GNUNET_assert (0 == op->suggest_id);
4519 if (0 == suggest_id)
4520 suggest_id++;
4521 op->suggest_id = suggest_id++;
4522 GNUNET_assert (NULL != op->timeout_task);
4523 GNUNET_SCHEDULER_cancel (op->timeout_task);
4524 op->timeout_task = NULL;
4525 env = GNUNET_MQ_msg_nested_mh (cmsg,
4526 GNUNET_MESSAGE_TYPE_SETU_REQUEST,
4527 op->context_msg);
4528 GNUNET_log (
4529 GNUNET_ERROR_TYPE_DEBUG,
4530 "Suggesting incoming request with accept id %u to listener %p of client %p\n",
4531 op->suggest_id,
4532 listener,
4533 listener->cs);
4534 cmsg->accept_id = htonl (op->suggest_id);
4535 cmsg->peer_id = op->peer;
4536 GNUNET_MQ_send (listener->cs->mq,
4537 env);
4538 /* NOTE: GNUNET_CADET_receive_done() will be called in
4539 #handle_client_accept() */
4540}
4541
4542
4543/**
4544 * Called when a client wants to create a new set. This is typically
4545 * the first request from a client, and includes the type of set
4546 * operation to be performed.
4547 *
4548 * @param cls client that sent the message
4549 * @param m message sent by the client
4550 */
4551static void
4552handle_client_create_set (void *cls,
4553 const struct GNUNET_SETU_CreateMessage *msg)
4554{
4555 struct ClientState *cs = cls;
4556 struct Set *set;
4557
4558 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4559 "Client created new set for union operation\n");
4560 if (NULL != cs->set)
4561 {
4562 /* There can only be one set per client */
4563 GNUNET_break (0);
4564 GNUNET_SERVICE_client_drop (cs->client);
4565 return;
4566 }
4567 set = GNUNET_new (struct Set);
4568 {
4569 struct MultiStrataEstimator *se;
4570
4571 se = strata_estimator_create (SE_STRATA_COUNT,
4572 SE_IBFS_TOTAL_SIZE,
4573 SE_IBF_HASH_NUM);
4574 if (NULL == se)
4575 {
4576 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
4577 "Failed to allocate strata estimator\n");
4578 GNUNET_free (set);
4579 GNUNET_SERVICE_client_drop (cs->client);
4580 return;
4581 }
4582 set->se = se;
4583 }
4584 set->content = GNUNET_new (struct SetContent);
4585 set->content->refcount = 1;
4586 set->content->elements = GNUNET_CONTAINER_multihashmap_create (1,
4587 GNUNET_YES);
4588 set->cs = cs;
4589 cs->set = set;
4590 GNUNET_SERVICE_client_continue (cs->client);
4591}
4592
4593
4594/**
4595 * Timeout happens iff:
4596 * - we suggested an operation to our listener,
4597 * but did not receive a response in time
4598 * - we got the channel from a peer but no #GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST
4599 *
4600 * @param cls channel context
4601 * @param tc context information (why was this task triggered now)
4602 */
4603static void
4604incoming_timeout_cb (void *cls)
4605{
4606 struct Operation *op = cls;
4607
4608 op->timeout_task = NULL;
4609 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4610 "Remote peer's incoming request timed out\n");
4611 incoming_destroy (op);
4612}
4613
4614
4615/**
4616 * Method called whenever another peer has added us to a channel the
4617 * other peer initiated. Only called (once) upon reception of data
4618 * from a channel we listen on.
4619 *
4620 * The channel context represents the operation itself and gets added
4621 * to a DLL, from where it gets looked up when our local listener
4622 * client responds to a proposed/suggested operation or connects and
4623 * associates with this operation.
4624 *
4625 * @param cls closure
4626 * @param channel new handle to the channel
4627 * @param source peer that started the channel
4628 * @return initial channel context for the channel
4629 * returns NULL on error
4630 */
4631static void *
4632channel_new_cb (void *cls,
4633 struct GNUNET_CADET_Channel *channel,
4634 const struct GNUNET_PeerIdentity *source)
4635{
4636 struct Listener *listener = cls;
4637 struct Operation *op;
4638
4639 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4640 "New incoming channel\n");
4641 op = GNUNET_new (struct Operation);
4642 op->listener = listener;
4643 op->peer = *source;
4644 op->channel = channel;
4645 op->mq = GNUNET_CADET_get_mq (op->channel);
4646 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
4647 UINT32_MAX);
4648 op->timeout_task = GNUNET_SCHEDULER_add_delayed (INCOMING_CHANNEL_TIMEOUT,
4649 &incoming_timeout_cb,
4650 op);
4651 GNUNET_CONTAINER_DLL_insert (listener->op_head,
4652 listener->op_tail,
4653 op);
4654 return op;
4655}
4656
4657
4658/**
4659 * Function called whenever a channel is destroyed. Should clean up
4660 * any associated state. It must NOT call
4661 * GNUNET_CADET_channel_destroy() on the channel.
4662 *
4663 * The peer_disconnect function is part of a a virtual table set initially either
4664 * when a peer creates a new channel with us, or once we create
4665 * a new channel ourselves (evaluate).
4666 *
4667 * Once we know the exact type of operation (union/intersection), the vt is
4668 * replaced with an operation specific instance (_GSS_[op]_vt).
4669 *
4670 * @param channel_ctx place where local state associated
4671 * with the channel is stored
4672 * @param channel connection to the other end (henceforth invalid)
4673 */
4674static void
4675channel_end_cb (void *channel_ctx,
4676 const struct GNUNET_CADET_Channel *channel)
4677{
4678 struct Operation *op = channel_ctx;
4679
4680 op->channel = NULL;
4681 _GSS_operation_destroy2 (op);
4682}
4683
4684
4685/**
4686 * Function called whenever an MQ-channel's transmission window size changes.
4687 *
4688 * The first callback in an outgoing channel will be with a non-zero value
4689 * and will mean the channel is connected to the destination.
4690 *
4691 * For an incoming channel it will be called immediately after the
4692 * #GNUNET_CADET_ConnectEventHandler, also with a non-zero value.
4693 *
4694 * @param cls Channel closure.
4695 * @param channel Connection to the other end (henceforth invalid).
4696 * @param window_size New window size. If the is more messages than buffer size
4697 * this value will be negative..
4698 */
4699static void
4700channel_window_cb (void *cls,
4701 const struct GNUNET_CADET_Channel *channel,
4702 int window_size)
4703{
4704 /* FIXME: not implemented, we could do flow control here... */
4705}
4706
4707
4708/**
4709 * Called when a client wants to create a new listener.
4710 *
4711 * @param cls client that sent the message
4712 * @param msg message sent by the client
4713 */
4714
4715static void
4716handle_client_listen (void *cls,
4717 const struct GNUNET_SETU_ListenMessage *msg)
4718{
4719 struct ClientState *cs = cls;
4720 struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
4721 GNUNET_MQ_hd_var_size (incoming_msg,
4722 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
4723 struct OperationRequestMessage,
4724 NULL),
4725 GNUNET_MQ_hd_var_size (union_p2p_ibf,
4726 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF,
4727 struct IBFMessage,
4728 NULL),
4729 GNUNET_MQ_hd_var_size (union_p2p_elements,
4730 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS,
4731 struct GNUNET_SETU_ElementMessage,
4732 NULL),
4733 GNUNET_MQ_hd_var_size (union_p2p_offer,
4734 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER,
4735 struct GNUNET_MessageHeader,
4736 NULL),
4737 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
4738 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY,
4739 struct InquiryMessage,
4740 NULL),
4741 GNUNET_MQ_hd_var_size (union_p2p_demand,
4742 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND,
4743 struct GNUNET_MessageHeader,
4744 NULL),
4745 GNUNET_MQ_hd_fixed_size (union_p2p_done,
4746 GNUNET_MESSAGE_TYPE_SETU_P2P_DONE,
4747 struct GNUNET_MessageHeader,
4748 NULL),
4749 GNUNET_MQ_hd_fixed_size (union_p2p_over,
4750 GNUNET_MESSAGE_TYPE_SETU_P2P_OVER,
4751 struct GNUNET_MessageHeader,
4752 NULL),
4753 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
4754 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE,
4755 struct GNUNET_MessageHeader,
4756 NULL),
4757 GNUNET_MQ_hd_var_size (union_p2p_request_full,
4758 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL,
4759 struct TransmitFullMessage,
4760 NULL),
4761 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4762 GNUNET_MESSAGE_TYPE_SETU_P2P_SE,
4763 struct StrataEstimatorMessage,
4764 NULL),
4765 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
4766 GNUNET_MESSAGE_TYPE_SETU_P2P_SEC,
4767 struct StrataEstimatorMessage,
4768 NULL),
4769 GNUNET_MQ_hd_var_size (union_p2p_full_element,
4770 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT,
4771 struct GNUNET_SETU_ElementMessage,
4772 NULL),
4773 GNUNET_MQ_hd_var_size (union_p2p_send_full,
4774 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL,
4775 struct TransmitFullMessage,
4776 NULL),
4777 GNUNET_MQ_handler_end ()
4778 };
4779 struct Listener *listener;
4780
4781 if (NULL != cs->listener)
4782 {
4783 /* max. one active listener per client! */
4784 GNUNET_break (0);
4785 GNUNET_SERVICE_client_drop (cs->client);
4786 return;
4787 }
4788 listener = GNUNET_new (struct Listener);
4789 listener->cs = cs;
4790 cs->listener = listener;
4791 listener->app_id = msg->app_id;
4792 GNUNET_CONTAINER_DLL_insert (listener_head,
4793 listener_tail,
4794 listener);
4795 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4796 "New listener created (port %s)\n",
4797 GNUNET_h2s (&listener->app_id));
4798 listener->open_port = GNUNET_CADET_open_port (cadet,
4799 &msg->app_id,
4800 &channel_new_cb,
4801 listener,
4802 &channel_window_cb,
4803 &channel_end_cb,
4804 cadet_handlers);
4805 GNUNET_SERVICE_client_continue (cs->client);
4806}
4807
4808
4809/**
4810 * Called when the listening client rejects an operation
4811 * request by another peer.
4812 *
4813 * @param cls client that sent the message
4814 * @param msg message sent by the client
4815 */
4816static void
4817handle_client_reject (void *cls,
4818 const struct GNUNET_SETU_RejectMessage *msg)
4819{
4820 struct ClientState *cs = cls;
4821 struct Operation *op;
4822
4823 op = get_incoming (ntohl (msg->accept_reject_id));
4824 if (NULL == op)
4825 {
4826 /* no matching incoming operation for this reject;
4827 could be that the other peer already disconnected... */
4828 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
4829 "Client rejected unknown operation %u\n",
4830 (unsigned int) ntohl (msg->accept_reject_id));
4831 GNUNET_SERVICE_client_continue (cs->client);
4832 return;
4833 }
4834 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4835 "Peer request (app %s) rejected by client\n",
4836 GNUNET_h2s (&cs->listener->app_id));
4837 _GSS_operation_destroy2 (op);
4838 GNUNET_SERVICE_client_continue (cs->client);
4839}
4840
4841
4842/**
4843 * Called when a client wants to add or remove an element to a set it inhabits.
4844 *
4845 * @param cls client that sent the message
4846 * @param msg message sent by the client
4847 */
4848static int
4849check_client_set_add (void *cls,
4850 const struct GNUNET_SETU_ElementMessage *msg)
4851{
4852 /* NOTE: Technically, we should probably check with the
4853 block library whether the element we are given is well-formed */
4854 return GNUNET_OK;
4855}
4856
4857
4858/**
4859 * Called when a client wants to add or remove an element to a set it inhabits.
4860 *
4861 * @param cls client that sent the message
4862 * @param msg message sent by the client
4863 */
4864static void
4865handle_client_set_add (void *cls,
4866 const struct GNUNET_SETU_ElementMessage *msg)
4867{
4868 struct ClientState *cs = cls;
4869 struct Set *set;
4870 struct GNUNET_SETU_Element el;
4871 struct ElementEntry *ee;
4872 struct GNUNET_HashCode hash;
4873
4874 if (NULL == (set = cs->set))
4875 {
4876 /* client without a set requested an operation */
4877 GNUNET_break (0);
4878 GNUNET_SERVICE_client_drop (cs->client);
4879 return;
4880 }
4881 GNUNET_SERVICE_client_continue (cs->client);
4882 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Executing mutation on set\n");
4883 el.size = ntohs (msg->header.size) - sizeof(*msg);
4884 el.data = &msg[1];
4885 el.element_type = ntohs (msg->element_type);
4886 GNUNET_SETU_element_hash (&el,
4887 &hash);
4888 ee = GNUNET_CONTAINER_multihashmap_get (set->content->elements,
4889 &hash);
4890 if (NULL == ee)
4891 {
4892 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4893 "Client inserts element %s of size %u\n",
4894 GNUNET_h2s (&hash),
4895 el.size);
4896 ee = GNUNET_malloc (el.size + sizeof(*ee));
4897 ee->element.size = el.size;
4898 GNUNET_memcpy (&ee[1], el.data, el.size);
4899 ee->element.data = &ee[1];
4900 ee->element.element_type = el.element_type;
4901 ee->remote = GNUNET_NO;
4902 ee->generation = set->current_generation;
4903 ee->element_hash = hash;
4904 GNUNET_break (GNUNET_YES ==
4905 GNUNET_CONTAINER_multihashmap_put (
4906 set->content->elements,
4907 &ee->element_hash,
4908 ee,
4909 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
4910 }
4911 else
4912 {
4913 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
4914 "Client inserted element %s of size %u twice (ignored)\n",
4915 GNUNET_h2s (&hash),
4916 el.size);
4917 /* same element inserted twice */
4918 return;
4919 }
4920 strata_estimator_insert (set->se,
4921 get_ibf_key (&ee->element_hash));
4922}
4923
4924
4925/**
4926 * Advance the current generation of a set,
4927 * adding exclusion ranges if necessary.
4928 *
4929 * @param set the set where we want to advance the generation
4930 */
4931static void
4932advance_generation (struct Set *set)
4933{
4934 set->content->latest_generation++;
4935 set->current_generation++;
4936}
4937
4938
4939/**
4940 * Called when a client wants to initiate a set operation with another
4941 * peer. Initiates the CADET connection to the listener and sends the
4942 * request.
4943 *
4944 * @param cls client that sent the message
4945 * @param msg message sent by the client
4946 * @return #GNUNET_OK if the message is well-formed
4947 */
4948static int
4949check_client_evaluate (void *cls,
4950 const struct GNUNET_SETU_EvaluateMessage *msg)
4951{
4952 /* FIXME: suboptimal, even if the context below could be NULL,
4953 there are malformed messages this does not check for... */
4954 return GNUNET_OK;
4955}
4956
4957
4958/**
4959 * Called when a client wants to initiate a set operation with another
4960 * peer. Initiates the CADET connection to the listener and sends the
4961 * request.
4962 *
4963 * @param cls client that sent the message
4964 * @param msg message sent by the client
4965 */
4966static void
4967handle_client_evaluate (void *cls,
4968 const struct GNUNET_SETU_EvaluateMessage *msg)
4969{
4970 struct ClientState *cs = cls;
4971 struct Operation *op = GNUNET_new (struct Operation);
4972
4973 const struct GNUNET_MQ_MessageHandler cadet_handlers[] = {
4974 GNUNET_MQ_hd_var_size (incoming_msg,
4975 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
4976 struct OperationRequestMessage,
4977 op),
4978 GNUNET_MQ_hd_var_size (union_p2p_ibf,
4979 GNUNET_MESSAGE_TYPE_SETU_P2P_IBF,
4980 struct IBFMessage,
4981 op),
4982 GNUNET_MQ_hd_var_size (union_p2p_elements,
4983 GNUNET_MESSAGE_TYPE_SETU_P2P_ELEMENTS,
4984 struct GNUNET_SETU_ElementMessage,
4985 op),
4986 GNUNET_MQ_hd_var_size (union_p2p_offer,
4987 GNUNET_MESSAGE_TYPE_SETU_P2P_OFFER,
4988 struct GNUNET_MessageHeader,
4989 op),
4990 GNUNET_MQ_hd_var_size (union_p2p_inquiry,
4991 GNUNET_MESSAGE_TYPE_SETU_P2P_INQUIRY,
4992 struct InquiryMessage,
4993 op),
4994 GNUNET_MQ_hd_var_size (union_p2p_demand,
4995 GNUNET_MESSAGE_TYPE_SETU_P2P_DEMAND,
4996 struct GNUNET_MessageHeader,
4997 op),
4998 GNUNET_MQ_hd_fixed_size (union_p2p_done,
4999 GNUNET_MESSAGE_TYPE_SETU_P2P_DONE,
5000 struct GNUNET_MessageHeader,
5001 op),
5002 GNUNET_MQ_hd_fixed_size (union_p2p_over,
5003 GNUNET_MESSAGE_TYPE_SETU_P2P_OVER,
5004 struct GNUNET_MessageHeader,
5005 op),
5006 GNUNET_MQ_hd_fixed_size (union_p2p_full_done,
5007 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_DONE,
5008 struct GNUNET_MessageHeader,
5009 op),
5010 GNUNET_MQ_hd_var_size (union_p2p_request_full,
5011 GNUNET_MESSAGE_TYPE_SETU_P2P_REQUEST_FULL,
5012 struct TransmitFullMessage,
5013 op),
5014 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
5015 GNUNET_MESSAGE_TYPE_SETU_P2P_SE,
5016 struct StrataEstimatorMessage,
5017 op),
5018 GNUNET_MQ_hd_var_size (union_p2p_strata_estimator,
5019 GNUNET_MESSAGE_TYPE_SETU_P2P_SEC,
5020 struct StrataEstimatorMessage,
5021 op),
5022 GNUNET_MQ_hd_var_size (union_p2p_full_element,
5023 GNUNET_MESSAGE_TYPE_SETU_P2P_FULL_ELEMENT,
5024 struct GNUNET_SETU_ElementMessage,
5025 op),
5026 GNUNET_MQ_hd_var_size (union_p2p_send_full,
5027 GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL,
5028 struct TransmitFullMessage,
5029 NULL),
5030 GNUNET_MQ_handler_end ()
5031 };
5032 struct Set *set;
5033 const struct GNUNET_MessageHeader *context;
5034
5035 if (NULL == (set = cs->set))
5036 {
5037 GNUNET_break (0);
5038 GNUNET_free (op);
5039 GNUNET_SERVICE_client_drop (cs->client);
5040 return;
5041 }
5042 op->salt = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
5043 UINT32_MAX);
5044 op->peer = msg->target_peer;
5045 op->client_request_id = ntohl (msg->request_id);
5046 op->byzantine = msg->byzantine;
5047 op->byzantine_lower_bound = ntohl (msg->byzantine_lower_bound);
5048 op->force_full = msg->force_full;
5049 op->force_delta = msg->force_delta;
5050 op->symmetric = msg->symmetric;
5051 op->rtt_bandwidth_tradeoff = msg->bandwidth_latency_tradeoff;
5052 op->ibf_bucket_number_factor = msg->ibf_bucket_number_factor;
5053 op->ibf_number_buckets_per_element = msg->ibf_number_of_buckets_per_element;
5054 op->byzantine_upper_bound = msg->byzantine_upper_bond;
5055 op->active_passive_switch_required = false;
5056 context = GNUNET_MQ_extract_nested_mh (msg);
5057
5058 /* create hashmap for message control */
5059 op->message_control_flow = GNUNET_CONTAINER_multihashmap_create (32,
5060 GNUNET_NO);
5061 op->inquiries_sent = GNUNET_CONTAINER_multihashmap_create (32,GNUNET_NO);
5062
5063#if MEASURE_PERFORMANCE
5064 /* load config */
5065 load_config (op);
5066#endif
5067
5068 /* Advance generation values, so that
5069 mutations won't interfere with the running operation. */
5070 op->set = set;
5071 op->generation_created = set->current_generation;
5072 advance_generation (set);
5073 GNUNET_CONTAINER_DLL_insert (set->ops_head,
5074 set->ops_tail,
5075 op);
5076 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5077 "Creating new CADET channel to port %s for set union\n",
5078 GNUNET_h2s (&msg->app_id));
5079 op->channel = GNUNET_CADET_channel_create (cadet,
5080 op,
5081 &msg->target_peer,
5082 &msg->app_id,
5083 &channel_window_cb,
5084 &channel_end_cb,
5085 cadet_handlers);
5086 op->mq = GNUNET_CADET_get_mq (op->channel);
5087 {
5088 struct GNUNET_MQ_Envelope *ev;
5089 struct OperationRequestMessage *msg;
5090
5091#if MEASURE_PERFORMANCE
5092 perf_store.operation_request.sent += 1;
5093#endif
5094 ev = GNUNET_MQ_msg_nested_mh (msg,
5095 GNUNET_MESSAGE_TYPE_SETU_P2P_OPERATION_REQUEST,
5096 context);
5097 if (NULL == ev)
5098 {
5099 /* the context message is too large */
5100 GNUNET_break (0);
5101 GNUNET_SERVICE_client_drop (cs->client);
5102 return;
5103 }
5104 op->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
5105 GNUNET_NO);
5106 /* copy the current generation's strata estimator for this operation */
5107 op->se = strata_estimator_dup (op->set->se);
5108 /* we started the operation, thus we have to send the operation request */
5109 op->phase = PHASE_EXPECT_SE;
5110
5111 op->salt_receive = (op->peer_site + 1) % 2;
5112 op->salt_send = op->peer_site; // FIXME?????
5113
5114
5115 LOG (GNUNET_ERROR_TYPE_DEBUG,
5116 "Initiating union operation evaluation\n");
5117 GNUNET_STATISTICS_update (_GSS_statistics,
5118 "# of total union operations",
5119 1,
5120 GNUNET_NO);
5121 GNUNET_STATISTICS_update (_GSS_statistics,
5122 "# of initiated union operations",
5123 1,
5124 GNUNET_NO);
5125 GNUNET_MQ_send (op->mq,
5126 ev);
5127 if (NULL != context)
5128 LOG (GNUNET_ERROR_TYPE_DEBUG,
5129 "sent op request with context message\n");
5130 else
5131 LOG (GNUNET_ERROR_TYPE_DEBUG,
5132 "sent op request without context message\n");
5133 initialize_key_to_element (op);
5134 op->initial_size = GNUNET_CONTAINER_multihashmap32_size (
5135 op->key_to_element);
5136
5137 }
5138 GNUNET_SERVICE_client_continue (cs->client);
5139}
5140
5141
5142/**
5143 * Handle a request from the client to cancel a running set operation.
5144 *
5145 * @param cls the client
5146 * @param msg the message
5147 */
5148static void
5149handle_client_cancel (void *cls,
5150 const struct GNUNET_SETU_CancelMessage *msg)
5151{
5152 struct ClientState *cs = cls;
5153 struct Set *set;
5154 struct Operation *op;
5155 int found;
5156
5157 if (NULL == (set = cs->set))
5158 {
5159 /* client without a set requested an operation */
5160 GNUNET_break (0);
5161 GNUNET_SERVICE_client_drop (cs->client);
5162 return;
5163 }
5164 found = GNUNET_NO;
5165 for (op = set->ops_head; NULL != op; op = op->next)
5166 {
5167 if (op->client_request_id == ntohl (msg->request_id))
5168 {
5169 found = GNUNET_YES;
5170 break;
5171 }
5172 }
5173 if (GNUNET_NO == found)
5174 {
5175 /* It may happen that the operation was already destroyed due to
5176 * the other peer disconnecting. The client may not know about this
5177 * yet and try to cancel the (just barely non-existent) operation.
5178 * So this is not a hard error.
5179 *///
5180 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
5181 "Client canceled non-existent op %u\n",
5182 (uint32_t) ntohl (msg->request_id));
5183 }
5184 else
5185 {
5186 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5187 "Client requested cancel for op %u\n",
5188 (uint32_t) ntohl (msg->request_id));
5189 _GSS_operation_destroy (op);
5190 }
5191 GNUNET_SERVICE_client_continue (cs->client);
5192}
5193
5194
5195/**
5196 * Handle a request from the client to accept a set operation that
5197 * came from a remote peer. We forward the accept to the associated
5198 * operation for handling
5199 *
5200 * @param cls the client
5201 * @param msg the message
5202 */
5203static void
5204handle_client_accept (void *cls,
5205 const struct GNUNET_SETU_AcceptMessage *msg)
5206{
5207 struct ClientState *cs = cls;
5208 struct Set *set;
5209 struct Operation *op;
5210 struct GNUNET_SETU_ResultMessage *result_message;
5211 struct GNUNET_MQ_Envelope *ev;
5212 struct Listener *listener;
5213
5214 if (NULL == (set = cs->set))
5215 {
5216 /* client without a set requested to accept */
5217 GNUNET_break (0);
5218 GNUNET_SERVICE_client_drop (cs->client);
5219 return;
5220 }
5221 op = get_incoming (ntohl (msg->accept_reject_id));
5222 if (NULL == op)
5223 {
5224 /* It is not an error if the set op does not exist -- it may
5225 * have been destroyed when the partner peer disconnected. */
5226 GNUNET_log (
5227 GNUNET_ERROR_TYPE_INFO,
5228 "Client %p accepted request %u of listener %p that is no longer active\n",
5229 cs,
5230 ntohl (msg->accept_reject_id),
5231 cs->listener);
5232 ev = GNUNET_MQ_msg (result_message,
5233 GNUNET_MESSAGE_TYPE_SETU_RESULT);
5234 result_message->request_id = msg->request_id;
5235 result_message->result_status = htons (GNUNET_SETU_STATUS_FAILURE);
5236 GNUNET_MQ_send (set->cs->mq, ev);
5237 GNUNET_SERVICE_client_continue (cs->client);
5238 return;
5239 }
5240 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5241 "Client accepting request %u\n",
5242 (uint32_t) ntohl (msg->accept_reject_id));
5243 listener = op->listener;
5244 op->listener = NULL;
5245 GNUNET_CONTAINER_DLL_remove (listener->op_head,
5246 listener->op_tail,
5247 op);
5248 op->set = set;
5249 GNUNET_CONTAINER_DLL_insert (set->ops_head,
5250 set->ops_tail,
5251 op);
5252 op->client_request_id = ntohl (msg->request_id);
5253 op->byzantine = msg->byzantine;
5254 op->byzantine_lower_bound = ntohl (msg->byzantine_lower_bound);
5255 op->force_full = msg->force_full;
5256 op->force_delta = msg->force_delta;
5257 op->symmetric = msg->symmetric;
5258 op->rtt_bandwidth_tradeoff = msg->bandwidth_latency_tradeoff;
5259 op->ibf_bucket_number_factor = msg->ibf_bucket_number_factor;
5260 op->ibf_number_buckets_per_element = msg->ibf_number_of_buckets_per_element;
5261 op->byzantine_upper_bound = msg->byzantine_upper_bond;
5262 op->active_passive_switch_required = false;
5263 /* create hashmap for message control */
5264 op->message_control_flow = GNUNET_CONTAINER_multihashmap_create (32,
5265 GNUNET_NO);
5266 op->inquiries_sent = GNUNET_CONTAINER_multihashmap_create (32,GNUNET_NO);
5267
5268#if MEASURE_PERFORMANCE
5269 /* load config */
5270 load_config (op);
5271#endif
5272
5273 /* Advance generation values, so that future mutations do not
5274 interfer with the running operation. */
5275 op->generation_created = set->current_generation;
5276 advance_generation (set);
5277 GNUNET_assert (NULL == op->se);
5278
5279 LOG (GNUNET_ERROR_TYPE_DEBUG,
5280 "accepting set union operation\n");
5281 GNUNET_STATISTICS_update (_GSS_statistics,
5282 "# of accepted union operations",
5283 1,
5284 GNUNET_NO);
5285 GNUNET_STATISTICS_update (_GSS_statistics,
5286 "# of total union operations",
5287 1,
5288 GNUNET_NO);
5289 {
5290 struct MultiStrataEstimator *se;
5291 struct GNUNET_MQ_Envelope *ev;
5292 struct StrataEstimatorMessage *strata_msg;
5293 char *buf;
5294 size_t len;
5295 uint16_t type;
5296
5297 op->se = strata_estimator_dup (op->set->se);
5298 op->demanded_hashes = GNUNET_CONTAINER_multihashmap_create (32,
5299 GNUNET_NO);
5300 op->salt_receive = (op->peer_site + 1) % 2;
5301 op->salt_send = op->peer_site; // FIXME?????
5302 initialize_key_to_element (op);
5303 op->initial_size = GNUNET_CONTAINER_multihashmap32_size (
5304 op->key_to_element);
5305
5306 /* kick off the operation */
5307 se = op->se;
5308
5309 uint8_t se_count = 1;
5310 if (op->initial_size > 0)
5311 {
5312 op->total_elements_size_local = 0;
5313 GNUNET_CONTAINER_multihashmap_iterate (op->set->content->elements,
5314 &
5315 determinate_avg_element_size_iterator,
5316 op);
5317 se_count = determine_strata_count (
5318 op->total_elements_size_local / op->initial_size,
5319 op->initial_size);
5320 }
5321 buf = GNUNET_malloc (se->stratas[0]->strata_count * IBF_BUCKET_SIZE
5322 * ((SE_IBFS_TOTAL_SIZE / 8) * se_count));
5323 len = strata_estimator_write (se,
5324 SE_IBFS_TOTAL_SIZE,
5325 se_count,
5326 buf);
5327#if MEASURE_PERFORMANCE
5328 perf_store.se.sent += 1;
5329 perf_store.se.sent_var_bytes += len;
5330#endif
5331
5332 if (len < se->stratas[0]->strata_count * IBF_BUCKET_SIZE
5333 * SE_IBFS_TOTAL_SIZE)
5334 type = GNUNET_MESSAGE_TYPE_SETU_P2P_SEC;
5335 else
5336 type = GNUNET_MESSAGE_TYPE_SETU_P2P_SE;
5337 ev = GNUNET_MQ_msg_extra (strata_msg,
5338 len,
5339 type);
5340 GNUNET_memcpy (&strata_msg[1],
5341 buf,
5342 len);
5343 GNUNET_free (buf);
5344 strata_msg->set_size
5345 = GNUNET_htonll (GNUNET_CONTAINER_multihashmap_size (
5346 op->set->content->elements));
5347 strata_msg->se_count = se_count;
5348 GNUNET_MQ_send (op->mq,
5349 ev);
5350 op->phase = PHASE_EXPECT_IBF;
5351 }
5352 /* Now allow CADET to continue, as we did not do this in
5353 #handle_incoming_msg (as we wanted to first see if the
5354 local client would accept the request). */
5355 GNUNET_CADET_receive_done (op->channel);
5356 GNUNET_SERVICE_client_continue (cs->client);
5357}
5358
5359
5360/**
5361 * Called to clean up, after a shutdown has been requested.
5362 *
5363 * @param cls closure, NULL
5364 */
5365static void
5366shutdown_task (void *cls)
5367{
5368 /* Delay actual shutdown to allow service to disconnect clients */
5369 in_shutdown = GNUNET_YES;
5370 if (0 == num_clients)
5371 {
5372 if (NULL != cadet)
5373 {
5374 GNUNET_CADET_disconnect (cadet);
5375 cadet = NULL;
5376 }
5377 }
5378 GNUNET_STATISTICS_destroy (_GSS_statistics,
5379 GNUNET_YES);
5380 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
5381 "handled shutdown request\n");
5382#if MEASURE_PERFORMANCE
5383 calculate_perf_store ();
5384#endif
5385}
5386
5387
5388/**
5389 * Function called by the service's run
5390 * method to run service-specific setup code.
5391 *
5392 * @param cls closure
5393 * @param cfg configuration to use
5394 * @param service the initialized service
5395 */
5396static void
5397run (void *cls,
5398 const struct GNUNET_CONFIGURATION_Handle *cfg,
5399 struct GNUNET_SERVICE_Handle *service)
5400{
5401 /* FIXME: need to modify SERVICE (!) API to allow
5402 us to run a shutdown task *after* clients were
5403 forcefully disconnected! */
5404 GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
5405 NULL);
5406 _GSS_statistics = GNUNET_STATISTICS_create ("setu",
5407 cfg);
5408 cadet = GNUNET_CADET_connect (cfg);
5409 if (NULL == cadet)
5410 {
5411 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
5412 _ ("Could not connect to CADET service\n"));
5413 GNUNET_SCHEDULER_shutdown ();
5414 return;
5415 }
5416}
5417
5418
5419/**
5420 * Define "main" method using service macro.
5421 */
5422GNUNET_SERVICE_MAIN (
5423 "set",
5424 GNUNET_SERVICE_OPTION_NONE,
5425 &run,
5426 &client_connect_cb,
5427 &client_disconnect_cb,
5428 NULL,
5429 GNUNET_MQ_hd_fixed_size (client_accept,
5430 GNUNET_MESSAGE_TYPE_SETU_ACCEPT,
5431 struct GNUNET_SETU_AcceptMessage,
5432 NULL),
5433 GNUNET_MQ_hd_var_size (client_set_add,
5434 GNUNET_MESSAGE_TYPE_SETU_ADD,
5435 struct GNUNET_SETU_ElementMessage,
5436 NULL),
5437 GNUNET_MQ_hd_fixed_size (client_create_set,
5438 GNUNET_MESSAGE_TYPE_SETU_CREATE,
5439 struct GNUNET_SETU_CreateMessage,
5440 NULL),
5441 GNUNET_MQ_hd_var_size (client_evaluate,
5442 GNUNET_MESSAGE_TYPE_SETU_EVALUATE,
5443 struct GNUNET_SETU_EvaluateMessage,
5444 NULL),
5445 GNUNET_MQ_hd_fixed_size (client_listen,
5446 GNUNET_MESSAGE_TYPE_SETU_LISTEN,
5447 struct GNUNET_SETU_ListenMessage,
5448 NULL),
5449 GNUNET_MQ_hd_fixed_size (client_reject,
5450 GNUNET_MESSAGE_TYPE_SETU_REJECT,
5451 struct GNUNET_SETU_RejectMessage,
5452 NULL),
5453 GNUNET_MQ_hd_fixed_size (client_cancel,
5454 GNUNET_MESSAGE_TYPE_SETU_CANCEL,
5455 struct GNUNET_SETU_CancelMessage,
5456 NULL),
5457 GNUNET_MQ_handler_end ());
5458
5459
5460/* end of gnunet-service-setu.c */
diff --git a/src/setu/gnunet-service-setu_protocol.h b/src/setu/gnunet-service-setu_protocol.h
deleted file mode 100644
index c896166ce..000000000
--- a/src/setu/gnunet-service-setu_protocol.h
+++ /dev/null
@@ -1,256 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2013, 2014 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @author Florian Dold
22 * @author Christian Grothoff
23 * @file set/gnunet-service-set_protocol.h
24 * @brief Peer-to-Peer messages for gnunet set
25 */
26#ifndef SET_PROTOCOL_H
27#define SET_PROTOCOL_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35struct OperationRequestMessage
36{
37 /**
38 * Type: #GNUNET_MESSAGE_TYPE_SET_P2P_OPERATION_REQUEST
39 */
40 struct GNUNET_MessageHeader header;
41
42 /**
43 * For Intersection: my element count
44 */
45 uint32_t element_count GNUNET_PACKED;
46
47 /**
48 * Application-specific identifier of the request.
49 */
50 struct GNUNET_HashCode app_idX;
51
52 /* rest: optional message */
53};
54
55
56/**
57 * Message containing buckets of an invertible bloom filter.
58 *
59 * If an IBF has too many buckets for an IBF message,
60 * it is split into multiple messages.
61 */
62struct IBFMessage
63{
64 /**
65 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
66 */
67 struct GNUNET_MessageHeader header;
68
69 /**
70 * Size of the whole ibf (number of buckets)
71 */
72 uint32_t ibf_size;
73
74 /**
75 * Offset of the strata in the rest of the message
76 */
77 uint32_t offset GNUNET_PACKED;
78
79 /**
80 * Salt used when hashing elements for this IBF.
81 */
82 uint16_t salt GNUNET_PACKED;
83
84 /**
85 * The bit length of the counter
86 */
87 uint16_t ibf_counter_bit_length;
88 /* rest: buckets */
89};
90
91
92/**
93estimate_best_mode_of_operation (uint64_t avg_element_size,
94uint64_t local_set_size,
95 uint64_t remote_set_size,
96uint64_t est_set_diff_remote,
97 uint64_t est_set_diff_local,)
98 **/
99
100
101struct InquiryMessage
102{
103 /**
104 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_IBF
105 */
106 struct GNUNET_MessageHeader header;
107
108 /**
109 * Salt used when hashing elements for this inquiry.
110 */
111 uint32_t salt GNUNET_PACKED;
112
113 /* rest: inquiry IBF keys */
114};
115
116
117/**
118 * During intersection, the first (and possibly second) message
119 * send it the number of elements in the set, to allow the peers
120 * to decide who should start with the Bloom filter.
121 */
122struct IntersectionElementInfoMessage
123{
124 /**
125 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_ELEMENT_INFO
126 */
127 struct GNUNET_MessageHeader header;
128
129 /**
130 * mutator used with this bloomfilter.
131 */
132 uint32_t sender_element_count GNUNET_PACKED;
133};
134
135
136/**
137 * Bloom filter messages exchanged for set intersection calculation.
138 */
139struct BFMessage
140{
141 /**
142 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_BF
143 */
144 struct GNUNET_MessageHeader header;
145
146 /**
147 * Number of elements the sender still has in the set.
148 */
149 uint32_t sender_element_count GNUNET_PACKED;
150
151 /**
152 * XOR of all hashes over all elements remaining in the set.
153 * Used to determine termination.
154 */
155 struct GNUNET_HashCode element_xor_hash;
156
157 /**
158 * Mutator used with this bloomfilter.
159 */
160 uint32_t sender_mutator GNUNET_PACKED;
161
162 /**
163 * Total length of the bloomfilter data.
164 */
165 uint32_t bloomfilter_total_length GNUNET_PACKED;
166
167 /**
168 * Number of bits (k-value) used in encoding the bloomfilter.
169 */
170 uint32_t bits_per_element GNUNET_PACKED;
171
172 /**
173 * rest: the sender's bloomfilter
174 */
175};
176
177
178/**
179 * Last message, send to confirm the final set. Contains the element
180 * count as it is possible that the peer determined that we were done
181 * by getting the empty set, which in that case also needs to be
182 * communicated.
183 */
184struct IntersectionDoneMessage
185{
186 /**
187 * Type: #GNUNET_MESSAGE_TYPE_SET_INTERSECTION_P2P_DONE
188 */
189 struct GNUNET_MessageHeader header;
190
191 /**
192 * Final number of elements in intersection.
193 */
194 uint32_t final_element_count GNUNET_PACKED;
195
196 /**
197 * XOR of all hashes over all elements remaining in the set.
198 */
199 struct GNUNET_HashCode element_xor_hash;
200};
201
202
203/**
204 * Strata estimator together with the peer's overall set size.
205 */
206struct StrataEstimatorMessage
207{
208 /**
209 * Type: #GNUNET_MESSAGE_TYPE_SET_UNION_P2P_SE(C)
210 */
211 struct GNUNET_MessageHeader header;
212
213 /**
214 * The number of ses transmitted
215 */
216 uint8_t se_count;
217
218 /**
219 * Size of the local set
220 */
221 uint64_t set_size GNUNET_PACKED;
222};
223
224
225/**
226 * Message which signals to other peer that we are sending full set
227 *
228 */
229struct TransmitFullMessage
230{
231 /**
232 * Type: #GNUNET_MESSAGE_TYPE_SETU_P2P_SEND_FULL
233 */
234 struct GNUNET_MessageHeader header;
235
236 /**
237 * Remote set difference calculated with strata estimator
238 */
239 uint32_t remote_set_difference;
240
241 /**
242 * Total remote set size
243 */
244 uint32_t remote_set_size;
245
246 /**
247 * Local set difference calculated with strata estimator
248 */
249 uint32_t local_set_difference;
250
251};
252
253
254GNUNET_NETWORK_STRUCT_END
255
256#endif
diff --git a/src/setu/gnunet-service-setu_strata_estimator.c b/src/setu/gnunet-service-setu_strata_estimator.c
deleted file mode 100644
index 43ccf3afd..000000000
--- a/src/setu/gnunet-service-setu_strata_estimator.c
+++ /dev/null
@@ -1,468 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/gnunet-service-setu_strata_estimator.c
22 * @brief invertible bloom filter
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "ibf.h"
30#include "gnunet-service-setu_strata_estimator.h"
31
32
33/**
34 * Should we try compressing the strata estimator? This will
35 * break compatibility with the 0.10.1-network.
36 */
37#define FAIL_10_1_COMPATIBILTIY 1
38
39/**
40 * Number of strata estimators in memory NOT transmitted
41 */
42
43#define MULTI_SE_BASE_COUNT 8
44
45/**
46 * The avg size of 1 se
47 * Based on the bsc thesis of Elias Summermatter (2021)
48 */
49
50#define AVG_BYTE_SIZE_SE 4221
51
52/**
53 * Calculates the optimal number of strata Estimators to send
54 * @param avg_element_size
55 * @param element_count
56 * @return
57 */
58uint8_t
59determine_strata_count (uint64_t avg_element_size, uint64_t element_count)
60{
61 uint64_t base_size = avg_element_size * element_count;
62 /* >67kb total size of elements in set */
63 if (base_size < AVG_BYTE_SIZE_SE * 16)
64 return 1;
65 /* >270kb total size of elements in set */
66 if (base_size < AVG_BYTE_SIZE_SE * 64)
67 return 2;
68 /* >1mb total size of elements in set */
69 if (base_size < AVG_BYTE_SIZE_SE * 256)
70 return 4;
71 return 8;
72}
73
74
75/**
76 * Modify an IBF key @a k_in based on the @a salt, returning a
77 * salted key in @a k_out.
78 */
79static void
80salt_key (const struct IBF_Key *k_in,
81 uint32_t salt,
82 struct IBF_Key *k_out)
83{
84 int s = (salt * 7) % 64;
85 uint64_t x = k_in->key_val;
86
87 /* rotate ibf key */
88 if (s > 0)
89 x = (x >> s) | (x << (64 - s));
90 k_out->key_val = x;
91}
92
93
94/**
95 * Reverse modification done in the salt_key function
96 */
97static void
98unsalt_key (const struct IBF_Key *k_in,
99 uint32_t salt,
100 struct IBF_Key *k_out)
101{
102 int s = (salt * 7) % 64;
103 uint64_t x = k_in->key_val;
104
105 x = (x << s) | (x >> (64 - s));
106 k_out->key_val = x;
107}
108
109
110/**
111 * Write the given strata estimator to the buffer.
112 *
113 * @param se strata estimator to serialize
114 * @param[out] buf buffer to write to, must be of appropriate size
115 * @return number of bytes written to @a buf
116 */
117size_t
118strata_estimator_write (struct MultiStrataEstimator *se,
119 uint16_t se_ibf_total_size,
120 uint8_t number_se_send,
121 void *buf)
122{
123 char *sbuf = buf;
124 unsigned int i;
125 size_t osize;
126 uint64_t sbuf_offset = 0;
127 se->size = number_se_send;
128
129 GNUNET_assert (NULL != se);
130 for (uint8_t strata_ctr = 0; strata_ctr < number_se_send; strata_ctr++)
131 {
132 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
133 {
134 ibf_write_slice (se->stratas[strata_ctr]->strata[i],
135 0,
136 se->stratas[strata_ctr]->ibf_size,
137 &sbuf[sbuf_offset],
138 8);
139 sbuf_offset += se->stratas[strata_ctr]->ibf_size * IBF_BUCKET_SIZE;
140 }
141 }
142 osize = ((se_ibf_total_size / 8) * number_se_send) * IBF_BUCKET_SIZE
143 * se->stratas[0]->strata_count;
144#if FAIL_10_1_COMPATIBILTIY
145 {
146 char *cbuf;
147 size_t nsize;
148
149 if (GNUNET_YES ==
150 GNUNET_try_compression (buf,
151 osize,
152 &cbuf,
153 &nsize))
154 {
155 GNUNET_memcpy (buf, cbuf, nsize);
156 osize = nsize;
157 GNUNET_free (cbuf);
158 }
159 }
160#endif
161 return osize;
162}
163
164
165/**
166 * Read strata from the buffer into the given strata
167 * estimator. The strata estimator must already be allocated.
168 *
169 * @param buf buffer to read from
170 * @param buf_len number of bytes in @a buf
171 * @param is_compressed is the data compressed?
172 * @param[out] se strata estimator to write to
173 * @return #GNUNET_OK on success
174 */
175int
176strata_estimator_read (const void *buf,
177 size_t buf_len,
178 int is_compressed,
179 uint8_t number_se_received,
180 uint16_t se_ibf_total_size,
181 struct MultiStrataEstimator *se)
182{
183 unsigned int i;
184 size_t osize;
185 char *dbuf;
186
187 dbuf = NULL;
188 if (GNUNET_YES == is_compressed)
189 {
190 osize = ((se_ibf_total_size / 8) * number_se_received) * IBF_BUCKET_SIZE
191 * se->stratas[0]->strata_count;
192 dbuf = GNUNET_decompress (buf,
193 buf_len,
194 osize);
195 if (NULL == dbuf)
196 {
197 GNUNET_break_op (0); /* bad compressed input data */
198 return GNUNET_SYSERR;
199 }
200 buf = dbuf;
201 buf_len = osize;
202 }
203
204 if (buf_len != se->stratas[0]->strata_count * ((se_ibf_total_size / 8)
205 * number_se_received)
206 * IBF_BUCKET_SIZE)
207 {
208 GNUNET_break (0); /* very odd error */
209 GNUNET_free (dbuf);
210 return GNUNET_SYSERR;
211 }
212
213 for (uint8_t strata_ctr = 0; strata_ctr < number_se_received; strata_ctr++)
214 {
215 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
216 {
217 ibf_read_slice (buf, 0, se->stratas[strata_ctr]->ibf_size,
218 se->stratas[strata_ctr]->strata[i], 8);
219 buf += se->stratas[strata_ctr]->ibf_size * IBF_BUCKET_SIZE;
220 }
221 }
222 se->size = number_se_received;
223 GNUNET_free (dbuf);
224 return GNUNET_OK;
225}
226
227
228/**
229 * Add a key to the strata estimator.
230 *
231 * @param se strata estimator to add the key to
232 * @param key key to add
233 */
234void
235strata_estimator_insert (struct MultiStrataEstimator *se,
236 struct IBF_Key key)
237{
238
239
240 /* count trailing '1'-bits of v */
241 for (int strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
242 {
243 unsigned int i;
244 uint64_t v;
245
246 struct IBF_Key salted_key;
247 salt_key (&key,
248 strata_ctr * (64 / MULTI_SE_BASE_COUNT),
249 &salted_key);
250 v = salted_key.key_val;
251 for (i = 0; v & 1; v >>= 1, i++)
252 {
253 ibf_insert (se->stratas[strata_ctr]->strata[i], salted_key);
254 }
255 }
256 /* empty */;
257
258}
259
260
261/**
262 * Remove a key from the strata estimator. (NOT USED)
263 *
264 * @param se strata estimator to remove the key from
265 * @param key key to remove
266 */
267void
268strata_estimator_remove (struct MultiStrataEstimator *se,
269 struct IBF_Key key)
270{
271
272 /* count trailing '1'-bits of v */
273 for (int strata_ctr = 0; strata_ctr < se->size; strata_ctr++)
274 {
275 uint64_t v;
276 unsigned int i;
277
278 struct IBF_Key unsalted_key;
279 unsalt_key (&key,
280 strata_ctr * (64 / MULTI_SE_BASE_COUNT),
281 &unsalted_key);
282
283 v = unsalted_key.key_val;
284 for (i = 0; v & 1; v >>= 1, i++)
285 {
286 /* empty */;
287 ibf_remove (se->stratas[strata_ctr]->strata[i], unsalted_key);
288 }
289 }
290}
291
292
293/**
294 * Create a new strata estimator with the given parameters.
295 *
296 * @param strata_count number of stratas, that is, number of ibfs in the estimator
297 * @param ibf_size size of each ibf stratum
298 * @param ibf_hashnum hashnum parameter of each ibf
299 * @return a freshly allocated, empty strata estimator, NULL on error
300 */
301struct MultiStrataEstimator *
302strata_estimator_create (unsigned int strata_count,
303 uint32_t ibf_size,
304 uint8_t ibf_hashnum)
305{
306 struct MultiStrataEstimator *se;
307 unsigned int i;
308 unsigned int j;
309 se = GNUNET_new (struct MultiStrataEstimator);
310
311 se->size = MULTI_SE_BASE_COUNT;
312 se->stratas = GNUNET_new_array (MULTI_SE_BASE_COUNT,struct StrataEstimator *);
313
314 uint8_t ibf_prime_sizes[] = {79,79,79,79,79,79,79,79};
315
316 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
317 {
318 se->stratas[strata_ctr] = GNUNET_new (struct StrataEstimator);
319 se->stratas[strata_ctr]->strata_count = strata_count;
320 se->stratas[strata_ctr]->ibf_size = ibf_prime_sizes[strata_ctr];
321 se->stratas[strata_ctr]->strata = GNUNET_new_array (strata_count * 4,
322 struct
323 InvertibleBloomFilter *);
324 for (i = 0; i < strata_count; i++)
325 {
326 se->stratas[strata_ctr]->strata[i] = ibf_create (
327 ibf_prime_sizes[strata_ctr], ibf_hashnum);
328 if (NULL == se->stratas[strata_ctr]->strata[i])
329 {
330 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
331 "Failed to allocate memory for strata estimator\n");
332 for (j = 0; j < i; j++)
333 ibf_destroy (se->stratas[strata_ctr]->strata[i]);
334 GNUNET_free (se);
335 return NULL;
336 }
337 }
338 }
339 return se;
340}
341
342
343/**
344 * Estimate set difference with two strata estimators,
345 * i.e. arrays of IBFs.
346 * Does not not modify its arguments.
347 *
348 * @param se1 first strata estimator
349 * @param se2 second strata estimator
350 * @return the estimated difference
351 */
352void
353strata_estimator_difference (const struct MultiStrataEstimator *se1,
354 const struct MultiStrataEstimator *se2)
355{
356 int avg_local_diff = 0;
357 int avg_remote_diff = 0;
358 uint8_t number_of_estimators = se1->size;
359
360 for (uint8_t strata_ctr = 0; strata_ctr < number_of_estimators; strata_ctr++)
361 {
362 GNUNET_assert (se1->stratas[strata_ctr]->strata_count ==
363 se2->stratas[strata_ctr]->strata_count);
364
365
366 for (int i = se1->stratas[strata_ctr]->strata_count - 1; i >= 0; i--)
367 {
368 struct InvertibleBloomFilter *diff;
369 /* number of keys decoded from the ibf */
370
371 /* FIXME: implement this without always allocating new IBFs */
372 diff = ibf_dup (se1->stratas[strata_ctr]->strata[i]);
373 diff->local_decoded_count = 0;
374 diff->remote_decoded_count = 0;
375
376 ibf_subtract (diff, se2->stratas[strata_ctr]->strata[i]);
377
378 for (int ibf_count = 0; GNUNET_YES; ibf_count++)
379 {
380 int more;
381
382 more = ibf_decode (diff, NULL, NULL);
383 if (GNUNET_NO == more)
384 {
385 se1->stratas[strata_ctr]->strata[0]->local_decoded_count +=
386 diff->local_decoded_count;
387 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count +=
388 diff->remote_decoded_count;
389 break;
390 }
391 /* Estimate if decoding fails or would not terminate */
392 if ((GNUNET_SYSERR == more) || (ibf_count > diff->size))
393 {
394 se1->stratas[strata_ctr]->strata[0]->local_decoded_count =
395 se1->stratas[strata_ctr]->strata[0]->local_decoded_count * (1 << (i
396 +
397 1));
398 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count =
399 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count * (1 << (i
400 +
401 1));
402 ibf_destroy (diff);
403 goto break_all_counting_loops;
404 }
405 }
406 ibf_destroy (diff);
407 }
408break_all_counting_loops:;
409 avg_local_diff += se1->stratas[strata_ctr]->strata[0]->local_decoded_count;
410 avg_remote_diff +=
411 se1->stratas[strata_ctr]->strata[0]->remote_decoded_count;
412 }
413 se1->stratas[0]->strata[0]->local_decoded_count = avg_local_diff
414 / number_of_estimators;
415 se1->stratas[0]->strata[0]->remote_decoded_count = avg_remote_diff
416 / number_of_estimators;
417}
418
419
420/**
421 * Make a copy of a strata estimator.
422 *
423 * @param se the strata estimator to copy
424 * @return the copy
425 */
426struct MultiStrataEstimator *
427strata_estimator_dup (struct MultiStrataEstimator *se)
428{
429 struct MultiStrataEstimator *c;
430 unsigned int i;
431
432 c = GNUNET_new (struct MultiStrataEstimator);
433 c->stratas = GNUNET_new_array (MULTI_SE_BASE_COUNT,struct StrataEstimator *);
434 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
435 {
436 c->stratas[strata_ctr] = GNUNET_new (struct StrataEstimator);
437 c->stratas[strata_ctr]->strata_count =
438 se->stratas[strata_ctr]->strata_count;
439 c->stratas[strata_ctr]->ibf_size = se->stratas[strata_ctr]->ibf_size;
440 c->stratas[strata_ctr]->strata = GNUNET_new_array (
441 se->stratas[strata_ctr]->strata_count,
442 struct
443 InvertibleBloomFilter *);
444 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
445 c->stratas[strata_ctr]->strata[i] = ibf_dup (
446 se->stratas[strata_ctr]->strata[i]);
447 }
448 return c;
449}
450
451
452/**
453 * Destroy a strata estimator, free all of its resources.
454 *
455 * @param se strata estimator to destroy.
456 */
457void
458strata_estimator_destroy (struct MultiStrataEstimator *se)
459{
460 unsigned int i;
461 for (uint8_t strata_ctr = 0; strata_ctr < MULTI_SE_BASE_COUNT; strata_ctr++)
462 {
463 for (i = 0; i < se->stratas[strata_ctr]->strata_count; i++)
464 ibf_destroy (se->stratas[strata_ctr]->strata[i]);
465 GNUNET_free (se->stratas[strata_ctr]->strata);
466 }
467 GNUNET_free (se);
468}
diff --git a/src/setu/gnunet-service-setu_strata_estimator.h b/src/setu/gnunet-service-setu_strata_estimator.h
deleted file mode 100644
index 4871a7fcd..000000000
--- a/src/setu/gnunet-service-setu_strata_estimator.h
+++ /dev/null
@@ -1,199 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-service-setu_strata_estimator.h
23 * @brief estimator of set difference
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#ifndef GNUNET_SERVICE_SETU_STRATA_ESTIMATOR_H
29#define GNUNET_SERVICE_SETU_STRATA_ESTIMATOR_H
30
31#include "platform.h"
32#include "gnunet_common.h"
33#include "gnunet_util_lib.h"
34
35#ifdef __cplusplus
36extern "C"
37{
38#if 0 /* keep Emacsens' auto-indent happy */
39}
40#endif
41#endif
42
43
44/**
45 * A handle to a strata estimator.
46 */
47struct StrataEstimator
48{
49 /**
50 * The IBFs of this strata estimator.
51 */
52 struct InvertibleBloomFilter **strata;
53
54 /**
55 * Size of the IBF array in @e strata
56 */
57 unsigned int strata_count;
58
59 /**
60 * Size of each IBF stratum (in bytes)
61 */
62 unsigned int ibf_size;
63};
64
65struct MultiStrataEstimator
66{
67 /**
68 * Array of strata estimators
69 */
70 struct StrataEstimator **stratas;
71
72 /**
73 * Number of strata estimators in struct
74 */
75 uint8_t size;
76
77};
78
79/**
80 * Deteminate how many strata estimators in the message are necessary
81 * @param avg_element_size
82 * @param element_count
83 * @return number of strata's
84 */
85
86uint8_t
87determine_strata_count (uint64_t avg_element_size,
88 uint64_t element_count);
89
90
91/**
92 * Write the given strata estimator to the buffer.
93 *
94 * @param se strata estimator to serialize
95 * @param[out] buf buffer to write to, must be of appropriate size
96 * @return number of bytes written to @a buf
97 */
98size_t
99strata_estimator_write (struct MultiStrataEstimator *se,
100 uint16_t se_ibf_total_size,
101 uint8_t number_se_send,
102 void *buf);
103
104
105/**
106 * Read strata from the buffer into the given strata
107 * estimator. The strata estimator must already be allocated.
108 *
109 * @param buf buffer to read from
110 * @param buf_len number of bytes in @a buf
111 * @param is_compressed is the data compressed?
112 * @param[out] se strata estimator to write to
113 * @return #GNUNET_OK on success
114 */
115int
116strata_estimator_read (const void *buf,
117 size_t buf_len,
118 int is_compressed,
119 uint8_t number_se_received,
120 uint16_t se_ibf_total_size,
121 struct MultiStrataEstimator *se);
122
123
124/**
125 * Create a new strata estimator with the given parameters.
126 *
127 * @param strata_count number of stratas, that is, number of ibfs in the estimator
128 * @param ibf_size size of each ibf stratum
129 * @param ibf_hashnum hashnum parameter of each ibf
130 * @return a freshly allocated, empty strata estimator, NULL on error
131 */
132struct MultiStrataEstimator *
133strata_estimator_create (unsigned int strata_count,
134 uint32_t ibf_size,
135 uint8_t ibf_hashnum);
136
137
138/**
139 * Get an estimation of the symmetric difference of the elements
140 * contained in both strata estimators.
141 *
142 * @param se1 first strata estimator
143 * @param se2 second strata estimator
144 * @return nothing
145 */
146void
147strata_estimator_difference (const struct MultiStrataEstimator *se1,
148 const struct MultiStrataEstimator *se2);
149
150
151/**
152 * Add a key to the strata estimator.
153 *
154 * @param se strata estimator to add the key to
155 * @param key key to add
156 */
157void
158strata_estimator_insert (struct MultiStrataEstimator *se,
159 struct IBF_Key key);
160
161
162/**
163 * Remove a key from the strata estimator.
164 *
165 * @param se strata estimator to remove the key from
166 * @param key key to remove
167 */
168void
169strata_estimator_remove (struct MultiStrataEstimator *se,
170 struct IBF_Key key);
171
172
173/**
174 * Destroy a strata estimator, free all of its resources.
175 *
176 * @param se strata estimator to destroy.
177 */
178void
179strata_estimator_destroy (struct MultiStrataEstimator *se);
180
181
182/**
183 * Make a copy of a strata estimator.
184 *
185 * @param se the strata estimator to copy
186 * @return the copy
187 */
188struct MultiStrataEstimator *
189strata_estimator_dup (struct MultiStrataEstimator *se);
190
191
192#if 0 /* keep Emacsens' auto-indent happy */
193{
194#endif
195#ifdef __cplusplus
196}
197#endif
198
199#endif
diff --git a/src/setu/gnunet-setu-ibf-profiler.c b/src/setu/gnunet-setu-ibf-profiler.c
deleted file mode 100644
index 6465b15b8..000000000
--- a/src/setu/gnunet-setu-ibf-profiler.c
+++ /dev/null
@@ -1,308 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/gnunet-set-ibf-profiler.c
23 * @brief tool for profiling the invertible bloom filter implementation
24 * @author Florian Dold
25 */
26
27#include "platform.h"
28#include "gnunet_util_lib.h"
29
30#include "ibf.h"
31
32static unsigned int asize = 10;
33static unsigned int bsize = 10;
34static unsigned int csize = 10;
35static unsigned int hash_num = 4;
36static unsigned int ibf_size = 80;
37
38/* FIXME: add parameter for this */
39static enum GNUNET_CRYPTO_Quality random_quality = GNUNET_CRYPTO_QUALITY_WEAK;
40
41static struct GNUNET_CONTAINER_MultiHashMap *set_a;
42static struct GNUNET_CONTAINER_MultiHashMap *set_b;
43/* common elements in a and b */
44static struct GNUNET_CONTAINER_MultiHashMap *set_c;
45
46static struct GNUNET_CONTAINER_MultiHashMap *key_to_hashcode;
47
48static struct InvertibleBloomFilter *ibf_a;
49static struct InvertibleBloomFilter *ibf_b;
50
51
52static void
53register_hashcode (struct GNUNET_HashCode *hash)
54{
55 struct GNUNET_HashCode replicated;
56 struct IBF_Key key;
57
58 key = ibf_key_from_hashcode (hash);
59 ibf_hashcode_from_key (key, &replicated);
60 (void) GNUNET_CONTAINER_multihashmap_put (
61 key_to_hashcode,
62 &replicated,
63 GNUNET_memdup (hash, sizeof *hash),
64 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
65}
66
67
68static void
69iter_hashcodes (struct IBF_Key key,
70 GNUNET_CONTAINER_MultiHashMapIteratorCallback iter,
71 void *cls)
72{
73 struct GNUNET_HashCode replicated;
74
75 ibf_hashcode_from_key (key, &replicated);
76 GNUNET_CONTAINER_multihashmap_get_multiple (key_to_hashcode,
77 &replicated,
78 iter,
79 cls);
80}
81
82
83static int
84insert_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
85{
86 struct InvertibleBloomFilter *ibf = cls;
87
88 ibf_insert (ibf, ibf_key_from_hashcode (key));
89 return GNUNET_YES;
90}
91
92
93static int
94remove_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
95{
96 struct GNUNET_CONTAINER_MultiHashMap *hashmap = cls;
97
98 /* if remove fails, there just was a collision with another key */
99 (void) GNUNET_CONTAINER_multihashmap_remove (hashmap, value, NULL);
100 return GNUNET_YES;
101}
102
103
104static void
105run (void *cls,
106 char *const *args,
107 const char *cfgfile,
108 const struct GNUNET_CONFIGURATION_Handle *cfg)
109{
110 struct GNUNET_HashCode id;
111 struct IBF_Key ibf_key;
112 int i;
113 int side;
114 int res;
115 struct GNUNET_TIME_Absolute start_time;
116 struct GNUNET_TIME_Relative delta_time;
117
118 set_a =
119 GNUNET_CONTAINER_multihashmap_create (((asize == 0) ? 1 : (asize + csize)),
120 GNUNET_NO);
121 set_b =
122 GNUNET_CONTAINER_multihashmap_create (((bsize == 0) ? 1 : (bsize + csize)),
123 GNUNET_NO);
124 set_c = GNUNET_CONTAINER_multihashmap_create (((csize == 0) ? 1 : csize),
125 GNUNET_NO);
126
127 key_to_hashcode =
128 GNUNET_CONTAINER_multihashmap_create (((asize + bsize + csize == 0)
129 ? 1
130 : (asize + bsize + csize)),
131 GNUNET_NO);
132
133 printf ("hash-num=%u, size=%u, #(A-B)=%u, #(B-A)=%u, #(A&B)=%u\n",
134 hash_num,
135 ibf_size,
136 asize,
137 bsize,
138 csize);
139
140 i = 0;
141 while (i < asize)
142 {
143 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
144 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
145 continue;
146 GNUNET_break (GNUNET_OK ==
147 GNUNET_CONTAINER_multihashmap_put (
148 set_a,
149 &id,
150 NULL,
151 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
152 register_hashcode (&id);
153 i++;
154 }
155 i = 0;
156 while (i < bsize)
157 {
158 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
159 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
160 continue;
161 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
162 continue;
163 GNUNET_break (GNUNET_OK ==
164 GNUNET_CONTAINER_multihashmap_put (
165 set_b,
166 &id,
167 NULL,
168 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
169 register_hashcode (&id);
170 i++;
171 }
172 i = 0;
173 while (i < csize)
174 {
175 GNUNET_CRYPTO_hash_create_random (random_quality, &id);
176 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id))
177 continue;
178 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id))
179 continue;
180 if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_c, &id))
181 continue;
182 GNUNET_break (GNUNET_OK ==
183 GNUNET_CONTAINER_multihashmap_put (
184 set_c,
185 &id,
186 NULL,
187 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
188 register_hashcode (&id);
189 i++;
190 }
191
192 ibf_a = ibf_create (ibf_size, hash_num);
193 ibf_b = ibf_create (ibf_size, hash_num);
194 if ((NULL == ibf_a) || (NULL == ibf_b))
195 {
196 /* insufficient memory */
197 GNUNET_break (0);
198 GNUNET_SCHEDULER_shutdown ();
199 return;
200 }
201
202
203 printf ("generated sets\n");
204
205 start_time = GNUNET_TIME_absolute_get ();
206
207 GNUNET_CONTAINER_multihashmap_iterate (set_a, &insert_iterator, ibf_a);
208 GNUNET_CONTAINER_multihashmap_iterate (set_b, &insert_iterator, ibf_b);
209 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_a);
210 GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_b);
211
212 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
213
214 printf ("encoded in: %s\n",
215 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
216
217 ibf_subtract (ibf_a, ibf_b);
218
219
220 start_time = GNUNET_TIME_absolute_get ();
221
222 for (i = 0; i <= asize + bsize; i++)
223 {
224 res = ibf_decode (ibf_a, &side, &ibf_key);
225 if (GNUNET_SYSERR == res)
226 {
227 printf ("decode failed, %u/%u elements left\n",
228 GNUNET_CONTAINER_multihashmap_size (set_a)
229 + GNUNET_CONTAINER_multihashmap_size (set_b),
230 asize + bsize);
231 return;
232 }
233 if (GNUNET_NO == res)
234 {
235 if ((0 == GNUNET_CONTAINER_multihashmap_size (set_b)) &&
236 (0 == GNUNET_CONTAINER_multihashmap_size (set_a)))
237 {
238 delta_time = GNUNET_TIME_absolute_get_duration (start_time);
239 printf ("decoded successfully in: %s\n",
240 GNUNET_STRINGS_relative_time_to_string (delta_time, GNUNET_NO));
241 }
242 else
243 {
244 printf ("decode missed elements (should never happen)\n");
245 }
246 return;
247 }
248
249 if (side == 1)
250 iter_hashcodes (ibf_key, remove_iterator, set_a);
251 if (side == -1)
252 iter_hashcodes (ibf_key, remove_iterator, set_b);
253 }
254 printf ("cyclic IBF, %u/%u elements left\n",
255 GNUNET_CONTAINER_multihashmap_size (set_a)
256 + GNUNET_CONTAINER_multihashmap_size (set_b),
257 asize + bsize);
258}
259
260
261int
262main (int argc, char **argv)
263{
264 struct GNUNET_GETOPT_CommandLineOption options[] = {
265 GNUNET_GETOPT_option_uint ('A',
266 "asize",
267 NULL,
268 gettext_noop ("number of element in set A-B"),
269 &asize),
270
271 GNUNET_GETOPT_option_uint ('B',
272 "bsize",
273 NULL,
274 gettext_noop ("number of element in set B-A"),
275 &bsize),
276
277 GNUNET_GETOPT_option_uint ('C',
278 "csize",
279 NULL,
280 gettext_noop (
281 "number of common elements in A and B"),
282 &csize),
283
284 GNUNET_GETOPT_option_uint ('k',
285 "hash-num",
286 NULL,
287 gettext_noop ("hash num"),
288 &hash_num),
289
290 GNUNET_GETOPT_option_uint ('s',
291 "ibf-size",
292 NULL,
293 gettext_noop ("ibf size"),
294 &ibf_size),
295
296 GNUNET_GETOPT_OPTION_END
297 };
298
299 GNUNET_PROGRAM_run2 (argc,
300 argv,
301 "gnunet-consensus-ibf",
302 "help",
303 options,
304 &run,
305 NULL,
306 GNUNET_YES);
307 return 0;
308}
diff --git a/src/setu/gnunet-setu-profiler.c b/src/setu/gnunet-setu-profiler.c
deleted file mode 100644
index 8d6a2dc8c..000000000
--- a/src/setu/gnunet-setu-profiler.c
+++ /dev/null
@@ -1,499 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file setu/gnunet-setu-profiler.c
23 * @brief profiling tool for set
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_statistics_service.h"
29#include "gnunet_setu_service.h"
30#include "gnunet_testbed_service.h"
31
32
33static int ret;
34
35static unsigned int num_a = 5;
36static unsigned int num_b = 5;
37static unsigned int num_c = 20;
38
39static char *op_str = "union";
40
41const static struct GNUNET_CONFIGURATION_Handle *config;
42
43struct SetInfo
44{
45 char *id;
46 struct GNUNET_SETU_Handle *set;
47 struct GNUNET_SETU_OperationHandle *oh;
48 struct GNUNET_CONTAINER_MultiHashMap *sent;
49 struct GNUNET_CONTAINER_MultiHashMap *received;
50 int done;
51} info1, info2;
52
53static struct GNUNET_CONTAINER_MultiHashMap *common_sent;
54
55static struct GNUNET_HashCode app_id;
56
57static struct GNUNET_PeerIdentity local_peer;
58
59static struct GNUNET_SETU_ListenHandle *set_listener;
60
61static int byzantine;
62static unsigned int force_delta;
63static unsigned int force_full;
64static unsigned int element_size = 32;
65
66/**
67 * Handle to the statistics service.
68 */
69static struct GNUNET_STATISTICS_Handle *statistics;
70
71/**
72 * The profiler will write statistics
73 * for all peers to the file with this name.
74 */
75static char *statistics_filename;
76
77/**
78 * The profiler will write statistics
79 * for all peers to this file.
80 */
81static FILE *statistics_file;
82
83
84static int
85map_remove_iterator (void *cls,
86 const struct GNUNET_HashCode *key,
87 void *value)
88{
89 struct GNUNET_CONTAINER_MultiHashMap *m = cls;
90 int ret;
91
92 GNUNET_assert (NULL != key);
93
94 ret = GNUNET_CONTAINER_multihashmap_remove_all (m, key);
95 if (GNUNET_OK != ret)
96 printf ("spurious element\n");
97 return GNUNET_YES;
98}
99
100
101/**
102 * Callback function to process statistic values.
103 *
104 * @param cls closure
105 * @param subsystem name of subsystem that created the statistic
106 * @param name the name of the datum
107 * @param value the current value
108 * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not
109 * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration
110 */
111static int
112statistics_result (void *cls,
113 const char *subsystem,
114 const char *name,
115 uint64_t value,
116 int is_persistent)
117{
118 if (NULL != statistics_file)
119 {
120 fprintf (statistics_file, "%s\t%s\t%lu\n", subsystem, name, (unsigned
121 long) value);
122 }
123 return GNUNET_OK;
124}
125
126
127static void
128statistics_done (void *cls,
129 int success)
130{
131 GNUNET_assert (GNUNET_YES == success);
132 if (NULL != statistics_file)
133 fclose (statistics_file);
134 GNUNET_SCHEDULER_shutdown ();
135}
136
137
138static void
139check_all_done (void)
140{
141 if ((info1.done == GNUNET_NO) || (info2.done == GNUNET_NO))
142 return;
143
144 GNUNET_CONTAINER_multihashmap_iterate (info1.received, map_remove_iterator,
145 info2.sent);
146 GNUNET_CONTAINER_multihashmap_iterate (info2.received, map_remove_iterator,
147 info1.sent);
148
149 printf ("set a: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
150 info1.sent));
151 printf ("set b: %d missing elements\n", GNUNET_CONTAINER_multihashmap_size (
152 info2.sent));
153
154 if (NULL == statistics_filename)
155 {
156 GNUNET_SCHEDULER_shutdown ();
157 return;
158 }
159
160 statistics_file = fopen (statistics_filename, "w");
161 GNUNET_STATISTICS_get (statistics, NULL, NULL,
162 &statistics_done,
163 &statistics_result, NULL);
164}
165
166
167static void
168set_result_cb (void *cls,
169 const struct GNUNET_SETU_Element *element,
170 uint64_t current_size,
171 enum GNUNET_SETU_Status status)
172{
173 struct SetInfo *info = cls;
174
175 GNUNET_assert (GNUNET_NO == info->done);
176 switch (status)
177 {
178 case GNUNET_SETU_STATUS_DONE:
179 info->done = GNUNET_YES;
180 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s done\n", info->id);
181 check_all_done ();
182 info->oh = NULL;
183 return;
184
185 case GNUNET_SETU_STATUS_FAILURE:
186 info->oh = NULL;
187 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "failure\n");
188 GNUNET_SCHEDULER_shutdown ();
189 return;
190
191 case GNUNET_SETU_STATUS_ADD_LOCAL:
192 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: local element\n", info->id);
193 break;
194 default:
195 GNUNET_assert (0);
196 }
197
198 if (element->size != element_size)
199 {
200 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
201 "wrong element size: %u, expected %u\n",
202 element->size,
203 (unsigned int) sizeof(struct GNUNET_HashCode));
204 GNUNET_assert (0);
205 }
206
207 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "set %s: got element (%s)\n",
208 info->id, GNUNET_h2s (element->data));
209 GNUNET_assert (NULL != element->data);
210 struct GNUNET_HashCode data_hash;
211 GNUNET_CRYPTO_hash (element->data, element_size, &data_hash);
212 GNUNET_CONTAINER_multihashmap_put (info->received,
213 &data_hash, NULL,
214 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
215}
216
217
218static void
219set_listen_cb (void *cls,
220 const struct GNUNET_PeerIdentity *other_peer,
221 const struct GNUNET_MessageHeader *context_msg,
222 struct GNUNET_SETU_Request *request)
223{
224 /* max. 2 options plus terminator */
225 struct GNUNET_SETU_Option opts[3] = { { 0 } };
226 unsigned int n_opts = 0;
227
228 if (NULL == request)
229 {
230 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
231 "listener failed\n");
232 return;
233 }
234 GNUNET_assert (NULL == info2.oh);
235 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
236 "set listen cb called\n");
237 if (byzantine)
238 {
239 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
240 GNUNET_SETU_OPTION_BYZANTINE };
241 }
242 GNUNET_assert (! (force_full && force_delta));
243 if (force_full)
244 {
245 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
246 GNUNET_SETU_OPTION_FORCE_FULL };
247 }
248 if (force_delta)
249 {
250 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
251 GNUNET_SETU_OPTION_FORCE_DELTA };
252 }
253
254 opts[n_opts].type = 0;
255 info2.oh = GNUNET_SETU_accept (request,
256 opts,
257 set_result_cb, &info2);
258 GNUNET_SETU_commit (info2.oh, info2.set);
259}
260
261
262static int
263set_insert_iterator (void *cls,
264 const struct GNUNET_HashCode *key,
265 void *value)
266{
267 struct GNUNET_SETU_Handle *set = cls;
268 struct GNUNET_SETU_Element el;
269
270 el.element_type = 0;
271 el.data = value;
272 el.size = element_size;
273 GNUNET_SETU_add_element (set, &el, NULL, NULL);
274 return GNUNET_YES;
275}
276
277
278static void
279handle_shutdown (void *cls)
280{
281 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
282 "Shutting down set profiler\n");
283 if (NULL != set_listener)
284 {
285 GNUNET_SETU_listen_cancel (set_listener);
286 set_listener = NULL;
287 }
288 if (NULL != info1.oh)
289 {
290 GNUNET_SETU_operation_cancel (info1.oh);
291 info1.oh = NULL;
292 }
293 if (NULL != info2.oh)
294 {
295 GNUNET_SETU_operation_cancel (info2.oh);
296 info2.oh = NULL;
297 }
298 if (NULL != info1.set)
299 {
300 GNUNET_SETU_destroy (info1.set);
301 info1.set = NULL;
302 }
303 if (NULL != info2.set)
304 {
305 GNUNET_SETU_destroy (info2.set);
306 info2.set = NULL;
307 }
308 GNUNET_STATISTICS_destroy (statistics, GNUNET_NO);
309}
310
311
312static void
313run (void *cls,
314 const struct GNUNET_CONFIGURATION_Handle *cfg,
315 struct GNUNET_TESTING_Peer *peer)
316{
317 unsigned int i;
318 struct GNUNET_HashCode hash;
319 /* max. 2 options plus terminator */
320 struct GNUNET_SETU_Option opts[3] = { { 0 } };
321 unsigned int n_opts = 0;
322
323 config = cfg;
324
325 GNUNET_assert (element_size > 0);
326
327 if (GNUNET_OK != GNUNET_CRYPTO_get_peer_identity (cfg, &local_peer))
328 {
329 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "could not retrieve host identity\n");
330 ret = 0;
331 return;
332 }
333
334 statistics = GNUNET_STATISTICS_create ("set-profiler", cfg);
335
336 GNUNET_SCHEDULER_add_shutdown (&handle_shutdown, NULL);
337
338 info1.id = "a";
339 info2.id = "b";
340
341 info1.sent = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
342 info2.sent = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
343 common_sent = GNUNET_CONTAINER_multihashmap_create (num_c + 1, GNUNET_NO);
344
345 info1.received = GNUNET_CONTAINER_multihashmap_create (num_a + 1, GNUNET_NO);
346 info2.received = GNUNET_CONTAINER_multihashmap_create (num_b + 1, GNUNET_NO);
347
348 for (i = 0; i < num_a; i++)
349 {
350 char *data = GNUNET_malloc (element_size);
351 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
352 GNUNET_CRYPTO_hash (data, element_size, &hash);
353 GNUNET_CONTAINER_multihashmap_put (info1.sent, &hash, data,
354 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
355 }
356
357 for (i = 0; i < num_b; i++)
358 {
359 char *data = GNUNET_malloc (element_size);
360 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
361 GNUNET_CRYPTO_hash (data, element_size, &hash);
362 GNUNET_CONTAINER_multihashmap_put (info2.sent, &hash, data,
363 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
364 }
365
366 for (i = 0; i < num_c; i++)
367 {
368 char *data = GNUNET_malloc (element_size);
369 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, data, element_size);
370 GNUNET_CRYPTO_hash (data, element_size, &hash);
371 GNUNET_CONTAINER_multihashmap_put (common_sent, &hash, data,
372 GNUNET_CONTAINER_MULTIHASHMAPOPTION_REPLACE);
373 }
374
375 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_STRONG, &app_id);
376
377 /* FIXME: also implement intersection etc. */
378 info1.set = GNUNET_SETU_create (config);
379 info2.set = GNUNET_SETU_create (config);
380
381 GNUNET_CONTAINER_multihashmap_iterate (info1.sent, set_insert_iterator,
382 info1.set);
383 GNUNET_CONTAINER_multihashmap_iterate (info2.sent, set_insert_iterator,
384 info2.set);
385 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
386 info1.set);
387 GNUNET_CONTAINER_multihashmap_iterate (common_sent, set_insert_iterator,
388 info2.set);
389
390 set_listener = GNUNET_SETU_listen (config,
391 &app_id,
392 &set_listen_cb,
393 NULL);
394
395
396 if (byzantine)
397 {
398 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
399 GNUNET_SETU_OPTION_BYZANTINE };
400 }
401 GNUNET_assert (! (force_full && force_delta));
402 if (force_full)
403 {
404 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
405 GNUNET_SETU_OPTION_FORCE_FULL };
406 }
407 if (force_delta)
408 {
409 opts[n_opts++] = (struct GNUNET_SETU_Option) { .type =
410 GNUNET_SETU_OPTION_FORCE_DELTA };
411 }
412
413 opts[n_opts].type = 0;
414
415 info1.oh = GNUNET_SETU_prepare (&local_peer, &app_id, NULL,
416 opts,
417 set_result_cb, &info1);
418 GNUNET_SETU_commit (info1.oh, info1.set);
419 GNUNET_SETU_destroy (info1.set);
420 info1.set = NULL;
421}
422
423
424static void
425pre_run (void *cls, char *const *args, const char *cfgfile,
426 const struct GNUNET_CONFIGURATION_Handle *cfg)
427{
428 if (0 != GNUNET_TESTING_peer_run ("set-profiler",
429 cfgfile,
430 &run, NULL))
431 ret = 2;
432}
433
434
435int
436main (int argc, char **argv)
437{
438 struct GNUNET_GETOPT_CommandLineOption options[] = {
439 GNUNET_GETOPT_option_uint ('A',
440 "num-first",
441 NULL,
442 gettext_noop ("number of values"),
443 &num_a),
444
445 GNUNET_GETOPT_option_uint ('B',
446 "num-second",
447 NULL,
448 gettext_noop ("number of values"),
449 &num_b),
450
451 GNUNET_GETOPT_option_flag ('b',
452 "byzantine",
453 gettext_noop ("use byzantine mode"),
454 &byzantine),
455
456 GNUNET_GETOPT_option_uint ('f',
457 "force-full",
458 NULL,
459 gettext_noop ("force sending full set"),
460 &force_full),
461
462 GNUNET_GETOPT_option_uint ('d',
463 "force-delta",
464 NULL,
465 gettext_noop ("number delta operation"),
466 &force_delta),
467
468 GNUNET_GETOPT_option_uint ('C',
469 "num-common",
470 NULL,
471 gettext_noop ("number of values"),
472 &num_c),
473
474 GNUNET_GETOPT_option_string ('x',
475 "operation",
476 NULL,
477 gettext_noop ("operation to execute"),
478 &op_str),
479
480 GNUNET_GETOPT_option_uint ('w',
481 "element-size",
482 NULL,
483 gettext_noop ("element size"),
484 &element_size),
485
486 GNUNET_GETOPT_option_filename ('s',
487 "statistics",
488 "FILENAME",
489 gettext_noop ("write statistics to file"),
490 &statistics_filename),
491
492 GNUNET_GETOPT_OPTION_END
493 };
494
495 GNUNET_PROGRAM_run2 (argc, argv, "gnunet-set-profiler",
496 "help",
497 options, &pre_run, NULL, GNUNET_YES);
498 return ret;
499}
diff --git a/src/setu/ibf.c b/src/setu/ibf.c
deleted file mode 100644
index a55e21764..000000000
--- a/src/setu/ibf.c
+++ /dev/null
@@ -1,585 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.c
23 * @brief implementation of the invertible bloom filter
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#include "platform.h"
29#include "ibf.h"
30#include "gnunet_util_lib.h"
31#define LOG(kind, ...) GNUNET_log_from (kind, "setu", __VA_ARGS__)
32
33
34/**
35 * Compute the key's hash from the key.
36 * Redefine to use a different hash function.
37 */
38#define IBF_KEY_HASH_VAL(k) (GNUNET_CRYPTO_crc32_n (&(k), sizeof(struct \
39 IBF_KeyHash)))
40
41/**
42 * Create a key from a hashcode.
43 *
44 * @param hash the hashcode
45 * @return a key
46 */
47struct IBF_Key
48ibf_key_from_hashcode (const struct GNUNET_HashCode *hash)
49{
50 return *(struct IBF_Key *) hash;
51}
52
53
54/**
55 * Create a hashcode from a key, by replicating the key
56 * until the hascode is filled
57 *
58 * @param key the key
59 * @param dst hashcode to store the result in
60 */
61void
62ibf_hashcode_from_key (struct IBF_Key key,
63 struct GNUNET_HashCode *dst)
64{
65 struct IBF_Key *p;
66 unsigned int i;
67 const unsigned int keys_per_hashcode = sizeof(struct GNUNET_HashCode)
68 / sizeof(struct IBF_Key);
69
70 p = (struct IBF_Key *) dst;
71 for (i = 0; i < keys_per_hashcode; i++)
72 *p++ = key;
73}
74
75
76/**
77 * Create an invertible bloom filter.
78 *
79 * @param size number of IBF buckets
80 * @param hash_num number of buckets one element is hashed in
81 * @return the newly created invertible bloom filter, NULL on error
82 */
83struct InvertibleBloomFilter *
84ibf_create (uint32_t size, uint8_t hash_num)
85{
86 struct InvertibleBloomFilter *ibf;
87
88 GNUNET_assert (0 != size);
89
90 ibf = GNUNET_new (struct InvertibleBloomFilter);
91 ibf->count = GNUNET_malloc_large (size * sizeof(uint64_t));
92 if (NULL == ibf->count)
93 {
94 GNUNET_free (ibf);
95 return NULL;
96 }
97 ibf->key_sum = GNUNET_malloc_large (size * sizeof(struct IBF_Key));
98 if (NULL == ibf->key_sum)
99 {
100 GNUNET_free (ibf->count);
101 GNUNET_free (ibf);
102 return NULL;
103 }
104 ibf->key_hash_sum = GNUNET_malloc_large (size * sizeof(struct IBF_KeyHash));
105 if (NULL == ibf->key_hash_sum)
106 {
107 GNUNET_free (ibf->key_sum);
108 GNUNET_free (ibf->count);
109 GNUNET_free (ibf);
110 return NULL;
111 }
112 ibf->size = size;
113 ibf->hash_num = hash_num;
114
115 return ibf;
116}
117
118
119/**
120 * Store unique bucket indices for the specified @a key in @a dst.
121 */
122static void
123ibf_get_indices (const struct InvertibleBloomFilter *ibf,
124 struct IBF_Key key,
125 int *dst)
126{
127 uint32_t filled;
128 uint32_t i;
129 uint32_t bucket;
130
131 bucket = GNUNET_CRYPTO_crc32_n (&key, sizeof key);
132 for (i = 0, filled = 0; filled < ibf->hash_num; i++)
133 {
134 uint64_t x;
135
136 for (unsigned int j = 0; j < filled; j++)
137 if (dst[j] == bucket % ibf->size)
138 goto try_next;
139 dst[filled++] = bucket % ibf->size;
140try_next:
141 x = ((uint64_t) bucket << 32) | i;
142 bucket = GNUNET_CRYPTO_crc32_n (&x, sizeof x);
143 }
144}
145
146
147static void
148ibf_insert_into (struct InvertibleBloomFilter *ibf,
149 struct IBF_Key key,
150 const int *buckets,
151 int side)
152{
153 for (unsigned int i = 0; i < ibf->hash_num; i++)
154 {
155 const int bucket = buckets[i];
156
157 ibf->count[bucket].count_val += side;
158 ibf->key_sum[bucket].key_val ^= key.key_val;
159 ibf->key_hash_sum[bucket].key_hash_val
160 ^= IBF_KEY_HASH_VAL (key);
161 }
162}
163
164
165/**
166 * Insert a key into an IBF.
167 *
168 * @param ibf the IBF
169 * @param key the element's hash code
170 */
171void
172ibf_insert (struct InvertibleBloomFilter *ibf,
173 struct IBF_Key key)
174{
175 int buckets[ibf->hash_num];
176
177 GNUNET_assert (ibf->hash_num <= ibf->size);
178 ibf_get_indices (ibf, key, buckets);
179 ibf_insert_into (ibf, key, buckets, 1);
180}
181
182
183/**
184 * Remove a key from an IBF.
185 *
186 * @param ibf the IBF
187 * @param key the element's hash code
188 */
189void
190ibf_remove (struct InvertibleBloomFilter *ibf,
191 struct IBF_Key key)
192{
193 int buckets[ibf->hash_num];
194
195 GNUNET_assert (ibf->hash_num <= ibf->size);
196 ibf_get_indices (ibf, key, buckets);
197 ibf_insert_into (ibf, key, buckets, -1);
198}
199
200
201/**
202 * Test is the IBF is empty, i.e. all counts, keys and key hashes are zero.
203 */
204static int
205ibf_is_empty (struct InvertibleBloomFilter *ibf)
206{
207 for (uint32_t i = 0; i < ibf->size; i++)
208 {
209 if (0 != ibf->count[i].count_val)
210 return GNUNET_NO;
211 if (0 != ibf->key_hash_sum[i].key_hash_val)
212 return GNUNET_NO;
213 if (0 != ibf->key_sum[i].key_val)
214 return GNUNET_NO;
215 }
216 return GNUNET_YES;
217}
218
219
220int
221ibf_decode (struct InvertibleBloomFilter *ibf,
222 int *ret_side,
223 struct IBF_Key *ret_id)
224{
225 struct IBF_KeyHash hash;
226 int buckets[ibf->hash_num];
227
228 for (uint32_t i = 0; i < ibf->size; i++)
229 {
230 int hit;
231
232 /* we can only decode from pure buckets */
233 if ( (1 != ibf->count[i].count_val) &&
234 (-1 != ibf->count[i].count_val) )
235 continue;
236
237 hash.key_hash_val = IBF_KEY_HASH_VAL (ibf->key_sum[i]);
238
239 /* test if the hash matches the key */
240 if (hash.key_hash_val != ibf->key_hash_sum[i].key_hash_val)
241 continue;
242
243 /* test if key in bucket hits its own location,
244 * if not, the key hash was subject to collision */
245 hit = GNUNET_NO;
246 ibf_get_indices (ibf, ibf->key_sum[i], buckets);
247 for (int j = 0; j < ibf->hash_num; j++)
248 if (buckets[j] == i)
249 hit = GNUNET_YES;
250
251 if (GNUNET_NO == hit)
252 continue;
253
254 if (1 == ibf->count[i].count_val)
255 {
256 ibf->remote_decoded_count++;
257 }
258 else
259 {
260 ibf->local_decoded_count++;
261 }
262
263
264 if (NULL != ret_side)
265 *ret_side = ibf->count[i].count_val;
266 if (NULL != ret_id)
267 *ret_id = ibf->key_sum[i];
268
269 /* insert on the opposite side, effectively removing the element */
270 ibf_insert_into (ibf, ibf->key_sum[i], buckets, -ibf->count[i].count_val);
271
272 return GNUNET_YES;
273 }
274
275 if (GNUNET_YES == ibf_is_empty (ibf))
276 return GNUNET_NO;
277 return GNUNET_SYSERR;
278}
279
280
281/**
282 * Returns the minimal bytes needed to store the counter of the IBF
283 *
284 * @param ibf the IBF
285 */
286uint8_t
287ibf_get_max_counter (struct InvertibleBloomFilter *ibf)
288{
289 long long max_counter = 0;
290 for (uint64_t i = 0; i < ibf->size; i++)
291 {
292 if (ibf->count[i].count_val > max_counter)
293 {
294 max_counter = ibf->count[i].count_val;
295 }
296 }
297 return 64 - __builtin_clzll (max_counter);
298}
299
300
301/**
302 * Write buckets from an ibf to a buffer.
303 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
304 *
305 * @param ibf the ibf to write
306 * @param start with which bucket to start
307 * @param count how many buckets to write
308 * @param buf buffer to write the data to
309 * @param max bit length of a counter for unpacking
310 */
311void
312ibf_write_slice (const struct InvertibleBloomFilter *ibf,
313 uint32_t start,
314 uint64_t count,
315 void *buf,
316 uint8_t counter_max_length)
317{
318 struct IBF_Key *key_dst;
319 struct IBF_KeyHash *key_hash_dst;
320
321 GNUNET_assert (start + count <= ibf->size);
322
323 /* copy keys */
324 key_dst = (struct IBF_Key *) buf;
325 GNUNET_memcpy (key_dst,
326 ibf->key_sum + start,
327 count * sizeof(*key_dst));
328 key_dst += count;
329 /* copy key hashes */
330 key_hash_dst = (struct IBF_KeyHash *) key_dst;
331 GNUNET_memcpy (key_hash_dst,
332 ibf->key_hash_sum + start,
333 count * sizeof(*key_hash_dst));
334 key_hash_dst += count;
335
336 /* pack and copy counter */
337 pack_counter (ibf,
338 start,
339 count,
340 (uint8_t *) key_hash_dst,
341 counter_max_length);
342
343
344}
345
346
347
348void
349pack_counter (const struct InvertibleBloomFilter *ibf,
350 uint32_t start,
351 uint64_t count,
352 uint8_t *buf,
353 uint8_t counter_max_length)
354{
355 uint8_t store_size = 0;
356 uint8_t store = 0;
357 uint16_t byte_ctr = 0;
358
359 /**
360 * Iterate over IBF bucket
361 */
362 for (uint64_t i = start; i< (count + start);)
363 {
364 uint64_t count_val_to_write = ibf->count[i].count_val;
365 uint8_t count_len_to_write = counter_max_length;
366
367 /**
368 * Pack and compose counters to byte values
369 */
370 while ((count_len_to_write + store_size) >= 8)
371 {
372 uint8_t bit_shift = 0;
373
374 /**
375 * Shift bits if more than a byte has to be written
376 * or the store size is not empty
377 */
378 if ((store_size > 0) || (count_len_to_write > 8))
379 {
380 uint8_t bit_unused = 8 - store_size;
381 bit_shift = count_len_to_write - bit_unused;
382 store = store << bit_unused;
383 }
384
385 buf[byte_ctr] = ((count_val_to_write >> bit_shift) | store) & 0xFF;
386 byte_ctr++;
387 count_len_to_write -= (8 - store_size);
388 count_val_to_write = count_val_to_write & ((1ULL <<
389 count_len_to_write) - 1);
390 store = 0;
391 store_size = 0;
392 }
393 store = (store << count_len_to_write) | count_val_to_write;
394 store_size = store_size + count_len_to_write;
395 count_len_to_write = 0;
396 i++;
397 }
398
399 /**
400 * Pack data left in story before finishing
401 */
402 if (store_size > 0)
403 {
404 buf[byte_ctr] = store << (8 - store_size);
405 byte_ctr++;
406 }
407
408}
409
410
411
412void
413unpack_counter (const struct InvertibleBloomFilter *ibf,
414 uint32_t start,
415 uint64_t count,
416 uint8_t *buf,
417 uint8_t counter_max_length)
418{
419 uint64_t ibf_counter_ctr = 0;
420 uint64_t store = 0;
421 uint64_t store_bit_ctr = 0;
422 uint64_t byte_ctr = 0;
423
424 /**
425 * Iterate over received bytes
426 */
427 while (true)
428 {
429 uint8_t byte_read = buf[byte_ctr];
430 uint8_t bit_to_read_left = 8;
431 byte_ctr++;
432
433 /**
434 * Pack data left in story before finishing
435 */
436 while (true)
437 {
438 /**
439 * Stop decoding when end is reached
440 */
441 if (ibf_counter_ctr > (count - 1))
442 return;
443
444 /*
445 * Unpack the counter
446 */
447 if ((store_bit_ctr + bit_to_read_left) >= counter_max_length)
448 {
449 uint8_t bytes_used = counter_max_length - store_bit_ctr;
450 if (store_bit_ctr > 0)
451 {
452 store = store << bytes_used;
453 }
454
455 uint8_t bytes_to_shift = bit_to_read_left - bytes_used;
456 uint64_t counter_part = byte_read >> bytes_to_shift;
457 store = store | counter_part;
458 ibf->count[ibf_counter_ctr + start].count_val = store;
459 byte_read = byte_read & ((1 << bytes_to_shift) - 1);
460 bit_to_read_left -= bytes_used;
461 ibf_counter_ctr++;
462 store = 0;
463 store_bit_ctr = 0;
464 }
465 else
466 {
467 store_bit_ctr += bit_to_read_left;
468 if (0 == store)
469 {
470 store = byte_read;
471 }
472 else
473 {
474 store = store << bit_to_read_left;
475 store = store | byte_read;
476 }
477 break;
478 }
479 }
480 }
481}
482
483
484/**
485 * Read buckets from a buffer into an ibf.
486 *
487 * @param buf pointer to the buffer to read from
488 * @param start which bucket to start at
489 * @param count how many buckets to read
490 * @param ibf the ibf to read from
491 * @param max bit length of a counter for unpacking
492 */
493void
494ibf_read_slice (const void *buf,
495 uint32_t start,
496 uint64_t count,
497 struct InvertibleBloomFilter *ibf,
498 uint8_t counter_max_length)
499{
500 struct IBF_Key *key_src;
501 struct IBF_KeyHash *key_hash_src;
502 struct IBF_Count *count_src;
503
504 GNUNET_assert (count > 0);
505 GNUNET_assert (start + count <= ibf->size);
506
507 /* copy keys */
508 key_src = (struct IBF_Key *) buf;
509 GNUNET_memcpy (ibf->key_sum + start,
510 key_src,
511 count * sizeof *key_src);
512 key_src += count;
513 /* copy key hashes */
514 key_hash_src = (struct IBF_KeyHash *) key_src;
515 GNUNET_memcpy (ibf->key_hash_sum + start,
516 key_hash_src,
517 count * sizeof *key_hash_src);
518 key_hash_src += count;
519
520 /* copy and unpack counts */
521 count_src = (struct IBF_Count *) key_hash_src;
522 unpack_counter (ibf,start,count,(uint8_t *) count_src,counter_max_length);
523}
524
525
526/**
527 * Subtract ibf2 from ibf1, storing the result in ibf1.
528 * The two IBF's must have the same parameters size and hash_num.
529 *
530 * @param ibf1 IBF that is subtracted from
531 * @param ibf2 IBF that will be subtracted from ibf1
532 */
533void
534ibf_subtract (struct InvertibleBloomFilter *ibf1,
535 const struct InvertibleBloomFilter *ibf2)
536{
537 GNUNET_assert (ibf1->size == ibf2->size);
538 GNUNET_assert (ibf1->hash_num == ibf2->hash_num);
539
540 for (uint32_t i = 0; i < ibf1->size; i++)
541 {
542 ibf1->count[i].count_val -= ibf2->count[i].count_val;
543 ibf1->key_hash_sum[i].key_hash_val ^= ibf2->key_hash_sum[i].key_hash_val;
544 ibf1->key_sum[i].key_val ^= ibf2->key_sum[i].key_val;
545 }
546}
547
548
549/**
550 * Create a copy of an IBF, the copy has to be destroyed properly.
551 *
552 * @param ibf the IBF to copy
553 */
554struct InvertibleBloomFilter *
555ibf_dup (const struct InvertibleBloomFilter *ibf)
556{
557 struct InvertibleBloomFilter *copy;
558
559 copy = GNUNET_malloc (sizeof *copy);
560 copy->hash_num = ibf->hash_num;
561 copy->size = ibf->size;
562 copy->key_hash_sum = GNUNET_memdup (ibf->key_hash_sum,
563 ibf->size * sizeof(struct IBF_KeyHash));
564 copy->key_sum = GNUNET_memdup (ibf->key_sum,
565 ibf->size * sizeof(struct IBF_Key));
566 copy->count = GNUNET_memdup (ibf->count,
567 ibf->size * sizeof(struct IBF_Count));
568 return copy;
569}
570
571
572/**
573 * Destroy all resources associated with the invertible bloom filter.
574 * No more ibf_*-functions may be called on ibf after calling destroy.
575 *
576 * @param ibf the intertible bloom filter to destroy
577 */
578void
579ibf_destroy (struct InvertibleBloomFilter *ibf)
580{
581 GNUNET_free (ibf->key_sum);
582 GNUNET_free (ibf->key_hash_sum);
583 GNUNET_free (ibf->count);
584 GNUNET_free (ibf);
585}
diff --git a/src/setu/ibf.h b/src/setu/ibf.h
deleted file mode 100644
index 5628405dc..000000000
--- a/src/setu/ibf.h
+++ /dev/null
@@ -1,310 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf.h
23 * @brief invertible bloom filter
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27
28#ifndef GNUNET_CONSENSUS_IBF_H
29#define GNUNET_CONSENSUS_IBF_H
30
31#include "platform.h"
32#include "gnunet_util_lib.h"
33
34#ifdef __cplusplus
35extern "C"
36{
37#if 0 /* keep Emacsens' auto-indent happy */
38}
39#endif
40#endif
41
42
43/**
44 * Keys that can be inserted into and removed from an IBF.
45 */
46struct IBF_Key
47{
48 uint64_t key_val;
49};
50
51
52/**
53 * Hash of an IBF key.
54 */
55struct IBF_KeyHash
56{
57 uint32_t key_hash_val;
58};
59
60
61/**
62 * Type of the count field of IBF buckets.
63 */
64struct IBF_Count
65{
66 int64_t count_val;
67};
68
69
70/**
71 * Size of one ibf bucket in bytes
72 */
73#define IBF_BUCKET_SIZE (sizeof(struct IBF_Count) + sizeof(struct IBF_Key) \
74 + sizeof(struct IBF_KeyHash))
75
76
77/**
78 * Invertible bloom filter (IBF).
79 *
80 * An IBF is a counting bloom filter that has the ability to restore
81 * the hashes of its stored elements with high probability.
82 */
83struct InvertibleBloomFilter
84{
85 /**
86 * How many cells does this IBF have?
87 */
88 uint32_t size;
89
90 /**
91 * In how many cells do we hash one element?
92 * Usually 4 or 3.
93 */
94 uint8_t hash_num;
95
96 /**
97 * If an IBF is decoded this count stores how many
98 * elements are on the local site. This is used
99 * to estimate the set difference on a site
100 */
101 int local_decoded_count;
102
103 /**
104 * If an IBF is decoded this count stores how many
105 * elements are on the remote site. This is used
106 * to estimate the set difference on a site
107 */
108 int remote_decoded_count;
109
110 /**
111 * Xor sums of the elements' keys, used to identify the elements.
112 * Array of 'size' elements.
113 */
114 struct IBF_Key *key_sum;
115
116 /**
117 * Xor sums of the hashes of the keys of inserted elements.
118 * Array of 'size' elements.
119 */
120 struct IBF_KeyHash *key_hash_sum;
121
122 /**
123 * How many times has a bucket been hit?
124 * Can be negative, as a result of IBF subtraction.
125 * Array of 'size' elements.
126 */
127 struct IBF_Count *count;
128};
129
130
131/**
132 * Write buckets from an ibf to a buffer.
133 * Exactly (IBF_BUCKET_SIZE*ibf->size) bytes are written to buf.
134 *
135 * @param ibf the ibf to write
136 * @param start with which bucket to start
137 * @param count how many buckets to write
138 * @param buf buffer to write the data to
139 */
140void
141ibf_write_slice (const struct InvertibleBloomFilter *ibf,
142 uint32_t start,
143 uint64_t count,
144 void *buf,
145 uint8_t counter_max_length);
146
147
148/**
149 * Read buckets from a buffer into an ibf.
150 *
151 * @param buf pointer to the buffer to read from
152 * @param start which bucket to start at
153 * @param count how many buckets to read
154 * @param ibf the ibf to write to
155 */
156void
157ibf_read_slice (const void *buf,
158 uint32_t start,
159 uint64_t count,
160 struct InvertibleBloomFilter *ibf,
161 uint8_t counter_max_length);
162
163
164/**
165 * Create a key from a hashcode.
166 *
167 * @param hash the hashcode
168 * @return a key
169 */
170struct IBF_Key
171ibf_key_from_hashcode (const struct GNUNET_HashCode *hash);
172
173
174/**
175 * Create a hashcode from a key, by replicating the key
176 * until the hascode is filled
177 *
178 * @param key the key
179 * @param dst hashcode to store the result in
180 */
181void
182ibf_hashcode_from_key (struct IBF_Key key, struct GNUNET_HashCode *dst);
183
184
185/**
186 * Create an invertible bloom filter.
187 *
188 * @param size number of IBF buckets
189 * @param hash_num number of buckets one element is hashed in, usually 3 or 4
190 * @return the newly created invertible bloom filter, NULL on error
191 */
192struct InvertibleBloomFilter *
193ibf_create (uint32_t size, uint8_t hash_num);
194
195
196/**
197 * Insert a key into an IBF.
198 *
199 * @param ibf the IBF
200 * @param key the element's hash code
201 */
202void
203ibf_insert (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
204
205
206/**
207 * Remove a key from an IBF.
208 *
209 * @param ibf the IBF
210 * @param key the element's hash code
211 */
212void
213ibf_remove (struct InvertibleBloomFilter *ibf, struct IBF_Key key);
214
215
216/**
217 * Subtract ibf2 from ibf1, storing the result in ibf1.
218 * The two IBF's must have the same parameters size and hash_num.
219 *
220 * @param ibf1 IBF that is subtracted from
221 * @param ibf2 IBF that will be subtracted from ibf1
222 */
223void
224ibf_subtract (struct InvertibleBloomFilter *ibf1,
225 const struct InvertibleBloomFilter *ibf2);
226
227
228/**
229 * Decode and remove an element from the IBF, if possible.
230 *
231 * @param ibf the invertible bloom filter to decode
232 * @param ret_side sign of the cell's count where the decoded element came from.
233 * A negative sign indicates that the element was recovered
234 * resides in an IBF that was previously subtracted from.
235 * @param ret_id receives the hash code of the decoded element, if successful
236 * @return #GNUNET_YES if decoding an element was successful,
237 * #GNUNET_NO if the IBF is empty,
238 * #GNUNET_SYSERR if the decoding has failed
239 */
240int
241ibf_decode (struct InvertibleBloomFilter *ibf,
242 int *ret_side,
243 struct IBF_Key *ret_id);
244
245
246/**
247 * Create a copy of an IBF, the copy has to be destroyed properly.
248 *
249 * @param ibf the IBF to copy
250 */
251struct InvertibleBloomFilter *
252ibf_dup (const struct InvertibleBloomFilter *ibf);
253
254
255/**
256 * Destroy all resources associated with the invertible bloom filter.
257 * No more ibf_*-functions may be called on ibf after calling destroy.
258 *
259 * @param ibf the intertible bloom filter to destroy
260 */
261void
262ibf_destroy (struct InvertibleBloomFilter *ibf);
263
264uint8_t
265ibf_get_max_counter (struct InvertibleBloomFilter *ibf);
266
267
268/**
269 * Packs the counter to transmit only the smallest possible amount of bytes and
270 * preventing overflow of the counter
271 * @param ibf the ibf to write
272 * @param start with which bucket to start
273 * @param count how many buckets to write
274 * @param buf buffer to write the data to
275 * @param max bit length of a counter for unpacking
276 */
277
278void
279pack_counter (const struct InvertibleBloomFilter *ibf,
280 uint32_t start,
281 uint64_t count,
282 uint8_t *buf,
283 uint8_t counter_max_length);
284
285/**
286 * Unpacks the counter to transmit only the smallest possible amount of bytes and
287 * preventing overflow of the counter
288 * @param ibf the ibf to write
289 * @param start with which bucket to start
290 * @param count how many buckets to write
291 * @param buf buffer to write the data to
292 * @param max bit length of a counter for unpacking
293 */
294
295void
296unpack_counter (const struct InvertibleBloomFilter *ibf,
297 uint32_t start,
298 uint64_t count,
299 uint8_t *buf,
300 uint8_t counter_max_length);
301
302
303#if 0 /* keep Emacsens' auto-indent happy */
304{
305#endif
306#ifdef __cplusplus
307}
308#endif
309
310#endif
diff --git a/src/setu/ibf_sim.c b/src/setu/ibf_sim.c
deleted file mode 100644
index 563ed0fb8..000000000
--- a/src/setu/ibf_sim.c
+++ /dev/null
@@ -1,143 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2013 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/ibf_sim.c
23 * @brief implementation of simulation for invertible bloom filter
24 * @author Florian Dold
25 *
26 * This code was used for some internal experiments, it is not
27 * build or shipped as part of the GNUnet system.
28 */
29#include "platform.h"
30#include <stdlib.h>
31#include <stdio.h>
32#include <string.h>
33
34#define MAX_IBF_DECODE 16
35
36/* report average over how many rounds? */
37#define ROUNDS 100000
38
39/* enable one of the three below */
40// simple fix
41#define FIX1 0
42// possibly slightly better fix for large IBF_DECODE values
43#define FIX2 1
44
45// SIGCOMM algorithm
46#define STRATA 0
47
48// print each value?
49#define VERBOSE 0
50// avoid assembly? (ASM is about 50% faster)
51#define SLOW 0
52
53int
54main (int argc, char **argv)
55{
56 unsigned int round;
57 unsigned int buckets[31]; // max is 2^31 as 'random' returns only between 0 and 2^31
58 unsigned int i;
59 int j;
60 unsigned int r;
61 unsigned int ret;
62 unsigned long long total;
63 unsigned int want;
64 double predict;
65
66 srandom (time (NULL));
67 total = 0;
68 want = atoi (argv[1]);
69 for (round = 0; round < ROUNDS; round++)
70 {
71 memset (buckets, 0, sizeof(buckets));
72 for (i = 0; i < want; i++)
73 {
74 /* FIXME: might want to use 'better' PRNG to avoid
75 PRNG-induced biases */
76 r = random ();
77 if (0 == r)
78 continue;
79#if SLOW
80 for (j = 0; (j < 31) && (0 == (r & (1 << j))); j++)
81 ;
82#else
83 /* use assembly / gcc */
84 j = __builtin_ffs (r) - 1;
85#endif
86 buckets[j]++;
87 }
88 ret = 0;
89 predict = 0.0;
90 for (j = 31; j >= 0; j--)
91 {
92#if FIX1
93 /* improved algorithm, for 1000 elements with IBF-DECODE 8, I
94 get 990/1000 elements on average over 1 million runs; key
95 idea being to stop short of the 'last' possible IBF as
96 otherwise a "lowball" per-chance would unduely influence the
97 result */if ((j > 0) &&
98 (buckets[j - 1] > MAX_IBF_DECODE))
99 {
100 ret *= (1 << (j + 1));
101 break;
102 }
103#endif
104#if FIX2
105 /* another improvement: don't just always cut off the last one,
106 but rather try to predict based on all previous values where
107 that "last" one is; additional prediction can only really
108 work if MAX_IBF_DECODE is sufficiently high */
109 if ((j > 0) &&
110 ((buckets[j - 1] > MAX_IBF_DECODE) ||
111 (predict > MAX_IBF_DECODE)))
112 {
113 ret *= (1 << (j + 1));
114 break;
115 }
116#endif
117#if STRATA
118 /* original algorithm, for 1000 elements with IBF-DECODE 8,
119 I get 920/1000 elements on average over 1 million runs */
120 if (buckets[j] > MAX_IBF_DECODE)
121 {
122 ret *= (1 << (j + 1));
123 break;
124 }
125#endif
126 ret += buckets[j];
127 predict = (buckets[j] + 2.0 * predict) / 2.0;
128 }
129#if VERBOSE
130 fprintf (stderr, "%u ", ret);
131#endif
132 total += ret;
133 }
134 fprintf (stderr, "\n");
135 fprintf (stdout, "average %llu\n", total / ROUNDS);
136 return 0;
137}
138
139
140/* TODO: should calculate stddev of the results to also be able to
141 say something about the stability of the results, outside of
142 large-scale averages -- gaining 8% precision at the expense of
143 50% additional variance might not be worth it... */
diff --git a/src/setu/meson.build b/src/setu/meson.build
deleted file mode 100644
index fe5611fd2..000000000
--- a/src/setu/meson.build
+++ /dev/null
@@ -1,50 +0,0 @@
1libgnunetsetu_src = ['setu_api.c']
2
3gnunetservicesetu_src = ['gnunet-service-setu.c',
4 'ibf.c',
5 'gnunet-service-setu_strata_estimator.c']
6
7configure_file(input : 'setu.conf.in',
8 output : 'setu.conf',
9 configuration : cdata,
10 install: true,
11 install_dir: pkgcfgdir)
12
13
14if get_option('monolith')
15 foreach p : libgnunetsetu_src + gnunetservicesetu_src
16 gnunet_src += 'setu/' + p
17 endforeach
18 subdir_done()
19endif
20
21libgnunetsetu = library('gnunetsetu',
22 libgnunetsetu_src,
23 soversion: '0',
24 version: '0.0.0',
25 dependencies: libgnunetutil_dep,
26 include_directories: [incdir, configuration_inc],
27 install: true,
28 install_dir: get_option('libdir'))
29pkg.generate(libgnunetsetu, url: 'https://www.gnunet.org',
30 description : 'Provides API for accessing the set union service')
31libgnunetsetu_dep = declare_dependency(link_with : libgnunetsetu)
32shared_module('gnunet_plugin_block_setu_test',
33 ['plugin_block_setu_test.c'],
34 dependencies: libgnunetutil_dep,
35 include_directories: [incdir, configuration_inc],
36 install:true,
37 install_dir: get_option('libdir')/'gnunet')
38executable ('gnunet-service-setu',
39 gnunetservicesetu_src,
40 dependencies: [libgnunetsetu_dep,
41 libgnunetutil_dep,
42 m_dep,
43 libgnunetstatistics_dep,
44 libgnunetcore_dep,
45 libgnunetcadet_dep,
46 libgnunetblock_dep],
47 include_directories: [incdir, configuration_inc],
48 install: true,
49 install_dir: get_option('libdir') / 'gnunet' / 'libexec')
50
diff --git a/src/setu/perf_setu_api.c b/src/setu/perf_setu_api.c
deleted file mode 100644
index 7f4d64f74..000000000
--- a/src/setu/perf_setu_api.c
+++ /dev/null
@@ -1,473 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_setu_api.c
23 * @brief testcase for setu_api.c
24 * @author Florian Dold
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_testing_lib.h"
30#include "gnunet_setu_service.h"
31#include <sys/sysinfo.h>
32#include <pthread.h>
33
34
35static struct GNUNET_PeerIdentity local_id;
36
37static struct GNUNET_HashCode app_id;
38
39static struct GNUNET_SETU_Handle *set1;
40
41static struct GNUNET_SETU_Handle *set2;
42
43static struct GNUNET_SETU_ListenHandle *listen_handle;
44
45static struct GNUNET_SETU_OperationHandle *oh1;
46
47static struct GNUNET_SETU_OperationHandle *oh2;
48
49static const struct GNUNET_CONFIGURATION_Handle *config;
50
51static int ret;
52
53static struct GNUNET_SCHEDULER_Task *tt;
54
55
56/**
57 * Handles configuration file for setu performance test
58 *
59 */
60static struct GNUNET_CONFIGURATION_Handle *setu_cfg;
61
62
63static void
64result_cb_set1 (void *cls,
65 const struct GNUNET_SETU_Element *element,
66 uint64_t size,
67 enum GNUNET_SETU_Status status)
68{
69 switch (status)
70 {
71 case GNUNET_SETU_STATUS_ADD_LOCAL:
72 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: got element\n");
73 break;
74
75 case GNUNET_SETU_STATUS_FAILURE:
76 GNUNET_break (0);
77 oh1 = NULL;
78 fprintf (stderr, "set 1: received failure status!\n");
79 ret = 1;
80 if (NULL != tt)
81 {
82 GNUNET_SCHEDULER_cancel (tt);
83 tt = NULL;
84 }
85 GNUNET_SCHEDULER_shutdown ();
86 break;
87
88 case GNUNET_SETU_STATUS_DONE:
89 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: done\n");
90 oh1 = NULL;
91 if (NULL != set1)
92 {
93 GNUNET_SETU_destroy (set1);
94 set1 = NULL;
95 }
96 if (NULL == set2)
97 {
98 GNUNET_SCHEDULER_cancel (tt);
99 tt = NULL;
100 GNUNET_SCHEDULER_shutdown ();
101 }
102 break;
103
104 default:
105 GNUNET_assert (0);
106 }
107}
108
109
110static void
111result_cb_set2 (void *cls,
112 const struct GNUNET_SETU_Element *element,
113 uint64_t size,
114 enum GNUNET_SETU_Status status)
115{
116 switch (status)
117 {
118 case GNUNET_SETU_STATUS_ADD_LOCAL:
119 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: got element\n");
120 break;
121
122 case GNUNET_SETU_STATUS_FAILURE:
123 GNUNET_break (0);
124 oh2 = NULL;
125 fprintf (stderr, "set 2: received failure status\n");
126 GNUNET_SCHEDULER_shutdown ();
127 ret = 1;
128 break;
129
130 case GNUNET_SETU_STATUS_DONE:
131 oh2 = NULL;
132 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: done\n");
133 GNUNET_SETU_destroy (set2);
134 set2 = NULL;
135 if (NULL == set1)
136 {
137 GNUNET_SCHEDULER_cancel (tt);
138 tt = NULL;
139 GNUNET_SCHEDULER_shutdown ();
140 }
141 break;
142
143 default:
144 GNUNET_assert (0);
145 }
146}
147
148
149static void
150listen_cb (void *cls,
151 const struct GNUNET_PeerIdentity *other_peer,
152 const struct GNUNET_MessageHeader *context_msg,
153 struct GNUNET_SETU_Request *request)
154{
155 GNUNET_assert (NULL != context_msg);
156 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
157 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "listen cb called\n");
158 oh2 = GNUNET_SETU_accept (request,
159 (struct GNUNET_SETU_Option[]){ 0 },
160 &result_cb_set2,
161 NULL);
162 GNUNET_SETU_commit (oh2, set2);
163}
164
165
166/**
167 * Start the set operation.
168 *
169 * @param cls closure, unused
170 */
171static void
172start (void *cls)
173{
174 struct GNUNET_MessageHeader context_msg;
175
176 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting reconciliation\n");
177 context_msg.size = htons (sizeof context_msg);
178 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
179 listen_handle = GNUNET_SETU_listen (config,
180 &app_id,
181 &listen_cb,
182 NULL);
183 oh1 = GNUNET_SETU_prepare (&local_id,
184 &app_id,
185 &context_msg,
186 (struct GNUNET_SETU_Option[]){ 0 },
187 &result_cb_set1,
188 NULL);
189 GNUNET_SETU_commit (oh1, set1);
190}
191
192
193/**
194 * Generate random byte stream
195 */
196
197unsigned char *
198gen_rdm_bytestream (size_t num_bytes)
199{
200 unsigned char *stream = GNUNET_malloc (num_bytes);
201 GNUNET_CRYPTO_random_block (GNUNET_CRYPTO_QUALITY_WEAK, stream, num_bytes);
202 return stream;
203}
204
205
206/**
207 * Generate random sets
208 */
209
210static void
211initRandomSets (int overlap, int set1_size, int set2_size, int
212 element_size_in_bytes)
213{
214 struct GNUNET_SETU_Element element;
215 element.element_type = 0;
216
217 // Add elements to both sets
218 for (int i = 0; i < overlap; i++)
219 {
220 element.data = gen_rdm_bytestream (element_size_in_bytes);
221 element.size = element_size_in_bytes;
222 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
223 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
224 set1_size--;
225 set2_size--;
226 }
227 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in both sets\n");
228
229 // Add other elements to set 1
230 while (set1_size>0)
231 {
232 element.data = gen_rdm_bytestream (element_size_in_bytes);
233 element.size = element_size_in_bytes;
234 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
235 set1_size--;
236 }
237 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in set1\n");
238
239 // Add other elements to set 2
240 while (set2_size > 0)
241 {
242 element.data = gen_rdm_bytestream (element_size_in_bytes);
243 element.size = element_size_in_bytes;
244
245 if (set2_size != 1)
246 {
247 GNUNET_SETU_add_element (set2, &element,NULL, NULL);
248 }
249 else
250 {
251 GNUNET_SETU_add_element (set2, &element,&start, NULL);
252 }
253
254 set2_size--;
255 }
256 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized elements in set2\n");
257}
258
259
260/**
261 * Function run on timeout.
262 *
263 * @param cls closure
264 */
265static void
266timeout_fail (void *cls)
267{
268 tt = NULL;
269 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, "Testcase failed with timeout\n");
270 GNUNET_SCHEDULER_shutdown ();
271 ret = 1;
272}
273
274
275/**
276 * Function run on shutdown.
277 *
278 * @param cls closure
279 */
280static void
281do_shutdown (void *cls)
282{
283 if (NULL != tt)
284 {
285 GNUNET_SCHEDULER_cancel (tt);
286 tt = NULL;
287 }
288 if (NULL != oh1)
289 {
290 GNUNET_SETU_operation_cancel (oh1);
291 oh1 = NULL;
292 }
293 if (NULL != oh2)
294 {
295 GNUNET_SETU_operation_cancel (oh2);
296 oh2 = NULL;
297 }
298 if (NULL != set1)
299 {
300 GNUNET_SETU_destroy (set1);
301 set1 = NULL;
302 }
303 if (NULL != set2)
304 {
305 GNUNET_SETU_destroy (set2);
306 set2 = NULL;
307 }
308 if (NULL != listen_handle)
309 {
310 GNUNET_SETU_listen_cancel (listen_handle);
311 listen_handle = NULL;
312 }
313}
314
315
316/**
317 * Signature of the 'main' function for a (single-peer) testcase that
318 * is run using 'GNUNET_TESTING_peer_run'.
319 *
320 * @param cls closure
321 * @param cfg configuration of the peer that was started
322 * @param peer identity of the peer that was created
323 */
324static void
325run (void *cls,
326 const struct GNUNET_CONFIGURATION_Handle *cfg,
327 struct GNUNET_TESTING_Peer *peer)
328{
329 struct GNUNET_SETU_OperationHandle *my_oh;
330
331 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
332 "Running preparatory tests\n");
333 tt = GNUNET_SCHEDULER_add_delayed (
334 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
335 &timeout_fail,
336 NULL);
337 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
338
339 config = cfg;
340 GNUNET_assert (GNUNET_OK == GNUNET_CRYPTO_get_peer_identity (cfg,
341 &local_id));
342 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
343 "my id (from CRYPTO): %s\n",
344 GNUNET_i2s (&local_id));
345 GNUNET_TESTING_peer_get_identity (peer,
346 &local_id);
347 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
348 "my id (from TESTING): %s\n",
349 GNUNET_i2s (&local_id));
350 set1 = GNUNET_SETU_create (cfg);
351 set2 = GNUNET_SETU_create (cfg);
352 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
353 "Created sets %p and %p for union operation\n",
354 set1,
355 set2);
356 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
357
358 /* test if canceling an uncommitted request works! */
359 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
360 "Launching and instantly stopping set operation\n");
361 my_oh = GNUNET_SETU_prepare (&local_id,
362 &app_id,
363 NULL,
364 (struct GNUNET_SETU_Option[]){ 0 },
365 NULL,
366 NULL);
367 GNUNET_SETU_operation_cancel (my_oh);
368
369 /* test the real set reconciliation */
370 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
371 "Running real set-reconciliation\n");
372 // init_set1 ();
373 // limit ~23800 element total
374 initRandomSets (490, 500,500,32);
375}
376
377
378void
379perf_thread ()
380{
381 GNUNET_TESTING_service_run ("perf_setu_api",
382 "arm",
383 "test_setu.conf",
384 &run,
385 NULL);
386
387}
388
389
390static void
391run_petf_thread (int total_runs)
392{
393 int core_count = get_nprocs_conf ();
394 pid_t child_pid, wpid;
395 int status = 0;
396
397// Father code (before child processes start)
398 for (int processed = 0; processed < total_runs;)
399 {
400 for (int id = 0; id < core_count; id++)
401 {
402 if (processed >= total_runs)
403 break;
404
405 if ((child_pid = fork ()) == 0)
406 {
407 perf_thread ();
408 exit (0);
409 }
410 processed += 1;
411 }
412 while ((wpid = wait (&status)) > 0)
413 ;
414
415 }
416}
417
418
419static void
420execute_perf ()
421{
422
423 /**
424 * Erase statfile
425 */
426 remove ("perf_stats.csv");
427 remove ("perf_failure_bucket_number_factor.csv");
428 for (int out_out_ctr = 3; out_out_ctr <= 3; out_out_ctr++)
429 {
430
431 for (int out_ctr = 20; out_ctr <= 20; out_ctr++)
432 {
433 float base = 0.1;
434 float x = out_ctr * base;
435 char factor[10];
436 char *buffer = gcvt (x, 4, factor);
437 setu_cfg = GNUNET_CONFIGURATION_create ();
438 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "IBF",
439 "BUCKET_NUMBER_FACTOR",
440 buffer); // Factor default=4
441 GNUNET_CONFIGURATION_set_value_number (setu_cfg, "IBF",
442 "NUMBER_PER_BUCKET", 3); // K default=4
443 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "PERFORMANCE",
444 "TRADEOFF", "2"); // default=0.25
445 GNUNET_CONFIGURATION_set_value_string (setu_cfg, "PERFORMANCE",
446 "MAX_SET_DIFF_FACTOR_DIFFERENTIAL",
447 "20000"); // default=0.25
448 GNUNET_CONFIGURATION_set_value_number (setu_cfg, "BOUNDARIES",
449 "UPPER_ELEMENT", 5000);
450
451
452 if (GNUNET_OK != GNUNET_CONFIGURATION_write (setu_cfg, "perf_setu.conf"))
453 GNUNET_log (
454 GNUNET_ERROR_TYPE_ERROR,
455 _ ("Failed to write subsystem default identifier map'.\n"));
456 run_petf_thread (100);
457 }
458
459 }
460 return;
461}
462
463
464int
465main (int argc, char **argv)
466{
467
468 GNUNET_log_setup ("perf_setu_api",
469 "WARNING",
470 NULL);
471 execute_perf ();
472 return 0;
473}
diff --git a/src/setu/plugin_block_setu_test.c b/src/setu/plugin_block_setu_test.c
deleted file mode 100644
index 178ad3314..000000000
--- a/src/setu/plugin_block_setu_test.c
+++ /dev/null
@@ -1,195 +0,0 @@
1/*
2 This file is part of GNUnet
3 Copyright (C) 2017 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file setu/plugin_block_setu_test.c
23 * @brief set test block, recognizes elements with non-zero first byte as invalid
24 * @author Christian Grothoff
25 */
26#include "platform.h"
27#include "gnunet_block_plugin.h"
28#include "gnunet_block_group_lib.h"
29
30
31/**
32 * Function called to validate a query.
33 *
34 * @param cls closure
35 * @param ctx block context
36 * @param type block type
37 * @param query original query (hash)
38 * @param xquery extrended query data (can be NULL, depending on type)
39 * @param xquery_size number of bytes in @a xquery
40 * @return #GNUNET_OK if the query is fine, #GNUNET_NO if not
41 */
42static enum GNUNET_GenericReturnValue
43block_plugin_setu_test_check_query (void *cls,
44 enum GNUNET_BLOCK_Type type,
45 const struct GNUNET_HashCode *query,
46 const void *xquery,
47 size_t xquery_size)
48{
49 if (GNUNET_BLOCK_TYPE_SETU_TEST != type)
50 {
51 GNUNET_break (0);
52 return GNUNET_SYSERR;
53 }
54 if (0 != xquery_size)
55 {
56 GNUNET_break_op (0);
57 return GNUNET_NO;
58 }
59 return GNUNET_OK;
60}
61
62
63/**
64 * Function called to validate a block for storage.
65 *
66 * @param cls closure
67 * @param type block type
68 * @param block block data to validate
69 * @param block_size number of bytes in @a block
70 * @return #GNUNET_OK if the block is fine, #GNUNET_NO if not
71 */
72static enum GNUNET_GenericReturnValue
73block_plugin_setu_test_check_block (void *cls,
74 enum GNUNET_BLOCK_Type type,
75 const void *block,
76 size_t block_size)
77{
78 if (GNUNET_BLOCK_TYPE_SETU_TEST != type)
79 {
80 GNUNET_break (0);
81 return GNUNET_SYSERR;
82 }
83 if ( (NULL == block) ||
84 (0 == block_size) ||
85 (0 != ((char *) block)[0]) )
86 return GNUNET_NO;
87 return GNUNET_OK;
88}
89
90
91/**
92 * Function called to validate a reply to a request. Note that it is assumed
93 * that the reply has already been matched to the key (and signatures checked)
94 * as it would be done with the GetKeyFunction and the
95 * BlockEvaluationFunction.
96 *
97 * @param cls closure
98 * @param type block type
99 * @param group which block group to use for evaluation
100 * @param query original query (hash)
101 * @param xquery extrended query data (can be NULL, depending on type)
102 * @param xquery_size number of bytes in @a xquery
103 * @param reply_block response to validate
104 * @param reply_block_size number of bytes in @a reply_block
105 * @return characterization of result
106 */
107static enum GNUNET_BLOCK_ReplyEvaluationResult
108block_plugin_setu_test_check_reply (void *cls,
109 enum GNUNET_BLOCK_Type type,
110 struct GNUNET_BLOCK_Group *group,
111 const struct GNUNET_HashCode *query,
112 const void *xquery,
113 size_t xquery_size,
114 const void *reply_block,
115 size_t reply_block_size)
116{
117 (void) cls;
118 (void) xquery;
119 (void) xquery_size;
120 if (GNUNET_BLOCK_TYPE_SETU_TEST != type)
121 {
122 GNUNET_break (0);
123 return GNUNET_BLOCK_REPLY_TYPE_NOT_SUPPORTED;
124 }
125 if ( (NULL == reply_block) ||
126 (0 == reply_block_size) ||
127 (0 != ((char *) reply_block)[0]) )
128 GNUNET_assert (0);
129 return GNUNET_BLOCK_REPLY_OK_MORE;
130}
131
132
133/**
134 * Function called to obtain the key for a block.
135 *
136 * @param cls closure
137 * @param type block type
138 * @param block block to get the key for
139 * @param block_size number of bytes in block
140 * @param key set to the key (query) for the given block
141 * @return #GNUNET_OK on success, #GNUNET_SYSERR if type not supported
142 * (or if extracting a key from a block of this type does not work)
143 */
144static enum GNUNET_GenericReturnValue
145block_plugin_setu_test_get_key (void *cls,
146 enum GNUNET_BLOCK_Type type,
147 const void *block,
148 size_t block_size,
149 struct GNUNET_HashCode *key)
150{
151 if (GNUNET_BLOCK_TYPE_SETU_TEST != type)
152 {
153 GNUNET_break (0);
154 return GNUNET_SYSERR;
155 }
156 return GNUNET_NO;
157}
158
159
160/**
161 * Entry point for the plugin.
162 */
163void *
164libgnunet_plugin_block_setu_test_init (void *cls)
165{
166 static enum GNUNET_BLOCK_Type types[] = {
167 GNUNET_BLOCK_TYPE_SETU_TEST,
168 GNUNET_BLOCK_TYPE_ANY /* end of list */
169 };
170 struct GNUNET_BLOCK_PluginFunctions *api;
171
172 api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions);
173 api->get_key = &block_plugin_setu_test_get_key;
174 api->check_query = &block_plugin_setu_test_check_query;
175 api->check_block = &block_plugin_setu_test_check_block;
176 api->check_reply = &block_plugin_setu_test_check_reply;
177 api->types = types;
178 return api;
179}
180
181
182/**
183 * Exit point from the plugin.
184 */
185void *
186libgnunet_plugin_block_setu_test_done (void *cls)
187{
188 struct GNUNET_BLOCK_PluginFunctions *api = cls;
189
190 GNUNET_free (api);
191 return NULL;
192}
193
194
195/* end of plugin_block_setu_test.c */
diff --git a/src/setu/setu.conf.in b/src/setu/setu.conf.in
deleted file mode 100644
index 6c48f6156..000000000
--- a/src/setu/setu.conf.in
+++ /dev/null
@@ -1,12 +0,0 @@
1[setu]
2START_ON_DEMAND = @START_ON_DEMAND@
3@UNIXONLY@PORT = 2106
4HOSTNAME = localhost
5BINARY = gnunet-service-setu
6ACCEPT_FROM = 127.0.0.1;
7ACCEPT_FROM6 = ::1;
8UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-setu.sock
9UNIX_MATCH_UID = YES
10UNIX_MATCH_GID = YES
11
12#PREFIX = valgrind
diff --git a/src/setu/setu.h b/src/setu/setu.h
deleted file mode 100644
index 7b606f12c..000000000
--- a/src/setu/setu.h
+++ /dev/null
@@ -1,365 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2014, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/set.h
22 * @brief messages used for the set union api
23 * @author Florian Dold
24 * @author Christian Grothoff
25 */
26#ifndef SET_H
27#define SET_H
28
29#include "platform.h"
30#include "gnunet_common.h"
31#include "gnunet_set_service.h"
32
33GNUNET_NETWORK_STRUCT_BEGIN
34
35/**
36 * Message sent by the client to the service to ask starting
37 * a new set to perform operations with. Includes the desired
38 * set operation type.
39 */
40struct GNUNET_SETU_CreateMessage
41{
42 /**
43 * Type: #GNUNET_MESSAGE_TYPE_SETU_CREATE
44 */
45 struct GNUNET_MessageHeader header;
46
47};
48
49
50/**
51 * Message sent by the client to the service to start listening for
52 * incoming requests to perform a certain type of set operation for a
53 * certain type of application.
54 */
55struct GNUNET_SETU_ListenMessage
56{
57 /**
58 * Type: #GNUNET_MESSAGE_TYPE_SETU_LISTEN
59 */
60 struct GNUNET_MessageHeader header;
61
62 /**
63 * Always zero.
64 */
65 uint32_t reserved GNUNET_PACKED;
66
67 /**
68 * application id
69 */
70 struct GNUNET_HashCode app_id;
71};
72
73
74/**
75 * Message sent by a listening client to the service to accept
76 * performing the operation with the other peer.
77 */
78struct GNUNET_SETU_AcceptMessage
79{
80 /**
81 * Type: #GNUNET_MESSAGE_TYPE_SETU_ACCEPT
82 */
83 struct GNUNET_MessageHeader header;
84
85 /**
86 * ID of the incoming request we want to accept.
87 */
88 uint32_t accept_reject_id GNUNET_PACKED;
89
90 /**
91 * Request ID to identify responses.
92 */
93 uint32_t request_id GNUNET_PACKED;
94
95 /**
96 * Always use delta operation instead of sending full sets,
97 * even it it's less efficient.
98 */
99 uint8_t force_delta;
100
101 /**
102 * Always send full sets, even if delta operations would
103 * be more efficient.
104 */
105 uint8_t force_full;
106
107 /**
108 * #GNUNET_YES to fail operations where Byzantine faults
109 * are suspected
110 */
111 uint8_t byzantine;
112
113 /**
114 * #GNUNET_YES to also send back set elements we are sending to
115 * the remote peer.
116 */
117 uint8_t symmetric;
118
119 /**
120 * Lower bound for the set size, used only when
121 * byzantine mode is enabled.
122 */
123 uint32_t byzantine_lower_bound;
124
125
126 /**
127 * Upper bound for the set size, used only when
128 * byzantine mode is enabled.
129 */
130 uint64_t byzantine_upper_bond;
131
132 /**
133 * Bandwidth latency tradeoff determines how much bytes a single RTT is
134 * worth, which is a performance setting
135 */
136 uint64_t bandwidth_latency_tradeoff;
137
138 /**
139 * The factor determines the number of buckets an IBF has which is
140 * multiplied by the estimated setsize default: 2
141 */
142 uint64_t ibf_bucket_number_factor;
143
144 /**
145 * This setting determines to how many IBF buckets an single elements
146 * is mapped to.
147 */
148 uint64_t ibf_number_of_buckets_per_element;
149
150};
151
152
153/**
154 * Message sent by a listening client to the service to reject
155 * performing the operation with the other peer.
156 */
157struct GNUNET_SETU_RejectMessage
158{
159 /**
160 * Type: #GNUNET_MESSAGE_TYPE_SETU_REJECT
161 */
162 struct GNUNET_MessageHeader header;
163
164 /**
165 * ID of the incoming request we want to reject.
166 */
167 uint32_t accept_reject_id GNUNET_PACKED;
168};
169
170
171/**
172 * A request for an operation with another client.
173 */
174struct GNUNET_SETU_RequestMessage
175{
176 /**
177 * Type: #GNUNET_MESSAGE_TYPE_SETU_REQUEST.
178 */
179 struct GNUNET_MessageHeader header;
180
181 /**
182 * ID of the to identify the request when accepting or
183 * rejecting it.
184 */
185 uint32_t accept_id GNUNET_PACKED;
186
187 /**
188 * Identity of the requesting peer.
189 */
190 struct GNUNET_PeerIdentity peer_id;
191
192 /* rest: context message, that is, application-specific
193 message to convince listener to pick up */
194};
195
196
197/**
198 * Message sent by client to service to initiate a set operation as a
199 * client (not as listener). A set (which determines the operation
200 * type) must already exist in association with this client.
201 */
202struct GNUNET_SETU_EvaluateMessage
203{
204 /**
205 * Type: #GNUNET_MESSAGE_TYPE_SETU_EVALUATE
206 */
207 struct GNUNET_MessageHeader header;
208
209 /**
210 * Id of our set to evaluate, chosen implicitly by the client when it
211 * calls #GNUNET_SETU_commit().
212 */
213 uint32_t request_id GNUNET_PACKED;
214
215 /**
216 * Peer to evaluate the operation with
217 */
218 struct GNUNET_PeerIdentity target_peer;
219
220 /**
221 * Application id
222 */
223 struct GNUNET_HashCode app_id;
224
225 /**
226 * Always use delta operation instead of sending full sets,
227 * even it it's less efficient.
228 */
229 uint8_t force_delta;
230
231 /**
232 * Always send full sets, even if delta operations would
233 * be more efficient.
234 */
235 uint8_t force_full;
236
237 /**
238 * #GNUNET_YES to fail operations where Byzantine faults
239 * are suspected
240 */
241 uint8_t byzantine;
242
243 /**
244 * Also return set elements we are sending to the remote peer.
245 */
246 uint8_t symmetric;
247
248 /**
249 * Lower bound for the set size, used only when
250 * byzantine mode is enabled.
251 */
252 uint32_t byzantine_lower_bound;
253
254 /**
255 * Upper bound for the set size, used only when
256 * byzantine mode is enabled.
257 */
258 uint64_t byzantine_upper_bond;
259
260 /**
261 * Bandwidth latency tradeoff determines how much bytes a single RTT is
262 * worth, which is a performance setting
263 */
264 uint64_t bandwidth_latency_tradeoff;
265
266 /**
267 * The factor determines the number of buckets an IBF has which is
268 * multiplied by the estimated setsize default: 2
269 */
270 uint64_t ibf_bucket_number_factor;
271
272 /**
273 * This setting determines to how many IBF buckets an single elements
274 * is mapped to.
275 */
276 uint64_t ibf_number_of_buckets_per_element;
277
278 /* rest: context message, that is, application-specific
279 message to convince listener to pick up */
280};
281
282
283/**
284 * Message sent by the service to the client to indicate an
285 * element that is removed (set intersection) or added
286 * (set union) or part of the final result, depending on
287 * options specified for the operation.
288 */
289struct GNUNET_SETU_ResultMessage
290{
291 /**
292 * Type: #GNUNET_MESSAGE_TYPE_SETU_RESULT
293 */
294 struct GNUNET_MessageHeader header;
295
296 /**
297 * Current set size.
298 */
299 uint64_t current_size;
300
301 /**
302 * id the result belongs to
303 */
304 uint32_t request_id GNUNET_PACKED;
305
306 /**
307 * Was the evaluation successful? Contains
308 * an `enum GNUNET_SETU_Status` in NBO.
309 */
310 uint16_t result_status GNUNET_PACKED;
311
312 /**
313 * Type of the element attached to the message, if any.
314 */
315 uint16_t element_type GNUNET_PACKED;
316
317 /* rest: the actual element */
318};
319
320
321/**
322 * Message sent by client to the service to add
323 * an element to the set.
324 */
325struct GNUNET_SETU_ElementMessage
326{
327 /**
328 * Type: #GNUNET_MESSAGE_TYPE_SETU_ADD
329 */
330 struct GNUNET_MessageHeader header;
331
332 /**
333 * Type of the element to add or remove.
334 */
335 uint16_t element_type GNUNET_PACKED;
336
337 /**
338 * For alignment, always zero.
339 */
340 uint16_t reserved GNUNET_PACKED;
341
342 /* rest: the actual element */
343};
344
345
346/**
347 * Sent to the service by the client in order to cancel a set operation.
348 */
349struct GNUNET_SETU_CancelMessage
350{
351 /**
352 * Type: #GNUNET_MESSAGE_TYPE_SETU_CANCEL
353 */
354 struct GNUNET_MessageHeader header;
355
356 /**
357 * ID of the request we want to cancel.
358 */
359 uint32_t request_id GNUNET_PACKED;
360};
361
362
363GNUNET_NETWORK_STRUCT_END
364
365#endif
diff --git a/src/setu/setu_api.c b/src/setu/setu_api.c
deleted file mode 100644
index 7fa144590..000000000
--- a/src/setu/setu_api.c
+++ /dev/null
@@ -1,911 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012-2016, 2020 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20/**
21 * @file set/setu_api.c
22 * @brief api for the set union service
23 * @author Florian Dold
24 * @author Christian Grothoff
25 * @author Elias Summermatter
26 */
27#include "platform.h"
28#include "gnunet_util_lib.h"
29#include "gnunet_protocols.h"
30#include "gnunet_setu_service.h"
31#include "setu.h"
32
33
34#define LOG(kind, ...) GNUNET_log_from (kind, "set-api", __VA_ARGS__)
35
36/**
37 * Opaque handle to a set.
38 */
39struct GNUNET_SETU_Handle
40{
41 /**
42 * Message queue for @e client.
43 */
44 struct GNUNET_MQ_Handle *mq;
45
46 /**
47 * Linked list of operations on the set.
48 */
49 struct GNUNET_SETU_OperationHandle *ops_head;
50
51 /**
52 * Linked list of operations on the set.
53 */
54 struct GNUNET_SETU_OperationHandle *ops_tail;
55
56 /**
57 * Should the set be destroyed once all operations are gone?
58 * #GNUNET_SYSERR if #GNUNET_SETU_destroy() must raise this flag,
59 * #GNUNET_YES if #GNUNET_SETU_destroy() did raise this flag.
60 */
61 int destroy_requested;
62
63 /**
64 * Has the set become invalid (e.g. service died)?
65 */
66 int invalid;
67
68};
69
70
71/**
72 * Handle for a set operation request from another peer.
73 */
74struct GNUNET_SETU_Request
75{
76 /**
77 * Id of the request, used to identify the request when
78 * accepting/rejecting it.
79 */
80 uint32_t accept_id;
81
82 /**
83 * Has the request been accepted already?
84 * #GNUNET_YES/#GNUNET_NO
85 */
86 int accepted;
87};
88
89
90/**
91 * Handle to an operation. Only known to the service after committing
92 * the handle with a set.
93 */
94struct GNUNET_SETU_OperationHandle
95{
96 /**
97 * Function to be called when we have a result,
98 * or an error.
99 */
100 GNUNET_SETU_ResultIterator result_cb;
101
102 /**
103 * Closure for @e result_cb.
104 */
105 void *result_cls;
106
107 /**
108 * Local set used for the operation,
109 * NULL if no set has been provided by conclude yet.
110 */
111 struct GNUNET_SETU_Handle *set;
112
113 /**
114 * Message sent to the server on calling conclude,
115 * NULL if conclude has been called.
116 */
117 struct GNUNET_MQ_Envelope *conclude_mqm;
118
119 /**
120 * Address of the request if in the conclude message,
121 * used to patch the request id into the message when the set is known.
122 */
123 uint32_t *request_id_addr;
124
125 /**
126 * Handles are kept in a linked list.
127 */
128 struct GNUNET_SETU_OperationHandle *prev;
129
130 /**
131 * Handles are kept in a linked list.
132 */
133 struct GNUNET_SETU_OperationHandle *next;
134
135 /**
136 * Request ID to identify the operation within the set.
137 */
138 uint32_t request_id;
139};
140
141
142/**
143 * Opaque handle to a listen operation.
144 */
145struct GNUNET_SETU_ListenHandle
146{
147 /**
148 * Message queue for the client.
149 */
150 struct GNUNET_MQ_Handle*mq;
151
152 /**
153 * Configuration handle for the listener, stored
154 * here to be able to reconnect transparently on
155 * connection failure.
156 */
157 const struct GNUNET_CONFIGURATION_Handle *cfg;
158
159 /**
160 * Function to call on a new incoming request,
161 * or on error.
162 */
163 GNUNET_SETU_ListenCallback listen_cb;
164
165 /**
166 * Closure for @e listen_cb.
167 */
168 void *listen_cls;
169
170 /**
171 * Application ID we listen for.
172 */
173 struct GNUNET_HashCode app_id;
174
175 /**
176 * Time to wait until we try to reconnect on failure.
177 */
178 struct GNUNET_TIME_Relative reconnect_backoff;
179
180 /**
181 * Task for reconnecting when the listener fails.
182 */
183 struct GNUNET_SCHEDULER_Task *reconnect_task;
184
185};
186
187
188/**
189 * Check that the given @a msg is well-formed.
190 *
191 * @param cls closure
192 * @param msg message to check
193 * @return #GNUNET_OK if message is well-formed
194 */
195static int
196check_result (void *cls,
197 const struct GNUNET_SETU_ResultMessage *msg)
198{
199 /* minimum size was already checked, everything else is OK! */
200 return GNUNET_OK;
201}
202
203
204/**
205 * Handle result message for a set operation.
206 *
207 * @param cls the set
208 * @param mh the message
209 */
210static void
211handle_result (void *cls,
212 const struct GNUNET_SETU_ResultMessage *msg)
213{
214 struct GNUNET_SETU_Handle *set = cls;
215 struct GNUNET_SETU_OperationHandle *oh;
216 struct GNUNET_SETU_Element e;
217 enum GNUNET_SETU_Status result_status;
218 int destroy_set;
219
220 GNUNET_assert (NULL != set->mq);
221 result_status = (enum GNUNET_SETU_Status) ntohs (msg->result_status);
222 LOG (GNUNET_ERROR_TYPE_DEBUG,
223 "Got result message with status %d\n",
224 result_status);
225 oh = GNUNET_MQ_assoc_get (set->mq,
226 ntohl (msg->request_id));
227 if (NULL == oh)
228 {
229 /* 'oh' can be NULL if we canceled the operation, but the service
230 did not get the cancel message yet. */
231 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
232 "Ignoring result from canceled operation\n");
233 return;
234 }
235
236 switch (result_status)
237 {
238 case GNUNET_SETU_STATUS_ADD_LOCAL:
239 case GNUNET_SETU_STATUS_ADD_REMOTE:
240 e.data = &msg[1];
241 e.size = ntohs (msg->header.size)
242 - sizeof(struct GNUNET_SETU_ResultMessage);
243 e.element_type = ntohs (msg->element_type);
244 if (NULL != oh->result_cb)
245 oh->result_cb (oh->result_cls,
246 &e,
247 GNUNET_ntohll (msg->current_size),
248 result_status);
249 return;
250 case GNUNET_SETU_STATUS_FAILURE:
251 case GNUNET_SETU_STATUS_DONE:
252 LOG (GNUNET_ERROR_TYPE_DEBUG,
253 "Treating result as final status\n");
254 GNUNET_MQ_assoc_remove (set->mq,
255 ntohl (msg->request_id));
256 GNUNET_CONTAINER_DLL_remove (set->ops_head,
257 set->ops_tail,
258 oh);
259 /* Need to do this calculation _before_ the result callback,
260 as IF the application still has a valid set handle, it
261 may trigger destruction of the set during the callback. */
262 destroy_set = (GNUNET_YES == set->destroy_requested) &&
263 (NULL == set->ops_head);
264 if (NULL != oh->result_cb)
265 {
266 oh->result_cb (oh->result_cls,
267 NULL,
268 GNUNET_ntohll (msg->current_size),
269 result_status);
270 }
271 else
272 {
273 LOG (GNUNET_ERROR_TYPE_DEBUG,
274 "No callback for final status\n");
275 }
276 if (destroy_set)
277 GNUNET_SETU_destroy (set);
278 GNUNET_free (oh);
279 return;
280 }
281}
282
283
284/**
285 * Destroy the given set operation.
286 *
287 * @param oh set operation to destroy
288 */
289static void
290set_operation_destroy (struct GNUNET_SETU_OperationHandle *oh)
291{
292 struct GNUNET_SETU_Handle *set = oh->set;
293 struct GNUNET_SETU_OperationHandle *h_assoc;
294
295 if (NULL != oh->conclude_mqm)
296 GNUNET_MQ_discard (oh->conclude_mqm);
297 /* is the operation already committed? */
298 if (NULL != set)
299 {
300 GNUNET_CONTAINER_DLL_remove (set->ops_head,
301 set->ops_tail,
302 oh);
303 h_assoc = GNUNET_MQ_assoc_remove (set->mq,
304 oh->request_id);
305 GNUNET_assert ((NULL == h_assoc) ||
306 (h_assoc == oh));
307 }
308 GNUNET_free (oh);
309}
310
311
312/**
313 * Cancel the given set operation. We need to send an explicit cancel
314 * message, as all operations one one set communicate using one
315 * handle.
316 *
317 * @param oh set operation to cancel
318 */
319void
320GNUNET_SETU_operation_cancel (struct GNUNET_SETU_OperationHandle *oh)
321{
322 struct GNUNET_SETU_Handle *set = oh->set;
323 struct GNUNET_SETU_CancelMessage *m;
324 struct GNUNET_MQ_Envelope *mqm;
325
326 LOG (GNUNET_ERROR_TYPE_DEBUG,
327 "Cancelling SET operation\n");
328 if (NULL != set)
329 {
330 mqm = GNUNET_MQ_msg (m, GNUNET_MESSAGE_TYPE_SETU_CANCEL);
331 m->request_id = htonl (oh->request_id);
332 GNUNET_MQ_send (set->mq, mqm);
333 }
334 set_operation_destroy (oh);
335 if ((NULL != set) &&
336 (GNUNET_YES == set->destroy_requested) &&
337 (NULL == set->ops_head))
338 {
339 LOG (GNUNET_ERROR_TYPE_DEBUG,
340 "Destroying set after operation cancel\n");
341 GNUNET_SETU_destroy (set);
342 }
343}
344
345
346/**
347 * We encountered an error communicating with the set service while
348 * performing a set operation. Report to the application.
349 *
350 * @param cls the `struct GNUNET_SETU_Handle`
351 * @param error error code
352 */
353static void
354handle_client_set_error (void *cls,
355 enum GNUNET_MQ_Error error)
356{
357 struct GNUNET_SETU_Handle *set = cls;
358
359 LOG (GNUNET_ERROR_TYPE_ERROR,
360 "Handling client set error %d\n",
361 error);
362 while (NULL != set->ops_head)
363 {
364 if ((NULL != set->ops_head->result_cb) &&
365 (GNUNET_NO == set->destroy_requested))
366 set->ops_head->result_cb (set->ops_head->result_cls,
367 NULL,
368 0,
369 GNUNET_SETU_STATUS_FAILURE);
370 set_operation_destroy (set->ops_head);
371 }
372 set->invalid = GNUNET_YES;
373}
374
375
376/**
377 * Create an empty set, supporting the specified operation.
378 *
379 * @param cfg configuration to use for connecting to the
380 * set service
381 * @return a handle to the set
382 */
383struct GNUNET_SETU_Handle *
384GNUNET_SETU_create (const struct GNUNET_CONFIGURATION_Handle *cfg)
385{
386 struct GNUNET_SETU_Handle *set = GNUNET_new (struct GNUNET_SETU_Handle);
387 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
388 GNUNET_MQ_hd_var_size (result,
389 GNUNET_MESSAGE_TYPE_SETU_RESULT,
390 struct GNUNET_SETU_ResultMessage,
391 set),
392 GNUNET_MQ_handler_end ()
393 };
394 struct GNUNET_MQ_Envelope *mqm;
395 struct GNUNET_SETU_CreateMessage *create_msg;
396
397 set->mq = GNUNET_CLIENT_connect (cfg,
398 "setu",
399 mq_handlers,
400 &handle_client_set_error,
401 set);
402 if (NULL == set->mq)
403 {
404 GNUNET_free (set);
405 return NULL;
406 }
407 mqm = GNUNET_MQ_msg (create_msg,
408 GNUNET_MESSAGE_TYPE_SETU_CREATE);
409 GNUNET_MQ_send (set->mq,
410 mqm);
411 return set;
412}
413
414
415/**
416 * Add an element to the given set. After the element has been added
417 * (in the sense of being transmitted to the set service), @a cont
418 * will be called. Multiple calls to GNUNET_SETU_add_element() can be
419 * queued.
420 *
421 * @param set set to add element to
422 * @param element element to add to the set
423 * @param cb continuation called after the element has been added
424 * @param cb_cls closure for @a cb
425 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
426 * set is invalid (e.g. the set service crashed)
427 */
428int
429GNUNET_SETU_add_element (struct GNUNET_SETU_Handle *set,
430 const struct GNUNET_SETU_Element *element,
431 GNUNET_SCHEDULER_TaskCallback cb,
432 void *cb_cls)
433{
434 struct GNUNET_MQ_Envelope *mqm;
435 struct GNUNET_SETU_ElementMessage *msg;
436
437 LOG (GNUNET_ERROR_TYPE_DEBUG,
438 "adding element of type %u to set %p\n",
439 (unsigned int) element->element_type,
440 set);
441 GNUNET_assert (NULL != set);
442 if (GNUNET_YES == set->invalid)
443 {
444 if (NULL != cb)
445 cb (cb_cls);
446 return GNUNET_SYSERR;
447 }
448 mqm = GNUNET_MQ_msg_extra (msg,
449 element->size,
450 GNUNET_MESSAGE_TYPE_SETU_ADD);
451 msg->element_type = htons (element->element_type);
452 GNUNET_memcpy (&msg[1],
453 element->data,
454 element->size);
455 GNUNET_MQ_notify_sent (mqm,
456 cb,
457 cb_cls);
458 GNUNET_MQ_send (set->mq,
459 mqm);
460 return GNUNET_OK;
461}
462
463
464/**
465 * Destroy the set handle if no operations are left, mark the set
466 * for destruction otherwise.
467 *
468 * @param set set handle to destroy
469 */
470void
471GNUNET_SETU_destroy (struct GNUNET_SETU_Handle *set)
472{
473 /* destroying set while iterator is active is currently
474 not supported; we should expand the API to allow
475 clients to explicitly cancel the iteration! */
476 GNUNET_assert (NULL != set);
477 if ((NULL != set->ops_head) ||
478 (GNUNET_SYSERR == set->destroy_requested))
479 {
480 LOG (GNUNET_ERROR_TYPE_DEBUG,
481 "Set operations are pending, delaying set destruction\n");
482 set->destroy_requested = GNUNET_YES;
483 return;
484 }
485 LOG (GNUNET_ERROR_TYPE_DEBUG,
486 "Really destroying set\n");
487 if (NULL != set->mq)
488 {
489 GNUNET_MQ_destroy (set->mq);
490 set->mq = NULL;
491 }
492 GNUNET_free (set);
493}
494
495
496struct GNUNET_SETU_OperationHandle *
497GNUNET_SETU_prepare (const struct GNUNET_PeerIdentity *other_peer,
498 const struct GNUNET_HashCode *app_id,
499 const struct GNUNET_MessageHeader *context_msg,
500 const struct GNUNET_SETU_Option options[],
501 GNUNET_SETU_ResultIterator result_cb,
502 void *result_cls)
503{
504 struct GNUNET_MQ_Envelope *mqm;
505 struct GNUNET_SETU_OperationHandle *oh;
506 struct GNUNET_SETU_EvaluateMessage *msg;
507
508 LOG (GNUNET_ERROR_TYPE_DEBUG,
509 "Client prepares set union operation\n");
510 oh = GNUNET_new (struct GNUNET_SETU_OperationHandle);
511 oh->result_cb = result_cb;
512 oh->result_cls = result_cls;
513 mqm = GNUNET_MQ_msg_nested_mh (msg,
514 GNUNET_MESSAGE_TYPE_SETU_EVALUATE,
515 context_msg);
516 msg->app_id = *app_id;
517 msg->target_peer = *other_peer;
518
519 /* Set default values */
520 msg->byzantine_upper_bond = UINT64_MAX;
521 msg->bandwidth_latency_tradeoff = 0;
522 msg->ibf_bucket_number_factor = 2;
523 msg->ibf_number_of_buckets_per_element = 3;
524
525
526 for (const struct GNUNET_SETU_Option *opt = options; opt->type != 0; opt++)
527 {
528 switch (opt->type)
529 {
530 case GNUNET_SETU_OPTION_BYZANTINE:
531 msg->byzantine = GNUNET_YES;
532 msg->byzantine_lower_bound = htonl (opt->v.num);
533 break;
534 case GNUNET_SETU_OPTION_CUSTOM_BYZANTINE_UPPER_BOUND:
535 msg->byzantine_upper_bond = htonl (opt->v.num);
536 break;
537 case GNUNET_SETU_OPTION_CUSTOM_BANDWIDTH_LATENCY_TRADEOFF:
538 msg->bandwidth_latency_tradeoff = htonl (opt->v.num);
539 break;
540 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKET_NUMBER_FACTOR:
541 msg->ibf_bucket_number_factor = htonl (opt->v.num);
542 break;
543 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKETS_PER_ELEMENT:
544 msg->ibf_number_of_buckets_per_element = htonl (opt->v.num);
545 break;
546 case GNUNET_SETU_OPTION_FORCE_FULL:
547 msg->force_full = GNUNET_YES;
548 break;
549 case GNUNET_SETU_OPTION_FORCE_DELTA:
550 msg->force_delta = GNUNET_YES;
551 break;
552 case GNUNET_SETU_OPTION_SYMMETRIC:
553 msg->symmetric = GNUNET_YES;
554 break;
555 default:
556 LOG (GNUNET_ERROR_TYPE_ERROR,
557 "Option with type %d not recognized\n",
558 (int) opt->type);
559 }
560 }
561 oh->conclude_mqm = mqm;
562 oh->request_id_addr = &msg->request_id;
563 return oh;
564}
565
566
567/**
568 * Connect to the set service in order to listen for requests.
569 *
570 * @param cls the `struct GNUNET_SETU_ListenHandle *` to connect
571 */
572static void
573listen_connect (void *cls);
574
575
576/**
577 * Check validity of request message for a listen operation
578 *
579 * @param cls the listen handle
580 * @param msg the message
581 * @return #GNUNET_OK if the message is well-formed
582 */
583static int
584check_request (void *cls,
585 const struct GNUNET_SETU_RequestMessage *msg)
586{
587 const struct GNUNET_MessageHeader *context_msg;
588
589 if (ntohs (msg->header.size) == sizeof(*msg))
590 return GNUNET_OK; /* no context message is OK */
591 context_msg = GNUNET_MQ_extract_nested_mh (msg);
592 if (NULL == context_msg)
593 {
594 /* malformed context message is NOT ok */
595 GNUNET_break_op (0);
596 return GNUNET_SYSERR;
597 }
598 return GNUNET_OK;
599}
600
601
602/**
603 * Handle request message for a listen operation
604 *
605 * @param cls the listen handle
606 * @param msg the message
607 */
608static void
609handle_request (void *cls,
610 const struct GNUNET_SETU_RequestMessage *msg)
611{
612 struct GNUNET_SETU_ListenHandle *lh = cls;
613 struct GNUNET_SETU_Request req;
614 const struct GNUNET_MessageHeader *context_msg;
615 struct GNUNET_MQ_Envelope *mqm;
616 struct GNUNET_SETU_RejectMessage *rmsg;
617
618 LOG (GNUNET_ERROR_TYPE_DEBUG,
619 "Processing incoming operation request with id %u\n",
620 ntohl (msg->accept_id));
621 /* we got another valid request => reset the backoff */
622 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
623 req.accept_id = ntohl (msg->accept_id);
624 req.accepted = GNUNET_NO;
625 context_msg = GNUNET_MQ_extract_nested_mh (msg);
626 /* calling #GNUNET_SETU_accept() in the listen cb will set req->accepted */
627 lh->listen_cb (lh->listen_cls,
628 &msg->peer_id,
629 context_msg,
630 &req);
631 if (GNUNET_YES == req.accepted)
632 return; /* the accept-case is handled in #GNUNET_SETU_accept() */
633 LOG (GNUNET_ERROR_TYPE_DEBUG,
634 "Rejected request %u\n",
635 ntohl (msg->accept_id));
636 mqm = GNUNET_MQ_msg (rmsg,
637 GNUNET_MESSAGE_TYPE_SETU_REJECT);
638 rmsg->accept_reject_id = msg->accept_id;
639 GNUNET_MQ_send (lh->mq,
640 mqm);
641}
642
643
644/**
645 * Our connection with the set service encountered an error,
646 * re-initialize with exponential back-off.
647 *
648 * @param cls the `struct GNUNET_SETU_ListenHandle *`
649 * @param error reason for the disconnect
650 */
651static void
652handle_client_listener_error (void *cls,
653 enum GNUNET_MQ_Error error)
654{
655 struct GNUNET_SETU_ListenHandle *lh = cls;
656
657 LOG (GNUNET_ERROR_TYPE_DEBUG,
658 "Listener broke down (%d), re-connecting\n",
659 (int) error);
660 GNUNET_MQ_destroy (lh->mq);
661 lh->mq = NULL;
662 lh->reconnect_task = GNUNET_SCHEDULER_add_delayed (lh->reconnect_backoff,
663 &listen_connect,
664 lh);
665 lh->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (lh->reconnect_backoff);
666}
667
668
669/**
670 * Connect to the set service in order to listen for requests.
671 *
672 * @param cls the `struct GNUNET_SETU_ListenHandle *` to connect
673 */
674static void
675listen_connect (void *cls)
676{
677 struct GNUNET_SETU_ListenHandle *lh = cls;
678 struct GNUNET_MQ_MessageHandler mq_handlers[] = {
679 GNUNET_MQ_hd_var_size (request,
680 GNUNET_MESSAGE_TYPE_SETU_REQUEST,
681 struct GNUNET_SETU_RequestMessage,
682 lh),
683 GNUNET_MQ_handler_end ()
684 };
685 struct GNUNET_MQ_Envelope *mqm;
686 struct GNUNET_SETU_ListenMessage *msg;
687
688 lh->reconnect_task = NULL;
689 GNUNET_assert (NULL == lh->mq);
690 lh->mq = GNUNET_CLIENT_connect (lh->cfg,
691 "setu",
692 mq_handlers,
693 &handle_client_listener_error,
694 lh);
695 if (NULL == lh->mq)
696 return;
697 mqm = GNUNET_MQ_msg (msg,
698 GNUNET_MESSAGE_TYPE_SETU_LISTEN);
699 msg->app_id = lh->app_id;
700 GNUNET_MQ_send (lh->mq,
701 mqm);
702}
703
704
705/**
706 * Wait for set operation requests for the given application id
707 *
708 * @param cfg configuration to use for connecting to
709 * the set service, needs to be valid for the lifetime of the listen handle
710 * @param app_id id of the application that handles set operation requests
711 * @param listen_cb called for each incoming request matching the operation
712 * and application id
713 * @param listen_cls handle for @a listen_cb
714 * @return a handle that can be used to cancel the listen operation
715 */
716struct GNUNET_SETU_ListenHandle *
717GNUNET_SETU_listen (const struct GNUNET_CONFIGURATION_Handle *cfg,
718 const struct GNUNET_HashCode *app_id,
719 GNUNET_SETU_ListenCallback listen_cb,
720 void *listen_cls)
721{
722 struct GNUNET_SETU_ListenHandle *lh;
723
724 LOG (GNUNET_ERROR_TYPE_DEBUG,
725 "Starting listener for app %s\n",
726 GNUNET_h2s (app_id));
727 lh = GNUNET_new (struct GNUNET_SETU_ListenHandle);
728 lh->listen_cb = listen_cb;
729 lh->listen_cls = listen_cls;
730 lh->cfg = cfg;
731 lh->app_id = *app_id;
732 lh->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
733 listen_connect (lh);
734 if (NULL == lh->mq)
735 {
736 GNUNET_free (lh);
737 return NULL;
738 }
739 return lh;
740}
741
742
743/**
744 * Cancel the given listen operation.
745 *
746 * @param lh handle for the listen operation
747 */
748void
749GNUNET_SETU_listen_cancel (struct GNUNET_SETU_ListenHandle *lh)
750{
751 LOG (GNUNET_ERROR_TYPE_DEBUG,
752 "Canceling listener %s\n",
753 GNUNET_h2s (&lh->app_id));
754 if (NULL != lh->mq)
755 {
756 GNUNET_MQ_destroy (lh->mq);
757 lh->mq = NULL;
758 }
759 if (NULL != lh->reconnect_task)
760 {
761 GNUNET_SCHEDULER_cancel (lh->reconnect_task);
762 lh->reconnect_task = NULL;
763 }
764 GNUNET_free (lh);
765}
766
767
768struct GNUNET_SETU_OperationHandle *
769GNUNET_SETU_accept (struct GNUNET_SETU_Request *request,
770 const struct GNUNET_SETU_Option options[],
771 GNUNET_SETU_ResultIterator result_cb,
772 void *result_cls)
773{
774 struct GNUNET_MQ_Envelope *mqm;
775 struct GNUNET_SETU_OperationHandle *oh;
776 struct GNUNET_SETU_AcceptMessage *msg;
777
778 GNUNET_assert (GNUNET_NO == request->accepted);
779 LOG (GNUNET_ERROR_TYPE_DEBUG,
780 "Client accepts set union operation with id %u\n",
781 request->accept_id);
782 request->accepted = GNUNET_YES;
783 mqm = GNUNET_MQ_msg (msg,
784 GNUNET_MESSAGE_TYPE_SETU_ACCEPT);
785 msg->accept_reject_id = htonl (request->accept_id);
786
787 /* Set default values */
788 msg->byzantine_upper_bond = UINT64_MAX;
789 msg->bandwidth_latency_tradeoff = 0;
790 msg->ibf_bucket_number_factor = 2;
791 msg->ibf_number_of_buckets_per_element = 3;
792
793 for (const struct GNUNET_SETU_Option *opt = options; opt->type != 0; opt++)
794 {
795 switch (opt->type)
796 {
797 case GNUNET_SETU_OPTION_BYZANTINE:
798 msg->byzantine = GNUNET_YES;
799 msg->byzantine_lower_bound = htonl (opt->v.num);
800 break;
801 case GNUNET_SETU_OPTION_CUSTOM_BYZANTINE_UPPER_BOUND:
802 msg->byzantine_upper_bond = htonl (opt->v.num);
803 break;
804 case GNUNET_SETU_OPTION_CUSTOM_BANDWIDTH_LATENCY_TRADEOFF:
805 msg->bandwidth_latency_tradeoff = htonl (opt->v.num);
806 break;
807 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKET_NUMBER_FACTOR:
808 msg->ibf_bucket_number_factor = htonl (opt->v.num);
809 break;
810 case GNUNET_SETU_OPTION_CUSTOM_IBF_BUCKETS_PER_ELEMENT:
811 msg->ibf_number_of_buckets_per_element = htonl (opt->v.num);
812 break;
813 case GNUNET_SETU_OPTION_FORCE_FULL:
814 msg->force_full = GNUNET_YES;
815 break;
816 case GNUNET_SETU_OPTION_FORCE_DELTA:
817 msg->force_delta = GNUNET_YES;
818 break;
819 case GNUNET_SETU_OPTION_SYMMETRIC:
820 msg->symmetric = GNUNET_YES;
821 break;
822 default:
823 LOG (GNUNET_ERROR_TYPE_ERROR,
824 "Option with type %d not recognized\n",
825 (int) opt->type);
826 }
827 }
828 oh = GNUNET_new (struct GNUNET_SETU_OperationHandle);
829 oh->result_cb = result_cb;
830 oh->result_cls = result_cls;
831 oh->conclude_mqm = mqm;
832 oh->request_id_addr = &msg->request_id;
833 return oh;
834}
835
836
837/**
838 * Commit a set to be used with a set operation.
839 * This function is called once we have fully constructed
840 * the set that we want to use for the operation. At this
841 * time, the P2P protocol can then begin to exchange the
842 * set information and call the result callback with the
843 * result information.
844 *
845 * @param oh handle to the set operation
846 * @param set the set to use for the operation
847 * @return #GNUNET_OK on success, #GNUNET_SYSERR if the
848 * set is invalid (e.g. the set service crashed)
849 */
850int
851GNUNET_SETU_commit (struct GNUNET_SETU_OperationHandle *oh,
852 struct GNUNET_SETU_Handle *set)
853{
854 if (NULL != oh->set)
855 {
856 /* Some other set was already committed for this
857 * operation, there is a logic bug in the client of this API */
858 GNUNET_break (0);
859 return GNUNET_OK;
860 }
861 GNUNET_assert (NULL != set);
862 if (GNUNET_YES == set->invalid)
863 return GNUNET_SYSERR;
864 LOG (GNUNET_ERROR_TYPE_DEBUG,
865 "Client commits to SET\n");
866 GNUNET_assert (NULL != oh->conclude_mqm);
867 oh->set = set;
868 GNUNET_CONTAINER_DLL_insert (set->ops_head,
869 set->ops_tail,
870 oh);
871 oh->request_id = GNUNET_MQ_assoc_add (set->mq,
872 oh);
873 *oh->request_id_addr = htonl (oh->request_id);
874 GNUNET_MQ_send (set->mq,
875 oh->conclude_mqm);
876 oh->conclude_mqm = NULL;
877 oh->request_id_addr = NULL;
878 return GNUNET_OK;
879}
880
881
882/**
883 * Hash a set element.
884 *
885 * @param element the element that should be hashed
886 * @param[out] ret_hash a pointer to where the hash of @a element
887 * should be stored
888 */
889void
890GNUNET_SETU_element_hash (const struct GNUNET_SETU_Element *element,
891 struct GNUNET_HashCode *ret_hash)
892{
893 struct GNUNET_HashContext *ctx = GNUNET_CRYPTO_hash_context_start ();
894
895 /* It's not guaranteed that the element data is always after the element header,
896 so we need to hash the chunks separately. */
897 GNUNET_CRYPTO_hash_context_read (ctx,
898 &element->size,
899 sizeof(uint16_t));
900 GNUNET_CRYPTO_hash_context_read (ctx,
901 &element->element_type,
902 sizeof(uint16_t));
903 GNUNET_CRYPTO_hash_context_read (ctx,
904 element->data,
905 element->size);
906 GNUNET_CRYPTO_hash_context_finish (ctx,
907 ret_hash);
908}
909
910
911/* end of setu_api.c */
diff --git a/src/setu/test_setu.conf b/src/setu/test_setu.conf
deleted file mode 100644
index 6d132fc1f..000000000
--- a/src/setu/test_setu.conf
+++ /dev/null
@@ -1,32 +0,0 @@
1@INLINE@ ../../contrib/conf/gnunet/no_forcestart.conf
2
3[PATHS]
4GNUNET_TEST_HOME = $GNUNET_TMP/test-gnunet-set/
5
6[setu]
7START_ON_DEMAND = YES
8#PREFIX = valgrind --leak-check=full
9#PREFIX = gdbserver :1234
10OPTIONS = -L INFO
11
12[transport]
13PLUGINS = unix
14OPTIONS = -LERROR
15
16[nat]
17RETURN_LOCAL_ADDRESSES = YES
18DISABLEV6 = YES
19USE_LOCALADDR = YES
20
21[peerinfo]
22NO_IO = YES
23
24[nat]
25# Use addresses from the local network interfaces (including loopback, but also others)
26USE_LOCALADDR = YES
27
28# Disable IPv6 support
29DISABLEV6 = NO
30
31# Do we use addresses from localhost address ranges? (::1, 127.0.0.0/8)
32RETURN_LOCAL_ADDRESSES = YES
diff --git a/src/setu/test_setu_api.c b/src/setu/test_setu_api.c
deleted file mode 100644
index 5a0c9d70d..000000000
--- a/src/setu/test_setu_api.c
+++ /dev/null
@@ -1,360 +0,0 @@
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2012 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
21/**
22 * @file set/test_setu_api.c
23 * @brief testcase for setu_api.c
24 * @author Florian Dold
25 */
26#include "platform.h"
27#include "gnunet_util_lib.h"
28#include "gnunet_testing_lib.h"
29#include "gnunet_setu_service.h"
30
31
32static struct GNUNET_PeerIdentity local_id;
33
34static struct GNUNET_HashCode app_id;
35
36static struct GNUNET_SETU_Handle *set1;
37
38static struct GNUNET_SETU_Handle *set2;
39
40static struct GNUNET_SETU_ListenHandle *listen_handle;
41
42static struct GNUNET_SETU_OperationHandle *oh1;
43
44static struct GNUNET_SETU_OperationHandle *oh2;
45
46static const struct GNUNET_CONFIGURATION_Handle *config;
47
48static int ret;
49
50static struct GNUNET_SCHEDULER_Task *tt;
51
52
53static void
54result_cb_set1 (void *cls,
55 const struct GNUNET_SETU_Element *element,
56 uint64_t size,
57 enum GNUNET_SETU_Status status)
58{
59 switch (status)
60 {
61 case GNUNET_SETU_STATUS_ADD_LOCAL:
62 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: got element\n");
63 break;
64
65 case GNUNET_SETU_STATUS_FAILURE:
66 GNUNET_break (0);
67 oh1 = NULL;
68 fprintf (stderr, "set 1: received failure status!\n");
69 ret = 1;
70 if (NULL != tt)
71 {
72 GNUNET_SCHEDULER_cancel (tt);
73 tt = NULL;
74 }
75 GNUNET_SCHEDULER_shutdown ();
76 break;
77
78 case GNUNET_SETU_STATUS_DONE:
79 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 1: done\n");
80 oh1 = NULL;
81 if (NULL != set1)
82 {
83 GNUNET_SETU_destroy (set1);
84 set1 = NULL;
85 }
86 if (NULL == set2)
87 {
88 GNUNET_SCHEDULER_cancel (tt);
89 tt = NULL;
90 GNUNET_SCHEDULER_shutdown ();
91 }
92 break;
93
94 default:
95 GNUNET_assert (0);
96 }
97}
98
99
100static void
101result_cb_set2 (void *cls,
102 const struct GNUNET_SETU_Element *element,
103 uint64_t size,
104 enum GNUNET_SETU_Status status)
105{
106 switch (status)
107 {
108 case GNUNET_SETU_STATUS_ADD_LOCAL:
109 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: got element\n");
110 break;
111
112 case GNUNET_SETU_STATUS_FAILURE:
113 GNUNET_break (0);
114 oh2 = NULL;
115 fprintf (stderr, "set 2: received failure status\n");
116 GNUNET_SCHEDULER_shutdown ();
117 ret = 1;
118 break;
119
120 case GNUNET_SETU_STATUS_DONE:
121 oh2 = NULL;
122 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "set 2: done\n");
123 GNUNET_SETU_destroy (set2);
124 set2 = NULL;
125 if (NULL == set1)
126 {
127 GNUNET_SCHEDULER_cancel (tt);
128 tt = NULL;
129 GNUNET_SCHEDULER_shutdown ();
130 }
131 break;
132
133 default:
134 GNUNET_assert (0);
135 }
136}
137
138
139static void
140listen_cb (void *cls,
141 const struct GNUNET_PeerIdentity *other_peer,
142 const struct GNUNET_MessageHeader *context_msg,
143 struct GNUNET_SETU_Request *request)
144{
145 GNUNET_assert (NULL != context_msg);
146 GNUNET_assert (ntohs (context_msg->type) == GNUNET_MESSAGE_TYPE_DUMMY);
147 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "listen cb called\n");
148 oh2 = GNUNET_SETU_accept (request,
149 (struct GNUNET_SETU_Option[]){ 0 },
150 &result_cb_set2,
151 NULL);
152 GNUNET_SETU_commit (oh2, set2);
153}
154
155
156/**
157 * Start the set operation.
158 *
159 * @param cls closure, unused
160 */
161static void
162start (void *cls)
163{
164 struct GNUNET_MessageHeader context_msg;
165
166 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting reconciliation\n");
167 context_msg.size = htons (sizeof context_msg);
168 context_msg.type = htons (GNUNET_MESSAGE_TYPE_DUMMY);
169 listen_handle = GNUNET_SETU_listen (config,
170 &app_id,
171 &listen_cb,
172 NULL);
173 oh1 = GNUNET_SETU_prepare (&local_id,
174 &app_id,
175 &context_msg,
176 (struct GNUNET_SETU_Option[]){ 0 },
177 &result_cb_set1,
178 NULL);
179 GNUNET_SETU_commit (oh1, set1);
180}
181
182
183/**
184 * Initialize the second set, continue
185 *
186 * @param cls closure, unused
187 */
188static void
189init_set2 (void *cls)
190{
191 struct GNUNET_SETU_Element element;
192
193 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initializing set 2\n");
194
195 element.element_type = 0;
196 element.data = "hello1";
197 element.size = strlen (element.data);
198 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
199 element.data = "quux";
200 element.size = strlen (element.data);
201 GNUNET_SETU_add_element (set2, &element, NULL, NULL);
202 element.data = "baz";
203 element.size = strlen (element.data);
204 GNUNET_SETU_add_element (set2, &element, &start, NULL);
205}
206
207
208/**
209 * Initialize the first set, continue.
210 */
211static void
212init_set1 (void)
213{
214 struct GNUNET_SETU_Element element;
215
216 element.element_type = 0;
217 element.data = "hello";
218 element.size = strlen (element.data);
219 GNUNET_SETU_add_element (set1, &element, NULL, NULL);
220 element.data = "bar";
221 element.size = strlen (element.data);
222 GNUNET_SETU_add_element (set1, &element, &init_set2, NULL);
223 GNUNET_log (GNUNET_ERROR_TYPE_INFO, "initialized set 1\n");
224}
225
226
227/**
228 * Function run on timeout.
229 *
230 * @param cls closure
231 */
232static void
233timeout_fail (void *cls)
234{
235 tt = NULL;
236 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, "Testcase failed with timeout\n");
237 GNUNET_SCHEDULER_shutdown ();
238 ret = 1;
239}
240
241
242/**
243 * Function run on shutdown.
244 *
245 * @param cls closure
246 */
247static void
248do_shutdown (void *cls)
249{
250 if (NULL != tt)
251 {
252 GNUNET_SCHEDULER_cancel (tt);
253 tt = NULL;
254 }
255 if (NULL != oh1)
256 {
257 GNUNET_SETU_operation_cancel (oh1);
258 oh1 = NULL;
259 }
260 if (NULL != oh2)
261 {
262 GNUNET_SETU_operation_cancel (oh2);
263 oh2 = NULL;
264 }
265 if (NULL != set1)
266 {
267 GNUNET_SETU_destroy (set1);
268 set1 = NULL;
269 }
270 if (NULL != set2)
271 {
272 GNUNET_SETU_destroy (set2);
273 set2 = NULL;
274 }
275 if (NULL != listen_handle)
276 {
277 GNUNET_SETU_listen_cancel (listen_handle);
278 listen_handle = NULL;
279 }
280}
281
282
283/**
284 * Signature of the 'main' function for a (single-peer) testcase that
285 * is run using 'GNUNET_TESTING_peer_run'.
286 *
287 * @param cls closure
288 * @param cfg configuration of the peer that was started
289 * @param peer identity of the peer that was created
290 */
291static void
292run (void *cls,
293 const struct GNUNET_CONFIGURATION_Handle *cfg,
294 struct GNUNET_TESTING_Peer *peer)
295{
296 struct GNUNET_SETU_OperationHandle *my_oh;
297
298 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
299 "Running preparatory tests\n");
300 tt = GNUNET_SCHEDULER_add_delayed (
301 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5),
302 &timeout_fail,
303 NULL);
304 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
305
306 config = cfg;
307 GNUNET_assert (GNUNET_OK == GNUNET_CRYPTO_get_peer_identity (cfg,
308 &local_id));
309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
310 "my id (from CRYPTO): %s\n",
311 GNUNET_i2s (&local_id));
312 GNUNET_TESTING_peer_get_identity (peer,
313 &local_id);
314 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
315 "my id (from TESTING): %s\n",
316 GNUNET_i2s (&local_id));
317 set1 = GNUNET_SETU_create (cfg);
318 set2 = GNUNET_SETU_create (cfg);
319 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
320 "Created sets %p and %p for union operation\n",
321 set1,
322 set2);
323 GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &app_id);
324
325 /* test if canceling an uncommitted request works! */
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Launching and instantly stopping set operation\n");
328 my_oh = GNUNET_SETU_prepare (&local_id,
329 &app_id,
330 NULL,
331 (struct GNUNET_SETU_Option[]){ 0 },
332 NULL,
333 NULL);
334 GNUNET_SETU_operation_cancel (my_oh);
335
336 /* test the real set reconciliation */
337 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
338 "Running real set-reconciliation\n");
339 init_set1 ();
340}
341
342
343int
344main (int argc, char **argv)
345{
346 GNUNET_log_setup ("test_setu_api",
347 "WARNING",
348 NULL);
349 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
350 "Launching peer\n");
351 if (0 !=
352 GNUNET_TESTING_peer_run ("test_setu_api",
353 "test_setu.conf",
354 &run,
355 NULL))
356 {
357 return 1;
358 }
359 return ret;
360}
generated by cgit v1.2.3 (git 2.39.1) at 2024-04-27 19:51:38 +0000