diff options
author | Florian Dold <florian.dold@gmail.com> | 2012-12-20 00:39:17 +0000 |
---|---|---|
committer | Florian Dold <florian.dold@gmail.com> | 2012-12-20 00:39:17 +0000 |
commit | a054f68e16155f955655d827c1b8d8af9c1b9cbd (patch) | |
tree | faa415bcc2748e5d72b2de1a62b710bc3a8bfda3 /src/consensus | |
parent | df625371bda1c30b9b8903867a34b42faab56f2e (diff) | |
download | gnunet-a054f68e16155f955655d827c1b8d8af9c1b9cbd.tar.gz gnunet-a054f68e16155f955655d827c1b8d8af9c1b9cbd.zip |
implemented the invertible bloom filter
Diffstat (limited to 'src/consensus')
-rw-r--r-- | src/consensus/Makefile.am | 14 | ||||
-rw-r--r-- | src/consensus/gnunet-consensus-ibf.c | 192 | ||||
-rw-r--r-- | src/consensus/ibf.c | 218 | ||||
-rw-r--r-- | src/consensus/ibf.h | 59 |
4 files changed, 356 insertions, 127 deletions
diff --git a/src/consensus/Makefile.am b/src/consensus/Makefile.am index 29c466901..7c28b4869 100644 --- a/src/consensus/Makefile.am +++ b/src/consensus/Makefile.am | |||
@@ -17,7 +17,8 @@ endif | |||
17 | 17 | ||
18 | bin_PROGRAMS = \ | 18 | bin_PROGRAMS = \ |
19 | gnunet-consensus \ | 19 | gnunet-consensus \ |
20 | gnunet-consensus-start-peers | 20 | gnunet-consensus-start-peers \ |
21 | gnunet-consensus-ibf | ||
21 | 22 | ||
22 | libexec_PROGRAMS = \ | 23 | libexec_PROGRAMS = \ |
23 | gnunet-service-consensus | 24 | gnunet-service-consensus |
@@ -26,7 +27,7 @@ lib_LTLIBRARIES = \ | |||
26 | libgnunetconsensus.la | 27 | libgnunetconsensus.la |
27 | 28 | ||
28 | gnunet_consensus_SOURCES = \ | 29 | gnunet_consensus_SOURCES = \ |
29 | gnunet-consensus.c | 30 | gnunet-consensus.c |
30 | gnunet_consensus_LDADD = \ | 31 | gnunet_consensus_LDADD = \ |
31 | $(top_builddir)/src/util/libgnunetutil.la \ | 32 | $(top_builddir)/src/util/libgnunetutil.la \ |
32 | $(top_builddir)/src/consensus/libgnunetconsensus.la \ | 33 | $(top_builddir)/src/consensus/libgnunetconsensus.la \ |
@@ -40,6 +41,13 @@ gnunet_consensus_start_peers_LDADD = \ | |||
40 | $(top_builddir)/src/consensus/libgnunetconsensus.la \ | 41 | $(top_builddir)/src/consensus/libgnunetconsensus.la \ |
41 | $(GN_LIBINTL) | 42 | $(GN_LIBINTL) |
42 | 43 | ||
44 | gnunet_consensus_ibf_SOURCES = \ | ||
45 | gnunet-consensus-ibf.c \ | ||
46 | ibf.c | ||
47 | gnunet_consensus_ibf_LDADD = \ | ||
48 | $(top_builddir)/src/util/libgnunetutil.la \ | ||
49 | $(GN_LIBINTL) | ||
50 | |||
43 | gnunet_service_consensus_SOURCES = \ | 51 | gnunet_service_consensus_SOURCES = \ |
44 | gnunet-service-consensus.c | 52 | gnunet-service-consensus.c |
45 | gnunet_service_consensus_LDADD = \ | 53 | gnunet_service_consensus_LDADD = \ |
@@ -69,6 +77,6 @@ test_consensus_api_LDADD = \ | |||
69 | $(top_builddir)/src/testing/libgnunettesting.la \ | 77 | $(top_builddir)/src/testing/libgnunettesting.la \ |
70 | $(top_builddir)/src/consensus/libgnunetconsensus.la | 78 | $(top_builddir)/src/consensus/libgnunetconsensus.la |
71 | 79 | ||
72 | |||
73 | EXTRA_DIST = \ | 80 | EXTRA_DIST = \ |
74 | test_consensus.conf | 81 | test_consensus.conf |
82 | |||
diff --git a/src/consensus/gnunet-consensus-ibf.c b/src/consensus/gnunet-consensus-ibf.c new file mode 100644 index 000000000..f29b91704 --- /dev/null +++ b/src/consensus/gnunet-consensus-ibf.c | |||
@@ -0,0 +1,192 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet. | ||
3 | (C) 2012 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file consensus/gnunet-consensus-ibf | ||
23 | * @brief tool for reconciling data with invertible bloom filters | ||
24 | * @author Florian Dold | ||
25 | */ | ||
26 | |||
27 | |||
28 | #include "platform.h" | ||
29 | #include "gnunet_common.h" | ||
30 | #include "gnunet_container_lib.h" | ||
31 | #include "gnunet_util_lib.h" | ||
32 | |||
33 | #include "ibf.h" | ||
34 | |||
35 | static unsigned int asize = 10; | ||
36 | static unsigned int bsize = 10; | ||
37 | static unsigned int csize = 10; | ||
38 | static unsigned int hash_num = 3; | ||
39 | static unsigned int ibf_size = 80; | ||
40 | |||
41 | |||
42 | static struct GNUNET_CONTAINER_MultiHashMap *set_a; | ||
43 | static struct GNUNET_CONTAINER_MultiHashMap *set_b; | ||
44 | /* common elements in a and b */ | ||
45 | static struct GNUNET_CONTAINER_MultiHashMap *set_c; | ||
46 | |||
47 | static struct InvertibleBloomFilter *ibf_a; | ||
48 | static struct InvertibleBloomFilter *ibf_b; | ||
49 | |||
50 | |||
51 | |||
52 | static int | ||
53 | insert_iterator (void *cls, | ||
54 | const struct GNUNET_HashCode *key, | ||
55 | void *value) | ||
56 | { | ||
57 | struct InvertibleBloomFilter *ibf = (struct InvertibleBloomFilter *) cls; | ||
58 | ibf_insert (ibf, key); | ||
59 | return GNUNET_YES; | ||
60 | } | ||
61 | |||
62 | |||
63 | static void | ||
64 | run (void *cls, char *const *args, const char *cfgfile, | ||
65 | const struct GNUNET_CONFIGURATION_Handle *cfg) | ||
66 | { | ||
67 | struct GNUNET_HashCode id; | ||
68 | int i; | ||
69 | int side; | ||
70 | int res; | ||
71 | |||
72 | set_a = GNUNET_CONTAINER_multihashmap_create (((asize == 0) ? 1 : (asize + csize)), | ||
73 | GNUNET_NO); | ||
74 | set_b = GNUNET_CONTAINER_multihashmap_create (((bsize == 0) ? 1 : (bsize + csize)), | ||
75 | GNUNET_NO); | ||
76 | set_c = GNUNET_CONTAINER_multihashmap_create (((csize == 0) ? 1 : csize), | ||
77 | GNUNET_NO); | ||
78 | |||
79 | printf ("hash-num=%u, size=%u, #(A-B)=%u, #(B-A)=%u, #(A&B)=%u\n", | ||
80 | hash_num, ibf_size, asize, bsize, csize); | ||
81 | |||
82 | i = 0; | ||
83 | while (i < asize) | ||
84 | { | ||
85 | GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &id); | ||
86 | if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id)) | ||
87 | continue; | ||
88 | printf("A: %s\n", GNUNET_h2s (&id)); | ||
89 | GNUNET_CONTAINER_multihashmap_put ( | ||
90 | set_a, &id, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
91 | i++; | ||
92 | } | ||
93 | i = 0; | ||
94 | while (i < bsize) | ||
95 | { | ||
96 | GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &id); | ||
97 | if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id)) | ||
98 | continue; | ||
99 | if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id)) | ||
100 | continue; | ||
101 | printf("B: %s\n", GNUNET_h2s (&id)); | ||
102 | GNUNET_CONTAINER_multihashmap_put ( | ||
103 | set_b, &id, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
104 | i++; | ||
105 | } | ||
106 | i = 0; | ||
107 | while (i < csize) | ||
108 | { | ||
109 | GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK, &id); | ||
110 | if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_a, &id)) | ||
111 | continue; | ||
112 | if (GNUNET_YES == GNUNET_CONTAINER_multihashmap_contains (set_b, &id)) | ||
113 | continue; | ||
114 | printf("C: %s\n", GNUNET_h2s (&id)); | ||
115 | GNUNET_CONTAINER_multihashmap_put ( | ||
116 | set_c, &id, NULL, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY); | ||
117 | i++; | ||
118 | } | ||
119 | |||
120 | |||
121 | ibf_a = ibf_create (ibf_size, hash_num, 0); | ||
122 | ibf_b = ibf_create (ibf_size, hash_num, 0); | ||
123 | |||
124 | GNUNET_CONTAINER_multihashmap_iterate (set_a, &insert_iterator, ibf_a); | ||
125 | GNUNET_CONTAINER_multihashmap_iterate (set_b, &insert_iterator, ibf_b); | ||
126 | GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_a); | ||
127 | GNUNET_CONTAINER_multihashmap_iterate (set_c, &insert_iterator, ibf_b); | ||
128 | |||
129 | |||
130 | printf ("-----------------\n"); | ||
131 | ibf_subtract (ibf_a, ibf_b); | ||
132 | |||
133 | for (;;) | ||
134 | { | ||
135 | res = ibf_decode (ibf_a, &side, &id); | ||
136 | if (GNUNET_SYSERR == res) | ||
137 | { | ||
138 | printf ("decode failed\n"); | ||
139 | return; | ||
140 | } | ||
141 | if (GNUNET_NO == res) | ||
142 | { | ||
143 | if ((0 == GNUNET_CONTAINER_multihashmap_size (set_b)) && | ||
144 | (0 == GNUNET_CONTAINER_multihashmap_size (set_a))) | ||
145 | printf ("decode succeeded\n"); | ||
146 | else | ||
147 | printf ("decode missed elements\n"); | ||
148 | return; | ||
149 | } | ||
150 | |||
151 | printf("R: %s\n", GNUNET_h2s (&id)); | ||
152 | printf("s: %d\n", side); | ||
153 | |||
154 | if (side == 1) | ||
155 | res = GNUNET_CONTAINER_multihashmap_remove (set_a, &id, NULL); | ||
156 | if (side == -1) | ||
157 | res = GNUNET_CONTAINER_multihashmap_remove (set_b, &id, NULL); | ||
158 | if (GNUNET_YES != res) | ||
159 | { | ||
160 | printf ("decoded wrong element\n"); | ||
161 | return; | ||
162 | } | ||
163 | } | ||
164 | } | ||
165 | |||
166 | int | ||
167 | main (int argc, char **argv) | ||
168 | { | ||
169 | static const struct GNUNET_GETOPT_CommandLineOption options[] = { | ||
170 | {'A', "asize", NULL, | ||
171 | gettext_noop ("number of element in set A-B"), 1, | ||
172 | &GNUNET_GETOPT_set_uint, &asize}, | ||
173 | {'B', "bsize", NULL, | ||
174 | gettext_noop ("number of element in set B-A"), 1, | ||
175 | &GNUNET_GETOPT_set_uint, &bsize}, | ||
176 | {'C', "csize", NULL, | ||
177 | gettext_noop ("number of common elements in A and B"), 1, | ||
178 | &GNUNET_GETOPT_set_uint, &csize}, | ||
179 | {'k', "hash-num", NULL, | ||
180 | gettext_noop ("hash num"), 1, | ||
181 | &GNUNET_GETOPT_set_uint, &hash_num}, | ||
182 | {'s', "ibf-size", NULL, | ||
183 | gettext_noop ("ibf size"), 1, | ||
184 | &GNUNET_GETOPT_set_uint, &ibf_size}, | ||
185 | GNUNET_GETOPT_OPTION_END | ||
186 | }; | ||
187 | GNUNET_PROGRAM_run2 (argc, argv, "gnunet-consensus-ibf", | ||
188 | "help", | ||
189 | options, &run, NULL, GNUNET_YES); | ||
190 | return 0; | ||
191 | } | ||
192 | |||
diff --git a/src/consensus/ibf.c b/src/consensus/ibf.c index d0cb218cc..12d0fd320 100644 --- a/src/consensus/ibf.c +++ b/src/consensus/ibf.c | |||
@@ -16,7 +16,7 @@ | |||
16 | along with GNUnet; see the file COPYING. If not, write to the | 16 | along with GNUnet; see the file COPYING. If not, write to the |
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | 17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, |
18 | Boston, MA 02111-1307, USA. | 18 | Boston, MA 02111-1307, USA. |
19 | */ | 19 | */ |
20 | 20 | ||
21 | 21 | ||
22 | /** | 22 | /** |
@@ -25,35 +25,25 @@ | |||
25 | * @author Florian Dold | 25 | * @author Florian Dold |
26 | */ | 26 | */ |
27 | 27 | ||
28 | #include "platform.h" | ||
29 | #include "gnunet_common.h" | ||
30 | #include "ibf.h" | 28 | #include "ibf.h" |
31 | 29 | ||
32 | |||
33 | struct PureCells | ||
34 | { | ||
35 | int index; | ||
36 | struct PureCells *next; | ||
37 | struct PureCells *prev; | ||
38 | }; | ||
39 | |||
40 | struct InvertibleBloomFilter | 30 | struct InvertibleBloomFilter |
41 | { | 31 | { |
42 | /** | 32 | /** |
43 | * How many cells does this IBF have? | 33 | * How many cells does this IBF have? |
44 | */ | 34 | */ |
45 | int size; | 35 | unsigned int size; |
46 | 36 | ||
47 | /** | 37 | /** |
48 | * In how many cells do we hash one element? | 38 | * In how many cells do we hash one element? |
49 | * Usually 4 or 3. | 39 | * Usually 4 or 3. |
50 | */ | 40 | */ |
51 | int hash_num; | 41 | unsigned int hash_num; |
52 | 42 | ||
53 | /** | 43 | /** |
54 | * Salt for mingling hashes | 44 | * Salt for mingling hashes |
55 | */ | 45 | */ |
56 | int salt; | 46 | uint32_t salt; |
57 | 47 | ||
58 | /** | 48 | /** |
59 | * How many times has a bucket been hit? | 49 | * How many times has a bucket been hit? |
@@ -64,21 +54,13 @@ struct InvertibleBloomFilter | |||
64 | /** | 54 | /** |
65 | * xor sums of the elements' hash codes, used to identify the elements. | 55 | * xor sums of the elements' hash codes, used to identify the elements. |
66 | */ | 56 | */ |
67 | GNUNET_HashCode *id_sum; | 57 | struct GNUNET_HashCode *id_sum; |
68 | 58 | ||
69 | /** | 59 | /** |
70 | * xor sums of the "hash of the hash". | 60 | * xor sums of the "hash of the hash". |
71 | */ | 61 | */ |
72 | GNUNET_HashCode *hash_sum; | 62 | struct GNUNET_HashCode *hash_sum; |
73 | |||
74 | struct PureCells *pure_head; | ||
75 | struct PureCells *pure_tail; | ||
76 | 63 | ||
77 | /** | ||
78 | * GNUNET_YES: fresh list is deprecated | ||
79 | * GNUNET_NO: fresh list is up to date | ||
80 | */ | ||
81 | int pure_fresh; | ||
82 | }; | 64 | }; |
83 | 65 | ||
84 | 66 | ||
@@ -86,145 +68,141 @@ struct InvertibleBloomFilter | |||
86 | * Create an invertible bloom filter. | 68 | * Create an invertible bloom filter. |
87 | */ | 69 | */ |
88 | struct InvertibleBloomFilter * | 70 | struct InvertibleBloomFilter * |
89 | ibf_create(int size, int hash_num) | 71 | ibf_create (unsigned int size, unsigned int hash_num, uint32_t salt) |
90 | { | 72 | { |
91 | struct InvertibleBloomFilter *ibf; | 73 | struct InvertibleBloomFilter *ibf; |
92 | 74 | ||
93 | ibf = GNUNET_malloc (sizeof (struct InvertibleBloomFilter)); | 75 | ibf = GNUNET_malloc (sizeof (struct InvertibleBloomFilter)); |
94 | ibf->count = GNUNET_malloc (size * sizeof uint8_t); | 76 | ibf->count = GNUNET_malloc (size * sizeof (uint8_t)); |
95 | ibf->id_sum = GNUNET_malloc (size * sizeof (struct GNUNET_HashCode)); | 77 | ibf->id_sum = GNUNET_malloc (size * sizeof (struct GNUNET_HashCode)); |
96 | ibf->hash_sum = GNUNET_malloc (size * sizeof (struct GNUNET_HashCode)); | 78 | ibf->hash_sum = GNUNET_malloc (size * sizeof (struct GNUNET_HashCode)); |
97 | ibf->size = size; | 79 | ibf->size = size; |
98 | ibf->hash_num = hash_num; | 80 | ibf->hash_num = hash_num; |
81 | |||
82 | return ibf; | ||
99 | } | 83 | } |
100 | 84 | ||
101 | 85 | ||
86 | |||
102 | /** | 87 | /** |
103 | * Insert an element into an IBF. | 88 | * Insert an element into an IBF. |
104 | */ | 89 | */ |
105 | void | 90 | void |
106 | ibf_insert (struct InvertibleBloomFilter *ibf, struct GNUNET_HashCode *id) | 91 | ibf_insert_on_side (struct InvertibleBloomFilter *ibf, |
92 | const struct GNUNET_HashCode *key, | ||
93 | int side) | ||
107 | { | 94 | { |
108 | struct GNUNET_HashCode key; | 95 | struct GNUNET_HashCode bucket_indices; |
109 | struct GNUNET_HashCode id_hash; | 96 | struct GNUNET_HashCode remove_key; |
97 | struct GNUNET_HashCode key_hash; | ||
110 | int i; | 98 | int i; |
111 | 99 | ||
112 | key = *id; | 100 | /* copy the key, if key and an entry in the IBF alias */ |
113 | GNUNET_hash (id, sizeof (struct GNUNET_HashCode), &id_hash); | 101 | remove_key = *key; |
114 | 102 | ||
115 | for (i = 0; i < ibf->hash_num; i++) | 103 | GNUNET_assert ((1 == side) || (-1 == side)); |
116 | { | ||
117 | int bucket; | ||
118 | int j; | ||
119 | if ((i != 0) && (i % 16) == 0) | ||
120 | { | ||
121 | GNUNET_hash (&key, sizeof (struct GNUNET_HashCode), &key); | ||
122 | } | ||
123 | bucket = hash.bits[i%16] % ibf->size; | ||
124 | |||
125 | /* count<0 can happen after ibf subtraction, but then no insert should be done */ | ||
126 | GNUNET_assert (ibf->count[bucket] >= 0); | ||
127 | 104 | ||
128 | ibf->count[bucket]++; | 105 | bucket_indices = remove_key; |
129 | 106 | GNUNET_CRYPTO_hash (key, sizeof (struct GNUNET_HashCode), &key_hash); | |
130 | for (j=0; j < 16; j++) | ||
131 | { | ||
132 | ibf->id_sum.bits[j] ^= &id; | ||
133 | ibf->hash_sum.bits[j] ^= &id_hash; | ||
134 | } | ||
135 | 107 | ||
108 | for (i = 0; i < ibf->hash_num; i++) | ||
109 | { | ||
110 | unsigned int bucket; | ||
111 | if ((i % 16) == 0) | ||
112 | GNUNET_CRYPTO_hash (&bucket_indices, sizeof (struct GNUNET_HashCode), | ||
113 | &bucket_indices); | ||
114 | |||
115 | bucket = bucket_indices.bits[i%16] % ibf->size; | ||
116 | |||
117 | printf ("inserting %s in bucket %u side %d\n", | ||
118 | GNUNET_h2s (&remove_key), bucket, side); | ||
119 | |||
120 | ibf->count[bucket] += side; | ||
121 | |||
122 | GNUNET_CRYPTO_hash_xor (&remove_key, &ibf->id_sum[bucket], | ||
123 | &ibf->id_sum[bucket]); | ||
124 | GNUNET_CRYPTO_hash_xor (&key_hash, &ibf->hash_sum[bucket], | ||
125 | &ibf->hash_sum[bucket]); | ||
136 | } | 126 | } |
137 | } | 127 | } |
138 | 128 | ||
139 | |||
140 | /** | 129 | /** |
141 | * Update the linked list of pure cells, if not fresh anymore | 130 | * Insert an element into an IBF. |
142 | */ | 131 | */ |
143 | void | 132 | void |
144 | update_pure (struct InvertibleBloomFilter *ibf) | 133 | ibf_insert (struct InvertibleBloomFilter *ibf, const struct GNUNET_HashCode *key) |
145 | { | 134 | { |
146 | if (GNUNET_YES == ibf->pure_fresh) | 135 | ibf_insert_on_side (ibf, key, 1); |
136 | } | ||
137 | |||
138 | static int | ||
139 | ibf_is_empty (struct InvertibleBloomFilter *ibf) | ||
140 | { | ||
141 | int i; | ||
142 | for (i = 0; i < ibf->size; i++) | ||
147 | { | 143 | { |
148 | return; | 144 | int j; |
145 | if (0 != ibf->count[i]) | ||
146 | return GNUNET_NO; | ||
147 | for (j = 0; j < 16; ++j) | ||
148 | { | ||
149 | if (0 != ibf->hash_sum[i].bits[j]) | ||
150 | return GNUNET_NO; | ||
151 | if (0 != ibf->id_sum[i].bits[j]) | ||
152 | return GNUNET_NO; | ||
153 | } | ||
149 | } | 154 | } |
150 | 155 | return GNUNET_YES; | |
151 | ibf->pure_fresh = GNUNET_YES; | ||
152 | } | 156 | } |
153 | 157 | ||
158 | |||
154 | /** | 159 | /** |
155 | * Decode and remove an element from the IBF, if possible. | 160 | * Decode and remove an element from the IBF, if possible. |
156 | * | 161 | * |
157 | * @param ibf the invertible bloom filter to decode | 162 | * @param ibf the invertible bloom filter to decode |
158 | * @param ret_id the hash code of the decoded element, if successful | 163 | * @param ret_id the hash code of the decoded element, if successful |
159 | * @param side sign of the cell's count where the decoded element came from. | 164 | * @param side sign of the cell's count where the decoded element came from. |
160 | * A negative sign indicates that the element was recovered resides in an IBF | 165 | * A negative sign indicates that the element was recovered |
161 | * that was previously subtracted from. | 166 | * resides in an IBF that was previously subtracted from. |
162 | * @return GNUNET_YES if decoding an element was successful, GNUNET_NO if the IBF is empty, | 167 | * @return GNUNET_YES if decoding an element was successful, |
163 | * GNUNET_SYSERR if the decoding has faile | 168 | * GNUNET_NO if the IBF is empty, |
169 | * GNUNET_SYSERR if the decoding has failed | ||
164 | */ | 170 | */ |
165 | int | 171 | int |
166 | ibf_decode (struct InvertibleBloomFilter *ibf, int *ret_side, struct GNUNET_HashCode *ret_id) | 172 | ibf_decode (struct InvertibleBloomFilter *ibf, |
173 | int *ret_side, struct GNUNET_HashCode *ret_id) | ||
167 | { | 174 | { |
168 | struct GNUNET_HashCode hash; | 175 | struct GNUNET_HashCode hash; |
169 | struct PureCells *pure; | 176 | int i; |
170 | int count; | ||
171 | 177 | ||
172 | GNUNET_assert (NULL != ibf); | 178 | GNUNET_assert (NULL != ibf); |
173 | GNUNET_assert (NULL != red_id); | 179 | GNUNET_assert (NULL != ret_id); |
174 | GNUNET_assert (NULL != side); | 180 | GNUNET_assert (NULL != ret_side); |
175 | 181 | ||
176 | update_pure (ibf); | 182 | for (i = 0; i < ibf->size; i++) |
177 | |||
178 | pure = ibf->pure_head; | ||
179 | ibf->pure_head = pure->next; | ||
180 | |||
181 | if (NULL == pure) | ||
182 | { | 183 | { |
183 | int i; | 184 | if ((1 != ibf->count[i]) && (-1 != ibf->count[i])) |
184 | for (i = 0; i < ibf->size; i++) | 185 | continue; |
185 | { | ||
186 | int j; | ||
187 | if (0 != ibf->count[i]) | ||
188 | return GNUNET_SYSERR; | ||
189 | for (j = 0; j < 16; ++j) | ||
190 | if ((0 != ibf->hash_sum[i].bits[j]) || (0 != ibf->id_sum[i].bits[j])) | ||
191 | return GNUNET_SYSERR; | ||
192 | return GNUNET_NO; | ||
193 | } | ||
194 | } | ||
195 | 186 | ||
196 | GNUNET_CRYPTO_hash (ibf->id_sum[pure->idx], sizeof (struct GNUNET_HashCode), &hash); | 187 | GNUNET_CRYPTO_hash (&ibf->id_sum[i], sizeof (struct GNUNET_HashCode), &hash); |
197 | 188 | ||
198 | if (0 == memcmp (&hash, ibf->hash_sum[pure->idx])) | 189 | if (0 != memcmp (&hash, &ibf->hash_sum[i], sizeof (struct GNUNET_HashCode))) |
199 | { | 190 | continue; |
200 | struct GNUNET_HashCode key; | ||
201 | int i; | ||
202 | 191 | ||
203 | *ret_side = ibf->count[pure->index]; | 192 | printf ("%d pure\n", i); |
204 | *ret_id = ibf->id_sum[pure->index]; | 193 | printf ("%d count\n", ibf->count[i]); |
205 | 194 | ||
206 | key = *ibf->id_sum[pure->index]; | 195 | *ret_side = ibf->count[i]; |
196 | *ret_id = ibf->id_sum[i]; | ||
207 | 197 | ||
208 | /* delete the item from all buckets */ | 198 | /* insert on the opposite side, effectively removing the element */ |
209 | for (i = 0; i < ibf->hash_num; i++) | 199 | ibf_insert_on_side (ibf, &ibf->id_sum[i], -ibf->count[i]); |
210 | { | 200 | |
211 | int bucket; | 201 | return GNUNET_YES; |
212 | int j; | ||
213 | if ((i != 0) && (i % 16) == 0) | ||
214 | { | ||
215 | GNUNET_hash (&key, sizeof (struct GNUNET_HashCode), &key); | ||
216 | } | ||
217 | bucket = hash.bits[i%16] % ibf->size; | ||
218 | |||
219 | ibf->count[bucket] -= count; | ||
220 | |||
221 | for (j=0; j < 16; j++) | ||
222 | { | ||
223 | ibf->id_sum.bits[j] ^= &id; | ||
224 | ibf->hash_sum.bits[j] ^= &id_hash; | ||
225 | } | ||
226 | return GNUNET_YES; | ||
227 | } | 202 | } |
203 | |||
204 | if (GNUNET_YES == ibf_is_empty (ibf)) | ||
205 | return GNUNET_NO; | ||
228 | return GNUNET_SYSERR; | 206 | return GNUNET_SYSERR; |
229 | } | 207 | } |
230 | 208 | ||
@@ -239,6 +217,22 @@ ibf_decode (struct InvertibleBloomFilter *ibf, int *ret_side, struct GNUNET_Hash | |||
239 | void | 217 | void |
240 | ibf_subtract (struct InvertibleBloomFilter *ibf1, struct InvertibleBloomFilter *ibf2) | 218 | ibf_subtract (struct InvertibleBloomFilter *ibf1, struct InvertibleBloomFilter *ibf2) |
241 | { | 219 | { |
242 | /* FIXME */ | 220 | int i; |
221 | |||
222 | GNUNET_assert (ibf1->size == ibf2->size); | ||
223 | GNUNET_assert (ibf1->hash_num == ibf2->hash_num); | ||
224 | GNUNET_assert (ibf1->salt == ibf2->salt); | ||
225 | |||
226 | for (i = 0; i < ibf1->size; i++) | ||
227 | { | ||
228 | printf ("ibf1->count[%d]=%d, ibf2->count[%d]=%d\n", i, ibf1->count[i], i, | ||
229 | ibf2->count[i]); | ||
230 | ibf1->count[i] -= ibf2->count[i]; | ||
231 | printf("diff: %d\n", ibf1->count[i]); | ||
232 | GNUNET_CRYPTO_hash_xor (&ibf1->id_sum[i], &ibf2->id_sum[i], | ||
233 | &ibf1->id_sum[i]); | ||
234 | GNUNET_CRYPTO_hash_xor (&ibf1->hash_sum[i], &ibf2->hash_sum[i], | ||
235 | &ibf1->hash_sum[i]); | ||
236 | } | ||
243 | } | 237 | } |
244 | 238 | ||
diff --git a/src/consensus/ibf.h b/src/consensus/ibf.h index 2c9931e69..12aef5d2f 100644 --- a/src/consensus/ibf.h +++ b/src/consensus/ibf.h | |||
@@ -16,8 +16,7 @@ | |||
16 | along with GNUnet; see the file COPYING. If not, write to the | 16 | along with GNUnet; see the file COPYING. If not, write to the |
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | 17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, |
18 | Boston, MA 02111-1307, USA. | 18 | Boston, MA 02111-1307, USA. |
19 | */ | 19 | */ |
20 | |||
21 | 20 | ||
22 | /** | 21 | /** |
23 | * @file consensus/ibf.h | 22 | * @file consensus/ibf.h |
@@ -25,6 +24,21 @@ | |||
25 | * @author Florian Dold | 24 | * @author Florian Dold |
26 | */ | 25 | */ |
27 | 26 | ||
27 | #ifndef GNUNET_CONSENSUS_IBF_H | ||
28 | #define GNUNET_CONSENSUS_IBF_H | ||
29 | |||
30 | #include "platform.h" | ||
31 | #include "gnunet_common.h" | ||
32 | #include "gnunet_util_lib.h" | ||
33 | |||
34 | #ifdef __cplusplus | ||
35 | extern "C" | ||
36 | { | ||
37 | #if 0 /* keep Emacsens' auto-indent happy */ | ||
38 | } | ||
39 | #endif | ||
40 | #endif | ||
41 | |||
28 | 42 | ||
29 | /** | 43 | /** |
30 | * Opaque handle to an invertible bloom filter (IBF). | 44 | * Opaque handle to an invertible bloom filter (IBF). |
@@ -32,19 +46,21 @@ | |||
32 | * An IBF is a counting bloom filter that has the ability to restore | 46 | * An IBF is a counting bloom filter that has the ability to restore |
33 | * the hashes of its stored elements with high probability. | 47 | * the hashes of its stored elements with high probability. |
34 | */ | 48 | */ |
35 | struct InvertibleBloomFilter | 49 | struct InvertibleBloomFilter; |
50 | |||
36 | 51 | ||
37 | /** | 52 | /** |
38 | * Create an invertible bloom filter. | 53 | * Create an invertible bloom filter. |
39 | * | 54 | * |
40 | * @param size number of IBF buckets | 55 | * @param size number of IBF buckets |
56 | * @param hash_num number of buckets one element is hashed in | ||
41 | * @param salt salt for mingling hashes, different salt may | 57 | * @param salt salt for mingling hashes, different salt may |
42 | * result in less (or more) collisions | 58 | * result in less (or more) collisions |
43 | * @param hash_num number of buckets one element is hashed in | ||
44 | * @return the newly created invertible bloom filter | 59 | * @return the newly created invertible bloom filter |
45 | */ | 60 | */ |
46 | struct InvertibleBloomFilter * | 61 | struct InvertibleBloomFilter * |
47 | ibf_create(int size, int salt, int hash_num); | 62 | ibf_create(unsigned int size, unsigned int hash_num, uint32_t salt); |
63 | |||
48 | 64 | ||
49 | /** | 65 | /** |
50 | * Insert an element into an IBF. | 66 | * Insert an element into an IBF. |
@@ -53,7 +69,8 @@ ibf_create(int size, int salt, int hash_num); | |||
53 | * @param id the element's hash code | 69 | * @param id the element's hash code |
54 | */ | 70 | */ |
55 | void | 71 | void |
56 | ibf_insert (struct InvertibleBloomFilter *ibf, GNUNET_HashCode *id); | 72 | ibf_insert (struct InvertibleBloomFilter *ibf, const struct GNUNET_HashCode *id); |
73 | |||
57 | 74 | ||
58 | /** | 75 | /** |
59 | * Subtract ibf2 from ibf1, storing the result in ibf1. | 76 | * Subtract ibf2 from ibf1, storing the result in ibf1. |
@@ -62,15 +79,20 @@ ibf_insert (struct InvertibleBloomFilter *ibf, GNUNET_HashCode *id); | |||
62 | void | 79 | void |
63 | ibf_subtract (struct InvertibleBloomFilter *ibf1, struct InvertibleBloomFilter *ibf2); | 80 | ibf_subtract (struct InvertibleBloomFilter *ibf1, struct InvertibleBloomFilter *ibf2); |
64 | 81 | ||
82 | |||
65 | /** | 83 | /** |
66 | * Decode and remove an element from the IBF, if possible. | 84 | * Decode and remove an element from the IBF, if possible. |
67 | * | 85 | * |
68 | * @param ibf the invertible bloom filter | 86 | * @param ibf the invertible bloom filter to decode |
69 | * @param the id of the element is written to this hash code | 87 | * @param ret_id the hash code of the decoded element, if successful |
70 | * @return GNUNET_YES if decoding an element was successful, GNUNET_NO if it failed to decode | 88 | * @param side sign of the cell's count where the decoded element came from. |
89 | * A negative sign indicates that the element was recovered resides in an IBF | ||
90 | * that was previously subtracted from. | ||
91 | * @return GNUNET_YES if decoding an element was successful, GNUNET_NO if the IBF is empty, | ||
92 | * GNUNET_SYSERR if the decoding has faile | ||
71 | */ | 93 | */ |
72 | int | 94 | int |
73 | ibf_decode (struct InvertibleBloomFilter *ibf, struct GNUNET_HashCode *ret_id); | 95 | ibf_decode (struct InvertibleBloomFilter *ibf, int *side, struct GNUNET_HashCode *ret_id); |
74 | 96 | ||
75 | 97 | ||
76 | /** | 98 | /** |
@@ -81,10 +103,13 @@ ibf_decode (struct InvertibleBloomFilter *ibf, struct GNUNET_HashCode *ret_id); | |||
81 | struct InvertibleBloomFilter * | 103 | struct InvertibleBloomFilter * |
82 | ibf_dup (struct InvertibleBloomFilter *ibf); | 104 | ibf_dup (struct InvertibleBloomFilter *ibf); |
83 | 105 | ||
106 | |||
84 | /* | 107 | /* |
85 | ibf_hton(); | 108 | ibf_hton (); |
86 | 109 | ||
87 | ibf_ntoh(); | 110 | ibf_ntoh (); |
111 | |||
112 | ibf_get_nbo_size (); | ||
88 | */ | 113 | */ |
89 | 114 | ||
90 | /** | 115 | /** |
@@ -96,3 +121,13 @@ ibf_ntoh(); | |||
96 | void | 121 | void |
97 | ibf_destroy (struct InvertibleBloomFilter *ibf); | 122 | ibf_destroy (struct InvertibleBloomFilter *ibf); |
98 | 123 | ||
124 | |||
125 | #if 0 /* keep Emacsens' auto-indent happy */ | ||
126 | { | ||
127 | #endif | ||
128 | #ifdef __cplusplus | ||
129 | } | ||
130 | #endif | ||
131 | |||
132 | #endif | ||
133 | |||