diff options
author | Christian Grothoff <christian@grothoff.org> | 2012-12-07 22:29:53 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2012-12-07 22:29:53 +0000 |
commit | 6304323162321afab21eadf5a0a9433dbeff4e50 (patch) | |
tree | 017b5268dfb2ef14b5ed5ef6391ad30cec088cf6 /src/datastore | |
parent | 9366ca95afdc80db816706bcaf6dc25237309d32 (diff) | |
download | gnunet-6304323162321afab21eadf5a0a9433dbeff4e50.tar.gz gnunet-6304323162321afab21eadf5a0a9433dbeff4e50.zip |
-implementing heap-based datastore backend
Diffstat (limited to 'src/datastore')
-rw-r--r-- | src/datastore/Makefile.am | 49 | ||||
-rw-r--r-- | src/datastore/datastore.conf.in | 3 | ||||
-rw-r--r-- | src/datastore/perf_plugin_datastore.c | 3 | ||||
-rw-r--r-- | src/datastore/perf_plugin_datastore_data_heap.conf | 7 | ||||
-rw-r--r-- | src/datastore/plugin_datastore_heap.c | 867 | ||||
-rw-r--r-- | src/datastore/plugin_datastore_template.c | 16 | ||||
-rw-r--r-- | src/datastore/test_datastore_api_data_heap.conf | 20 | ||||
-rw-r--r-- | src/datastore/test_plugin_datastore_data_heap.conf | 6 |
8 files changed, 969 insertions, 2 deletions
diff --git a/src/datastore/Makefile.am b/src/datastore/Makefile.am index 1b17fd3ed..eb97996bd 100644 --- a/src/datastore/Makefile.am +++ b/src/datastore/Makefile.am | |||
@@ -87,6 +87,7 @@ plugin_LTLIBRARIES = \ | |||
87 | $(SQLITE_PLUGIN) \ | 87 | $(SQLITE_PLUGIN) \ |
88 | $(MYSQL_PLUGIN) \ | 88 | $(MYSQL_PLUGIN) \ |
89 | $(POSTGRES_PLUGIN) \ | 89 | $(POSTGRES_PLUGIN) \ |
90 | libgnunet_plugin_datastore_heap.la \ | ||
90 | libgnunet_plugin_datastore_template.la | 91 | libgnunet_plugin_datastore_template.la |
91 | 92 | ||
92 | 93 | ||
@@ -100,6 +101,15 @@ libgnunet_plugin_datastore_sqlite_la_LDFLAGS = \ | |||
100 | $(GN_PLUGIN_LDFLAGS) | 101 | $(GN_PLUGIN_LDFLAGS) |
101 | 102 | ||
102 | 103 | ||
104 | libgnunet_plugin_datastore_heap_la_SOURCES = \ | ||
105 | plugin_datastore_heap.c | ||
106 | libgnunet_plugin_datastore_heap_la_LIBADD = \ | ||
107 | $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \ | ||
108 | $(LTLIBINTL) | ||
109 | libgnunet_plugin_datastore_heap_la_LDFLAGS = \ | ||
110 | $(GN_PLUGIN_LDFLAGS) | ||
111 | |||
112 | |||
103 | libgnunet_plugin_datastore_mysql_la_SOURCES = \ | 113 | libgnunet_plugin_datastore_mysql_la_SOURCES = \ |
104 | plugin_datastore_mysql.c | 114 | plugin_datastore_mysql.c |
105 | libgnunet_plugin_datastore_mysql_la_LIBADD = \ | 115 | libgnunet_plugin_datastore_mysql_la_LIBADD = \ |
@@ -132,6 +142,11 @@ libgnunet_plugin_datastore_template_la_LDFLAGS = \ | |||
132 | $(GN_PLUGIN_LDFLAGS) | 142 | $(GN_PLUGIN_LDFLAGS) |
133 | 143 | ||
134 | check_PROGRAMS = \ | 144 | check_PROGRAMS = \ |
145 | test_datastore_api_heap \ | ||
146 | test_datastore_api_management_heap \ | ||
147 | perf_datastore_api_heap \ | ||
148 | perf_plugin_datastore_heap \ | ||
149 | test_plugin_datastore_heap \ | ||
135 | $(SQLITE_TESTS) \ | 150 | $(SQLITE_TESTS) \ |
136 | $(MYSQL_TESTS) \ | 151 | $(MYSQL_TESTS) \ |
137 | $(POSTGRES_TESTS) | 152 | $(POSTGRES_TESTS) |
@@ -140,6 +155,40 @@ if ENABLE_TEST_RUN | |||
140 | TESTS = $(check_PROGRAMS) | 155 | TESTS = $(check_PROGRAMS) |
141 | endif | 156 | endif |
142 | 157 | ||
158 | test_datastore_api_heap_SOURCES = \ | ||
159 | test_datastore_api.c | ||
160 | test_datastore_api_heap_LDADD = \ | ||
161 | $(top_builddir)/src/testing/libgnunettesting.la \ | ||
162 | $(top_builddir)/src/datastore/libgnunetdatastore.la \ | ||
163 | $(top_builddir)/src/util/libgnunetutil.la | ||
164 | |||
165 | test_datastore_api_management_heap_SOURCES = \ | ||
166 | test_datastore_api_management.c | ||
167 | test_datastore_api_management_heap_LDADD = \ | ||
168 | $(top_builddir)/src/testing/libgnunettesting.la \ | ||
169 | $(top_builddir)/src/datastore/libgnunetdatastore.la \ | ||
170 | $(top_builddir)/src/util/libgnunetutil.la | ||
171 | |||
172 | perf_datastore_api_heap_SOURCES = \ | ||
173 | perf_datastore_api.c | ||
174 | perf_datastore_api_heap_LDADD = \ | ||
175 | $(top_builddir)/src/testing/libgnunettesting.la \ | ||
176 | $(top_builddir)/src/datastore/libgnunetdatastore.la \ | ||
177 | $(top_builddir)/src/util/libgnunetutil.la | ||
178 | |||
179 | perf_plugin_datastore_heap_SOURCES = \ | ||
180 | perf_plugin_datastore.c | ||
181 | perf_plugin_datastore_heap_LDADD = \ | ||
182 | $(top_builddir)/src/testing/libgnunettesting.la \ | ||
183 | $(top_builddir)/src/util/libgnunetutil.la | ||
184 | |||
185 | test_plugin_datastore_heap_SOURCES = \ | ||
186 | test_plugin_datastore.c | ||
187 | test_plugin_datastore_heap_LDADD = \ | ||
188 | $(top_builddir)/src/testing/libgnunettesting.la \ | ||
189 | $(top_builddir)/src/util/libgnunetutil.la | ||
190 | |||
191 | |||
143 | test_datastore_api_sqlite_SOURCES = \ | 192 | test_datastore_api_sqlite_SOURCES = \ |
144 | test_datastore_api.c | 193 | test_datastore_api.c |
145 | test_datastore_api_sqlite_LDADD = \ | 194 | test_datastore_api_sqlite_LDADD = \ |
diff --git a/src/datastore/datastore.conf.in b/src/datastore/datastore.conf.in index 809a51a3f..7a92070d7 100644 --- a/src/datastore/datastore.conf.in +++ b/src/datastore/datastore.conf.in | |||
@@ -29,4 +29,5 @@ CONFIG = ~/.my.cnf | |||
29 | # PORT = 3306 | 29 | # PORT = 3306 |
30 | 30 | ||
31 | 31 | ||
32 | 32 | [datastore-heap] | |
33 | HASHMAPSIZE = 1024 | ||
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c index fec090571..111e57b59 100644 --- a/src/datastore/perf_plugin_datastore.c +++ b/src/datastore/perf_plugin_datastore.c | |||
@@ -164,7 +164,8 @@ iterate_zeros (void *cls, const struct GNUNET_HashCode * key, uint32_t size, | |||
164 | hits[i / 8] |= (1 << (i % 8)); | 164 | hits[i / 8] |= (1 << (i % 8)); |
165 | 165 | ||
166 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, | 166 | GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, |
167 | "Found result type=%u, priority=%u, size=%u, expire=%llu\n", | 167 | "Found result %d type=%u, priority=%u, size=%u, expire=%llu\n", |
168 | i, | ||
168 | type, priority, size, (unsigned long long) expiration.abs_value); | 169 | type, priority, size, (unsigned long long) expiration.abs_value); |
169 | crc->cnt++; | 170 | crc->cnt++; |
170 | if (crc->cnt == PUT_10 / 4 - 1) | 171 | if (crc->cnt == PUT_10 / 4 - 1) |
diff --git a/src/datastore/perf_plugin_datastore_data_heap.conf b/src/datastore/perf_plugin_datastore_data_heap.conf new file mode 100644 index 000000000..7ee215c0a --- /dev/null +++ b/src/datastore/perf_plugin_datastore_data_heap.conf | |||
@@ -0,0 +1,7 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | SERVICEHOME = /tmp/perf-gnunet-datastore-heap/ | ||
4 | |||
5 | |||
6 | [datastore] | ||
7 | DATABASE = heap | ||
diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c new file mode 100644 index 000000000..c0ed073fc --- /dev/null +++ b/src/datastore/plugin_datastore_heap.c | |||
@@ -0,0 +1,867 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet | ||
3 | (C) 2012 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datastore/plugin_datastore_heap.c | ||
23 | * @brief heap-based datastore backend; usually we want the datastore | ||
24 | * to be persistent, and storing data in the heap is obviously | ||
25 | * NOT going to be persistent; still, this plugin is useful for | ||
26 | * testing/benchmarking --- but never for production! | ||
27 | * @author Christian Grothoff | ||
28 | */ | ||
29 | |||
30 | #include "platform.h" | ||
31 | #include "gnunet_datastore_plugin.h" | ||
32 | |||
33 | |||
34 | /** | ||
35 | * A value that we are storing. | ||
36 | */ | ||
37 | struct Value | ||
38 | { | ||
39 | |||
40 | /** | ||
41 | * Key for the value. | ||
42 | */ | ||
43 | struct GNUNET_HashCode key; | ||
44 | |||
45 | /** | ||
46 | * Pointer to the value's data (allocated at the end of this struct). | ||
47 | */ | ||
48 | const void *data; | ||
49 | |||
50 | /** | ||
51 | * Entry for this value in the 'expire' heap. | ||
52 | */ | ||
53 | struct GNUNET_CONTAINER_HeapNode *expire_heap; | ||
54 | |||
55 | /** | ||
56 | * Entry for this value in the 'replication' heap. | ||
57 | */ | ||
58 | struct GNUNET_CONTAINER_HeapNode *replication_heap; | ||
59 | |||
60 | /** | ||
61 | * Expiration time for this value. | ||
62 | */ | ||
63 | struct GNUNET_TIME_Absolute expiration; | ||
64 | |||
65 | /** | ||
66 | * Offset of this value in the array of the 'struct ZeroAnonByType'; | ||
67 | * only used if anonymity is zero. | ||
68 | */ | ||
69 | unsigned int zero_anon_offset; | ||
70 | |||
71 | /** | ||
72 | * Number of bytes in 'data'. | ||
73 | */ | ||
74 | uint32_t size; | ||
75 | |||
76 | /** | ||
77 | * Priority of the value. | ||
78 | */ | ||
79 | uint32_t priority; | ||
80 | |||
81 | /** | ||
82 | * Anonymity level for the value. | ||
83 | */ | ||
84 | uint32_t anonymity; | ||
85 | |||
86 | /** | ||
87 | * Replication level for the value. | ||
88 | */ | ||
89 | uint32_t replication; | ||
90 | |||
91 | /** | ||
92 | * Type of 'data'. | ||
93 | */ | ||
94 | enum GNUNET_BLOCK_Type type; | ||
95 | |||
96 | }; | ||
97 | |||
98 | |||
99 | /** | ||
100 | * We organize 0-anonymity values in arrays "by type". | ||
101 | */ | ||
102 | struct ZeroAnonByType | ||
103 | { | ||
104 | |||
105 | /** | ||
106 | * We keep these in a DLL. | ||
107 | */ | ||
108 | struct ZeroAnonByType *next; | ||
109 | |||
110 | /** | ||
111 | * We keep these in a DLL. | ||
112 | */ | ||
113 | struct ZeroAnonByType *prev; | ||
114 | |||
115 | /** | ||
116 | * Array of 0-anonymity items of the given type. | ||
117 | */ | ||
118 | struct Value **array; | ||
119 | |||
120 | /** | ||
121 | * Allocated size of the array. | ||
122 | */ | ||
123 | unsigned int array_size; | ||
124 | |||
125 | /** | ||
126 | * First unused offset in 'array'. | ||
127 | */ | ||
128 | unsigned int array_pos; | ||
129 | |||
130 | /** | ||
131 | * Type of all of the values in 'array'. | ||
132 | */ | ||
133 | enum GNUNET_BLOCK_Type type; | ||
134 | }; | ||
135 | |||
136 | |||
137 | /** | ||
138 | * Context for all functions in this plugin. | ||
139 | */ | ||
140 | struct Plugin | ||
141 | { | ||
142 | /** | ||
143 | * Our execution environment. | ||
144 | */ | ||
145 | struct GNUNET_DATASTORE_PluginEnvironment *env; | ||
146 | |||
147 | /** | ||
148 | * Mapping from keys to 'struct Value's. | ||
149 | */ | ||
150 | struct GNUNET_CONTAINER_MultiHashMap *keyvalue; | ||
151 | |||
152 | /** | ||
153 | * Heap organized by minimum expiration time. | ||
154 | */ | ||
155 | struct GNUNET_CONTAINER_Heap *by_expiration; | ||
156 | |||
157 | /** | ||
158 | * Heap organized by maximum replication value. | ||
159 | */ | ||
160 | struct GNUNET_CONTAINER_Heap *by_replication; | ||
161 | |||
162 | /** | ||
163 | * Head of list of arrays containing zero-anonymity values by type. | ||
164 | */ | ||
165 | struct ZeroAnonByType *zero_head; | ||
166 | |||
167 | /** | ||
168 | * Tail of list of arrays containing zero-anonymity values by type. | ||
169 | */ | ||
170 | struct ZeroAnonByType *zero_tail; | ||
171 | |||
172 | /** | ||
173 | * Size of all values we're storing. | ||
174 | */ | ||
175 | unsigned long long size; | ||
176 | |||
177 | }; | ||
178 | |||
179 | |||
180 | /** | ||
181 | * Get an estimate of how much space the database is | ||
182 | * currently using. | ||
183 | * | ||
184 | * @param cls our "struct Plugin*" | ||
185 | * @return number of bytes used on disk | ||
186 | */ | ||
187 | static unsigned long long | ||
188 | heap_plugin_estimate_size (void *cls) | ||
189 | { | ||
190 | struct Plugin *plugin = cls; | ||
191 | |||
192 | return plugin->size; | ||
193 | } | ||
194 | |||
195 | |||
196 | /** | ||
197 | * Store an item in the datastore. | ||
198 | * | ||
199 | * @param cls closure | ||
200 | * @param key key for the item | ||
201 | * @param size number of bytes in data | ||
202 | * @param data content stored | ||
203 | * @param type type of the content | ||
204 | * @param priority priority of the content | ||
205 | * @param anonymity anonymity-level for the content | ||
206 | * @param replication replication-level for the content | ||
207 | * @param expiration expiration time for the content | ||
208 | * @param msg set to error message | ||
209 | * @return GNUNET_OK on success | ||
210 | */ | ||
211 | static int | ||
212 | heap_plugin_put (void *cls, | ||
213 | const struct GNUNET_HashCode * key, | ||
214 | uint32_t size, | ||
215 | const void *data, | ||
216 | enum GNUNET_BLOCK_Type type, | ||
217 | uint32_t priority, uint32_t anonymity, | ||
218 | uint32_t replication, | ||
219 | struct GNUNET_TIME_Absolute expiration, char **msg) | ||
220 | { | ||
221 | struct Plugin *plugin = cls; | ||
222 | struct Value *value; | ||
223 | |||
224 | value = GNUNET_malloc (sizeof (struct Value) + size); | ||
225 | value->key = *key; | ||
226 | value->data = &value[1]; | ||
227 | value->expire_heap = GNUNET_CONTAINER_heap_insert (plugin->by_expiration, | ||
228 | value, | ||
229 | expiration.abs_value); | ||
230 | value->replication_heap = GNUNET_CONTAINER_heap_insert (plugin->by_replication, | ||
231 | value, | ||
232 | replication); | ||
233 | value->expiration = expiration; | ||
234 | if (0 == anonymity) | ||
235 | { | ||
236 | struct ZeroAnonByType *zabt; | ||
237 | |||
238 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
239 | if (zabt->type == type) | ||
240 | break; | ||
241 | if (NULL == zabt) | ||
242 | { | ||
243 | zabt = GNUNET_malloc (sizeof (struct ZeroAnonByType)); | ||
244 | zabt->type = type; | ||
245 | GNUNET_CONTAINER_DLL_insert (plugin->zero_head, | ||
246 | plugin->zero_tail, | ||
247 | zabt); | ||
248 | } | ||
249 | if (zabt->array_size == zabt->array_pos) | ||
250 | { | ||
251 | GNUNET_array_grow (zabt->array, | ||
252 | zabt->array_size, | ||
253 | zabt->array_size * 2 + 4); | ||
254 | } | ||
255 | value->zero_anon_offset = zabt->array_pos; | ||
256 | zabt->array[zabt->array_pos++] = value; | ||
257 | } | ||
258 | value->size = size; | ||
259 | value->priority = priority; | ||
260 | value->anonymity = anonymity; | ||
261 | value->replication = replication; | ||
262 | value->type = type; | ||
263 | memcpy (&value[1], data, size); | ||
264 | GNUNET_CONTAINER_multihashmap_put (plugin->keyvalue, | ||
265 | &value->key, | ||
266 | value, | ||
267 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
268 | plugin->size += size; | ||
269 | return GNUNET_OK; | ||
270 | } | ||
271 | |||
272 | |||
273 | /** | ||
274 | * Delete the given value, removing it from the plugin's data | ||
275 | * structures. | ||
276 | * | ||
277 | * @param plugin the plugin | ||
278 | * @param value value to delete | ||
279 | */ | ||
280 | static void | ||
281 | delete_value (struct Plugin *plugin, | ||
282 | struct Value *value) | ||
283 | { | ||
284 | GNUNET_assert (GNUNET_YES == | ||
285 | GNUNET_CONTAINER_multihashmap_remove (plugin->keyvalue, | ||
286 | &value->key, | ||
287 | value)); | ||
288 | GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (value->expire_heap)); | ||
289 | GNUNET_assert (value == GNUNET_CONTAINER_heap_remove_node (value->replication_heap)); | ||
290 | if (0 == value->anonymity) | ||
291 | { | ||
292 | struct ZeroAnonByType *zabt; | ||
293 | |||
294 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
295 | if (zabt->type == value->type) | ||
296 | break; | ||
297 | GNUNET_assert (NULL != zabt); | ||
298 | zabt->array[value->zero_anon_offset] = zabt->array[--zabt->array_pos]; | ||
299 | zabt->array[value->zero_anon_offset]->zero_anon_offset = value->zero_anon_offset; | ||
300 | if (0 == zabt->array_pos) | ||
301 | { | ||
302 | GNUNET_array_grow (zabt->array, | ||
303 | zabt->array_size, | ||
304 | 0); | ||
305 | GNUNET_CONTAINER_DLL_remove (plugin->zero_head, | ||
306 | plugin->zero_tail, | ||
307 | zabt); | ||
308 | GNUNET_free (zabt); | ||
309 | } | ||
310 | } | ||
311 | plugin->size -= value->size; | ||
312 | GNUNET_free (value); | ||
313 | } | ||
314 | |||
315 | |||
316 | /** | ||
317 | * Closure for iterator called during 'get_key'. | ||
318 | */ | ||
319 | struct GetContext | ||
320 | { | ||
321 | |||
322 | /** | ||
323 | * Desired result offset / number of results. | ||
324 | */ | ||
325 | uint64_t offset; | ||
326 | |||
327 | /** | ||
328 | * The plugin. | ||
329 | */ | ||
330 | struct Plugin *plugin; | ||
331 | |||
332 | /** | ||
333 | * Requested value hash. | ||
334 | */ | ||
335 | const struct GNUNET_HashCode * vhash; | ||
336 | |||
337 | /** | ||
338 | * Requested type. | ||
339 | */ | ||
340 | enum GNUNET_BLOCK_Type type; | ||
341 | |||
342 | /** | ||
343 | * Function to call with the result. | ||
344 | */ | ||
345 | PluginDatumProcessor proc; | ||
346 | |||
347 | /** | ||
348 | * Closure for 'proc'. | ||
349 | */ | ||
350 | void *proc_cls; | ||
351 | }; | ||
352 | |||
353 | |||
354 | /** | ||
355 | * Test if a value matches the specification from the 'get' context | ||
356 | * | ||
357 | * @param gc query | ||
358 | * @param value the value to check against the query | ||
359 | * @return GNUNET_YES if the value matches | ||
360 | */ | ||
361 | static int | ||
362 | match (const struct GetContext *gc, | ||
363 | struct Value *value) | ||
364 | { | ||
365 | struct GNUNET_HashCode vh; | ||
366 | |||
367 | if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) && | ||
368 | (gc->type != value->type) ) | ||
369 | return GNUNET_NO; | ||
370 | if (NULL != gc->vhash) | ||
371 | { | ||
372 | GNUNET_CRYPTO_hash (&value[1], value->size, &vh); | ||
373 | if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode))) | ||
374 | return GNUNET_NO; | ||
375 | } | ||
376 | return GNUNET_YES; | ||
377 | } | ||
378 | |||
379 | |||
380 | /** | ||
381 | * Count number of matching values. | ||
382 | * | ||
383 | * @param cls the 'struct GetContext' | ||
384 | * @param key unused | ||
385 | * @param val the 'struct Value' | ||
386 | * @return GNUNET_YES (continue iteration) | ||
387 | */ | ||
388 | static int | ||
389 | count_iterator (void *cls, | ||
390 | const struct GNUNET_HashCode *key, | ||
391 | void *val) | ||
392 | { | ||
393 | struct GetContext *gc = cls; | ||
394 | struct Value *value = val; | ||
395 | |||
396 | if (GNUNET_NO == match (gc, value)) | ||
397 | return GNUNET_OK; | ||
398 | gc->offset++; | ||
399 | return GNUNET_OK; | ||
400 | } | ||
401 | |||
402 | |||
403 | /** | ||
404 | * Obtain matching value at 'offset'. | ||
405 | * | ||
406 | * @param cls the 'struct GetContext' | ||
407 | * @param key unused | ||
408 | * @param val the 'struct Value' | ||
409 | * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found | ||
410 | */ | ||
411 | static int | ||
412 | get_iterator (void *cls, | ||
413 | const struct GNUNET_HashCode *key, | ||
414 | void *val) | ||
415 | { | ||
416 | struct GetContext *gc = cls; | ||
417 | struct Value *value = val; | ||
418 | |||
419 | if (GNUNET_NO == match (gc, value)) | ||
420 | return GNUNET_OK; | ||
421 | if (0 != gc->offset--) | ||
422 | return GNUNET_OK; | ||
423 | if (GNUNET_NO == | ||
424 | gc->proc (gc->proc_cls, | ||
425 | key, | ||
426 | value->size, | ||
427 | &value[1], | ||
428 | value->type, | ||
429 | value->priority, | ||
430 | value->anonymity, | ||
431 | value->expiration, | ||
432 | (uint64_t) (long) value)) | ||
433 | delete_value (gc->plugin, value); | ||
434 | return GNUNET_NO; | ||
435 | } | ||
436 | |||
437 | |||
438 | /** | ||
439 | * Get one of the results for a particular key in the datastore. | ||
440 | * | ||
441 | * @param cls closure | ||
442 | * @param offset offset of the result (modulo num-results); | ||
443 | * specific ordering does not matter for the offset | ||
444 | * @param key maybe NULL (to match all entries) | ||
445 | * @param vhash hash of the value, maybe NULL (to | ||
446 | * match all values that have the right key). | ||
447 | * Note that for DBlocks there is no difference | ||
448 | * betwen key and vhash, but for other blocks | ||
449 | * there may be! | ||
450 | * @param type entries of which type are relevant? | ||
451 | * Use 0 for any type. | ||
452 | * @param proc function to call on each matching value; | ||
453 | * will be called with NULL if nothing matches | ||
454 | * @param proc_cls closure for proc | ||
455 | */ | ||
456 | static void | ||
457 | heap_plugin_get_key (void *cls, uint64_t offset, | ||
458 | const struct GNUNET_HashCode *key, | ||
459 | const struct GNUNET_HashCode *vhash, | ||
460 | enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, | ||
461 | void *proc_cls) | ||
462 | { | ||
463 | struct Plugin *plugin = cls; | ||
464 | struct GetContext gc; | ||
465 | |||
466 | gc.plugin = plugin; | ||
467 | gc.offset = 0; | ||
468 | gc.vhash = vhash; | ||
469 | gc.type = type; | ||
470 | gc.proc = proc; | ||
471 | gc.proc_cls = proc_cls; | ||
472 | if (NULL == key) | ||
473 | { | ||
474 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
475 | &count_iterator, | ||
476 | &gc); | ||
477 | if (0 == gc.offset) | ||
478 | { | ||
479 | proc (proc_cls, | ||
480 | NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
481 | return; | ||
482 | } | ||
483 | gc.offset = offset % gc.offset; | ||
484 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
485 | &get_iterator, | ||
486 | &gc); | ||
487 | } | ||
488 | else | ||
489 | { | ||
490 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, | ||
491 | key, | ||
492 | &count_iterator, | ||
493 | &gc); | ||
494 | if (0 == gc.offset) | ||
495 | { | ||
496 | proc (proc_cls, | ||
497 | NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
498 | return; | ||
499 | } | ||
500 | gc.offset = offset % gc.offset; | ||
501 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, | ||
502 | key, | ||
503 | &get_iterator, | ||
504 | &gc); | ||
505 | } | ||
506 | } | ||
507 | |||
508 | |||
509 | /** | ||
510 | * Get a random item for replication. Returns a single, not expired, | ||
511 | * random item from those with the highest replication counters. The | ||
512 | * item's replication counter is decremented by one IF it was positive | ||
513 | * before. Call 'proc' with all values ZERO or NULL if the datastore | ||
514 | * is empty. | ||
515 | * | ||
516 | * @param cls closure | ||
517 | * @param proc function to call the value (once only). | ||
518 | * @param proc_cls closure for proc | ||
519 | */ | ||
520 | static void | ||
521 | heap_plugin_get_replication (void *cls, | ||
522 | PluginDatumProcessor proc, | ||
523 | void *proc_cls) | ||
524 | { | ||
525 | struct Plugin *plugin = cls; | ||
526 | struct Value *value; | ||
527 | |||
528 | value = GNUNET_CONTAINER_heap_remove_root (plugin->by_replication); | ||
529 | if (NULL == value) | ||
530 | { | ||
531 | proc (proc_cls, | ||
532 | NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
533 | return; | ||
534 | } | ||
535 | if (value->replication > 0) | ||
536 | { | ||
537 | value->replication--; | ||
538 | value->replication_heap = GNUNET_CONTAINER_heap_insert (plugin->by_replication, | ||
539 | value, | ||
540 | value->replication); | ||
541 | } | ||
542 | else | ||
543 | { | ||
544 | /* need a better way to pick a random item, replication level is always 0 */ | ||
545 | value->replication_heap = GNUNET_CONTAINER_heap_insert (plugin->by_replication, | ||
546 | value, | ||
547 | value->replication); | ||
548 | value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication); | ||
549 | } | ||
550 | if (GNUNET_NO == | ||
551 | proc (proc_cls, | ||
552 | &value->key, | ||
553 | value->size, | ||
554 | &value[1], | ||
555 | value->type, | ||
556 | value->priority, | ||
557 | value->anonymity, | ||
558 | value->expiration, | ||
559 | (uint64_t) (long) value)) | ||
560 | delete_value (plugin, value); | ||
561 | } | ||
562 | |||
563 | |||
564 | /** | ||
565 | * Get a random item for expiration. Call 'proc' with all values ZERO | ||
566 | * or NULL if the datastore is empty. | ||
567 | * | ||
568 | * @param cls closure | ||
569 | * @param proc function to call the value (once only). | ||
570 | * @param proc_cls closure for proc | ||
571 | */ | ||
572 | static void | ||
573 | heap_plugin_get_expiration (void *cls, PluginDatumProcessor proc, | ||
574 | void *proc_cls) | ||
575 | { | ||
576 | struct Plugin *plugin = cls; | ||
577 | struct Value *value; | ||
578 | |||
579 | value = GNUNET_CONTAINER_heap_peek (plugin->by_expiration); | ||
580 | if (NULL == value) | ||
581 | { | ||
582 | proc (proc_cls, | ||
583 | NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
584 | return; | ||
585 | } | ||
586 | if (GNUNET_NO == | ||
587 | proc (proc_cls, | ||
588 | &value->key, | ||
589 | value->size, | ||
590 | &value[1], | ||
591 | value->type, | ||
592 | value->priority, | ||
593 | value->anonymity, | ||
594 | value->expiration, | ||
595 | (uint64_t) (long) value)) | ||
596 | delete_value (plugin, value); | ||
597 | } | ||
598 | |||
599 | |||
600 | /** | ||
601 | * Update the priority for a particular key in the datastore. If | ||
602 | * the expiration time in value is different than the time found in | ||
603 | * the datastore, the higher value should be kept. For the | ||
604 | * anonymity level, the lower value is to be used. The specified | ||
605 | * priority should be added to the existing priority, ignoring the | ||
606 | * priority in value. | ||
607 | * | ||
608 | * @param cls our "struct Plugin*" | ||
609 | * @param uid unique identifier of the datum | ||
610 | * @param delta by how much should the priority | ||
611 | * change? If priority + delta < 0 the | ||
612 | * priority should be set to 0 (never go | ||
613 | * negative). | ||
614 | * @param expire new expiration time should be the | ||
615 | * MAX of any existing expiration time and | ||
616 | * this value | ||
617 | * @param msg set to error message | ||
618 | * @return GNUNET_OK on success | ||
619 | */ | ||
620 | static int | ||
621 | heap_plugin_update (void *cls, | ||
622 | uint64_t uid, | ||
623 | int delta, | ||
624 | struct GNUNET_TIME_Absolute expire, char **msg) | ||
625 | { | ||
626 | struct Plugin *plugin = cls; | ||
627 | struct Value *value; | ||
628 | |||
629 | value = (struct Value*) (long) uid; | ||
630 | GNUNET_assert (NULL != value); | ||
631 | if (value->expiration.abs_value != expire.abs_value) | ||
632 | { | ||
633 | value->expiration = expire; | ||
634 | GNUNET_CONTAINER_heap_update_cost (plugin->by_expiration, | ||
635 | value->expire_heap, | ||
636 | expire.abs_value); | ||
637 | } | ||
638 | if ( (delta < 0) && (value->priority < - delta) ) | ||
639 | value->priority = 0; | ||
640 | else | ||
641 | value->priority += delta; | ||
642 | return GNUNET_OK; | ||
643 | } | ||
644 | |||
645 | |||
646 | /** | ||
647 | * Call the given processor on an item with zero anonymity. | ||
648 | * | ||
649 | * @param cls our "struct Plugin*" | ||
650 | * @param offset offset of the result (modulo num-results); | ||
651 | * specific ordering does not matter for the offset | ||
652 | * @param type entries of which type should be considered? | ||
653 | * Use 0 for any type. | ||
654 | * @param proc function to call on each matching value; | ||
655 | * will be called with NULL if no value matches | ||
656 | * @param proc_cls closure for proc | ||
657 | */ | ||
658 | static void | ||
659 | heap_plugin_get_zero_anonymity (void *cls, uint64_t offset, | ||
660 | enum GNUNET_BLOCK_Type type, | ||
661 | PluginDatumProcessor proc, void *proc_cls) | ||
662 | { | ||
663 | struct Plugin *plugin = cls; | ||
664 | struct ZeroAnonByType *zabt; | ||
665 | struct Value *value; | ||
666 | uint64_t count; | ||
667 | |||
668 | count = 0; | ||
669 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
670 | { | ||
671 | if ( (type != GNUNET_BLOCK_TYPE_ANY) && | ||
672 | (type != zabt->type) ) | ||
673 | continue; | ||
674 | count += zabt->array_pos; | ||
675 | } | ||
676 | if (0 == count) | ||
677 | { | ||
678 | proc (proc_cls, | ||
679 | NULL, 0, NULL, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); | ||
680 | return; | ||
681 | } | ||
682 | offset = offset % count; | ||
683 | for (zabt = plugin->zero_head; NULL != zabt; zabt = zabt->next) | ||
684 | { | ||
685 | if ( (type != GNUNET_BLOCK_TYPE_ANY) && | ||
686 | (type != zabt->type) ) | ||
687 | continue; | ||
688 | if (offset >= zabt->array_pos) | ||
689 | { | ||
690 | offset -= zabt->array_pos; | ||
691 | continue; | ||
692 | } | ||
693 | break; | ||
694 | } | ||
695 | value = zabt->array[offset]; | ||
696 | if (GNUNET_NO == | ||
697 | proc (proc_cls, | ||
698 | &value->key, | ||
699 | value->size, | ||
700 | &value[1], | ||
701 | value->type, | ||
702 | value->priority, | ||
703 | value->anonymity, | ||
704 | value->expiration, | ||
705 | (uint64_t) (long) value)) | ||
706 | delete_value (plugin, value); | ||
707 | } | ||
708 | |||
709 | |||
710 | /** | ||
711 | * Drop database. | ||
712 | */ | ||
713 | static void | ||
714 | heap_plugin_drop (void *cls) | ||
715 | { | ||
716 | /* nothing needs to be done */ | ||
717 | } | ||
718 | |||
719 | |||
720 | /** | ||
721 | * Closure for the 'return_value' function. | ||
722 | */ | ||
723 | struct GetAllContext | ||
724 | { | ||
725 | /** | ||
726 | * Function to call. | ||
727 | */ | ||
728 | PluginKeyProcessor proc; | ||
729 | |||
730 | /** | ||
731 | * Closure for 'proc'. | ||
732 | */ | ||
733 | void *proc_cls; | ||
734 | }; | ||
735 | |||
736 | |||
737 | /** | ||
738 | * Callback invoked to call callback on each value. | ||
739 | * | ||
740 | * @param cls the plugin | ||
741 | * @param key unused | ||
742 | * @param val the value | ||
743 | * @return GNUNET_OK (continue to iterate) | ||
744 | */ | ||
745 | static int | ||
746 | return_value (void *cls, | ||
747 | const struct GNUNET_HashCode *key, | ||
748 | void *val) | ||
749 | { | ||
750 | struct GetAllContext *gac = cls; | ||
751 | |||
752 | gac->proc (gac->proc_cls, | ||
753 | key, | ||
754 | 1); | ||
755 | return GNUNET_OK; | ||
756 | } | ||
757 | |||
758 | |||
759 | /** | ||
760 | * Get all of the keys in the datastore. | ||
761 | * | ||
762 | * @param cls closure | ||
763 | * @param proc function to call on each key | ||
764 | * @param proc_cls closure for proc | ||
765 | */ | ||
766 | static void | ||
767 | heap_get_keys (void *cls, | ||
768 | PluginKeyProcessor proc, | ||
769 | void *proc_cls) | ||
770 | { | ||
771 | struct Plugin *plugin = cls; | ||
772 | struct GetAllContext gac; | ||
773 | |||
774 | gac.proc = proc; | ||
775 | gac.proc_cls = proc_cls; | ||
776 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
777 | &return_value, | ||
778 | &gac); | ||
779 | } | ||
780 | |||
781 | |||
782 | /** | ||
783 | * Entry point for the plugin. | ||
784 | * | ||
785 | * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" | ||
786 | * @return our "struct Plugin*" | ||
787 | */ | ||
788 | void * | ||
789 | libgnunet_plugin_datastore_heap_init (void *cls) | ||
790 | { | ||
791 | struct GNUNET_DATASTORE_PluginEnvironment *env = cls; | ||
792 | struct GNUNET_DATASTORE_PluginFunctions *api; | ||
793 | struct Plugin *plugin; | ||
794 | unsigned long long esize; | ||
795 | |||
796 | if (GNUNET_OK != | ||
797 | GNUNET_CONFIGURATION_get_value_number (env->cfg, | ||
798 | "datastore-heap", | ||
799 | "HASHMAPSIZE", | ||
800 | &esize)) | ||
801 | esize = 128 * 1024; | ||
802 | plugin = GNUNET_malloc (sizeof (struct Plugin)); | ||
803 | plugin->env = env; | ||
804 | plugin->keyvalue = GNUNET_CONTAINER_multihashmap_create (esize, GNUNET_YES); | ||
805 | plugin->by_expiration = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | ||
806 | plugin->by_replication = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MAX); | ||
807 | api = GNUNET_malloc (sizeof (struct GNUNET_DATASTORE_PluginFunctions)); | ||
808 | api->cls = plugin; | ||
809 | api->estimate_size = &heap_plugin_estimate_size; | ||
810 | api->put = &heap_plugin_put; | ||
811 | api->update = &heap_plugin_update; | ||
812 | api->get_key = &heap_plugin_get_key; | ||
813 | api->get_replication = &heap_plugin_get_replication; | ||
814 | api->get_expiration = &heap_plugin_get_expiration; | ||
815 | api->get_zero_anonymity = &heap_plugin_get_zero_anonymity; | ||
816 | api->drop = &heap_plugin_drop; | ||
817 | api->get_keys = &heap_get_keys; | ||
818 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap", | ||
819 | _("Heap database running\n")); | ||
820 | return api; | ||
821 | } | ||
822 | |||
823 | |||
824 | /** | ||
825 | * Callback invoked to free all value. | ||
826 | * | ||
827 | * @param cls the plugin | ||
828 | * @param key unused | ||
829 | * @param val the value | ||
830 | * @return GNUNET_OK (continue to iterate) | ||
831 | */ | ||
832 | static int | ||
833 | free_value (void *cls, | ||
834 | const struct GNUNET_HashCode *key, | ||
835 | void *val) | ||
836 | { | ||
837 | struct Plugin *plugin = cls; | ||
838 | struct Value *value = val; | ||
839 | |||
840 | delete_value (plugin, value); | ||
841 | return GNUNET_OK; | ||
842 | } | ||
843 | |||
844 | |||
845 | /** | ||
846 | * Exit point from the plugin. | ||
847 | * @param cls our "struct Plugin*" | ||
848 | * @return always NULL | ||
849 | */ | ||
850 | void * | ||
851 | libgnunet_plugin_datastore_heap_done (void *cls) | ||
852 | { | ||
853 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | ||
854 | struct Plugin *plugin = api->cls; | ||
855 | |||
856 | GNUNET_CONTAINER_multihashmap_iterate (plugin->keyvalue, | ||
857 | &free_value, | ||
858 | plugin); | ||
859 | GNUNET_CONTAINER_multihashmap_destroy (plugin->keyvalue); | ||
860 | GNUNET_CONTAINER_heap_destroy (plugin->by_expiration); | ||
861 | GNUNET_CONTAINER_heap_destroy (plugin->by_replication); | ||
862 | GNUNET_free (plugin); | ||
863 | GNUNET_free (api); | ||
864 | return NULL; | ||
865 | } | ||
866 | |||
867 | /* end of plugin_datastore_heap.c */ | ||
diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c index 1e30abe18..2978de4fe 100644 --- a/src/datastore/plugin_datastore_template.c +++ b/src/datastore/plugin_datastore_template.c | |||
@@ -213,6 +213,21 @@ template_plugin_drop (void *cls) | |||
213 | 213 | ||
214 | 214 | ||
215 | /** | 215 | /** |
216 | * Get all of the keys in the datastore. | ||
217 | * | ||
218 | * @param cls closure | ||
219 | * @param proc function to call on each key | ||
220 | * @param proc_cls closure for proc | ||
221 | */ | ||
222 | static void | ||
223 | template_get_keys (void *cls, | ||
224 | PluginKeyProcessor proc, | ||
225 | void *proc_cls) | ||
226 | { | ||
227 | } | ||
228 | |||
229 | |||
230 | /** | ||
216 | * Entry point for the plugin. | 231 | * Entry point for the plugin. |
217 | * | 232 | * |
218 | * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" | 233 | * @param cls the "struct GNUNET_DATASTORE_PluginEnvironment*" |
@@ -237,6 +252,7 @@ libgnunet_plugin_datastore_template_init (void *cls) | |||
237 | api->get_expiration = &template_plugin_get_expiration; | 252 | api->get_expiration = &template_plugin_get_expiration; |
238 | api->get_zero_anonymity = &template_plugin_get_zero_anonymity; | 253 | api->get_zero_anonymity = &template_plugin_get_zero_anonymity; |
239 | api->drop = &template_plugin_drop; | 254 | api->drop = &template_plugin_drop; |
255 | api->get_keys = &template_get_keys; | ||
240 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", | 256 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", |
241 | _("Template database running\n")); | 257 | _("Template database running\n")); |
242 | return api; | 258 | return api; |
diff --git a/src/datastore/test_datastore_api_data_heap.conf b/src/datastore/test_datastore_api_data_heap.conf new file mode 100644 index 000000000..0e6994d5b --- /dev/null +++ b/src/datastore/test_datastore_api_data_heap.conf | |||
@@ -0,0 +1,20 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | SERVICEHOME = /tmp/test-gnunet-datastore-heap/ | ||
4 | |||
5 | [TESTING] | ||
6 | WEAKRANDOM = YES | ||
7 | |||
8 | [arm] | ||
9 | PORT = 42466 | ||
10 | DEFAULTSERVICES = | ||
11 | |||
12 | [statistics] | ||
13 | PORT = 22667 | ||
14 | |||
15 | [resolver] | ||
16 | PORT = 42464 | ||
17 | |||
18 | [datastore] | ||
19 | QUOTA = 10 MB | ||
20 | DATABASE = heap | ||
diff --git a/src/datastore/test_plugin_datastore_data_heap.conf b/src/datastore/test_plugin_datastore_data_heap.conf new file mode 100644 index 000000000..6e352b95b --- /dev/null +++ b/src/datastore/test_plugin_datastore_data_heap.conf | |||
@@ -0,0 +1,6 @@ | |||
1 | @INLINE@ test_defaults.conf | ||
2 | [PATHS] | ||
3 | SERVICEHOME = /tmp/test-gnunet-datastore-plugin-heap/ | ||
4 | |||
5 | [datastore] | ||
6 | DATABASE = heap | ||