diff options
author | Christian Grothoff <christian@grothoff.org> | 2012-11-12 14:40:11 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2012-11-12 14:40:11 +0000 |
commit | f397c283914604f4d526e7b9fc438c7ccc56bded (patch) | |
tree | 149a834894fa2c436dfa0bfe109dc03f50123c9a /src | |
parent | 386e076d4958b21dcc100661ec1343054904949a (diff) | |
download | gnunet-f397c283914604f4d526e7b9fc438c7ccc56bded.tar.gz gnunet-f397c283914604f4d526e7b9fc438c7ccc56bded.zip |
-adding plugin to keep datacache entirely in the heap
Diffstat (limited to 'src')
-rw-r--r-- | src/datacache/Makefile.am | 10 | ||||
-rw-r--r-- | src/datacache/datacache.c | 4 | ||||
-rw-r--r-- | src/datacache/plugin_datacache_heap.c | 395 |
3 files changed, 407 insertions, 2 deletions
diff --git a/src/datacache/Makefile.am b/src/datacache/Makefile.am index 0f10e6c1b..99e333dc9 100644 --- a/src/datacache/Makefile.am +++ b/src/datacache/Makefile.am | |||
@@ -44,6 +44,7 @@ plugin_LTLIBRARIES = \ | |||
44 | $(SQLITE_PLUGIN) \ | 44 | $(SQLITE_PLUGIN) \ |
45 | $(MYSQL_PLUGIN) \ | 45 | $(MYSQL_PLUGIN) \ |
46 | $(POSTGRES_PLUGIN) \ | 46 | $(POSTGRES_PLUGIN) \ |
47 | libgnunet_plugin_datacache_heap.la \ | ||
47 | libgnunet_plugin_datacache_template.la | 48 | libgnunet_plugin_datacache_template.la |
48 | 49 | ||
49 | 50 | ||
@@ -56,6 +57,15 @@ libgnunet_plugin_datacache_sqlite_la_LIBADD = \ | |||
56 | libgnunet_plugin_datacache_sqlite_la_LDFLAGS = \ | 57 | libgnunet_plugin_datacache_sqlite_la_LDFLAGS = \ |
57 | $(GN_PLUGIN_LDFLAGS) | 58 | $(GN_PLUGIN_LDFLAGS) |
58 | 59 | ||
60 | libgnunet_plugin_datacache_heap_la_SOURCES = \ | ||
61 | plugin_datacache_heap.c | ||
62 | libgnunet_plugin_datacache_heap_la_LIBADD = \ | ||
63 | $(top_builddir)/src/statistics/libgnunetstatistics.la \ | ||
64 | $(top_builddir)/src/util/libgnunetutil.la $(XLIBS) \ | ||
65 | $(LTLIBINTL) | ||
66 | libgnunet_plugin_datacache_heap_la_LDFLAGS = \ | ||
67 | $(GN_PLUGIN_LDFLAGS) | ||
68 | |||
59 | libgnunet_plugin_datacache_mysql_la_SOURCES = \ | 69 | libgnunet_plugin_datacache_mysql_la_SOURCES = \ |
60 | plugin_datacache_mysql.c | 70 | plugin_datacache_mysql.c |
61 | libgnunet_plugin_datacache_mysql_la_LIBADD = \ | 71 | libgnunet_plugin_datacache_mysql_la_LIBADD = \ |
diff --git a/src/datacache/datacache.c b/src/datacache/datacache.c index 1af4a148f..53f7cc864 100644 --- a/src/datacache/datacache.c +++ b/src/datacache/datacache.c | |||
@@ -249,9 +249,9 @@ GNUNET_DATACACHE_put (struct GNUNET_DATACACHE_Handle *h, | |||
249 | uint32_t used; | 249 | uint32_t used; |
250 | 250 | ||
251 | used = h->api->put (h->api->cls, key, size, data, type, discard_time); | 251 | used = h->api->put (h->api->cls, key, size, data, type, discard_time); |
252 | if (used == 0) | 252 | if (0 == used) |
253 | { | 253 | { |
254 | GNUNET_break (0); | 254 | /* error or duplicate */ |
255 | return GNUNET_SYSERR; | 255 | return GNUNET_SYSERR; |
256 | } | 256 | } |
257 | LOG (GNUNET_ERROR_TYPE_DEBUG, "Stored data under key `%s' in cache\n", | 257 | LOG (GNUNET_ERROR_TYPE_DEBUG, "Stored data under key `%s' in cache\n", |
diff --git a/src/datacache/plugin_datacache_heap.c b/src/datacache/plugin_datacache_heap.c new file mode 100644 index 000000000..2f55bcf53 --- /dev/null +++ b/src/datacache/plugin_datacache_heap.c | |||
@@ -0,0 +1,395 @@ | |||
1 | /* | ||
2 | This file is part of GNUnet | ||
3 | (C) 2012 Christian Grothoff (and other contributing authors) | ||
4 | |||
5 | GNUnet is free software; you can redistribute it and/or modify | ||
6 | it under the terms of the GNU General Public License as published | ||
7 | by the Free Software Foundation; either version 3, or (at your | ||
8 | option) any later version. | ||
9 | |||
10 | GNUnet is distributed in the hope that it will be useful, but | ||
11 | WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
13 | General Public License for more details. | ||
14 | |||
15 | You should have received a copy of the GNU General Public License | ||
16 | along with GNUnet; see the file COPYING. If not, write to the | ||
17 | Free Software Foundation, Inc., 59 Temple Place - Suite 330, | ||
18 | Boston, MA 02111-1307, USA. | ||
19 | */ | ||
20 | |||
21 | /** | ||
22 | * @file datacache/plugin_datacache_heap.c | ||
23 | * @brief heap-only implementation of a database backend for the datacache | ||
24 | * @author Christian Grothoff | ||
25 | */ | ||
26 | #include "platform.h" | ||
27 | #include "gnunet_util_lib.h" | ||
28 | #include "gnunet_datacache_plugin.h" | ||
29 | |||
30 | #define LOG(kind,...) GNUNET_log_from (kind, "datacache-heap", __VA_ARGS__) | ||
31 | |||
32 | #define LOG_STRERROR_FILE(kind,op,fn) GNUNET_log_from_strerror_file (kind, "datacache-heap", op, fn) | ||
33 | |||
34 | |||
35 | /** | ||
36 | * Context for all functions in this plugin. | ||
37 | */ | ||
38 | struct Plugin | ||
39 | { | ||
40 | /** | ||
41 | * Our execution environment. | ||
42 | */ | ||
43 | struct GNUNET_DATACACHE_PluginEnvironment *env; | ||
44 | |||
45 | /** | ||
46 | * Our hash map. | ||
47 | */ | ||
48 | struct GNUNET_CONTAINER_MultiHashMap *map; | ||
49 | |||
50 | /** | ||
51 | * Heap for expirations. | ||
52 | */ | ||
53 | struct GNUNET_CONTAINER_Heap *heap; | ||
54 | |||
55 | }; | ||
56 | |||
57 | |||
58 | /** | ||
59 | * Entry in the hash map. | ||
60 | */ | ||
61 | struct Value | ||
62 | { | ||
63 | /** | ||
64 | * Key for the entry. | ||
65 | */ | ||
66 | struct GNUNET_HashCode key; | ||
67 | |||
68 | /** | ||
69 | * Expiration time. | ||
70 | */ | ||
71 | struct GNUNET_TIME_Absolute discard_time; | ||
72 | |||
73 | /** | ||
74 | * Corresponding node in the heap. | ||
75 | */ | ||
76 | struct GNUNET_CONTAINER_HeapNode *hn; | ||
77 | |||
78 | /** | ||
79 | * Payload (actual payload follows this struct) | ||
80 | */ | ||
81 | size_t size; | ||
82 | |||
83 | /** | ||
84 | * Type of the block. | ||
85 | */ | ||
86 | enum GNUNET_BLOCK_Type type; | ||
87 | |||
88 | }; | ||
89 | |||
90 | |||
91 | /** | ||
92 | * Closure for 'put_cb'. | ||
93 | */ | ||
94 | struct PutContext | ||
95 | { | ||
96 | /** | ||
97 | * Expiration time for the new value. | ||
98 | */ | ||
99 | struct GNUNET_TIME_Absolute discard_time; | ||
100 | |||
101 | /** | ||
102 | * Data for the new value. | ||
103 | */ | ||
104 | const char *data; | ||
105 | |||
106 | /** | ||
107 | * Heap from the plugin. | ||
108 | */ | ||
109 | struct GNUNET_CONTAINER_Heap *heap; | ||
110 | |||
111 | /** | ||
112 | * Number of bytes in 'data'. | ||
113 | */ | ||
114 | size_t size; | ||
115 | |||
116 | /** | ||
117 | * Type of the node. | ||
118 | */ | ||
119 | enum GNUNET_BLOCK_Type type; | ||
120 | |||
121 | /** | ||
122 | * Value to set to GNUNET_YES if an equivalent block was found. | ||
123 | */ | ||
124 | int found; | ||
125 | }; | ||
126 | |||
127 | |||
128 | /** | ||
129 | * Function called during PUT to detect if an equivalent block | ||
130 | * already exists. | ||
131 | * | ||
132 | * @param cls the 'struct PutContext' | ||
133 | * @param key the key for the value(s) | ||
134 | * @param value an existing value | ||
135 | * @return GNUNET_YES if not found (to continue to iterate) | ||
136 | */ | ||
137 | static int | ||
138 | put_cb (void *cls, | ||
139 | const struct GNUNET_HashCode *key, | ||
140 | void *value) | ||
141 | { | ||
142 | struct PutContext *put_ctx = cls; | ||
143 | struct Value *val = value; | ||
144 | |||
145 | if ( (val->size == put_ctx->size) && | ||
146 | (val->type == put_ctx->type) && | ||
147 | (0 == memcmp (&val[1], put_ctx->data, put_ctx->size)) ) | ||
148 | { | ||
149 | put_ctx->found = GNUNET_YES; | ||
150 | val->discard_time = GNUNET_TIME_absolute_max (val->discard_time, | ||
151 | put_ctx->discard_time); | ||
152 | GNUNET_CONTAINER_heap_update_cost (put_ctx->heap, | ||
153 | val->hn, | ||
154 | val->discard_time.abs_value); | ||
155 | return GNUNET_NO; | ||
156 | } | ||
157 | if (val->type == put_ctx->type) | ||
158 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
159 | "Got another value for key %s and type %d (size %u vs %u)\n", | ||
160 | GNUNET_h2s (key), | ||
161 | val->type, | ||
162 | (unsigned int) val->size, | ||
163 | (unsigned int) put_ctx->size); | ||
164 | return GNUNET_YES; | ||
165 | } | ||
166 | |||
167 | |||
168 | /** | ||
169 | * Store an item in the datastore. | ||
170 | * | ||
171 | * @param cls closure (our "struct Plugin") | ||
172 | * @param key key to store data under | ||
173 | * @param size number of bytes in data | ||
174 | * @param data data to store | ||
175 | * @param type type of the value | ||
176 | * @param discard_time when to discard the value in any case | ||
177 | * @return 0 on error, number of bytes used otherwise | ||
178 | */ | ||
179 | static size_t | ||
180 | heap_plugin_put (void *cls, const struct GNUNET_HashCode * key, size_t size, | ||
181 | const char *data, enum GNUNET_BLOCK_Type type, | ||
182 | struct GNUNET_TIME_Absolute discard_time) | ||
183 | { | ||
184 | struct Plugin *plugin = cls; | ||
185 | struct Value *val; | ||
186 | struct PutContext put_ctx; | ||
187 | |||
188 | put_ctx.found = GNUNET_NO; | ||
189 | put_ctx.heap = plugin->heap; | ||
190 | put_ctx.data = data; | ||
191 | put_ctx.size = size; | ||
192 | put_ctx.discard_time = discard_time; | ||
193 | put_ctx.type = type; | ||
194 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->map, | ||
195 | key, | ||
196 | put_cb, | ||
197 | &put_ctx); | ||
198 | if (GNUNET_YES == put_ctx.found) | ||
199 | return 0; | ||
200 | val = GNUNET_malloc (sizeof (struct Value) + size); | ||
201 | memcpy (&val[1], data, size); | ||
202 | val->key = *key; | ||
203 | val->type = type; | ||
204 | val->discard_time = discard_time; | ||
205 | val->size = size; | ||
206 | (void) GNUNET_CONTAINER_multihashmap_put (plugin->map, | ||
207 | &val->key, | ||
208 | val, | ||
209 | GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); | ||
210 | val->hn = GNUNET_CONTAINER_heap_insert (plugin->heap, | ||
211 | val, | ||
212 | val->discard_time.abs_value); | ||
213 | return size; | ||
214 | } | ||
215 | |||
216 | |||
217 | /** | ||
218 | * Closure for 'get_cb'. | ||
219 | */ | ||
220 | struct GetContext | ||
221 | { | ||
222 | /** | ||
223 | * Function to call for each result. | ||
224 | */ | ||
225 | GNUNET_DATACACHE_Iterator iter; | ||
226 | |||
227 | /** | ||
228 | * Closure for 'iter'. | ||
229 | */ | ||
230 | void *iter_cls; | ||
231 | |||
232 | /** | ||
233 | * Number of results found. | ||
234 | */ | ||
235 | unsigned int cnt; | ||
236 | |||
237 | /** | ||
238 | * Block type requested. | ||
239 | */ | ||
240 | enum GNUNET_BLOCK_Type type; | ||
241 | }; | ||
242 | |||
243 | |||
244 | |||
245 | /** | ||
246 | * Function called during GET to find matching blocks. | ||
247 | * Only matches by type. | ||
248 | * | ||
249 | * @param cls the 'struct GetContext' | ||
250 | * @param key the key for the value(s) | ||
251 | * @param value an existing value | ||
252 | * @return GNUNET_YES to continue to iterate | ||
253 | */ | ||
254 | static int | ||
255 | get_cb (void *cls, | ||
256 | const struct GNUNET_HashCode *key, | ||
257 | void *value) | ||
258 | { | ||
259 | struct GetContext *get_ctx = cls; | ||
260 | struct Value *val = value; | ||
261 | int ret; | ||
262 | |||
263 | if ( (get_ctx->type != val->type) && | ||
264 | (GNUNET_BLOCK_TYPE_ANY != get_ctx->type) ) | ||
265 | return GNUNET_OK; | ||
266 | ret = get_ctx->iter (get_ctx->iter_cls, | ||
267 | val->discard_time, | ||
268 | key, | ||
269 | val->size, | ||
270 | (const char *) &val[1], | ||
271 | val->type); | ||
272 | |||
273 | get_ctx->cnt++; | ||
274 | return ret; | ||
275 | } | ||
276 | |||
277 | |||
278 | /** | ||
279 | * Iterate over the results for a particular key | ||
280 | * in the datastore. | ||
281 | * | ||
282 | * @param cls closure (our "struct Plugin") | ||
283 | * @param key | ||
284 | * @param type entries of which type are relevant? | ||
285 | * @param iter maybe NULL (to just count) | ||
286 | * @param iter_cls closure for iter | ||
287 | * @return the number of results found | ||
288 | */ | ||
289 | static unsigned int | ||
290 | heap_plugin_get (void *cls, const struct GNUNET_HashCode * key, | ||
291 | enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, | ||
292 | void *iter_cls) | ||
293 | { | ||
294 | struct Plugin *plugin = cls; | ||
295 | struct GetContext get_ctx; | ||
296 | |||
297 | get_ctx.type = type; | ||
298 | get_ctx.iter = iter; | ||
299 | get_ctx.iter_cls = iter_cls; | ||
300 | get_ctx.cnt = 0; | ||
301 | GNUNET_CONTAINER_multihashmap_get_multiple (plugin->map, | ||
302 | key, | ||
303 | get_cb, | ||
304 | &get_ctx); | ||
305 | return get_ctx.cnt; | ||
306 | } | ||
307 | |||
308 | |||
309 | /** | ||
310 | * Delete the entry with the lowest expiration value | ||
311 | * from the datacache right now. | ||
312 | * | ||
313 | * @param cls closure (our "struct Plugin") | ||
314 | * @return GNUNET_OK on success, GNUNET_SYSERR on error | ||
315 | */ | ||
316 | static int | ||
317 | heap_plugin_del (void *cls) | ||
318 | { | ||
319 | struct Plugin *plugin = cls; | ||
320 | struct Value *val; | ||
321 | |||
322 | val = GNUNET_CONTAINER_heap_remove_root (plugin->heap); | ||
323 | if (NULL == val) | ||
324 | return GNUNET_SYSERR; | ||
325 | GNUNET_assert (GNUNET_YES == | ||
326 | GNUNET_CONTAINER_multihashmap_remove (plugin->map, | ||
327 | &val->key, | ||
328 | val)); | ||
329 | plugin->env->delete_notify (plugin->env->cls, | ||
330 | &val->key, | ||
331 | val->size); | ||
332 | GNUNET_free (val); | ||
333 | return GNUNET_OK; | ||
334 | } | ||
335 | |||
336 | |||
337 | /** | ||
338 | * Entry point for the plugin. | ||
339 | * | ||
340 | * @param cls closure (the "struct GNUNET_DATACACHE_PluginEnvironmnet") | ||
341 | * @return the plugin's closure (our "struct Plugin") | ||
342 | */ | ||
343 | void * | ||
344 | libgnunet_plugin_datacache_heap_init (void *cls) | ||
345 | { | ||
346 | struct GNUNET_DATACACHE_PluginEnvironment *env = cls; | ||
347 | struct GNUNET_DATACACHE_PluginFunctions *api; | ||
348 | struct Plugin *plugin; | ||
349 | |||
350 | plugin = GNUNET_malloc (sizeof (struct Plugin)); | ||
351 | plugin->map = GNUNET_CONTAINER_multihashmap_create (1024, /* FIXME: base on quota! */ | ||
352 | GNUNET_YES); | ||
353 | plugin->heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); | ||
354 | plugin->env = env; | ||
355 | api = GNUNET_malloc (sizeof (struct GNUNET_DATACACHE_PluginFunctions)); | ||
356 | api->cls = plugin; | ||
357 | api->get = &heap_plugin_get; | ||
358 | api->put = &heap_plugin_put; | ||
359 | api->del = &heap_plugin_del; | ||
360 | LOG (GNUNET_ERROR_TYPE_INFO, _("Heap datacache running\n")); | ||
361 | return api; | ||
362 | } | ||
363 | |||
364 | |||
365 | /** | ||
366 | * Exit point from the plugin. | ||
367 | * | ||
368 | * @param cls closure (our "struct Plugin") | ||
369 | * @return NULL | ||
370 | */ | ||
371 | void * | ||
372 | libgnunet_plugin_datacache_heap_done (void *cls) | ||
373 | { | ||
374 | struct GNUNET_DATACACHE_PluginFunctions *api = cls; | ||
375 | struct Plugin *plugin = api->cls; | ||
376 | struct Value *val; | ||
377 | |||
378 | while (NULL != (val = GNUNET_CONTAINER_heap_remove_root (plugin->heap))) | ||
379 | { | ||
380 | GNUNET_assert (GNUNET_YES == | ||
381 | GNUNET_CONTAINER_multihashmap_remove (plugin->map, | ||
382 | &val->key, | ||
383 | val)); | ||
384 | GNUNET_free (val); | ||
385 | } | ||
386 | GNUNET_CONTAINER_heap_destroy (plugin->heap); | ||
387 | GNUNET_CONTAINER_multihashmap_destroy (plugin->map); | ||
388 | GNUNET_free (plugin); | ||
389 | GNUNET_free (api); | ||
390 | return NULL; | ||
391 | } | ||
392 | |||
393 | |||
394 | |||
395 | /* end of plugin_datacache_heap.c */ | ||