aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristian Grothoff <christian@grothoff.org>2011-12-15 14:55:51 +0000
committerChristian Grothoff <christian@grothoff.org>2011-12-15 14:55:51 +0000
commitf04c8750cca2dcfd35aceae3379b83637b8aadfc (patch)
tree2721f97b3b2dc947aec26c8f46a7dca334ec4ff8
parentd767c4cb4f08364e3dd7a4d58acedfe24098429c (diff)
downloadgnunet-f04c8750cca2dcfd35aceae3379b83637b8aadfc.tar.gz
gnunet-f04c8750cca2dcfd35aceae3379b83637b8aadfc.zip
adding support for detection quota changes / missing bloomfilter files and reconstruction/recovery code
-rw-r--r--src/datastore/gnunet-service-datastore.c83
-rw-r--r--src/include/gnunet_datastore_plugin.h33
2 files changed, 114 insertions, 2 deletions
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 327244ffd..8555fad3a 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -1444,6 +1444,24 @@ cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client)
1444 1444
1445 1445
1446/** 1446/**
1447 * Adds a given key to the bloomfilter 'count' times.
1448 *
1449 * @param cls the bloomfilter
1450 * @param key key to add
1451 * @param count number of times to add key
1452 */
1453static void
1454add_key_to_bloomfilter (void *cls,
1455 const GNUNET_HashCode *key,
1456 unsigned int count)
1457{
1458 struct GNUNET_CONTAINER_BloomFilter *bf = cls;
1459 while (0 < count--)
1460 GNUNET_CONTAINER_bloomfilter_add (bf, key);
1461}
1462
1463
1464/**
1447 * Process datastore requests. 1465 * Process datastore requests.
1448 * 1466 *
1449 * @param cls closure 1467 * @param cls closure
@@ -1477,6 +1495,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
1477 }; 1495 };
1478 char *fn; 1496 char *fn;
1479 unsigned int bf_size; 1497 unsigned int bf_size;
1498 int refresh_bf;
1480 1499
1481 cfg = c; 1500 cfg = c;
1482 if (GNUNET_OK != 1501 if (GNUNET_OK !=
@@ -1509,9 +1528,58 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
1509 fn = NULL; 1528 fn = NULL;
1510 } 1529 }
1511 if (fn != NULL) 1530 if (fn != NULL)
1512 filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ 1531 {
1532 if (GNUNET_YES == GNUNET_DISK_file_test (fn))
1533 {
1534 filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */
1535 if (NULL == filter)
1536 {
1537 /* file exists but not valid, remove and try again, but refresh */
1538 if (0 != UNLINK (fn))
1539 {
1540 /* failed to remove, run without file */
1541 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1542 _("Failed to remove bogus bloomfilter file `%s'\n"),
1543 fn);
1544 GNUNET_free (fn);
1545 fn = NULL;
1546 filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */
1547 refresh_bf = GNUNET_YES;
1548 }
1549 else
1550 {
1551 /* try again after remove */
1552 filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */
1553 refresh_bf = GNUNET_YES;
1554 if (NULL == filter)
1555 {
1556 /* failed yet again, give up on using file */
1557 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1558 _("Failed to remove bogus bloomfilter file `%s'\n"),
1559 fn);
1560 GNUNET_free (fn);
1561 fn = NULL;
1562 filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */
1563 }
1564 }
1565 }
1566 else
1567 {
1568 /* normal case: have an existing valid bf file, no need to refresh */
1569 refresh_bf = GNUNET_NO;
1570 }
1571 }
1572 else
1573 {
1574 filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */
1575 refresh_bf = GNUNET_YES;
1576 }
1577 }
1513 else 1578 else
1579 {
1514 filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */ 1580 filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */
1581 refresh_bf = GNUNET_YES;
1582 }
1515 GNUNET_free_non_null (fn); 1583 GNUNET_free_non_null (fn);
1516 if (filter == NULL) 1584 if (filter == NULL)
1517 { 1585 {
@@ -1542,6 +1610,19 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
1542 &process_stat_in, plugin); 1610 &process_stat_in, plugin);
1543 GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL); 1611 GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
1544 GNUNET_SERVER_add_handlers (server, handlers); 1612 GNUNET_SERVER_add_handlers (server, handlers);
1613 if (GNUNET_YES == refresh_bf)
1614 {
1615 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1616 _("Rebuilding bloomfilter. Please be patient.\n"));
1617 if (NULL != plugin->api->get_keys)
1618 plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
1619 else
1620 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1621 _("Plugin does not support get_keys function. Please fix!\n"));
1622
1623 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1624 _("Bloomfilter construction complete.\n"));
1625 }
1545 expired_kill_task = 1626 expired_kill_task =
1546 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, 1627 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
1547 &delete_expired, NULL); 1628 &delete_expired, NULL);
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index fb1dff51d..bbf0ce2aa 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -135,6 +135,29 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
135 135
136 136
137/** 137/**
138 * An processor over a set of keys stored in the datastore.
139 *
140 * @param cls closure
141 * @param key key in the data store
142 * @param count how many values are stored under this key in the datastore
143 */
144typedef void (*PluginKeyProcessor) (void *cls,
145 const GNUNET_HashCode *key,
146 unsigned int count);
147
148
149/**
150 * Get all of the keys in the datastore.
151 *
152 * @param cls closure
153 * @param proc function to call on each key
154 * @param proc_cls closure for proc
155 */
156typedef void (*PluginGetKeys) (void *cls,
157 PluginKeyProcessor proc, void *proc_cls);
158
159
160/**
138 * Get one of the results for a particular key in the datastore. 161 * Get one of the results for a particular key in the datastore.
139 * 162 *
140 * @param cls closure 163 * @param cls closure
@@ -148,6 +171,8 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
148 * there may be! 171 * there may be!
149 * @param type entries of which type are relevant? 172 * @param type entries of which type are relevant?
150 * Use 0 for any type. 173 * Use 0 for any type.
174 * @param min find the smallest key that is larger than the given min,
175 * NULL for no minimum (return smallest key)
151 * @param proc function to call on the matching value; 176 * @param proc function to call on the matching value;
152 * proc should be called with NULL if there is no result 177 * proc should be called with NULL if there is no result
153 * @param proc_cls closure for proc 178 * @param proc_cls closure for proc
@@ -159,7 +184,6 @@ typedef void (*PluginGetKey) (void *cls, uint64_t offset,
159 PluginDatumProcessor proc, void *proc_cls); 184 PluginDatumProcessor proc, void *proc_cls);
160 185
161 186
162
163/** 187/**
164 * Get a random item (additional constraints may apply depending on 188 * Get a random item (additional constraints may apply depending on
165 * the specific implementation). Calls 'proc' with all values ZERO or 189 * the specific implementation). Calls 'proc' with all values ZERO or
@@ -174,6 +198,8 @@ typedef void (*PluginGetRandom) (void *cls, PluginDatumProcessor proc,
174 void *proc_cls); 198 void *proc_cls);
175 199
176 200
201
202
177/** 203/**
178 * Update the priority for a particular key in the datastore. If 204 * Update the priority for a particular key in the datastore. If
179 * the expiration time in value is different than the time found in 205 * the expiration time in value is different than the time found in
@@ -296,6 +322,11 @@ struct GNUNET_DATASTORE_PluginFunctions
296 */ 322 */
297 PluginDrop drop; 323 PluginDrop drop;
298 324
325 /**
326 * Iterate over all keys in the database.
327 */
328 PluginGetKeys get_keys;
329
299}; 330};
300 331
301 332