adding support for detection quota changes / missing bloomfilter files and reconstruction/recovery code

2 files changed, 114 insertions, 2 deletions

diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 327244ffd..8555fad3a 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -1444,6 +1444,24 @@ cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client)
 
 
 /**
+ * Adds a given key to the bloomfilter 'count' times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
+ */
+static void
+add_key_to_bloomfilter (void *cls,
+                        const GNUNET_HashCode *key,
+                        unsigned int count)
+{
+  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+  while (0 < count--)
+    GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
+/**
  * Process datastore requests.
  *
  * @param cls closure
@@ -1477,6 +1495,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   };
   char *fn;
   unsigned int bf_size;
+  int refresh_bf;
 
   cfg = c;
   if (GNUNET_OK !=
@@ -1509,9 +1528,58 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
     fn = NULL;
   }
   if (fn != NULL)
-    filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+  {
+    if (GNUNET_YES == GNUNET_DISK_file_test (fn))
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+      if (NULL == filter)
+      {
+        /* file exists but not valid, remove and try again, but refresh */
+        if (0 != UNLINK (fn))
+        {
+          /* failed to remove, run without file */
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      _("Failed to remove bogus bloomfilter file `%s'\n"),
+                      fn);
+          GNUNET_free (fn);
+          fn = NULL;
+          filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);        /* approx. 3% false positives at max use */
+          refresh_bf = GNUNET_YES;
+        }
+        else
+        {
+          /* try again after remove */
+          filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+          refresh_bf = GNUNET_YES;
+          if (NULL == filter)
+          {
+            /* failed yet again, give up on using file */
+            GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                        _("Failed to remove bogus bloomfilter file `%s'\n"),
+                        fn);
+            GNUNET_free (fn);
+            fn = NULL;
+            filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);        /* approx. 3% false positives at max use */
+          }
+        }
+      }
+      else
+      {
+        /* normal case: have an existing valid bf file, no need to refresh */
+        refresh_bf = GNUNET_NO;
+      }
+    }
+    else
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+      refresh_bf = GNUNET_YES;
+    }
+  }
   else
+  {
     filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5);      /* approx. 3% false positives at max use */
+    refresh_bf = GNUNET_YES;
+  }
   GNUNET_free_non_null (fn);
   if (filter == NULL)
   {
@@ -1542,6 +1610,19 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
                              &process_stat_in, plugin);
   GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
+  if (GNUNET_YES == refresh_bf)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Rebuilding bloomfilter.  Please be patient.\n"));
+    if (NULL != plugin->api->get_keys)
+      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);  
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                  _("Plugin does not support get_keys function. Please fix!\n"));
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                _("Bloomfilter construction complete.\n"));
+  }
   expired_kill_task =
       GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                           &delete_expired, NULL);
diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index fb1dff51d..bbf0ce2aa 100644
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -135,6 +135,29 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
 
 
 /**
+ * An processor over a set of keys stored in the datastore.
+ *
+ * @param cls closure
+ * @param key key in the data store
+ * @param count how many values are stored under this key in the datastore
+ */
+typedef void (*PluginKeyProcessor) (void *cls, 
+                                   const GNUNET_HashCode *key,
+                                   unsigned int count);
+
+
+/**
+ * Get all of the keys in the datastore.
+ *
+ * @param cls closure
+ * @param proc function to call on each key
+ * @param proc_cls closure for proc
+ */
+typedef void (*PluginGetKeys) (void *cls,
+                               PluginKeyProcessor proc, void *proc_cls);
+
+
+/**
  * Get one of the results for a particular key in the datastore.
  *
  * @param cls closure
@@ -148,6 +171,8 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
  *        there may be!
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
+ * @param min find the smallest key that is larger than the given min,
+ *            NULL for no minimum (return smallest key)
  * @param proc function to call on the matching value;
  *        proc should be called with NULL if there is no result
  * @param proc_cls closure for proc
@@ -159,7 +184,6 @@ typedef void (*PluginGetKey) (void *cls, uint64_t offset,
                               PluginDatumProcessor proc, void *proc_cls);
 
 
-
 /**
  * Get a random item (additional constraints may apply depending on
  * the specific implementation).  Calls 'proc' with all values ZERO or
@@ -174,6 +198,8 @@ typedef void (*PluginGetRandom) (void *cls, PluginDatumProcessor proc,
                                  void *proc_cls);
 
 
+
+
 /**
  * Update the priority for a particular key in the datastore.  If
  * the expiration time in value is different than the time found in
@@ -296,6 +322,11 @@ struct GNUNET_DATASTORE_PluginFunctions
    */
   PluginDrop drop;
 
+  /**
+   * Iterate over all keys in the database.
+   */
+  PluginGetKeys get_keys;
+  
 };