train hacks

6 files changed, 397 insertions, 224 deletions

diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 40ea153de..01778b1b7 100644
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -42,6 +42,13 @@
  */
 #define MAX_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
 
+#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
+
+/**
+ * After how many payload-changing operations
+ * do we sync our statistics?
+ */
+#define MAX_STAT_SYNC_LAG 50
 
 
 /**
@@ -109,6 +116,7 @@ struct ReservationList
 };
 
 
+
 /**
  * Our datastore plugin (NULL if not available).
  */
@@ -141,6 +149,24 @@ static unsigned long long cache_size;
  * How much space have we currently reserved?
  */
 static unsigned long long reserved;
+  
+/**
+ * How much data are we currently storing
+ * in the database?
+ */
+static unsigned long long payload;
+
+/**
+ * Number of updates that were made to the
+ * payload value since we last synchronized
+ * it with the statistics service.
+ */
+static unsigned int lastSync;
+
+/**
+ * Did we get an answer from statistics?
+ */
+static int stats_worked;
 
 /**
  * Identity of the task that is used to delete
@@ -165,6 +191,23 @@ static struct GNUNET_STATISTICS_Handle *stats;
 
 
 /**
+ * Synchronize our utilization statistics with the 
+ * statistics service.
+ */
+static void 
+sync_stats ()
+{
+  GNUNET_STATISTICS_set (stats,
+                         QUOTA_STAT_NAME,
+                         payload,
+                         GNUNET_YES);
+  lastSync = 0;
+}
+
+
+
+
+/**
  * Function called once the transmit operation has
  * either failed or succeeded.
  *
@@ -242,6 +285,12 @@ static struct TransmitCallbackContext *tcc_tail;
 static int cleaning_done;
 
 /**
+ * Handle for pending get request.
+ */
+static struct GNUNET_STATISTICS_GetHandle *stat_get;
+
+
+/**
  * Task that is used to remove expired entries from
  * the datastore.  This task will schedule itself
  * again automatically to always delete all expired
@@ -731,7 +780,7 @@ handle_reserve (void *cls,
 #endif
   amount = GNUNET_ntohll(msg->amount);
   entries = ntohl(msg->entries);
-  used = plugin->api->get_size (plugin->api->cls) + reserved;
+  used = payload + reserved;
   req = amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
   if (used + req > quota)
     {
@@ -931,13 +980,13 @@ execute_put (struct GNUNET_SERVER_Client *client,
                    (GNUNET_SYSERR == ret) ? GNUNET_SYSERR : GNUNET_OK, 
                    msg);
   GNUNET_free_non_null (msg);
-  if (quota - reserved - cache_size < plugin->api->get_size (plugin->api->cls))
+  if (quota - reserved - cache_size < payload)
     {
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                   _("Need %llu bytes more space (%llu allowed, using %llu)\n"),
                   (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
                   (unsigned long long) (quota - reserved - cache_size),
-                  (unsigned long long) plugin->api->get_size (plugin->api->cls));
+                  (unsigned long long) payload);
       manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
     }
 }
@@ -1351,6 +1400,78 @@ handle_drop (void *cls,
 
 
 /**
+ * Function called by plugins to notify us about a
+ * change in their disk utilization.
+ *
+ * @param cls closure (NULL)
+ * @param delta change in disk utilization, 
+ *        0 for "reset to empty"
+ */
+static void
+disk_utilization_change_cb (void *cls,
+                            int delta)
+{
+  if ( (delta < 0) &&
+       (payload < -delta) )
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _("Datastore payload inaccurate (%lld < %lld).  Trying to fix.\n"),
+                  (long long) payload,
+                  (long long) -delta);
+      payload = plugin->api->get_size (plugin->api->cls);
+      sync_stats ();
+      return;
+    }
+  payload += delta;
+  lastSync++;
+  if (lastSync >= MAX_STAT_SYNC_LAG)
+    sync_stats ();
+}
+
+
+/**
+ * Callback function to process statistic values.
+ *
+ * @param cls closure (struct Plugin*)
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+process_stat_in (void *cls,
+                 const char *subsystem,
+                 const char *name,
+                 uint64_t value,
+                 int is_persistent)
+{
+  GNUNET_assert (stats_worked == GNUNET_NO);
+  stats_worked = GNUNET_YES;
+  payload += value;
+#if DEBUG_SQLITE
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Notification from statistics about existing payload (%llu), new payload is %llu\n",
+              value,
+              payload);
+#endif
+  return GNUNET_OK;
+}
+
+
+static void
+process_stat_done (void *cls,
+                   int success)
+{
+  struct DatastorePlugin *plugin = cls;
+
+  stat_get = NULL;
+  if (stats_worked == GNUNET_NO) 
+    payload = plugin->api->get_size (plugin->api->cls);
+}
+
+
+/**
  * Load the datastore plugin.
  */
 static struct DatastorePlugin *
@@ -1373,6 +1494,8 @@ load_plugin ()
   ret = GNUNET_malloc (sizeof(struct DatastorePlugin));
   ret->env.cfg = cfg;
   ret->env.sched = sched;  
+  ret->env.duc = &disk_utilization_change_cb;
+  ret->env.cls = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               _("Loading `%s' datastore plugin\n"), name);
   GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
@@ -1426,6 +1549,13 @@ unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       GNUNET_CONTAINER_bloomfilter_free (filter);
       filter = NULL;
     }
+  if (lastSync > 0)
+    sync_stats ();
+  if (stat_get != NULL)
+    {
+      GNUNET_STATISTICS_get_cancel (stat_get);
+      stat_get = NULL;
+    }
   if (stats != NULL)
     {
       GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
@@ -1614,6 +1744,13 @@ run (void *cls,
         }
       return;
     }
+  stat_get = GNUNET_STATISTICS_get (stats,
+                                    "datastore",
+                                    QUOTA_STAT_NAME,
+                                    GNUNET_TIME_UNIT_SECONDS,
+                                    &process_stat_done,
+                                    &process_stat_in,
+                                    plugin);
   GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
   expired_kill_task
diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c
index 17cd009bc..4d39d85ac 100644
--- a/src/datastore/perf_plugin_datastore.c
+++ b/src/datastore/perf_plugin_datastore.c
@@ -81,6 +81,20 @@ struct CpsRunContext
 };
 
 
+/**
+ * Function called by plugins to notify us about a
+ * change in their disk utilization.
+ *
+ * @param cls closure (NULL)
+ * @param delta change in disk utilization, 
+ *        0 for "reset to empty"
+ */
+static void
+disk_utilization_change_cb (void *cls,
+                            int delta)
+{
+}
+
              
 static void
 putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k)
@@ -331,6 +345,8 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg,
     }
   env.cfg = cfg;
   env.sched = sched;  
+  env.duc = &disk_utilization_change_cb;
+  env.cls = NULL;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               _("Loading `%s' datastore plugin\n"), name);
   GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
diff --git a/src/datastore/plugin_datastore.h b/src/datastore/plugin_datastore.h
index e8f433671..fa08501cc 100644
--- a/src/datastore/plugin_datastore.h
+++ b/src/datastore/plugin_datastore.h
@@ -22,10 +22,6 @@
  * @file datastore/plugin_datastore.h
  * @brief API for the database backend plugins.
  * @author Christian Grothoff
- *
- * TODO:
- * - consider defining enumeration or at least typedef
- *   for the type of "type" (instead of using uint32_t)
  */
 #ifndef PLUGIN_DATASTORE_H
 #define PLUGIN_DATASTORE_H
@@ -39,12 +35,24 @@
 
 /**
  * How many bytes of overhead will we assume per entry
- * in the SQlite DB?
+ * in any DB (for reservations)?
  */
 #define GNUNET_DATASTORE_ENTRY_OVERHEAD 256
 
 
 /**
+ * Function invoked to notify service of disk utilization
+ * changes.
+ *
+ * @param cls closure
+ * @param delta change in disk utilization, 
+ *        0 for "reset to empty"
+ */
+typedef void (*DiskUtilizationChange)(void *cls,
+                                      int delta);
+
+
+/**
  * The datastore service will pass a pointer to a struct
  * of this type as the first and only argument to the
  * entry point of each datastore plugin.
@@ -61,6 +69,16 @@ struct GNUNET_DATASTORE_PluginEnvironment
    */
   struct GNUNET_SCHEDULER_Handle *sched;
 
+  /**
+   * Function to call on disk utilization change.
+   */
+  DiskUtilizationChange duc;
+
+  /**
+   * Closure.
+   */
+  void *cls;
+
 };
 
 
diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c
index c216e989c..ea6cc2322 100644
--- a/src/datastore/plugin_datastore_mysql.c
+++ b/src/datastore/plugin_datastore_mysql.c
@@ -191,7 +191,6 @@
                                   " ORDER BY expire DESC,vkey DESC LIMIT 1)"\
                                   "ORDER BY expire DESC,vkey DESC LIMIT 1"
 
-// #define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn072"
 
 
 struct GNUNET_MysqlStatementHandle
@@ -344,12 +343,10 @@ struct Plugin
 #define UPDATE_ENTRY "UPDATE gn090 SET prio=prio+?,expire=IF(expire>=?,expire,?) WHERE vkey=?"
   struct GNUNET_MysqlStatementHandle *update_entry;
 
-  struct GNUNET_MysqlStatementHandle *iter[4];
+#define SELECT_SIZE "SELECT SUM(BIT_LENGTH(value) DIV 8) FROM gn072"
+  struct GNUNET_MysqlStatementHandle *get_size;
 
-  /**
-   * Size of the mysql database on disk.
-   */
-  unsigned long long content_size;
+  struct GNUNET_MysqlStatementHandle *iter[4];
 
 };
 
@@ -957,8 +954,11 @@ do_delete_entry_by_vkey (struct Plugin *plugin,
   return ret;
 }
 
+
 static int
-return_ok (void *cls, unsigned int num_values, MYSQL_BIND * values)
+return_ok (void *cls, 
+           unsigned int num_values, 
+           MYSQL_BIND * values)
 {
   return GNUNET_OK;
 }
@@ -1189,7 +1189,9 @@ mysql_next_request_cont (void *next_cls,
     {
       do_delete_value (plugin, vkey);
       do_delete_entry_by_vkey (plugin, vkey);
-      plugin->content_size -= length;
+      if (length != 0)
+        plugin->env->duc (plugin->env->cls,
+                          - length);
     }
   return;
  END_SET:
@@ -1279,14 +1281,29 @@ iterateHelper (struct Plugin *plugin,
  * Get an estimate of how much space the database is
  * currently using.
  *
- * @param cls our "struct Plugin*"
+ * @param cls our "struct Plugin *"
  * @return number of bytes used on disk
  */
 static unsigned long long
 mysql_plugin_get_size (void *cls)
 {
   struct Plugin *plugin = cls;
-  return plugin->content_size;
+  MYSQL_BIND cbind[1];
+  long long total;
+
+  memset (cbind, 0, sizeof (cbind));
+  total = 0;
+  cbind[0].buffer_type = MYSQL_TYPE_LONGLONG;
+  cbind[0].buffer = &total;
+  cbind[0].is_unsigned = GNUNET_NO;
+  if (GNUNET_OK != 
+      prepared_statement_run_select (plugin,
+                                     plugin->get_size,
+                                     1, cbind, 
+                                     &return_ok, NULL,
+                                     -1))
+    return 0;
+  return total;
 }
 
 
@@ -1373,7 +1390,9 @@ mysql_plugin_put (void *cls,
               vkey,
               (unsigned int) size);
 #endif
-  plugin->content_size += size;
+  if (size > 0)
+    plugin->env->duc (plugin->env->cls,
+                      size);
   return GNUNET_OK;
 }
 
@@ -1804,8 +1823,8 @@ mysql_plugin_drop (void *cls)
                                    "DROP TABLE gn090")) ||
       (GNUNET_OK != run_statement (plugin,
                                    "DROP TABLE gn072")))
-    return;                     /* error */
-  plugin->content_size = 0;
+    return;           /* error */
+  plugin->env->duc (plugin->env->cls, 0);
 }
 
 
@@ -1865,6 +1884,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls)
       || PINIT (plugin->select_entry_by_hash_vhash_and_type,
                 SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE)
       || PINIT (plugin->count_entry_by_hash, COUNT_ENTRY_BY_HASH)
+      || PINIT (plugin->get_size, SELECT_SIZE)
       || PINIT (plugin->count_entry_by_hash_and_vhash, COUNT_ENTRY_BY_HASH_AND_VHASH)
       || PINIT (plugin->count_entry_by_hash_and_type, COUNT_ENTRY_BY_HASH_AND_TYPE)
       || PINIT (plugin->count_entry_by_hash_vhash_and_type,
diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c
index 889309784..54bdde542 100644
--- a/src/datastore/plugin_datastore_postgres.c
+++ b/src/datastore/plugin_datastore_postgres.c
@@ -82,26 +82,104 @@
 #define BUSY_TIMEOUT GNUNET_TIME_UNIT_SECONDS
 
 
+/**
+ * Closure for 'postgres_next_request_cont'.
+ */
 struct NextRequestClosure
 {
+  /**
+   * Global plugin data.
+   */
   struct Plugin *plugin;
+  
+  /**
+   * Function to call for each matching entry.
+   */
   PluginIterator iter;
+  
+  /**
+   * Closure for 'iter'.
+   */
   void *iter_cls;
+  
+  /**
+   * Parameters for the prepared statement.
+   */
   const char *paramValues[5];
+  
+  /**
+   * Name of the prepared statement to run.
+   */
   const char *pname;
+  
+  /**
+   * Size of values pointed to by paramValues.
+   */
   int paramLengths[5];
+  
+  /**
+   * Number of paramters in paramValues/paramLengths.
+   */
   int nparams; 
+  
+  /**
+   * Current time (possible parameter), big-endian.
+   */
   uint64_t bnow;
+  
+  /**
+   * Key (possible parameter)
+   */
   GNUNET_HashCode key;
+  
+  /**
+   * Hash of value (possible parameter)
+   */
   GNUNET_HashCode vhash;
+  
+  /**
+   * Number of entries found so far
+   */
   long long count;
+  
+  /**
+   * Offset this iteration starts at.
+   */
   uint64_t off;
+  
+  /**
+   * Current offset to use in query, big-endian.
+   */
   uint64_t blimit_off;
+  
+  /**
+   *  Overall number of matching entries.
+   */
   unsigned long long total;
+  
+  /**
+   * Expiration value of previous result (possible parameter), big-endian.
+   */
   uint64_t blast_expire;
+  
+  /**
+   * Row ID of last result (possible paramter), big-endian.
+   */
   uint32_t blast_rowid;
+  
+  /**
+   * Priority of last result (possible parameter), big-endian.
+   */
   uint32_t blast_prio;
+  
+  /**
+   * Type of block (possible paramter), big-endian.
+   */
   uint32_t btype;
+  
+  /**
+   * Flag set to GNUNET_YES to stop iteration.
+   */
   int end_it;
 };
 
@@ -131,10 +209,6 @@ struct Plugin
    */
   GNUNET_SCHEDULER_TaskIdentifier next_task;
 
-  unsigned long long payload;
-
-  unsigned int lastSync;
-  
 };
 
 
@@ -143,6 +217,12 @@ struct Plugin
  * the desired status code.  If not, log an error, clear the
  * result and return GNUNET_SYSERR.
  * 
+ * @param plugin global context
+ * @param ret result to check
+ * @param expected_status expected return value
+ * @param command name of SQL command that was run
+ * @param args arguments to SQL command
+ * @param line line number for error reporting
  * @return GNUNET_OK if the result is acceptable
  */
 static int
@@ -173,6 +253,10 @@ check_result (struct Plugin *plugin,
 
 /**
  * Run simple SQL statement (without results).
+ *
+ * @param plugin global context
+ * @param sql statement to run
+ * @param line code line for error reporting
  */
 static int
 pq_exec (struct Plugin *plugin,
@@ -190,6 +274,12 @@ pq_exec (struct Plugin *plugin,
 
 /**
  * Prepare SQL statement.
+ *
+ * @param plugin global context
+ * @param sql SQL code to prepare
+ * @param nparams number of parameters in sql
+ * @param line code line for error reporting
+ * @return GNUNET_OK on success
  */
 static int
 pq_prepare (struct Plugin *plugin,
@@ -207,6 +297,8 @@ pq_prepare (struct Plugin *plugin,
 
 /**
  * @brief Get a database handle
+ *
+ * @param plugin global context
  * @return GNUNET_OK on success, GNUNET_SYSERR on error
  */
 static int
@@ -413,6 +505,8 @@ init_connection (struct Plugin *plugin)
  * Delete the row identified by the given rowid (qid
  * in postgres).
  *
+ * @param plugin global context
+ * @param rowid which row to delete
  * @return GNUNET_OK on success
  */
 static int
@@ -450,11 +544,32 @@ static unsigned long long
 postgres_plugin_get_size (void *cls)
 {
   struct Plugin *plugin = cls;
-  double ret;
+  unsigned long long total;
+  PGresult *ret;
 
-  ret = plugin->payload;
-  return (unsigned long long) (ret * 1.00);
-  /* benchmarking shows XX% overhead */
+  ret = PQexecParams (plugin->dbh,
+                      "SELECT SUM(LENGTH(value))+256*COUNT(*) FROM gn090",
+                      0, NULL, NULL, NULL, NULL, 1);
+  if (GNUNET_OK != check_result (plugin,
+                                 ret,
+                                 PGRES_TUPLES_OK,
+                                 "PQexecParams",
+                                 "get_size",
+                                 __LINE__))
+    {
+      return 0;
+    }
+  if ((PQntuples (ret) != 1) ||
+      (PQnfields (ret) != 1) ||
+      (PQgetlength (ret, 0, 0) != sizeof (unsigned long long)))
+    {
+      GNUNET_break (0);
+      PQclear (ret);
+      return 0;
+    }
+  total = GNUNET_ntohll (*(const unsigned long long *) PQgetvalue (ret, 0, 0));
+  PQclear (ret);
+  return total;
 }
 
 
@@ -518,13 +633,12 @@ postgres_plugin_put (void *cls,
                                  "PQexecPrepared", "put", __LINE__))
     return GNUNET_SYSERR;
   PQclear (ret);
-  plugin->payload += size;
+  plugin->env->duc (plugin->env->cls, size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
 #if DEBUG_POSTGRES
   GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                    "datastore-postgres",
-                   "Stored %u bytes in database, new payload is %llu\n",
-                   (unsigned int) size,
-                   (unsigned long long) plugin->payload);
+                   "Stored %u bytes in database\n",
+                   (unsigned int) size);
 #endif
   return GNUNET_OK;
 }
@@ -695,18 +809,16 @@ postgres_next_request_cont (void *next_cls,
 #if DEBUG_POSTGRES
           GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                            "datastore-postgres",
-                           "Deleting %u bytes from database, current payload is %llu\n",
-                           (unsigned int) size,
-                           (unsigned long long) plugin->payload);
+                           "Deleting %u bytes from database\n",
+                           (unsigned int) size);
 #endif
-          GNUNET_assert (plugin->payload >= size);
-          plugin->payload -= size;
+          plugin->env->duc (plugin->env->cls,
+                            - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
 #if DEBUG_POSTGRES
           GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                            "datastore-postgres",
-                           "Deleted %u bytes from database, new payload is %llu\n",
-                           (unsigned int) size,
-                           (unsigned long long) plugin->payload);
+                           "Deleted %u bytes from database\n",
+                           (unsigned int) size);
 #endif
         }
     }
@@ -803,11 +915,15 @@ postgres_plugin_update (void *cls,
  * Call a method for each key in the database and
  * call the callback method on it.
  *
+ * @param plugin global context
  * @param type entries of which type should be considered?
+ * @param is_asc ascending or descending iteration?
+ * @param iter_select which SELECT method should be used?
  * @param iter maybe NULL (to just count); iter
  *     should return GNUNET_SYSERR to abort the
  *     iteration, GNUNET_NO to delete the entry and
  *     continue and GNUNET_OK to continue iterating
+ * @param iter_cls closure for 'iter'
  */
 static void
 postgres_iterate (struct Plugin *plugin,
@@ -1123,7 +1239,6 @@ postgres_plugin_iter_ascending_expiration (void *cls,
 }
 
 
-
 /**
  * Select a subset of the items in the datastore and call
  * the given iterator for each of them.
@@ -1148,7 +1263,6 @@ postgres_plugin_iter_migration_order (void *cls,
 }
 
 
-
 /**
  * Select a subset of the items in the datastore and call
  * the given iterator for each of them.
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index ca1f4e4ae..076d468ee 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -25,19 +25,11 @@
  */
 
 #include "platform.h"
-#include "gnunet_statistics_service.h"
 #include "plugin_datastore.h"
 #include <sqlite3.h>
 
 #define DEBUG_SQLITE GNUNET_NO
 
-/**
- * After how many payload-changing operations
- * do we sync our statistics?
- */
-#define MAX_STAT_SYNC_LAG 50
-
-#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
 
 /**
  * Log an error message at log-level 'level' that indicates
@@ -123,16 +115,6 @@ struct Plugin
   sqlite3_stmt *insertContent;
 
   /**
-   * Handle to the statistics service.
-   */
-  struct GNUNET_STATISTICS_Handle *statistics;
-
-  /**
-   * Handle for pending get request.
-   */
-  struct GNUNET_STATISTICS_GetHandle *stat_get;
-
-  /**
    * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
    */
   struct NextContext *next_task_nc;
@@ -141,29 +123,12 @@ struct Plugin
    * Pending task with scheduler for running the next request.
    */
   GNUNET_SCHEDULER_TaskIdentifier next_task;
-  
-  /**
-   * How much data are we currently storing
-   * in the database?
-   */
-  unsigned long long payload;
-
-  /**
-   * Number of updates that were made to the
-   * payload value since we last synchronized
-   * it with the statistics service.
-   */
-  unsigned int lastSync;
 
   /**
    * Should the database be dropped on shutdown?
    */
   int drop_on_shutdown;
 
-  /**
-   * Did we get an answer from statistics?
-   */
-  int stats_worked;
 };
 
 
@@ -267,12 +232,7 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
           return GNUNET_SYSERR;
         }
       /* database is new or got deleted, reset payload to zero! */
-      if (plugin->stat_get != NULL)
-        {
-          GNUNET_STATISTICS_get_cancel (plugin->stat_get);
-          plugin->stat_get = NULL;
-        }
-      plugin->payload = 0;
+      plugin->env->duc (plugin->env->cls, 0);
     }
   plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
 #ifdef ENABLE_NLS
@@ -375,22 +335,6 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
 
 /**
- * Synchronize our utilization statistics with the 
- * statistics service.
- * @param plugin the plugin context (state for this module)
- */
-static void 
-sync_stats (struct Plugin *plugin)
-{
-  GNUNET_STATISTICS_set (plugin->statistics,
-                         QUOTA_STAT_NAME,
-                         plugin->payload,
-                         GNUNET_YES);
-  plugin->lastSync = 0;
-}
-
-
-/**
  * Shutdown database connection and associate data
  * structures.
  * @param plugin the plugin context (state for this module)
@@ -398,8 +342,6 @@ sync_stats (struct Plugin *plugin)
 static void
 database_shutdown (struct Plugin *plugin)
 {
-  if (plugin->lastSync > 0)
-    sync_stats (plugin);
   if (plugin->updPrio != NULL)
     sqlite3_finalize (plugin->updPrio);
   if (plugin->insertContent != NULL)
@@ -410,20 +352,6 @@ database_shutdown (struct Plugin *plugin)
 
 
 /**
- * Get an estimate of how much space the database is
- * currently using.
- *
- * @param cls our plugin context
- * @return number of bytes used on disk
- */
-static unsigned long long sqlite_plugin_get_size (void *cls)
-{
-  struct Plugin *plugin = cls;
-  return plugin->payload;
-}
-
-
-/**
  * Delete the database entry with the given
  * row identifier.
  *
@@ -661,23 +589,15 @@ sqlite_next_request_cont (void *cls,
   if ( (ret == GNUNET_NO) &&
        (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
     {
-      if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD)
-        plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
-      else
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    _("Datastore payload inaccurate, please fix and restart!\n"));
-      plugin->lastSync++; 
+      plugin->env->duc (plugin->env->cls,
+                        - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
 #if DEBUG_SQLITE
-      if (ret == GNUNET_NO)
-        GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
-                         "sqlite",
-                         "Removed entry %llu (%u bytes), new payload is %llu\n",
-                         (unsigned long long) rowid,
-                         size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
-                         plugin->payload);
+      GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
+                       "sqlite",
+                       "Removed entry %llu (%u bytes)\n",
+                       (unsigned long long) rowid,
+                       size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
 #endif
-      if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
-        sync_stats (plugin);
     }
 }
 
@@ -798,17 +718,14 @@ sqlite_plugin_put (void *cls,
     LOG_SQLITE (plugin, NULL,
                 GNUNET_ERROR_TYPE_ERROR |
                 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
-  plugin->lastSync++;
-  plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
+  plugin->env->duc (plugin->env->cls,
+                    size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
 #if DEBUG_SQLITE
   GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
                    "sqlite",
-                   "Stored new entry (%u bytes), new payload is %llu\n",
-                   size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
-                   plugin->payload);
+                   "Stored new entry (%u bytes)\n",
+           size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
 #endif
-  if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
-    sync_stats (plugin);
   return GNUNET_OK;
 }
 
@@ -1574,81 +1491,50 @@ sqlite_plugin_drop (void *cls)
 }
 
 
-/**
- * Callback function to process statistic values.
- *
- * @param cls closure
- * @param subsystem name of subsystem that created the statistic
- * @param name the name of the datum
- * @param value the current value
- * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
- * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
- */
-static int
-process_stat_in (void *cls,
-                 const char *subsystem,
-                 const char *name,
-                 uint64_t value,
-                 int is_persistent)
-{
-  struct Plugin *plugin = cls;
-
-  plugin->stats_worked = GNUNET_YES;
-  plugin->payload += value;
-#if DEBUG_SQLITE
-  GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
-                   "sqlite",
-                   "Notification from statistics about existing payload (%llu), new payload is %llu\n",
-                   value,
-                   plugin->payload);
-#endif
-  return GNUNET_OK;
-}
-
-
-static void
-process_stat_done (void *cls,
-                   int success)
+static unsigned long long
+sqlite_plugin_get_size (void *cls)
 {
   struct Plugin *plugin = cls;
   sqlite3_stmt *stmt;
   uint64_t pages;
   uint64_t page_size;
 
-  plugin->stat_get = NULL;
-  if ( (plugin->stats_worked == GNUNET_NO) &&
-       (SQLITE_VERSION_NUMBER >= 3006000) )
-   {
-      CHECK (SQLITE_OK ==
-             sqlite3_exec (plugin->dbh,
-                           "VACUUM", NULL, NULL, ENULL));
-      CHECK (SQLITE_OK ==
-             sqlite3_exec (plugin->dbh,
-                           "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
-      CHECK (SQLITE_OK ==
-             sq_prepare (plugin->dbh,
-                         "PRAGMA page_count",
-                         &stmt));
-      if (SQLITE_ROW ==
-          sqlite3_step (stmt))
-        pages = sqlite3_column_int64 (stmt, 0);
-      else
-        pages = 0;
-      sqlite3_finalize (stmt);
-      CHECK (SQLITE_OK ==
-             sq_prepare (plugin->dbh,
-                         "PRAGMA page_size",
-                         &stmt));
-      CHECK (SQLITE_ROW ==
-             sqlite3_step (stmt));
-      page_size = sqlite3_column_int64 (stmt, 0);
-      sqlite3_finalize (stmt);
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
-                  (unsigned long long) pages,
-                  (unsigned long long) page_size);
-      plugin->payload = pages * page_size;
+  if (SQLITE_VERSION_NUMBER < 3006000)
+    {
+      GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
+                       "datastore-sqlite",
+                       _("sqlite version to old to determine size, assuming zero\n"));
+      return 0;
     }
+  CHECK (SQLITE_OK ==
+         sqlite3_exec (plugin->dbh,
+                       "VACUUM", NULL, NULL, ENULL));
+  CHECK (SQLITE_OK ==
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_count",
+                     &stmt));
+  if (SQLITE_ROW ==
+      sqlite3_step (stmt))
+    pages = sqlite3_column_int64 (stmt, 0);
+  else
+    pages = 0;
+  sqlite3_finalize (stmt);
+  CHECK (SQLITE_OK ==
+         sq_prepare (plugin->dbh,
+                     "PRAGMA page_size",
+                     &stmt));
+  CHECK (SQLITE_ROW ==
+         sqlite3_step (stmt));
+  page_size = sqlite3_column_int64 (stmt, 0);
+  sqlite3_finalize (stmt);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
+              (unsigned long long) pages,
+              (unsigned long long) page_size);
+  return  pages * page_size;
 }
                                          
 
@@ -1669,16 +1555,6 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
     return NULL; /* can only initialize once! */
   memset (&plugin, 0, sizeof(struct Plugin));
   plugin.env = env;
-  plugin.statistics = GNUNET_STATISTICS_create (env->sched,
-                                                "ds-sqlite",
-                                                env->cfg);
-  plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics,
-                                           "ds-sqlite",
-                                           QUOTA_STAT_NAME,
-                                           GNUNET_TIME_UNIT_SECONDS,
-                                           &process_stat_done,
-                                           &process_stat_in,
-                                           &plugin);
   if (GNUNET_OK !=
       database_setup (env->cfg, &plugin))
     {
@@ -1717,11 +1593,6 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
   struct GNUNET_DATASTORE_PluginFunctions *api = cls;
   struct Plugin *plugin = api->cls;
 
-  if (plugin->stat_get != NULL)
-    {
-      GNUNET_STATISTICS_get_cancel (plugin->stat_get);
-      plugin->stat_get = NULL;
-    }
   if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
     {
       GNUNET_SCHEDULER_cancel (plugin->env->sched,
@@ -1735,10 +1606,7 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
   if (plugin->drop_on_shutdown)
     fn = GNUNET_strdup (plugin->fn);
   database_shutdown (plugin);
-  GNUNET_STATISTICS_destroy (plugin->statistics,
-                             GNUNET_NO);
   plugin->env = NULL; 
-  plugin->payload = 0;
   GNUNET_free (api);
   if (fn != NULL)
     {