From 1d5caebcb5ec2617f92550cf1789037313d5c9e0 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 3 Apr 2010 10:38:02 +0000 Subject: fix datastore quota management -- especially if stats is not working --- src/datastore/gnunet-service-datastore.c | 2 +- src/datastore/plugin_datastore_sqlite.c | 60 ++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 8 deletions(-) (limited to 'src/datastore') diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index c3370e0a5..9b23f6105 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c @@ -297,7 +297,6 @@ expired_processor (void *cls, { struct GNUNET_TIME_Absolute now; - expired_kill_task = GNUNET_SCHEDULER_NO_TASK; if (key == NULL) { expired_kill_task @@ -343,6 +342,7 @@ static void delete_expired (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { + expired_kill_task = GNUNET_SCHEDULER_NO_TASK; plugin->api->iter_ascending_expiration (plugin->api->cls, 0, &expired_processor, diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index 1d38f8326..cecdf094f 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c @@ -29,7 +29,7 @@ #include "plugin_datastore.h" #include -#define DEBUG_SQLITE GNUNET_NO +#define DEBUG_SQLITE GNUNET_YES /** * After how many payload-changing operations @@ -159,6 +159,11 @@ struct Plugin * Should the database be dropped on shutdown? */ int drop_on_shutdown; + + /** + * Did we get an answer from statistics? + */ + int stats_worked; }; @@ -283,11 +288,15 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL)); + CHECK (SQLITE_OK == + sqlite3_exec (plugin->dbh, + "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL)); CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "PRAGMA count_changes=OFF", NULL, NULL, ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, ENULL)); + sqlite3_exec (plugin->dbh, + "PRAGMA page_size=4092", NULL, NULL, ENULL)); CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); @@ -642,7 +651,11 @@ sqlite_next_request_cont (void *cls, if ( (ret == GNUNET_NO) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) { - plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); + if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD) + plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); + else + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Datastore payload inaccurate, please fix and restart!\n")); plugin->lastSync++; #if DEBUG_SQLITE if (ret == GNUNET_NO) @@ -651,7 +664,7 @@ sqlite_next_request_cont (void *cls, "Removed entry %llu (%u bytes), new payload is %llu\n", (unsigned long long) rowid, size + GNUNET_DATASTORE_ENTRY_OVERHEAD, - (unsigned long long) plugin->payload); + plugin->payload); #endif if (plugin->lastSync >= MAX_STAT_SYNC_LAG) sync_stats (plugin); @@ -779,7 +792,7 @@ sqlite_plugin_put (void *cls, "sqlite", "Stored new entry (%u bytes), new payload is %llu\n", size + GNUNET_DATASTORE_ENTRY_OVERHEAD, - (unsigned long long) plugin->payload); + plugin->payload); #endif if (plugin->lastSync >= MAX_STAT_SYNC_LAG) sync_stats (plugin); @@ -1566,13 +1579,15 @@ process_stat_in (void *cls, int is_persistent) { struct Plugin *plugin = cls; + + plugin->stats_worked = GNUNET_YES; plugin->payload += value; #if DEBUG_SQLITE GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, "sqlite", "Notification from statistics about existing payload (%llu), new payload is %llu\n", value, - (unsigned long long) plugin->payload); + plugin->payload); #endif return GNUNET_OK; } @@ -1583,7 +1598,35 @@ process_stat_done (void *cls, int success) { struct Plugin *plugin = cls; + sqlite3_stmt *stmt; + uint64_t pages; + uint64_t free_pages; + plugin->stat_get = NULL; + if (plugin->stats_worked == GNUNET_NO) + { + CHECK (SQLITE_OK == + sq_prepare (plugin->dbh, + "PRAGMA page_count", + &stmt)); + CHECK (SQLITE_ROW == + sqlite3_step (stmt)); + pages = sqlite3_column_int64 (stmt, 0); + sqlite3_finalize (stmt); + CHECK (SQLITE_OK == + sq_prepare (plugin->dbh, + "PRAGMA freelist_count", + &stmt)); + CHECK (SQLITE_ROW == + sqlite3_step (stmt)); + free_pages = sqlite3_column_int64 (stmt, 0); + sqlite3_finalize (stmt); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Using sqlite page utilization to estimate payload (%llu pages total, %llu free)\n"), + (unsigned long long) pages, + (unsigned long long) free_pages); + plugin->payload = (pages - free_pages) * 4092LL; + } } @@ -1610,7 +1653,7 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics, "datastore-sqlite", QUOTA_STAT_NAME, - GNUNET_TIME_UNIT_MINUTES, + GNUNET_TIME_UNIT_SECONDS, &process_stat_done, &process_stat_in, &plugin); @@ -1635,6 +1678,9 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) api->drop = &sqlite_plugin_drop; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite", _("Sqlite database running\n")); + fprintf (stderr, + "Initial payload: %llu\n", + plugin.payload); return api; } -- cgit v1.2.3