diff options
author | Christian Grothoff <christian@grothoff.org> | 2010-04-03 10:38:02 +0000 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2010-04-03 10:38:02 +0000 |
commit | 1d5caebcb5ec2617f92550cf1789037313d5c9e0 (patch) | |
tree | 518346ea8aaeee3a9c2f42bd171195bae483ea86 /src/datastore | |
parent | e940210d88f0aa021a8fc57abe20fe2ad98d0f74 (diff) | |
download | gnunet-1d5caebcb5ec2617f92550cf1789037313d5c9e0.tar.gz gnunet-1d5caebcb5ec2617f92550cf1789037313d5c9e0.zip |
fix datastore quota management -- especially if stats is not working
Diffstat (limited to 'src/datastore')
-rw-r--r-- | src/datastore/gnunet-service-datastore.c | 2 | ||||
-rw-r--r-- | src/datastore/plugin_datastore_sqlite.c | 60 |
2 files changed, 54 insertions, 8 deletions
diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index c3370e0a5..9b23f6105 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c | |||
@@ -297,7 +297,6 @@ expired_processor (void *cls, | |||
297 | { | 297 | { |
298 | struct GNUNET_TIME_Absolute now; | 298 | struct GNUNET_TIME_Absolute now; |
299 | 299 | ||
300 | expired_kill_task = GNUNET_SCHEDULER_NO_TASK; | ||
301 | if (key == NULL) | 300 | if (key == NULL) |
302 | { | 301 | { |
303 | expired_kill_task | 302 | expired_kill_task |
@@ -343,6 +342,7 @@ static void | |||
343 | delete_expired (void *cls, | 342 | delete_expired (void *cls, |
344 | const struct GNUNET_SCHEDULER_TaskContext *tc) | 343 | const struct GNUNET_SCHEDULER_TaskContext *tc) |
345 | { | 344 | { |
345 | expired_kill_task = GNUNET_SCHEDULER_NO_TASK; | ||
346 | plugin->api->iter_ascending_expiration (plugin->api->cls, | 346 | plugin->api->iter_ascending_expiration (plugin->api->cls, |
347 | 0, | 347 | 0, |
348 | &expired_processor, | 348 | &expired_processor, |
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index 1d38f8326..cecdf094f 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c | |||
@@ -29,7 +29,7 @@ | |||
29 | #include "plugin_datastore.h" | 29 | #include "plugin_datastore.h" |
30 | #include <sqlite3.h> | 30 | #include <sqlite3.h> |
31 | 31 | ||
32 | #define DEBUG_SQLITE GNUNET_NO | 32 | #define DEBUG_SQLITE GNUNET_YES |
33 | 33 | ||
34 | /** | 34 | /** |
35 | * After how many payload-changing operations | 35 | * After how many payload-changing operations |
@@ -159,6 +159,11 @@ struct Plugin | |||
159 | * Should the database be dropped on shutdown? | 159 | * Should the database be dropped on shutdown? |
160 | */ | 160 | */ |
161 | int drop_on_shutdown; | 161 | int drop_on_shutdown; |
162 | |||
163 | /** | ||
164 | * Did we get an answer from statistics? | ||
165 | */ | ||
166 | int stats_worked; | ||
162 | }; | 167 | }; |
163 | 168 | ||
164 | 169 | ||
@@ -285,9 +290,13 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
285 | "PRAGMA synchronous=OFF", NULL, NULL, ENULL)); | 290 | "PRAGMA synchronous=OFF", NULL, NULL, ENULL)); |
286 | CHECK (SQLITE_OK == | 291 | CHECK (SQLITE_OK == |
287 | sqlite3_exec (plugin->dbh, | 292 | sqlite3_exec (plugin->dbh, |
293 | "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL)); | ||
294 | CHECK (SQLITE_OK == | ||
295 | sqlite3_exec (plugin->dbh, | ||
288 | "PRAGMA count_changes=OFF", NULL, NULL, ENULL)); | 296 | "PRAGMA count_changes=OFF", NULL, NULL, ENULL)); |
289 | CHECK (SQLITE_OK == | 297 | CHECK (SQLITE_OK == |
290 | sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, ENULL)); | 298 | sqlite3_exec (plugin->dbh, |
299 | "PRAGMA page_size=4092", NULL, NULL, ENULL)); | ||
291 | 300 | ||
292 | CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); | 301 | CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); |
293 | 302 | ||
@@ -642,7 +651,11 @@ sqlite_next_request_cont (void *cls, | |||
642 | if ( (ret == GNUNET_NO) && | 651 | if ( (ret == GNUNET_NO) && |
643 | (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) | 652 | (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) |
644 | { | 653 | { |
645 | plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | 654 | if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD) |
655 | plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | ||
656 | else | ||
657 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
658 | _("Datastore payload inaccurate, please fix and restart!\n")); | ||
646 | plugin->lastSync++; | 659 | plugin->lastSync++; |
647 | #if DEBUG_SQLITE | 660 | #if DEBUG_SQLITE |
648 | if (ret == GNUNET_NO) | 661 | if (ret == GNUNET_NO) |
@@ -651,7 +664,7 @@ sqlite_next_request_cont (void *cls, | |||
651 | "Removed entry %llu (%u bytes), new payload is %llu\n", | 664 | "Removed entry %llu (%u bytes), new payload is %llu\n", |
652 | (unsigned long long) rowid, | 665 | (unsigned long long) rowid, |
653 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, | 666 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, |
654 | (unsigned long long) plugin->payload); | 667 | plugin->payload); |
655 | #endif | 668 | #endif |
656 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) | 669 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) |
657 | sync_stats (plugin); | 670 | sync_stats (plugin); |
@@ -779,7 +792,7 @@ sqlite_plugin_put (void *cls, | |||
779 | "sqlite", | 792 | "sqlite", |
780 | "Stored new entry (%u bytes), new payload is %llu\n", | 793 | "Stored new entry (%u bytes), new payload is %llu\n", |
781 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, | 794 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, |
782 | (unsigned long long) plugin->payload); | 795 | plugin->payload); |
783 | #endif | 796 | #endif |
784 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) | 797 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) |
785 | sync_stats (plugin); | 798 | sync_stats (plugin); |
@@ -1566,13 +1579,15 @@ process_stat_in (void *cls, | |||
1566 | int is_persistent) | 1579 | int is_persistent) |
1567 | { | 1580 | { |
1568 | struct Plugin *plugin = cls; | 1581 | struct Plugin *plugin = cls; |
1582 | |||
1583 | plugin->stats_worked = GNUNET_YES; | ||
1569 | plugin->payload += value; | 1584 | plugin->payload += value; |
1570 | #if DEBUG_SQLITE | 1585 | #if DEBUG_SQLITE |
1571 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | 1586 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, |
1572 | "sqlite", | 1587 | "sqlite", |
1573 | "Notification from statistics about existing payload (%llu), new payload is %llu\n", | 1588 | "Notification from statistics about existing payload (%llu), new payload is %llu\n", |
1574 | value, | 1589 | value, |
1575 | (unsigned long long) plugin->payload); | 1590 | plugin->payload); |
1576 | #endif | 1591 | #endif |
1577 | return GNUNET_OK; | 1592 | return GNUNET_OK; |
1578 | } | 1593 | } |
@@ -1583,7 +1598,35 @@ process_stat_done (void *cls, | |||
1583 | int success) | 1598 | int success) |
1584 | { | 1599 | { |
1585 | struct Plugin *plugin = cls; | 1600 | struct Plugin *plugin = cls; |
1601 | sqlite3_stmt *stmt; | ||
1602 | uint64_t pages; | ||
1603 | uint64_t free_pages; | ||
1604 | |||
1586 | plugin->stat_get = NULL; | 1605 | plugin->stat_get = NULL; |
1606 | if (plugin->stats_worked == GNUNET_NO) | ||
1607 | { | ||
1608 | CHECK (SQLITE_OK == | ||
1609 | sq_prepare (plugin->dbh, | ||
1610 | "PRAGMA page_count", | ||
1611 | &stmt)); | ||
1612 | CHECK (SQLITE_ROW == | ||
1613 | sqlite3_step (stmt)); | ||
1614 | pages = sqlite3_column_int64 (stmt, 0); | ||
1615 | sqlite3_finalize (stmt); | ||
1616 | CHECK (SQLITE_OK == | ||
1617 | sq_prepare (plugin->dbh, | ||
1618 | "PRAGMA freelist_count", | ||
1619 | &stmt)); | ||
1620 | CHECK (SQLITE_ROW == | ||
1621 | sqlite3_step (stmt)); | ||
1622 | free_pages = sqlite3_column_int64 (stmt, 0); | ||
1623 | sqlite3_finalize (stmt); | ||
1624 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1625 | _("Using sqlite page utilization to estimate payload (%llu pages total, %llu free)\n"), | ||
1626 | (unsigned long long) pages, | ||
1627 | (unsigned long long) free_pages); | ||
1628 | plugin->payload = (pages - free_pages) * 4092LL; | ||
1629 | } | ||
1587 | } | 1630 | } |
1588 | 1631 | ||
1589 | 1632 | ||
@@ -1610,7 +1653,7 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) | |||
1610 | plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics, | 1653 | plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics, |
1611 | "datastore-sqlite", | 1654 | "datastore-sqlite", |
1612 | QUOTA_STAT_NAME, | 1655 | QUOTA_STAT_NAME, |
1613 | GNUNET_TIME_UNIT_MINUTES, | 1656 | GNUNET_TIME_UNIT_SECONDS, |
1614 | &process_stat_done, | 1657 | &process_stat_done, |
1615 | &process_stat_in, | 1658 | &process_stat_in, |
1616 | &plugin); | 1659 | &plugin); |
@@ -1635,6 +1678,9 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) | |||
1635 | api->drop = &sqlite_plugin_drop; | 1678 | api->drop = &sqlite_plugin_drop; |
1636 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, | 1679 | GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, |
1637 | "sqlite", _("Sqlite database running\n")); | 1680 | "sqlite", _("Sqlite database running\n")); |
1681 | fprintf (stderr, | ||
1682 | "Initial payload: %llu\n", | ||
1683 | plugin.payload); | ||
1638 | return api; | 1684 | return api; |
1639 | } | 1685 | } |
1640 | 1686 | ||