diff options
Diffstat (limited to 'src/datastore/plugin_datastore_sqlite.c')
-rw-r--r-- | src/datastore/plugin_datastore_sqlite.c | 230 |
1 files changed, 49 insertions, 181 deletions
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index ca1f4e4ae..076d468ee 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c | |||
@@ -25,19 +25,11 @@ | |||
25 | */ | 25 | */ |
26 | 26 | ||
27 | #include "platform.h" | 27 | #include "platform.h" |
28 | #include "gnunet_statistics_service.h" | ||
29 | #include "plugin_datastore.h" | 28 | #include "plugin_datastore.h" |
30 | #include <sqlite3.h> | 29 | #include <sqlite3.h> |
31 | 30 | ||
32 | #define DEBUG_SQLITE GNUNET_NO | 31 | #define DEBUG_SQLITE GNUNET_NO |
33 | 32 | ||
34 | /** | ||
35 | * After how many payload-changing operations | ||
36 | * do we sync our statistics? | ||
37 | */ | ||
38 | #define MAX_STAT_SYNC_LAG 50 | ||
39 | |||
40 | #define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore") | ||
41 | 33 | ||
42 | /** | 34 | /** |
43 | * Log an error message at log-level 'level' that indicates | 35 | * Log an error message at log-level 'level' that indicates |
@@ -123,16 +115,6 @@ struct Plugin | |||
123 | sqlite3_stmt *insertContent; | 115 | sqlite3_stmt *insertContent; |
124 | 116 | ||
125 | /** | 117 | /** |
126 | * Handle to the statistics service. | ||
127 | */ | ||
128 | struct GNUNET_STATISTICS_Handle *statistics; | ||
129 | |||
130 | /** | ||
131 | * Handle for pending get request. | ||
132 | */ | ||
133 | struct GNUNET_STATISTICS_GetHandle *stat_get; | ||
134 | |||
135 | /** | ||
136 | * Closure of the 'next_task' (must be freed if 'next_task' is cancelled). | 118 | * Closure of the 'next_task' (must be freed if 'next_task' is cancelled). |
137 | */ | 119 | */ |
138 | struct NextContext *next_task_nc; | 120 | struct NextContext *next_task_nc; |
@@ -141,29 +123,12 @@ struct Plugin | |||
141 | * Pending task with scheduler for running the next request. | 123 | * Pending task with scheduler for running the next request. |
142 | */ | 124 | */ |
143 | GNUNET_SCHEDULER_TaskIdentifier next_task; | 125 | GNUNET_SCHEDULER_TaskIdentifier next_task; |
144 | |||
145 | /** | ||
146 | * How much data are we currently storing | ||
147 | * in the database? | ||
148 | */ | ||
149 | unsigned long long payload; | ||
150 | |||
151 | /** | ||
152 | * Number of updates that were made to the | ||
153 | * payload value since we last synchronized | ||
154 | * it with the statistics service. | ||
155 | */ | ||
156 | unsigned int lastSync; | ||
157 | 126 | ||
158 | /** | 127 | /** |
159 | * Should the database be dropped on shutdown? | 128 | * Should the database be dropped on shutdown? |
160 | */ | 129 | */ |
161 | int drop_on_shutdown; | 130 | int drop_on_shutdown; |
162 | 131 | ||
163 | /** | ||
164 | * Did we get an answer from statistics? | ||
165 | */ | ||
166 | int stats_worked; | ||
167 | }; | 132 | }; |
168 | 133 | ||
169 | 134 | ||
@@ -267,12 +232,7 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
267 | return GNUNET_SYSERR; | 232 | return GNUNET_SYSERR; |
268 | } | 233 | } |
269 | /* database is new or got deleted, reset payload to zero! */ | 234 | /* database is new or got deleted, reset payload to zero! */ |
270 | if (plugin->stat_get != NULL) | 235 | plugin->env->duc (plugin->env->cls, 0); |
271 | { | ||
272 | GNUNET_STATISTICS_get_cancel (plugin->stat_get); | ||
273 | plugin->stat_get = NULL; | ||
274 | } | ||
275 | plugin->payload = 0; | ||
276 | } | 236 | } |
277 | plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), | 237 | plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), |
278 | #ifdef ENABLE_NLS | 238 | #ifdef ENABLE_NLS |
@@ -375,22 +335,6 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, | |||
375 | 335 | ||
376 | 336 | ||
377 | /** | 337 | /** |
378 | * Synchronize our utilization statistics with the | ||
379 | * statistics service. | ||
380 | * @param plugin the plugin context (state for this module) | ||
381 | */ | ||
382 | static void | ||
383 | sync_stats (struct Plugin *plugin) | ||
384 | { | ||
385 | GNUNET_STATISTICS_set (plugin->statistics, | ||
386 | QUOTA_STAT_NAME, | ||
387 | plugin->payload, | ||
388 | GNUNET_YES); | ||
389 | plugin->lastSync = 0; | ||
390 | } | ||
391 | |||
392 | |||
393 | /** | ||
394 | * Shutdown database connection and associate data | 338 | * Shutdown database connection and associate data |
395 | * structures. | 339 | * structures. |
396 | * @param plugin the plugin context (state for this module) | 340 | * @param plugin the plugin context (state for this module) |
@@ -398,8 +342,6 @@ sync_stats (struct Plugin *plugin) | |||
398 | static void | 342 | static void |
399 | database_shutdown (struct Plugin *plugin) | 343 | database_shutdown (struct Plugin *plugin) |
400 | { | 344 | { |
401 | if (plugin->lastSync > 0) | ||
402 | sync_stats (plugin); | ||
403 | if (plugin->updPrio != NULL) | 345 | if (plugin->updPrio != NULL) |
404 | sqlite3_finalize (plugin->updPrio); | 346 | sqlite3_finalize (plugin->updPrio); |
405 | if (plugin->insertContent != NULL) | 347 | if (plugin->insertContent != NULL) |
@@ -410,20 +352,6 @@ database_shutdown (struct Plugin *plugin) | |||
410 | 352 | ||
411 | 353 | ||
412 | /** | 354 | /** |
413 | * Get an estimate of how much space the database is | ||
414 | * currently using. | ||
415 | * | ||
416 | * @param cls our plugin context | ||
417 | * @return number of bytes used on disk | ||
418 | */ | ||
419 | static unsigned long long sqlite_plugin_get_size (void *cls) | ||
420 | { | ||
421 | struct Plugin *plugin = cls; | ||
422 | return plugin->payload; | ||
423 | } | ||
424 | |||
425 | |||
426 | /** | ||
427 | * Delete the database entry with the given | 355 | * Delete the database entry with the given |
428 | * row identifier. | 356 | * row identifier. |
429 | * | 357 | * |
@@ -661,23 +589,15 @@ sqlite_next_request_cont (void *cls, | |||
661 | if ( (ret == GNUNET_NO) && | 589 | if ( (ret == GNUNET_NO) && |
662 | (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) | 590 | (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) |
663 | { | 591 | { |
664 | if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD) | 592 | plugin->env->duc (plugin->env->cls, |
665 | plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); | 593 | - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); |
666 | else | ||
667 | GNUNET_log (GNUNET_ERROR_TYPE_WARNING, | ||
668 | _("Datastore payload inaccurate, please fix and restart!\n")); | ||
669 | plugin->lastSync++; | ||
670 | #if DEBUG_SQLITE | 594 | #if DEBUG_SQLITE |
671 | if (ret == GNUNET_NO) | 595 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, |
672 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | 596 | "sqlite", |
673 | "sqlite", | 597 | "Removed entry %llu (%u bytes)\n", |
674 | "Removed entry %llu (%u bytes), new payload is %llu\n", | 598 | (unsigned long long) rowid, |
675 | (unsigned long long) rowid, | 599 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); |
676 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, | ||
677 | plugin->payload); | ||
678 | #endif | 600 | #endif |
679 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) | ||
680 | sync_stats (plugin); | ||
681 | } | 601 | } |
682 | } | 602 | } |
683 | 603 | ||
@@ -798,17 +718,14 @@ sqlite_plugin_put (void *cls, | |||
798 | LOG_SQLITE (plugin, NULL, | 718 | LOG_SQLITE (plugin, NULL, |
799 | GNUNET_ERROR_TYPE_ERROR | | 719 | GNUNET_ERROR_TYPE_ERROR | |
800 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); | 720 | GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); |
801 | plugin->lastSync++; | 721 | plugin->env->duc (plugin->env->cls, |
802 | plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD; | 722 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); |
803 | #if DEBUG_SQLITE | 723 | #if DEBUG_SQLITE |
804 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | 724 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, |
805 | "sqlite", | 725 | "sqlite", |
806 | "Stored new entry (%u bytes), new payload is %llu\n", | 726 | "Stored new entry (%u bytes)\n", |
807 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD, | 727 | size + GNUNET_DATASTORE_ENTRY_OVERHEAD); |
808 | plugin->payload); | ||
809 | #endif | 728 | #endif |
810 | if (plugin->lastSync >= MAX_STAT_SYNC_LAG) | ||
811 | sync_stats (plugin); | ||
812 | return GNUNET_OK; | 729 | return GNUNET_OK; |
813 | } | 730 | } |
814 | 731 | ||
@@ -1574,81 +1491,50 @@ sqlite_plugin_drop (void *cls) | |||
1574 | } | 1491 | } |
1575 | 1492 | ||
1576 | 1493 | ||
1577 | /** | 1494 | static unsigned long long |
1578 | * Callback function to process statistic values. | 1495 | sqlite_plugin_get_size (void *cls) |
1579 | * | ||
1580 | * @param cls closure | ||
1581 | * @param subsystem name of subsystem that created the statistic | ||
1582 | * @param name the name of the datum | ||
1583 | * @param value the current value | ||
1584 | * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not | ||
1585 | * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration | ||
1586 | */ | ||
1587 | static int | ||
1588 | process_stat_in (void *cls, | ||
1589 | const char *subsystem, | ||
1590 | const char *name, | ||
1591 | uint64_t value, | ||
1592 | int is_persistent) | ||
1593 | { | ||
1594 | struct Plugin *plugin = cls; | ||
1595 | |||
1596 | plugin->stats_worked = GNUNET_YES; | ||
1597 | plugin->payload += value; | ||
1598 | #if DEBUG_SQLITE | ||
1599 | GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, | ||
1600 | "sqlite", | ||
1601 | "Notification from statistics about existing payload (%llu), new payload is %llu\n", | ||
1602 | value, | ||
1603 | plugin->payload); | ||
1604 | #endif | ||
1605 | return GNUNET_OK; | ||
1606 | } | ||
1607 | |||
1608 | |||
1609 | static void | ||
1610 | process_stat_done (void *cls, | ||
1611 | int success) | ||
1612 | { | 1496 | { |
1613 | struct Plugin *plugin = cls; | 1497 | struct Plugin *plugin = cls; |
1614 | sqlite3_stmt *stmt; | 1498 | sqlite3_stmt *stmt; |
1615 | uint64_t pages; | 1499 | uint64_t pages; |
1616 | uint64_t page_size; | 1500 | uint64_t page_size; |
1617 | 1501 | ||
1618 | plugin->stat_get = NULL; | 1502 | if (SQLITE_VERSION_NUMBER < 3006000) |
1619 | if ( (plugin->stats_worked == GNUNET_NO) && | 1503 | { |
1620 | (SQLITE_VERSION_NUMBER >= 3006000) ) | 1504 | GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING, |
1621 | { | 1505 | "datastore-sqlite", |
1622 | CHECK (SQLITE_OK == | 1506 | _("sqlite version to old to determine size, assuming zero\n")); |
1623 | sqlite3_exec (plugin->dbh, | 1507 | return 0; |
1624 | "VACUUM", NULL, NULL, ENULL)); | ||
1625 | CHECK (SQLITE_OK == | ||
1626 | sqlite3_exec (plugin->dbh, | ||
1627 | "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL)); | ||
1628 | CHECK (SQLITE_OK == | ||
1629 | sq_prepare (plugin->dbh, | ||
1630 | "PRAGMA page_count", | ||
1631 | &stmt)); | ||
1632 | if (SQLITE_ROW == | ||
1633 | sqlite3_step (stmt)) | ||
1634 | pages = sqlite3_column_int64 (stmt, 0); | ||
1635 | else | ||
1636 | pages = 0; | ||
1637 | sqlite3_finalize (stmt); | ||
1638 | CHECK (SQLITE_OK == | ||
1639 | sq_prepare (plugin->dbh, | ||
1640 | "PRAGMA page_size", | ||
1641 | &stmt)); | ||
1642 | CHECK (SQLITE_ROW == | ||
1643 | sqlite3_step (stmt)); | ||
1644 | page_size = sqlite3_column_int64 (stmt, 0); | ||
1645 | sqlite3_finalize (stmt); | ||
1646 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1647 | _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"), | ||
1648 | (unsigned long long) pages, | ||
1649 | (unsigned long long) page_size); | ||
1650 | plugin->payload = pages * page_size; | ||
1651 | } | 1508 | } |
1509 | CHECK (SQLITE_OK == | ||
1510 | sqlite3_exec (plugin->dbh, | ||
1511 | "VACUUM", NULL, NULL, ENULL)); | ||
1512 | CHECK (SQLITE_OK == | ||
1513 | sqlite3_exec (plugin->dbh, | ||
1514 | "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL)); | ||
1515 | CHECK (SQLITE_OK == | ||
1516 | sq_prepare (plugin->dbh, | ||
1517 | "PRAGMA page_count", | ||
1518 | &stmt)); | ||
1519 | if (SQLITE_ROW == | ||
1520 | sqlite3_step (stmt)) | ||
1521 | pages = sqlite3_column_int64 (stmt, 0); | ||
1522 | else | ||
1523 | pages = 0; | ||
1524 | sqlite3_finalize (stmt); | ||
1525 | CHECK (SQLITE_OK == | ||
1526 | sq_prepare (plugin->dbh, | ||
1527 | "PRAGMA page_size", | ||
1528 | &stmt)); | ||
1529 | CHECK (SQLITE_ROW == | ||
1530 | sqlite3_step (stmt)); | ||
1531 | page_size = sqlite3_column_int64 (stmt, 0); | ||
1532 | sqlite3_finalize (stmt); | ||
1533 | GNUNET_log (GNUNET_ERROR_TYPE_INFO, | ||
1534 | _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"), | ||
1535 | (unsigned long long) pages, | ||
1536 | (unsigned long long) page_size); | ||
1537 | return pages * page_size; | ||
1652 | } | 1538 | } |
1653 | 1539 | ||
1654 | 1540 | ||
@@ -1669,16 +1555,6 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) | |||
1669 | return NULL; /* can only initialize once! */ | 1555 | return NULL; /* can only initialize once! */ |
1670 | memset (&plugin, 0, sizeof(struct Plugin)); | 1556 | memset (&plugin, 0, sizeof(struct Plugin)); |
1671 | plugin.env = env; | 1557 | plugin.env = env; |
1672 | plugin.statistics = GNUNET_STATISTICS_create (env->sched, | ||
1673 | "ds-sqlite", | ||
1674 | env->cfg); | ||
1675 | plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics, | ||
1676 | "ds-sqlite", | ||
1677 | QUOTA_STAT_NAME, | ||
1678 | GNUNET_TIME_UNIT_SECONDS, | ||
1679 | &process_stat_done, | ||
1680 | &process_stat_in, | ||
1681 | &plugin); | ||
1682 | if (GNUNET_OK != | 1558 | if (GNUNET_OK != |
1683 | database_setup (env->cfg, &plugin)) | 1559 | database_setup (env->cfg, &plugin)) |
1684 | { | 1560 | { |
@@ -1717,11 +1593,6 @@ libgnunet_plugin_datastore_sqlite_done (void *cls) | |||
1717 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; | 1593 | struct GNUNET_DATASTORE_PluginFunctions *api = cls; |
1718 | struct Plugin *plugin = api->cls; | 1594 | struct Plugin *plugin = api->cls; |
1719 | 1595 | ||
1720 | if (plugin->stat_get != NULL) | ||
1721 | { | ||
1722 | GNUNET_STATISTICS_get_cancel (plugin->stat_get); | ||
1723 | plugin->stat_get = NULL; | ||
1724 | } | ||
1725 | if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK) | 1596 | if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK) |
1726 | { | 1597 | { |
1727 | GNUNET_SCHEDULER_cancel (plugin->env->sched, | 1598 | GNUNET_SCHEDULER_cancel (plugin->env->sched, |
@@ -1735,10 +1606,7 @@ libgnunet_plugin_datastore_sqlite_done (void *cls) | |||
1735 | if (plugin->drop_on_shutdown) | 1606 | if (plugin->drop_on_shutdown) |
1736 | fn = GNUNET_strdup (plugin->fn); | 1607 | fn = GNUNET_strdup (plugin->fn); |
1737 | database_shutdown (plugin); | 1608 | database_shutdown (plugin); |
1738 | GNUNET_STATISTICS_destroy (plugin->statistics, | ||
1739 | GNUNET_NO); | ||
1740 | plugin->env = NULL; | 1609 | plugin->env = NULL; |
1741 | plugin->payload = 0; | ||
1742 | GNUNET_free (api); | 1610 | GNUNET_free (api); |
1743 | if (fn != NULL) | 1611 | if (fn != NULL) |
1744 | { | 1612 | { |