aboutsummaryrefslogtreecommitdiff
path: root/src/datastore/plugin_datastore_sqlite.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/datastore/plugin_datastore_sqlite.c')
-rw-r--r--src/datastore/plugin_datastore_sqlite.c230
1 files changed, 49 insertions, 181 deletions
diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c
index ca1f4e4ae..076d468ee 100644
--- a/src/datastore/plugin_datastore_sqlite.c
+++ b/src/datastore/plugin_datastore_sqlite.c
@@ -25,19 +25,11 @@
25 */ 25 */
26 26
27#include "platform.h" 27#include "platform.h"
28#include "gnunet_statistics_service.h"
29#include "plugin_datastore.h" 28#include "plugin_datastore.h"
30#include <sqlite3.h> 29#include <sqlite3.h>
31 30
32#define DEBUG_SQLITE GNUNET_NO 31#define DEBUG_SQLITE GNUNET_NO
33 32
34/**
35 * After how many payload-changing operations
36 * do we sync our statistics?
37 */
38#define MAX_STAT_SYNC_LAG 50
39
40#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
41 33
42/** 34/**
43 * Log an error message at log-level 'level' that indicates 35 * Log an error message at log-level 'level' that indicates
@@ -123,16 +115,6 @@ struct Plugin
123 sqlite3_stmt *insertContent; 115 sqlite3_stmt *insertContent;
124 116
125 /** 117 /**
126 * Handle to the statistics service.
127 */
128 struct GNUNET_STATISTICS_Handle *statistics;
129
130 /**
131 * Handle for pending get request.
132 */
133 struct GNUNET_STATISTICS_GetHandle *stat_get;
134
135 /**
136 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled). 118 * Closure of the 'next_task' (must be freed if 'next_task' is cancelled).
137 */ 119 */
138 struct NextContext *next_task_nc; 120 struct NextContext *next_task_nc;
@@ -141,29 +123,12 @@ struct Plugin
141 * Pending task with scheduler for running the next request. 123 * Pending task with scheduler for running the next request.
142 */ 124 */
143 GNUNET_SCHEDULER_TaskIdentifier next_task; 125 GNUNET_SCHEDULER_TaskIdentifier next_task;
144
145 /**
146 * How much data are we currently storing
147 * in the database?
148 */
149 unsigned long long payload;
150
151 /**
152 * Number of updates that were made to the
153 * payload value since we last synchronized
154 * it with the statistics service.
155 */
156 unsigned int lastSync;
157 126
158 /** 127 /**
159 * Should the database be dropped on shutdown? 128 * Should the database be dropped on shutdown?
160 */ 129 */
161 int drop_on_shutdown; 130 int drop_on_shutdown;
162 131
163 /**
164 * Did we get an answer from statistics?
165 */
166 int stats_worked;
167}; 132};
168 133
169 134
@@ -267,12 +232,7 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
267 return GNUNET_SYSERR; 232 return GNUNET_SYSERR;
268 } 233 }
269 /* database is new or got deleted, reset payload to zero! */ 234 /* database is new or got deleted, reset payload to zero! */
270 if (plugin->stat_get != NULL) 235 plugin->env->duc (plugin->env->cls, 0);
271 {
272 GNUNET_STATISTICS_get_cancel (plugin->stat_get);
273 plugin->stat_get = NULL;
274 }
275 plugin->payload = 0;
276 } 236 }
277 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir), 237 plugin->fn = GNUNET_STRINGS_to_utf8 (afsdir, strlen (afsdir),
278#ifdef ENABLE_NLS 238#ifdef ENABLE_NLS
@@ -375,22 +335,6 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg,
375 335
376 336
377/** 337/**
378 * Synchronize our utilization statistics with the
379 * statistics service.
380 * @param plugin the plugin context (state for this module)
381 */
382static void
383sync_stats (struct Plugin *plugin)
384{
385 GNUNET_STATISTICS_set (plugin->statistics,
386 QUOTA_STAT_NAME,
387 plugin->payload,
388 GNUNET_YES);
389 plugin->lastSync = 0;
390}
391
392
393/**
394 * Shutdown database connection and associate data 338 * Shutdown database connection and associate data
395 * structures. 339 * structures.
396 * @param plugin the plugin context (state for this module) 340 * @param plugin the plugin context (state for this module)
@@ -398,8 +342,6 @@ sync_stats (struct Plugin *plugin)
398static void 342static void
399database_shutdown (struct Plugin *plugin) 343database_shutdown (struct Plugin *plugin)
400{ 344{
401 if (plugin->lastSync > 0)
402 sync_stats (plugin);
403 if (plugin->updPrio != NULL) 345 if (plugin->updPrio != NULL)
404 sqlite3_finalize (plugin->updPrio); 346 sqlite3_finalize (plugin->updPrio);
405 if (plugin->insertContent != NULL) 347 if (plugin->insertContent != NULL)
@@ -410,20 +352,6 @@ database_shutdown (struct Plugin *plugin)
410 352
411 353
412/** 354/**
413 * Get an estimate of how much space the database is
414 * currently using.
415 *
416 * @param cls our plugin context
417 * @return number of bytes used on disk
418 */
419static unsigned long long sqlite_plugin_get_size (void *cls)
420{
421 struct Plugin *plugin = cls;
422 return plugin->payload;
423}
424
425
426/**
427 * Delete the database entry with the given 355 * Delete the database entry with the given
428 * row identifier. 356 * row identifier.
429 * 357 *
@@ -661,23 +589,15 @@ sqlite_next_request_cont (void *cls,
661 if ( (ret == GNUNET_NO) && 589 if ( (ret == GNUNET_NO) &&
662 (GNUNET_OK == delete_by_rowid (plugin, rowid)) ) 590 (GNUNET_OK == delete_by_rowid (plugin, rowid)) )
663 { 591 {
664 if (plugin->payload >= size + GNUNET_DATASTORE_ENTRY_OVERHEAD) 592 plugin->env->duc (plugin->env->cls,
665 plugin->payload -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD); 593 - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
666 else
667 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
668 _("Datastore payload inaccurate, please fix and restart!\n"));
669 plugin->lastSync++;
670#if DEBUG_SQLITE 594#if DEBUG_SQLITE
671 if (ret == GNUNET_NO) 595 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
672 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 596 "sqlite",
673 "sqlite", 597 "Removed entry %llu (%u bytes)\n",
674 "Removed entry %llu (%u bytes), new payload is %llu\n", 598 (unsigned long long) rowid,
675 (unsigned long long) rowid, 599 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
676 size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
677 plugin->payload);
678#endif 600#endif
679 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
680 sync_stats (plugin);
681 } 601 }
682} 602}
683 603
@@ -798,17 +718,14 @@ sqlite_plugin_put (void *cls,
798 LOG_SQLITE (plugin, NULL, 718 LOG_SQLITE (plugin, NULL,
799 GNUNET_ERROR_TYPE_ERROR | 719 GNUNET_ERROR_TYPE_ERROR |
800 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset"); 720 GNUNET_ERROR_TYPE_BULK, "sqlite3_reset");
801 plugin->lastSync++; 721 plugin->env->duc (plugin->env->cls,
802 plugin->payload += size + GNUNET_DATASTORE_ENTRY_OVERHEAD; 722 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
803#if DEBUG_SQLITE 723#if DEBUG_SQLITE
804 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, 724 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
805 "sqlite", 725 "sqlite",
806 "Stored new entry (%u bytes), new payload is %llu\n", 726 "Stored new entry (%u bytes)\n",
807 size + GNUNET_DATASTORE_ENTRY_OVERHEAD, 727 size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
808 plugin->payload);
809#endif 728#endif
810 if (plugin->lastSync >= MAX_STAT_SYNC_LAG)
811 sync_stats (plugin);
812 return GNUNET_OK; 729 return GNUNET_OK;
813} 730}
814 731
@@ -1574,81 +1491,50 @@ sqlite_plugin_drop (void *cls)
1574} 1491}
1575 1492
1576 1493
1577/** 1494static unsigned long long
1578 * Callback function to process statistic values. 1495sqlite_plugin_get_size (void *cls)
1579 *
1580 * @param cls closure
1581 * @param subsystem name of subsystem that created the statistic
1582 * @param name the name of the datum
1583 * @param value the current value
1584 * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1585 * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1586 */
1587static int
1588process_stat_in (void *cls,
1589 const char *subsystem,
1590 const char *name,
1591 uint64_t value,
1592 int is_persistent)
1593{
1594 struct Plugin *plugin = cls;
1595
1596 plugin->stats_worked = GNUNET_YES;
1597 plugin->payload += value;
1598#if DEBUG_SQLITE
1599 GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG,
1600 "sqlite",
1601 "Notification from statistics about existing payload (%llu), new payload is %llu\n",
1602 value,
1603 plugin->payload);
1604#endif
1605 return GNUNET_OK;
1606}
1607
1608
1609static void
1610process_stat_done (void *cls,
1611 int success)
1612{ 1496{
1613 struct Plugin *plugin = cls; 1497 struct Plugin *plugin = cls;
1614 sqlite3_stmt *stmt; 1498 sqlite3_stmt *stmt;
1615 uint64_t pages; 1499 uint64_t pages;
1616 uint64_t page_size; 1500 uint64_t page_size;
1617 1501
1618 plugin->stat_get = NULL; 1502 if (SQLITE_VERSION_NUMBER < 3006000)
1619 if ( (plugin->stats_worked == GNUNET_NO) && 1503 {
1620 (SQLITE_VERSION_NUMBER >= 3006000) ) 1504 GNUNET_log_from (GNUNET_ERROR_TYPE_WARNING,
1621 { 1505 "datastore-sqlite",
1622 CHECK (SQLITE_OK == 1506 _("sqlite version to old to determine size, assuming zero\n"));
1623 sqlite3_exec (plugin->dbh, 1507 return 0;
1624 "VACUUM", NULL, NULL, ENULL));
1625 CHECK (SQLITE_OK ==
1626 sqlite3_exec (plugin->dbh,
1627 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
1628 CHECK (SQLITE_OK ==
1629 sq_prepare (plugin->dbh,
1630 "PRAGMA page_count",
1631 &stmt));
1632 if (SQLITE_ROW ==
1633 sqlite3_step (stmt))
1634 pages = sqlite3_column_int64 (stmt, 0);
1635 else
1636 pages = 0;
1637 sqlite3_finalize (stmt);
1638 CHECK (SQLITE_OK ==
1639 sq_prepare (plugin->dbh,
1640 "PRAGMA page_size",
1641 &stmt));
1642 CHECK (SQLITE_ROW ==
1643 sqlite3_step (stmt));
1644 page_size = sqlite3_column_int64 (stmt, 0);
1645 sqlite3_finalize (stmt);
1646 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1647 _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1648 (unsigned long long) pages,
1649 (unsigned long long) page_size);
1650 plugin->payload = pages * page_size;
1651 } 1508 }
1509 CHECK (SQLITE_OK ==
1510 sqlite3_exec (plugin->dbh,
1511 "VACUUM", NULL, NULL, ENULL));
1512 CHECK (SQLITE_OK ==
1513 sqlite3_exec (plugin->dbh,
1514 "PRAGMA auto_vacuum=INCREMENTAL", NULL, NULL, ENULL));
1515 CHECK (SQLITE_OK ==
1516 sq_prepare (plugin->dbh,
1517 "PRAGMA page_count",
1518 &stmt));
1519 if (SQLITE_ROW ==
1520 sqlite3_step (stmt))
1521 pages = sqlite3_column_int64 (stmt, 0);
1522 else
1523 pages = 0;
1524 sqlite3_finalize (stmt);
1525 CHECK (SQLITE_OK ==
1526 sq_prepare (plugin->dbh,
1527 "PRAGMA page_size",
1528 &stmt));
1529 CHECK (SQLITE_ROW ==
1530 sqlite3_step (stmt));
1531 page_size = sqlite3_column_int64 (stmt, 0);
1532 sqlite3_finalize (stmt);
1533 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1534 _("Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1535 (unsigned long long) pages,
1536 (unsigned long long) page_size);
1537 return pages * page_size;
1652} 1538}
1653 1539
1654 1540
@@ -1669,16 +1555,6 @@ libgnunet_plugin_datastore_sqlite_init (void *cls)
1669 return NULL; /* can only initialize once! */ 1555 return NULL; /* can only initialize once! */
1670 memset (&plugin, 0, sizeof(struct Plugin)); 1556 memset (&plugin, 0, sizeof(struct Plugin));
1671 plugin.env = env; 1557 plugin.env = env;
1672 plugin.statistics = GNUNET_STATISTICS_create (env->sched,
1673 "ds-sqlite",
1674 env->cfg);
1675 plugin.stat_get = GNUNET_STATISTICS_get (plugin.statistics,
1676 "ds-sqlite",
1677 QUOTA_STAT_NAME,
1678 GNUNET_TIME_UNIT_SECONDS,
1679 &process_stat_done,
1680 &process_stat_in,
1681 &plugin);
1682 if (GNUNET_OK != 1558 if (GNUNET_OK !=
1683 database_setup (env->cfg, &plugin)) 1559 database_setup (env->cfg, &plugin))
1684 { 1560 {
@@ -1717,11 +1593,6 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
1717 struct GNUNET_DATASTORE_PluginFunctions *api = cls; 1593 struct GNUNET_DATASTORE_PluginFunctions *api = cls;
1718 struct Plugin *plugin = api->cls; 1594 struct Plugin *plugin = api->cls;
1719 1595
1720 if (plugin->stat_get != NULL)
1721 {
1722 GNUNET_STATISTICS_get_cancel (plugin->stat_get);
1723 plugin->stat_get = NULL;
1724 }
1725 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK) 1596 if (plugin->next_task != GNUNET_SCHEDULER_NO_TASK)
1726 { 1597 {
1727 GNUNET_SCHEDULER_cancel (plugin->env->sched, 1598 GNUNET_SCHEDULER_cancel (plugin->env->sched,
@@ -1735,10 +1606,7 @@ libgnunet_plugin_datastore_sqlite_done (void *cls)
1735 if (plugin->drop_on_shutdown) 1606 if (plugin->drop_on_shutdown)
1736 fn = GNUNET_strdup (plugin->fn); 1607 fn = GNUNET_strdup (plugin->fn);
1737 database_shutdown (plugin); 1608 database_shutdown (plugin);
1738 GNUNET_STATISTICS_destroy (plugin->statistics,
1739 GNUNET_NO);
1740 plugin->env = NULL; 1609 plugin->env = NULL;
1741 plugin->payload = 0;
1742 GNUNET_free (api); 1610 GNUNET_free (api);
1743 if (fn != NULL) 1611 if (fn != NULL)
1744 { 1612 {