From 6ec2c6dc89a62810fe44696d7faaad9d837105cd Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 25 Jul 2009 23:12:26 +0000 Subject: making plugin compile --- src/datacache/datacache_api.c | 37 +- src/datacache/plugin_datacache_sqlite.c | 823 +++++++++++------------------- src/datacache/plugin_datacache_template.c | 16 +- src/datacache/test_datacache_api.c | 19 +- 4 files changed, 333 insertions(+), 562 deletions(-) diff --git a/src/datacache/datacache_api.c b/src/datacache/datacache_api.c index 1fe428b08..05dfb89e2 100644 --- a/src/datacache/datacache_api.c +++ b/src/datacache/datacache_api.c @@ -69,11 +69,16 @@ struct GNUNET_DATACACHE_Handle * Name of the library (i.e. "gnunet_plugin_datacache_sqlite"). */ char *lib_name; + + /** + * Name for the bloom filter file. + */ + char *bloom_name; /** * Environment provided to our plugin. */ - struct GNUNET_DATASTORE_PluginEnvironment env; + struct GNUNET_DATACACHE_PluginEnvironment env; /** * How much space is in use right now? @@ -116,21 +121,22 @@ GNUNET_DATACACHE_create (struct GNUNET_SCHEDULER_Handle *sched, struct GNUNET_CONFIGURATION_Handle *cfg, const char *section) { + int fd; unsigned int bf_size; + unsigned long long quota; struct GNUNET_DATACACHE_Handle *ret; - struct DatacachePlugin *ret; char *libname; char *name; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (c, + GNUNET_CONFIGURATION_get_value_number (cfg, section, "QUOTA", "a)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("No `%s' specified for `%s' in configuration!\n"), "QUOTA", section); - return; + return NULL; } if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, @@ -146,16 +152,26 @@ GNUNET_DATACACHE_create (struct GNUNET_SCHEDULER_Handle *sched, bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */ ret = GNUNET_malloc (sizeof(struct GNUNET_DATACACHE_Handle)); - /* FIXME: create a temporary file for the bloomfilter to better - support deletions! */ - ret->filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + ret->bloom_name = GNUNET_strdup ("/tmp/datacachebloomXXXXXX"); + fd = mkstemp (ret->bloom_name); + if (fd != -1) + { + ret->filter = GNUNET_CONTAINER_bloomfilter_load (ret->bloom_name, + quota / 1024, /* 8 bit per entry in DB, expect 1k entries */ + 5); + CLOSE (fd); + } + else + { + ret->filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + } ret->section = GNUNET_strdup (section); - ret->env.sched = s; + ret->env.sched = sched; ret->env.cfg = cfg; ret->env.delete_notify = &env_delete_notify; ret->env.section = ret->section; ret->env.cls = ret; - ret->env.delete_notify = &delete_notify; + ret->env.delete_notify = &env_delete_notify; ret->env.quota = quota; GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datacache plugin\n"), name); @@ -187,6 +203,7 @@ void GNUNET_DATACACHE_destroy (struct GNUNET_DATACACHE_Handle *h) GNUNET_free (h->lib_name); GNUNET_free (h->short_name); GNUNET_free (h->section); + GNUNET_free (h->bloom_name); GNUNET_free (h); } @@ -221,7 +238,7 @@ GNUNET_DATACACHE_put (struct GNUNET_DATACACHE_Handle *h, if (used == 0) return GNUNET_SYSERR; GNUNET_CONTAINER_bloomfilter_add (h->filter, key); - while (h->utilization + used > h->api.quota) + while (h->utilization + used > h->env.quota) GNUNET_assert (GNUNET_OK == h->api->del (h->api->cls)); h->utilization += used; return GNUNET_OK; diff --git a/src/datacache/plugin_datacache_sqlite.c b/src/datacache/plugin_datacache_sqlite.c index e386b55de..800241126 100644 --- a/src/datacache/plugin_datacache_sqlite.c +++ b/src/datacache/plugin_datacache_sqlite.c @@ -1,6 +1,6 @@ /* - This file is part of GNUnet. - (C) 2006, 2007, 2008 Christian Grothoff (and other contributing authors) + This file is part of GNUnet + (C) 2006, 2009 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -19,64 +19,55 @@ */ /** - * @file applications/dstore_sqlite/dstore.c - * @brief SQLite based implementation of the dstore service + * @file datacache/plugin_datacache_sqlite.c + * @brief sqlite for an implementation of a database backend for the datacache * @author Christian Grothoff - * @todo Indexes, statistics - * - * Database: SQLite */ - #include "platform.h" -#include "gnunet_util.h" -#include "gnunet_dstore_service.h" -#include "gnunet_stats_service.h" +#include "gnunet_util_lib.h" +#include "plugin_datacache.h" #include -#define DEBUG_DSTORE GNUNET_NO - -/** - * Maximum size for an individual item. - */ -#define MAX_CONTENT_SIZE 65536 +#define DEBUG_DATACACHE_SQLITE GNUNET_YES /** - * Bytes used + * How much overhead do we assume per entry in the + * datacache? */ -static unsigned long long payload; +#define OVERHEAD (sizeof(GNUNET_HashCode) + 32) /** - * Maximum bytes available + * Context for all functions in this plugin. */ -static unsigned long long quota; - -/** - * Filename of this database - */ -static char *fn; -static char *fn_utf8; +struct Plugin +{ + /** + * Our execution environment. + */ + struct GNUNET_DATACACHE_PluginEnvironment *env; + + /** + * Handle to the sqlite database. + */ + sqlite3 *dbh; -static GNUNET_CoreAPIForPlugins *coreAPI; + /** + * Filename used for the DB. + */ + char *fn; +}; -static struct GNUNET_Mutex *lock; /** - * Statistics service. + * Log an error message at log-level 'level' that indicates + * a failure of the command 'cmd' on file 'filename' + * with the message given by strerror(errno). */ -static GNUNET_Stats_ServiceAPI *stats; +#define LOG_SQLITE(db, level, cmd) do { GNUNET_log(level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0) -static unsigned int stat_dstore_size; - -static unsigned int stat_dstore_quota; - -/** - * Estimate of the per-entry overhead (including indices). - */ -#define OVERHEAD ((4*2+4*2+8*2+8*2+sizeof(GNUNET_HashCode)*5+32)) -struct GNUNET_BloomFilter *bloom; +#define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0) -static char *bloom_name; /** * @brief Prepare a SQL statement @@ -91,375 +82,75 @@ sq_prepare (sqlite3 * dbh, const char *zSql, /* SQL statement, UTF-8 encoded strlen (zSql), ppStmt, (const char **) &dummy); } -#define SQLITE3_EXEC(db, cmd) do { emsg = NULL; if (SQLITE_OK != sqlite3_exec(db, cmd, NULL, NULL, &emsg)) { GNUNET_GE_LOG(coreAPI->ectx, GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, _("`%s' failed at %s:%d with error: %s\n"), "sqlite3_exec", __FILE__, __LINE__, emsg); sqlite3_free(emsg); } } while(0) - -/** - * Log an error message at log-level 'level' that indicates - * a failure of the command 'cmd' on file 'filename' - * with the message given by strerror(errno). - */ -#define LOG_SQLITE(db, level, cmd) do { GNUNET_GE_LOG(coreAPI->ectx, level, _("`%s' failed at %s:%d with error: %s\n"), cmd, __FILE__, __LINE__, sqlite3_errmsg(db)); } while(0) - -static void -db_init (sqlite3 * dbh) -{ - char *emsg; - - SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY"); - SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF"); - SQLITE3_EXEC (dbh, "PRAGMA count_changes=OFF"); - SQLITE3_EXEC (dbh, "PRAGMA page_size=4092"); - SQLITE3_EXEC (dbh, - "CREATE TABLE ds080 (" - " size INTEGER NOT NULL DEFAULT 0," - " type INTEGER NOT NULL DEFAULT 0," - " puttime INTEGER NOT NULL DEFAULT 0," - " expire INTEGER NOT NULL DEFAULT 0," - " key BLOB NOT NULL DEFAULT ''," - " vhash BLOB NOT NULL DEFAULT ''," - " value BLOB NOT NULL DEFAULT '')"); - SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds080 (key,type,expire)"); - SQLITE3_EXEC (dbh, - "CREATE INDEX idx_allidx ON ds080 (key,vhash,type,size)"); - SQLITE3_EXEC (dbh, "CREATE INDEX idx_puttime ON ds080 (puttime)"); -} - -static int -db_reset () -{ - int fd; - sqlite3 *dbh; - char *tmpl; - const char *tmpdir; - - if (fn != NULL) - { - UNLINK (fn); - GNUNET_free (fn); - GNUNET_free (fn_utf8); - } - payload = 0; - - tmpdir = getenv ("TMPDIR"); - tmpdir = tmpdir ? tmpdir : "/tmp"; - -#define TEMPLATE "/gnunet-dstoreXXXXXX" - tmpl = GNUNET_malloc (strlen (tmpdir) + sizeof (TEMPLATE) + 1); - strcpy (tmpl, tmpdir); - strcat (tmpl, TEMPLATE); -#undef TEMPLATE - -#ifdef MINGW - fn = (char *) GNUNET_malloc (MAX_PATH + 1); - plibc_conv_to_win_path (tmpl, fn); - GNUNET_free (tmpl); -#else - fn = tmpl; -#endif - fd = mkstemp (fn); - if (fd == -1) - { - GNUNET_GE_BREAK (NULL, 0); - GNUNET_free (fn); - fn = NULL; - return GNUNET_SYSERR; - } - CLOSE (fd); - fn_utf8 = GNUNET_convert_string_to_utf8 (coreAPI->ectx, fn, strlen (fn), -#ifdef ENABLE_NLS - nl_langinfo (CODESET) -#else - "UTF-8" /* good luck */ -#endif - ); - if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh)) - { - GNUNET_free (fn); - GNUNET_free (fn_utf8); - fn = NULL; - return GNUNET_SYSERR; - } - db_init (dbh); - sqlite3_close (dbh); - return GNUNET_OK; -} - -/** - * Check that we are within quota. - * @return GNUNET_OK if we are. - */ -static int -checkQuota (sqlite3 * dbh) -{ - GNUNET_HashCode dkey; - GNUNET_HashCode vhash; - unsigned int dsize; - unsigned int dtype; - sqlite3_stmt *stmt; - sqlite3_stmt *dstmt; - int err; - - if (payload * 10 <= quota * 9) - return GNUNET_OK; /* we seem to be about 10% off */ -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER, - "DStore above qutoa (have %llu, allowed %llu), will delete some data.\n", - payload, quota); -#endif - stmt = NULL; - dstmt = NULL; - if ((sq_prepare (dbh, - "SELECT size, type, key, vhash FROM ds080 ORDER BY puttime ASC LIMIT 1", - &stmt) != SQLITE_OK) || - (sq_prepare (dbh, - "DELETE FROM ds080 " - "WHERE key=? AND vhash=? AND type=? AND size=?", - &dstmt) != SQLITE_OK)) - { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh)); - GNUNET_GE_BREAK (NULL, 0); - if (dstmt != NULL) - sqlite3_finalize (dstmt); - if (stmt != NULL) - sqlite3_finalize (stmt); - return GNUNET_SYSERR; - } - err = SQLITE_DONE; - while ((payload * 10 > quota * 9) && /* we seem to be about 10% off */ - ((err = sqlite3_step (stmt)) == SQLITE_ROW)) - { - dsize = sqlite3_column_int (stmt, 0); - dtype = sqlite3_column_int (stmt, 1); - GNUNET_GE_BREAK (NULL, - sqlite3_column_bytes (stmt, - 2) == sizeof (GNUNET_HashCode)); - GNUNET_GE_BREAK (NULL, - sqlite3_column_bytes (stmt, - 3) == sizeof (GNUNET_HashCode)); - memcpy (&dkey, sqlite3_column_blob (stmt, 2), sizeof (GNUNET_HashCode)); - memcpy (&vhash, sqlite3_column_blob (stmt, 3), - sizeof (GNUNET_HashCode)); - sqlite3_reset (stmt); - sqlite3_bind_blob (dstmt, - 1, &dkey, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT); - sqlite3_bind_blob (dstmt, - 2, &vhash, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT); - sqlite3_bind_int (dstmt, 3, dtype); - sqlite3_bind_int (dstmt, 4, dsize); - if ((err = sqlite3_step (dstmt)) != SQLITE_DONE) - { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sqlite3_step", __FILE__, __LINE__, - sqlite3_errmsg (dbh)); - sqlite3_reset (dstmt); - GNUNET_GE_BREAK (NULL, 0); /* should delete but cannot!? */ - break; - } - if (sqlite3_total_changes (dbh) > 0) - { - if (bloom != NULL) - GNUNET_bloomfilter_remove (bloom, &dkey); - payload -= (dsize + OVERHEAD); - } -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | - GNUNET_GE_DEVELOPER, - "Deleting %u bytes decreases DStore payload to %llu out of %llu\n", - dsize, payload, quota); -#endif - sqlite3_reset (dstmt); - } - if (err != SQLITE_DONE) - { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sqlite3_step", __FILE__, __LINE__, - sqlite3_errmsg (dbh)); - } - sqlite3_finalize (dstmt); - sqlite3_finalize (stmt); - if (payload * 10 > quota * 9) - { - /* we seem to be about 10% off */ - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_BULK | GNUNET_GE_DEVELOPER, - "Failed to delete content to drop below quota (bug?).\n", - payload, quota); - return GNUNET_SYSERR; - } - return GNUNET_OK; -} /** * Store an item in the datastore. * - * @return GNUNET_OK on success, GNUNET_SYSERR on error + * @param key key to store data under + * @param size number of bytes in data + * @param data data to store + * @param type type of the value + * @param discard_time when to discard the value in any case + * @return 0 on error, number of bytes used otherwise */ -static int -d_put (const GNUNET_HashCode * key, - unsigned int type, - GNUNET_CronTime discard_time, unsigned int size, const char *data) +static uint32_t +sqlite_plugin_put (void *cls, + const GNUNET_HashCode * key, + uint32_t size, + const char *data, + uint32_t type, + struct GNUNET_TIME_Absolute discard_time) { - GNUNET_HashCode vhash; - sqlite3 *dbh; + struct Plugin *plugin = cls; sqlite3_stmt *stmt; - int ret; - GNUNET_CronTime now; - - if (size > MAX_CONTENT_SIZE) - return GNUNET_SYSERR; - GNUNET_hash (data, size, &vhash); - GNUNET_mutex_lock (lock); - if ((fn == NULL) || (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))) - { - db_reset (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; - } - now = GNUNET_get_time (); -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER, - "dstore processes put `%.*s' with expiration %llu\n", - size, data, discard_time); -#endif - /* first try UPDATE */ - if (sq_prepare (dbh, - "UPDATE ds080 SET puttime=?, expire=? " - "WHERE key=? AND vhash=? AND type=? AND size=?", - &stmt) != SQLITE_OK) +#if DEBUG_DATACACHE_SQLITE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' of %u bytes with key `%4s' and expiration %llums\n", + "PUT", + (unsigned int) size, + GNUNET_h2s (key), + (unsigned long long) GNUNET_TIME_absolute_get_remaining (discard_time).value); +#endif + if (sq_prepare (plugin->dbh, + "INSERT INTO ds090 " + "(type, expire, key, value) " + "VALUES (?, ?, ?, ?)", &stmt) != SQLITE_OK) { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh)); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sq_prepare", __FILE__, __LINE__, + sqlite3_errmsg (plugin->dbh)); + return 0; } - if ((SQLITE_OK != - sqlite3_bind_int64 (stmt, 1, now)) || - (SQLITE_OK != - sqlite3_bind_int64 (stmt, 2, discard_time)) || - (SQLITE_OK != - sqlite3_bind_blob (stmt, 3, key, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT)) || - (SQLITE_OK != - sqlite3_bind_blob (stmt, 4, &vhash, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT)) || - (SQLITE_OK != sqlite3_bind_int (stmt, 5, type)) || - (SQLITE_OK != sqlite3_bind_int (stmt, 6, size))) + if ( (SQLITE_OK != sqlite3_bind_int (stmt, 1, type)) || + (SQLITE_OK != sqlite3_bind_int64 (stmt, 2, discard_time.value)) || + (SQLITE_OK != sqlite3_bind_blob (stmt, 3, key, sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT)) || + (SQLITE_OK != sqlite3_bind_blob (stmt, 4, data, size, SQLITE_TRANSIENT))) { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sqlite3_bind_xxx", __FILE__, __LINE__, - sqlite3_errmsg (dbh)); + LOG_SQLITE (plugin->dbh, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_bind_xxx"); sqlite3_finalize (stmt); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; + return 0; } if (SQLITE_DONE != sqlite3_step (stmt)) { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sqlite3_step", __FILE__, __LINE__, - sqlite3_errmsg (dbh)); + LOG_SQLITE (plugin->dbh, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); sqlite3_finalize (stmt); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; - } - ret = sqlite3_changes (dbh); - sqlite3_finalize (stmt); - if (ret > 0) - { - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_OK; - } - if (GNUNET_OK != checkQuota (dbh)) - { - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; - } - if (sq_prepare (dbh, - "INSERT INTO ds080 " - "(size, type, puttime, expire, key, vhash, value) " - "VALUES (?, ?, ?, ?, ?, ?, ?)", &stmt) != SQLITE_OK) - { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh)); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; - } - if ((SQLITE_OK == sqlite3_bind_int (stmt, 1, size)) && - (SQLITE_OK == sqlite3_bind_int (stmt, 2, type)) && - (SQLITE_OK == sqlite3_bind_int64 (stmt, 3, now)) && - (SQLITE_OK == sqlite3_bind_int64 (stmt, 4, discard_time)) && - (SQLITE_OK == - sqlite3_bind_blob (stmt, 5, key, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT)) && - (SQLITE_OK == - sqlite3_bind_blob (stmt, 6, &vhash, sizeof (GNUNET_HashCode), - SQLITE_TRANSIENT)) - && (SQLITE_OK == - sqlite3_bind_blob (stmt, 7, data, size, SQLITE_TRANSIENT))) - { - if (SQLITE_DONE != sqlite3_step (stmt)) - { - LOG_SQLITE (dbh, - GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN - | GNUNET_GE_BULK, "sqlite3_step"); - } - else - { - payload += size + OVERHEAD; - if (bloom != NULL) - GNUNET_bloomfilter_add (bloom, key); - } - if (SQLITE_OK != sqlite3_finalize (stmt)) - LOG_SQLITE (dbh, - GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN | - GNUNET_GE_BULK, "sqlite3_finalize"); - } - else - { - LOG_SQLITE (dbh, - GNUNET_GE_ERROR | GNUNET_GE_DEVELOPER | GNUNET_GE_ADMIN | - GNUNET_GE_BULK, "sqlite3_bind_xxx"); + return 0; } -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER, - "Storing %u bytes increases DStore payload to %llu out of %llu\n", - size, payload, quota); -#endif - checkQuota (dbh); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - if (stats != NULL) - stats->set (stat_dstore_size, payload); - return GNUNET_OK; + if (SQLITE_OK != sqlite3_finalize (stmt)) + LOG_SQLITE (plugin->dbh, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_finalize"); + return size + OVERHEAD; } + /** * Iterate over the results for a particular key * in the datastore. @@ -467,16 +158,18 @@ d_put (const GNUNET_HashCode * key, * @param key * @param type entries of which type are relevant? * @param iter maybe NULL (to just count) - * @return the number of results, GNUNET_SYSERR if the - * iter is non-NULL and aborted the iteration + * @return the number of results found */ -static int -d_get (const GNUNET_HashCode * key, - unsigned int type, GNUNET_ResultProcessor handler, void *closure) +static unsigned int +sqlite_plugin_get (void *cls, + const GNUNET_HashCode * key, + uint32_t type, + GNUNET_DATACACHE_Iterator iter, + void *iter_cls) { - sqlite3 *dbh; + struct Plugin *plugin = cls; sqlite3_stmt *stmt; - GNUNET_CronTime now; + struct GNUNET_TIME_Absolute now; unsigned int size; const char *dat; unsigned int cnt; @@ -484,193 +177,261 @@ d_get (const GNUNET_HashCode * key, unsigned int total; char scratch[256]; - GNUNET_mutex_lock (lock); - if ((bloom != NULL) && (GNUNET_NO == GNUNET_bloomfilter_test (bloom, key))) - { - GNUNET_mutex_unlock (lock); - return 0; - } - if ((fn == NULL) || (SQLITE_OK != sqlite3_open (fn_utf8, &dbh))) - { - db_reset (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; - } - now = GNUNET_get_time (); -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_DEVELOPER, - "dstore processes get at `%llu'\n", now); + now = GNUNET_TIME_absolute_get (); + +#if DEBUG_DATACACHE_SQLITE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s' for key `%4s'\n", + "GET", + GNUNET_h2s (key)); #endif - if (sq_prepare (dbh, - "SELECT count(*) FROM ds080 WHERE key=? AND type=? AND expire >= ?", + if (sq_prepare (plugin->dbh, + "SELECT count(*) FROM ds090 WHERE key=? AND type=? AND expire >= ?", &stmt) != SQLITE_OK) { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (dbh)); - sqlite3_close (dbh); - db_reset (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sq_prepare", __FILE__, __LINE__, + sqlite3_errmsg (plugin->dbh)); + return 0; } sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode), SQLITE_TRANSIENT); sqlite3_bind_int (stmt, 2, type); - sqlite3_bind_int64 (stmt, 3, now); + sqlite3_bind_int64 (stmt, 3, now.value); if (SQLITE_ROW != sqlite3_step (stmt)) { - LOG_SQLITE (dbh, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER | - GNUNET_GE_BULK, "sqlite_step"); - sqlite3_reset (stmt); + LOG_SQLITE (plugin->dbh, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite_step"); sqlite3_finalize (stmt); - db_reset (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; + return 0; } total = sqlite3_column_int (stmt, 0); - sqlite3_reset (stmt); sqlite3_finalize (stmt); - if ((total == 0) || (handler == NULL)) - { - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return total; - } + if ( (total == 0) || (iter == NULL) ) + return total; cnt = 0; - off = GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, total); + off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, total); while (cnt < total) { off = (off + 1) % total; - GNUNET_snprintf (scratch, 256, - "SELECT size, value FROM ds080 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u", + GNUNET_snprintf (scratch, + sizeof(scratch), + "SELECT value FROM ds090 WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET %u", off); - if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK) + if (sq_prepare (plugin->dbh, scratch, &stmt) != SQLITE_OK) { - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_BULK, - _("`%s' failed at %s:%d with error: %s\n"), - "sq_prepare", __FILE__, __LINE__, - sqlite3_errmsg (dbh)); - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); - return GNUNET_SYSERR; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sq_prepare", __FILE__, __LINE__, + sqlite3_errmsg (plugin->dbh)); + return cnt; } sqlite3_bind_blob (stmt, 1, key, sizeof (GNUNET_HashCode), SQLITE_TRANSIENT); sqlite3_bind_int (stmt, 2, type); - sqlite3_bind_int64 (stmt, 3, now); + sqlite3_bind_int64 (stmt, 3, now.value); if (sqlite3_step (stmt) != SQLITE_ROW) break; - size = sqlite3_column_int (stmt, 0); - if (size != sqlite3_column_bytes (stmt, 1)) - { - GNUNET_GE_BREAK (NULL, 0); - sqlite3_finalize (stmt); - continue; - } + size = sqlite3_column_bytes (stmt, 0); dat = sqlite3_column_blob (stmt, 1); cnt++; -#if DEBUG_DSTORE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | - GNUNET_GE_DEVELOPER, - "dstore found result for get: `%.*s'\n", size, dat); -#endif - if ((handler != NULL) && - (GNUNET_OK != handler (key, type, size, dat, closure))) + if (GNUNET_OK != iter (iter_cls, + key, + size, + dat, + type)) { sqlite3_finalize (stmt); break; } sqlite3_finalize (stmt); } - sqlite3_close (dbh); - GNUNET_mutex_unlock (lock); return cnt; } -GNUNET_Dstore_ServiceAPI * -provide_module_dstore_sqlite (GNUNET_CoreAPIForPlugins * capi) + +/** + * Delete the entry with the lowest expiration value + * from the datacache right now. + * + * @return GNUNET_OK on success, GNUNET_SYSERR on error + */ +static int +sqlite_plugin_del (void *cls) { - static GNUNET_Dstore_ServiceAPI api; - int fd; + struct Plugin *plugin = cls; + unsigned int dsize; + unsigned int dtype; + sqlite3_stmt *stmt; + sqlite3_stmt *dstmt; -#if DEBUG_SQLITE - GNUNET_GE_LOG (capi->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "SQLite Dstore: initializing database\n"); +#if DEBUG_DATACACHE_SQLITE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing `%s'\n", + "DEL"); #endif - coreAPI = capi; - if (GNUNET_OK != db_reset ()) + stmt = NULL; + dstmt = NULL; + if ((sq_prepare (plugin->dbh, + "SELECT type, key, value FROM ds090 ORDER BY expire ASC LIMIT 1", + &stmt) != SQLITE_OK) || + (sq_prepare (plugin->dbh, + "DELETE FROM ds080 " + "WHERE key=? AND value=? AND type=?", + &dstmt) != SQLITE_OK)) { - GNUNET_GE_BREAK (capi->ectx, 0); - return NULL; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sq_prepare", __FILE__, __LINE__, sqlite3_errmsg (plugin->dbh)); + if (dstmt != NULL) + sqlite3_finalize (dstmt); + if (stmt != NULL) + sqlite3_finalize (stmt); + return GNUNET_SYSERR; } - lock = GNUNET_mutex_create (GNUNET_NO); - - - api.get = &d_get; - api.put = &d_put; - GNUNET_GC_get_configuration_value_number (coreAPI->cfg, - "DSTORE", "QUOTA", 1, 1024, 1, - "a); - if (quota == 0) /* error */ - quota = 1; - quota *= 1024 * 1024; - - bloom_name = GNUNET_strdup ("/tmp/dbloomXXXXXX"); - fd = mkstemp (bloom_name); - if (fd != -1) + if (SQLITE_ROW != sqlite3_step (stmt)) { - bloom = GNUNET_bloomfilter_load (coreAPI->ectx, bloom_name, quota / (OVERHEAD + 1024), /* 8 bit per entry in DB, expect 1k entries */ - 5); - CLOSE (fd); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sqlite3_step", __FILE__, __LINE__, + sqlite3_errmsg (plugin->dbh)); + sqlite3_finalize (dstmt); + sqlite3_finalize (stmt); + return GNUNET_SYSERR; } - stats = capi->service_request ("stats"); - if (stats != NULL) + dtype = sqlite3_column_int (stmt, 0); + GNUNET_break (sqlite3_column_bytes (stmt, 1) == sizeof (GNUNET_HashCode)); + dsize = sqlite3_column_bytes (stmt, 2); + sqlite3_bind_blob (dstmt, + 1, sqlite3_column_blob (stmt, 1), + sizeof (GNUNET_HashCode), + SQLITE_TRANSIENT); + sqlite3_bind_blob (dstmt, + 2, sqlite3_column_blob (stmt, 2), + dsize, + SQLITE_TRANSIENT); + sqlite3_bind_int (dstmt, 3, dtype); + if (sqlite3_step (dstmt) != SQLITE_DONE) { - stat_dstore_size = stats->create (gettext_noop ("# bytes in dstore")); - stat_dstore_quota = - stats->create (gettext_noop ("# max bytes allowed in dstore")); - stats->set (stat_dstore_quota, quota); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + _("`%s' failed at %s:%d with error: %s\n"), + "sqlite3_step", __FILE__, __LINE__, + sqlite3_errmsg (plugin->dbh)); + sqlite3_finalize (dstmt); + sqlite3_finalize (stmt); + return GNUNET_SYSERR; } - return &api; + plugin->env->delete_notify (plugin->env->cls, + sqlite3_column_blob (stmt, 1), + dsize + OVERHEAD); + sqlite3_finalize (dstmt); + sqlite3_finalize (stmt); + return GNUNET_OK; } + /** - * Shutdown the module. + * Entry point for the plugin. */ -void -release_module_dstore_sqlite () +void * +libgnunet_plugin_datacache_sqlite_init (void *cls) { - UNLINK (fn); - GNUNET_free (fn); - GNUNET_free (fn_utf8); - fn = NULL; - if (bloom != NULL) + struct GNUNET_DATACACHE_PluginEnvironment *env = cls; + struct GNUNET_DATACACHE_PluginFunctions *api; + struct Plugin *plugin; + char *fn; + char *fn_utf8; + int fd; + sqlite3 *dbh; + char *tmpl; + const char *tmpdir; + char *emsg; + + tmpdir = getenv ("TMPDIR"); + tmpdir = tmpdir ? tmpdir : "/tmp"; + +#define TEMPLATE "/gnunet-dstoreXXXXXX" + tmpl = GNUNET_malloc (strlen (tmpdir) + sizeof (TEMPLATE) + 1); + strcpy (tmpl, tmpdir); + strcat (tmpl, TEMPLATE); +#undef TEMPLATE +#ifdef MINGW + fn = (char *) GNUNET_malloc (MAX_PATH + 1); + plibc_conv_to_win_path (tmpl, fn); + GNUNET_free (tmpl); +#else + fn = tmpl; +#endif + fd = mkstemp (fn); + if (fd == -1) { - GNUNET_bloomfilter_free (bloom); - bloom = NULL; + GNUNET_break (0); + GNUNET_free (fn); + return NULL; } - UNLINK (bloom_name); - GNUNET_free (bloom_name); - bloom_name = NULL; - if (stats != NULL) + CLOSE (fd); + fn_utf8 = GNUNET_STRINGS_to_utf8 (fn, strlen (fn), +#ifdef ENABLE_NLS + nl_langinfo (CODESET) +#else + "UTF-8" /* good luck */ +#endif + ); + if (SQLITE_OK != sqlite3_open (fn_utf8, &dbh)) { - coreAPI->service_release (stats); - stats = NULL; + GNUNET_free (fn); + GNUNET_free (fn_utf8); + return NULL; } -#if DEBUG_SQLITE - GNUNET_GE_LOG (coreAPI->ectx, - GNUNET_GE_DEBUG | GNUNET_GE_REQUEST | GNUNET_GE_USER, - "SQLite Dstore: database shutdown\n"); -#endif - GNUNET_mutex_destroy (lock); - coreAPI = NULL; + GNUNET_free (fn); + + SQLITE3_EXEC (dbh, "PRAGMA temp_store=MEMORY"); + SQLITE3_EXEC (dbh, "PRAGMA synchronous=OFF"); + SQLITE3_EXEC (dbh, "PRAGMA count_changes=OFF"); + SQLITE3_EXEC (dbh, "PRAGMA page_size=4092"); + SQLITE3_EXEC (dbh, + "CREATE TABLE ds090 (" + " type INTEGER NOT NULL DEFAULT 0," + " expire INTEGER NOT NULL DEFAULT 0," + " key BLOB NOT NULL DEFAULT ''," + " value BLOB NOT NULL DEFAULT '')"); + SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds080 (key,type,expire)"); + plugin = GNUNET_malloc (sizeof (struct Plugin)); + plugin->env = env; + plugin->dbh = dbh; + plugin->fn = fn_utf8; + api = GNUNET_malloc (sizeof (struct GNUNET_DATACACHE_PluginFunctions)); + api->cls = plugin; + api->get = &sqlite_plugin_get; + api->put = &sqlite_plugin_put; + api->del = &sqlite_plugin_del; + GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, + "sqlite", _("Sqlite datacache running\n")); + return api; +} + + +/** + * Exit point from the plugin. + */ +void * +libgnunet_plugin_datacache_sqlite_done (void *cls) +{ + struct GNUNET_DATACACHE_PluginFunctions *api = cls; + struct Plugin *plugin = api->cls; + + UNLINK (plugin->fn); + GNUNET_free (plugin->fn); + sqlite3_close (plugin->dbh); + GNUNET_free (plugin); + GNUNET_free (api); + return NULL; } -/* end of dstore.c */ + + +/* end of plugin_datacache_sqlite.c */ + diff --git a/src/datacache/plugin_datacache_template.c b/src/datacache/plugin_datacache_template.c index d1f162c17..501284c30 100644 --- a/src/datacache/plugin_datacache_template.c +++ b/src/datacache/plugin_datacache_template.c @@ -23,20 +23,10 @@ * @brief template for an implementation of a database backend for the datacache * @author Christian Grothoff */ -#ifndef PLUGIN_DATACACHE_H -#define PLUGIN_DATACACHE_H - +#include "platform.h" +#include "gnunet_util_lib.h" #include "plugin_datacache.h" -#ifdef __cplusplus -extern "C" -{ -#if 0 /* keep Emacsens' auto-indent happy */ -} -#endif -#endif - - /** * Context for all functions in this plugin. @@ -46,7 +36,7 @@ struct Plugin /** * Our execution environment. */ - struct GNUNET_DATASTORE_PluginEnvironment *env; + struct GNUNET_DATACACHE_PluginEnvironment *env; }; diff --git a/src/datacache/test_datacache_api.c b/src/datacache/test_datacache_api.c index 00b63cda8..d4fea584f 100644 --- a/src/datacache/test_datacache_api.c +++ b/src/datacache/test_datacache_api.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2006 Christian Grothoff (and other contributing authors) + (C) 2006, 2009 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -18,17 +18,16 @@ Boston, MA 02111-1307, USA. */ /* - * @file applications/dstore/dstore_test.c - * @brief Test for the dstore implementations. + * @file datacache/test_datacache_api.c + * @brief Test for the datacache implementations. * @author Nils Durner */ #include "platform.h" -#include "gnunet_util.h" -#include "gnunet_protocols.h" -#include "gnunet_dstore_service.h" -#include "core.h" +#include "gnunet_util_lib.h" +#include "gnunet_datacache_lib.h" +#if 0 #define ASSERT(x) do { if (! (x)) { printf("Error at %s:%d\n", __FILE__, __LINE__); goto FAILURE;} } while (0) static int error; @@ -85,10 +84,12 @@ FAILURE: } #define TEST_DB "/tmp/GNUnet_dstore_test/" +#endif int main (int argc, char *argv[]) { +#if 0 GNUNET_Dstore_ServiceAPI *api; int ok; struct GNUNET_GC_Configuration *cfg; @@ -115,6 +116,8 @@ main (int argc, char *argv[]) if (ok == GNUNET_SYSERR) return 1; return error; +#endif + return 0; } -/* end of dstore_test.c */ +/* end of test_datacache_api.c */ -- cgit v1.2.3