From a58d36b8da7afa42410bac54f57d5f3b6b6c4391 Mon Sep 17 00:00:00 2001 From: David Barksdale Date: Sun, 16 Apr 2017 20:46:21 -0500 Subject: [datastore] Create remove plugin API call The only use of vhash in the get_key call was for removing, split that out into its own function. This simplifies the get_key call and removes the need for some indexes, speeding up insertion into the database. --- src/datastore/gnunet-service-datastore.c | 80 +++++------- src/datastore/plugin_datastore_heap.c | 202 ++++++++++++++++++++--------- src/datastore/plugin_datastore_mysql.c | 209 +++++++++++++++--------------- src/datastore/plugin_datastore_postgres.c | 90 ++++++++++--- src/datastore/plugin_datastore_sqlite.c | 110 +++++++++++++--- src/datastore/plugin_datastore_template.c | 39 ++++-- src/datastore/test_plugin_datastore.c | 40 +++++- src/include/gnunet_datastore_plugin.h | 47 ++++++- 8 files changed, 552 insertions(+), 265 deletions(-) diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index d965ad8e0..53ba858e4 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c @@ -880,7 +880,6 @@ handle_get (void *cls, GNUNET_ntohll (msg->next_uid), msg->random, NULL, - NULL, ntohl (msg->type), &transmit_item, client); @@ -932,7 +931,6 @@ handle_get_key (void *cls, GNUNET_ntohll (msg->next_uid), msg->random, &msg->key, - NULL, ntohl (msg->type), &transmit_item, client); @@ -1001,50 +999,46 @@ handle_get_zero_anonymity (void *cls, /** - * Callback function that will cause the item that is passed - * in to be deleted (by returning #GNUNET_NO). + * Remove continuation. * * @param cls closure * @param key key for the content * @param size number of bytes in data - * @param data content stored - * @param type type of the content - * @param priority priority of the content - * @param anonymity anonymity-level for the content - * @param replication replication-level for the content - * @param expiration expiration time for the content - * @param uid unique identifier for the datum - * @return #GNUNET_OK to keep the item - * #GNUNET_NO to delete the item + * @param status #GNUNET_OK if removed, #GNUNET_NO if not found, + * or #GNUNET_SYSERROR if error + * @param msg error message on error */ -static int -remove_callback (void *cls, - const struct GNUNET_HashCode *key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - uint32_t replication, - struct GNUNET_TIME_Absolute expiration, - uint64_t uid) +static void +remove_continuation (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + int status, + const char *msg) { struct GNUNET_SERVICE_Client *client = cls; - if (NULL == key) + if (GNUNET_SYSERR == status) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "REMOVE request failed: %s.\n", + msg); + transmit_status (client, + GNUNET_NO, + msg); + return; + } + if (GNUNET_NO == status) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "No further matches for REMOVE request.\n"); + "Content not found for REMOVE request.\n"); transmit_status (client, GNUNET_NO, _("Content not found")); - return GNUNET_OK; /* last item */ + return; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Item %llu matches REMOVE request for key `%s' and type %u.\n", - (unsigned long long) uid, - GNUNET_h2s (key), - type); + "Item matches REMOVE request for key `%s'.\n", + GNUNET_h2s (key)); GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes removed (explicit request)"), size, @@ -1054,7 +1048,6 @@ remove_callback (void *cls, transmit_status (client, GNUNET_OK, NULL); - return GNUNET_NO; } @@ -1090,26 +1083,19 @@ handle_remove (void *cls, const struct DataMessage *dm) { struct GNUNET_SERVICE_Client *client = cls; - struct GNUNET_HashCode vhash; GNUNET_STATISTICS_update (stats, gettext_noop ("# REMOVE requests received"), 1, GNUNET_NO); - GNUNET_CRYPTO_hash (&dm[1], - ntohl (dm->size), - &vhash); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Processing REMOVE request for `%s' of type %u\n", - GNUNET_h2s (&dm->key), - (uint32_t) ntohl (dm->type)); - plugin->api->get_key (plugin->api->cls, - 0, - false, - &dm->key, - &vhash, - (enum GNUNET_BLOCK_Type) ntohl (dm->type), - &remove_callback, - client); + "Processing REMOVE request for `%s'\n", + GNUNET_h2s (&dm->key)); + plugin->api->remove_key (plugin->api->cls, + &dm->key, + ntohl (dm->size), + &dm[1], + &remove_continuation, + client); GNUNET_SERVICE_client_continue (client); } diff --git a/src/datastore/plugin_datastore_heap.c b/src/datastore/plugin_datastore_heap.c index 6dbc15ebd..9950f7ab2 100644 --- a/src/datastore/plugin_datastore_heap.c +++ b/src/datastore/plugin_datastore_heap.c @@ -432,11 +432,6 @@ struct GetContext */ struct Value *value; - /** - * Requested value hash. - */ - const struct GNUNET_HashCode *vhash; - /** * Requested type. */ @@ -465,17 +460,10 @@ get_iterator (void *cls, { struct GetContext *gc = cls; struct Value *value = val; - struct GNUNET_HashCode vh; if ( (gc->type != GNUNET_BLOCK_TYPE_ANY) && (gc->type != value->type) ) return GNUNET_OK; - if (NULL != gc->vhash) - { - GNUNET_CRYPTO_hash (&value[1], value->size, &vh); - if (0 != memcmp (&vh, gc->vhash, sizeof (struct GNUNET_HashCode))) - return GNUNET_OK; - } if (gc->random) { gc->value = value; @@ -498,23 +486,20 @@ get_iterator (void *cls, * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key maybe NULL (to match all entries) - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. - * @param proc function to call on each matching value; + * @param proc function to call on the matching value; * will be called with NULL if nothing matches - * @param proc_cls closure for proc + * @param proc_cls closure for @a proc */ static void -heap_plugin_get_key (void *cls, uint64_t next_uid, bool random, - const struct GNUNET_HashCode *key, - const struct GNUNET_HashCode *vhash, - enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, - void *proc_cls) +heap_plugin_get_key (void *cls, + uint64_t next_uid, + bool random, + const struct GNUNET_HashCode *key, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, + void *proc_cls) { struct Plugin *plugin = cls; struct GetContext gc; @@ -522,7 +507,6 @@ heap_plugin_get_key (void *cls, uint64_t next_uid, bool random, gc.value = NULL; gc.next_uid = next_uid; gc.random = random; - gc.vhash = vhash; gc.type = type; if (NULL == key) { @@ -542,20 +526,17 @@ heap_plugin_get_key (void *cls, uint64_t next_uid, bool random, proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); return; } - if (GNUNET_NO == - proc (proc_cls, - &gc.value->key, - gc.value->size, - &gc.value[1], - gc.value->type, - gc.value->priority, - gc.value->anonymity, - gc.value->replication, - gc.value->expiration, - (uint64_t) (intptr_t) gc.value)) - { - delete_value (plugin, gc.value); - } + GNUNET_assert (GNUNET_OK == + proc (proc_cls, + &gc.value->key, + gc.value->size, + &gc.value[1], + gc.value->type, + gc.value->priority, + gc.value->anonymity, + gc.value->replication, + gc.value->expiration, + (uint64_t) (intptr_t) gc.value)); } @@ -599,18 +580,17 @@ heap_plugin_get_replication (void *cls, value->replication); value = GNUNET_CONTAINER_heap_walk_get_next (plugin->by_replication); } - if (GNUNET_NO == - proc (proc_cls, - &value->key, - value->size, - &value[1], - value->type, - value->priority, - value->anonymity, - value->replication, - value->expiration, - (uint64_t) (intptr_t) value)) - delete_value (plugin, value); + GNUNET_assert (GNUNET_OK == + proc (proc_cls, + &value->key, + value->size, + &value[1], + value->type, + value->priority, + value->anonymity, + value->replication, + value->expiration, + (uint64_t) (intptr_t) value)); } @@ -690,18 +670,17 @@ heap_plugin_get_zero_anonymity (void *cls, uint64_t next_uid, proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0); return; } - if (GNUNET_NO == - proc (proc_cls, - &value->key, - value->size, - &value[1], - value->type, - value->priority, - value->anonymity, - value->replication, - value->expiration, - (uint64_t) (intptr_t) value)) - delete_value (plugin, value); + GNUNET_assert (GNUNET_OK == + proc (proc_cls, + &value->key, + value->size, + &value[1], + value->type, + value->priority, + value->anonymity, + value->replication, + value->expiration, + (uint64_t) (intptr_t) value)); } @@ -778,6 +757,102 @@ heap_get_keys (void *cls, } +/** + * Closure for iterator called during 'remove_key'. + */ +struct RemoveContext +{ + + /** + * Value found. + */ + struct Value *value; + + /** + * Size of data. + */ + uint32_t size; + + /** + * Data to remove. + */ + const void *data; + +}; + + +/** + * Obtain the matching value with the lowest uid >= next_uid. + * + * @param cls the 'struct GetContext' + * @param key unused + * @param val the 'struct Value' + * @return GNUNET_YES (continue iteration), GNUNET_NO if result was found + */ +static int +remove_iterator (void *cls, + const struct GNUNET_HashCode *key, + void *val) +{ + struct RemoveContext *rc = cls; + struct Value *value = val; + + if (value->size != rc->size) + return GNUNET_YES; + if (0 != memcmp (value->data, rc->data, rc->size)) + return GNUNET_YES; + rc->value = value; + return GNUNET_NO; +} + + +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +static void +heap_plugin_remove_key (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls) +{ + struct Plugin *plugin = cls; + struct RemoveContext rc; + + rc.value = NULL; + rc.size = size; + rc.data = data; + GNUNET_CONTAINER_multihashmap_get_multiple (plugin->keyvalue, + key, + &remove_iterator, + &rc); + if (NULL == rc.value) + { + cont (cont_cls, + key, + size, + GNUNET_NO, + NULL); + return; + } + delete_value (plugin, + rc.value); + cont (cont_cls, + key, + size, + GNUNET_OK, + NULL); +} + + /** * Entry point for the plugin. * @@ -813,6 +888,7 @@ libgnunet_plugin_datastore_heap_init (void *cls) api->get_zero_anonymity = &heap_plugin_get_zero_anonymity; api->drop = &heap_plugin_drop; api->get_keys = &heap_get_keys; + api->remove_key = &heap_plugin_remove_key; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "heap", _("Heap database running\n")); return api; diff --git a/src/datastore/plugin_datastore_mysql.c b/src/datastore/plugin_datastore_mysql.c index edc459272..708e35860 100644 --- a/src/datastore/plugin_datastore_mysql.c +++ b/src/datastore/plugin_datastore_mysql.c @@ -150,6 +150,12 @@ struct Plugin #define DELETE_ENTRY_BY_UID "DELETE FROM gn090 WHERE uid=?" struct GNUNET_MYSQL_StatementHandle *delete_entry_by_uid; +#define DELETE_ENTRY_BY_HASH_VALUE "DELETE FROM gn090 "\ + "WHERE hash = ? AND "\ + "value = ? "\ + "LIMIT 1" + struct GNUNET_MYSQL_StatementHandle *delete_entry_by_hash_value; + #define RESULT_COLUMNS "repl, type, prio, anonLevel, expire, hash, value, uid" #define SELECT_ENTRY "SELECT " RESULT_COLUMNS " FROM gn090 "\ @@ -159,22 +165,13 @@ struct Plugin struct GNUNET_MYSQL_StatementHandle *select_entry; #define SELECT_ENTRY_BY_HASH "SELECT " RESULT_COLUMNS " FROM gn090 "\ - "FORCE INDEX (idx_hash) "\ + "FORCE INDEX (idx_hash_type_uid) "\ "WHERE hash=? AND "\ "uid >= ? AND "\ "(rvalue >= ? OR 0 = ?) "\ "ORDER BY uid LIMIT 1" struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash; -#define SELECT_ENTRY_BY_HASH_AND_VHASH "SELECT " RESULT_COLUMNS " FROM gn090 "\ - "FORCE INDEX (idx_hash_vhash) "\ - "WHERE hash = ? AND "\ - "vhash = ? AND "\ - "uid >= ? AND "\ - "(rvalue >= ? OR 0 = ?) "\ - "ORDER BY uid LIMIT 1" - struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_vhash; - #define SELECT_ENTRY_BY_HASH_AND_TYPE "SELECT " RESULT_COLUMNS " FROM gn090 "\ "FORCE INDEX (idx_hash_type_uid) "\ "WHERE hash = ? AND "\ @@ -184,17 +181,6 @@ struct Plugin "ORDER BY uid LIMIT 1" struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_and_type; -#define SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE "SELECT " RESULT_COLUMNS " "\ - "FROM gn090 "\ - "FORCE INDEX (idx_hash_vhash) "\ - "WHERE hash = ? AND "\ - "vhash = ? AND "\ - "type = ? AND "\ - "uid >= ? AND "\ - "(rvalue >= ? OR 0 = ?) "\ - "ORDER BY uid LIMIT 1" - struct GNUNET_MYSQL_StatementHandle *select_entry_by_hash_vhash_and_type; - #define UPDATE_ENTRY "UPDATE gn090 SET "\ "prio = prio + ?, "\ "repl = repl + ?, "\ @@ -552,11 +538,6 @@ execute_select (struct Plugin *plugin, * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key key to match, never NULL - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. * @param proc function to call on the matching value, @@ -568,7 +549,6 @@ mysql_plugin_get_key (void *cls, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, - const struct GNUNET_HashCode *vhash, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls) @@ -602,79 +582,37 @@ mysql_plugin_get_key (void *cls, } else if (type != GNUNET_BLOCK_TYPE_ANY) { - if (NULL != vhash) - { - struct GNUNET_MY_QueryParam params_select[] = { - GNUNET_MY_query_param_auto_from_type (key), - GNUNET_MY_query_param_auto_from_type (vhash), - GNUNET_MY_query_param_uint32 (&type), - GNUNET_MY_query_param_uint64 (&next_uid), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_end - }; - - execute_select (plugin, - plugin->select_entry_by_hash_vhash_and_type, - proc, - proc_cls, - params_select); - } - else - { - struct GNUNET_MY_QueryParam params_select[] = { - GNUNET_MY_query_param_auto_from_type (key), - GNUNET_MY_query_param_uint32 (&type), - GNUNET_MY_query_param_uint64 (&next_uid), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_end - }; - - execute_select (plugin, - plugin->select_entry_by_hash_and_type, - proc, - proc_cls, - params_select); - } + struct GNUNET_MY_QueryParam params_select[] = { + GNUNET_MY_query_param_auto_from_type (key), + GNUNET_MY_query_param_uint32 (&type), + GNUNET_MY_query_param_uint64 (&next_uid), + GNUNET_MY_query_param_uint64 (&rvalue), + GNUNET_MY_query_param_uint64 (&rvalue), + GNUNET_MY_query_param_end + }; + + execute_select (plugin, + plugin->select_entry_by_hash_and_type, + proc, + proc_cls, + params_select); } else { - if (NULL != vhash) - { - struct GNUNET_MY_QueryParam params_select[] = { - GNUNET_MY_query_param_auto_from_type (key), - GNUNET_MY_query_param_auto_from_type (vhash), - GNUNET_MY_query_param_uint64 (&next_uid), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_end - }; - - execute_select (plugin, - plugin->select_entry_by_hash_and_vhash, - proc, - proc_cls, - params_select); - } - else - { - struct GNUNET_MY_QueryParam params_select[] = { - GNUNET_MY_query_param_auto_from_type (key), - GNUNET_MY_query_param_uint64 (&next_uid), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_uint64 (&rvalue), - GNUNET_MY_query_param_end - }; - - execute_select (plugin, - plugin->select_entry_by_hash, - proc, - proc_cls, - params_select); - } - } + struct GNUNET_MY_QueryParam params_select[] = { + GNUNET_MY_query_param_auto_from_type (key), + GNUNET_MY_query_param_uint64 (&next_uid), + GNUNET_MY_query_param_uint64 (&rvalue), + GNUNET_MY_query_param_uint64 (&rvalue), + GNUNET_MY_query_param_end + }; + execute_select (plugin, + plugin->select_entry_by_hash, + proc, + proc_cls, + params_select); + } } @@ -1097,6 +1035,69 @@ mysql_plugin_drop (void *cls) } +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +static void +mysql_plugin_remove_key (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls) +{ + struct Plugin *plugin = cls; + struct GNUNET_MY_QueryParam params_delete[] = { + GNUNET_MY_query_param_auto_from_type (key), + GNUNET_MY_query_param_fixed_size (data, size), + GNUNET_MY_query_param_end + }; + + if (GNUNET_OK != + GNUNET_MY_exec_prepared (plugin->mc, + plugin->delete_entry_by_hash_value, + params_delete)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Removing key `%s' from gn090 table failed\n", + GNUNET_h2s (key)); + cont (cont_cls, + key, + size, + GNUNET_SYSERR, + _("MySQL statement run failure")); + return; + } + + MYSQL_STMT *stmt = GNUNET_MYSQL_statement_get_stmt (plugin->delete_entry_by_hash_value); + my_ulonglong rows = mysql_stmt_affected_rows (stmt); + + if (0 == rows) + { + cont (cont_cls, + key, + size, + GNUNET_NO, + NULL); + return; + } + plugin->env->duc (plugin->env->cls, + -size); + cont (cont_cls, + key, + size, + GNUNET_OK, + NULL); +} + + /** * Entry point for the plugin. * @@ -1132,24 +1133,20 @@ libgnunet_plugin_datastore_mysql_init (void *cls) " hash BINARY(64) NOT NULL DEFAULT ''," " vhash BINARY(64) NOT NULL DEFAULT ''," " value BLOB NOT NULL DEFAULT ''," " uid BIGINT NOT NULL AUTO_INCREMENT," - " PRIMARY KEY (uid)," " INDEX idx_hash (hash(64))," - " INDEX idx_hash_uid (hash(64),uid)," - " INDEX idx_hash_vhash (hash(64),vhash(64))," + " PRIMARY KEY (uid)," " INDEX idx_hash_type_uid (hash(64),type,rvalue)," - " INDEX idx_prio (prio)," " INDEX idx_repl_rvalue (repl,rvalue)," + " INDEX idx_prio (prio)," + " INDEX idx_repl_rvalue (repl,rvalue)," " INDEX idx_expire (expire)," " INDEX idx_anonLevel_type_rvalue (anonLevel,type,rvalue)" ") ENGINE=InnoDB") || MRUNS ("SET AUTOCOMMIT = 1") || PINIT (plugin->insert_entry, INSERT_ENTRY) || PINIT (plugin->delete_entry_by_uid, DELETE_ENTRY_BY_UID) || + PINIT (plugin->delete_entry_by_hash_value, DELETE_ENTRY_BY_HASH_VALUE) || PINIT (plugin->select_entry, SELECT_ENTRY) || PINIT (plugin->select_entry_by_hash, SELECT_ENTRY_BY_HASH) || - PINIT (plugin->select_entry_by_hash_and_vhash, - SELECT_ENTRY_BY_HASH_AND_VHASH) || PINIT (plugin->select_entry_by_hash_and_type, SELECT_ENTRY_BY_HASH_AND_TYPE) || - PINIT (plugin->select_entry_by_hash_vhash_and_type, - SELECT_ENTRY_BY_HASH_VHASH_AND_TYPE) || PINIT (plugin->get_size, SELECT_SIZE) || PINIT (plugin->update_entry, UPDATE_ENTRY) || PINIT (plugin->dec_repl, DEC_REPL) || @@ -1158,7 +1155,8 @@ libgnunet_plugin_datastore_mysql_init (void *cls) PINIT (plugin->select_priority, SELECT_IT_PRIORITY) || PINIT (plugin->max_repl, SELECT_MAX_REPL) || PINIT (plugin->get_all_keys, GET_ALL_KEYS) || - PINIT (plugin->select_replication, SELECT_IT_REPLICATION)) + PINIT (plugin->select_replication, SELECT_IT_REPLICATION) || + false) { GNUNET_MYSQL_context_destroy (plugin->mc); GNUNET_free (plugin); @@ -1177,6 +1175,7 @@ libgnunet_plugin_datastore_mysql_init (void *cls) api->get_zero_anonymity = &mysql_plugin_get_zero_anonymity; api->get_keys = &mysql_plugin_get_keys; api->drop = &mysql_plugin_drop; + api->remove_key = &mysql_plugin_remove_key; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "mysql", _("Mysql database running\n")); return api; diff --git a/src/datastore/plugin_datastore_postgres.c b/src/datastore/plugin_datastore_postgres.c index 349848ae6..b6aeb0be6 100644 --- a/src/datastore/plugin_datastore_postgres.c +++ b/src/datastore/plugin_datastore_postgres.c @@ -117,9 +117,6 @@ init_connection (struct Plugin *plugin) if ((GNUNET_OK != GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)")) || - (GNUNET_OK != - GNUNET_POSTGRES_exec (plugin->dbh, - "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)")) || (GNUNET_OK != GNUNET_POSTGRES_exec (plugin->dbh, "CREATE INDEX IF NOT EXISTS idx_prio ON gn090 (prio)")) || @@ -183,9 +180,8 @@ init_connection (struct Plugin *plugin) "WHERE oid >= $1::bigint AND " "(rvalue >= $2 OR 0 = $3::smallint) AND " "(hash = $4 OR 0 = $5::smallint) AND " - "(vhash = $6 OR 0 = $7::smallint) AND " - "(type = $8 OR 0 = $9::smallint) " - "ORDER BY oid ASC LIMIT 1", 9)) || + "(type = $6 OR 0 = $7::smallint) " + "ORDER BY oid ASC LIMIT 1", 7)) || (GNUNET_OK != GNUNET_POSTGRES_prepare (plugin->dbh, "put", "INSERT INTO gn090 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " @@ -222,6 +218,10 @@ init_connection (struct Plugin *plugin) "ORDER BY repl DESC,RANDOM() LIMIT 1", 0)) || (GNUNET_OK != GNUNET_POSTGRES_prepare (plugin->dbh, "delrow", "DELETE FROM gn090 " "WHERE oid=$1", 1)) || + (GNUNET_OK != + GNUNET_POSTGRES_prepare (plugin->dbh, "remove", "DELETE FROM gn090 " + "WHERE hash = $1 AND " + "value = $2", 2)) || (GNUNET_OK != GNUNET_POSTGRES_prepare (plugin->dbh, "get_keys", "SELECT hash FROM gn090", 0))) { @@ -536,11 +536,6 @@ process_result (struct Plugin *plugin, * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key maybe NULL (to match all entries) - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. * @param proc function to call on the matching value; @@ -552,7 +547,6 @@ postgres_plugin_get_key (void *cls, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, - const struct GNUNET_HashCode *vhash, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls) @@ -561,7 +555,6 @@ postgres_plugin_get_key (void *cls, uint32_t utype = type; uint16_t use_rvalue = random; uint16_t use_key = NULL != key; - uint16_t use_vhash = NULL != vhash; uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type; uint64_t rvalue; struct GNUNET_PQ_QueryParam params[] = { @@ -570,8 +563,6 @@ postgres_plugin_get_key (void *cls, GNUNET_PQ_query_param_uint16 (&use_rvalue), GNUNET_PQ_query_param_auto_from_type (key), GNUNET_PQ_query_param_uint16 (&use_key), - GNUNET_PQ_query_param_auto_from_type (vhash), - GNUNET_PQ_query_param_uint16 (&use_vhash), GNUNET_PQ_query_param_uint32 (&utype), GNUNET_PQ_query_param_uint16 (&use_type), GNUNET_PQ_query_param_end @@ -853,6 +844,74 @@ postgres_plugin_drop (void *cls) } +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +static void +postgres_plugin_remove_key (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls) +{ + struct Plugin *plugin = cls; + PGresult *ret; + struct GNUNET_PQ_QueryParam params[] = { + GNUNET_PQ_query_param_auto_from_type (key), + GNUNET_PQ_query_param_fixed_size (data, size), + GNUNET_PQ_query_param_end + }; + ret = GNUNET_PQ_exec_prepared (plugin->dbh, + "remove", + params); + if (GNUNET_OK != + GNUNET_POSTGRES_check_result (plugin->dbh, + ret, + PGRES_COMMAND_OK, + "PQexecPrepared", + "remove")) + { + cont (cont_cls, + key, + size, + GNUNET_SYSERR, + _("Postgress exec failure")); + return; + } + /* What an awful API, this function really does return a string */ + bool affected = 0 != strcmp ("0", PQcmdTuples (ret)); + PQclear (ret); + if (!affected) + { + cont (cont_cls, + key, + size, + GNUNET_NO, + NULL); + return; + } + plugin->env->duc (plugin->env->cls, + - (size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); + GNUNET_log_from (GNUNET_ERROR_TYPE_DEBUG, + "datastore-postgres", + "Deleted %u bytes from database\n", + (unsigned int) size); + cont (cont_cls, + key, + size, + GNUNET_OK, + NULL); +} + + /** * Entry point for the plugin. * @@ -883,6 +942,7 @@ libgnunet_plugin_datastore_postgres_init (void *cls) api->get_zero_anonymity = &postgres_plugin_get_zero_anonymity; api->get_keys = &postgres_plugin_get_keys; api->drop = &postgres_plugin_drop; + api->remove_key = &postgres_plugin_remove_key; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "datastore-postgres", _("Postgres database running\n")); diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index 469dd7717..bcaf27d99 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c @@ -87,6 +87,11 @@ struct Plugin */ sqlite3 *dbh; + /** + * Precompiled SQL for remove_key. + */ + sqlite3_stmt *remove; + /** * Precompiled SQL for deletion. */ @@ -183,10 +188,6 @@ create_indices (sqlite3 * dbh) if ((SQLITE_OK != sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_hash ON gn090 (hash)", NULL, NULL, NULL)) || - (SQLITE_OK != - sqlite3_exec (dbh, - "CREATE INDEX IF NOT EXISTS idx_hash_vhash ON gn090 (hash,vhash)", - NULL, NULL, NULL)) || (SQLITE_OK != sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS idx_expire_repl ON gn090 (expire ASC,repl DESC)", @@ -415,15 +416,21 @@ database_setup (const struct GNUNET_CONFIGURATION_Handle *cfg, "WHERE _ROWID_ >= ? AND " "(rvalue >= ? OR 0 = ?) AND " "(hash = ? OR 0 = ?) AND " - "(vhash = ? OR 0 = ?) AND " "(type = ? OR 0 = ?) " "ORDER BY _ROWID_ ASC LIMIT 1", &plugin->get)) || (SQLITE_OK != sq_prepare (plugin->dbh, "DELETE FROM gn090 WHERE _ROWID_ = ?", - &plugin->delRow)) - ) + &plugin->delRow)) || + (SQLITE_OK != + sq_prepare (plugin->dbh, + "DELETE FROM gn090 " + "WHERE hash = ? AND " + "value = ? " + "LIMIT 1", + &plugin->remove)) || + false) { LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, @@ -448,6 +455,8 @@ database_shutdown (struct Plugin *plugin) sqlite3_stmt *stmt; #endif + if (NULL != plugin->remove) + sqlite3_finalize (plugin->remove); if (NULL != plugin->delRow) sqlite3_finalize (plugin->delRow); if (NULL != plugin->update) @@ -845,15 +854,10 @@ sqlite_plugin_get_zero_anonymity (void *cls, * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key maybe NULL (to match all entries) - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. - * @param proc function to call on each matching value; - * will be called once with a NULL value at the end + * @param proc function to call on the matching value; + * will be called with NULL if nothing matches * @param proc_cls closure for @a proc */ static void @@ -861,7 +865,6 @@ sqlite_plugin_get_key (void *cls, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, - const struct GNUNET_HashCode *vhash, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls) @@ -872,15 +875,12 @@ sqlite_plugin_get_key (void *cls, uint32_t type32 = (uint32_t) type; uint16_t use_type = GNUNET_BLOCK_TYPE_ANY != type; uint16_t use_key = NULL != key; - uint16_t use_vhash = NULL != vhash; struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&next_uid), GNUNET_SQ_query_param_uint64 (&rvalue), GNUNET_SQ_query_param_uint16 (&use_rvalue), GNUNET_SQ_query_param_auto_from_type (key), GNUNET_SQ_query_param_uint16 (&use_key), - GNUNET_SQ_query_param_auto_from_type (vhash), - GNUNET_SQ_query_param_uint16 (&use_vhash), GNUNET_SQ_query_param_uint32 (&type32), GNUNET_SQ_query_param_uint16 (&use_type), GNUNET_SQ_query_param_end @@ -1185,6 +1185,79 @@ sqlite_plugin_drop (void *cls) } +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +static void +sqlite_plugin_remove_key (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls) +{ + struct Plugin *plugin = cls; + struct GNUNET_SQ_QueryParam params[] = { + GNUNET_SQ_query_param_auto_from_type (key), + GNUNET_SQ_query_param_fixed_size (data, size), + GNUNET_SQ_query_param_end + }; + + if (GNUNET_OK != + GNUNET_SQ_bind (plugin->remove, + params)) + { + cont (cont_cls, + key, + size, + GNUNET_SYSERR, + "bind failed"); + return; + } + if (SQLITE_DONE != sqlite3_step (plugin->remove)) + { + LOG_SQLITE (plugin, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + GNUNET_SQ_reset (plugin->dbh, + plugin->remove); + cont (cont_cls, + key, + size, + GNUNET_SYSERR, + "sqlite3_step failed"); + return; + } + int changes = sqlite3_changes (plugin->dbh); + GNUNET_SQ_reset (plugin->dbh, + plugin->remove); + if (0 == changes) + { + cont (cont_cls, + key, + size, + GNUNET_NO, + NULL); + return; + } + if (NULL != plugin->env->duc) + plugin->env->duc (plugin->env->cls, + -(size + GNUNET_DATASTORE_ENTRY_OVERHEAD)); + cont (cont_cls, + key, + size, + GNUNET_OK, + NULL); +} + + /** * Get an estimate of how much space the database is * currently using. @@ -1286,6 +1359,7 @@ libgnunet_plugin_datastore_sqlite_init (void *cls) api->get_zero_anonymity = &sqlite_plugin_get_zero_anonymity; api->get_keys = &sqlite_plugin_get_keys; api->drop = &sqlite_plugin_drop; + api->remove_key = &sqlite_plugin_remove_key; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "sqlite", _("Sqlite database running\n")); diff --git a/src/datastore/plugin_datastore_template.c b/src/datastore/plugin_datastore_template.c index 704d586bc..16bda45d4 100644 --- a/src/datastore/plugin_datastore_template.c +++ b/src/datastore/plugin_datastore_template.c @@ -99,11 +99,6 @@ template_plugin_put (void *cls, * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key maybe NULL (to match all entries) - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. * @param proc function to call on each matching value; @@ -111,10 +106,12 @@ template_plugin_put (void *cls, * @param proc_cls closure for proc */ static void -template_plugin_get_key (void *cls, uint64_t next_uid, bool random, - const struct GNUNET_HashCode * key, - const struct GNUNET_HashCode * vhash, - enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, +template_plugin_get_key (void *cls, + uint64_t next_uid, + bool random, + const struct GNUNET_HashCode *key, + enum GNUNET_BLOCK_Type type, + PluginDatumProcessor proc, void *proc_cls) { GNUNET_break (0); @@ -203,6 +200,29 @@ template_get_keys (void *cls, } +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +static void +template_plugin_remove_key (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls) +{ + GNUNET_break (0); + cont (cont_cls, key, size, GNUNET_SYSERR, "not implemented"); +} + + /** * Entry point for the plugin. * @@ -228,6 +248,7 @@ libgnunet_plugin_datastore_template_init (void *cls) api->get_zero_anonymity = &template_plugin_get_zero_anonymity; api->drop = &template_plugin_drop; api->get_keys = &template_get_keys; + api->remove_key = &template_plugin_remove_key; GNUNET_log_from (GNUNET_ERROR_TYPE_INFO, "template", _("Template database running\n")); return api; diff --git a/src/datastore/test_plugin_datastore.c b/src/datastore/test_plugin_datastore.c index 0c34a5f66..d460daed7 100644 --- a/src/datastore/test_plugin_datastore.c +++ b/src/datastore/test_plugin_datastore.c @@ -52,6 +52,7 @@ enum RunPhase RP_ITER_ZERO, RP_REPL_GET, RP_EXPI_GET, + RP_REMOVE, RP_DROP }; @@ -153,7 +154,7 @@ do_put (struct CpsRunContext *crc) /* most content is 32k */ size = 32 * 1024; - if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + if (0 != i && GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); size = size - (size & 7); /* always multiple of 8 */ @@ -220,6 +221,25 @@ iterate_one_shot (void *cls, } +static void +remove_continuation (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + int status, + const char *msg) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (NULL != key); + GNUNET_assert (32768 == size); + GNUNET_assert (GNUNET_OK == status); + GNUNET_assert (NULL == msg); + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, + crc); +} + + /** * Function called when the service shuts * down. Unloads our datastore plugin. @@ -303,7 +323,6 @@ test (void *cls) 0, false, &key, - NULL, GNUNET_BLOCK_TYPE_ANY, &iterate_one_shot, crc); @@ -324,6 +343,23 @@ test (void *cls) case RP_EXPI_GET: crc->api->get_expiration (crc->api->cls, &iterate_one_shot, crc); break; + case RP_REMOVE: + { + struct GNUNET_HashCode key; + uint32_t size = 32768; + char value[size]; + + gen_key (0, &key); + memset (value, 0, size); + value[0] = crc->i; + crc->api->remove_key (crc->api->cls, + &key, + size, + value, + &remove_continuation, + crc); + break; + } case RP_DROP: crc->api->drop (crc->api->cls); GNUNET_SCHEDULER_add_now (&cleaning_task, crc); diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h index 28c8241b1..36a3fbec2 100644 --- a/src/include/gnunet_datastore_plugin.h +++ b/src/include/gnunet_datastore_plugin.h @@ -212,11 +212,6 @@ typedef void * @param next_uid return the result with lowest uid >= next_uid * @param random if true, return a random result instead of using next_uid * @param key maybe NULL (to match all entries) - * @param vhash hash of the value, maybe NULL (to - * match all values that have the right key). - * Note that for DBlocks there is no difference - * betwen key and vhash, but for other blocks - * there may be! * @param type entries of which type are relevant? * Use 0 for any type. * @param proc function to call on the matching value; @@ -228,12 +223,48 @@ typedef void uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, - const struct GNUNET_HashCode *vhash, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls); +/** + * Remove continuation. + * + * @param cls closure + * @param key key for the content removed + * @param size number of bytes removed + * @param status #GNUNET_OK if removed, #GNUNET_NO if not found, + * or #GNUNET_SYSERROR if error + * @param msg error message on error + */ +typedef void +(*PluginRemoveCont) (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + int status, + const char *msg); + + +/** + * Remove a particular key in the datastore. + * + * @param cls closure + * @param key key for the content + * @param size number of bytes in data + * @param data content stored + * @param cont continuation called with success or failure status + * @param cont_cls continuation closure for @a cont + */ +typedef void +(*PluginRemoveKey) (void *cls, + const struct GNUNET_HashCode *key, + uint32_t size, + const void *data, + PluginRemoveCont cont, + void *cont_cls); + + /** * Get a random item (additional constraints may apply depending on * the specific implementation). Calls @a proc with all values ZERO or @@ -339,6 +370,10 @@ struct GNUNET_DATASTORE_PluginFunctions */ PluginGetKeys get_keys; + /** + * Function to remove an item from the database. + */ + PluginRemoveKey remove_key; }; #endif -- cgit v1.2.3