From 7eb7bd8666aeb2e855cd22b1ea7f44b87bb60400 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 17 Apr 2018 14:42:46 +0200 Subject: use namestore API for zone import instead of using plugin directly --- src/include/gnunet_namestore_plugin.h | 29 ---- src/include/gnunet_namestore_service.h | 5 +- src/namestore/Makefile.am | 1 + src/namestore/gnunet-namestore.c | 3 + src/namestore/gnunet-zoneimport.c | 279 ++++++++++++++---------------- src/namestore/plugin_namestore_flat.c | 43 ----- src/namestore/plugin_namestore_postgres.c | 77 --------- src/namestore/plugin_namestore_sqlite.c | 44 ----- 8 files changed, 137 insertions(+), 344 deletions(-) diff --git a/src/include/gnunet_namestore_plugin.h b/src/include/gnunet_namestore_plugin.h index d1c68cd23..3ebf48987 100644 --- a/src/include/gnunet_namestore_plugin.h +++ b/src/include/gnunet_namestore_plugin.h @@ -145,35 +145,6 @@ struct GNUNET_NAMESTORE_PluginFunctions void *iter_cls); - /** - * Start a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_NO if transactions are not supported, - * #GNUNET_SYSERR on internal errors - */ - int - (*begin_transaction) (void *cls); - - - /** - * Try to commit a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ - int - (*commit_transaction) (void *cls); - - - /** - * Rollback a transaction. - * - * @param cls closure - */ - void - (*rollback_transaction) (void *cls); - }; diff --git a/src/include/gnunet_namestore_service.h b/src/include/gnunet_namestore_service.h index 6d3c07f1d..f8c2eaf3b 100644 --- a/src/include/gnunet_namestore_service.h +++ b/src/include/gnunet_namestore_service.h @@ -1,6 +1,6 @@ /* This file is part of GNUnet - Copyright (C) 2012, 2013 GNUnet e.V. + Copyright (C) 2012, 2013, 2018 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -128,7 +128,6 @@ GNUNET_NAMESTORE_records_store (struct GNUNET_NAMESTORE_Handle *h, void *cont_cls); - /** * Process a record that was stored in the namestore. * @@ -153,7 +152,7 @@ typedef void * @param pkey private key of the zone * @param nick the nick name to set * @param cont continuation to call when done - * @param cont_cls closure for 'cont' + * @param cont_cls closure for @a cont * @return handle to abort the request */ struct GNUNET_NAMESTORE_QueueEntry * diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am index 50b60d020..fd8f8054f 100644 --- a/src/namestore/Makefile.am +++ b/src/namestore/Makefile.am @@ -138,6 +138,7 @@ endif gnunet_zoneimport_SOURCES = \ gnunet-zoneimport.c gnunet_zoneimport_LDADD = \ + libgnunetnamestore.la \ $(top_builddir)/src/identity/libgnunetidentity.la \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ $(top_builddir)/src/dns/libgnunetdnsparser.la \ diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c index 2eff995e0..9a1805af4 100644 --- a/src/namestore/gnunet-namestore.c +++ b/src/namestore/gnunet-namestore.c @@ -394,6 +394,7 @@ display_record (void *cls, struct GNUNET_TIME_Relative rt; (void) cls; + (void) zone_key; if ( (NULL != name) && (0 != strcmp (name, rname)) ) { @@ -509,6 +510,7 @@ get_existing_record (void *cls, struct GNUNET_GNSRECORD_Data *rde; (void) cls; + (void) zone_key; add_qe = NULL; if (0 != strcmp (rec_name, name)) { @@ -656,6 +658,7 @@ handle_reverse_lookup (void *cls, const struct GNUNET_GNSRECORD_Data *rd) { (void) cls; + (void) zone; reverse_qe = NULL; if (NULL == label) FPRINTF (stdout, diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index 4d595e554..0148f42a7 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include @@ -163,6 +163,11 @@ struct Request */ char *label; + /** + * Namestore operation pending for this record. + */ + struct GNUNET_NAMESTORE_QueueEntry *qe; + /** * Zone responsible for this request. */ @@ -210,9 +215,9 @@ struct Request static struct GNUNET_IDENTITY_Handle *id; /** - * Namestore plugin. + * Namestore handle. */ -static struct GNUNET_NAMESTORE_PluginFunctions *ns; +static struct GNUNET_NAMESTORE_Handle *ns; /** * Context for DNS resolution. @@ -270,11 +275,6 @@ static struct GNUNET_SCHEDULER_Task *t; */ static char *dns_server; -/** - * Name of the database plugin (for loading/unloading). - */ -static char *db_lib_name; - /** * Head of list of zones we are managing. */ @@ -285,16 +285,6 @@ static struct Zone *zone_head; */ static struct Zone *zone_tail; -/** - * Set to #GNUNET_YES if we are currently in a DB transaction. - */ -static int in_transaction; - -/** - * Flag set if we should use transactions to batch DB operations. - */ -static int use_transactions; - /** * Callback for #for_all_records @@ -789,6 +779,41 @@ process_record (void *cls, } +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure with our `struct Request` + * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate) + * #GNUNET_NO if content was already there or not found + * #GNUNET_YES (or other positive value) on success + * @param emsg NULL on success, otherwise an error message + */ +static void +store_completed_cb (void *cls, + int32_t success, + const char *emsg) +{ + struct Request *req = cls; + + req->qe = NULL; + pending--; + if (GNUNET_SYSERR == success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to store zone data for `%s': %s\n", + req->hostname, + emsg); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Stored records under `%s'\n", + req->label); + } +} + + /** * Function called with the result of a DNS resolution. * @@ -835,7 +860,6 @@ process_result (void *cls, GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); - pending--; GNUNET_DNSSTUB_resolve_cancel (req->rs); req->rs = NULL; p = GNUNET_DNSPARSER_parse ((const char *) dns, @@ -849,9 +873,11 @@ process_result (void *cls, { failures++; insert_sorted (req); + pending--; return; } insert_sorted (req); + pending--; return; } /* Free old/legacy records */ @@ -899,34 +925,13 @@ process_result (void *cls, /* convert linked list into array */ for (rec = req->rec_head; NULL != rec; rec =rec->next) rd[off++] = rec->grd; - if ( (! in_transaction) && - (GNUNET_YES == use_transactions) ) - { - /* not all plugins support transactions, but if one does, - remember we need to eventually commit... */ - if (GNUNET_OK == - ns->begin_transaction (ns->cls)) - in_transaction = GNUNET_YES; - } - if (GNUNET_OK != - ns->store_records (ns->cls, - &req->zone->key, - req->label, - rd_count, - rd)) - { - if (0 != rd_count) - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to store zone data for `%s'\n", - req->hostname); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Stored %u records under `%s'\n", - (unsigned int) rd_count, - req->label); - } + req->qe = GNUNET_NAMESTORE_records_store (ns, + &req->zone->key, + req->label, + rd_count, + rd, + &store_completed_cb, + req); } insert_sorted (req); } @@ -947,6 +952,8 @@ submit_req (struct Request *req) static struct GNUNET_TIME_Absolute last_request; struct GNUNET_TIME_Absolute now; + if (NULL != req->qe) + return GNUNET_NO; /* namestore op still pending */ if (NULL != req->rs) { GNUNET_break (0); @@ -978,25 +985,6 @@ submit_req (struct Request *req) } -/** - * If we are currently in a transaction, commit it. - */ -static void -finish_transaction () -{ - if (! in_transaction) - return; - if (GNUNET_OK != - ns->commit_transaction (ns->cls)) - { - GNUNET_break (0); - GNUNET_SCHEDULER_shutdown (); - return; - } - in_transaction = GNUNET_NO; -} - - /** * Process as many requests as possible from the queue. * @@ -1006,7 +994,6 @@ static void process_queue (void *cls) { struct Request *req; - static unsigned int cnt; (void) cls; t = NULL; @@ -1033,7 +1020,6 @@ process_queue (void *cls) "Waiting until %s for next record (`%s') to expire\n", GNUNET_STRINGS_absolute_time_to_string (req->expires), req->hostname); - finish_transaction (); if (NULL != t) GNUNET_SCHEDULER_cancel (t); t = GNUNET_SCHEDULER_add_at (req->expires, @@ -1042,8 +1028,6 @@ process_queue (void *cls) } else { - if (0 == cnt++ % TRANSACTION_SYNC_FREQ) - finish_transaction (); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling for 1ms\n"); if (NULL != t) @@ -1077,14 +1061,10 @@ do_shutdown (void *cls) GNUNET_SCHEDULER_cancel (t); t = NULL; } - finish_transaction (); if (NULL != ns) { - GNUNET_break (NULL == - GNUNET_PLUGIN_unload (db_lib_name, - ns)); - GNUNET_free (db_lib_name); - db_lib_name = NULL; + GNUNET_NAMESTORE_disconnect (ns); + ns = NULL; } if (NULL != ctx) { @@ -1120,26 +1100,46 @@ do_shutdown (void *cls) /** - * Function called for each matching record. + * Function called if #GNUNET_NAMESTORE_records_lookup() failed. + * Continues resolution based on assumption namestore has no data. * - * @param cls `struct Request *` - * @param zone_key private key of the zone - * @param label name that is being mapped (at most 255 characters long) - * @param rd_count number of entries in @a rd array - * @param rd array of records with data to store + * @param cls a `struct Request` */ static void -import_records (void *cls, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key, - const char *label, - unsigned int rd_count, - const struct GNUNET_GNSRECORD_Data *rd) +ns_lookup_error_cb (void *cls) { struct Request *req = cls; - GNUNET_break (0 == memcmp (private_key, + req->qe = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to load data from namestore for `%s'\n", + req->label); + insert_sorted (req); +} + + +/** + * Process a record that was stored in the namestore. + * + * @param cls a `struct Request *` + * @param zone private key of the zone + * @param label label of the records + * @param rd_count number of entries in @a rd array, 0 if label was deleted + * @param rd array of records with data to store + */ +static void +ns_lookup_result_cb (void *cls, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, + const char *label, + unsigned int rd_count, + const struct GNUNET_GNSRECORD_Data *rd) +{ + struct Request *req = cls; + + req->qe = NULL; + GNUNET_break (0 == memcmp (zone, &req->zone->key, - sizeof (*private_key))); + sizeof (*zone))); GNUNET_break (0 == strcasecmp (label, req->label)); for (unsigned int i=0;idata, rd->data_size); } + if (0 == rd_count) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Empty record set in namestore for `%s'\n", + req->label); + } + else + { + unsigned int pos = 0; + + req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; + for (struct Record *rec = req->rec_head; + NULL != rec; + rec = rec->next) + { + struct GNUNET_TIME_Absolute at; + + at.abs_value_us = rec->grd.expiration_time; + req->expires = GNUNET_TIME_absolute_min (req->expires, + at); + pos++; + } + if (0 == pos) + req->expires = GNUNET_TIME_UNIT_ZERO_ABS; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Hot-start with %u existing records for `%s'\n", + pos, + req->label); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Adding `%s' to worklist to start at %s\n", + req->hostname, + GNUNET_STRINGS_absolute_time_to_string (req->expires)); + insert_sorted (req); } @@ -1236,45 +1270,13 @@ queue (const char *hostname) req->id = p.id; req->label = GNUNET_strndup (hostname, dot - hostname); - if (GNUNET_OK != - ns->lookup_records (ns->cls, - &req->zone->key, - req->label, - &import_records, - req)) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to load data from namestore for `%s'\n", - req->label); - } - else - { - unsigned int rd_count = 0; - - req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; - for (struct Record *rec = req->rec_head; - NULL != rec; - rec = rec->next) - { - struct GNUNET_TIME_Absolute at; - - at.abs_value_us = rec->grd.expiration_time; - req->expires = GNUNET_TIME_absolute_min (req->expires, - at); - rd_count++; - } - if (0 == rd_count) - req->expires = GNUNET_TIME_UNIT_ZERO_ABS; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Hot-start with %u existing records for `%s'\n", - rd_count, - req->label); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Adding `%s' to worklist to start at %s\n", - req->hostname, - GNUNET_STRINGS_absolute_time_to_string (req->expires)); - insert_sorted (req); + req->qe = GNUNET_NAMESTORE_records_lookup (ns, + &req->zone->key, + req->label, + &ns_lookup_error_cb, + req, + &ns_lookup_result_cb, + req); } @@ -1390,8 +1392,6 @@ run (void *cls, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) { - char *database; - (void) cls; (void) args; (void) cfgfile; @@ -1403,22 +1403,9 @@ run (void *cls, "Failed to initialize GNUnet DNS STUB\n"); return; } - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "namestore", - "database", - &database)) - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "No database backend configured\n"); - - GNUNET_asprintf (&db_lib_name, - "libgnunet_plugin_namestore_%s", - database); - ns = GNUNET_PLUGIN_load (db_lib_name, - (void *) cfg); - GNUNET_free (database); GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); + ns = GNUNET_NAMESTORE_connect (cfg); if (NULL == ns) { GNUNET_SCHEDULER_shutdown (); @@ -1448,10 +1435,6 @@ main (int argc, "IP", "which DNS server should be used", &dns_server)), - GNUNET_GETOPT_option_flag ('e', - "enable-transactions", - "enable use of transactions to reduce disk IO", - &use_transactions), GNUNET_GETOPT_OPTION_END }; diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c index 170adb49e..305fe7ba1 100644 --- a/src/namestore/plugin_namestore_flat.c +++ b/src/namestore/plugin_namestore_flat.c @@ -666,46 +666,6 @@ namestore_flat_zone_to_name (void *cls, } -/** - * Start a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_NO if transactions are not supported, - * #GNUNET_SYSERR on internal errors - */ -static int -namestore_flat_begin_transaction (void *cls) -{ - return GNUNET_NO; -} - - -/** - * Try to commit a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ -static int -namestore_flat_commit_transaction (void *cls) -{ - GNUNET_break (0); - return GNUNET_SYSERR; -} - - -/** - * Rollback a transaction. - * - * @param cls closure - */ -static void -namestore_flat_rollback_transaction (void *cls) -{ - GNUNET_break (0); -} - - /** * Entry point for the plugin. * @@ -736,9 +696,6 @@ libgnunet_plugin_namestore_flat_init (void *cls) api->iterate_records = &namestore_flat_iterate_records; api->zone_to_name = &namestore_flat_zone_to_name; api->lookup_records = &namestore_flat_lookup_records; - api->begin_transaction = &namestore_flat_begin_transaction; - api->commit_transaction = &namestore_flat_commit_transaction; - api->rollback_transaction = &namestore_flat_rollback_transaction; GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("flat file database running\n")); return api; diff --git a/src/namestore/plugin_namestore_postgres.c b/src/namestore/plugin_namestore_postgres.c index 378a88f51..a9c19d517 100644 --- a/src/namestore/plugin_namestore_postgres.c +++ b/src/namestore/plugin_namestore_postgres.c @@ -517,80 +517,6 @@ database_shutdown (struct Plugin *plugin) } -/** - * Start a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_NO if transactions are not supported, - * #GNUNET_SYSERR on internal errors - */ -static int -namestore_postgres_begin_transaction (void *cls) -{ - struct Plugin *plugin = cls; - PGresult *result; - ExecStatusType ex; - - result = PQexec (plugin->dbh, - "START TRANSACTION ISOLATION LEVEL SERIALIZABLE"); - if (PGRES_COMMAND_OK != - (ex = PQresultStatus (result))) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to start transaction (%s): %s\n", - PQresStatus (ex), - PQerrorMessage (plugin->dbh)); - GNUNET_break (0); - PQclear (result); - return GNUNET_SYSERR; - } - PQclear (result); - return GNUNET_OK; -} - - -/** - * Try to commit a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ -static int -namestore_postgres_commit_transaction (void *cls) -{ - struct Plugin *plugin = cls; - PGresult *result; - ExecStatusType status; - int ret; - - result = PQexec (plugin->dbh, - "COMMIT"); - status = PQresultStatus (result); - ret = (PGRES_COMMAND_OK == status) ? GNUNET_OK : GNUNET_SYSERR; - PQclear (result); - return ret; -} - - -/** - * Rollback a transaction. - * - * @param cls closure - */ -static void -namestore_postgres_rollback_transaction (void *cls) -{ - struct Plugin *plugin = cls; - PGresult *result; - - result = PQexec (plugin->dbh, - "ROLLBACK"); - GNUNET_break (PGRES_COMMAND_OK == - PQresultStatus (result)); - PQclear (result); -} - - /** * Entry point for the plugin. * @@ -619,9 +545,6 @@ libgnunet_plugin_namestore_postgres_init (void *cls) api->iterate_records = &namestore_postgres_iterate_records; api->zone_to_name = &namestore_postgres_zone_to_name; api->lookup_records = &namestore_postgres_lookup_records; - api->begin_transaction = &namestore_postgres_begin_transaction; - api->commit_transaction = &namestore_postgres_commit_transaction; - api->rollback_transaction = &namestore_postgres_rollback_transaction; LOG (GNUNET_ERROR_TYPE_INFO, "Postgres namestore plugin running\n"); return api; diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c index 1ebb6bfc7..5ad84688c 100644 --- a/src/namestore/plugin_namestore_sqlite.c +++ b/src/namestore/plugin_namestore_sqlite.c @@ -772,47 +772,6 @@ namestore_sqlite_zone_to_name (void *cls, } -/** - * Start a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_NO if transactions are not supported, - * #GNUNET_SYSERR on internal errors - */ -static int -namestore_sqlite_begin_transaction (void *cls) -{ - return GNUNET_NO; -} - - -/** - * Try to commit a transaction. - * - * @param cls closure - * @return #GNUNET_OK on success, #GNUNET_SYSERR on failure - */ -static int -namestore_sqlite_commit_transaction (void *cls) -{ - GNUNET_break (0); - return GNUNET_SYSERR; -} - - -/** - * Rollback a transaction. - * - * @param cls closure - */ -static void -namestore_sqlite_rollback_transaction (void *cls) -{ - GNUNET_break (0); -} - - - /** * Entry point for the plugin. * @@ -841,9 +800,6 @@ libgnunet_plugin_namestore_sqlite_init (void *cls) api->iterate_records = &namestore_sqlite_iterate_records; api->zone_to_name = &namestore_sqlite_zone_to_name; api->lookup_records = &namestore_sqlite_lookup_records; - api->begin_transaction = &namestore_sqlite_begin_transaction; - api->commit_transaction = &namestore_sqlite_commit_transaction; - api->rollback_transaction = &namestore_sqlite_rollback_transaction; LOG (GNUNET_ERROR_TYPE_INFO, _("Sqlite database running\n")); return api; -- cgit v1.2.3