From 4fe4c850dec0aff53510de7fdd7fce5fa870ae40 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 10 May 2018 11:24:16 +0200 Subject: separate zonemaster monitoring functionality from main iterate-and-publish operation --- src/zonemaster/.gitignore | 1 + src/zonemaster/Makefile.am | 17 +- src/zonemaster/gnunet-service-zonemaster-monitor.c | 484 +++++++++++++++++++++ src/zonemaster/gnunet-service-zonemaster.c | 198 +-------- src/zonemaster/zonemaster.conf.in | 17 + 5 files changed, 519 insertions(+), 198 deletions(-) create mode 100644 src/zonemaster/gnunet-service-zonemaster-monitor.c diff --git a/src/zonemaster/.gitignore b/src/zonemaster/.gitignore index 1a0e22b66..cde57fa0a 100644 --- a/src/zonemaster/.gitignore +++ b/src/zonemaster/.gitignore @@ -1 +1,2 @@ gnunet-service-zonemaster +gnunet-service-zonemaster-monitor diff --git a/src/zonemaster/Makefile.am b/src/zonemaster/Makefile.am index 21f986498..b655b9ef3 100644 --- a/src/zonemaster/Makefile.am +++ b/src/zonemaster/Makefile.am @@ -1,5 +1,5 @@ # This Makefile.am is in the public domain -AM_CPPFLAGS = -I$(top_srcdir)/src/include +AM_CPPFLAGS = -I$(top_srcdir)/src/include plugindir = $(libdir)/gnunet @@ -20,11 +20,11 @@ if USE_COVERAGE endif libexec_PROGRAMS = \ - gnunet-service-zonemaster + gnunet-service-zonemaster \ + gnunet-service-zonemaster-monitor gnunet_service_zonemaster_SOURCES = \ gnunet-service-zonemaster.c - gnunet_service_zonemaster_LDADD = \ $(top_builddir)/src/dht/libgnunetdht.la \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ @@ -33,3 +33,14 @@ gnunet_service_zonemaster_LDADD = \ $(top_builddir)/src/namestore/libgnunetnamestore.la \ $(GN_LIBINTL) + +gnunet_service_zonemaster_monitor_SOURCES = \ + gnunet-service-zonemaster-monitor.c + +gnunet_service_zonemaster_monitor_LDADD = \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + $(top_builddir)/src/statistics/libgnunetstatistics.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/namestore/libgnunetnamestore.la \ + $(GN_LIBINTL) diff --git a/src/zonemaster/gnunet-service-zonemaster-monitor.c b/src/zonemaster/gnunet-service-zonemaster-monitor.c new file mode 100644 index 000000000..a0c178511 --- /dev/null +++ b/src/zonemaster/gnunet-service-zonemaster-monitor.c @@ -0,0 +1,484 @@ +/* + This file is part of GNUnet. + Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ + +/** + * @file zonemaster/gnunet-service-zonemaster-monitor.c + * @brief monitor namestore changes and publish them immediately to GNUnet name system + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_util_lib.h" +#include "gnunet_dht_service.h" +#include "gnunet_namestore_service.h" +#include "gnunet_statistics_service.h" + + +#define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename) + + +/** + * How often should we (re)publish each record before + * it expires? + */ +#define PUBLISH_OPS_PER_EXPIRATION 4 + +/** + * How many pending DHT operations do we allow at most? + */ +#define DHT_QUEUE_LIMIT 2000 + +/** + * How many events may the namestore give us before it has to wait + * for us to keep up? + */ +#define NAMESTORE_QUEUE_LIMIT 50 + +/** + * What replication level do we use for DHT PUT operations? + */ +#define DHT_GNS_REPLICATION_LEVEL 5 + + +/** + * Handle for DHT PUT activity triggered from the namestore monitor. + */ +struct DhtPutActivity +{ + /** + * Kept in a DLL. + */ + struct DhtPutActivity *next; + + /** + * Kept in a DLL. + */ + struct DhtPutActivity *prev; + + /** + * Handle for the DHT PUT operation. + */ + struct GNUNET_DHT_PutHandle *ph; + + /** + * When was this PUT initiated? + */ + struct GNUNET_TIME_Absolute start_date; +}; + + +/** + * Handle to the statistics service + */ +static struct GNUNET_STATISTICS_Handle *statistics; + +/** + * Our handle to the DHT + */ +static struct GNUNET_DHT_Handle *dht_handle; + +/** + * Our handle to the namestore service + */ +static struct GNUNET_NAMESTORE_Handle *namestore_handle; + +/** + * Handle to monitor namestore changes to instant propagation. + */ +static struct GNUNET_NAMESTORE_ZoneMonitor *zmon; + +/** + * Head of monitor activities; kept in a DLL. + */ +static struct DhtPutActivity *ma_head; + +/** + * Tail of monitor activities; kept in a DLL. + */ +static struct DhtPutActivity *ma_tail; + +/** + * Number of entries in the DHT queue #ma_head. + */ +static unsigned int ma_queue_length; + +/** + * Optimize block insertion by caching map of private keys to + * public keys in memory? + */ +static int cache_keys; + + +/** + * Task run during shutdown. + * + * @param cls unused + * @param tc unused + */ +static void +shutdown_task (void *cls) +{ + struct DhtPutActivity *ma; + + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Shutting down!\n"); + while (NULL != (ma = ma_head)) + { + GNUNET_DHT_put_cancel (ma->ph); + ma_queue_length--; + GNUNET_CONTAINER_DLL_remove (ma_head, + ma_tail, + ma); + GNUNET_free (ma); + } + if (NULL != statistics) + { + GNUNET_STATISTICS_destroy (statistics, + GNUNET_NO); + statistics = NULL; + } + if (NULL != zmon) + { + GNUNET_NAMESTORE_zone_monitor_stop (zmon); + zmon = NULL; + } + if (NULL != namestore_handle) + { + GNUNET_NAMESTORE_disconnect (namestore_handle); + namestore_handle = NULL; + } + if (NULL != dht_handle) + { + GNUNET_DHT_disconnect (dht_handle); + dht_handle = NULL; + } +} + + +/** + * Continuation called from DHT once the PUT operation triggered + * by a monitor is done. + * + * @param cls a `struct DhtPutActivity` + */ +static void +dht_put_monitor_continuation (void *cls) +{ + struct DhtPutActivity *ma = cls; + + GNUNET_NAMESTORE_zone_monitor_next (zmon, + 1); + ma_queue_length--; + GNUNET_CONTAINER_DLL_remove (ma_head, + ma_tail, + ma); + GNUNET_free (ma); +} + + +/** + * Convert namestore records from the internal format to that + * suitable for publication (removes private records, converts + * to absolute expiration time). + * + * @param rd input records + * @param rd_count size of the @a rd and @a rd_public arrays + * @param rd_public where to write the converted records + * @return number of records written to @a rd_public + */ +static unsigned int +convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd, + unsigned int rd_count, + struct GNUNET_GNSRECORD_Data *rd_public) +{ + struct GNUNET_TIME_Absolute now; + unsigned int rd_public_count; + + rd_public_count = 0; + now = GNUNET_TIME_absolute_get (); + for (unsigned int i=0;ipurpose.size) + + sizeof (struct GNUNET_CRYPTO_EcdsaSignature) + + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey); + GNUNET_GNSRECORD_query_from_private_key (key, + label, + &query); + GNUNET_STATISTICS_update (statistics, + "DHT put operations initiated", + 1, + GNUNET_NO); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n", + rd_public_count, + label, + GNUNET_STRINGS_absolute_time_to_string (expire), + GNUNET_h2s (&query)); + ret = GNUNET_DHT_put (dht_handle, + &query, + DHT_GNS_REPLICATION_LEVEL, + GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE, + GNUNET_BLOCK_TYPE_GNS_NAMERECORD, + block_size, + block, + expire, + cont, + cont_cls); + GNUNET_free (block); + return ret; +} + + +/** + * Process a record that was stored in the namestore + * (invoked by the monitor). + * + * @param cls closure, NULL + * @param zone private key of the zone; NULL on disconnect + * @param label label of the records; NULL on disconnect + * @param rd_count number of entries in @a rd array, 0 if label was deleted + * @param rd array of records with data to store + */ +static void +handle_monitor_event (void *cls, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, + const char *label, + unsigned int rd_count, + const struct GNUNET_GNSRECORD_Data *rd) +{ + struct GNUNET_GNSRECORD_Data rd_public[rd_count]; + unsigned int rd_public_count; + struct DhtPutActivity *ma; + + (void) cls; + GNUNET_STATISTICS_update (statistics, + "Namestore monitor events received", + 1, + GNUNET_NO); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received %u records for label `%s' via namestore monitor\n", + rd_count, + label); + /* filter out records that are not public, and convert to + absolute expiration time. */ + rd_public_count = convert_records_for_export (rd, + rd_count, + rd_public); + if (0 == rd_public_count) + { + GNUNET_NAMESTORE_zone_monitor_next (zmon, + 1); + return; /* nothing to do */ + } + ma = GNUNET_new (struct DhtPutActivity); + ma->start_date = GNUNET_TIME_absolute_get (); + ma->ph = perform_dht_put (zone, + label, + rd, + rd_count, + &dht_put_monitor_continuation, + ma); + if (NULL == ma->ph) + { + /* PUT failed, do not remember operation */ + GNUNET_free (ma); + GNUNET_NAMESTORE_zone_monitor_next (zmon, + 1); + return; + } + GNUNET_CONTAINER_DLL_insert_tail (ma_head, + ma_tail, + ma); + ma_queue_length++; + if (ma_queue_length > DHT_QUEUE_LIMIT) + { + ma = ma_head; + GNUNET_CONTAINER_DLL_remove (ma_head, + ma_tail, + ma); + GNUNET_DHT_put_cancel (ma->ph); + ma_queue_length--; + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "DHT PUT unconfirmed after %s, aborting PUT\n", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date), + GNUNET_YES)); + GNUNET_free (ma); + } +} + + +/** + * The zone monitor encountered an IPC error trying to to get in + * sync. Restart from the beginning. + * + * @param cls NULL + */ +static void +handle_monitor_error (void *cls) +{ + (void) cls; + GNUNET_STATISTICS_update (statistics, + "Namestore monitor errors encountered", + 1, + GNUNET_NO); +} + + +/** + * Performe zonemaster duties: watch namestore, publish records. + * + * @param cls closure + * @param server the initialized server + * @param c configuration to use + */ +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *c, + struct GNUNET_SERVICE_Handle *service) +{ + unsigned long long max_parallel_bg_queries = 128; + + (void) cls; + (void) service; + namestore_handle = GNUNET_NAMESTORE_connect (c); + if (NULL == namestore_handle) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to connect to the namestore!\n")); + GNUNET_SCHEDULER_shutdown (); + return; + } + cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c, + "namestore", + "CACHE_KEYS"); + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (c, + "zonemaster", + "MAX_PARALLEL_BACKGROUND_QUERIES", + &max_parallel_bg_queries)) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Number of allowed parallel background queries: %llu\n", + max_parallel_bg_queries); + } + if (0 == max_parallel_bg_queries) + max_parallel_bg_queries = 1; + dht_handle = GNUNET_DHT_connect (c, + (unsigned int) max_parallel_bg_queries); + if (NULL == dht_handle) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Could not connect to DHT!\n")); + GNUNET_SCHEDULER_add_now (&shutdown_task, + NULL); + return; + } + + /* Schedule periodic put for our records. */ + statistics = GNUNET_STATISTICS_create ("zonemaster-mon", + c); + zmon = GNUNET_NAMESTORE_zone_monitor_start (c, + NULL, + GNUNET_NO, + &handle_monitor_error, + NULL, + &handle_monitor_event, + NULL, + NULL /* sync_cb */, + NULL); + GNUNET_NAMESTORE_zone_monitor_next (zmon, + NAMESTORE_QUEUE_LIMIT - 1); + GNUNET_break (NULL != zmon); + GNUNET_SCHEDULER_add_shutdown (&shutdown_task, + NULL); +} + + +/** + * Define "main" method using service macro. + */ +GNUNET_SERVICE_MAIN +("zonemaster-monitor", + GNUNET_SERVICE_OPTION_NONE, + &run, + NULL, + NULL, + NULL, + GNUNET_MQ_handler_end()); + + +/* end of gnunet-service-zonemaster-monitor.c */ diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index 8220e6eab..482d3a203 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - Copyright (C) 2012, 2013, 2014, 2017 GNUnet e.V. + Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -29,8 +29,6 @@ #include "gnunet_dht_service.h" #include "gnunet_namestore_service.h" #include "gnunet_statistics_service.h" -#include "gnunet_namestore_plugin.h" -#include "gnunet_signatures.h" #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename) @@ -89,11 +87,6 @@ */ #define LATE_ITERATION_SPEEDUP_FACTOR 2 -/** - * How long until a DHT PUT attempt should time out? - */ -#define DHT_OPERATION_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60) - /** * What replication level do we use for DHT PUT operations? */ @@ -147,21 +140,6 @@ static struct GNUNET_NAMESTORE_Handle *namestore_handle; */ static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter; -/** - * Handle to monitor namestore changes to instant propagation. - */ -static struct GNUNET_NAMESTORE_ZoneMonitor *zmon; - -/** - * Head of monitor activities; kept in a DLL. - */ -static struct DhtPutActivity *ma_head; - -/** - * Tail of monitor activities; kept in a DLL. - */ -static struct DhtPutActivity *ma_tail; - /** * Head of iteration put activities; kept in a DLL. */ @@ -281,15 +259,6 @@ shutdown_task (void *cls) (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutting down!\n"); - while (NULL != (ma = ma_head)) - { - GNUNET_DHT_put_cancel (ma->ph); - ma_queue_length--; - GNUNET_CONTAINER_DLL_remove (ma_head, - ma_tail, - ma); - GNUNET_free (ma); - } while (NULL != (ma = it_head)) { GNUNET_DHT_put_cancel (ma->ph); @@ -316,11 +285,6 @@ shutdown_task (void *cls) GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter); namestore_iter = NULL; } - if (NULL != zmon) - { - GNUNET_NAMESTORE_zone_monitor_stop (zmon); - zmon = NULL; - } if (NULL != namestore_handle) { GNUNET_NAMESTORE_disconnect (namestore_handle); @@ -361,27 +325,6 @@ static void publish_zone_dht_start (void *cls); -/** - * Continuation called from DHT once the PUT operation triggered - * by a monitor is done. - * - * @param cls a `struct DhtPutActivity` - */ -static void -dht_put_monitor_continuation (void *cls) -{ - struct DhtPutActivity *ma = cls; - - GNUNET_NAMESTORE_zone_monitor_next (zmon, - 1); - ma_queue_length--; - GNUNET_CONTAINER_DLL_remove (ma_head, - ma_tail, - ma); - GNUNET_free (ma); -} - - /** * Calculate #target_iteration_velocity_per_record. */ @@ -883,131 +826,6 @@ publish_zone_dht_start (void *cls) } -/** - * Process a record that was stored in the namestore - * (invoked by the monitor). - * - * @param cls closure, NULL - * @param zone private key of the zone; NULL on disconnect - * @param label label of the records; NULL on disconnect - * @param rd_count number of entries in @a rd array, 0 if label was deleted - * @param rd array of records with data to store - */ -static void -handle_monitor_event (void *cls, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, - const char *label, - unsigned int rd_count, - const struct GNUNET_GNSRECORD_Data *rd) -{ - struct GNUNET_GNSRECORD_Data rd_public[rd_count]; - unsigned int rd_public_count; - struct DhtPutActivity *ma; - - (void) cls; - GNUNET_STATISTICS_update (statistics, - "Namestore monitor events received", - 1, - GNUNET_NO); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received %u records for label `%s' via namestore monitor\n", - rd_count, - label); - /* filter out records that are not public, and convert to - absolute expiration time. */ - rd_public_count = convert_records_for_export (rd, - rd_count, - rd_public); - if (0 == rd_public_count) - { - GNUNET_NAMESTORE_zone_monitor_next (zmon, - 1); - return; /* nothing to do */ - } - num_public_records++; - ma = GNUNET_new (struct DhtPutActivity); - ma->start_date = GNUNET_TIME_absolute_get (); - ma->ph = perform_dht_put (zone, - label, - rd, - rd_count, - &dht_put_monitor_continuation, - ma); - if (NULL == ma->ph) - { - /* PUT failed, do not remember operation */ - GNUNET_free (ma); - GNUNET_NAMESTORE_zone_monitor_next (zmon, - 1); - return; - } - GNUNET_CONTAINER_DLL_insert_tail (ma_head, - ma_tail, - ma); - ma_queue_length++; - if (ma_queue_length > DHT_QUEUE_LIMIT) - { - ma = ma_head; - GNUNET_CONTAINER_DLL_remove (ma_head, - ma_tail, - ma); - GNUNET_DHT_put_cancel (ma->ph); - ma_queue_length--; - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "DHT PUT unconfirmed after %s, aborting PUT\n", - GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date), - GNUNET_YES)); - GNUNET_free (ma); - } -} - - -/** - * The zone monitor is now in SYNC with the current state of the - * name store. Start to perform periodic iterations. - * - * @param cls NULL - */ -static void -monitor_sync_event (void *cls) -{ - (void) cls; - if ( (NULL == zone_publish_task) && - (NULL == namestore_iter) ) - zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, - NULL); -} - - -/** - * The zone monitor encountered an IPC error trying to to get in - * sync. Restart from the beginning. - * - * @param cls NULL - */ -static void -handle_monitor_error (void *cls) -{ - (void) cls; - GNUNET_STATISTICS_update (statistics, - "Namestore monitor errors encountered", - 1, - GNUNET_NO); - if (NULL != zone_publish_task) - { - GNUNET_SCHEDULER_cancel (zone_publish_task); - zone_publish_task = NULL; - } - if (NULL != namestore_iter) - { - GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter); - namestore_iter = NULL; - } - zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, - NULL); -} - - /** * Performe zonemaster duties: watch namestore, publish records. * @@ -1084,18 +902,8 @@ run (void *cls, "Target zone iteration velocity (μs)", target_iteration_velocity_per_record.rel_value_us, GNUNET_NO); - zmon = GNUNET_NAMESTORE_zone_monitor_start (c, - NULL, - GNUNET_NO, - &handle_monitor_error, - NULL, - &handle_monitor_event, - NULL, - &monitor_sync_event, - NULL); - GNUNET_NAMESTORE_zone_monitor_next (zmon, - NAMESTORE_QUEUE_LIMIT - 1); - GNUNET_break (NULL != zmon); + zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start, + NULL); GNUNET_SCHEDULER_add_shutdown (&shutdown_task, NULL); } diff --git a/src/zonemaster/zonemaster.conf.in b/src/zonemaster/zonemaster.conf.in index 871eb9c8f..6179b87be 100644 --- a/src/zonemaster/zonemaster.conf.in +++ b/src/zonemaster/zonemaster.conf.in @@ -23,3 +23,20 @@ ZONE_PUBLISH_TIME_WINDOW = 4 h # USE_CACHE = YES # PREFIX = valgrind --leak-check=full --track-origins=yes + + + +[zonemaster-monitor] +AUTOSTART = @AUTOSTART@ +FORCESTART = YES +HOSTNAME = localhost +BINARY = gnunet-service-zonemaster-monitor +UNIXPATH = $GNUNET_USER_RUNTIME_DIR/gnunet-service-zonemaster-monitor.sock +@JAVAPORT@PORT = 2124 + +# Do we require users that want to access GNS to run this process +# (usually not a good idea) +UNIX_MATCH_UID = NO + +# Do we require users that want to access GNS to be in the 'gnunet' group? +UNIX_MATCH_GID = NO -- cgit v1.2.3