12 files changed, 178 insertions, 47 deletions

diff --git a/contrib/timeout_watchdog.c b/contrib/timeout_watchdog.c
index 7a0376b9f..fc61a7cc7 100644
--- a/contrib/timeout_watchdog.c
+++ b/contrib/timeout_watchdog.c
@@ -33,12 +33,14 @@
 
 static pid_t child;
 
+
 static void
 sigchld_handler (int val)
 {
   int status = 0;
   int ret = 0;
 
+  (void) val;
   waitpid (child, &status, 0);
   if (WIFEXITED (status) != 0)
     {
@@ -53,6 +55,7 @@ sigchld_handler (int val)
   exit (ret);
 }
 
+
 static void
 sigint_handler (int val)
 {
@@ -60,8 +63,10 @@ sigint_handler (int val)
   exit (val);
 }
 
+
 int
-main (int argc, char *argv[])
+main (int argc,
+      char *argv[])
 {
   int timeout = 0;
   pid_t gpid = 0;
diff --git a/src/credential/gnunet-service-credential.c b/src/credential/gnunet-service-credential.c
index a2c339363..399371a2e 100644
--- a/src/credential/gnunet-service-credential.c
+++ b/src/credential/gnunet-service-credential.c
@@ -961,9 +961,11 @@ collect_next (void *cls)
   struct VerifyRequestHandle *vrh = cls;
   vrh->collect_next_task = NULL;
   GNUNET_assert (NULL != vrh->cred_collection_iter);
-  GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter);
+  GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter,
+                                       1);
 }
 
+
 static void
 handle_cred_collection_cb (void *cls,
                            const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
diff --git a/src/gns/gns.conf.in b/src/gns/gns.conf.in
index d48a213e9..2e49a4c60 100644
--- a/src/gns/gns.conf.in
+++ b/src/gns/gns.conf.in
@@ -22,9 +22,6 @@ MAX_PARALLEL_BACKGROUND_QUERIES = 1000
 # called via NSS or other mechanisms).
 INTERCEPT_DNS = YES
 
-# Using caching (start with namestore), or always ask DHT?
-USE_CACHE = YES
-
 # PREFIX = valgrind --leak-check=full --track-origins=yes
 
 # Zones
diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c
index 90cd47e1d..c376ddfcc 100644
--- a/src/gns/gnunet-service-gns.c
+++ b/src/gns/gnunet-service-gns.c
@@ -229,14 +229,14 @@ GNS_get_tld (const char *name)
 /**
  * Task run during shutdown.
  *
- * @param cls unused
- * @param tc unused
+ * @param cls unused, NULL
  */
 static void
 shutdown_task (void *cls)
 {
   struct GNS_TopLevelDomain *tld;
 
+  (void) cls;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Shutting down!\n");
   GNS_interceptor_done ();
@@ -283,6 +283,7 @@ client_disconnect_cb (void *cls,
   struct ClientLookupHandle *clh;
   struct GnsClient *gc = app_ctx;
 
+  (void) cls;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client %p disconnected\n",
               client);
@@ -314,6 +315,8 @@ client_connect_cb (void *cls,
                    struct GNUNET_MQ_Handle *mq)
 {
   struct GnsClient *gc;
+
+  (void) cls;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Client %p connected\n",
               client);
@@ -383,6 +386,7 @@ check_lookup (void *cls,
   size_t msg_size;
   const char* name;
 
+  (void) cls;
   msg_size = ntohs (l_msg->header.size);
   if (msg_size < sizeof (struct LookupMessage))
   {
@@ -479,6 +483,8 @@ read_service_conf (void *cls,
   struct GNUNET_CRYPTO_EcdsaPublicKey pk;
   struct GNS_TopLevelDomain *tld;
 
+  (void) cls;
+  (void) section;
   if (option[0] != '.')
     return;
   if (GNUNET_OK !=
diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c
index 745a2f3bd..92e03bc69 100644
--- a/src/gns/gnunet-service-gns_resolver.c
+++ b/src/gns/gnunet-service-gns_resolver.c
@@ -501,7 +501,7 @@ static struct CacheOps *co_tail;
 /**
  * Use namecache
  */
-static int use_cache;
+static int disable_cache;
 
 /**
  * Global configuration.
@@ -2478,7 +2478,7 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh)
   GNUNET_GNSRECORD_query_from_public_key (&ac->authority_info.gns_authority,
                                           ac->label,
                                           &query);
-  if (GNUNET_YES == use_cache)
+  if (GNUNET_YES != disable_cache)
   {
     rh->namecache_qe
       = GNUNET_NAMECACHE_lookup_block (namecache_handle,
@@ -2489,7 +2489,8 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh)
   }
   else
   {
-    start_dht_request (rh, &query);
+    start_dht_request (rh,
+                       &query);
   }
 }
 
@@ -2816,13 +2817,11 @@ GNS_resolver_init (struct GNUNET_NAMECACHE_Handle *nc,
   dht_lookup_heap =
     GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
   max_allowed_background_queries = max_bg_queries;
-  if (GNUNET_SYSERR == (use_cache =
-                        GNUNET_CONFIGURATION_get_value_yesno (c,
-                                                              "gns",
-                                                              "USE_CACHE")))
-    use_cache = GNUNET_YES;
-  if (GNUNET_NO == use_cache)
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+  disable_cache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                                        "namecache",
+                                                        "DISABLE");
+  if (GNUNET_YES == disable_cache)
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Namecache disabled\n");
   vpn_handle = GNUNET_VPN_connect (cfg);
 }
diff --git a/src/identity-provider/gnunet-service-identity-provider.c b/src/identity-provider/gnunet-service-identity-provider.c
index 572202c3f..6afb7bb05 100644
--- a/src/identity-provider/gnunet-service-identity-provider.c
+++ b/src/identity-provider/gnunet-service-identity-provider.c
@@ -2190,12 +2190,15 @@ attr_iter_cb (void *cls,
 
   if (rd_count != 1)
   {
-    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+                                         1);
     return;
   }
 
-  if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type) {
-    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+  if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type)
+  {
+    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+                                         1);
     return;
   }
   attr_ver = ntohl(*((uint32_t*)rd->data));
@@ -2209,8 +2212,10 @@ attr_iter_cb (void *cls,
                                             rd->data_size-sizeof (uint32_t),
                                             key,
                                             (void**)&attr_ser);
-  if (GNUNET_SYSERR == msg_extra_len) {
-    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+  if (GNUNET_SYSERR == msg_extra_len)
+  {
+    GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+                                         1);
     return;
   }
 
@@ -2253,13 +2258,15 @@ iterate_after_abe_bootstrap (void *cls,
                                                      ai);
 }
 
-void
+
+static void
 iterate_next_after_abe_bootstrap (void *cls,
                                   struct GNUNET_ABE_AbeMasterKey *abe_key)
 {
   struct AttributeIterator *ai = cls;
   ai->abe_key = abe_key;
-  GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it);
+  GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it,
+                                       1);
 }
 
 
diff --git a/src/identity-provider/plugin_rest_openid_connect.c b/src/identity-provider/plugin_rest_openid_connect.c
index 7e8054e60..125bb98f6 100644
--- a/src/identity-provider/plugin_rest_openid_connect.c
+++ b/src/identity-provider/plugin_rest_openid_connect.c
@@ -1076,16 +1076,19 @@ namestore_iteration_callback (
     }
   }
 
-  GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it);
+  GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it,
+                                       1);
 }
 
+
 /**
  * Iteration over all results finished, build final
  * response.
  *
  * @param cls the `struct RequestHandle`
  */
-static void namestore_iteration_finished (void *cls)
+static void
+namestore_iteration_finished (void *cls)
 {
   struct RequestHandle *handle = cls;
   struct GNUNET_HashCode cache_key;
diff --git a/src/namecache/namecache.conf.in b/src/namecache/namecache.conf.in
index 236cafecd..becd34187 100644
--- a/src/namecache/namecache.conf.in
+++ b/src/namecache/namecache.conf.in
@@ -11,6 +11,11 @@ ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
 DATABASE = sqlite
 
+# Disables use of caching by GNS. Useful for systems that
+# publish very large zones and are CPU bound, if they do not
+# also do a large number of lookups.
+DISABLE = NO
+
 [namecache-sqlite]
 FILENAME = $GNUNET_DATA_HOME/namecache/sqlite.db
 
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 06a50132b..b9061ed9f 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -257,6 +257,12 @@ static struct GNUNET_NotificationContext *monitor_nc;
  */
 static int cache_keys;
 
+/**
+ * Use the namecache? Doing so creates additional cryptographic
+ * operations whenever we touch a record.
+ */
+static int disable_namecache;
+
 
 /**
  * Task run during shutdown.
@@ -281,8 +287,11 @@ cleanup_task (void *cls)
                                  cop);
     GNUNET_free (cop);
   }
-  GNUNET_NAMECACHE_disconnect (namecache);
-  namecache = NULL;
+  if (NULL != namecache)
+  {
+    GNUNET_NAMECACHE_disconnect (namecache);
+    namecache = NULL;
+  }
   GNUNET_break (NULL == GNUNET_PLUGIN_unload (db_lib_name,
                                               GSN_database));
   GNUNET_free (db_lib_name);
@@ -715,6 +724,14 @@ refresh_block (struct NamestoreClient *nc,
                          rid);
     return; /* no data, no need to update cache */
   }
+  if (GNUNET_YES == disable_namecache)
+  {
+    GNUNET_STATISTICS_update (statistics,
+                              "Namecache updates skipped (NC disabled)",
+                              1,
+                              GNUNET_NO);
+    return;
+  }
   exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
                                                           res);
   if (cache_keys)
@@ -1692,13 +1709,20 @@ run (void *cls,
   (void) service;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Starting namestore service\n");
-  GSN_cfg = cfg;
-  monitor_nc = GNUNET_notification_context_create (1);
-  namecache = GNUNET_NAMECACHE_connect (cfg);
-  /* Loading database plugin */
   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg,
                                                      "namestore",
                                                      "CACHE_KEYS");
+  disable_namecache = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                                            "namecache",
+                                                            "DISABLE");
+  GSN_cfg = cfg;
+  monitor_nc = GNUNET_notification_context_create (1);
+  if (GNUNET_NO == disable_namecache)
+  {
+    namecache = GNUNET_NAMECACHE_connect (cfg);
+    GNUNET_assert (NULL != namecache);
+  }
+  /* Loading database plugin */
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
                                              "namestore",
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index 503262487..55779e06a 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -951,10 +951,10 @@ store_completed_cb (void *cls,
 
       delta = GNUNET_TIME_absolute_get_duration (last);
       last = GNUNET_TIME_absolute_get ();
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Processed 1000 records in %s\n",
-                  GNUNET_STRINGS_relative_time_to_string (delta,
-                                                          GNUNET_YES));
+      fprintf (stderr,
+               "Processed 1000 records in %s\n",
+               GNUNET_STRINGS_relative_time_to_string (delta,
+                                                       GNUNET_YES));
     }
   }
   free_records (req);
@@ -1590,10 +1590,10 @@ process_stdin (void *cls)
 
       delta = GNUNET_TIME_absolute_get_duration (last);
       last = GNUNET_TIME_absolute_get ();
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Imported 1000 records in %s\n",
-                  GNUNET_STRINGS_relative_time_to_string (delta,
-                                                          GNUNET_YES));
+      fprintf (stderr,
+               "Imported 1000 records in %s\n",
+               GNUNET_STRINGS_relative_time_to_string (delta,
+                                                       GNUNET_YES));
     }
     queue (hn);
   }
diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c
index e9ed1cc4f..f061ab7d1 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -583,7 +583,7 @@ iterate_zones (void *cls,
   (void) key;
   if (0 == ic->limit)
     return GNUNET_NO;
-  if ( (NULL != it->zone) &&
+  if ( (NULL != ic->zone) &&
        (0 != memcmp (entry->private_key,
                      ic->zone,
                      sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) )
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index 518d5f572..25baf4396 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -272,6 +272,14 @@ publish_zone_dht_start (void *cls);
 
 
 /**
+ * How often do we measure the delta between desired zone
+ * iteration speed and actual speed, and tell statistics
+ * service about it?
+ */
+#define DELTA_INTERVAL 100
+
+
+/**
  * Continuation called from DHT once the PUT operation is done.
  *
  * @param cls closure, NULL if called from regular iteration,
@@ -283,7 +291,11 @@ dht_put_continuation (void *cls,
                       int success)
 {
   struct MonitorActivity *ma = cls;
+  static unsigned long long put_cnt;
+  static struct GNUNET_TIME_Absolute last_put_100;
+  static struct GNUNET_TIME_Relative sub_delta;
   struct GNUNET_TIME_Relative next_put_interval;
+  struct GNUNET_TIME_Relative delay;
 
   num_public_records++;
   if (NULL == ma)
@@ -300,20 +312,91 @@ dht_put_continuation (void *cls,
                                                        LATE_ITERATION_SPEEDUP_FACTOR);
     }
     else
+    {
       next_put_interval = put_interval;
+    }
     next_put_interval = GNUNET_TIME_relative_min (next_put_interval,
                                                   MAXIMUM_ZONE_ITERATION_INTERVAL);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "PUT complete, next PUT in %s!\n",
                 GNUNET_STRINGS_relative_time_to_string (next_put_interval,
                                                         GNUNET_YES));
-
-    GNUNET_STATISTICS_set (statistics,
-                           "Current zone iteration interval (ms)",
-                           next_put_interval.rel_value_us / 1000LL,
-                           GNUNET_NO);
+    /* compute velocities and delay corrections to apply */
+    if (0 == put_cnt)
+      last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
+    put_cnt++;
+    if (0 == put_cnt % DELTA_INTERVAL)
+    {
+      struct GNUNET_TIME_Relative delta;
+      unsigned long long pct = 0;
+          
+      /* How fast were we really? */
+      delta = GNUNET_TIME_absolute_get_duration (last_put_100);
+      delta.rel_value_us /= DELTA_INTERVAL;
+      last_put_100 = GNUNET_TIME_absolute_get ();
+      /* Tell statistics actual vs. desired speed */
+      GNUNET_STATISTICS_set (statistics,
+                             "Target zone iteration velocity (μs)",
+                             next_put_interval.rel_value_us,
+                             GNUNET_NO);
+      GNUNET_STATISTICS_set (statistics,
+                             "Current zone iteration velocity (μs)",
+                             delta.rel_value_us,
+                             GNUNET_NO);
+      /* update "sub_delta" based on difference, taking
+         previous sub_delta into account! */
+      if (next_put_interval.rel_value_us > delta.rel_value_us)
+      {
+        /* We were too fast, reduce sub_delta! */
+        struct GNUNET_TIME_Relative corr;
+
+        corr = GNUNET_TIME_relative_subtract (next_put_interval,
+                                              delta);
+        if (sub_delta.rel_value_us > delta.rel_value_us)
+        {
+          /* Reduce sub_delta by corr */
+          sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
+                                                     corr);
+        }
+        else
+        {
+          /* We're doing fine with waiting the full time, this
+             should theoretically only happen if we run at
+             infinite speed. */
+          sub_delta = GNUNET_TIME_UNIT_ZERO;
+        }
+      }
+      else if (next_put_interval.rel_value_us < delta.rel_value_us)
+      {
+        /* We were too slow, increase sub_delta! */
+        struct GNUNET_TIME_Relative corr;
+
+        corr = GNUNET_TIME_relative_subtract (delta,
+                                              next_put_interval);
+        sub_delta = GNUNET_TIME_relative_add (sub_delta,
+                                              corr);
+        if (sub_delta.rel_value_us > next_put_interval.rel_value_us)
+        {
+          /* CPU overload detected, we cannot go at desired speed,
+             as this would mean using a negative delay. */
+          sub_delta = next_put_interval;
+          /* compute how much faster we would want to be for
+             the desired velocity */
+          if (0 == next_put_interval.rel_value_us)
+            pct = UINT64_MAX; /* desired speed is infinity ... */
+          else
+            pct = sub_delta.rel_value_us * 100 / next_put_interval.rel_value_us;
+        }
+      }
+      GNUNET_STATISTICS_set (statistics,
+                             "% speed increase needed for target velocity",
+                             pct,
+                             GNUNET_NO);
+    } /* end of periodic velocity calculations */
+    delay = GNUNET_TIME_relative_subtract (next_put_interval,
+                                           sub_delta);
     GNUNET_assert (NULL == zone_publish_task);
-    zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
+    zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
                                                       &publish_zone_dht_next,
                                                       NULL);
   }