misc zoneimporter fixes

4 files changed, 173 insertions, 117 deletions

diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c
index 6f5f2d952..e0f64a6b0 100644
--- a/src/namecache/plugin_namecache_sqlite.c
+++ b/src/namecache/plugin_namecache_sqlite.c
@@ -136,13 +136,16 @@ create_indices (sqlite3 * dbh)
 {
   /* create indices */
   if ( (SQLITE_OK !=
-        sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)",
+        sqlite3_exec (dbh,
+                      "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)",
                       NULL, NULL, NULL)) ||
        (SQLITE_OK !=
-        sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)",
+        sqlite3_exec (dbh,
+                      "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)",
                       NULL, NULL, NULL)) )
     LOG (GNUNET_ERROR_TYPE_ERROR,
-         "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
+         "Failed to create indices: %s\n",
+         sqlite3_errmsg (dbh));
 }
 
 
@@ -208,28 +211,44 @@ database_setup (struct Plugin *plugin)
     return GNUNET_SYSERR;
   }
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA temp_store=MEMORY",
+                       NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA synchronous=NORMAL",
+                       NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA legacy_file_format=OFF",
+                       NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
-                       NULL, ENULL));
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA auto_vacuum=INCREMENTAL",
+                       NULL, NULL,
+                       ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL,
-                       NULL, ENULL));
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA encoding=\"UTF-8\"",
+                       NULL, NULL,
+                       ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA locking_mode=EXCLUSIVE",
+                       NULL, NULL,
                        ENULL));
   CHECK (SQLITE_OK ==
-         sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
+         sqlite3_exec (plugin->dbh,
+                       "PRAGMA page_size=4092",
+                       NULL, NULL,
                        ENULL));
 
-  CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
+  CHECK (SQLITE_OK ==
+         sqlite3_busy_timeout (plugin->dbh,
+                               BUSY_TIMEOUT_MS));
 
 
   /* Create tables */
@@ -237,17 +256,19 @@ database_setup (struct Plugin *plugin)
          sq_prepare (plugin->dbh,
                      "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns096blocks'",
                      &stmt));
-  if ((sqlite3_step (stmt) == SQLITE_DONE) &&
-      (sqlite3_exec
-       (plugin->dbh,
-        "CREATE TABLE ns096blocks ("
-        " query BLOB NOT NULL,"
-        " block BLOB NOT NULL,"
-        " expiration_time INT8 NOT NULL"
-        ")",
-        NULL, NULL, NULL) != SQLITE_OK))
+  if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
+       (SQLITE_OK !=
+        sqlite3_exec (plugin->dbh,
+                      "CREATE TABLE ns096blocks ("
+                      " query BLOB NOT NULL,"
+                      " block BLOB NOT NULL,"
+                      " expiration_time INT8 NOT NULL"
+                      ")",
+                      NULL, NULL, NULL)) )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+    LOG_SQLITE (plugin,
+                GNUNET_ERROR_TYPE_ERROR,
+                "sqlite3_exec");
     sqlite3_finalize (stmt);
     return GNUNET_SYSERR;
   }
@@ -411,11 +432,11 @@ namecache_sqlite_cache_block (void *cls,
   GNUNET_CRYPTO_hash (&block->derived_key,
                       sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
                       &query);
-  fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-           "Caching new version of block %s (expires %s)\n",
-           GNUNET_h2s (&query),
-           GNUNET_STRINGS_absolute_time_to_string (expiration));
   expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Caching new version of block %s (expires %s)\n",
+              GNUNET_h2s (&query),
+              GNUNET_STRINGS_absolute_time_to_string (expiration));
   if (block_size > 64 * 65536)
   {
     GNUNET_break (0);
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index 994eae2c8..06a50132b 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -708,11 +708,15 @@ refresh_block (struct NamestoreClient *nc,
                              &res);
     GNUNET_free (nick);
   }
-
-  exp_time = (0 == res_count)
-    ? GNUNET_TIME_UNIT_ZERO_ABS
-    : GNUNET_GNSRECORD_record_get_expiration_time (res_count,
-                                                   res);
+  if (0 == res_count)
+  {
+    send_store_response (nc,
+                         GNUNET_OK,
+                         rid);
+    return; /* no data, no need to update cache */
+  }
+  exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
+                                                          res);
   if (cache_keys)
     block = GNUNET_GNSRECORD_block_create2 (zone_key,
                                             exp_time,
diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c
index 6622a7676..503262487 100644
--- a/src/namestore/gnunet-zoneimport.c
+++ b/src/namestore/gnunet-zoneimport.c
@@ -48,6 +48,16 @@
 #define MAX_RETRIES 5
 
 /**
+ * How many DNS requests do we at most issue in rapid series?
+ */
+#define MAX_SERIES 10
+
+/**
+ * How long do we wait at least between series of requests?
+ */
+#define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
+
+/**
  * How many requests do we request from NAMESTORE in one batch
  * during our initial iteration?
  */
@@ -200,11 +210,16 @@ static struct GNUNET_NAMESTORE_Handle *ns;
 static struct GNUNET_DNSSTUB_Context *ctx;
 
 /**
- * The number of queries that are outstanding
+ * The number of DNS queries that are outstanding
  */
 static unsigned int pending;
 
 /**
+ * The number of NAMESTORE record store operations that are outstanding
+ */
+static unsigned int pending_rs;
+
+/**
  * Number of lookups we performed overall.
  */
 static unsigned int lookups;
@@ -692,7 +707,7 @@ process_record (void *cls,
                        req->hostname))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "DNS returned record for `%s' of type %u while resolving `%s'\n",
+                "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
                 rec->name,
                 (unsigned int) rec->type,
                 req->hostname);
@@ -910,7 +925,11 @@ store_completed_cb (void *cls,
   struct Request *req = cls;
 
   req->qe = NULL;
-  pending--;
+  pending_rs--;
+  if (NULL == t)
+    t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+                                      &process_queue,
+                                      NULL);
   if (GNUNET_SYSERR == success)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -967,6 +986,10 @@ process_result (void *cls,
                                  req_tail,
                                  req);
     pending--;
+    if (NULL == t)
+      t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+                                        &process_queue,
+                                        NULL);
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Stub gave up on DNS reply for `%s'\n",
                 req->hostname);
@@ -981,12 +1004,17 @@ process_result (void *cls,
     return;
   }
   if (req->id != dns->id)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "DNS ID did not match request, ignoring reply\n");
     return;
+  }
   GNUNET_CONTAINER_DLL_remove (req_head,
                                req_tail,
                                req);
   GNUNET_DNSSTUB_resolve_cancel (req->rs);
   req->rs = NULL;
+  pending--;
   p = GNUNET_DNSPARSER_parse ((const char *) dns,
                               dns_len);
   if (NULL == p)
@@ -998,11 +1026,17 @@ process_result (void *cls,
     {
       failures++;
       insert_sorted (req);
-      pending--;
+      if (NULL == t)
+        t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+                                          &process_queue,
+                                          NULL);
       return;
     }
     insert_sorted (req);
-    pending--;
+    if (NULL == t)
+      t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+                                        &process_queue,
+                                        NULL);
     return;
   }
   /* import new records */
@@ -1047,6 +1081,7 @@ process_result (void *cls,
     /* convert linked list into array */
     for (rec = req->rec_head; NULL != rec; rec =rec->next)
       rd[off++] = rec->grd;
+    pending_rs++;
     req->qe = GNUNET_NAMESTORE_records_store (ns,
                                               &req->zone->key,
                                               get_label (req),
@@ -1054,69 +1089,13 @@ process_result (void *cls,
                                               rd,
                                               &store_completed_cb,
                                               req);
+    GNUNET_assert (NULL != req->qe);
   }
   insert_sorted (req);
 }
 
 
 /**
- * Submit a request to DNS unless we need to slow down because
- * we are at the rate limit.
- *
- * @param req request to submit
- * @return #GNUNET_OK if request was submitted
- *         #GNUNET_NO if request was already submitted
- *         #GNUNET_SYSERR if we are at the rate limit
- */
-static int
-submit_req (struct Request *req)
-{
-  static struct GNUNET_TIME_Absolute last_request;
-  struct GNUNET_TIME_Absolute now;
-  void *raw;
-  size_t raw_size;
-
-  if (NULL != req->qe)
-    return GNUNET_NO; /* namestore op still pending */
-  if (NULL != req->rs)
-  {
-    GNUNET_break (0);
-    return GNUNET_NO; /* already submitted */
-  }
-  now = GNUNET_TIME_absolute_get ();
-  if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
-       (pending >= THRESH) )
-    return GNUNET_SYSERR;
-  GNUNET_CONTAINER_DLL_insert (req_head,
-                               req_tail,
-                               req);
-  GNUNET_assert (NULL == req->rs);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Requesting resolution for `%s'\n",
-              req->hostname);
-  raw = build_dns_query (req,
-                         &raw_size);
-  if (NULL == raw)
-  {
-    GNUNET_break (0);
-    free_request (req);
-    return GNUNET_SYSERR;
-  }
-  req->rs = GNUNET_DNSSTUB_resolve (ctx,
-                                    raw,
-                                    raw_size,
-                                    &process_result,
-                                    req);
-  GNUNET_assert (NULL != req->rs);
-  req->issue_num++;
-  last_request = now;
-  lookups++;
-  pending++;
-  return GNUNET_OK;
-}
-
-
-/**
  * Process as many requests as possible from the queue.
  *
  * @param cls NULL
@@ -1125,23 +1104,60 @@ static void
 process_queue (void *cls)
 {
   struct Request *req;
+  unsigned int series;
+  void *raw;
+  size_t raw_size;
 
   (void) cls;
+  series = 0;
   t = NULL;
-  while (1)
+  while (pending + pending_rs < THRESH)
   {
     req = GNUNET_CONTAINER_heap_peek (req_heap);
     if (NULL == req)
       break;
+    if (NULL != req->qe)
+      return; /* namestore op still pending */
+    if (NULL != req->rs)
+    {
+      GNUNET_break (0);
+      return; /* already submitted */
+    }
     if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
       break;
-    if (GNUNET_OK != submit_req (req))
-      break;
     GNUNET_assert (req ==
                    GNUNET_CONTAINER_heap_remove_root (req_heap));
     req->hn = NULL;
+    GNUNET_CONTAINER_DLL_insert (req_head,
+                                 req_tail,
+                                 req);
+    GNUNET_assert (NULL == req->rs);
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                "Requesting resolution for `%s'\n",
+                req->hostname);
+    raw = build_dns_query (req,
+                           &raw_size);
+    if (NULL == raw)
+    {
+      GNUNET_break (0);
+      free_request (req);
+      continue;
+    }
+    req->rs = GNUNET_DNSSTUB_resolve (ctx,
+                                      raw,
+                                      raw_size,
+                                      &process_result,
+                                      req);
+    GNUNET_assert (NULL != req->rs);
+    req->issue_num++;
+    lookups++;
+    pending++;
+    series++;
+    if (series > MAX_SERIES)
+      break;
   }
-
+  if (pending + pending_rs >= THRESH)
+    return; /* wait for replies */
   req = GNUNET_CONTAINER_heap_peek (req_heap);
   if (NULL == req)
     return;
@@ -1156,17 +1172,15 @@ process_queue (void *cls)
     t = GNUNET_SCHEDULER_add_at (req->expires,
                                  &process_queue,
                                  NULL);
+    return;
   }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Throttling for 1ms\n");
-    if (NULL != t)
-      GNUNET_SCHEDULER_cancel (t);
-    t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
-                                      &process_queue,
-                                      NULL);
-  }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Throttling\n");
+  if (NULL != t)
+    GNUNET_SCHEDULER_cancel (t);
+  t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+                                    &process_queue,
+                                    NULL);
 }
 
 
@@ -1512,7 +1526,6 @@ iterate_zones (void *cls)
   GNUNET_assert (NULL != zone_tail);
   if (zone_tail == last)
   {
-    GNUNET_assert (NULL == t);
     /* Done iterating over relevant zones in NAMESTORE, move
        rest of hash map to work queue as well. */
     GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
@@ -1750,12 +1763,13 @@ main (int argc,
                       NULL);
   GNUNET_free ((void*) argv);
   fprintf (stderr,
-           "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n",
+           "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u/%u pending on shutdown\n",
            rejects,
            lookups,
            records,
            failures,
-           pending);
+           pending,
+           pending_rs);
   return 0;
 }
 
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index f40f25c3f..518d5f572 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -181,6 +181,13 @@ static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 static int first_zone_iteration;
 
 /**
+ * Optimize block insertion by caching map of private keys to
+ * public keys in memory?
+ */
+static int cache_keys;
+
+
+/**
  * Task run during shutdown.
  *
  * @param cls unused
@@ -388,11 +395,18 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 
   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
                                                         rd_public);
-  block = GNUNET_GNSRECORD_block_create (key,
-                                         expire,
-                                         label,
-                                         rd_public,
-                                         rd_public_count);
+  if (cache_keys)
+    block = GNUNET_GNSRECORD_block_create2 (key,
+                                            expire,
+                                            label,
+                                            rd_public,
+                                            rd_public_count);
+  else
+    block = GNUNET_GNSRECORD_block_create (key,
+                                           expire,
+                                           label,
+                                           rd_public,
+                                           rd_public_count);
   if (NULL == block)
   {
     GNUNET_break (0);
@@ -585,6 +599,7 @@ put_gns_record (void *cls,
 static void
 publish_zone_dht_start (void *cls)
 {
+  (void) cls;
   zone_publish_task = NULL;
   GNUNET_STATISTICS_update (statistics,
                             "Full zone iterations launched",
@@ -736,7 +751,9 @@ run (void *cls,
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-
+  cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
+                                                     "namestore",
+                                                     "CACHE_KEYS");
   put_interval = INITIAL_PUT_INTERVAL;
   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
   if (GNUNET_OK ==