-fix notification processing zonemaster; give zonemaster some time in GNS tests; fix GNS tests for new gnunet.org IP

6 files changed, 100 insertions, 84 deletions

diff --git a/src/gns/test_gns_dht_lookup.sh b/src/gns/test_gns_dht_lookup.sh
index 8d446c507..da87d8477 100755
--- a/src/gns/test_gns_dht_lookup.sh
+++ b/src/gns/test_gns_dht_lookup.sh
@@ -40,14 +40,12 @@ gnunet-arm -i zonemaster -c test_gns_lookup.conf
 #gnunet-arm -r -c test_gns_lookup.conf
 #gnunet-arm -i zonemaster
 #gnunet-arm -i gns -c test_gns_lookup.conf
-gnunet-arm -I -c test_gns_lookup.conf
 #gnunet-identity -D $OTHER_EGO -c test_gns_lookup.conf
 #gnunet-namestore -z $MY_EGO -d -n b -t PKEY -V $DELEGATED_PKEY  -e never -c test_gns_lookup.conf
 #gnunet-namestore -z $OTHER_EGO -d -n www -t A -V $TEST_IP  -e never -c test_gns_lookup.conf
 RES_IP=`$DO_TIMEOUT gnunet-gns --raw -u www.b.$MY_EGO -t A -c test_gns_lookup.conf`
 RES_IP_REL=`$DO_TIMEOUT gnunet-gns --raw -u www2.b.$MY_EGO -t A -c test_gns_lookup.conf`
 #gnunet-namestore -z $MY_EGO -d -n b -t PKEY -V $DELEGATED_PKEY  -e never -c test_gns_lookup.conf
-gnunet-arm -I -c test_gns_lookup.conf
 gnunet-arm -e -c test_gns_lookup.conf
 rm -rf `gnunet-config -c test_gns_lookup.conf -f -s paths -o GNUNET_TEST_HOME`
 
diff --git a/src/gns/test_gns_gns2dns_lookup.sh b/src/gns/test_gns_gns2dns_lookup.sh
index 9419330e6..43a4756d3 100755
--- a/src/gns/test_gns_gns2dns_lookup.sh
+++ b/src/gns/test_gns_gns2dns_lookup.sh
@@ -18,8 +18,6 @@ rm -rf `gnunet-config -c test_gns_lookup.conf -f -s paths -o GNUNET_TEST_HOME`
 TEST_IP_ALT2="147.87.255.218"
 # IP address of 'www.gnunet.org'
 TEST_IP="147.87.255.218"
-# IP address of 'gnunet.org'
-TEST_IP_ALT="131.159.74.67"
 # IPv6 address of 'gnunet.org'
 TEST_IP6="2a07:6b47:100:464::9357:ffdb"
 # permissive DNS resolver we will use for the test
@@ -66,6 +64,8 @@ gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_NAME -t GNS2DNS -V $TEST_RECOR
 
 sleep 1
 
+gnunet-gns -u $TEST_RECORD_NAME.$MY_EGO -t GNS2DNS -c test_gns_lookup.conf
+
 # lookup 'www.gnunet.org', IPv4
 RES_IP=`$DO_TIMEOUT gnunet-gns --raw -u $TEST_DOMAIN -t A -c test_gns_lookup.conf`
 # lookup 'www.gnunet.org', IPv6
@@ -99,7 +99,7 @@ else
   ret=1
 fi
 
-if echo "$RES_IP_ALT" | grep "$TEST_IP_ALT" > /dev/null
+if echo "$RES_IP_ALT" | grep "$TEST_IP" > /dev/null
 then
   echo "PASS: Resolved $TEST_DOMAIN_ALT to $RES_IP_ALT."
 else
diff --git a/src/gns/test_gns_gns2dns_zkey_lookup.sh b/src/gns/test_gns_gns2dns_zkey_lookup.sh
index ee3f3a748..03549314e 100755
--- a/src/gns/test_gns_gns2dns_zkey_lookup.sh
+++ b/src/gns/test_gns_gns2dns_zkey_lookup.sh
@@ -18,8 +18,6 @@ rm -rf `gnunet-config -c test_gns_lookup.conf -f -s paths -o GNUNET_TEST_HOME`
 TEST_IP_ALT2="147.87.255.218"
 # IP address of 'www.gnunet.org'
 TEST_IP="147.87.255.218"
-# IP address of 'www.gnunet.org'
-TEST_IP_ALT="131.159.74.67"
 # IPv6 address of 'gnunet.org'
 TEST_IP6="2a07:6b47:100:464::9357:ffdb"
 # permissive DNS resolver we will use for the test
@@ -66,6 +64,7 @@ gnunet-namestore -p -z $MY_EGO -a -n $TEST_RESOLVER_LABEL -t A -V $TEST_IP_GNS2D
 gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_NAME -t GNS2DNS -V $TEST_RECORD_GNS2DNS -e never -c test_gns_lookup.conf
 
 sleep 1
+
 # lookup 'www.gnunet.org', IPv4
 RES_IP=`$DO_TIMEOUT gnunet-gns --raw -u $TEST_DOMAIN -t A -c test_gns_lookup.conf`
 # lookup 'www.gnunet.org', IPv6
@@ -99,7 +98,7 @@ else
   ret=1
 fi
 
-if echo "$RES_IP_ALT" | grep "$TEST_IP_ALT" > /dev/null
+if echo "$RES_IP_ALT" | grep "$TEST_IP" > /dev/null
 then
   echo "PASS: Resolved $TEST_DOMAIN_ALT to $RES_IP_ALT."
 else
diff --git a/src/gns/test_gns_redirect_lookup.sh b/src/gns/test_gns_redirect_lookup.sh
index 4ee5c0e3e..90729713d 100755
--- a/src/gns/test_gns_redirect_lookup.sh
+++ b/src/gns/test_gns_redirect_lookup.sh
@@ -26,7 +26,7 @@ fi
 rm -rf `gnunet-config -c test_gns_lookup.conf -f -s paths -o GNUNET_TEST_HOME`
 
 TEST_IP_PLUS="127.0.0.1"
-TEST_IP_DNS="131.159.74.67"
+TEST_IP_DNS="147.87.255.218"
 TEST_RECORD_REDIRECT_SERVER="server"
 TEST_RECORD_REDIRECT_PLUS="server.+"
 TEST_RECORD_REDIRECT_DNS="gnunet.org"
@@ -48,7 +48,7 @@ gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_NAME_DNS -t REDIRECT -V $TEST_
 gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_NAME_PLUS -t REDIRECT -V $TEST_RECORD_REDIRECT_PLUS -e never -c test_gns_lookup.conf
 gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_NAME_ZKEY -t REDIRECT -V $TEST_RECORD_REDIRECT_ZKEY -e never -c test_gns_lookup.conf
 gnunet-namestore -p -z $MY_EGO -a -n $TEST_RECORD_REDIRECT_SERVER -t A -V $TEST_IP_PLUS -e never -c test_gns_lookup.conf
-sleep 3
+sleep 1
 RES_REDIRECT=`$DO_TIMEOUT gnunet-gns --raw -u $TEST_DOMAIN_PLUS -t A -c test_gns_lookup.conf`
 RES_REDIRECT_RAW=`$DO_TIMEOUT gnunet-gns --raw -u $TEST_DOMAIN_PLUS -t REDIRECT -c test_gns_lookup.conf`
 RES_REDIRECT_ZKEY=`$DO_TIMEOUT gnunet-gns --raw -u $TEST_DOMAIN_ZKEY -t A -c test_gns_lookup.conf`
diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c
index 029abe208..0413fff32 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -387,6 +387,17 @@ static struct CacheOperation *cop_head;
 static struct CacheOperation *cop_tail;
 
 
+static void
+free_job (struct OpenSignJob *job)
+{
+  if (job->block != job->block_priv)
+    GNUNET_free (job->block_priv);
+  GNUNET_free (job->block);
+  if (NULL != job->label)
+    GNUNET_free (job->label);
+  GNUNET_free (job);
+}
+
 /**
  * Task run during shutdown.
  *
@@ -398,6 +409,7 @@ shutdown_task (void *cls)
 {
   struct DhtPutActivity *ma;
   struct CacheOperation *cop;
+  struct OpenSignJob *job;
 
   (void) cls;
   in_shutdown == GNUNET_YES;
@@ -416,6 +428,22 @@ shutdown_task (void *cls)
     GNUNET_free (cop);
   }
 
+  while (NULL != (job = jobs_head))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Removing incomplete jobs\n");
+    GNUNET_CONTAINER_DLL_remove (jobs_head, jobs_tail, job);
+    free_job (job);
+  }
+
+  while (NULL != (job = results_head))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Removing incomplete jobs\n");
+    GNUNET_CONTAINER_DLL_remove (results_head, results_tail, job);
+    free_job (job);
+  }
+
   while (NULL != (ma = it_head))
   {
     if (NULL != ma->ph)
@@ -771,18 +799,6 @@ dht_put_continuation (void *cls)
 }
 
 
-static void
-free_job (struct OpenSignJob *job)
-{
-  if (job->block != job->block_priv)
-    GNUNET_free (job->block_priv);
-  GNUNET_free (job->block);
-  if (NULL != job->label)
-    GNUNET_free (job->label);
-  GNUNET_free (job);
-}
-
-
 /**
  * Store GNS records in the DHT.
  *
@@ -850,8 +866,7 @@ perform_dht_put (const struct GNUNET_IDENTITY_PrivateKey *key,
   block_size = GNUNET_GNSRECORD_block_get_size (block);
   GNUNET_assert (0 == pthread_mutex_lock (&jobs_lock));
   struct OpenSignJob *job = GNUNET_new (struct OpenSignJob);
-  job->block = GNUNET_malloc (block_size); // FIXME this does not need to be copied, can be freed by worker
-  memcpy (job->block, block, block_size);
+  job->block = block;
   job->block_size = block_size;
   job->block_priv = block_priv;
   job->zone = *key;
@@ -859,10 +874,10 @@ perform_dht_put (const struct GNUNET_IDENTITY_PrivateKey *key,
   job->label = GNUNET_strdup (label);
   job->expire_pub = expire_pub;
   GNUNET_CONTAINER_DLL_insert (jobs_head, jobs_tail, job);
+  GNUNET_assert (0 == pthread_cond_signal (&empty_jobs));
   GNUNET_assert (0 == pthread_mutex_unlock (&jobs_lock));
-  pthread_cond_signal (&empty_jobs);
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Storing %u record(s) for label `%s' in DHT with expiration `%s'\n",
+              "Creating job with %u record(s) for label `%s', expiration `%s'\n",
               rd_public_count,
               label,
               GNUNET_STRINGS_absolute_time_to_string (expire));
@@ -884,8 +899,6 @@ initiate_put_from_pipe_trigger (void *cls)
   ssize_t nf_count;
 
   pipe_read_task = NULL;
-  GNUNET_assert (0 == pthread_mutex_lock (&results_lock));
-  job = results_head;
   np_fh = GNUNET_DISK_pipe_handle (notification_pipe,
                                    GNUNET_DISK_PIPE_END_READ);
   pipe_read_task =
@@ -895,50 +908,54 @@ initiate_put_from_pipe_trigger (void *cls)
       notification_pipe_cb,
       NULL);
   /* empty queue */
-  while (GNUNET_SYSERR !=
-         (nf_count = GNUNET_DISK_file_read (np_fh, buf, sizeof (buf))))
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Read %lld notifications from pipe\n",
-                (long long) nf_count);
-  if (NULL == job)
+  nf_count = GNUNET_DISK_file_read (np_fh, buf, sizeof (buf));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Read %lld notifications from pipe\n",
+              (long long) nf_count);
+  for (int i = 0; i < nf_count; i++)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Hmm... no results. Back to sleep.\n");
+    GNUNET_assert (0 == pthread_mutex_lock (&results_lock));
+    job = results_head;
+    if (NULL == job)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Hmm... no results. Back to sleep.\n");
+      GNUNET_assert (0 == pthread_mutex_unlock (&results_lock));
+      return;
+    }
+    GNUNET_CONTAINER_DLL_remove (results_head, results_tail, job);
     GNUNET_assert (0 == pthread_mutex_unlock (&results_lock));
-    return;
-  }
-  GNUNET_CONTAINER_DLL_remove (results_head, results_tail, job);
-  GNUNET_assert (0 == pthread_mutex_unlock (&results_lock));
-  GNUNET_GNSRECORD_query_from_private_key (&job->zone,
-                                           job->label,
-                                           &query);
-  GNUNET_STATISTICS_update (statistics,
-                            "DHT put operations initiated",
-                            1,
-                            GNUNET_NO);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Storing record(s) for label `%s' in DHT under key %s\n",
-              job->label,
-              GNUNET_h2s (&query));
-  job->ma->ph = GNUNET_DHT_put (dht_handle,
-                                &query,
-                                DHT_GNS_REPLICATION_LEVEL,
-                                GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
-                                GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
-                                job->block_size,
-                                job->block,
-                                job->expire_pub,
-                                &dht_put_continuation,
-                                job->ma);
-  if (NULL == job->ma->ph)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "Could not perform DHT PUT, is the DHT running?\n");
-    GNUNET_free (job->ma);
+    GNUNET_GNSRECORD_query_from_private_key (&job->zone,
+                                             job->label,
+                                             &query);
+    GNUNET_STATISTICS_update (statistics,
+                              "DHT put operations initiated",
+                              1,
+                              GNUNET_NO);
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Storing record(s) for label `%s' in DHT under key %s\n",
+                job->label,
+                GNUNET_h2s (&query));
+    job->ma->ph = GNUNET_DHT_put (dht_handle,
+                                  &query,
+                                  DHT_GNS_REPLICATION_LEVEL,
+                                  GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
+                                  GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
+                                  job->block_size,
+                                  job->block,
+                                  job->expire_pub,
+                                  &dht_put_continuation,
+                                  job->ma);
+    if (NULL == job->ma->ph)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Could not perform DHT PUT, is the DHT running?\n");
+      GNUNET_free (job->ma);
+      free_job (job);
+      return;
+    }
+    refresh_block (job->block_priv);
     free_job (job);
-    return;
   }
-  refresh_block (job->block_priv);
-  free_job (job);
   return;
 }
 
@@ -1213,8 +1230,8 @@ perform_dht_put_monitor (const struct GNUNET_IDENTITY_PrivateKey *key,
   job->label = GNUNET_strdup (label);
   job->expire_pub = expire_pub;
   GNUNET_CONTAINER_DLL_insert (jobs_head, jobs_tail, job);
+  GNUNET_assert (0 == pthread_cond_signal (&empty_jobs));
   GNUNET_assert (0 == pthread_mutex_unlock (&jobs_lock));
-  pthread_cond_signal (&empty_jobs);
 }
 
 
@@ -1305,25 +1322,27 @@ sign_worker (void *cls)
   {
     GNUNET_assert (0 == pthread_mutex_lock (&jobs_lock));
     while (NULL == jobs_head)
-      pthread_cond_wait (&empty_jobs, &jobs_lock);
+      GNUNET_assert (0 == pthread_cond_wait (&empty_jobs, &jobs_lock));
+    if (GNUNET_YES == in_shutdown)
+    {
+      GNUNET_assert (0 == pthread_mutex_unlock (&jobs_lock));
+      return NULL;
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "Taking on Job for %s\n", jobs_head->label);
     job = jobs_head;
     GNUNET_CONTAINER_DLL_remove (jobs_head, jobs_tail, job);
     GNUNET_assert (0 == pthread_mutex_unlock (&jobs_lock));
-    if (NULL != job)
-    {
-      GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block);
-      if (job->block != job->block_priv)
-        GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block_priv);
-      GNUNET_assert (0 == pthread_mutex_lock (&results_lock));
-      GNUNET_CONTAINER_DLL_insert (results_head, results_tail, job);
-      GNUNET_assert (0 == pthread_mutex_unlock (&results_lock));
-      job = NULL;
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Done, notifying main thread through pipe!\n");
-      GNUNET_DISK_file_write (fh, "!", 1);
-    }
+    GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block);
+    if (job->block != job->block_priv)
+      GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block_priv);
+    GNUNET_assert (0 == pthread_mutex_lock (&results_lock));
+    GNUNET_CONTAINER_DLL_insert (results_head, results_tail, job);
+    GNUNET_assert (0 == pthread_mutex_unlock (&results_lock));
+    job = NULL;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Done, notifying main thread through pipe!\n");
+    GNUNET_DISK_file_write (fh, "!", 1);
   }
   return NULL;
 }
diff --git a/src/zonemaster/zonemaster.conf.in b/src/zonemaster/zonemaster.conf.in
index 9c920c476..549394d6c 100644
--- a/src/zonemaster/zonemaster.conf.in
+++ b/src/zonemaster/zonemaster.conf.in
@@ -6,7 +6,7 @@ HOSTNAME = localhost
 BINARY = gnunet-service-zonemaster
 UNIXPATH = $GNUNET_USER_RUNTIME_DIR/gnunet-service-zonemaster.sock
 @JAVAPORT@PORT = 2123
-WORKER_COUNT = 10
+WORKER_COUNT = 4
 
 # Do we require users that want to access GNS to run this process
 # (usually not a good idea)