add test case for NO topology restrictions, connect in ring. good for maximum number of peers testing

4 files changed, 306 insertions, 59 deletions

diff --git a/src/testing/Makefile.am b/src/testing/Makefile.am
index 163b4f9c9..3f68411f5 100644
--- a/src/testing/Makefile.am
+++ b/src/testing/Makefile.am
@@ -38,6 +38,7 @@ check_PROGRAMS = \
  test_testing_topology_small_world_torus \
  test_testing_topology_erdos_renyi \
  test_testing_topology_internat \
+ test_testing_topology_none \
  test_testing_topology_scale_free
  
 TESTS = \
@@ -168,6 +169,12 @@ test_testing_topology_scale_free_SOURCES = \
 test_testing_topology_scale_free_LDADD = \
  $(top_builddir)/src/testing/libgnunettesting.la \
  $(top_builddir)/src/util/libgnunetutil.la
+
+test_testing_topology_none_SOURCES = \
+ test_testing_topology.c
+test_testing_topology_none_LDADD = \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/util/libgnunetutil.la
  
  
 EXTRA_DIST = \
diff --git a/src/testing/test_testing_data_topology_none.conf b/src/testing/test_testing_data_topology_none.conf
new file mode 100644
index 000000000..9ff8ca390
--- /dev/null
+++ b/src/testing/test_testing_data_topology_none.conf
@@ -0,0 +1,56 @@
+[PATHS]
+SERVICEHOME = /tmp/test-gnunet-testing/
+DEFAULTCONFIG = test_testing_data_topology_clique.conf
+
+[resolver]
+PORT = 0
+
+[transport]
+PORT = 0
+PLUGINS = tcp
+#PREFIX = xterm -e xterm -T transport -e gdb --args
+#BINARY = /home/mrwiggles/documents/research/gnunet/gnunet-ng/src/transport/.libs/gnunet-service-transport
+#PREFIX = valgrind --tool=memcheck --log-file=logs%p
+#PREFIX = valgrind --tool=callgrind --log-file=callgrind.%p
+#DEBUG = YES
+
+[topology]
+PORT = 0
+
+[arm]
+PORT = 0
+DEFAULTSERVICES = 
+
+[statistics]
+PORT = 0
+
+[transport-tcp]
+PORT = 2568
+
+[peerinfo]
+PORT = 0
+#DEBUG = YES
+#PREFIX = xterm -e xterm -T peerinfo -e gdb --args
+#BINARY = /home/mrwiggles/documents/research/gnunet/gnunet-ng/src/peerinfo/.libs/gnunet-service-peerinfo
+#PREFIX = valgrind --tool=memcheck --log-file=peerinfo%p
+
+[core]
+PORT = 0
+#PREFIX = xterm -e xterm -T CORE -e gdb --args
+#PREFIX = valgrind --tool=memcheck --log-file=logs%p
+#DEBUG = YES
+
+[hostlist]
+PORT = 0
+
+[fs]
+PORT = 0
+
+[testing]
+NUM_PEERS = 200
+WEAKRANDOM = YES
+TOPOLOGY = NONE
+F2F = NO
+BLACKLIST_TOPOLOGY = NONE
+CONNECT_TOPOLOGY = RING
+#DEBUG = YES
diff --git a/src/testing/test_testing_topology.c b/src/testing/test_testing_topology.c
index ba9084e31..0c569880b 100644
--- a/src/testing/test_testing_topology.c
+++ b/src/testing/test_testing_topology.c
@@ -25,7 +25,7 @@
 #include "gnunet_testing_lib.h"
 #include "gnunet_core_service.h"
 
-#define VERBOSE GNUNET_NO
+#define VERBOSE GNUNET_YES
 
 /**
  * How long until we fail the whole testcase?
@@ -37,6 +37,8 @@
  */
 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 500)
 
+#define SECONDS_PER_PEER_START 45
+
 #define DEFAULT_NUM_PEERS 4
 
 #define MAX_OUTSTANDING_CONNECTIONS 300
@@ -83,6 +85,10 @@ static int transmit_ready_failed;
 
 static int transmit_ready_called;
 
+static unsigned int modnum;
+
+static unsigned int dotnum;
+
 static enum GNUNET_TESTING_Topology topology;
 
 static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
@@ -196,7 +202,7 @@ finish_testing ()
     }
 #if VERBOSE
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "transmit_ready's scheduled %d, failed %d, transmit_ready's called %d\n", transmit_ready_scheduled, transmit_ready_failed, transmit_ready_called);
+                      "Transmit_ready's scheduled %d, failed %d, transmit_ready's called %d\n", transmit_ready_scheduled, transmit_ready_failed, transmit_ready_called);
 #endif
 
 #if VERBOSE
@@ -221,13 +227,13 @@ disconnect_cores (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
   struct TestMessageContext *pos = cls;
 
   /* Disconnect from the respective cores */
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Disconnecting from peer 1 `%4s'\n", GNUNET_i2s (&pos->peer1->id));
 #endif
   if (pos->peer1handle != NULL)
     GNUNET_CORE_disconnect(pos->peer1handle);
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Disconnecting from peer 2 `%4s'\n", GNUNET_i2s (&pos->peer2->id));
 #endif
@@ -253,8 +259,26 @@ process_mtype (void *cls,
   if (pos->uid != ntohl(msg->uid))
     return GNUNET_OK;
 
-  total_messages_received++;
 #if VERBOSE
+    if ((total_messages_received) % modnum == 0)
+      {
+        if (total_messages_received == 0)
+          fprintf (stdout, "0%%");
+        else
+          fprintf (stdout, "%d%%",
+                   (int) (((float) total_messages_received /
+                           expected_messages) * 100));
+
+      }
+    else if (total_messages_received % dotnum == 0)
+      {
+        fprintf (stdout, ".");
+      }
+    fflush (stdout);
+#endif
+
+  total_messages_received++;
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Received message from `%4s', type %d.\n", GNUNET_i2s (peer), ntohs(message->type));
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -263,6 +287,9 @@ process_mtype (void *cls,
 
   if (total_messages_received == expected_messages)
     {
+#if VERBOSE
+      fprintf(stdout, "100%%]\n");
+#endif
       GNUNET_SCHEDULER_cancel (sched, die_task);
       GNUNET_SCHEDULER_add_now (sched, &finish_testing, NULL);
     }
@@ -328,7 +355,7 @@ transmit_ready (void *cls, size_t size, void *buf)
   m->header.size = htons (sizeof (struct GNUNET_TestMessage));
   m->uid = htonl(pos->uid);
   transmit_ready_called++;
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "transmit ready for peer %s\ntransmit_ready's scheduled %d, transmit_ready's called %d\n", GNUNET_i2s(&pos->peer1->id), transmit_ready_scheduled, transmit_ready_called);
 #endif
@@ -353,7 +380,7 @@ init_notify_peer2 (void *cls,
 {
   struct TestMessageContext *pos = cls;
 
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Core connection to `%4s' established, scheduling message send\n",
               GNUNET_i2s (my_identity));
@@ -388,7 +415,7 @@ init_notify_peer1 (void *cls,
   struct TestMessageContext *pos = cls;
   total_server_connections++;
 
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Core connection to `%4s' established, setting up handles\n",
               GNUNET_i2s (my_identity));
@@ -474,11 +501,28 @@ topology_callback (void *cls,
   struct TestMessageContext *temp_context;
   if (emsg == NULL)
     {
-      total_connections++;
 #if VERBOSE
+    if ((total_connections) % modnum == 0)
+      {
+        if (total_connections == 0)
+          fprintf (stdout, "0%%");
+        else
+          fprintf (stdout, "%d%%",
+                   (int) (((float) total_connections /
+                           expected_connections) * 100));
+
+      }
+    else if (total_connections % dotnum == 0)
+      {
+        fprintf (stdout, ".");
+      }
+    fflush (stdout);
+#endif
+      total_connections++;
+#if VERBOSE > 1
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s\n",
-               first_daemon->shortname,
-               second_daemon->shortname);
+                 first_daemon->shortname,
+                 second_daemon->shortname);
 #endif
       temp_context = GNUNET_malloc(sizeof(struct TestMessageContext));
       temp_context->peer1 = first_daemon;
@@ -505,15 +549,21 @@ topology_callback (void *cls,
   if (total_connections == expected_connections)
     {
 #if VERBOSE
+      fprintf(stdout, "100%%]\n");
+#endif
+#if VERBOSE
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Created %d total connections, which is our target number!  Calling send messages.\n",
                   total_connections);
 #endif
-
+      modnum = expected_messages / 4;
+      dotnum = (expected_messages / 50) + 1;
+#if VERBOSE
+      fprintf(stdout, "Test message progress: [");
+#endif
       GNUNET_SCHEDULER_cancel (sched, die_task);
       die_task = GNUNET_SCHEDULER_NO_TASK;
       GNUNET_SCHEDULER_add_now (sched, &send_test_messages, test_messages);
-      //GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1), &send_test_messages, test_messages);
     }
   else if (total_connections + failed_connections == expected_connections)
     {
@@ -522,7 +572,6 @@ topology_callback (void *cls,
           GNUNET_SCHEDULER_cancel (sched, die_task);
           die_task = GNUNET_SCHEDULER_NO_TASK;
           GNUNET_SCHEDULER_add_now (sched, &send_test_messages, test_messages);
-          //GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1), &send_test_messages, test_messages);
         }
       else
         {
@@ -533,7 +582,7 @@ topology_callback (void *cls,
     }
   else
     {
-#if VERBOSE
+#if VERBOSE > 1
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Have %d total connections, %d failed connections, Want %d (at least %d)\n",
                   total_connections, failed_connections, expected_connections, expected_connections - (unsigned int)(fail_percentage * expected_connections));
@@ -548,7 +597,7 @@ connect_topology ()
   if ((pg != NULL) && (peers_left == 0))
     {
       expected_connections = GNUNET_TESTING_connect_topology (pg, connection_topology, connect_topology_option, connect_topology_option_modifier);
-#if VERBOSE
+#if VERBOSE > 1
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Have %d expected connections\n", expected_connections);
 #endif
@@ -564,6 +613,11 @@ connect_topology ()
   die_task = GNUNET_SCHEDULER_add_delayed (sched,
                                            TEST_TIMEOUT,
                                            &end_badly, "from connect topology (timeout)");
+  modnum = expected_connections / 4;
+  dotnum = (expected_connections / 50) + 1;
+#if VERBOSE
+  fprintf(stdout, "Peer connection progress: [");
+#endif
 }
 
 static void
@@ -576,6 +630,7 @@ create_topology ()
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Topology set up, now starting peers!\n");
 #endif
+      fprintf(stdout, "Daemon start progress [");
       GNUNET_TESTING_daemons_continue_startup(pg);
     }
   else
@@ -604,16 +659,36 @@ peers_started_callback (void *cls,
       return;
     }
   GNUNET_assert (id != NULL);
-#if VERBOSE
+#if VERBOSE > 1
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
               (num_peers - peers_left) + 1, num_peers);
 #endif
+#if VERBOSE
+    if ((num_peers - peers_left) % modnum == 0)
+      {
+        if (num_peers - peers_left == 0)
+          fprintf (stdout, "0%%");
+        else
+          fprintf (stdout, "%d%%",
+                   (int) (((float) (num_peers - peers_left) /
+                           num_peers) * 100));
+
+      }
+    else if ((num_peers - peers_left) % dotnum == 0)
+      {
+        fprintf (stdout, ".");
+      }
+    fflush (stdout);
+#endif
   peers_left--;
   if (peers_left == 0)
     {
 #if VERBOSE
+      fprintf(stdout, "100%%]\n");
+#endif
+#if VERBOSE
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "All %d daemons started, now creating topology!\n",
+                  "All %d daemons started, now connecting peers!\n",
                   num_peers);
 #endif
       GNUNET_SCHEDULER_cancel (sched, die_task);
@@ -647,15 +722,34 @@ void hostkey_callback (void *cls,
       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg);
     }
 
-#if VERBOSE
+#if VERBOSE > 1
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Hostkey created for peer `%s'\n",
-                GNUNET_i2s(id));
+                "Hostkey (%d/%d) created for peer `%s'\n",
+                num_peers - peers_left, num_peers, GNUNET_i2s(id));
+#endif
+
+#if VERBOSE
+    if ((num_peers - peers_left) % modnum == 0)
+      {
+        if (num_peers - peers_left == 0)
+          fprintf (stdout, "0%%");
+        else
+          fprintf (stdout, "%d%%",
+                   (int) (((float) (num_peers - peers_left) /
+                           num_peers) * 100));
+
+      }
+    else if ((num_peers - peers_left) % dotnum == 0)
+      {
+        fprintf (stdout, ".");
+      }
+    fflush (stdout);
 #endif
     peers_left--;
     if (peers_left == 0)
       {
 #if VERBOSE
+        fprintf(stdout, "100%%]\n");
         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                     "All %d hostkeys created, now creating topology!\n",
                     num_peers);
@@ -664,9 +758,8 @@ void hostkey_callback (void *cls,
         /* Set up task in case topology creation doesn't finish
          * within a reasonable amount of time */
         die_task = GNUNET_SCHEDULER_add_delayed (sched,
-                                                 GNUNET_TIME_relative_multiply
-                                                 (GNUNET_TIME_UNIT_MINUTES, 5),
-                                                 &end_badly, "from hostkey_callback");
+                                                 TIMEOUT,
+                                                 &end_badly, "from create_topology");
         GNUNET_SCHEDULER_add_now(sched, &create_topology, NULL);
         ok = 0;
       }
@@ -766,15 +859,16 @@ run (void *cls,
   main_cfg = cfg;
 
   peers_left = num_peers;
-
+  modnum = num_peers / 4;
+  dotnum = (num_peers / 50) + 1;
+  fprintf (stdout, "Hostkey generation progress: \[");
   /* Set up a task to end testing if peer start fails */
   die_task = GNUNET_SCHEDULER_add_delayed (sched,
-                                           GNUNET_TIME_relative_multiply
-                                           (GNUNET_TIME_UNIT_MINUTES, 5),
-                                           &end_badly, "didn't start all daemons in reasonable amount of time!!!");
+                                           GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers),
+                                           &end_badly, "didn't generate all hostkeys within a reasonable amount of time!!!");
 
   pg = GNUNET_TESTING_daemons_start (sched, cfg,
-                                     peers_left, TIMEOUT, &hostkey_callback, NULL, &peers_started_callback, NULL,
+                                     peers_left, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), &hostkey_callback, NULL, &peers_started_callback, NULL,
                                      &topology_callback, NULL, NULL);
 
 }
diff --git a/src/testing/testing_group.c b/src/testing/testing_group.c
index 831657305..6770cf446 100644
--- a/src/testing/testing_group.c
+++ b/src/testing/testing_group.c
@@ -31,8 +31,6 @@
 
 #define VERBOSE_TESTING GNUNET_NO
 
-#define VERBOSE_TOPOLOGY GNUNET_NO
-
 #define DEBUG_CHURN GNUNET_NO
 
 /**
@@ -52,6 +50,8 @@
 
 #define MAX_OUTSTANDING_CONNECTIONS 50
 
+#define MAX_CONCURRENT_HOSTKEYS 16
+
 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
 
 #define CONNECT_ATTEMPTS 8
@@ -318,6 +318,11 @@ struct GNUNET_TESTING_PeerGroup
    * At what time should we fail the peer startup process?
    */
   struct GNUNET_TIME_Absolute max_timeout;
+
+  /**
+   * How many peers are being started right now?
+   */
+  unsigned int starting;
 };
 
 /**
@@ -640,10 +645,8 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
  * @param first index of the first peer
  * @param second index of the second peer
  *
- * @return the number of connections added (can be 0, 1 or 2)
- *         technically should only be 0 or 2, but the small price
- *         of iterating over the lists (hashmaps in the future)
- *         for being sure doesn't bother me!
+ * @return the number of connections added
+ *         technically should only be 0 or 2
  *
  */
 static int
@@ -2057,8 +2060,8 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * by the topology.  This will only have an effect once peers
  * are started if the FRIENDS_ONLY option is set in the base
  * config.  Also takes an optional restrict topology which
- * disallows direct TCP connections UNLESS they are specified in
- * the restricted topology.
+ * disallows connections based on a particular transport
+ * UNLESS they are specified in the restricted topology.
  *
  * @param pg the peer group struct representing the running peers
  * @param topology which topology to connect the peers in
@@ -2086,49 +2089,49 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
   switch (topology)
     {
     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating clique topology\n"));
 #endif
       num_connections = create_clique (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) topology\n"));
 #endif
       num_connections = create_small_world_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) topology\n"));
 #endif
       num_connections = create_small_world (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating ring topology\n"));
 #endif
       num_connections = create_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating 2d torus topology\n"));
 #endif
       num_connections = create_2d_torus (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi topology\n"));
 #endif
       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating InterNAT topology\n"));
 #endif
@@ -2149,38 +2152,43 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       num_connections = create_line (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating no allowed topology (all peers can connect at core level)\n"));
+#endif
       num_connections = 0;
       break;
     default:
       num_connections = 0;
       break;
     }
-  if (num_connections < 1)
+
+  if (num_connections < 0)
     return GNUNET_SYSERR;
 
   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
     {
       ret = create_and_copy_friend_files(pg);
-    }
-
-  if (ret != GNUNET_OK)
-    {
+      if (ret != GNUNET_OK)
+        {
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Failed during friend file copying!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Failed during friend file copying!\n"));
 #endif
-      return GNUNET_SYSERR;
-    }
-  else
-    {
+          return GNUNET_SYSERR;
+        }
+      else
+        {
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Friend files created/copied successfully!\n"));
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                          _("Friend files created/copied successfully!\n"));
 #endif
+        }
     }
 
   /* Use the create clique method to initially set all connections as blacklisted. */
-  create_clique (pg, &blacklist_connections);
+  if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
+    create_clique (pg, &blacklist_connections);
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
   switch (restrict_topology)
@@ -2249,7 +2257,10 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       unblacklisted_connections = create_line (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
-      /* Fall through */
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating no blacklist topology (all peers can connect at transport level)\n"));
+#endif
     default:
       break;
     }
@@ -2798,6 +2809,69 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
     }
 }
 
+struct InternalStartContext
+{
+  struct PeerData *peer;
+  struct GNUNET_SCHEDULER_Handle *sched;
+  const struct GNUNET_CONFIGURATION_Handle *pcfg;
+  struct GNUNET_TIME_Relative timeout;
+  GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
+  void *hostkey_cls;
+  GNUNET_TESTING_NotifyDaemonRunning cb;
+  void *cb_cls;
+  const char *hostname;
+};
+
+
+/**
+ * Prototype of a function that will be called whenever
+ * a daemon was started by the testing library.
+ *
+ * @param cls closure
+ * @param id identifier for the daemon, NULL on error
+ * @param d handle for the daemon
+ * @param emsg error message (NULL on success)
+ */
+static void internal_hostkey_callback (void *cls,
+                                       const struct GNUNET_PeerIdentity *id,
+                                       struct GNUNET_TESTING_Daemon *d,
+                                       const char *emsg)
+{
+  struct InternalStartContext *internal_context = cls;
+  internal_context->peer->pg->starting--;
+  internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
+  GNUNET_free(internal_context);
+}
+
+static void
+internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
+{
+  struct InternalStartContext *internal_context = cls;
+
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    {
+      GNUNET_free(internal_context);
+      return;
+    }
+
+  if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
+    {
+      internal_context->peer->pg->starting++;
+      internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->sched,
+                                                                    internal_context->pcfg,
+                                                                    internal_context->timeout,
+                                                                    internal_context->hostname,
+                                                                    &internal_hostkey_callback,
+                                                                    internal_context,
+                                                                    internal_context->cb,
+                                                                    internal_context->cb_cls);
+    }
+  else
+    {
+      GNUNET_SCHEDULER_add_delayed(internal_context->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
+    }
+}
+
 /**
  * Start count gnunetd processes with the same set of transports and
  * applications.  The port numbers (any option called "PORT") will be
@@ -2842,6 +2916,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
   char *baseservicehome;
   char *newservicehome;
   char *tmpdir;
+  struct InternalStartContext *internal_context;
   struct GNUNET_CONFIGURATION_Handle *pcfg;
   unsigned int off;
   unsigned int hostcnt;
@@ -2968,6 +3043,20 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
       pg->peers[off].pg = pg;
+      internal_context = GNUNET_malloc(sizeof(struct InternalStartContext));
+      internal_context->sched = sched;
+      internal_context->peer = &pg->peers[off];
+      internal_context->pcfg = pcfg;
+      internal_context->timeout = timeout;
+      internal_context->hostname = hostname;
+      internal_context->hostkey_callback = hostkey_callback;
+      internal_context->hostkey_cls = hostkey_cls;
+      internal_context->cb = cb;
+      internal_context->cb_cls = cb_cls;
+
+      GNUNET_SCHEDULER_add_now (sched, &internal_start, internal_context);
+
+      /*
       pg->peers[off].daemon = GNUNET_TESTING_daemon_start (sched,
                                                            pcfg,
                                                            timeout,
@@ -2978,6 +3067,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       if (NULL == pg->peers[off].daemon)
         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                     _("Could not start peer number %u!\n"), off);
+      */
 
     }
   return pg;