start service

14 files changed, 402 insertions, 51 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index c3cf0836a..d912ac173 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -22,6 +22,7 @@ SUBDIRS = \
   transport \
   peerinfo-tool \
   core \
+  nse \
   testing \
   nse \
   dv \
diff --git a/src/dht/gnunet-dht-driver.c b/src/dht/gnunet-dht-driver.c
index 7c602cfca..6039e196b 100644
--- a/src/dht/gnunet-dht-driver.c
+++ b/src/dht/gnunet-dht-driver.c
@@ -3354,8 +3354,8 @@ hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
 }
 
 static void
-run(void *cls, char * const *args, const char *cfgfile,
-    const struct GNUNET_CONFIGURATION_Handle *cfg)
+run (void *cls, char * const *args, const char *cfgfile,
+     const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   struct stat frstat;
   struct GNUNET_DHTLOG_TrialInfo trial_info;
diff --git a/src/dht/gnunet-dht-put.c b/src/dht/gnunet-dht-put.c
index 6bd9f9045..02a4fc217 100644
--- a/src/dht/gnunet-dht-put.c
+++ b/src/dht/gnunet-dht-put.c
@@ -149,18 +149,11 @@ run (void *cls,
                                        expiration_seconds));
 
   if (verbose)
-    fprintf (stderr,
-             _("Issuing put request for `%s' with data `%s'!\n"),
+    fprintf (stderr, _("Issuing put request for `%s' with data `%s'!\n"),
              query_key, data);
-  GNUNET_DHT_put (dht_handle,
-                  &key, 
-                  DEFAULT_PUT_REPLICATION,
-                  GNUNET_DHT_RO_NONE,
-                  query_type, 
-                  strlen (data), data,
-                  expiration, 
-                  timeout,
-                  &message_sent_cont, NULL);
+  GNUNET_DHT_put (dht_handle, &key, DEFAULT_PUT_REPLICATION,
+                  GNUNET_DHT_RO_NONE, query_type, strlen (data), data,
+                  expiration, timeout, &message_sent_cont, NULL);
 
 }
 
diff --git a/src/include/gnunet_peerinfo_service.h b/src/include/gnunet_peerinfo_service.h
index 1411168c0..ff4d99d95 100644
--- a/src/include/gnunet_peerinfo_service.h
+++ b/src/include/gnunet_peerinfo_service.h
@@ -89,6 +89,45 @@ GNUNET_PEERINFO_add_peer (struct GNUNET_PEERINFO_Handle *h,
 
 
 /**
+ * Connect to the peerinfo service.
+ *
+ * @param cfg configuration to use
+ * @return NULL on error (configuration related, actual connection
+ *         etablishment may happen asynchronously).
+ */
+struct GNUNET_PEERINFO_Handle *
+GNUNET_PEERINFO_standalone_connect (const struct GNUNET_CONFIGURATION_Handle *cfg);
+
+
+/**
+ * Disconnect from the peerinfo service.  Note that all iterators must
+ * have completed or have been cancelled by the time this function is
+ * called (otherwise, calling this function is a serious error).
+ * Furthermore, if 'GNUNET_PEERINFO_add_peer' operations are still
+ * pending, they will be cancelled silently on disconnect.
+ *
+ * @param h handle to disconnect
+ */
+void
+GNUNET_PEERINFO_standalone_disconnect (struct GNUNET_PEERINFO_Handle *h);
+
+
+/**
+ * Add a host to the persistent list.  This method operates in
+ * semi-reliable mode: if the transmission is not completed by
+ * the time 'GNUNET_PEERINFO_disconnect' is called, it will be
+ * aborted.  Furthermore, if a second HELLO is added for the
+ * same peer before the first one was transmitted, PEERINFO may
+ * merge the two HELLOs prior to transmission to the service.
+ *
+ * @param h handle to the peerinfo service
+ * @param hello the verified (!) HELLO message
+ */
+void
+GNUNET_PEERINFO_standalone_add_peer (struct GNUNET_PEERINFO_Handle *h,
+                          const struct GNUNET_HELLO_Message *hello);
+
+/**
  * Type of an iterator over the hosts.  Note that each
  * host will be called with each available protocol.
  *
@@ -144,9 +183,41 @@ GNUNET_PEERINFO_iterate (struct GNUNET_PEERINFO_Handle *h,
 void
 GNUNET_PEERINFO_iterate_cancel (struct GNUNET_PEERINFO_IteratorContext *ic);
 
+/**
+ * Call a method for each known matching host and change its trust
+ * value.  The callback method will be invoked once for each matching
+ * host and then finally once with a NULL pointer.  After that final
+ * invocation, the iterator context must no longer be used.
+ *
+ * Instead of calling this function with 'peer == NULL'
+ * it is often better to use 'GNUNET_PEERINFO_notify'.
+ *
+ * @param h handle to the peerinfo service
+ * @param peer restrict iteration to this peer only (can be NULL)
+ * @param timeout how long to wait until timing out
+ * @param callback the method to call for each peer
+ * @param callback_cls closure for callback
+ * @return NULL on error (in this case, 'callback' is never called!),
+ *         otherwise an iterator context
+ */
+struct GNUNET_PEERINFO_IteratorContext *
+GNUNET_PEERINFO_standalone_iterate (struct GNUNET_PEERINFO_Handle *h,
+                         const struct GNUNET_PeerIdentity *peer,
+                         struct GNUNET_TIME_Relative timeout,
+                         GNUNET_PEERINFO_Processor callback,
+                         void *callback_cls);
+
 
 
 /**
+ * Cancel an iteration over peer information.
+ *
+ * @param ic context of the iterator to cancel
+ */
+void
+GNUNET_PEERINFO_standalone_iterate_cancel (struct GNUNET_PEERINFO_IteratorContext *ic);
+
+/**
  * Handle for notifications about changes to the set of known peers.
  */
 struct GNUNET_PEERINFO_NotifyContext;
diff --git a/src/include/gnunet_testing_lib.h b/src/include/gnunet_testing_lib.h
index 0b998cf95..9d85f3af9 100644
--- a/src/include/gnunet_testing_lib.h
+++ b/src/include/gnunet_testing_lib.h
@@ -514,12 +514,27 @@ GNUNET_TESTING_daemon_start_stopped (struct GNUNET_TESTING_Daemon *daemon,
                                      void *cb_cls);
 
 /**
- * Stops a GNUnet daemon.
+ * Starts a GNUnet daemon's service.
  *
  * @param d the daemon for which the service should be started
  * @param service the name of the service to start
- * @param timeout how long to wait for process for shutdown to complete
- * @param cb function called once the daemon was stopped
+ * @param timeout how long to wait for process for startup
+ * @param cb function called once gnunet-arm returns
+ * @param cb_cls closure for cb
+ */
+void
+GNUNET_TESTING_daemon_start_service (struct GNUNET_TESTING_Daemon *d,
+                                     char *service,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_TESTING_NotifyDaemonRunning cb, void *cb_cls);
+
+/**
+ * Starts a GNUnet daemon's service which has been previously turned off.
+ *
+ * @param d the daemon for which the service should be started
+ * @param service the name of the service to start
+ * @param timeout how long to wait for process for startup
+ * @param cb function called once gnunet-arm returns
  * @param cb_cls closure for cb
  */
 void
@@ -759,6 +774,23 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
                               struct GNUNET_TIME_Relative timeout,
                               GNUNET_TESTING_NotifyCompletion cb,
                               void *cb_cls);
+/*
+ * Start a given service for each of the peers in the peer group.
+ *
+ * @param pg handle for the peer group
+ * @param service the service to start
+ * @param timeout how long to wait for operations to finish before
+ *        giving up
+ * @param cb function to call once finished
+ * @param cb_cls closure for cb
+ *
+ */
+void
+GNUNET_TESTING_daemons_start_service (struct GNUNET_TESTING_PeerGroup *pg,
+                                      char *service,
+                                      struct GNUNET_TIME_Relative timeout,
+                                      GNUNET_TESTING_NotifyCompletion cb,
+                                      void *cb_cls);
 
 /**
  * Callback function to process statistic values.
diff --git a/src/nse/nse-profiler.c b/src/nse/nse-profiler.c
index f019165d1..3b7371ccc 100644
--- a/src/nse/nse-profiler.c
+++ b/src/nse/nse-profiler.c
@@ -507,6 +507,13 @@ churn_peers (void *cls,
 
 
 static void
+nse_started_cb(void *cls, const char *emsg)
+{
+  GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
+  disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
+}
+
+static void
 my_cb (void *cls,
        const char *emsg)
 {
@@ -538,10 +545,13 @@ my_cb (void *cls,
       GNUNET_free (buf);
     }
   peers_running = GNUNET_TESTING_daemons_running(pg);
-  GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
-  disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
-}
+  GNUNET_TESTING_daemons_start_service (pg,
+                                        "nse",
+                                        wait_time,
+                                        &nse_started_cb,
+                                        NULL);
 
+}
 
 /**
  * Function that will be called whenever two daemons are connected by
diff --git a/src/nse/nse_profiler_test.conf b/src/nse/nse_profiler_test.conf
index 7da26b232..ad113b24f 100644
--- a/src/nse/nse_profiler_test.conf
+++ b/src/nse/nse_profiler_test.conf
@@ -11,6 +11,12 @@ BINARY = gnunet-service-nse
 AUTOSTART = YES
 DEBUG = NO
 CONFIG = $DEFAULTCONFIG
+# Overriding network settings for faster testing (do NOT use
+# these values in production just because they are here)
+WORKDELAY = 10000
+INTERVAL = 15000
+WORKBITS = 0
+PROOFFILE = $SERVICEHOME/nse.proof
 
 [arm]
 PORT = 0
@@ -19,7 +25,7 @@ UNIXPATH = /tmp/test-nse-service-arm.unix
 #DEBUG = YES
 
 [statistics]
-AUTOSTART = NO
+AUTOSTART = YES
 
 [fs]
 AUTOSTART = NO
@@ -34,6 +40,15 @@ AUTOSTART = NO
 PORT = 0
 AUTOSTART = YES
 
+[nat]
+DISABLEV6 = YES 
+BINDTO = 127.0.0.1
+ENABLE_UPNP = NO
+BEHIND_NAT = NO
+ALLOW_NAT = NO
+INTERNAL_ADDRESS = 127.0.0.1
+EXTERNAL_ADDRESS = 127.0.0.1
+
 [transport-unix]
 PORT = 11111
 
@@ -55,11 +70,15 @@ PORT = 0
 plugins = unix
 
 [testing]
-NUM_PEERS = 1000
+NUM_PEERS = 100
 WEAKRANDOM = YES
 TOPOLOGY = NONE
 CONNECT_TOPOLOGY = SMALL_WORLD_RING
-PERCENTAGE = 4
+#CONNECT_TOPOLOGY = ERDOS_RENYI
+#CONNECT_TOPOLOGY_OPTION = CONNECT_MINIMUM
+#CONNECT_TOPOLOGY_OPTION_MODIFIER = 20
+PERCENTAGE = 3
+#PROBABILITY = .1
 F2F = NO
 CONNECT_TIMEOUT = 60
 CONNECT_ATTEMPTS = 3
@@ -74,12 +93,17 @@ MAX_OUTSTANDING_CONNECTIONS = 200
 #NUM_PEERINFO_PER_HOST = 10
 #SINGLE_STATISTICS_PER_HOST = YES
 #NUM_STATISTICS_PER_HOST = 10
+DELETE_FILES = NO
 
 [nse-profiler]
-#OUTPUT_FILE = nse_output_1000_peers.dat
-#TOPOLOGY_OUTPUT_FILE = nse_topo_1000_peers
-#ROUND0 = 75
-#ROUND1 = 750
+#OUTPUT_FILE = nse_output_100_peers.dat
+TOPOLOGY_OUTPUT_FILE = nse_topo_100_peers
+DATA_OUTPUT_FILE = nse_topo_100_peers_stats
+#ROUND0 = 90
+#ROUND1 = 100
+#ROUND2 = 50
+#ROUND3 = 100
 #ROUND2 = 500
 #ROUND3 = 1000
-WAIT_TIME = 7200
+WAIT_TIME = 120
+CONNECTION_LIMIT = 10
diff --git a/src/nse/test_nse.conf b/src/nse/test_nse.conf
index 475f87eaf..ab64dffe2 100644
--- a/src/nse/test_nse.conf
+++ b/src/nse/test_nse.conf
@@ -11,7 +11,7 @@ BINARY = gnunet-service-nse
 AUTOSTART = YES
 DEBUG = YES
 CONFIG = $DEFAULTCONFIG
-
+PROOFFILE = $SERVICEHOME/proof.nse
 # Overriding network settings for faster testing (do NOT use
 # these values in production just because they are here)
 WORKDELAY = 1
diff --git a/src/peerinfo/Makefile.am b/src/peerinfo/Makefile.am
index d58425730..2a078536f 100644
--- a/src/peerinfo/Makefile.am
+++ b/src/peerinfo/Makefile.am
@@ -13,7 +13,8 @@ lib_LTLIBRARIES = libgnunetpeerinfo.la
 
 libgnunetpeerinfo_la_SOURCES = \
   peerinfo_api.c peerinfo.h \
-  peerinfo_api_notify.c
+  peerinfo_api_notify.c \
+  standalone-peerinfo.c
 libgnunetpeerinfo_la_LIBADD = \
   $(top_builddir)/src/hello/libgnunethello.la \
   $(top_builddir)/src/util/libgnunetutil.la \
diff --git a/src/peerinfo/gnunet-service-peerinfo.c b/src/peerinfo/gnunet-service-peerinfo.c
index d5d153b65..0d86e483c 100644
--- a/src/peerinfo/gnunet-service-peerinfo.c
+++ b/src/peerinfo/gnunet-service-peerinfo.c
@@ -51,6 +51,8 @@
  */
 #define DATA_HOST_CLEAN_FREQ GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 60)
 
+#define WRITE_TO_DISK GNUNET_NO
+
 /**
  * In-memory cache of known hosts.
  */
@@ -113,6 +115,7 @@ make_info_message (const struct HostEntry *he)
 }
 
 
+#if WRITE_TO_DISK
 /**
  * Address iterator that causes expired entries to be discarded.
  *
@@ -157,6 +160,7 @@ get_host_filename (const struct GNUNET_PeerIdentity *id)
                    "%s%s%s", networkIdDirectory, DIR_SEPARATOR_STR, &fil);
   return fn;
 }
+#endif
 
 
 /**
@@ -187,12 +191,14 @@ static void
 add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
 {
   struct HostEntry *entry;
+#if WRITE_TO_DISK
   char buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
   const struct GNUNET_HELLO_Message *hello;
   struct GNUNET_HELLO_Message *hello_clean;
   int size;
   struct GNUNET_TIME_Absolute now;
   char *fn;
+#endif
 
   entry = GNUNET_CONTAINER_multihashmap_get (hostmap,
                                              &identity->hashPubKey);
@@ -204,7 +210,7 @@ add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
                             GNUNET_NO);
   entry = GNUNET_malloc (sizeof (struct HostEntry));
   entry->identity = *identity;
-
+#if WRITE_TO_DISK
   fn = get_host_filename (identity);
   if (GNUNET_DISK_file_test (fn) == GNUNET_YES)
     {
@@ -230,6 +236,7 @@ add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
         }
     }
   GNUNET_free (fn);
+#endif
   GNUNET_CONTAINER_multihashmap_put (hostmap,
                                      &identity->hashPubKey,
                                      entry,
@@ -237,7 +244,7 @@ add_host_to_known_hosts (const struct GNUNET_PeerIdentity *identity)
   notify_all (entry);
 }
 
-
+#if WRITE_TO_DISK
 /**
  * Remove a file that should not be there.  LOG
  * success or failure.
@@ -319,7 +326,7 @@ cron_scan_directory_data_hosts (void *cls,
   GNUNET_SCHEDULER_add_delayed (DATA_HOST_FREQ,
                                 &cron_scan_directory_data_hosts, NULL);
 }
-
+#endif
 
 /**
  * Bind a host address (hello) to a hostId.
@@ -331,7 +338,9 @@ static void
 bind_address (const struct GNUNET_PeerIdentity *peer,
               const struct GNUNET_HELLO_Message *hello)
 {
+#if WRITE_TO_DISK
   char *fn;
+#endif
   struct HostEntry *host;
   struct GNUNET_HELLO_Message *mrg;
   struct GNUNET_TIME_Absolute delta;
@@ -359,6 +368,7 @@ bind_address (const struct GNUNET_PeerIdentity *peer,
       GNUNET_free (host->hello);
       host->hello = mrg;
     }
+#if WRITE_TO_DISK
   fn = get_host_filename (peer);
   if (GNUNET_OK == GNUNET_DISK_directory_create_for_file (fn))
     {
@@ -374,6 +384,7 @@ bind_address (const struct GNUNET_PeerIdentity *peer,
         
     }
   GNUNET_free (fn);
+#endif
   notify_all (host);
 }
 
@@ -417,7 +428,7 @@ add_to_tc (void *cls,
   return GNUNET_YES;
 }
 
-
+#if WRITE_TO_DISK
 /**
  * @brief delete expired HELLO entries in data/hosts/
  */
@@ -478,6 +489,7 @@ cron_clean_data_hosts (void *cls,
   GNUNET_SCHEDULER_add_delayed (DATA_HOST_CLEAN_FREQ,
                                 &cron_clean_data_hosts, NULL);
 }
+#endif
 
 
 /**
@@ -683,6 +695,7 @@ run (void *cls,
   hostmap = GNUNET_CONTAINER_multihashmap_create (1024);
   stats = GNUNET_STATISTICS_create ("peerinfo", cfg);
   notify_list = GNUNET_SERVER_notification_context_create (server, 0);
+#if WRITE_TO_DISK
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                           "peerinfo",
@@ -693,6 +706,7 @@ run (void *cls,
                                       &cron_scan_directory_data_hosts, NULL);
   GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                       &cron_clean_data_hosts, NULL);
+#endif
   GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
                                 &shutdown_task, NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
diff --git a/src/peerinfo/test_peerinfo_api.c b/src/peerinfo/test_peerinfo_api.c
index 460a3764d..15efeccad 100644
--- a/src/peerinfo/test_peerinfo_api.c
+++ b/src/peerinfo/test_peerinfo_api.c
@@ -44,6 +44,8 @@ static struct GNUNET_PEERINFO_Handle *h;
 
 static unsigned int retries;
 
+struct GNUNET_PeerIdentity pid;
+
 static int
 check_it (void *cls,
           const char *tname,
@@ -83,7 +85,6 @@ static void
 add_peer ()
 {
   struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pkey;
-  struct GNUNET_PeerIdentity pid;
   struct GNUNET_HELLO_Message *h2;
   size_t agc;
 
@@ -91,7 +92,7 @@ add_peer ()
   memset (&pkey, 32, sizeof (pkey));
   GNUNET_CRYPTO_hash (&pkey, sizeof (pkey), &pid.hashPubKey);
   h2 = GNUNET_HELLO_create (&pkey, &address_generator, &agc);
-  GNUNET_PEERINFO_add_peer (h, h2);
+  GNUNET_PEERINFO_standalone_add_peer (h, h2);
   GNUNET_free (h2);
 
 }
@@ -108,7 +109,7 @@ process (void *cls,
 
   if (err_msg != NULL)
   {
-          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Error in communication with PEERINFO service\n"));
   }
 
@@ -121,8 +122,8 @@ process (void *cls,
           /* try again */
           retries++;      
           add_peer ();
-          ic = GNUNET_PEERINFO_iterate (h,
-                                        NULL,
+          ic = GNUNET_PEERINFO_standalone_iterate (h,
+                                        &pid,
                                         GNUNET_TIME_relative_multiply
                                         (GNUNET_TIME_UNIT_SECONDS, 15), 
                                         &process, cls);
@@ -130,7 +131,7 @@ process (void *cls,
         }
       GNUNET_assert (peer == NULL);
       GNUNET_assert (2 == *ok);
-      GNUNET_PEERINFO_disconnect (h);
+      GNUNET_PEERINFO_standalone_disconnect (h);
       h = NULL;
       *ok = 0;
       return;
@@ -153,11 +154,11 @@ run (void *cls,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
   cfg = c;
-  h = GNUNET_PEERINFO_connect (cfg);
+  h = GNUNET_PEERINFO_standalone_connect (cfg);
   GNUNET_assert (h != NULL);
   add_peer ();
-  ic = GNUNET_PEERINFO_iterate (h,
-                                NULL,
+  ic = GNUNET_PEERINFO_standalone_iterate (h,
+                                &pid,
                                 GNUNET_TIME_relative_multiply
                                 (GNUNET_TIME_UNIT_SECONDS, 15),
                                 &process, cls);
diff --git a/src/testing/testing.c b/src/testing/testing.c
index 79c67e89c..a079390ad 100644
--- a/src/testing/testing.c
+++ b/src/testing/testing.c
@@ -1043,12 +1043,12 @@ GNUNET_TESTING_daemon_running (struct GNUNET_TESTING_Daemon *daemon)
 
 
 /**
- * Stops a GNUnet daemon.
+ * Starts a GNUnet daemon service which has been previously stopped.
  *
  * @param d the daemon for which the service should be started
  * @param service the name of the service to start
  * @param timeout how long to wait for process for shutdown to complete
- * @param cb function called once the daemon was stopped
+ * @param cb function called once the service starts
  * @param cb_cls closure for cb
  */
 void
@@ -1069,13 +1069,9 @@ GNUNET_TESTING_daemon_start_stopped_service (struct GNUNET_TESTING_Daemon *d,
       d->phase = SP_START_DONE;
     }
 
-#if DEBUG_TESTING
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              _("Terminating peer `%4s'\n"), GNUNET_i2s (&d->id));
-#endif
   if (d->churned_services == NULL)
     {
-      d->dead_cb(d->dead_cb_cls, "No service has been churned off yet!!");
+      d->cb(d->cb_cls, &d->id, d->cfg, d, "No service has been churned off yet!!");
       return;
     }
   d->phase = SP_SERVICE_START;
@@ -1132,6 +1128,85 @@ GNUNET_TESTING_daemon_start_stopped_service (struct GNUNET_TESTING_Daemon *d,
 }
 
 /**
+ * Starts a GNUnet daemon's service.
+ *
+ * @param d the daemon for which the service should be started
+ * @param service the name of the service to start
+ * @param timeout how long to wait for process for startup
+ * @param cb function called once gnunet-arm returns
+ * @param cb_cls closure for cb
+ */
+void
+GNUNET_TESTING_daemon_start_service (struct GNUNET_TESTING_Daemon *d,
+                                     char *service,
+                                     struct GNUNET_TIME_Relative timeout,
+                                     GNUNET_TESTING_NotifyDaemonRunning cb, void *cb_cls)
+{
+  char *arg;
+  d->cb = cb;
+  d->cb_cls = cb_cls;
+
+  GNUNET_assert(service != NULL);
+  GNUNET_assert(d->running == GNUNET_YES);
+  GNUNET_assert(d->phase == SP_START_DONE);
+
+#if DEBUG_TESTING
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Starting service %s for peer `%4s'\n"), service, GNUNET_i2s (&d->id));
+#endif
+
+  d->phase = SP_SERVICE_START;
+
+  /* Check if this is a local or remote process */
+  if (NULL != d->hostname)
+    {
+#if DEBUG_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Starting gnunet-arm with config `%s' on host `%s'.\n",
+                  d->cfgfile, d->hostname);
+#endif
+
+      if (d->username != NULL)
+        GNUNET_asprintf (&arg, "%s@%s", d->username, d->hostname);
+      else
+        arg = GNUNET_strdup (d->hostname);
+
+      d->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh",
+#if !DEBUG_TESTING
+                                         "-q",
+#endif
+                                         arg, "gnunet-arm",
+#if DEBUG_TESTING
+                                         "-L", "DEBUG",
+#endif
+                                         "-c", d->cfgfile, "-i", service, "-q",
+                                         NULL);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Starting gnunet-arm with command ssh %s gnunet-arm -c %s -i %s -q\n",
+                  arg, "gnunet-arm", d->cfgfile, service);
+      GNUNET_free (arg);
+    }
+  else
+    {
+#if DEBUG_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Starting gnunet-arm with config `%s' locally.\n",
+                  d->cfgfile);
+#endif
+      d->proc = GNUNET_OS_start_process (NULL, NULL, "gnunet-arm",
+                                         "gnunet-arm",
+#if DEBUG_TESTING
+                                         "-L", "DEBUG",
+#endif
+                                         "-c", d->cfgfile, "-i", service, "-q",
+                                         NULL);
+    }
+
+  d->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
+  d->task = GNUNET_SCHEDULER_add_now (&start_fsm, d);
+}
+
+/**
  * Start a peer that has previously been stopped using the daemon_stop
  * call (and files weren't deleted and the allow restart flag)
  *
diff --git a/src/testing/testing_group.c b/src/testing/testing_group.c
index 7986d1b5f..ba4b29150 100644
--- a/src/testing/testing_group.c
+++ b/src/testing/testing_group.c
@@ -279,6 +279,33 @@ struct PeerRestartContext
   struct GNUNET_TESTING_Daemon *daemon;
 };
 
+struct ServiceStartContext
+{
+  struct GNUNET_TESTING_PeerGroup *pg;
+  unsigned int remaining;
+  GNUNET_TESTING_NotifyCompletion cb;
+  unsigned int outstanding;
+  char *service;
+  struct GNUNET_TIME_Relative timeout;
+  void *cb_cls;
+};
+
+/**
+ * Individual shutdown context for a particular peer.
+ */
+struct PeerServiceStartContext
+{
+  /**
+   * Pointer to the high level start context.
+   */
+  struct ServiceStartContext *start_ctx;
+
+  /**
+   * The daemon handle for the peer to start the service on.
+   */
+  struct GNUNET_TESTING_Daemon *daemon;
+};
+
 struct CreateTopologyContext
 {
 
@@ -5605,6 +5632,66 @@ schedule_churn_restart(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     }
 }
 
+/**
+ * Callback for informing us about a successful
+ * or unsuccessful churn start call.
+ *
+ * @param cls a struct ServiceStartContext *startup_ctx
+ * @param id the peer identity of the started peer
+ * @param cfg the handle to the configuration of the peer
+ * @param d handle to the daemon for the peer
+ * @param emsg NULL on success, non-NULL on failure
+ *
+ */
+void
+service_start_callback(void *cls,
+                       const struct GNUNET_PeerIdentity *id,
+                       const struct GNUNET_CONFIGURATION_Handle *cfg,
+                       struct GNUNET_TESTING_Daemon *d,
+                       const char *emsg)
+{
+  struct ServiceStartContext *startup_ctx = (struct ServiceStartContext *)cls;
+
+  if (emsg != NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Service start failed with error `%s'\n", emsg);
+    }
+
+  startup_ctx->outstanding--;
+  startup_ctx->remaining--;
+
+  if (startup_ctx->remaining == 0)
+    {
+      startup_ctx->cb (startup_ctx->cb_cls, NULL);
+      GNUNET_free (startup_ctx->service);
+      GNUNET_free (startup_ctx);
+    }
+}
+
+static void
+schedule_service_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerServiceStartContext *peer_ctx = cls;
+  struct ServiceStartContext *startup_ctx = peer_ctx->start_ctx;
+
+  if (startup_ctx->outstanding > startup_ctx->pg->max_concurrent_ssh)
+    GNUNET_SCHEDULER_add_delayed (
+                                  GNUNET_TIME_relative_multiply (
+                                                                 GNUNET_TIME_UNIT_MILLISECONDS,
+                                                                 100),
+                                  &schedule_service_start, peer_ctx);
+  else
+    {
+
+      GNUNET_TESTING_daemon_start_service (peer_ctx->daemon,
+                                           startup_ctx->service,
+                                           startup_ctx->timeout,
+                                           &service_start_callback, startup_ctx);
+      GNUNET_free (peer_ctx);
+    }
+}
+
 
 static void
 internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
@@ -6635,6 +6722,48 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   GNUNET_free_non_null (stopped_permute);
 }
 
+/*
+ * Start a given service for each of the peers in the peer group.
+ *
+ * @param pg handle for the peer group
+ * @param service the service to start
+ * @param timeout how long to wait for operations to finish before
+ *        giving up
+ * @param cb function to call once finished
+ * @param cb_cls closure for cb
+ *
+ */
+void
+GNUNET_TESTING_daemons_start_service (struct GNUNET_TESTING_PeerGroup *pg,
+                                      char *service,
+                                      struct GNUNET_TIME_Relative timeout,
+                                      GNUNET_TESTING_NotifyCompletion cb,
+                                      void *cb_cls)
+{
+  struct ServiceStartContext *start_ctx;
+  struct PeerServiceStartContext *peer_start_ctx;
+  unsigned int i;
+
+  GNUNET_assert(service != NULL);
+
+  start_ctx = GNUNET_malloc(sizeof(struct ServiceStartContext));
+  start_ctx->pg = pg;
+  start_ctx->remaining = pg->total;
+  start_ctx->cb = cb;
+  start_ctx->cb_cls = cb_cls;
+  start_ctx->service = GNUNET_strdup(service);
+
+  for (i = 0; i < pg->total; i++)
+    {
+#if DEBUG_START
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up service %s on peer %d!\n", service, stopped_arr[stopped_permute[i]]);
+#endif
+      peer_start_ctx = GNUNET_malloc (sizeof (struct PeerServiceStartContext));
+      peer_start_ctx->start_ctx = start_ctx;
+      peer_start_ctx->daemon = pg->peers[i].daemon;
+      GNUNET_SCHEDULER_add_now (&schedule_service_start, peer_start_ctx);
+    }
+}
 
 /**
  * Restart all peers in the given group.
diff --git a/src/util/service.c b/src/util/service.c
index abf3b2ab1..4d88aff43 100644
--- a/src/util/service.c
+++ b/src/util/service.c
@@ -1630,8 +1630,8 @@ GNUNET_SERVICE_run (int argc,
       (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number(sctx.cfg, "testing", "skew_variance", &skew_variance)))
     {
       clock_offset = skew_offset - skew_variance;
-#if 1
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Skewing clock by %ll\n", clock_offset);
+#if DEBUG_SERVICE
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Skewing clock by %dll\n", clock_offset);
 #endif
     }
   /* actually run service */