6 files changed, 332 insertions, 78 deletions

diff --git a/src/include/gnunet_testbed_service.h b/src/include/gnunet_testbed_service.h
index 91fb424c4..55e503232 100644
--- a/src/include/gnunet_testbed_service.h
+++ b/src/include/gnunet_testbed_service.h
@@ -133,19 +133,53 @@ GNUNET_TESTBED_host_destroy (struct GNUNET_TESTBED_Host *host);
 
 
 /**
+ * The handle for whether a host is habitable or not
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle;
+
+
+/**
+ * Callbacks of this type are called by GNUNET_TESTBED_is_host_habitable to
+ * inform whether the given host is habitable or not. The Handle returned by
+ * GNUNET_TESTBED_is_host_habitable() is invalid after this callback is called
+ *
+ * @param cls the closure given to GNUNET_TESTBED_is_host_habitable()
+ * @param status GNUNET_YES if it is habitable; GNUNET_NO if not
+ */
+typedef void (*GNUNET_TESTBED_HostHabitableCallback) (void *cls,
+                                                      int status);
+
+
+/**
  * Checks whether a host can be used to start testbed service
  *
  * @param host the host to check
- * @param config the configuration handle to lookup the path of the testbed helper
- * @return GNUNET_YES if testbed service can be started on the given host
- *           remotely; GNUNET_NO if not
- */
-int
+ * @param config the configuration handle to lookup the path of the testbed
+ *          helper
+ * @param cb the callback to call to inform about habitability of the given host
+ * @param cb_cls the closure for the callback
+ * @return NULL upon any error or a handle which can be passed to
+ *           GNUNET_TESTBED_is_host_habitable_cancel()
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle *
 GNUNET_TESTBED_is_host_habitable (const struct GNUNET_TESTBED_Host *host,
-                                  const struct GNUNET_CONFIGURATION_Handle *config);
+                                  const struct GNUNET_CONFIGURATION_Handle
+                                  *config,
+                                  GNUNET_TESTBED_HostHabitableCallback cb,
+                                  void *cb_cls);
 
 
 /**
+ * Function to cancel a request started using GNUNET_TESTBED_is_host_habitable()
+ *
+ * @param struct handle the habitability check handle
+ */
+void
+GNUNET_TESTBED_is_host_habitable_cancel (struct
+                                         GNUNET_TESTBED_HostHabitableCheckHandle
+                                         *handle);
+
+/**
  * Obtain the host's hostname.
  *
  * @param host handle to the host, NULL means 'localhost'
diff --git a/src/testbed/gnunet-testbed-profiler.c b/src/testbed/gnunet-testbed-profiler.c
index 083926cbf..9eaf702a3 100644
--- a/src/testbed/gnunet-testbed-profiler.c
+++ b/src/testbed/gnunet-testbed-profiler.c
@@ -153,6 +153,11 @@ struct DLLOperation *dll_op_tail;
 struct GNUNET_TESTBED_Operation *topology_op;
 
 /**
+ * The handle for whether a host is habitable or not
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle **hc_handles;
+
+/**
  * Abort task identifier
  */
 static GNUNET_SCHEDULER_TaskIdentifier abort_task;
@@ -253,6 +258,14 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   shutdown_task = GNUNET_SCHEDULER_NO_TASK;
   if (GNUNET_SCHEDULER_NO_TASK != abort_task)
     GNUNET_SCHEDULER_cancel (abort_task);
+  if (NULL != hc_handles)
+  {
+    for (nhost = 0; nhost < num_hosts; nhost++)
+      if (NULL != hc_handles[num_hosts])
+        GNUNET_TESTBED_is_host_habitable_cancel (hc_handles[num_hosts]);
+    GNUNET_free (hc_handles);
+    hc_handles = NULL;
+  }
   if (GNUNET_SCHEDULER_NO_TASK != register_hosts_task)
     GNUNET_SCHEDULER_cancel (register_hosts_task);
   if (NULL != reg_handle)
@@ -688,6 +701,35 @@ status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *config, int stat
 
 
 /**
+ * Callbacks of this type are called by GNUNET_TESTBED_is_host_habitable to
+ * inform whether the given host is habitable or not. The Handle returned by
+ * GNUNET_TESTBED_is_host_habitable() is invalid after this callback is called
+ *
+ * @param cls NULL
+ * @param status GNUNET_YES if it is habitable; GNUNET_NO if not
+ */
+static void 
+host_habitable_cb (void *cls, int status)
+{
+  struct GNUNET_TESTBED_HostHabitableCheckHandle **hc_handle = cls;
+  static unsigned int hosts_checked;
+
+  *hc_handle = NULL;
+  if (++hosts_checked < num_hosts)
+    return;
+  GNUNET_free (hc_handles);
+  hc_handles = NULL;
+  mc_proc = 
+      GNUNET_TESTBED_controller_start (GNUNET_TESTBED_host_get_hostname_ 
+                                       (hosts[0]),
+                                       hosts[0],
+                                       cfg,
+                                       status_cb,
+                                       NULL);
+}
+
+
+/**
  * Main function that will be run by the scheduler.
  *
  * @param cls closure
@@ -717,12 +759,22 @@ run (void *cls, char *const *args, const char *cfgfile,
     fprintf (stderr, _("No hosts loaded. Need at least one host\n"));
     return;
   }
+  hc_handles = GNUNET_malloc (sizeof (struct
+                                      GNUNET_TESTBED_HostHabitableCheckHandle *) 
+                              * num_hosts);
   for (nhost = 0; nhost < num_hosts; nhost++)
-  {
-    if (GNUNET_YES != GNUNET_TESTBED_is_host_habitable (hosts[nhost], config))
+  {    
+    if (NULL == (hc_handles[nhost] = GNUNET_TESTBED_is_host_habitable (hosts[nhost], config,
+                                                                       &host_habitable_cb,
+                                                                       &hc_handles[nhost])))
     {
       fprintf (stderr, _("Host %s cannot start testbed\n"),
                GNUNET_TESTBED_host_get_hostname_ (hosts[nhost]));
+      for (nhost = 0; nhost < num_hosts; nhost++)
+        if (NULL != hc_handles[num_hosts])
+          GNUNET_TESTBED_is_host_habitable_cancel (hc_handles[num_hosts]);
+      GNUNET_free (hc_handles);
+      hc_handles = NULL;
       break;
     }
   }
@@ -733,13 +785,6 @@ run (void *cls, char *const *args, const char *cfgfile,
     return;
   }
   cfg = GNUNET_CONFIGURATION_dup (config);
-  mc_proc = 
-      GNUNET_TESTBED_controller_start (GNUNET_TESTBED_host_get_hostname_ 
-                                       (hosts[0]),
-                                       hosts[0],
-                                       cfg,
-                                       status_cb,
-                                       NULL);
   abort_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                     (GNUNET_TIME_UNIT_SECONDS, 5), &do_abort,
diff --git a/src/testbed/test_testbed_api_3peers_3controllers.c b/src/testbed/test_testbed_api_3peers_3controllers.c
index 3e75469ff..0bdcb6147 100644
--- a/src/testbed/test_testbed_api_3peers_3controllers.c
+++ b/src/testbed/test_testbed_api_3peers_3controllers.c
@@ -144,6 +144,11 @@ static struct GNUNET_CONFIGURATION_Handle *cfg2;
 static struct GNUNET_TESTBED_Operation *common_operation;
 
 /**
+ * The handle for whether a host is habitable or not
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle *hc_handle;
+
+/**
  * Abort task identifier
  */
 static GNUNET_SCHEDULER_TaskIdentifier abort_task;
@@ -256,6 +261,8 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   if (GNUNET_SCHEDULER_NO_TASK != abort_task)
     GNUNET_SCHEDULER_cancel (abort_task);
+  if (NULL != hc_handle)
+    GNUNET_TESTBED_is_host_habitable_cancel (hc_handle);
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == delayed_connect_task);
   if (NULL != reg_handle)
     GNUNET_TESTBED_cancel_registration (reg_handle);
@@ -854,6 +861,23 @@ status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *config, int stat
 
 
 /**
+ * Callbacks of this type are called by GNUNET_TESTBED_is_host_habitable to
+ * inform whether the given host is habitable or not. The Handle returned by
+ * GNUNET_TESTBED_is_host_habitable() is invalid after this callback is called
+ *
+ * @param cls NULL
+ * @param status GNUNET_YES if it is habitable; GNUNET_NO if not
+ */
+static void 
+host_habitable_cb (void *cls, int status)
+{
+  hc_handle = NULL;
+  cp1 = GNUNET_TESTBED_controller_start ("127.0.0.1", host, cfg, status_cb,
+                                         NULL);
+}
+
+
+/**
  * Main run function.
  *
  * @param cls NULL
@@ -872,7 +896,9 @@ run (void *cls, char *const *args, const char *cfgfile,
     abort_test();
     return;
   }
-  if (GNUNET_YES != GNUNET_TESTBED_is_host_habitable (host, config))
+  if (NULL == (hc_handle = GNUNET_TESTBED_is_host_habitable (host, config,
+                                                             &host_habitable_cb,
+                                                             NULL)))
   {
     GNUNET_TESTBED_host_destroy (host);
     host = NULL;
@@ -884,8 +910,6 @@ run (void *cls, char *const *args, const char *cfgfile,
     return;
   }
   cfg = GNUNET_CONFIGURATION_dup (config);
-  cp1 = GNUNET_TESTBED_controller_start ("127.0.0.1", host, cfg, status_cb,
-                                         NULL);
   abort_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                     (GNUNET_TIME_UNIT_MINUTES, 3), &do_abort,
diff --git a/src/testbed/test_testbed_api_controllerlink.c b/src/testbed/test_testbed_api_controllerlink.c
index da9fd000b..3867df27b 100644
--- a/src/testbed/test_testbed_api_controllerlink.c
+++ b/src/testbed/test_testbed_api_controllerlink.c
@@ -243,6 +243,11 @@ static struct GNUNET_TESTBED_Peer *slave2_peer;
 static struct GNUNET_TESTBED_Peer *master_peer;
 
 /**
+ * The handle for whether a host is habitable or not
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle *hc_handle;
+
+/**
  * Event mask
  */
 uint64_t event_mask;
@@ -264,6 +269,8 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   if (GNUNET_SCHEDULER_NO_TASK != abort_task)
     GNUNET_SCHEDULER_cancel (abort_task);
+  if (NULL != hc_handle)
+    GNUNET_TESTBED_is_host_habitable_cancel (hc_handle);
   if (NULL != slave3)
     GNUNET_TESTBED_host_destroy (slave3);
   if (NULL != slave2)
@@ -632,6 +639,23 @@ status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *config,
 
 
 /**
+ * Callbacks of this type are called by GNUNET_TESTBED_is_host_habitable to
+ * inform whether the given host is habitable or not. The Handle returned by
+ * GNUNET_TESTBED_is_host_habitable() is invalid after this callback is called
+ *
+ * @param cls NULL
+ * @param status GNUNET_YES if it is habitable; GNUNET_NO if not
+ */
+static void 
+host_habitable_cb (void *cls, int status)
+{
+  hc_handle = NULL;
+  cp = GNUNET_TESTBED_controller_start ("127.0.0.1", host, cfg, status_cb,
+                                        NULL);  
+}
+
+
+/**
  * Main run function.
  *
  * @param cls NULL
@@ -645,7 +669,9 @@ run (void *cls, char *const *args, const char *cfgfile,
 {
   host = GNUNET_TESTBED_host_create (NULL, NULL, 0);
   GNUNET_assert (NULL != host);
-  if (GNUNET_YES != GNUNET_TESTBED_is_host_habitable (host, config))
+  if (NULL == (hc_handle = GNUNET_TESTBED_is_host_habitable (host, config,
+                                                             &host_habitable_cb,
+                                                             NULL)))
   {
     GNUNET_TESTBED_host_destroy (host);
     host = NULL;
@@ -656,9 +682,7 @@ run (void *cls, char *const *args, const char *cfgfile,
     result = SUCCESS;
     return;
   }
-  cfg = GNUNET_CONFIGURATION_dup (config);
-  cp = GNUNET_TESTBED_controller_start ("127.0.0.1", host, cfg, status_cb,
-                                        NULL);
+  cfg = GNUNET_CONFIGURATION_dup (config);  
   abort_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                     (GNUNET_TIME_UNIT_MINUTES, 5), &do_abort,
diff --git a/src/testbed/testbed_api_hosts.c b/src/testbed/testbed_api_hosts.c
index 42b8e3864..e0455d316 100644
--- a/src/testbed/testbed_api_hosts.c
+++ b/src/testbed/testbed_api_hosts.c
@@ -482,74 +482,201 @@ GNUNET_TESTBED_is_host_registered_ (const struct GNUNET_TESTBED_Host *host,
 
 
 /**
+ * The handle for whether a host is habitable or not
+ */
+struct GNUNET_TESTBED_HostHabitableCheckHandle
+{
+  /* /\** */
+  /*  * The host to check */
+  /*  *\/ */
+  /* const struct GNUNET_TESTBED_Host *host; */
+
+  /* /\** */
+  /*  * the configuration handle to lookup the path of the testbed helper */
+  /*  *\/ */
+  /* const struct GNUNET_CONFIGURATION_Handle *cfg; */
+  
+  /**
+   * The callback to call once we have the status
+   */
+  GNUNET_TESTBED_HostHabitableCallback cb;
+
+  /**
+   * The callback closure
+   */
+  void *cb_cls;
+
+  /**
+   * The process handle for the SSH process
+   */
+  struct GNUNET_OS_Process *auxp;
+
+  /**
+   * The SSH destination address string
+   */
+  char *ssh_addr;
+
+  /**
+   * The destination port string
+   */
+  char *portstr;
+
+  /**
+   * The path for hte testbed helper binary
+   */
+  char *helper_binary_path;
+
+  /**
+   * Task id for the habitability check task
+   */
+  GNUNET_SCHEDULER_TaskIdentifier habitability_check_task;
+
+  /**
+   * How long we wait before checking the process status. Should grow
+   * exponentially
+   */
+  struct GNUNET_TIME_Relative wait_time;
+
+};
+
+
+/**
+ * Task for checking whether a host is habitable or not
+ *
+ * @param cls GNUNET_TESTBED_HostHabitableCheckHandle
+ * @param tc the scheduler task context
+ */
+static void
+habitability_check (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_TESTBED_HostHabitableCheckHandle *h = cls;
+  void *cb_cls;
+  GNUNET_TESTBED_HostHabitableCallback cb;  
+  unsigned long code;
+  enum GNUNET_OS_ProcessStatusType type;
+  int ret;
+
+  h->habitability_check_task = GNUNET_SCHEDULER_NO_TASK;
+  ret = GNUNET_OS_process_status (h->auxp, &type, &code);
+  if (GNUNET_SYSERR == ret)
+  {
+    GNUNET_break (0);
+    ret = GNUNET_NO;
+    goto call_cb;
+  }
+  if (GNUNET_NO == ret)
+  {
+    h->wait_time = GNUNET_TIME_STD_BACKOFF (h->wait_time);
+    h->habitability_check_task =
+        GNUNET_SCHEDULER_add_delayed (h->wait_time,
+                                      &habitability_check, h);
+    return;
+  }
+  GNUNET_OS_process_destroy (h->auxp);
+  h->auxp = NULL;
+  ret = (0 != code) ? GNUNET_NO : GNUNET_YES;
+  
+ call_cb:
+  GNUNET_free (h->ssh_addr);
+  GNUNET_free (h->portstr);
+  GNUNET_free (h->helper_binary_path);
+  if (NULL != h->auxp)
+    GNUNET_OS_process_destroy (h->auxp);
+  cb = h->cb;
+  cb_cls = h->cb_cls;
+  GNUNET_free (h);
+  if (NULL != cb)
+    cb (cb_cls, ret);
+}
+
+
+/**
  * Checks whether a host can be used to start testbed service
  *
  * @param host the host to check
- * @param config the configuration handle to lookup the path of the testbed helper
- * @return GNUNET_YES if testbed service can be started on the given host
- *           remotely; GNUNET_NO if not
+ * @param config the configuration handle to lookup the path of the testbed
+ *          helper
+ * @param cb the callback to call to inform about habitability of the given host
+ * @param cb_cls the closure for the callback
+ * @return NULL upon any error or a handle which can be passed to
+ *           GNUNET_TESTBED_is_host_habitable_cancel()
  */
-int
+struct GNUNET_TESTBED_HostHabitableCheckHandle *
 GNUNET_TESTBED_is_host_habitable (const struct GNUNET_TESTBED_Host *host,
-                                  const struct GNUNET_CONFIGURATION_Handle *config)
+                                  const struct GNUNET_CONFIGURATION_Handle
+                                  *config,
+                                  GNUNET_TESTBED_HostHabitableCallback cb,
+                                  void *cb_cls)
 {
+  struct GNUNET_TESTBED_HostHabitableCheckHandle *h;
   char *remote_args[11];
-  char *helper_binary_path;
-  char *portstr;
-  char *ssh_addr;
   const char *hostname;
-  struct GNUNET_OS_Process *auxp;
-  unsigned long code;
-  enum GNUNET_OS_ProcessStatusType type;
-  int ret;
   unsigned int argp;
 
-  portstr = NULL;
-  ssh_addr = NULL;
+  h = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_HostHabitableCheckHandle));
+  h->cb = cb;
+  h->cb_cls = cb_cls;
   hostname = (NULL == host->hostname) ? "127.0.0.1" : host->hostname;
   if (NULL == host->username)
-    ssh_addr = GNUNET_strdup (hostname);
+    h->ssh_addr = GNUNET_strdup (hostname);
   else
-    GNUNET_asprintf (&ssh_addr, "%s@%s", host->username, hostname);
+    GNUNET_asprintf (&h->ssh_addr, "%s@%s", host->username, hostname);
   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (config, "testbed",
                                                           "HELPER_BINARY_PATH",
-                                                          &helper_binary_path))
-      helper_binary_path = GNUNET_OS_get_libexec_binary_path (HELPER_TESTBED_BINARY);
+                                                          &h->helper_binary_path))
+    h->helper_binary_path =
+        GNUNET_OS_get_libexec_binary_path (HELPER_TESTBED_BINARY);
   argp = 0;
   remote_args[argp++] = "ssh";
-  GNUNET_asprintf (&portstr, "%u", host->port);
+  GNUNET_asprintf (&h->portstr, "%u", host->port);
   remote_args[argp++] = "-p";
-  remote_args[argp++] = portstr;
+  remote_args[argp++] = h->portstr;
   remote_args[argp++] = "-o";
   remote_args[argp++] = "BatchMode=yes";
   remote_args[argp++] = "-o";
   remote_args[argp++] = "NoHostAuthenticationForLocalhost=yes";
-  remote_args[argp++] = ssh_addr;
+  remote_args[argp++] = h->ssh_addr;
   remote_args[argp++] = "stat";
-  remote_args[argp++] = helper_binary_path;
+  remote_args[argp++] = h->helper_binary_path;
   remote_args[argp++] = NULL;
   GNUNET_assert (argp == 11);
-  auxp =
+  h->auxp =
       GNUNET_OS_start_process_vap (GNUNET_NO, GNUNET_OS_INHERIT_STD_ERR, NULL,
                                    NULL, "ssh", remote_args);
-  if (NULL == auxp)
-  {
-    GNUNET_free (ssh_addr);
-    GNUNET_free (portstr);
-    return GNUNET_NO;
-  }
-  do
+  if (NULL == h->auxp)
   {
-    ret = GNUNET_OS_process_status (auxp, &type, &code);
-    GNUNET_assert (GNUNET_SYSERR != ret);
-    (void) usleep (300);
+    GNUNET_break (0);         /* Cannot exec SSH? */
+    GNUNET_free (h->ssh_addr);
+    GNUNET_free (h->portstr);
+    GNUNET_free (h->helper_binary_path);
+    GNUNET_free (h);
+    return NULL;
   }
-  while (GNUNET_NO == ret);  
-  GNUNET_OS_process_destroy (auxp);
-  GNUNET_free (ssh_addr);
-  GNUNET_free (portstr);
-  GNUNET_free (helper_binary_path);
-  return (0 != code) ? GNUNET_NO : GNUNET_YES;
+  h->wait_time = GNUNET_TIME_STD_BACKOFF (h->wait_time);
+  h->habitability_check_task =
+      GNUNET_SCHEDULER_add_delayed (h->wait_time,
+                                    &habitability_check, h);
+  return h;
 }
 
+
+/**
+ * Function to cancel a request started using GNUNET_TESTBED_is_host_habitable()
+ *
+ * @param struct handle the habitability check handle
+ */
+void
+GNUNET_TESTBED_is_host_habitable_cancel (struct
+                                         GNUNET_TESTBED_HostHabitableCheckHandle
+                                         *handle)
+{
+  GNUNET_SCHEDULER_cancel (handle->habitability_check_task);
+  (void) GNUNET_OS_process_kill (handle->auxp, SIGTERM);
+  (void) GNUNET_OS_process_wait (handle->auxp);
+  GNUNET_OS_process_destroy (handle->auxp);
+  GNUNET_free (handle->ssh_addr);
+  GNUNET_free (handle->portstr);
+  GNUNET_free (handle->helper_binary_path);
+  GNUNET_free (handle);
+}
 /* end of testbed_api_hosts.c */
diff --git a/src/testbed/testbed_api_testbed.c b/src/testbed/testbed_api_testbed.c
index 61dc5e741..05debd98c 100644
--- a/src/testbed/testbed_api_testbed.c
+++ b/src/testbed/testbed_api_testbed.c
@@ -969,23 +969,23 @@ GNUNET_TESTBED_create_va (struct GNUNET_TESTBED_Controller *controller,
                           enum GNUNET_TESTBED_TopologyOption underlay_topology,
                           va_list va)
 {
-  unsigned int nhost;
-
-  GNUNET_assert (underlay_topology < GNUNET_TESTBED_TOPOLOGY_NONE);
-  if (num_hosts != 0)
-  {
-    for (nhost = 0; nhost < num_hosts; nhost++)
-    {
-      if (GNUNET_YES != GNUNET_TESTBED_is_host_habitable (hosts[nhost], cfg))
-      {
-        LOG (GNUNET_ERROR_TYPE_ERROR, _("Host %s cannot start testbed\n"),
-             GNUNET_TESTBED_host_get_hostname_ (hosts[nhost]));
-        break;
-      }
-    }
-    if (num_hosts != nhost)
-      return NULL;
-  }
+  /* unsigned int nhost; */
+
+  /* GNUNET_assert (underlay_topology < GNUNET_TESTBED_TOPOLOGY_NONE); */
+  /* if (num_hosts != 0) */
+  /* { */
+  /*   for (nhost = 0; nhost < num_hosts; nhost++) */
+  /*   { */
+  /*     if (GNUNET_YES != GNUNET_TESTBED_is_host_habitable (hosts[nhost], cfg)) */
+  /*     { */
+  /*       LOG (GNUNET_ERROR_TYPE_ERROR, _("Host %s cannot start testbed\n"), */
+  /*            GNUNET_TESTBED_host_get_hostname_ (hosts[nhost])); */
+  /*       break; */
+  /*     } */
+  /*   } */
+  /*   if (num_hosts != nhost) */
+  /*     return NULL; */
+  /* } */
   /* We need controller callback here to get operation done events while
      linking hosts */
   GNUNET_break (0);