Call TestMaster callback upon timeout set through configuration

fixes #2833

5 files changed, 302 insertions, 18 deletions

diff --git a/src/testbed/Makefile.am b/src/testbed/Makefile.am
index bc703ebcb..8b5cfbd9f 100644
--- a/src/testbed/Makefile.am
+++ b/src/testbed/Makefile.am
@@ -109,6 +109,7 @@ check_PROGRAMS = \
   test_testbed_api_operations \
   test_testbed_api_testbed_run \
   test_testbed_api_test \
+  test_testbed_api_test_timeout \
   test_gnunet_helper_testbed \
   test_testbed_api_topology \
   test_testbed_api_topology_clique \
@@ -135,6 +136,7 @@ if ENABLE_TEST_RUN
   test_testbed_api_3peers_3controllers \
   test_testbed_api_testbed_run \
   test_testbed_api_test \
+  test_testbed_api_test_timeout \
   test_testbed_api_statistics \
   test_testbed_api_topology \
   test_testbed_api_topology_clique \
@@ -207,6 +209,12 @@ test_testbed_api_test_LDADD = \
  $(top_builddir)/src/util/libgnunetutil.la \
  libgnunettestbed.la
 
+test_testbed_api_test_timeout_SOURCES = \
+ test_testbed_api_test_timeout.c
+test_testbed_api_test_timeout_LDADD = \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ libgnunettestbed.la
+
 test_testbed_api_topology_SOURCES = \
  test_testbed_api_topology.c
 test_testbed_api_topology_LDADD = \
@@ -288,6 +296,7 @@ test_testbed_api_statistics_LDADD = \
 
 EXTRA_DIST = \
   test_testbed_api.conf \
+  test_testbed_api_test_timeout.conf \
   test_testbed_api_testbed_run_topologyring.conf \
   test_testbed_api_testbed_run_topologyclique.conf \
   test_testbed_api_testbed_run_topologyline.conf \
diff --git a/src/testbed/test_testbed_api_test_timeout.c b/src/testbed/test_testbed_api_test_timeout.c
new file mode 100644
index 000000000..537837536
--- /dev/null
+++ b/src/testbed/test_testbed_api_test_timeout.c
@@ -0,0 +1,115 @@
+/*
+  This file is part of GNUnet
+  (C) 2008--2012 Christian Grothoff (and other contributing authors)
+
+  GNUnet is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 3, or (at your
+  option) any later version.
+
+  GNUnet is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with GNUnet; see the file COPYING.  If not, write to the
+  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+  Boston, MA 02111-1307, USA.
+*/
+
+/**
+ * @file src/testbed/test_testbed_api_test.c
+ * @brief testing cases for testing notications via test master callback upon
+ *          timeout while setting up testbed using functions
+ *          GNUNET_TESTBED_test_run()
+ * @author Sree Harsha Totakura <sreeharsha@totakura.in>
+ */
+
+#include "platform.h"
+#include "gnunet_common.h"
+#include "gnunet_testbed_service.h"
+
+
+/**
+ * Generic logging shortcut
+ */
+#define LOG(kind,...)                           \
+  GNUNET_log (kind, __VA_ARGS__)
+
+/**
+ * Number of peers we want to start
+ */
+#define NUM_PEERS 25
+
+/**
+ * Testing result
+ */
+static int result;
+
+
+/**
+ * shortcut to exit during failure
+ */
+#define FAIL_TEST(cond) do {                                            \
+    if (!(cond)) {                                                      \
+      GNUNET_break(0);                                                  \
+      GNUNET_SCHEDULER_shutdown ();                                     \
+      return;                                                           \
+    }                                                                   \
+  } while (0)
+
+
+/**
+ * Controller event callback
+ *
+ * @param cls NULL
+ * @param event the controller event
+ */
+static void
+controller_event_cb (void *cls,
+                     const struct GNUNET_TESTBED_EventInformation *event)
+{
+  FAIL_TEST (0);
+}
+
+
+/**
+ * Signature of a main function for a testcase.
+ *
+ * @param cls closure
+ * @param num_peers number of peers in 'peers'
+ * @param peers handle to peers run in the testbed
+ */
+static void
+test_master (void *cls, unsigned int num_peers,
+             struct GNUNET_TESTBED_Peer **peers_)
+{
+  FAIL_TEST (NULL == cls);
+  FAIL_TEST (0 == num_peers);
+  FAIL_TEST (NULL == peers_);
+  result = GNUNET_OK;
+  GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
+ * Main function
+ */
+int
+main (int argc, char **argv)
+{
+  uint64_t event_mask;
+
+  result = GNUNET_SYSERR;
+  event_mask = 0;
+  (void) GNUNET_TESTBED_test_run ("test_testbed_api_test",
+                                  "test_testbed_api_test_timeout.conf", NUM_PEERS,
+                                  event_mask, &controller_event_cb, NULL,
+                                  &test_master, NULL);
+  if (GNUNET_OK != result)
+    return 1;
+  return 0;
+}
+
+/* end of test_testbed_api_test.c */
diff --git a/src/testbed/test_testbed_api_test_timeout.conf b/src/testbed/test_testbed_api_test_timeout.conf
new file mode 100644
index 000000000..074de3d83
--- /dev/null
+++ b/src/testbed/test_testbed_api_test_timeout.conf
@@ -0,0 +1,92 @@
+[testbed]
+AUTOSTART = NO
+PORT = 12113
+ACCEPT_FROM = 127.0.0.1;
+HOSTNAME = localhost
+OVERLAY_TOPOLOGY = CLIQUE
+SETUP_TIMEOUT = 2 s
+#PREFIX = xterm -geometry 100x85 -T peer1 -e libtool --mode=execute gdb --args
+
+[fs]
+AUTOSTART = NO
+
+[resolver]
+AUTOSTART = NO
+
+[mesh]
+AUTOSTART = NO
+
+[dht]
+AUTOSTART = NO
+
+[block]
+plugins = dht test
+
+[dhtcache]
+QUOTA = 1 MB
+DATABASE = sqlite
+
+[transport]
+PLUGINS = tcp
+ACCEPT_FROM6 = ::1;
+ACCEPT_FROM = 127.0.0.1;
+NEIGHBOUR_LIMIT = 50
+PORT = 12365
+
+[ats]
+WAN_QUOTA_OUT = 3932160
+WAN_QUOTA_IN = 3932160
+
+[core]
+PORT = 12092
+AUTOSTART = YES
+USE_EPHEMERAL_KEYS = NO
+
+[arm]
+DEFAULTSERVICES = core transport
+PORT = 12366
+
+[transport-tcp]
+TIMEOUT = 300 s
+PORT = 12368
+
+[TESTING]
+NUM_PEERS = 5
+WEAKRANDOM = YES
+HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat
+MAX_CONCURRENT_SSH = 10
+USE_PROGRESSBARS = YES
+PEERGROUP_TIMEOUT = 2400 s
+
+[gnunetd]
+HOSTKEY = $SERVICEHOME/.hostkey
+
+[PATHS]
+SERVICEHOME = /tmp/test-testbed/
+
+[dns]
+AUTOSTART = NO
+
+[nse]
+AUTOSTART = NO
+
+[vpn]
+AUTOSTART = NO
+
+[nat]
+RETURN_LOCAL_ADDRESSES = YES
+
+[gns-helper-service-w32]
+AUTOSTART = NO
+
+[consensus]
+AUTOSTART = NO
+
+[gns]
+AUTOSTART = NO
+
+[statistics]
+AUTOSTART = NO
+
+[peerinfo]
+NO_IO = YES
diff --git a/src/testbed/testbed.conf.in b/src/testbed/testbed.conf.in
index 379cd22d1..5f91958ba 100644
--- a/src/testbed/testbed.conf.in
+++ b/src/testbed/testbed.conf.in
@@ -4,16 +4,44 @@ AUTOSTART = NO
 HOSTNAME = localhost
 HOME = $SERVICEHOME
 BINARY = gnunet-service-testbed
+
+# How long should operations wait?
 OPERATION_TIMEOUT = 30 s
-# Set this to the path where the testbed helper is installed
+
+# Set this to the path where the testbed helper is installed.  By default the
+# helper binary is searched in @prefix@/lib/gnunet/libexec/
 # HELPER_BINARY_PATH = @prefix@/lib/gnunet/libexec/gnunet-helper-testbed
+
+# Add your local network address here. For example, if you want to running
+# testbed on a group of hosts connected to network 192.168.1.0/24, then set
+#   ACCEPT_FROM = 127.0.0.1; 192.168.1.0/24;
+# Multiple network addresses can be given.  They should be separated by `;'
 ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
+
 UNIXPATH = /tmp/gnunet-service-testbed.sock
 UNIX_MATCH_UID = YES
 UNIX_MATCH_GID = YES
+
+# How many maximum number of operations can be run in parallel.  This number
+# should be decreased if the system is getting overloaded and to keep reduce the
+# load of testbed.
 MAX_PARALLEL_OPERATIONS = 1000
-MAX_PARALLEL_SERVICE_CONNECTIONS = 256
 MAX_PARALLEL_TOPOLOGY_CONFIG_OPERATIONS = 1
+
+# How many maximum number of handles to peers' services should be kept open at
+# any time.  This number also keeps a check on the number of open descriptors as
+# opening a service connection results in opening a file descriptor.
+MAX_PARALLEL_SERVICE_CONNECTIONS = 256
+
+# Size of the internal testbed cache.  It is used to cache handles to peers
+# while trying to connect them.
 CACHE_SIZE = 30
+
+# Maximum number of file descriptors a testbed controller is permitted to keep
+# open.
 MAX_OPEN_FDS = 512
+
+# How long should we wait for testbed to setup while using helper functions
+# GNUNET_TESTBED_test_run() and GNUNET_TESTBED_run()
+SETUP_TIMEOUT = 5 m
\ No newline at end of file
diff --git a/src/testbed/testbed_api_testbed.c b/src/testbed/testbed_api_testbed.c
index 3bd6f1db8..96be90e83 100644
--- a/src/testbed/testbed_api_testbed.c
+++ b/src/testbed/testbed_api_testbed.c
@@ -45,6 +45,11 @@
   LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
 
 /**
+ * The default setup timeout in seconds
+ */
+#define DEFAULT_SETUP_TIMEOUT 300
+
+/**
  * DLL of operations
  */
 struct DLLOperation
@@ -229,6 +234,11 @@ struct RunContext
   GNUNET_SCHEDULER_TaskIdentifier shutdown_run_task;
 
   /**
+   * Task to be run of a timeout
+   */
+  GNUNET_SCHEDULER_TaskIdentifier timeout_task;
+
+  /**
    * The event mask for the controller
    */
   uint64_t event_mask;
@@ -457,6 +467,11 @@ shutdown_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_SCHEDULER_cancel (rc->register_hosts_task);
     rc->register_hosts_task = GNUNET_SCHEDULER_NO_TASK;
   }
+  if (GNUNET_SCHEDULER_NO_TASK != rc->timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (rc->timeout_task);
+    rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  }
   if (NULL != rc->reg_handle)
   {
     GNUNET_TESTBED_cancel_registration (rc->reg_handle);
@@ -499,22 +514,16 @@ shutdown_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
 
 /**
- * Task to call master task
+ * call test master callback
  *
  * @param cls the run context
  * @param tc the task context
  */
 static void
-call_master (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+call_master (struct RunContext *rc)
 {
-  struct RunContext *rc = cls;
-
-  if (NULL != rc->topology_operation)
-  {
-    DEBUG ("Overlay topology generated in %s\n", prof_time (rc));
-    GNUNET_TESTBED_operation_done (rc->topology_operation);
-    rc->topology_operation = NULL;
-  }
+  GNUNET_SCHEDULER_cancel (rc->timeout_task);
+  rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
   if (NULL != rc->test_master)
     rc->test_master (rc->test_master_cls, rc->num_peers, rc->peers);
 }
@@ -535,9 +544,11 @@ topology_completion_callback (void *cls, unsigned int nsuccess,
 {
   struct RunContext *rc = cls;
 
+  DEBUG ("Overlay topology generated in %s\n", prof_time (rc));
+  GNUNET_TESTBED_operation_done (rc->topology_operation);
+  rc->topology_operation = NULL;
   rc->state = RC_READY;
-  GNUNET_SCHEDULER_add_continuation (&call_master, rc,
-                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+  call_master (rc);
 }
 
 
@@ -706,8 +717,7 @@ call_cc:
     }
   }
   rc->state = RC_READY;
-  GNUNET_SCHEDULER_add_continuation (&call_master, rc,
-                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+  call_master (rc);
 }
 
 
@@ -800,8 +810,8 @@ controller_status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   if (status != GNUNET_OK)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Controller crash detected. Shutting down.\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("Controller crash detected. Shutting down.\n"));
     rc->cproc = NULL;
     if (NULL != rc->peers)
     {
@@ -949,6 +959,26 @@ host_habitable_cb (void *cls, const struct GNUNET_TESTBED_Host *host,
 
 
 /**
+ * Task run upon timeout while setting up the testbed
+ *
+ * @param cls the RunContext
+ * @param tc the task context
+ */
+static void
+timeout_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct RunContext *rc = cls;
+  
+  rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  LOG (GNUNET_ERROR_TYPE_ERROR, _("Shutting down testbed due to timeout while setup.\n"));
+  shutdown_now (rc);
+   if (NULL != rc->test_master)
+    rc->test_master (rc->test_master_cls, 0, NULL);
+   rc->test_master = NULL;
+}
+
+
+/**
  * Convenience method for running a testbed with
  * a single call.  Underlay and overlay topology
  * are configured using the "UNDERLAY" and "OVERLAY"
@@ -985,6 +1015,7 @@ GNUNET_TESTBED_run (const char *host_filename,
 {
   struct RunContext *rc;
   char *topology;
+  struct GNUNET_TIME_Relative timeout;
   unsigned long long random_links;
   unsigned int hid;
   unsigned int nhost;
@@ -1106,6 +1137,15 @@ GNUNET_TESTBED_run (const char *host_filename,
     rc->cproc =
         GNUNET_TESTBED_controller_start ("127.0.0.1", rc->h, rc->cfg,
                                          &controller_status_cb, rc);
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED",
+                                                        "SETUP_TIMEOUT",
+                                                        &timeout))
+  {
+    timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                             DEFAULT_SETUP_TIMEOUT);
+  }
+  rc->timeout_task =
+      GNUNET_SCHEDULER_add_delayed (timeout, &timeout_task, rc);
   rc->shutdown_run_task =
       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_run,
                                     rc);