Implement GNUNET_TESTBED_get_statistics() - resolves #2662

Test case for testing GNUNET_TESTBED_get_statistics()
Fix waiting for SHUTDOWN_PEERS reply when controller crashes

7 files changed, 481 insertions, 18 deletions

diff --git a/src/testbed/Makefile.am b/src/testbed/Makefile.am
index 2189de3cb..bc703ebcb 100644
--- a/src/testbed/Makefile.am
+++ b/src/testbed/Makefile.am
@@ -120,7 +120,8 @@ check_PROGRAMS = \
   test_testbed_api_testbed_run_topology2dtorus \
   test_testbed_api_testbed_run_topologysmallworld \
   test_testbed_api_testbed_run_topologyfromfile \
-  test_testbed_api_testbed_run_topologyscalefree
+  test_testbed_api_testbed_run_topologyscalefree \
+  test_testbed_api_statistics
 
 if ENABLE_TEST_RUN
  TESTS = \
@@ -134,6 +135,7 @@ if ENABLE_TEST_RUN
   test_testbed_api_3peers_3controllers \
   test_testbed_api_testbed_run \
   test_testbed_api_test \
+  test_testbed_api_statistics \
   test_testbed_api_topology \
   test_testbed_api_topology_clique \
   test_testbed_api_testbed_run_topologyrandom \
@@ -278,6 +280,12 @@ test_testbed_api_testbed_run_topologyscalefree_LDADD = \
  $(top_builddir)/src/util/libgnunetutil.la \
  libgnunettestbed.la
 
+test_testbed_api_statistics_SOURCES = \
+ test_testbed_api_statistics.c
+test_testbed_api_statistics_LDADD = \
+ $(top_builddir)/src/util/libgnunetutil.la \
+ libgnunettestbed.la
+
 EXTRA_DIST = \
   test_testbed_api.conf \
   test_testbed_api_testbed_run_topologyring.conf \
diff --git a/src/testbed/test_testbed_api_statistics.conf b/src/testbed/test_testbed_api_statistics.conf
new file mode 100644
index 000000000..36de44147
--- /dev/null
+++ b/src/testbed/test_testbed_api_statistics.conf
@@ -0,0 +1,93 @@
+[testbed]
+AUTOSTART = NO
+PORT = 12113
+ACCEPT_FROM = 127.0.0.1;
+HOSTNAME = localhost
+NEIGHBOUR_LIMIT = 100
+OVERLAY_TOPOLOGY = 2D_TORUS
+#PREFIX = xterm -geometry 100x85 -T peer1 -e libtool --mode=execute gdb --args
+
+[fs]
+AUTOSTART = NO
+
+[resolver]
+AUTOSTART = NO
+
+[mesh]
+AUTOSTART = NO
+
+[dht]
+AUTOSTART = NO
+
+[block]
+plugins = dht test
+
+[dhtcache]
+QUOTA = 1 MB
+DATABASE = sqlite
+
+[transport]
+PLUGINS = tcp
+ACCEPT_FROM6 = ::1;
+ACCEPT_FROM = 127.0.0.1;
+NEIGHBOUR_LIMIT = 50
+PORT = 12365
+
+[ats]
+WAN_QUOTA_OUT = 3932160
+WAN_QUOTA_IN = 3932160
+
+[core]
+PORT = 12092
+AUTOSTART = YES
+USE_EPHEMERAL_KEYS = NO
+
+[arm]
+DEFAULTSERVICES = core transport
+PORT = 12366
+
+[transport-tcp]
+TIMEOUT = 300 s
+PORT = 12368
+
+[TESTING]
+NUM_PEERS = 5
+WEAKRANDOM = YES
+HOSTKEYSFILE = ../../contrib/testing_hostkeys.dat
+MAX_CONCURRENT_SSH = 10
+USE_PROGRESSBARS = YES
+PEERGROUP_TIMEOUT = 2400 s
+
+[gnunetd]
+HOSTKEY = $SERVICEHOME/.hostkey
+
+[PATHS]
+SERVICEHOME = /tmp/test-testbed/
+
+[dns]
+AUTOSTART = NO
+
+[nse]
+AUTOSTART = NO
+
+[vpn]
+AUTOSTART = NO
+
+[nat]
+RETURN_LOCAL_ADDRESSES = YES
+
+[gns-helper-service-w32]
+AUTOSTART = NO
+
+[consensus]
+AUTOSTART = NO
+
+[gns]
+AUTOSTART = NO
+
+[statistics]
+AUTOSTART = YES
+PORT = 30
+
+[peerinfo]
+NO_IO = YES
diff --git a/src/testbed/testbed_api.c b/src/testbed/testbed_api.c
index 0c891b447..117b39833 100644
--- a/src/testbed/testbed_api.c
+++ b/src/testbed/testbed_api.c
@@ -47,13 +47,13 @@
  * Generic logging shorthand
  */
 #define LOG(kind, ...)                          \
-  GNUNET_log_from (kind, "testbed-api", __VA_ARGS__);
+  GNUNET_log_from (kind, "testbed-api", __VA_ARGS__)
 
 /**
  * Debug logging
  */
 #define LOG_DEBUG(...)                          \
-  LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
 
 /**
  * Relative time seconds shorthand
diff --git a/src/testbed/testbed_api_operations.c b/src/testbed/testbed_api_operations.c
index 3e7eb91bd..d213d3334 100644
--- a/src/testbed/testbed_api_operations.c
+++ b/src/testbed/testbed_api_operations.c
@@ -280,6 +280,23 @@ GNUNET_TESTBED_operation_queue_destroy_ (struct OperationQueue *queue)
 
 
 /**
+ * Destroys the operation queue if it is empty.  If not empty return GNUNET_NO.
+ *
+ * @param queue the queue to destroy if empty
+ * @return GNUNET_YES if the queue is destroyed.  GNUNET_NO if not (because it
+ *           is not empty)
+ */
+int
+GNUNET_TESTBED_operation_queue_destroy_empty_ (struct OperationQueue *q)
+{
+  if (NULL != q->head)
+    return GNUNET_NO;
+  GNUNET_TESTBED_operation_queue_destroy_ (q);
+  return GNUNET_YES;
+}
+
+
+/**
  * Function to reset the maximum number of operations in the given queue. If
  * max_active is lesser than the number of currently active operations, the
  * active operations are not stopped immediately.
diff --git a/src/testbed/testbed_api_operations.h b/src/testbed/testbed_api_operations.h
index 61b45e236..403dd889c 100644
--- a/src/testbed/testbed_api_operations.h
+++ b/src/testbed/testbed_api_operations.h
@@ -59,6 +59,17 @@ GNUNET_TESTBED_operation_queue_destroy_ (struct OperationQueue *queue);
 
 
 /**
+ * Destroys the operation queue if it is empty.  If not empty return GNUNET_NO.
+ *
+ * @param queue the queue to destroy if empty
+ * @return GNUNET_YES if the queue is destroyed.  GNUNET_NO if not (because it
+ *           is not empty)
+ */
+int
+GNUNET_TESTBED_operation_queue_destroy_empty_ (struct OperationQueue *q);
+
+
+/**
  * Function to reset the maximum number of operations in the given queue. If
  * max_active is lesser than the number of currently active operations, the
  * active operations are not stopped immediately.
diff --git a/src/testbed/testbed_api_statistics.c b/src/testbed/testbed_api_statistics.c
index f013c0302..d6af1bce2 100644
--- a/src/testbed/testbed_api_statistics.c
+++ b/src/testbed/testbed_api_statistics.c
@@ -1,6 +1,6 @@
 /*
       This file is part of GNUnet
-      (C) 2008--2012 Christian Grothoff (and other contributing authors)
+      (C) 2008--2013 Christian Grothoff (and other contributing authors)
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
@@ -25,8 +25,324 @@
  * @author Sree Harsha Totakura
  */
 #include "platform.h"
+#include "gnunet_common.h"
 #include "gnunet_testbed_service.h"
 
+#include "testbed_api_operations.h"
+
+
+/**
+ * Generic logging shorthand
+ */
+#define LOG(kind,...)                           \
+  GNUNET_log_from (kind, "testbed-api-statistics", __VA_ARGS__)
+
+/**
+ * Debug logging shorthand
+ */
+#define LOG_DEBUG(...)                          \
+  LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
+
+
+/**
+ * Context information for use in GNUNET_TESTBED_get_statistics()
+ */
+struct GetStatsContext
+{
+  /**
+   * The main operation we generate while creating this context
+   */
+  struct GNUNET_TESTBED_Operation *main_op;
+
+  /**
+   * The service connect operations we create to open connection to the
+   * statistics service of each given peer
+   */
+  struct  GNUNET_TESTBED_Operation **ops;
+
+  /**
+   * The array of peers whose statistics services are to be accessed
+   */
+  struct GNUNET_TESTBED_Peer **peers;
+
+  /**
+   * The iterator to call with statistics information
+   */
+  GNUNET_TESTBED_StatisticsIterator proc;
+
+  /**
+   * The callback to call when we are done iterating through all peers'
+   * statistics services
+   */
+  GNUNET_TESTBED_OperationCompletionCallback cont;
+
+  /**
+   * The closure for the above callbacks
+   */
+  void *cb_cls;
+
+  /**
+   * The task for calling the continuation callback
+   */
+  GNUNET_SCHEDULER_TaskIdentifier call_completion_task_id;
+  
+  /**
+   * The number of peers present in the peers array.  This number also
+   * represents the number of service connect operations in the ops array
+   */
+  unsigned int num_peers;
+
+  /**
+   * How many peers' statistics have we iterated through
+   */
+  unsigned int num_completed;
+
+};
+
+
+/**
+ * Context information with respect to a particular peer
+ */
+struct PeerGetStatsContext
+{
+  /**
+   * The GetStatsContext which is associated with this context
+   */
+  struct GetStatsContext *sc;
+
+  /**
+   * The handle from GNUNET_STATISTICS_get()
+   */
+  struct GNUNET_STATISTICS_GetHandle *get_handle;
+
+  /**
+   * The index of this peer in the peers array of GetStatsContext
+   */
+  unsigned int peer_index;
+};
+
+
+/**
+ * A no-wait operation queue
+ */
+static struct OperationQueue *no_wait_queue;
+
+
+/**
+ * Call statistics operation completion.  We call it in a separate task because
+ * the iteration_completion_cb() cannot destroy statistics handle which will be
+ * the case if the user calles GNUNET_TESTBED_operation_done() on the
+ * get_statistics operation.
+ *
+ * @param cls the GetStatsContext
+ * @param tc the scheduler task context
+ */
+static void
+call_completion_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GetStatsContext *sc = cls;
+
+  GNUNET_assert (sc->call_completion_task_id != GNUNET_SCHEDULER_NO_TASK);
+  sc->call_completion_task_id = GNUNET_SCHEDULER_NO_TASK;
+  LOG_DEBUG ("Calling get_statistics() continuation callback\n");
+  sc->cont (sc->cb_cls, sc->main_op, NULL);
+}
+
+
+/**
+ * Continuation called by the "get_all" and "get" functions.
+ *
+ * @param cls the PeerGetStatsContext
+ * @param success GNUNET_OK if statistics were
+ *        successfully obtained, GNUNET_SYSERR if not.
+ */
+static void
+iteration_completion_cb (void *cls, int success)
+{
+  struct PeerGetStatsContext *peer_sc = cls;
+  struct GetStatsContext *sc;
+
+  if (NULL == peer_sc->get_handle)
+  {
+    /* We are being called after we cancelled the GetHandle */
+    GNUNET_assert (GNUNET_SYSERR == success);
+    return;
+  }
+  GNUNET_break (GNUNET_OK == success);
+  sc = peer_sc->sc;
+  peer_sc->get_handle = NULL; 
+  sc->num_completed++;
+  if (sc->num_completed == sc->num_peers)
+  {
+    LOG_DEBUG ("Scheduling to call iteration completion callback\n");
+    sc->call_completion_task_id =
+        GNUNET_SCHEDULER_add_now (&call_completion_task, sc);
+  }
+}
+
+
+/**
+ * Callback function to process statistic values.
+ *
+ * @param cls the PeerGetStatsContext
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+iterator_cb (void *cls, const char *subsystem,
+             const char *name, uint64_t value,
+             int is_persistent)
+{
+  struct PeerGetStatsContext *peer_sc = cls;
+  struct GetStatsContext *sc;
+  struct GNUNET_TESTBED_Peer *peer;
+  int ret;
+
+  sc = peer_sc->sc;
+  peer = sc->peers[peer_sc->peer_index];
+  LOG_DEBUG ("Peer %u: [%s,%s] -> %lu\n", peer_sc->peer_index,
+             subsystem, name, (unsigned long) value);
+  ret = sc->proc (sc->cb_cls, peer,
+                  subsystem, name, value, is_persistent);  
+  if (GNUNET_SYSERR == ret)
+    LOG_DEBUG ("Aborting iteration for peer %u\n", peer_sc->peer_index);
+  return ret;
+}
+
+
+/**
+ * Called after opening a connection to the statistics service of a peer
+ *
+ * @param cls the PeerGetStatsContext
+ * @param op the operation that has been finished
+ * @param ca_result the service handle returned from GNUNET_TESTBED_ConnectAdapter()
+ * @param emsg error message in case the operation has failed; will be NULL if
+ *          operation has executed successfully.
+ */
+static void
+service_connect_comp (void *cls,
+                      struct GNUNET_TESTBED_Operation *op,
+                      void *ca_result,
+                      const char *emsg)
+{
+  struct PeerGetStatsContext *peer_sc = cls;
+  struct GNUNET_STATISTICS_Handle *h = ca_result;
+  
+  LOG_DEBUG ("Retrieving statistics of peer %u\n", peer_sc->peer_index);
+  peer_sc->get_handle =
+      GNUNET_STATISTICS_get (h, NULL, NULL,
+                             GNUNET_TIME_UNIT_FOREVER_REL,
+                             &iteration_completion_cb,
+                             iterator_cb, peer_sc);
+}
+
+
+/**
+ * Adapter function called to establish a connection to the statistics service
+ * of a peer.
+ *
+ * @param cls the PeerGetStatsContext
+ * @param cfg configuration of the peer to connect to; will be available until
+ *          GNUNET_TESTBED_operation_done() is called on the operation returned
+ *          from GNUNET_TESTBED_service_connect()
+ * @return service handle to return in 'op_result', NULL on error
+ */
+static void * 
+statistics_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
+{
+  struct PeerGetStatsContext *peer_sc = cls;
+  
+  LOG_DEBUG ("Connecting to statistics service of peer %u\n",
+             peer_sc->peer_index);
+  return GNUNET_STATISTICS_create ("<testbed-api>", cfg);
+}
+
+
+/**
+ * Adapter function called to destroy statistics connection
+ *
+ * @param cls the PeerGetStatsContext
+ * @param op_result service handle returned from the connect adapter
+ */
+static void 
+statistics_da (void *cls, void *op_result)
+{
+  struct PeerGetStatsContext *peer_sc = cls;
+
+  if (NULL != peer_sc->get_handle)
+  {
+    GNUNET_STATISTICS_get_cancel (peer_sc->get_handle);
+    peer_sc->get_handle = NULL;
+  }
+  GNUNET_STATISTICS_destroy ((struct GNUNET_STATISTICS_Handle *) op_result,
+                             GNUNET_NO);
+  GNUNET_free (peer_sc);
+}
+
+
+/**
+ * Function called when get_statistics operation is ready
+ *
+ * @param cls the GetStatsContext
+ */
+static void
+opstart_get_stats (void *cls)
+{
+  struct GetStatsContext *sc = cls;
+  struct PeerGetStatsContext *peer_sc;
+  unsigned int peer;
+
+  LOG_DEBUG ("Starting get_statistics operation\n");
+  sc->ops = GNUNET_malloc (sc->num_peers * 
+                           sizeof (struct GNUNET_TESTBED_Operation *));
+  for (peer = 0; peer < sc->num_peers; peer++)
+  {
+    peer_sc = GNUNET_malloc (sizeof (struct PeerGetStatsContext));
+    peer_sc->sc = sc;
+    peer_sc->peer_index = peer;
+    sc->ops[peer] = 
+        GNUNET_TESTBED_service_connect (sc, sc->peers[peer], "statistics",
+                                        &service_connect_comp,
+                                        peer_sc,
+                                        &statistics_ca,
+                                        &statistics_da,
+                                        peer_sc);
+  }
+}
+
+
+/**
+ * Function called when get_statistics operation is cancelled or marked as done
+ *
+ * @param cls the GetStatsContext
+ */
+static void
+oprelease_get_stats (void *cls)
+{
+  struct GetStatsContext *sc = cls;
+  unsigned int peer;
+  
+  LOG_DEBUG ("Cleaning up get_statistics operation\n");
+  if (GNUNET_SCHEDULER_NO_TASK != sc->call_completion_task_id)
+    GNUNET_SCHEDULER_cancel (sc->call_completion_task_id);
+  if (NULL != sc->ops)
+  {
+    for (peer = 0; peer < sc->num_peers; peer++)
+    {
+      if (NULL != sc->ops[peer])
+        GNUNET_TESTBED_operation_done (sc->ops[peer]);
+    }
+    GNUNET_free (sc->ops);
+  }
+  GNUNET_free (sc);
+  if (GNUNET_YES == 
+      GNUNET_TESTBED_operation_queue_destroy_empty_ (no_wait_queue))
+    no_wait_queue = NULL;
+}
+
 
 /**
  * Convenience method that iterates over all (running) peers
@@ -46,10 +362,25 @@ GNUNET_TESTBED_get_statistics (unsigned int num_peers,
                                GNUNET_TESTBED_OperationCompletionCallback cont,
                                void *cls)
 {
-  // FIXME: not implemented, but clients will kind-of work if we do this:
-  GNUNET_break (0);
-  cont (cls, NULL, "not implemented");
-  return NULL;
+  struct GetStatsContext *sc;
+
+  GNUNET_assert (NULL != proc);
+  GNUNET_assert (NULL != cont);
+  if (NULL == no_wait_queue)
+    no_wait_queue =
+        GNUNET_TESTBED_operation_queue_create_ (UINT_MAX);
+  sc = GNUNET_malloc (sizeof (struct GetStatsContext));
+  sc->peers = peers;
+  sc->proc = proc;
+  sc->cont = cont;
+  sc->cb_cls = cls;
+  sc->num_peers = num_peers;
+  sc->main_op =
+      GNUNET_TESTBED_operation_create_ (sc, &opstart_get_stats,
+                                        &oprelease_get_stats);
+  GNUNET_TESTBED_operation_queue_insert_ (no_wait_queue, sc->main_op);
+  GNUNET_TESTBED_operation_begin_wait_ (sc->main_op);
+  return sc->main_op;
 }
 
 
diff --git a/src/testbed/testbed_api_testbed.c b/src/testbed/testbed_api_testbed.c
index 24ca2653c..3bd6f1db8 100644
--- a/src/testbed/testbed_api_testbed.c
+++ b/src/testbed/testbed_api_testbed.c
@@ -800,18 +800,21 @@ controller_status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   if (status != GNUNET_OK)
   {
-    switch (rc->state)
-    {
-    case RC_INIT:
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Testbed startup failed\n");
-      return;
-    default:
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   "Controller crash detected. Shutting down.\n");
-      rc->cproc = NULL;
-      shutdown_now (rc);
-      return;
+    rc->cproc = NULL;
+    if (NULL != rc->peers)
+    {
+      GNUNET_free (rc->peers);
+      rc->peers = NULL;
+    }
+    if (NULL != rc->c)
+    {
+      GNUNET_TESTBED_controller_disconnect (rc->c);
+      rc->c = NULL;
     }
+    shutdown_now (rc);
+    return;
   }
   GNUNET_CONFIGURATION_destroy (rc->cfg);
   rc->cfg = GNUNET_CONFIGURATION_dup (cfg);