-topology test case relaxed to accommodated overlay timeouts

-bounded overlay connects
-topology API to retry on timeouts
-test case configuration to include bounds on topology
-cleanup of occs during shutdown

5 files changed, 137 insertions, 52 deletions

diff --git a/src/testbed/gnunet-service-testbed.c b/src/testbed/gnunet-service-testbed.c
index 9c4b90653..2371d2825 100644
--- a/src/testbed/gnunet-service-testbed.c
+++ b/src/testbed/gnunet-service-testbed.c
@@ -58,7 +58,7 @@
 /**
  * Default timeout for operations which may take some time
  */
-#define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60)
+#define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
 
 /**
  * The main context information associated with the client which started us
@@ -497,6 +497,16 @@ struct Peer
 struct OverlayConnectContext
 {
   /**
+   * The next pointer for maintaining a DLL
+   */
+  struct OverlayConnectContext *next;
+
+  /**
+   * The prev pointer for maintaining a DLL
+   */
+  struct OverlayConnectContext *prev;
+  
+  /**
    * The client which has requested for overlay connection
    */
   struct GNUNET_SERVER_Client *client;
@@ -573,6 +583,11 @@ struct OverlayConnectContext
   GNUNET_SCHEDULER_TaskIdentifier timeout_task;
 
   /**
+   * The id of the cleanup task
+   */
+  GNUNET_SCHEDULER_TaskIdentifier cleanup_task;
+
+  /**
    * The id of peer A
    */
   uint32_t peer_id;
@@ -777,6 +792,16 @@ static struct MessageQueue *mq_head;
 static struct MessageQueue *mq_tail;
 
 /**
+ * DLL head for OverlayConnectContext DLL - to be used to clean up during shutdown
+ */
+static struct OverlayConnectContext *occq_head;
+
+/**
+ * DLL tail for OverlayConnectContext DLL
+ */
+static struct OverlayConnectContext *occq_tail;
+
+/**
  * Array of hosts
  */
 static struct GNUNET_TESTBED_Host **host_list;
@@ -865,6 +890,7 @@ transmit_ready_notify (void *cls, size_t size, void *buf)
   size = ntohs (mq_entry->msg->size);
   memcpy (buf, mq_entry->msg, size);
   GNUNET_free (mq_entry->msg);
+  GNUNET_SERVER_client_drop (mq_entry->client);
   GNUNET_CONTAINER_DLL_remove (mq_head, mq_tail, mq_entry);
   GNUNET_free (mq_entry);
   mq_entry = mq_head;
@@ -899,6 +925,7 @@ queue_message (struct GNUNET_SERVER_Client *client,
   mq_entry = GNUNET_malloc (sizeof (struct MessageQueue));
   mq_entry->msg = msg;
   mq_entry->client = client;
+  GNUNET_SERVER_client_keep (client);
   LOG_DEBUG ("Queueing message of type %u, size %u for sending\n", type,
              ntohs (msg->size));
   GNUNET_CONTAINER_DLL_insert_tail (mq_head, mq_tail, mq_entry);
@@ -2537,16 +2564,13 @@ handle_peer_get_config (void *cls, struct GNUNET_SERVER_Client *client,
 
 
 /**
- * Task for cleaing up overlay connect context structure
+ * Cleanup overlay connect context structure
  *
- * @param cls the overlay connect context
- * @param tc the task context
+ * @param occ the overlay connect context
  */
 static void
-occ_cleanup (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleanup_occ (struct OverlayConnectContext *occ)
 {
-  struct OverlayConnectContext *occ = cls;
-
   LOG_DEBUG ("Cleaning up occ\n");
   GNUNET_free_non_null (occ->emsg);
   GNUNET_free_non_null (occ->hello);
@@ -2555,6 +2579,10 @@ occ_cleanup (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_TESTBED_forward_operation_msg_cancel_ (occ->opc);
   if (GNUNET_SCHEDULER_NO_TASK != occ->send_hello_task)
     GNUNET_SCHEDULER_cancel (occ->send_hello_task);
+  if (GNUNET_SCHEDULER_NO_TASK != occ->cleanup_task)
+    GNUNET_SCHEDULER_cancel (occ->cleanup_task);
+  if (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task)
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
   if (NULL != occ->ch)
     GNUNET_CORE_disconnect (occ->ch);
   if (NULL != occ->ghh)
@@ -2563,11 +2591,28 @@ occ_cleanup (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_TRANSPORT_disconnect (occ->p1th);
   if (NULL != occ->p2th)
     GNUNET_TRANSPORT_disconnect (occ->p2th);
+  GNUNET_CONTAINER_DLL_remove (occq_head, occq_tail, occ);
   GNUNET_free (occ);
 }
 
 
 /**
+ * Task for cleaing up overlay connect context structure
+ *
+ * @param cls the overlay connect context
+ * @param tc the task context
+ */
+static void
+do_cleanup_occ (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct OverlayConnectContext *occ = cls;
+  
+  occ->cleanup_task = GNUNET_SCHEDULER_NO_TASK;
+  cleanup_occ (occ);
+}
+
+
+/**
  * Task which will be run when overlay connect request has been timed out
  *
  * @param cls the OverlayConnectContext
@@ -2584,7 +2629,7 @@ timeout_overlay_connect (void *cls,
        "Timeout while connecting peers %u and %u\n",
        occ->peer_id, occ->other_peer_id);
   send_operation_fail_msg (occ->client, occ->op_id, occ->emsg);
-  occ_cleanup (occ, tc);
+  cleanup_occ (occ);
 }
 
 
@@ -2652,7 +2697,8 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
   msg->peer2 = htonl (occ->other_peer_id);
   msg->operation_id = GNUNET_htonll (occ->op_id);
   queue_message (occ->client, &msg->header);
-  GNUNET_SCHEDULER_add_now (&occ_cleanup, occ);
+  occ->cleanup_task = GNUNET_SCHEDULER_add_now (&do_cleanup_occ, occ);
+  //cleanup_occ (occ);
 }
 
 
@@ -2949,11 +2995,12 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
                         const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TESTBED_OverlayConnectMessage *msg;
-  struct OverlayConnectContext *occ;
   const struct GNUNET_CORE_MessageHandler no_handlers[] = {
     {NULL, 0, 0}
   };
   struct Peer *peer;
+  struct OverlayConnectContext *occ;
+  struct GNUNET_TESTBED_Controller *peer2_controller;
   uint64_t operation_id;
   uint32_t p1;
   uint32_t p2; 
@@ -3069,13 +3116,8 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));
-  GNUNET_SERVER_client_keep (client);
-  occ->client = client;
-  occ->peer_id = p1;
-  occ->other_peer_id = p2;
-  occ->peer = peer_list[p1];
-  occ->op_id = GNUNET_ntohll (msg->operation_id);  
+
+  peer2_controller = NULL;
   if ((p2 >= peer_list_size) || (NULL == peer_list[p2]))
   {   
     if ((peer2_host_id >= slave_list_size)
@@ -3084,8 +3126,7 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
       struct GNUNET_TESTBED_NeedControllerConfig *reply;
 
       LOG_DEBUG ("Need controller configuration for connecting peers %u and %u\n",
-                 occ->peer_id, occ->other_peer_id);
-      GNUNET_free (occ);
+                 p1, p2);
       reply = GNUNET_malloc (sizeof (struct
                                      GNUNET_TESTBED_NeedControllerConfig)); 
       reply->header.size = htons (sizeof (struct
@@ -3099,12 +3140,11 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     }
     else
     {
-      occ->peer2_controller = slave_list[peer2_host_id]->controller;
-      if (NULL == occ->peer2_controller)
+      //occ->peer2_controller = slave_list[peer2_host_id]->controller;
+      peer2_controller = slave_list[peer2_host_id]->controller;
+      if (NULL == peer2_controller)
       {
         GNUNET_break (0);       /* What's going on? */
-        GNUNET_SERVER_client_drop (client);
-        GNUNET_free (occ);
         GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
         return;
       }
@@ -3112,9 +3152,18 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
   }
   else
   {
-    if (GNUNET_YES == peer_list[occ->other_peer_id]->is_remote)
-      occ->peer2_controller = peer_list[occ->other_peer_id]->details.remote.slave->controller;
+    if (GNUNET_YES == peer_list[p2]->is_remote)
+      peer2_controller = peer_list[p2]->details.remote.slave->controller;
   }
+  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));
+  GNUNET_CONTAINER_DLL_insert_tail (occq_head, occq_tail, occ);
+  GNUNET_SERVER_client_keep (client);
+  occ->client = client;
+  occ->peer_id = p1;
+  occ->other_peer_id = p2;
+  occ->peer = peer_list[p1];
+  occ->op_id = GNUNET_ntohll (msg->operation_id);
+  occ->peer2_controller = peer2_controller;
   /* Get the identity of the second peer */
   if (NULL != occ->peer2_controller)
   {
@@ -3449,6 +3498,7 @@ static void
 shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct LCFContextQueue *lcfq;
+  struct OverlayConnectContext *occ;
   uint32_t id;
 
   shutdown_task_id = GNUNET_SCHEDULER_NO_TASK;
@@ -3472,6 +3522,8 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_CONTAINER_DLL_remove (lcfq_head, lcfq_tail, lcfq);
     GNUNET_free (lcfq);
   }
+  while (NULL != (occ = occq_head))
+    cleanup_occ (occ);
   /* Clear peer list */
   for (id = 0; id < peer_list_size; id++)
     if (NULL != peer_list[id])
@@ -3605,8 +3657,9 @@ testbed_run (void *cls, struct GNUNET_SERVER_Handle *server,
   GNUNET_SERVER_disconnect_notify (server, &client_disconnect_cb, NULL);
   ss_map = GNUNET_CONTAINER_multihashmap_create (5, GNUNET_NO);
   shutdown_task_id =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
-                                    &shutdown_task, NULL);
+      GNUNET_SCHEDULER_add_delayed_with_priority (GNUNET_TIME_UNIT_FOREVER_REL,
+                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                                  &shutdown_task, NULL);
   LOG_DEBUG ("Testbed startup complete\n");
   event_mask = 1LL << GNUNET_TESTBED_ET_OPERATION_FINISHED;
 }
diff --git a/src/testbed/test_testbed_api.conf b/src/testbed/test_testbed_api.conf
index 2fdefcc1f..dfdfdb93b 100644
--- a/src/testbed/test_testbed_api.conf
+++ b/src/testbed/test_testbed_api.conf
@@ -3,6 +3,8 @@ AUTOSTART = NO
 PORT = 12113
 ACCEPT_FROM = 127.0.0.1;
 HOSTNAME = localhost
+NEIGHBOUR_LIMIT = 100
+MAX_PARALLEL_OVERLAY_CONNECT_OPERATIONS = 10
 #PREFIX = xterm -geometry 100x85 -T peer1 -e libtool --mode=execute gdb --args
 
 [fs]
diff --git a/src/testbed/test_testbed_api_topology.c b/src/testbed/test_testbed_api_topology.c
index 91a9f4bde..8e673d293 100644
--- a/src/testbed/test_testbed_api_topology.c
+++ b/src/testbed/test_testbed_api_topology.c
@@ -44,6 +44,11 @@ static struct GNUNET_TESTBED_Peer **peers;
 static struct GNUNET_TESTBED_Operation *op;
 
 /**
+ * Shutdown task
+ */
+static GNUNET_SCHEDULER_TaskIdentifier shutdown_task;
+
+/**
  * Testing result
  */
 static int result;
@@ -63,6 +68,7 @@ static unsigned int overlay_connects;
 static void
 do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
+  shutdown_task = GNUNET_SCHEDULER_NO_TASK;
   if (NULL != op)
   {
     GNUNET_TESTBED_operation_done (op);
@@ -91,6 +97,9 @@ controller_event_cb (void *cls,
       GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
     }
     break;
+  case GNUNET_TESTBED_ET_OPERATION_FINISHED:
+    GNUNET_assert (NULL != event->details.operation_finished.emsg);
+    break;
   default:
     GNUNET_break (0);
     if ((GNUNET_TESTBED_ET_OPERATION_FINISHED == event->type) && 
@@ -99,7 +108,8 @@ controller_event_cb (void *cls,
                   "An operation failed with error: %s\n",
                   event->details.operation_finished.emsg);
     result = GNUNET_SYSERR;
-    GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
+    GNUNET_SCHEDULER_cancel (shutdown_task);
+    shutdown_task = GNUNET_SCHEDULER_add_now (&do_shutdown, NULL);
   }  
 }
 
@@ -128,6 +138,9 @@ test_master (void *cls, unsigned int num_peers,
                                                   GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI,
                                                   NUM_PEERS);
   GNUNET_assert (NULL != op);
+  shutdown_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                                (GNUNET_TIME_UNIT_SECONDS, 120),
+                                                do_shutdown, NULL);
 }
 
 
diff --git a/src/testbed/testbed_api_peers.c b/src/testbed/testbed_api_peers.c
index 3c3f478ea..07103ca89 100644
--- a/src/testbed/testbed_api_peers.c
+++ b/src/testbed/testbed_api_peers.c
@@ -357,7 +357,7 @@ oprelease_overlay_connect (void *cls)
 {
   struct OperationContext *opc = cls;
 
-  if (OPC_STATE_FINISHED != opc->state)
+  if (OPC_STATE_STARTED == opc->state)
   {
     GNUNET_free (opc->data);
     GNUNET_CONTAINER_DLL_remove (opc->c->ocq_head, opc->c->ocq_tail, opc);
@@ -720,8 +720,8 @@ GNUNET_TESTBED_overlay_connect (void *op_cls,
   opc->op =
       GNUNET_TESTBED_operation_create_ (opc, &opstart_overlay_connect,
                                         &oprelease_overlay_connect);
-  GNUNET_TESTBED_operation_queue_insert_ (opc->c->opq_parallel_operations,
-                                          opc->op);
+  /* GNUNET_TESTBED_operation_queue_insert_ (opc->c->opq_parallel_operations, */
+  /*                                         opc->op); */
   GNUNET_TESTBED_operation_queue_insert_
       (opc->c->opq_parallel_overlay_connect_operations, opc->op);
   return opc->op;
diff --git a/src/testbed/testbed_api_topology.c b/src/testbed/testbed_api_topology.c
index 22e9207f3..0e237cde3 100644
--- a/src/testbed/testbed_api_topology.c
+++ b/src/testbed/testbed_api_topology.c
@@ -35,11 +35,29 @@
 #define LOG(kind,...)                                           \
   GNUNET_log_from (kind, "testbed-api-topology", __VA_ARGS__)
 
+
+/**
+ * Context information for topology operations
+ */
+struct TopologyContext;
+
+
 /**
  * Representation of an overlay link
  */
 struct OverlayLink
 {
+
+  /**
+   * An operation corresponding to this link
+   */
+  struct GNUNET_TESTBED_Operation *op;
+
+  /**
+   * The topology context this link is a part of
+   */  
+  struct TopologyContext *tc;
+
   /**
    * position of peer A's handle in peers array
    */
@@ -69,13 +87,6 @@ struct TopologyContext
   struct OverlayLink *link_array;
 
   /**
-   * An array of operations resulting from the links we try to establish; the
-   * number of operations in this array is equal to link_array_size (1 link = 1
-   * operation)
-   */
-  struct GNUNET_TESTBED_Operation **link_ops;
-
-  /**
    * The operation closure
    */
   void *op_cls;
@@ -101,15 +112,23 @@ overlay_link_completed (void *cls,
                         struct GNUNET_TESTBED_Operation *op, 
                         const char *emsg)
 {
-  struct GNUNET_TESTBED_Operation **link_op = cls;
+  struct OverlayLink *link = cls;
+  struct TopologyContext *tc;
 
-  GNUNET_assert (*link_op == op);
+  GNUNET_assert (op == link->op);
   GNUNET_TESTBED_operation_done (op);
-  *link_op = NULL;
+  link->op = NULL;  
   if (NULL != emsg)
   {
+    tc = link->tc;
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Error while establishing a link: %s\n", emsg);
+         "Error while establishing a link: %s -- Retrying\n", emsg);
+    link->op =
+        GNUNET_TESTBED_overlay_connect (tc->op_cls,
+                                        &overlay_link_completed,
+                                        link,
+                                        tc->peers[link->A],
+                                        tc->peers[link->B]);
     return;
   }
 }
@@ -127,13 +146,11 @@ opstart_overlay_configure_topology (void *cls)
   struct TopologyContext *tc = cls;
   unsigned int p;
   
-  tc->link_ops = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_Operations *)
-                                * tc->link_array_size);
   for (p = 0; p < tc->link_array_size; p++)
   {
-    tc->link_ops[p] =
+    tc->link_array[p].op =
         GNUNET_TESTBED_overlay_connect (tc->op_cls, &overlay_link_completed,
-                                        &tc->link_ops[p],
+                                        &tc->link_array[p],
                                         tc->peers[tc->link_array[p].A],
                                         tc->peers[tc->link_array[p].B]);                                                  
   }
@@ -151,14 +168,13 @@ oprelease_overlay_configure_topology (void *cls)
   struct TopologyContext *tc = cls;
   unsigned int p;
   
-  if (NULL != tc->link_ops)
+  if (NULL != tc->link_array)
   {
     for (p = 0; p < tc->link_array_size; p++)
-      if (NULL != tc->link_ops[p])
-        GNUNET_TESTBED_operation_cancel (tc->link_ops[p]);      
-    GNUNET_free (tc->link_ops);    
+      if (NULL != tc->link_array[p].op)
+        GNUNET_TESTBED_operation_cancel (tc->link_array[p].op);
+    GNUNET_free (tc->link_array);
   }
-  GNUNET_free_non_null (tc->link_array);
   GNUNET_free (tc);
 }
 
@@ -242,7 +258,7 @@ GNUNET_TESTBED_overlay_configure_topology_va (void *op_cls,
   c = peers[0]->controller;
   tc = GNUNET_malloc (sizeof (struct TopologyContext));
   tc->peers = peers;
-  tc->op_cls = tc->op_cls;
+  tc->op_cls = op_cls;
   switch (topo)
   {
   case GNUNET_TESTBED_TOPOLOGY_LINE:
@@ -272,6 +288,7 @@ GNUNET_TESTBED_overlay_configure_topology_va (void *op_cls,
       } while (A_rand == B_rand);      
       tc->link_array[cnt].A = A_rand;
       tc->link_array[cnt].B = B_rand;
+      tc->link_array[cnt].tc = tc;
     }
     break;
   default: