Merge branch 'master' of ssh://gnunet.org/gnunet

14 files changed, 633 insertions, 192 deletions

diff --git a/src/cadet/.gitignore b/src/cadet/.gitignore
index 096ee06eb..154eabf3b 100644
--- a/src/cadet/.gitignore
+++ b/src/cadet/.gitignore
@@ -19,3 +19,4 @@ test_cadet_5_speed_reliable
 test_cadet_5_speed_reliable_backwards
 test_cadet_local
 test_cadet_single
+gnunet-service-cadet-new
diff --git a/src/cadet/cadet.conf.in b/src/cadet/cadet.conf.in
index 48fd03329..a6d762786 100644
--- a/src/cadet/cadet.conf.in
+++ b/src/cadet/cadet.conf.in
@@ -3,7 +3,7 @@ FORCESTART = YES
 AUTOSTART = @AUTOSTART@
 @JAVAPORT@PORT = 2096
 HOSTNAME = localhost
-BINARY = gnunet-service-cadet
+BINARY = gnunet-service-cadet-new
 # PREFIX = valgrind --leak-check=yes
 ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
diff --git a/src/cadet/gnunet-service-cadet-new.c b/src/cadet/gnunet-service-cadet-new.c
index 97489f3fd..f24c9f518 100644
--- a/src/cadet/gnunet-service-cadet-new.c
+++ b/src/cadet/gnunet-service-cadet-new.c
@@ -183,6 +183,11 @@ unsigned long long ratchet_messages;
  */
 struct GNUNET_TIME_Relative ratchet_time;
 
+/**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+struct GNUNET_TIME_Relative keepalive_period;
+
 
 /**
  * Send a message to a client.
@@ -1335,6 +1340,18 @@ run (void *cls,
                                "need delay value");
     ratchet_time = GNUNET_TIME_UNIT_HOURS;
   }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_time (c,
+                                           "CADET",
+                                           "REFRESH_CONNECTION_TIME",
+                                           &keepalive_period))
+  {
+    GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_WARNING,
+                               "CADET",
+                               "REFRESH_CONNECTION_TIME",
+                               "need delay value");
+    keepalive_period = GNUNET_TIME_UNIT_MINUTES;
+  }
 
   my_private_key = GNUNET_CRYPTO_eddsa_key_create_from_configuration (c);
   if (NULL == my_private_key)
diff --git a/src/cadet/gnunet-service-cadet-new.h b/src/cadet/gnunet-service-cadet-new.h
index 4a76c578b..721044ac4 100644
--- a/src/cadet/gnunet-service-cadet-new.h
+++ b/src/cadet/gnunet-service-cadet-new.h
@@ -226,6 +226,11 @@ extern unsigned long long ratchet_messages;
 extern struct GNUNET_TIME_Relative ratchet_time;
 
 /**
+ * How frequently do we send KEEPALIVE messages on idle connections?
+ */
+extern struct GNUNET_TIME_Relative keepalive_period;
+
+/**
  * Signal that shutdown is happening: prevent recovery measures.
  */
 extern int shutting_down;
diff --git a/src/cadet/gnunet-service-cadet-new_channel.c b/src/cadet/gnunet-service-cadet-new_channel.c
index c3d5ef194..e55a9a77d 100644
--- a/src/cadet/gnunet-service-cadet-new_channel.c
+++ b/src/cadet/gnunet-service-cadet-new_channel.c
@@ -25,13 +25,19 @@
  * @author Christian Grothoff
  *
  * TODO:
- * - FIXME: send ACKs back to loopback clients!
- *
+ * - Optimize ACKs by using 'mid_futures' properly!
+ * - calculate current RTT if possible, use that for initial retransmissions
+ *   (NOTE: needs us to learn which connection the tunnel uses for the message!)
  * - introduce shutdown so we can have half-closed channels, modify
  *   destroy to include MID to have FIN-ACK equivalents, etc.
  * - estimate max bandwidth using bursts and use to for CONGESTION CONTROL!
+ *   (and figure out how/where to use this!)
  * - check that '0xFFULL' really is sufficient for flow control!
+ *   (this is right now a big HACK!)
  * - revisit handling of 'unreliable' traffic!
+ *   (has not seen enough review)
+ * - revisit handling of 'unbuffered' traffic!
+ *   (has not seen enough review)
  * - revisit handling of 'out-of-order' option, especially in combination with/without 'reliable'.
  * - figure out flow control without ACKs (unreliable traffic!)
  */
@@ -59,6 +65,23 @@
  */
 #define TIMEOUT_CLOSED_PORT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
 
+/**
+ * How long do we wait at least before retransmitting ever?
+ */
+#define MIN_RTT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 75)
+
+/**
+ * Maximum message ID into the future we accept for out-of-order messages.
+ * If the message is more than this into the future, we drop it.  This is
+ * important both to detect values that are actually in the past, as well
+ * as to limit adversarially triggerable memory consumption.
+ *
+ * Note that right now we have "max_pending_messages = 4" hard-coded in
+ * the logic below, so a value of 4 would suffice here. But we plan to
+ * allow larger windows in the future...
+ */
+#define MAX_OUT_OF_ORDER_DISTANCE 1024
+
 
 /**
  * All the states a connection can be in.
@@ -272,6 +295,8 @@ struct CadetChannel
 
   /**
    * Bitfield of already-received messages past @e mid_recv.
+   *
+   * FIXME: not yet properly used (bits here are never set!)
    */
   uint64_t mid_futures;
 
@@ -526,6 +551,7 @@ send_channel_open (void *cls)
                                   &msgcc.header,
                                   &channel_open_sent_cb,
                                   ch);
+  GNUNET_assert (NULL == ch->retry_control_task);
 }
 
 
@@ -753,6 +779,11 @@ send_channel_data_ack (struct CadetChannel *ch)
   msg.futures = GNUNET_htonll (ch->mid_futures);
   if (NULL != ch->last_control_qe)
     GCT_send_cancel (ch->last_control_qe);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending DATA_ACK %u:%llX via %s\n",
+       (unsigned int) ntohl (msg.mid.mid),
+       (unsigned long long) ch->mid_futures,
+       GCCH_2s (ch));
   ch->last_control_qe = GCT_send (ch->t,
                                   &msg.header,
                                   &send_ack_cb,
@@ -837,9 +868,16 @@ send_ack_to_client (struct CadetChannel *ch,
   struct GNUNET_CADET_LocalAck *ack;
   struct CadetChannelClient *ccc;
 
+  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
+  if (NULL == ccc)
+  {
+    /* This can happen if we are just getting ACKs after
+       our local client already disconnected. */
+    GNUNET_assert (GNUNET_YES == ch->destroy);
+    return;
+  }
   env = GNUNET_MQ_msg (ack,
                        GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
-  ccc = (GNUNET_YES == to_owner) ? ch->owner : ch->dest;
   ack->ccn = ccc->ccn;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Sending CADET_LOCAL_ACK to %s (%s) at ccn %X (%u/%u pending)\n",
@@ -1060,23 +1098,11 @@ is_before (void *cls,
   if (delta > (uint32_t) INT_MAX)
   {
     /* in overflow range, we can safely assume we wrapped around */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "%u > %u => %p > %p\n",
-                (unsigned int) v1,
-                (unsigned int) v2,
-                m1,
-                m2);
     return GNUNET_NO;
   }
   else
   {
     /* result is small, thus v2 > v1, thus e1 < e2 */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "%u < %u => %p < %p\n",
-                (unsigned int) v1,
-                (unsigned int) v2,
-                m1,
-                m2);
     return GNUNET_YES;
   }
 }
@@ -1097,6 +1123,11 @@ GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
   struct GNUNET_CADET_LocalData *ld;
   struct CadetChannelClient *ccc;
   size_t payload_size;
+  struct CadetOutOfOrderMessage *com;
+  int duplicate;
+  uint32_t mid_min;
+  uint32_t mid_max;
+  uint32_t mid_msg;
 
   GNUNET_assert (GNUNET_NO == ch->is_loopback);
   if ( (GNUNET_YES == ch->destroy) &&
@@ -1127,8 +1158,9 @@ GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
          (msg->mid.mid == ch->mid_recv.mid) ) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Giving %u bytes of payload from %s to client %s\n",
+         "Giving %u bytes of payload with MID %u from %s to client %s\n",
          (unsigned int) payload_size,
+         ntohl (msg->mid.mid),
          GCCH_2s (ch),
          GSC_2s (ccc->c));
     ccc->client_ready = GNUNET_NO;
@@ -1136,52 +1168,148 @@ GCCH_handle_channel_plaintext_data (struct CadetChannel *ch,
                         env);
     ch->mid_recv.mid = htonl (1 + ntohl (ch->mid_recv.mid));
     ch->mid_futures >>= 1;
+    if (GNUNET_YES == ch->reliable)
+      send_channel_data_ack (ch);
+    return;
   }
-  else
+
+  /* check if message ought to be dropped because it is anicent/too distant/duplicate */
+  mid_min = ntohl (ch->mid_recv.mid);
+  mid_max = mid_min + MAX_OUT_OF_ORDER_DISTANCE;
+  mid_msg = ntohl (msg->mid.mid);
+  if ( ( (uint32_t) (mid_msg - mid_min) > MAX_OUT_OF_ORDER_DISTANCE) ||
+       ( (uint32_t) (mid_max - mid_msg) > MAX_OUT_OF_ORDER_DISTANCE) )
   {
-    struct CadetOutOfOrderMessage *com;
-    int duplicate;
-
-    /* FIXME-SECURITY: if the element is WAY too far ahead,
-       drop it (can't buffer too much!) */
-
-    com = GNUNET_new (struct CadetOutOfOrderMessage);
-    com->mid = msg->mid;
-    com->env = env;
-    duplicate = GNUNET_NO;
-    GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
-                                        is_before,
-                                        &duplicate,
-                                        ccc->head_recv,
-                                        ccc->tail_recv,
-                                        com);
-    if (GNUNET_YES == duplicate)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
-           (unsigned int) payload_size,
-           GCCH_2s (ch),
-           ntohl (msg->mid.mid));
-      GNUNET_STATISTICS_update (stats,
-                                "# duplicate DATA",
-                                1,
-                                GNUNET_NO);
-      GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
-                                   ccc->tail_recv,
-                                   com);
-      GNUNET_free (com);
-      return;
-    }
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Queued %s payload of %u bytes on %s (mid %u, need %u first)\n",
-         (GNUNET_YES == ccc->client_ready)
-         ? "out-of-order"
-         : "client-not-ready",
+         "Duplicate ancient or future payload of %u bytes on %s (mid %u) dropped\n",
          (unsigned int) payload_size,
          GCCH_2s (ch),
-         ntohl (msg->mid.mid),
-         ntohl (ch->mid_recv.mid));
+         ntohl (msg->mid.mid));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA (ancient or future)",
+                              1,
+                              GNUNET_NO);
+    GNUNET_MQ_discard (env);
+    return;
+  }
+
+  /* Insert message into sorted out-of-order queue */
+  com = GNUNET_new (struct CadetOutOfOrderMessage);
+  com->mid = msg->mid;
+  com->env = env;
+  duplicate = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetOutOfOrderMessage,
+                                      is_before,
+                                      &duplicate,
+                                      ccc->head_recv,
+                                      ccc->tail_recv,
+                                      com);
+  if (GNUNET_YES == duplicate)
+  {
+    /* Duplicate within the queue, drop also */
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Duplicate payload of %u bytes on %s (mid %u) dropped\n",
+         (unsigned int) payload_size,
+         GCCH_2s (ch),
+         ntohl (msg->mid.mid));
+    GNUNET_STATISTICS_update (stats,
+                              "# duplicate DATA",
+                              1,
+                              GNUNET_NO);
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    GNUNET_MQ_discard (com->env);
+    GNUNET_free (com);
+    if (GNUNET_YES == ch->reliable)
+      send_channel_data_ack (ch);
+    return;
   }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queued %s payload of %u bytes on %s-%X(%p) (mid %u, need %u first)\n",
+       (GNUNET_YES == ccc->client_ready)
+       ? "out-of-order"
+       : "client-not-ready",
+       (unsigned int) payload_size,
+       GCCH_2s (ch),
+       ntohl (ccc->ccn.channel_of_client),
+       ccc,
+       ntohl (msg->mid.mid),
+       ntohl (ch->mid_recv.mid));
+  send_channel_data_ack (ch);
+}
+
+
+/**
+ * Function called once the tunnel has sent one of our messages.
+ * If the message is unreliable, simply frees the `crm`. If the
+ * message was reliable, calculate retransmission time and
+ * wait for ACK (or retransmit).
+ *
+ * @param cls the `struct CadetReliableMessage` that was sent
+ */
+static void
+data_sent_cb (void *cls);
+
+
+/**
+ * We need to retry a transmission, the last one took too long to
+ * be acknowledged.
+ *
+ * @param cls the `struct CadetChannel` where we need to retransmit
+ */
+static void
+retry_transmission (void *cls)
+{
+  struct CadetChannel *ch = cls;
+  struct CadetReliableMessage *crm = ch->head_sent;
+
+  ch->retry_data_task = NULL;
+  GNUNET_assert (NULL == crm->qe);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Retrying transmission on %s of message %u\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid));
+  crm->qe = GCT_send (ch->t,
+                      &crm->data_message->header,
+                      &data_sent_cb,
+                      crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
+}
+
+
+/**
+ * We got an PLAINTEXT_DATA_ACK for a message in our queue, remove it from
+ * the queue and tell our client that it can send more.
+ *
+ * @param ch the channel that got the PLAINTEXT_DATA_ACK
+ * @param crm the message that got acknowledged
+ */
+static void
+handle_matching_ack (struct CadetChannel *ch,
+                     struct CadetReliableMessage *crm)
+{
+  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
+                               ch->tail_sent,
+                               crm);
+  ch->pending_messages--;
+  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
+       GCCH_2s (ch),
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       ch->pending_messages);
+  if (NULL != crm->qe)
+  {
+    GCT_send_cancel (crm->qe);
+    crm->qe = NULL;
+  }
+  GNUNET_free (crm->data_message);
+  GNUNET_free (crm);
+  send_ack_to_client (ch,
+                      (NULL == ch->owner)
+                      ? GNUNET_NO
+                      : GNUNET_YES);
 }
 
 
@@ -1197,6 +1325,11 @@ GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
                                         const struct GNUNET_CADET_ChannelDataAckMessage *ack)
 {
   struct CadetReliableMessage *crm;
+  struct CadetReliableMessage *crmn;
+  int found;
+  uint32_t mid_base;
+  uint64_t mid_mask;
+  unsigned int delta;
 
   GNUNET_break (GNUNET_NO == ch->is_loopback);
   if (GNUNET_NO == ch->reliable)
@@ -1205,12 +1338,32 @@ GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
     GNUNET_break_op (0);
     return;
   }
+  mid_base = ntohl (ack->mid.mid);
+  mid_mask = GNUNET_htonll (ack->futures);
+  found = GNUNET_NO;
   for (crm = ch->head_sent;
         NULL != crm;
-       crm = crm->next)
+       crm = crmn)
+  {
+    crmn = crm->next;
     if (ack->mid.mid == crm->data_message->mid.mid)
-      break;
-  if (NULL == crm)
+    {
+      handle_matching_ack (ch,
+                           crm);
+      found = GNUNET_YES;
+      continue;
+    }
+    delta = (unsigned int) (ntohl (crm->data_message->mid.mid) - mid_base) - 1;
+    if (delta >= 64)
+      continue;
+    if (0 != (mid_mask & (1LLU << delta)))
+    {
+      handle_matching_ack (ch,
+                           crm);
+      found = GNUNET_YES;
+    }
+  }
+  if (GNUNET_NO == found)
   {
     /* ACK for message we already dropped, might have been a
        duplicate ACK? Ignore. */
@@ -1223,26 +1376,16 @@ GCCH_handle_channel_plaintext_data_ack (struct CadetChannel *ch,
                               GNUNET_NO);
     return;
   }
-  GNUNET_CONTAINER_DLL_remove (ch->head_sent,
-                               ch->tail_sent,
-                               crm);
-  GNUNET_free (crm->data_message);
-  GNUNET_free (crm);
-  ch->pending_messages--;
-  send_ack_to_client (ch,
-                      (NULL == ch->owner)
-                      ? GNUNET_NO
-                      : GNUNET_YES);
-  GNUNET_assert (ch->pending_messages < ch->max_pending_messages);
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received DATA_ACK on %s for message %u (%u ACKs pending)\n",
-       GCCH_2s (ch),
-       (unsigned int) ntohl (ack->mid.mid),
-       ch->pending_messages);
-  send_ack_to_client (ch,
-                      (NULL == ch->owner)
-                      ? GNUNET_NO
-                      : GNUNET_YES);
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
+  if (NULL != ch->head_sent)
+    ch->retry_data_task
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
 }
 
 
@@ -1285,35 +1428,22 @@ GCCH_handle_remote_destroy (struct CadetChannel *ch)
 
 
 /**
- * Function called once the tunnel has sent one of our messages.
- * If the message is unreliable, simply frees the `crm`. If the
- * message was reliable, calculate retransmission time and
- * wait for ACK (or retransmit).
- *
- * @param cls the `struct CadetReliableMessage` that was sent
- */
-static void
-data_sent_cb (void *cls);
-
-
-/**
- * We need to retry a transmission, the last one took too long to
- * be acknowledged.
+ * Test if element @a e1 comes before element @a e2.
  *
- * @param cls the `struct CadetChannel` where we need to retransmit
+ * @param cls closure, to a flag where we indicate duplicate packets
+ * @param crm1 an element of to sort
+ * @param crm2 another element to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
  */
-static void
-retry_transmission (void *cls)
+static int
+cmp_crm_by_next_retry (void *cls,
+                       struct CadetReliableMessage *crm1,
+                       struct CadetReliableMessage *crm2)
 {
-  struct CadetChannel *ch = cls;
-  struct CadetReliableMessage *crm = ch->head_sent;
-
-  ch->retry_data_task = NULL;
-  GNUNET_assert (NULL == crm->qe);
-  crm->qe = GCT_send (ch->t,
-                      &crm->data_message->header,
-                      &data_sent_cb,
-                      crm);
+  if (crm1->next_retry.abs_value_us <
+      crm2->next_retry.abs_value_us)
+    return GNUNET_YES;
+  return GNUNET_NO;
 }
 
 
@@ -1330,9 +1460,9 @@ data_sent_cb (void *cls)
 {
   struct CadetReliableMessage *crm = cls;
   struct CadetChannel *ch = crm->ch;
-  struct CadetReliableMessage *off;
 
   GNUNET_assert (GNUNET_NO == ch->is_loopback);
+  GNUNET_assert (NULL != crm->qe);
   crm->qe = NULL;
   GNUNET_CONTAINER_DLL_remove (ch->head_sent,
                                ch->tail_sent,
@@ -1350,42 +1480,33 @@ data_sent_cb (void *cls)
   }
   if (0 == crm->retry_delay.rel_value_us)
     crm->retry_delay = ch->expected_delay;
+  else
+    crm->retry_delay = GNUNET_TIME_STD_BACKOFF (crm->retry_delay);
+  crm->retry_delay = GNUNET_TIME_relative_max (crm->retry_delay,
+                                               MIN_RTT_DELAY);
   crm->next_retry = GNUNET_TIME_relative_to_absolute (crm->retry_delay);
 
-  /* find position for re-insertion into the DLL */
-  if ( (NULL == ch->head_sent) ||
-       (crm->next_retry.abs_value_us < ch->head_sent->next_retry.abs_value_us) )
+  GNUNET_CONTAINER_DLL_insert_sorted (struct CadetReliableMessage,
+                                      cmp_crm_by_next_retry,
+                                      NULL,
+                                      ch->head_sent,
+                                      ch->tail_sent,
+                                      crm);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Message %u sent, next transmission on %s in %s\n",
+       (unsigned int) ntohl (crm->data_message->mid.mid),
+       GCCH_2s (ch),
+       GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (ch->head_sent->next_retry),
+                                               GNUNET_YES));
+  if (crm == ch->head_sent)
   {
-    /* insert at HEAD, also (re)schedule retry task! */
-    GNUNET_CONTAINER_DLL_insert (ch->head_sent,
-                                 ch->tail_sent,
-                                 crm);
+    /* We are the new head, need to reschedule retry task */
     if (NULL != ch->retry_data_task)
       GNUNET_SCHEDULER_cancel (ch->retry_data_task);
-    GNUNET_assert (NULL == crm->qe);
     ch->retry_data_task
-      = GNUNET_SCHEDULER_add_delayed (crm->retry_delay,
-                                      &retry_transmission,
-                                      ch);
-    return;
-  }
-  for (off = ch->head_sent; NULL != off; off = off->next)
-    if (crm->next_retry.abs_value_us < off->next_retry.abs_value_us)
-      break;
-  if (NULL == off)
-  {
-    /* insert at tail */
-    GNUNET_CONTAINER_DLL_insert_tail (ch->head_sent,
-                                      ch->tail_sent,
-                                      crm);
-  }
-  else
-  {
-    /* insert before off */
-    GNUNET_CONTAINER_DLL_insert_after (ch->head_sent,
-                                       ch->tail_sent,
-                                       off->prev,
-                                       crm);
+      = GNUNET_SCHEDULER_add_at (ch->head_sent->next_retry,
+                                 &retry_transmission,
+                                 ch);
   }
 }
 
@@ -1446,11 +1567,23 @@ GCCH_handle_local_data (struct CadetChannel *ch,
     GNUNET_memcpy (&ld[1],
                    buf,
                    buf_len);
-    /* FIXME: this does not provide for flow control! */
-    GSC_send_to_client (receiver->c,
-                        env);
-    send_ack_to_client (ch,
-                        to_owner);
+    if (GNUNET_YES == receiver->client_ready)
+    {
+      GSC_send_to_client (receiver->c,
+                          env);
+      send_ack_to_client (ch,
+                          to_owner);
+    }
+    else
+    {
+      struct CadetOutOfOrderMessage *oom;
+
+      oom = GNUNET_new (struct CadetOutOfOrderMessage);
+      oom->env = env;
+      GNUNET_CONTAINER_DLL_insert_tail (receiver->head_recv,
+                                        receiver->tail_recv,
+                                        oom);
+    }
     return GNUNET_OK;
   }
 
@@ -1471,13 +1604,20 @@ GCCH_handle_local_data (struct CadetChannel *ch,
                                ch->tail_sent,
                                crm);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending %u bytes from local client to %s\n",
+       "Sending %u bytes from local client to %s with MID %u\n",
        buf_len,
-       GCCH_2s (ch));
+       GCCH_2s (ch),
+       ntohl (crm->data_message->mid.mid));
+  if (NULL != ch->retry_data_task)
+  {
+    GNUNET_SCHEDULER_cancel (ch->retry_data_task);
+    ch->retry_data_task = NULL;
+  }
   crm->qe = GCT_send (ch->t,
                       &crm->data_message->header,
                       &data_sent_cb,
                       crm);
+  GNUNET_assert (NULL == ch->retry_data_task);
   return GNUNET_OK;
 }
 
@@ -1509,11 +1649,42 @@ GCCH_handle_local_ack (struct CadetChannel *ch,
   if (NULL == com)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Got LOCAL_ACK, %s-%X ready to receive more data (but none pending)!\n",
+         "Got LOCAL_ACK, %s-%X ready to receive more data, but none pending on %s-%X(%p)!\n",
          GSC_2s (ccc->c),
-         ntohl (ccc->ccn.channel_of_client));
+         ntohl (client_ccn.channel_of_client),
+         GCCH_2s (ch),
+         ntohl (ccc->ccn.channel_of_client),
+         ccc);
     return; /* none pending */
   }
+  if (GNUNET_YES == ch->is_loopback)
+  {
+    int to_owner;
+
+    /* Messages are always in-order, just send */
+    GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
+                                 ccc->tail_recv,
+                                 com);
+    GSC_send_to_client (ccc->c,
+                        com->env);
+    /* Notify sender that we can receive more */
+    if (ccc->ccn.channel_of_client ==
+        ch->owner->ccn.channel_of_client)
+    {
+      to_owner = GNUNET_NO;
+    }
+    else
+    {
+      GNUNET_assert (ccc->ccn.channel_of_client ==
+                     ch->dest->ccn.channel_of_client);
+      to_owner = GNUNET_YES;
+    }
+    send_ack_to_client (ch,
+                        to_owner);
+    GNUNET_free (com);
+    return;
+  }
+
   if ( (com->mid.mid != ch->mid_recv.mid) &&
        (GNUNET_NO == ch->out_of_order) )
   {
@@ -1526,11 +1697,12 @@ GCCH_handle_local_ack (struct CadetChannel *ch,
     return; /* missing next one in-order */
   }
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got LOCAL ACK, passing payload message to %s-%X on %s\n",
-              GSC_2s (ccc->c),
-              ntohl (ccc->ccn.channel_of_client),
-              GCCH_2s (ch));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got LOCAL ACK, passing payload message %u to %s-%X on %s\n",
+       ntohl (com->mid.mid),
+       GSC_2s (ccc->c),
+       ntohl (ccc->ccn.channel_of_client),
+       GCCH_2s (ch));
 
   /* all good, pass next message to client */
   GNUNET_CONTAINER_DLL_remove (ccc->head_recv,
@@ -1552,9 +1724,9 @@ GCCH_handle_local_ack (struct CadetChannel *ch,
        urgently waiting for an ACK from us. (As we have an inherent
        maximum of 64 bits, and 15 is getting too close for comfort.)
        So we should send one now. */
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Sender on %s likely blocked on flow-control, sending ACK now.\n",
-                GCCH_2s (ch));
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Sender on %s likely blocked on flow-control, sending ACK now.\n",
+         GCCH_2s (ch));
     if (GNUNET_YES == ch->reliable)
       send_channel_data_ack (ch);
   }
diff --git a/src/cadet/gnunet-service-cadet-new_connection.c b/src/cadet/gnunet-service-cadet-new_connection.c
index 60389008c..58922bc1e 100644
--- a/src/cadet/gnunet-service-cadet-new_connection.c
+++ b/src/cadet/gnunet-service-cadet-new_connection.c
@@ -27,16 +27,18 @@
  * @author Christian Grothoff
  *
  * TODO:
- * - Optimization: keepalive messages / timeout (timeout to be done @ peer level!)
+ * - Implement: keepalive messages / timeout (timeout to be done @ peer level!)
  * - Optimization: keep performance metrics (?)
  */
 #include "platform.h"
+#include "gnunet-service-cadet-new.h"
 #include "gnunet-service-cadet-new_channel.h"
 #include "gnunet-service-cadet-new_connection.h"
 #include "gnunet-service-cadet-new_paths.h"
 #include "gnunet-service-cadet-new_peer.h"
 #include "gnunet-service-cadet-new_tunnels.h"
 #include "gnunet_cadet_service.h"
+#include "gnunet_statistics_service.h"
 #include "cadet_protocol.h"
 
 
@@ -119,6 +121,11 @@ struct CadetConnection
   struct GNUNET_SCHEDULER_Task *task;
 
   /**
+   * Queue entry for keepalive messages.
+   */
+  struct CadetTunnelQueueEntry *keepalive_qe;
+
+  /**
    * Function to call once we are ready to transmit.
    */
   GCC_ReadyCallback ready_cb;
@@ -184,6 +191,11 @@ GCC_destroy (struct CadetConnection *cc)
     GNUNET_SCHEDULER_cancel (cc->task);
     cc->task = NULL;
   }
+  if (NULL != cc->keepalive_qe)
+  {
+    GCT_send_cancel (cc->keepalive_qe);
+    cc->keepalive_qe = NULL;
+  }
   GCPP_del_connection (cc->path,
                        cc->off,
                        cc);
@@ -209,7 +221,72 @@ GCC_get_ct (struct CadetConnection *cc)
 
 
 /**
- * A CADET_CONNECTION_ACK was received for this connection, implying
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ */
+static void
+send_keepalive (void *cls);
+
+
+/**
+ * Keepalive was transmitted.  Remember this, and possibly
+ * schedule the next one.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ */
+static void
+keepalive_done (void *cls)
+{
+  struct CadetConnection *cc = cls;
+
+  cc->keepalive_qe = NULL;
+  if ( (GNUNET_YES == cc->mqm_ready) &&
+       (NULL == cc->task) )
+    cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                             &send_keepalive,
+                                             cc);
+}
+
+
+/**
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE through the
+ * tunnel to prevent it from timing out.
+ *
+ * @param cls the `struct CadetConnection` to keep alive.
+ */
+static void
+send_keepalive (void *cls)
+{
+  struct CadetConnection *cc = cls;
+  struct GNUNET_MessageHeader msg;
+
+  cc->task = NULL;
+  GNUNET_assert (NULL != cc->ct);
+  GNUNET_assert (GNUNET_YES == cc->mqm_ready);
+  GNUNET_assert (NULL == cc->keepalive_qe);
+  LOG (GNUNET_ERROR_TYPE_INFO,
+       "Sending KEEPALIVE on behalf of %s via %s\n",
+       GCC_2s (cc),
+       GCT_2s (cc->ct->t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives sent",
+                            1,
+                            GNUNET_NO);
+  msg.size = htons (sizeof (msg));
+  msg.type = htons (GNUNET_MESSAGE_TYPE_CADET_CHANNEL_KEEPALIVE);
+
+  cc->keepalive_qe
+    = GCT_send (cc->ct->t,
+                &msg,
+                &keepalive_done,
+                cc);
+}
+
+
+/**
+ * A #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE_ACK was received for this connection, implying
  * that the end-to-end connection is up.  Process it.
  *
  * @param cc the connection that got the ACK.
@@ -227,15 +304,18 @@ GCC_handle_connection_create_ack (struct CadetConnection *cc)
     GNUNET_SCHEDULER_cancel (cc->task);
     cc->task = NULL;
   }
-#if FIXME_KEEPALIVE
-  cc->task = GNUNET_SCHEDULER_add_delayed (cc->keepalive_period,
-                                           &send_keepalive,
-                                           cc);
-#endif
   cc->state = CADET_CONNECTION_READY;
   if (GNUNET_YES == cc->mqm_ready)
+  {
     cc->ready_cb (cc->ready_cb_cls,
                   GNUNET_YES);
+    if ( (NULL == cc->keepalive_qe) &&
+         (GNUNET_YES == cc->mqm_ready) &&
+         (NULL == cc->task) )
+      cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                               &send_keepalive,
+                                               cc);
+  }
 }
 
 
@@ -288,7 +368,8 @@ GCC_handle_encrypted (struct CadetConnection *cc,
 
 
 /**
- * Send a CREATE message to the first hop.
+ * Send a #GNUNET_MESSAGE_TYPE_CADET_CONNECTION_CREATE message to the
+ * first hop.
  *
  * @param cls the `struct CadetConnection` to initiate
  */
@@ -459,6 +540,19 @@ manage_first_hop_mq (void *cls,
   case CADET_CONNECTION_READY:
     cc->ready_cb (cc->ready_cb_cls,
                   GNUNET_YES);
+    if ( (NULL == cc->keepalive_qe) &&
+         (GNUNET_YES == cc->mqm_ready) &&
+         (NULL == cc->task) )
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Scheduling keepalive for %s in %s\n",
+           GCC_2s (cc),
+           GNUNET_STRINGS_relative_time_to_string (keepalive_period,
+                                                   GNUNET_YES));
+      cc->task = GNUNET_SCHEDULER_add_delayed (keepalive_period,
+                                               &send_keepalive,
+                                               cc);
+    }
     break;
   }
 }
@@ -610,6 +704,11 @@ GCC_transmit (struct CadetConnection *cc,
   GNUNET_assert (GNUNET_YES == cc->mqm_ready);
   GNUNET_assert (CADET_CONNECTION_READY == cc->state);
   cc->mqm_ready = GNUNET_NO;
+  if (NULL != cc->task)
+  {
+    GNUNET_SCHEDULER_cancel (cc->task);
+    cc->task = NULL;
+  }
   GCP_send (cc->mq_man,
             env);
 }
diff --git a/src/cadet/gnunet-service-cadet-new_paths.c b/src/cadet/gnunet-service-cadet-new_paths.c
index 685656ec3..8a4d7bbf8 100644
--- a/src/cadet/gnunet-service-cadet-new_paths.c
+++ b/src/cadet/gnunet-service-cadet-new_paths.c
@@ -24,6 +24,11 @@
  * @author Christian Grothoff
  *
  * TODO:
+ * - currently only allowing one unique connection per path,
+ *   but need to allow 2 in case WE are establishing one from A to B
+ *   while at the same time B establishes one to A.
+ *   Also, must not ASSERT if B establishes a 2nd one to us.
+ *   Need to have some reasonable tie-breaking to only keep ONE.
  * - path desirability score calculations are not done
  */
 #include "platform.h"
diff --git a/src/cadet/gnunet-service-cadet-new_tunnels.c b/src/cadet/gnunet-service-cadet-new_tunnels.c
index fd8335486..03067a605 100644
--- a/src/cadet/gnunet-service-cadet-new_tunnels.c
+++ b/src/cadet/gnunet-service-cadet-new_tunnels.c
@@ -24,6 +24,8 @@
  * @author Christian Grothoff
  *
  * FIXME:
+ * - implement keepalive
+ * - implement rekeying
  * - check KX estate machine -- make sure it is never stuck!
  * - clean up KX logic, including adding sender authentication
  * - implement connection management (evaluate, kill old ones,
@@ -1961,7 +1963,7 @@ GCT_consider_path (struct CadetTunnel *t,
 
 
 /**
- * NOT IMPLEMENTED.
+ * We got a keepalive. Track in statistics.
  *
  * @param cls the `struct CadetTunnel` for which we decrypted the message
  * @param msg  the message we received on the tunnel
@@ -1972,7 +1974,13 @@ handle_plaintext_keepalive (void *cls,
 {
   struct CadetTunnel *t = cls;
 
-  GNUNET_break (0); // FIXME
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Received KEEPALIVE on tunnel %s\n",
+       GCT_2s (t));
+  GNUNET_STATISTICS_update (stats,
+                            "# keepalives received",
+                            1,
+                            GNUNET_NO);
 }
 
 
diff --git a/src/cadet/test_cadet.c b/src/cadet/test_cadet.c
index 3a1042eba..efc4f96b3 100644
--- a/src/cadet/test_cadet.c
+++ b/src/cadet/test_cadet.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
+     Copyright (C) 2011, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -64,7 +64,7 @@ static int test;
 /**
  * String with test name
  */
-char *test_name;
+static char *test_name;
 
 /**
  * Flag to send traffic leaf->root in speed tests to test BCK_ACK logic.
@@ -79,32 +79,32 @@ static int ok;
 /**
  * Number of events expected to conclude the test successfully.
  */
-int ok_goal;
+static int ok_goal;
 
 /**
  * Size of each test packet
  */
-size_t size_payload = sizeof (struct GNUNET_MessageHeader) + sizeof (uint32_t);
+static size_t size_payload = sizeof (struct GNUNET_MessageHeader) + sizeof (uint32_t);
 
 /**
  * Operation to get peer ids.
  */
-struct GNUNET_TESTBED_Operation *t_op[2];
+static struct GNUNET_TESTBED_Operation *t_op[2];
 
 /**
  * Peer ids.
  */
-struct GNUNET_PeerIdentity *p_id[2];
+static struct GNUNET_PeerIdentity *p_id[2];
 
 /**
  * Port ID
  */
-struct GNUNET_HashCode port;
+static struct GNUNET_HashCode port;
 
 /**
  * Peer ids counter.
  */
-unsigned int p_ids;
+static unsigned int p_ids;
 
 /**
  * Is the setup initialized?
@@ -345,12 +345,21 @@ shutdown_task (void *cls)
  *          operation has executed successfully.
  */
 static void
-stats_cont (void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
+stats_cont (void *cls,
+            struct GNUNET_TESTBED_Operation *op,
+            const char *emsg)
 {
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, " KA sent: %u, KA received: %u\n",
-              ka_sent, ka_received);
-  if (KEEPALIVE == test && (ka_sent < 2 || ka_sent > ka_received + 1))
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              " KA sent: %u, KA received: %u\n",
+              ka_sent,
+              ka_received);
+  if ( (KEEPALIVE == test) &&
+       ( (ka_sent < 2) ||
+         (ka_sent > ka_received + 1)) )
+  {
+    GNUNET_break (0);
     ok--;
+  }
   GNUNET_TESTBED_operation_done (stats_op);
 
   if (NULL != disconnect_task)
@@ -439,10 +448,11 @@ gather_stats_and_exit (void *cls)
 static void
 abort_test (long line)
 {
-  if (disconnect_task != NULL)
+  if (NULL != disconnect_task)
   {
     GNUNET_SCHEDULER_cancel (disconnect_task);
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Aborting test from %ld\n", line);
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Aborting test from %ld\n", line);
     disconnect_task = GNUNET_SCHEDULER_add_now (&disconnect_cadet_peers,
                                                 (void *) line);
   }
diff --git a/src/core/core_api.c b/src/core/core_api.c
index 18b07068f..afae20850 100644
--- a/src/core/core_api.c
+++ b/src/core/core_api.c
@@ -336,7 +336,6 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
   env = GNUNET_MQ_msg (smr,
                        GNUNET_MESSAGE_TYPE_CORE_SEND_REQUEST);
   smr->priority = htonl ((uint32_t) priority);
-  // smr->deadline = GNUNET_TIME_absolute_hton (deadline);
   smr->peer = pr->peer;
   smr->reserved = htonl (0);
   smr->size = htons (msize);
@@ -349,7 +348,6 @@ core_mq_send_impl (struct GNUNET_MQ_Handle *mq,
                                      GNUNET_MESSAGE_TYPE_CORE_SEND,
                                      msg);
   sm->priority = htonl ((uint32_t) priority);
-  // sm->deadline = GNUNET_TIME_absolute_hton (deadline);
   sm->peer = pr->peer;
   sm->cork = htonl ((uint32_t) cork);
   sm->reserved = htonl (0);
diff --git a/src/include/gnunet_container_lib.h b/src/include/gnunet_container_lib.h
index d2f8c5d9c..c77d82fd3 100644
--- a/src/include/gnunet_container_lib.h
+++ b/src/include/gnunet_container_lib.h
@@ -2092,8 +2092,8 @@ GNUNET_CONTAINER_multihashmap32_iterator_destroy (struct GNUNET_CONTAINER_MultiH
                         element, \
                         head)) ) \
   { \
-    /* insert at head, e;e,emt < head */ \
-    GNUNET_CONTAINER_DLL_insert (head, \
+    /* insert at head, element < head */ \
+    GNUNET_CONTAINER_DLL_insert (head,                                \
                                  tail, \
                                  element); \
   } \
@@ -2109,17 +2109,21 @@ GNUNET_CONTAINER_multihashmap32_iterator_destroy (struct GNUNET_CONTAINER_MultiH
                       element, \
                       pos)) \
         break; /* element < pos */ \
-      if (NULL == pos) /* => element > tail */ \
-        GNUNET_CONTAINER_DLL_insert_tail (head, \
-                                          tail, \
-                                          element); \
-      else /* prev < element < pos */ \
-        GNUNET_CONTAINER_DLL_insert_after (head, \
-                                           tail, \
-                                           element, \
-                                           pos->prev); \
+    if (NULL == pos) /* => element > tail */ \
+    { \
+      GNUNET_CONTAINER_DLL_insert_tail (head,                             \
+                                        tail, \
+                                        element); \
+    } \
+    else /* prev < element < pos */ \
+    { \
+      GNUNET_CONTAINER_DLL_insert_after (head, \
+                                         tail, \
+                                         pos->prev, \
+                                         element); \
     } \
-  } while (0)
+  } \
+} while (0)
 
 
 /* ******************** Heap *************** */
diff --git a/src/util/.gitignore b/src/util/.gitignore
index f207e07bf..a0cb43874 100644
--- a/src/util/.gitignore
+++ b/src/util/.gitignore
@@ -22,6 +22,7 @@ test_connection_timeout.nc
 test_connection_timeout_no_connect.nc
 test_connection_transmit_cancel.nc
 test_container_bloomfilter
+test_container_dll
 test_container_heap
 test_container_meta_data
 test_container_multihashmap
diff --git a/src/util/Makefile.am b/src/util/Makefile.am
index f49aee17f..c666b017d 100644
--- a/src/util/Makefile.am
+++ b/src/util/Makefile.am
@@ -273,6 +273,7 @@ check_PROGRAMS = \
  test_common_logging \
  test_configuration \
  test_container_bloomfilter \
+ test_container_dll \
  test_container_meta_data \
  test_container_multihashmap \
  test_container_multihashmap32 \
@@ -398,6 +399,11 @@ test_container_bloomfilter_SOURCES = \
 test_container_bloomfilter_LDADD = \
  libgnunetutil.la
 
+test_container_dll_SOURCES = \
+ test_container_dll.c
+test_container_dll_LDADD = \
+ libgnunetutil.la
+
 test_container_meta_data_SOURCES = \
  test_container_meta_data.c
 test_container_meta_data_LDADD = \
diff --git a/src/util/test_container_dll.c b/src/util/test_container_dll.c
new file mode 100644
index 000000000..8ad08423a
--- /dev/null
+++ b/src/util/test_container_dll.c
@@ -0,0 +1,115 @@
+/*
+ This file is part of GNUnet.
+ Copyright (C) 2017 GNUnet e.V.
+
+ GNUnet is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3, or (at your
+ option) any later version.
+
+ GNUnet is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GNUnet; see the file COPYING.  If not, write to the
+ Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @author Christian Grothoff
+ * @file util/test_container_dll.c
+ * @brief Test of DLL operations
+ */
+
+#include "platform.h"
+#include "gnunet_util_lib.h"
+
+/**
+ * Element in the DLL.
+ */
+struct Element
+{
+  /**
+   * Required pointer to previous element.
+   */
+  struct Element *prev;
+
+  /**
+   * Required pointer to next element.
+   */
+  struct Element *next;
+
+  /**
+   * Used to sort.
+   */
+  unsigned int value;
+};
+
+
+/**
+ * Compare two elements.
+ *
+ * @param cls closure, NULL
+ * @param e1 an element of to sort
+ * @param e2 another element to sort
+ * @return #GNUNET_YES if @e1 < @e2, otherwise #GNUNET_NO
+ */
+static int
+cmp_elem (void *cls,
+          struct Element *e1,
+          struct Element *e2)
+{
+  if (e1->value == e2->value)
+    return 0;
+  return (e1->value < e2->value) ? 1 : -1;
+}
+
+
+int
+main (int argc, char **argv)
+{
+  unsigned int values2[] = {
+    4, 5, 8, 6, 9, 3, 7, 2, 6, 1, 0
+  };
+  unsigned int values[] = {
+    1, 3, 2, 0
+  };
+  struct Element *head = NULL;
+  struct Element *tail = NULL;
+  struct Element *e;
+  unsigned int want;
+
+  GNUNET_log_setup ("test-container-dll",
+                    "WARNING",
+                    NULL);
+  for (unsigned int off=0;
+       0 != values[off];
+       off++)
+  {
+    e = GNUNET_new (struct Element);
+    e->value = values[off];
+    GNUNET_CONTAINER_DLL_insert_sorted (struct Element,
+                                        cmp_elem,
+                                        NULL,
+                                        head,
+                                        tail,
+                                        e);
+  }
+
+  want = 1;
+  while (NULL != (e = head))
+  {
+    GNUNET_assert (e->value == want);
+    GNUNET_CONTAINER_DLL_remove (head,
+                                 tail,
+                                 e);
+    GNUNET_free (e);
+    want++;
+  }
+  return 0;
+}
+
+/* end of test_container_heap.c */